solid_queue 1.1.0 → 1.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e843e842397e1d8141e0457b5b278026b71effe6ebf83d8300d2c4098db56adf
-  data.tar.gz: a5db37bdea8dacec796f2dcb912ab8f2a69c929487dff081d8a53fe10c20086c
+  metadata.gz: 9066bd5266e43075385bfd3365de2512400960f5cfa7f780dba69e4a3259c07a
+  data.tar.gz: 0a7103f485e445563814874e3113b6ac6dca84c8333bad418c449ebaf3fac1c9
 SHA512:
-  metadata.gz: a6831f7114c24d68ae8ed7b5aebfd4d3df0cc2c7b4e0046e6275a5bf5e46f8da8f5b79e9a303ae908e3581e8b96a132776f4c1a3f55bbf550dd008541e679665
-  data.tar.gz: e51b90234b3a60355a96c9163bfc7d95a1f5eb95fcf7a2dc1faf553f04a7c242da2d48493ba5b6574c2dbf4b0d259cb69af1dda576475f1510a31f8325be5f8b
+  metadata.gz: 952b71b5cd59ebd79eb51c44f7ed509bf3d4959c010dc0441cff37c0a1bd2ccea97054007bd3a197b287a158342e8791318c841f0d1d9b3dd347986da68bb53a
+  data.tar.gz: 1309ce242499f430667d9677b7ff8807e1b43af33873d9b00575c4c9e13be24824d9520ca24db432901da2616c4ba5a530db83fd29c09c47b820fff3586a75b7

data/README.md

@@ -6,6 +6,34 @@ Besides regular job enqueuing and processing, Solid Queue supports delayed jobs,
 
 Solid Queue can be used with SQL databases such as MySQL, PostgreSQL or SQLite, and it leverages the `FOR UPDATE SKIP LOCKED` clause, if available, to avoid blocking and waiting on locks when polling jobs. It relies on Active Job for retries, discarding, error handling, serialization, or delays, and it's compatible with Ruby on Rails's multi-threading.
 
+## Table of contents
+
+- [Installation](#installation)
+  - [Usage in development and other non-production environments](#usage-in-development-and-other-non-production-environments)
+  - [Single database configuration](#single-database-configuration)
+  - [Dashboard UI Setup](#dashboard-ui-setup)
+  - [Incremental adoption](#incremental-adoption)
+  - [High performance requirements](#high-performance-requirements)
+- [Configuration](#configuration)
+  - [Workers, dispatchers and scheduler](#workers-dispatchers-and-scheduler)
+  - [Queue order and priorities](#queue-order-and-priorities)
+  - [Queues specification and performance](#queues-specification-and-performance)
+  - [Threads, processes and signals](#threads-processes-and-signals)
+  - [Database configuration](#database-configuration)
+  - [Other configuration settings](#other-configuration-settings)
+- [Lifecycle hooks](#lifecycle-hooks)
+- [Errors when enqueuing](#errors-when-enqueuing)
+- [Concurrency controls](#concurrency-controls)
+  - [Performance considerations](#performance-considerations)
+- [Failed jobs and retries](#failed-jobs-and-retries)
+  - [Error reporting on jobs](#error-reporting-on-jobs)
+- [Puma plugin](#puma-plugin)
+- [Jobs and transactional integrity](#jobs-and-transactional-integrity)
+- [Recurring tasks](#recurring-tasks)
+- [Inspiration](#inspiration)
+- [License](#license)
+
+
 ## Installation
 
 Solid Queue is configured by default in new Rails 8 applications. But if you're running an earlier version, you can add it manually following these steps:
@@ -13,6 +41,8 @@ Solid Queue is configured by default in new Rails 8 applications. But if you're
 1. `bundle add solid_queue`
 2. `bin/rails solid_queue:install`
 
+(Note: The minimum supported version of Rails is 7.1 and Ruby is 3.1.6.)
+
 This will configure Solid Queue as the production Active Job backend, create the configuration files `config/queue.yml` and `config/recurring.yml`, and create the `db/queue_schema.rb`. It'll also create a `bin/jobs` executable wrapper that you can use to start Solid Queue.
 
 Once you've done that, you will then have to add the configuration for the queue database in `config/database.yml`. If you're using SQLite, it'll look like this:
@@ -43,8 +73,6 @@ production:
     migrations_paths: db/queue_migrate
 ```
 
-Note: Calling `bin/rails solid_queue:install` will automatically add `config.solid_queue.connects_to = { database: { writing: :queue } }` to `config/environments/production.rb`, so no additional configuration is needed there (although you must make sure that you use the `queue` name in `database.yml` for this to match!). But if you want to use Solid Queue in a different environment (like staging or even development), you'll have to manually add that `config.solid_queue.connects_to` line to the respective environment file. And, as always, make sure that the name you're using for the database in `config/database.yml` matches the name you use in `config.solid_queue.connects_to`.
-
 Then run `db:prepare` in production to ensure the database is created and the schema is loaded.
 
 Now you're ready to start processing jobs by running `bin/jobs` on the server that's doing the work. This will start processing jobs in all queues using the default configuration. See [below](#configuration) to learn more about configuring Solid Queue.
@@ -53,6 +81,72 @@ For small projects, you can run Solid Queue on the same machine as your webserve
 
 **Note**: future changes to the schema will come in the form of regular migrations.
 
+### Usage in development and other non-production environments
+
+Calling `bin/rails solid_queue:install` will automatically add `config.solid_queue.connects_to = { database: { writing: :queue } }` to `config/environments/production.rb`. In order to use Solid Queue in other environments (such as development or staging), you'll need to add a similar configuration(s).
+
+For example, if you're using SQLite in development, update `database.yml` as follows:
+
+```diff
+development:
++ primary:
+    <<: *default
+    database: storage/development.sqlite3
++  queue:
++    <<: *default
++    database: storage/development_queue.sqlite3
++    migrations_paths: db/queue_migrate
+```
+
+Next, add the following to `development.rb`
+
+```ruby
+  # Use Solid Queue in Development.
+  config.active_job.queue_adapter = :solid_queue
+  config.solid_queue.connects_to = { database: { writing: :queue } }
+```
+
+Once you've added this, run `db:prepare` to create the Solid Queue database and load the schema.
+
+Finally, in order for jobs to be processed, you'll need to have Solid Queue running. In Development, this can be done via [the Puma plugin](#puma-plugin) as well. In `puma.rb` update the following line:
+
+```ruby
+# You can either set the env var, or check for development
+plugin :solid_queue if ENV["SOLID_QUEUE_IN_PUMA"] || Rails.env.development?
+```
+
+You can also just use `bin/jobs`, but in this case you might want to [set a different logger for Solid Queue](#other-configuration-settings) because the default logger will log to `log/development.log` and you won't see anything when you run `bin/jobs`. For example:
+```ruby
+config.solid_queue.logger = ActiveSupport::Logger.new(STDOUT)
+```
+
+**Note about Action Cable**: If you use Action Cable (or anything dependent on Action Cable, such as Turbo Streams), you will also need to update it to use a database.
+
+In `config/cable.yml`
+
+```diff
+development:
+-  adapter: async
++ adapter: solid_cable
++  connects_to:
++    database:
++      writing: cable
++  polling_interval: 0.1.seconds
++  message_retention: 1.day
+```
+
+In `config/database.yml`
+
+```diff
+development:
+  primary:
+    <<: *default
+    database: storage/development.sqlite3
++  cable:
++    <<: *default
++    database: storage/development_cable.sqlite3
++    migrations_paths: db/cable_migrate
+```
 
 ### Single database configuration
 
@@ -64,7 +158,11 @@ Running Solid Queue in a separate database is recommended, but it's also possibl
 
 You won't have multiple databases, so `database.yml` doesn't need to have primary and queue database.
 
-## Incremental adoption
+### Dashboard ui setup
+
+For viewing information about your jobs via a UI, we recommend taking a look at [mission_control-jobs](https://github.com/rails/mission_control-jobs), a dashboard where, among other things, you can examine and retry/discard failed jobs.
+
+### Incremental adoption
 
 If you're planning to adopt Solid Queue incrementally by switching one job at the time, you can do so by leaving the `config.active_job.queue_adapter` set to your old backend, and then set the `queue_adapter` directly in the jobs you're moving:
 
@@ -77,7 +175,7 @@ class MyJob < ApplicationJob
 end
 ```
 
-## High performance requirements
+### High performance requirements
 
 Solid Queue was designed for the highest throughput when used with MySQL 8+ or PostgreSQL 9.5+, as they support `FOR UPDATE SKIP LOCKED`. You can use it with older versions, but in that case, you might run into lock waits if you run multiple workers for the same queue. You can also use it with SQLite on smaller applications.
 
@@ -86,6 +184,7 @@ Solid Queue was designed for the highest throughput when used with MySQL 8+ or P
 ### Workers, dispatchers and scheduler
 
 We have several types of actors in Solid Queue:
+
 - _Workers_ are in charge of picking jobs ready to run from queues and processing them. They work off the `solid_queue_ready_executions` table.
 - _Dispatchers_ are in charge of selecting jobs scheduled to run in the future that are due and _dispatching_ them, which is simply moving them from the `solid_queue_scheduled_executions` table over to the `solid_queue_ready_executions` table so that workers can pick them up. On top of that, they do some maintenance work related to [concurrency controls](#concurrency-controls).
 - The _scheduler_ manages [recurring tasks](#recurring-tasks), enqueuing jobs for them when they're due.
@@ -99,6 +198,7 @@ By default, Solid Queue will try to find your configuration under `config/queue.
 bin/jobs -c config/calendar.yml
 ```
 
+You can also skip all recurring tasks by setting the environment variable `SOLID_QUEUE_SKIP_RECURRING=true`. This is useful for environments like staging, review apps, or development where you don't want any recurring jobs to run. This is equivalent to using the `--skip-recurring` option with `bin/jobs`.
 
 This is what this configuration looks like:
 
@@ -153,6 +253,7 @@ Here's an overview of the different options:
   Check the sections below on [how queue order behaves combined with priorities](#queue-order-and-priorities), and [how the way you specify the queues per worker might affect performance](#queues-specification-and-performance).
 
 - `threads`: this is the max size of the thread pool that each worker will have to run jobs. Each worker will fetch this number of jobs from their queue(s), at most and will post them to the thread pool to be run. By default, this is `3`. Only workers have this setting.
+It is recommended to set this value less than or equal to the queue database's connection pool size minus 2, as each worker thread uses one connection, and two additional connections are reserved for polling and heartbeat.
 - `processes`: this is the number of worker processes that will be forked by the supervisor with the settings given. By default, this is `1`, just a single process. This setting is useful if you want to dedicate more than one CPU core to a queue or queues with the same configuration. Only workers have this setting.
 - `concurrency_maintenance`: whether the dispatcher will perform the concurrency maintenance work. This is `true` by default, and it's useful if you don't use any [concurrency controls](#concurrency-controls) and want to disable it or if you run multiple dispatchers and want some of them to just dispatch jobs without doing anything else.
 
@@ -220,7 +321,7 @@ and then remove the paused ones. Pausing in general should be something rare, us
 Do this:
 
 ```yml
-queues: background, backend
+queues: [ background, backend ]
 ```
 
 instead of this:
@@ -250,33 +351,6 @@ You can configure the database used by Solid Queue via the `config.solid_queue.c
 
 All the options available to Active Record for multiple databases can be used here.
 
-## Lifecycle hooks
-
-In Solid queue, you can hook into two different points in the supervisor's life:
-- `start`: after the supervisor has finished booting and right before it forks workers and dispatchers.
-- `stop`: after receiving a signal (`TERM`, `INT` or `QUIT`) and right before starting graceful or immediate shutdown.
-
-And into two different points in a worker's life:
-- `worker_start`: after the worker has finished booting and right before it starts the polling loop.
-- `worker_stop`: after receiving a signal (`TERM`, `INT` or `QUIT`) and right before starting graceful or immediate shutdown (which is just `exit!`).
-
-You can use the following methods with a block to do this:
-```ruby
-SolidQueue.on_start
-SolidQueue.on_stop
-
-SolidQueue.on_worker_start
-SolidQueue.on_worker_stop
-```
-
-For example:
-```ruby
-SolidQueue.on_start { start_metrics_server }
-SolidQueue.on_stop { stop_metrics_server }
-```
-
-These can be called several times to add multiple hooks, but it needs to happen before Solid Queue is started. An initializer would be a good place to do this.
-
 ### Other configuration settings
 
 _Note_: The settings in this section should be set in your `config/application.rb` or your environment config like this: `config.solid_queue.silence_polling = true`
@@ -299,9 +373,58 @@ There are several settings that control how Solid Queue works that you can set a
 - `silence_polling`: whether to silence Active Record logs emitted when polling for both workers and dispatchers—defaults to `true`.
 - `supervisor_pidfile`: path to a pidfile that the supervisor will create when booting to prevent running more than one supervisor in the same host, or in case you want to use it for a health check. It's `nil` by default.
 - `preserve_finished_jobs`: whether to keep finished jobs in the `solid_queue_jobs` table—defaults to `true`.
-- `clear_finished_jobs_after`: period to keep finished jobs around, in case `preserve_finished_jobs` is true—defaults to 1 day. **Note:** Right now, there's no automatic cleanup of finished jobs. You'd need to do this by periodically invoking `SolidQueue::Job.clear_finished_in_batches`, but this will happen automatically in the near future.
+- `clear_finished_jobs_after`: period to keep finished jobs around, in case `preserve_finished_jobs` is true — defaults to 1 day. When installing Solid Queue, [a recurring job](#recurring-tasks) is automatically configured to clear finished jobs every hour on the 12th minute in batches. You can edit the `recurring.yml` configuration to change this as you see fit.
 - `default_concurrency_control_period`: the value to be used as the default for the `duration` parameter in [concurrency controls](#concurrency-controls). It defaults to 3 minutes.
 
+
+## Lifecycle hooks
+
+In Solid queue, you can hook into two different points in the supervisor's life:
+- `start`: after the supervisor has finished booting and right before it forks workers and dispatchers.
+- `stop`: after receiving a signal (`TERM`, `INT` or `QUIT`) and right before starting graceful or immediate shutdown.
+
+And into two different points in the worker's, dispatcher's and scheduler's life:
+- `(worker|dispatcher|scheduler)_start`: after the worker/dispatcher/scheduler has finished booting and right before it starts the polling loop or loading the recurring schedule.
+- `(worker|dispatcher|scheduler)_stop`: after receiving a signal (`TERM`, `INT` or `QUIT`) and right before starting graceful or immediate shutdown (which is just `exit!`).
+
+Each of these hooks has an instance of the supervisor/worker/dispatcher/scheduler yielded to the block so that you may read its configuration for logging or metrics reporting purposes.
+
+You can use the following methods with a block to do this:
+```ruby
+SolidQueue.on_start
+SolidQueue.on_stop
+
+SolidQueue.on_worker_start
+SolidQueue.on_worker_stop
+
+SolidQueue.on_dispatcher_start
+SolidQueue.on_dispatcher_stop
+
+SolidQueue.on_scheduler_start
+SolidQueue.on_scheduler_stop
+```
+
+For example:
+```ruby
+SolidQueue.on_start do |supervisor|
+  MyMetricsReporter.process_name = supervisor.name
+
+  start_metrics_server
+end
+
+SolidQueue.on_stop do |_supervisor|
+  stop_metrics_server
+end
+
+SolidQueue.on_worker_start do |worker|
+  MyMetricsReporter.process_name = worker.name
+  MyMetricsReporter.queues = worker.queues.join(',')
+end
+```
+
+These can be called several times to add multiple hooks, but it needs to happen before Solid Queue is started. An initializer would be a good place to do this.
+
+
 ## Errors when enqueuing
 
 Solid Queue will raise a `SolidQueue::Job::EnqueueError` for any Active Record errors that happen when enqueuing a job. The reason for not raising `ActiveJob::EnqueueError` is that this one gets handled by Active Job, causing `perform_later` to return `false` and set `job.enqueue_error`, yielding the job to a block that you need to pass to `perform_later`. This works very well for your own jobs, but makes failure very hard to handle for jobs enqueued by Rails or other gems, such as `Turbo::Streams::BroadcastJob` or `ActiveStorage::AnalyzeJob`, because you don't control the call to `perform_later` in that cases.
@@ -310,11 +433,13 @@ In the case of recurring tasks, if such error is raised when enqueuing the job c
 
 ## Concurrency controls
 
-Solid Queue extends Active Job with concurrency controls, that allows you to limit how many jobs of a certain type or with certain arguments can run at the same time. When limited in this way, jobs will be blocked from running, and they'll stay blocked until another job finishes and unblocks them, or after the set expiry time (concurrency limit's _duration_) elapses. Jobs are never discarded or lost, only blocked.
+Solid Queue extends Active Job with concurrency controls, that allows you to limit how many jobs of a certain type or with certain arguments can run at the same time. When limited in this way, **by default, jobs will be blocked from running**, and they'll stay blocked until another job finishes and unblocks them, or after the set expiry time (concurrency limit's _duration_) elapses.
+
+**Alternatively, jobs can be configured to be discarded instead of blocked**. This means that if a job with certain arguments has already been enqueued, other jobs with the same characteristics (in the same concurrency _class_) won't be enqueued.
 
 ```ruby
 class MyJob < ApplicationJob
-  limits_concurrency to: max_concurrent_executions, key: ->(arg1, arg2, **) { ... }, duration: max_interval_to_guarantee_concurrency_limit, group: concurrency_group
+  limits_concurrency to: max_concurrent_executions, key: ->(arg1, arg2, **) { ... }, duration: max_interval_to_guarantee_concurrency_limit, group: concurrency_group, on_conflict: on_conflict_behaviour
 
   # ...
 ```
@@ -322,10 +447,19 @@ class MyJob < ApplicationJob
 - `to` is `1` by default.
 - `duration` is set to `SolidQueue.default_concurrency_control_period` by default, which itself defaults to `3 minutes`, but that you can configure as well.
 - `group` is used to control the concurrency of different job classes together. It defaults to the job class name.
+- `on_conflict` controls behaviour when enqueuing a job that conflicts with the concurrency limits configured. It can be set to one of the following:
+  - (default) `:block`: the job is blocked and is dispatched when another job completes and unblocks it, or when the duration expires.
+  - `:discard`: the job is discarded. When you choose this option, bear in mind that if a job runs and fails to remove the concurrency lock (or _semaphore_, read below to know more about this), all jobs conflicting with it will be discarded up to the interval defined by `duration` has elapsed.
 
 When a job includes these controls, we'll ensure that, at most, the number of jobs (indicated as `to`) that yield the same `key` will be performed concurrently, and this guarantee will last for `duration` for each job enqueued. Note that there's no guarantee about _the order of execution_, only about jobs being performed at the same time (overlapping).
 
-The concurrency limits use the concept of semaphores when enqueuing, and work as follows: when a job is enqueued, we check if it specifies concurrency controls. If it does, we check the semaphore for the computed concurrency key. If the semaphore is open, we claim it and we set the job as _ready_. Ready means it can be picked up by workers for execution. When the job finishes executing (be it successfully or unsuccessfully, resulting in a failed execution), we signal the semaphore and try to unblock the next job with the same key, if any. Unblocking the next job doesn't mean running that job right away, but moving it from _blocked_ to _ready_. Since something can happen that prevents the first job from releasing the semaphore and unblocking the next job (for example, someone pulling a plug in the machine where the worker is running), we have the `duration` as a failsafe. Jobs that have been blocked for more than duration are candidates to be released, but only as many of them as the concurrency rules allow, as each one would need to go through the semaphore dance check. This means that the `duration` is not really about the job that's enqueued or being run, it's about the jobs that are blocked waiting.
+The concurrency limits use the concept of semaphores when enqueuing, and work as follows: when a job is enqueued, we check if it specifies concurrency controls. If it does, we check the semaphore for the computed concurrency key. If the semaphore is open, we claim it and we set the job as _ready_. Ready means it can be picked up by workers for execution. When the job finishes executing (be it successfully or unsuccessfully, resulting in a failed execution), we signal the semaphore and try to unblock the next job with the same key, if any. Unblocking the next job doesn't mean running that job right away, but moving it from _blocked_ to _ready_. If you're using the `discard` behaviour for `on_conflict`, jobs enqueued while the semaphore is closed will be discarded.
+
+Since something can happen that prevents the first job from releasing the semaphore and unblocking the next job (for example, someone pulling a plug in the machine where the worker is running), we have the `duration` as a failsafe. Jobs that have been blocked for more than `duration` are candidates to be released, but only as many of them as the concurrency rules allow, as each one would need to go through the semaphore dance check. This means that the `duration` is not really about the job that's enqueued or being run, it's about the jobs that are blocked waiting, or about the jobs that would get discarded while the semaphore is closed.
+
+It's important to note that after one or more candidate jobs are unblocked (either because a job finishes or because `duration` expires and a semaphore is released), the `duration` timer for the still blocked jobs is reset. This happens indirectly via the expiration time of the semaphore, which is updated.
+
+When using `discard` as the behaviour to handle conflicts, you might have jobs discarded for up to the `duration` interval if something happens and a running job fails to release the semaphore.
 
 
 For example:
@@ -358,12 +492,63 @@ class Bundle::RebundlePostingsJob < ApplicationJob
 
 In this case, if we have a `Box::MovePostingsByContactToDesignatedBoxJob` job enqueued for a contact record with id `123` and another `Bundle::RebundlePostingsJob` job enqueued simultaneously for a bundle record that references contact `123`, only one of them will be allowed to proceed. The other one will stay blocked until the first one finishes (or 15 minutes pass, whatever happens first).
 
-Note that the `duration` setting depends indirectly on the value for `concurrency_maintenance_interval` that you set for your dispatcher(s), as that'd be the frequency with which blocked jobs are checked and unblocked. In general, you should set `duration` in a way that all your jobs would finish well under that duration and think of the concurrency maintenance task as a failsafe in case something goes wrong.
+Note that the `duration` setting depends indirectly on the value for `concurrency_maintenance_interval` that you set for your dispatcher(s), as that'd be the frequency with which blocked jobs are checked and unblocked (at which point, only one job per concurrency key, at most, is unblocked). In general, you should set `duration` in a way that all your jobs would finish well under that duration and think of the concurrency maintenance task as a failsafe in case something goes wrong.
 
-Jobs are unblocked in order of priority but queue order is not taken into account for unblocking jobs. That means that if you have a group of jobs that share a concurrency group but are in different queues, or jobs of the same class that you enqueue in different queues, the queue order you set for a worker is not taken into account when unblocking blocked ones. The reason is that a job that runs unblocks the next one, and the job itself doesn't know about a particular worker's queue order (you could even have different workers with different queue orders), it can only know about priority. Once blocked jobs are unblocked and available for polling, they'll be picked up by a worker following its queue order.
+Jobs are unblocked in order of priority but **queue order is not taken into account for unblocking jobs**. That means that if you have a group of jobs that share a concurrency group but are in different queues, or jobs of the same class that you enqueue in different queues, the queue order you set for a worker is not taken into account when unblocking blocked ones. The reason is that a job that runs unblocks the next one, and the job itself doesn't know about a particular worker's queue order (you could even have different workers with different queue orders), it can only know about priority. Once blocked jobs are unblocked and available for polling, they'll be picked up by a worker following its queue order.
 
 Finally, failed jobs that are automatically or manually retried work in the same way as new jobs that get enqueued: they get in the queue for getting an open semaphore, and whenever they get it, they'll be run. It doesn't matter if they had already gotten an open semaphore in the past.
 
+### Scheduled jobs
+
+Jobs set to run in the future (via Active Job's `wait` or `wait_until` options) have concurrency limits enforced when they're due, not when they're scheduled. For example, consider this job:
+```ruby
+class DeliverAnnouncementToContactJob < ApplicationJob
+  limits_concurrency to: 1, key: ->(contact) { contact.account }, duration: 5.minutes
+
+  def perform(contact)
+    # ...
+```
+
+If several jobs are enqueued like this:
+
+```ruby
+DeliverAnnouncementToContactJob.set(wait: 10.minutes).perform_later(contact)
+DeliverAnnouncementToContactJob.set(wait: 10.minutes).perform_later(contact)
+DeliverAnnouncementToContactJob.set(wait: 30.minutes).perform_later(contact)
+```
+
+The 3 jobs will go into the scheduled queue and will wait there until they're due. Then, 10 minutes after, the first two jobs will be enqueued and the second one most likely will be blocked because the first one will be running first. Then, assuming the jobs are fast and finish in a few seconds, when the third job is due, it'll be enqueued normally.
+
+Normally scheduled jobs are enqueued in batches, but with concurrency controls, jobs need to be enqueued one by one. This has an impact on performance, similarly to the impact of concurrency controls in bulk enqueuing. Read below for more details. I'd generally advise against mixing concurrency controls with waiting/scheduling in the future.
+
+### Performance considerations
+
+Concurrency controls introduce significant overhead (blocked executions need to be created and promoted to ready, semaphores need to be created and updated) so you should consider carefully whether you need them. For throttling purposes, where you plan to have `limit` significantly larger than 1, I'd encourage relying on a limited number of workers per queue instead. For example:
+
+```ruby
+class ThrottledJob < ApplicationJob
+  queue_as :throttled
+```
+
+```yml
+production:
+  workers:
+    - queues: throttled
+      threads: 1
+      polling_interval: 1
+    - queues: default
+      threads: 5
+      polling_interval: 0.1
+      processes: 3
+```
+
+Or something similar to that depending on your setup. You can also assign a different queue to a job on the moment of enqueuing so you can decide whether to enqueue a job in the throttled queue or another queue depending on the arguments, or pass a block to `queue_as` as explained [here](https://guides.rubyonrails.org/active_job_basics.html#queues).
+
+
+In addition, mixing concurrency controls with **bulk enqueuing** (Active Job's `perform_all_later`) is not a good idea because concurrency controlled job needs to be enqueued one by one to ensure concurrency limits are respected, so you lose all the benefits of bulk enqueuing.
+
+When jobs that have concurrency controls and `on_conflict: :discard` are enqueued in bulk, the ones that fail to be enqueued and are discarded would have `successfully_enqueued` set to `false`. The total count of jobs enqueued returned by `perform_all_later` will exclude these jobs as expected.
+
 ## Failed jobs and retries
 
 Solid Queue doesn't include any automatic retry mechanism, it [relies on Active Job for this](https://edgeguides.rubyonrails.org/active_job_basics.html#retrying-or-discarding-failed-jobs). Jobs that fail will be kept in the system, and a _failed execution_ (a record in the `solid_queue_failed_executions` table) will be created for these. The job will stay there until manually discarded or re-enqueued. You can do this in a console as:
@@ -375,8 +560,6 @@ failed_execution.retry # This will re-enqueue the job as if it was enqueued for
 failed_execution.discard # This will delete the job from the system
 ```
 
-However, we recommend taking a look at [mission_control-jobs](https://github.com/rails/mission_control-jobs), a dashboard where, among other things, you can examine and retry/discard failed jobs.
-
 ### Error reporting on jobs
 
 Some error tracking services that integrate with Rails, such as Sentry or Rollbar, hook into [Active Job](https://guides.rubyonrails.org/active_job_basics.html#exceptions) and automatically report not handled errors that happen during job execution. However, if your error tracking system doesn't, or if you need some custom reporting, you can hook into Active Job yourself. A possible way of doing this would be:
@@ -412,6 +595,13 @@ plugin :solid_queue
 ```
 to your `puma.rb` configuration.
 
+If you're using Puma in development but you don't want to use Solid Queue in development, make sure you avoid the plugin being used, for example using an environment variable like this:
+```ruby
+plugin :solid_queue if ENV["SOLID_QUEUE_IN_PUMA"]
+```
+that you set in production only. This is what Rails 8's default Puma config looks like. Otherwise, if you're using Puma in development but not Solid Queue, starting Puma would start also Solid Queue supervisor and it'll most likely fail because it won't be properly configured.
+
+**Note**: phased restarts are not supported currently because the plugin requires [app preloading](https://github.com/puma/puma?tab=readme-ov-file#cluster-mode) to work.
 
 ## Jobs and transactional integrity
 :warning: Having your jobs in the same ACID-compliant database as your application data enables a powerful yet sharp tool: taking advantage of transactional integrity to ensure some action in your app is not committed unless your job is also committed and vice versa, and ensuring that your job won't be enqueued until the transaction within which you're enqueuing it is committed. This can be very powerful and useful, but it can also backfire if you base some of your logic on this behaviour, and in the future, you move to another active job backend, or if you simply move Solid Queue to its own database, and suddenly the behaviour changes under you. Because this can be quite tricky and many people shouldn't need to worry about it, by default Solid Queue is configured in a different database as the main app.
@@ -450,6 +640,8 @@ Solid Queue supports defining recurring tasks that run at specific times in the
 bin/jobs --recurring_schedule_file=config/schedule.yml
 ```
 
+You can completely disable recurring tasks by setting the environment variable `SOLID_QUEUE_SKIP_RECURRING=true` or by using the `--skip-recurring` option with `bin/jobs`.
+
 The configuration itself looks like this:
 
 ```yml
@@ -477,9 +669,15 @@ MyJob.perform_later(42, status: "custom_status")
 
 - `priority`: a numeric priority value to be used when enqueuing the job.
 
-
 Tasks are enqueued at their corresponding times by the scheduler, and each task schedules the next one. This is pretty much [inspired by what GoodJob does](https://github.com/bensheldon/good_job/blob/994ecff5323bf0337e10464841128fda100750e6/lib/good_job/cron_manager.rb).
 
+For recurring tasks defined as a `command`, you can also change the job class that runs them as follows:
+```ruby
+Rails.application.config.after_initialize do # or to_prepare
+  SolidQueue::RecurringTask.default_job_class = MyRecurringCommandJob
+end
+```
+
 It's possible to run multiple schedulers with the same `recurring_tasks` configuration, for example, if you have multiple servers for redundancy, and you run the `scheduler` in more than one of them. To avoid enqueuing duplicate tasks at the same time, an entry in a new `solid_queue_recurring_executions` table is created in the same transaction as the job is enqueued. This table has a unique index on `task_key` and `run_at`, ensuring only one entry per task per time will be created. This only works if you have `preserve_finished_jobs` set to `true` (the default), and the guarantee applies as long as you keep the jobs around.
 
 **Note**: a single recurring schedule is supported, so you can have multiple schedulers using the same schedule, but not multiple schedulers using different configurations.

data/Rakefile

@@ -8,14 +8,36 @@ load "rails/tasks/engine.rake"
 load "rails/tasks/statistics.rake"
 
 require "bundler/gem_tasks"
+require "rake/tasklib"
 
-def databases
-  %w[ mysql postgres sqlite ]
-end
+class TestHelpers < Rake::TaskLib
+  def initialize(databases)
+    @databases = databases
+    define
+  end
 
-task :test do
-  databases.each do |database|
+  def define
+    desc "Run tests for all databases (mysql, postgres, sqlite)"
+    task :test do
+      @databases.each { |database| run_test_for_database(database) }
+    end
+
+    namespace :test do
+      @databases.each do |database|
+        desc "Run tests for #{database} database"
+        task database do
+          run_test_for_database(database)
+        end
+      end
+    end
+  end
+
+  private
+
+  def run_test_for_database(database)
     sh("TARGET_DB=#{database} bin/setup")
     sh("TARGET_DB=#{database} bin/rails test")
   end
 end
+
+TestHelpers.new(%w[ mysql postgres sqlite ])

data/app/models/solid_queue/blocked_execution.rb

@@ -12,7 +12,7 @@ module SolidQueue
     class << self
       def unblock(limit)
         SolidQueue.instrument(:release_many_blocked, limit: limit) do |payload|
-          expired.distinct.limit(limit).pluck(:concurrency_key).then do |concurrency_keys|
+          expired.order(:concurrency_key).distinct.limit(limit).pluck(:concurrency_key).then do |concurrency_keys|
             payload[:size] = release_many releasable(concurrency_keys)
           end
         end

data/app/models/solid_queue/claimed_execution.rb

@@ -39,7 +39,10 @@ class SolidQueue::ClaimedExecution < SolidQueue::Execution
     def fail_all_with(error)
       SolidQueue.instrument(:fail_many_claimed) do |payload|
         includes(:job).tap do |executions|
-          executions.each { |execution| execution.failed_with(error) }
+          executions.each do |execution|
+            execution.failed_with(error)
+            execution.unblock_next_job
+          end
 
           payload[:process_ids] = executions.map(&:process_id).uniq
           payload[:job_ids] = executions.map(&:job_id).uniq
@@ -67,7 +70,7 @@ class SolidQueue::ClaimedExecution < SolidQueue::Execution
       raise result.error
     end
   ensure
-    job.unblock_next_blocked_job
+    unblock_next_job
   end
 
   def release
@@ -90,9 +93,13 @@ class SolidQueue::ClaimedExecution < SolidQueue::Execution
     end
   end
 
+  def unblock_next_job
+    job.unblock_next_blocked_job
+  end
+
   private
     def execute
-      ActiveJob::Base.execute(job.arguments)
+      ActiveJob::Base.execute(job.arguments.merge("provider_job_id" => job.id))
       Result.new(true, nil)
     rescue Exception => e
       Result.new(false, e)

data/app/models/solid_queue/job/clearable.rb

@@ -10,9 +10,10 @@ module SolidQueue
       end
 
       class_methods do
-        def clear_finished_in_batches(batch_size: 500, finished_before: SolidQueue.clear_finished_jobs_after.ago, class_name: nil)
+        def clear_finished_in_batches(batch_size: 500, finished_before: SolidQueue.clear_finished_jobs_after.ago, class_name: nil, sleep_between_batches: 0)
           loop do
             records_deleted = clearable(finished_before: finished_before, class_name: class_name).limit(batch_size).delete_all
+            sleep(sleep_between_batches) if sleep_between_batches > 0
             break if records_deleted == 0
           end
         end

data/app/models/solid_queue/job/concurrency_controls.rb

@@ -34,6 +34,10 @@ module SolidQueue
       end
 
       private
+        def concurrency_on_conflict
+          job_class.concurrency_on_conflict.to_s.inquiry
+        end
+
         def acquire_concurrency_lock
           return true unless concurrency_limited?
 
@@ -46,6 +50,14 @@ module SolidQueue
           Semaphore.signal(self)
         end
 
+        def handle_concurrency_conflict
+          if concurrency_on_conflict.discard?
+            destroy
+          else
+            block
+          end
+        end
+
         def block
           BlockedExecution.create_or_find_by!(job_id: id)
         end

data/app/models/solid_queue/job/executable.rb

@@ -67,7 +67,7 @@ module SolidQueue
       def dispatch
         if acquire_concurrency_lock then ready
         else
-          block
+          handle_concurrency_conflict
         end
       end
 

data/app/models/solid_queue/job.rb

@@ -29,7 +29,8 @@ module SolidQueue
         active_job.scheduled_at = scheduled_at
 
         create_from_active_job(active_job).tap do |enqueued_job|
-          active_job.provider_job_id = enqueued_job.id
+          active_job.provider_job_id = enqueued_job.id if enqueued_job.persisted?
+          active_job.successfully_enqueued = enqueued_job.persisted?
         end
       end
 
@@ -49,7 +50,7 @@ module SolidQueue
         def create_all_from_active_jobs(active_jobs)
           job_rows = active_jobs.map { |job| attributes_from_active_job(job) }
           insert_all(job_rows)
-          where(active_job_id: active_jobs.map(&:job_id))
+          where(active_job_id: active_jobs.map(&:job_id)).order(id: :asc)
         end
 
         def attributes_from_active_job(active_job)

data/app/models/solid_queue/recurring_task.rb

@@ -12,6 +12,8 @@ module SolidQueue
 
     scope :static, -> { where(static: true) }
 
+    has_many :recurring_executions, foreign_key: :task_key, primary_key: :key
+
     mattr_accessor :default_job_class
     self.default_job_class = RecurringJob
 
@@ -53,6 +55,18 @@ module SolidQueue
       parsed_schedule.next_time.utc
     end
 
+    def previous_time
+      parsed_schedule.previous_time.utc
+    end
+
+    def last_enqueued_time
+      if recurring_executions.loaded?
+        recurring_executions.map(&:run_at).max
+      else
+        recurring_executions.maximum(:run_at)
+      end
+    end
+
     def enqueue(at:)
       SolidQueue.instrument(:enqueue_recurring_task, task: key, at: at) do |payload|
         active_job = if using_solid_queue_adapter?
@@ -116,7 +130,6 @@ module SolidQueue
             active_job.run_callbacks(:enqueue) do
               Job.enqueue(active_job)
             end
-            active_job.successfully_enqueued = true
           end
         end
       end

data/app/models/solid_queue/scheduled_execution.rb

@@ -14,7 +14,7 @@ module SolidQueue
       def dispatch_next_batch(batch_size)
         transaction do
           job_ids = next_batch(batch_size).non_blocking_lock.pluck(:job_id)
-          if job_ids.empty? then []
+          if job_ids.empty? then 0
           else
             SolidQueue.instrument(:dispatch_scheduled, batch_size: batch_size) do |payload|
               payload[:size] = dispatch_jobs(job_ids)