good_job 3.1.0 → 3.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bc6e3cffa2189a101181b7ceb278de97ded27d14dd6da915ea8fe3729907dd88
-  data.tar.gz: 0504d2f2efd09d1d52669597e364f62b69edc739d3c76b8bcfdd1933ea7cb8da
+  metadata.gz: 9ecbb75f2764a72686acb96367389abb1900321b5cf1292d6522aa8094c68451
+  data.tar.gz: 43a5305a9f2610c1544d6e52c61f0b338771b07d92d6493fed0058ad33bbc045
 SHA512:
-  metadata.gz: c10f675a3635f481e370502828ed0902aff3464e19f161bf5f300463a2671ecac09643eb0943030eda30163171b67986b0f9e31ca423ad5da9e4ffd5b23a7df7
-  data.tar.gz: 3ab12e91adb910040a4007140054cc048a0f83e34e54ee649e52ea673a27b95cfb267b56fae98a7605616e4c3f55d17af403c99bf9ba2a5d504dffdd917a7cbb
+  metadata.gz: 0e637579c190a3f3d1aba6c02f6a71e2193d9e5c4bb449a3d4d9571093150a25a1c293ac3f5c541d2fae09aeffff771692fa7ef58bc8ed672f61a2c9d35a7e74
+  data.tar.gz: bc669b4d1628f7ff4b885787c65a38a57ad020aa168c3ae85f0c6bb26c9f0e19b64947682b0c99cf3eb28c2d660e1d79ee6095000a72340f09229d1679fa5dfe

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## [v3.2.0](https://github.com/bensheldon/good_job/tree/v3.2.0) (2022-07-12)
+
+[Full Changelog](https://github.com/bensheldon/good_job/compare/v3.1.0...v3.2.0)
+
+**Merged pull requests:**
+
+- Ordered queue handling by workers [\#665](https://github.com/bensheldon/good_job/pull/665) ([jrochkind](https://github.com/jrochkind))
+
 ## [v3.1.0](https://github.com/bensheldon/good_job/tree/v3.1.0) (2022-07-11)
 
 [Full Changelog](https://github.com/bensheldon/good_job/compare/v3.0.2...v3.1.0)

data/README.md

@@ -665,7 +665,7 @@ By default, GoodJob creates a single thread execution pool that will execute job
 
     A pool is configured with the following syntax `<participating_queues>:<thread_count>`:
 
-    - `<participating_queues>`: either `queue1,queue2` (only those queues), `*` (all) or `-queue1,queue2` (all except those queues).
+    - `<participating_queues>`: either `queue1,queue2` (only those queues), `+queue1,queue2` (only those queues, and processed in order), `*` (all) or `-queue1,queue2` (all except those queues).
     - `<thread_count>`: a count overriding for this specific pool the global `max-threads`.
 
     Pool configurations are separated with a semicolon (;) in the `queues` configuration
@@ -683,6 +683,12 @@ By default, GoodJob creates a single thread execution pool that will execute job
     - `-transactional_messages,batch_processing`: execute jobs enqueued on _any_ queue _excluding_ `transactional_messages` or `batch_processing`, with up to 2 threads.
     - `*`: execute jobs on any queue, with up to 5 threads (as configured by `--max-threads=5`).
 
+    When a pool is performing jobs from multiple queues, jobs will be performed from specified queues, ordered by priority and creation time. To perform jobs from queues in the queues' given order, use the `+` modifier. In this example, jobs in `batch_processing` will be performed only when there are no jobs in `transactional_messages`:
+
+    ```bash
+    bundle exec good_job --queues="+transactional_messages,batch_processing"
+    ```
+
     Configuration can be injected by environment variables too:
 
     ```bash

data/lib/good_job/job_performer.rb

@@ -24,7 +24,7 @@ module GoodJob
     # Perform the next eligible job
     # @return [Object, nil] Returns job result or +nil+ if no job was found
     def next
-      job_query.perform_with_advisory_lock
+      job_query.perform_with_advisory_lock(parsed_queues: parsed_queues)
     end
 
     # Tests whether this performer should be used in GoodJob's current state.

data/lib/good_job/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 module GoodJob
   # GoodJob gem version.
-  VERSION = '3.1.0'
+  VERSION = '3.2.0'
 end

data/lib/models/good_job/execution.rb

@@ -29,15 +29,21 @@ module GoodJob
     #     not match.
     #   - +{ include: Array<String> }+ indicates the listed queue names should
     #     match.
+    #   - +{ include: Array<String>, ordered_queues: true }+ indicates the listed
+    #     queue names should match, and dequeue should respect queue order.
     # @example
     #   GoodJob::Execution.queue_parser('-queue1,queue2')
     #   => { exclude: [ 'queue1', 'queue2' ] }
     def self.queue_parser(string)
       string = string.presence || '*'
 
-      if string.first == '-'
+      case string.first
+      when '-'
         exclude_queues = true
         string = string[1..-1]
+      when '+'
+        ordered_queues = true
+        string = string[1..-1]
       end
 
       queues = string.split(',').map(&:strip)
@@ -46,6 +52,11 @@ module GoodJob
         { all: true }
       elsif exclude_queues
         { exclude: queues }
+      elsif ordered_queues
+        {
+          include: queues,
+          ordered_queues: true,
+        }
       else
         { include: queues }
       end
@@ -95,10 +106,34 @@ module GoodJob
     scope :creation_ordered, -> { order('created_at ASC') }
 
     # Order jobs for de-queueing
-    # @!method dequeue_ordered
+    # @!method dequeueing_ordered
     # @!scope class
-    # @return [ActiveRecord:Relation]
-    scope :dequeue_ordered, -> { priority_ordered.creation_ordered }
+    # @param parsed_queues [Hash]
+    #   optional output of .queue_parser, parsed queues, will be used for
+    #   ordered queues.
+    # @return [ActiveRecord::Relation]
+    scope :dequeueing_ordered, (lambda do |parsed_queues|
+      relation = self
+      relation = relation.queue_ordered(parsed_queues[:include]) if parsed_queues && parsed_queues[:ordered_queues] && parsed_queues[:include]
+      relation = relation.priority_ordered.creation_ordered
+
+      relation
+    end)
+
+    # Order jobs in order of queues in array param
+    # @!method queue_ordered
+    # @!scope class
+    # @param queues [Array<string] ordered names of queues
+    # @return [ActiveRecord::Relation]
+    scope :queue_ordered, (lambda do |queues|
+      clauses = queues.map.with_index do |queue_name, index|
+        "WHEN queue_name = '#{queue_name}' THEN #{index}"
+      end
+
+      order(
+        Arel.sql("(CASE #{clauses.join(' ')} ELSE #{queues.length} END)")
+      )
+    end)
 
     # Order jobs by scheduled or created (oldest first).
     # @!method schedule_ordered
@@ -165,8 +200,8 @@ module GoodJob
     #   return value for the job's +#perform+ method, and the exception the job
     #   raised, if any (if the job raised, then the second array entry will be
     #   +nil+). If there were no jobs to execute, returns +nil+.
-    def self.perform_with_advisory_lock
-      unfinished.dequeue_ordered.only_scheduled.limit(1).with_advisory_lock(unlock_session: true) do |executions|
+    def self.perform_with_advisory_lock(parsed_queues: nil)
+      unfinished.dequeueing_ordered(parsed_queues).only_scheduled.limit(1).with_advisory_lock(unlock_session: true) do |executions|
         execution = executions.first
         break if execution.blank?
         break :unlocked unless execution&.executable?

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: good_job
 version: !ruby/object:Gem::Version
-  version: 3.1.0
+  version: 3.2.0
 platform: ruby
 authors:
 - Ben Sheldon
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-07-11 00:00:00.000000000 Z
+date: 2022-07-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activejob