riverqueue 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8aa1b2a14e085df2b2a7e79fa6c6cd5b19aeb2c893a4fd369b99e469dd5916b9
-  data.tar.gz: 3cebd975dcadb223ecc163918b2e8bb0ce67bb671be99c6f706d3d9ec2da6507
+  metadata.gz: 46f45a18aa1b8884c93295ba1b023751c826d8feeac212949e18f388f3a6b6cc
+  data.tar.gz: 679dde3151af410e5e8f8949d80750d68f247e300af1e6f93b43270d81dc5d40
 SHA512:
-  metadata.gz: 868459a39a19fe9aacec7552b03148d97228ff58bdfb6b79a6e724c800cc97ef10017097a46f5132684d5d4e97ea9c788b83c813dcb6dc812835e9c0c9ec9645
-  data.tar.gz: '05538541ee6466adc722af9d6ab4cbf4c1a081a59c9eb887089b8abea8474415d61dfa9fdf827e4863ee393874258f09e6c54b06ffac87ddc90931f346f12f0b'
+  metadata.gz: a7507d5e78f99ef0bb6cda3a91479da386bb5e66c18c40a26c98ed5aec42bb3b5425b3753e797759e4ca4117f9a01636d9784fd1b27d160af215e5e5216ac998
+  data.tar.gz: a65b3a772b4d85b823256fe2bebb656a816cff1ba87777c77d9393670dbdfe3be383095bb49f3446d8110a143cb1ba68b9c7530d64da44a9e9093265be9aa8ba

data/lib/client.rb

@@ -1,5 +1,4 @@
 require "digest"
-require "fnv"
 require "time"
 
 module River
@@ -25,9 +24,8 @@ module River
   # River drivers are found in separate gems like `riverqueue-sequel` to help
   # minimize transient dependencies.
   class Client
-    def initialize(driver, advisory_lock_prefix: nil)
+    def initialize(driver)
       @driver = driver
-      @advisory_lock_prefix = check_advisory_lock_prefix_bounds(advisory_lock_prefix)
       @time_now_utc = -> { Time.now.utc } # for test time stubbing
     end
 
@@ -73,11 +71,8 @@ module River
     #
     # Returns an instance of InsertResult.
     def insert(args, insert_opts: EMPTY_INSERT_OPTS)
-      insert_params, unique_opts = make_insert_params(args, insert_opts)
-      check_unique_job(insert_params, unique_opts) do
-        job = @driver.job_insert(insert_params)
-        InsertResult.new(job)
-      end
+      insert_params = make_insert_params(args, insert_opts)
+      insert_and_check_unique_job(insert_params)
     end
 
     # Inserts many new jobs as part of a single batch operation for improved
@@ -122,30 +117,20 @@ module River
     #
     # See also JobArgsHash for an easy way to insert a job from a hash.
     #
-    # Unique job insertion isn't supported with bulk insertion because it'd run
-    # the risk of major lock contention.
-    #
     # Returns the number of jobs inserted.
     def insert_many(args)
       all_params = args.map do |arg|
         if arg.is_a?(InsertManyParams)
-          make_insert_params(arg.args, arg.insert_opts || EMPTY_INSERT_OPTS, is_insert_many: true).first # unique opts ignored on batch insert
+          make_insert_params(arg.args, arg.insert_opts || EMPTY_INSERT_OPTS)
         else # jobArgs
-          make_insert_params(arg, EMPTY_INSERT_OPTS, is_insert_many: true).first # unique opts ignored on batch insert
+          make_insert_params(arg, EMPTY_INSERT_OPTS)
         end
       end
 
       @driver.job_insert_many(all_params)
-    end
-
-    private def check_advisory_lock_prefix_bounds(advisory_lock_prefix)
-      return nil if advisory_lock_prefix.nil?
-
-      # 2**32-1 is 0xffffffff (the largest number that's four bytes)
-      if advisory_lock_prefix < 0 || advisory_lock_prefix > 2**32 - 1
-        raise ArgumentError, "advisory lock prefix must fit inside four bytes"
+        .map do |job, unique_skipped_as_duplicate|
+        InsertResult.new(job, unique_skipped_as_duplicated: unique_skipped_as_duplicate)
       end
-      advisory_lock_prefix
     end
 
     # Default states that are used during a unique insert. Can be overridden by
@@ -153,123 +138,101 @@ module River
     DEFAULT_UNIQUE_STATES = [
       JOB_STATE_AVAILABLE,
       JOB_STATE_COMPLETED,
+      JOB_STATE_PENDING,
       JOB_STATE_RETRYABLE,
       JOB_STATE_RUNNING,
       JOB_STATE_SCHEDULED
     ].freeze
     private_constant :DEFAULT_UNIQUE_STATES
 
+    REQUIRED_UNIQUE_STATES = [
+      JOB_STATE_AVAILABLE,
+      JOB_STATE_PENDING,
+      JOB_STATE_RUNNING,
+      JOB_STATE_SCHEDULED
+    ].freeze
+    private_constant :REQUIRED_UNIQUE_STATES
+
     EMPTY_INSERT_OPTS = InsertOpts.new.freeze
     private_constant :EMPTY_INSERT_OPTS
 
-    private def check_unique_job(insert_params, unique_opts, &block)
-      return block.call if unique_opts.nil?
-
-      any_unique_opts = false
-      get_params = Driver::JobGetByKindAndUniquePropertiesParam.new(kind: insert_params.kind)
-      unique_key = ""
+    private def insert_and_check_unique_job(insert_params)
+      job, unique_skipped_as_duplicate = @driver.job_insert(insert_params)
+      InsertResult.new(job, unique_skipped_as_duplicated: unique_skipped_as_duplicate)
+    end
 
-      # It's extremely important here that this lock string format and algorithm
-      # match the one in the main River library _exactly_. Don't change them
-      # unless they're updated everywhere.
-      if unique_opts.by_args
-        any_unique_opts = true
-        get_params.encoded_args = insert_params.encoded_args
-        unique_key += "&args=#{insert_params.encoded_args}"
-      end
+    private def make_insert_params(args, insert_opts)
+      raise "args should respond to `#kind`" if !args.respond_to?(:kind)
 
-      if unique_opts.by_period
-        lower_period_bound = truncate_time(@time_now_utc.call, unique_opts.by_period).utc
+      # ~all objects in Ruby respond to `#to_json`, so check non-nil instead.
+      args_json = args.to_json
+      raise "args should return non-nil from `#to_json`" if !args_json
 
-        any_unique_opts = true
-        get_params.created_at = [lower_period_bound, lower_period_bound + unique_opts.by_period]
-        unique_key += "&period=#{lower_period_bound.strftime("%FT%TZ")}"
+      args_insert_opts = if args.respond_to?(:insert_opts)
+        args_with_insert_opts = args #: _JobArgsWithInsertOpts # rubocop:disable Layout/LeadingCommentSpace
+        args_with_insert_opts.insert_opts || EMPTY_INSERT_OPTS
+      else
+        EMPTY_INSERT_OPTS
       end
 
-      if unique_opts.by_queue
-        any_unique_opts = true
-        get_params.queue = insert_params.queue
-        unique_key += "&queue=#{insert_params.queue}"
-      end
+      scheduled_at = insert_opts.scheduled_at || args_insert_opts.scheduled_at
 
-      if unique_opts.by_state
-        any_unique_opts = true
-        get_params.state = unique_opts.by_state
-        unique_key += "&state=#{unique_opts.by_state.join(",")}"
-      else
-        get_params.state = DEFAULT_UNIQUE_STATES
-        unique_key += "&state=#{DEFAULT_UNIQUE_STATES.join(",")}"
+      insert_params = Driver::JobInsertParams.new(
+        encoded_args: args_json,
+        kind: args.kind,
+        max_attempts: insert_opts.max_attempts || args_insert_opts.max_attempts || MAX_ATTEMPTS_DEFAULT,
+        priority: insert_opts.priority || args_insert_opts.priority || PRIORITY_DEFAULT,
+        queue: insert_opts.queue || args_insert_opts.queue || QUEUE_DEFAULT,
+        scheduled_at: scheduled_at&.utc || Time.now,
+        state: scheduled_at ? JOB_STATE_SCHEDULED : JOB_STATE_AVAILABLE,
+        tags: validate_tags(insert_opts.tags || args_insert_opts.tags || [])
+      )
+
+      unique_opts = insert_opts.unique_opts || args_insert_opts.unique_opts
+      if unique_opts
+        unique_key, unique_states = make_unique_key_and_bitmask(insert_params, unique_opts)
+        insert_params.unique_key = unique_key
+        insert_params.unique_states = unique_states
       end
+      insert_params
+    end
 
-      return block.call unless any_unique_opts
+    private def make_unique_key_and_bitmask(insert_params, unique_opts)
+      unique_key = ""
 
-      # fast path
-      if !unique_opts.by_state || unique_opts.by_state.sort == DEFAULT_UNIQUE_STATES
-        unique_key_hash = Digest::SHA256.digest(unique_key)
-        job, unique_skipped_as_duplicate = @driver.job_insert_unique(insert_params, unique_key_hash)
-        return InsertResult.new(job, unique_skipped_as_duplicated: unique_skipped_as_duplicate)
+      # It's extremely important here that this unique key format and algorithm
+      # match the one in the main River library _exactly_. Don't change them
+      # unless they're updated everywhere.
+      unless unique_opts.exclude_kind
+        unique_key += "&kind=#{insert_params.kind}"
       end
 
-      @driver.transaction do
-        lock_str = "unique_key"
-        lock_str += "kind=#{insert_params.kind}"
-        lock_str += unique_key
-
-        lock_key = if @advisory_lock_prefix.nil?
-          FNV.fnv1_hash(lock_str, size: 64)
+      if unique_opts.by_args
+        parsed_args = JSON.parse(insert_params.encoded_args)
+        filtered_args = if unique_opts.by_args.is_a?(Array)
+          parsed_args.select { |k, _| unique_opts.by_args.include?(k) }
         else
-          # Steep should be able to tell that this is not nil, but it can't.
-          prefix = @advisory_lock_prefix #: Integer # rubocop:disable Layout/LeadingCommentSpace
-          prefix << 32 | FNV.fnv1_hash(lock_str, size: 32)
+          parsed_args
         end
 
-        # Packs a uint64 then unpacks to int64, which we need to do to keep the
-        # value within the bounds of Postgres' bigint. Overflow is okay because
-        # we can use the full bigint space (including negative numbers) for the
-        # advisory lock.
-        lock_key = uint64_to_int64(lock_key)
-
-        @driver.advisory_lock(lock_key)
-
-        existing_job = @driver.job_get_by_kind_and_unique_properties(get_params)
-        return InsertResult.new(existing_job, unique_skipped_as_duplicated: true) if existing_job
-
-        block.call
+        encoded_args = JSON.generate(filtered_args.sort.to_h)
+        unique_key += "&args=#{encoded_args}"
       end
-    end
 
-    private def make_insert_params(args, insert_opts, is_insert_many: false)
-      raise "args should respond to `#kind`" if !args.respond_to?(:kind)
+      if unique_opts.by_period
+        lower_period_bound = truncate_time(@time_now_utc.call, unique_opts.by_period).utc
 
-      # ~all objects in Ruby respond to `#to_json`, so check non-nil instead.
-      args_json = args.to_json
-      raise "args should return non-nil from `#to_json`" if !args_json
+        unique_key += "&period=#{lower_period_bound.strftime("%FT%TZ")}"
+      end
 
-      args_insert_opts = if args.respond_to?(:insert_opts)
-        args_with_insert_opts = args #: _JobArgsWithInsertOpts # rubocop:disable Layout/LeadingCommentSpace
-        args_with_insert_opts.insert_opts || EMPTY_INSERT_OPTS
-      else
-        EMPTY_INSERT_OPTS
+      if unique_opts.by_queue
+        unique_key += "&queue=#{insert_params.queue}"
       end
 
-      scheduled_at = insert_opts.scheduled_at || args_insert_opts.scheduled_at
-      unique_opts = insert_opts.unique_opts || args_insert_opts.unique_opts
+      unique_key_hash = Digest::SHA256.digest(unique_key)
+      unique_states = validate_unique_states(unique_opts.by_state || DEFAULT_UNIQUE_STATES)
 
-      raise ArgumentError, "unique opts can't be used with `#insert_many`" if is_insert_many && unique_opts
-
-      [
-        Driver::JobInsertParams.new(
-          encoded_args: args_json,
-          kind: args.kind,
-          max_attempts: insert_opts.max_attempts || args_insert_opts.max_attempts || MAX_ATTEMPTS_DEFAULT,
-          priority: insert_opts.priority || args_insert_opts.priority || PRIORITY_DEFAULT,
-          queue: insert_opts.queue || args_insert_opts.queue || QUEUE_DEFAULT,
-          scheduled_at: scheduled_at&.utc, # database defaults to now
-          state: scheduled_at ? JOB_STATE_SCHEDULED : JOB_STATE_AVAILABLE,
-          tags: validate_tags(insert_opts.tags || args_insert_opts.tags)
-        ),
-        unique_opts
-      ]
+      [unique_key_hash, UniqueBitmask.from_states(unique_states)]
     end
 
     # Truncates the given time down to the interval. For example:
@@ -290,11 +253,18 @@ module River
     private_constant :TAG_RE
 
     private def validate_tags(tags)
-      tags&.each do |tag|
+      tags.each do |tag|
         raise ArgumentError, "tags should be 255 characters or less" if tag.length > 255
         raise ArgumentError, "tag should match regex #{TAG_RE.inspect}" unless TAG_RE.match(tag)
       end
     end
+
+    private def validate_unique_states(states)
+      REQUIRED_UNIQUE_STATES.each do |required_state|
+        raise ArgumentError, "by_state should include required state #{required_state}" unless states.include?(required_state)
+      end
+      states
+    end
   end
 
   # A single job to insert that's part of an #insert_many batch insert. Unlike
@@ -322,7 +292,7 @@ module River
     # job matching unique property already being present.
     attr_reader :unique_skipped_as_duplicated
 
-    def initialize(job, unique_skipped_as_duplicated: false)
+    def initialize(job, unique_skipped_as_duplicated:)
       @job = job
       @unique_skipped_as_duplicated = unique_skipped_as_duplicated
     end

data/lib/driver.rb

@@ -4,29 +4,6 @@ module River
   # considered to be for internal use only and subject to change. API stability
   # is not guaranteed.
   module Driver
-    # Parameters for looking up a job by kind and unique properties.
-    class JobGetByKindAndUniquePropertiesParam
-      attr_accessor :created_at
-      attr_accessor :encoded_args
-      attr_accessor :kind
-      attr_accessor :queue
-      attr_accessor :state
-
-      def initialize(
-        kind:,
-        created_at: nil,
-        encoded_args: nil,
-        queue: nil,
-        state: nil
-      )
-        self.kind = kind
-        self.created_at = created_at
-        self.encoded_args = encoded_args
-        self.queue = queue
-        self.state = state
-      end
-    end
-
     # Insert parameters for a job. This is sent to underlying drivers and is meant
     # for internal use only. Its interface is subject to change.
     class JobInsertParams
@@ -38,6 +15,8 @@ module River
       attr_accessor :scheduled_at
       attr_accessor :state
       attr_accessor :tags
+      attr_accessor :unique_key
+      attr_accessor :unique_states
 
       def initialize(
         encoded_args:,
@@ -47,7 +26,9 @@ module River
         queue:,
         scheduled_at:,
         state:,
-        tags:
+        tags:,
+        unique_key: nil,
+        unique_states: nil
       )
         self.encoded_args = encoded_args
         self.kind = kind
@@ -57,6 +38,8 @@ module River
         self.scheduled_at = scheduled_at
         self.state = state
         self.tags = tags
+        self.unique_key = unique_key
+        self.unique_states = unique_states
       end
     end
   end

data/lib/insert_opts.rb

@@ -114,23 +114,33 @@ module River
     # Unlike other unique options, ByState gets a default when it's not set for
     # user convenience. The default is equivalent to:
     #
-    #   by_state: [River::JOB_STATE_AVAILABLE, River::JOB_STATE_COMPLETED, River::JOB_STATE_RUNNING, River::JOB_STATE_RETRYABLE, River::JOB_STATE_SCHEDULED]
+    #   by_state: [River::JOB_STATE_AVAILABLE, River::JOB_STATE_COMPLETED, River::JOB_STATE_PENDING, River::JOB_STATE_RUNNING, River::JOB_STATE_RETRYABLE, River::JOB_STATE_SCHEDULED]
     #
     # With this setting, any jobs of the same kind that have been completed or
     # discarded, but not yet cleaned out by the system, won't count towards the
     # uniqueness of a new insert.
+    #
+    # The pending, scheduled, available, and running states are required when
+    # customizing this list.
     attr_accessor :by_state
 
+    # Indicates that the job kind should not be considered for uniqueness. This
+    # is useful when you want to enforce uniqueness based on other properties
+    # across multiple worker types.
+    attr_accessor :exclude_kind
+
     def initialize(
       by_args: nil,
       by_period: nil,
       by_queue: nil,
-      by_state: nil
+      by_state: nil,
+      exclude_kind: nil
     )
       self.by_args = by_args
       self.by_period = by_period
       self.by_queue = by_queue
       self.by_state = by_state
+      self.exclude_kind = exclude_kind
     end
   end
 end

data/lib/job.rb

@@ -3,6 +3,7 @@ module River
   JOB_STATE_CANCELLED = "cancelled"
   JOB_STATE_COMPLETED = "completed"
   JOB_STATE_DISCARDED = "discarded"
+  JOB_STATE_PENDING = "pending"
   JOB_STATE_RETRYABLE = "retryable"
   JOB_STATE_RUNNING = "running"
   JOB_STATE_SCHEDULED = "scheduled"
@@ -113,6 +114,9 @@ module River
     # configuration.
     attr_accessor :unique_key
 
+    # A list of states that the job must be in to be considered for uniqueness.
+    attr_accessor :unique_states
+
     def initialize(
       id:,
       args:,
@@ -132,7 +136,8 @@ module River
       errors: nil,
       finalized_at: nil,
       tags: nil,
-      unique_key: nil
+      unique_key: nil,
+      unique_states: nil
     )
       self.id = id
       self.args = args
@@ -151,6 +156,7 @@ module River
       self.state = state
       self.tags = tags
       self.unique_key = unique_key
+      self.unique_states = unique_states
     end
   end
 

data/lib/riverqueue.rb

@@ -1,11 +1,11 @@
 require "json"
 
-require_relative "fnv"
 require_relative "insert_opts"
 require_relative "job"
 
 require_relative "client"
 require_relative "driver"
+require_relative "unique_bitmask"
 
 module River
 end

data/lib/unique_bitmask.rb

@@ -0,0 +1,41 @@
+module River
+  class UniqueBitmask
+    JOB_STATE_BIT_POSITIONS = {
+      ::River::JOB_STATE_AVAILABLE => 7,
+      ::River::JOB_STATE_CANCELLED => 6,
+      ::River::JOB_STATE_COMPLETED => 5,
+      ::River::JOB_STATE_DISCARDED => 4,
+      ::River::JOB_STATE_PENDING => 3,
+      ::River::JOB_STATE_RETRYABLE => 2,
+      ::River::JOB_STATE_RUNNING => 1,
+      ::River::JOB_STATE_SCHEDULED => 0
+    }.freeze
+    private_constant :JOB_STATE_BIT_POSITIONS
+
+    def self.from_states(states)
+      val = 0
+
+      states.each do |state|
+        bit_index = JOB_STATE_BIT_POSITIONS[state]
+
+        bit_position = 7 - (bit_index % 8)
+        val |= 1 << bit_position
+      end
+
+      format("%08b", val)
+    end
+
+    def self.to_states(mask)
+      states = [] #: Array[jobStateAll] # rubocop:disable Layout/LeadingCommentSpace
+
+      JOB_STATE_BIT_POSITIONS.each do |state, bit_index|
+        bit_position = 7 - (bit_index % 8)
+        if (mask & (1 << bit_position)) != 0
+          states << state
+        end
+      end
+
+      states.sort
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: riverqueue
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - Blake Gentry
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-08-31 00:00:00.000000000 Z
+date: 2024-12-20 00:00:00.000000000 Z
 dependencies: []
 description: River is a fast job queue for Go. Use this gem in conjunction with gems
   riverqueue-activerecord or riverqueue-sequel to insert jobs in Ruby which will be
@@ -21,10 +21,10 @@ extra_rdoc_files: []
 files:
 - lib/client.rb
 - lib/driver.rb
-- lib/fnv.rb
 - lib/insert_opts.rb
 - lib/job.rb
 - lib/riverqueue.rb
+- lib/unique_bitmask.rb
 homepage: https://riverqueue.com
 licenses:
 - LGPL-3.0-or-later
@@ -48,7 +48,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.20
+rubygems_version: 3.5.16
 signing_key:
 specification_version: 4
 summary: River is a fast job queue for Go.

data/lib/fnv.rb

@@ -1,35 +0,0 @@
-module River
-  # FNV is the Fowler–Noll–Vo hash function, a simple hash that's very easy to
-  # implement, and hash the perfect characteristics for use with the 64 bits of
-  # available space in a PG advisory lock.
-  #
-  # I'm implemented it myself so that the River gem can stay dependency free
-  # (and because it's quite easy to do).
-  module FNV
-    def self.fnv1_hash(str, size:)
-      hash = OFFSET_BASIS.fetch(size)
-      mask = (2**size - 1).to_int # creates a mask of 1s of `size` bits long like 0xffffffff
-      prime = PRIME.fetch(size)
-
-      str.each_byte do |byte|
-        hash *= prime
-        hash &= mask
-        hash ^= byte
-      end
-
-      hash
-    end
-
-    OFFSET_BASIS = {
-      32 => 0x811c9dc5,
-      64 => 0xcbf29ce484222325
-    }.freeze
-    private_constant :OFFSET_BASIS
-
-    PRIME = {
-      32 => 0x01000193,
-      64 => 0x00000100000001B3
-    }.freeze
-    private_constant :PRIME
-  end
-end