riverqueue 0.13.0 → 0.14.0

data/lib/client_runtime.rb

@@ -28,10 +28,18 @@ module River
       @condition = ConditionVariable.new
       @config = config
       @driver = driver
+      @fetching_queues = {}
+      @leader = false
+      @maintenance_condition = ConditionVariable.new
+      @maintenance_generation = 0
       @mutex = Mutex.new
+      @notifications_disabled = false
       @producer_threads = {}
       @queue_configs = config.queues.dup
+      @queue_generations = Hash.new(0)
+      @queue_paused = {}
       @removed_queues = {}
+      @resign_requested = false
       @running = {}
       @started = false
       @stopped = true
@@ -98,12 +106,21 @@ module River
 
     def publish_queue(kind, queue)
       event = Event.new(kind, nil, queue, nil)
-      @mutex.synchronize { @subscriptions.dup }.each { |subscription| subscription.publish(event) }
+      subscriptions = @mutex.synchronize do
+        paused = kind == EVENT_QUEUE_PAUSED
+        return if @queue_paused[queue.name] == paused
+
+        @queue_paused[queue.name] = paused
+        @subscriptions.dup
+      end
+      subscriptions.each { |subscription| subscription.publish(event) }
     end
 
     def healthy?
       @mutex.synchronize do
-        @stop_requested || (@producer_threads.all? { |name, thread| @removed_queues[name] || thread.alive? } && (!@maintenance_thread || @maintenance_thread.alive?))
+        @stop_requested || (@producer_threads.all? { |name, thread| @removed_queues[name] || thread.alive? } &&
+          (!@maintenance_thread || @maintenance_thread.alive?) &&
+          (!@notification_thread || @notifications_disabled || @notification_thread.alive?))
       end
     end
 
@@ -183,6 +200,7 @@ module River
       end
 
       begin
+        start_notifications
         queues = @mutex.synchronize { @queue_configs.dup }
         queues.each { |name, queue_config| start_producer(name, queue_config) }
         start_maintenance unless @queue_configs.empty? && @periodic_jobs.empty? && @config.maintenance_services.empty?
@@ -207,6 +225,7 @@ module River
 
         @stop_requested = true
         @condition.broadcast
+        @maintenance_condition.broadcast
         @running.values.each { |entry| entry[:thread].raise(Interrupted) if cancel && entry[:working] }
         @threads.dup
       end
@@ -225,6 +244,9 @@ module River
         @threads.clear
         @producer_threads.clear
         @maintenance_thread = nil
+        @notification_thread = nil
+        @leader = false
+        @resign_requested = false
       end
 
       self
@@ -245,21 +267,23 @@ module River
     end
 
     def wake
-      @mutex.synchronize { @condition.broadcast }
+      wake_queue("*")
     end
 
     private def begin_work(id)
-      should_interrupt = @mutex.synchronize do
+      interruption = @mutex.synchronize do
         entry = @running.fetch(id)
         if @stop_requested
-          true
+          Interrupted
+        elsif entry[:cancelled]
+          JobCancelError
         else
           entry[:working] = true
-          false
+          nil
         end
       end
 
-      raise Interrupted if should_interrupt
+      raise interruption if interruption
     end
 
     private def check_remote_cancellations(queue)
@@ -303,6 +327,8 @@ module River
           finish_work(row.id)
         end
 
+        return [:completed, nil] if finish_transactional_completion(job, timing)
+
         finalize_hooks = @config.plugins.any? { |plugin| plugin.respond_to?(:job_finalize) }
         if finalize_hooks
           raise JobCancelError if @driver.job_get_cancelled_ids([row.id]).include?(row.id)
@@ -322,22 +348,26 @@ module River
         end
 
         [:completed, nil]
-      rescue JobSnoozeError => error
-        job.__capture_resumable_metadata!
-        [finish_snoozed(row, job, error, timing), error]
-      rescue JobCancelError => error
-        job.__capture_resumable_metadata!
-        finish_failed(row, job, error, timing, cancelled: true)
-        [:cancelled, error]
-      rescue Interrupted => error
-        job.__capture_resumable_metadata!
-        [finish_interrupted(row, job, timing), error]
       rescue => error
+        return [:completed, nil] if finish_transactional_completion(job, timing)
+
         job.__capture_resumable_metadata!
-        [finish_failed(row, job, error, timing, worker: worker), error]
+        outcome = case error
+        when JobSnoozeError
+          finish_snoozed(row, job, error, timing)
+        when JobCancelError
+          finish_failed(row, job, error, timing, cancelled: true)
+          :cancelled
+        when Interrupted
+          finish_interrupted(row, job, timing)
+        else
+          finish_failed(row, job, error, timing, worker: worker)
+        end
+        [outcome, error]
       ensure
         @mutex.synchronize do
           @running.delete(row.id)
+          @queue_generations[row.queue] += 1
           @condition.broadcast
         end
       end
@@ -410,6 +440,35 @@ module River
       (updated&.state == JOB_STATE_CANCELLED) ? :cancelled : :snoozed
     end
 
+    private def finish_transactional_completion(job, timing)
+      return false unless job.__completion_attempted
+
+      completed = @driver.job_get_by_id(job.row.id)
+      unless completed
+        snapshot = job.__completion_snapshot
+        return false unless snapshot
+
+        publish(EVENT_JOB_COMPLETED, snapshot, timing)
+        return true
+      end
+
+      # A retry can replace a committed attempt before its worker or middleware
+      # returns. Only a still-running original attempt may have rolled back and
+      # need normal finalization; never finalize or publish a replacement.
+      return true unless completed.attempt == job.row.attempt && completed.attempted_at == job.row.attempted_at
+      return false if completed.state == JOB_STATE_RUNNING
+      return true unless completed.state == JOB_STATE_COMPLETED
+
+      # Work middleware (including persisted logging) can add metadata after the
+      # worker commits. Save it without changing the committed state or timestamp.
+      # Guard the write too: a retry can happen after the read above.
+      metadata = job.metadata_updates
+      completed = @driver.job_metadata_merge(job.row.id, metadata, completed) || completed unless metadata.empty?
+
+      publish(EVENT_JOB_COMPLETED, completed, timing)
+      true
+    end
+
     private def finish_work(id)
       @mutex.synchronize do
         entry = @running[id]
@@ -425,15 +484,22 @@ module River
       return perform_work(worker, job) if @config.plugins.empty?
 
       operation = -> do
+        invoke_plugins(:work_begin, job)
+
         error = nil
-        begin
-          invoke_plugins(:work_begin, job)
+        result = begin
           perform_work(worker, job)
-        rescue => error
-          raise
-        ensure
-          invoke_plugins(:work_end, job, error)
+        rescue => work_error
+          error = work_error
+          nil
+        end
+
+        @config.plugins.each do |plugin|
+          error = plugin.work_end(job, error) if plugin.respond_to?(:work_end)
         end
+        raise error if error
+
+        result
       end
       @config.plugins.reverse_each do |plugin|
         next unless plugin.respond_to?(:work)
@@ -457,16 +523,39 @@ module River
         end
       end
 
-      @mutex.synchronize { @running[row.id] = {queue: row.queue, thread: thread, working: false} }
+      @mutex.synchronize do
+        @running[row.id] = {cancelled: @fetching_queues.fetch(row.queue).include?(row.id), queue: row.queue, thread: thread, working: false}
+      end
       gate.push(true)
     end
 
     private def maintenance_loop
       leader = false
+      next_election = 0.0
       next_schedule = next_rescue = next_cleanup = Time.at(0)
       until stopping?
+        generation, resign = @mutex.synchronize { [@maintenance_generation, @resign_requested] }
+        if resign && leader
+          @driver.leader_release(@config.id)
+          leader = false
+          @mutex.synchronize do
+            @leader = false
+            @resign_requested = false
+          end
+          # Give followers a chance to acquire the term we just released.
+          next_election = monotonic_now + 5
+        end
+        if !leader && (delay = next_election - monotonic_now).positive?
+          wait_for_maintenance(delay, generation)
+          next
+        end
+
         now = Time.now.utc
         leader = leader ? @driver.leader_renew(@config.id, now: now) : @driver.leader_acquire(@config.id, now: now)
+        @mutex.synchronize do
+          @leader = leader
+          @resign_requested = false unless leader
+        end
         if leader
           if now >= next_schedule
             @driver.job_schedule(now: now)
@@ -496,7 +585,7 @@ module River
           end
         end
 
-        wait(5)
+        wait_for_maintenance(5, generation)
       end
     rescue => error
       @config.logger.error("River maintenance stopped: #{error.full_message}")
@@ -536,6 +625,55 @@ module River
       ((count + 1 + (1 << 63)) % (1 << 64)) - (1 << 63)
     end
 
+    private def notification_loop(ready)
+      listener = nil #: untyped
+      retry_delay = 0.1
+      begin
+        until notifications_stopping?
+          begin
+            unless listener
+              listener = @driver.notification_listener
+              @mutex.synchronize { @notifications_disabled = listener.nil? }
+              ready << true if ready
+              ready = nil
+              return unless listener
+
+              # Poll durable state after reconnecting: Postgres messages sent
+              # while disconnected cannot be replayed.
+              wake
+              wake_maintenance
+            end
+            listener.poll(0.1).each { |topic, payload| receive_notification(topic, payload) }
+            retry_delay = 0.1
+          rescue => error
+            if ready
+              ready << error
+              ready = nil
+              return
+            end
+            @config.logger.error("River notification receiver failed: #{error.full_message}")
+            # SQLite's cursor survives temporary read errors. Reopening it would
+            # skip requests committed while the database was unavailable.
+            unless listener.is_a?(Driver::NotificationListener::SQLite)
+              listener&.close
+              listener = nil
+            end
+            sleep(retry_delay) unless notifications_stopping?
+            retry_delay = [retry_delay * 2, 1.0].min
+          end
+        end
+      ensure
+        ready << true if ready
+        listener&.close
+      end
+    end
+
+    private def notifications_stopping?
+      @mutex.synchronize do
+        @stop_requested && @running.empty? && @producer_threads.values.none?(&:alive?)
+      end
+    end
+
     private def perform_work(worker, job)
       timeout = worker_timeout(worker, job)
       result = timeout ? Timeout.timeout(timeout) { worker.work(job) } : worker.work(job)
@@ -545,7 +683,7 @@ module River
 
     private def periodic_jobs_changed
       start_maintenance
-      wake
+      wake_maintenance
     end
 
     private def producer_loop(queue, queue_config)
@@ -556,6 +694,7 @@ module River
       last_fetch = 0.0
       poll_interval = queue_config.resolved_fetch_poll_interval(@config)
       loop do
+        generation = @mutex.synchronize { @queue_generations[queue] }
         draining = queue_stopping?(queue)
         break if draining && running_count(queue).zero?
 
@@ -573,18 +712,25 @@ module River
             wait(sleep_for)
           end
           next if queue_stopping?(queue)
+        end
 
-          # A pause may arrive during cooldown; only read the queue immediately
-          # before fetching, and avoid the read entirely when all slots are busy.
-          unless @driver.queue_get(queue)&.paused_at
+        # Refresh controls even when every worker slot is occupied. A pause
+        # can also arrive during cooldown, so read immediately before fetching.
+        queue_state = @driver.queue_get(queue)
+        publish_queue(queue_state.paused_at ? EVENT_QUEUE_PAUSED : EVENT_QUEUE_RESUMED, queue_state) if queue_state
+        if capacity.positive? && !queue_state&.paused_at
+          @mutex.synchronize { @fetching_queues[queue] = [] }
+          begin
             jobs = @driver.job_get_available(attempted_by: @config.id, max: capacity, queue: queue, **fetch_options)
             last_fetch = monotonic_now
             jobs.each { |job| launch(job) }
             next unless jobs.empty?
+          ensure
+            @mutex.synchronize { @fetching_queues.delete(queue) }
           end
         end
 
-        wait(poll_interval)
+        wait_for_queue(queue, poll_interval, generation)
       end
     rescue => error
       @config.logger.error("River producer for #{queue.inspect} stopped: #{error.full_message}")
@@ -615,6 +761,52 @@ module River
       @mutex.synchronize { @stop_requested || @removed_queues[queue] }
     end
 
+    private def receive_notification(topic, payload)
+      message = JSON.parse(payload)
+      return unless message.is_a?(Hash)
+
+      case topic
+      when "river_insert"
+        wake_queue(message["queue"]) if message["queue"].is_a?(String)
+      when "river_control"
+        queue = message["queue"]
+        return unless queue.is_a?(String)
+
+        case message["action"]
+        when "cancel"
+          return unless message["job_id"].is_a?(Integer)
+
+          @mutex.synchronize do
+            entry = @running[message["job_id"]]
+            if entry && entry[:queue] == queue
+              entry[:cancelled] = true
+              entry[:thread].raise(JobCancelError) if entry[:working]
+            elsif (pending = @fetching_queues[queue])
+              # A claim may have committed before its workers are registered.
+              pending << message["job_id"]
+            end
+          end
+        when "metadata_changed", "pause", "resume"
+          wake_queue(queue)
+        end
+      when "river_leadership"
+        case message["action"]
+        when "request_resign"
+          @mutex.synchronize do
+            if @leader
+              @resign_requested = true
+              @maintenance_generation += 1
+              @maintenance_condition.broadcast
+            end
+          end
+        when "resigned"
+          wake_maintenance if message["leader_id"].is_a?(String) && message["leader_id"] != @config.id
+        end
+      end
+    rescue JSON::ParserError, TypeError => error
+      @config.logger.warn("River ignored invalid notification: #{error.message}")
+    end
+
     private def remove_subscription(subscription)
       @mutex.synchronize { @subscriptions.delete(subscription) }
     end
@@ -646,10 +838,10 @@ module River
     end
 
     private def run_periodic(now)
-      due = @periodic_jobs.due(now) do |job, error|
+      due = @periodic_jobs.__due_with_times(now) do |job, error|
         @config.logger.error("River periodic job schedule failed (id=#{job.id.inspect}): #{error.full_message}")
       end
-      due.each do |periodic_job|
+      due.each do |periodic_job, scheduled_at|
         value = periodic_job.constructor.call
         next unless value
 
@@ -657,7 +849,7 @@ module River
         opts = (opts || InsertOpts.new).dup
         opts.metadata = (opts.metadata || {}).merge("periodic" => true)
         opts.metadata["river:periodic_job_id"] = periodic_job.id if periodic_job.id && !periodic_job.id.empty?
-        @client.insert(args, insert_opts: opts)
+        @client.__insert_with_default_scheduled_at(args, opts, scheduled_at)
       rescue => error
         @config.logger.error("River periodic job failed to insert: #{error.full_message}")
       end
@@ -680,11 +872,28 @@ module River
       end
     end
 
+    private def start_notifications
+      return if @config.poll_only || !@driver.respond_to?(:notification_listener)
+
+      ready = ::Queue.new
+      @mutex.synchronize do
+        return if @stop_requested
+
+        thread = Thread.new { notification_loop(ready) }
+        @notification_thread = thread
+        @threads << thread
+      end
+      result = ready.pop
+      raise result if result.is_a?(Exception)
+    end
+
     private def start_producer(name, queue_config)
-      @driver.queue_upsert(name)
+      queue = @driver.queue_upsert(name)
       @mutex.synchronize do
         return if @stop_requested || @removed_queues[name] || @producer_threads.key?(name)
 
+        @queue_paused[name] = !queue.paused_at.nil? if queue
+
         # Shutdown must see every launched thread, including a producer whose
         # database setup overlapped a stop or queue removal.
         thread = Thread.new { producer_loop(name, queue_config) }
@@ -701,6 +910,21 @@ module River
       @mutex.synchronize { @condition.wait(@mutex, duration) unless @stop_requested }
     end
 
+    private def wait_for_maintenance(duration, generation)
+      @mutex.synchronize do
+        @maintenance_condition.wait(@mutex, duration) if !@stop_requested && @maintenance_generation == generation
+      end
+    end
+
+    private def wait_for_queue(queue, duration, generation)
+      deadline = monotonic_now + duration
+      @mutex.synchronize do
+        while !@stop_requested && !@removed_queues[queue] && @queue_generations[queue] == generation && (remaining = deadline - monotonic_now).positive?
+          @condition.wait(@mutex, remaining)
+        end
+      end
+    end
+
     private def wait_for_running_jobs(queue, duration)
       @mutex.synchronize do
         # Draining still needs a poll interval after stop is requested. Check
@@ -709,6 +933,20 @@ module River
       end
     end
 
+    private def wake_maintenance
+      @mutex.synchronize do
+        @maintenance_generation += 1
+        @maintenance_condition.broadcast
+      end
+    end
+
+    private def wake_queue(queue)
+      @mutex.synchronize do
+        @queue_configs.each_key { |name| @queue_generations[name] += 1 if queue == "*" || queue == name }
+        @condition.broadcast
+      end
+    end
+
     private def worker_timeout(worker, job)
       timeout = worker.respond_to?(:timeout) ? worker.timeout(job) : @config.job_timeout
       timeout = Float(timeout) unless timeout.nil?

data/lib/config.rb

@@ -40,22 +40,52 @@ module River
     end
   end
 
+  # Lazily generates a client ID once per process. The process ID check
+  # regenerates it in forked children without needing a fork hook.
+  class DefaultClientID # :nodoc:
+    HOSTNAME_MAX_LENGTH = 80
+
+    def initialize
+      @mutex = Mutex.new
+      @state = nil
+    end
+
+    def value
+      pid = Process.pid
+      @state&.[](pid) || @mutex.synchronize { @state&.[](pid) || generate(pid) }
+    end
+
+    # The PID and ID are swapped in as one frozen hash so a lock-free reader
+    # never pairs one process's PID with another's ID.
+    private def generate(pid)
+      id = "#{Socket.gethostname[0, HOSTNAME_MAX_LENGTH]}-#{pid}-#{SecureRandom.hex(6)}".freeze
+      @state = {pid => id}.freeze
+      id
+    end
+  end
+  private_constant :DefaultClientID
+
   # Complete configuration for a River client and its worker runtime.
   class Config
     # Configured values used by Client and its runtime.
     attr_reader :cancelled_job_retention_period, :completed_job_retention_period,
       :discarded_job_retention_period, :error_handler, :fetch_cooldown,
-      :fetch_only_known_kinds, :fetch_poll_interval, :id, :job_timeout,
+      :fetch_only_known_kinds, :fetch_poll_interval, :job_timeout,
       :leader_election_disabled, :logger, :maintenance_services,
-      :periodic_jobs, :plugins, :queues, :retry_policy, :workers
+      :periodic_jobs, :plugins, :poll_only, :queues, :retry_policy, :workers
 
     # Creates a client configuration.
     #
     # Configure queues and workers here to enable job processing; a client with
     # neither may still be used for insertion and administrative operations.
-    # +fetch_only_known_kinds+ leaves unregistered kinds untouched in shared
-    # queues. +leader_election_disabled+ prevents this client's maintenance;
-    # another eligible client must run scheduling, rescue, and cleanup.
+    # An explicit +id+ is used exactly as given and must be unique among all
+    # running clients. Without one, each process using this config generates
+    # its own ID; see #id. +fetch_only_known_kinds+ leaves unregistered kinds
+    # untouched in shared queues. +leader_election_disabled+ prevents this
+    # client's maintenance; another eligible client must run scheduling,
+    # rescue, and cleanup.
+    # +poll_only+ disables notification receiving, including leadership
+    # resignation requests. Job and queue state are still polled.
     def initialize(
       queues: {},
       workers: Workers.new,
@@ -70,6 +100,7 @@ module River
       plugins: [],
       maintenance_services: [],
       periodic_jobs: [],
+      poll_only: false,
       cancelled_job_retention_period: 86_400,
       completed_job_retention_period: 86_400,
       discarded_job_retention_period: 604_800,
@@ -82,13 +113,15 @@ module River
       @fetch_cooldown = Float(fetch_cooldown)
       @fetch_only_known_kinds = fetch_only_known_kinds
       @fetch_poll_interval = Float(fetch_poll_interval)
-      @id = id || "#{Socket.gethostname}-#{Process.pid}-#{SecureRandom.hex(6)}"
+      @default_id = DefaultClientID.new
+      @id = id
       @job_timeout = job_timeout.nil? ? nil : Float(job_timeout)
       @leader_election_disabled = leader_election_disabled
       @logger = logger || Logger.new($stdout, level: Logger::WARN)
       @maintenance_services = maintenance_services.dup.freeze
       @periodic_jobs = periodic_jobs.dup.freeze
       @plugins = plugins.dup.freeze
+      @poll_only = poll_only
       @queues = normalize_queues(queues).freeze
       @retry_policy = retry_policy
       @workers = workers
@@ -96,10 +129,22 @@ module River
       validate
     end
 
+    # Returns the client ID recorded in +attempted_by+ and used for leadership.
+    #
+    # An explicitly configured ID is returned unchanged. Otherwise, the ID is
+    # generated on first use in each process and stays stable for the life of
+    # that process, so clients built from a config before a fork (for example,
+    # by a preloading Puma, Unicorn, or Pitchfork server) get distinct IDs in
+    # each forked child instead of sharing the parent's.
+    def id
+      @id || @default_id.value
+    end
+
     # Returns a new Config with the supplied values replacing this config's
-    # corresponding settings.
+    # corresponding settings. Without an +id+ override, the copy shares this
+    # config's ID, including its per-process generated default.
     def with(**overrides)
-      Config.new(id: id,
+      copy = Config.new(id: @id,
         cancelled_job_retention_period: cancelled_job_retention_period,
         completed_job_retention_period: completed_job_retention_period,
         discarded_job_retention_period: discarded_job_retention_period,
@@ -113,9 +158,16 @@ module River
         maintenance_services: maintenance_services,
         periodic_jobs: periodic_jobs,
         plugins: plugins,
+        poll_only: poll_only,
         queues: queues,
         retry_policy: retry_policy,
         workers: workers, **overrides)
+      copy.adopt_default_id(@default_id) unless overrides.key?(:id)
+      copy
+    end
+
+    protected def adopt_default_id(default_id)
+      @default_id = default_id
     end
 
     private def normalize_queues(queues)
@@ -136,7 +188,7 @@ module River
 
     private def validate
       raise ArgumentError, "fetch intervals and job_timeout must be finite" unless [fetch_cooldown, fetch_poll_interval, job_timeout].compact.all?(&:finite?)
-      raise ArgumentError, "id must be between 1 and 127 characters" unless (1...128).cover?(id.length)
+      raise ArgumentError, "id must be between 1 and 127 characters" if (explicit_id = @id) && !(1...128).cover?(explicit_id.length)
       raise ArgumentError, "fetch_cooldown must be at least 0.001 seconds" if fetch_cooldown < 0.001
       raise ArgumentError, "fetch_poll_interval cannot be less than fetch_cooldown" if fetch_poll_interval < fetch_cooldown
       raise ArgumentError, "job_timeout must be greater than zero or nil" if job_timeout && job_timeout <= 0

data/lib/driver/job_row_decoder.rb

@@ -23,7 +23,7 @@ module River::Driver
     def self.error_time(value)
       return ZERO_TIME unless value.is_a?(String)
 
-      # Keep Ruby's existing acceptance of PostgreSQL-style timestamps too.
+      # Keep Ruby's existing acceptance of Postgres-style timestamps too.
       Time.parse(value).utc
     rescue ArgumentError
       ZERO_TIME
@@ -63,7 +63,7 @@ module River::Driver
     end
 
     def decoded(field, value, type: nil, strings: false, default: nil)
-      # Sequel's PostgreSQL JSON and array wrappers delegate to Ruby values.
+      # Sequel's Postgres JSON and array wrappers delegate to Ruby values.
       value = value.__getobj__ if value.respond_to?(:__getobj__)
       return default if value.nil?