riverqueue 0.11.0 → 0.12.0

data/lib/migrator.rb

@@ -0,0 +1,189 @@
+# frozen_string_literal: true
+
+module River
+  # Runs River's canonical SQL migrations, committing each version separately.
+  # Use a dedicated migration connection and stop workers before downgrading.
+  class Migrator
+    Migration = Data.define(:version, :name, :sql_up, :sql_down)
+    Status = Data.define(:version, :name, :applied)
+
+    attr_reader :migrations
+
+    # Creates a migrator for an installed River driver. PostgreSQL schemas must
+    # already exist and use simple SQL identifiers; SQLite uses its main schema.
+    # The optional migrations_path is a root containing backend/line/*.sql.
+    def initialize(driver, line: "main", migrations_path: File.expand_path("../migration", __dir__.to_s), schema: nil)
+      @backend = driver.migration_backend
+      @driver = driver
+      raise ArgumentError, "unsupported migration backend" unless [:postgresql, :sqlite].include?(@backend)
+      raise ArgumentError, "invalid migration line" unless line.match?(/\A[a-z][a-z0-9_]*\z/)
+      raise ArgumentError, "SQLite does not support a migration schema option" if @backend == :sqlite && schema
+
+      @line = line
+      @mutex = Mutex.new
+      @requested_schema = schema
+
+      directory = File.join(migrations_path, @backend.to_s, line)
+      @migrations = Dir.glob(File.join(directory, "*.up.sql")).sort.map do |path|
+        version, name = File.basename(path).delete_suffix(".up.sql").split("_", 2)
+        Migration.new(Integer(version.to_s, 10), name, File.read(path).freeze, File.read(path.sub(/\.up\.sql\z/, ".down.sql")).freeze)
+      end.freeze
+      unless @migrations.any? && @migrations.map(&:version) == (1..@migrations.length).to_a
+        raise ArgumentError, "migration files must contain contiguous versions starting at 1"
+      end
+    end
+
+    # Applies pending migrations up (all by default) or down (one by default).
+    # target is the version to end at; target: 0 removes the migration line.
+    # An up target that is already applied is a no-op.
+    # steps limits the number applied. dry_run returns the plan without writes.
+    # Returns Migration objects for the versions applied or planned.
+    def migrate(direction: :up, dry_run: false, steps: nil, target: nil)
+      raise ArgumentError, "direction must be up or down" unless [:up, :down].include?(direction)
+      raise ArgumentError, "steps must be positive" if steps && (!steps.is_a?(Integer) || steps <= 0)
+      raise ArgumentError, "target must be a bundled version or zero" if target && (!target.is_a?(Integer) || !(0..migrations.length).cover?(target))
+
+      session(lock: !dry_run) do
+        existing = existing_versions
+        current = existing.last || 0
+        destination = target || ((direction == :up) ? migrations.length : 0)
+        if direction == :down && destination > current
+          raise ArgumentError, "target is in the opposite direction"
+        end
+
+        plan = if direction == :up
+          migrations.select { |migration| migration.version > current && migration.version <= destination }
+        else
+          migrations.reverse.select { |migration| migration.version <= current && migration.version > destination }
+        end
+        limit = steps || ((direction == :down && target.nil?) ? 1 : plan.length)
+        plan = plan.first(limit)
+
+        unless dry_run
+          plan.each do |migration|
+            execute((@backend == :sqlite) ? "BEGIN IMMEDIATE" : "BEGIN")
+            committed = false
+            begin
+              raise River::Error, "migration state changed concurrently; rerun migration" unless existing_versions == existing
+
+              if direction == :down && @line == "main" && line_column?
+                raise River::Error, "remove non-main migration lines before downgrading main" if query("SELECT version FROM #{table} WHERE line <> 'main'").any?
+              end
+
+              sql = (direction == :up) ? migration.sql_up : migration.sql_down
+              execute(sql.gsub("/* TEMPLATE: schema */", @schema ? %("#{@schema}".) : ""))
+
+              if direction == :up
+                columns, values = line_column? ? ["line, version", "'#{@line}', #{migration.version}"] : ["version", migration.version.to_s]
+                execute("INSERT INTO #{table} (#{columns}) VALUES (#{values})")
+                existing += [migration.version]
+              else
+                unless @line == "main" && migration.version == 1
+                  filter = line_column? ? " AND line = '#{@line}'" : ""
+                  execute("DELETE FROM #{table} WHERE version = #{migration.version}#{filter}")
+                end
+
+                existing -= [migration.version]
+              end
+
+              execute("COMMIT")
+              committed = true
+            ensure
+              execute("ROLLBACK") unless committed
+            end
+          end
+        end
+
+        plan
+      end
+    end
+
+    # Returns the bundled versions with their database-applied status.
+    def status
+      session do
+        existing = existing_versions
+        migrations.map { |migration| Status.new(migration.version, migration.name, existing.include?(migration.version)) }
+      end
+    end
+
+    private def execute(sql)
+      if @backend == :postgresql
+        @connection.exec(sql)
+      else
+        @connection.execute_batch(sql)
+      end
+    end
+
+    private def existing_versions
+      exists = if @backend == :postgresql
+        query("SELECT to_regclass('#{table}') AS name").first.fetch("name")
+      else
+        query("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'river_migration'").any?
+      end
+      if @line != "main"
+        unless exists && line_column? && query("SELECT version FROM #{table} WHERE line = 'main' AND version >= 7").any?
+          raise River::Error, "migrate main to version 7 or later before migrating #{@line}"
+        end
+      end
+
+      return [] unless exists
+
+      filter = line_column? ? " WHERE line = '#{@line}'" : ""
+      versions = query("SELECT version FROM #{table}#{filter} ORDER BY version").map { |row| row.fetch("version").to_i }
+      unless versions == (1..versions.length).to_a && versions.all? { |version| version <= migrations.length }
+        raise River::Error, "database migration history is incomplete or newer than this gem"
+      end
+
+      versions
+    end
+
+    private def line_column?
+      if @backend == :postgresql
+        query("SELECT column_name FROM information_schema.columns WHERE table_schema = '#{@schema}' AND table_name = 'river_migration' AND column_name = 'line'").any?
+      else
+        query("PRAGMA table_info(river_migration)").any? { |row| row.fetch("name") == "line" }
+      end
+    end
+
+    private def query(sql)
+      if @backend == :postgresql
+        @connection.exec(sql).to_a
+      else
+        rows = @connection.execute2(sql)
+        columns = rows.shift
+        rows.map { |row| row.is_a?(Hash) ? row : columns.zip(row).to_h }
+      end
+    end
+
+    private def session(lock: false)
+      @mutex.synchronize do
+        @driver.migration_connection do |connection|
+          @connection = connection
+          @schema = if @backend == :postgresql
+            @requested_schema || query("SELECT current_schema() AS name").first.fetch("name")
+          end
+          if @backend == :postgresql
+            raise ArgumentError, "schema must be a simple SQL identifier" unless @schema&.match?(/\A[a-zA-Z_][a-zA-Z0-9_]*\z/)
+
+            if lock
+              locked = query("SELECT pg_try_advisory_lock(hashtext(current_database()), hashtext('river_migrate:#{@schema}')) AS locked").first.fetch("locked")
+              raise River::Error, "another Ruby migrator holds the schema lock" unless [true, "t"].include?(locked)
+            end
+          end
+
+          begin
+            yield
+          ensure
+            if @backend == :postgresql && lock
+              query("SELECT pg_advisory_unlock(hashtext(current_database()), hashtext('river_migrate:#{@schema}'))")
+            end
+          end
+        end
+      end
+    end
+
+    private def table
+      @schema ? %("#{@schema}".river_migration) : "river_migration"
+    end
+  end
+end

data/lib/params.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module River
+  # Result of Client#job_delete_many.
+  JobDeleteManyResult = Data.define(:jobs)
+
+  # Position in a job listing, including the ordering value so pagination still
+  # works after the job at the end of the previous page has been deleted.
+  JobListCursor = Data.define(:id, :sort_by, :sort_order, :value)
+
+  # A page of jobs and the cursor for fetching the next page.
+  JobListResult = Data.define(:jobs, :last_cursor)
+
+  # A River queue record persisted in the database.
+  Queue = Data.define(:name, :created_at, :metadata, :paused_at, :updated_at)
+
+  # Result of Client#queue_list.
+  QueueListResult = Data.define(:queues)
+
+  # Filtering, ordering, and pagination parameters for Client#job_list and
+  # Client#job_delete_many.
+  class JobListParams
+    # Normalized filtering, pagination, and ordering values.
+    attr_reader :after, :after_id, :ids, :kinds, :limit, :metadata, :priorities, :queues, :sort_by,
+      :sort_order, :states, :tags_all, :tags_any
+
+    # Creates job-list parameters. Pass JobListResult#last_cursor as +after+ and
+    # keep the same filters and ordering for subsequent pages. +after_id+ is a
+    # shortcut for ID-ordered listings. Null timestamps always sort last.
+    # Kind, queue, and state filters accept symbols or strings.
+    def initialize(after: nil, after_id: nil, ids: nil, kinds: nil, limit: 100, priorities: nil,
+      metadata: nil, queues: nil, sort_by: :id, sort_order: :asc, states: nil, tags_all: nil, tags_any: nil)
+      @after = after
+      @after_id = after_id.nil? ? nil : Integer(after_id)
+      @ids = ids
+      @kinds = kinds&.map(&:to_s)
+      @limit = Integer(limit)
+      @metadata = metadata
+      @priorities = priorities
+      @queues = queues&.map(&:to_s)
+      @sort_by = sort_by.to_sym
+      @sort_order = sort_order.to_sym
+      @states = states&.map(&:to_s)
+      @tags_all = tags_all
+      @tags_any = tags_any
+
+      raise ArgumentError, "limit must be between 1 and 10,000" unless (1..10_000).cover?(@limit)
+      raise ArgumentError, "invalid sort field" unless [:id, :scheduled_at, :finalized_at].include?(@sort_by)
+      raise ArgumentError, "invalid sort order" unless [:asc, :desc].include?(@sort_order)
+      raise ArgumentError, "use either after or after_id" if after && after_id
+      raise ArgumentError, "after_id requires sorting by id; use after for timestamp ordering" if after_id && @sort_by != :id
+      if after && (!after.is_a?(JobListCursor) || after.sort_by != @sort_by || after.sort_order != @sort_order)
+        raise ArgumentError, "after must be a JobListCursor with the same ordering"
+      end
+    end
+
+    def filters?
+      [after, after_id, ids, kinds, metadata, priorities, queues, states, tags_all, tags_any].any? do |value|
+        value.respond_to?(:empty?) ? !value.empty? : !value.nil?
+      end
+    end
+  end
+
+  # Fields to change with Client#job_update. Omitted fields are left unchanged;
+  # explicitly passing nil clears nullable fields.
+  class JobUpdateParams
+    UNSET = Object.new.freeze
+
+    # Values to apply, with omitted fields represented internally by UNSET.
+    attr_reader :attempt, :attempted_at, :attempted_by, :errors, :finalized_at,
+      :max_attempts, :metadata, :state
+
+    # Creates a partial set of updates for a persisted job.
+    def initialize(attempt: UNSET, attempted_at: UNSET, attempted_by: UNSET,
+      errors: UNSET, finalized_at: UNSET, max_attempts: UNSET, metadata: UNSET,
+      state: UNSET)
+      @attempt = attempt
+      @attempted_at = attempted_at
+      @attempted_by = attempted_by
+      @errors = errors
+      @finalized_at = finalized_at
+      @max_attempts = max_attempts
+      @metadata = metadata
+      @state = state.is_a?(Symbol) ? state.to_s : state
+    end
+
+    def each
+      return enum_for(:each) unless block_given?
+
+      instance_variables.each do |ivar|
+        value = instance_variable_get(ivar)
+        yield ivar.to_s.delete_prefix("@").to_sym, value unless value.equal?(UNSET)
+      end
+    end
+  end
+end

data/lib/periodic_cron.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module River
+  # A cron schedule for PeriodicJob. Add the optional +fugit+ gem to use it.
+  class PeriodicCron
+    # Parses a cron expression once. +timezone+ defaults to UTC; use an IANA
+    # name such as America/New_York for local calendar schedules. Specify the
+    # timezone here, not inside +expression+. Fugit is loaded only on construction.
+    def initialize(expression, timezone: "UTC")
+      require "fugit"
+
+      raise ArgumentError, "cron expression must be a String" unless expression.is_a?(String)
+      raise ArgumentError, "timezone must be a nonempty name without whitespace" unless timezone.is_a?(String) && /\A\S+\z/.match?(timezone)
+
+      @cron = Object.const_get(:Fugit).const_get(:Cron).do_parse("#{expression} #{timezone}")
+    end
+
+    # Returns a UTC Time strictly after +time+. Calendar and daylight-saving
+    # rules are provided by Fugit; this helper does not enqueue or persist jobs.
+    def next(time)
+      @cron.next_time(time).to_t.getutc
+    end
+  end
+end

data/lib/periodic_job.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+module River
+  # Definition of a recurring job and the schedule used to construct it.
+  class PeriodicJob
+    # Callable that produces job arguments, or an arguments/options pair.
+    attr_reader :constructor
+
+    # Stable string identifier used for removal and durable Pro scheduling.
+    attr_reader :id
+
+    # Whether the job should run immediately when its scheduler starts.
+    attr_reader :run_on_start
+
+    # Object or callable that computes the next run time.
+    attr_reader :schedule
+
+    # Creates a periodic job. +schedule+ may respond to +next(time)+ or be a
+    # callable, and must return a Time strictly after its argument. Supply a
+    # factory block or a +constructor+ callable (not both),
+    # invoked whenever the job is due. +id+ accepts a symbol or string; nil
+    # leaves the registration anonymous.
+    def initialize(schedule:, constructor: nil, id: nil, run_on_start: false, &block)
+      raise ArgumentError, "use constructor or a block, not both" if constructor && block
+
+      @constructor = constructor || block
+      raise ArgumentError, "a factory block or callable constructor is required" unless @constructor.respond_to?(:call)
+      raise ArgumentError, "schedule must respond to next or call" unless schedule.respond_to?(:next) || schedule.respond_to?(:call)
+      @id = id&.to_s
+      @run_on_start = run_on_start
+      @schedule = schedule
+    end
+
+    # Returns the next scheduled Time after +now+.
+    def next_at(now)
+      next_time = schedule.respond_to?(:next) ? schedule.next(now) : schedule.call(now)
+      raise ArgumentError, "schedule must return a future Time" unless next_time.is_a?(Time) && next_time > now
+      next_time
+    end
+  end
+
+  # A fixed-duration schedule suitable for PeriodicJob.
+  class PeriodicInterval
+    # Creates an interval measured in seconds.
+    def initialize(seconds)
+      @seconds = Float(seconds, exception: true) #: Float
+      raise ArgumentError, "period must be finite" unless @seconds.finite?
+      raise ArgumentError, "period must be greater than zero" unless @seconds.positive?
+    end
+
+    # Returns the next Time one interval after +time+.
+    def next(time)
+      time + @seconds
+    end
+  end
+
+  # Thread-safe collection used to change a client's periodic jobs at runtime.
+  class PeriodicJobBundle
+    def initialize(jobs, wake:)
+      @jobs = {}
+      @mutex = Mutex.new
+      @next_handle = 0
+      @wake = wake
+      add_many(jobs)
+    end
+
+    # Adds a periodic job and returns a handle that can be passed to #remove.
+    def add(job)
+      add_many([job]).fetch(0)
+    end
+
+    # Adds periodic jobs atomically and returns their handles in input order.
+    # Invalid schedules or duplicate IDs leave the registry unchanged.
+    def add_many(jobs)
+      now = Time.now.utc
+      entries = jobs.map { |job| {job: job, next_at: job.run_on_start ? now : job.next_at(now)} }
+      handles = @mutex.synchronize do
+        ids = @jobs.values.to_h { |entry| [entry[:job].id, true] }
+        jobs.each do |job|
+          next unless job.id
+          raise ArgumentError, "periodic job ID is already registered: #{job.id}" if ids[job.id]
+          ids[job.id] = true
+        end
+        entries.map do |entry|
+          @next_handle += 1
+          @jobs[@next_handle] = entry
+          @next_handle
+        end
+      end
+      @wake.call unless handles.empty?
+      handles
+    end
+
+    # Removes all periodic jobs from the bundle and returns self.
+    def clear
+      @mutex.synchronize { @jobs.clear }
+      self
+    end
+
+    def due(now)
+      # Application schedules may reenter the registry, so evaluate outside its
+      # lock and only advance registrations that haven't been removed or claimed.
+      entries = @mutex.synchronize { @jobs.map { |handle, entry| [handle, entry, entry[:next_at]] } }
+      entries.filter_map do |handle, entry, next_at|
+        next if next_at > now
+
+        following = entry[:job].next_at(now)
+        @mutex.synchronize do
+          next unless @jobs[handle].equal?(entry) && entry[:next_at] == next_at
+          entry[:next_at] = following
+          entry[:job]
+        end
+      rescue => error
+        raise unless block_given?
+        yield(entry[:job], error)
+        nil
+      end
+    end
+
+    # Removes the job associated with +handle+, returning the PeriodicJob or
+    # nil when no such handle exists.
+    def remove(handle)
+      @mutex.synchronize { @jobs.delete(handle)&.fetch(:job) }
+    end
+
+    # Removes the periodic job with +id+. Returns whether a job was removed.
+    def remove_by_id(id)
+      id = id&.to_s
+      @mutex.synchronize do
+        pair = @jobs.find { |_handle, entry| entry[:job].id == id }
+        !!(pair && @jobs.delete(pair.first))
+      end
+    end
+  end
+end

data/lib/resumable.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+module River
+  RESUMABLE_CURSOR_METADATA_KEY = "river:resumable_cursor"
+  RESUMABLE_STEP_METADATA_KEY = "river:resumable_step"
+
+  # Execution state for resumable steps. Applications normally interact with
+  # this through Job#resumable_step and Job#resumable_step_cursor.
+  class ResumableState
+    attr_reader :all_step_names
+    attr_accessor :completed_step
+    attr_reader :cursors
+    attr_accessor :cursors_dirty
+    attr_accessor :resume_matched
+    attr_reader :resume_step
+    attr_accessor :step_name
+
+    def initialize(metadata)
+      @all_step_names = {}
+      @completed_step = nil
+      @cursors = (metadata[RESUMABLE_CURSOR_METADATA_KEY] || {}).dup
+      @cursors_dirty = false
+      @resume_step = metadata[RESUMABLE_STEP_METADATA_KEY]
+      @step_name = nil
+
+      @resume_matched = @resume_step.to_s.empty?
+    end
+
+    def register(name)
+      raise Error, "duplicate resumable step name #{name.inspect}" if all_step_names.key?(name)
+
+      all_step_names[name] = true
+    end
+  end
+
+  class Job
+    # Internal runtime boundary: attach progress only to attempts that did not
+    # complete, matching River Go's persisted metadata format.
+    def __capture_resumable_metadata!
+      if @resumable_state.cursors_dirty
+        @metadata_updates[RESUMABLE_CURSOR_METADATA_KEY] = @resumable_state.cursors.empty? ? nil : @resumable_state.cursors.dup
+      end
+
+      if @resumable_state.completed_step
+        @metadata_updates[RESUMABLE_STEP_METADATA_KEY] = @resumable_state.completed_step
+      end
+    end
+
+    # Internal runtime boundary: validate that the recorded resume point still
+    # exists in the worker after it has returned.
+    def __finish_resumable_work!
+      if @resumable_state.resume_step && !@resumable_state.resume_matched
+        raise Error, "resumable step #{@resumable_state.resume_step.inspect} not found in worker"
+      end
+    end
+
+    # Immediately checkpoints the current step and cursor progress. Wrap the
+    # call in Driver#transaction alongside application writes when an atomic
+    # checkpoint is needed.
+    def resumable_checkpoint(cursor: RESUMABLE_CURSOR_UNSET)
+      step_name = @resumable_state.step_name
+      raise Error, "resumable step can only be persisted inside a resumable step" unless step_name
+      raise Error, "job must be running" unless row.state == JOB_STATE_RUNNING
+
+      cursors = @resumable_state.cursors.dup
+      cursors[step_name] = JSON.parse(JSON.generate(cursor)) unless cursor.equal?(RESUMABLE_CURSOR_UNSET)
+      updates = {
+        RESUMABLE_STEP_METADATA_KEY => step_name,
+        RESUMABLE_CURSOR_METADATA_KEY => cursors.empty? ? nil : cursors
+      }
+
+      updated = client.driver.job_metadata_merge(row.id, updates) || raise(NotFoundError, "job not found: #{row.id}")
+      # The database owns checkpointed progress, including rollback of an
+      # enclosing application transaction. Only progress made after this write
+      # belongs in the attempt's deferred updates; replaying the checkpoint on
+      # failure could otherwise commit progress whose application writes rolled back.
+      @resumable_state.cursors.replace(cursors)
+      @resumable_state.cursors_dirty = false
+      @resumable_state.completed_step = nil
+      @row = updated
+    end
+
+    # Records JSON-compatible cursor data for the current step. It is persisted
+    # with the failed attempt so that its retry can continue from this value.
+    def resumable_set_cursor(cursor)
+      step_name = @resumable_state.step_name
+      raise Error, "resumable cursor can only be set inside a resumable step" unless step_name
+
+      @resumable_state.cursors[step_name] = JSON.parse(JSON.generate(cursor))
+      @resumable_state.cursors_dirty = true
+      cursor
+    end
+
+    # Runs a named step, skipping it on retry when a previous attempt already
+    # completed it. Names accept symbols or strings and must be unique within a
+    # worker invocation. Steps cannot be nested: a nested resume point would be
+    # unreachable when its enclosing step is skipped on retry. Persisted
+    # checkpoints always use string names.
+    # Exceptions propagate immediately, just as they do outside a step.
+    def resumable_step(name, &block)
+      run_resumable_step(name.to_s, cursor: false, default: nil, &block)
+    end
+
+    # Runs a named step with the cursor saved by #resumable_set_cursor during a
+    # previous failed attempt. Names accept symbols or strings.
+    def resumable_step_cursor(name, default: nil, &block)
+      run_resumable_step(name.to_s, cursor: true, default: default, &block)
+    end
+
+    RESUMABLE_CURSOR_UNSET = Object.new.freeze
+    private_constant :RESUMABLE_CURSOR_UNSET
+
+    private def initialize_resumable_state
+      @resumable_state = ResumableState.new(row.metadata)
+    end
+
+    private def run_resumable_step(name, cursor:, default:)
+      raise Error, "resumable steps cannot be nested" if @resumable_state.step_name
+      raise ArgumentError, "resumable step name must be non-empty" if name.empty?
+      @resumable_state.register(name)
+
+      unless @resumable_state.resume_matched
+        if name == @resumable_state.resume_step
+          @resumable_state.completed_step = name
+          @resumable_state.resume_matched = true
+          return unless cursor && @resumable_state.cursors.key?(name)
+        else
+          return
+        end
+      end
+
+      @resumable_state.step_name = name
+      begin
+        value = (cursor && @resumable_state.cursors.key?(name)) ? @resumable_state.cursors[name] : default
+        result = cursor ? yield(value) : yield
+        @resumable_state.completed_step = name
+        if cursor && @resumable_state.cursors.key?(name)
+          @resumable_state.cursors.delete(name)
+          @resumable_state.cursors_dirty = true
+        end
+
+        result
+      ensure
+        @resumable_state.step_name = nil
+      end
+    end
+  end
+end

data/lib/riverqueue/testing/minitest.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require "minitest"
+require "riverqueue/testing"
+
+module River
+  module Testing
+    # Include in Minitest::Test (or an individual test class) to use River
+    # assertions with Minitest's failure reporting and assertion count.
+    module Minitest
+      include Assertions
+
+      private def river_assert(condition, message)
+        assert(condition, message)
+      end
+    end
+  end
+end