solid_objects 0.15.2 → 0.16.0

data/lib/solid_objects/redrive_manager.rb

@@ -0,0 +1,130 @@
+# rbs_inline: enabled
+
+require "digest"
+
+module SolidObjects
+  class RedriveManager
+    RUNNING = "running"
+    COMPLETED = "completed"
+    CANCELLED = "cancelled"
+
+    # @rbs (scope: DeadLetterScope, filters: Hash[String, untyped], authorization_context: untyped) -> RedriveTask
+    def start(scope:, filters:, authorization_context:)
+      scope.authorize!(:redrive, authorization_context:)
+      active_scope = "#{scope.kind}:#{Digest::SHA256.hexdigest(filters.to_json)}"
+      running = Redrive.find_by(active_scope:)
+      return task_for(running) if running
+
+      task_for(open_task(scope:, filters:, active_scope:, authorization_context:))
+    rescue ActiveRecord::RecordNotUnique
+      task_for(Redrive.find_by!(active_scope:))
+    end
+
+    # @rbs (String, ?authorization_context: untyped) -> RedriveTask
+    def find(id, authorization_context: nil)
+      authorize!(:inspect, authorization_context:, resource_id: id)
+      task_for(Redrive.find(id))
+    end
+
+    # @rbs (?status: Symbol | String | nil, ?authorization_context: untyped) -> Array[RedriveTask]
+    def all(status: nil, authorization_context: nil)
+      authorize!(:inspect, authorization_context:)
+      relation = Redrive.order(started_at: :desc, id: :desc)
+      relation = relation.where(status: status.to_s) if status
+      relation.map { |record| task_for(record) }
+    end
+
+    # @rbs (String, ?authorization_context: untyped) -> RedriveTask
+    def cancel(id, authorization_context: nil)
+      authorize!(:cancel, authorization_context:, resource_id: id)
+      record = Redrive.find(id)
+      return task_for(record) unless record.status == RUNNING
+
+      audit(record, action: "redrive.cancel") if close(record, status: CANCELLED)
+      task_for(record)
+    end
+
+    # @rbs (Redrive, status: String) -> bool
+    def close(record, status:)
+      changed = Redrive.where(id: record.id, status: RUNNING).update_all(
+        status:,
+        active_scope: nil,
+        finished_at: SolidObjects.database_adapter.database_now,
+        updated_at: SolidObjects.database_adapter.database_now
+      )
+      record.reload if changed.positive?
+      changed.positive?
+    end
+
+    # @rbs (Redrive, action: String) -> void
+    def audit(record, action:)
+      AdministrationAudit.record(
+        action:,
+        kind: record.kind,
+        subject_id: record.id,
+        filters: record.filters,
+        actor: record.actor
+      )
+    end
+
+    # @rbs (Redrive) -> RedriveTask
+    def task_for(record)
+      RedriveTask.new(
+        id: record.id,
+        kind: record.kind,
+        filters: Serialization.readonly_copy(record.filters),
+        status: record.status,
+        moved: record.moved,
+        remaining: remaining_for(record),
+        started_at: record.started_at,
+        finished_at: record.finished_at
+      )
+    end
+
+    private
+
+    # @rbs (scope: DeadLetterScope, filters: Hash[String, untyped], active_scope: String, authorization_context: untyped) -> Redrive
+    def open_task(scope:, filters:, active_scope:, authorization_context:)
+      record = Redrive.create!(
+        id: "redrive_#{SecureRandom.uuid}",
+        kind: scope.kind,
+        filters:,
+        status: RUNNING,
+        active_scope:,
+        moved: 0,
+        move_limit: filters["limit"],
+        actor: AdministrationAudit.identity(authorization_context),
+        started_at: SolidObjects.database_adapter.database_now
+      )
+      audit(record, action: "redrive.start")
+      record
+    end
+
+    # @rbs (Redrive) -> Integer
+    def remaining_for(record)
+      return 0 unless record.status == RUNNING
+
+      matching = DeadLetterScope
+        .for_kind(record.kind)
+        .matching(record.filters, dead_before: record.started_at)
+        .count
+      limit = record.move_limit
+      return matching unless limit
+
+      [ matching, limit - record.moved ].min
+    end
+
+    # @rbs (Symbol, authorization_context: untyped, ?resource_id: String?) -> void
+    def authorize!(action, authorization_context:, resource_id: nil)
+      authorized = SolidObjects.configuration.authorize_administration.call(
+        action: action.to_s,
+        resource: "redrives",
+        resource_id:,
+        authorization_context:
+      )
+      return if authorized
+
+      raise Unauthorized, "actor administration is not authorized"
+    end
+  end
+end

data/lib/solid_objects/redrive_runner.rb

@@ -0,0 +1,60 @@
+# rbs_inline: enabled
+
+module SolidObjects
+  class RedriveRunner
+    # @rbs () -> bool
+    def run_once
+      SolidObjects.database_adapter.transaction do
+        record = claim
+        next false unless record
+
+        advance(record)
+      end
+    end
+
+    private
+
+    # @rbs () -> Redrive?
+    def claim
+      relation = Redrive.where(status: RedriveManager::RUNNING).order(:started_at, :id)
+      SolidObjects.database_adapter.lock_candidates(relation).first
+    end
+
+    # @rbs (Redrive) -> bool
+    def advance(record)
+      moved = move_batch(record)
+      return true if moved.positive?
+
+      finish(record)
+      false
+    end
+
+    # @rbs (Redrive) -> Integer
+    def move_batch(record)
+      scope = DeadLetterScope.for_kind(record.kind)
+      configured = SolidObjects.configuration.redrive_batch_size
+      limit = record.move_limit
+      size = limit ? [ configured, limit - record.moved ].min : configured
+      return 0 unless size.positive?
+
+      identifiers = scope
+        .matching(record.filters, dead_before: record.started_at)
+        .order(:id)
+        .limit(size)
+        .pluck(:id)
+      return 0 if identifiers.empty?
+
+      revived = scope.revive_all(identifiers)
+      record.update!(moved: record.moved + revived)
+      revived
+    end
+
+    # @rbs (Redrive) -> void
+    def finish(record)
+      manager = SolidObjects.redrives
+      return unless manager.close(record, status: RedriveManager::COMPLETED)
+
+      manager.audit(record, action: "redrive.finish")
+    end
+  end
+end

data/lib/solid_objects/redrive_task.rb

@@ -0,0 +1,14 @@
+# rbs_inline: enabled
+
+module SolidObjects
+  RedriveTask = Data.define(
+    :id, :kind, :filters, :status, :moved, :remaining, :started_at, :finished_at
+  )
+
+  class RedriveTask
+    # @rbs (?authorization_context: untyped) -> RedriveTask
+    def cancel(authorization_context: nil)
+      SolidObjects.redrives.cancel(id, authorization_context:)
+    end
+  end
+end

data/lib/solid_objects/reference.rb

@@ -67,6 +67,11 @@ module SolidObjects
       SolidObjects.client.destroy(self, authorization_context:)
     end
 
+    # @rbs (idempotency_key: String, ?authorization_context: untyped) -> MessageReference?
+    def find_by(idempotency_key:, authorization_context: nil)
+      SolidObjects.client.find_by(reference: self, idempotency_key:, authorization_context:)
+    end
+
     # @rbs (?authorization_context: untyped) -> StateSnapshot
     def snapshot(authorization_context: nil)
       SolidObjects.client.snapshot(self, authorization_context:)

data/lib/solid_objects/schema_bootstrap.rb

@@ -0,0 +1,27 @@
+# rbs_inline: enabled
+
+require "active_record"
+require "active_support/core_ext/string/inflections"
+
+module SolidObjects
+  module SchemaBootstrap
+    class << self
+      # @rbs (?connection: untyped) -> void
+      def install(connection: nil)
+        migrations.each do |migration_class|
+          migration = migration_class.new
+          migration.define_singleton_method(:connection) { connection } if connection
+          migration.migrate(:up)
+        end
+      end
+
+      # @rbs () -> Array[Class]
+      def migrations
+        Dir[File.expand_path("../../db/migrate/*.rb", __dir__)].sort.map do |file|
+          require file
+          Object.const_get(File.basename(file, ".rb").sub(/\A\d+_/, "").camelize)
+        end
+      end
+    end
+  end
+end

data/lib/solid_objects/supervisor.rb

@@ -31,6 +31,7 @@ module SolidObjects
       @started = false
       @cleaned_up_at = nil
       @retention = nil
+      @redrive = nil
       @lifecycle = Thread::Mutex.new
     end
 
@@ -50,6 +51,7 @@ module SolidObjects
       @threads = components.map { |component| supervise(component) }
       @monitor = Thread.new { monitor_loop }
       @retention = Thread.new { retention_loop }
+      @redrive = Thread.new { redrive_loop }
       SolidObjects.instrument(:"supervisor.started", component_count: components.length)
     end
 
@@ -64,6 +66,7 @@ module SolidObjects
         @lifecycle.synchronize { @started = false }
         stop_monitor
         stop_retention
+        stop_redrive
         components.each(&:request_shutdown)
         join_until_timeout
         components.reject(&:stopped?).each(&:stop)
@@ -204,6 +207,46 @@ module SolidObjects
       [ backoff, interval ].min
     end
 
+    # @rbs () -> void
+    def redrive_loop
+      while @started
+        advance_redrive ? pause_between_batches : wait_for_next_redrive
+      end
+    end
+
+    # @rbs () -> bool
+    def advance_redrive
+      RedriveRunner.new.run_once
+    rescue => error
+      SolidObjects.instrument(
+        :"supervisor.redrive_failed",
+        error_class: error.class.name,
+        error_message: error.message
+      )
+      false
+    end
+
+    # @rbs () -> void
+    def pause_between_batches
+      pause = SolidObjects.configuration.redrive_batch_pause
+      sleep pause if pause.positive?
+    end
+
+    # @rbs () -> void
+    def wait_for_next_redrive
+      sleep SolidObjects.configuration.supervisor_monitor_interval
+    end
+
+    # @rbs () -> void
+    def stop_redrive
+      redrive = @redrive
+      @redrive = nil
+      return unless redrive
+
+      redrive.join(SolidObjects.configuration.shutdown_timeout)
+      redrive.kill if redrive.alive?
+    end
+
     # @rbs () -> void
     def stop_retention
       retention = @retention

data/lib/solid_objects/test_helper.rb

@@ -31,6 +31,8 @@ module SolidObjects
       # @rbs () -> Array[Class]
       def actor_owned_models
         [
+          AdministrationEvent,
+          Redrive,
           DeadLetter,
           ClaimedMessage,
           ReadyMessage,

data/lib/solid_objects/version.rb

@@ -1,5 +1,5 @@
 # rbs_inline: enabled
 
 module SolidObjects
-  VERSION = "0.15.2"
+  VERSION = "0.16.0"
 end

data/lib/solid_objects/wake_up.rb

@@ -2,6 +2,8 @@
 
 module SolidObjects
   class WakeUp
+    include ReportsWakeUpCapability
+
     class Watch
       # @rbs @wake_up: WakeUp
       # @rbs @generation: Integer
@@ -29,6 +31,16 @@ module SolidObjects
       @generation = 0
     end
 
+    # @rbs () -> WakeUpCapability
+    def default_capability
+      WakeUpCapability.new(
+        adapter: :in_process,
+        crosses_processes: false,
+        measured_floor_ms: nil,
+        reason: "in-process signalling, which a commit in another process cannot reach"
+      )
+    end
+
     # @rbs () -> void
     def signal
       mutex.synchronize do

data/lib/solid_objects/wake_up_adapters/postgresql.rb

@@ -10,6 +10,8 @@ module SolidObjects
     # when one is available, so a missed or failed notification costs latency
     # rather than correctness.
     class Postgresql
+      include ReportsWakeUpCapability
+
       CHANNEL = "solid_objects_wake_up"
       FAILED_WAIT_INTERVAL = 0.05
 
@@ -19,6 +21,16 @@ module SolidObjects
 
       attr_reader :channel
 
+      # @rbs () -> WakeUpCapability
+      def default_capability
+        WakeUpCapability.new(
+          adapter: :postgresql_notify,
+          crosses_processes: true,
+          measured_floor_ms: 2.9,
+          reason: "PostgreSQL LISTEN carries the signal between processes"
+        )
+      end
+
       # @rbs (?channel: String) -> void
       def initialize(channel: CHANNEL)
         @channel = channel

data/lib/solid_objects/wake_up_adapters/redis.rb

@@ -12,6 +12,8 @@ module SolidObjects
     # polling interval remains the upper bound, so a missed or failed
     # notification costs latency rather than correctness.
     class Redis
+      include ReportsWakeUpCapability
+
       class Watch
         # @rbs @adapter: Redis
         # @rbs @generation: Integer
@@ -43,6 +45,16 @@ module SolidObjects
 
       attr_reader :channel
 
+      # @rbs () -> WakeUpCapability
+      def default_capability
+        WakeUpCapability.new(
+          adapter: :redis,
+          crosses_processes: true,
+          measured_floor_ms: 5.7,
+          reason: "Redis carries the signal between processes"
+        )
+      end
+
       # @rbs (?channel: String, ?url: String?, ?client: untyped) -> void
       def initialize(channel: CHANNEL, url: nil, client: nil)
         @channel = channel

data/lib/solid_objects/wake_up_adapters.rb

@@ -2,22 +2,217 @@
 
 module SolidObjects
   module WakeUpAdapters
+    POSTGRESQL_FLOOR_MS = 2.9
+    REDIS_FLOOR_MS = 5.7
+    REDIS_URL_VARIABLE = "SOLID_OBJECTS_REDIS_URL"
+    PROBE_TIMEOUT_SECONDS = 2.0
+    PROBE_CHANNEL = "solid_objects_wake_up_probe"
+
+    @pooled_warning_mutex = Thread::Mutex.new
+    @pooled_warning_emitted = false
+
     module_function
 
-    # Returns the best wake-up strategy for a connection: cross-process
-    # notifications where the database provides them, and the in-process
-    # default everywhere else.
-    #
-    # This is deliberately not the default. A notification adapter opens a
-    # connection per waiting thread outside the pool, and `LISTEN` does not
-    # survive a transaction-pooling proxy such as PgBouncer, so adopting it is
-    # a deployment decision rather than an upgrade side effect.
-    #
+    NAMES = %i[automatic in_process postgresql redis].freeze
+
     # @rbs (?untyped) -> untyped
     def for(connection = Record.connection)
-      return Postgresql.new if DatabaseAdapter.family(connection) == :postgresql
+      select(connection)
+    end
+
+    # @rbs (untyped) -> untyped
+    def build(setting)
+      return select if setting.nil? || setting == :automatic
+      return named(setting) if setting.is_a?(Symbol)
+
+      configured(setting)
+    end
+
+    # @rbs (Symbol) -> untyped
+    def named(name)
+      case name
+      when :in_process then labelled(WakeUp.new, :in_process, false, nil, "in-process signalling was requested")
+      when :postgresql then requested_postgresql
+      when :redis then requested_redis
+      else
+        raise ArgumentError, "unknown wake_up_adapter #{name.inspect}, expected one of #{NAMES.join(", ")} or an adapter"
+      end
+    end
+
+    # @rbs () -> untyped
+    def requested_postgresql
+      family = DatabaseAdapter.family(Record.connection)
+      unless family == :postgresql
+        return unavailable_selection(
+          "wake_up_adapter :postgresql needs a database with a notification " \
+          "channel, and #{family || "this database"} provides none"
+        )
+      end
+
+      labelled(Postgresql.new, :postgresql_notify, true, POSTGRESQL_FLOOR_MS, "PostgreSQL LISTEN was requested")
+    end
+
+    # @rbs () -> untyped
+    def requested_redis
+      url = redis_url
+      unless url
+        return unavailable_selection(
+          "wake_up_adapter :redis needs #{REDIS_URL_VARIABLE}, which is not set"
+        )
+      end
+
+      redis_adapter(url, "Redis was requested")
+    end
+
+    # @rbs (String, String) -> untyped
+    def redis_adapter(url, reason)
+      return unavailable_selection("#{REDIS_URL_VARIABLE} is set, and the redis gem is not installed") unless redis_installed?
+
+      labelled(Redis.new(url:), :redis, true, REDIS_FLOOR_MS, reason)
+    end
+
+    # @rbs () -> bool
+    def redis_installed?
+      require "redis"
+      true
+    rescue LoadError
+      false
+    end
+
+    # @rbs (String) -> untyped
+    def unavailable_selection(reason)
+      SolidObjects.configuration.logger.warn(
+        event: "solid_objects.wake_up.unavailable",
+        reason:
+      )
+      polling_adapter(reason)
+    end
+
+    # @rbs (untyped) -> untyped
+    def configured(adapter)
+      return adapter unless adapter.respond_to?(:capability=)
+      return adapter if adapter.respond_to?(:default_capability)
+
+      labelled(adapter, :configured, true, nil, "an adapter was configured, so selection did not run")
+    end
+
+    # @rbs (untyped, Symbol, bool, Numeric?, String) -> untyped
+    def labelled(adapter, name, crosses_processes, floor, reason)
+      adapter.capability = WakeUpCapability.new(
+        adapter: name,
+        crosses_processes:,
+        measured_floor_ms: floor,
+        reason:
+      )
+      adapter
+    end
+
+    # @rbs (?untyped) -> untyped
+    def select(connection = Record.connection)
+      url = redis_url
+      return redis_selection(url) if url
+
+      family = DatabaseAdapter.family(connection)
+      return postgresql_selection if family == :postgresql
+
+      polling_selection(family)
+    end
+
+    # @rbs () -> bool
+    def notifications_deliver?
+      probe = Postgresql.new(channel: PROBE_CHANNEL)
+      return false unless probe.listen
+      return false unless notify_probe_channel
+
+      probe.wait(timeout: PROBE_TIMEOUT_SECONDS)
+    rescue
+      false
+    ensure
+      probe&.stop
+    end
+
+    # @rbs () -> bool
+    def notify_probe_channel
+      connection = Record.connection_pool.send(:new_connection)
+      connection.execute("NOTIFY #{connection.quote_table_name(PROBE_CHANNEL)}")
+      true
+    ensure
+      disconnect_probe(connection)
+    end
+
+    # @rbs (untyped) -> void
+    def disconnect_probe(connection)
+      connection&.disconnect!
+    rescue
+      nil
+    end
+
+    # @rbs () -> void
+    def reset_pooled_warning!
+      @pooled_warning_mutex.synchronize { @pooled_warning_emitted = false }
+    end
+
+    # @rbs () -> String?
+    def redis_url
+      value = ENV[REDIS_URL_VARIABLE].to_s
+      value.empty? ? nil : value
+    end
+
+    # @rbs (String) -> untyped
+    def redis_selection(url)
+      redis_adapter(
+        url,
+        "#{REDIS_URL_VARIABLE} is set, so Redis carries the signal between processes"
+      )
+    end
+
+    # @rbs () -> untyped
+    def postgresql_selection
+      return pooled_selection unless notifications_deliver?
+
+      labelled(
+        Postgresql.new, :postgresql_notify, true, POSTGRESQL_FLOOR_MS,
+        "a probe notification arrived, so PostgreSQL LISTEN carries the signal between processes"
+      )
+    end
+
+    # @rbs () -> untyped
+    def pooled_selection
+      warn_pooled_session_once
+      polling_adapter(
+        "a probe notification did not arrive, so LISTEN cannot carry the signal " \
+        "between processes; a transaction pooler such as PgBouncer is the usual cause"
+      )
+    end
+
+    # @rbs (Symbol?) -> untyped
+    def polling_selection(family)
+      polling_adapter(
+        "#{family || "this database"} has no notification channel and " \
+        "#{REDIS_URL_VARIABLE} is not set"
+      )
+    end
+
+    # @rbs (String) -> untyped
+    def polling_adapter(reason)
+      labelled(
+        WakeUp.new, :polling, false,
+        SolidObjects.configuration.idle_polling_interval * 1_000, reason
+      )
+    end
+
+    # @rbs () -> void
+    def warn_pooled_session_once
+      @pooled_warning_mutex.synchronize do
+        return if @pooled_warning_emitted
 
-      WakeUp.new
+        SolidObjects.configuration.logger.warn(
+          event: "solid_objects.wake_up.pooled_session",
+          reason: "PostgreSQL notifications were not selected because a probe " \
+            "notification did not arrive"
+        )
+        @pooled_warning_emitted = true
+      end
     end
   end
 end

data/lib/solid_objects/wake_up_capability.rb

@@ -0,0 +1,15 @@
+# rbs_inline: enabled
+
+module SolidObjects
+  WakeUpCapability = Data.define(:adapter, :crosses_processes, :measured_floor_ms, :reason)
+
+  module ReportsWakeUpCapability
+    # @rbs (WakeUpCapability) -> void
+    attr_writer :capability
+
+    # @rbs () -> WakeUpCapability
+    def capability
+      @capability ||= default_capability
+    end
+  end
+end