solid_objects 0.15.2 → 0.16.0

data/lib/solid_objects/configuration.rb

@@ -31,6 +31,10 @@ module SolidObjects
     # @rbs @instance_retention_by_actor_type: Hash[String, Numeric]
     # @rbs @process_retention: Numeric
     # @rbs @prune_batch_size: Integer
+    # @rbs @retained_idempotency_keys: Integer
+    # @rbs @retained_idempotency_keys_bytes: Integer
+    # @rbs @redrive_batch_size: Integer
+    # @rbs @redrive_batch_pause: Float
     # @rbs @worker_count: Integer
     # @rbs @effect_worker_count: Integer
     # @rbs @broadcast_worker_count: Integer
@@ -49,6 +53,7 @@ module SolidObjects
     # @rbs @authorize_subscription: Proc
     # @rbs @authorize_administration: Proc
     # @rbs @authorize_transmission: Proc
+    # @rbs @administration_identity: Proc
     # @rbs @transmission_actor_type_resolver: Proc
 
     attr_accessor :table_name_prefix,
@@ -80,6 +85,10 @@ module SolidObjects
       :instance_retention_by_actor_type,
       :process_retention,
       :prune_batch_size,
+      :retained_idempotency_keys,
+      :retained_idempotency_keys_bytes,
+      :redrive_batch_size,
+      :redrive_batch_pause,
       :worker_count,
       :effect_worker_count,
       :broadcast_worker_count,
@@ -98,6 +107,7 @@ module SolidObjects
       :authorize_subscription,
       :authorize_administration,
       :authorize_transmission,
+      :administration_identity,
       :transmission_actor_type_resolver
 
     # @rbs @additional_components: Array[untyped]
@@ -134,6 +144,10 @@ module SolidObjects
       @instance_retention_by_actor_type = {}
       @process_retention = 7.days
       @prune_batch_size = 1_000
+      @retained_idempotency_keys = 64
+      @retained_idempotency_keys_bytes = 16.kilobytes
+      @redrive_batch_size = 100
+      @redrive_batch_pause = 0.05
       @worker_count = 1
       @effect_worker_count = 1
       @broadcast_worker_count = 1
@@ -141,7 +155,7 @@ module SolidObjects
       @connects_to = nil
       @stream_signing_secret = nil
       @broadcast_adapter = nil
-      @wake_up_adapter = nil
+      @wake_up_adapter = :automatic
       @component_path_resolver = nil
       @component_authorization_context = ->(controller:) { controller }
       @payload_authorization_context = ->(connection:) { connection }
@@ -156,6 +170,7 @@ module SolidObjects
       @authorize_subscription = ->(**) { false }
       @authorize_administration = ->(**) { false }
       @authorize_transmission = ->(**) { false }
+      @administration_identity = ->(authorization_context) { authorization_context.to_s }
       @transmission_actor_type_resolver = ->(actor_type) { actor_type }
       @additional_components = []
     end
@@ -226,6 +241,9 @@ module SolidObjects
       positive_values.each do |name, value|
         raise ArgumentError, "#{name} must be positive" unless value.positive?
       end
+      if redrive_batch_pause.negative?
+        raise ArgumentError, "redrive_batch_pause must not be negative"
+      end
       if warn_state_bytes > max_state_bytes
         raise ArgumentError, "warn_state_bytes must not exceed max_state_bytes"
       end
@@ -237,6 +255,7 @@ module SolidObjects
         raise ArgumentError, "actor type cannot be empty" if actor_type.to_s.empty?
         raise ArgumentError, "instance retention must be positive" unless retention.positive?
       end
+      validate_wake_up_adapter!
       unless component_path_resolver.nil? || component_path_resolver.respond_to?(:call)
         raise ArgumentError, "component_path_resolver must respond to call"
       end
@@ -246,12 +265,35 @@ module SolidObjects
       unless payload_authorization_context.respond_to?(:call)
         raise ArgumentError, "payload_authorization_context must respond to call"
       end
+      unless administration_identity.respond_to?(:call)
+        raise ArgumentError, "administration_identity must respond to call"
+      end
 
       self
     end
 
     private
 
+    # @rbs () -> void
+    def validate_wake_up_adapter!
+      return if wake_up_adapter.nil?
+      return validate_wake_up_object! unless wake_up_adapter.is_a?(Symbol)
+      return if WakeUpAdapters::NAMES.include?(wake_up_adapter)
+
+      raise ArgumentError,
+        "unknown wake_up_adapter #{wake_up_adapter.inspect}, " \
+        "expected one of #{WakeUpAdapters::NAMES.join(", ")} or an adapter"
+    end
+
+    # @rbs () -> void
+    def validate_wake_up_object!
+      %i[signal wait].each do |method_name|
+        next if wake_up_adapter.respond_to?(method_name)
+
+        raise ArgumentError, "wake_up_adapter must respond to #{method_name}"
+      end
+    end
+
     # @rbs () -> Hash[Symbol, Numeric]
     def positive_values
       {
@@ -274,7 +316,10 @@ module SolidObjects
         shutdown_timeout:,
         message_retention:,
         process_retention:,
-        prune_batch_size:
+        prune_batch_size:,
+        retained_idempotency_keys:,
+        retained_idempotency_keys_bytes:,
+        redrive_batch_size:
       }
     end
 

data/lib/solid_objects/context.rb

@@ -5,7 +5,7 @@ module SolidObjects
 
   module Context
     STORAGE_KEY = :solid_objects_context
-    Frame = Data.define(:actor, :message, :authorization_context)
+    Frame = Data.define(:actor, :message, :authorization_context, :instance_id)
 
     class << self
       # @rbs () -> Frame?
@@ -28,10 +28,16 @@ module SolidObjects
         current&.authorization_context
       end
 
-      # @rbs (actor: Actor?, message: MessageContext?, authorization_context: untyped) { () -> untyped } -> untyped
-      def with(actor:, message:, authorization_context: nil)
+      # @rbs () -> Integer?
+      def current_instance_id
+        current&.instance_id
+      end
+
+      # @rbs (actor: Actor?, message: MessageContext?, ?authorization_context: untyped, ?instance_id: Integer?) { () -> untyped } -> untyped
+      def with(actor:, message:, authorization_context: nil, instance_id: nil)
         previous = current
-        ActiveSupport::IsolatedExecutionState[STORAGE_KEY] = Frame.new(actor:, message:, authorization_context:)
+        ActiveSupport::IsolatedExecutionState[STORAGE_KEY] =
+          Frame.new(actor:, message:, authorization_context:, instance_id:)
         yield
       ensure
         ActiveSupport::IsolatedExecutionState[STORAGE_KEY] = previous

data/lib/solid_objects/dead_letter_manager.rb

@@ -11,10 +11,48 @@ module SolidObjects
     # @rbs (Integer, ?authorization_context: untyped) -> MessageReference
     def retry(dead_letter_id, authorization_context: nil)
       authorize!(:retry, authorization_context:, dead_letter_id:)
-      dead_letter = DeadLetter.find(dead_letter_id)
-      return MessageReference.from_message(Message.find(dead_letter.retried_message_id)) if dead_letter.retried_message_id
+      actor = AdministrationAudit.identity(authorization_context)
+      SolidObjects.database_adapter.transaction do
+        dead_letter = DeadLetter.find(dead_letter_id)
+        reference = if dead_letter.retried_message_id
+          MessageReference.from_message(Message.find(dead_letter.retried_message_id))
+        else
+          enqueue_retry(dead_letter)
+        end
+        AdministrationAudit.record(
+          action: "dead_letter.retry",
+          kind: "message",
+          subject_id: dead_letter.id,
+          actor:
+        )
+        reference
+      end
+    end
+
+    # @rbs () -> DeadLetterScope
+    def effects
+      @effects ||= DeadLetterScope.new(
+        model: Effect,
+        resource: "effect_dead_letters",
+        identifier: :effect_id,
+        kind: "effect"
+      )
+    end
 
-      original_message = dead_letter.message
+    # @rbs () -> DeadLetterScope
+    def broadcasts
+      @broadcasts ||= DeadLetterScope.new(
+        model: Broadcast,
+        resource: "broadcast_dead_letters",
+        identifier: :broadcast_id,
+        kind: "broadcast"
+      )
+    end
+
+    private
+
+    # @rbs (DeadLetter) -> MessageReference
+    def enqueue_retry(dead_letter)
       message_reference = Mailbox.new.enqueue(
         reference: Reference.new(
           actor_type: dead_letter.actor_type,
@@ -22,15 +60,13 @@ module SolidObjects
         ),
         operation: dead_letter.operation,
         arguments: dead_letter.arguments,
-        delivery_mode: original_message.delivery_mode,
+        delivery_mode: dead_letter.message.delivery_mode,
         idempotency_key: "dead-letter:#{dead_letter.id}"
       )
       dead_letter.update!(retried_message_id: message_reference.id)
       message_reference
     end
 
-    private
-
     # @rbs (Symbol, authorization_context: untyped, ?dead_letter_id: Integer?) -> void
     def authorize!(action, authorization_context:, dead_letter_id: nil)
       authorized = SolidObjects.configuration.authorize_administration.call(

data/lib/solid_objects/dead_letter_scope.rb

@@ -0,0 +1,143 @@
+# rbs_inline: enabled
+
+module SolidObjects
+  class DeadLetterScope
+    DEAD = "dead"
+    PENDING = "pending"
+
+    # @rbs @model: untyped
+    # @rbs @resource: String
+    # @rbs @identifier: Symbol
+
+    attr_reader :resource, :kind
+
+    # @rbs (model: untyped, resource: String, identifier: Symbol, kind: String) -> void
+    def initialize(model:, resource:, identifier:, kind:)
+      @model = model
+      @resource = resource
+      @identifier = identifier
+      @kind = kind
+    end
+
+    # @rbs (String) -> DeadLetterScope
+    def self.for_kind(kind)
+      return SolidObjects.dead_letters.effects if kind == "effect"
+      return SolidObjects.dead_letters.broadcasts if kind == "broadcast"
+
+      raise ArgumentError, "unknown dead letter kind #{kind.inspect}"
+    end
+
+    # @rbs (?authorization_context: untyped) -> Array[DeadRow]
+    def all(authorization_context: nil)
+      authorize!(:inspect, authorization_context:)
+      dead.includes(:instance).order(updated_at: :desc, id: :desc).map do |row|
+        DeadRow.new(
+          id: row.public_send(identifier),
+          kind:,
+          actor_type: row.instance.actor_type,
+          actor_id: row.instance.actor_id,
+          status: row.status,
+          attempt_count: row.attempt_count,
+          available_at: row.available_at,
+          failed_at: row.updated_at,
+          error: row.error
+        )
+      end
+    end
+
+    # @rbs (String, ?authorization_context: untyped) -> untyped
+    def retry(identifier_value, authorization_context: nil)
+      authorize!(:retry, authorization_context:, resource_id: identifier_value)
+      actor = AdministrationAudit.identity(authorization_context)
+      SolidObjects.database_adapter.transaction do
+        row = model.find_by!(identifier => identifier_value)
+        revive(row) if row.status == DEAD
+        AdministrationAudit.record(
+          action: "dead_letter.retry",
+          kind: kind,
+          subject_id: identifier_value,
+          actor:
+        )
+        row
+      end
+    end
+
+    # @rbs (?actor_type: String?, ?failed_after: untyped, ?limit: Integer?, ?authorization_context: untyped) -> RedriveTask
+    def redrive(actor_type: nil, failed_after: nil, limit: nil, authorization_context: nil)
+      if failed_after && !failed_after.respond_to?(:utc)
+        raise ArgumentError, "failed_after must be a time"
+      end
+      if limit && !(limit.is_a?(Integer) && limit.positive?)
+        raise ArgumentError, "limit must be a positive integer"
+      end
+
+      SolidObjects.redrives.start(
+        scope: self,
+        filters: {
+          "actor_type" => actor_type,
+          "failed_after" => failed_after&.utc&.iso8601(6),
+          "limit" => limit
+        },
+        authorization_context:
+      )
+    end
+
+    # @rbs () -> ActiveRecord::Relation[untyped]
+    def dead
+      model.where(status: DEAD)
+    end
+
+    # @rbs (Hash[String, untyped], ?dead_before: untyped) -> ActiveRecord::Relation[untyped]
+    def matching(filters, dead_before: nil)
+      relation = dead
+      actor_type = filters["actor_type"]
+      failed_after = filters["failed_after"]
+      relation = relation.joins(:instance).where(Instance.table_name => { actor_type: }) if actor_type
+      relation = relation.where(updated_at: Time.parse(failed_after)..) if failed_after
+      relation = relation.where(updated_at: ..dead_before) if dead_before
+      relation
+    end
+
+    # @rbs (Array[untyped]) -> Integer
+    def revive_all(identifiers)
+      model.where(id: identifiers, status: DEAD).update_all(revival_attributes)
+    end
+
+    # @rbs (untyped) -> Integer
+    def revive(row)
+      model.where(id: row.id, status: DEAD).update_all(revival_attributes).tap do
+        row.reload
+      end
+    end
+
+    # @rbs (Symbol, authorization_context: untyped, ?resource_id: String?) -> void
+    def authorize!(action, authorization_context:, resource_id: nil)
+      authorized = SolidObjects.configuration.authorize_administration.call(
+        action: action.to_s,
+        resource: resource,
+        resource_id: resource_id,
+        authorization_context:
+      )
+      return if authorized
+
+      raise Unauthorized, "actor administration is not authorized"
+    end
+
+    private
+
+    attr_reader :model, :identifier
+
+    # @rbs () -> Hash[Symbol, untyped]
+    def revival_attributes
+      now = SolidObjects.database_adapter.database_now
+      {
+        status: PENDING,
+        attempt_count: 0,
+        available_at: now,
+        claimed_by: nil,
+        claimed_at: nil,
+        updated_at: now
+      }
+    end
+  end
+end

data/lib/solid_objects/dead_row.rb

@@ -0,0 +1,15 @@
+# rbs_inline: enabled
+
+module SolidObjects
+  DeadRow = Data.define(
+    :id,
+    :kind,
+    :actor_type,
+    :actor_id,
+    :status,
+    :attempt_count,
+    :available_at,
+    :failed_at,
+    :error
+  )
+end

data/lib/solid_objects/doctor.rb

@@ -65,11 +65,11 @@ module SolidObjects
       instances: %w[
         id actor_type actor_id state state_version next_message_sequence
         activation_owner_id activation_token activation_expires_at
-        activation_generation
+        activation_generation state_revision completed_idempotency_keys
       ],
       messages: %w[
-        id instance_id delivery_mode arguments sequence attempt_count request_id
-        result error rejection completed_at rejected_at
+        id instance_id operation delivery_mode arguments sequence attempt_count
+        request_id result error rejection completed_at rejected_at
       ],
       ready_messages: %w[id message_id instance_id sequence available_at],
       claimed_messages: %w[
@@ -77,10 +77,15 @@ module SolidObjects
         activation_generation claimed_at
       ],
       reminders: %w[id instance_id operation next_run_at status],
-      effects: %w[id message_id instance_id effect_id status available_at],
+      effects: %w[
+        id message_id instance_id effect_id status available_at
+        success_operation failure_operation
+      ],
       effect_recoveries: %w[effect_id instance_id recovery_operation status_operation recovery_timeout retired_at],
       broadcasts: %w[id message_id instance_id broadcast_id status available_at],
-      dead_letters: %w[id message_id instance_id actor_type actor_id attempts]
+      dead_letters: %w[id message_id instance_id actor_type actor_id operation attempts],
+      administration_events: %w[id action kind subject_id actor occurred_at],
+      redrives: %w[id kind filters status active_scope moved move_limit started_at finished_at]
     }.freeze
 
     class ProbeActor < Actor
@@ -113,6 +118,7 @@ module SolidObjects
         schema_check,
         check_authorization,
         check_database_server,
+        check_wake_up,
         schema_check.failed? ? skipped_runtime : check_runtime,
         ready_for_round_trip?(configuration_check, schema_check) ?
           check_sync_round_trip :
@@ -208,6 +214,19 @@ module SolidObjects
       warn_check(:database_server, "#{error.class}: #{error.message}")
     end
 
+    # @rbs () -> Check
+    def check_wake_up
+      capability = SolidObjects.wake_up.capability
+      floor = capability.measured_floor_ms
+      summary = "#{capability.adapter}: #{capability.reason}"
+      summary += ", floor #{floor} ms" if floor
+      return pass(:wake_up, summary) if capability.crosses_processes
+
+      warn_check(:wake_up, "#{summary}; a commit in one process cannot wake another")
+    rescue => error
+      warn_check(:wake_up, "#{error.class}: #{error.message}")
+    end
+
     # @rbs () -> Check
     def check_runtime
       cutoff = SolidObjects.database_adapter.database_now -

data/lib/solid_objects/errors.rb

@@ -263,4 +263,15 @@ module SolidObjects
 
   class ActorCallCycle < Error
   end
+
+  class MessagePruned < Error
+    # @rbs @idempotency_key: String
+    attr_reader :idempotency_key
+
+    # @rbs (String) -> void
+    def initialize(idempotency_key)
+      @idempotency_key = idempotency_key
+      super("the message for idempotency key #{idempotency_key.inspect} was pruned")
+    end
+  end
 end

data/lib/solid_objects/executor.rb

@@ -111,7 +111,8 @@ module SolidObjects
           state: state_after.value,
           state_version: actor.class.state_version,
           state_revision: locked_message.sequence,
-          last_used_at: SolidObjects.database_adapter.database_now
+          last_used_at: SolidObjects.database_adapter.database_now,
+          completed_idempotency_keys: remembered_keys(instance, locked_message)
         )
         locked_message.update!(
           result: serialized_result,
@@ -280,6 +281,8 @@ module SolidObjects
     # @rbs (Instance, Array[Actor::ReminderIntent]) -> Array[Hash[Symbol, untyped]]
     def schedule_reminders(instance, intents)
       intents.filter_map do |intent|
+        next cancel_reminder(instance, intent) unless intent.is_a?(Actor::ReminderIntent)
+
         reminder = Reminder.find_or_initialize_by(instance:, name: intent.name)
         previous_run_at = reminder.next_run_at
         reminder.assign_attributes(
@@ -300,6 +303,21 @@ module SolidObjects
       end
     end
 
+    # A cancel reports nothing, because a reminder that no longer exists did not
+    # move. Deleting the row rather than marking it keeps a later schedule of the
+    # same name free of a tombstone.
+    # @rbs (Instance, Actor::UnscheduleIntent | Actor::UnscheduleAllIntent) -> nil
+    def cancel_reminder(instance, intent)
+      scope = Reminder.where(instance:)
+      scope = if intent.is_a?(Actor::UnscheduleAllIntent)
+        scope.where(operation: intent.operation)
+      else
+        scope.where(name: intent.name)
+      end
+      scope.delete_all
+      nil
+    end
+
     # Arguments are omitted deliberately: a reminder carries application data
     # and this event exists to be logged.
     # @rbs (Reminder, Time?) -> Hash[Symbol, untyped]?
@@ -377,7 +395,7 @@ module SolidObjects
       error_details = serialized_error(error)
       dead = false
 
-      activation.lease.fenced_transaction do
+      activation.lease.fenced_transaction do |instance|
         claimed_message = matching_claim!
         locked_message = Message.lock.find(message.id)
         now = SolidObjects.database_adapter.database_now
@@ -387,6 +405,7 @@ module SolidObjects
         if error.is_a?(NonRetryableError) ||
             locked_message.attempt_count >= locked_message.max_attempts
           create_dead_letter(message: locked_message, error_details:, now:)
+          instance.update!(completed_idempotency_keys: remembered_keys(instance, locked_message)) if locked_message.idempotency_key
           dead = true
         else
           ReadyMessage.create!(
@@ -422,7 +441,10 @@ module SolidObjects
         claimed_message = matching_claim!
         locked_message = Message.lock.find(message.id)
         now = SolidObjects.database_adapter.database_now
-        instance.update!(last_used_at: now)
+        instance.update!(
+          last_used_at: now,
+          completed_idempotency_keys: remembered_keys(instance, locked_message)
+        )
         locked_message.update!(
           result: nil,
           rejection: rejection_data,
@@ -453,6 +475,22 @@ module SolidObjects
       raise LostActivation, "message claim changed"
     end
 
+    # @rbs (Instance, Message) -> Array[Hash[String, untyped]]
+    def remembered_keys(instance, message)
+      remembered = Array(instance.completed_idempotency_keys)
+      key = message.idempotency_key
+      return remembered unless key
+
+      entry = { "key" => key, "operation" => message.operation, "arguments" => message.arguments }
+      return remembered if remembered.last == entry
+
+      kept = (remembered.reject { |value| value["key"] == key } + [ entry ])
+        .last(SolidObjects.configuration.retained_idempotency_keys)
+      limit = SolidObjects.configuration.retained_idempotency_keys_bytes
+      kept.shift while kept.any? && kept.to_json.bytesize > limit
+      kept
+    end
+
     # @rbs (Exception) -> Hash[String, untyped]
     def serialized_error(error)
       Serialization.dump(

data/lib/solid_objects/message_reference.rb

@@ -35,19 +35,26 @@ module SolidObjects
 
     # @rbs () -> String
     def status
-      message = Message.find(id)
-      return "rejected" if message.rejected?
-      return "completed" if message.completed?
-      return "dead" if message.dead?
-      return "claimed" if message.claimed?
-      return "ready" if message.ready?
-
-      "unknown"
+      Message.uncached { status_of(Message.find(id)) }
     end
 
     # @rbs () -> untyped
     def result
-      Message.find(id).result
+      Message.uncached { Message.find(id).result }
+    end
+
+    # @rbs () -> Outcome
+    def outcome
+      Message.uncached do
+        message = Message.find(id)
+        Outcome.new(
+          status: status_of(message),
+          result: Serialization.readonly_copy(message.result),
+          error: ErrorRecord.from(message.error),
+          rejection: RejectionRecord.from(message.rejection),
+          attempts: message.attempt_count
+        )
+      end
     end
 
     # @rbs (?timeout: Numeric, ?authorization_context: untyped) -> untyped
@@ -58,5 +65,18 @@ module SolidObjects
         authorization_context:
       )
     end
+
+    private
+
+    # @rbs (Message) -> String
+    def status_of(message)
+      return "rejected" if message.rejected?
+      return "completed" if message.completed?
+      return "dead" if message.dead?
+      return "claimed" if message.claimed?
+      return "ready" if message.ready?
+
+      "unknown"
+    end
   end
 end

data/lib/solid_objects/outcome.rb

@@ -0,0 +1,31 @@
+# rbs_inline: enabled
+
+module SolidObjects
+  ErrorRecord = Data.define(:class_name, :message, :backtrace) do
+    # @rbs (Hash[String, untyped]?) -> ErrorRecord?
+    def self.from(error)
+      return nil if error.blank?
+
+      new(
+        class_name: error["class"],
+        message: error["message"],
+        backtrace: Serialization.readonly_copy(Array(error["backtrace"]))
+      )
+    end
+  end
+
+  RejectionRecord = Data.define(:code, :message, :details) do
+    # @rbs (Hash[String, untyped]?) -> RejectionRecord?
+    def self.from(rejection)
+      return nil if rejection.blank?
+
+      new(
+        code: rejection["code"],
+        message: rejection["message"],
+        details: Serialization.readonly_copy(rejection["details"])
+      )
+    end
+  end
+
+  Outcome = Data.define(:status, :result, :error, :rejection, :attempts)
+end

data/lib/solid_objects/process_registry.rb

@@ -61,7 +61,9 @@ module SolidObjects
 
       # @rbs () -> void
       def warn_if_polling_is_only_cross_process_wake_up
-        return if SolidObjects.configuration.wake_up_adapter
+        wake_up = SolidObjects.wake_up
+        return unless wake_up.respond_to?(:capability)
+        return if wake_up.capability.crosses_processes
 
         polling_warning_mutex.synchronize do
           return if polling_warning_emitted?