solid_objects 0.2.1 → 0.3.0

data/lib/solid_objects/client.rb

@@ -50,16 +50,57 @@ module SolidObjects
         arguments,
         authorization_context:
       )
-      message_reference = mailbox.enqueue(
+      reject_sync_inside_transaction!(reference, message_name)
+      SyncDeadline.with(timeout:) do
+        message_reference = enqueue_sync(
+          reference,
+          message_name,
+          arguments,
+          idempotency_key:,
+          timeout:
+        )
+        SynchronousInvocation.new.call(message_reference, timeout:)
+      end
+    end
+
+    # @rbs (MessageReference, timeout: Numeric, ?authorization_context: untyped) -> untyped
+    def wait(message_reference, timeout:, authorization_context: nil)
+      message = Message.find(message_reference.id)
+      validate_message_reference!(message_reference, message)
+      reference = Reference.new(
+        actor_type: message.actor_type,
+        actor_id: message.actor_id
+      )
+      actor_class = SolidObjects.registry.fetch(reference.actor_type)
+      query = actor_class.definition.queries.key?(message.message_name.to_sym)
+      actor_message = actor_class.definition.messages.key?(message.message_name.to_sym)
+      unless query || actor_message
+        raise UnknownMessage, "unknown message #{message.message_name.inspect}"
+      end
+      authorize!(
+        query ? SolidObjects.configuration.authorize_query : SolidObjects.configuration.authorize_message,
         reference,
-        message_name,
-        arguments,
-        kind: "sync",
-        idempotency_key:
+        message.message_name,
+        message.arguments,
+        authorization_context:
       )
+      reject_sync_inside_transaction!(reference, message.message_name)
       SynchronousInvocation.new.call(message_reference, timeout:)
     end
 
+    # @rbs (Reference, ?authorization_context: untyped) -> StateSnapshot
+    def snapshot(reference, authorization_context: nil)
+      SolidObjects.registry.fetch(reference.actor_type)
+      authorize!(
+        SolidObjects.configuration.authorize_query,
+        reference,
+        "__snapshot__",
+        {},
+        authorization_context:
+      )
+      StateSnapshot.new(reference)
+    end
+
     # @rbs (Reference, ?authorization_context: untyped) -> bool
     def destroy(reference, authorization_context: nil)
       raise ActorCallCycle, "actors cannot synchronously destroy another actor" if Context.current_actor
@@ -93,6 +134,58 @@ module SolidObjects
 
     attr_reader :mailbox
 
+    # @rbs (Reference, Symbol | String, Hash[Symbol | String, untyped], idempotency_key: String?, timeout: Numeric) -> MessageReference
+    def enqueue_sync(reference, message_name, arguments, idempotency_key:, timeout:)
+      mailbox.enqueue(
+        reference,
+        message_name,
+        arguments,
+        kind: "sync",
+        idempotency_key:
+      )
+    rescue DatabaseDeadlineExceeded
+      SolidObjects.instrument(
+        :"sync.enqueue_timeout",
+        actor_type: reference.actor_type,
+        actor_id: reference.actor_id,
+        message_name: message_name.to_s
+      )
+      raise SyncEnqueueTimeout.new(
+        timeout:,
+        actor_type: reference.actor_type,
+        actor_id: reference.actor_id,
+        message_name: message_name.to_s
+      )
+    end
+
+    # @rbs (MessageReference, Message) -> void
+    def validate_message_reference!(message_reference, message)
+      valid = message_reference.request_id == message.request_id &&
+        message_reference.actor_type == message.actor_type &&
+        message_reference.actor_id == message.actor_id &&
+        message_reference.sequence == message.sequence
+      return if valid
+
+      raise ActorDestroyed, "message reference no longer identifies this invocation"
+    end
+
+    # @rbs (Reference, Symbol | String) -> void
+    def reject_sync_inside_transaction!(reference, message_name)
+      return unless SolidObjects::Record.connection.transaction_open?
+
+      SolidObjects.instrument(
+        :"sync.transaction_rejected",
+        actor_type: reference.actor_type,
+        actor_id: reference.actor_id,
+        message_name: message_name.to_s
+      )
+      raise SyncInsideTransaction.new(
+        actor_type: reference.actor_type,
+        actor_id: reference.actor_id,
+        message_name: message_name.to_s
+      )
+    end
+
     # @rbs (Proc, Reference, Symbol | String, Hash[Symbol | String, untyped], authorization_context: untyped) -> void
     def authorize!(hook, reference, message_name, arguments, authorization_context:)
       authorized = hook.call(

data/lib/solid_objects/commit_action_registry.rb

@@ -0,0 +1,42 @@
+# rbs_inline: enabled
+
+module SolidObjects
+  CommitActionContext = Data.define(
+    :message_id,
+    :request_id,
+    :actor_type,
+    :actor_id,
+    :activation_generation
+  )
+
+  class CommitActionRegistry
+    # @rbs @handlers: Hash[String, Proc]
+    # @rbs @mutex: Mutex
+
+    # @rbs () -> void
+    def initialize
+      @handlers = {}
+      @mutex = Mutex.new
+    end
+
+    # @rbs (String | Symbol, Proc) -> Proc
+    def register(name, handler)
+      action_name = name.to_s
+      raise ArgumentError, "commit action name cannot be empty" if action_name.empty?
+
+      mutex.synchronize { handlers[action_name] = handler }
+      handler
+    end
+
+    # @rbs (String | Symbol) -> Proc
+    def fetch(name)
+      handlers.fetch(name.to_s) do
+        raise UnknownCommitAction, "no commit action registered for #{name.inspect}"
+      end
+    end
+
+    private
+
+    attr_reader :handlers, :mutex
+  end
+end

data/lib/solid_objects/configuration.rb

@@ -20,6 +20,11 @@ module SolidObjects
     # @rbs @process_heartbeat_interval: Float
     # @rbs @process_alive_threshold: Float
     # @rbs @shutdown_timeout: Float
+    # @rbs @message_retention: Numeric
+    # @rbs @message_retention_by_actor_type: Hash[String, Numeric]
+    # @rbs @instance_retention_by_actor_type: Hash[String, Numeric]
+    # @rbs @process_retention: Numeric
+    # @rbs @prune_batch_size: Integer
     # @rbs @worker_count: Integer
     # @rbs @effect_worker_count: Integer
     # @rbs @broadcast_worker_count: Integer
@@ -53,6 +58,11 @@ module SolidObjects
       :process_heartbeat_interval,
       :process_alive_threshold,
       :shutdown_timeout,
+      :message_retention,
+      :message_retention_by_actor_type,
+      :instance_retention_by_actor_type,
+      :process_retention,
+      :prune_batch_size,
       :worker_count,
       :effect_worker_count,
       :broadcast_worker_count,
@@ -88,6 +98,11 @@ module SolidObjects
       @process_heartbeat_interval = 15.0
       @process_alive_threshold = 60.0
       @shutdown_timeout = 15.0
+      @message_retention = 30.days
+      @message_retention_by_actor_type = {}
+      @instance_retention_by_actor_type = {}
+      @process_retention = 7.days
+      @prune_batch_size = 1_000
       @worker_count = 1
       @effect_worker_count = 1
       @broadcast_worker_count = 1
@@ -128,6 +143,14 @@ module SolidObjects
       positive_values.each do |name, value|
         raise ArgumentError, "#{name} must be positive" unless value.positive?
       end
+      message_retention_by_actor_type.each do |actor_type, retention|
+        raise ArgumentError, "actor type cannot be empty" if actor_type.to_s.empty?
+        raise ArgumentError, "message retention must be positive" unless retention.positive?
+      end
+      instance_retention_by_actor_type.each do |actor_type, retention|
+        raise ArgumentError, "actor type cannot be empty" if actor_type.to_s.empty?
+        raise ArgumentError, "instance retention must be positive" unless retention.positive?
+      end
 
       self
     end
@@ -151,7 +174,10 @@ module SolidObjects
         max_attempts:,
         process_heartbeat_interval:,
         process_alive_threshold:,
-        shutdown_timeout:
+        shutdown_timeout:,
+        message_retention:,
+        process_retention:,
+        prune_batch_size:
       }
     end
 

data/lib/solid_objects/database_adapter.rb

@@ -54,9 +54,22 @@ module SolidObjects
 
     # @rbs () { () -> untyped } -> untyped
     def transaction(&block)
+      raise DatabaseDeadlineExceeded, "synchronous invocation deadline expired" if SyncDeadline.expired?
+
       with_connection do |connection|
-        connection.transaction(requires_new: true, &block)
+        with_transaction_deadline(connection) do
+          connection.transaction(requires_new: true) do
+            configure_transaction_deadline(connection)
+            block.call
+          end
+        end
       end
+    rescue => error
+      raise unless deadline_error?(error)
+
+      raise DatabaseDeadlineExceeded,
+        "database lock wait exceeded the synchronous invocation deadline",
+        cause: error
     end
 
     # @rbs (ActiveRecord::Relation[untyped]) -> ActiveRecord::Relation[untyped]
@@ -68,6 +81,20 @@ module SolidObjects
 
     attr_reader :connection_pool, :fixed_connection
 
+    # @rbs (untyped) { () -> untyped } -> untyped
+    def with_transaction_deadline(_connection)
+      yield
+    end
+
+    # @rbs (untyped) -> void
+    def configure_transaction_deadline(_connection)
+    end
+
+    # @rbs (Exception) -> bool
+    def deadline_error?(_error)
+      false
+    end
+
     # @rbs () { (untyped) -> untyped } -> untyped
     def with_connection
       return yield fixed_connection unless connection_pool

data/lib/solid_objects/database_adapters/mysql.rb

@@ -17,6 +17,52 @@ module SolidObjects
       def current_time_expression
         "CURRENT_TIMESTAMP(6)"
       end
+
+      private
+
+      # @rbs (untyped) { () -> untyped } -> untyped
+      def with_transaction_deadline(connection)
+        return yield unless SyncDeadline.active?
+
+        previous_lock_timeout = connection.select_value(
+          "SELECT @@SESSION.innodb_lock_wait_timeout"
+        ).to_i
+        previous_execution_timeout = connection.select_value(
+          "SELECT @@SESSION.max_execution_time"
+        ).to_i
+        connection.execute(
+          "SET SESSION innodb_lock_wait_timeout = #{SyncDeadline.remaining_seconds}"
+        )
+        connection.execute(
+          "SET SESSION max_execution_time = #{SyncDeadline.remaining_milliseconds}"
+        )
+        yield
+      ensure
+        if previous_lock_timeout
+          connection.execute(
+            "SET SESSION innodb_lock_wait_timeout = #{previous_lock_timeout}"
+          )
+        end
+        if previous_execution_timeout
+          connection.execute(
+            "SET SESSION max_execution_time = #{previous_execution_timeout}"
+          )
+        end
+      end
+
+      # @rbs (Exception) -> bool
+      def deadline_error?(error)
+        return false unless SyncDeadline.active?
+        return true if error.is_a?(ActiveRecord::LockWaitTimeout)
+
+        cause = error
+        while cause
+          return true if cause.respond_to?(:error_number) && cause.error_number == 3024
+
+          cause = cause.cause
+        end
+        false
+      end
     end
   end
 end

data/lib/solid_objects/database_adapters/postgresql.rb

@@ -12,6 +12,35 @@ module SolidObjects
       def claim_lock
         "FOR UPDATE SKIP LOCKED"
       end
+
+      private
+
+      # @rbs (untyped) -> void
+      def configure_transaction_deadline(connection)
+        return unless SyncDeadline.active?
+
+        connection.execute(
+          "SET LOCAL lock_timeout = #{SyncDeadline.remaining_milliseconds}"
+        )
+        connection.execute(
+          "SET LOCAL statement_timeout = #{SyncDeadline.remaining_milliseconds}"
+        )
+      end
+
+      # @rbs (Exception) -> bool
+      def deadline_error?(error)
+        return false unless SyncDeadline.active?
+        return true if error.is_a?(ActiveRecord::LockWaitTimeout)
+
+        cause = error
+        while cause
+          result = cause.respond_to?(:result) ? cause.result : nil
+          return true if result&.error_field(PG::Result::PG_DIAG_SQLSTATE) == "57014"
+
+          cause = cause.cause
+        end
+        false
+      end
     end
   end
 end

data/lib/solid_objects/database_adapters/sqlite.rb

@@ -7,6 +7,34 @@ module SolidObjects
       def current_time_expression
         "STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')"
       end
+
+      private
+
+      # @rbs (untyped) { () -> untyped } -> untyped
+      def with_transaction_deadline(connection)
+        return yield unless SyncDeadline.active?
+
+        previous_timeout = connection.select_value("PRAGMA busy_timeout").to_i
+        connection.execute(
+          "PRAGMA busy_timeout = #{SyncDeadline.remaining_milliseconds}"
+        )
+        yield
+      ensure
+        connection.execute("PRAGMA busy_timeout = #{previous_timeout}") if previous_timeout
+      end
+
+      # @rbs (Exception) -> bool
+      def deadline_error?(error)
+        return false unless SyncDeadline.active?
+
+        cause = error
+        while cause
+          return true if cause.class.name.match?(/BusyException|BusyError/)
+
+          cause = cause.cause
+        end
+        false
+      end
     end
   end
 end

data/lib/solid_objects/errors.rb

@@ -4,6 +4,9 @@ module SolidObjects
   class Error < StandardError
   end
 
+  class NonRetryableError < Error
+  end
+
   class UnsupportedDatabase < Error
   end
 
@@ -40,7 +43,148 @@ module SolidObjects
   class ActorDestroyed < LostActivation
   end
 
+  class DatabaseDeadlineExceeded < Error
+  end
+
+  class SyncEnqueueTimeout < Error
+    # @rbs @timeout: Numeric
+    # @rbs @actor_type: String
+    # @rbs @actor_id: String
+    # @rbs @message_name: String
+
+    attr_reader :timeout, :actor_type, :actor_id, :message_name
+
+    # @rbs (timeout: Numeric, actor_type: String, actor_id: String, message_name: String) -> void
+    def initialize(timeout:, actor_type:, actor_id:, message_name:)
+      @timeout = timeout
+      @actor_type = actor_type
+      @actor_id = actor_id
+      @message_name = message_name
+      super(
+        "actor invocation could not be durably enqueued within #{timeout} seconds for " \
+          "#{actor_type}(#{actor_id.inspect}).#{message_name}"
+      )
+    end
+  end
+
   class SyncTimeout < Error
+    # @rbs @timeout: Numeric
+    # @rbs @actor_type: String
+    # @rbs @actor_id: String
+    # @rbs @message_name: String
+    # @rbs @message_id: Integer
+    # @rbs @request_id: String
+    # @rbs @sequence: Integer
+    # @rbs @status: String
+    # @rbs @waiting_on: String
+    # @rbs @activation: Hash[String, untyped]
+    # @rbs @blocker: Hash[String, untyped]?
+
+    attr_reader :timeout,
+      :actor_type,
+      :actor_id,
+      :message_name,
+      :message_id,
+      :request_id,
+      :sequence,
+      :status,
+      :waiting_on,
+      :activation,
+      :blocker
+
+    # @rbs (timeout: Numeric, actor_type: String, actor_id: String, message_name: String, message_id: Integer, request_id: String, sequence: Integer, status: String, waiting_on: String, activation: Hash[String, untyped], blocker: Hash[String, untyped]?) -> void
+    def initialize(
+      timeout:,
+      actor_type:,
+      actor_id:,
+      message_name:,
+      message_id:,
+      request_id:,
+      sequence:,
+      status:,
+      waiting_on:,
+      activation:,
+      blocker:
+    )
+      @timeout = timeout
+      @actor_type = actor_type
+      @actor_id = actor_id
+      @message_name = message_name
+      @message_id = message_id
+      @request_id = request_id
+      @sequence = sequence
+      @status = status
+      @waiting_on = waiting_on
+      @activation = Serialization.readonly_copy(activation)
+      @blocker = Serialization.readonly_copy(blocker)
+      super(
+        "actor invocation timed out after #{timeout} seconds for " \
+          "#{actor_type}(#{actor_id.inspect}).#{message_name} " \
+          "message_id=#{message_id} sequence=#{sequence} status=#{status} waiting_on=#{waiting_on}"
+      )
+    end
+
+    # @rbs () -> MessageReference
+    def message_reference
+      MessageReference.new(
+        id: message_id,
+        request_id:,
+        actor_type:,
+        actor_id:,
+        sequence:
+      )
+    end
+  end
+
+  class SyncInsideTransaction < Error
+    # @rbs @actor_type: String
+    # @rbs @actor_id: String
+    # @rbs @message_name: String
+
+    attr_reader :actor_type, :actor_id, :message_name
+
+    # @rbs (actor_type: String, actor_id: String, message_name: String) -> void
+    def initialize(actor_type:, actor_id:, message_name:)
+      @actor_type = actor_type
+      @actor_id = actor_id
+      @message_name = message_name
+      super(
+        "cannot synchronously invoke #{actor_type}(#{actor_id.inspect}).#{message_name} " \
+          "inside an open database transaction; call it before the transaction or enqueue it with async"
+      )
+    end
+  end
+
+  class ApplicationWriteForbidden < NonRetryableError
+    # @rbs @actor_type: String
+    # @rbs @actor_id: String
+    # @rbs @message_name: String
+
+    attr_reader :actor_type, :actor_id, :message_name
+
+    # @rbs (actor_type: String, actor_id: String, message_name: String) -> void
+    def initialize(actor_type:, actor_id:, message_name:)
+      @actor_type = actor_type
+      @actor_id = actor_id
+      @message_name = message_name
+      super(
+        "#{actor_type}(#{actor_id.inspect}).#{message_name} attempted an Active Record write; " \
+          "mutate actor state, stage a commit action, or emit a durable effect instead"
+      )
+    end
+  end
+
+  class CommitActionUnavailable < NonRetryableError
+    # @rbs (actor_type: String, actor_id: String, message_name: String) -> void
+    def initialize(actor_type:, actor_id:, message_name:)
+      super(
+        "#{actor_type}(#{actor_id.inspect}).#{message_name} staged a commit action, " \
+          "but Solid Objects uses a separate database from Active Record"
+      )
+    end
+  end
+
+  class UnknownCommitAction < NonRetryableError
   end
 
   class Rejected < Error

data/lib/solid_objects/executor.rb

@@ -24,9 +24,7 @@ module SolidObjects
       )
 
       SolidObjects.instrument(:"message.started", **instrumentation_payload)
-      result = Context.with(actor:, message: message_context) do
-        actor.invoke(message.message_name, message.arguments)
-      end
+      result = invoke_actor(message_context)
       ensure_query_did_not_mutate_state!(state_before)
       observable_changes = changed_observables(observables_before, actor.observable_values)
       complete(result, observable_changes)
@@ -54,6 +52,13 @@ module SolidObjects
       activation.actor
     end
 
+    # @rbs (MessageContext) -> untyped
+    def invoke_actor(message_context)
+      Context.with(actor:, message: message_context) do
+        actor.invoke(message.message_name, message.arguments)
+      end
+    end
+
     # @rbs (Hash[String, untyped]) -> void
     def ensure_query_did_not_mutate_state!(state_before)
       return unless message.message_kind == "sync"
@@ -81,6 +86,7 @@ module SolidObjects
         max_bytes: SolidObjects.configuration.max_result_bytes
       )
       effect_intents = actor.drain_effect_intents
+      commit_action_intents = actor.drain_commit_action_intents
       reminder_intents = actor.drain_reminder_intents
       outbound_message_intents = actor.drain_outbound_message_intents
       enqueued_effects = []
@@ -88,6 +94,7 @@ module SolidObjects
       activation.lease.fenced_transaction do |instance|
         claimed_message = matching_claim!
         locked_message = Message.lock.find(message.id)
+        execute_commit_actions(commit_action_intents)
         instance.update!(
           state: serialized_state,
           state_version: actor.class.state_version,
@@ -128,6 +135,58 @@ module SolidObjects
       SolidObjects.wake_up.signal
     end
 
+    # @rbs (Array[Actor::CommitActionIntent]) -> void
+    def execute_commit_actions(intents)
+      ensure_application_database_is_shared! if intents.any?
+      context = CommitActionContext.new(
+        message_id: message.id,
+        request_id: message.request_id,
+        actor_type: message.actor_type,
+        actor_id: message.actor_id,
+        activation_generation: activation.lease.generation
+      )
+
+      intents.each do |intent|
+        handler = SolidObjects.commit_action_registry.fetch(intent.name)
+        execute_commit_action(intent, handler, context)
+      end
+    end
+
+    # @rbs (Actor::CommitActionIntent, Proc, CommitActionContext) -> untyped
+    def execute_commit_action(intent, handler, context)
+      payload = {
+        commit_action_name: intent.name,
+        message_id: message.id,
+        request_id: message.request_id,
+        actor_type: message.actor_type,
+        actor_id: message.actor_id,
+        activation_generation: activation.lease.generation
+      }
+      SolidObjects.instrument(:"commit_action.started", **payload)
+      handler.call(intent.arguments, context).tap do
+        SolidObjects.instrument(:"commit_action.completed", **payload)
+      end
+    rescue => error
+      SolidObjects.instrument(
+        :"commit_action.failed",
+        **payload,
+        error_class: error.class.name,
+        error_message: error.message
+      )
+      raise
+    end
+
+    # @rbs () -> void
+    def ensure_application_database_is_shared!
+      return if SolidObjects::Record.connection_pool.equal?(ActiveRecord::Base.connection_pool)
+
+      raise CommitActionUnavailable.new(
+        actor_type: message.actor_type,
+        actor_id: message.actor_id,
+        message_name: message.message_name
+      )
+    end
+
     # @rbs (Message, Instance, Array[Actor::EffectIntent]) -> Array[Effect]
     def enqueue_effects(locked_message, instance, intents)
       intents.map do |intent|
@@ -218,7 +277,8 @@ module SolidObjects
         locked_message.update!(error: error_details, last_failed_at: now)
         claimed_message.destroy!
 
-        if locked_message.attempt_count >= locked_message.max_attempts
+        if error.is_a?(NonRetryableError) ||
+            locked_message.attempt_count >= locked_message.max_attempts
           create_dead_letter(locked_message, error_details, now)
           dead = true
         else