solid_objects 0.14.6 → 0.15.0

data/lib/solid_objects/actor.rb

@@ -2,7 +2,9 @@
 
 module SolidObjects
   class Actor
-    EffectIntent = Data.define(:name, :arguments, :success_operation, :failure_operation)
+    EffectIntent = Data.define(:effect_id, :name, :arguments, :success_operation, :failure_operation,
+      :recovery_operation, :status_operation, :recovery_timeout)
+    EffectRecoveryIntent = Data.define(:effect_id, :request_id)
     CommitActionIntent = Data.define(:name, :arguments)
     # The reminders table holds a name in 191 characters.
     REMINDER_NAME_LIMIT = 191
@@ -143,6 +145,7 @@ module SolidObjects
     # @rbs @actor_id: String
     # @rbs @state: State
     # @rbs @effect_intents: Array[EffectIntent]
+    # @rbs @effect_recovery_intents: Array[EffectRecoveryIntent]
     # @rbs @commit_action_intents: Array[CommitActionIntent]
     # @rbs @reminder_intents: Array[ReminderIntent]
     # @rbs @outbound_message_intents: Array[OutboundMessageIntent]
@@ -154,6 +157,7 @@ module SolidObjects
       @actor_id = actor_id
       @state = state
       @effect_intents = []
+      @effect_recovery_intents = []
       @commit_action_intents = []
       @reminder_intents = []
       @outbound_message_intents = []
@@ -175,18 +179,36 @@ module SolidObjects
       raise Rejected.new(code: rejection_code, message:, details:)
     end
 
-    # @rbs (Symbol | String, ?on_success: Symbol | String?, ?on_failure: Symbol | String?, **untyped) -> nil
-    def emit(name, on_success: nil, on_failure: nil, **arguments)
+    # @rbs (Symbol | String, ?on_success: Symbol | String?, ?on_failure: Symbol | String?, ?on_recovery: Symbol | String?, ?on_status: Symbol | String?, ?recovery_timeout: Numeric?, **untyped) -> effect_handle
+    def emit(name, on_success: nil, on_failure: nil, on_recovery: nil, on_status: nil, recovery_timeout: nil, **arguments)
       validate_effect_callback!(on_success)
       validate_effect_callback!(on_failure)
+      validate_effect_callback!(on_recovery)
+      validate_effect_callback!(on_status)
+      validate_recovery_timeout!(timeout: recovery_timeout, operation: on_recovery)
+      effect_id = SecureRandom.uuid
       EffectIntent.new(
+        effect_id:,
         name: name.to_s,
         arguments: Serialization.dump(arguments),
         success_operation: on_success&.to_s,
-        failure_operation: on_failure&.to_s
+        failure_operation: on_failure&.to_s,
+        recovery_operation: on_recovery&.to_s,
+        status_operation: on_status&.to_s,
+        recovery_timeout: recovery_timeout&.to_f
       ).tap do |intent|
         effect_intents << intent
       end
+      { "effect_id" => effect_id }
+    end
+
+    # @rbs (effect_handle) -> nil
+    def request_effect_recovery(handle)
+      unless handle.is_a?(Hash) && handle["effect_id"].is_a?(String) && !handle.fetch("effect_id").empty?
+        raise InvalidPayload, "expected an effect handle returned by emit"
+      end
+
+      effect_recovery_intents << EffectRecoveryIntent.new(effect_id: handle.fetch("effect_id"), request_id: SecureRandom.uuid)
       nil
     end
 
@@ -360,6 +382,11 @@ module SolidObjects
       effect_intents.shift(effect_intents.length)
     end
 
+    # @rbs () -> Array[EffectRecoveryIntent]
+    def drain_effect_recovery_intents
+      effect_recovery_intents.shift(effect_recovery_intents.length)
+    end
+
     # @rbs () -> Array[CommitActionIntent]
     def drain_commit_action_intents
       commit_action_intents.shift(commit_action_intents.length)
@@ -378,6 +405,7 @@ module SolidObjects
     # @rbs () -> void
     def discard_intents
       effect_intents.clear
+      effect_recovery_intents.clear
       commit_action_intents.clear
       reminder_intents.clear
       outbound_message_intents.clear
@@ -386,6 +414,7 @@ module SolidObjects
     private
 
     attr_reader :effect_intents,
+      :effect_recovery_intents,
       :commit_action_intents,
       :reminder_intents,
       :outbound_message_intents
@@ -412,5 +441,15 @@ module SolidObjects
 
       raise UnknownMessage, "unknown effect callback operation #{operation.inspect}"
     end
+
+    # @rbs (timeout: Numeric?, operation: String | Symbol?) -> void
+    def validate_recovery_timeout!(timeout:, operation:)
+      return if timeout.nil?
+
+      unless timeout.is_a?(Numeric) && timeout.real? && timeout.to_f.finite? && timeout.to_f.positive?
+        raise ArgumentError, "recovery_timeout must be a positive finite duration in seconds"
+      end
+      raise ArgumentError, "recovery_timeout requires on_recovery" unless operation
+    end
   end
 end

data/lib/solid_objects/actor_signatures.rb

@@ -0,0 +1,80 @@
+# rbs_inline: enabled
+
+require "solid_objects"
+require "rbs"
+require "pathname"
+
+module SolidObjects
+  class ActorSignatures
+    # @rbs (actors: Array[Class], signatures: Array[String]) -> String
+    def self.generate(actors:, signatures:)
+      new(signatures:).generate(actors:)
+    end
+
+    # @rbs @builder: untyped
+
+    # @rbs (signatures: Array[String]) -> void
+    def initialize(signatures:)
+      loader = RBS::EnvironmentLoader.new
+      loader.add(path: Pathname.new(File.expand_path("../../sig", __dir__)))
+      signatures.sort.each { |path| loader.add(path: Pathname.new(path)) }
+      environment = RBS::Environment.from_loader(loader).resolve_type_names
+      @builder = RBS::DefinitionBuilder.new(env: environment)
+    end
+
+    # @rbs (actors: Array[Class]) -> String
+    def generate(actors:)
+      actors.uniq.sort_by { |actor| actor.name.to_s }.map { |actor| actor_signature(actor) }.join("\n")
+    end
+
+    private
+
+    # @rbs (untyped) -> String
+    def actor_signature(actor)
+      unless actor < Actor && actor.name
+        raise ArgumentError, "actor signatures require named SolidObjects::Actor subclasses"
+      end
+
+      name = RBS::TypeName.parse("::#{actor.name}")
+      definition = @builder.build_instance(name)
+      if definition.type_params.any?
+        raise ArgumentError, "generic actor classes require application-owned dispatcher signatures"
+      end
+      messages = actor.definition.messages.keys.sort
+      methods = messages.map do |operation|
+        method = definition.methods[operation]
+        unless method && method.accessibility == :public
+          raise ArgumentError, "declare a public RBS signature for #{actor.name}##{operation}"
+        end
+        types = method.method_types.map { |type| staged_type(type, actor.name, operation).to_s }
+        "    def #{operation}: #{types.join("\n      | ")}"
+      end
+      callback_names = messages.flat_map { |operation| [ operation.inspect, operation.to_s.inspect ] }
+      callbacks = (callback_names + [ "nil" ]).join(" | ")
+      <<~RBS
+        class #{name}
+          interface _SolidObjectsOperations
+        #{methods.join("\n")}
+            def public_send: (Symbol | String, **untyped) -> nil
+          end
+
+          def schedule: (at: Time, ?every: Numeric?, ?missed: Symbol | String, ?key: (String | Symbol | Integer)?) -> #{name}::_SolidObjectsOperations
+          def transmit: () -> #{name}::_SolidObjectsOperations
+          def emit: (Symbol | String, ?on_success: (#{callbacks}), ?on_failure: (#{callbacks}), ?on_recovery: (#{callbacks}), ?on_status: (#{callbacks}), ?recovery_timeout: Numeric?, **untyped) -> SolidObjects::effect_handle
+        end
+      RBS
+    end
+
+    # @rbs (untyped, String, Symbol) -> untyped
+    def staged_type(method_type, actor_name, operation)
+      function = method_type.type
+      if !function.is_a?(RBS::Types::Function) || method_type.block ||
+          function.required_positionals.any? || function.optional_positionals.any? ||
+          function.rest_positionals || function.trailing_positionals.any?
+        raise ArgumentError, "#{actor_name}##{operation} must declare keyword-only arguments without a block"
+      end
+
+      method_type.update(type: function.update(return_type: RBS::Types::Bases::Nil.new(location: nil)))
+    end
+  end
+end

data/lib/solid_objects/database_adapter.rb

@@ -111,6 +111,19 @@ module SolidObjects
       ActiveSupport::IsolatedExecutionState[TRANSACTION_CLOCK] ||= read_database_now
     end
 
+    # @rbs () -> Time
+    def database_clock_now
+      value = with_connection do |connection|
+        expression = case self.class.family(connection)
+        when :postgresql then "clock_timestamp()"
+        when :mysql then "CURRENT_TIMESTAMP(6)"
+        else "STRFTIME('%Y-%m-%d %H:%M:%f', 'now')"
+        end
+        connection.select_value("SELECT #{expression}")
+      end
+      value.is_a?(Time) ? value.utc : Time.parse("#{value} UTC").utc
+    end
+
     # @rbs () { () -> untyped } -> untyped
     def with_lock_retry
       yield

data/lib/solid_objects/doctor.rb

@@ -78,6 +78,7 @@ module SolidObjects
       ],
       reminders: %w[id instance_id operation next_run_at status],
       effects: %w[id message_id instance_id effect_id status available_at],
+      effect_recoveries: %w[effect_id instance_id recovery_operation status_operation recovery_timeout retired_at],
       broadcasts: %w[id message_id instance_id broadcast_id status available_at],
       dead_letters: %w[id message_id instance_id actor_type actor_id attempts]
     }.freeze

data/lib/solid_objects/effect_executor.rb

@@ -45,12 +45,16 @@ module SolidObjects
       effect = claim_next
       return false unless effect
 
+      heartbeat = ProcessHeartbeat.new(process_registry:)
+      heartbeat.start
       result = deliver(effect)
       complete(effect, result)
       true
     rescue => error
       fail_effect(effect, error) if effect
       false
+    ensure
+      heartbeat&.stop
     end
 
     # @rbs () -> void
@@ -113,6 +117,7 @@ module SolidObjects
 
     # @rbs () -> Effect?
     def claim_next
+      EffectRecoveryCoordinator.new.recover_available
       database_adapter.transaction do
         now = database_adapter.database_now
         effect = database_adapter.lock_candidates(
@@ -178,17 +183,18 @@ module SolidObjects
       )
       result_message = nil
       database_adapter.transaction do
+        Instance.lock.find(effect.instance_id)
         locked_effect = Effect.lock.find(effect.id)
         verify_claim!(locked_effect)
         result_message = enqueue_result_message(
           effect: locked_effect,
           operation: locked_effect.success_operation,
           outcome: "success",
-          arguments: {
-            "effect_id" => locked_effect.effect_id,
-            "arguments" => locked_effect.arguments,
-            "result" => serialized_result
-          }
+          arguments: EffectPayload.success(
+            effect_id: locked_effect.effect_id,
+            arguments: locked_effect.arguments,
+            result: serialized_result
+          )
         )
         locked_effect.update!(
           status: "completed",
@@ -213,24 +219,21 @@ module SolidObjects
     def fail_effect(effect, error)
       result_message = nil
       database_adapter.transaction do
+        Instance.lock.find(effect.instance_id)
         locked_effect = Effect.lock.find(effect.id)
         verify_claim!(locked_effect)
         dead = locked_effect.attempt_count >= locked_effect.max_attempts
-        error_details = {
-          "class" => error.class.name,
-          "message" => error.message.to_s.byteslice(0, 8_192),
-          "backtrace" => Array(error.backtrace).first(50)
-        }
+        error_details = EffectPayload.error(error)
         if dead
           result_message = enqueue_result_message(
             effect: locked_effect,
             operation: locked_effect.failure_operation,
             outcome: "failure",
-            arguments: {
-              "effect_id" => locked_effect.effect_id,
-              "arguments" => locked_effect.arguments,
-              "error" => error_details
-            }
+            arguments: EffectPayload.failure(
+              effect_id: locked_effect.effect_id,
+              arguments: locked_effect.arguments,
+              error: error_details
+            )
           )
         end
         locked_effect.update!(

data/lib/solid_objects/effect_payload.rb

@@ -0,0 +1,42 @@
+# rbs_inline: enabled
+
+module SolidObjects
+  module EffectPayload
+    class << self
+      # @rbs [Arguments] (effect_id: String, arguments: Arguments) -> effect_retired_payload[Arguments]
+      def retired(effect_id:, arguments:)
+        { "effect_id" => effect_id, "arguments" => arguments, "outcome" => EffectRecoveryOutcome::RETIRED }
+      end
+
+      # @rbs [Arguments, Result] (effect_id: String, arguments: Arguments, result: Result) -> effect_completed_recovery_payload[Arguments, Result]
+      def recovery_completed(effect_id:, arguments:, result:)
+        { "effect_id" => effect_id, "arguments" => arguments, "outcome" => EffectRecoveryOutcome::COMPLETED, "result" => result }
+      end
+
+      # @rbs (effect_id: String, outcome: effect_observation_outcome) -> effect_observation_payload
+      def recovery_observation(effect_id:, outcome:)
+        { "effect_id" => effect_id, "outcome" => outcome }
+      end
+
+      # @rbs [Arguments, Result] (effect_id: String, arguments: Arguments, result: Result) -> effect_success_payload[Arguments, Result]
+      def success(effect_id:, arguments:, result:)
+        { "effect_id" => effect_id, "arguments" => arguments, "result" => result }
+      end
+
+      # @rbs [Arguments] (effect_id: String, arguments: Arguments, error: effect_error) -> effect_failure_payload[Arguments]
+      def failure(effect_id:, arguments:, error:)
+        { "effect_id" => effect_id, "arguments" => arguments, "error" => error }
+      end
+
+      # @rbs (Exception) -> effect_error
+      def error(exception)
+        message = exception.message.to_s.byteslice(0, 8_192) # : String
+        {
+          "class" => exception.class.name,
+          "message" => message,
+          "backtrace" => Array(exception.backtrace).first(50)
+        }
+      end
+    end
+  end
+end

data/lib/solid_objects/effect_recovery_coordinator.rb

@@ -0,0 +1,143 @@
+# rbs_inline: enabled
+
+module SolidObjects
+  module EffectRecoveryOutcome
+    RETIRED = "retired".freeze #: "retired"
+    DEFERRED = "deferred".freeze #: "deferred"
+    PENDING = "pending".freeze #: "pending"
+    COMPLETED = "completed".freeze #: "completed"
+    DEAD = "dead".freeze #: "dead"
+    ALREADY_RETIRED = "already_retired".freeze #: "already_retired"
+    MISSING = "missing".freeze #: "missing"
+  end
+
+  class EffectRecoveryCoordinator
+    # @rbs (instance: Instance, intents: Array[Actor::EffectRecoveryIntent]) -> void
+    def check(instance:, intents:)
+      return if intents.empty?
+
+      effect_ids = intents.map(&:effect_id).uniq.sort
+      effects = Effect.where(instance_id: instance.id, effect_id: effect_ids).order(:effect_id).lock.to_a.index_by(&:effect_id)
+      recoveries = EffectRecovery.where(instance_id: instance.id, effect_id: effect_ids).order(:effect_id).lock.to_a.index_by(&:effect_id)
+      effect_ids.each do |effect_id|
+        recovery = recoveries[effect_id]
+        unless recovery&.recovery_operation && recovery.status_operation
+          raise InvalidPayload, "effect recovery requires an owned handle with on_recovery and on_status"
+        end
+      end
+      owner_ids = effects.values.filter_map(&:claimed_by).uniq.sort
+      owners = Process.where(id: owner_ids).order(:id).lock.to_a.index_by(&:id)
+      now = SolidObjects.database_adapter.database_clock_now
+      intents.each do |intent|
+        check_one(instance:, intent:, recovery: recoveries.fetch(intent.effect_id), effect: effects[intent.effect_id], owners:, now:)
+      end
+    end
+
+    # @rbs () -> void
+    def recover_available
+      recovery_candidates.each do |candidate|
+        notification = SolidObjects.database_adapter.transaction do
+          instance = Instance.lock.find_by(id: candidate.instance_id)
+          next unless instance
+
+          effect = Effect.lock.find_by(effect_id: candidate.effect_id, instance_id: instance.id)
+          recovery = EffectRecovery.lock.find_by(effect_id: candidate.effect_id, instance_id: instance.id)
+          next unless effect && recovery
+          next if recovery.retired_at || effect.status != "processing"
+
+          owner = Process.lock.find_by(id: effect.claimed_by) if effect.claimed_by
+          now = SolidObjects.database_adapter.database_clock_now
+          timeout = [ SolidObjects.configuration.process_alive_threshold, recovery.recovery_timeout || 0 ].max
+          next if owner && owner.last_heartbeat_at > now - timeout
+
+          retire(instance:, effect:, recovery:, now:)
+        end
+        Mailbox.new.announce(notification) if notification
+      end
+    end
+
+    private
+
+    # @rbs () -> ActiveRecord::Relation[EffectRecovery]
+    def recovery_candidates
+      effects = Effect.table_name
+      owners = Process.table_name
+      bindings = EffectRecovery.table_name
+      heartbeat = case DatabaseAdapter.family(Record.connection)
+      when :postgresql then "EXTRACT(EPOCH FROM #{owners}.last_heartbeat_at)"
+      when :mysql then "UNIX_TIMESTAMP(#{owners}.last_heartbeat_at)"
+      else "CAST(STRFTIME('%s', #{owners}.last_heartbeat_at) AS REAL)"
+      end
+      now = SolidObjects.database_adapter.database_clock_now.to_f
+      threshold = SolidObjects.configuration.process_alive_threshold
+      EffectRecovery.joins("INNER JOIN #{effects} ON #{effects}.effect_id = #{bindings}.effect_id")
+        .joins("LEFT JOIN #{owners} ON #{owners}.id = #{effects}.claimed_by")
+        .where(retired_at: nil).where.not(recovery_operation: nil)
+        .where("#{effects}.status = ?", "processing")
+        .where("#{owners}.id IS NULL OR #{heartbeat} <= ? - CASE WHEN #{bindings}.recovery_timeout > ? THEN #{bindings}.recovery_timeout ELSE ? END", now, threshold, threshold)
+        .order(:effect_id).limit(SolidObjects.configuration.claim_scan_limit)
+    end
+
+    # @rbs (instance: Instance, intent: Actor::EffectRecoveryIntent, recovery: EffectRecovery, effect: Effect?, owners: Hash[String, Process], now: Time) -> void
+    def check_one(instance:, intent:, recovery:, effect:, owners:, now:)
+      key = "effect:#{intent.effect_id}:check:#{intent.request_id}"
+      return if Message.where(instance_id: instance.id, idempotency_key: key).exists?
+
+      outcome = observe(effect:, recovery:, owners:, now:)
+      retire(instance:, effect:, recovery:, now:) if outcome == EffectRecoveryOutcome::RETIRED
+      operation = recovery.status_operation
+      unless operation && SolidObjects.registry.fetch(instance.actor_type).definition.messages.key?(operation.to_sym)
+        raise UnknownMessage, "unknown effect status operation #{operation.inspect}"
+      end
+      arguments = case outcome
+      when EffectRecoveryOutcome::RETIRED
+        EffectPayload.retired(effect_id: intent.effect_id, arguments: effect.arguments)
+      when EffectRecoveryOutcome::COMPLETED
+        EffectPayload.recovery_completed(effect_id: intent.effect_id, arguments: effect.arguments, result: effect.result)
+      else
+        EffectPayload.recovery_observation(effect_id: intent.effect_id, outcome:)
+      end
+      Mailbox.new.enqueue_in_transaction(
+        reference: Reference.new(actor_type: instance.actor_type, actor_id: instance.actor_id),
+        operation:,
+        arguments:,
+        delivery_mode: "internal",
+        idempotency_key: key
+      )
+    end
+
+    # @rbs (effect: Effect?, recovery: EffectRecovery, owners: Hash[String, Process], now: Time) -> String
+    def observe(effect:, recovery:, owners:, now:)
+      return EffectRecoveryOutcome::ALREADY_RETIRED if recovery.retired_at
+      return EffectRecoveryOutcome::MISSING unless effect
+      return EffectRecoveryOutcome::PENDING if effect.status == "pending"
+      return EffectRecoveryOutcome::COMPLETED if effect.status == "completed"
+      return EffectRecoveryOutcome::DEAD if effect.status == "dead"
+
+      owner = owners[effect.claimed_by]
+      timeout = [ SolidObjects.configuration.process_alive_threshold, recovery.recovery_timeout || 0 ].max
+      return EffectRecoveryOutcome::DEFERRED if owner && owner.last_heartbeat_at > now - timeout
+
+      EffectRecoveryOutcome::RETIRED
+    end
+
+    # @rbs (instance: Instance, effect: Effect, recovery: EffectRecovery, now: Time) -> Message
+    def retire(instance:, effect:, recovery:, now:)
+      actor_class = SolidObjects.registry.fetch(instance.actor_type)
+      operation = recovery.recovery_operation
+      unless operation && actor_class.definition.messages.key?(operation.to_sym)
+        raise UnknownMessage, "unknown effect recovery operation #{operation.inspect}"
+      end
+
+      effect.update!(status: "completed", completed_at: now, claimed_by: nil, claimed_at: nil)
+      recovery.update!(retired_at: now)
+      Mailbox.new.enqueue_in_transaction(
+        reference: Reference.new(actor_type: instance.actor_type, actor_id: instance.actor_id),
+        operation:,
+        arguments: EffectPayload.retired(effect_id: effect.effect_id, arguments: effect.arguments),
+        delivery_mode: "internal",
+        idempotency_key: "effect:#{effect.effect_id}:recovery"
+      )
+    end
+  end
+end

data/lib/solid_objects/executor.rb

@@ -90,6 +90,7 @@ module SolidObjects
         max_bytes: SolidObjects.configuration.max_result_bytes
       )
       effect_intents = actor.drain_effect_intents
+      recovery_intents = actor.drain_effect_recovery_intents
       commit_action_intents = actor.drain_commit_action_intents
       reminder_intents = actor.drain_reminder_intents
       outbound_message_intents = actor.drain_outbound_message_intents
@@ -130,6 +131,7 @@ module SolidObjects
           observable_changes:,
           state_changed:
         )
+        EffectRecoveryCoordinator.new.check(instance:, intents: recovery_intents)
         claimed_message.destroy!
       end
 
@@ -241,10 +243,10 @@ module SolidObjects
     # @rbs (message: Message, instance: Instance, intents: Array[Actor::EffectIntent]) -> Array[Effect]
     def enqueue_effects(message:, instance:, intents:)
       intents.map do |intent|
-        Effect.create!(
+        effect = Effect.create!(
           message:,
           instance:,
-          effect_id: SecureRandom.uuid,
+          effect_id: intent.effect_id,
           name: intent.name,
           arguments: intent.arguments,
           success_operation: intent.success_operation,
@@ -253,6 +255,16 @@ module SolidObjects
           max_attempts: SolidObjects.configuration.max_attempts,
           available_at: SolidObjects.database_adapter.database_now
         )
+        if intent.recovery_operation || intent.status_operation
+          EffectRecovery.create!(
+            effect_id: intent.effect_id,
+            instance:,
+            recovery_operation: intent.recovery_operation,
+            status_operation: intent.status_operation,
+            recovery_timeout: intent.recovery_timeout
+          )
+        end
+        effect
       end
     end
 

data/lib/solid_objects/process_heartbeat.rb

@@ -0,0 +1,64 @@
+# rbs_inline: enabled
+
+module SolidObjects
+  class ProcessHeartbeat
+    # @rbs @process_registry: ProcessRegistry
+    # @rbs @mutex: Thread::Mutex
+    # @rbs @condition: Thread::ConditionVariable
+    # @rbs @stopped: bool
+    # @rbs @thread: Thread?
+
+    # @rbs (process_registry: ProcessRegistry) -> void
+    def initialize(process_registry:)
+      @process_registry = process_registry
+      @mutex = Thread::Mutex.new
+      @condition = Thread::ConditionVariable.new
+      @stopped = false
+      @thread = nil
+    end
+
+    # @rbs () -> void
+    def start
+      @thread = Thread.new do
+        Thread.current.report_on_exception = false
+        loop do
+          break if wait_for_interval
+
+          Record.connection_pool.with_connection { process_registry.heartbeat }
+        rescue => error
+          report_failure(error)
+        end
+      end
+    end
+
+    # @rbs () -> void
+    def stop
+      mutex.synchronize do
+        @stopped = true
+        condition.broadcast
+      end
+      @thread&.join
+    end
+
+    private
+
+    attr_reader :process_registry, :mutex, :condition
+
+    # @rbs (Exception) -> void
+    def report_failure(error)
+      payload = { process_id: process_registry.process_record&.id, error_class: error.class.name }
+      SolidObjects.configuration.logger.warn({ event: "solid_objects.process.heartbeat_failed", **payload })
+      SolidObjects.instrument(:"process.heartbeat_failed", **payload)
+    rescue
+      nil
+    end
+
+    # @rbs () -> bool
+    def wait_for_interval
+      mutex.synchronize do
+        condition.wait(mutex, SolidObjects.configuration.process_heartbeat_interval) unless @stopped
+        @stopped
+      end
+    end
+  end
+end

data/lib/solid_objects/process_pruner.rb

@@ -43,7 +43,7 @@ module SolidObjects
       Process.where(
         shutdown_state: "stopped",
         stopped_at: ...(now - SolidObjects.configuration.process_retention)
-      )
+      ).where.not(id: Effect.where.not(claimed_by: nil).select(:claimed_by))
     end
   end
 end

data/lib/solid_objects/process_registry.rb

@@ -10,6 +10,7 @@ module SolidObjects
     class << self
       # @rbs (?now: Time) -> Integer
       def cleanup_dead(now: SolidObjects.database_adapter.database_now)
+        EffectRecoveryCoordinator.new.recover_available
         stale_at = now - SolidObjects.configuration.process_alive_threshold
         dead_processes = Process
           .where.not(shutdown_state: "stopped")
@@ -32,12 +33,14 @@ module SolidObjects
             process_id: nil,
             activation_token: nil
           )
-          Effect.where(claimed_by: process_record.id).update_all(
-            status: "pending",
-            claimed_by: nil,
-            claimed_at: nil,
-            available_at: now
-          )
+          Effect.where(claimed_by: process_record.id)
+            .where.not(effect_id: EffectRecovery.where.not(recovery_operation: nil).select(:effect_id))
+            .update_all(
+              status: "pending",
+              claimed_by: nil,
+              claimed_at: nil,
+              available_at: now
+            )
           Reminder.where(claimed_by: process_record.id).update_all(
             claimed_by: nil,
             claimed_at: nil

data/lib/solid_objects/test_helper.rb

@@ -35,6 +35,7 @@ module SolidObjects
           ClaimedMessage,
           ReadyMessage,
           Broadcast,
+          EffectRecovery,
           Effect,
           Reminder,
           Message,

data/lib/solid_objects/version.rb

@@ -1,5 +1,5 @@
 # rbs_inline: enabled
 
 module SolidObjects
-  VERSION = "0.14.6"
+  VERSION = "0.15.0"
 end

data/lib/solid_objects.rb

@@ -55,6 +55,9 @@ require "solid_objects/wake_up_adapters/redis"
 require "solid_objects/wake_up_adapters"
 require "solid_objects/polling_backoff"
 require "solid_objects/effect_registry"
+require "solid_objects/effect_payload"
+require "solid_objects/effect_recovery_coordinator"
+require "solid_objects/process_heartbeat"
 require "solid_objects/commit_action_registry"
 require "solid_objects/lease"
 require "solid_objects/lease_renewer"