solid_objects 0.14.7 → 0.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d36e77947a8de94216c9174cc3a34a878803bdf242efa0b10f6067d48b4e7cda
-  data.tar.gz: 178955e77093552df5569ee5c0badfb6035a51a479c7bcfbd1025465ac5947a4
+  metadata.gz: 30a65b9701ec2d566a8d62c2ee428ec61cbfb5bc24d1390905a46943594b78cf
+  data.tar.gz: 39daa8fe6b2586902eee4e9bd6fbdeef4e2666651e707d647a91e5644243bb79
 SHA512:
-  metadata.gz: 491698862debab8266650dbdc21a86561d3fa7dd51764edd469a7b5173a1b2a24a2542d39a584796d3433a2bc66794d7d5251ebfe73d7c991d539f2a4c92d287
-  data.tar.gz: 705ce841595e49cbc892535afd15fb96b8ac4134bb01a1c92de8d12061a6fe5717b745d989decf82fb1c25a19966b9e50a88afb30a1426c23055e1333c00f2cc
+  metadata.gz: 6035b50982322dc2d90c6a6c286dc4ea626402003389f6bfdb66d26fdccb1aa4418cc088accedba0030bafa31e2eb4bb699203c0c2f608ba6e5849c4946f11b3
+  data.tar.gz: 43615ab44e86f76db4818a89a0d16971a55c3de959331675aa6d47640ea79db7f3d386c3ce4eefdae9a9c3fdce7aa2527f21754a67a4ab0cede9d45cefb09851

data/CHANGELOG.md

@@ -1,5 +1,20 @@
 # Changelog
 
+## 0.15.0 - 2026-09-15
+
+- Maintain effect-owner heartbeats during long-running handlers, completion, and
+  failure handling, so healthy external I/O cannot trigger abandoned recovery.
+  Report failed heartbeat updates and retry on the next configured interval
+  without consuming effect attempts.
+- Return a stable effect handle from every `emit`. Wrappers must return it;
+  operations relying on an implicit `nil` result should return `nil` explicitly.
+- Add abandoned effect recovery with `on_recovery`, optional `on_status`, staged
+  `request_effect_recovery`, and an extending `recovery_timeout` in seconds.
+  Retirement and durable callbacks share the claim-locking transaction. Add
+  frozen outcome constants and public RBS envelopes. Install the new recovery
+  binding migration before upgrading runtime processes. External actions still
+  require idempotency; retirement does not cancel an old handler or remote call.
+
 ## 0.14.7 - 2026-09-14
 
 - Publish RBS contracts for effect callback envelopes and Ruby error summaries.

data/README.md

@@ -149,6 +149,7 @@ SQL and should be allowed to enjoy that.
 - One successful turn commits actor state and staged reminders, messages, effects, commit actions, and broadcasts together.
 - Fencing prevents stale Ruby code from committing, but it cannot stop that code from continuing to run.
 - External effects can repeat and must deduplicate with the stable effect ID or another durable idempotency key.
+- [Effect recovery](docs/effect-recovery.md) uses stable `emit` handles and `on_recovery` to retire abandoned work atomically with a durable actor callback.
 - Actor handlers may read application records but cannot write them directly. Use `commit_action` for bounded same-database writes and `emit` for external I/O.
 - `async`, reminders, effects, and broadcasts need `bundle exec solid_objects start`. Pending work remains in SQL while it is down.
 - One hot identity is intentionally sequential. There are no transactions across actor identities.

data/app/models/solid_objects/effect_recovery.rb

@@ -0,0 +1,10 @@
+# rbs_inline: enabled
+
+module SolidObjects
+  class EffectRecovery < Record
+    self.table_name = SolidObjects.table_name(:effect_recoveries)
+    self.primary_key = "effect_id"
+
+    belongs_to :instance, class_name: "SolidObjects::Instance"
+  end
+end

data/benchmark/support.rb

@@ -428,9 +428,11 @@ module SolidObjectsBenchmark
       require_relative "../db/migrate/20260805000000_create_solid_objects_tables"
       require_relative "../db/migrate/20260806000000_add_state_revision_to_solid_objects_instances"
       require_relative "../db/migrate/20260813000000_rename_message_dispatch_columns"
+      require_relative "../db/migrate/20260915000000_add_solid_objects_effect_recoveries"
       CreateSolidObjectsTables.new.migrate(:up)
       AddStateRevisionToSolidObjectsInstances.new.migrate(:up)
       RenameMessageDispatchColumns.new.migrate(:up)
+      AddSolidObjectsEffectRecoveries.new.migrate(:up)
     end
 
     # @rbs () -> void
@@ -444,6 +446,7 @@ module SolidObjectsBenchmark
         claimed_message
         reminder
         effect
+        effect_recovery
         broadcast
         dead_letter
       ].each do |model|

data/db/migrate/20260915000000_add_solid_objects_effect_recoveries.rb

@@ -0,0 +1,17 @@
+# rbs_inline: enabled
+
+class AddSolidObjectsEffectRecoveries < ActiveRecord::Migration[7.1]
+  # @rbs () -> void
+  def change
+    create_table SolidObjects.table_name(:effect_recoveries), id: :string, limit: 36, primary_key: :effect_id do |definition|
+      definition.references :instance, null: false,
+        foreign_key: { to_table: SolidObjects.table_name(:instances), on_delete: :cascade, name: "fk_so_effect_recoveries_instance" }
+      definition.string :recovery_operation, limit: 191
+      definition.string :status_operation, limit: 191
+      definition.float :recovery_timeout
+      definition.datetime :retired_at, precision: 6
+      definition.timestamps precision: 6, null: false
+      definition.check_constraint "recovery_timeout IS NULL OR recovery_timeout > 0", name: "chk_so_effect_recoveries_timeout"
+    end
+  end
+end

data/docs/effect-recovery.md

@@ -0,0 +1,203 @@
+# Effect recovery coordination
+
+`emit` returns a JSON-serializable handle containing the public `effect_id`.
+Registering `on_recovery` opts the effect into retirement when its owner has
+stopped heartbeating. `on_status` is optional and receives responses only to
+explicit `request_effect_recovery(handle)` intents. Normal success and failure
+retain their existing callbacks.
+
+## Watchdog using supported APIs
+
+```ruby
+class ReportExport < SolidObjects::Actor
+  attribute :revision, default: 0
+  attribute :export_effect, default: nil
+  attribute :artifact_key, default: ""
+  attribute :applied_effect_id, default: nil
+
+  def start
+    self.revision += 1
+    self.export_effect = emit(:build_report,
+      revision: revision,
+      on_success: :export_finished,
+      on_failure: :export_failed,
+      on_recovery: :recover_export,
+      on_status: :inspect_export,
+      recovery_timeout: 120)
+    schedule(at: Time.now + 30, key: "export-watchdog").watchdog
+    nil
+  end
+
+  def watchdog
+    request_effect_recovery(export_effect) if export_effect
+  end
+
+  def recover_export(effect_id:, arguments:, outcome:)
+    return unless effect_id == export_effect&.fetch("effect_id")
+    return unless arguments.fetch("revision") == revision
+
+    start
+  end
+
+  def export_finished(effect_id:, arguments:, result:)
+    apply_export_result(effect_id:, arguments:, result:)
+  end
+
+  def export_failed(effect_id:, arguments:, error:)
+  end
+
+  def inspect_export(effect_id:, outcome:, arguments: nil, result: nil)
+    return unless effect_id == export_effect&.fetch("effect_id")
+
+    case outcome
+    when SolidObjects::EffectRecoveryOutcome::COMPLETED
+      apply_export_result(effect_id:, arguments:, result:)
+    when SolidObjects::EffectRecoveryOutcome::DEFERRED, SolidObjects::EffectRecoveryOutcome::PENDING
+      schedule(at: Time.now + 30, key: "export-watchdog").watchdog
+    end
+  end
+
+  private
+
+  def apply_export_result(effect_id:, arguments:, result:)
+    return unless effect_id == export_effect&.fetch("effect_id")
+    return unless arguments.fetch("revision") == revision
+    return if applied_effect_id == effect_id
+
+    self.artifact_key = result.fetch("artifact_key")
+    self.applied_effect_id = effect_id
+  end
+end
+```
+
+Register `build_report` through the ordinary effect registry. Its successful
+result in this example is `{ "artifact_key" => "reports/example.pdf" }`.
+The library builds the retirement payload, including `"outcome" => "retired"`;
+the effect handler does not return that outcome itself. Ruby actor operations
+receive keywords. `recover_export` needs handle/revision guards but no outcome
+guard because only a new retirement invokes it. The same guarded result helper
+handles success and completed-status repair, preventing duplicate application.
+Only recovery emits a replacement; status observations never do.
+
+The watchdog is optional: `on_recovery` alone enables automatic retirement.
+`on_status` alone does not enable retirement, polling, or subscriptions. Explicit
+checks require both bindings persisted by `emit` and cannot replace either.
+
+## Public envelopes and timeout
+
+`SolidObjects::effect_handle` describes `{ "effect_id" => String }`.
+`SolidObjects::effect_retired_payload[Arguments]` requires the effect ID, original
+arguments, and `"outcome" => "retired"`. Status uses
+`SolidObjects::effect_recovery_payload[Arguments, Result]`, a record union.
+Every variant has an effect ID; retired and completed require original arguments;
+only completed has a recorded result (including `nil`). Other Ruby observations
+contain only the effect ID and outcome.
+
+Strict packaged-consumer tests verify the constant literals and individual
+records. Steep 2.0 does not narrow this string-keyed record union after comparing
+`payload["outcome"]` with `COMPLETED`; accessing `result` through the union still
+fails its return-type check. Use the concrete completed/retired record in typed
+helpers after validating the discriminator, with an explicit type assertion if
+needed. The library retains precise records rather than weakening them to an
+untyped hash. Ordinary Ruby keyword dispatch needs no payload hydration.
+
+| Frozen `SolidObjects::EffectRecoveryOutcome` constant | Wire value | Meaning |
+| --- | --- | --- |
+| `RETIRED` | `"retired"` | This check retired the effect; separate recovery owns replacement. |
+| `DEFERRED` | `"deferred"` | Fresh owner; preserve its claim and attempts. |
+| `PENDING` | `"pending"` | Initial execution or retry remains with the scheduler. |
+| `COMPLETED` | `"completed"` | Original arguments and recorded result are available. |
+| `DEAD` | `"dead"` | Preserve terminal failure and its existing callback. |
+| `ALREADY_RETIRED` | `"already_retired"` | Earlier retirement; no additional recovery notification. |
+| `MISSING` | `"missing"` | Owned binding exists but effect data was pruned. |
+
+`recovery_timeout` must be positive finite seconds and requires `on_recovery`.
+Fractional durations are allowed. Omission uses the current runtime
+`process_alive_threshold`, normally 60 seconds. Smaller positive values are
+floored at that runtime threshold; changing configuration changes the effective
+floor even for existing effects. Database lookup errors surface as errors,
+never as missing/stale observations.
+
+Effect workers maintain their process heartbeat while the handler waits on
+external I/O and while committing success or failure. A long-running healthy
+handler therefore remains protected beyond the recovery timeout. This requires
+an available database connection for the heartbeat, as well as runtime threads
+that can continue running.
+
+Failed updates emit `solid_objects.process.heartbeat_failed` and retry at the
+configured heartbeat interval without consuming effect attempts. If an outage
+lasts beyond the freshness window, recovery can still be permitted; retries do
+not cancel external work or extend the configured window.
+
+## Compatibility and installation
+
+Upgrade all effect workers and process cleanup roles before emitting effects
+with recovery enabled. Older runtimes do not honor the persisted bindings or
+the new lock protocol.
+
+Run `solid_objects:install:migrations` and your application's normal migration
+process before starting upgraded workers. The additive migration creates the
+durable binding table; it does not change existing effect status constraints.
+
+`emit` now returns its handle, including without recovery options. Callers may
+ignore it. Wrappers must return `super`; operations whose last expression used
+to be `emit` may now return the handle to callers. End those operations with
+`nil` if their previous result must remain unchanged. The return-value change is
+intentional and is not strictly backward compatible.
+
+## Transaction and lock protocol
+
+Emission, actor state, the effect, and its recovery binding share the actor's
+fenced commit. An explicit check executes on that same connection. Automatic
+recovery performs one independent library transaction per candidate, with no
+application transaction waiting on a second connection.
+
+The lock order is originating instance, effect rows ordered by public effect
+ID, recovery binding rows in the same order, then owner processes ordered by ID.
+Completion and failure must acquire the instance before the effect. Pending
+claims lock only their effect and do not subsequently acquire an instance lock.
+Mailbox insertion reuses the instance lock already held by the decision.
+Multiple checks in one actor commit lock all their effects and bindings before
+locking any processes. Unlocked candidate reads are hints, never decisions.
+
+Automatic passes prefilter owner freshness and the effective per-effect timeout
+using database time, and visit at most `claim_scan_limit` stale candidates. Fresh
+actors and owners are not locked, including owners protected by extended grace.
+Remaining stale effects are revisited on later polls. Every candidate still
+undergoes the authoritative locked recheck, and successful retirement announces
+the committed mailbox work through the existing wake-up mechanism.
+
+The decision samples database wall time after obtaining the owner lock. The
+effective timeout is the larger of the runtime's `process_alive_threshold` and
+the effect's persisted `recovery_timeout`, in seconds. A heartbeat newer than
+the cutoff is fresh; equality is stale. A stopped or draining process with
+fresh heartbeat evidence still protects an opted-in effect until that timeout.
+Cleanup preserves opted-in claims, and process pruning excludes processes that
+still own effects. Later effect polling and process cleanup revisit deferred
+effects without resetting their last heartbeat.
+
+Retirement stores a durable `retired_at` in `effect_recoveries` and moves the
+effect into the existing terminal `completed` storage state, clearing its
+claim. The recovery record distinguishes retirement from successful completion;
+no success callback is generated. All recovery observations consult that record
+before interpreting the effect row. A late completion or failure is rejected by
+the existing processing/claim fence. This representation avoids rewriting the
+existing effect-status constraint across adapters.
+
+The retirement record and the recovery mailbox message commit atomically. A
+winning explicit check additionally enqueues its status response after the
+retirement notification. Failure to insert either message rolls back the whole
+decision. Retirement is deduplicated per effect; check responses use separate
+per-request idempotency keys. Wake-up signals are delivery hints after commit.
+
+Recovery bindings survive effect/message pruning and remain until the originating
+instance is destroyed or pruned. They do not prevent normal message or instance
+retention. Within that lifetime, a removed non-retired effect reports `missing`
+and a retirement record reports `already_retired`. A handle without an owned
+binding raises an error, without disclosing another actor's state or recreating
+a destroyed actor. Status-response message idempotency follows normal mailbox
+retention; callers cannot supply or reuse internal check request IDs.
+
+An owner heartbeat measures process liveness, not effect progress. Retirement
+does not cancel the old handler or prove a remote request stopped. External
+actions still require idempotency across retries and replacement generations.

data/docs/roadmap.md

@@ -14,6 +14,11 @@
   dead letters, and tail retry
 - Transactional effects with success/failure actor messages carrying the
   originally staged arguments for callback correlation
+- Stable `emit` handles and opted-in abandoned effect retirement coordinated
+  with effect claims, with atomic recovery/status mailbox notifications and
+  per-effect extending heartbeat timeouts and heartbeats throughout long-running
+  effect handlers. See [effect recovery](effect-recovery.md)
+  for the SQL lock protocol, retention boundary, and external idempotency limit.
 - Public RBS effect success/failure envelopes and error records, checked against
   the runtime constructors and a packaged consumer with strict Steep diagnostics
 - Actor-to-actor asynchronous outbox delivery. Effects and broadcasts use

data/examples/at_least_once/boot.rb

@@ -23,7 +23,7 @@ module AtLeastOnceBoot
     require "solid_objects/database_adapter"
     %w[
       record process instance message ready_message claimed_message
-      reminder effect broadcast dead_letter
+      reminder effect effect_recovery broadcast dead_letter
     ].each { |model| require File.join(ROOT, "app/models/solid_objects", model) }
 
     SolidObjects.configuration.authorize_message = ->(**) { true }
@@ -40,8 +40,10 @@ module AtLeastOnceBoot
     require File.join(ROOT, "db/migrate/20260805000000_create_solid_objects_tables")
     require File.join(ROOT, "db/migrate/20260806000000_add_state_revision_to_solid_objects_instances")
     require File.join(ROOT, "db/migrate/20260813000000_rename_message_dispatch_columns")
+    require File.join(ROOT, "db/migrate/20260915000000_add_solid_objects_effect_recoveries")
     CreateSolidObjectsTables.new.migrate(:up)
     AddStateRevisionToSolidObjectsInstances.new.migrate(:up)
     RenameMessageDispatchColumns.new.migrate(:up)
+    AddSolidObjectsEffectRecoveries.new.migrate(:up)
   end
 end

data/lib/solid_objects/actor.rb

@@ -2,7 +2,9 @@
 
 module SolidObjects
   class Actor
-    EffectIntent = Data.define(:name, :arguments, :success_operation, :failure_operation)
+    EffectIntent = Data.define(:effect_id, :name, :arguments, :success_operation, :failure_operation,
+      :recovery_operation, :status_operation, :recovery_timeout)
+    EffectRecoveryIntent = Data.define(:effect_id, :request_id)
     CommitActionIntent = Data.define(:name, :arguments)
     # The reminders table holds a name in 191 characters.
     REMINDER_NAME_LIMIT = 191
@@ -143,6 +145,7 @@ module SolidObjects
     # @rbs @actor_id: String
     # @rbs @state: State
     # @rbs @effect_intents: Array[EffectIntent]
+    # @rbs @effect_recovery_intents: Array[EffectRecoveryIntent]
     # @rbs @commit_action_intents: Array[CommitActionIntent]
     # @rbs @reminder_intents: Array[ReminderIntent]
     # @rbs @outbound_message_intents: Array[OutboundMessageIntent]
@@ -154,6 +157,7 @@ module SolidObjects
       @actor_id = actor_id
       @state = state
       @effect_intents = []
+      @effect_recovery_intents = []
       @commit_action_intents = []
       @reminder_intents = []
       @outbound_message_intents = []
@@ -175,18 +179,36 @@ module SolidObjects
       raise Rejected.new(code: rejection_code, message:, details:)
     end
 
-    # @rbs (Symbol | String, ?on_success: Symbol | String?, ?on_failure: Symbol | String?, **untyped) -> nil
-    def emit(name, on_success: nil, on_failure: nil, **arguments)
+    # @rbs (Symbol | String, ?on_success: Symbol | String?, ?on_failure: Symbol | String?, ?on_recovery: Symbol | String?, ?on_status: Symbol | String?, ?recovery_timeout: Numeric?, **untyped) -> effect_handle
+    def emit(name, on_success: nil, on_failure: nil, on_recovery: nil, on_status: nil, recovery_timeout: nil, **arguments)
       validate_effect_callback!(on_success)
       validate_effect_callback!(on_failure)
+      validate_effect_callback!(on_recovery)
+      validate_effect_callback!(on_status)
+      validate_recovery_timeout!(timeout: recovery_timeout, operation: on_recovery)
+      effect_id = SecureRandom.uuid
       EffectIntent.new(
+        effect_id:,
         name: name.to_s,
         arguments: Serialization.dump(arguments),
         success_operation: on_success&.to_s,
-        failure_operation: on_failure&.to_s
+        failure_operation: on_failure&.to_s,
+        recovery_operation: on_recovery&.to_s,
+        status_operation: on_status&.to_s,
+        recovery_timeout: recovery_timeout&.to_f
       ).tap do |intent|
         effect_intents << intent
       end
+      { "effect_id" => effect_id }
+    end
+
+    # @rbs (effect_handle) -> nil
+    def request_effect_recovery(handle)
+      unless handle.is_a?(Hash) && handle["effect_id"].is_a?(String) && !handle.fetch("effect_id").empty?
+        raise InvalidPayload, "expected an effect handle returned by emit"
+      end
+
+      effect_recovery_intents << EffectRecoveryIntent.new(effect_id: handle.fetch("effect_id"), request_id: SecureRandom.uuid)
       nil
     end
 
@@ -360,6 +382,11 @@ module SolidObjects
       effect_intents.shift(effect_intents.length)
     end
 
+    # @rbs () -> Array[EffectRecoveryIntent]
+    def drain_effect_recovery_intents
+      effect_recovery_intents.shift(effect_recovery_intents.length)
+    end
+
     # @rbs () -> Array[CommitActionIntent]
     def drain_commit_action_intents
       commit_action_intents.shift(commit_action_intents.length)
@@ -378,6 +405,7 @@ module SolidObjects
     # @rbs () -> void
     def discard_intents
       effect_intents.clear
+      effect_recovery_intents.clear
       commit_action_intents.clear
       reminder_intents.clear
       outbound_message_intents.clear
@@ -386,6 +414,7 @@ module SolidObjects
     private
 
     attr_reader :effect_intents,
+      :effect_recovery_intents,
       :commit_action_intents,
       :reminder_intents,
       :outbound_message_intents
@@ -412,5 +441,15 @@ module SolidObjects
 
       raise UnknownMessage, "unknown effect callback operation #{operation.inspect}"
     end
+
+    # @rbs (timeout: Numeric?, operation: String | Symbol?) -> void
+    def validate_recovery_timeout!(timeout:, operation:)
+      return if timeout.nil?
+
+      unless timeout.is_a?(Numeric) && timeout.real? && timeout.to_f.finite? && timeout.to_f.positive?
+        raise ArgumentError, "recovery_timeout must be a positive finite duration in seconds"
+      end
+      raise ArgumentError, "recovery_timeout requires on_recovery" unless operation
+    end
   end
 end

data/lib/solid_objects/actor_signatures.rb

@@ -60,7 +60,7 @@ module SolidObjects
 
           def schedule: (at: Time, ?every: Numeric?, ?missed: Symbol | String, ?key: (String | Symbol | Integer)?) -> #{name}::_SolidObjectsOperations
           def transmit: () -> #{name}::_SolidObjectsOperations
-          def emit: (Symbol | String, ?on_success: (#{callbacks}), ?on_failure: (#{callbacks}), **untyped) -> nil
+          def emit: (Symbol | String, ?on_success: (#{callbacks}), ?on_failure: (#{callbacks}), ?on_recovery: (#{callbacks}), ?on_status: (#{callbacks}), ?recovery_timeout: Numeric?, **untyped) -> SolidObjects::effect_handle
         end
       RBS
     end

data/lib/solid_objects/database_adapter.rb

@@ -111,6 +111,19 @@ module SolidObjects
       ActiveSupport::IsolatedExecutionState[TRANSACTION_CLOCK] ||= read_database_now
     end
 
+    # @rbs () -> Time
+    def database_clock_now
+      value = with_connection do |connection|
+        expression = case self.class.family(connection)
+        when :postgresql then "clock_timestamp()"
+        when :mysql then "CURRENT_TIMESTAMP(6)"
+        else "STRFTIME('%Y-%m-%d %H:%M:%f', 'now')"
+        end
+        connection.select_value("SELECT #{expression}")
+      end
+      value.is_a?(Time) ? value.utc : Time.parse("#{value} UTC").utc
+    end
+
     # @rbs () { () -> untyped } -> untyped
     def with_lock_retry
       yield

data/lib/solid_objects/doctor.rb

@@ -78,6 +78,7 @@ module SolidObjects
       ],
       reminders: %w[id instance_id operation next_run_at status],
       effects: %w[id message_id instance_id effect_id status available_at],
+      effect_recoveries: %w[effect_id instance_id recovery_operation status_operation recovery_timeout retired_at],
       broadcasts: %w[id message_id instance_id broadcast_id status available_at],
       dead_letters: %w[id message_id instance_id actor_type actor_id attempts]
     }.freeze

data/lib/solid_objects/effect_executor.rb

@@ -45,12 +45,16 @@ module SolidObjects
       effect = claim_next
       return false unless effect
 
+      heartbeat = ProcessHeartbeat.new(process_registry:)
+      heartbeat.start
       result = deliver(effect)
       complete(effect, result)
       true
     rescue => error
       fail_effect(effect, error) if effect
       false
+    ensure
+      heartbeat&.stop
     end
 
     # @rbs () -> void
@@ -113,6 +117,7 @@ module SolidObjects
 
     # @rbs () -> Effect?
     def claim_next
+      EffectRecoveryCoordinator.new.recover_available
       database_adapter.transaction do
         now = database_adapter.database_now
         effect = database_adapter.lock_candidates(
@@ -178,6 +183,7 @@ module SolidObjects
       )
       result_message = nil
       database_adapter.transaction do
+        Instance.lock.find(effect.instance_id)
         locked_effect = Effect.lock.find(effect.id)
         verify_claim!(locked_effect)
         result_message = enqueue_result_message(
@@ -213,6 +219,7 @@ module SolidObjects
     def fail_effect(effect, error)
       result_message = nil
       database_adapter.transaction do
+        Instance.lock.find(effect.instance_id)
         locked_effect = Effect.lock.find(effect.id)
         verify_claim!(locked_effect)
         dead = locked_effect.attempt_count >= locked_effect.max_attempts

data/lib/solid_objects/effect_payload.rb

@@ -3,6 +3,21 @@
 module SolidObjects
   module EffectPayload
     class << self
+      # @rbs [Arguments] (effect_id: String, arguments: Arguments) -> effect_retired_payload[Arguments]
+      def retired(effect_id:, arguments:)
+        { "effect_id" => effect_id, "arguments" => arguments, "outcome" => EffectRecoveryOutcome::RETIRED }
+      end
+
+      # @rbs [Arguments, Result] (effect_id: String, arguments: Arguments, result: Result) -> effect_completed_recovery_payload[Arguments, Result]
+      def recovery_completed(effect_id:, arguments:, result:)
+        { "effect_id" => effect_id, "arguments" => arguments, "outcome" => EffectRecoveryOutcome::COMPLETED, "result" => result }
+      end
+
+      # @rbs (effect_id: String, outcome: effect_observation_outcome) -> effect_observation_payload
+      def recovery_observation(effect_id:, outcome:)
+        { "effect_id" => effect_id, "outcome" => outcome }
+      end
+
       # @rbs [Arguments, Result] (effect_id: String, arguments: Arguments, result: Result) -> effect_success_payload[Arguments, Result]
       def success(effect_id:, arguments:, result:)
         { "effect_id" => effect_id, "arguments" => arguments, "result" => result }

data/lib/solid_objects/effect_recovery_coordinator.rb

@@ -0,0 +1,143 @@
+# rbs_inline: enabled
+
+module SolidObjects
+  module EffectRecoveryOutcome
+    RETIRED = "retired".freeze #: "retired"
+    DEFERRED = "deferred".freeze #: "deferred"
+    PENDING = "pending".freeze #: "pending"
+    COMPLETED = "completed".freeze #: "completed"
+    DEAD = "dead".freeze #: "dead"
+    ALREADY_RETIRED = "already_retired".freeze #: "already_retired"
+    MISSING = "missing".freeze #: "missing"
+  end
+
+  class EffectRecoveryCoordinator
+    # @rbs (instance: Instance, intents: Array[Actor::EffectRecoveryIntent]) -> void
+    def check(instance:, intents:)
+      return if intents.empty?
+
+      effect_ids = intents.map(&:effect_id).uniq.sort
+      effects = Effect.where(instance_id: instance.id, effect_id: effect_ids).order(:effect_id).lock.to_a.index_by(&:effect_id)
+      recoveries = EffectRecovery.where(instance_id: instance.id, effect_id: effect_ids).order(:effect_id).lock.to_a.index_by(&:effect_id)
+      effect_ids.each do |effect_id|
+        recovery = recoveries[effect_id]
+        unless recovery&.recovery_operation && recovery.status_operation
+          raise InvalidPayload, "effect recovery requires an owned handle with on_recovery and on_status"
+        end
+      end
+      owner_ids = effects.values.filter_map(&:claimed_by).uniq.sort
+      owners = Process.where(id: owner_ids).order(:id).lock.to_a.index_by(&:id)
+      now = SolidObjects.database_adapter.database_clock_now
+      intents.each do |intent|
+        check_one(instance:, intent:, recovery: recoveries.fetch(intent.effect_id), effect: effects[intent.effect_id], owners:, now:)
+      end
+    end
+
+    # @rbs () -> void
+    def recover_available
+      recovery_candidates.each do |candidate|
+        notification = SolidObjects.database_adapter.transaction do
+          instance = Instance.lock.find_by(id: candidate.instance_id)
+          next unless instance
+
+          effect = Effect.lock.find_by(effect_id: candidate.effect_id, instance_id: instance.id)
+          recovery = EffectRecovery.lock.find_by(effect_id: candidate.effect_id, instance_id: instance.id)
+          next unless effect && recovery
+          next if recovery.retired_at || effect.status != "processing"
+
+          owner = Process.lock.find_by(id: effect.claimed_by) if effect.claimed_by
+          now = SolidObjects.database_adapter.database_clock_now
+          timeout = [ SolidObjects.configuration.process_alive_threshold, recovery.recovery_timeout || 0 ].max
+          next if owner && owner.last_heartbeat_at > now - timeout
+
+          retire(instance:, effect:, recovery:, now:)
+        end
+        Mailbox.new.announce(notification) if notification
+      end
+    end
+
+    private
+
+    # @rbs () -> ActiveRecord::Relation[EffectRecovery]
+    def recovery_candidates
+      effects = Effect.table_name
+      owners = Process.table_name
+      bindings = EffectRecovery.table_name
+      heartbeat = case DatabaseAdapter.family(Record.connection)
+      when :postgresql then "EXTRACT(EPOCH FROM #{owners}.last_heartbeat_at)"
+      when :mysql then "UNIX_TIMESTAMP(#{owners}.last_heartbeat_at)"
+      else "CAST(STRFTIME('%s', #{owners}.last_heartbeat_at) AS REAL)"
+      end
+      now = SolidObjects.database_adapter.database_clock_now.to_f
+      threshold = SolidObjects.configuration.process_alive_threshold
+      EffectRecovery.joins("INNER JOIN #{effects} ON #{effects}.effect_id = #{bindings}.effect_id")
+        .joins("LEFT JOIN #{owners} ON #{owners}.id = #{effects}.claimed_by")
+        .where(retired_at: nil).where.not(recovery_operation: nil)
+        .where("#{effects}.status = ?", "processing")
+        .where("#{owners}.id IS NULL OR #{heartbeat} <= ? - CASE WHEN #{bindings}.recovery_timeout > ? THEN #{bindings}.recovery_timeout ELSE ? END", now, threshold, threshold)
+        .order(:effect_id).limit(SolidObjects.configuration.claim_scan_limit)
+    end
+
+    # @rbs (instance: Instance, intent: Actor::EffectRecoveryIntent, recovery: EffectRecovery, effect: Effect?, owners: Hash[String, Process], now: Time) -> void
+    def check_one(instance:, intent:, recovery:, effect:, owners:, now:)
+      key = "effect:#{intent.effect_id}:check:#{intent.request_id}"
+      return if Message.where(instance_id: instance.id, idempotency_key: key).exists?
+
+      outcome = observe(effect:, recovery:, owners:, now:)
+      retire(instance:, effect:, recovery:, now:) if outcome == EffectRecoveryOutcome::RETIRED
+      operation = recovery.status_operation
+      unless operation && SolidObjects.registry.fetch(instance.actor_type).definition.messages.key?(operation.to_sym)
+        raise UnknownMessage, "unknown effect status operation #{operation.inspect}"
+      end
+      arguments = case outcome
+      when EffectRecoveryOutcome::RETIRED
+        EffectPayload.retired(effect_id: intent.effect_id, arguments: effect.arguments)
+      when EffectRecoveryOutcome::COMPLETED
+        EffectPayload.recovery_completed(effect_id: intent.effect_id, arguments: effect.arguments, result: effect.result)
+      else
+        EffectPayload.recovery_observation(effect_id: intent.effect_id, outcome:)
+      end
+      Mailbox.new.enqueue_in_transaction(
+        reference: Reference.new(actor_type: instance.actor_type, actor_id: instance.actor_id),
+        operation:,
+        arguments:,
+        delivery_mode: "internal",
+        idempotency_key: key
+      )
+    end
+
+    # @rbs (effect: Effect?, recovery: EffectRecovery, owners: Hash[String, Process], now: Time) -> String
+    def observe(effect:, recovery:, owners:, now:)
+      return EffectRecoveryOutcome::ALREADY_RETIRED if recovery.retired_at
+      return EffectRecoveryOutcome::MISSING unless effect
+      return EffectRecoveryOutcome::PENDING if effect.status == "pending"
+      return EffectRecoveryOutcome::COMPLETED if effect.status == "completed"
+      return EffectRecoveryOutcome::DEAD if effect.status == "dead"
+
+      owner = owners[effect.claimed_by]
+      timeout = [ SolidObjects.configuration.process_alive_threshold, recovery.recovery_timeout || 0 ].max
+      return EffectRecoveryOutcome::DEFERRED if owner && owner.last_heartbeat_at > now - timeout
+
+      EffectRecoveryOutcome::RETIRED
+    end
+
+    # @rbs (instance: Instance, effect: Effect, recovery: EffectRecovery, now: Time) -> Message
+    def retire(instance:, effect:, recovery:, now:)
+      actor_class = SolidObjects.registry.fetch(instance.actor_type)
+      operation = recovery.recovery_operation
+      unless operation && actor_class.definition.messages.key?(operation.to_sym)
+        raise UnknownMessage, "unknown effect recovery operation #{operation.inspect}"
+      end
+
+      effect.update!(status: "completed", completed_at: now, claimed_by: nil, claimed_at: nil)
+      recovery.update!(retired_at: now)
+      Mailbox.new.enqueue_in_transaction(
+        reference: Reference.new(actor_type: instance.actor_type, actor_id: instance.actor_id),
+        operation:,
+        arguments: EffectPayload.retired(effect_id: effect.effect_id, arguments: effect.arguments),
+        delivery_mode: "internal",
+        idempotency_key: "effect:#{effect.effect_id}:recovery"
+      )
+    end
+  end
+end

data/lib/solid_objects/executor.rb

@@ -90,6 +90,7 @@ module SolidObjects
         max_bytes: SolidObjects.configuration.max_result_bytes
       )
       effect_intents = actor.drain_effect_intents
+      recovery_intents = actor.drain_effect_recovery_intents
       commit_action_intents = actor.drain_commit_action_intents
       reminder_intents = actor.drain_reminder_intents
       outbound_message_intents = actor.drain_outbound_message_intents
@@ -130,6 +131,7 @@ module SolidObjects
           observable_changes:,
           state_changed:
         )
+        EffectRecoveryCoordinator.new.check(instance:, intents: recovery_intents)
         claimed_message.destroy!
       end
 
@@ -241,10 +243,10 @@ module SolidObjects
     # @rbs (message: Message, instance: Instance, intents: Array[Actor::EffectIntent]) -> Array[Effect]
     def enqueue_effects(message:, instance:, intents:)
       intents.map do |intent|
-        Effect.create!(
+        effect = Effect.create!(
           message:,
           instance:,
-          effect_id: SecureRandom.uuid,
+          effect_id: intent.effect_id,
           name: intent.name,
           arguments: intent.arguments,
           success_operation: intent.success_operation,
@@ -253,6 +255,16 @@ module SolidObjects
           max_attempts: SolidObjects.configuration.max_attempts,
           available_at: SolidObjects.database_adapter.database_now
         )
+        if intent.recovery_operation || intent.status_operation
+          EffectRecovery.create!(
+            effect_id: intent.effect_id,
+            instance:,
+            recovery_operation: intent.recovery_operation,
+            status_operation: intent.status_operation,
+            recovery_timeout: intent.recovery_timeout
+          )
+        end
+        effect
       end
     end
 

data/lib/solid_objects/process_heartbeat.rb

@@ -0,0 +1,64 @@
+# rbs_inline: enabled
+
+module SolidObjects
+  class ProcessHeartbeat
+    # @rbs @process_registry: ProcessRegistry
+    # @rbs @mutex: Thread::Mutex
+    # @rbs @condition: Thread::ConditionVariable
+    # @rbs @stopped: bool
+    # @rbs @thread: Thread?
+
+    # @rbs (process_registry: ProcessRegistry) -> void
+    def initialize(process_registry:)
+      @process_registry = process_registry
+      @mutex = Thread::Mutex.new
+      @condition = Thread::ConditionVariable.new
+      @stopped = false
+      @thread = nil
+    end
+
+    # @rbs () -> void
+    def start
+      @thread = Thread.new do
+        Thread.current.report_on_exception = false
+        loop do
+          break if wait_for_interval
+
+          Record.connection_pool.with_connection { process_registry.heartbeat }
+        rescue => error
+          report_failure(error)
+        end
+      end
+    end
+
+    # @rbs () -> void
+    def stop
+      mutex.synchronize do
+        @stopped = true
+        condition.broadcast
+      end
+      @thread&.join
+    end
+
+    private
+
+    attr_reader :process_registry, :mutex, :condition
+
+    # @rbs (Exception) -> void
+    def report_failure(error)
+      payload = { process_id: process_registry.process_record&.id, error_class: error.class.name }
+      SolidObjects.configuration.logger.warn({ event: "solid_objects.process.heartbeat_failed", **payload })
+      SolidObjects.instrument(:"process.heartbeat_failed", **payload)
+    rescue
+      nil
+    end
+
+    # @rbs () -> bool
+    def wait_for_interval
+      mutex.synchronize do
+        condition.wait(mutex, SolidObjects.configuration.process_heartbeat_interval) unless @stopped
+        @stopped
+      end
+    end
+  end
+end

data/lib/solid_objects/process_pruner.rb

@@ -43,7 +43,7 @@ module SolidObjects
       Process.where(
         shutdown_state: "stopped",
         stopped_at: ...(now - SolidObjects.configuration.process_retention)
-      )
+      ).where.not(id: Effect.where.not(claimed_by: nil).select(:claimed_by))
     end
   end
 end

data/lib/solid_objects/process_registry.rb

@@ -10,6 +10,7 @@ module SolidObjects
     class << self
       # @rbs (?now: Time) -> Integer
       def cleanup_dead(now: SolidObjects.database_adapter.database_now)
+        EffectRecoveryCoordinator.new.recover_available
         stale_at = now - SolidObjects.configuration.process_alive_threshold
         dead_processes = Process
           .where.not(shutdown_state: "stopped")
@@ -32,12 +33,14 @@ module SolidObjects
             process_id: nil,
             activation_token: nil
           )
-          Effect.where(claimed_by: process_record.id).update_all(
-            status: "pending",
-            claimed_by: nil,
-            claimed_at: nil,
-            available_at: now
-          )
+          Effect.where(claimed_by: process_record.id)
+            .where.not(effect_id: EffectRecovery.where.not(recovery_operation: nil).select(:effect_id))
+            .update_all(
+              status: "pending",
+              claimed_by: nil,
+              claimed_at: nil,
+              available_at: now
+            )
           Reminder.where(claimed_by: process_record.id).update_all(
             claimed_by: nil,
             claimed_at: nil

data/lib/solid_objects/test_helper.rb

@@ -35,6 +35,7 @@ module SolidObjects
           ClaimedMessage,
           ReadyMessage,
           Broadcast,
+          EffectRecovery,
           Effect,
           Reminder,
           Message,

data/lib/solid_objects/version.rb

@@ -1,5 +1,5 @@
 # rbs_inline: enabled
 
 module SolidObjects
-  VERSION = "0.14.7"
+  VERSION = "0.15.0"
 end

data/lib/solid_objects.rb

@@ -56,6 +56,8 @@ require "solid_objects/wake_up_adapters"
 require "solid_objects/polling_backoff"
 require "solid_objects/effect_registry"
 require "solid_objects/effect_payload"
+require "solid_objects/effect_recovery_coordinator"
+require "solid_objects/process_heartbeat"
 require "solid_objects/commit_action_registry"
 require "solid_objects/lease"
 require "solid_objects/lease_renewer"

data/sig/generated/lib/solid_objects/actor.rbs

@@ -3,6 +3,8 @@
 module SolidObjects
   class Actor
     class EffectIntent < Data
+      attr_reader effect_id(): untyped
+
       attr_reader name(): untyped
 
       attr_reader arguments(): untyped
@@ -11,12 +13,31 @@ module SolidObjects
 
       attr_reader failure_operation(): untyped
 
-      def self.new: (untyped name, untyped arguments, untyped success_operation, untyped failure_operation) -> instance
-                  | (name: untyped, arguments: untyped, success_operation: untyped, failure_operation: untyped) -> instance
+      attr_reader recovery_operation(): untyped
+
+      attr_reader status_operation(): untyped
+
+      attr_reader recovery_timeout(): untyped
+
+      def self.new: (untyped effect_id, untyped name, untyped arguments, untyped success_operation, untyped failure_operation, untyped recovery_operation, untyped status_operation, untyped recovery_timeout) -> instance
+                  | (effect_id: untyped, name: untyped, arguments: untyped, success_operation: untyped, failure_operation: untyped, recovery_operation: untyped, status_operation: untyped, recovery_timeout: untyped) -> instance
 
-      def self.members: () -> [ :name, :arguments, :success_operation, :failure_operation ]
+      def self.members: () -> [ :effect_id, :name, :arguments, :success_operation, :failure_operation, :recovery_operation, :status_operation, :recovery_timeout ]
 
-      def members: () -> [ :name, :arguments, :success_operation, :failure_operation ]
+      def members: () -> [ :effect_id, :name, :arguments, :success_operation, :failure_operation, :recovery_operation, :status_operation, :recovery_timeout ]
+    end
+
+    class EffectRecoveryIntent < Data
+      attr_reader effect_id(): untyped
+
+      attr_reader request_id(): untyped
+
+      def self.new: (untyped effect_id, untyped request_id) -> instance
+                  | (effect_id: untyped, request_id: untyped) -> instance
+
+      def self.members: () -> [ :effect_id, :request_id ]
+
+      def members: () -> [ :effect_id, :request_id ]
     end
 
     class CommitActionIntent < Data
@@ -139,6 +160,8 @@ module SolidObjects
 
     @commit_action_intents: Array[CommitActionIntent]
 
+    @effect_recovery_intents: Array[EffectRecoveryIntent]
+
     @effect_intents: Array[EffectIntent]
 
     @state: State
@@ -158,8 +181,11 @@ module SolidObjects
     # @rbs (Symbol | String, String, ?details: Hash[String | Symbol, untyped]) -> bot
     def reject: (Symbol | String, String, ?details: Hash[String | Symbol, untyped]) -> bot
 
-    # @rbs (Symbol | String, ?on_success: Symbol | String?, ?on_failure: Symbol | String?, **untyped) -> nil
-    def emit: (Symbol | String, ?on_success: Symbol | String?, ?on_failure: Symbol | String?, **untyped) -> nil
+    # @rbs (Symbol | String, ?on_success: Symbol | String?, ?on_failure: Symbol | String?, ?on_recovery: Symbol | String?, ?on_status: Symbol | String?, ?recovery_timeout: Numeric?, **untyped) -> effect_handle
+    def emit: (Symbol | String, ?on_success: Symbol | String?, ?on_failure: Symbol | String?, ?on_recovery: Symbol | String?, ?on_status: Symbol | String?, ?recovery_timeout: Numeric?, **untyped) -> effect_handle
+
+    # @rbs (effect_handle) -> nil
+    def request_effect_recovery: (effect_handle) -> nil
 
     # @rbs () -> OperationDispatcher
     def transmit: () -> OperationDispatcher
@@ -216,6 +242,9 @@ module SolidObjects
     # @rbs () -> Array[EffectIntent]
     def drain_effect_intents: () -> Array[EffectIntent]
 
+    # @rbs () -> Array[EffectRecoveryIntent]
+    def drain_effect_recovery_intents: () -> Array[EffectRecoveryIntent]
+
     # @rbs () -> Array[CommitActionIntent]
     def drain_commit_action_intents: () -> Array[CommitActionIntent]
 
@@ -232,6 +261,8 @@ module SolidObjects
 
     attr_reader effect_intents: untyped
 
+    attr_reader effect_recovery_intents: untyped
+
     attr_reader commit_action_intents: untyped
 
     attr_reader reminder_intents: untyped
@@ -246,5 +277,8 @@ module SolidObjects
 
     # @rbs (Symbol | String?) -> void
     def validate_effect_callback!: (Symbol | String?) -> void
+
+    # @rbs (timeout: Numeric?, operation: String | Symbol?) -> void
+    def validate_recovery_timeout!: (timeout: Numeric?, operation: String | Symbol?) -> void
   end
 end

data/sig/generated/lib/solid_objects/database_adapter.rbs

@@ -55,6 +55,9 @@ module SolidObjects
     # @rbs () -> Time
     def database_now: () -> Time
 
+    # @rbs () -> Time
+    def database_clock_now: () -> Time
+
     # @rbs () { () -> untyped } -> untyped
     def with_lock_retry: () { () -> untyped } -> untyped
 

data/sig/generated/lib/solid_objects/effect_payload.rbs

@@ -2,6 +2,15 @@
 
 module SolidObjects
   module EffectPayload
+    # @rbs [Arguments] (effect_id: String, arguments: Arguments) -> effect_retired_payload[Arguments]
+    def self.retired: [Arguments] (effect_id: String, arguments: Arguments) -> effect_retired_payload[Arguments]
+
+    # @rbs [Arguments, Result] (effect_id: String, arguments: Arguments, result: Result) -> effect_completed_recovery_payload[Arguments, Result]
+    def self.recovery_completed: [Arguments, Result] (effect_id: String, arguments: Arguments, result: Result) -> effect_completed_recovery_payload[Arguments, Result]
+
+    # @rbs (effect_id: String, outcome: effect_observation_outcome) -> effect_observation_payload
+    def self.recovery_observation: (effect_id: String, outcome: effect_observation_outcome) -> effect_observation_payload
+
     # @rbs [Arguments, Result] (effect_id: String, arguments: Arguments, result: Result) -> effect_success_payload[Arguments, Result]
     def self.success: [Arguments, Result] (effect_id: String, arguments: Arguments, result: Result) -> effect_success_payload[Arguments, Result]
 

data/sig/generated/lib/solid_objects/effect_recovery_coordinator.rbs

@@ -0,0 +1,41 @@
+# Generated from lib/solid_objects/effect_recovery_coordinator.rb with RBS::Inline
+
+module SolidObjects
+  module EffectRecoveryOutcome
+    RETIRED: "retired"
+
+    DEFERRED: "deferred"
+
+    PENDING: "pending"
+
+    COMPLETED: "completed"
+
+    DEAD: "dead"
+
+    ALREADY_RETIRED: "already_retired"
+
+    MISSING: "missing"
+  end
+
+  class EffectRecoveryCoordinator
+    # @rbs (instance: Instance, intents: Array[Actor::EffectRecoveryIntent]) -> void
+    def check: (instance: Instance, intents: Array[Actor::EffectRecoveryIntent]) -> void
+
+    # @rbs () -> void
+    def recover_available: () -> void
+
+    private
+
+    # @rbs () -> ActiveRecord::Relation[EffectRecovery]
+    def recovery_candidates: () -> ActiveRecord::Relation[EffectRecovery]
+
+    # @rbs (instance: Instance, intent: Actor::EffectRecoveryIntent, recovery: EffectRecovery, effect: Effect?, owners: Hash[String, Process], now: Time) -> void
+    def check_one: (instance: Instance, intent: Actor::EffectRecoveryIntent, recovery: EffectRecovery, effect: Effect?, owners: Hash[String, Process], now: Time) -> void
+
+    # @rbs (effect: Effect?, recovery: EffectRecovery, owners: Hash[String, Process], now: Time) -> String
+    def observe: (effect: Effect?, recovery: EffectRecovery, owners: Hash[String, Process], now: Time) -> String
+
+    # @rbs (instance: Instance, effect: Effect, recovery: EffectRecovery, now: Time) -> Message
+    def retire: (instance: Instance, effect: Effect, recovery: EffectRecovery, now: Time) -> Message
+  end
+end

data/sig/generated/lib/solid_objects/process_heartbeat.rbs

@@ -0,0 +1,38 @@
+# Generated from lib/solid_objects/process_heartbeat.rb with RBS::Inline
+
+module SolidObjects
+  class ProcessHeartbeat
+    @process_registry: ProcessRegistry
+
+    @mutex: Thread::Mutex
+
+    @condition: Thread::ConditionVariable
+
+    @stopped: bool
+
+    @thread: Thread?
+
+    # @rbs (process_registry: ProcessRegistry) -> void
+    def initialize: (process_registry: ProcessRegistry) -> void
+
+    # @rbs () -> void
+    def start: () -> void
+
+    # @rbs () -> void
+    def stop: () -> void
+
+    private
+
+    attr_reader process_registry: untyped
+
+    attr_reader mutex: untyped
+
+    attr_reader condition: untyped
+
+    # @rbs (Exception) -> void
+    def report_failure: (Exception) -> void
+
+    # @rbs () -> bool
+    def wait_for_interval: () -> bool
+  end
+end

data/sig/generated/models/solid_objects/effect_recovery.rbs

@@ -0,0 +1,6 @@
+# Generated from app/models/solid_objects/effect_recovery.rb with RBS::Inline
+
+module SolidObjects
+  class EffectRecovery < Record
+  end
+end

data/sig/public/effect_payload.rbs

@@ -1,4 +1,18 @@
 module SolidObjects
+  type effect_handle = { "effect_id" => String }
+
+  type effect_retired_payload[Arguments] = {
+    "effect_id" => String, "arguments" => Arguments, "outcome" => "retired"
+  }
+
+  type effect_completed_recovery_payload[Arguments, Result] = {
+    "effect_id" => String, "arguments" => Arguments, "outcome" => "completed", "result" => Result
+  }
+
+  type effect_observation_outcome = "deferred" | "pending" | "dead" | "already_retired" | "missing"
+  type effect_observation_payload = { "effect_id" => String, "outcome" => effect_observation_outcome }
+  type effect_recovery_payload[Arguments, Result] = effect_retired_payload[Arguments] | effect_completed_recovery_payload[Arguments, Result] | effect_observation_payload
+
   type effect_error = {
     "class" => String?,
     "message" => String,

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: solid_objects
 version: !ruby/object:Gem::Version
-  version: 0.14.7
+  version: 0.15.0
 platform: ruby
 authors:
 - Lucas Carlson
@@ -303,6 +303,7 @@ files:
 - app/models/solid_objects/claimed_message.rb
 - app/models/solid_objects/dead_letter.rb
 - app/models/solid_objects/effect.rb
+- app/models/solid_objects/effect_recovery.rb
 - app/models/solid_objects/instance.rb
 - app/models/solid_objects/message.rb
 - app/models/solid_objects/process.rb
@@ -330,6 +331,7 @@ files:
 - db/migrate/20260805000000_create_solid_objects_tables.rb
 - db/migrate/20260806000000_add_state_revision_to_solid_objects_instances.rb
 - db/migrate/20260813000000_rename_message_dispatch_columns.rb
+- db/migrate/20260915000000_add_solid_objects_effect_recoveries.rb
 - docs/adr/0001-postgresql-backend.md
 - docs/adr/0002-jsonb-actor-state.md
 - docs/adr/0003-mailbox-ordering.md
@@ -350,6 +352,7 @@ files:
 - docs/dashboard.md
 - docs/database-schema.md
 - docs/development.md
+- docs/effect-recovery.md
 - docs/fit.md
 - docs/implementation-plan.md
 - docs/local-testing.md
@@ -418,6 +421,7 @@ files:
 - lib/solid_objects/dom_identity.rb
 - lib/solid_objects/effect_executor.rb
 - lib/solid_objects/effect_payload.rb
+- lib/solid_objects/effect_recovery_coordinator.rb
 - lib/solid_objects/effect_registry.rb
 - lib/solid_objects/engine.rb
 - lib/solid_objects/errors.rb
@@ -433,6 +437,7 @@ files:
 - lib/solid_objects/operation_dispatcher.rb
 - lib/solid_objects/payload_broadcast.rb
 - lib/solid_objects/polling_backoff.rb
+- lib/solid_objects/process_heartbeat.rb
 - lib/solid_objects/process_pruner.rb
 - lib/solid_objects/process_registry.rb
 - lib/solid_objects/reference.rb
@@ -509,6 +514,7 @@ files:
 - sig/generated/lib/solid_objects/dom_identity.rbs
 - sig/generated/lib/solid_objects/effect_executor.rbs
 - sig/generated/lib/solid_objects/effect_payload.rbs
+- sig/generated/lib/solid_objects/effect_recovery_coordinator.rbs
 - sig/generated/lib/solid_objects/effect_registry.rbs
 - sig/generated/lib/solid_objects/engine.rbs
 - sig/generated/lib/solid_objects/errors.rbs
@@ -524,6 +530,7 @@ files:
 - sig/generated/lib/solid_objects/operation_dispatcher.rbs
 - sig/generated/lib/solid_objects/payload_broadcast.rbs
 - sig/generated/lib/solid_objects/polling_backoff.rbs
+- sig/generated/lib/solid_objects/process_heartbeat.rbs
 - sig/generated/lib/solid_objects/process_pruner.rbs
 - sig/generated/lib/solid_objects/process_registry.rbs
 - sig/generated/lib/solid_objects/reference.rbs
@@ -559,6 +566,7 @@ files:
 - sig/generated/models/solid_objects/claimed_message.rbs
 - sig/generated/models/solid_objects/dead_letter.rbs
 - sig/generated/models/solid_objects/effect.rbs
+- sig/generated/models/solid_objects/effect_recovery.rbs
 - sig/generated/models/solid_objects/instance.rbs
 - sig/generated/models/solid_objects/message.rbs
 - sig/generated/models/solid_objects/process.rbs