solid_objects 0.16.1 → 0.17.0

data/lib/solid_objects/actor_snapshot.rb

@@ -7,6 +7,7 @@ module SolidObjects
     # @rbs @actor: Actor
     # @rbs @instance_id: Integer
     # @rbs @revision: Integer
+    # @rbs @state_data: Hash[String, json_value]
     # @rbs @observable_values: Hash[String, untyped]?
     # @rbs @observable_value_cache: Hash[String, untyped]
 
@@ -24,6 +25,7 @@ module SolidObjects
       end
       @instance_id = @instance&.id || 0
       @revision = @instance&.state_revision || 0
+      @state_data = State.new(actor_class.definition.state_definition, migrated_state).to_h
       @actor = build_actor
       @observable_values = nil
       @observable_value_cache = {}
@@ -43,14 +45,23 @@ module SolidObjects
         Serialization.readonly_copy(actor.observable_value(observable_name))
     end
 
+    # @rbs () -> Actor
+    def build_actor
+      actor_class.new(
+        actor_id: reference.actor_id,
+        state: State.new(actor_class.definition.state_definition, @state_data),
+        instance_id: instance&.id
+      )
+    end
+
     private
 
     attr_reader :instance
 
-    # @rbs () -> Actor
-    def build_actor
+    # @rbs () -> Hash[String, json_value]
+    def migrated_state
       state_version = instance&.state_version || actor_class.state_version
-      state_data = ApplicationWriteGuard.call(
+      ApplicationWriteGuard.call(
         actor_type: reference.actor_type,
         actor_id: reference.actor_id,
         operation: "state_migration"
@@ -60,11 +71,6 @@ module SolidObjects
           instance&.state || {}
         )
       end
-      actor_class.new(
-        actor_id: reference.actor_id,
-        state: State.new(actor_class.definition.state_definition, state_data),
-        instance_id: @instance&.id
-      )
     end
   end
 end

data/lib/solid_objects/broadcast_executor.rb

@@ -41,6 +41,7 @@ module SolidObjects
       broadcast = claim_next
       return false unless broadcast
 
+      Telemetry.outbox(broadcast)
       broadcast_adapter.call(broadcast)
       complete(broadcast)
       true

data/lib/solid_objects/client.rb

@@ -94,6 +94,19 @@ module SolidObjects
       raise SyncDiagnostics.new.database_contention_for(message_reference, timeout:)
     end
 
+    # @rbs (MessageReference, ?authorization_context: untyped) -> Message
+    def read_message(message_reference, authorization_context: nil)
+      Message.uncached do
+        message = Message.includes(:ready_message, :claimed_message, :dead_letter).find(message_reference.id)
+        validate_message_reference!(message_reference, message)
+        raise Unauthorized, "message result is not authorized" unless authorized_to_read?(message, authorization_context:)
+
+        message
+      end
+    rescue ActiveRecord::RecordNotFound, ActorDestroyed
+      raise Unauthorized, "message result is not authorized"
+    end
+
     # @rbs (?reference: Reference?, ?request_id: String?, ?idempotency_key: String?, ?authorization_context: untyped) -> MessageReference?
     def find_by(reference: nil, request_id: nil, idempotency_key: nil, authorization_context: nil)
       unless [ request_id, idempotency_key ].compact.one?
@@ -193,7 +206,8 @@ module SolidObjects
         :"sync.enqueue_timeout",
         actor_type: reference.actor_type,
         actor_id: reference.actor_id,
-        operation: operation.to_s
+        operation: operation.to_s,
+        timeout_milliseconds: (timeout * 1000).round
       )
       raise SyncEnqueueTimeout.new(
         timeout:,

data/lib/solid_objects/configuration.rb

@@ -40,6 +40,7 @@ module SolidObjects
     # @rbs @broadcast_worker_count: Integer
     # @rbs @reminder_scheduler_count: Integer
     # @rbs @connects_to: Hash[Symbol, untyped]?
+    # @rbs @instrumentation: event_observer?
     # @rbs @logger: untyped
     # @rbs @stream_signing_secret: String?
     # @rbs @broadcast_adapter: Proc?
@@ -110,6 +111,8 @@ module SolidObjects
       :administration_identity,
       :transmission_actor_type_resolver
 
+    attr_accessor :instrumentation #: event_observer?
+
     # @rbs @additional_components: Array[untyped]
     attr_reader :additional_components
 
@@ -159,6 +162,7 @@ module SolidObjects
       @component_path_resolver = nil
       @component_authorization_context = ->(controller:) { controller }
       @payload_authorization_context = ->(connection:) { connection }
+      @instrumentation = nil
       @logger = if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
         Rails.logger
       else

data/lib/solid_objects/database_adapters/sqlite.rb

@@ -22,7 +22,13 @@ module SolidObjects
       def transaction(&block)
         return with_lock_retry { super } if SyncDeadline.active?
 
-        with_busy_retry { super }
+        with_busy_retry do
+          with_connection do |connection|
+            timeout = configured_busy_handler_timeout(connection)
+            connection.raw_connection.busy_handler_timeout = timeout if timeout
+            super
+          end
+        end
       end
 
       # A write outside a synchronous deadline has no Ruby-level budget, so it

data/lib/solid_objects/diagnostics.rb

@@ -0,0 +1,83 @@
+# rbs_inline: enabled
+
+module SolidObjects
+  class Diagnostics
+    # @rbs @reference: Reference
+    # @rbs (Reference) -> void
+    def initialize(reference)
+      @reference = reference
+    end
+
+    # @rbs (?limit: Integer, ?authorization_context: untyped) -> actor_diagnostics
+    def summary(limit: 100, authorization_context: nil)
+      authorize!(:inspect, authorization_context:)
+      raise ArgumentError, "diagnostic limit must be an integer between 1 and 100" unless limit.is_a?(Integer) && limit.between?(1, 100)
+
+      instance = Instance.find_by(actor_type: reference.actor_type, actor_id: reference.actor_id)
+      now = SolidObjects.database_adapter.database_now
+      instance_id = instance&.id
+      mailbox = ReadyMessage.where(instance_id:).order(:available_at).limit(limit + 1).pluck(:available_at) +
+        ClaimedMessage.where(instance_id:).order(:claimed_at).limit(limit + 1).pluck(:claimed_at)
+      outbox = Effect.where(instance_id:, status: %w[pending processing]).order(:available_at).limit(limit + 1).pluck(:available_at) +
+        Broadcast.where(instance_id:, status: %w[pending processing]).order(:available_at).limit(limit + 1).pluck(:available_at)
+      reminders = Reminder.where(instance_id:, status: %w[scheduled paused]).order(:next_run_at).limit(limit + 1).pluck(:next_run_at)
+      retries = ReadyMessage.joins(:message).where(instance_id:).where.not(Message.table_name => { error: nil }).order(:available_at).limit(limit + 1).pluck(:available_at)
+      recovery_messages = Message.where(instance_id:, delivery_mode: "internal").where("idempotency_key LIKE ?", "effect:%:recovery").select(:id)
+      recovery_failures = DeadLetter.where(instance_id:, message_id: recovery_messages).order(:last_failed_at).limit(limit + 1).pluck(:last_failed_at)
+      result = {
+        "actorType" => reference.actor_type,
+        "actorId" => reference.actor_id,
+        "incarnation" => instance_id&.to_s,
+        "revision" => instance&.state_revision&.to_s,
+        "adapter" => DatabaseAdapter.family(Record.connection).to_s,
+        "occurredAt" => now.utc.iso8601(3),
+        "limit" => limit,
+        "mailbox" => summarize(mailbox, now:, limit:),
+        "outbox" => summarize(outbox, now:, limit:),
+        "reminders" => summarize(reminders, now:, limit:),
+        "retries" => summarize(retries, now:, limit:),
+        "recoveryFailures" => summarize(recovery_failures, now:, limit:)
+      }
+      SolidObjects.instrument(:"mailbox.depth", actor_type: reference.actor_type, actor_id: reference.actor_id, instance_id:, count: result.fetch("mailbox").fetch("sampled"), truncated: result.fetch("mailbox").fetch("truncated"), depth: result.fetch("mailbox").fetch("truncated") ? nil : result.fetch("mailbox").fetch("sampled"))
+      Serialization.readonly_copy(result)
+    end
+
+    # @rbs (?authorization_context: untyped) { (portable_event) -> void } -> Proc
+    def observe(authorization_context: nil, &block)
+      raise ArgumentError, "an actor observer requires a block" unless block
+
+      authorize!(:observe, authorization_context:)
+      SolidObjects.observer_registry.subscribe do |notification|
+        payload = notification.payload
+        next unless payload[:actor_type] == reference.actor_type && payload[:actor_id] == reference.actor_id
+
+        Telemetry.deliver(block, Telemetry.event(notification.name.delete_prefix("solid_objects.").to_sym, payload))
+      end
+    end
+
+    private
+
+    attr_reader :reference #: Reference
+
+    # @rbs (Symbol, ?authorization_context: untyped) -> void
+    def authorize!(action, authorization_context: nil)
+      allowed = SolidObjects.configuration.authorize_administration.call(
+        action:,
+        resource: "actor_diagnostics",
+        resource_id: [ reference.actor_type, reference.actor_id ].to_json,
+        authorization_context:
+      )
+      raise Unauthorized, "actor diagnostics are not authorized" unless allowed
+    end
+
+    # @rbs (Array[Time?], now: Time, limit: Integer) -> diagnostic_summary
+    def summarize(timestamps, now:, limit:)
+      oldest = timestamps.compact.min
+      {
+        "sampled" => [ timestamps.length, limit ].min,
+        "truncated" => timestamps.length > limit,
+        "oldestAgeMilliseconds" => oldest ? [ ((now - oldest) * 1000).round, 0 ].max : nil
+      }
+    end
+  end
+end

data/lib/solid_objects/effect_executor.rb

@@ -137,6 +137,7 @@ module SolidObjects
 
     # @rbs (Effect) -> untyped
     def deliver(effect)
+      Telemetry.outbox(effect)
       return deliver_actor_message(effect) if effect.name == ACTOR_MESSAGE_EFFECT
 
       handler = SolidObjects.effect_registry.fetch(effect.name)
@@ -208,6 +209,9 @@ module SolidObjects
       Mailbox.new.announce(result_message) if result_message
       SolidObjects.instrument(
         :"effect.completed",
+        instance_id: effect.instance_id,
+        actor_type: effect.instance.actor_type,
+        actor_id: effect.instance.actor_id,
         effect_id: effect.effect_id,
         effect_name: effect.name,
         message_id: effect.message_id,

data/lib/solid_objects/errors.rb

@@ -7,6 +7,9 @@ module SolidObjects
   class NonRetryableError < Error
   end
 
+  class QueryMutatedState < NonRetryableError
+  end
+
   class UnsupportedDatabase < Error
   end
 

data/lib/solid_objects/executor.rb

@@ -4,12 +4,14 @@ module SolidObjects
   class Executor
     # @rbs @activation: Activation
     # @rbs @message: Message
+    # @rbs @started_at: Float
     # @rbs @completion_transaction: untyped
 
     # @rbs (activation: Activation, message: Message) -> void
     def initialize(activation:, message:)
       @activation = activation
       @message = message
+      @started_at = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
     end
 
     # @rbs () -> bool
@@ -24,11 +26,12 @@ module SolidObjects
         request_id: message.request_id
       )
 
+      SolidObjects.instrument(:"recovery.reclaimed", **instrumentation_payload) if message.attempt_count > 1 && message.error.nil?
       SolidObjects.instrument(:"message.started", **instrumentation_payload)
       result = invoke_actor(message_context)
       observable_changes = changed_observables(observables_before, actor.observable_values)
       state_after = actor.state.to_h_with_byte_size
-      ensure_query_did_not_mutate_state!(state_before, state_after.value)
+      ensure_query_is_read_only!(state_before, state_after.value)
       complete(
         result,
         observable_changes,
@@ -66,13 +69,12 @@ module SolidObjects
       end
     end
 
-    # @rbs (Hash[String, untyped], Hash[String, untyped]) -> void
-    def ensure_query_did_not_mutate_state!(state_before, state_after)
-      return unless message.delivery_mode == "sync"
+    # @rbs (Hash[String, json_value], Hash[String, json_value]) -> void
+    def ensure_query_is_read_only!(state_before, state_after)
       return unless actor.class.definition.queries.key?(message.operation.to_sym)
-      return if state_after == state_before
 
-      raise InvalidActor, "query #{message.operation.inspect} mutated actor state"
+      raise QueryMutatedState, "query #{message.operation.inspect} mutated actor state" if state_after != state_before
+      raise QueryMutatedState, "query #{message.operation.inspect} staged durable work" if actor.intent_count.positive?
     end
 
     # @rbs (Hash[String, untyped], Hash[String, untyped]) -> Hash[String, untyped]
@@ -86,7 +88,7 @@ module SolidObjects
     def complete(result, observable_changes, state_after:, state_changed:)
       ensure_state_fits!(state_after.byte_size)
       serialized_result = Serialization.dump(
-        (message.delivery_mode == "sync") ? result : nil,
+        result,
         max_bytes: SolidObjects.configuration.max_result_bytes
       )
       effect_intents = actor.drain_effect_intents
@@ -156,8 +158,9 @@ module SolidObjects
           actor_id: message.actor_id
         )
       end
+      SolidObjects.instrument_after_commit(:"recovery.completed", **instrumentation_payload) if recovery_message?
       report_large_state(state_after.byte_size)
-      SolidObjects.instrument_after_commit(:"message.completed", **instrumentation_payload)
+      SolidObjects.instrument_after_commit(:"message.completed", **instrumentation_payload, revision: message.sequence, duration_milliseconds: elapsed_milliseconds)
       SolidObjects.wake_up.signal
     rescue CommittedTransactionError
       raise
@@ -209,11 +212,8 @@ module SolidObjects
     # @rbs (intent: Actor::CommitActionIntent, handler: Proc, context: CommitActionContext) -> untyped
     def execute_commit_action(intent:, handler:, context:)
       payload = {
+        **instrumentation_payload,
         commit_action_name: intent.name,
-        message_id: message.id,
-        request_id: message.request_id,
-        actor_type: message.actor_type,
-        actor_id: message.actor_id,
         activation_generation: activation.lease.generation
       }
       SolidObjects.instrument(:"commit_action.started", **payload)
@@ -224,8 +224,7 @@ module SolidObjects
       SolidObjects.instrument(
         :"commit_action.failed",
         **payload,
-        error_class: error.class.name,
-        error_message: error.message
+        error_class: error.class.name
       )
       raise
     end
@@ -421,8 +420,13 @@ module SolidObjects
         :"message.failed",
         **instrumentation_payload,
         error_class: error.class.name,
+        duration_milliseconds: elapsed_milliseconds,
+        retryable: !error.is_a?(NonRetryableError),
+        outcome: dead ? "dead" : "retrying",
         dead:
       )
+      SolidObjects.instrument_after_commit(:"recovery.failed", **instrumentation_payload) if dead && recovery_message?
+      SolidObjects.instrument_after_commit(dead ? :"dead_letter.created" : :"message.retry", **instrumentation_payload)
       SolidObjects.wake_up.signal
     end
 
@@ -458,7 +462,8 @@ module SolidObjects
       SolidObjects.instrument_after_commit(
         :"message.rejected",
         **instrumentation_payload,
-        code: rejection.code
+        code: rejection.code,
+        duration_milliseconds: elapsed_milliseconds
       )
       SolidObjects.wake_up.signal
     end
@@ -520,15 +525,28 @@ module SolidObjects
       )
     end
 
+    # @rbs () -> bool
+    def recovery_message?
+      message.delivery_mode == "internal" && message.idempotency_key.to_s.start_with?("effect:") && message.idempotency_key.to_s.end_with?(":recovery")
+    end
+
+    # @rbs () -> (Integer | Float)
+    def elapsed_milliseconds
+      ((::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - @started_at) * 1000).round(3)
+    end
+
     # @rbs () -> Hash[Symbol, untyped]
     def instrumentation_payload
       {
+        instance_id: message.instance_id,
         message_id: message.id,
         actor_type: message.actor_type,
         actor_id: message.actor_id,
         sequence: message.sequence,
         attempt: message.attempt_count,
-        request_id: message.request_id
+        request_id: message.request_id,
+        operation: message.operation,
+        delivery_mode: message.delivery_mode
       }
     end
   end

data/lib/solid_objects/instrumentation.rb

@@ -4,7 +4,28 @@ module SolidObjects
   module Instrumentation
     # @rbs (Symbol, **untyped) { (Hash[Symbol, untyped]) -> untyped } -> untyped
     def instrument(event, **payload, &block)
-      ActiveSupport::Notifications.instrument("solid_objects.#{event}", payload, &block)
+      executed = false
+      application_error = nil
+      result = nil
+      begin
+        ActiveSupport::Notifications.instrument("solid_objects.#{event}", payload) do
+          executed = true
+          begin
+            result = block&.call(payload)
+          rescue => error
+            application_error = error
+            raise
+          end
+        end
+      rescue => error
+        raise application_error if application_error
+
+        report_instrumentation_failure(event, error) unless event == :"instrumentation.failed"
+        result = block&.call(payload) unless executed
+      ensure
+        Telemetry.emit(event, payload)
+      end
+      result
     end
 
     # @rbs (Symbol, **untyped) -> void

data/lib/solid_objects/log_subscriber.rb

@@ -12,7 +12,7 @@ module SolidObjects
             {
               event: event.name,
               duration_ms: event.duration.round(2)
-            }.merge(event.payload)
+            }.merge(event.payload.except(:error_message, :arguments, :state, :credentials, :response, :exception, :exception_object))
           )
         end
       end

data/lib/solid_objects/mailbox.rb

@@ -79,11 +79,15 @@ module SolidObjects
     def announce(message)
       SolidObjects.instrument(
         :"message.enqueued",
+        instance_id: message.instance_id,
+        attempt: message.attempt_count,
         message_id: message.id,
         actor_type: message.actor_type,
         actor_id: message.actor_id,
         sequence: message.sequence,
-        request_id: message.request_id
+        request_id: message.request_id,
+        operation: message.operation,
+        delivery_mode: message.delivery_mode
       )
       SolidObjects.wake_up.signal
     end

data/lib/solid_objects/message_reference.rb

@@ -33,20 +33,20 @@ module SolidObjects
       freeze
     end
 
-    # @rbs () -> String
-    def status
-      Message.uncached { status_of(Message.find(id)) }
+    # @rbs (?authorization_context: untyped) -> String
+    def status(authorization_context: nil)
+      status_of(SolidObjects.client.read_message(self, authorization_context:))
     end
 
-    # @rbs () -> untyped
-    def result
-      Message.uncached { Message.find(id).result }
+    # @rbs (?authorization_context: untyped) -> json_value
+    def result(authorization_context: nil)
+      SolidObjects.client.read_message(self, authorization_context:).result!
     end
 
-    # @rbs () -> Outcome
-    def outcome
+    # @rbs (?authorization_context: untyped) -> Outcome
+    def outcome(authorization_context: nil)
       Message.uncached do
-        message = Message.find(id)
+        message = SolidObjects.client.read_message(self, authorization_context:)
         Outcome.new(
           status: status_of(message),
           result: Serialization.readonly_copy(message.result),

data/lib/solid_objects/observer_registry.rb

@@ -0,0 +1,47 @@
+# rbs_inline: enabled
+
+module SolidObjects
+  class ObserverRegistry
+    MAXIMUM_OBSERVERS = 1000
+
+    # @rbs @subscriptions: Array[Object]
+    # @rbs @mutex: Mutex
+
+    # @rbs () -> void
+    def initialize
+      @subscriptions = []
+      @mutex = Mutex.new
+    end
+
+    # @rbs () { (ActiveSupport::Notifications::Event) -> void } -> Proc
+    def subscribe(&listener)
+      subscription = mutex.synchronize do
+        raise ArgumentError, "at most #{MAXIMUM_OBSERVERS} local observers may be registered" if subscriptions.length >= MAXIMUM_OBSERVERS
+
+        ActiveSupport::Notifications.subscribe(/\Asolid_objects\./, &listener).tap { |created| subscriptions << created }
+      end
+      -> { unsubscribe(subscription) }
+    end
+
+    # @rbs () -> void
+    def clear
+      mutex.synchronize do
+        subscriptions.each { |subscription| ActiveSupport::Notifications.unsubscribe(subscription) }
+        subscriptions.clear
+      end
+    end
+
+    private
+
+    attr_reader :subscriptions #: Array[Object]
+    attr_reader :mutex #: Mutex
+
+    # @rbs (Object) -> void
+    def unsubscribe(subscription)
+      mutex.synchronize do
+        ActiveSupport::Notifications.unsubscribe(subscription)
+        subscriptions.delete(subscription)
+      end
+    end
+  end
+end

data/lib/solid_objects/payload_broadcast.rb

@@ -2,7 +2,6 @@
 
 module SolidObjects
   class PayloadBroadcast
-    MAXIMUM_PAYLOAD_BYTES = 1_048_576
     REVISION_OBSERVABLE = "solid_objects.revision"
 
     # @rbs @snapshot: ActorSnapshot
@@ -42,7 +41,7 @@ module SolidObjects
     def rendered_payload(handler)
       payload = Serialization.dump(
         evaluated_payload(handler),
-        max_bytes: MAXIMUM_PAYLOAD_BYTES
+        max_bytes: SolidObjects.configuration.max_payload_bytes
       )
       return payload if payload.is_a?(Hash) || payload.is_a?(Array)
 
@@ -56,10 +55,12 @@ module SolidObjects
     # unaffected.
     # @rbs (ActorDefinition::Handler) -> untyped
     def evaluated_payload(handler)
-      actor = snapshot.actor
-      actor.instance_exec(actor, authorization_context, &handler.block)
+      actor = snapshot.build_actor
+      actor.read_projection("payload.#{name}") do
+        actor.instance_exec(actor, authorization_context, &handler.block)
+      end
     rescue NameError => error
-      raise unless class_level_receiver?(error)
+      raise unless class_level_receiver?(error, actor)
 
       raise InvalidPayloadBroadcast,
         "payload broadcast #{name.inspect} called #{error.name.inspect} on the " \
@@ -70,9 +71,9 @@ module SolidObjects
     # Distinguishes a block that relied on the old class-level receiver from an
     # ordinary typo, so the one behaviour change reports itself instead of
     # surfacing as an unexplained NameError.
-    # @rbs (NameError[untyped]) -> bool
-    def class_level_receiver?(error)
-      error.receiver.equal?(snapshot.actor) &&
+    # @rbs (NameError[BasicObject], Actor?) -> bool
+    def class_level_receiver?(error, actor)
+      error.receiver.equal?(actor) &&
         snapshot.actor_class.respond_to?(error.name)
     rescue ArgumentError, NameError
       false

data/lib/solid_objects/reference.rb

@@ -16,6 +16,25 @@ module SolidObjects
       freeze
     end
 
+    # @rbs (?limit: Integer, ?authorization_context: untyped) -> actor_diagnostics
+    def diagnostics(limit: 100, authorization_context: nil)
+      Diagnostics.new(self).summary(limit:, authorization_context:)
+    end
+
+    # @rbs (?authorization_context: untyped) { (portable_event) -> void } -> Proc
+    def observe(authorization_context: nil, &block)
+      Diagnostics.new(self).observe(authorization_context:, &block)
+    end
+
+    # @rbs (String, ?authorization_context: untyped) { (portable_event) -> void } -> Proc
+    def on(name, authorization_context: nil, &block)
+      raise ArgumentError, "an actor observer requires a block" unless block
+
+      observe(authorization_context:) do |event|
+        block.call(event) if event.fetch("name") == "solid_objects.#{name}"
+      end
+    end
+
     # @rbs (?available_at: Time?, ?idempotency_key: String?, ?authorization_context: untyped) -> OperationDispatcher
     def async(available_at: nil, idempotency_key: nil, authorization_context: nil)
       OperationDispatcher.new(

data/lib/solid_objects/reminder_scheduler.rb

@@ -166,9 +166,14 @@ module SolidObjects
       mailbox.announce(message)
       SolidObjects.instrument(
         :"reminder.enqueued",
+        instance_id: reminder.instance_id,
+        message_id: message.id,
+        attempt: message.attempt_count,
+        lateness_milliseconds: [ (((now || database_adapter.database_now) - reminder.next_run_at) * 1000).round, 0 ].max,
         reminder_id: reminder.id,
         actor_type: reminder.actor_type,
         actor_id: reminder.actor_id,
+        operation: reminder.operation,
         occurrence: reminder.occurrence
       )
       MessageReference.from_message(message)

data/lib/solid_objects/state_snapshot.rb

@@ -10,6 +10,7 @@ module SolidObjects
       actor_snapshot = ActorSnapshot.new(reference)
       @actor_class = actor_snapshot.actor_class
       @data = Serialization.readonly_copy(actor_snapshot.actor.state.to_h)
+      SolidObjects.instrument(:"snapshot.read", actor_type: reference.actor_type, actor_id: reference.actor_id, instance_id: actor_snapshot.instance_id.zero? ? nil : actor_snapshot.instance_id, revision: actor_snapshot.revision)
     end
 
     # @rbs () -> Hash[String, untyped]

data/lib/solid_objects/supervisor.rb

@@ -97,8 +97,7 @@ module SolidObjects
         rescue => error
           SolidObjects.instrument(
             :"supervisor.monitor_failed",
-            error_class: error.class.name,
-            error_message: error.message
+            error_class: error.class.name
           )
         end
         sleep SolidObjects.configuration.supervisor_monitor_interval
@@ -161,8 +160,7 @@ module SolidObjects
           failures += 1
           SolidObjects.instrument(
             :"supervisor.retention_failed",
-            error_class: error.class.name,
-            error_message: error.message
+            error_class: error.class.name
           )
         end
         wait_for_next_retention(failures)
@@ -220,8 +218,7 @@ module SolidObjects
     rescue => error
       SolidObjects.instrument(
         :"supervisor.redrive_failed",
-        error_class: error.class.name,
-        error_message: error.message
+        error_class: error.class.name
       )
       false
     end