solid_objects 0.2.1 → 0.3.0

data/lib/solid_objects/instance_pruner.rb

@@ -0,0 +1,97 @@
+# rbs_inline: enabled
+
+module SolidObjects
+  class InstancePruner
+    # @rbs @now: Time
+    # @rbs @batch_size: Integer
+
+    # @rbs (?now: Time, ?batch_size: Integer) -> void
+    def initialize(
+      now: SolidObjects.database_adapter.database_now,
+      batch_size: SolidObjects.configuration.prune_batch_size
+    )
+      @now = now
+      @batch_size = batch_size
+    end
+
+    # @rbs () -> Integer
+    def preview
+      policy_relations.sum(&:count)
+    end
+
+    # @rbs () -> Integer
+    def prune
+      policy_relations.sum { |relation| prune_relation(relation) }.tap do |count|
+        SolidObjects.instrument(:"instances.pruned", count:)
+      end
+    end
+
+    private
+
+    attr_reader :now, :batch_size
+
+    # @rbs () -> Array[ActiveRecord::Relation[Instance]]
+    def policy_relations
+      SolidObjects.configuration.instance_retention_by_actor_type.map do |actor_type, retention|
+        prunable
+          .where(actor_type: actor_type.to_s)
+          .where("COALESCE(last_used_at, created_at) < ?", now - retention)
+      end
+    end
+
+    # @rbs () -> ActiveRecord::Relation[Instance]
+    def prunable
+      Instance
+        .where(activation_owner_id: nil, paused_at: nil)
+        .where.not(id: ReadyMessage.select(:instance_id))
+        .where.not(id: ClaimedMessage.select(:instance_id))
+        .where.not(
+          id: Reminder.where(status: "scheduled").select(:instance_id)
+        )
+        .where.not(
+          id: Effect.where.not(status: "completed").select(:instance_id)
+        )
+        .where.not(
+          id: Broadcast.where.not(status: "delivered").select(:instance_id)
+        )
+        .where.not(id: DeadLetter.select(:instance_id))
+    end
+
+    # @rbs (ActiveRecord::Relation[Instance]) -> Integer
+    def prune_relation(relation)
+      deleted = 0
+
+      loop do
+        instance_ids = relation.limit(batch_size).pluck(:id)
+        break if instance_ids.empty?
+
+        instance_ids.each do |instance_id|
+          deleted += prune_instance(instance_id, relation)
+        end
+      end
+
+      deleted
+    end
+
+    # @rbs (Integer, ActiveRecord::Relation[Instance]) -> Integer
+    def prune_instance(instance_id, relation)
+      expired_actor = SolidObjects.database_adapter.transaction do
+        instance = Instance.lock.find_by(id: instance_id)
+        next unless instance
+        next unless relation.where(id: instance_id).exists?
+
+        identity = {
+          instance_id:,
+          actor_type: instance.actor_type,
+          actor_id: instance.actor_id
+        }
+        instance.delete
+        identity
+      end
+      return 0 unless expired_actor
+
+      SolidObjects.instrument(:"actor.expired", **expired_actor)
+      1
+    end
+  end
+end

data/lib/solid_objects/message_pruner.rb

@@ -0,0 +1,97 @@
+# rbs_inline: enabled
+
+module SolidObjects
+  class MessagePruner
+    # @rbs @now: Time
+    # @rbs @batch_size: Integer
+
+    # @rbs (?now: Time, ?batch_size: Integer) -> void
+    def initialize(
+      now: SolidObjects.database_adapter.database_now,
+      batch_size: SolidObjects.configuration.prune_batch_size
+    )
+      @now = now
+      @batch_size = batch_size
+    end
+
+    # @rbs () -> Integer
+    def preview
+      policy_relations.sum(&:count)
+    end
+
+    # @rbs () -> Integer
+    def prune
+      policy_relations.sum { |relation| prune_relation(relation) }.tap do |count|
+        SolidObjects.instrument(:"messages.pruned", count:)
+      end
+    end
+
+    private
+
+    attr_reader :now, :batch_size
+
+    # @rbs () -> Array[ActiveRecord::Relation[Message]]
+    def policy_relations
+      overrides = retention_overrides
+      relations = overrides.map do |actor_type, retention|
+        prunable.where(actor_type:, completed_at: ...retention_cutoff(retention))
+      end
+      default_relation = prunable.where(completed_at: ...retention_cutoff(default_retention))
+      default_relation = default_relation.where.not(actor_type: overrides.keys) if overrides.any?
+      relations << default_relation
+    end
+
+    # @rbs () -> ActiveRecord::Relation[Message]
+    def prunable
+      Message
+        .where.not(id: ReadyMessage.select(:message_id))
+        .where.not(id: ClaimedMessage.select(:message_id))
+        .where.not(id: DeadLetter.select(:message_id))
+        .where.not(
+          id: DeadLetter
+            .where.not(retried_message_id: nil)
+            .select(:retried_message_id)
+        )
+        .where.not(
+          id: Effect
+            .where.not(status: "completed")
+            .select(:message_id)
+        )
+        .where.not(
+          id: Broadcast
+            .where.not(status: "delivered")
+            .select(:message_id)
+        )
+    end
+
+    # @rbs (ActiveRecord::Relation[Message]) -> Integer
+    def prune_relation(relation)
+      deleted = 0
+
+      loop do
+        message_ids = relation.limit(batch_size).pluck(:id)
+        break if message_ids.empty?
+
+        deleted += Message.where(id: message_ids).delete_all
+      end
+
+      deleted
+    end
+
+    # @rbs () -> Numeric
+    def default_retention
+      SolidObjects.configuration.message_retention
+    end
+
+    # @rbs () -> Hash[String, Numeric]
+    def retention_overrides
+      SolidObjects.configuration.message_retention_by_actor_type
+        .to_h { |actor_type, retention| [ actor_type.to_s, retention ] }
+    end
+
+    # @rbs (Numeric) -> Time
+    def retention_cutoff(retention)
+      now - retention
+    end
+  end
+end

data/lib/solid_objects/message_reference.rb

@@ -49,5 +49,14 @@ module SolidObjects
     def result
       Message.find(id).result
     end
+
+    # @rbs (?timeout: Numeric, ?authorization_context: untyped) -> untyped
+    def wait(timeout: 5.seconds, authorization_context: nil)
+      SolidObjects.client.wait(
+        self,
+        timeout:,
+        authorization_context:
+      )
+    end
   end
 end

data/lib/solid_objects/process_pruner.rb

@@ -0,0 +1,49 @@
+# rbs_inline: enabled
+
+module SolidObjects
+  class ProcessPruner
+    # @rbs @now: Time
+    # @rbs @batch_size: Integer
+
+    # @rbs (?now: Time, ?batch_size: Integer) -> void
+    def initialize(
+      now: SolidObjects.database_adapter.database_now,
+      batch_size: SolidObjects.configuration.prune_batch_size
+    )
+      @now = now
+      @batch_size = batch_size
+    end
+
+    # @rbs () -> Integer
+    def preview
+      prunable.count
+    end
+
+    # @rbs () -> Integer
+    def prune
+      deleted = 0
+
+      loop do
+        process_ids = prunable.limit(batch_size).pluck(:id)
+        break if process_ids.empty?
+
+        deleted += Process.where(id: process_ids).delete_all
+      end
+
+      SolidObjects.instrument(:"processes.pruned", count: deleted)
+      deleted
+    end
+
+    private
+
+    attr_reader :now, :batch_size
+
+    # @rbs () -> ActiveRecord::Relation[Process]
+    def prunable
+      Process.where(
+        shutdown_state: "stopped",
+        stopped_at: ...(now - SolidObjects.configuration.process_retention)
+      )
+    end
+  end
+end

data/lib/solid_objects/reference.rb

@@ -58,6 +58,11 @@ module SolidObjects
       SolidObjects.client.destroy(self, authorization_context:)
     end
 
+    # @rbs (?authorization_context: untyped) -> StateSnapshot
+    def snapshot(authorization_context: nil)
+      SolidObjects.client.snapshot(self, authorization_context:)
+    end
+
     # @rbs (Symbol, *untyped, **untyped) -> untyped
     def method_missing(name, *arguments, **keywords)
       return super if arguments.any? || block_given?

data/lib/solid_objects/state_snapshot.rb

@@ -0,0 +1,41 @@
+# rbs_inline: enabled
+
+module SolidObjects
+  class StateSnapshot
+    # @rbs @actor_class: Class
+    # @rbs @data: Hash[String, untyped]
+
+    # @rbs (Reference) -> void
+    def initialize(reference)
+      actor_snapshot = ActorSnapshot.new(reference)
+      @actor_class = actor_snapshot.actor_class
+      @data = Serialization.readonly_copy(actor_snapshot.actor.state.to_h)
+    end
+
+    # @rbs () -> Hash[String, untyped]
+    def to_h
+      data
+    end
+
+    # @rbs (Symbol, *untyped, **untyped) -> untyped
+    def method_missing(name, *arguments, **keywords)
+      return data.fetch(name.to_s) if arguments.empty? && keywords.empty? && attribute?(name)
+
+      super
+    end
+
+    # @rbs (Symbol, bool) -> bool
+    def respond_to_missing?(name, include_private = false)
+      attribute?(name) || super
+    end
+
+    private
+
+    attr_reader :actor_class, :data
+
+    # @rbs (Symbol | String) -> bool
+    def attribute?(name)
+      actor_class.definition.state_definition.to_h.key?(name.to_sym)
+    end
+  end
+end

data/lib/solid_objects/sync_deadline.rb

@@ -0,0 +1,57 @@
+# rbs_inline: enabled
+
+module SolidObjects
+  class SyncDeadline
+    KEY = :solid_objects_sync_deadline
+
+    class << self
+      # @rbs (timeout: Numeric) { () -> untyped } -> untyped
+      def with(timeout:)
+        previous_deadline = ActiveSupport::IsolatedExecutionState[KEY]
+        ActiveSupport::IsolatedExecutionState[KEY] = monotonic_now + timeout.to_f
+        yield
+      ensure
+        ActiveSupport::IsolatedExecutionState[KEY] = previous_deadline
+      end
+
+      # @rbs () -> bool
+      def active?
+        !deadline.nil?
+      end
+
+      # @rbs () -> bool
+      def expired?
+        active? && !remaining.positive?
+      end
+
+      # @rbs () -> Float
+      def remaining
+        return Float::INFINITY unless deadline
+
+        deadline - monotonic_now
+      end
+
+      # @rbs () -> Integer
+      def remaining_milliseconds
+        [ (remaining * 1_000).floor, 1 ].max
+      end
+
+      # @rbs () -> Integer
+      def remaining_seconds
+        [ remaining.ceil, 1 ].max
+      end
+
+      private
+
+      # @rbs () -> Float?
+      def deadline
+        ActiveSupport::IsolatedExecutionState[KEY]
+      end
+
+      # @rbs () -> Float
+      def monotonic_now
+        ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      end
+    end
+  end
+end

data/lib/solid_objects/sync_diagnostics.rb

@@ -0,0 +1,133 @@
+# rbs_inline: enabled
+
+module SolidObjects
+  class SyncDiagnostics
+    # @rbs (Message, timeout: Numeric) -> SyncTimeout
+    def call(message, timeout:)
+      message = Message.uncached { Message.find(message.id) }
+      instance = Instance.uncached { Instance.find(message.instance_id) }
+      blocker = earlier_blocker(message)
+      status = message_status(message)
+      waiting_on = waiting_reason(message, instance, blocker)
+      activation = activation_details(instance)
+      error = SyncTimeout.new(
+        timeout:,
+        actor_type: message.actor_type,
+        actor_id: message.actor_id,
+        message_name: message.message_name,
+        message_id: message.id,
+        request_id: message.request_id,
+        sequence: message.sequence,
+        status:,
+        waiting_on:,
+        activation:,
+        blocker: blocker_details(blocker)
+      )
+      SolidObjects.instrument(
+        :"sync.timeout",
+        message_id: message.id,
+        request_id: message.request_id,
+        actor_type: message.actor_type,
+        actor_id: message.actor_id,
+        sequence: message.sequence,
+        status:,
+        waiting_on:,
+        activation_owner_id: activation["owner_id"],
+        activation_generation: activation["generation"]
+      )
+      error
+    end
+
+    private
+
+    # @rbs (Message) -> String
+    def message_status(message)
+      return "rejected" if message.rejected?
+      return "completed" if message.completed?
+      return "dead" if message.dead?
+      return "claimed" if ClaimedMessage.where(message_id: message.id).exists?
+      return "ready" if ReadyMessage.where(message_id: message.id).exists?
+
+      "unknown"
+    end
+
+    # @rbs (Message, Instance, Message?) -> String
+    def waiting_reason(message, instance, blocker)
+      return "actor_paused" if instance.paused_at
+      return "activation_held" if live_activation?(instance)
+      return "earlier_message" if blocker
+      return "message_claimed" if ClaimedMessage.where(message_id: message.id).exists?
+
+      ready_message = ReadyMessage.find_by(message_id: message.id)
+      return "not_yet_available" if future?(ready_message)
+      return "ready_unclaimed" if ready_message
+
+      "unknown"
+    end
+
+    # @rbs (ReadyMessage?) -> bool
+    def future?(ready_message)
+      ready_message&.available_at.present? &&
+        ready_message.available_at > SolidObjects.database_adapter.database_now
+    end
+
+    # @rbs (Instance) -> bool
+    def live_activation?(instance)
+      instance.activation_owner_id.present? &&
+        instance.activation_expires_at.present? &&
+        instance.activation_expires_at > SolidObjects.database_adapter.database_now
+    end
+
+    # @rbs (Message) -> Message?
+    def earlier_blocker(message)
+      ready = Message
+        .joins(:ready_message)
+        .where(instance_id: message.instance_id, sequence: ...message.sequence)
+        .order(:sequence)
+        .first
+      claimed = Message
+        .joins(:claimed_message)
+        .where(instance_id: message.instance_id, sequence: ...message.sequence)
+        .order(:sequence)
+        .first
+
+      [ ready, claimed ].compact.min_by(&:sequence)
+    end
+
+    # @rbs (Instance) -> Hash[String, untyped]
+    def activation_details(instance)
+      process_record = Process.find_by(id: instance.activation_owner_id)
+      {
+        "owner_id" => instance.activation_owner_id,
+        "generation" => instance.activation_generation,
+        "expires_at" => instance.activation_expires_at&.iso8601(6),
+        "process" => process_details(process_record)
+      }
+    end
+
+    # @rbs (Process?) -> Hash[String, untyped]?
+    def process_details(process_record)
+      return unless process_record
+
+      {
+        "kind" => process_record.kind,
+        "hostname" => process_record.hostname,
+        "pid" => process_record.pid,
+        "last_heartbeat_at" => process_record.last_heartbeat_at&.iso8601(6),
+        "shutdown_state" => process_record.shutdown_state
+      }
+    end
+
+    # @rbs (Message?) -> Hash[String, untyped]?
+    def blocker_details(blocker)
+      return unless blocker
+
+      {
+        "message_id" => blocker.id,
+        "sequence" => blocker.sequence,
+        "message_name" => blocker.message_name,
+        "status" => message_status(blocker)
+      }
+    end
+  end
+end

data/lib/solid_objects/synchronous_invocation.rb

@@ -2,12 +2,26 @@
 
 require "solid_objects/activation_manager"
 require "solid_objects/lease_renewer"
+require "solid_objects/sync_diagnostics"
 
 module SolidObjects
   class SynchronousInvocation
     # @rbs (MessageReference, timeout: Numeric) -> untyped
     def call(message_reference, timeout:)
-      deadline = monotonic_now + timeout.to_f
+      return call_before_deadline(message_reference, timeout:) if SyncDeadline.active?
+
+      SyncDeadline.with(timeout:) do
+        call_before_deadline(message_reference, timeout:)
+      end
+    rescue ActiveRecord::RecordNotFound
+      raise ActorDestroyed, "actor was destroyed while waiting for its result"
+    end
+
+    private
+
+    # @rbs (MessageReference, timeout: Numeric) -> untyped
+    def call_before_deadline(message_reference, timeout:)
+      deadline = monotonic_now + SyncDeadline.remaining
 
       loop do
         message = load_message(message_reference)
@@ -15,17 +29,22 @@ module SolidObjects
 
         remaining = deadline - monotonic_now
         unless remaining.positive?
-          raise SyncTimeout, "actor invocation timed out after #{timeout} seconds"
+          message = load_message(message_reference)
+          return completed_result(message) if message.completed? || message.dead?
+
+          raise SyncDiagnostics.new.call(message, timeout:)
         end
 
         processed = assist(message, deadline:)
-        wait(remaining) if processed.zero?
+        remaining = deadline - monotonic_now
+        wait(remaining) if processed.zero? && remaining.positive?
       end
-    rescue ActiveRecord::RecordNotFound
-      raise ActorDestroyed, "actor was destroyed while waiting for its result"
-    end
+    rescue DatabaseDeadlineExceeded
+      message = load_message(message_reference)
+      return completed_result(message) if message.completed? || message.dead?
 
-    private
+      raise SyncDiagnostics.new.call(message, timeout:)
+    end
 
     # @rbs (MessageReference) -> Message
     def load_message(message_reference)

data/lib/solid_objects/test_helper.rb

@@ -0,0 +1,78 @@
+# rbs_inline: enabled
+
+require "solid_objects"
+
+module SolidObjects
+  module TestHelper
+    class << self
+      # @rbs (Class) -> void
+      def included(test_case)
+        test_case.use_transactional_tests = false if test_case.respond_to?(:use_transactional_tests=)
+        test_case.setup { reset_actors! }
+        test_case.teardown { reset_actors! }
+      end
+
+      # @rbs () -> void
+      def reset_actors!
+        SolidObjects.reset_caller_process!
+        Instance.delete_all
+        Process.delete_all
+      end
+    end
+
+    # @rbs () -> void
+    def reset_actors!
+      TestHelper.reset_actors!
+    end
+
+    # @rbs (?roles: Array[Symbol], ?max_passes: Integer) -> Integer
+    def drain_solid_objects(
+      roles: [ :reminders, :actors, :effects, :broadcasts ],
+      max_passes: 1_000
+    )
+      runners = build_solid_objects_runners(roles)
+      total = 0
+
+      max_passes.times do
+        processed = runners.sum { |runner| solid_objects_runner_result(runner) }
+        break if processed.zero?
+
+        total += processed
+      end
+
+      total
+    ensure
+      runners&.uniq&.each(&:stop)
+    end
+
+    private
+
+    # @rbs (Array[Symbol]) -> Array[Worker | EffectExecutor | ReminderScheduler | BroadcastExecutor]
+    def build_solid_objects_runners(roles)
+      unknown_roles = roles - [ :actors, :effects, :reminders, :broadcasts ]
+      raise ArgumentError, "unknown Solid Objects test roles: #{unknown_roles.join(", ")}" if unknown_roles.any?
+
+      runners = {}
+      runners[:actors] = Worker.new if roles.include?(:actors)
+      runners[:effects] = EffectExecutor.new if roles.include?(:effects)
+      runners[:reminders] = ReminderScheduler.new if roles.include?(:reminders)
+      runners[:broadcasts] = BroadcastExecutor.new if roles.include?(:broadcasts)
+
+      [
+        runners[:reminders],
+        runners[:actors],
+        runners[:effects],
+        runners[:actors],
+        runners[:broadcasts]
+      ].compact
+    end
+
+    # @rbs (Worker | EffectExecutor | ReminderScheduler | BroadcastExecutor) -> Integer
+    def solid_objects_runner_result(runner)
+      result = runner.run_once
+      return result if result.is_a?(Integer)
+
+      result ? 1 : 0
+    end
+  end
+end

data/lib/solid_objects/version.rb

@@ -1,5 +1,5 @@
 # rbs_inline: enabled
 
 module SolidObjects
-  VERSION = "0.2.1"
+  VERSION = "0.3.0"
 end

data/lib/solid_objects/worker.rb

@@ -43,7 +43,7 @@ module SolidObjects
     rescue ActorDestroyed
       release_activation(activation) if activation
       0
-    rescue StateMigrationError, LostActivation => error
+    rescue StateMigrationError, ApplicationWriteForbidden, LostActivation => error
       release_activation(activation) if activation
       SolidObjects.configuration.logger.error(
         event: "solid_objects.worker.error",