smith-agents 0.4.4 → 0.5.0

data/lib/smith/workflow/split_step_persistence/boundary.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+module Smith
+  class Workflow
+    module SplitStepPersistence
+      module Boundary
+        def run_persisted!(...)
+          ensure_no_split_step_boundary!
+          super
+        end
+
+        def advance_persisted!(...)
+          ensure_no_split_step_boundary!
+          super
+        end
+
+        def clear_persisted!(...)
+          ensure_no_split_step_boundary!
+          super
+        end
+
+        def initialize_copy(source)
+          super
+          copied_split_step_phase = @split_step_phase
+          detach_copied_runtime_state!(source)
+          return unless copied_split_step_phase
+
+          @split_step_phase = :copied_boundary
+          reset_copied_execution_state!
+          reset_copied_dispatch_state!
+        end
+        private :initialize_copy
+
+        private
+
+        def reset_copied_execution_state!
+          @split_step_token = nil
+          @split_step_prepared_descriptor = nil
+          @split_step_transaction_identity = nil
+          @split_step_execution_thread = nil
+          @split_step_active_execution_authorization = nil
+          @split_step_execution_result = nil
+          @split_step_advance_permit = false
+          clear_split_step_execution_verification!
+          clear_split_step_execution_authorization!
+        end
+
+        def reset_copied_dispatch_state!
+          @split_step_dispatch_token = nil
+          @split_step_dispatch_descriptor = nil
+          @split_step_attempted_dispatch_descriptor = nil
+          @split_step_dispatch_thread = nil
+          @split_step_pre_dispatch_payload = nil
+          @split_step_attempted_dispatch_payload = nil
+          @split_step_preparation_thread = nil
+          @split_step_persist_permit = false
+        end
+
+        def detach_copied_runtime_state!(source)
+          source_messages = source.instance_variable_get(:@session_messages)
+          @split_step_mutex = Mutex.new
+          @persisted_keys_mutex = Mutex.new
+          @tool_results_mutex = Mutex.new
+          @usage_mutex = Mutex.new
+          detach_split_step_execution_state!
+          @session_messages = snapshot_value(source_messages || [])
+        end
+
+        def ensure_split_step_execution_allowed!
+          @split_step_mutex.synchronize do
+            return unless @split_step_phase
+
+            if split_step_advance_permitted?
+              @split_step_advance_permit = false
+              return
+            end
+          end
+
+          raise WorkflowError, "use execute_prepared_step! for the active split-step boundary"
+        end
+
+        def claim_split_step_advance!
+          @split_step_mutex.synchronize do
+            unless @split_step_phase
+              @split_step_phase = :ordinary_execution
+              return :ordinary
+            end
+            if split_step_advance_permitted?
+              @split_step_advance_permit = false
+              return :split_step
+            end
+
+            raise WorkflowError, "use execute_prepared_step! for the active split-step boundary"
+          end
+        end
+
+        def release_split_step_advance!(claim)
+          return unless claim == :ordinary
+
+          @split_step_mutex.synchronize do
+            @split_step_phase = nil if @split_step_phase == :ordinary_execution
+          end
+        end
+
+        def ensure_no_split_step_boundary!
+          return unless @split_step_phase
+
+          raise WorkflowError, "a split-step persistence boundary is already active"
+        end
+
+        def guard_split_step_subclass_execution!
+          @split_step_mutex.synchronize do
+            return unless @split_step_phase
+            return if split_step_advance_permitted?
+          end
+
+          raise WorkflowError, "use execute_prepared_step! for the active split-step boundary"
+        end
+
+        def split_step_advance_permitted?
+          @split_step_phase == :executing &&
+            @split_step_execution_thread.equal?(Thread.current) &&
+            @split_step_advance_permit
+        end
+      end
+    end
+  end
+end

data/lib/smith/workflow/split_step_persistence/boundary_reset.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Smith
+  class Workflow
+    module SplitStepPersistence
+      module BoundaryReset
+        private
+
+        def clear_split_step_boundary!
+          @split_step_phase = nil
+          @split_step_transition_name = nil
+          @split_step_transition = nil
+          @split_step_transition_signature = nil
+          @split_step_origin_state = nil
+          @split_step_token = nil
+          @split_step_active_execution_authorization = nil
+          @split_step_execution_result = nil
+          clear_split_step_descriptor_state!
+          @split_step_persistence_key = nil
+          @split_step_adapter = nil
+          remove_instance_variable(:@split_step_persistence_ttl) if
+            instance_variable_defined?(:@split_step_persistence_ttl)
+          @split_step_preparation_payload = nil
+          @split_step_checkpoint_digest = nil
+          @split_step_checkpoint_version = nil
+          @split_step_preparation_thread = nil
+          @split_step_persist_permit = false
+        end
+
+        def clear_split_step_descriptor_state!
+          @split_step_prepared_descriptor = nil
+          @split_step_transaction_identity = nil
+          @split_step_preparation_digest = nil
+          @split_step_pending_descriptor = nil
+          @split_step_dispatch_started = false
+          @split_step_dispatch_token = nil
+          @split_step_dispatch_descriptor = nil
+          @split_step_attempted_dispatch_descriptor = nil
+          @split_step_dispatch_thread = nil
+          @split_step_pre_dispatch_payload = nil
+          @split_step_attempted_dispatch_payload = nil
+          clear_split_step_execution_verification!
+          clear_split_step_execution_authorization!
+          remove_instance_variable(:@split_step_definition_digest) if
+            instance_variable_defined?(:@split_step_definition_digest)
+        end
+      end
+    end
+  end
+end

data/lib/smith/workflow/split_step_persistence/canonical_payload_digest.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require "digest"
+require "dry-initializer"
+require "json"
+
+module Smith
+  class Workflow
+    module SplitStepPersistence
+      class CanonicalPayloadDigest
+        extend Dry::Initializer
+
+        MAX_BYTES = 4 * 1024 * 1024
+        MAX_NODES = 100_000
+
+        param :payload
+
+        def self.call(payload) = new(payload).call
+
+        def call
+          if payload.bytesize > MAX_BYTES
+            raise WorkflowError, "split-step preparation payload exceeds maximum bytes #{MAX_BYTES}"
+          end
+
+          @nodes = 0
+          canonical = canonicalize(JSON.parse(payload))
+          Digest::SHA256.hexdigest(JSON.generate(canonical))
+        end
+
+        private
+
+        def canonicalize(value)
+          visit!
+          case value
+          when Hash then canonical_hash(value)
+          when Array then value.map { |item| canonicalize(item) }
+          when Float then value == value.to_i ? value.to_i : value
+          else value
+          end
+        end
+
+        def canonical_hash(value)
+          value.keys.sort.to_h { |key| [key, canonicalize(value.fetch(key))] }
+        end
+
+        def visit!
+          @nodes += 1
+          return if @nodes <= MAX_NODES
+
+          raise WorkflowError, "split-step preparation payload exceeds maximum size #{MAX_NODES}"
+        end
+      end
+    end
+  end
+end

data/lib/smith/workflow/split_step_persistence/checkpoint.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+module Smith
+  class Workflow
+    module SplitStepPersistence
+      module Checkpoint
+        def complete_persisted_step!
+          completion_claimed = false
+          previous_phase = claim_split_step_completion!
+          completion_claimed = true
+          completed = false
+          payload = @split_step_adapter.fetch(@split_step_persistence_key)
+          unless persisted_split_step_checkpoint?(payload)
+            recover_rolled_back_split_step_checkpoint!(payload)
+            raise WorkflowError, "the persisted split-step checkpoint is not committed"
+          end
+
+          @split_step_mutex.synchronize do
+            @persistence_version = @split_step_checkpoint_version
+            clear_step_in_progress!
+            clear_split_step_boundary!
+          end
+          completed = true
+          self
+        ensure
+          restore_incomplete_checkpoint!(previous_phase) if completion_claimed && !completed
+        end
+
+        def persist!(key = nil, adapter: Smith.persistence_adapter)
+          checkpoint_claimed = false
+          resolved_key, store = resolve_split_step_target(key, adapter)
+          checkpoint_claimed = claim_split_step_checkpoint!
+          result = super(resolved_key, adapter: store) do |payload, next_version|
+            capture_split_step_payload!(payload, next_version, checkpoint_claimed:)
+          end
+          @split_step_mutex.synchronize { @split_step_phase = :checkpointed } if checkpoint_claimed
+          result
+        rescue StandardError
+          mark_split_step_checkpoint_unknown! if checkpoint_claimed
+          raise
+        end
+
+        private
+
+        def resolve_split_step_target(key, adapter)
+          resolved_key = if @split_step_persistence_key
+                           candidate = candidate_split_step_persistence_key(key)
+                           if candidate.equal?(@split_step_persistence_key)
+                             candidate
+                           else
+                             normalize_split_step_persistence_key(candidate)
+                           end
+                         else
+                           resolve_persistence_key!(key)
+                         end
+          store = persistence_adapter!(adapter)
+          validate_split_step_target!(resolved_key, store)
+          resolved_key = @split_step_persistence_key if @split_step_persistence_key
+          @persistence_key = resolved_key
+          [resolved_key, store]
+        end
+
+        def claim_split_step_checkpoint!
+          @split_step_mutex.synchronize do
+            if split_step_preparation_persist_permitted?
+              @split_step_persist_permit = false
+              return false
+            end
+            return false unless @split_step_phase
+
+            if %i[executed checkpoint_retryable].include?(@split_step_phase)
+              @split_step_phase = :checkpointing
+              return true
+            end
+
+            raise WorkflowError, "the active split-step boundary cannot be checkpointed"
+          end
+        end
+
+        def split_step_preparation_persist_permitted?
+          @split_step_phase == :preparing &&
+            @split_step_preparation_thread.equal?(Thread.current) &&
+            @split_step_persist_permit
+        end
+
+        def validate_split_step_target!(key, adapter)
+          return unless @split_step_persistence_key
+          return if key == @split_step_persistence_key && adapter.equal?(@split_step_adapter)
+
+          raise WorkflowError, "the split-step persistence target cannot change"
+        end
+
+        def capture_split_step_payload!(payload, next_version, checkpoint_claimed:)
+          return capture_split_step_preparation_payload!(payload, next_version) if @split_step_phase == :preparing
+
+          validate_split_step_marker!(payload, expected: true) if checkpoint_claimed
+          return payload unless checkpoint_claimed
+
+          checkpoint_payload = split_step_checkpoint_payload(payload)
+          checkpoint_digest = Digest::SHA256.hexdigest(checkpoint_payload)
+          validate_split_step_retry_payload!(checkpoint_digest)
+          @split_step_checkpoint_digest = checkpoint_digest
+          @split_step_checkpoint_version = next_version
+          checkpoint_payload
+        end
+
+        def validate_split_step_retry_payload!(checkpoint_digest)
+          return unless @split_step_checkpoint_digest
+          return if @split_step_checkpoint_digest == checkpoint_digest
+
+          @split_step_mutex.synchronize do
+            @split_step_phase = :checkpoint_retryable if @split_step_phase == :checkpointing
+          end
+          raise WorkflowError, "the reconciled split-step checkpoint payload changed"
+        end
+      end
+    end
+  end
+end

data/lib/smith/workflow/split_step_persistence/checkpoint_state.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Smith
+  class Workflow
+    module SplitStepPersistence
+      module CheckpointState
+        private
+
+        def claim_split_step_completion!
+          @split_step_mutex.synchronize do
+            unless %i[checkpoint_unknown checkpointed].include?(@split_step_phase)
+              raise WorkflowError, "no persisted split-step checkpoint is awaiting completion"
+            end
+            if Smith::PersistenceAdapters.supports?(@split_step_adapter, :transaction_open?) &&
+               @split_step_adapter.transaction_open?
+              raise WorkflowError, "the split-step checkpoint transaction is still open"
+            end
+
+            previous_phase = @split_step_phase
+            @split_step_phase = :confirming_checkpoint
+            previous_phase
+          end
+        end
+
+        def restore_incomplete_checkpoint!(previous_phase)
+          @split_step_mutex.synchronize do
+            @split_step_phase = previous_phase if @split_step_phase == :confirming_checkpoint
+          end
+        end
+
+        def recover_rolled_back_split_step_checkpoint!(payload)
+          return unless persisted_split_step_payload?(payload, @split_step_preparation_payload)
+
+          preparation_version = JSON.parse(payload).fetch("persistence_version")
+          @split_step_mutex.synchronize do
+            return unless @split_step_phase == :confirming_checkpoint
+
+            @persistence_version = preparation_version
+            @split_step_phase = :checkpoint_retryable
+          end
+        end
+
+        def mark_split_step_checkpoint_unknown!
+          @split_step_mutex.synchronize do
+            @split_step_phase = :checkpoint_unknown if @split_step_phase == :checkpointing
+          end
+        end
+      end
+    end
+  end
+end

data/lib/smith/workflow/split_step_persistence/definition_boundary.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Smith
+  class Workflow
+    module SplitStepPersistence
+      module DefinitionBoundary
+        private
+
+        def restart_safe_split_step? = !effective_definition_digest.nil?
+
+        def ensure_split_step_definition_current!
+          return if effective_definition_digest == self.class.definition_digest
+
+          raise WorkflowError, "the prepared workflow definition has changed"
+        end
+      end
+    end
+  end
+end

data/lib/smith/workflow/split_step_persistence/dispatch_boundary.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Smith
+  class Workflow
+    module SplitStepPersistence
+      module DispatchBoundary
+        private
+
+        def dispatch_store!(...)
+          @split_step_dispatch_started = true if @split_step_phase == :preparing
+          super
+        end
+      end
+    end
+  end
+end

data/lib/smith/workflow/split_step_persistence/dispatch_claim.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+module Smith
+  class Workflow
+    module SplitStepPersistence
+      module DispatchClaim
+        def claim_prepared_step_dispatch!
+          ensure_restart_safe_dispatch_claim!
+          ensure_split_step_definition_current!
+          transaction_identity = TransactionIdentity.capture(@split_step_adapter)
+          claim_dispatch_intent!
+          payload, descriptor = build_split_step_dispatch_claim
+          dispatch_started = true
+          replace_exact_dispatch!(payload)
+          accept_split_step_dispatch!(payload, descriptor, transaction_identity)
+          descriptor
+        rescue PersistencePayloadConflict
+          reject_split_step_dispatch!
+          raise
+        rescue StandardError
+          if dispatch_started
+            mark_split_step_dispatch_unknown!(payload, descriptor)
+          else
+            restore_unclaimed_dispatch!
+          end
+          raise
+        end
+
+        private
+
+        def ensure_restart_safe_dispatch_claim!
+          return if restart_safe_split_step?
+
+          raise WorkflowError, "restart-safe dispatch claiming requires a definition_digest"
+        end
+
+        def split_step_dispatch_payload(token)
+          document = JSON.parse(@split_step_preparation_payload)
+          document["split_step_phase"] = "dispatching"
+          document["split_step_dispatch_token"] = token
+          JSON.generate(document)
+        end
+
+        def build_split_step_dispatch_claim
+          token = SecureRandom.uuid.freeze
+          payload = split_step_dispatch_payload(token)
+          [payload, build_split_step_dispatch(token, payload)]
+        end
+
+        def replace_exact_dispatch!(payload)
+          @split_step_adapter.replace_exact(
+            @split_step_persistence_key,
+            payload,
+            expected_payload: @split_step_preparation_payload,
+            ttl: nil
+          )
+        end
+
+        def build_split_step_dispatch(token, payload)
+          PreparedStepDispatch.new(
+            prepared_step: @split_step_prepared_descriptor,
+            token:,
+            dispatch_digest: CanonicalPayloadDigest.call(payload)
+          )
+        end
+
+        def accept_split_step_dispatch!(payload, descriptor, transaction_identity)
+          @split_step_mutex.synchronize do
+            unless active_split_step_dispatch_claim?
+              raise WorkflowError, "the prepared dispatch claim is no longer active"
+            end
+
+            @split_step_pre_dispatch_payload = @split_step_preparation_payload
+            @split_step_preparation_payload = payload.freeze
+            @split_step_dispatch_token = descriptor.token
+            @split_step_dispatch_descriptor = descriptor
+            @split_step_dispatch_thread = nil
+            @split_step_phase = transaction_identity ? :dispatch_claimed_uncommitted : :dispatch_claimed
+          end
+        end
+
+        def reject_split_step_dispatch!
+          @split_step_mutex.synchronize do
+            next unless active_split_step_dispatch_claim?
+
+            @split_step_dispatch_thread = nil
+            @split_step_phase = :dispatch_rejected
+          end
+        end
+
+        def restore_unclaimed_dispatch!
+          @split_step_mutex.synchronize do
+            next unless active_split_step_dispatch_claim?
+
+            @split_step_dispatch_thread = nil
+            @split_step_phase = :prepared
+          end
+        end
+
+        def mark_split_step_dispatch_unknown!(payload, descriptor)
+          @split_step_mutex.synchronize do
+            next unless active_split_step_dispatch_claim?
+
+            @split_step_attempted_dispatch_payload = payload&.freeze
+            @split_step_dispatch_token = descriptor&.token
+            @split_step_attempted_dispatch_descriptor = descriptor
+            @split_step_dispatch_thread = nil
+            @split_step_phase = :dispatch_unknown
+          end
+        end
+
+        def active_split_step_dispatch_claim?
+          @split_step_phase == :claiming_dispatch && @split_step_dispatch_thread.equal?(Thread.current)
+        end
+      end
+    end
+  end
+end

data/lib/smith/workflow/split_step_persistence/dispatch_confirmation.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Smith
+  class Workflow
+    module SplitStepPersistence
+      module DispatchConfirmation
+        def confirm_prepared_step_dispatch!
+          confirmation_claimed = false
+          rolled_back = false
+          claim_dispatch_confirmation!
+          confirmation_claimed = true
+          payload = @split_step_adapter.fetch(@split_step_persistence_key)
+          return confirm_committed_dispatch! if persisted_split_step_payload?(payload, @split_step_preparation_payload)
+
+          rolled_back = recover_rolled_back_dispatch!(payload)
+          raise WorkflowError, "the persisted split-step dispatch claim is not committed"
+        rescue StandardError
+          mark_unconfirmed_dispatch_unknown! if confirmation_claimed && !rolled_back
+          raise
+        end
+
+        private
+
+        def claim_dispatch_confirmation!
+          @split_step_mutex.synchronize do
+            unless @split_step_phase == :dispatch_claimed_uncommitted
+              raise WorkflowError, "no uncommitted split-step dispatch claim is awaiting confirmation"
+            end
+            if @split_step_adapter.transaction_open?
+              raise WorkflowError, "the split-step dispatch transaction is still open"
+            end
+
+            @split_step_phase = :confirming_dispatch
+          end
+        end
+
+        def confirm_committed_dispatch!
+          @split_step_mutex.synchronize do
+            @split_step_phase = :dispatch_claimed
+            clear_split_step_dispatch_transaction!
+          end
+          self
+        end
+
+        def recover_rolled_back_dispatch!(payload)
+          return false unless persisted_split_step_payload?(payload, @split_step_pre_dispatch_payload)
+
+          @split_step_mutex.synchronize do
+            return false unless @split_step_phase == :confirming_dispatch
+
+            @split_step_preparation_payload = @split_step_pre_dispatch_payload
+            @split_step_phase = :prepared
+            clear_split_step_dispatch_transaction!
+            true
+          end
+        end
+
+        def mark_unconfirmed_dispatch_unknown!
+          @split_step_mutex.synchronize do
+            @split_step_phase = :dispatch_unknown if @split_step_phase == :confirming_dispatch
+          end
+        end
+
+        def clear_split_step_dispatch_transaction!
+          @split_step_pre_dispatch_payload = nil
+        end
+      end
+    end
+  end
+end

data/lib/smith/workflow/split_step_persistence/dispatch_verification.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Smith
+  class Workflow
+    module SplitStepPersistence
+      module DispatchVerification
+        private
+
+        def claim_dispatch_intent!
+          @split_step_mutex.synchronize do
+            raise WorkflowError, "no persisted step is prepared" unless @split_step_phase == :prepared
+
+            ensure_prepared_split_step_transition_matches!
+            @split_step_phase = :claiming_dispatch
+            @split_step_dispatch_thread = Thread.current
+          end
+        end
+
+        def verify_split_step_dispatch_available!(verification_token)
+          payload = @split_step_adapter.fetch(@split_step_persistence_key)
+          return if persisted_split_step_payload?(payload, @split_step_preparation_payload)
+
+          @split_step_mutex.synchronize do
+            if active_split_step_execution_verification?(verification_token)
+              @split_step_phase = :dispatch_unknown
+              @split_step_execution_previous_phase = nil
+              clear_split_step_execution_verification!
+            end
+          end
+          raise PersistencePayloadConflict.new(key: @split_step_persistence_key)
+        end
+      end
+    end
+  end
+end