smith-agents 0.4.5 → 0.5.0

data/lib/smith/workflow/split_step_persistence/boundary.rb

@@ -21,19 +21,50 @@ module Smith
 
         def initialize_copy(source)
           super
-          @split_step_mutex = Mutex.new
-          return unless source.instance_variable_get(:@split_step_phase)
+          copied_split_step_phase = @split_step_phase
+          detach_copied_runtime_state!(source)
+          return unless copied_split_step_phase
 
           @split_step_phase = :copied_boundary
+          reset_copied_execution_state!
+          reset_copied_dispatch_state!
+        end
+        private :initialize_copy
+
+        private
+
+        def reset_copied_execution_state!
           @split_step_token = nil
+          @split_step_prepared_descriptor = nil
+          @split_step_transaction_identity = nil
           @split_step_execution_thread = nil
+          @split_step_active_execution_authorization = nil
+          @split_step_execution_result = nil
           @split_step_advance_permit = false
+          clear_split_step_execution_verification!
+          clear_split_step_execution_authorization!
+        end
+
+        def reset_copied_dispatch_state!
+          @split_step_dispatch_token = nil
+          @split_step_dispatch_descriptor = nil
+          @split_step_attempted_dispatch_descriptor = nil
+          @split_step_dispatch_thread = nil
+          @split_step_pre_dispatch_payload = nil
+          @split_step_attempted_dispatch_payload = nil
           @split_step_preparation_thread = nil
           @split_step_persist_permit = false
         end
-        private :initialize_copy
 
-        private
+        def detach_copied_runtime_state!(source)
+          source_messages = source.instance_variable_get(:@session_messages)
+          @split_step_mutex = Mutex.new
+          @persisted_keys_mutex = Mutex.new
+          @tool_results_mutex = Mutex.new
+          @usage_mutex = Mutex.new
+          detach_split_step_execution_state!
+          @session_messages = snapshot_value(source_messages || [])
+        end
 
         def ensure_split_step_execution_allowed!
           @split_step_mutex.synchronize do
@@ -91,24 +122,6 @@ module Smith
             @split_step_execution_thread.equal?(Thread.current) &&
             @split_step_advance_permit
         end
-
-        def clear_split_step_boundary!
-          @split_step_phase = nil
-          @split_step_transition_name = nil
-          @split_step_transition = nil
-          @split_step_transition_signature = nil
-          @split_step_origin_state = nil
-          @split_step_token = nil
-          @split_step_persistence_key = nil
-          @split_step_adapter = nil
-          remove_instance_variable(:@split_step_persistence_ttl) if
-            instance_variable_defined?(:@split_step_persistence_ttl)
-          @split_step_preparation_payload = nil
-          @split_step_checkpoint_digest = nil
-          @split_step_checkpoint_version = nil
-          @split_step_preparation_thread = nil
-          @split_step_persist_permit = false
-        end
       end
     end
   end

data/lib/smith/workflow/split_step_persistence/boundary_reset.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Smith
+  class Workflow
+    module SplitStepPersistence
+      module BoundaryReset
+        private
+
+        def clear_split_step_boundary!
+          @split_step_phase = nil
+          @split_step_transition_name = nil
+          @split_step_transition = nil
+          @split_step_transition_signature = nil
+          @split_step_origin_state = nil
+          @split_step_token = nil
+          @split_step_active_execution_authorization = nil
+          @split_step_execution_result = nil
+          clear_split_step_descriptor_state!
+          @split_step_persistence_key = nil
+          @split_step_adapter = nil
+          remove_instance_variable(:@split_step_persistence_ttl) if
+            instance_variable_defined?(:@split_step_persistence_ttl)
+          @split_step_preparation_payload = nil
+          @split_step_checkpoint_digest = nil
+          @split_step_checkpoint_version = nil
+          @split_step_preparation_thread = nil
+          @split_step_persist_permit = false
+        end
+
+        def clear_split_step_descriptor_state!
+          @split_step_prepared_descriptor = nil
+          @split_step_transaction_identity = nil
+          @split_step_preparation_digest = nil
+          @split_step_pending_descriptor = nil
+          @split_step_dispatch_started = false
+          @split_step_dispatch_token = nil
+          @split_step_dispatch_descriptor = nil
+          @split_step_attempted_dispatch_descriptor = nil
+          @split_step_dispatch_thread = nil
+          @split_step_pre_dispatch_payload = nil
+          @split_step_attempted_dispatch_payload = nil
+          clear_split_step_execution_verification!
+          clear_split_step_execution_authorization!
+          remove_instance_variable(:@split_step_definition_digest) if
+            instance_variable_defined?(:@split_step_definition_digest)
+        end
+      end
+    end
+  end
+end

data/lib/smith/workflow/split_step_persistence/canonical_payload_digest.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require "digest"
+require "dry-initializer"
+require "json"
+
+module Smith
+  class Workflow
+    module SplitStepPersistence
+      class CanonicalPayloadDigest
+        extend Dry::Initializer
+
+        MAX_BYTES = 4 * 1024 * 1024
+        MAX_NODES = 100_000
+
+        param :payload
+
+        def self.call(payload) = new(payload).call
+
+        def call
+          if payload.bytesize > MAX_BYTES
+            raise WorkflowError, "split-step preparation payload exceeds maximum bytes #{MAX_BYTES}"
+          end
+
+          @nodes = 0
+          canonical = canonicalize(JSON.parse(payload))
+          Digest::SHA256.hexdigest(JSON.generate(canonical))
+        end
+
+        private
+
+        def canonicalize(value)
+          visit!
+          case value
+          when Hash then canonical_hash(value)
+          when Array then value.map { |item| canonicalize(item) }
+          when Float then value == value.to_i ? value.to_i : value
+          else value
+          end
+        end
+
+        def canonical_hash(value)
+          value.keys.sort.to_h { |key| [key, canonicalize(value.fetch(key))] }
+        end
+
+        def visit!
+          @nodes += 1
+          return if @nodes <= MAX_NODES
+
+          raise WorkflowError, "split-step preparation payload exceeds maximum size #{MAX_NODES}"
+        end
+      end
+    end
+  end
+end

data/lib/smith/workflow/split_step_persistence/checkpoint.rb

@@ -91,11 +91,9 @@ module Smith
         end
 
         def capture_split_step_payload!(payload, next_version, checkpoint_claimed:)
-          validate_split_step_marker!(payload, expected: true) if @split_step_phase == :preparing || checkpoint_claimed
-          if @split_step_phase == :preparing
-            payload = split_step_preparation_payload(payload)
-            @split_step_preparation_payload = payload
-          end
+          return capture_split_step_preparation_payload!(payload, next_version) if @split_step_phase == :preparing
+
+          validate_split_step_marker!(payload, expected: true) if checkpoint_claimed
           return payload unless checkpoint_claimed
 
           checkpoint_payload = split_step_checkpoint_payload(payload)

data/lib/smith/workflow/split_step_persistence/definition_boundary.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Smith
+  class Workflow
+    module SplitStepPersistence
+      module DefinitionBoundary
+        private
+
+        def restart_safe_split_step? = !effective_definition_digest.nil?
+
+        def ensure_split_step_definition_current!
+          return if effective_definition_digest == self.class.definition_digest
+
+          raise WorkflowError, "the prepared workflow definition has changed"
+        end
+      end
+    end
+  end
+end

data/lib/smith/workflow/split_step_persistence/dispatch_boundary.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Smith
+  class Workflow
+    module SplitStepPersistence
+      module DispatchBoundary
+        private
+
+        def dispatch_store!(...)
+          @split_step_dispatch_started = true if @split_step_phase == :preparing
+          super
+        end
+      end
+    end
+  end
+end

data/lib/smith/workflow/split_step_persistence/dispatch_claim.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+module Smith
+  class Workflow
+    module SplitStepPersistence
+      module DispatchClaim
+        def claim_prepared_step_dispatch!
+          ensure_restart_safe_dispatch_claim!
+          ensure_split_step_definition_current!
+          transaction_identity = TransactionIdentity.capture(@split_step_adapter)
+          claim_dispatch_intent!
+          payload, descriptor = build_split_step_dispatch_claim
+          dispatch_started = true
+          replace_exact_dispatch!(payload)
+          accept_split_step_dispatch!(payload, descriptor, transaction_identity)
+          descriptor
+        rescue PersistencePayloadConflict
+          reject_split_step_dispatch!
+          raise
+        rescue StandardError
+          if dispatch_started
+            mark_split_step_dispatch_unknown!(payload, descriptor)
+          else
+            restore_unclaimed_dispatch!
+          end
+          raise
+        end
+
+        private
+
+        def ensure_restart_safe_dispatch_claim!
+          return if restart_safe_split_step?
+
+          raise WorkflowError, "restart-safe dispatch claiming requires a definition_digest"
+        end
+
+        def split_step_dispatch_payload(token)
+          document = JSON.parse(@split_step_preparation_payload)
+          document["split_step_phase"] = "dispatching"
+          document["split_step_dispatch_token"] = token
+          JSON.generate(document)
+        end
+
+        def build_split_step_dispatch_claim
+          token = SecureRandom.uuid.freeze
+          payload = split_step_dispatch_payload(token)
+          [payload, build_split_step_dispatch(token, payload)]
+        end
+
+        def replace_exact_dispatch!(payload)
+          @split_step_adapter.replace_exact(
+            @split_step_persistence_key,
+            payload,
+            expected_payload: @split_step_preparation_payload,
+            ttl: nil
+          )
+        end
+
+        def build_split_step_dispatch(token, payload)
+          PreparedStepDispatch.new(
+            prepared_step: @split_step_prepared_descriptor,
+            token:,
+            dispatch_digest: CanonicalPayloadDigest.call(payload)
+          )
+        end
+
+        def accept_split_step_dispatch!(payload, descriptor, transaction_identity)
+          @split_step_mutex.synchronize do
+            unless active_split_step_dispatch_claim?
+              raise WorkflowError, "the prepared dispatch claim is no longer active"
+            end
+
+            @split_step_pre_dispatch_payload = @split_step_preparation_payload
+            @split_step_preparation_payload = payload.freeze
+            @split_step_dispatch_token = descriptor.token
+            @split_step_dispatch_descriptor = descriptor
+            @split_step_dispatch_thread = nil
+            @split_step_phase = transaction_identity ? :dispatch_claimed_uncommitted : :dispatch_claimed
+          end
+        end
+
+        def reject_split_step_dispatch!
+          @split_step_mutex.synchronize do
+            next unless active_split_step_dispatch_claim?
+
+            @split_step_dispatch_thread = nil
+            @split_step_phase = :dispatch_rejected
+          end
+        end
+
+        def restore_unclaimed_dispatch!
+          @split_step_mutex.synchronize do
+            next unless active_split_step_dispatch_claim?
+
+            @split_step_dispatch_thread = nil
+            @split_step_phase = :prepared
+          end
+        end
+
+        def mark_split_step_dispatch_unknown!(payload, descriptor)
+          @split_step_mutex.synchronize do
+            next unless active_split_step_dispatch_claim?
+
+            @split_step_attempted_dispatch_payload = payload&.freeze
+            @split_step_dispatch_token = descriptor&.token
+            @split_step_attempted_dispatch_descriptor = descriptor
+            @split_step_dispatch_thread = nil
+            @split_step_phase = :dispatch_unknown
+          end
+        end
+
+        def active_split_step_dispatch_claim?
+          @split_step_phase == :claiming_dispatch && @split_step_dispatch_thread.equal?(Thread.current)
+        end
+      end
+    end
+  end
+end

data/lib/smith/workflow/split_step_persistence/dispatch_confirmation.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Smith
+  class Workflow
+    module SplitStepPersistence
+      module DispatchConfirmation
+        def confirm_prepared_step_dispatch!
+          confirmation_claimed = false
+          rolled_back = false
+          claim_dispatch_confirmation!
+          confirmation_claimed = true
+          payload = @split_step_adapter.fetch(@split_step_persistence_key)
+          return confirm_committed_dispatch! if persisted_split_step_payload?(payload, @split_step_preparation_payload)
+
+          rolled_back = recover_rolled_back_dispatch!(payload)
+          raise WorkflowError, "the persisted split-step dispatch claim is not committed"
+        rescue StandardError
+          mark_unconfirmed_dispatch_unknown! if confirmation_claimed && !rolled_back
+          raise
+        end
+
+        private
+
+        def claim_dispatch_confirmation!
+          @split_step_mutex.synchronize do
+            unless @split_step_phase == :dispatch_claimed_uncommitted
+              raise WorkflowError, "no uncommitted split-step dispatch claim is awaiting confirmation"
+            end
+            if @split_step_adapter.transaction_open?
+              raise WorkflowError, "the split-step dispatch transaction is still open"
+            end
+
+            @split_step_phase = :confirming_dispatch
+          end
+        end
+
+        def confirm_committed_dispatch!
+          @split_step_mutex.synchronize do
+            @split_step_phase = :dispatch_claimed
+            clear_split_step_dispatch_transaction!
+          end
+          self
+        end
+
+        def recover_rolled_back_dispatch!(payload)
+          return false unless persisted_split_step_payload?(payload, @split_step_pre_dispatch_payload)
+
+          @split_step_mutex.synchronize do
+            return false unless @split_step_phase == :confirming_dispatch
+
+            @split_step_preparation_payload = @split_step_pre_dispatch_payload
+            @split_step_phase = :prepared
+            clear_split_step_dispatch_transaction!
+            true
+          end
+        end
+
+        def mark_unconfirmed_dispatch_unknown!
+          @split_step_mutex.synchronize do
+            @split_step_phase = :dispatch_unknown if @split_step_phase == :confirming_dispatch
+          end
+        end
+
+        def clear_split_step_dispatch_transaction!
+          @split_step_pre_dispatch_payload = nil
+        end
+      end
+    end
+  end
+end

data/lib/smith/workflow/split_step_persistence/dispatch_verification.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Smith
+  class Workflow
+    module SplitStepPersistence
+      module DispatchVerification
+        private
+
+        def claim_dispatch_intent!
+          @split_step_mutex.synchronize do
+            raise WorkflowError, "no persisted step is prepared" unless @split_step_phase == :prepared
+
+            ensure_prepared_split_step_transition_matches!
+            @split_step_phase = :claiming_dispatch
+            @split_step_dispatch_thread = Thread.current
+          end
+        end
+
+        def verify_split_step_dispatch_available!(verification_token)
+          payload = @split_step_adapter.fetch(@split_step_persistence_key)
+          return if persisted_split_step_payload?(payload, @split_step_preparation_payload)
+
+          @split_step_mutex.synchronize do
+            if active_split_step_execution_verification?(verification_token)
+              @split_step_phase = :dispatch_unknown
+              @split_step_execution_previous_phase = nil
+              clear_split_step_execution_verification!
+            end
+          end
+          raise PersistencePayloadConflict.new(key: @split_step_persistence_key)
+        end
+      end
+    end
+  end
+end

data/lib/smith/workflow/split_step_persistence/execution.rb

@@ -5,74 +5,67 @@ module Smith
     module SplitStepPersistence
       module Execution
         def execute_prepared_step!
-          execution_started = false
+          authorization = ExecutionAuthorization
+                          .instance_method(:authorize_prepared_step_execution!)
+                          .bind_call(self)
+          result = Execution.instance_method(:execute_authorized_prepared_step!).bind_call(self, authorization)
+          result.step_snapshot
+        end
+
+        def execute_authorized_prepared_step!(authorization)
+          authorization = validate_split_step_execution_authorization!(authorization)
           step = nil
-          claim_split_step_execution_verification!
+          result = nil
+          execution_thread = Thread.current
+          execution_active = false
           begin
-            verify_split_step_preparation_available!
-            activate_split_step_execution!
-            execution_started = true
+            activate_split_step_execution!(authorization, execution_thread)
+            execution_active = true
             step = execute_claimed_split_step_transition!
+            result = consume_split_step_execution_result!(execution_thread)
           ensure
-            execution_started ? finish_split_step_execution!(step) : restore_unverified_execution!
+            finish_split_step_execution!(step, authorization, execution_thread) if execution_active
           end
-          step
+          result
         end
 
-        def prepared_persisted_step? = @split_step_mutex.synchronize { @split_step_phase == :prepared }
-
-        private
-
-        def claim_split_step_execution_verification!
+        def prepared_persisted_step?
           @split_step_mutex.synchronize do
-            raise WorkflowError, "no persisted step is prepared" unless @split_step_phase == :prepared
-            raise WorkflowError, "the prepared transition no longer matches the workflow" unless
-              prepared_split_step_transition_matches?
-
-            @split_step_phase = :verifying_execution
+            expected = restart_safe_split_step? ? :dispatch_claimed : :prepared
+            [expected, :execution_authorized].include?(@split_step_phase)
           end
         end
 
-        def prepared_split_step_transition_matches?
-          transition = pending_split_step_transition
-          transition_name = @next_transition_name || transition&.name
-          transition_name == @split_step_transition_name &&
-            transition.equal?(@split_step_transition) &&
-            split_step_transition_signature(transition) == @split_step_transition_signature
-        end
+        private
 
-        def activate_split_step_execution!
+        def activate_split_step_execution!(authorization, execution_thread)
           @split_step_mutex.synchronize do
-            unless @split_step_phase == :verifying_execution
-              raise WorkflowError, "the prepared execution claim is no longer active"
+            unless active_split_step_execution_authorization?(authorization)
+              raise WorkflowError, "the prepared-step execution authorization is no longer active"
             end
 
+            ensure_split_step_definition_current!
+            ensure_prepared_split_step_transition_matches!
+            PreparedStepExecutionAuthorization.instance_method(:activate_execution!)
+                                              .bind_call(authorization, execution_thread)
+
+            @split_step_active_execution_authorization = authorization
+            @split_step_execution_result = nil
             @split_step_phase = :executing
-            @split_step_execution_thread = Thread.current
+            clear_split_step_execution_authorization!
+            @split_step_execution_thread = execution_thread
             @split_step_advance_permit = true
           end
         end
 
-        def restore_unverified_execution!
+        def restore_unverified_execution!(verification_token)
           @split_step_mutex.synchronize do
-            @split_step_phase = :prepared if @split_step_phase == :verifying_execution
-          end
-        end
-
-        def verify_split_step_preparation_available!
-          payload = @split_step_adapter.fetch(@split_step_persistence_key)
-          durable = persisted_split_step_payload?(payload, @split_step_preparation_payload)
-          live = persisted_split_step_payload?(current_split_step_preparation_payload, @split_step_preparation_payload)
-          stable = split_step_transition_signature(@split_step_transition) == @split_step_transition_signature
-          return if durable && live && stable
+            next unless active_split_step_execution_verification?(verification_token)
 
-          raise WorkflowError, "the persisted split-step preparation is no longer available"
-        end
-
-        def current_split_step_preparation_payload
-          split_step_preparation_payload(
-            JSON.generate(to_state.merge(persistence_version: @persistence_version))
-          )
+            @split_step_phase = @split_step_execution_previous_phase
+            @split_step_execution_previous_phase = nil
+            clear_split_step_execution_verification!
+          end
         end
 
         def execute_claimed_split_step_transition!
@@ -82,12 +75,21 @@ module Smith
           raise WorkflowError, "prepared execution did not return the claimed transition"
         end
 
-        def finish_split_step_execution!(step)
+        def finish_split_step_execution!(step, authorization, execution_thread)
           @split_step_mutex.synchronize do
+            next unless @split_step_phase == :executing &&
+                        @split_step_execution_thread.equal?(execution_thread)
+
             @split_step_execution_thread = nil
             @split_step_advance_permit = false
+            @split_step_execution_previous_phase = nil
+            @split_step_active_execution_authorization = nil
+            @split_step_execution_result = nil
             @split_step_phase = step ? :executed : :attempted
           end
+        ensure
+          PreparedStepExecutionAuthorization.instance_method(:close_execution!)
+                                            .bind_call(authorization, execution_thread)
         end
 
         def resolve_split_step_advance_transition