smith-agents 0.4.4 → 0.5.0

data/lib/smith/workflow/split_step_persistence/execution.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+module Smith
+  class Workflow
+    module SplitStepPersistence
+      module Execution
+        def execute_prepared_step!
+          authorization = ExecutionAuthorization
+                          .instance_method(:authorize_prepared_step_execution!)
+                          .bind_call(self)
+          result = Execution.instance_method(:execute_authorized_prepared_step!).bind_call(self, authorization)
+          result.step_snapshot
+        end
+
+        def execute_authorized_prepared_step!(authorization)
+          authorization = validate_split_step_execution_authorization!(authorization)
+          step = nil
+          result = nil
+          execution_thread = Thread.current
+          execution_active = false
+          begin
+            activate_split_step_execution!(authorization, execution_thread)
+            execution_active = true
+            step = execute_claimed_split_step_transition!
+            result = consume_split_step_execution_result!(execution_thread)
+          ensure
+            finish_split_step_execution!(step, authorization, execution_thread) if execution_active
+          end
+          result
+        end
+
+        def prepared_persisted_step?
+          @split_step_mutex.synchronize do
+            expected = restart_safe_split_step? ? :dispatch_claimed : :prepared
+            [expected, :execution_authorized].include?(@split_step_phase)
+          end
+        end
+
+        private
+
+        def activate_split_step_execution!(authorization, execution_thread)
+          @split_step_mutex.synchronize do
+            unless active_split_step_execution_authorization?(authorization)
+              raise WorkflowError, "the prepared-step execution authorization is no longer active"
+            end
+
+            ensure_split_step_definition_current!
+            ensure_prepared_split_step_transition_matches!
+            PreparedStepExecutionAuthorization.instance_method(:activate_execution!)
+                                              .bind_call(authorization, execution_thread)
+
+            @split_step_active_execution_authorization = authorization
+            @split_step_execution_result = nil
+            @split_step_phase = :executing
+            clear_split_step_execution_authorization!
+            @split_step_execution_thread = execution_thread
+            @split_step_advance_permit = true
+          end
+        end
+
+        def restore_unverified_execution!(verification_token)
+          @split_step_mutex.synchronize do
+            next unless active_split_step_execution_verification?(verification_token)
+
+            @split_step_phase = @split_step_execution_previous_phase
+            @split_step_execution_previous_phase = nil
+            clear_split_step_execution_verification!
+          end
+        end
+
+        def execute_claimed_split_step_transition!
+          step = Workflow.instance_method(:advance!).bind_call(self)
+          return step if accepted_split_step_result?(step)
+
+          raise WorkflowError, "prepared execution did not return the claimed transition"
+        end
+
+        def finish_split_step_execution!(step, authorization, execution_thread)
+          @split_step_mutex.synchronize do
+            next unless @split_step_phase == :executing &&
+                        @split_step_execution_thread.equal?(execution_thread)
+
+            @split_step_execution_thread = nil
+            @split_step_advance_permit = false
+            @split_step_execution_previous_phase = nil
+            @split_step_active_execution_authorization = nil
+            @split_step_execution_result = nil
+            @split_step_phase = step ? :executed : :attempted
+          end
+        ensure
+          PreparedStepExecutionAuthorization.instance_method(:close_execution!)
+                                            .bind_call(authorization, execution_thread)
+        end
+
+        def resolve_split_step_advance_transition
+          @split_step_mutex.synchronize do
+            unless @split_step_phase == :executing && @split_step_execution_thread.equal?(Thread.current)
+              return NO_SPLIT_TRANSITION
+            end
+
+            name = @next_transition_name
+            @next_transition_name = nil if name
+            raise UnresolvedTransitionError.new(name, self.class, @state) if name && @split_step_transition.nil?
+
+            @split_step_transition
+          end
+        end
+
+        def accepted_split_step_result?(step)
+          return false unless step.is_a?(Hash)
+          return false unless step[:transition] == @split_step_transition_name
+          return false unless step[:from] == @split_step_origin_state
+
+          step[:error] ? accepted_split_step_failure? : step.key?(:to) && @state == step[:to]
+        end
+
+        def accepted_split_step_failure?
+          failure_name = @split_step_transition&.failure_transition || :fail
+          failure_transition = self.class.find_transition(failure_name)
+          return false unless failure_transition
+
+          @state == failure_transition.to ||
+            (@state == @split_step_origin_state && @next_transition_name == failure_name)
+        end
+      end
+    end
+  end
+end

data/lib/smith/workflow/split_step_persistence/execution_authorization.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module Smith
+  class Workflow
+    module SplitStepPersistence
+      module ExecutionAuthorization
+        def authorize_prepared_step_execution!
+          verification_token = claim_split_step_execution_verification!
+          authorized = false
+          verify_claimed_split_step_execution!(verification_token)
+          authorization = build_split_step_execution_authorization
+          activate_split_step_execution_authorization!(authorization, verification_token)
+          authorized = true
+          authorization
+        ensure
+          restore_unverified_execution!(verification_token) if verification_token && !authorized
+        end
+
+        def release_prepared_step_execution!(authorization)
+          authorization = validate_split_step_execution_authorization!(authorization)
+          @split_step_mutex.synchronize do
+            unless active_split_step_execution_authorization?(authorization)
+              raise WorkflowError, "the prepared-step execution authorization is no longer active"
+            end
+
+            @split_step_phase = @split_step_execution_previous_phase
+            clear_split_step_execution_authorization!
+          end
+          PreparedStepExecutionAuthorization
+            .instance_method(:close_execution!)
+            .bind_call(authorization)
+          self
+        end
+
+        private
+
+        def build_split_step_execution_authorization
+          PreparedStepExecutionAuthorization.new(
+            prepared_step: @split_step_prepared_descriptor,
+            dispatch_claim: @split_step_dispatch_descriptor,
+            execution_bindings: ExecutionBindingSnapshot.capture(
+              @split_step_transition,
+              workflow_class: self.class
+            )
+          )
+        end
+
+        def activate_split_step_execution_authorization!(authorization, verification_token)
+          @split_step_mutex.synchronize do
+            unless active_split_step_execution_verification?(verification_token)
+              raise WorkflowError, "the prepared execution verification is no longer active"
+            end
+
+            @split_step_execution_authorization = authorization
+            clear_split_step_execution_verification!
+            @split_step_phase = :execution_authorized
+          end
+        end
+
+        def validate_split_step_execution_authorization!(authorization)
+          return authorization if authorization.is_a?(PreparedStepExecutionAuthorization)
+
+          raise ArgumentError,
+                "authorization must be a Smith::Workflow::PreparedStepExecutionAuthorization"
+        end
+
+        def active_split_step_execution_authorization?(authorization)
+          @split_step_phase == :execution_authorized &&
+            @split_step_execution_authorization.equal?(authorization) &&
+            authorization.issued_in_current_process?
+        end
+
+        def active_split_step_execution_verification?(verification_token)
+          @split_step_phase == :verifying_execution &&
+            @split_step_execution_verification_token.equal?(verification_token)
+        end
+
+        def clear_split_step_execution_verification!
+          @split_step_execution_verification_token = nil
+        end
+
+        def clear_split_step_execution_authorization!
+          @split_step_execution_authorization = nil
+          @split_step_execution_previous_phase = nil
+        end
+      end
+    end
+  end
+end

data/lib/smith/workflow/split_step_persistence/execution_binding_collector.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require_relative "../../agent"
+require_relative "../../agent/registry"
+
+module Smith
+  class Workflow
+    module SplitStepPersistence
+      class ExecutionBindingCollector
+        MAX_BINDINGS = 10_000
+
+        def initialize
+          @requests = {}
+          @bindings = {}
+        end
+
+        def capture(transition, workflow_class:)
+          capture_binding(transition.agent_name, workflow_class:, transition:, role: :agent)
+          capture_fanout(transition, workflow_class:)
+          capture_optimization(transition, workflow_class:)
+          capture_orchestration(transition, workflow_class:)
+        end
+
+        def fetch!(name, workflow_class:, transition_name:, role:)
+          @bindings.fetch(name.to_s) do
+            raise WorkflowError,
+                  "execution authorization does not contain #{role} :#{name} " \
+                  "for workflow #{workflow_class}, transition :#{transition_name}"
+          end
+        end
+
+        def resolve!
+          @bindings = Agent::Registry.capture_bindings!(@requests.values)
+
+          @requests.freeze
+          self
+        end
+
+        def each(&)
+          @bindings.each(&)
+        end
+
+        def freeze
+          @requests.freeze
+          @bindings.freeze
+          super
+        end
+
+        private
+
+        def capture_fanout(transition, workflow_class:)
+          transition.fanout_config&.fetch(:branches, nil)&.each_value do |agent|
+            capture_binding(agent, workflow_class:, transition:, role: :fanout_agent)
+          end
+        end
+
+        def capture_optimization(transition, workflow_class:)
+          return unless (config = transition.optimization_config)
+
+          capture_binding(config.fetch(:generator), workflow_class:, transition:, role: :generator)
+          capture_binding(config.fetch(:evaluator), workflow_class:, transition:, role: :evaluator)
+        end
+
+        def capture_orchestration(transition, workflow_class:)
+          return unless (config = transition.orchestrator_config)
+
+          capture_binding(config.fetch(:orchestrator), workflow_class:, transition:, role: :orchestrator)
+          capture_binding(config.fetch(:worker), workflow_class:, transition:, role: :worker)
+        end
+
+        def capture_binding(name, workflow_class:, transition:, role:)
+          return unless name
+
+          key = name.to_s
+          return if @requests.key?(key)
+          if @requests.length >= MAX_BINDINGS
+            raise WorkflowError, "execution authorization exceeds maximum agent bindings #{MAX_BINDINGS}"
+          end
+
+          @requests[key] = {
+            name:,
+            workflow_class:,
+            transition_name: transition.name,
+            role:
+          }.freeze
+        end
+      end
+    end
+  end
+end

data/lib/smith/workflow/split_step_persistence/execution_binding_snapshot.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+require_relative "execution_binding_collector"
+require_relative "execution_workflow_snapshot"
+
+module Smith
+  class Workflow
+    module SplitStepPersistence
+      class ExecutionBindingSnapshot
+        MAX_NESTED_WORKFLOWS = 128
+        MAX_TRANSITIONS = 10_000
+
+        def self.capture(transition, workflow_class:)
+          new(transition, workflow_class:).capture
+        end
+
+        def initialize(transition, workflow_class:)
+          @transition = transition
+          @workflow_class = workflow_class
+          @bindings = ExecutionBindingCollector.new
+          @visited_workflows = {}.compare_by_identity
+          @visited_workflows[workflow_class] = true
+          @workflow_snapshots = {}.compare_by_identity
+          @workflow_queue = []
+          @workflow_queue_index = 0
+          @transition_count = 0
+        end
+
+        def capture
+          capture_transition(@transition, workflow_class: @workflow_class)
+          drain_workflow_queue
+          @bindings.resolve!
+          @bindings.freeze
+          @workflow_snapshots.freeze
+          freeze
+        end
+
+        def fetch!(name, workflow_class:, transition_name:, role:)
+          @bindings.fetch!(name, workflow_class:, transition_name:, role:)
+        end
+
+        def each_agent_binding(&)
+          @bindings.each(&)
+        end
+
+        def verify_workflow!(workflow_class)
+          snapshot = @workflow_snapshots.fetch(workflow_class) do
+            raise WorkflowError, "execution authorization does not contain nested workflow #{workflow_class}"
+          end
+          snapshot.verify!
+        end
+
+        private
+
+        def capture_transition(transition, workflow_class:)
+          return unless transition
+
+          visit_transition!
+          @bindings.capture(transition, workflow_class:)
+          enqueue_nested_workflow(transition.workflow_class)
+        end
+
+        def enqueue_nested_workflow(workflow_class)
+          return unless workflow_class
+          return if @visited_workflows.key?(workflow_class)
+
+          @visited_workflows[workflow_class] = true
+          if @visited_workflows.length > MAX_NESTED_WORKFLOWS
+            raise WorkflowError,
+                  "execution authorization exceeds maximum nested workflows #{MAX_NESTED_WORKFLOWS}"
+          end
+
+          @workflow_queue << workflow_class
+        end
+
+        def drain_workflow_queue
+          while @workflow_queue_index < @workflow_queue.length
+            workflow_class = @workflow_queue.fetch(@workflow_queue_index)
+            @workflow_queue_index += 1
+            @workflow_snapshots[workflow_class] = ExecutionWorkflowSnapshot.capture(workflow_class) do |transition|
+              capture_transition(transition, workflow_class:)
+            end
+          end
+        end
+
+        def visit_transition!
+          @transition_count += 1
+          return if @transition_count <= MAX_TRANSITIONS
+
+          raise WorkflowError, "execution authorization exceeds maximum transitions #{MAX_TRANSITIONS}"
+        end
+      end
+    end
+  end
+end

data/lib/smith/workflow/split_step_persistence/execution_result_capture.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Smith
+  class Workflow
+    module SplitStepPersistence
+      module ExecutionResultCapture
+        private
+
+        def capture_split_step_execution_result!(step)
+          result = prepare_split_step_execution_result(step)
+          commit_split_step_execution_result!(result)
+        end
+
+        def prepare_split_step_execution_result(step)
+          return unless active_split_step_execution_thread?
+
+          PreparedStepExecutionResult.from_step(step)
+        end
+
+        def commit_split_step_execution_result!(result)
+          return unless result
+
+          @split_step_mutex.synchronize do
+            return unless active_split_step_execution_thread?
+            raise WorkflowError, "prepared-step execution produced more than one result" if @split_step_execution_result
+
+            @split_step_execution_result = result
+          end
+        end
+
+        def consume_split_step_execution_result!(execution_thread)
+          @split_step_mutex.synchronize do
+            unless @split_step_phase == :executing && @split_step_execution_thread.equal?(execution_thread)
+              raise WorkflowError, "prepared-step execution is no longer active"
+            end
+
+            @split_step_execution_result ||
+              raise(WorkflowError, "prepared-step execution did not produce a result")
+          end
+        end
+
+        def active_split_step_execution_thread?
+          @split_step_phase == :executing && @split_step_execution_thread.equal?(Thread.current)
+        end
+      end
+    end
+  end
+end

data/lib/smith/workflow/split_step_persistence/execution_verification.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Smith
+  class Workflow
+    module SplitStepPersistence
+      module ExecutionVerification
+        private
+
+        def claim_split_step_execution_verification!
+          @split_step_mutex.synchronize do
+            ensure_split_step_definition_current!
+            expected = restart_safe_split_step? ? :dispatch_claimed : :prepared
+            raise WorkflowError, "no persisted step is prepared for execution" unless @split_step_phase == expected
+
+            ensure_prepared_split_step_transition_matches!
+
+            token = Object.new.freeze
+            @split_step_execution_previous_phase = expected
+            @split_step_execution_verification_token = token
+            @split_step_phase = :verifying_execution
+            token
+          end
+        end
+
+        def verify_claimed_split_step_execution!(verification_token)
+          if restart_safe_split_step?
+            verify_split_step_dispatch_available!(verification_token)
+          else
+            verify_split_step_preparation_available!
+          end
+        end
+
+        def prepared_split_step_transition_matches?
+          transition = pending_split_step_transition
+          transition_name = @next_transition_name || transition&.name
+          transition_name == @split_step_transition_name &&
+            transition.equal?(@split_step_transition) &&
+            split_step_transition_signature(transition) == @split_step_transition_signature
+        end
+
+        def ensure_prepared_split_step_transition_matches!
+          return if prepared_split_step_transition_matches?
+
+          raise WorkflowError, "the prepared transition no longer matches the workflow"
+        end
+
+        def verify_split_step_preparation_available!
+          payload = @split_step_adapter.fetch(@split_step_persistence_key)
+          durable = persisted_split_step_payload?(payload, @split_step_preparation_payload)
+          live = persisted_split_step_payload?(current_split_step_preparation_payload, @split_step_preparation_payload)
+          stable = split_step_transition_signature(@split_step_transition) == @split_step_transition_signature
+          return if durable && live && stable
+
+          raise WorkflowError, "the persisted split-step preparation is no longer available"
+        end
+
+        def current_split_step_preparation_payload
+          split_step_preparation_payload(
+            JSON.generate(to_state.merge(persistence_version: @persistence_version))
+          )
+        end
+      end
+    end
+  end
+end

data/lib/smith/workflow/split_step_persistence/execution_workflow_snapshot.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require "dry-initializer"
+
+require_relative "../../errors"
+require_relative "../graph"
+require_relative "transition_contract"
+
+module Smith
+  class Workflow
+    module SplitStepPersistence
+      class ExecutionWorkflowSnapshot
+        extend Dry::Initializer
+
+        param :workflow_class
+
+        def self.capture(workflow_class, &)
+          new(workflow_class).capture(&)
+        end
+
+        def capture
+          @initial_state_signature = initial_state_signature
+          @transitions = reachable_definition_transitions.map do |transition|
+            yield transition if block_given?
+            [transition.name, transition, TransitionContract.capture(transition)].freeze
+          end.freeze
+          freeze
+        end
+
+        def verify!
+          unless initial_state_signature == @initial_state_signature
+            raise WorkflowError, "authorized nested workflow definition changed before execution"
+          end
+
+          index = 0
+          reachable_definition_transitions.each do |transition|
+            expected = @transitions[index]
+            verify_transition!(transition, expected)
+            index += 1
+          end
+          return true if index == @transitions.length
+
+          raise WorkflowError, "authorized nested workflow definition changed before execution"
+        end
+
+        private
+
+        def reachable_definition_transitions
+          workflow_class.graph.reachable_transitions.map do |transition|
+            workflow_class.transition_at(transition.definition_index)
+          end
+        rescue IndexError
+          raise WorkflowError, "authorized nested workflow definition changed before execution"
+        end
+
+        def initial_state_signature
+          TransitionContractSignature.new(
+            value: workflow_class.initial_state,
+            max_depth: TransitionContract::MAX_DEPTH,
+            max_nodes: TransitionContract::MAX_NODES,
+            max_bytes: TransitionContract::MAX_BYTES
+          ).call
+        end
+
+        def verify_transition!(transition, expected)
+          matches = expected &&
+                    transition.name == expected.fetch(0) &&
+                    transition.equal?(expected.fetch(1)) &&
+                    TransitionContract.signature(transition) == expected.fetch(2)
+          return if matches
+
+          raise WorkflowError, "authorized nested workflow definition changed before execution"
+        end
+      end
+    end
+  end
+end

data/lib/smith/workflow/split_step_persistence/inheritance.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Smith
+  class Workflow
+    module SplitStepPersistence
+      module Inheritance
+        def inherited(subclass)
+          super
+          subclass.prepend(SubclassBoundary)
+        end
+
+        def prepend(*features)
+          result = super
+          guard = SubclassBoundary.guard
+          guard.freeze
+          Module.instance_method(:prepend).bind_call(self, guard)
+          result
+        end
+      end
+    end
+  end
+end

data/lib/smith/workflow/split_step_persistence/payloads.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Smith
+  class Workflow
+    module SplitStepPersistence
+      module Payloads
+        private
+
+        def persisted_split_step_payload?(payload, expected)
+          return false unless payload.is_a?(String) && expected.is_a?(String)
+
+          CanonicalPayloadDigest.call(payload) == CanonicalPayloadDigest.call(expected)
+        rescue WorkflowError, JSON::ParserError, TypeError
+          false
+        end
+
+        def persisted_split_step_checkpoint?(payload)
+          return false unless payload && @split_step_checkpoint_digest
+
+          @split_step_checkpoint_digest == Digest::SHA256.hexdigest(payload)
+        end
+
+        def validate_split_step_marker!(payload, expected:)
+          marker = JSON.parse(payload).fetch("step_in_progress")
+          return if marker == expected
+
+          raise WorkflowError, "split-step persistence serialized an invalid step_in_progress marker"
+        rescue JSON::ParserError, KeyError, TypeError
+          raise WorkflowError, "split-step persistence requires an explicit step_in_progress marker"
+        end
+
+        def split_step_checkpoint_payload(payload)
+          document = JSON.parse(payload)
+          document["step_in_progress"] = false
+          JSON.generate(document)
+        end
+
+        def split_step_preparation_payload(payload)
+          document = JSON.parse(payload)
+          document["split_step_token"] = @split_step_token
+          if restart_safe_split_step?
+            document["split_step_phase"] = "prepared"
+            document["split_step_persistence_identity"] = @split_step_adapter.persistence_identity
+          end
+          JSON.generate(document)
+        end
+
+        def split_step_transition_signature(transition)
+          transition && TransitionContract.signature(transition)
+        end
+      end
+    end
+  end
+end