smith-agents 0.10.0 → 0.11.0

data/lib/smith/workflow/evaluator_optimizer.rb

@@ -3,6 +3,7 @@
 require "json"
 
 require_relative "agent_result"
+require_relative "execution_result_snapshot"
 require_relative "optimization_state"
 
 module Smith
@@ -24,6 +25,7 @@ module Smith
           transition_name: transition.name,
           role: :evaluator
         )
+        pend_evaluations
         run_optimization_loop(state)
       end
 
@@ -41,10 +43,11 @@ module Smith
 
       def run_optimization_round(state, round)
         generate_candidate!(state, round)
-        invoke_before_eval(state)
-        evaluation = normalize_evaluation(evaluate_candidate(state))
+        rejection = before_eval_rejection(state)
+        evaluation = rejection || normalize_evaluation(evaluate_candidate(state))
         validate_evaluation_structure!(evaluation)
         validate_evaluation_fields!(evaluation, state.config)
+        record_evaluation(state, evaluation, round, rejection ? :before_eval : :evaluator)
 
         return state.candidate if evaluation[:accept]
 
@@ -61,6 +64,88 @@ module Smith
         nil
       end
 
+      # Each valid round's evaluation, in round order: the round, whether the
+      # evaluator or a before_eval rejection gave it, and the verdict as
+      # normalized, copied and frozen so neither the loop nor a host holding a
+      # record can change it. Recorded before any exit, so the verdict that
+      # ends the loop (accepted, converged, or under the threshold) is kept.
+      # The loop's state carries a frozen copy, so a callable reads the
+      # verdicts so far and cannot rewrite the step's.
+      def record_evaluation(state, evaluation, round, source)
+        records = @pending_evaluations.values.last
+        record = { round: round, source: source, verdict: frozen_verdict(evaluation) }.freeze
+        within_step_record_limits!(@pending_evaluations.values.flatten(1) << record)
+        records << record
+        state.evaluations = records.dup.freeze
+      end
+
+      # The verdicts ride on the step's record, which a split step snapshots
+      # under ExecutionResultSnapshot's limits. A round whose verdict would
+      # take the step's verdicts past them fails here, the same way in every
+      # run mode, and the verdicts already recorded still fit a failed step's
+      # record.
+      def within_step_record_limits!(records)
+        ExecutionResultSnapshot.new({ evaluations: records }).call
+      rescue WorkflowError => e
+        raise WorkflowError, "evaluations exceed a step record's limits: #{e.message}"
+      end
+
+      # A verdict holds JSON values only, as a provider's structured output
+      # does: anything else (an object a before_eval returned, a key that is
+      # not a String or Symbol, a non-finite Float) fails the round here, the
+      # same way in every run mode, rather than when a split step snapshots it.
+      def frozen_verdict(value)
+        case value
+        when Hash then value.to_h { |key, nested| [verdict_key(key), frozen_verdict(nested)] }.freeze
+        when Array then value.map { |nested| frozen_verdict(nested) }.freeze
+        when String then StringSnapshot.copy(value, freeze: true)
+        else verdict_scalar(value)
+        end
+      end
+
+      def verdict_scalar(value)
+        case value
+        when Float then finite_verdict_number(value)
+        when Symbol, Integer, true, false, nil then value
+        else raise WorkflowError, "evaluation must hold JSON values; got #{value.class}"
+        end
+      end
+
+      def verdict_key(key)
+        return key if key.is_a?(Symbol)
+        return StringSnapshot.copy(key, freeze: true) if key.is_a?(String)
+
+        raise WorkflowError, "evaluation keys must be Strings or Symbols; got #{key.class}"
+      end
+
+      def finite_verdict_number(value)
+        return value if value.finite?
+
+        raise WorkflowError, "evaluation must hold finite numbers; got #{value}"
+      end
+
+      # An optimize step's record carries every verdict its loops recorded
+      # under :evaluations, whether the step completed or failed, each naming
+      # the attempt that gave it (`@step_attempt`, the retry policy's): a step
+      # `retry_on` runs again starts its loop afresh, rounds counting from 0
+      # again, and an earlier attempt's verdicts were given and paid for all
+      # the same. The key is there whenever a loop ran; a step that failed
+      # before its loop (an agent not registered, a guardrail) and any other
+      # step have none. The carrier is transient: the loop's list keyed by its
+      # attempt, cleared once the step's record is committed or its failure
+      # captured, and at every step's start.
+      def pend_evaluations
+        (@pending_evaluations ||= {})[@step_attempt || 1] = []
+      end
+
+      def fold_pending_evaluations(step)
+        return unless @pending_evaluations
+
+        step[:evaluations] = @pending_evaluations.flat_map do |attempt, records|
+          records.map { |record| { attempt: attempt, **record }.freeze }
+        end.freeze
+      end
+
       # :raise => WorkflowError(message); :return_last => state.candidate;
       # callable => mode.call(state). Default :raise preserves legacy
       # behavior for hosts that don't opt in to graceful exits.
@@ -127,21 +212,34 @@ module Smith
       # evaluator sees the same seed_messages + inject_state context.
       # Default nil keeps the legacy candidate-only payload.
       def build_evaluator_input(state)
-        return [{ role: :user, content: state.candidate.to_s }] unless state.config[:evaluator_context] == :inject_state
+        content = candidate_content(state.candidate)
+        return [{ role: :user, content: }] unless state.config[:evaluator_context] == :inject_state
 
         prior = Array(state.prepared_input).dup
-        prior.push(role: :user, content: state.candidate.to_s)
+        prior.push(role: :user, content:)
+      end
+
+      # A structured candidate (an output_schema agent's Hash or Array) gets
+      # the JSON serialisation the provider boundary applies; text stays as is.
+      def candidate_content(candidate)
+        candidate.is_a?(Hash) || candidate.is_a?(Array) ? json_message_content(candidate) : candidate.to_s
       end
 
       # Runs after candidate generation, before evaluator invocation.
-      # Receives (state, @context); @context is mutable. Return value
-      # is discarded. Raised exceptions bubble through the standard
-      # step failure path.
-      def invoke_before_eval(state)
+      # Receives (state, @context); @context is mutable. A returned Hash
+      # whose accept (Symbol or String key) is false is the round's
+      # evaluation, validated like evaluator output, and the evaluator is
+      # not called; any other return value is discarded. Raised exceptions
+      # bubble through the standard step failure path.
+      def before_eval_rejection(state)
         callback = state.config[:before_eval]
         return unless callback
 
-        callback.call(state, @context)
+        verdict = callback.call(state, @context)
+        return unless verdict.is_a?(Hash)
+
+        evaluation = normalize_evaluation(verdict)
+        evaluation if evaluation.key?(:accept) && evaluation[:accept].equal?(false)
       end
 
       def invoke_with_evaluator_schema(evaluator_class, schema, input)
@@ -195,7 +293,7 @@ module Smith
         return prepared_input if round.zero?
 
         (prepared_input&.dup || []).push(
-          { role: :assistant, content: prior_candidate.to_s },
+          { role: :assistant, content: candidate_content(prior_candidate) },
           {
             role: :user,
             content: "[smith:refinement-round] #{round + 1}\n[smith:evaluator-feedback]\n#{feedback}"

data/lib/smith/workflow/execution.rb

@@ -34,7 +34,9 @@ module Smith
         # Reset the per-step agent-attribution carrier so only a serial agent
         # step that actually runs (below) leaves model/provider for complete_step;
         # a deterministic step never sets it and must not inherit a stale value.
+        # The optimizer's verdict carrier resets for the same reason.
         @pending_agent_execution = nil
+        @pending_evaluations = nil
         output = with_scoped_artifacts { run_with_retry_policy(transition) }
         StepCompletion.instance_method(:complete_step).bind_call(self, transition, output)
       end

data/lib/smith/workflow/failure_reconstructor.rb

@@ -18,6 +18,9 @@ module Smith
         "bounded_completion_error" => ->(record) { Smith::BoundedCompletionError.new(record[:error_message]) },
         "persisted_failure_invalid" => ->(record) { Smith::PersistedFailureInvalid.new(record[:error_message]) },
         "deadline_exceeded" => ->(record) { Smith::DeadlineExceeded.new(record[:error_message]) },
+        "provider_permanent_failure" => ->(record) { provider_permanent_failure(record) },
+        "budget_exceeded" => ->(record) { Smith::BudgetExceeded.new(record[:error_message]) },
+        "guardrail_failed" => ->(record) { Smith::GuardrailFailed.new(record[:error_message]) },
         "agent_error" => ->(record) { Smith::AgentError.new(record[:error_message]) },
         "workflow_error" => ->(record) { Smith::WorkflowError.new(record[:error_message]) },
         "other" => ->(record) { RuntimeError.new(record[:error_message]) }
@@ -25,7 +28,8 @@ module Smith
       SPECIAL_CLASS_BUILDERS = {
         "Smith::Workflow::Composite::BranchFailure" => lambda { |record|
           Smith::Workflow::Composite::BranchFailure.from_details(record[:error_details])
-        }
+        },
+        "Smith::BlankAgentOutputError" => ->(record) { blank_agent_output_failure(record) }
       }.freeze
       private_constant :FAMILY_BUILDERS, :SPECIAL_CLASS_BUILDERS
 
@@ -73,7 +77,19 @@ module Smith
         Smith::ToolFailureNotificationFailed.from_details(record.fetch(:error_details))
       end
 
-      private_class_method :deterministic_step_failure, :tool_guardrail_failure, :tool_failure_notification_failure
+      def self.provider_permanent_failure(record)
+        Smith::ProviderPermanentFailure.from_details(record.fetch(:error_details), message: record[:error_message])
+      end
+
+      def self.blank_agent_output_failure(record)
+        details = record[:error_details]
+        return Smith::AgentError.new(record[:error_message]) if details.nil?
+
+        Smith::BlankAgentOutputError.from_details(details)
+      end
+
+      private_class_method :deterministic_step_failure, :tool_guardrail_failure, :tool_failure_notification_failure,
+                           :provider_permanent_failure, :blank_agent_output_failure
     end
   end
 end

data/lib/smith/workflow/failure_record.rb

@@ -20,6 +20,9 @@ module Smith
         [Smith::PersistedFailureInvalid, "persisted_failure_invalid"],
         [Smith::ToolGuardrailFailed, "tool_guardrail_failed"],
         [Smith::DeadlineExceeded, "deadline_exceeded"],
+        [Smith::ProviderPermanentFailure, "provider_permanent_failure"],
+        [Smith::BudgetExceeded, "budget_exceeded"],
+        [Smith::GuardrailFailed, "guardrail_failed"],
         [Smith::AgentError, "agent_error"],
         [Smith::WorkflowError, "workflow_error"]
       ].freeze
@@ -27,10 +30,7 @@ module Smith
       EXCEPTION_CAUSE = Exception.instance_method(:cause)
       EXCEPTION_MESSAGE = Exception.instance_method(:message)
       MODULE_MATCH = Module.instance_method(:===)
-      MODULE_NAME = Module.instance_method(:name)
-      OBJECT_CLASS = Object.instance_method(:class)
-      private_constant :ERROR_FAMILIES, :BOOLEAN_VALUES, :EXCEPTION_CAUSE, :EXCEPTION_MESSAGE, :MODULE_MATCH,
-                       :MODULE_NAME, :OBJECT_CLASS
+      private_constant :ERROR_FAMILIES, :BOOLEAN_VALUES, :EXCEPTION_CAUSE, :EXCEPTION_MESSAGE, :MODULE_MATCH
 
       extend Dry::Initializer
 
@@ -52,7 +52,7 @@ module Smith
 
       def error_attributes(error)
         {
-          error_class: DiagnosticText.capture(error_class_name(error), max_bytes: 512),
+          error_class: DiagnosticText.error_class_name(error),
           error_family: error_family(error),
           error_message: error_message(error),
           error_retryable: retryable_value(error),
@@ -70,7 +70,7 @@ module Smith
       def cause_attributes(error)
         cause = uncertainty_cause(error)
         {
-          error_cause_class: cause && DiagnosticText.capture(error_class_name(cause), max_bytes: 512),
+          error_cause_class: cause && DiagnosticText.error_class_name(cause),
           error_cause_family: cause && error_family(cause),
           error_cause_message: cause && error_message(cause)
         }
@@ -89,11 +89,6 @@ module Smith
         ERROR_FAMILIES.find { |error_class, _| MODULE_MATCH.bind_call(error_class, error) }&.last || "other"
       end
 
-      def error_class_name(error)
-        error_class = OBJECT_CLASS.bind_call(error)
-        MODULE_NAME.bind_call(error_class) || "anonymous_error"
-      end
-
       # Never captures blank text: restore requires non-empty messages, so
       # a blank live message takes the shared deterministic placeholder.
       def error_message(error)

data/lib/smith/workflow/failure_record_validator.rb

@@ -20,14 +20,25 @@ module Smith
         "Smith::AgentError" => "agent_error",
         "Smith::BlankAgentOutputError" => "agent_error",
         "Smith::DeadlineExceeded" => "deadline_exceeded",
+        "Smith::ProviderPermanentFailure" => "provider_permanent_failure",
+        "Smith::BudgetExceeded" => "budget_exceeded",
+        "Smith::GuardrailFailed" => "guardrail_failed",
         "Smith::WorkflowError" => "workflow_error",
         "Smith::UnresolvedTransitionError" => "workflow_error",
         "Smith::Workflow::Composite::BranchFailure" => "workflow_error"
       }.freeze
+      # Records written before these classes had their own families carry
+      # "other" and no details; they keep restoring exactly as they did.
+      LEGACY_ERROR_FAMILIES = {
+        "Smith::ProviderPermanentFailure" => "other",
+        "Smith::BudgetExceeded" => "other",
+        "Smith::GuardrailFailed" => "other"
+      }.freeze
       KNOWN_FAMILIES = %w[
         deterministic_step_failure tool_guardrail_failed tool_failure_notification_failed tool_capture_failed
         tool_outcome_uncertain tool_execution_not_admitted bounded_completion_error persisted_failure_invalid
-        deadline_exceeded agent_error workflow_error other
+        deadline_exceeded provider_permanent_failure budget_exceeded guardrail_failed agent_error workflow_error
+        other
       ].freeze
       RETRY_FORBIDDEN_FAMILIES = %w[
         tool_capture_failed tool_failure_notification_failed tool_outcome_uncertain tool_execution_not_admitted
@@ -40,11 +51,15 @@ module Smith
         },
         "Smith::Workflow::Composite::BranchFailure" => lambda { |details|
           Smith::Workflow::Composite::BranchFailure.from_details(details)
+        },
+        "Smith::ProviderPermanentFailure" => ->(details) { Smith::ProviderPermanentFailure.from_details(details) },
+        "Smith::BlankAgentOutputError" => lambda { |details|
+          Smith::BlankAgentOutputError.from_details(details) unless details.nil?
         }
       }.freeze
       BOOLEAN_VALUES = [true, false].freeze
-      private_constant :KNOWN_ERROR_FAMILIES, :KNOWN_FAMILIES, :RETRY_FORBIDDEN_FAMILIES, :DETAIL_VALIDATORS,
-                       :BOOLEAN_VALUES
+      private_constant :KNOWN_ERROR_FAMILIES, :LEGACY_ERROR_FAMILIES, :KNOWN_FAMILIES, :RETRY_FORBIDDEN_FAMILIES,
+                       :DETAIL_VALIDATORS, :BOOLEAN_VALUES
 
       extend Dry::Initializer
 
@@ -74,10 +89,13 @@ module Smith
       def validate_class_family!
         expected = KNOWN_ERROR_FAMILIES[snapshot[:error_class]]
         return unless expected && family != expected
+        return if legacy_family?
 
         reject!("persisted workflow failure class and family disagree")
       end
 
+      def legacy_family? = LEGACY_ERROR_FAMILIES[snapshot[:error_class]] == family
+
       def validate_retry_policy!
         forbidden = snapshot[:error_retry_forbidden]
         return if forbidden.nil?
@@ -103,6 +121,8 @@ module Smith
       end
 
       def validate_details!
+        return if legacy_family?
+
         validator = DETAIL_VALIDATORS[snapshot[:error_class]]
         validator&.call(snapshot[:error_details])
       end

data/lib/smith/workflow/guardrail_integration.rb

@@ -24,6 +24,8 @@ module Smith
 
       def handle_step_failure(transition, error)
         step = { transition: transition.name, from: transition.from, to: transition.to, error: error }
+        fold_pending_evaluations(step)
+        @pending_evaluations = nil
         # Staged, not emitted: this rescue runs under the step snapshot's
         # interrupt mask, and host StepFailed handlers must not execute
         # unkillable. with_step_context flushes after the mask closes.

data/lib/smith/workflow/optimization_state.rb

@@ -3,10 +3,10 @@
 module Smith
   class Workflow
     OptimizationState = Struct.new(
-      :config, :prepared_input, :candidate, :feedback, :last_score, :generator_class, :evaluator_class
+      :config, :prepared_input, :candidate, :feedback, :last_score, :generator_class, :evaluator_class, :evaluations
     ) do
       def initialize(config, prepared_input)
-        super(config, prepared_input, nil, nil, nil, nil, nil)
+        super(config, prepared_input, nil, nil, nil, nil, nil, [].freeze)
       end
     end
   end

data/lib/smith/workflow/persistence.rb

@@ -100,11 +100,12 @@ module Smith
         @usage_entries = restore_usage_entries(normalized)
         @last_output = restore_last_output(normalized)
         # Backward-compat: pre-patch states have no last_agent_execution key and
-        # restore to nil. @pending_agent_execution is transient (nil between
-        # steps), never persisted, but must be initialized because from_state
-        # allocates and bypasses #initialize.
+        # restore to nil. @pending_agent_execution and @pending_evaluations are
+        # transient (nil between steps), never persisted, but must be
+        # initialized because from_state allocates and bypasses #initialize.
         @last_agent_execution = restore_last_agent_execution(normalized)
         @pending_agent_execution = nil
+        @pending_evaluations = nil
         @last_failed_step = restore_last_failed_step(normalized)
         # Restore the optimistic-locking version from the persisted payload.
         # Backward-compat: pre-versioning payloads have no key, restore to 0
@@ -160,16 +161,17 @@ module Smith
 
         raise Smith::StepInProgressOnRestore.new(
           workflow: self.class.name,
-          persistence_key: normalized[:persistence_key]
+          persistence_key: normalized[:persistence_key],
+          state: normalized[:state],
+          transition: normalized[:next_transition_name]
         )
       end
 
       def validate_raw_step_in_progress!(state)
         return unless state.is_a?(Hash)
 
-        marker = state[:step_in_progress] || state["step_in_progress"]
-        persistence_key = state[:persistence_key] || state["persistence_key"]
-        validate_step_in_progress!(step_in_progress: marker, persistence_key:)
+        fields = %i[step_in_progress persistence_key state next_transition_name]
+        validate_step_in_progress!(fields.to_h { |field| [field, state[field] || state[field.to_s]] })
       end
 
       def validate_seed_digest!(normalized)
@@ -233,7 +235,9 @@ module Smith
         end
         return unless value.is_a?(Hash)
 
-        symbolize_keys(value)
+        execution = symbolize_keys(value)
+        execution[:provider] = execution[:provider].to_sym if execution[:provider].is_a?(String)
+        execution
       end
 
       # Symbolize ONLY the top-level keys of last_failed_step + the
@@ -430,7 +434,20 @@ module Smith
         config = self.class.budget
         return nil unless config
 
-        Budget::Ledger.new(limits: config, consumed: consumed)
+        Budget::Ledger.new(limits: config, consumed: declared_budget_consumed(consumed, config))
+      end
+
+      # A dimension the workflow no longer declares has no limit, so its
+      # consumed entry (a key an earlier version accepted and ignored) is
+      # dropped rather than refused.
+      def declared_budget_consumed(consumed, config)
+        return consumed unless consumed.is_a?(Hash)
+
+        declared = config.each_key.to_h { |dimension| [dimension.to_s, dimension] }
+        consumed.each_with_object({}) do |(key, amount), kept|
+          dimension = declared[key.to_s]
+          kept[dimension] = amount if dimension
+        end
       end
 
       def persisted_context

data/lib/smith/workflow/retry_execution.rb

@@ -5,14 +5,18 @@ module Smith
     module RetryExecution
       private
 
+      # `@step_attempt` is the attempt under way, from 1, which the optimizer
+      # names its verdicts by; transient, set here before every attempt.
       def run_with_retry_policy(transition)
         config = transition.retry_config
+        @step_attempt = 1
         return run_guarded_step(transition) unless config
 
         schedule = retry_schedule(config)
         attempt = 0
         begin
           attempt += 1
+          @step_attempt = attempt
           run_guarded_step(transition)
         rescue StandardError => e
           raise unless retry_transition_error?(config, e, attempt)

data/lib/smith/workflow/run_result.rb

@@ -33,6 +33,16 @@ module Smith
         steps.reverse.map { |step| step[:error] }.compact.first
       end
 
+      # Every verdict an optimize step's loops recorded, in step, attempt and
+      # round order, each a frozen record naming its step's transition. Like
+      # `steps`, they belong to the run that executed the step: a terminal
+      # restore has none.
+      def evaluations
+        steps.flat_map do |step|
+          (step[:evaluations] || []).map { |evaluation| { transition: step[:transition], **evaluation }.freeze }
+        end
+      end
+
       def failed_transition
         failure_detail&.fetch(:transition)
       end

data/lib/smith/workflow/step_completion.rb

@@ -16,6 +16,7 @@ module Smith
         end
 
         @pending_agent_execution = nil
+        fold_pending_evaluations(step)
         result = SplitStepPersistence
                  .instance_method(:prepare_split_step_execution_result)
                  .bind_call(self, step)
@@ -23,6 +24,9 @@ module Smith
         SplitStepPersistence
           .instance_method(:commit_split_step_execution_result!)
           .bind_call(self, result)
+        # Cleared only once the record is committed: a completion that fails
+        # before this folds the same verdicts into its failure record.
+        @pending_evaluations = nil
         @state = transition.to
         @next_transition_name = @router_next_transition || transition.success_transition
         @router_next_transition = nil

data/lib/smith/workflow.rb

@@ -44,7 +44,7 @@ module Smith
       @step_count = 0
       @next_transition_name = nil
       @ledger = ledger || build_ledger
-      @created_at = created_at || Time.now.utc.iso8601
+      @created_at = created_at || Time.now.utc.iso8601(6)
       @updated_at = @created_at
       @total_cost = 0.0
       @total_tokens = 0
@@ -63,9 +63,11 @@ module Smith
       # fallback resolution), exposed to the following deterministic step as
       # `last_agent_model` / `last_agent_provider`. `@pending_agent_execution`
       # is the transient per-step carrier from `execute_serial_step` to
-      # `complete_step`; it is never persisted.
+      # `complete_step`; it is never persisted. `@pending_evaluations` is the
+      # optimizer's, carrying a loop's verdicts into its step's record.
       @last_agent_execution = nil
       @pending_agent_execution = nil
+      @pending_evaluations = nil
       @last_failed_step = nil
       # Optimistic-locking version. Incremented on each persist!; restored
       # from the persisted payload. Adapters that support store_versioned
@@ -120,7 +122,7 @@ module Smith
                       execute_step(transition)
                     end
       @step_count += 1
-      @updated_at = Time.now.utc.iso8601
+      @updated_at = Time.now.utc.iso8601(6)
       record_step_snapshot(step_result)
       step_result
     rescue UnresolvedTransitionError => e

data/lib/smith.rb

@@ -59,8 +59,8 @@ module Smith
 
   # Persistence TTL in Integer seconds. nil (default) means workflows
   # persist indefinitely. Adapters that natively support TTL (Redis,
-  # CacheStore, Memory) pass this through; ActiveRecordStore TTL is
-  # deferred (would need an `expires_at` column + sweeper).
+  # CacheStore, Memory) pass this through; ActiveRecordStore has no expiry
+  # column and refuses a non-nil TTL.
   # Per-workflow `Workflow.persistence_ttl 1.day.to_i` DSL overrides this.
   setting :persistence_ttl, default: nil
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: smith-agents
 version: !ruby/object:Gem::Version
-  version: 0.10.0
+  version: 0.11.0
 platform: ruby
 authors:
 - Samuel Ralak