smith-agents 0.10.0 → 0.12.0

data/lib/smith/agent/invocation_preparation.rb

@@ -15,23 +15,43 @@ module Smith
       def bridge_workflow_inputs(agent_class)
         return {} unless @context.is_a?(Hash)
 
-        declared = agent_class.inputs || []
-        user_declared = declared - Smith::Agent::RESERVED_INPUT_NAMES
+        user_declared = (agent_class.inputs || []) - Smith::Agent::RESERVED_INPUT_NAMES
+        reject_untrusted_inputs!(agent_class, user_declared)
         user_declared.to_h do |name|
           [name, @context[name]]
         end
       end
 
-      def add_prepared_input(chat, prepared_input)
+      # A key the run's context, or that of a workflow it is nested in,
+      # marks untrusted never reaches an agent as a declared input, where it
+      # would sit beside the agent's instructions. Reserved input names are
+      # never read from the context, so they are not checked.
+      def reject_untrusted_inputs!(agent_class, declared)
+        untrusted = declared & untrusted_context_keys
+        return if untrusted.empty?
+
+        raise WorkflowError, "agent #{agent_class.register_as.inspect} declares untrusted context key " \
+                             "#{untrusted.first.inspect} as an input"
+      end
+
+      # The messages, with their content parts (ContentParts) as `provider`
+      # takes them.
+      def add_prepared_input(chat, prepared_input, provider = nil)
         return unless prepared_input
 
         prepared_input = provider_safe_prepared_input(prepared_input)
-        system_messages, other_messages = prepared_input.partition do |message|
-          message_role(message) == :system
+        check_cache_breakpoints!(chat, prepared_input, provider)
+        with_built_contents do
+          prepared_input = prepared_input.map do |message|
+            Message.parts?(message) ? provider_parts_message(message, provider) : message
+          end
+          system_messages, other_messages = prepared_input.partition do |message|
+            message_role(message) == :system
+          end
+
+          merge_system_messages!(chat, system_messages) if system_messages.any?
+          other_messages.each { |message| add_message(chat, message) }
         end
-
-        merge_system_messages!(chat, system_messages) if system_messages.any?
-        other_messages.each { |message| add_message(chat, message) }
       end
 
       def provider_safe_prepared_input(prepared_input)
@@ -41,13 +61,16 @@ module Smith
         messages + [{ role: :user, content: WORKFLOW_CONTINUATION_MESSAGE }]
       end
 
+      # The input ends with the output the run last added to its transcript,
+      # so the provider gets a user turn after it rather than an assistant
+      # turn to continue.
       def workflow_handoff?(messages)
         message = messages.reverse_each.find { |candidate| message_role(candidate) != :system }
         return false unless message
         return false unless message_role(message) == :assistant
-        return false unless defined?(@last_output) && !@last_output.nil?
+        return false unless defined?(@last_transcript_output) && !@last_transcript_output.nil?
 
-        message_content(message) == @last_output
+        message_content(message) == @last_transcript_output
       end
 
       def merge_system_messages!(chat, prepared_system_messages)
@@ -93,12 +116,12 @@ module Smith
       # { text:, ...attachments } and opens each value as a file, so replaying a prior structured
       # output to the next agent in a workflow session raises Errno::ENOENT. Serialize non-string
       # content to JSON so the provider sees the prior output as text; the session store keeps the raw
-      # value (last_output stays structured), only this provider-facing copy is serialized. Genuine
-      # multimodal attachments are supplied through the provider's own with: mechanism, never as a bare
-      # Hash message content in a workflow session.
+      # value (last_output stays structured), only this provider-facing copy is serialized. The only
+      # content that passes as it is: what Smith built from content parts for this very chat
+      # (ContentParts), never a RubyLLM object a session, output or host put there.
       def provider_safe_message(attributes)
         content = attributes[:content]
-        return attributes if content.nil? || content.is_a?(String)
+        return attributes if content.nil? || content.is_a?(String) || built_content?(content)
 
         attributes.merge(content: json_message_content(content))
       end

data/lib/smith/agent/lifecycle.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative "output_integrity"
 require_relative "provider_completion"
 require_relative "usage_tracking"
 
@@ -8,6 +9,7 @@ module Smith
     module Lifecycle
       include ProviderCompletion
       include UsageTracking
+      include OutputIntegrity
 
       private
 
@@ -18,12 +20,21 @@ module Smith
         instance.after_completion(result, context)
       end
 
-      def invoke_agent(agent_class, prepared_input, output_schema: agent_class.output_schema)
+      # One invocation: the provider call, then whatever OutputIntegrity
+      # asks for until the response can be the output.
+      def invoke_agent(agent_class, prepared_input, output_schema: effective_output_schema(agent_class))
         check_deadline!
-        attempt = complete_with_provider(agent_class, prepared_input, output_schema:)
-        snapshot_and_finalize(
-          agent_class, attempt.completion, attempt.model_reference, attempt_id: attempt.attempt_id
-        )
+        plan = output_plan(agent_class, prepared_input, output_schema)
+        attempt = complete_with_provider(agent_class, prepared_input, shape: plan.shape)
+        loop do
+          result = account_attempt(agent_class, attempt.completion, attempt.model_reference,
+                                   attempt_id: attempt.attempt_id)
+          follow_up = output_follow_up(plan, attempt, result)
+          return finalize_output(agent_class, result) unless follow_up
+
+          settle_rejected(result)
+          attempt = follow_up.call
+        end
       end
     end
   end

data/lib/smith/agent/observed_model.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Smith
+  class Agent
+    # The model a chat actually holds (after RubyLLM resolved it), else the
+    # reference the attempt was made with.
+    module ObservedModel
+      private
+
+      def observed_model_reference(chat, fallback:)
+        model = observable_model(chat)
+        return fallback unless model.respond_to?(:id) && !model.id.to_s.empty?
+
+        ModelReference.coerce(model.id, provider: observed_provider(model, fallback))
+      rescue StandardError
+        fallback
+      end
+
+      def observable_model(chat)
+        resolved_chat = chat.respond_to?(:to_llm) ? chat.to_llm : chat
+        resolved_chat.model if resolved_chat.respond_to?(:model)
+      end
+
+      def observed_provider(model, fallback)
+        model.respond_to?(:provider) && !model.provider.to_s.empty? ? model.provider : fallback.provider
+      end
+    end
+  end
+end

data/lib/smith/agent/output_contract_check.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Smith
+  class Agent
+    # The problems an agent's `output_contract` finds in its output, as
+    # "path: problem" lines. A contract's `call(hash)` returns true or false,
+    # or a result answering `success?` and `errors`: a Hash of messages by
+    # key (nested as deep as the output), an Array of messages, or anything
+    # with `to_h` giving such a Hash (as dry-validation's errors do). A
+    # failure with no messages still counts.
+    module OutputContractCheck
+      FAILED = "/: fails the output contract"
+
+      module_function
+
+      def problems(contract, output)
+        return [] unless contract
+
+        verdict = contract.call(output)
+        return verdict ? [] : [FAILED] if [true, false].include?(verdict)
+        return [] if readable!(verdict).success?
+
+        lines = messages(verdict.respond_to?(:errors) ? verdict.errors : nil)
+        lines.empty? ? [FAILED] : lines.first(OutputFailure::MAX_PROBLEMS)
+      end
+
+      def readable!(verdict)
+        return verdict if verdict.respond_to?(:success?)
+
+        raise WorkflowError, "output_contract must return true, false, or a result answering success? and errors"
+      end
+
+      def messages(errors)
+        case errors
+        when Hash then lines(errors)
+        when Array then errors.map { |message| "/: #{message}" }
+        when nil then []
+        else errors.respond_to?(:to_h) && !errors.is_a?(String) ? lines(errors.to_h) : ["/: #{errors}"]
+        end
+      end
+
+      # { fields: { title: ["is missing"] } } becomes "/fields/title: is missing".
+      def lines(errors, path = "")
+        errors.flat_map do |key, value|
+          at = key.nil? ? path : "#{path}/#{key}"
+          next lines(value, at) if value.is_a?(Hash)
+
+          Array(value).map { |message| "#{at.empty? ? "/" : at}: #{message}" }
+        end
+      end
+    end
+  end
+end

data/lib/smith/agent/output_integrity.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+require "json"
+
+require_relative "output_contract_check"
+require_relative "output_recovery"
+
+module Smith
+  class Agent
+    # Accepting an agent's response as its output. A refused response takes
+    # the agent's refusal fallback once or raises OutputRefused; a cut-off
+    # one is retried once with a larger output limit or raises
+    # OutputTruncated (unless the agent accepts it). The response then goes
+    # through the agent's `after_completion`; a structured output that is
+    # not a JSON object, breaks a schema constraint or fails the agent's
+    # output contract is sent back for repair up to `output_repairs`, then
+    # raises OutputInvalid.
+    #
+    # No follow-up call is made once a tool has run in the step (the tools
+    # would run again), once the call has spent what its step reserved, or
+    # past MAX_CALLS; a follow-up that fails raises the output failure it
+    # was making up for, with the provider's error as its cause. Every
+    # response is billed; one that is set aside settles with the
+    # invocation's failed attempts, and an :output_rejected trace names it.
+    module OutputIntegrity
+      include OutputRecovery
+
+      # What one invocation has tried so far.
+      Plan = Struct.new(:agent_class, :input, :output_schema, :json_schema, :repairs_left, :retried, :fell_back,
+                        :output_limit, :calls) do
+        def shape = ProviderRequest::Shape.new(output_schema:, output_limit:)
+      end
+      private_constant :Plan
+
+      private
+
+      # The agent's `output_schema`, else a static RubyLLM `schema` (a schema
+      # given as a block is resolved by RubyLLM per chat, and not checked).
+      def effective_output_schema(agent_class)
+        return agent_class.output_schema if agent_class.output_schema
+
+        schema = agent_class.schema if agent_class.respond_to?(:schema)
+        schema unless schema.is_a?(Proc)
+      end
+
+      def output_plan(agent_class, prepared_input, output_schema)
+        Plan.new(agent_class:, input: prepared_input, output_schema:, repairs_left: agent_class.output_repairs,
+                 json_schema: output_schema && Providers::SchemaConstraints.json_schema(output_schema), calls: 1)
+      end
+
+      # nil when the output is accepted; otherwise a callable that makes the
+      # next attempt. Raises the output failure when nothing is left to try.
+      def output_follow_up(plan, attempt, result)
+        failure = stop_failure(plan, attempt, result) || content_failure(plan, attempt, result)
+        return unless failure
+
+        follow_up = recovery(plan, attempt, result, failure)
+        record_rejection(plan, attempt, failure, follow_up)
+        raise failure unless follow_up
+
+        follow_up
+      end
+
+      def stop_failure(plan, attempt, result)
+        case Providers::StopReason.classify(stop_reason_of(attempt))
+        when :refused then output_failure(OutputRefused, plan, attempt, result)
+        when :truncated
+          accepted = plan.agent_class.on_truncation[:policy] == :accept
+          output_failure(OutputTruncated, plan, attempt, result) unless accepted
+        end
+      end
+
+      # After the agent's `after_completion`, which sees every response the
+      # provider did not refuse or cut off. A blank output is left to
+      # BlankAgentOutputError.
+      def content_failure(plan, attempt, result)
+        result.content = run_after_completion(plan.agent_class, result.content, @context)
+        return if blank_agent_output?(result.content)
+
+        parse_structured_output!(plan, result)
+        problems = output_problems(plan, result.content)
+        output_failure(OutputInvalid, plan, attempt, result, problems) unless problems.empty?
+      end
+
+      # A structured output that reached Smith as JSON text (its schema
+      # parsing bypassed) is read as RubyLLM reads a schema response.
+      def parse_structured_output!(plan, result)
+        return unless (plan.output_schema || plan.agent_class.output_contract) && result.content.is_a?(String)
+
+        parsed = JSON.parse(result.content)
+        result.content = parsed if parsed.is_a?(Hash)
+      rescue JSON::ParserError
+        nil
+      end
+
+      def output_problems(plan, content)
+        contract = plan.agent_class.output_contract
+        return [] unless plan.output_schema || contract
+        return ["/: is not a JSON object"] unless content.is_a?(Hash)
+
+        schema = plan.json_schema && Providers::SchemaConstraints.problems(plan.json_schema["schema"], content)
+        Array(schema) + OutputContractCheck.problems(contract, content)
+      end
+
+      # A billed response that is not the invocation's output settles with
+      # its failed attempts.
+      def settle_rejected(agent_result)
+        add_failed_agent_results([agent_result])
+        Thread.current[:smith_last_agent_result] = nil
+      end
+
+      def record_rejection(plan, attempt, failure, follow_up)
+        Smith::Trace.record(
+          type: :output_rejected,
+          data: { agent: plan.agent_class.register_as, attempt_id: attempt.attempt_id,
+                  error_class: failure.class.name, stop_reason: failure.stop_reason,
+                  problem_count: failure.problems.length, follow_up: follow_up ? next_step(failure) : :none }
+        )
+      end
+
+      def next_step(failure)
+        case failure
+        when OutputRefused then :fallback
+        when OutputTruncated then :retry
+        else :repair
+        end
+      end
+
+      def stop_reason_of(attempt) = attempt.completion.response_facts[:stop_reason]
+
+      def output_failure(error_class, plan, attempt, result, problems = [])
+        error_class.new(agent_name: plan.agent_class.register_as, model_used: result.model_used,
+                        stop_reason: stop_reason_of(attempt), problems:)
+      end
+    end
+  end
+end

data/lib/smith/agent/output_policy.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Smith
+  class Agent
+    # What an agent does with a response that cannot be its output:
+    #
+    #   on_truncation :fail                         # the default: OutputTruncated
+    #   on_truncation :accept                       # keep the cut-off output
+    #   on_truncation retry_with: { max_tokens: 32_000 }
+    #   on_refusal :fail                            # the default: OutputRefused
+    #   on_refusal fallback: "anthropic/claude-opus-5-5"
+    #   output_contract ReviewContract              # checks an output_schema agent's Hash
+    #   output_repairs 1                            # the default; 0 turns repair off
+    #
+    # A retry, a fallback and each repair are one more provider call, billed
+    # like any other.
+    module OutputPolicy
+      MAX_REPAIRS = 3
+      FAIL = { policy: :fail }.freeze
+      private_constant :FAIL
+
+      def on_truncation(policy = nil, retry_with: nil)
+        return @on_truncation || FAIL if policy.nil? && retry_with.nil?
+
+        @on_truncation = if one_of!("on_truncation", policy, %i[fail accept], retry_with)
+                           { policy: :retry, max_tokens: retry_limit(retry_with) }.freeze
+                         else
+                           { policy: }.freeze
+                         end
+      end
+
+      def on_refusal(policy = nil, fallback: nil)
+        return @on_refusal || FAIL if policy.nil? && fallback.nil?
+
+        @on_refusal = if one_of!("on_refusal", policy, %i[fail], fallback)
+                        { policy: :fallback, model: refusal_fallback(fallback) }.freeze
+                      else
+                        FAIL
+                      end
+      end
+
+      # Any object (or a class whose instances are such objects) whose
+      # `call(hash)` returns a result answering `success?` and `errors`, as
+      # a dry-validation contract does.
+      def output_contract(contract = nil)
+        return @output_contract if contract.nil?
+
+        checker = contract.is_a?(Class) ? contract.new : contract
+        raise ArgumentError, "output_contract must respond to call" unless checker.respond_to?(:call)
+
+        @output_contract = checker
+      end
+
+      def output_repairs(count = nil)
+        return @output_repairs || 1 if count.nil?
+        unless count.is_a?(Integer) && count.between?(0, MAX_REPAIRS)
+          raise ArgumentError, "output_repairs must be an Integer from 0 to #{MAX_REPAIRS}"
+        end
+
+        @output_repairs = count
+      end
+
+      private
+
+      # True when the option is given; otherwise the policy must be one of
+      # `policies`.
+      def one_of!(macro, policy, policies, option)
+        raise ArgumentError, "#{macro} takes a policy or an option, not both" if policy && option
+        return true if option
+        return false if policies.include?(policy)
+
+        raise ArgumentError, "#{macro} takes #{policies.map(&:inspect).join(" or ")}, or an option"
+      end
+
+      def retry_limit(options)
+        limit = options.is_a?(Hash) && options.keys == [:max_tokens] ? options[:max_tokens] : nil
+        return limit if limit.is_a?(Integer) && limit.positive?
+
+        raise ArgumentError, "on_truncation retry_with: takes { max_tokens: a positive Integer }"
+      end
+
+      def refusal_fallback(model)
+        reference = ModelReference.coerce(model)
+        return reference if reference.provider
+
+        raise ArgumentError, "on_refusal fallback: must name a provider, as in \"anthropic/<model>\""
+      rescue Dry::Struct::Error => e
+        raise ArgumentError, "invalid on_refusal fallback: #{e.message}"
+      end
+    end
+  end
+end

data/lib/smith/agent/output_recovery.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Smith
+  class Agent
+    # The follow-up call an output failure may get (OutputIntegrity): a
+    # refusal fallback, a truncation retry, or a repair, each within the
+    # agent's settings and never once a tool has run in the step, once the
+    # call has spent what its step reserved, or past MAX_CALLS calls.
+    module OutputRecovery
+      MAX_CALLS = 3 + OutputPolicy::MAX_REPAIRS
+      REPAIR_NOTE = "Your previous reply could not be used as the required output:\n%s\n" \
+                    "Reply again with output that fixes these problems."
+      REPAIR_TASK = "Reply with the required output."
+      # Why a streamed step's rejected text is withdrawn (Smith::Streaming).
+      STREAM_RESETS = { OutputRefused => :fallback, OutputTruncated => :truncation }.freeze
+      private_constant :MAX_CALLS, :REPAIR_NOTE, :REPAIR_TASK, :STREAM_RESETS
+
+      private
+
+      def recovery(plan, attempt, result, failure)
+        return if tools_ran? || plan.calls >= MAX_CALLS || budget_spent?(result)
+
+        case failure
+        when OutputRefused then refusal_recovery(plan, attempt, failure)
+        when OutputTruncated then truncation_recovery(plan, attempt, failure)
+        else repair_recovery(plan, attempt, result, failure)
+        end
+      end
+
+      def tools_ran? = Tool.current_tool_execution_tracker&.started? || false
+
+      def refusal_recovery(plan, attempt, failure)
+        fallback = plan.agent_class.on_refusal[:model]
+        return if fallback.nil? || plan.fell_back || fallback.same_candidate?(attempt.model_reference)
+
+        plan.fell_back = true
+        -> { follow_up_attempt(plan, attempt, fallback, failure) }
+      end
+
+      def truncation_recovery(plan, attempt, failure)
+        limit = retry_limit(plan, attempt)
+        return unless limit
+
+        plan.retried = true
+        plan.output_limit = limit
+        -> { follow_up_attempt(plan, attempt, attempt.model_reference, failure) }
+      end
+
+      # The larger limit to retry with, or nil: no retry is declared or left,
+      # the provider has no output limit Smith can set, the response ran out
+      # of context rather than output, or the limit is not larger than the
+      # one the request already had.
+      def retry_limit(plan, attempt)
+        return unless retry_allowed?(plan, attempt)
+
+        limit = plan.agent_class.on_truncation[:max_tokens]
+        current = plan.output_limit || Providers::OutputLimit.current(attempt.model_reference.provider,
+                                                                      agent_params(plan.agent_class),
+                                                                      attempt.model_reference)
+        limit if current.nil? || limit > current
+      end
+
+      def retry_allowed?(plan, attempt)
+        plan.agent_class.on_truncation[:policy] == :retry && !plan.retried &&
+          Providers::OutputLimit.supported?(attempt.model_reference.provider) &&
+          stop_reason_of(attempt) != "model_context_window_exceeded"
+      end
+
+      def agent_params(agent_class)
+        params = agent_class.params if agent_class.respond_to?(:params)
+        params.is_a?(Hash) ? params : {}
+      end
+
+      def repair_recovery(plan, attempt, result, failure)
+        return unless plan.repairs_left.positive?
+
+        plan.repairs_left -= 1
+        plan.input = repair_input(plan.input, result.content, failure.problems)
+        -> { follow_up_attempt(plan, attempt, attempt.model_reference, failure) }
+      end
+
+      # The step's messages as the provider got them (with any continuation
+      # turn), the rejected reply as the assistant's, and the problems.
+      def repair_input(input, content, problems)
+        note = format(REPAIR_NOTE, problems.map { |problem| "- #{problem}" }.join("\n"))
+        messages = provider_safe_prepared_input(Array(input))
+        messages = [{ role: :user, content: REPAIR_TASK }] if messages.empty?
+        messages + [{ role: :assistant, content: rejected_text(content) }, { role: :user, content: note }]
+      end
+
+      def rejected_text(content)
+        content.is_a?(String) ? content : JSON.generate(content)
+      rescue JSON::GeneratorError, Encoding::UndefinedConversionError
+        content.to_s
+      end
+
+      def follow_up_attempt(plan, previous, model_reference, pending)
+        check_deadline!
+        plan.calls += 1
+        withdraw_streamed(STREAM_RESETS.fetch(pending.class, :repair))
+        attempt = attempt_model(plan.agent_class, plan.input, model_reference,
+                                shape: plan.shape, attempt_index: previous.attempt_index.to_i + 1)
+        return attempt if attempt.success?
+
+        fail_follow_up(plan, attempt, pending)
+      end
+
+      def fail_follow_up(plan, attempt, pending)
+        account_failed_attempt(attempt.error, attempt.model_reference, plan.agent_class, attempt_id: attempt.attempt_id)
+        raise pending, cause: attempt.error
+      end
+
+      # Whether the responses so far have reached what the call's step
+      # reserved in any budget dimension.
+      def budget_spent?(result)
+        reservation = Thread.current[:smith_call_reservation]
+        return false unless reservation
+
+        spent = extract_actuals([*Array(Thread.current[:smith_failed_agent_results]), result])
+        reservation.amounts.any? do |dimension, amount|
+          used = Workflow::BudgetIntegration::COST_DIMENSIONS.include?(dimension) ? spent[:cost] : spent[:tokens]
+          used >= amount
+        end
+      end
+    end
+  end
+end

data/lib/smith/agent/provider_attempt.rb

@@ -4,6 +4,7 @@ require "dry-struct"
 
 require_relative "../types"
 require_relative "model_reference"
+require_relative "provider_usage"
 
 module Smith
   class Agent
@@ -17,13 +18,25 @@ module Smith
       # completion call started.
       attribute :attempt_id, Types::String.optional.default(nil)
       attribute :duration_ms, Types::Integer.optional.default(nil)
+      # The candidate's position in the model chain, the registered name of
+      # the agent that made the attempt, and the provider-reported usage its
+      # usage entries record (nil when the provider reported none).
+      attribute :attempt_index, Types::Integer.optional.default(nil)
+      attribute :agent_name, (Types::Symbol | Types::String).optional.default(nil)
+      attribute :usage, Types.Instance(ProviderUsage).optional.default(nil)
+      # True for a batch result that answered a request other than the one
+      # the call renders now (BatchExchange); nil otherwise.
+      attribute :request_changed, Types::Bool.optional.default(nil)
+      # True for a response a host's response memo kept (ResponseMemo): the
+      # call sent nothing.
+      attribute :memo, Types::Bool.optional.default(nil)
 
-      def self.success(completion:, model_reference:, attempt_id: nil, duration_ms: nil)
-        new(completion:, model_reference:, error: nil, attempt_id:, duration_ms:)
+      def self.success(completion:, model_reference:, **facts)
+        new(completion:, model_reference:, error: nil, **facts)
       end
 
-      def self.failure(error:, model_reference:, attempt_id: nil, duration_ms: nil)
-        new(completion: nil, model_reference:, error:, attempt_id:, duration_ms:)
+      def self.failure(error:, model_reference:, **facts)
+        new(completion: nil, model_reference:, error:, **facts)
       end
 
       def success?