smith-agents 0.11.0 → 0.12.0

data/lib/smith/providers/request_holds.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+require "ruby_llm"
+
+module Smith
+  module Providers
+    # Where a host's budget ledger (a workflow's `budget_ledger`) meets
+    # RubyLLM: each request a step's call makes over the wire (each round of
+    # a tool loop, each fallback model, each follow-up, streamed or not, and
+    # OpenAI's Responses route, which never reaches the base class) goes
+    # through `Provider#complete`. A call's first request is held before
+    # anything is sent. A later round of a tool loop continues a call whose
+    # tools already ran: it is held only when its step can pause for budget
+    # (`pausable`), and otherwise goes on, reconciled all the same. Each
+    # request is then reconciled with what it spent (its usage; its hold
+    # when it answered without usage or its stream failed part way) or, if
+    # it failed before anything came back, its hold is released. Only the
+    # call's own provider object is held, so a tool's own RubyLLM call is
+    # not. While a batch step captures its request, the hold is noted on the
+    # capture instead, for the batch to reserve.
+    module RequestHolds
+      HELD = :smith_request_holds
+      private_constant :HELD
+
+      Holder = Struct.new(:provider, :ledger, :attempt_id, :pausable, :rounds, :busy)
+      # Whether a streamed request's response began.
+      Stream = Struct.new(:started)
+
+      module_function
+
+      # Runs the block with the provider's requests held on `ledger`, keyed
+      # "<attempt_id>:<round>".
+      def within(provider, ledger, attempt_id, pausable: false, rounds: 0)
+        outer = Fiber[HELD]
+        Fiber[HELD] = Holder.new(provider, ledger, attempt_id, pausable, rounds, false)
+        yield Fiber[HELD]
+      ensure
+        Fiber[HELD] = outer
+      end
+
+      # The holder of a request the provider is about to make, unless one is
+      # under way (the base class's hook inside OpenAI's).
+      def holder_for(provider)
+        holder = Fiber[HELD]
+        holder if holder&.provider.equal?(provider) && !holder.busy
+      end
+
+      def hold(provider, args, kwargs, mode: :sync)
+        Budget::RequestHold.for_request(
+          provider: provider.slug, model: kwargs.fetch(:model), messages: args.first, tools: kwargs[:tools],
+          schema: kwargs[:schema], params: kwargs[:params], mode:
+        )
+      end
+
+      # The request (`request`: the provider's own `complete`), held.
+      def complete(provider, args, kwargs, block, &request)
+        exchange = BatchExchange.exchange_for(provider)
+        return capture(provider, exchange, args, kwargs, block, &request) if exchange
+
+        ResponseMemo.round(provider, args.first) { send_held(provider, args, kwargs, block, &request) }
+      end
+
+      def send_held(provider, args, kwargs, block, &request)
+        holder = holder_for(provider)
+        return request.call(*args, **kwargs, &block) unless holder
+
+        held(holder, hold(provider, args, kwargs), kwargs.fetch(:model).id) do |stream|
+          next request.call(*args, **kwargs) unless block
+
+          request.call(*args, **kwargs) do |chunk|
+            stream.started = true
+            block.call(chunk)
+          end
+        end
+      end
+
+      def capture(provider, exchange, args, kwargs, block, &request)
+        exchange.hold ||= hold(provider, args, kwargs, mode: :batch) if exchange.holding && exchange.result.nil?
+        request.call(*args, **kwargs, &block)
+      end
+
+      def held(holder, amount, model_id)
+        key, reserved = open_round(holder, amount)
+        stream = Stream.new(false)
+        begin
+          message = busy(holder) { yield(stream) }
+        rescue Exception => e # rubocop:disable Lint/RescueException -- an interrupted request settles too
+          settle_failed(holder, key, spent(holder.provider, model_id, e) || (amount if stream.started), reserved)
+          raise
+        end
+        Budget::HostLedger.settle(holder.ledger, key, spent(holder.provider, model_id, message) || amount)
+        message
+      end
+
+      # The request's key and whether it is held: a call's first request, or
+      # any round of a step that can pause for budget.
+      def open_round(holder, amount)
+        holder.rounds += 1
+        key = "#{holder.attempt_id}:#{holder.rounds}"
+        reserved = holder.rounds == 1 || holder.pausable
+        Budget::HostLedger.admit!(holder.ledger, amount, key) if reserved
+        [key, reserved]
+      end
+
+      def busy(holder)
+        holder.busy = true
+        yield
+      ensure
+        holder.busy = false
+      end
+
+      # A failed request reconciles what it spent; a held one that spent
+      # nothing is released, an unheld one settles nothing.
+      def settle_failed(holder, key, spent, reserved)
+        Budget::HostLedger.settle(holder.ledger, key, spent) if spent || reserved
+      end
+
+      # What a request spent, from its reply (or the error that reported
+      # usage), priced for the request's model; nil when it reported none.
+      def spent(provider, model_id, reply)
+        usage = Agent::ProviderUsage.from_message(reply)
+        Budget::HostLedger.amount(usage, model: model_id, provider: provider.slug) if usage
+      end
+
+      # Prepended on RubyLLM::Provider, and on RubyLLM::Providers::OpenAI
+      # outside Smith's Responses routing.
+      module Hook
+        def complete(*args, **kwargs, &block)
+          RequestHolds.complete(self, args, kwargs, block) do |*sent, **options, &chunks|
+            super(*sent, **options, &chunks)
+          end
+        end
+      end
+    end
+  end
+end
+
+RubyLLM::Provider.prepend(Smith::Providers::RequestHolds::Hook)
+RubyLLM::Providers::OpenAI.prepend(Smith::Providers::RequestHolds::Hook)

data/lib/smith/providers/response_facts.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require_relative "../diagnostic_text"
+
+module Smith
+  module Providers
+    # The stop reason and the provider's own message id of one provider
+    # response, read from the raw body RubyLLM keeps on the message. Each
+    # provider's body is recognized by its own marker, never guessed from
+    # loose field names:
+    #
+    #   Anthropic Messages      "type" => "message"          stop_reason, id
+    #   OpenAI Chat Completions "object" => "chat.completion" choices[0].finish_reason, id
+    #   OpenAI Responses        "object" => "response"       incomplete_details.reason, else status; id
+    #   Gemini                  "candidates" => [...]         candidates[0].finishReason, responseId
+    #   Bedrock Converse        "stopReason" => String        stopReason
+    #
+    # OpenAI reports a refusal on the message rather than as a finish
+    # reason (Chat Completions `message.refusal`, a Responses `refusal`
+    # content item); it is read as the stop reason "refusal". Gemini blocks a
+    # prompt with `promptFeedback.blockReason` and no candidates; that is read
+    # as the stop reason "prompt_blocked".
+    #
+    # Both facts are metadata: bounded Strings, absent when the body is of
+    # none of these shapes, does not carry them, or cannot be read (streamed
+    # responses keep no raw body). Reading never raises.
+    module ResponseFacts
+      MAX_BYTES = 256
+      REFUSAL = "refusal"
+      PROMPT_BLOCKED = "prompt_blocked"
+
+      KEPT = "smith.response"
+      # A response Smith made or kept itself (a response memo's, a test
+      # stub's) carries its facts as a raw body of this shape.
+      Kept = Struct.new(:body)
+
+      READERS = [
+        [->(body) { body["object"] == KEPT }, ->(body) { [body["stop_reason"], body["id"]] }],
+        [->(body) { body["type"] == "message" }, ->(body) { [body["stop_reason"], body["id"]] }],
+        [->(body) { body["object"] == "chat.completion" },
+         lambda do |body|
+           choice = first_entry(body["choices"])
+           [(REFUSAL if chat_refusal?(choice)) || choice&.fetch("finish_reason", nil), body["id"]]
+         end],
+        [->(body) { body["object"] == "response" },
+         lambda do |body|
+           details = body["incomplete_details"]
+           reason = (REFUSAL if responses_refusal?(body["output"])) || (details["reason"] if details.is_a?(Hash))
+           [reason || body["status"], body["id"]]
+         end],
+        [->(body) { !body.key?("candidates") && body.dig("promptFeedback", "blockReason").is_a?(String) },
+         ->(body) { [PROMPT_BLOCKED, body["responseId"]] }],
+        [->(body) { body["candidates"].is_a?(Array) },
+         ->(body) { [first_entry(body["candidates"])&.fetch("finishReason", nil), body["responseId"]] }],
+        [->(body) { body["stopReason"].is_a?(String) }, ->(body) { [body["stopReason"], nil] }]
+      ].freeze
+      private_constant :READERS
+
+      module_function
+
+      def kept(stop_reason:, id:) = Kept.new({ "object" => KEPT, "stop_reason" => stop_reason, "id" => id })
+
+      def from_message(message)
+        body = raw_body(message)
+        return {} unless body.instance_of?(Hash)
+
+        _, reader = READERS.find { |(matches, _)| matches.call(body) }
+        return {} unless reader
+
+        stop_reason, message_id = reader.call(body)
+        { stop_reason: bounded(stop_reason), provider_message_id: bounded(message_id) }.compact
+      rescue StandardError
+        {}
+      end
+
+      def raw_body(message)
+        return unless message.respond_to?(:raw)
+
+        raw = message.raw
+        raw.respond_to?(:body) ? raw.body : raw
+      end
+
+      def chat_refusal?(choice)
+        message = choice&.fetch("message", nil)
+        message.is_a?(Hash) && message["refusal"].is_a?(String) && !message["refusal"].empty?
+      end
+
+      def responses_refusal?(output)
+        return false unless output.is_a?(Array)
+
+        output.any? do |item|
+          item.is_a?(Hash) && item["content"].is_a?(Array) &&
+            item["content"].any? { |part| part.is_a?(Hash) && part["type"] == REFUSAL }
+        end
+      end
+
+      def first_entry(list)
+        list.first if list.is_a?(Array) && list.first.is_a?(Hash)
+      end
+
+      def bounded(value)
+        return unless value.is_a?(String) && !value.empty?
+
+        DiagnosticText.capture(value, max_bytes: MAX_BYTES)
+      end
+    end
+  end
+end

data/lib/smith/providers/schema_constraint_check.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+require_relative "schema_rules"
+
+module Smith
+  module Providers
+    # Checks a parsed structured output against the constraints in its JSON
+    # schema that SchemaConstraints may have kept from the provider:
+    # numeric bounds and multiples, string lengths, array sizes and
+    # uniqueness, and required properties, through properties, items,
+    # local `$ref`s, `allOf`, `anyOf` (one alternative must accept the
+    # value) and `oneOf` (exactly one must). When no alternative accepts a
+    # value, the problems of the one alternative of its type are reported
+    # if there is one. Each schema and value pair is judged once, so the
+    # walk stays polynomial however deeply the output nests; it reports at
+    # most MAX_PROBLEMS "path: problem" lines and stops descending after
+    # MAX_VISITS visits.
+    class SchemaConstraintCheck
+      MAX_PROBLEMS = 20
+      MAX_DEPTH = 64
+      MAX_VISITS = 1_000_000
+      include SchemaRules
+
+      def initialize(root)
+        @root = root
+      end
+
+      def call(value)
+        @found = []
+        @visits = 0
+        @verdicts = {}
+        check(@root, value, "", 0)
+        @found.first(MAX_PROBLEMS)
+      end
+
+      private
+
+      def check(schema, value, path, depth)
+        return unless schema.is_a?(Hash) && depth <= MAX_DEPTH && @found.length < MAX_PROBLEMS
+        return if (@visits += 1) > MAX_VISITS
+
+        schema = resolve(schema)
+        check_alternatives(schema, value, path, depth) if schema["anyOf"] || schema["oneOf"]
+        Array(schema["allOf"]).each { check(_1, value, path, depth + 1) }
+        check_value(schema, value, path, depth)
+      end
+
+      def check_value(schema, value, path, depth)
+        own_problems(schema, value).each { |problem| report(path, problem) }
+        check_children(schema, value, path, depth)
+      end
+
+      def report(path, problem) = @found << "#{path.empty? ? "/" : path}: #{problem}"
+
+      def check_alternatives(schema, value, path, depth)
+        alternatives = Array(schema["anyOf"] || schema["oneOf"]).map { resolve(_1) }
+        accepted = alternatives.count { accepts?(_1, value, depth + 1) }
+        return if schema["anyOf"] ? accepted.positive? : accepted == 1
+        return report(path, "matches more than one of the allowed forms") if accepted > 1
+
+        report_closest(alternatives, value, path, depth)
+      end
+
+      def report_closest(alternatives, value, path, depth)
+        typed = alternatives.select { type?(_1["type"], value) }
+        return report(path, "matches none of the allowed forms") unless typed.length == 1
+
+        check(typed.first, value, path, depth + 1)
+      end
+
+      # Whether the alternative accepts the value, judged once per schema
+      # and value.
+      def accepts?(schema, value, depth)
+        @verdicts.fetch([schema.object_id, value.object_id]) do |key|
+          @verdicts[key] = type?(schema["type"], value) && trial(schema, value, depth)
+        end
+      end
+
+      def trial(schema, value, depth)
+        outer = @found
+        @found = []
+        check(schema, value, "", depth)
+        @found.empty?
+      ensure
+        @found = outer
+      end
+
+      def check_children(schema, value, path, depth)
+        case value
+        when Hash
+          Hash(schema["properties"]).each do |name, child|
+            check(child, value[name], "#{path}/#{name}", depth + 1) if value.key?(name)
+          end
+        when Array
+          value.each_with_index do |item, index|
+            check(item_schema(schema, index), item, "#{path}/#{index}", depth + 1)
+          end
+        end
+      end
+
+      def item_schema(schema, index)
+        tuple = schema["prefixItems"] || (schema["items"] if schema["items"].is_a?(Array))
+        Array(tuple)[index] || (schema["items"] unless schema["items"].is_a?(Array))
+      end
+
+      def own_problems(schema, value)
+        case value
+        when Integer, Float then number_problems(schema, value)
+        when String then size_problems(schema, value.length, "characters", "minLength", "maxLength")
+        when Array then array_problems(schema, value)
+        when Hash then Array(schema["required"]).reject { value.key?(_1) }.map { "is missing #{_1}" }
+        else []
+        end
+      end
+
+      def number_problems(schema, number)
+        # NUMBER_RULES: [keyword, broken?(number, limit), message]
+        NUMBER_RULES.filter_map do |keyword, broken, message|
+          limit = schema[keyword]
+          format(message, limit) if numeric?(limit) && broken.call(number, limit)
+        end
+      end
+
+      # A local JSON pointer `$ref` ("#" for the whole schema,
+      # "#/$defs/name", "#/definitions/name", or deeper, as
+      # "#/$defs/a/properties/b"); any other reference is left as the schema
+      # it sits in.
+      def resolve(schema)
+        ref = schema["$ref"]
+        return @root if ref == "#"
+        return schema unless ref.is_a?(String) && ref.start_with?("#/")
+
+        target = ref.delete_prefix("#/").split("/").reduce(@root) do |level, part|
+          key = part.gsub("~1", "/").gsub("~0", "~")
+          level.is_a?(Hash) ? level[key] : nil
+        end
+        target.is_a?(Hash) ? target : schema
+      end
+    end
+  end
+end

data/lib/smith/providers/schema_constraints.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+require "json"
+
+require_relative "schema_constraint_check"
+
+module Smith
+  module Providers
+    # Structured-output schemas and the constraints providers do not enforce.
+    #
+    # Anthropic's structured outputs reject numeric constraints (`minimum`,
+    # `maximum`, `exclusiveMinimum`, `exclusiveMaximum`, `multipleOf`),
+    # string lengths (`minLength`, `maxLength`) and array sizes beyond a
+    # `minItems` of 0 or 1 (`maxItems`, `uniqueItems`), and need
+    # `additionalProperties: false` on every object. `for_provider` sends
+    # such a schema without them; `problems` checks them on the parsed
+    # output, for every provider (SchemaConstraintCheck), so one schema
+    # serves both the request and the check. Walks are bounded in depth.
+    module SchemaConstraints
+      STRIPPED = %w[minimum maximum exclusiveMinimum exclusiveMaximum multipleOf minLength maxLength maxItems
+                    uniqueItems].freeze
+      NESTED_LISTS = %w[anyOf allOf oneOf prefixItems].freeze
+      NESTED_MAPS = %w[properties $defs definitions].freeze
+      NESTED_SCHEMAS = %w[items not].freeze
+      MAX_DEPTH = 64
+
+      module_function
+
+      # The JSON schema payload ({ "schema" => ..., with "name" and the
+      # like when given }, String keys) of an output schema: a RubyLLM
+      # schema class or instance, or a Hash. nil when it has none.
+      def json_schema(output_schema)
+        instance = output_schema.is_a?(Class) ? output_schema.new : output_schema
+        raw = instance.respond_to?(:to_json_schema) ? instance.to_json_schema : instance
+        return unless raw.is_a?(Hash)
+
+        schema = JSON.parse(JSON.generate(raw))
+        schema["schema"].is_a?(Hash) ? schema : { "schema" => schema }
+      rescue StandardError
+        nil
+      end
+
+      # The schema payload to send to `provider`: for Anthropic, without the
+      # constraints it rejects and with closed objects; unchanged otherwise.
+      def for_provider(payload, provider)
+        return payload unless provider.to_s == "anthropic"
+
+        payload.merge("schema" => strip(payload.fetch("schema"), 0))
+      end
+
+      # "path: problem" lines for each constraint the value breaks, at most
+      # SchemaConstraintCheck::MAX_PROBLEMS.
+      def problems(schema, value) = SchemaConstraintCheck.new(schema).call(value)
+
+      def strip(schema, depth)
+        return schema unless schema.is_a?(Hash) && depth <= MAX_DEPTH
+
+        copy = schema.reject { |key, setting| STRIPPED.include?(key) || (key == "minItems" && setting.to_i > 1) }
+        close_object!(copy)
+        copy.to_h { |key, setting| [key, strip_nested(key, setting, depth)] }
+      end
+
+      def strip_nested(key, setting, depth)
+        if (NESTED_LISTS.include?(key) || key == "items") && setting.is_a?(Array)
+          setting.map { strip(_1, depth + 1) }
+        elsif NESTED_MAPS.include?(key) && setting.is_a?(Hash) then setting.transform_values { strip(_1, depth + 1) }
+        elsif NESTED_SCHEMAS.include?(key) then strip(setting, depth + 1)
+        else setting
+        end
+      end
+
+      # An object gets `additionalProperties: false`. One that allows other
+      # properties, or declares none (a free-form map, which closed could
+      # only be `{}`), cannot be sent and is refused before the call.
+      def close_object!(schema)
+        return unless Array(schema["type"]).include?("object") || schema.key?("properties")
+
+        open = schema.key?("additionalProperties") && schema["additionalProperties"] != false
+        if open || Hash(schema["properties"]).empty?
+          raise Smith::WorkflowError, "Anthropic structured outputs need every object to declare its properties " \
+                                      "and allow no others"
+        end
+        schema["additionalProperties"] = false
+      end
+    end
+  end
+end

data/lib/smith/providers/schema_rules.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require "bigdecimal"
+
+module Smith
+  module Providers
+    # The JSON schema rules SchemaConstraintCheck applies: numeric bounds
+    # and multiples, array sizes and uniqueness, and JSON types.
+    module SchemaRules
+      NUMBER_RULES = [
+        ["minimum", ->(number, limit) { number < limit }, "is below the minimum %s"],
+        ["maximum", ->(number, limit) { number > limit }, "is above the maximum %s"],
+        ["exclusiveMinimum", ->(number, limit) { number <= limit }, "must be above %s"],
+        ["exclusiveMaximum", ->(number, limit) { number >= limit }, "must be below %s"],
+        ["multipleOf", ->(number, divisor) { divisor.positive? && !SchemaRules.multiple?(number, divisor) },
+         "is not a multiple of %s"]
+      ].freeze
+      TYPES = {
+        "object" => ->(value) { value.is_a?(Hash) }, "array" => ->(value) { value.is_a?(Array) },
+        "string" => ->(value) { value.is_a?(String) },
+        "integer" => ->(value) { value.is_a?(Integer) || SchemaRules.integral_float?(value) },
+        "number" => ->(value) { value.is_a?(Integer) || value.is_a?(Float) },
+        "boolean" => ->(value) { [true, false].include?(value) }, "null" => lambda(&:nil?)
+      }.freeze
+
+      # Compared as the decimals written in the schema and the output, so
+      # 5 is a multiple of 0.01 and 0.3 of 0.1.
+      def self.integral_float?(value) = value.is_a?(Float) && value.finite? && value == value.floor
+
+      def self.multiple?(number, divisor)
+        quotient = BigDecimal(number.to_s) / BigDecimal(divisor.to_s)
+        (quotient - quotient.round).abs < BigDecimal("1e-9")
+      end
+
+      private
+
+      def numeric?(setting) = setting.is_a?(Integer) || (setting.is_a?(Float) && setting.finite?)
+
+      def array_problems(schema, array)
+        problems = size_problems(schema, array.length, "items", "minItems", "maxItems")
+        problems << "has duplicate items" if schema["uniqueItems"] == true && duplicates?(array)
+        problems
+      end
+
+      # 1 and 1.0 are the same JSON number.
+      def duplicates?(array)
+        normalized = array.map { |item| SchemaRules.integral_float?(item) ? item.to_i : item }
+        normalized.uniq.length != normalized.length
+      end
+
+      def size_problems(schema, size, unit, min_key, max_key)
+        [
+          ("has fewer than #{schema[min_key]} #{unit}" if schema[min_key].is_a?(Integer) && size < schema[min_key]),
+          ("has more than #{schema[max_key]} #{unit}" if schema[max_key].is_a?(Integer) && size > schema[max_key])
+        ].compact
+      end
+
+      def type?(type, value)
+        return true if type.nil?
+        return Array(type).any? { type?(_1, value) } if type.is_a?(Array)
+
+        TYPES.fetch(type, ->(_value) { true }).call(value)
+      end
+    end
+  end
+end

data/lib/smith/providers/stop_reason.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Smith
+  module Providers
+    # What a provider's stop reason says about how a response ended: cut off
+    # at the output limit (:truncated), declined (:refused), or nothing Smith
+    # acts on (nil, which includes an absent or unknown stop reason).
+    #
+    #   truncated  max_tokens, model_context_window_exceeded (Anthropic,
+    #              Bedrock), length (OpenAI Chat Completions),
+    #              max_output_tokens (OpenAI Responses), MAX_TOKENS (Gemini),
+    #              output_limit_reached and stream_incomplete (Smith, for
+    #              a streamed reply that ended without a stop reason, with
+    #              or without its output reaching the request's limit)
+    #   refused    refusal (Anthropic; OpenAI's message-level refusal),
+    #              prompt_blocked (a Gemini prompt block),
+    #              content_filter (OpenAI), SAFETY, RECITATION, BLOCKLIST,
+    #              PROHIBITED_CONTENT, SPII, IMAGE_SAFETY (Gemini),
+    #              content_filtered, guardrail_intervened (Bedrock)
+    module StopReason
+      TRUNCATED = %w[
+        max_tokens model_context_window_exceeded length max_output_tokens MAX_TOKENS output_limit_reached
+        stream_incomplete
+      ].freeze
+      REFUSED = %w[
+        refusal prompt_blocked content_filter SAFETY RECITATION BLOCKLIST PROHIBITED_CONTENT SPII IMAGE_SAFETY
+        content_filtered guardrail_intervened
+      ].freeze
+
+      module_function
+
+      def classify(stop_reason)
+        return :truncated if TRUNCATED.include?(stop_reason)
+
+        :refused if REFUSED.include?(stop_reason)
+      end
+    end
+  end
+end