smith-agents 0.11.0 → 0.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +712 -0
- data/README.md +622 -15
- data/UPSTREAM_PROPOSAL.md +1 -0
- data/docs/CONFIGURATION.md +80 -14
- data/docs/PATTERNS.md +30 -13
- data/docs/PERSISTENCE.md +733 -5
- data/docs/TOOLS_AND_GUARDRAILS.md +24 -3
- data/lib/generators/smith/install/install_generator.rb +67 -2
- data/lib/generators/smith/install/templates/create_workflow_states.rb.tt +31 -0
- data/lib/generators/smith/install/templates/smith.rb.tt +39 -8
- data/lib/generators/smith/install/templates/workflow_state.rb.tt +19 -0
- data/lib/smith/agent/completion.rb +32 -6
- data/lib/smith/agent/completion_usage_recording.rb +23 -14
- data/lib/smith/agent/content_bytes.rb +98 -0
- data/lib/smith/agent/content_parts.rb +149 -0
- data/lib/smith/agent/invocation_preparation.rb +37 -14
- data/lib/smith/agent/lifecycle.rb +16 -5
- data/lib/smith/agent/observed_model.rb +29 -0
- data/lib/smith/agent/output_contract_check.rb +53 -0
- data/lib/smith/agent/output_integrity.rb +137 -0
- data/lib/smith/agent/output_policy.rb +92 -0
- data/lib/smith/agent/output_recovery.rb +129 -0
- data/lib/smith/agent/provider_attempt.rb +6 -0
- data/lib/smith/agent/provider_call_timing.rb +13 -4
- data/lib/smith/agent/provider_completion.rb +35 -32
- data/lib/smith/agent/provider_failure_handling.rb +5 -1
- data/lib/smith/agent/provider_request.rb +36 -0
- data/lib/smith/agent/provider_streaming.rb +155 -0
- data/lib/smith/agent/provider_usage.rb +70 -3
- data/lib/smith/agent/usage_entry_recording.rb +4 -2
- data/lib/smith/agent/usage_traces.rb +9 -2
- data/lib/smith/agent/usage_tracking.rb +64 -51
- data/lib/smith/agent.rb +5 -0
- data/lib/smith/artifacts/active_record_store.rb +79 -0
- data/lib/smith/artifacts/file.rb +45 -18
- data/lib/smith/artifacts/memory.rb +47 -36
- data/lib/smith/artifacts/s3.rb +126 -0
- data/lib/smith/artifacts/scoped_store.rb +55 -7
- data/lib/smith/artifacts/store_rules.rb +49 -0
- data/lib/smith/artifacts.rb +98 -1
- data/lib/smith/batch/message_shape.rb +36 -0
- data/lib/smith/batch.rb +166 -0
- data/lib/smith/budget/amount.rb +25 -0
- data/lib/smith/budget/continuation.rb +153 -0
- data/lib/smith/budget/host_ledger.rb +137 -0
- data/lib/smith/budget/ledger.rb +4 -2
- data/lib/smith/budget/ledger_state_transition.rb +7 -5
- data/lib/smith/budget/request_hold.rb +211 -0
- data/lib/smith/budget/spent_reservation.rb +18 -0
- data/lib/smith/child_run.rb +41 -0
- data/lib/smith/child_runs_failed.rb +56 -0
- data/lib/smith/content.rb +96 -0
- data/lib/smith/context/session.rb +14 -2
- data/lib/smith/context/state_injection.rb +10 -5
- data/lib/smith/context.rb +28 -0
- data/lib/smith/doctor/checks/artifacts.rb +103 -0
- data/lib/smith/doctor/checks/configuration.rb +0 -11
- data/lib/smith/doctor/checks/durability.rb +20 -18
- data/lib/smith/doctor/checks/state_model.rb +63 -0
- data/lib/smith/doctor/installer.rb +9 -2
- data/lib/smith/doctor.rb +4 -0
- data/lib/smith/errors.rb +249 -2
- data/lib/smith/idempotency_key/canonical.rb +74 -0
- data/lib/smith/idempotency_key.rb +145 -0
- data/lib/smith/launchers/inline.rb +46 -0
- data/lib/smith/message.rb +76 -0
- data/lib/smith/models/tool_routing.rb +47 -44
- data/lib/smith/output_failure.rb +130 -0
- data/lib/smith/persistence_adapters/active_record_exact_store.rb +43 -0
- data/lib/smith/pricing/catalog_validation.rb +91 -0
- data/lib/smith/pricing.rb +95 -64
- data/lib/smith/providers/anthropic/batches.rb +116 -0
- data/lib/smith/providers/batch_exchange.rb +77 -0
- data/lib/smith/providers/openai/responses.rb +45 -20
- data/lib/smith/providers/output_limit.rb +68 -0
- data/lib/smith/providers/request_holds.rb +139 -0
- data/lib/smith/providers/response_facts.rb +108 -0
- data/lib/smith/providers/schema_constraint_check.rb +141 -0
- data/lib/smith/providers/schema_constraints.rb +87 -0
- data/lib/smith/providers/schema_rules.rb +66 -0
- data/lib/smith/providers/stop_reason.rb +39 -0
- data/lib/smith/providers/stream_facts.rb +192 -0
- data/lib/smith/response_memo.rb +152 -0
- data/lib/smith/setting_constructors.rb +47 -0
- data/lib/smith/streaming/json_field.rb +136 -0
- data/lib/smith/streaming/json_string.rb +91 -0
- data/lib/smith/streaming/session.rb +77 -0
- data/lib/smith/streaming/think_tags.rb +67 -0
- data/lib/smith/streaming.rb +64 -0
- data/lib/smith/testing/checks.rb +58 -0
- data/lib/smith/testing/hooks.rb +85 -0
- data/lib/smith/testing/minitest.rb +37 -0
- data/lib/smith/testing/rspec.rb +52 -0
- data/lib/smith/testing/stub.rb +110 -0
- data/lib/smith/testing.rb +158 -0
- data/lib/smith/tool/scoped_context.rb +9 -12
- data/lib/smith/tool.rb +14 -0
- data/lib/smith/trace/logger.rb +1 -1
- data/lib/smith/trace/memory.rb +1 -1
- data/lib/smith/trace/open_telemetry.rb +1 -1
- data/lib/smith/trace.rb +11 -4
- data/lib/smith/untrusted.rb +74 -0
- data/lib/smith/version.rb +2 -2
- data/lib/smith/workflow/active_clock.rb +61 -0
- data/lib/smith/workflow/agent_result.rb +37 -0
- data/lib/smith/workflow/artifact_integration.rb +57 -7
- data/lib/smith/workflow/await_step.rb +42 -0
- data/lib/smith/workflow/batch_charges.rb +74 -0
- data/lib/smith/workflow/batch_execution.rb +149 -0
- data/lib/smith/workflow/batch_holds.rb +73 -0
- data/lib/smith/workflow/batch_wait.rb +134 -0
- data/lib/smith/workflow/budget_integration.rb +11 -5
- data/lib/smith/workflow/budget_pause.rb +51 -0
- data/lib/smith/workflow/child_run_declarations.rb +127 -0
- data/lib/smith/workflow/child_run_execution.rb +150 -0
- data/lib/smith/workflow/child_run_results.rb +55 -0
- data/lib/smith/workflow/composite/effects.rb +37 -6
- data/lib/smith/workflow/composite/error.rb +1 -1
- data/lib/smith/workflow/composite/error_evidence.rb +9 -9
- data/lib/smith/workflow/composite/input.rb +11 -4
- data/lib/smith/workflow/context_keys.rb +77 -0
- data/lib/smith/workflow/deadline_enforcement.rb +1 -1
- data/lib/smith/workflow/deterministic_execution.rb +61 -10
- data/lib/smith/workflow/deterministic_step.rb +55 -6
- data/lib/smith/workflow/dsl.rb +59 -0
- data/lib/smith/workflow/durability.rb +139 -30
- data/lib/smith/workflow/evaluator_optimizer.rb +12 -6
- data/lib/smith/workflow/event_integration.rb +9 -0
- data/lib/smith/workflow/execution.rb +30 -23
- data/lib/smith/workflow/execution_binding_resolution.rb +13 -0
- data/lib/smith/workflow/failure_reconstructor.rb +27 -4
- data/lib/smith/workflow/failure_record.rb +6 -1
- data/lib/smith/workflow/failure_record_validator.rb +11 -4
- data/lib/smith/workflow/fanout_execution.rb +5 -8
- data/lib/smith/workflow/graph/batch_diagnostic.rb +83 -0
- data/lib/smith/workflow/graph/branch_limit_diagnostic.rb +41 -0
- data/lib/smith/workflow/graph/budget_ledger_diagnostic.rb +126 -0
- data/lib/smith/workflow/graph/durable_retry_diagnostic.rb +50 -0
- data/lib/smith/workflow/graph/execution_successors.rb +16 -2
- data/lib/smith/workflow/graph/inherited_untrusted_diagnostics.rb +77 -0
- data/lib/smith/workflow/graph/isolated_input_diagnostic.rb +50 -0
- data/lib/smith/workflow/graph/nested_readiness_diagnostics.rb +43 -4
- data/lib/smith/workflow/graph/output_key_diagnostic.rb +53 -0
- data/lib/smith/workflow/graph/output_schema_diagnostic.rb +49 -0
- data/lib/smith/workflow/graph/retry_policy_diagnostic.rb +1 -0
- data/lib/smith/workflow/graph/run_launcher_diagnostic.rb +30 -0
- data/lib/smith/workflow/graph/runtime_binding_diagnostic_builder.rb +9 -3
- data/lib/smith/workflow/graph/runtime_binding_diagnostics.rb +7 -3
- data/lib/smith/workflow/graph/runtime_readiness_metrics.rb +8 -1
- data/lib/smith/workflow/graph/runtime_readiness_report_builder.rb +9 -2
- data/lib/smith/workflow/graph/runtime_readiness_traversal.rb +7 -4
- data/lib/smith/workflow/graph/stream_diagnostic.rb +95 -0
- data/lib/smith/workflow/graph/transition_contract.rb +19 -0
- data/lib/smith/workflow/graph/transition_contract_attributes.rb +25 -2
- data/lib/smith/workflow/graph/transition_contract_configurations.rb +24 -3
- data/lib/smith/workflow/graph/transition_diagnostics.rb +11 -24
- data/lib/smith/workflow/graph/transition_snapshot.rb +40 -39
- data/lib/smith/workflow/graph/untrusted_input_diagnostic.rb +57 -0
- data/lib/smith/workflow/graph/waiting_child_diagnostic.rb +24 -0
- data/lib/smith/workflow/graph.rb +12 -0
- data/lib/smith/workflow/guarded_step_execution.rb +48 -3
- data/lib/smith/workflow/message_batch.rb +6 -0
- data/lib/smith/workflow/message_roles.rb +25 -0
- data/lib/smith/workflow/message_value_normalizer.rb +2 -1
- data/lib/smith/workflow/nested_execution.rb +47 -4
- data/lib/smith/workflow/orchestrator_worker.rb +9 -3
- data/lib/smith/workflow/parallel_execution.rb +10 -0
- data/lib/smith/workflow/persistence.rb +136 -13
- data/lib/smith/workflow/remembered_calls.rb +92 -0
- data/lib/smith/workflow/retry_execution.rb +68 -9
- data/lib/smith/workflow/retry_wait.rb +134 -0
- data/lib/smith/workflow/split_step_persistence/checkpoint.rb +3 -0
- data/lib/smith/workflow/split_step_persistence/checkpoint_state.rb +7 -0
- data/lib/smith/workflow/split_step_persistence/composite_branch_execution.rb +4 -1
- data/lib/smith/workflow/split_step_persistence/composite_preparation.rb +3 -1
- data/lib/smith/workflow/split_step_persistence/dispatch_claim.rb +1 -0
- data/lib/smith/workflow/split_step_persistence/preparation.rb +5 -0
- data/lib/smith/workflow/split_step_persistence/state_snapshot.rb +1 -0
- data/lib/smith/workflow/step_completion.rb +36 -7
- data/lib/smith/workflow/step_context.rb +21 -0
- data/lib/smith/workflow/step_io.rb +132 -0
- data/lib/smith/workflow/step_stream.rb +52 -0
- data/lib/smith/workflow/thread_context_snapshot.rb +2 -0
- data/lib/smith/workflow/transition.rb +267 -13
- data/lib/smith/workflow/transition_actionability.rb +2 -0
- data/lib/smith/workflow/usage_entry.rb +24 -3
- data/lib/smith/workflow/wait_completion.rb +131 -0
- data/lib/smith/workflow/wait_execution.rb +180 -0
- data/lib/smith/workflow/wait_record.rb +157 -0
- data/lib/smith/workflow.rb +113 -17
- data/lib/smith.rb +101 -45
- metadata +88 -1
data/lib/smith/pricing.rb
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative "pricing/catalog_validation"
|
|
4
|
+
|
|
3
5
|
module Smith
|
|
4
6
|
module Pricing
|
|
5
7
|
# Compute provider cost for a single agent call. Two pricing shapes
|
|
@@ -18,6 +20,18 @@ module Smith
|
|
|
18
20
|
# long-context premium pricing (Gemini 2.5 Pro: $1.25/$10 below
|
|
19
21
|
# 200K input tokens, $2.50/$15 above).
|
|
20
22
|
#
|
|
23
|
+
# Both shapes accept two optional rates beside input and output:
|
|
24
|
+
# `:cache_read_cost_per_token` and `:cache_write_cost_per_token`, for
|
|
25
|
+
# input read from or written to a prompt cache, which providers report
|
|
26
|
+
# apart from uncached input. An entry may also carry `:batch_discount`,
|
|
27
|
+
# the fraction a provider's batch API takes off (0.5 for half price),
|
|
28
|
+
# applied to the whole cost of a call made in `mode: :batch`. These are
|
|
29
|
+
# configured per catalog: a missing cache rate prices those tokens at 0,
|
|
30
|
+
# and a missing batch discount is 0, so a batch call costs the full
|
|
31
|
+
# rate. Tiers are
|
|
32
|
+
# chosen on the call's total input, uncached plus cache read plus cache
|
|
33
|
+
# write, since that is what long-context premiums are measured against.
|
|
34
|
+
#
|
|
21
35
|
# Catalog keys are normalized once per assigned catalog object:
|
|
22
36
|
# ["provider", "model"] / [:provider, :model] -> provider-qualified
|
|
23
37
|
# "provider/model" (first slash splits) -> provider-qualified
|
|
@@ -25,21 +39,58 @@ module Smith
|
|
|
25
39
|
#
|
|
26
40
|
# Lookup policy: a provider-qualified lookup reads only qualified
|
|
27
41
|
# entries and NEVER prices from a legacy model-only rate; unpriced
|
|
28
|
-
# usage stays visibly unpriced (nil cost).
|
|
42
|
+
# usage stays visibly unpriced (nil cost). A count that is not a
|
|
43
|
+
# non-negative Integer prices as 0. This method never raises,
|
|
29
44
|
# so accounting paths cannot mask a provider error with a pricing
|
|
30
45
|
# configuration error. The legacy-only-key policy is enforced at
|
|
31
46
|
# catalog admission time via `validate_catalog!`.
|
|
32
|
-
|
|
47
|
+
# rubocop:disable Metrics/ParameterLists
|
|
48
|
+
def self.compute_cost(model:, input_tokens:, output_tokens:, provider: nil, cache_read_tokens: 0,
|
|
49
|
+
cache_write_tokens: 0, mode: :sync)
|
|
50
|
+
counts = [input_tokens, output_tokens, cache_read_tokens, cache_write_tokens].map { |count| token_count(count) }
|
|
51
|
+
rates = rates_for(model, provider, counts[0] + counts[2] + counts[3])
|
|
52
|
+
return nil unless rates
|
|
53
|
+
|
|
54
|
+
apply_mode(priced_tokens(rates, *counts), rates.fetch(:entry), mode)
|
|
55
|
+
end
|
|
56
|
+
# rubocop:enable Metrics/ParameterLists
|
|
57
|
+
|
|
58
|
+
# True when the call's cache tokens were priced at 0 because its rates
|
|
59
|
+
# have no cache rate for them; false when they were priced, absent, or
|
|
60
|
+
# the call is unpriced altogether.
|
|
61
|
+
def self.cache_unpriced?(model:, input_tokens:, provider: nil, cache_read_tokens: 0, cache_write_tokens: 0)
|
|
62
|
+
read = token_count(cache_read_tokens)
|
|
63
|
+
write = token_count(cache_write_tokens)
|
|
64
|
+
rates = rates_for(model, provider, token_count(input_tokens) + read + write)
|
|
65
|
+
return false unless rates
|
|
66
|
+
|
|
67
|
+
(read.positive? && rates[:cache_read].nil?) || (write.positive? && rates[:cache_write].nil?)
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# A count that is not a non-negative Integer prices as 0, so accounting
|
|
71
|
+
# never raises on a provider's missing or malformed count.
|
|
72
|
+
def self.token_count(count)
|
|
73
|
+
count.is_a?(Integer) && count.positive? ? count : 0
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def self.rates_for(model, provider, total_input_tokens)
|
|
33
77
|
catalog = Smith.config.pricing
|
|
34
78
|
return nil unless catalog.is_a?(Hash)
|
|
35
79
|
|
|
36
80
|
entry = lookup_entry(normalized_index(catalog), model, provider)
|
|
37
|
-
|
|
81
|
+
resolve_rates(entry, total_input_tokens) if entry.is_a?(Hash)
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def self.priced_tokens(rates, input_tokens, output_tokens, cache_read_tokens, cache_write_tokens)
|
|
85
|
+
(input_tokens * rates[:input]) + (output_tokens * rates[:output]) +
|
|
86
|
+
(cache_read_tokens * (rates[:cache_read] || 0)) + (cache_write_tokens * (rates[:cache_write] || 0))
|
|
87
|
+
end
|
|
38
88
|
|
|
39
|
-
|
|
40
|
-
return
|
|
89
|
+
def self.apply_mode(cost, entry, mode)
|
|
90
|
+
return cost unless mode == :batch
|
|
41
91
|
|
|
42
|
-
|
|
92
|
+
discount = value_at(entry, :batch_discount)
|
|
93
|
+
discount.is_a?(Numeric) ? cost * (1 - discount) : cost
|
|
43
94
|
end
|
|
44
95
|
|
|
45
96
|
def self.lookup_entry(index, model, provider)
|
|
@@ -52,42 +103,18 @@ module Smith
|
|
|
52
103
|
# is assigned (`config.pricing = Smith::Pricing.validate_catalog!(catalog)`)
|
|
53
104
|
# so misconfiguration fails at configuration time instead of inside
|
|
54
105
|
# an in-flight accounting path. Rejects legacy model-only keys,
|
|
55
|
-
# unrecognized key shapes, non-Hash entries,
|
|
56
|
-
#
|
|
106
|
+
# unrecognized key shapes, non-Hash entries, keys that collide after
|
|
107
|
+
# normalization, optional rates that are not non-negative Integers or
|
|
108
|
+
# Floats, and a batch discount out of range or inside a tier. Returns
|
|
109
|
+
# the catalog for assignment chaining.
|
|
57
110
|
def self.validate_catalog!(catalog = Smith.config.pricing)
|
|
58
|
-
|
|
59
|
-
raise Smith::PricingConfigurationError, "pricing catalog must be a Hash" unless catalog.is_a?(Hash)
|
|
60
|
-
|
|
61
|
-
catalog.each_with_object({}) do |(key, entry), seen|
|
|
62
|
-
provider, model = validated_key!(key)
|
|
63
|
-
validate_entry!(key, entry)
|
|
64
|
-
reject_collision!(seen, key, provider, model)
|
|
65
|
-
end
|
|
66
|
-
catalog
|
|
111
|
+
CatalogValidation.validate!(catalog)
|
|
67
112
|
end
|
|
68
113
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
raise Smith::PricingConfigurationError, "pricing for #{model} must use a provider-qualified catalog key"
|
|
75
|
-
end
|
|
76
|
-
|
|
77
|
-
def self.validate_entry!(key, entry)
|
|
78
|
-
return if entry.is_a?(Hash)
|
|
79
|
-
|
|
80
|
-
raise Smith::PricingConfigurationError, "pricing entry for #{key.inspect} must be a Hash"
|
|
81
|
-
end
|
|
82
|
-
|
|
83
|
-
def self.reject_collision!(seen, key, provider, model)
|
|
84
|
-
previous = seen[[provider, model]]
|
|
85
|
-
if previous
|
|
86
|
-
raise Smith::PricingConfigurationError,
|
|
87
|
-
"pricing catalog keys #{previous.inspect} and #{key.inspect} collide as #{provider}/#{model}"
|
|
88
|
-
end
|
|
89
|
-
|
|
90
|
-
seen[[provider, model]] = key
|
|
114
|
+
# A catalog key read as a Symbol or a String, as a catalog loaded from
|
|
115
|
+
# JSON or YAML has.
|
|
116
|
+
def self.value_at(source, name)
|
|
117
|
+
source.key?(name) ? source[name] : source[name.to_s]
|
|
91
118
|
end
|
|
92
119
|
|
|
93
120
|
# The catalog is treated as immutable once assigned: normalization
|
|
@@ -148,39 +175,43 @@ module Smith
|
|
|
148
175
|
text unless text.nil? || text.empty?
|
|
149
176
|
end
|
|
150
177
|
|
|
151
|
-
# Returns
|
|
152
|
-
#
|
|
153
|
-
# shape is the legacy default.
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
178
|
+
# Returns { input:, output:, cache_read:, cache_write:, entry: } or nil
|
|
179
|
+
# if no applicable input and output rate. Tiered shape is recognized by
|
|
180
|
+
# the presence of a :tiers key; flat shape is the legacy default. A tier
|
|
181
|
+
# without its own cache rate uses the entry's.
|
|
182
|
+
def self.resolve_rates(entry, total_input_tokens)
|
|
183
|
+
tiers = value_at(entry, :tiers)
|
|
184
|
+
rates = if tiers.is_a?(Array) && !tiers.empty?
|
|
185
|
+
tier = tiers.find { |candidate| candidate.is_a?(Hash) && tier_applies?(candidate, total_input_tokens) }
|
|
186
|
+
tier && rates_from(tier, defaults: entry)
|
|
187
|
+
else
|
|
188
|
+
rates_from(entry)
|
|
189
|
+
end
|
|
190
|
+
rates&.merge(entry:)
|
|
164
191
|
end
|
|
165
192
|
|
|
166
|
-
def self.
|
|
167
|
-
|
|
168
|
-
|
|
193
|
+
def self.rates_from(source, defaults: nil)
|
|
194
|
+
rates = %i[input output cache_read cache_write].to_h { |name| [name, rate_at(source, defaults, name)] }
|
|
195
|
+
rates if rates[:input] && rates[:output]
|
|
196
|
+
end
|
|
169
197
|
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
198
|
+
# A tier's own rate, else, for a cache rate, the entry's.
|
|
199
|
+
def self.rate_at(source, defaults, name)
|
|
200
|
+
rate = value_at(source, :"#{name}_cost_per_token")
|
|
201
|
+
rate = value_at(defaults, :"#{name}_cost_per_token") if rate.nil? && defaults && CACHE_RATES.include?(name)
|
|
202
|
+
rate if rate.is_a?(Numeric)
|
|
175
203
|
end
|
|
176
204
|
|
|
177
205
|
def self.tier_applies?(tier, input_tokens)
|
|
178
|
-
maximum = tier
|
|
206
|
+
maximum = value_at(tier, :max_input_tokens)
|
|
179
207
|
maximum.nil? || input_tokens <= maximum
|
|
180
208
|
end
|
|
181
209
|
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
210
|
+
CACHE_RATES = %i[cache_read cache_write].freeze
|
|
211
|
+
private_constant :CACHE_RATES
|
|
212
|
+
|
|
213
|
+
private_class_method :rate_at, :lookup_entry, :normalized_index, :build_normalized_index, :canonical_key,
|
|
214
|
+
:array_key, :text_key, :scalar_key, :resolve_rates, :rates_from, :tier_applies?,
|
|
215
|
+
:priced_tokens, :apply_mode, :token_count, :rates_for, :value_at
|
|
185
216
|
end
|
|
186
217
|
end
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "ruby_llm"
|
|
5
|
+
|
|
6
|
+
module Smith
|
|
7
|
+
module Providers
|
|
8
|
+
module Anthropic
|
|
9
|
+
# Anthropic's Message Batches API, over RubyLLM's configured Anthropic
|
|
10
|
+
# connection (its key and base URL), for the host to submit the
|
|
11
|
+
# requests Smith's batch queue took, read a batch's status, collect its
|
|
12
|
+
# results and cancel it. Smith calls none of these: the host schedules
|
|
13
|
+
# them, then resumes each run with the results of its wait.
|
|
14
|
+
#
|
|
15
|
+
# batches = Smith::Providers::Anthropic::Batches.new
|
|
16
|
+
# batch_id = batches.submit(queued) # [{ custom_id:, request: }, ...], as enqueued
|
|
17
|
+
# batches.status(batch_id) # => { status: "in_progress" | "canceling" | "ended", counts: {...} }
|
|
18
|
+
# batches.results(batch_id) { |custom_id, result| ... }
|
|
19
|
+
#
|
|
20
|
+
# A batch takes at most 100,000 requests and 256 MB, and every request
|
|
21
|
+
# in it the same per-call headers (a beta feature's, say), which never
|
|
22
|
+
# override the connection's own (its key and API version). The
|
|
23
|
+
# connection makes no retries of its own, so a submit is never sent
|
|
24
|
+
# twice; the host's schedule tries again.
|
|
25
|
+
class Batches
|
|
26
|
+
PATH = "v1/messages/batches"
|
|
27
|
+
MAX_REQUESTS = 100_000
|
|
28
|
+
MAX_BYTES = 256 * 1024 * 1024
|
|
29
|
+
BATCH_ID = /\A[A-Za-z0-9_-]{1,128}\z/
|
|
30
|
+
|
|
31
|
+
def initialize(config: RubyLLM.config)
|
|
32
|
+
config = config.dup
|
|
33
|
+
config.max_retries = 0
|
|
34
|
+
@connection = RubyLLM::Providers::Anthropic.new(config).connection
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Submits the requests; returns the batch id.
|
|
38
|
+
def submit(requests)
|
|
39
|
+
entries = Array(requests).map { |entry| entry.transform_keys(&:to_sym) }
|
|
40
|
+
body = { requests: entries.map { |entry| { custom_id: entry.fetch(:custom_id), params: params(entry) } } }
|
|
41
|
+
check_batch!(entries, body)
|
|
42
|
+
headers = shared_headers(entries)
|
|
43
|
+
@connection.post(PATH, body) { |request| add_headers(request.headers, headers) }.body.fetch("id")
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# The batch's processing status ("in_progress", "canceling" or
|
|
47
|
+
# "ended") and its request counts.
|
|
48
|
+
def status(batch_id)
|
|
49
|
+
body = @connection.get("#{PATH}/#{id!(batch_id)}").body
|
|
50
|
+
{ status: body.fetch("processing_status"), counts: body["request_counts"], ended_at: body["ended_at"],
|
|
51
|
+
expires_at: body["expires_at"] }
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# Yields each result of an ended batch as [custom_id, result].
|
|
55
|
+
def results(batch_id)
|
|
56
|
+
return enum_for(:results, batch_id) unless block_given?
|
|
57
|
+
|
|
58
|
+
body = @connection.get("#{PATH}/#{id!(batch_id)}/results").body
|
|
59
|
+
entries(body).each { |entry| yield entry.fetch("custom_id"), entry.fetch("result") }
|
|
60
|
+
nil
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# Asks the provider to stop the batch; returns its status.
|
|
64
|
+
def cancel(batch_id)
|
|
65
|
+
@connection.post("#{PATH}/#{id!(batch_id)}/cancel", {}).body.fetch("processing_status")
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
private
|
|
69
|
+
|
|
70
|
+
def params(entry)
|
|
71
|
+
request = entry.fetch(:request).transform_keys(&:to_s)
|
|
72
|
+
raise ArgumentError, "a request for #{request["provider"]} cannot go in an Anthropic batch" unless
|
|
73
|
+
request["provider"] == "anthropic"
|
|
74
|
+
|
|
75
|
+
request.fetch("params")
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def check_batch!(entries, body)
|
|
79
|
+
unless entries.length.between?(1, MAX_REQUESTS)
|
|
80
|
+
raise ArgumentError, "a batch takes 1 to #{MAX_REQUESTS} requests, not #{entries.length}"
|
|
81
|
+
end
|
|
82
|
+
return if JSON.generate(body).bytesize <= MAX_BYTES
|
|
83
|
+
|
|
84
|
+
raise ArgumentError, "a batch takes at most #{MAX_BYTES} bytes; split the requests"
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def shared_headers(entries)
|
|
88
|
+
headers = entries.map { |entry| entry.fetch(:request).transform_keys(&:to_s).fetch("headers", {}).to_h }.uniq
|
|
89
|
+
raise ArgumentError, "the requests of one batch must share their headers" if headers.length > 1
|
|
90
|
+
|
|
91
|
+
headers.first
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# The requests' own headers, under the connection's (its key and API
|
|
95
|
+
# version win).
|
|
96
|
+
def add_headers(request_headers, headers)
|
|
97
|
+
headers.each { |name, value| request_headers[name] = value unless request_headers.key?(name) }
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
def id!(batch_id)
|
|
101
|
+
return batch_id if batch_id.is_a?(String) && BATCH_ID.match?(batch_id)
|
|
102
|
+
|
|
103
|
+
raise ArgumentError, "not a batch id: #{batch_id.inspect}"
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
# The results file: one JSON object per line.
|
|
107
|
+
def entries(body)
|
|
108
|
+
return body if body.is_a?(Array)
|
|
109
|
+
return [body] if body.is_a?(Hash)
|
|
110
|
+
|
|
111
|
+
body.to_s.each_line.filter_map { |line| JSON.parse(line) unless line.strip.empty? }
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
end
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "ruby_llm"
|
|
4
|
+
|
|
5
|
+
module Smith
|
|
6
|
+
module Providers
|
|
7
|
+
# Where batch mode meets RubyLLM: a provider's synchronous response
|
|
8
|
+
# (`sync_response`), which a call reaches with the exact payload and
|
|
9
|
+
# per-call headers it would post. While a batch step captures, the
|
|
10
|
+
# payload is thrown out as the batch request before anything is sent;
|
|
11
|
+
# while it delivers, the batch result is parsed by the provider's own
|
|
12
|
+
# parser instead of a response from the wire. A call that renders
|
|
13
|
+
# another request than the one sent (its digest differs: a date in its
|
|
14
|
+
# prompt, data read afresh) takes the result all the same and the
|
|
15
|
+
# delivery says so, unless its step declares another `on_request_change`
|
|
16
|
+
# (then BatchRequestChanged, before the result is read). Only the call's
|
|
17
|
+
# own provider object is affected, so any other RubyLLM call (a tool's)
|
|
18
|
+
# goes over the wire as ever.
|
|
19
|
+
module BatchExchange
|
|
20
|
+
HELD = :smith_batch_exchange
|
|
21
|
+
CAPTURED = :smith_batch_request
|
|
22
|
+
private_constant :HELD
|
|
23
|
+
|
|
24
|
+
# A capture notes its request's hold (RequestHolds) when its step has
|
|
25
|
+
# a host's budget ledger (`holding`).
|
|
26
|
+
Exchange = Struct.new(:provider, :result, :digest, :changed, :on_change, :holding, :hold)
|
|
27
|
+
# What a provider's response parser reads.
|
|
28
|
+
Response = Struct.new(:body)
|
|
29
|
+
|
|
30
|
+
module_function
|
|
31
|
+
|
|
32
|
+
# Runs the block with `result` (a batch result) answering its call,
|
|
33
|
+
# whose request was the one with `digest`; returns the block's value
|
|
34
|
+
# and whether the call rendered another request. Unless `on_change` is
|
|
35
|
+
# :keep, a call that renders another request raises
|
|
36
|
+
# BatchRequestChanged instead of taking the result.
|
|
37
|
+
def deliver(provider, result, digest, on_change: :keep, &)
|
|
38
|
+
exchange = Exchange.new(provider, result, digest, false, on_change)
|
|
39
|
+
[within(exchange, &), exchange.changed]
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def within(exchange)
|
|
43
|
+
outer = Fiber[HELD]
|
|
44
|
+
Fiber[HELD] = exchange
|
|
45
|
+
yield
|
|
46
|
+
ensure
|
|
47
|
+
Fiber[HELD] = outer
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def exchange_for(provider)
|
|
51
|
+
exchange = Fiber[HELD]
|
|
52
|
+
exchange if exchange&.provider.equal?(provider)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Prepended on RubyLLM::Provider.
|
|
56
|
+
module Hook
|
|
57
|
+
private
|
|
58
|
+
|
|
59
|
+
def sync_response(connection, payload, additional_headers = {})
|
|
60
|
+
exchange = BatchExchange.exchange_for(self)
|
|
61
|
+
return super unless exchange
|
|
62
|
+
|
|
63
|
+
unless exchange.result
|
|
64
|
+
throw(CAPTURED, Smith::Batch.request(provider: slug, params: payload, headers: additional_headers))
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
exchange.changed = Smith::Batch.changed?(exchange.digest, payload, additional_headers)
|
|
68
|
+
raise Smith::BatchRequestChanged if exchange.changed && exchange.on_change != :keep
|
|
69
|
+
|
|
70
|
+
parse_completion_response(Response.new(Smith::Batch.message!(exchange.result).to_h))
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
RubyLLM::Provider.prepend(Smith::Providers::BatchExchange::Hook)
|
|
@@ -51,21 +51,11 @@ module Smith
|
|
|
51
51
|
# vendored helpers, POSTs via the provider's Faraday connection,
|
|
52
52
|
# parses the response back into a RubyLLM::Message.
|
|
53
53
|
#
|
|
54
|
-
#
|
|
55
|
-
#
|
|
56
|
-
#
|
|
57
|
-
# message so the host can either disable streaming or fall back
|
|
58
|
-
# to `openai_api_mode = :off`.
|
|
54
|
+
# This path does not stream (the stream_response port from PR #770
|
|
55
|
+
# is not vendored): a call given a block completes, then passes the
|
|
56
|
+
# block the reply's whole text as one chunk.
|
|
59
57
|
def self.complete(provider, messages, tools:, temperature:, model:, params: {}, headers: {},
|
|
60
|
-
schema: nil, thinking: nil, tool_prefs: nil, &
|
|
61
|
-
if block
|
|
62
|
-
raise NotImplementedError,
|
|
63
|
-
"Smith::Providers::OpenAI::Responses does not yet support streaming. " \
|
|
64
|
-
"Streaming over /v1/responses needs a separate stream_response port from PR #770. " \
|
|
65
|
-
"Workaround: pass no block (sync only), or set Smith.config.openai_api_mode = :off " \
|
|
66
|
-
"to route via chat-completions with graceful tool-dropping."
|
|
67
|
-
end
|
|
68
|
-
|
|
58
|
+
schema: nil, thinking: nil, tool_prefs: nil, &)
|
|
69
59
|
payload = render_response_payload(
|
|
70
60
|
messages,
|
|
71
61
|
tools: tools,
|
|
@@ -86,7 +76,15 @@ module Smith
|
|
|
86
76
|
merged_headers.each { |k, v| req.headers[k] = v }
|
|
87
77
|
end
|
|
88
78
|
|
|
89
|
-
parse_response_response(http_response, provider: provider)
|
|
79
|
+
with_whole_chunk(parse_response_response(http_response, provider: provider), &)
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# SMITH-AUTHORED: a block gets the reply's whole text as one chunk.
|
|
83
|
+
def self.with_whole_chunk(message)
|
|
84
|
+
text = message&.content
|
|
85
|
+
yield ::RubyLLM::Chunk.new(role: :assistant, content: text, model_id: message.model_id) if
|
|
86
|
+
block_given? && text.is_a?(String) && !text.empty?
|
|
87
|
+
message
|
|
90
88
|
end
|
|
91
89
|
|
|
92
90
|
# rubocop:disable Metrics/ParameterLists
|
|
@@ -141,11 +139,9 @@ module Smith
|
|
|
141
139
|
content: response_output_text(data),
|
|
142
140
|
thinking: ::RubyLLM::Thinking.build(text: response_reasoning_text(outputs)),
|
|
143
141
|
tool_calls: ToolsExtensions.parse_response_tool_calls(outputs),
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
cache_creation_tokens: usage.dig("input_tokens_details", "cache_write_tokens") || 0,
|
|
148
|
-
thinking_tokens: usage.dig("output_tokens_details", "reasoning_tokens"),
|
|
142
|
+
# SMITH-AUTHORED: every usage count from response_usage (see the
|
|
143
|
+
# SMITH-AUTHORED helpers below and UPSTREAM_PROPOSAL.md).
|
|
144
|
+
**response_usage(usage),
|
|
149
145
|
model_id: data["model"],
|
|
150
146
|
raw: response
|
|
151
147
|
)
|
|
@@ -286,6 +282,7 @@ module Smith
|
|
|
286
282
|
end
|
|
287
283
|
|
|
288
284
|
# ---- SMITH-AUTHORED helpers (inlined from PR #770 chat.rb) -------
|
|
285
|
+
|
|
289
286
|
#
|
|
290
287
|
# Upstream PR #770 keeps these on the chat module (which is mixed
|
|
291
288
|
# into the provider class so they're available as instance methods
|
|
@@ -294,6 +291,34 @@ module Smith
|
|
|
294
291
|
# inlined as class methods with the provider passed in where
|
|
295
292
|
# @config access is needed.
|
|
296
293
|
|
|
294
|
+
# SMITH-AUTHORED, not from PR #770. The Responses API counts cached
|
|
295
|
+
# and cache-written input inside `input_tokens`; RubyLLM reports
|
|
296
|
+
# every other provider's input with those taken out, and Smith totals
|
|
297
|
+
# and prices the cache counts beside it, so the same is done here.
|
|
298
|
+
# Cache counts that are not non-negative Integers are dropped, and
|
|
299
|
+
# counts larger than the reported input are capped to it, so the
|
|
300
|
+
# input total the provider reported is kept.
|
|
301
|
+
def self.response_usage(usage)
|
|
302
|
+
details = usage["input_tokens_details"].is_a?(Hash) ? usage["input_tokens_details"] : {}
|
|
303
|
+
input, cached, written = uncached_input(
|
|
304
|
+
usage["input_tokens"], usage_count(details["cached_tokens"]), usage_count(details["cache_write_tokens"])
|
|
305
|
+
)
|
|
306
|
+
{ input_tokens: input, output_tokens: usage["output_tokens"], cached_tokens: cached,
|
|
307
|
+
cache_creation_tokens: written, thinking_tokens: usage.dig("output_tokens_details", "reasoning_tokens") }
|
|
308
|
+
end
|
|
309
|
+
|
|
310
|
+
def self.uncached_input(input, cached, written)
|
|
311
|
+
return [input, cached, written] unless input.is_a?(Integer)
|
|
312
|
+
|
|
313
|
+
cached &&= [cached, input].min
|
|
314
|
+
written &&= [written, input - cached.to_i].min
|
|
315
|
+
[input - cached.to_i - written.to_i, cached, written]
|
|
316
|
+
end
|
|
317
|
+
|
|
318
|
+
def self.usage_count(value)
|
|
319
|
+
value if value.is_a?(Integer) && value >= 0
|
|
320
|
+
end
|
|
321
|
+
|
|
297
322
|
def self.format_role(role, provider: nil)
|
|
298
323
|
case role
|
|
299
324
|
when :system
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Smith
|
|
4
|
+
module Providers
|
|
5
|
+
# A provider's own parameter for the most tokens a response may hold,
|
|
6
|
+
# merged into the request when an agent retries a cut-off response
|
|
7
|
+
# (`on_truncation retry_with: { max_tokens: n }`). A provider not listed
|
|
8
|
+
# here is not retried.
|
|
9
|
+
module OutputLimit
|
|
10
|
+
PROVIDERS = %w[anthropic openai gemini vertexai bedrock].freeze
|
|
11
|
+
PARAM_PATHS = {
|
|
12
|
+
"anthropic" => [%i[max_tokens]],
|
|
13
|
+
"openai" => [%i[max_output_tokens], %i[max_completion_tokens], %i[max_tokens]],
|
|
14
|
+
"gemini" => [%i[generationConfig maxOutputTokens]],
|
|
15
|
+
"vertexai" => [%i[generationConfig maxOutputTokens]],
|
|
16
|
+
"bedrock" => [%i[inferenceConfig maxTokens]]
|
|
17
|
+
}.freeze
|
|
18
|
+
ANTHROPIC_DEFAULT = 4096
|
|
19
|
+
|
|
20
|
+
module_function
|
|
21
|
+
|
|
22
|
+
def supported?(provider) = PROVIDERS.include?(provider.to_s)
|
|
23
|
+
|
|
24
|
+
# `chat_params` tells the OpenAI paths apart: Smith routes a chat
|
|
25
|
+
# whose params carry `openai_api_mode: :responses` to the Responses
|
|
26
|
+
# API, and a chat that already sets the older `max_tokens` keeps it.
|
|
27
|
+
def params(provider, chat_params, max_tokens)
|
|
28
|
+
case provider.to_s
|
|
29
|
+
when "anthropic" then { max_tokens: }
|
|
30
|
+
when "openai" then { openai_key(chat_params) => max_tokens }
|
|
31
|
+
when "gemini", "vertexai" then { generationConfig: { maxOutputTokens: max_tokens } }
|
|
32
|
+
when "bedrock" then { inferenceConfig: { maxTokens: max_tokens } }
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# The limit a request already had: the agent's own parameter, else for
|
|
37
|
+
# Anthropic the model's maximum as RubyLLM sends it; nil when unknown.
|
|
38
|
+
def current(provider, chat_params, model_reference)
|
|
39
|
+
set = PARAM_PATHS.fetch(provider.to_s, []).filter_map { |path| lookup(chat_params, path) }.first
|
|
40
|
+
return set if set.is_a?(Integer)
|
|
41
|
+
|
|
42
|
+
anthropic_default(model_reference.model_id) if provider.to_s == "anthropic"
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def openai_key(chat_params)
|
|
46
|
+
return :max_output_tokens if responses?(chat_params)
|
|
47
|
+
|
|
48
|
+
lookup(chat_params, %i[max_tokens]) ? :max_tokens : :max_completion_tokens
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def responses?(chat_params) = lookup(chat_params, %i[openai_api_mode]).to_s == "responses"
|
|
52
|
+
|
|
53
|
+
def lookup(params, path)
|
|
54
|
+
path.reduce(params) do |level, key|
|
|
55
|
+
break unless level.is_a?(Hash)
|
|
56
|
+
|
|
57
|
+
level.key?(key) ? level[key] : level[key.to_s]
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def anthropic_default(model_id)
|
|
62
|
+
::RubyLLM.models.find(model_id.to_s, "anthropic").max_tokens || ANTHROPIC_DEFAULT
|
|
63
|
+
rescue StandardError
|
|
64
|
+
ANTHROPIC_DEFAULT
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
end
|