smith-agents 0.8.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +433 -0
- data/README.md +16 -6
- data/docs/CONFIGURATION.md +26 -6
- data/docs/PATTERNS.md +59 -0
- data/docs/PERSISTENCE.md +5 -1
- data/lib/smith/agent/completion_usage_recording.rb +21 -3
- data/lib/smith/agent/fallback_configuration.rb +26 -3
- data/lib/smith/agent/invocation_preparation.rb +24 -0
- data/lib/smith/agent/lifecycle.rb +4 -2
- data/lib/smith/agent/provider_attempt.rb +17 -4
- data/lib/smith/agent/provider_call_timing.rb +129 -0
- data/lib/smith/agent/provider_completion.rb +26 -8
- data/lib/smith/agent/provider_failure_handling.rb +2 -2
- data/lib/smith/agent/provider_usage.rb +6 -0
- data/lib/smith/agent/usage_entry_recording.rb +27 -4
- data/lib/smith/agent/usage_traces.rb +49 -0
- data/lib/smith/agent/usage_tracking.rb +26 -20
- data/lib/smith/agent.rb +9 -1
- data/lib/smith/attribution.rb +115 -0
- data/lib/smith/budget/amount_representation.rb +20 -0
- data/lib/smith/budget/ledger.rb +6 -2
- data/lib/smith/context/state_injection.rb +12 -2
- data/lib/smith/diagnostic_text.rb +10 -1
- data/lib/smith/doctor/checks/durability.rb +16 -0
- data/lib/smith/doctor/checks/persistence_capabilities.rb +13 -7
- data/lib/smith/errors.rb +55 -4
- data/lib/smith/event.rb +10 -2
- data/lib/smith/events/bus.rb +63 -5
- data/lib/smith/events/step_completed.rb +3 -0
- data/lib/smith/events/step_failed.rb +25 -0
- data/lib/smith/events/subscription.rb +8 -0
- data/lib/smith/persistence_adapters/active_record_exact_store.rb +8 -3
- data/lib/smith/persistence_adapters/active_record_store.rb +4 -7
- data/lib/smith/persistence_adapters.rb +10 -0
- data/lib/smith/provider_permanent_failure.rb +44 -0
- data/lib/smith/tool/capture.rb +9 -1
- data/lib/smith/tool.rb +10 -1
- data/lib/smith/trace/logger.rb +1 -0
- data/lib/smith/trace/memory.rb +40 -4
- data/lib/smith/trace/open_telemetry.rb +47 -3
- data/lib/smith/trace.rb +24 -5
- data/lib/smith/version.rb +1 -1
- data/lib/smith/workflow/budget_integration.rb +3 -3
- data/lib/smith/workflow/composite/effects.rb +64 -33
- data/lib/smith/workflow/deterministic_execution.rb +3 -2
- data/lib/smith/workflow/deterministic_step.rb +15 -2
- data/lib/smith/workflow/dsl.rb +40 -6
- data/lib/smith/workflow/durability.rb +1 -1
- data/lib/smith/workflow/evaluator_optimizer.rb +111 -11
- data/lib/smith/workflow/event_integration.rb +48 -1
- data/lib/smith/workflow/execution.rb +10 -0
- data/lib/smith/workflow/failure_reconstructor.rb +18 -2
- data/lib/smith/workflow/failure_record.rb +6 -11
- data/lib/smith/workflow/failure_record_validator.rb +23 -3
- data/lib/smith/workflow/fanout_execution.rb +1 -1
- data/lib/smith/workflow/guardrail_integration.rb +27 -0
- data/lib/smith/workflow/optimization_state.rb +2 -2
- data/lib/smith/workflow/parallel_execution.rb +6 -0
- data/lib/smith/workflow/persistence.rb +55 -6
- data/lib/smith/workflow/prepared_branch_execution.rb +13 -4
- data/lib/smith/workflow/retry_execution.rb +4 -0
- data/lib/smith/workflow/run_result.rb +10 -0
- data/lib/smith/workflow/split_step_persistence/composite_branch_execution.rb +9 -1
- data/lib/smith/workflow/split_step_persistence/state_snapshot.rb +1 -0
- data/lib/smith/workflow/step_completion.rb +13 -0
- data/lib/smith/workflow/step_context.rb +46 -0
- data/lib/smith/workflow/thread_context_snapshot.rb +1 -0
- data/lib/smith/workflow/usage_entry.rb +32 -4
- data/lib/smith/workflow.rb +30 -3
- data/lib/smith.rb +9 -4
- metadata +5 -1
|
@@ -5,9 +5,18 @@ module Smith
|
|
|
5
5
|
module CompletionUsageRecording
|
|
6
6
|
private
|
|
7
7
|
|
|
8
|
-
|
|
8
|
+
# Returns [invocation_cost, fully_priced]. The cost is the sum of the
|
|
9
|
+
# recorded entries' costs (nil when nothing was priced): per-response
|
|
10
|
+
# pricing is what tiered catalogs bill, so the invocation cost comes
|
|
11
|
+
# from this sum, never from pricing the aggregate token totals (which
|
|
12
|
+
# would resolve the wrong tier for multi-response tool loops).
|
|
13
|
+
# fully_priced is true only when every provider response carried usage
|
|
14
|
+
# and every usage priced; a partial sum is still returned (it is what
|
|
15
|
+
# was verifiably billed) but callers must not present it as the
|
|
16
|
+
# complete invocation cost.
|
|
17
|
+
def record_completion_usage(agent_class, completion, attempt_kind, model_reference, attempt_id: nil)
|
|
9
18
|
model_reference = coerce_model_reference(model_reference)
|
|
10
|
-
completion.provider_usages.
|
|
19
|
+
costs = completion.provider_usages.map do |usage|
|
|
11
20
|
result = Workflow::AgentResult.new(
|
|
12
21
|
content: nil,
|
|
13
22
|
input_tokens: usage.input_tokens,
|
|
@@ -17,8 +26,17 @@ module Smith
|
|
|
17
26
|
provider_used: model_reference.provider
|
|
18
27
|
)
|
|
19
28
|
compute_agent_cost(result)
|
|
20
|
-
record_usage(agent_class, result, attempt_kind, model_reference)
|
|
29
|
+
record_usage(agent_class, result, attempt_kind, model_reference, attempt_id:)
|
|
30
|
+
result.cost
|
|
21
31
|
end
|
|
32
|
+
summarize_invocation_costs(costs, completion)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def summarize_invocation_costs(costs, completion)
|
|
36
|
+
priced = costs.compact
|
|
37
|
+
invocation_cost = priced.empty? ? nil : priced.sum
|
|
38
|
+
fully_priced = completion.usage_complete && !costs.empty? && priced.length == costs.length
|
|
39
|
+
[invocation_cost, fully_priced]
|
|
22
40
|
end
|
|
23
41
|
end
|
|
24
42
|
end
|
|
@@ -3,15 +3,38 @@
|
|
|
3
3
|
module Smith
|
|
4
4
|
class Agent
|
|
5
5
|
module FallbackConfiguration
|
|
6
|
-
|
|
6
|
+
attr_reader :fallback_models_block
|
|
7
|
+
|
|
8
|
+
def fallback_models(*models, &block)
|
|
9
|
+
return configure_dynamic_fallback_models(models, block) if block
|
|
7
10
|
return @fallback_models_list if models.empty?
|
|
8
11
|
|
|
9
|
-
|
|
10
|
-
@fallback_models_list =
|
|
12
|
+
@fallback_models_block = nil
|
|
13
|
+
@fallback_models_list = qualified_fallback_references(models)
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
# Evaluated per invocation with the workflow context. The entries pass
|
|
17
|
+
# the static form's validation and de-duplication: O(F) time and space
|
|
18
|
+
# for F returned entries.
|
|
19
|
+
def resolve_fallback_models(context)
|
|
20
|
+
return @fallback_models_list || [].freeze unless fallback_models_block
|
|
21
|
+
|
|
22
|
+
qualified_fallback_references([fallback_models_block.call(context)])
|
|
11
23
|
end
|
|
12
24
|
|
|
13
25
|
private
|
|
14
26
|
|
|
27
|
+
def configure_dynamic_fallback_models(models, block)
|
|
28
|
+
raise ArgumentError, "fallback_models can take model entries OR a block, not both" if models.any?
|
|
29
|
+
|
|
30
|
+
@fallback_models_list = nil
|
|
31
|
+
@fallback_models_block = block
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def qualified_fallback_references(models)
|
|
35
|
+
models.flatten.compact.map { qualified_fallback_reference(_1) }.uniq(&:key).freeze
|
|
36
|
+
end
|
|
37
|
+
|
|
15
38
|
def qualified_fallback_reference(model)
|
|
16
39
|
reference = ModelReference.coerce(model)
|
|
17
40
|
return reference if reference.provider
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "json"
|
|
4
|
+
|
|
3
5
|
module Smith
|
|
4
6
|
class Agent
|
|
5
7
|
module InvocationPreparation
|
|
@@ -82,9 +84,31 @@ module Smith
|
|
|
82
84
|
else
|
|
83
85
|
message
|
|
84
86
|
end
|
|
87
|
+
attributes = provider_safe_message(attributes) if attributes.is_a?(Hash)
|
|
85
88
|
chat.add_message(attributes)
|
|
86
89
|
end
|
|
87
90
|
|
|
91
|
+
# A structured agent output recorded as a session message (StepCompletion#append_accepted_output)
|
|
92
|
+
# carries a Hash/Array content. RubyLLM's Message#normalize_content treats a Hash content as
|
|
93
|
+
# { text:, ...attachments } and opens each value as a file, so replaying a prior structured
|
|
94
|
+
# output to the next agent in a workflow session raises Errno::ENOENT. Serialize non-string
|
|
95
|
+
# content to JSON so the provider sees the prior output as text; the session store keeps the raw
|
|
96
|
+
# value (last_output stays structured), only this provider-facing copy is serialized. Genuine
|
|
97
|
+
# multimodal attachments are supplied through the provider's own with: mechanism, never as a bare
|
|
98
|
+
# Hash message content in a workflow session.
|
|
99
|
+
def provider_safe_message(attributes)
|
|
100
|
+
content = attributes[:content]
|
|
101
|
+
return attributes if content.nil? || content.is_a?(String)
|
|
102
|
+
|
|
103
|
+
attributes.merge(content: json_message_content(content))
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def json_message_content(content)
|
|
107
|
+
JSON.generate(content)
|
|
108
|
+
rescue StandardError
|
|
109
|
+
content.to_s
|
|
110
|
+
end
|
|
111
|
+
|
|
88
112
|
def message_role(message)
|
|
89
113
|
message_attribute(message, :role)&.to_sym
|
|
90
114
|
end
|
|
@@ -20,8 +20,10 @@ module Smith
|
|
|
20
20
|
|
|
21
21
|
def invoke_agent(agent_class, prepared_input, output_schema: agent_class.output_schema)
|
|
22
22
|
check_deadline!
|
|
23
|
-
|
|
24
|
-
snapshot_and_finalize(
|
|
23
|
+
attempt = complete_with_provider(agent_class, prepared_input, output_schema:)
|
|
24
|
+
snapshot_and_finalize(
|
|
25
|
+
agent_class, attempt.completion, attempt.model_reference, attempt_id: attempt.attempt_id
|
|
26
|
+
)
|
|
25
27
|
end
|
|
26
28
|
end
|
|
27
29
|
end
|
|
@@ -4,6 +4,7 @@ require "dry-struct"
|
|
|
4
4
|
|
|
5
5
|
require_relative "../types"
|
|
6
6
|
require_relative "model_reference"
|
|
7
|
+
require_relative "provider_usage"
|
|
7
8
|
|
|
8
9
|
module Smith
|
|
9
10
|
class Agent
|
|
@@ -11,13 +12,25 @@ module Smith
|
|
|
11
12
|
attribute :completion, Types::Any.optional
|
|
12
13
|
attribute :model_reference, Types.Instance(ModelReference)
|
|
13
14
|
attribute :error, Types.Instance(StandardError).optional
|
|
15
|
+
# One attempt = one chat completion (including any provider tool loop).
|
|
16
|
+
# `attempt_id` joins the attempt's usage entries to its single measured
|
|
17
|
+
# duration; `duration_ms` is nil when the attempt failed before the
|
|
18
|
+
# completion call started.
|
|
19
|
+
attribute :attempt_id, Types::String.optional.default(nil)
|
|
20
|
+
attribute :duration_ms, Types::Integer.optional.default(nil)
|
|
21
|
+
# The candidate's position in the model chain, the registered name of
|
|
22
|
+
# the agent that made the attempt, and the provider-reported usage its
|
|
23
|
+
# usage entries record (nil when the provider reported none).
|
|
24
|
+
attribute :attempt_index, Types::Integer.optional.default(nil)
|
|
25
|
+
attribute :agent_name, (Types::Symbol | Types::String).optional.default(nil)
|
|
26
|
+
attribute :usage, Types.Instance(ProviderUsage).optional.default(nil)
|
|
14
27
|
|
|
15
|
-
def self.success(completion:, model_reference
|
|
16
|
-
new(completion:, model_reference:, error: nil)
|
|
28
|
+
def self.success(completion:, model_reference:, **facts)
|
|
29
|
+
new(completion:, model_reference:, error: nil, **facts)
|
|
17
30
|
end
|
|
18
31
|
|
|
19
|
-
def self.failure(error:, model_reference
|
|
20
|
-
new(completion: nil, model_reference:, error
|
|
32
|
+
def self.failure(error:, model_reference:, **facts)
|
|
33
|
+
new(completion: nil, model_reference:, error:, **facts)
|
|
21
34
|
end
|
|
22
35
|
|
|
23
36
|
def success?
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../diagnostic_text"
|
|
4
|
+
|
|
5
|
+
module Smith
|
|
6
|
+
class Agent
|
|
7
|
+
# Monotonic timing and trace emission for one provider attempt: the
|
|
8
|
+
# attempt's single measured duration wraps the whole chat completion
|
|
9
|
+
# (including any provider-side tool loop). Per-network-round timing
|
|
10
|
+
# belongs to the host's RubyLLM notification subscriptions, not Smith.
|
|
11
|
+
module ProviderCallTiming
|
|
12
|
+
EXCEPTION_CAUSE = Exception.instance_method(:cause)
|
|
13
|
+
private_constant :EXCEPTION_CAUSE
|
|
14
|
+
|
|
15
|
+
class Timer
|
|
16
|
+
def self.start
|
|
17
|
+
new
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def initialize
|
|
21
|
+
@started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
22
|
+
@stopped_at = nil
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# Idempotent: the first stop wins, so a rescue-path stop after a
|
|
26
|
+
# success-path stop cannot stretch the measurement.
|
|
27
|
+
def stop
|
|
28
|
+
@stopped_at ||= Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
29
|
+
self
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def elapsed_ms
|
|
33
|
+
ending = @stopped_at || Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
34
|
+
((ending - @started_at) * 1000).round
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
private
|
|
39
|
+
|
|
40
|
+
# One `:provider_call` trace per attempt: success, provider failure, or
|
|
41
|
+
# aborted (a non-provider error that re-raises). Usage entries from the
|
|
42
|
+
# same attempt share its attempt_id; join there instead of summing
|
|
43
|
+
# durations across entries. duration_ms is present only when the timed
|
|
44
|
+
# provider call actually started: a failure before dispatch (model
|
|
45
|
+
# resolution, chat construction) emits its attempt without a duration.
|
|
46
|
+
# input_tokens and output_tokens total the attempt's usage entries, so
|
|
47
|
+
# a host can record the whole attempt when it ends.
|
|
48
|
+
def record_provider_call_trace(attempt, aborted: false)
|
|
49
|
+
Smith::Trace.record(
|
|
50
|
+
type: :provider_call,
|
|
51
|
+
data: {
|
|
52
|
+
model: attempt.model_reference.model_id,
|
|
53
|
+
provider: attempt.model_reference.provider,
|
|
54
|
+
duration_ms: attempt.duration_ms,
|
|
55
|
+
attempt_id: attempt.attempt_id,
|
|
56
|
+
attempt_index: attempt.attempt_index,
|
|
57
|
+
outcome: provider_call_outcome(attempt, aborted),
|
|
58
|
+
**provider_call_error_classes(attempt.error),
|
|
59
|
+
agent_name: attempt.agent_name,
|
|
60
|
+
**provider_call_usage(attempt.usage)
|
|
61
|
+
}.compact
|
|
62
|
+
)
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Token counts are metadata, never content, so the content policy
|
|
66
|
+
# leaves them in place.
|
|
67
|
+
def provider_call_usage(usage)
|
|
68
|
+
return {} unless usage
|
|
69
|
+
|
|
70
|
+
{ input_tokens: usage.input_tokens, output_tokens: usage.output_tokens }
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def provider_call_outcome(attempt, aborted)
|
|
74
|
+
return :aborted if aborted
|
|
75
|
+
return :success if attempt.success?
|
|
76
|
+
|
|
77
|
+
:failure
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# Class names only, never a message: the same bounded identifier the
|
|
81
|
+
# failed :transition trace carries, for the error and its direct cause.
|
|
82
|
+
def provider_call_error_classes(error)
|
|
83
|
+
return {} unless error
|
|
84
|
+
|
|
85
|
+
cause = provider_call_error_cause(error)
|
|
86
|
+
{
|
|
87
|
+
error_class: DiagnosticText.error_class_name(error),
|
|
88
|
+
error_cause_class: cause && DiagnosticText.error_class_name(cause)
|
|
89
|
+
}
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# A transport error wraps what actually happened (never connected, or
|
|
93
|
+
# lost the connection after sending); Faraday keeps it as
|
|
94
|
+
# wrapped_exception when the error was not raised inside a rescue.
|
|
95
|
+
def provider_call_error_cause(error)
|
|
96
|
+
cause = EXCEPTION_CAUSE.bind_call(error)
|
|
97
|
+
cause ||= error.wrapped_exception if error.respond_to?(:wrapped_exception)
|
|
98
|
+
cause if cause.is_a?(Exception)
|
|
99
|
+
rescue StandardError
|
|
100
|
+
nil
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def completed_provider_attempt(completion, observed_reference, timer, facts)
|
|
104
|
+
attempt = ProviderAttempt.success(
|
|
105
|
+
completion:, model_reference: observed_reference, duration_ms: timer.elapsed_ms,
|
|
106
|
+
usage: ProviderUsage.sum(completion.provider_usages), **facts
|
|
107
|
+
)
|
|
108
|
+
record_provider_call_trace(attempt)
|
|
109
|
+
attempt
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
# Aborted (non-provider) attempts emit a :provider_call too, so the
|
|
113
|
+
# prefix-accounted usage entries stamped with this attempt_id always
|
|
114
|
+
# have their join target; the error then propagates unchanged. A
|
|
115
|
+
# provider failure's own reported usage is accounted beside the
|
|
116
|
+
# completed prefix, so its trace totals both.
|
|
117
|
+
def failed_provider_attempt(error, observed_reference, timer, facts, prefix_usages)
|
|
118
|
+
aborted = !provider_failure?(error)
|
|
119
|
+
usages = aborted ? prefix_usages : [*prefix_usages, ProviderUsage.from_message(error)].compact
|
|
120
|
+
attempt = ProviderAttempt.failure(
|
|
121
|
+
error:, model_reference: observed_reference, duration_ms: timer&.elapsed_ms,
|
|
122
|
+
usage: ProviderUsage.sum(usages), **facts
|
|
123
|
+
)
|
|
124
|
+
record_provider_call_trace(attempt, aborted:)
|
|
125
|
+
attempt
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
end
|
|
129
|
+
end
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
require_relative "provider_failure_handling"
|
|
4
4
|
require_relative "invocation_preparation"
|
|
5
5
|
require_relative "provider_attempt"
|
|
6
|
+
require_relative "provider_call_timing"
|
|
6
7
|
require_relative "provider_candidate_sequence"
|
|
7
8
|
|
|
8
9
|
module Smith
|
|
@@ -10,6 +11,7 @@ module Smith
|
|
|
10
11
|
module ProviderCompletion
|
|
11
12
|
include ProviderFailureHandling
|
|
12
13
|
include InvocationPreparation
|
|
14
|
+
include ProviderCallTiming
|
|
13
15
|
|
|
14
16
|
private
|
|
15
17
|
|
|
@@ -17,13 +19,14 @@ module Smith
|
|
|
17
19
|
candidates = ProviderCandidateSequence.new(build_model_chain(agent_class))
|
|
18
20
|
candidates.each do |model_reference, index|
|
|
19
21
|
check_deadline! if index.positive?
|
|
20
|
-
attempt = attempt_model(agent_class, prepared_input, model_reference, output_schema:)
|
|
21
|
-
return
|
|
22
|
+
attempt = attempt_model(agent_class, prepared_input, model_reference, output_schema:, attempt_index: index)
|
|
23
|
+
return attempt if attempt.success?
|
|
22
24
|
|
|
23
25
|
candidates.suppress(account_failed_provider(attempt, model_reference))
|
|
24
26
|
handle_provider_failure!(
|
|
25
27
|
attempt.error, attempt.model_reference, agent_class,
|
|
26
|
-
fallback_available: candidates.fallback_available
|
|
28
|
+
fallback_available: candidates.fallback_available?,
|
|
29
|
+
attempt_id: attempt.attempt_id
|
|
27
30
|
)
|
|
28
31
|
end
|
|
29
32
|
|
|
@@ -65,6 +68,8 @@ module Smith
|
|
|
65
68
|
end
|
|
66
69
|
|
|
67
70
|
def fallback_model_references(agent_class)
|
|
71
|
+
return agent_class.resolve_fallback_models(@context || {}) if agent_class.fallback_models_block
|
|
72
|
+
|
|
68
73
|
Array(agent_class.fallback_models).map { |model| ModelReference.coerce(model) }
|
|
69
74
|
end
|
|
70
75
|
|
|
@@ -79,20 +84,33 @@ module Smith
|
|
|
79
84
|
raise Smith::AgentError, "invalid model block result for #{agent_class}: #{e.message}"
|
|
80
85
|
end
|
|
81
86
|
|
|
82
|
-
|
|
87
|
+
# rubocop:disable Metrics/AbcSize -- one provider attempt is a single
|
|
88
|
+
# cohesive lifecycle (identity, prepared chat, observed model, timed
|
|
89
|
+
# completion, prefix accounting on failure); splitting it would scatter
|
|
90
|
+
# the rescue-path accounting away from what it accounts for.
|
|
91
|
+
def attempt_model(agent_class, prepared_input, model_reference, output_schema:, attempt_index:)
|
|
92
|
+
facts = { attempt_id: SecureRandom.uuid, attempt_index:, agent_name: agent_class.register_as }
|
|
83
93
|
chat = prepared_attempt_chat(agent_class, prepared_input, model_reference, output_schema:)
|
|
84
94
|
message_count = chat_message_count(chat)
|
|
85
95
|
observed_reference = observed_model_reference(chat, fallback: model_reference)
|
|
86
|
-
|
|
96
|
+
timer = ProviderCallTiming::Timer.start
|
|
97
|
+
response = chat.complete
|
|
98
|
+
timer.stop
|
|
99
|
+
completion = Completion.from_messages(response: response, messages: new_chat_messages(chat, message_count))
|
|
87
100
|
|
|
88
|
-
|
|
101
|
+
completed_provider_attempt(completion, observed_reference, timer, facts)
|
|
89
102
|
rescue StandardError => e
|
|
103
|
+
timer&.stop
|
|
90
104
|
observed_reference ||= observed_model_reference(chat, fallback: model_reference)
|
|
91
|
-
|
|
105
|
+
prefix_usages = account_completed_prefix(
|
|
106
|
+
agent_class, observed_reference, new_chat_messages(chat, message_count), attempt_id: facts[:attempt_id]
|
|
107
|
+
)
|
|
108
|
+
attempt = failed_provider_attempt(e, observed_reference, timer, facts, prefix_usages)
|
|
92
109
|
raise unless provider_failure?(e)
|
|
93
110
|
|
|
94
|
-
|
|
111
|
+
attempt
|
|
95
112
|
end
|
|
113
|
+
# rubocop:enable Metrics/AbcSize
|
|
96
114
|
|
|
97
115
|
def observed_model_reference(chat, fallback:)
|
|
98
116
|
model = observable_model(chat)
|
|
@@ -21,8 +21,8 @@ module Smith
|
|
|
21
21
|
|
|
22
22
|
private
|
|
23
23
|
|
|
24
|
-
def handle_provider_failure!(error, model_reference, agent_class, fallback_available:)
|
|
25
|
-
account_failed_attempt(error, model_reference, agent_class)
|
|
24
|
+
def handle_provider_failure!(error, model_reference, agent_class, fallback_available:, attempt_id: nil)
|
|
25
|
+
account_failed_attempt(error, model_reference, agent_class, attempt_id:)
|
|
26
26
|
if completed_tool_calls?
|
|
27
27
|
raise Smith::ToolOutcomeUncertain.new(
|
|
28
28
|
"provider failed after tool execution began; retry or fallback could replay an uncertain outcome"
|
|
@@ -20,6 +20,12 @@ module Smith
|
|
|
20
20
|
new(input_tokens:, output_tokens:)
|
|
21
21
|
end
|
|
22
22
|
|
|
23
|
+
def self.sum(usages)
|
|
24
|
+
return if usages.empty?
|
|
25
|
+
|
|
26
|
+
new(input_tokens: usages.sum(&:input_tokens), output_tokens: usages.sum(&:output_tokens))
|
|
27
|
+
end
|
|
28
|
+
|
|
23
29
|
def total_tokens
|
|
24
30
|
input_tokens + output_tokens
|
|
25
31
|
end
|
|
@@ -5,15 +5,22 @@ module Smith
|
|
|
5
5
|
module UsageEntryRecording
|
|
6
6
|
private
|
|
7
7
|
|
|
8
|
-
def record_usage(agent_class, agent_result, attempt_kind, model_reference)
|
|
8
|
+
def record_usage(agent_class, agent_result, attempt_kind, model_reference, attempt_id: nil)
|
|
9
9
|
return unless agent_result.usage_known?
|
|
10
10
|
|
|
11
11
|
model_reference = coerce_model_reference(model_reference)
|
|
12
|
-
entry = build_usage_entry(agent_class, agent_result, attempt_kind, model_reference)
|
|
12
|
+
entry = build_usage_entry(agent_class, agent_result, attempt_kind, model_reference, attempt_id:)
|
|
13
13
|
accumulate_usage(agent_result, entry)
|
|
14
14
|
end
|
|
15
15
|
|
|
16
|
-
|
|
16
|
+
# Attribution (transition, branch key, optimizer round) is read from the
|
|
17
|
+
# ambient context of the recording thread, which is the thread that ran
|
|
18
|
+
# the provider call: a fan-out branch records under its own overlay.
|
|
19
|
+
# transition and branch_key are recorded as Symbols even when a host
|
|
20
|
+
# seeded Strings through Attribution.with, because from_h symbolizes
|
|
21
|
+
# them on restore: recording the same way keeps a restored entry equal
|
|
22
|
+
# to the recorded one.
|
|
23
|
+
def build_usage_entry(agent_class, agent_result, attempt_kind, model_reference, attempt_id: nil)
|
|
17
24
|
Workflow::UsageEntry.new(
|
|
18
25
|
usage_id: SecureRandom.uuid,
|
|
19
26
|
agent_name: agent_class.register_as,
|
|
@@ -23,10 +30,26 @@ module Smith
|
|
|
23
30
|
output_tokens: agent_result.output_tokens,
|
|
24
31
|
cost: agent_result.cost,
|
|
25
32
|
attempt_kind: attempt_kind,
|
|
26
|
-
recorded_at: Time.now.utc.iso8601
|
|
33
|
+
recorded_at: Time.now.utc.iso8601(6),
|
|
34
|
+
attempt_id: attempt_id,
|
|
35
|
+
**ambient_attribution_fields
|
|
27
36
|
)
|
|
28
37
|
end
|
|
29
38
|
|
|
39
|
+
def ambient_attribution_fields
|
|
40
|
+
attribution = Smith::Attribution.ambient
|
|
41
|
+
{
|
|
42
|
+
transition: symbolized_attribution(attribution.transition),
|
|
43
|
+
branch_key: symbolized_attribution(attribution.branch_key),
|
|
44
|
+
round: attribution.round,
|
|
45
|
+
workflow: attribution.workflow
|
|
46
|
+
}
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def symbolized_attribution(value)
|
|
50
|
+
value.is_a?(String) ? value.to_sym : value
|
|
51
|
+
end
|
|
52
|
+
|
|
30
53
|
def accumulate_usage(agent_result, entry)
|
|
31
54
|
@usage_mutex.synchronize do
|
|
32
55
|
@total_tokens = (@total_tokens || 0) + agent_result.input_tokens + agent_result.output_tokens
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Smith
|
|
4
|
+
class Agent
|
|
5
|
+
# Trace emission for a completed agent invocation's usage facts, kept
|
|
6
|
+
# apart from the accounting itself (UsageTracking) so recording rows and
|
|
7
|
+
# emitting observability stay separate concerns.
|
|
8
|
+
module UsageTraces
|
|
9
|
+
private
|
|
10
|
+
|
|
11
|
+
def emit_token_usage(agent_result)
|
|
12
|
+
return unless agent_result.usage_known?
|
|
13
|
+
|
|
14
|
+
Smith::Trace.record(
|
|
15
|
+
type: :token_usage,
|
|
16
|
+
data: {
|
|
17
|
+
input_tokens: agent_result.input_tokens,
|
|
18
|
+
output_tokens: agent_result.output_tokens,
|
|
19
|
+
model: agent_result.model_used,
|
|
20
|
+
provider: agent_result.provider_used
|
|
21
|
+
}.compact
|
|
22
|
+
)
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# One :cost trace per completed agent invocation. The cost is the sum
|
|
26
|
+
# of the invocation's per-response entry costs (what tiered catalogs
|
|
27
|
+
# actually bill), never the aggregate token totals priced as one call.
|
|
28
|
+
# Token counts remain the invocation aggregates. Billed failed and
|
|
29
|
+
# partial attempts appear only in usage entries, so summing :cost
|
|
30
|
+
# traces is not a spend total. Unpriced usage emits nothing, and the
|
|
31
|
+
# caller gates out partially metered or partially priced invocations
|
|
32
|
+
# so an incomplete figure is never presented as the invocation cost.
|
|
33
|
+
def emit_cost_trace(agent_result, invocation_cost)
|
|
34
|
+
return if invocation_cost.nil?
|
|
35
|
+
|
|
36
|
+
Smith::Trace.record(
|
|
37
|
+
type: :cost,
|
|
38
|
+
data: {
|
|
39
|
+
cost: invocation_cost,
|
|
40
|
+
model: agent_result.model_used,
|
|
41
|
+
provider: agent_result.provider_used,
|
|
42
|
+
input_tokens: agent_result.input_tokens,
|
|
43
|
+
output_tokens: agent_result.output_tokens
|
|
44
|
+
}.compact
|
|
45
|
+
)
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
end
|
|
@@ -2,31 +2,36 @@
|
|
|
2
2
|
|
|
3
3
|
require_relative "completion_usage_recording"
|
|
4
4
|
require_relative "usage_entry_recording"
|
|
5
|
+
require_relative "usage_traces"
|
|
5
6
|
|
|
6
7
|
module Smith
|
|
7
8
|
class Agent
|
|
8
9
|
module UsageTracking
|
|
9
10
|
include CompletionUsageRecording
|
|
10
11
|
include UsageEntryRecording
|
|
12
|
+
include UsageTraces
|
|
11
13
|
|
|
12
14
|
private
|
|
13
15
|
|
|
14
|
-
def account_failed_attempt(error, model_reference, agent_class)
|
|
16
|
+
def account_failed_attempt(error, model_reference, agent_class, attempt_id: nil)
|
|
15
17
|
return unless error.respond_to?(:input_tokens) && error.respond_to?(:output_tokens)
|
|
16
18
|
|
|
17
19
|
input = error.input_tokens
|
|
18
20
|
output = error.output_tokens
|
|
19
21
|
return unless input.is_a?(Integer) && output.is_a?(Integer)
|
|
20
22
|
|
|
21
|
-
record_failed_usage(agent_class, model_reference, input, output, :
|
|
23
|
+
record_failed_usage(agent_class, model_reference, input, output, attempt_id:)
|
|
22
24
|
end
|
|
23
25
|
|
|
24
|
-
|
|
26
|
+
# Returns the provider usages it recorded, so the attempt's trace can
|
|
27
|
+
# carry the same totals its usage entries hold.
|
|
28
|
+
def account_completed_prefix(agent_class, model_reference, messages, attempt_id: nil)
|
|
25
29
|
completion = Completion.from_messages(response: nil, messages: messages)
|
|
26
|
-
record_completion_usage(agent_class, completion, :partial_attempt, model_reference)
|
|
30
|
+
record_completion_usage(agent_class, completion, :partial_attempt, model_reference, attempt_id:)
|
|
31
|
+
completion.provider_usages
|
|
27
32
|
end
|
|
28
33
|
|
|
29
|
-
def record_failed_usage(agent_class, model_reference, input_tokens, output_tokens,
|
|
34
|
+
def record_failed_usage(agent_class, model_reference, input_tokens, output_tokens, attempt_id: nil)
|
|
30
35
|
model_reference = coerce_model_reference(model_reference)
|
|
31
36
|
cost = Smith::Pricing.compute_cost(
|
|
32
37
|
model: model_reference.model_id,
|
|
@@ -44,10 +49,10 @@ module Smith
|
|
|
44
49
|
)
|
|
45
50
|
Thread.current[:smith_failed_agent_results] ||= []
|
|
46
51
|
Thread.current[:smith_failed_agent_results] << agent_result
|
|
47
|
-
record_usage(agent_class, agent_result,
|
|
52
|
+
record_usage(agent_class, agent_result, :failed_attempt, model_reference, attempt_id:)
|
|
48
53
|
end
|
|
49
54
|
|
|
50
|
-
def snapshot_and_finalize(agent_class, completion, model_reference)
|
|
55
|
+
def snapshot_and_finalize(agent_class, completion, model_reference, attempt_id: nil)
|
|
51
56
|
model_reference = coerce_model_reference(model_reference)
|
|
52
57
|
agent_result = Workflow::AgentResult.new(
|
|
53
58
|
content: completion.content,
|
|
@@ -59,8 +64,7 @@ module Smith
|
|
|
59
64
|
)
|
|
60
65
|
Thread.current[:smith_last_agent_result] = agent_result
|
|
61
66
|
emit_token_usage(agent_result)
|
|
62
|
-
|
|
63
|
-
record_completion_usage(agent_class, completion, :completed_attempt, model_reference)
|
|
67
|
+
account_completion!(agent_class, completion, model_reference, agent_result, attempt_id)
|
|
64
68
|
|
|
65
69
|
agent_result.content = run_after_completion(agent_class, agent_result.content, @context)
|
|
66
70
|
raise_blank_output!(agent_class, agent_result)
|
|
@@ -83,18 +87,20 @@ module Smith
|
|
|
83
87
|
false
|
|
84
88
|
end
|
|
85
89
|
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
90
|
+
# Records one usage entry per provider response and settles the
|
|
91
|
+
# invocation's cost from the recorded per-response sum: that sum is
|
|
92
|
+
# what tiered catalogs bill, so it becomes agent_result.cost (which
|
|
93
|
+
# budget settlement and result surfaces read) and the recorded entries,
|
|
94
|
+
# the budget ledger, and the :cost trace all agree. The trace emits
|
|
95
|
+
# only for fully metered, fully priced invocations, so a partial
|
|
96
|
+
# figure is never presented as the invocation cost; the partial sum
|
|
97
|
+
# still settles the budget because it is what was verifiably billed.
|
|
98
|
+
def account_completion!(agent_class, completion, model_reference, agent_result, attempt_id)
|
|
99
|
+
invocation_cost, fully_priced = record_completion_usage(
|
|
100
|
+
agent_class, completion, :completed_attempt, model_reference, attempt_id:
|
|
97
101
|
)
|
|
102
|
+
agent_result.cost = invocation_cost
|
|
103
|
+
emit_cost_trace(agent_result, invocation_cost) if fully_priced
|
|
98
104
|
end
|
|
99
105
|
|
|
100
106
|
def compute_agent_cost(agent_result)
|
data/lib/smith/agent.rb
CHANGED
|
@@ -11,9 +11,10 @@ module Smith
|
|
|
11
11
|
EXECUTION_IDENTITY_UNSET = Object.new.freeze
|
|
12
12
|
TOOL_BUDGET_EXHAUSTION_UNSET = Object.new.freeze
|
|
13
13
|
TOOL_BUDGET_EXHAUSTION_POLICIES = %i[raise complete].freeze
|
|
14
|
+
BUDGET_KEYS = %i[token_limit cost wall_clock tool_calls total_tokens total_cost].freeze
|
|
14
15
|
|
|
15
16
|
private_constant :EXECUTION_IDENTITY_UNSET
|
|
16
|
-
private_constant :TOOL_BUDGET_EXHAUSTION_UNSET, :TOOL_BUDGET_EXHAUSTION_POLICIES
|
|
17
|
+
private_constant :TOOL_BUDGET_EXHAUSTION_UNSET, :TOOL_BUDGET_EXHAUSTION_POLICIES, :BUDGET_KEYS
|
|
17
18
|
|
|
18
19
|
# Reserved input names auto-injected by the normalizer into
|
|
19
20
|
# runtime_context. User-side `inputs :name` calls cannot redeclare
|
|
@@ -30,6 +31,7 @@ module Smith
|
|
|
30
31
|
subclass.instance_variable_set(:@output_schema_class, @output_schema_class)
|
|
31
32
|
subclass.instance_variable_set(:@data_volume, @data_volume)
|
|
32
33
|
subclass.instance_variable_set(:@fallback_models_list, @fallback_models_list&.dup&.freeze)
|
|
34
|
+
subclass.instance_variable_set(:@fallback_models_block, @fallback_models_block)
|
|
33
35
|
subclass.instance_variable_set(:@model_block, @model_block)
|
|
34
36
|
subclass.instance_variable_set(:@tool_budget_exhaustion, @tool_budget_exhaustion)
|
|
35
37
|
subclass.instance_variable_set(:@execution_identity, nil)
|
|
@@ -48,6 +50,12 @@ module Smith
|
|
|
48
50
|
def budget(**opts)
|
|
49
51
|
return @budget_config if opts.empty?
|
|
50
52
|
|
|
53
|
+
unknown = opts.keys - BUDGET_KEYS
|
|
54
|
+
unless unknown.empty?
|
|
55
|
+
raise ArgumentError, "agent budget does not accept #{unknown.map(&:inspect).join(", ")}; " \
|
|
56
|
+
"accepted keys are #{BUDGET_KEYS.map(&:inspect).join(", ")}"
|
|
57
|
+
end
|
|
58
|
+
|
|
51
59
|
@budget_config = opts
|
|
52
60
|
end
|
|
53
61
|
|