active_harness 0.2.30 → 0.2.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/active_harness/agent/cost.rb +13 -15
- data/lib/active_harness/agent/prompt.rb +7 -1
- data/lib/active_harness/agent.rb +42 -16
- data/lib/active_harness/result.rb +51 -11
- data/lib/active_harness.rb +1 -1
- data/lib/generators/active_harness/install/templates/controllers/ai_controller.rb +2 -2
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: d6d0a8c421353c670b4f0ba80cab409a05c97a1505382fd35a0822b39d66691a
|
|
4
|
+
data.tar.gz: a312b8db19958a621d26aee139d9bbe284f269690991b02008a2e6716fb08d7b
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: bc0be024611a21164721a379741f48cf2547ebb4b5bd64f15bb3cdf5bae41e8bdbbe7f0d576c41ddd24a5167488bd2db2729cb659d699f3aceffda2791664998
|
|
7
|
+
data.tar.gz: 329d8648ae84605f6ecbd2ee9112a9a0abf39a4d3f4a2fb98ee6814b79e7bd8167a1fd807e1a994dc5d299475b02b603e48d65a4270fa01b9082ca5ecf499900
|
|
@@ -2,25 +2,23 @@ module ActiveHarness
|
|
|
2
2
|
class Agent
|
|
3
3
|
private
|
|
4
4
|
|
|
5
|
-
#
|
|
6
|
-
#
|
|
5
|
+
# Builds a CostBreakdown for a single request from token usage and
|
|
6
|
+
# pricing data from ActiveHarness::Costs.
|
|
7
7
|
#
|
|
8
|
-
# Returns
|
|
8
|
+
# Returns CostBreakdown (input, output, total in USD),
|
|
9
9
|
# or nil if usage is absent or the model is not found in the pricing registry.
|
|
10
|
-
def calculate_cost(
|
|
11
|
-
return nil
|
|
10
|
+
def calculate_cost(pricing, tokens)
|
|
11
|
+
return nil unless pricing && tokens
|
|
12
|
+
return nil unless pricing.input_per_million && pricing.output_per_million
|
|
12
13
|
|
|
13
|
-
|
|
14
|
-
|
|
14
|
+
input_cost = (tokens.input.to_f / 1_000_000) * pricing.input_per_million
|
|
15
|
+
output_cost = (tokens.output.to_f / 1_000_000) * pricing.output_per_million
|
|
15
16
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
output_cost: output_cost.round(8),
|
|
22
|
-
total_cost: (input_cost + output_cost).round(8)
|
|
23
|
-
}
|
|
17
|
+
CostBreakdown.new(
|
|
18
|
+
input: input_cost.round(8),
|
|
19
|
+
output: output_cost.round(8),
|
|
20
|
+
total: (input_cost + output_cost).round(8)
|
|
21
|
+
)
|
|
24
22
|
rescue StandardError
|
|
25
23
|
nil
|
|
26
24
|
end
|
|
@@ -57,7 +57,13 @@ module ActiveHarness
|
|
|
57
57
|
obj.instance_variable_set(:@params, @params)
|
|
58
58
|
obj.instance_variable_set(:@config, @config)
|
|
59
59
|
obj.instance_variable_set(:@memory, @memory)
|
|
60
|
-
obj.instance_variable_set(:@context_window,
|
|
60
|
+
obj.instance_variable_set(:@context_window, context_window_for_prompt)
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def context_window_for_prompt
|
|
64
|
+
Costs.find(model_list.to_a.first&.dig(:model).to_s)&.context_window
|
|
65
|
+
rescue StandardError
|
|
66
|
+
nil
|
|
61
67
|
end
|
|
62
68
|
end
|
|
63
69
|
end
|
data/lib/active_harness/agent.rb
CHANGED
|
@@ -54,7 +54,6 @@ module ActiveHarness
|
|
|
54
54
|
:params,
|
|
55
55
|
:memory
|
|
56
56
|
attr_reader :result,
|
|
57
|
-
:context_window,
|
|
58
57
|
:token_stream,
|
|
59
58
|
:event_stream
|
|
60
59
|
|
|
@@ -77,9 +76,8 @@ module ActiveHarness
|
|
|
77
76
|
@context = context
|
|
78
77
|
@params = params
|
|
79
78
|
@memory = memory
|
|
80
|
-
@models_override
|
|
81
|
-
@
|
|
82
|
-
@token_stream = streams[:token]
|
|
79
|
+
@models_override = Array(models) if models
|
|
80
|
+
@token_stream = streams[:token]
|
|
83
81
|
@event_stream = streams[:agent]
|
|
84
82
|
fire(:setup)
|
|
85
83
|
end
|
|
@@ -146,31 +144,59 @@ module ActiveHarness
|
|
|
146
144
|
end
|
|
147
145
|
|
|
148
146
|
def build_result(response, entry, attempts, elapsed)
|
|
149
|
-
raw
|
|
150
|
-
processed
|
|
151
|
-
|
|
152
|
-
|
|
147
|
+
raw = response[:content]
|
|
148
|
+
processed = parse_output(raw)
|
|
149
|
+
raw_usage = response[:usage]
|
|
150
|
+
model_cost = lookup_model_cost(entry)
|
|
153
151
|
|
|
154
152
|
Result.new(
|
|
155
153
|
input: @input,
|
|
156
154
|
output: raw,
|
|
157
155
|
processed: processed,
|
|
158
156
|
system_prompt: @system_prompt,
|
|
159
|
-
|
|
160
|
-
model: entry[:model],
|
|
161
|
-
temperature: entry[:temperature],
|
|
157
|
+
model: build_model_info(entry, model_cost),
|
|
162
158
|
model_list: model_list,
|
|
163
159
|
attempts: attempts,
|
|
164
160
|
execution_time: elapsed,
|
|
165
|
-
usage:
|
|
166
|
-
|
|
167
|
-
|
|
161
|
+
usage: build_usage(raw_usage, model_cost)
|
|
162
|
+
)
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
def build_model_info(entry, model_cost)
|
|
166
|
+
pricing = if model_cost&.input_per_million && model_cost&.output_per_million
|
|
167
|
+
ModelPricing.new(
|
|
168
|
+
input: (model_cost.input_per_million / 1_000_000.0).round(10),
|
|
169
|
+
output: (model_cost.output_per_million / 1_000_000.0).round(10)
|
|
170
|
+
)
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
ModelInfo.new(
|
|
174
|
+
name: entry[:model],
|
|
175
|
+
provider: entry[:provider],
|
|
176
|
+
temperature: entry[:temperature],
|
|
177
|
+
context_window: model_cost&.context_window,
|
|
178
|
+
pricing: pricing
|
|
179
|
+
)
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
def build_usage(raw_usage, model_cost)
|
|
183
|
+
return nil if raw_usage.nil?
|
|
184
|
+
|
|
185
|
+
tokens = TokenCounts.new(
|
|
186
|
+
input: raw_usage[:input_tokens],
|
|
187
|
+
output: raw_usage[:output_tokens],
|
|
188
|
+
total: raw_usage[:total_tokens]
|
|
189
|
+
)
|
|
190
|
+
|
|
191
|
+
UsageInfo.new(
|
|
192
|
+
tokens: tokens,
|
|
193
|
+
cost: calculate_cost(model_cost, tokens)
|
|
168
194
|
)
|
|
169
195
|
end
|
|
170
196
|
|
|
171
|
-
def
|
|
197
|
+
def lookup_model_cost(entry)
|
|
172
198
|
return nil unless entry
|
|
173
|
-
Costs.find(entry[:model])
|
|
199
|
+
Costs.find(entry[:model].to_s)
|
|
174
200
|
rescue StandardError
|
|
175
201
|
nil
|
|
176
202
|
end
|
|
@@ -1,24 +1,64 @@
|
|
|
1
1
|
require "json"
|
|
2
2
|
|
|
3
3
|
module ActiveHarness
|
|
4
|
-
#
|
|
4
|
+
# Value objects for structured result data.
|
|
5
|
+
|
|
6
|
+
# Pricing rates for a model (per-token, from Costs registry).
|
|
7
|
+
# nil when the model is not found in the pricing registry.
|
|
8
|
+
ModelPricing = Struct.new(:input, :output, keyword_init: true)
|
|
9
|
+
|
|
10
|
+
# Static model metadata resolved at call time.
|
|
11
|
+
ModelInfo = Struct.new(
|
|
12
|
+
:name,
|
|
13
|
+
:provider,
|
|
14
|
+
:temperature,
|
|
15
|
+
:context_window,
|
|
16
|
+
:pricing, # ModelPricing or nil
|
|
17
|
+
keyword_init: true
|
|
18
|
+
) do
|
|
19
|
+
def to_s; name.to_s; end
|
|
20
|
+
def inspect
|
|
21
|
+
parts = ["name=#{name.inspect}", "provider=#{provider.inspect}"]
|
|
22
|
+
parts << "temperature=#{temperature}" if temperature
|
|
23
|
+
parts << "context_window=#{context_window}" if context_window
|
|
24
|
+
parts << "pricing=#{pricing.inspect}" if pricing
|
|
25
|
+
"#<ModelInfo #{parts.join(' ')}>"
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# Token counts for a single call.
|
|
30
|
+
TokenCounts = Struct.new(:input, :output, :total, keyword_init: true)
|
|
31
|
+
|
|
32
|
+
# Monetary cost of a single call in USD.
|
|
33
|
+
# nil when pricing data is unavailable.
|
|
34
|
+
CostBreakdown = Struct.new(:input, :output, :total, keyword_init: true)
|
|
35
|
+
|
|
36
|
+
# Combined token + cost stats for a single agent call.
|
|
37
|
+
# tokens is always present (raw provider data).
|
|
38
|
+
# cost is nil when pricing is unavailable.
|
|
39
|
+
UsageInfo = Struct.new(:tokens, :cost, keyword_init: true)
|
|
40
|
+
|
|
41
|
+
# Result returned by Agent#call (accessible via agent.result).
|
|
5
42
|
#
|
|
6
|
-
#
|
|
7
|
-
#
|
|
8
|
-
#
|
|
9
|
-
#
|
|
43
|
+
# result.input — original input string
|
|
44
|
+
# result.output — raw string from the provider
|
|
45
|
+
# result.processed — parsed Hash/Array for :json agents, raw string for :text
|
|
46
|
+
# result.system_prompt — resolved system prompt string
|
|
47
|
+
# result.model — ModelInfo (name, provider, temperature, context_window, pricing)
|
|
48
|
+
# result.model_list — full model chain proxy
|
|
49
|
+
# result.attempts — Array of failed attempt entries before success
|
|
50
|
+
# result.execution_time — wall-clock seconds for the successful call
|
|
51
|
+
# result.usage — UsageInfo (tokens + cost), nil for streaming without usage
|
|
10
52
|
Result = Struct.new(
|
|
11
53
|
:input,
|
|
12
54
|
:output,
|
|
13
55
|
:processed,
|
|
14
56
|
:system_prompt,
|
|
15
|
-
:
|
|
16
|
-
:temperature,
|
|
57
|
+
:model, # ModelInfo
|
|
17
58
|
:model_list,
|
|
18
59
|
:attempts,
|
|
19
60
|
:execution_time,
|
|
20
|
-
:usage,
|
|
21
|
-
:
|
|
22
|
-
|
|
23
|
-
keyword_init: true)
|
|
61
|
+
:usage, # UsageInfo or nil
|
|
62
|
+
keyword_init: true
|
|
63
|
+
)
|
|
24
64
|
end
|
data/lib/active_harness.rb
CHANGED
|
@@ -12,7 +12,7 @@ class AiSupportController < ApplicationController
|
|
|
12
12
|
|
|
13
13
|
render json: {
|
|
14
14
|
output: result.output,
|
|
15
|
-
model: result.model,
|
|
15
|
+
model: result.model.name,
|
|
16
16
|
time: result.execution_time
|
|
17
17
|
}
|
|
18
18
|
end
|
|
@@ -30,7 +30,7 @@ class AiSupportController < ApplicationController
|
|
|
30
30
|
|
|
31
31
|
render json: {
|
|
32
32
|
output: result.output,
|
|
33
|
-
model: result.model,
|
|
33
|
+
model: result.model.name,
|
|
34
34
|
time: result.execution_time,
|
|
35
35
|
turns: memory.size
|
|
36
36
|
}
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: active_harness
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.2.
|
|
4
|
+
version: 0.2.31
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- the-teacher
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-06-
|
|
11
|
+
date: 2026-06-12 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: concurrent-ruby
|