active_harness 0.2.30 → 0.2.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/active_harness/agent/cost.rb +13 -15
- data/lib/active_harness/agent/prompt.rb +7 -1
- data/lib/active_harness/agent.rb +42 -16
- data/lib/active_harness/{costs.rb → pricing.rb} +18 -18
- data/lib/active_harness/result.rb +51 -11
- data/lib/active_harness.rb +2 -2
- data/lib/generators/active_harness/install/templates/controllers/ai_controller.rb +2 -2
- metadata +3 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 231f48053aa92c1172f16cf7b6d03b222eccc9a967092d78488718dafa006b2b
|
|
4
|
+
data.tar.gz: 1f4f83487faff2a5b6b5dad31e8c902718c6179a0fec1162425b62640454b2da
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 23b68426a66c24f9f1c5759b64a01cefd4293a5f94f11f1a30806447d55db1d85cc54d4de4a3a18b9254db32b5aeb63c502938adbe023cb4d03962dcc4d8247b
|
|
7
|
+
data.tar.gz: 93ca0e61a02ce50bb5beb10fbb4c98630ef0c90efc9642b3d1c8d74f666559cdebcc965b4f29bcda32c9b7ac5609c0a6ea49790d5f62384a2ecacbab941fd0b2
|
|
@@ -2,25 +2,23 @@ module ActiveHarness
|
|
|
2
2
|
class Agent
|
|
3
3
|
private
|
|
4
4
|
|
|
5
|
-
#
|
|
6
|
-
#
|
|
5
|
+
# Builds a CostBreakdown for a single request from token usage and
|
|
6
|
+
# pricing data from ActiveHarness::Pricing.
|
|
7
7
|
#
|
|
8
|
-
# Returns
|
|
8
|
+
# Returns CostBreakdown (input, output, total in USD),
|
|
9
9
|
# or nil if usage is absent or the model is not found in the pricing registry.
|
|
10
|
-
def calculate_cost(
|
|
11
|
-
return nil
|
|
10
|
+
def calculate_cost(pricing, tokens)
|
|
11
|
+
return nil unless pricing && tokens
|
|
12
|
+
return nil unless pricing.input_per_million && pricing.output_per_million
|
|
12
13
|
|
|
13
|
-
|
|
14
|
-
|
|
14
|
+
input_cost = (tokens.input.to_f / 1_000_000) * pricing.input_per_million
|
|
15
|
+
output_cost = (tokens.output.to_f / 1_000_000) * pricing.output_per_million
|
|
15
16
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
output_cost: output_cost.round(8),
|
|
22
|
-
total_cost: (input_cost + output_cost).round(8)
|
|
23
|
-
}
|
|
17
|
+
CostBreakdown.new(
|
|
18
|
+
input: input_cost.round(8),
|
|
19
|
+
output: output_cost.round(8),
|
|
20
|
+
total: (input_cost + output_cost).round(8)
|
|
21
|
+
)
|
|
24
22
|
rescue StandardError
|
|
25
23
|
nil
|
|
26
24
|
end
|
|
@@ -57,7 +57,13 @@ module ActiveHarness
|
|
|
57
57
|
obj.instance_variable_set(:@params, @params)
|
|
58
58
|
obj.instance_variable_set(:@config, @config)
|
|
59
59
|
obj.instance_variable_set(:@memory, @memory)
|
|
60
|
-
obj.instance_variable_set(:@context_window,
|
|
60
|
+
obj.instance_variable_set(:@context_window, context_window_for_prompt)
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def context_window_for_prompt
|
|
64
|
+
Pricing.find(model_list.to_a.first&.dig(:model).to_s)&.context_window
|
|
65
|
+
rescue StandardError
|
|
66
|
+
nil
|
|
61
67
|
end
|
|
62
68
|
end
|
|
63
69
|
end
|
data/lib/active_harness/agent.rb
CHANGED
|
@@ -54,7 +54,6 @@ module ActiveHarness
|
|
|
54
54
|
:params,
|
|
55
55
|
:memory
|
|
56
56
|
attr_reader :result,
|
|
57
|
-
:context_window,
|
|
58
57
|
:token_stream,
|
|
59
58
|
:event_stream
|
|
60
59
|
|
|
@@ -77,9 +76,8 @@ module ActiveHarness
|
|
|
77
76
|
@context = context
|
|
78
77
|
@params = params
|
|
79
78
|
@memory = memory
|
|
80
|
-
@models_override
|
|
81
|
-
@
|
|
82
|
-
@token_stream = streams[:token]
|
|
79
|
+
@models_override = Array(models) if models
|
|
80
|
+
@token_stream = streams[:token]
|
|
83
81
|
@event_stream = streams[:agent]
|
|
84
82
|
fire(:setup)
|
|
85
83
|
end
|
|
@@ -146,31 +144,59 @@ module ActiveHarness
|
|
|
146
144
|
end
|
|
147
145
|
|
|
148
146
|
def build_result(response, entry, attempts, elapsed)
|
|
149
|
-
raw
|
|
150
|
-
processed
|
|
151
|
-
|
|
152
|
-
|
|
147
|
+
raw = response[:content]
|
|
148
|
+
processed = parse_output(raw)
|
|
149
|
+
raw_usage = response[:usage]
|
|
150
|
+
model_cost = lookup_model_cost(entry)
|
|
153
151
|
|
|
154
152
|
Result.new(
|
|
155
153
|
input: @input,
|
|
156
154
|
output: raw,
|
|
157
155
|
processed: processed,
|
|
158
156
|
system_prompt: @system_prompt,
|
|
159
|
-
|
|
160
|
-
model: entry[:model],
|
|
161
|
-
temperature: entry[:temperature],
|
|
157
|
+
model: build_model_info(entry, model_cost),
|
|
162
158
|
model_list: model_list,
|
|
163
159
|
attempts: attempts,
|
|
164
160
|
execution_time: elapsed,
|
|
165
|
-
usage:
|
|
166
|
-
|
|
167
|
-
|
|
161
|
+
usage: build_usage(raw_usage, model_cost)
|
|
162
|
+
)
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
def build_model_info(entry, model_cost)
|
|
166
|
+
pricing = if model_cost&.input_per_million && model_cost&.output_per_million
|
|
167
|
+
ModelPricing.new(
|
|
168
|
+
input: (model_cost.input_per_million / 1_000_000.0).round(10),
|
|
169
|
+
output: (model_cost.output_per_million / 1_000_000.0).round(10)
|
|
170
|
+
)
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
ModelInfo.new(
|
|
174
|
+
name: entry[:model],
|
|
175
|
+
provider: entry[:provider],
|
|
176
|
+
temperature: entry[:temperature],
|
|
177
|
+
context_window: model_cost&.context_window,
|
|
178
|
+
pricing: pricing
|
|
179
|
+
)
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
def build_usage(raw_usage, model_cost)
|
|
183
|
+
return nil if raw_usage.nil?
|
|
184
|
+
|
|
185
|
+
tokens = TokenCounts.new(
|
|
186
|
+
input: raw_usage[:input_tokens],
|
|
187
|
+
output: raw_usage[:output_tokens],
|
|
188
|
+
total: raw_usage[:total_tokens]
|
|
189
|
+
)
|
|
190
|
+
|
|
191
|
+
UsageInfo.new(
|
|
192
|
+
tokens: tokens,
|
|
193
|
+
cost: calculate_cost(model_cost, tokens)
|
|
168
194
|
)
|
|
169
195
|
end
|
|
170
196
|
|
|
171
|
-
def
|
|
197
|
+
def lookup_model_cost(entry)
|
|
172
198
|
return nil unless entry
|
|
173
|
-
|
|
199
|
+
Pricing.find(entry[:model].to_s)
|
|
174
200
|
rescue StandardError
|
|
175
201
|
nil
|
|
176
202
|
end
|
|
@@ -10,28 +10,28 @@ module ActiveHarness
|
|
|
10
10
|
# by ActiveHarness (files present in lib/active_harness/providers/).
|
|
11
11
|
#
|
|
12
12
|
# Data source priority:
|
|
13
|
-
# 1. {project_root}/tmp/active_harness/
|
|
13
|
+
# 1. {project_root}/tmp/active_harness/pricing.json — fetched cache (refreshed once per day)
|
|
14
14
|
# 2. lib/active_harness/data/models.json — bundled fallback (ships with gem)
|
|
15
15
|
#
|
|
16
16
|
# Usage:
|
|
17
17
|
#
|
|
18
18
|
# # Fetch fresh data and save to tmp cache (also called automatically when stale)
|
|
19
|
-
# ActiveHarness::
|
|
19
|
+
# ActiveHarness::Pricing.update
|
|
20
20
|
#
|
|
21
21
|
# # All models (auto-updates cache if missing or older than 24h)
|
|
22
|
-
# ActiveHarness::
|
|
22
|
+
# ActiveHarness::Pricing.all
|
|
23
23
|
#
|
|
24
24
|
# # Single model by ID
|
|
25
|
-
# ActiveHarness::
|
|
25
|
+
# ActiveHarness::Pricing.find("gpt-4o")
|
|
26
26
|
#
|
|
27
27
|
# # By provider — method or bracket syntax
|
|
28
|
-
# ActiveHarness::
|
|
29
|
-
# ActiveHarness::
|
|
28
|
+
# ActiveHarness::Pricing.providers.openai
|
|
29
|
+
# ActiveHarness::Pricing.providers[:anthropic]
|
|
30
30
|
#
|
|
31
31
|
# # List providers that have data
|
|
32
|
-
# ActiveHarness::
|
|
32
|
+
# ActiveHarness::Pricing.providers.list
|
|
33
33
|
#
|
|
34
|
-
module
|
|
34
|
+
module Pricing
|
|
35
35
|
BUNDLED_DATA_FILE = File.expand_path("data/models.json", __dir__).freeze
|
|
36
36
|
MODELS_DEV_URL = "https://models.dev/api.json"
|
|
37
37
|
CACHE_TTL = 86_400 # 24 hours in seconds
|
|
@@ -54,7 +54,7 @@ module ActiveHarness
|
|
|
54
54
|
}.freeze
|
|
55
55
|
|
|
56
56
|
# Value object representing the pricing for a single model.
|
|
57
|
-
|
|
57
|
+
ModelPrice = Struct.new(
|
|
58
58
|
:id,
|
|
59
59
|
:name,
|
|
60
60
|
:provider,
|
|
@@ -89,31 +89,31 @@ module ActiveHarness
|
|
|
89
89
|
parts << "output=$#{output_per_million}/M" if output_per_million
|
|
90
90
|
parts << "ctx=#{context_window}" if context_window
|
|
91
91
|
parts << "cats=#{categories.join(',')}" if categories.any?
|
|
92
|
-
"#<
|
|
92
|
+
"#<ModelPrice #{parts.join(' ')}>"
|
|
93
93
|
end
|
|
94
94
|
end
|
|
95
95
|
|
|
96
96
|
# Proxy object that exposes providers as methods and via [].
|
|
97
97
|
class ProvidersProxy
|
|
98
98
|
def [](name)
|
|
99
|
-
ActiveHarness::
|
|
99
|
+
ActiveHarness::Pricing.for_provider(name.to_s)
|
|
100
100
|
end
|
|
101
101
|
|
|
102
102
|
def list
|
|
103
|
-
ActiveHarness::
|
|
103
|
+
ActiveHarness::Pricing.provider_names
|
|
104
104
|
end
|
|
105
105
|
|
|
106
106
|
def method_missing(name, *args, &block)
|
|
107
107
|
provider = name.to_s
|
|
108
|
-
if ActiveHarness::
|
|
109
|
-
ActiveHarness::
|
|
108
|
+
if ActiveHarness::Pricing.provider_names.include?(provider)
|
|
109
|
+
ActiveHarness::Pricing.for_provider(provider)
|
|
110
110
|
else
|
|
111
111
|
super
|
|
112
112
|
end
|
|
113
113
|
end
|
|
114
114
|
|
|
115
115
|
def respond_to_missing?(name, include_private = false)
|
|
116
|
-
ActiveHarness::
|
|
116
|
+
ActiveHarness::Pricing.provider_names.include?(name.to_s) || super
|
|
117
117
|
end
|
|
118
118
|
end
|
|
119
119
|
|
|
@@ -154,7 +154,7 @@ module ActiveHarness
|
|
|
154
154
|
end
|
|
155
155
|
|
|
156
156
|
# Fetches fresh pricing data from models.dev, filters to supported providers,
|
|
157
|
-
# and writes the result to {project_root}/tmp/active_harness/
|
|
157
|
+
# and writes the result to {project_root}/tmp/active_harness/pricing.json.
|
|
158
158
|
# Returns the number of models saved, or raises on HTTP failure.
|
|
159
159
|
def update
|
|
160
160
|
raw_api = fetch_models_dev
|
|
@@ -176,7 +176,7 @@ module ActiveHarness
|
|
|
176
176
|
|
|
177
177
|
# Path to the per-project cache file.
|
|
178
178
|
def cache_file
|
|
179
|
-
File.join(project_root, "tmp", "active_harness", "
|
|
179
|
+
File.join(project_root, "tmp", "active_harness", "pricing.json")
|
|
180
180
|
end
|
|
181
181
|
|
|
182
182
|
# Names of providers supported by ActiveHarness (derived from providers/ directory).
|
|
@@ -267,7 +267,7 @@ module ActiveHarness
|
|
|
267
267
|
|
|
268
268
|
def build_cost(raw)
|
|
269
269
|
standard = raw.dig(:pricing, :text_tokens, :standard) || {}
|
|
270
|
-
|
|
270
|
+
ModelPrice.new(
|
|
271
271
|
id: raw[:id],
|
|
272
272
|
name: raw[:name],
|
|
273
273
|
provider: raw[:provider],
|
|
@@ -1,24 +1,64 @@
|
|
|
1
1
|
require "json"
|
|
2
2
|
|
|
3
3
|
module ActiveHarness
|
|
4
|
-
#
|
|
4
|
+
# Value objects for structured result data.
|
|
5
|
+
|
|
6
|
+
# Pricing rates for a model (per-token, from Pricing registry).
|
|
7
|
+
# nil when the model is not found in the pricing registry.
|
|
8
|
+
ModelPricing = Struct.new(:input, :output, keyword_init: true)
|
|
9
|
+
|
|
10
|
+
# Static model metadata resolved at call time.
|
|
11
|
+
ModelInfo = Struct.new(
|
|
12
|
+
:name,
|
|
13
|
+
:provider,
|
|
14
|
+
:temperature,
|
|
15
|
+
:context_window,
|
|
16
|
+
:pricing, # ModelPricing or nil
|
|
17
|
+
keyword_init: true
|
|
18
|
+
) do
|
|
19
|
+
def to_s; name.to_s; end
|
|
20
|
+
def inspect
|
|
21
|
+
parts = ["name=#{name.inspect}", "provider=#{provider.inspect}"]
|
|
22
|
+
parts << "temperature=#{temperature}" if temperature
|
|
23
|
+
parts << "context_window=#{context_window}" if context_window
|
|
24
|
+
parts << "pricing=#{pricing.inspect}" if pricing
|
|
25
|
+
"#<ModelInfo #{parts.join(' ')}>"
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# Token counts for a single call.
|
|
30
|
+
TokenCounts = Struct.new(:input, :output, :total, keyword_init: true)
|
|
31
|
+
|
|
32
|
+
# Monetary cost of a single call in USD.
|
|
33
|
+
# nil when pricing data is unavailable.
|
|
34
|
+
CostBreakdown = Struct.new(:input, :output, :total, keyword_init: true)
|
|
35
|
+
|
|
36
|
+
# Combined token + cost stats for a single agent call.
|
|
37
|
+
# tokens is always present (raw provider data).
|
|
38
|
+
# cost is nil when pricing is unavailable.
|
|
39
|
+
UsageInfo = Struct.new(:tokens, :cost, keyword_init: true)
|
|
40
|
+
|
|
41
|
+
# Result returned by Agent#call (accessible via agent.result).
|
|
5
42
|
#
|
|
6
|
-
#
|
|
7
|
-
#
|
|
8
|
-
#
|
|
9
|
-
#
|
|
43
|
+
# result.input — original input string
|
|
44
|
+
# result.output — raw string from the provider
|
|
45
|
+
# result.processed — parsed Hash/Array for :json agents, raw string for :text
|
|
46
|
+
# result.system_prompt — resolved system prompt string
|
|
47
|
+
# result.model — ModelInfo (name, provider, temperature, context_window, pricing)
|
|
48
|
+
# result.model_list — full model chain proxy
|
|
49
|
+
# result.attempts — Array of failed attempt entries before success
|
|
50
|
+
# result.execution_time — wall-clock seconds for the successful call
|
|
51
|
+
# result.usage — UsageInfo (tokens + cost), nil for streaming without usage
|
|
10
52
|
Result = Struct.new(
|
|
11
53
|
:input,
|
|
12
54
|
:output,
|
|
13
55
|
:processed,
|
|
14
56
|
:system_prompt,
|
|
15
|
-
:
|
|
16
|
-
:temperature,
|
|
57
|
+
:model, # ModelInfo
|
|
17
58
|
:model_list,
|
|
18
59
|
:attempts,
|
|
19
60
|
:execution_time,
|
|
20
|
-
:usage,
|
|
21
|
-
:
|
|
22
|
-
|
|
23
|
-
keyword_init: true)
|
|
61
|
+
:usage, # UsageInfo or nil
|
|
62
|
+
keyword_init: true
|
|
63
|
+
)
|
|
24
64
|
end
|
data/lib/active_harness.rb
CHANGED
|
@@ -21,7 +21,7 @@ require_relative "active_harness/providers/azure"
|
|
|
21
21
|
require_relative "active_harness/providers/bedrock"
|
|
22
22
|
require_relative "active_harness/providers/vertexai"
|
|
23
23
|
require_relative "active_harness/providers/custom"
|
|
24
|
-
require_relative "active_harness/
|
|
24
|
+
require_relative "active_harness/pricing"
|
|
25
25
|
require_relative "active_harness/memory"
|
|
26
26
|
require_relative "active_harness/agent"
|
|
27
27
|
require_relative "active_harness/tribunal"
|
|
@@ -30,7 +30,7 @@ require_relative "active_harness/pipeline"
|
|
|
30
30
|
require_relative "active_harness/railtie" if defined?(Rails::Railtie)
|
|
31
31
|
|
|
32
32
|
module ActiveHarness
|
|
33
|
-
VERSION = "0.2.
|
|
33
|
+
VERSION = "0.2.32"
|
|
34
34
|
|
|
35
35
|
class << self
|
|
36
36
|
# Configure ActiveHarness.
|
|
@@ -12,7 +12,7 @@ class AiSupportController < ApplicationController
|
|
|
12
12
|
|
|
13
13
|
render json: {
|
|
14
14
|
output: result.output,
|
|
15
|
-
model: result.model,
|
|
15
|
+
model: result.model.name,
|
|
16
16
|
time: result.execution_time
|
|
17
17
|
}
|
|
18
18
|
end
|
|
@@ -30,7 +30,7 @@ class AiSupportController < ApplicationController
|
|
|
30
30
|
|
|
31
31
|
render json: {
|
|
32
32
|
output: result.output,
|
|
33
|
-
model: result.model,
|
|
33
|
+
model: result.model.name,
|
|
34
34
|
time: result.execution_time,
|
|
35
35
|
turns: memory.size
|
|
36
36
|
}
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: active_harness
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.2.
|
|
4
|
+
version: 0.2.32
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- the-teacher
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-06-
|
|
11
|
+
date: 2026-06-12 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: concurrent-ruby
|
|
@@ -43,7 +43,6 @@ files:
|
|
|
43
43
|
- lib/active_harness/configuration.rb
|
|
44
44
|
- lib/active_harness/core/errors.rb
|
|
45
45
|
- lib/active_harness/core/hooks.rb
|
|
46
|
-
- lib/active_harness/costs.rb
|
|
47
46
|
- lib/active_harness/data/models.json
|
|
48
47
|
- lib/active_harness/http/client.rb
|
|
49
48
|
- lib/active_harness/http/retry_policy.rb
|
|
@@ -57,6 +56,7 @@ files:
|
|
|
57
56
|
- lib/active_harness/pipeline/README.md
|
|
58
57
|
- lib/active_harness/pipeline/hooks.rb
|
|
59
58
|
- lib/active_harness/pipeline/step.rb
|
|
59
|
+
- lib/active_harness/pricing.rb
|
|
60
60
|
- lib/active_harness/providers/PROVIDER_CONTRACT.md
|
|
61
61
|
- lib/active_harness/providers/anthropic.rb
|
|
62
62
|
- lib/active_harness/providers/azure.rb
|