active_harness 0.2.29 → 0.2.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/active_harness/agent/cost.rb +13 -15
- data/lib/active_harness/agent/prompt.rb +7 -1
- data/lib/active_harness/agent.rb +42 -16
- data/lib/active_harness/pipeline/step.rb +25 -3
- data/lib/active_harness/pipeline.rb +21 -2
- data/lib/active_harness/result.rb +51 -11
- data/lib/active_harness.rb +1 -1
- data/lib/generators/active_harness/install/templates/controllers/ai_controller.rb +2 -2
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: d6d0a8c421353c670b4f0ba80cab409a05c97a1505382fd35a0822b39d66691a
|
|
4
|
+
data.tar.gz: a312b8db19958a621d26aee139d9bbe284f269690991b02008a2e6716fb08d7b
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: bc0be024611a21164721a379741f48cf2547ebb4b5bd64f15bb3cdf5bae41e8bdbbe7f0d576c41ddd24a5167488bd2db2729cb659d699f3aceffda2791664998
|
|
7
|
+
data.tar.gz: 329d8648ae84605f6ecbd2ee9112a9a0abf39a4d3f4a2fb98ee6814b79e7bd8167a1fd807e1a994dc5d299475b02b603e48d65a4270fa01b9082ca5ecf499900
|
|
@@ -2,25 +2,23 @@ module ActiveHarness
|
|
|
2
2
|
class Agent
|
|
3
3
|
private
|
|
4
4
|
|
|
5
|
-
#
|
|
6
|
-
#
|
|
5
|
+
# Builds a CostBreakdown for a single request from token usage and
|
|
6
|
+
# pricing data from ActiveHarness::Costs.
|
|
7
7
|
#
|
|
8
|
-
# Returns
|
|
8
|
+
# Returns CostBreakdown (input, output, total in USD),
|
|
9
9
|
# or nil if usage is absent or the model is not found in the pricing registry.
|
|
10
|
-
def calculate_cost(
|
|
11
|
-
return nil
|
|
10
|
+
def calculate_cost(pricing, tokens)
|
|
11
|
+
return nil unless pricing && tokens
|
|
12
|
+
return nil unless pricing.input_per_million && pricing.output_per_million
|
|
12
13
|
|
|
13
|
-
|
|
14
|
-
|
|
14
|
+
input_cost = (tokens.input.to_f / 1_000_000) * pricing.input_per_million
|
|
15
|
+
output_cost = (tokens.output.to_f / 1_000_000) * pricing.output_per_million
|
|
15
16
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
output_cost: output_cost.round(8),
|
|
22
|
-
total_cost: (input_cost + output_cost).round(8)
|
|
23
|
-
}
|
|
17
|
+
CostBreakdown.new(
|
|
18
|
+
input: input_cost.round(8),
|
|
19
|
+
output: output_cost.round(8),
|
|
20
|
+
total: (input_cost + output_cost).round(8)
|
|
21
|
+
)
|
|
24
22
|
rescue StandardError
|
|
25
23
|
nil
|
|
26
24
|
end
|
|
@@ -57,7 +57,13 @@ module ActiveHarness
|
|
|
57
57
|
obj.instance_variable_set(:@params, @params)
|
|
58
58
|
obj.instance_variable_set(:@config, @config)
|
|
59
59
|
obj.instance_variable_set(:@memory, @memory)
|
|
60
|
-
obj.instance_variable_set(:@context_window,
|
|
60
|
+
obj.instance_variable_set(:@context_window, context_window_for_prompt)
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def context_window_for_prompt
|
|
64
|
+
Costs.find(model_list.to_a.first&.dig(:model).to_s)&.context_window
|
|
65
|
+
rescue StandardError
|
|
66
|
+
nil
|
|
61
67
|
end
|
|
62
68
|
end
|
|
63
69
|
end
|
data/lib/active_harness/agent.rb
CHANGED
|
@@ -54,7 +54,6 @@ module ActiveHarness
|
|
|
54
54
|
:params,
|
|
55
55
|
:memory
|
|
56
56
|
attr_reader :result,
|
|
57
|
-
:context_window,
|
|
58
57
|
:token_stream,
|
|
59
58
|
:event_stream
|
|
60
59
|
|
|
@@ -77,9 +76,8 @@ module ActiveHarness
|
|
|
77
76
|
@context = context
|
|
78
77
|
@params = params
|
|
79
78
|
@memory = memory
|
|
80
|
-
@models_override
|
|
81
|
-
@
|
|
82
|
-
@token_stream = streams[:token]
|
|
79
|
+
@models_override = Array(models) if models
|
|
80
|
+
@token_stream = streams[:token]
|
|
83
81
|
@event_stream = streams[:agent]
|
|
84
82
|
fire(:setup)
|
|
85
83
|
end
|
|
@@ -146,31 +144,59 @@ module ActiveHarness
|
|
|
146
144
|
end
|
|
147
145
|
|
|
148
146
|
def build_result(response, entry, attempts, elapsed)
|
|
149
|
-
raw
|
|
150
|
-
processed
|
|
151
|
-
|
|
152
|
-
|
|
147
|
+
raw = response[:content]
|
|
148
|
+
processed = parse_output(raw)
|
|
149
|
+
raw_usage = response[:usage]
|
|
150
|
+
model_cost = lookup_model_cost(entry)
|
|
153
151
|
|
|
154
152
|
Result.new(
|
|
155
153
|
input: @input,
|
|
156
154
|
output: raw,
|
|
157
155
|
processed: processed,
|
|
158
156
|
system_prompt: @system_prompt,
|
|
159
|
-
|
|
160
|
-
model: entry[:model],
|
|
161
|
-
temperature: entry[:temperature],
|
|
157
|
+
model: build_model_info(entry, model_cost),
|
|
162
158
|
model_list: model_list,
|
|
163
159
|
attempts: attempts,
|
|
164
160
|
execution_time: elapsed,
|
|
165
|
-
usage:
|
|
166
|
-
|
|
167
|
-
|
|
161
|
+
usage: build_usage(raw_usage, model_cost)
|
|
162
|
+
)
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
def build_model_info(entry, model_cost)
|
|
166
|
+
pricing = if model_cost&.input_per_million && model_cost&.output_per_million
|
|
167
|
+
ModelPricing.new(
|
|
168
|
+
input: (model_cost.input_per_million / 1_000_000.0).round(10),
|
|
169
|
+
output: (model_cost.output_per_million / 1_000_000.0).round(10)
|
|
170
|
+
)
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
ModelInfo.new(
|
|
174
|
+
name: entry[:model],
|
|
175
|
+
provider: entry[:provider],
|
|
176
|
+
temperature: entry[:temperature],
|
|
177
|
+
context_window: model_cost&.context_window,
|
|
178
|
+
pricing: pricing
|
|
179
|
+
)
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
def build_usage(raw_usage, model_cost)
|
|
183
|
+
return nil if raw_usage.nil?
|
|
184
|
+
|
|
185
|
+
tokens = TokenCounts.new(
|
|
186
|
+
input: raw_usage[:input_tokens],
|
|
187
|
+
output: raw_usage[:output_tokens],
|
|
188
|
+
total: raw_usage[:total_tokens]
|
|
189
|
+
)
|
|
190
|
+
|
|
191
|
+
UsageInfo.new(
|
|
192
|
+
tokens: tokens,
|
|
193
|
+
cost: calculate_cost(model_cost, tokens)
|
|
168
194
|
)
|
|
169
195
|
end
|
|
170
196
|
|
|
171
|
-
def
|
|
197
|
+
def lookup_model_cost(entry)
|
|
172
198
|
return nil unless entry
|
|
173
|
-
Costs.find(entry[:model])
|
|
199
|
+
Costs.find(entry[:model].to_s)
|
|
174
200
|
rescue StandardError
|
|
175
201
|
nil
|
|
176
202
|
end
|
|
@@ -21,15 +21,37 @@ module ActiveHarness
|
|
|
21
21
|
lam ? @stop_if = lam : @stop_if
|
|
22
22
|
end
|
|
23
23
|
|
|
24
|
+
# DSL: define how to extract the new payload from a result.
|
|
25
|
+
# When provided, the step always updates the payload — even if stop_if is also set.
|
|
26
|
+
# The block receives the Result and must return the new payload value.
|
|
27
|
+
#
|
|
28
|
+
# step :laundry do
|
|
29
|
+
# use PromptLaundryPipeline
|
|
30
|
+
# transform { |result| result.output }
|
|
31
|
+
# stop_if ->(result) { result.processed["stopped"] == true }
|
|
32
|
+
# end
|
|
33
|
+
def transform(&block)
|
|
34
|
+
@transform_block = block if block
|
|
35
|
+
@transform_block
|
|
36
|
+
end
|
|
37
|
+
|
|
24
38
|
# True if agent_class is a Tribunal subclass — tribunal steps do not update payload.
|
|
25
39
|
def tribunal?
|
|
26
40
|
@agent_class.is_a?(Class) && @agent_class <= ActiveHarness::Tribunal
|
|
27
41
|
end
|
|
28
42
|
|
|
29
|
-
#
|
|
30
|
-
#
|
|
43
|
+
# Returns true when this step should update the pipeline payload after execution.
|
|
44
|
+
# A step transforms when:
|
|
45
|
+
# - an explicit transform block is defined (overrides default), OR
|
|
46
|
+
# - no stop_if and not a tribunal (legacy default)
|
|
31
47
|
def transform?
|
|
32
|
-
!tribunal? && @stop_if.nil?
|
|
48
|
+
@transform_block ? true : (!tribunal? && @stop_if.nil?)
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# Extract the new payload value from result.
|
|
52
|
+
# Uses the user-defined transform block when present; falls back to result.output.
|
|
53
|
+
def extract_payload(result)
|
|
54
|
+
@transform_block ? @transform_block.call(result) : result.output
|
|
33
55
|
end
|
|
34
56
|
end
|
|
35
57
|
end
|
|
@@ -140,6 +140,20 @@ module ActiveHarness
|
|
|
140
140
|
@stopped
|
|
141
141
|
end
|
|
142
142
|
|
|
143
|
+
# Wraps pipeline outcome into a Result so a pipeline can be used as a step
|
|
144
|
+
# inside another pipeline, matching the same interface as Agent and Tribunal.
|
|
145
|
+
#
|
|
146
|
+
# output — final payload (nil when stopped)
|
|
147
|
+
# processed — { "stopped" => bool, "stopped_at" => step_name_string_or_nil }
|
|
148
|
+
def result
|
|
149
|
+
Result.new(
|
|
150
|
+
input: @original_input,
|
|
151
|
+
output: @output,
|
|
152
|
+
processed: { "stopped" => @stopped, "stopped_at" => @stopped_at&.to_s },
|
|
153
|
+
execution_time: @execution_time
|
|
154
|
+
)
|
|
155
|
+
end
|
|
156
|
+
|
|
143
157
|
# Execute all steps sequentially. Returns self for chaining.
|
|
144
158
|
def call
|
|
145
159
|
config = self.class.pipeline_config
|
|
@@ -155,7 +169,7 @@ module ActiveHarness
|
|
|
155
169
|
|
|
156
170
|
@step_results[step.name] = result
|
|
157
171
|
@context[step.name] = result
|
|
158
|
-
@payload = result
|
|
172
|
+
@payload = step.extract_payload(result) if step.transform?
|
|
159
173
|
|
|
160
174
|
fire(:after_step, step.name, result, config)
|
|
161
175
|
fire_step(:after_step, step.name, result, config)
|
|
@@ -210,7 +224,12 @@ module ActiveHarness
|
|
|
210
224
|
end
|
|
211
225
|
|
|
212
226
|
def execute_step(step)
|
|
213
|
-
streams = {
|
|
227
|
+
streams = {
|
|
228
|
+
token: @token_stream,
|
|
229
|
+
agent: @agent_event_stream,
|
|
230
|
+
tribunal: @tribunal_event_stream,
|
|
231
|
+
pipeline: @pipeline_event_stream
|
|
232
|
+
}.compact
|
|
214
233
|
step.agent_class.new(
|
|
215
234
|
input: @payload,
|
|
216
235
|
context: @context.dup,
|
|
@@ -1,24 +1,64 @@
|
|
|
1
1
|
require "json"
|
|
2
2
|
|
|
3
3
|
module ActiveHarness
|
|
4
|
-
#
|
|
4
|
+
# Value objects for structured result data.
|
|
5
|
+
|
|
6
|
+
# Pricing rates for a model (per-token, from Costs registry).
|
|
7
|
+
# nil when the model is not found in the pricing registry.
|
|
8
|
+
ModelPricing = Struct.new(:input, :output, keyword_init: true)
|
|
9
|
+
|
|
10
|
+
# Static model metadata resolved at call time.
|
|
11
|
+
ModelInfo = Struct.new(
|
|
12
|
+
:name,
|
|
13
|
+
:provider,
|
|
14
|
+
:temperature,
|
|
15
|
+
:context_window,
|
|
16
|
+
:pricing, # ModelPricing or nil
|
|
17
|
+
keyword_init: true
|
|
18
|
+
) do
|
|
19
|
+
def to_s; name.to_s; end
|
|
20
|
+
def inspect
|
|
21
|
+
parts = ["name=#{name.inspect}", "provider=#{provider.inspect}"]
|
|
22
|
+
parts << "temperature=#{temperature}" if temperature
|
|
23
|
+
parts << "context_window=#{context_window}" if context_window
|
|
24
|
+
parts << "pricing=#{pricing.inspect}" if pricing
|
|
25
|
+
"#<ModelInfo #{parts.join(' ')}>"
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# Token counts for a single call.
|
|
30
|
+
TokenCounts = Struct.new(:input, :output, :total, keyword_init: true)
|
|
31
|
+
|
|
32
|
+
# Monetary cost of a single call in USD.
|
|
33
|
+
# nil when pricing data is unavailable.
|
|
34
|
+
CostBreakdown = Struct.new(:input, :output, :total, keyword_init: true)
|
|
35
|
+
|
|
36
|
+
# Combined token + cost stats for a single agent call.
|
|
37
|
+
# tokens is always present (raw provider data).
|
|
38
|
+
# cost is nil when pricing is unavailable.
|
|
39
|
+
UsageInfo = Struct.new(:tokens, :cost, keyword_init: true)
|
|
40
|
+
|
|
41
|
+
# Result returned by Agent#call (accessible via agent.result).
|
|
5
42
|
#
|
|
6
|
-
#
|
|
7
|
-
#
|
|
8
|
-
#
|
|
9
|
-
#
|
|
43
|
+
# result.input — original input string
|
|
44
|
+
# result.output — raw string from the provider
|
|
45
|
+
# result.processed — parsed Hash/Array for :json agents, raw string for :text
|
|
46
|
+
# result.system_prompt — resolved system prompt string
|
|
47
|
+
# result.model — ModelInfo (name, provider, temperature, context_window, pricing)
|
|
48
|
+
# result.model_list — full model chain proxy
|
|
49
|
+
# result.attempts — Array of failed attempt entries before success
|
|
50
|
+
# result.execution_time — wall-clock seconds for the successful call
|
|
51
|
+
# result.usage — UsageInfo (tokens + cost), nil for streaming without usage
|
|
10
52
|
Result = Struct.new(
|
|
11
53
|
:input,
|
|
12
54
|
:output,
|
|
13
55
|
:processed,
|
|
14
56
|
:system_prompt,
|
|
15
|
-
:
|
|
16
|
-
:temperature,
|
|
57
|
+
:model, # ModelInfo
|
|
17
58
|
:model_list,
|
|
18
59
|
:attempts,
|
|
19
60
|
:execution_time,
|
|
20
|
-
:usage,
|
|
21
|
-
:
|
|
22
|
-
|
|
23
|
-
keyword_init: true)
|
|
61
|
+
:usage, # UsageInfo or nil
|
|
62
|
+
keyword_init: true
|
|
63
|
+
)
|
|
24
64
|
end
|
data/lib/active_harness.rb
CHANGED
|
@@ -12,7 +12,7 @@ class AiSupportController < ApplicationController
|
|
|
12
12
|
|
|
13
13
|
render json: {
|
|
14
14
|
output: result.output,
|
|
15
|
-
model: result.model,
|
|
15
|
+
model: result.model.name,
|
|
16
16
|
time: result.execution_time
|
|
17
17
|
}
|
|
18
18
|
end
|
|
@@ -30,7 +30,7 @@ class AiSupportController < ApplicationController
|
|
|
30
30
|
|
|
31
31
|
render json: {
|
|
32
32
|
output: result.output,
|
|
33
|
-
model: result.model,
|
|
33
|
+
model: result.model.name,
|
|
34
34
|
time: result.execution_time,
|
|
35
35
|
turns: memory.size
|
|
36
36
|
}
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: active_harness
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.2.
|
|
4
|
+
version: 0.2.31
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- the-teacher
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-06-
|
|
11
|
+
date: 2026-06-12 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: concurrent-ruby
|