active_harness 0.3.7 → 0.3.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/active_harness/agent/goal_check.rb +26 -0
- data/lib/active_harness/agent/state.rb +90 -0
- data/lib/active_harness/agent.rb +734 -0
- data/lib/active_harness/core/errors.rb +11 -0
- data/lib/active_harness/http/client.rb +58 -11
- data/lib/active_harness/http/streaming_client.rb +33 -1
- data/lib/active_harness/pipeline/hooks.rb +1 -1
- data/lib/active_harness/providers/anthropic.rb +57 -8
- data/lib/active_harness/providers/base.rb +6 -0
- data/lib/active_harness/providers/deepseek.rb +7 -4
- data/lib/active_harness/providers/gpustack.rb +7 -4
- data/lib/active_harness/providers/groq.rb +7 -4
- data/lib/active_harness/providers/hetzner.rb +7 -4
- data/lib/active_harness/providers/mistral.rb +7 -4
- data/lib/active_harness/providers/ollama.rb +7 -4
- data/lib/active_harness/providers/ollama_cloud.rb +7 -4
- data/lib/active_harness/providers/openai.rb +7 -4
- data/lib/active_harness/providers/openrouter.rb +7 -4
- data/lib/active_harness/providers/tool_calling.rb +95 -0
- data/lib/active_harness/providers/xai.rb +7 -4
- data/lib/active_harness/railtie.rb +1 -1
- data/lib/active_harness/request/hooks.rb +1 -1
- data/lib/active_harness/request/providers.rb +15 -0
- data/lib/active_harness/request.rb +39 -7
- data/lib/active_harness/result.rb +4 -0
- data/lib/active_harness/tool/path_policy.rb +28 -0
- data/lib/active_harness/tool.rb +348 -0
- data/lib/active_harness/tribunal/hooks.rb +1 -1
- data/lib/active_harness.rb +4 -1
- data/lib/generators/active_harness/agent/agent_generator.rb +16 -0
- data/lib/generators/active_harness/agent/templates/agent.rb.tt +14 -0
- data/lib/generators/active_harness/goal/goal_generator.rb +16 -0
- data/lib/generators/active_harness/goal/templates/goal.rb.tt +14 -0
- data/lib/generators/active_harness/install/install_generator.rb +30 -25
- data/lib/generators/active_harness/install/templates/agents/files_agent.rb +39 -0
- data/lib/generators/active_harness/install/templates/concerns/agent_tracing.rb +36 -0
- data/lib/generators/active_harness/install/templates/concerns/pipeline_tracing.rb +28 -0
- data/lib/generators/active_harness/install/templates/concerns/request_tracing.rb +44 -0
- data/lib/generators/active_harness/install/templates/concerns/tool_tracing.rb +21 -0
- data/lib/generators/active_harness/install/templates/concerns/tracing_log.rb +18 -0
- data/lib/generators/active_harness/install/templates/concerns/tribunal_tracing.rb +28 -0
- data/lib/generators/active_harness/install/templates/controllers/ai_controller.rb +23 -2
- data/lib/generators/active_harness/install/templates/goals/answer_present_goal.rb +22 -0
- data/lib/generators/active_harness/install/templates/pipelines/support_pipeline.rb +8 -18
- data/lib/generators/active_harness/install/templates/prompts/files_prompt.rb +9 -0
- data/lib/generators/active_harness/install/templates/prompts/support_prompt.rb +3 -0
- data/lib/generators/active_harness/install/templates/requests/files_request.rb +18 -0
- data/lib/generators/active_harness/install/templates/requests/support_guard_request.rb +12 -0
- data/lib/generators/active_harness/install/templates/requests/support_request.rb +6 -0
- data/lib/generators/active_harness/install/templates/tools/ls_tool.rb +40 -0
- data/lib/generators/active_harness/install/templates/tools/read_file_tool.rb +43 -0
- data/lib/generators/active_harness/install/templates/tribunals/support_guard_tribunal.rb +7 -0
- data/lib/generators/active_harness/pipeline/templates/pipeline.rb.tt +5 -0
- data/lib/generators/active_harness/request/templates/request.rb.tt +2 -0
- data/lib/generators/active_harness/tool/templates/tool.rb.tt +22 -0
- data/lib/generators/active_harness/tool/tool_generator.rb +16 -0
- data/lib/generators/active_harness/tribunal/templates/tribunal.rb.tt +3 -0
- metadata +31 -7
|
@@ -5,11 +5,14 @@ module ActiveHarness
|
|
|
5
5
|
# xAI (Grok) — OpenAI-compatible API.
|
|
6
6
|
# https://docs.x.ai/api
|
|
7
7
|
class XAI < Base
|
|
8
|
-
|
|
8
|
+
include ToolCalling
|
|
9
|
+
|
|
10
|
+
def call(model:, messages:, temperature: 0.7, stream: nil, tools: nil)
|
|
9
11
|
headers = { "Content-Type" => "application/json", "Authorization" => "Bearer #{api_key}" }
|
|
10
|
-
body = { model: model, messages: messages, temperature: temperature }
|
|
12
|
+
body = { model: model, messages: openai_messages(messages), temperature: temperature }
|
|
13
|
+
body[:tools] = openai_tools(tools) if tools?(tools)
|
|
11
14
|
|
|
12
|
-
return call_streaming(url: config.xai_api_url, headers: headers, body: body, stream: stream, provider: :xai, model: model) if stream
|
|
15
|
+
return call_streaming(url: config.xai_api_url, headers: headers, body: body, stream: stream, provider: :xai, model: model) if stream && !tools?(tools)
|
|
13
16
|
|
|
14
17
|
raw = post_json(URI(config.xai_api_url), headers: headers, body: body)
|
|
15
18
|
data = parse!(raw)
|
|
@@ -19,7 +22,7 @@ module ActiveHarness
|
|
|
19
22
|
content: data.dig("choices", 0, "message", "content").to_s.strip,
|
|
20
23
|
provider: :xai,
|
|
21
24
|
model: data["model"] || model,
|
|
22
|
-
usage: extract_usage_openai(data)
|
|
25
|
+
usage: extract_usage_openai(data), **openai_tool_calls_entry(data)
|
|
23
26
|
}
|
|
24
27
|
end
|
|
25
28
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
module ActiveHarness
|
|
2
2
|
class Railtie < Rails::Railtie
|
|
3
|
-
APP_AI_DIRS = %w[requests prompts tribunals pipelines memory].freeze
|
|
3
|
+
APP_AI_DIRS = %w[concerns requests prompts tribunals pipelines memory agents tools goals].freeze
|
|
4
4
|
|
|
5
5
|
initializer "active_harness.autoload_paths", before: :set_autoload_paths do |app|
|
|
6
6
|
APP_AI_DIRS.each do |dir|
|
|
@@ -65,6 +65,12 @@ module ActiveHarness
|
|
|
65
65
|
provider = resolve_provider(entry[:provider])
|
|
66
66
|
messages = build_messages(system_prompt, @input)
|
|
67
67
|
opts = { model: entry[:model], messages: messages }
|
|
68
|
+
if @tools && !@tools.empty?
|
|
69
|
+
unless provider.respond_to?(:supports_tools?) && provider.supports_tools?
|
|
70
|
+
raise Errors::InvalidRequestError, "Provider #{entry[:provider].inspect} does not support tool calling"
|
|
71
|
+
end
|
|
72
|
+
opts[:tools] = @tools
|
|
73
|
+
end
|
|
68
74
|
opts[:temperature] = entry[:temperature] if entry[:temperature]
|
|
69
75
|
opts[:stream] = @token if @token
|
|
70
76
|
opts[:name] = entry[:name] if entry[:name]
|
|
@@ -138,6 +144,15 @@ module ActiveHarness
|
|
|
138
144
|
# See ActiveHarness::Memory for examples.
|
|
139
145
|
messages = []
|
|
140
146
|
messages << { role: "system", content: system_prompt } if system_prompt
|
|
147
|
+
|
|
148
|
+
if @messages
|
|
149
|
+
# Conversation mode (used by Agent): the system prompt of the Request goes first,
|
|
150
|
+
# unless the conversation already carries its own system message.
|
|
151
|
+
convo = @messages.map { |m| m.dup }
|
|
152
|
+
messages.shift if messages.any? && convo.any? { |m| (m[:role] || m["role"]).to_s == "system" }
|
|
153
|
+
return messages + convo
|
|
154
|
+
end
|
|
155
|
+
|
|
141
156
|
messages << { role: "user", content: input }
|
|
142
157
|
messages
|
|
143
158
|
end
|
|
@@ -17,7 +17,9 @@ module ActiveHarness
|
|
|
17
17
|
memory: nil,
|
|
18
18
|
models: nil,
|
|
19
19
|
token: nil,
|
|
20
|
-
stream: nil
|
|
20
|
+
stream: nil,
|
|
21
|
+
messages: nil,
|
|
22
|
+
tools: nil
|
|
21
23
|
)
|
|
22
24
|
new(
|
|
23
25
|
input: input,
|
|
@@ -26,7 +28,9 @@ module ActiveHarness
|
|
|
26
28
|
memory: memory,
|
|
27
29
|
models: models,
|
|
28
30
|
token: token,
|
|
29
|
-
stream: stream
|
|
31
|
+
stream: stream,
|
|
32
|
+
messages: messages,
|
|
33
|
+
tools: tools
|
|
30
34
|
).call
|
|
31
35
|
end
|
|
32
36
|
|
|
@@ -54,7 +58,9 @@ module ActiveHarness
|
|
|
54
58
|
attr_accessor :input,
|
|
55
59
|
:context,
|
|
56
60
|
:params,
|
|
57
|
-
:memory
|
|
61
|
+
:memory,
|
|
62
|
+
:messages,
|
|
63
|
+
:tools
|
|
58
64
|
attr_reader :result,
|
|
59
65
|
:token,
|
|
60
66
|
:stream
|
|
@@ -71,9 +77,13 @@ module ActiveHarness
|
|
|
71
77
|
memory: nil,
|
|
72
78
|
models: nil,
|
|
73
79
|
token: nil,
|
|
74
|
-
stream: nil
|
|
80
|
+
stream: nil,
|
|
81
|
+
messages: nil,
|
|
82
|
+
tools: nil
|
|
75
83
|
)
|
|
76
84
|
@input = input
|
|
85
|
+
@messages = messages
|
|
86
|
+
@tools = tools
|
|
77
87
|
@config = self.class.request_config
|
|
78
88
|
normalize_input!
|
|
79
89
|
@context = context
|
|
@@ -91,11 +101,18 @@ module ActiveHarness
|
|
|
91
101
|
# Optionally accepts input and stream callback inline:
|
|
92
102
|
# request.call("What is the capital of Japan?")
|
|
93
103
|
# request.call("...", stream: ->(token) { print token })
|
|
94
|
-
|
|
104
|
+
#
|
|
105
|
+
# Agent extension: pass +messages:+ (a conversation instead of a single input)
|
|
106
|
+
# and +tools:+ (neutral tool schemas). The model may then answer with
|
|
107
|
+
# +result.tool_calls+; nothing is executed here — running tools is the Agent's job.
|
|
108
|
+
def call(input = nil, token: nil, stream: nil, messages: nil, tools: nil)
|
|
95
109
|
if input
|
|
96
110
|
@input = input
|
|
97
111
|
normalize_input!
|
|
98
112
|
end
|
|
113
|
+
@messages = messages if messages
|
|
114
|
+
@tools = tools if tools
|
|
115
|
+
@input = last_user_content(@messages) if @input.nil? && @messages
|
|
99
116
|
@token = token if token
|
|
100
117
|
@stream = stream if stream
|
|
101
118
|
fire(:before_call)
|
|
@@ -147,7 +164,9 @@ module ActiveHarness
|
|
|
147
164
|
|
|
148
165
|
def build_result(response, entry, attempts, elapsed)
|
|
149
166
|
raw = response[:content]
|
|
150
|
-
|
|
167
|
+
tool_calls = response[:tool_calls]
|
|
168
|
+
tool_calls = nil if tool_calls && tool_calls.empty?
|
|
169
|
+
processed = tool_calls ? raw : parse_output(raw)
|
|
151
170
|
raw_usage = response[:usage]
|
|
152
171
|
model_cost = lookup_model_cost(entry)
|
|
153
172
|
|
|
@@ -160,10 +179,23 @@ module ActiveHarness
|
|
|
160
179
|
model_list: model_list,
|
|
161
180
|
attempts: attempts,
|
|
162
181
|
execution_time: elapsed,
|
|
163
|
-
usage: build_usage(raw_usage, model_cost)
|
|
182
|
+
usage: build_usage(raw_usage, model_cost),
|
|
183
|
+
tool_calls: tool_calls,
|
|
184
|
+
message: tool_calls || @messages ? assistant_message(raw, tool_calls) : nil
|
|
164
185
|
)
|
|
165
186
|
end
|
|
166
187
|
|
|
188
|
+
def assistant_message(content, tool_calls)
|
|
189
|
+
msg = { role: :assistant, content: content.to_s }
|
|
190
|
+
msg[:tool_calls] = tool_calls if tool_calls
|
|
191
|
+
msg
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
def last_user_content(messages)
|
|
195
|
+
last = Array(messages).reverse.find { |m| (m[:role] || m["role"]).to_s == "user" }
|
|
196
|
+
last && (last[:content] || last["content"])
|
|
197
|
+
end
|
|
198
|
+
|
|
167
199
|
def build_model_info(entry, model_cost)
|
|
168
200
|
pricing = if model_cost&.input_per_million && model_cost&.output_per_million
|
|
169
201
|
ModelPricing.new(
|
|
@@ -49,6 +49,8 @@ module ActiveHarness
|
|
|
49
49
|
# result.attempts — Array of failed attempt entries before success
|
|
50
50
|
# result.execution_time — wall-clock seconds for the successful call
|
|
51
51
|
# result.usage — UsageInfo (tokens + cost), nil for streaming without usage
|
|
52
|
+
# result.tool_calls — Array of { id:, name:, arguments: } the model asked for (nil when none)
|
|
53
|
+
# result.message — the assistant message in neutral shape, ready to append to a messages list
|
|
52
54
|
Result = Struct.new(
|
|
53
55
|
:input,
|
|
54
56
|
:output,
|
|
@@ -59,6 +61,8 @@ module ActiveHarness
|
|
|
59
61
|
:attempts,
|
|
60
62
|
:execution_time,
|
|
61
63
|
:usage, # UsageInfo or nil
|
|
64
|
+
:tool_calls, # Array<Hash> or nil
|
|
65
|
+
:message, # { role: :assistant, content:, tool_calls: } or nil
|
|
62
66
|
keyword_init: true
|
|
63
67
|
)
|
|
64
68
|
end
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
module ActiveHarness
|
|
2
|
+
class Tool
|
|
3
|
+
# Reusable path checks — implemented once, correctly (symlinks, "..", the
|
|
4
|
+
# trailing-slash prefix bug). The including tool must define ALLOWED_PATHS;
|
|
5
|
+
# a missing constant raises NameError, which fails closed (an :error result).
|
|
6
|
+
#
|
|
7
|
+
# class LsTool < ActiveHarness::Tool
|
|
8
|
+
# include ActiveHarness::Tool::PathPolicy
|
|
9
|
+
# ALLOWED_PATHS = %w[/srv/app/workspace].freeze
|
|
10
|
+
#
|
|
11
|
+
# step(:resolve_path, transform: true) { |input| input.merge(path: safe_path!(input[:path])) }
|
|
12
|
+
# end
|
|
13
|
+
module PathPolicy
|
|
14
|
+
# Returns a canonical absolute path inside ALLOWED_PATHS, or raises Denied.
|
|
15
|
+
def safe_path!(path)
|
|
16
|
+
deny!("path must not contain NUL") if path.to_s.include?("\0")
|
|
17
|
+
|
|
18
|
+
real = File.realpath(File.expand_path(path.to_s)) # follows symlinks
|
|
19
|
+
roots = self.class::ALLOWED_PATHS.map { |r| File.realpath(r) }
|
|
20
|
+
ok = roots.any? { |r| real == r || real.start_with?(r + File::SEPARATOR) }
|
|
21
|
+
deny!("#{path} is outside allowed directories", detail: "resolved to #{real}") unless ok
|
|
22
|
+
real
|
|
23
|
+
rescue Errno::ENOENT
|
|
24
|
+
deny!("#{path} does not exist")
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
@@ -0,0 +1,348 @@
|
|
|
1
|
+
require "json"
|
|
2
|
+
|
|
3
|
+
module ActiveHarness
|
|
4
|
+
# One step of a tool call, as recorded in ToolResult#steps.
|
|
5
|
+
#
|
|
6
|
+
# name — step name (:perform for the action itself)
|
|
7
|
+
# status — :ok | :denied | :error
|
|
8
|
+
# message — safe text; the ONLY thing the model may see
|
|
9
|
+
# detail — internals (paths, policy names, exception class) for traces/logs only
|
|
10
|
+
# ms — duration in milliseconds
|
|
11
|
+
StepResult = Struct.new(:name, :status, :message, :detail, :ms, keyword_init: true)
|
|
12
|
+
|
|
13
|
+
# Outcome of Tool.call. Never an exception for expected failures.
|
|
14
|
+
#
|
|
15
|
+
# status — :ok | :denied | :error
|
|
16
|
+
# input — arguments as the tool received them
|
|
17
|
+
# output — value returned by +perform+ (nil unless :ok)
|
|
18
|
+
# failed_step — StepResult of the step that stopped the call (nil when :ok)
|
|
19
|
+
# steps — StepResult of every step that ran
|
|
20
|
+
# outcome_unknown — true when the tool reported that a side effect may have happened
|
|
21
|
+
ToolResult = Struct.new(:status, :input, :output, :failed_step, :steps, :outcome_unknown, keyword_init: true) do
|
|
22
|
+
def ok?; status == :ok; end
|
|
23
|
+
def denied?; status == :denied; end
|
|
24
|
+
def error?; status == :error; end
|
|
25
|
+
|
|
26
|
+
# Text for the `tool` message sent back to the model.
|
|
27
|
+
def model_message
|
|
28
|
+
case status
|
|
29
|
+
when :ok
|
|
30
|
+
output.is_a?(String) ? output : JSON.generate(output)
|
|
31
|
+
else
|
|
32
|
+
failed_step&.message.to_s
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def execution_time
|
|
37
|
+
(steps || []).sum { |s| s.ms.to_i } / 1000.0
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# A privileged action behind named, ordered steps (checks and transforms).
|
|
42
|
+
# The model never gets a shell — it can only call tools you wrote, and every
|
|
43
|
+
# call passes through your checks first.
|
|
44
|
+
#
|
|
45
|
+
# class LsTool < ActiveHarness::Tool
|
|
46
|
+
# scope :os
|
|
47
|
+
# description "List files in a directory"
|
|
48
|
+
# param :path, "Directory to list"
|
|
49
|
+
#
|
|
50
|
+
# step(:no_hidden) { |input| deny!("hidden dirs are off limits") if input[:path].include?("/.") }
|
|
51
|
+
#
|
|
52
|
+
# def perform(input)
|
|
53
|
+
# Dir.children(input[:path]).first(200)
|
|
54
|
+
# end
|
|
55
|
+
# end
|
|
56
|
+
#
|
|
57
|
+
# LsTool.call(input: { path: "/srv" }, context: { user: current_user }) # => ToolResult
|
|
58
|
+
class Tool
|
|
59
|
+
# Raised (via #deny!) from any step or hook: the call is refused. +message+ is
|
|
60
|
+
# safe for the model; +detail+ stays in the trace.
|
|
61
|
+
class Denied < StandardError
|
|
62
|
+
attr_reader :detail
|
|
63
|
+
|
|
64
|
+
def initialize(message = nil, detail: nil)
|
|
65
|
+
super(message)
|
|
66
|
+
@detail = detail
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# Raised (via #unknown_outcome!) when a side effect may or may not have happened
|
|
71
|
+
# (e.g. a timeout after a write was sent). Result: status :error, outcome_unknown: true.
|
|
72
|
+
# Deliberately NOT a Denied, so it never matches `rescue Denied`.
|
|
73
|
+
class UnknownOutcome < StandardError
|
|
74
|
+
attr_reader :detail
|
|
75
|
+
|
|
76
|
+
def initialize(message = nil, detail: nil)
|
|
77
|
+
super(message)
|
|
78
|
+
@detail = detail
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
Step = Struct.new(:name, :block, :transform)
|
|
83
|
+
PERFORM = Step.new(:perform, nil, true)
|
|
84
|
+
|
|
85
|
+
VALID_HOOKS = %i[before_step after_step before_perform after_perform].freeze
|
|
86
|
+
VALID_STEP_HOOKS = %i[before_step after_step].freeze
|
|
87
|
+
|
|
88
|
+
GENERIC_ERROR_MESSAGE = "tool failed, try another approach or finish".freeze
|
|
89
|
+
UNKNOWN_OUTCOME_MESSAGE = "result unknown, verify before retrying".freeze
|
|
90
|
+
|
|
91
|
+
# -------------------------------------------------------------------------
|
|
92
|
+
# Class-level DSL
|
|
93
|
+
# -------------------------------------------------------------------------
|
|
94
|
+
class << self
|
|
95
|
+
# --- what the model sees ---
|
|
96
|
+
|
|
97
|
+
# Required namespace: :os, :web, :ruby ... Full name is "scope::name".
|
|
98
|
+
def scope(value = nil)
|
|
99
|
+
value ? @scope = value.to_sym : @scope
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# Defaults to the class name without "Tool": LsTool => "ls", FetchUrlTool => "fetch_url".
|
|
103
|
+
def tool_name(value = nil)
|
|
104
|
+
value ? @tool_name = value.to_s : (@tool_name || default_name)
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def full_name
|
|
108
|
+
raise ArgumentError, "#{name || 'Tool'} must declare a scope (e.g. `scope :os`)" unless scope
|
|
109
|
+
|
|
110
|
+
"#{scope}::#{tool_name}"
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# Name on the wire: providers allow only [a-zA-Z0-9_-] in function names.
|
|
114
|
+
def wire_name
|
|
115
|
+
full_name.sub("::", "__")
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def description(value = nil)
|
|
119
|
+
value ? @description = value : @description
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
# Declares one argument for the model.
|
|
123
|
+
#
|
|
124
|
+
# param :path, "Directory to list" # type: :string, required: true
|
|
125
|
+
# param :limit, "Max entries", type: :integer, required: false
|
|
126
|
+
# param :kind, "Entry kind", enum: %w[file dir]
|
|
127
|
+
def param(name, description, type: :string, required: true, enum: nil)
|
|
128
|
+
params << { name: name.to_sym, description: description, type: type, required: required, enum: enum }.reject { |_, v| v.nil? }
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
def params
|
|
132
|
+
@params ||= []
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
# Argument list (same data as +param+ builds); may be overridden by hand.
|
|
136
|
+
def schema
|
|
137
|
+
params
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
# Provider-neutral function schema; providers map it to their wire format.
|
|
141
|
+
# { name: "os__ls", description: "...", parameters: { type: "object", properties: {...}, required: [...] } }
|
|
142
|
+
def to_schema
|
|
143
|
+
props = {}
|
|
144
|
+
required = []
|
|
145
|
+
schema.each do |arg|
|
|
146
|
+
prop = { type: (arg[:type] || :string).to_s }
|
|
147
|
+
prop[:description] = arg[:description] if arg[:description]
|
|
148
|
+
prop[:enum] = arg[:enum] if arg[:enum]
|
|
149
|
+
props[arg[:name].to_s] = prop
|
|
150
|
+
required << arg[:name].to_s if arg.fetch(:required, true)
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
{
|
|
154
|
+
name: wire_name,
|
|
155
|
+
description: description.to_s,
|
|
156
|
+
parameters: { type: "object", properties: props, required: required }
|
|
157
|
+
}
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
# Checks arguments against +schema+. Returns an error message String or nil.
|
|
161
|
+
# Used by the Agent before running the tool (tools themselves get raw input).
|
|
162
|
+
def validate_input(input)
|
|
163
|
+
return "arguments must be an object" unless input.is_a?(Hash)
|
|
164
|
+
|
|
165
|
+
known = schema.map { |a| a[:name].to_sym }
|
|
166
|
+
extra = input.keys.map(&:to_sym) - known
|
|
167
|
+
return "unknown argument: #{extra.first}" unless extra.empty?
|
|
168
|
+
|
|
169
|
+
schema.each do |arg|
|
|
170
|
+
key = arg[:name].to_sym
|
|
171
|
+
value = input.key?(key) ? input[key] : input[key.to_s]
|
|
172
|
+
return "missing required argument: #{key}" if arg.fetch(:required, true) && value.nil?
|
|
173
|
+
if arg[:enum] && !value.nil? && !arg[:enum].include?(value)
|
|
174
|
+
return "invalid value for #{key}: expected one of #{arg[:enum].join(', ')}"
|
|
175
|
+
end
|
|
176
|
+
end
|
|
177
|
+
nil
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
# --- what the model does NOT see: ordered, named steps ---
|
|
181
|
+
|
|
182
|
+
# step :name do |input| ... end — a block
|
|
183
|
+
# step :name — calls the instance method of the same name
|
|
184
|
+
# step :name, transform: true { ... } — the return value replaces the payload for later steps and perform
|
|
185
|
+
def step(name, transform: false, &block)
|
|
186
|
+
steps << Step.new(name.to_sym, block, transform)
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
def steps
|
|
190
|
+
@steps ||= []
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
def hooks
|
|
194
|
+
@hooks ||= {}
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
def step_hooks
|
|
198
|
+
@step_hooks ||= {}
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
# Hooks run through instance_exec on the tool instance (they see context, payload, output).
|
|
202
|
+
#
|
|
203
|
+
# on :before_step, :resolve_path do |payload| ... end # per-step form
|
|
204
|
+
# on :before_step do |step_name, payload| ... end
|
|
205
|
+
# on :after_step do |step_name, step_result| ... end
|
|
206
|
+
# on :before_perform do |payload| ... end
|
|
207
|
+
# on :after_perform do |output| ... end
|
|
208
|
+
def on(event, step_name = nil, &block)
|
|
209
|
+
if step_name
|
|
210
|
+
raise ArgumentError, "Per-step hooks support: #{VALID_STEP_HOOKS.join(', ')}. Got :#{event}" unless VALID_STEP_HOOKS.include?(event)
|
|
211
|
+
|
|
212
|
+
((step_hooks[step_name.to_sym] ||= {})[event] ||= []) << block
|
|
213
|
+
else
|
|
214
|
+
raise ArgumentError, "Unknown Tool hook :#{event}. Valid: #{VALID_HOOKS.join(', ')}" unless VALID_HOOKS.include?(event)
|
|
215
|
+
|
|
216
|
+
(hooks[event] ||= []) << block
|
|
217
|
+
end
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
def before(event, step_name = nil, &block)
|
|
221
|
+
on(:"before_#{event}", step_name, &block)
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
def after(event, step_name = nil, &block)
|
|
225
|
+
on(:"after_#{event}", step_name, &block)
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
def inherited(subclass)
|
|
229
|
+
super
|
|
230
|
+
subclass.instance_variable_set(:@scope, @scope)
|
|
231
|
+
subclass.instance_variable_set(:@steps, steps.dup)
|
|
232
|
+
subclass.instance_variable_set(:@params, params.map(&:dup))
|
|
233
|
+
subclass.instance_variable_set(:@hooks, hooks.each_with_object({}) { |(k, v), h| h[k] = v.dup })
|
|
234
|
+
subclass.instance_variable_set(:@step_hooks, step_hooks.each_with_object({}) { |(k, v), h| h[k] = v.transform_values(&:dup) })
|
|
235
|
+
end
|
|
236
|
+
|
|
237
|
+
# Same style as Request.call.
|
|
238
|
+
def call(input:, context: {})
|
|
239
|
+
new(context: context).call(input)
|
|
240
|
+
end
|
|
241
|
+
|
|
242
|
+
private
|
|
243
|
+
|
|
244
|
+
def default_name
|
|
245
|
+
return nil unless name
|
|
246
|
+
|
|
247
|
+
base = name.split("::").last.sub(/Tool\z/, "")
|
|
248
|
+
base.gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2').gsub(/([a-z\d])([A-Z])/, '\1_\2').downcase
|
|
249
|
+
end
|
|
250
|
+
end
|
|
251
|
+
|
|
252
|
+
include Core::HookRunner
|
|
253
|
+
|
|
254
|
+
# -------------------------------------------------------------------------
|
|
255
|
+
# Instance API
|
|
256
|
+
# -------------------------------------------------------------------------
|
|
257
|
+
attr_reader :context
|
|
258
|
+
attr_accessor :payload, :output
|
|
259
|
+
|
|
260
|
+
def initialize(context: {})
|
|
261
|
+
@context = context
|
|
262
|
+
end
|
|
263
|
+
|
|
264
|
+
# Entry point used by the Agent loop. Never raises for expected failures.
|
|
265
|
+
def call(input)
|
|
266
|
+
@payload = input
|
|
267
|
+
@output = nil
|
|
268
|
+
steps = []
|
|
269
|
+
|
|
270
|
+
(self.class.steps + [PERFORM]).each do |step|
|
|
271
|
+
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
272
|
+
status, message, detail, unknown = :ok, nil, nil, false
|
|
273
|
+
|
|
274
|
+
begin
|
|
275
|
+
fire(:before_step, step.name, @payload)
|
|
276
|
+
fire_step(:before_step, step.name, @payload)
|
|
277
|
+
fire(:before_perform, @payload) if step.name == :perform
|
|
278
|
+
|
|
279
|
+
value = run_step(step)
|
|
280
|
+
@payload = value if step.transform
|
|
281
|
+
|
|
282
|
+
if step.name == :perform
|
|
283
|
+
@output = value
|
|
284
|
+
fire(:after_perform, @output)
|
|
285
|
+
end
|
|
286
|
+
rescue Denied => e
|
|
287
|
+
status, message, detail = :denied, e.message, e.detail
|
|
288
|
+
rescue UnknownOutcome => e
|
|
289
|
+
status, message, detail, unknown = :error, UNKNOWN_OUTCOME_MESSAGE, e.detail || e.message, true
|
|
290
|
+
rescue StandardError => e
|
|
291
|
+
status, message, detail = :error, GENERIC_ERROR_MESSAGE, "#{e.class}: #{e.message}"
|
|
292
|
+
end
|
|
293
|
+
|
|
294
|
+
steps << StepResult.new(name: step.name, status: status, message: message, detail: detail, ms: elapsed_ms(started))
|
|
295
|
+
fire(:after_step, step.name, steps.last)
|
|
296
|
+
fire_step(:after_step, step.name, steps.last)
|
|
297
|
+
|
|
298
|
+
next if status == :ok
|
|
299
|
+
|
|
300
|
+
return ToolResult.new(status: status, input: input, failed_step: steps.last, steps: steps, outcome_unknown: unknown)
|
|
301
|
+
end
|
|
302
|
+
|
|
303
|
+
ToolResult.new(status: :ok, input: input, output: @output, steps: steps, outcome_unknown: false)
|
|
304
|
+
end
|
|
305
|
+
|
|
306
|
+
# The action itself. Always the last step; its return value is the output.
|
|
307
|
+
def perform(_input)
|
|
308
|
+
raise NotImplementedError, "#{self.class.name} must implement #perform"
|
|
309
|
+
end
|
|
310
|
+
|
|
311
|
+
# Refuse the call. +msg+ is shown to the model; +detail+ only goes to the trace.
|
|
312
|
+
def deny!(msg, detail: nil)
|
|
313
|
+
raise Denied.new(msg, detail: detail)
|
|
314
|
+
end
|
|
315
|
+
|
|
316
|
+
# The side effect may have happened — the model is told to verify before retrying.
|
|
317
|
+
def unknown_outcome!(msg = nil, detail: nil)
|
|
318
|
+
raise UnknownOutcome.new(msg || "unknown outcome", detail: detail)
|
|
319
|
+
end
|
|
320
|
+
|
|
321
|
+
private
|
|
322
|
+
|
|
323
|
+
def run_step(step)
|
|
324
|
+
if step.name == :perform
|
|
325
|
+
perform(@payload)
|
|
326
|
+
elsif step.block
|
|
327
|
+
instance_exec(@payload, &step.block)
|
|
328
|
+
else
|
|
329
|
+
send(step.name, @payload)
|
|
330
|
+
end
|
|
331
|
+
end
|
|
332
|
+
|
|
333
|
+
def fire(event, *args)
|
|
334
|
+
run_hooks(self.class.hooks, event, *args)
|
|
335
|
+
end
|
|
336
|
+
|
|
337
|
+
# Per-step hook: receives only the data, not the step name.
|
|
338
|
+
def fire_step(event, step_name, data)
|
|
339
|
+
run_hooks(self.class.step_hooks[step_name] || {}, event, data)
|
|
340
|
+
end
|
|
341
|
+
|
|
342
|
+
def elapsed_ms(started)
|
|
343
|
+
((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1000).round
|
|
344
|
+
end
|
|
345
|
+
end
|
|
346
|
+
end
|
|
347
|
+
|
|
348
|
+
require_relative "tool/path_policy"
|
data/lib/active_harness.rb
CHANGED
|
@@ -6,6 +6,7 @@ require_relative "active_harness/http/client"
|
|
|
6
6
|
require_relative "active_harness/http/streaming_client"
|
|
7
7
|
require_relative "active_harness/http/retry_policy"
|
|
8
8
|
require_relative "active_harness/providers/base"
|
|
9
|
+
require_relative "active_harness/providers/tool_calling"
|
|
9
10
|
require_relative "active_harness/providers/openai"
|
|
10
11
|
require_relative "active_harness/providers/openrouter"
|
|
11
12
|
require_relative "active_harness/providers/groq"
|
|
@@ -37,11 +38,13 @@ require_relative "active_harness/memory"
|
|
|
37
38
|
require_relative "active_harness/request"
|
|
38
39
|
require_relative "active_harness/tribunal"
|
|
39
40
|
require_relative "active_harness/pipeline"
|
|
41
|
+
require_relative "active_harness/tool"
|
|
42
|
+
require_relative "active_harness/agent"
|
|
40
43
|
|
|
41
44
|
require_relative "active_harness/railtie" if defined?(Rails::Railtie)
|
|
42
45
|
|
|
43
46
|
module ActiveHarness
|
|
44
|
-
VERSION = "0.3.
|
|
47
|
+
VERSION = "0.3.9"
|
|
45
48
|
|
|
46
49
|
class << self
|
|
47
50
|
# Configure ActiveHarness.
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
require "rails/generators"
|
|
2
|
+
|
|
3
|
+
module ActiveHarness
|
|
4
|
+
module Generators
|
|
5
|
+
class AgentGenerator < Rails::Generators::NamedBase
|
|
6
|
+
source_root File.expand_path("templates", __dir__)
|
|
7
|
+
|
|
8
|
+
desc "Creates an ActiveHarness agent in app/ai/agents/"
|
|
9
|
+
|
|
10
|
+
def create_agent
|
|
11
|
+
template "agent.rb.tt",
|
|
12
|
+
"app/ai/agents/#{file_name}_agent.rb"
|
|
13
|
+
end
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
class <%= class_name %>Agent < ActiveHarness::Agent
|
|
2
|
+
# The Request that talks to the model each turn (owns model chain, retries, fallback).
|
|
3
|
+
request <%= class_name %>Request
|
|
4
|
+
|
|
5
|
+
# The only tools the model may call (default deny: anything else is "unknown tool").
|
|
6
|
+
# tools ExampleTool
|
|
7
|
+
|
|
8
|
+
# Optional: judge the final answer; without it the first answer without tool calls ends the run.
|
|
9
|
+
# goal <%= class_name %>Goal
|
|
10
|
+
|
|
11
|
+
# Hard limits so a looping model cannot run or spend forever.
|
|
12
|
+
max_turns 10
|
|
13
|
+
max_cost 0.50
|
|
14
|
+
end
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
require "rails/generators"
|
|
2
|
+
|
|
3
|
+
module ActiveHarness
|
|
4
|
+
module Generators
|
|
5
|
+
class GoalGenerator < Rails::Generators::NamedBase
|
|
6
|
+
source_root File.expand_path("templates", __dir__)
|
|
7
|
+
|
|
8
|
+
desc "Creates an ActiveHarness goal check in app/ai/goals/"
|
|
9
|
+
|
|
10
|
+
def create_goal
|
|
11
|
+
template "goal.rb.tt",
|
|
12
|
+
"app/ai/goals/#{file_name}_goal.rb"
|
|
13
|
+
end
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# A goal check judges the agent's final answer: override #call(candidate:, goal:, state:)
|
|
2
|
+
# to return a GoalCheckResult. Plug it in with `goal <%= class_name %>Goal`.
|
|
3
|
+
# Prefer deterministic checks (tests, schemas, counts) to asking an LLM.
|
|
4
|
+
class <%= class_name %>Goal < ActiveHarness::GoalCheck
|
|
5
|
+
def call(candidate:, goal:, state:)
|
|
6
|
+
if candidate.to_s.strip.empty?
|
|
7
|
+
# :continue sends the feedback to the model for another turn (up to max_goal_retries).
|
|
8
|
+
# Other statuses: :achieved, :replan, :needs_input, :failed.
|
|
9
|
+
ActiveHarness::GoalCheckResult.new(status: :continue, feedback: "The answer is empty. Answer the question.")
|
|
10
|
+
else
|
|
11
|
+
ActiveHarness::GoalCheckResult.new(status: :achieved)
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
end
|