active_harness 0.1.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/active_harness/agent/hooks.rb +75 -0
- data/lib/active_harness/agent/models.rb +147 -0
- data/lib/active_harness/agent/output_parser.rb +57 -0
- data/lib/active_harness/agent/prompt.rb +58 -0
- data/lib/active_harness/agent/providers.rb +54 -0
- data/lib/active_harness/agent.rb +107 -228
- data/lib/active_harness/core/errors.rb +22 -28
- data/lib/active_harness/http/client.rb +8 -19
- data/lib/active_harness/http/streaming_client.rb +60 -0
- data/lib/active_harness/memory/adapter/base.rb +36 -0
- data/lib/active_harness/memory/adapter/file.rb +141 -0
- data/lib/active_harness/memory.rb +212 -0
- data/lib/active_harness/pipeline/step.rb +36 -0
- data/lib/active_harness/pipeline.rb +207 -0
- data/lib/active_harness/providers/PROVIDER_CONTRACT.md +54 -0
- data/lib/active_harness/providers/anthropic.rb +76 -4
- data/lib/active_harness/providers/base.rb +41 -13
- data/lib/active_harness/providers/gemini.rb +61 -0
- data/lib/active_harness/providers/groq.rb +64 -0
- data/lib/active_harness/providers/openai.rb +39 -47
- data/lib/active_harness/providers/openrouter.rb +40 -54
- data/lib/active_harness/railtie.rb +12 -0
- data/lib/active_harness/result.rb +10 -0
- data/lib/active_harness/tribunal.rb +216 -0
- data/lib/active_harness.rb +17 -46
- data/lib/generators/active_harness/agent/agent_generator.rb +16 -0
- data/lib/generators/active_harness/agent/templates/agent.rb.tt +8 -0
- data/lib/generators/active_harness/install/install_generator.rb +54 -0
- data/lib/generators/active_harness/install/templates/agents/test_support_agent.rb +10 -0
- data/lib/generators/active_harness/install/templates/agents/test_support_guard_agent.rb +11 -0
- data/lib/generators/active_harness/install/templates/controllers/ai_controller.rb +105 -0
- data/lib/generators/active_harness/install/templates/memory/test_support_memory.rb +16 -0
- data/lib/generators/active_harness/install/templates/pipelines/test_support_pipeline.rb +31 -0
- data/lib/generators/active_harness/install/templates/prompts/test_support_guard_prompt.rb +9 -0
- data/lib/generators/active_harness/install/templates/prompts/test_support_prompt.rb +5 -0
- data/lib/generators/active_harness/install/templates/tribunals/test_support_guard_tribunal.rb +11 -0
- data/lib/generators/active_harness/memory/memory_generator.rb +16 -0
- data/lib/generators/active_harness/memory/templates/memory.rb.tt +12 -0
- data/lib/generators/active_harness/pipeline/pipeline_generator.rb +16 -0
- data/lib/generators/active_harness/pipeline/templates/pipeline.rb.tt +19 -0
- data/lib/generators/active_harness/prompt/prompt_generator.rb +16 -0
- data/lib/generators/active_harness/prompt/templates/prompt.rb.tt +5 -0
- data/lib/generators/active_harness/tribunal/templates/tribunal.rb.tt +7 -0
- data/lib/generators/active_harness/tribunal/tribunal_generator.rb +16 -0
- metadata +42 -72
- data/LICENSE +0 -21
- data/README.md +0 -113
- data/lib/active_harness/core/configuration.rb +0 -55
- data/lib/active_harness/core/version.rb +0 -3
- data/lib/active_harness/http/retry_policy.rb +0 -47
- data/lib/active_harness/models/model_request.rb +0 -14
- data/lib/active_harness/models/model_response.rb +0 -13
- data/lib/active_harness/payload.rb +0 -47
- data/lib/active_harness/pipeline/engine.rb +0 -251
- data/lib/active_harness/pipeline/fallback_runner.rb +0 -76
- data/lib/active_harness/pipeline/guard_runner.rb +0 -125
- data/lib/active_harness/pipeline/output_parser.rb +0 -43
- data/lib/active_harness/pipeline/prompt_builder.rb +0 -46
- data/lib/active_harness/pipeline/provider_registry.rb +0 -16
- data/lib/active_harness/prompts/guard_system_prompt.rb +0 -33
- data/lib/active_harness/providers/google.rb +0 -11
- data/lib/active_harness/rate_limit/request_limiter.rb +0 -50
- data/lib/active_harness/rate_limit/risk_holdback.rb +0 -69
- data/lib/active_harness/results/debug_result.rb +0 -19
- data/lib/active_harness/results/input_result.rb +0 -27
- data/lib/active_harness/results/result.rb +0 -55
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
require "json"
|
|
2
|
+
require "fileutils"
|
|
3
|
+
|
|
4
|
+
module ActiveHarness
|
|
5
|
+
class Memory
|
|
6
|
+
module Adapter
|
|
7
|
+
# Persists memory as JSON files on disk.
|
|
8
|
+
#
|
|
9
|
+
# Each session is stored in one file:
|
|
10
|
+
# <path>/<session_id>.json (no namespace)
|
|
11
|
+
# <path>/<session_id>/<namespace>.json (with namespace)
|
|
12
|
+
#
|
|
13
|
+
# Options:
|
|
14
|
+
# path — base directory (default: "storage/ai/memory")
|
|
15
|
+
# filename — String or Proc(session_id) (default: "<session_id>.json")
|
|
16
|
+
# pretty — format JSON with indentation (default: false)
|
|
17
|
+
# compact — store only q/a keys (default: false)
|
|
18
|
+
# encoding — file encoding (default: "UTF-8")
|
|
19
|
+
# storage_size — max turns kept in file (default: 1000)
|
|
20
|
+
# eviction_percent — % of oldest turns to drop (default: 10)
|
|
21
|
+
# on_trim — Proc called with trimmed turns (default: nil)
|
|
22
|
+
class File < Base
|
|
23
|
+
DEFAULT_PATH = "storage/ai/memory"
|
|
24
|
+
DEFAULT_STORAGE_SIZE = 1000
|
|
25
|
+
DEFAULT_TRIM_PERCENT = 10
|
|
26
|
+
|
|
27
|
+
def initialize(opts = {})
|
|
28
|
+
@path = opts.fetch(:path, DEFAULT_PATH)
|
|
29
|
+
@filename_opt = opts[:filename]
|
|
30
|
+
@pretty = opts.fetch(:pretty, false)
|
|
31
|
+
@compact = opts.fetch(:compact, false)
|
|
32
|
+
@encoding = opts.fetch(:encoding, "UTF-8")
|
|
33
|
+
@storage_size = opts.fetch(:storage_size, DEFAULT_STORAGE_SIZE)
|
|
34
|
+
@trim_percent = opts.fetch(:eviction_percent, DEFAULT_TRIM_PERCENT)
|
|
35
|
+
@on_trim = opts[:on_trim]
|
|
36
|
+
@namespace = opts[:namespace]
|
|
37
|
+
|
|
38
|
+
@session_id = nil
|
|
39
|
+
@turns = []
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def open(session_id)
|
|
43
|
+
@session_id = session_id
|
|
44
|
+
@turns = load_from_disk
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def read
|
|
48
|
+
@turns.dup
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def write(turn)
|
|
52
|
+
@turns << turn
|
|
53
|
+
trim_if_needed!
|
|
54
|
+
flush!
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def close
|
|
58
|
+
# File adapter writes immediately on each write, nothing to flush.
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def delete
|
|
62
|
+
path = file_path
|
|
63
|
+
::FileUtils.rm_f(path)
|
|
64
|
+
# remove parent dir only if it's a namespace dir and now empty
|
|
65
|
+
dir = ::File.dirname(path)
|
|
66
|
+
if @namespace && Dir.exist?(dir) && Dir.empty?(dir)
|
|
67
|
+
Dir.rmdir(dir)
|
|
68
|
+
end
|
|
69
|
+
@turns = []
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# -----------------------------------------------------------------------
|
|
73
|
+
private
|
|
74
|
+
# -----------------------------------------------------------------------
|
|
75
|
+
|
|
76
|
+
def file_path
|
|
77
|
+
name = resolve_filename
|
|
78
|
+
if @namespace
|
|
79
|
+
::File.join(@path, @session_id.to_s, "#{@namespace}.json")
|
|
80
|
+
elsif @filename_opt
|
|
81
|
+
::File.join(@path, name)
|
|
82
|
+
else
|
|
83
|
+
::File.join(@path, name)
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def resolve_filename
|
|
88
|
+
case @filename_opt
|
|
89
|
+
when Proc then @filename_opt.call(@session_id)
|
|
90
|
+
when String then @filename_opt
|
|
91
|
+
else "#{@session_id}.json"
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def load_from_disk
|
|
96
|
+
path = file_path
|
|
97
|
+
return [] unless ::File.exist?(path)
|
|
98
|
+
|
|
99
|
+
raw = ::File.read(path, encoding: @encoding)
|
|
100
|
+
data = JSON.parse(raw, symbolize_names: true)
|
|
101
|
+
turns = Array(data[:turns])
|
|
102
|
+
|
|
103
|
+
# Normalise compact format (q/a) to full format (request/response)
|
|
104
|
+
turns.map do |t|
|
|
105
|
+
if t.key?(:q)
|
|
106
|
+
{ request: t[:q], response: t[:a] }
|
|
107
|
+
else
|
|
108
|
+
t
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
rescue JSON::ParserError
|
|
112
|
+
[]
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def flush!
|
|
116
|
+
path = file_path
|
|
117
|
+
::FileUtils.mkdir_p(::File.dirname(path))
|
|
118
|
+
|
|
119
|
+
turns_to_write = if @compact
|
|
120
|
+
@turns.map { |t| { q: t[:request], a: t[:response] } }
|
|
121
|
+
else
|
|
122
|
+
@turns
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
data = { session_id: @session_id, turns: turns_to_write }
|
|
126
|
+
json = @pretty ? JSON.pretty_generate(data) : JSON.generate(data)
|
|
127
|
+
::File.write(path, json, encoding: @encoding)
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
def trim_if_needed!
|
|
131
|
+
return unless @storage_size
|
|
132
|
+
return if @turns.size <= @storage_size
|
|
133
|
+
|
|
134
|
+
count = [@turns.size * @trim_percent / 100, 1].max
|
|
135
|
+
trimmed = @turns.shift(count)
|
|
136
|
+
@on_trim&.call(trimmed)
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
end
|
|
140
|
+
end
|
|
141
|
+
end
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
require_relative "memory/adapter/base"
|
|
2
|
+
require_relative "memory/adapter/file"
|
|
3
|
+
|
|
4
|
+
module ActiveHarness
|
|
5
|
+
# Conversational memory for agents.
|
|
6
|
+
#
|
|
7
|
+
# Memory only records the history of request/response turns.
|
|
8
|
+
# It does NOT automatically inject history into LLM messages.
|
|
9
|
+
# Injection is always manual — you control when and how context is used.
|
|
10
|
+
#
|
|
11
|
+
# --- Recording (automatic) ---
|
|
12
|
+
#
|
|
13
|
+
# When a Memory object is passed to an agent, the agent automatically
|
|
14
|
+
# saves each successful turn (request + response) after the call.
|
|
15
|
+
#
|
|
16
|
+
# memory = ActiveHarness::Memory.new(session_id: "u42", depth: 8)
|
|
17
|
+
# SupportAgent.call(input: "Hello", memory: memory)
|
|
18
|
+
# # => turn is saved to storage/ai/memory/u42.json
|
|
19
|
+
#
|
|
20
|
+
# --- Manual injection patterns ---
|
|
21
|
+
#
|
|
22
|
+
# Option A: prepend history to input in a before_call hook
|
|
23
|
+
#
|
|
24
|
+
# on :before_call do
|
|
25
|
+
# history = @memory&.to_messages
|
|
26
|
+
# if history&.any?
|
|
27
|
+
# lines = history.map { |m| "#{m[:role]}: #{m[:content]}" }.join("\n")
|
|
28
|
+
# @input = "Previous conversation:\n#{lines}\n\nUser: #{@input}"
|
|
29
|
+
# end
|
|
30
|
+
# end
|
|
31
|
+
#
|
|
32
|
+
# Option B: inject history into the system prompt
|
|
33
|
+
#
|
|
34
|
+
# on :after_system_prompt do |prompt|
|
|
35
|
+
# history = @memory&.to_messages
|
|
36
|
+
# if history&.any?
|
|
37
|
+
# lines = history.map { |m| "#{m[:role]}: #{m[:content]}" }.join("\n")
|
|
38
|
+
# @system_prompt = "#{prompt}\n\nConversation so far:\n#{lines}"
|
|
39
|
+
# end
|
|
40
|
+
# end
|
|
41
|
+
#
|
|
42
|
+
# Option C: use a prompt class that reads @memory directly
|
|
43
|
+
#
|
|
44
|
+
# class SupportPrompt
|
|
45
|
+
# def call
|
|
46
|
+
# base = "You are a helpful assistant."
|
|
47
|
+
# return base unless @memory&.size&.positive?
|
|
48
|
+
# history = @memory.to_messages
|
|
49
|
+
# .map { |m| "#{m[:role]}: #{m[:content]}" }
|
|
50
|
+
# .join("\n")
|
|
51
|
+
# "#{base}\n\nConversation so far:\n#{history}"
|
|
52
|
+
# end
|
|
53
|
+
# end
|
|
54
|
+
#
|
|
55
|
+
class Memory
|
|
56
|
+
ADAPTERS = {
|
|
57
|
+
file: ->(**opts) { Adapter::File.new(**opts) }
|
|
58
|
+
}.freeze
|
|
59
|
+
|
|
60
|
+
attr_reader :session_id
|
|
61
|
+
|
|
62
|
+
# -------------------------------------------------------------------------
|
|
63
|
+
# Constructor
|
|
64
|
+
# -------------------------------------------------------------------------
|
|
65
|
+
# session_id — required; uniquely identifies this conversation
|
|
66
|
+
# depth — how many past turns to inject into messages (nil = all)
|
|
67
|
+
# adapter — :file (default), or an adapter instance
|
|
68
|
+
# enabled — false disables all reads and writes (no-op mode)
|
|
69
|
+
# read_only — true: load history but never write new turns
|
|
70
|
+
# namespace — isolates history per-agent within a session
|
|
71
|
+
# on_trim — Proc called with trimmed turns on storage trim
|
|
72
|
+
# async — write to adapter in a background thread
|
|
73
|
+
# **adapter_opts — forwarded to the adapter (path, storage_size, etc.)
|
|
74
|
+
def initialize(
|
|
75
|
+
session_id:,
|
|
76
|
+
depth: nil,
|
|
77
|
+
adapter: :file,
|
|
78
|
+
enabled: true,
|
|
79
|
+
read_only: false,
|
|
80
|
+
namespace: nil,
|
|
81
|
+
on_trim: nil,
|
|
82
|
+
async: false,
|
|
83
|
+
**adapter_opts
|
|
84
|
+
)
|
|
85
|
+
@session_id = session_id
|
|
86
|
+
@depth = depth
|
|
87
|
+
@enabled = enabled
|
|
88
|
+
@read_only = read_only
|
|
89
|
+
@namespace = namespace
|
|
90
|
+
@async = async
|
|
91
|
+
@turns = []
|
|
92
|
+
@loaded = false
|
|
93
|
+
|
|
94
|
+
adapter_opts[:namespace] = namespace if namespace
|
|
95
|
+
adapter_opts[:on_trim] = on_trim if on_trim
|
|
96
|
+
|
|
97
|
+
@adapter = resolve_adapter(adapter, adapter_opts)
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
# -------------------------------------------------------------------------
|
|
101
|
+
# Public API
|
|
102
|
+
# -------------------------------------------------------------------------
|
|
103
|
+
|
|
104
|
+
# Load history from storage into RAM.
|
|
105
|
+
# Called automatically by the agent at the start of #call.
|
|
106
|
+
# After loading, history is available via #turns and #to_messages
|
|
107
|
+
# for manual injection in hooks or prompt classes.
|
|
108
|
+
def load
|
|
109
|
+
return unless @enabled
|
|
110
|
+
return if @loaded
|
|
111
|
+
|
|
112
|
+
@adapter.open(@session_id)
|
|
113
|
+
@turns = @adapter.read
|
|
114
|
+
@loaded = true
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
# Record a turn after a successful agent call.
|
|
118
|
+
def record(request:, response:, **meta)
|
|
119
|
+
return unless @enabled
|
|
120
|
+
return if @read_only
|
|
121
|
+
|
|
122
|
+
turn = { request: request.to_s, response: response.to_s }
|
|
123
|
+
turn.merge!(meta) unless meta.empty?
|
|
124
|
+
turn[:at] ||= Time.now.utc.strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
125
|
+
|
|
126
|
+
@turns << turn
|
|
127
|
+
|
|
128
|
+
if @async
|
|
129
|
+
Thread.new { safe_write(turn) }
|
|
130
|
+
else
|
|
131
|
+
safe_write(turn)
|
|
132
|
+
# keep RAM in sync with what adapter stored (adapter may have trimmed)
|
|
133
|
+
@turns = @adapter.read
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# Returns messages array for LLM consumption, respecting depth.
|
|
138
|
+
# Optional filters:
|
|
139
|
+
# filter: ->(turn) { turn[:agent] == "SupportAgent" }
|
|
140
|
+
# since: Time.now - 3600
|
|
141
|
+
def to_messages(filter: nil, since: nil)
|
|
142
|
+
turns = @turns.dup
|
|
143
|
+
turns.select! { |t| filter.call(t) } if filter
|
|
144
|
+
turns.select! { |t| after?(t, since) } if since
|
|
145
|
+
turns = turns.last(@depth) if @depth
|
|
146
|
+
|
|
147
|
+
turns.flat_map do |t|
|
|
148
|
+
[
|
|
149
|
+
{ role: "user", content: t[:request] },
|
|
150
|
+
{ role: "assistant", content: t[:response] }
|
|
151
|
+
]
|
|
152
|
+
end
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
# All stored turns without depth/filter trimming.
|
|
156
|
+
def turns
|
|
157
|
+
@turns.dup
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
# Number of turns currently in memory.
|
|
161
|
+
def size
|
|
162
|
+
@turns.size
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
# Clear in-RAM turns (does not touch the storage file/key).
|
|
166
|
+
def clear
|
|
167
|
+
@turns = []
|
|
168
|
+
@loaded = false
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
# Delete the session from storage backend entirely.
|
|
172
|
+
def delete
|
|
173
|
+
return unless @enabled
|
|
174
|
+
|
|
175
|
+
@adapter.open(@session_id) unless @loaded
|
|
176
|
+
@adapter.delete
|
|
177
|
+
clear
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
# Flush and close the adapter.
|
|
181
|
+
def close
|
|
182
|
+
@adapter.close
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
# -------------------------------------------------------------------------
|
|
186
|
+
private
|
|
187
|
+
# -------------------------------------------------------------------------
|
|
188
|
+
|
|
189
|
+
def resolve_adapter(adapter, opts)
|
|
190
|
+
case adapter
|
|
191
|
+
when Symbol
|
|
192
|
+
factory = ADAPTERS[adapter]
|
|
193
|
+
raise ArgumentError, "Unknown adapter: #{adapter.inspect}. Available: #{ADAPTERS.keys.join(', ')}" unless factory
|
|
194
|
+
factory.call(**opts)
|
|
195
|
+
else
|
|
196
|
+
# assume an adapter instance
|
|
197
|
+
adapter
|
|
198
|
+
end
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
def safe_write(turn)
|
|
202
|
+
@adapter.open(@session_id) unless @loaded
|
|
203
|
+
@adapter.write(turn)
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
def after?(turn, time)
|
|
207
|
+
return true unless turn[:at]
|
|
208
|
+
turn_time = Time.parse(turn[:at].to_s) rescue nil
|
|
209
|
+
turn_time ? turn_time >= time : true
|
|
210
|
+
end
|
|
211
|
+
end
|
|
212
|
+
end
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
module ActiveHarness
|
|
2
|
+
class Pipeline
|
|
3
|
+
class Step
|
|
4
|
+
attr_reader :name, :agent_class
|
|
5
|
+
|
|
6
|
+
def initialize(name, agent_class = nil, &block)
|
|
7
|
+
@name = name
|
|
8
|
+
@agent_class = agent_class
|
|
9
|
+
@stop_if = nil
|
|
10
|
+
instance_eval(&block) if block_given?
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
# DSL: use TranslationAgent
|
|
14
|
+
def use(klass)
|
|
15
|
+
@agent_class = klass
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
# DSL (inside block): stop_if ->(result) { ... }
|
|
19
|
+
# Getter (external): step.stop_if → lambda or nil
|
|
20
|
+
def stop_if(lam = nil)
|
|
21
|
+
lam ? @stop_if = lam : @stop_if
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# True if agent_class is a Tribunal subclass — tribunal steps do not update payload.
|
|
25
|
+
def tribunal?
|
|
26
|
+
@agent_class.is_a?(Class) && @agent_class <= ActiveHarness::Tribunal
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# Transform steps update payload to result.output after execution.
|
|
30
|
+
# Guard steps (stop_if present) and tribunal steps leave payload unchanged.
|
|
31
|
+
def transform?
|
|
32
|
+
!tribunal? && @stop_if.nil?
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
module ActiveHarness
|
|
2
|
+
# Sequential pipeline that chains agents and tribunals.
|
|
3
|
+
# Each step receives the current payload and can transform it or stop the pipeline.
|
|
4
|
+
#
|
|
5
|
+
# Usage (subclass with DSL):
|
|
6
|
+
#
|
|
7
|
+
# class SupportPipeline < ActiveHarness::Pipeline
|
|
8
|
+
# step :injection_guard do
|
|
9
|
+
# use InjectionGuardAgent
|
|
10
|
+
# stop_if ->(result) { result.parsed["detected"] == true }
|
|
11
|
+
# end
|
|
12
|
+
#
|
|
13
|
+
# step :translate, TranslationAgent # shorthand — no stop_if
|
|
14
|
+
#
|
|
15
|
+
# step :safety_tribunal do
|
|
16
|
+
# use SafetyTribunal
|
|
17
|
+
# stop_if ->(result) { result.verdict == false }
|
|
18
|
+
# end
|
|
19
|
+
#
|
|
20
|
+
# on :before_step do |step_name, payload| ... end
|
|
21
|
+
# on :after_step do |step_name, result| ... end
|
|
22
|
+
# on :before_step, :translate do |payload| ... end
|
|
23
|
+
# on :after_step, :translate do |result| ... end
|
|
24
|
+
# on :stopped do |step_name, result| ... end
|
|
25
|
+
# on :complete do |last_result| ... end
|
|
26
|
+
# end
|
|
27
|
+
#
|
|
28
|
+
# pipeline = SupportPipeline.new(input: "...", context: { user_id: 1 })
|
|
29
|
+
# pipeline.call
|
|
30
|
+
# pipeline.output # => final payload string (nil if stopped)
|
|
31
|
+
# pipeline.stopped? # => false
|
|
32
|
+
# pipeline.step_results # => { translate: <Result>, ... }
|
|
33
|
+
#
|
|
34
|
+
class Pipeline
|
|
35
|
+
VALID_HOOKS = %i[before_step after_step stopped complete].freeze
|
|
36
|
+
VALID_STEP_HOOKS = %i[before_step after_step].freeze
|
|
37
|
+
|
|
38
|
+
# -------------------------------------------------------------------------
|
|
39
|
+
# Class-level DSL
|
|
40
|
+
# -------------------------------------------------------------------------
|
|
41
|
+
class << self
|
|
42
|
+
# Define a step in the pipeline.
|
|
43
|
+
#
|
|
44
|
+
# Shorthand (agent only, no stop_if):
|
|
45
|
+
# step :translate, TranslationAgent
|
|
46
|
+
#
|
|
47
|
+
# Full block form:
|
|
48
|
+
# step :injection_guard do
|
|
49
|
+
# use InjectionGuardAgent
|
|
50
|
+
# stop_if ->(result) { result.parsed["detected"] == true }
|
|
51
|
+
# end
|
|
52
|
+
def step(name, agent_class = nil, &block)
|
|
53
|
+
pipeline_config[:steps] << Pipeline::Step.new(name, agent_class, &block)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# Register a global or per-step hook.
|
|
57
|
+
#
|
|
58
|
+
# Global hooks fire on every step:
|
|
59
|
+
# on :before_step do |step_name, payload| ... end
|
|
60
|
+
# on :after_step do |step_name, result| ... end
|
|
61
|
+
# on :stopped do |step_name, result| ... end
|
|
62
|
+
# on :complete do |last_result| ... end
|
|
63
|
+
#
|
|
64
|
+
# Per-step hooks fire only for the named step (no step_name passed):
|
|
65
|
+
# on :before_step, :translate do |payload| ... end
|
|
66
|
+
# on :after_step, :translate do |result| ... end
|
|
67
|
+
def on(event, step_name = nil, &block)
|
|
68
|
+
if step_name
|
|
69
|
+
unless VALID_STEP_HOOKS.include?(event)
|
|
70
|
+
raise ArgumentError,
|
|
71
|
+
"Per-step hooks support: #{VALID_STEP_HOOKS.join(", ")}. Got :#{event}"
|
|
72
|
+
end
|
|
73
|
+
pipeline_config[:step_hooks][step_name] ||= {}
|
|
74
|
+
pipeline_config[:step_hooks][step_name][event] = block
|
|
75
|
+
else
|
|
76
|
+
unless VALID_HOOKS.include?(event)
|
|
77
|
+
raise ArgumentError,
|
|
78
|
+
"Unknown Pipeline hook :#{event}. Valid: #{VALID_HOOKS.join(", ")}"
|
|
79
|
+
end
|
|
80
|
+
pipeline_config[:hooks][event] = block
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# Rails-style aliases for +on+:
|
|
85
|
+
#
|
|
86
|
+
# Global:
|
|
87
|
+
# before :step do |name, payload| end # → on :before_step
|
|
88
|
+
# after :step do |name, result| end # → on :after_step
|
|
89
|
+
# callback :stopped do |name, result| end # → on :stopped
|
|
90
|
+
# callback :complete do |result| end # → on :complete
|
|
91
|
+
#
|
|
92
|
+
# Per-step:
|
|
93
|
+
# after :step, :translate do |result| end
|
|
94
|
+
# before :step, :translate do |payload| end
|
|
95
|
+
def before(event, step_name = nil, &block)
|
|
96
|
+
on(:"before_#{event}", step_name, &block)
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def after(event, step_name = nil, &block)
|
|
100
|
+
on(:"after_#{event}", step_name, &block)
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def callback(event, &block)
|
|
104
|
+
on(event, &block)
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def pipeline_config
|
|
108
|
+
@pipeline_config ||= { steps: [], hooks: {}, step_hooks: {} }
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# Each subclass gets its own isolated config.
|
|
112
|
+
def inherited(subclass)
|
|
113
|
+
subclass.instance_variable_set(
|
|
114
|
+
:@pipeline_config,
|
|
115
|
+
{ steps: [], hooks: {}, step_hooks: {} }
|
|
116
|
+
)
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
# -------------------------------------------------------------------------
|
|
121
|
+
# Instance API
|
|
122
|
+
# -------------------------------------------------------------------------
|
|
123
|
+
attr_reader :original_input, :output, :stopped_at, :stop_reason,
|
|
124
|
+
:execution_time, :step_results, :context
|
|
125
|
+
|
|
126
|
+
def initialize(input:, context: {}, memory: nil)
|
|
127
|
+
@original_input = input
|
|
128
|
+
@payload = input
|
|
129
|
+
@context = context.dup
|
|
130
|
+
@memory = memory
|
|
131
|
+
@step_results = {}
|
|
132
|
+
@stopped = false
|
|
133
|
+
@stopped_at = nil
|
|
134
|
+
@stop_reason = nil
|
|
135
|
+
@execution_time = nil
|
|
136
|
+
@output = nil
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def stopped?
|
|
140
|
+
@stopped
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
# Execute all steps sequentially. Returns self for chaining.
|
|
144
|
+
def call
|
|
145
|
+
config = self.class.pipeline_config
|
|
146
|
+
t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
147
|
+
|
|
148
|
+
@memory&.load
|
|
149
|
+
|
|
150
|
+
config[:steps].each do |step|
|
|
151
|
+
fire_global(:before_step, step.name, @payload, config)
|
|
152
|
+
fire_step(:before_step, step.name, @payload, config)
|
|
153
|
+
|
|
154
|
+
result = execute_step(step)
|
|
155
|
+
|
|
156
|
+
@step_results[step.name] = result
|
|
157
|
+
@context[step.name] = result
|
|
158
|
+
@payload = result.output if step.transform?
|
|
159
|
+
|
|
160
|
+
fire_global(:after_step, step.name, result, config)
|
|
161
|
+
fire_step(:after_step, step.name, result, config)
|
|
162
|
+
|
|
163
|
+
if step.stop_if && step.stop_if.call(result)
|
|
164
|
+
@stopped = true
|
|
165
|
+
@stopped_at = step.name
|
|
166
|
+
@stop_reason = result
|
|
167
|
+
config[:hooks][:stopped]&.call(step.name, result)
|
|
168
|
+
break
|
|
169
|
+
end
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
@execution_time = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - t0).round(3)
|
|
173
|
+
@output = @payload unless @stopped
|
|
174
|
+
|
|
175
|
+
unless @stopped
|
|
176
|
+
@memory&.record(
|
|
177
|
+
request: @original_input,
|
|
178
|
+
response: @output,
|
|
179
|
+
pipeline: self.class.name
|
|
180
|
+
)
|
|
181
|
+
|
|
182
|
+
last_result = @step_results[@step_results.keys.last]
|
|
183
|
+
config[:hooks][:complete]&.call(last_result)
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
self
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
private
|
|
190
|
+
|
|
191
|
+
def execute_step(step)
|
|
192
|
+
step.agent_class.new(input: @payload, context: @context.dup).call
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
# Global hook: receives (step_name, data)
|
|
196
|
+
def fire_global(event, step_name, data, config)
|
|
197
|
+
config[:hooks][event]&.call(step_name, data)
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
# Per-step hook: receives (data) only
|
|
201
|
+
def fire_step(event, step_name, data, config)
|
|
202
|
+
config[:step_hooks][step_name]&.dig(event)&.call(data)
|
|
203
|
+
end
|
|
204
|
+
end
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
require_relative "pipeline/step"
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# Provider Contract
|
|
2
|
+
|
|
3
|
+
Each provider class must inherit from `Providers::Base` and implement a single public method:
|
|
4
|
+
|
|
5
|
+
```ruby
|
|
6
|
+
def call(model:, messages:, temperature: 0.7) → Hash
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
## Return value
|
|
10
|
+
|
|
11
|
+
A plain Hash with exactly these keys:
|
|
12
|
+
|
|
13
|
+
| Key | Type | Description |
|
|
14
|
+
| ----------- | ------ | -------------------------------------------------- |
|
|
15
|
+
| `:content` | String | The model's text reply (stripped) |
|
|
16
|
+
| `:provider` | Symbol | Provider identifier, e.g. `:openai`, `:openrouter` |
|
|
17
|
+
| `:model` | String | Actual model name returned by the API |
|
|
18
|
+
|
|
19
|
+
```ruby
|
|
20
|
+
{
|
|
21
|
+
content: "Washington, D.C.",
|
|
22
|
+
provider: :openrouter,
|
|
23
|
+
model: "mistralai/mistral-nemo"
|
|
24
|
+
}
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
## Errors
|
|
28
|
+
|
|
29
|
+
All exceptions must be subclasses of `ActiveHarness::Errors::ProviderError` and carry:
|
|
30
|
+
|
|
31
|
+
| Attribute | Type | Description |
|
|
32
|
+
| ------------ | ------------- | ----------------------------------------------------------------------- |
|
|
33
|
+
| `message` | String | Human-readable error text from the API |
|
|
34
|
+
| `error_code` | String or nil | Raw code from the API response (`"429"`, `"invalid_api_key"`, etc.) |
|
|
35
|
+
| `metadata` | Hash or nil | Extra data returned by the API (rate-limit timing, provider name, etc.) |
|
|
36
|
+
|
|
37
|
+
```ruby
|
|
38
|
+
raise Errors::RateLimitError.new(msg, error_code: code, metadata: metadata)
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
### Error classes and retry behaviour
|
|
42
|
+
|
|
43
|
+
| Class | Retryable | Typical trigger |
|
|
44
|
+
| -------------------------- | --------- | -------------------------------- |
|
|
45
|
+
| `TimeoutError` | yes | Network open/read timeout |
|
|
46
|
+
| `RateLimitError` | yes | HTTP 429 / `rate_limit_exceeded` |
|
|
47
|
+
| `ServerError` | yes | API `server_error` type |
|
|
48
|
+
| `ProviderUnavailableError` | yes | HTTP 5xx, host unreachable |
|
|
49
|
+
| `InvalidRequestError` | no | Bad request, unknown model, etc. |
|
|
50
|
+
| `InvalidApiKeyError` | no | Missing or invalid API key |
|
|
51
|
+
| `SafetyBlockedError` | no | Content policy violation |
|
|
52
|
+
|
|
53
|
+
Retryable errors cause the agent to move to the next model in the chain.
|
|
54
|
+
Non-retryable errors abort the chain immediately and are re-raised.
|