active_harness 0.2.42 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/active_harness/configuration.rb +3 -1
- data/lib/active_harness/core/errors.rb +2 -2
- data/lib/active_harness/core/hooks.rb +2 -2
- data/lib/active_harness/memory.rb +11 -11
- data/lib/active_harness/pipeline/README.md +15 -15
- data/lib/active_harness/pipeline/step.rb +1 -1
- data/lib/active_harness/pipeline.rb +18 -18
- data/lib/active_harness/providers/PROVIDER_CONTRACT.md +1 -1
- data/lib/active_harness/providers/azure.rb +1 -1
- data/lib/active_harness/providers/bedrock.rb +2 -2
- data/lib/active_harness/providers/custom.rb +1 -1
- data/lib/active_harness/providers/images/openrouter.rb +49 -1
- data/lib/active_harness/providers/vertexai.rb +2 -2
- data/lib/active_harness/railtie.rb +1 -1
- data/lib/active_harness/{agent → request}/cost.rb +1 -1
- data/lib/active_harness/{agent → request}/custom_llm_backend.rb +4 -4
- data/lib/active_harness/{agent → request}/hooks.rb +4 -4
- data/lib/active_harness/{agent → request}/image.rb +7 -7
- data/lib/active_harness/{agent → request}/models.rb +15 -15
- data/lib/active_harness/{agent → request}/output_parser.rb +1 -1
- data/lib/active_harness/{agent → request}/prompt.rb +7 -7
- data/lib/active_harness/{agent → request}/providers.rb +1 -1
- data/lib/active_harness/{agent → request}/transcription.rb +7 -7
- data/lib/active_harness/{agent.rb → request.rb} +19 -19
- data/lib/active_harness/result.rb +3 -3
- data/lib/active_harness/tribunal/dsl.rb +7 -7
- data/lib/active_harness/tribunal/hooks.rb +9 -9
- data/lib/active_harness/tribunal.rb +62 -62
- data/lib/active_harness.rb +2 -2
- data/lib/generators/active_harness/install/install_generator.rb +1 -1
- data/lib/generators/active_harness/install/templates/controllers/ai_controller.rb +3 -3
- data/lib/generators/active_harness/install/templates/initializers/active_harness.rb +2 -2
- data/lib/generators/active_harness/install/templates/pipelines/support_pipeline.rb +2 -2
- data/lib/generators/active_harness/install/templates/{agents/support_guard_agent.rb → requests/support_guard_request.rb} +1 -1
- data/lib/generators/active_harness/install/templates/{agents/support_agent.rb → requests/support_request.rb} +1 -1
- data/lib/generators/active_harness/install/templates/tribunals/support_guard_tribunal.rb +3 -3
- data/lib/generators/active_harness/pipeline/templates/pipeline.rb.tt +1 -1
- data/lib/generators/active_harness/request/request_generator.rb +16 -0
- data/lib/generators/active_harness/{agent/templates/agent.rb.tt → request/templates/request.rb.tt} +1 -1
- data/lib/generators/active_harness/tribunal/templates/tribunal.rb.tt +1 -1
- metadata +17 -17
- data/lib/generators/active_harness/agent/agent_generator.rb +0 -16
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
module ActiveHarness
|
|
2
|
-
class
|
|
2
|
+
class Request
|
|
3
3
|
class << self
|
|
4
4
|
# Array-style model list:
|
|
5
5
|
#
|
|
@@ -8,10 +8,10 @@ module ActiveHarness
|
|
|
8
8
|
# { provider: :openrouter, model: "mistralai/mistral-nemo" }
|
|
9
9
|
# ]
|
|
10
10
|
def models(list)
|
|
11
|
-
|
|
11
|
+
request_config[:models] = Array(list)
|
|
12
12
|
end
|
|
13
13
|
|
|
14
|
-
# Output format for this
|
|
14
|
+
# Output format for this request.
|
|
15
15
|
#
|
|
16
16
|
# format :text # default — output is returned as-is
|
|
17
17
|
# format :json # output is parsed; result.processed is a Ruby Hash/Array
|
|
@@ -20,7 +20,7 @@ module ActiveHarness
|
|
|
20
20
|
raise ArgumentError, "Unknown format :#{type}. Valid values: :text, :json"
|
|
21
21
|
end
|
|
22
22
|
|
|
23
|
-
|
|
23
|
+
request_config[:format] = type
|
|
24
24
|
end
|
|
25
25
|
|
|
26
26
|
# Block-based model DSL:
|
|
@@ -32,16 +32,16 @@ module ActiveHarness
|
|
|
32
32
|
def model(&block)
|
|
33
33
|
config = ModelConfig.new
|
|
34
34
|
config.instance_eval(&block)
|
|
35
|
-
|
|
35
|
+
request_config[:model] = config.to_h
|
|
36
36
|
end
|
|
37
37
|
end
|
|
38
38
|
|
|
39
|
-
# Public instance API — returns a ModelList proxy for this
|
|
39
|
+
# Public instance API — returns a ModelList proxy for this request instance.
|
|
40
40
|
#
|
|
41
|
-
# Allows adding models at runtime before calling the
|
|
41
|
+
# Allows adding models at runtime before calling the request:
|
|
42
42
|
#
|
|
43
|
-
#
|
|
44
|
-
#
|
|
43
|
+
# request.models.prepend([{ provider: :openrouter, model: "..." }])
|
|
44
|
+
# request.models.push([{ provider: :openrouter, model: "..." }])
|
|
45
45
|
#
|
|
46
46
|
# Any prepended/pushed models are combined with the class-defined chain:
|
|
47
47
|
# [prepended...] + [class chain / constructor override] + [pushed...]
|
|
@@ -69,7 +69,7 @@ module ActiveHarness
|
|
|
69
69
|
end
|
|
70
70
|
|
|
71
71
|
# ---------------------------------------------------------------------------
|
|
72
|
-
# ModelList — mutable proxy returned by
|
|
72
|
+
# ModelList — mutable proxy returned by request.models
|
|
73
73
|
# ---------------------------------------------------------------------------
|
|
74
74
|
class ModelList
|
|
75
75
|
def initialize(base_models)
|
|
@@ -93,7 +93,7 @@ module ActiveHarness
|
|
|
93
93
|
|
|
94
94
|
# Replace the entire chain, discarding prepended/appended too.
|
|
95
95
|
#
|
|
96
|
-
#
|
|
96
|
+
# request.models.replace([{ provider: :openrouter, model: "..." }])
|
|
97
97
|
def replace(models)
|
|
98
98
|
@base = Array(models)
|
|
99
99
|
@prepended = []
|
|
@@ -104,9 +104,9 @@ module ActiveHarness
|
|
|
104
104
|
# Insert a single model entry at an arbitrary position in the final chain.
|
|
105
105
|
# Position 0 is the same as prepend; -1 appends after the last element.
|
|
106
106
|
#
|
|
107
|
-
#
|
|
108
|
-
#
|
|
109
|
-
#
|
|
107
|
+
# request.models.insert(0, { provider: :openrouter, model: "fast-model" })
|
|
108
|
+
# request.models.insert(2, { provider: :openrouter, model: "mid-model" })
|
|
109
|
+
# request.models.insert(-1, { provider: :openrouter, model: "last-resort" })
|
|
110
110
|
def insert(position, model)
|
|
111
111
|
to_a.tap { |list| list.insert(position, model) }.then do |new_list|
|
|
112
112
|
@prepended = []
|
|
@@ -127,7 +127,7 @@ module ActiveHarness
|
|
|
127
127
|
end
|
|
128
128
|
|
|
129
129
|
# ---------------------------------------------------------------------------
|
|
130
|
-
# DSL builder — used by
|
|
130
|
+
# DSL builder — used by Request.model { ... }
|
|
131
131
|
# ---------------------------------------------------------------------------
|
|
132
132
|
class ModelConfig
|
|
133
133
|
def initialize
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
module ActiveHarness
|
|
2
|
-
class
|
|
2
|
+
class Request
|
|
3
3
|
class << self
|
|
4
|
-
# System prompt for this
|
|
4
|
+
# System prompt for this request.
|
|
5
5
|
# Accepts:
|
|
6
6
|
# - a String → used as-is
|
|
7
7
|
# - a Class → instantiated and resolved via #call or #text
|
|
@@ -11,7 +11,7 @@ module ActiveHarness
|
|
|
11
11
|
# system_prompt MyPromptClass
|
|
12
12
|
# system_prompt -> { "Dynamic prompt built at #{Time.now}" }
|
|
13
13
|
def system_prompt(text_or_class)
|
|
14
|
-
|
|
14
|
+
request_config[:system_prompt] = text_or_class
|
|
15
15
|
end
|
|
16
16
|
alias prompt system_prompt
|
|
17
17
|
end
|
|
@@ -32,13 +32,13 @@ module ActiveHarness
|
|
|
32
32
|
when Proc then instance_exec(&sp)
|
|
33
33
|
when Class
|
|
34
34
|
instance = sp.new
|
|
35
|
-
|
|
35
|
+
inject_request_state(instance)
|
|
36
36
|
if instance.respond_to?(:call) then instance.call
|
|
37
37
|
elsif instance.respond_to?(:text) then instance.text
|
|
38
38
|
else instance.to_s
|
|
39
39
|
end
|
|
40
40
|
else
|
|
41
|
-
|
|
41
|
+
inject_request_state(sp)
|
|
42
42
|
if sp.respond_to?(:call) then sp.call
|
|
43
43
|
elsif sp.respond_to?(:text) then sp.text
|
|
44
44
|
else sp.to_s
|
|
@@ -49,9 +49,9 @@ module ActiveHarness
|
|
|
49
49
|
prompt
|
|
50
50
|
end
|
|
51
51
|
|
|
52
|
-
# Injects
|
|
52
|
+
# Injects request state into a prompt class instance before #call.
|
|
53
53
|
# Available in prompt classes: @input, @context, @params, @memory, @context_window, @config
|
|
54
|
-
def
|
|
54
|
+
def inject_request_state(obj)
|
|
55
55
|
obj.instance_variable_set(:@input, @input)
|
|
56
56
|
obj.instance_variable_set(:@context, @context)
|
|
57
57
|
obj.instance_variable_set(:@params, @params)
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
module ActiveHarness
|
|
2
|
-
class
|
|
2
|
+
class Request
|
|
3
3
|
class << self
|
|
4
|
-
# Mark this
|
|
4
|
+
# Mark this request as an audio transcription request.
|
|
5
5
|
#
|
|
6
|
-
# class
|
|
6
|
+
# class TranscriptionRequest < ActiveHarness::Request
|
|
7
7
|
# transcribe true
|
|
8
8
|
#
|
|
9
9
|
# model do
|
|
@@ -16,13 +16,13 @@ module ActiveHarness
|
|
|
16
16
|
# result.output — transcribed text (String)
|
|
17
17
|
# result.processed — same as output (format :text default)
|
|
18
18
|
def transcribe(value = true)
|
|
19
|
-
|
|
19
|
+
request_config[:transcribe] = value
|
|
20
20
|
end
|
|
21
21
|
|
|
22
|
-
# Default ISO-639-1 language hint for all models in this
|
|
22
|
+
# Default ISO-639-1 language hint for all models in this request's chain.
|
|
23
23
|
# Can be overridden per-model via: use provider: :openrouter, model: "...", language: "ja"
|
|
24
24
|
def language(default_language)
|
|
25
|
-
|
|
25
|
+
request_config[:transcribe_language] = default_language
|
|
26
26
|
end
|
|
27
27
|
end
|
|
28
28
|
|
|
@@ -38,7 +38,7 @@ module ActiveHarness
|
|
|
38
38
|
|
|
39
39
|
alias_method :_normalize_input_before_transcribe, :normalize_input!
|
|
40
40
|
|
|
41
|
-
# @input is a file path for transcription
|
|
41
|
+
# @input is a file path for transcription requests, not free text —
|
|
42
42
|
# stripping/collapsing whitespace could corrupt a path. Skip it.
|
|
43
43
|
def normalize_input!
|
|
44
44
|
return if @config[:transcribe]
|
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
require "json"
|
|
2
2
|
|
|
3
3
|
module ActiveHarness
|
|
4
|
-
class
|
|
4
|
+
class Request
|
|
5
5
|
# -------------------------------------------------------------------------
|
|
6
6
|
# Class-level DSL — core
|
|
7
7
|
# -------------------------------------------------------------------------
|
|
8
8
|
class << self
|
|
9
9
|
# Class-level entry point.
|
|
10
10
|
#
|
|
11
|
-
#
|
|
12
|
-
#
|
|
11
|
+
# SupportRequest.call(input: "Hi")
|
|
12
|
+
# SupportRequest.call(input: "Hi", context: { user_id: 42 })
|
|
13
13
|
def call(
|
|
14
14
|
input: nil,
|
|
15
15
|
context: {},
|
|
@@ -31,12 +31,12 @@ module ActiveHarness
|
|
|
31
31
|
end
|
|
32
32
|
|
|
33
33
|
# Each subclass gets its own isolated config hash.
|
|
34
|
-
def
|
|
35
|
-
@
|
|
34
|
+
def request_config
|
|
35
|
+
@request_config ||= {}
|
|
36
36
|
end
|
|
37
37
|
|
|
38
38
|
def inherited(subclass)
|
|
39
|
-
subclass.instance_variable_set(:@
|
|
39
|
+
subclass.instance_variable_set(:@request_config, {})
|
|
40
40
|
end
|
|
41
41
|
|
|
42
42
|
# Automatically strip and collapse whitespace in @input before each call.
|
|
@@ -44,7 +44,7 @@ module ActiveHarness
|
|
|
44
44
|
#
|
|
45
45
|
# normalize_input false
|
|
46
46
|
def normalize_input(value = true)
|
|
47
|
-
|
|
47
|
+
request_config[:normalize_input] = value
|
|
48
48
|
end
|
|
49
49
|
end
|
|
50
50
|
|
|
@@ -74,7 +74,7 @@ module ActiveHarness
|
|
|
74
74
|
stream: nil
|
|
75
75
|
)
|
|
76
76
|
@input = input
|
|
77
|
-
@config = self.class.
|
|
77
|
+
@config = self.class.request_config
|
|
78
78
|
normalize_input!
|
|
79
79
|
@context = context
|
|
80
80
|
@params = params
|
|
@@ -89,8 +89,8 @@ module ActiveHarness
|
|
|
89
89
|
# Raises Errors::AllModelsFailed if every model in the chain fails.
|
|
90
90
|
#
|
|
91
91
|
# Optionally accepts input and stream callback inline:
|
|
92
|
-
#
|
|
93
|
-
#
|
|
92
|
+
# request.call("What is the capital of Japan?")
|
|
93
|
+
# request.call("...", stream: ->(token) { print token })
|
|
94
94
|
def call(input = nil, token: nil, stream: nil)
|
|
95
95
|
if input
|
|
96
96
|
@input = input
|
|
@@ -205,13 +205,13 @@ module ActiveHarness
|
|
|
205
205
|
end
|
|
206
206
|
end
|
|
207
207
|
|
|
208
|
-
require_relative "
|
|
209
|
-
require_relative "
|
|
210
|
-
require_relative "
|
|
211
|
-
require_relative "
|
|
212
|
-
require_relative "
|
|
213
|
-
require_relative "
|
|
214
|
-
require_relative "
|
|
215
|
-
require_relative "
|
|
216
|
-
require_relative "
|
|
208
|
+
require_relative "request/prompt"
|
|
209
|
+
require_relative "request/hooks"
|
|
210
|
+
require_relative "request/models"
|
|
211
|
+
require_relative "request/providers"
|
|
212
|
+
require_relative "request/output_parser"
|
|
213
|
+
require_relative "request/custom_llm_backend"
|
|
214
|
+
require_relative "request/cost"
|
|
215
|
+
require_relative "request/image"
|
|
216
|
+
require_relative "request/transcription"
|
|
217
217
|
|
|
@@ -33,16 +33,16 @@ module ActiveHarness
|
|
|
33
33
|
# nil when pricing data is unavailable.
|
|
34
34
|
CostBreakdown = Struct.new(:input, :output, :total, keyword_init: true)
|
|
35
35
|
|
|
36
|
-
# Combined token + cost stats for a single
|
|
36
|
+
# Combined token + cost stats for a single request call.
|
|
37
37
|
# tokens is always present (raw provider data).
|
|
38
38
|
# cost is nil when pricing is unavailable.
|
|
39
39
|
UsageInfo = Struct.new(:tokens, :cost, keyword_init: true)
|
|
40
40
|
|
|
41
|
-
# Result returned by
|
|
41
|
+
# Result returned by Request#call (accessible via request.result).
|
|
42
42
|
#
|
|
43
43
|
# result.input — original input string
|
|
44
44
|
# result.output — raw string from the provider
|
|
45
|
-
# result.processed — parsed Hash/Array for :json
|
|
45
|
+
# result.processed — parsed Hash/Array for :json requests, raw string for :text
|
|
46
46
|
# result.system_prompt — resolved system prompt string
|
|
47
47
|
# result.model — ModelInfo (name, provider, temperature, context_window, pricing)
|
|
48
48
|
# result.model_list — full model chain proxy
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
module ActiveHarness
|
|
2
2
|
class Tribunal
|
|
3
3
|
class << self
|
|
4
|
-
# Declare
|
|
4
|
+
# Declare requests at the class level.
|
|
5
5
|
#
|
|
6
|
-
#
|
|
7
|
-
#
|
|
8
|
-
def
|
|
9
|
-
tribunal_config[:
|
|
6
|
+
# requests PolitenessRequest, ConstructivenessRequest
|
|
7
|
+
# requests [PolitenessRequest, ConstructivenessRequest]
|
|
8
|
+
def requests(*list)
|
|
9
|
+
tribunal_config[:requests] = list.flatten
|
|
10
10
|
end
|
|
11
11
|
|
|
12
12
|
# Class-level process block — defines how to compute the verdict from all results.
|
|
@@ -25,8 +25,8 @@ module ActiveHarness
|
|
|
25
25
|
# :majority — verdict true when more than half of successful results evaluate to true
|
|
26
26
|
#
|
|
27
27
|
# Options:
|
|
28
|
-
# may_fail: N — tolerate up to N
|
|
29
|
-
# (default: nil — raise only when all
|
|
28
|
+
# may_fail: N — tolerate up to N request errors before raising AllRequestsFailed
|
|
29
|
+
# (default: nil — raise only when all requests fail, preserving legacy behavior)
|
|
30
30
|
#
|
|
31
31
|
# The block receives a single Result and must return a truthy/falsy value.
|
|
32
32
|
#
|
|
@@ -2,9 +2,9 @@ module ActiveHarness
|
|
|
2
2
|
class Tribunal
|
|
3
3
|
VALID_HOOKS = %i[
|
|
4
4
|
before_call
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
5
|
+
before_request
|
|
6
|
+
after_request
|
|
7
|
+
request_error
|
|
8
8
|
after_call
|
|
9
9
|
before_verdict
|
|
10
10
|
after_verdict
|
|
@@ -14,9 +14,9 @@ module ActiveHarness
|
|
|
14
14
|
# Class-level hook registration.
|
|
15
15
|
#
|
|
16
16
|
# on :before_call do ... end
|
|
17
|
-
# on :
|
|
18
|
-
# on :
|
|
19
|
-
# on :
|
|
17
|
+
# on :before_request do |request| ... end
|
|
18
|
+
# on :after_request do |result| ... end
|
|
19
|
+
# on :request_error do |name, error| ... end
|
|
20
20
|
# on :after_call do |results, errors| ... end
|
|
21
21
|
# on :before_verdict do |results| results end # transform hook
|
|
22
22
|
# on :after_verdict do |verdict| ... end
|
|
@@ -32,12 +32,12 @@ module ActiveHarness
|
|
|
32
32
|
# Rails-style aliases for +on+:
|
|
33
33
|
#
|
|
34
34
|
# before :call do ... end # → on :before_call
|
|
35
|
-
# before :
|
|
35
|
+
# before :request do |request| end # → on :before_request
|
|
36
36
|
# before :verdict do |results| end # → on :before_verdict (transform)
|
|
37
37
|
# after :call do |r, e| end # → on :after_call
|
|
38
|
-
# after :
|
|
38
|
+
# after :request do |result| end # → on :after_request
|
|
39
39
|
# after :verdict do |verdict| end # → on :after_verdict
|
|
40
|
-
# callback :
|
|
40
|
+
# callback :request_error do |name, e| end # → on :request_error
|
|
41
41
|
def before(event, &block)
|
|
42
42
|
on(:"before_#{event}", &block)
|
|
43
43
|
end
|
|
@@ -11,19 +11,19 @@ module ActiveHarness
|
|
|
11
11
|
#
|
|
12
12
|
# Direct usage:
|
|
13
13
|
# tribunal = ActiveHarness::Tribunal.new(
|
|
14
|
-
# input:
|
|
15
|
-
# context:
|
|
16
|
-
#
|
|
17
|
-
# timeout:
|
|
14
|
+
# input: "Is this message toxic?",
|
|
15
|
+
# context: { user_id: 42 },
|
|
16
|
+
# requests: [ToxicityRequest, BiasRequest, SpamRequest],
|
|
17
|
+
# timeout: 7
|
|
18
18
|
# )
|
|
19
|
-
# tribunal.on(:
|
|
19
|
+
# tribunal.on(:after_request) { |result| puts result.model }
|
|
20
20
|
# tribunal.process { |results| results.all? { |r| r.processed["result"] == true } }
|
|
21
21
|
# tribunal.call
|
|
22
22
|
#
|
|
23
23
|
# Subclass with DSL:
|
|
24
24
|
# class ContentQualityTribunal < ActiveHarness::Tribunal
|
|
25
|
-
#
|
|
26
|
-
# on(:
|
|
25
|
+
# requests PolitenessRequest, ConstructivenessRequest
|
|
26
|
+
# on(:after_request) { |result| puts result.model }
|
|
27
27
|
# process { |results| results.all? { |r| r.processed["result"] == true } }
|
|
28
28
|
# end
|
|
29
29
|
# ContentQualityTribunal.new(input: "...").call
|
|
@@ -35,11 +35,11 @@ module ActiveHarness
|
|
|
35
35
|
class << self
|
|
36
36
|
# Each subclass gets its own isolated config hash.
|
|
37
37
|
def tribunal_config
|
|
38
|
-
@tribunal_config ||= {
|
|
38
|
+
@tribunal_config ||= { requests: [], hooks: {} }
|
|
39
39
|
end
|
|
40
40
|
|
|
41
41
|
def inherited(subclass)
|
|
42
|
-
subclass.instance_variable_set(:@tribunal_config, {
|
|
42
|
+
subclass.instance_variable_set(:@tribunal_config, { requests: [], hooks: {} })
|
|
43
43
|
end
|
|
44
44
|
end
|
|
45
45
|
|
|
@@ -54,7 +54,7 @@ module ActiveHarness
|
|
|
54
54
|
:errors,
|
|
55
55
|
:verdict,
|
|
56
56
|
:execution_time,
|
|
57
|
-
:
|
|
57
|
+
:request_execution_times,
|
|
58
58
|
:token,
|
|
59
59
|
:stream
|
|
60
60
|
|
|
@@ -63,7 +63,7 @@ module ActiveHarness
|
|
|
63
63
|
context: {},
|
|
64
64
|
params: {},
|
|
65
65
|
memory: nil,
|
|
66
|
-
|
|
66
|
+
requests: nil,
|
|
67
67
|
timeout: 7,
|
|
68
68
|
token: nil,
|
|
69
69
|
stream: nil,
|
|
@@ -71,28 +71,28 @@ module ActiveHarness
|
|
|
71
71
|
)
|
|
72
72
|
config = self.class.tribunal_config
|
|
73
73
|
|
|
74
|
-
@input
|
|
75
|
-
@context
|
|
76
|
-
@params
|
|
77
|
-
@memory
|
|
78
|
-
@
|
|
79
|
-
@timeout
|
|
80
|
-
@process_block
|
|
81
|
-
@strategy
|
|
82
|
-
@evaluate_block
|
|
83
|
-
@may_fail
|
|
84
|
-
@hooks
|
|
85
|
-
@token
|
|
86
|
-
@stream
|
|
87
|
-
@results
|
|
88
|
-
@errors
|
|
89
|
-
@verdict
|
|
90
|
-
@execution_time
|
|
91
|
-
@
|
|
74
|
+
@input = input
|
|
75
|
+
@context = context
|
|
76
|
+
@params = params
|
|
77
|
+
@memory = memory
|
|
78
|
+
@requests = requests || config[:requests]
|
|
79
|
+
@timeout = timeout
|
|
80
|
+
@process_block = config[:process]
|
|
81
|
+
@strategy = config[:strategy]
|
|
82
|
+
@evaluate_block = config[:evaluate_block]
|
|
83
|
+
@may_fail = may_fail == :_unset ? config[:may_fail] : may_fail
|
|
84
|
+
@hooks = config[:hooks].transform_values { |v| Array(v).dup }
|
|
85
|
+
@token = token
|
|
86
|
+
@stream = stream
|
|
87
|
+
@results = []
|
|
88
|
+
@errors = []
|
|
89
|
+
@verdict = nil
|
|
90
|
+
@execution_time = nil
|
|
91
|
+
@request_execution_times = []
|
|
92
92
|
end
|
|
93
93
|
|
|
94
94
|
# Returns a Result with processed: { "verdict" => @verdict } so the pipeline
|
|
95
|
-
# can handle
|
|
95
|
+
# can handle requests and tribunals through the same interface.
|
|
96
96
|
def result
|
|
97
97
|
Result.new(
|
|
98
98
|
input: @input,
|
|
@@ -102,55 +102,55 @@ module ActiveHarness
|
|
|
102
102
|
)
|
|
103
103
|
end
|
|
104
104
|
|
|
105
|
-
# Run all
|
|
105
|
+
# Run all requests in parallel, then compute the verdict.
|
|
106
106
|
# Returns self so calls can be chained: tribunal.call.verdict
|
|
107
107
|
#
|
|
108
108
|
# Accepts an optional input to update payload before running — matches
|
|
109
|
-
# the
|
|
109
|
+
# the Request#call(input) interface so tribunals work as pipeline executors.
|
|
110
110
|
#
|
|
111
111
|
# Behaviour on failure:
|
|
112
|
-
# - If some
|
|
112
|
+
# - If some requests fail/timeout, their errors are in #errors and
|
|
113
113
|
# #results contains only successful results.
|
|
114
|
-
# - If ALL
|
|
114
|
+
# - If ALL requests fail/timeout, raises Errors::AllRequestsFailed.
|
|
115
115
|
def call(input = nil, token: nil, stream: nil)
|
|
116
116
|
@input = input if input
|
|
117
117
|
@token = token if token
|
|
118
118
|
@stream = stream if stream
|
|
119
|
-
|
|
119
|
+
requests = resolve_requests
|
|
120
120
|
fire(:before_call)
|
|
121
121
|
|
|
122
122
|
started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
123
123
|
|
|
124
|
-
futures =
|
|
125
|
-
fire(:
|
|
124
|
+
futures = requests.each_with_index.map do |request, index|
|
|
125
|
+
fire(:before_request, request, index)
|
|
126
126
|
t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
127
|
-
future = Concurrent::Future.execute {
|
|
127
|
+
future = Concurrent::Future.execute { request.call }
|
|
128
128
|
[future, t0]
|
|
129
129
|
end
|
|
130
130
|
|
|
131
|
-
@results
|
|
132
|
-
@errors
|
|
133
|
-
@
|
|
131
|
+
@results = []
|
|
132
|
+
@errors = []
|
|
133
|
+
@request_execution_times = []
|
|
134
134
|
|
|
135
135
|
futures.each_with_index do |(future, t0), index|
|
|
136
136
|
future.wait(@timeout)
|
|
137
137
|
elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - t0).round(3)
|
|
138
|
-
@
|
|
138
|
+
@request_execution_times << { request: requests[index].class.name, time: elapsed }
|
|
139
139
|
|
|
140
140
|
if future.fulfilled?
|
|
141
141
|
value = future.value
|
|
142
|
-
result = value.is_a?(ActiveHarness::
|
|
142
|
+
result = value.is_a?(ActiveHarness::Request) ? value.result : value
|
|
143
143
|
@results << result
|
|
144
|
-
fire(:
|
|
144
|
+
fire(:after_request, result, index)
|
|
145
145
|
elsif future.incomplete?
|
|
146
146
|
error = Errors::TimeoutError.new(
|
|
147
|
-
"
|
|
147
|
+
"Request #{requests[index].class.name} timed out after #{@timeout}s"
|
|
148
148
|
)
|
|
149
|
-
@errors << {
|
|
150
|
-
fire(:
|
|
149
|
+
@errors << { request: requests[index].class.name, error: error }
|
|
150
|
+
fire(:request_error, requests[index].class.name, error, index)
|
|
151
151
|
else
|
|
152
|
-
@errors << {
|
|
153
|
-
fire(:
|
|
152
|
+
@errors << { request: requests[index].class.name, error: future.reason }
|
|
153
|
+
fire(:request_error, requests[index].class.name, future.reason, index)
|
|
154
154
|
end
|
|
155
155
|
end
|
|
156
156
|
|
|
@@ -158,7 +158,7 @@ module ActiveHarness
|
|
|
158
158
|
|
|
159
159
|
fire(:after_call, @results, @errors)
|
|
160
160
|
|
|
161
|
-
# If all
|
|
161
|
+
# If all requests failed, raise an exception.
|
|
162
162
|
# Otherwise, compute the verdict based on successful results.
|
|
163
163
|
check_failure_threshold!
|
|
164
164
|
|
|
@@ -174,26 +174,26 @@ module ActiveHarness
|
|
|
174
174
|
|
|
175
175
|
def check_failure_threshold!
|
|
176
176
|
if !@may_fail.nil? && @errors.size > @may_fail
|
|
177
|
-
raise Errors::
|
|
178
|
-
"Too many
|
|
177
|
+
raise Errors::AllRequestsFailed,
|
|
178
|
+
"Too many requests failed (#{@errors.size} > may_fail: #{@may_fail}) — #{error_summary}"
|
|
179
179
|
elsif @results.empty?
|
|
180
|
-
raise Errors::
|
|
180
|
+
raise Errors::AllRequestsFailed, "All requests failed — #{error_summary}"
|
|
181
181
|
end
|
|
182
182
|
end
|
|
183
183
|
|
|
184
184
|
def error_summary
|
|
185
|
-
@errors.map { |e| "#{e[:
|
|
185
|
+
@errors.map { |e| "#{e[:request]}: #{e[:error].message}" }.join("; ")
|
|
186
186
|
end
|
|
187
187
|
|
|
188
|
-
def
|
|
189
|
-
@
|
|
190
|
-
if
|
|
191
|
-
|
|
188
|
+
def resolve_requests
|
|
189
|
+
@requests.map do |request|
|
|
190
|
+
if request.is_a?(Class)
|
|
191
|
+
request.new(input: @input, context: @context.dup, params: @params, token: @token, stream: @stream)
|
|
192
192
|
else
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
193
|
+
request.input = @input if @input
|
|
194
|
+
request.instance_variable_set(:@token, @token) if @token
|
|
195
|
+
request.instance_variable_set(:@stream, @stream) if @stream
|
|
196
|
+
request
|
|
197
197
|
end
|
|
198
198
|
end
|
|
199
199
|
end
|
data/lib/active_harness.rb
CHANGED
|
@@ -27,14 +27,14 @@ require_relative "active_harness/providers/audio/openai"
|
|
|
27
27
|
require_relative "active_harness/providers/audio/openrouter"
|
|
28
28
|
require "active_harness_pricing"
|
|
29
29
|
require_relative "active_harness/memory"
|
|
30
|
-
require_relative "active_harness/
|
|
30
|
+
require_relative "active_harness/request"
|
|
31
31
|
require_relative "active_harness/tribunal"
|
|
32
32
|
require_relative "active_harness/pipeline"
|
|
33
33
|
|
|
34
34
|
require_relative "active_harness/railtie" if defined?(Rails::Railtie)
|
|
35
35
|
|
|
36
36
|
module ActiveHarness
|
|
37
|
-
VERSION = "0.
|
|
37
|
+
VERSION = "0.3.0"
|
|
38
38
|
|
|
39
39
|
class << self
|
|
40
40
|
# Configure ActiveHarness.
|
|
@@ -7,7 +7,7 @@ module ActiveHarness
|
|
|
7
7
|
|
|
8
8
|
source_root File.expand_path("templates", __dir__)
|
|
9
9
|
|
|
10
|
-
APP_AI_DIRS = %w[
|
|
10
|
+
APP_AI_DIRS = %w[requests prompts tribunals pipelines memory].freeze
|
|
11
11
|
|
|
12
12
|
def create_structure
|
|
13
13
|
APP_AI_DIRS.each do |dir|
|
|
@@ -8,7 +8,7 @@ class AiSupportController < ApplicationController
|
|
|
8
8
|
# body: { input: "What is your return policy?" }
|
|
9
9
|
# ---------------------------------------------------------------------------
|
|
10
10
|
def agent
|
|
11
|
-
result =
|
|
11
|
+
result = SupportRequest.call(input: params.require(:input))
|
|
12
12
|
|
|
13
13
|
render json: {
|
|
14
14
|
output: result.output,
|
|
@@ -26,7 +26,7 @@ class AiSupportController < ApplicationController
|
|
|
26
26
|
# ---------------------------------------------------------------------------
|
|
27
27
|
def agent_memory
|
|
28
28
|
memory = AppMemory.new(session_id: params.require(:session_id))
|
|
29
|
-
result =
|
|
29
|
+
result = SupportRequest.call(input: params.require(:input), memory: memory)
|
|
30
30
|
|
|
31
31
|
render json: {
|
|
32
32
|
output: result.output,
|
|
@@ -94,7 +94,7 @@ class AiSupportController < ApplicationController
|
|
|
94
94
|
|
|
95
95
|
sse = ActionController::Live::SSE.new(response.stream, event: "message")
|
|
96
96
|
|
|
97
|
-
|
|
97
|
+
SupportRequest.call(
|
|
98
98
|
input: input,
|
|
99
99
|
stream: ->(token) { sse.write({ token: token }.to_json) }
|
|
100
100
|
)
|