agent-harness 0.5.5 → 0.5.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.release-please-manifest.json +1 -1
- data/CHANGELOG.md +24 -0
- data/README.md +76 -1
- data/lib/agent_harness/command_executor.rb +453 -32
- data/lib/agent_harness/docker_command_executor.rb +23 -3
- data/lib/agent_harness/error_taxonomy.rb +10 -0
- data/lib/agent_harness/errors.rb +5 -0
- data/lib/agent_harness/orchestration/conductor.rb +40 -16
- data/lib/agent_harness/orchestration/provider_manager.rb +21 -13
- data/lib/agent_harness/provider_health_check.rb +216 -58
- data/lib/agent_harness/provider_runtime.rb +132 -0
- data/lib/agent_harness/providers/adapter.rb +157 -0
- data/lib/agent_harness/providers/aider.rb +21 -0
- data/lib/agent_harness/providers/anthropic.rb +21 -0
- data/lib/agent_harness/providers/base.rb +83 -11
- data/lib/agent_harness/providers/codex.rb +75 -8
- data/lib/agent_harness/providers/cursor.rb +47 -2
- data/lib/agent_harness/providers/gemini.rb +53 -0
- data/lib/agent_harness/providers/github_copilot.rb +34 -6
- data/lib/agent_harness/providers/kilocode.rb +39 -0
- data/lib/agent_harness/providers/mistral_vibe.rb +4 -0
- data/lib/agent_harness/providers/opencode.rb +91 -1
- data/lib/agent_harness/providers/registry.rb +54 -0
- data/lib/agent_harness/version.rb +1 -1
- data/lib/agent_harness.rb +78 -6
- metadata +22 -1
|
@@ -35,12 +35,20 @@ module AgentHarness
|
|
|
35
35
|
#
|
|
36
36
|
# @param command [Array<String>, String] command to execute
|
|
37
37
|
# @param timeout [Integer, nil] timeout in seconds
|
|
38
|
+
# @param idle_timeout [Integer, Float, nil] idle timeout in seconds based on output activity
|
|
38
39
|
# @param env [Hash] environment variables to set in the container
|
|
39
40
|
# @param stdin_data [String, nil] data to send to stdin
|
|
40
41
|
# @return [Result] execution result
|
|
41
|
-
def execute(command, timeout: nil, env: {}, stdin_data: nil)
|
|
42
|
+
def execute(command, timeout: nil, idle_timeout: nil, env: {}, stdin_data: nil, **execution_options)
|
|
42
43
|
docker_cmd = build_docker_command(command, env: env, stdin_data: stdin_data)
|
|
43
|
-
super(
|
|
44
|
+
super(
|
|
45
|
+
docker_cmd,
|
|
46
|
+
timeout: timeout,
|
|
47
|
+
idle_timeout: idle_timeout,
|
|
48
|
+
env: {},
|
|
49
|
+
stdin_data: stdin_data,
|
|
50
|
+
**execution_options
|
|
51
|
+
)
|
|
44
52
|
end
|
|
45
53
|
|
|
46
54
|
# Check if a binary exists inside the container
|
|
@@ -62,11 +70,23 @@ module AgentHarness
|
|
|
62
70
|
|
|
63
71
|
def build_docker_command(command, env:, stdin_data:)
|
|
64
72
|
cmd = ["docker", "exec"]
|
|
73
|
+
unset_env_keys = []
|
|
65
74
|
|
|
66
|
-
env.each
|
|
75
|
+
env.each do |key, value|
|
|
76
|
+
if value.nil?
|
|
77
|
+
unset_env_keys << key
|
|
78
|
+
next
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
cmd.push("--env", "#{key}=#{value}")
|
|
82
|
+
end
|
|
67
83
|
cmd.push("-i") if stdin_data
|
|
68
84
|
|
|
69
85
|
cmd.push(@container_id)
|
|
86
|
+
unless unset_env_keys.empty?
|
|
87
|
+
cmd.push("env")
|
|
88
|
+
unset_env_keys.each { |key| cmd.push("-u", key) }
|
|
89
|
+
end
|
|
70
90
|
|
|
71
91
|
cmd.concat(normalize_command(command))
|
|
72
92
|
end
|
|
@@ -39,6 +39,11 @@ module AgentHarness
|
|
|
39
39
|
action: :retry_with_backoff,
|
|
40
40
|
retryable: true
|
|
41
41
|
},
|
|
42
|
+
idle_timeout: {
|
|
43
|
+
description: "Operation exceeded idle timeout",
|
|
44
|
+
action: :escalate,
|
|
45
|
+
retryable: false
|
|
46
|
+
},
|
|
42
47
|
sandbox_failure: {
|
|
43
48
|
description: "Sandbox setup failed",
|
|
44
49
|
action: :escalate,
|
|
@@ -58,6 +63,9 @@ module AgentHarness
|
|
|
58
63
|
# @param patterns [Hash<Symbol, Array<Regexp>>] provider-specific patterns
|
|
59
64
|
# @return [Symbol] error category
|
|
60
65
|
def classify(error, patterns = {})
|
|
66
|
+
return :idle_timeout if error.is_a?(IdleTimeoutError)
|
|
67
|
+
return :timeout if error.is_a?(TimeoutError)
|
|
68
|
+
|
|
61
69
|
message = error.message.to_s.downcase
|
|
62
70
|
|
|
63
71
|
# Check provider-specific patterns first
|
|
@@ -112,6 +120,8 @@ module AgentHarness
|
|
|
112
120
|
|
|
113
121
|
def classify_generic(message)
|
|
114
122
|
case message
|
|
123
|
+
when /idle.?timeout/i
|
|
124
|
+
:idle_timeout
|
|
115
125
|
when /rate.?limit|too many requests|429/i
|
|
116
126
|
:rate_limited
|
|
117
127
|
when /quota|usage.?limit|billing/i
|
data/lib/agent_harness/errors.rb
CHANGED
|
@@ -22,6 +22,11 @@ module AgentHarness
|
|
|
22
22
|
# Execution errors
|
|
23
23
|
class TimeoutError < Error; end
|
|
24
24
|
|
|
25
|
+
# Raised when a duration argument is invalid (non-positive)
|
|
26
|
+
class InvalidDurationError < ArgumentError; end
|
|
27
|
+
|
|
28
|
+
class IdleTimeoutError < TimeoutError; end
|
|
29
|
+
|
|
25
30
|
class CommandExecutionError < Error; end
|
|
26
31
|
|
|
27
32
|
# Rate limiting and circuit breaker errors
|
|
@@ -33,13 +33,14 @@ module AgentHarness
|
|
|
33
33
|
# @param prompt [String] the prompt to send
|
|
34
34
|
# @param provider [Symbol, nil] preferred provider
|
|
35
35
|
# @param model [String, nil] model to use
|
|
36
|
+
# @param executor [CommandExecutor, nil] per-request executor override
|
|
36
37
|
# @param options [Hash] additional options
|
|
37
38
|
# @return [Response] the response
|
|
38
39
|
# @raise [NoProvidersAvailableError] if all providers fail
|
|
39
|
-
def send_message(prompt, provider: nil, model: nil, **options)
|
|
40
|
+
def send_message(prompt, provider: nil, model: nil, executor: nil, **options)
|
|
40
41
|
provider_name = provider || @config.default_provider
|
|
41
42
|
|
|
42
|
-
with_orchestration(provider_name, model, options) do |selected_provider|
|
|
43
|
+
with_orchestration(provider_name, model, executor, options) do |selected_provider|
|
|
43
44
|
selected_provider.send_message(prompt: prompt, model: model, **options)
|
|
44
45
|
end
|
|
45
46
|
end
|
|
@@ -50,8 +51,8 @@ module AgentHarness
|
|
|
50
51
|
# @param provider [Symbol] the provider to use
|
|
51
52
|
# @param options [Hash] additional options
|
|
52
53
|
# @return [Response] the response
|
|
53
|
-
def execute_direct(prompt, provider:, **options)
|
|
54
|
-
provider_instance = @provider_manager.get_provider(provider)
|
|
54
|
+
def execute_direct(prompt, provider:, executor: nil, **options)
|
|
55
|
+
provider_instance = @provider_manager.get_provider(provider, executor: executor)
|
|
55
56
|
provider_instance.send_message(prompt: prompt, **options)
|
|
56
57
|
end
|
|
57
58
|
|
|
@@ -77,7 +78,7 @@ module AgentHarness
|
|
|
77
78
|
|
|
78
79
|
private
|
|
79
80
|
|
|
80
|
-
def with_orchestration(provider_name, model, options)
|
|
81
|
+
def with_orchestration(provider_name, model, executor, options)
|
|
81
82
|
retries = 0
|
|
82
83
|
retry_config = @config.orchestration_config.retry_config
|
|
83
84
|
max_retries = retry_config.max_attempts
|
|
@@ -85,7 +86,7 @@ module AgentHarness
|
|
|
85
86
|
|
|
86
87
|
begin
|
|
87
88
|
# Select provider (may return different provider based on health)
|
|
88
|
-
provider = @provider_manager.select_provider(provider_name)
|
|
89
|
+
provider = @provider_manager.select_provider(provider_name, executor: executor)
|
|
89
90
|
provider_name = provider.class.provider_name
|
|
90
91
|
attempted_providers << provider_name
|
|
91
92
|
|
|
@@ -98,7 +99,9 @@ module AgentHarness
|
|
|
98
99
|
|
|
99
100
|
# Record success
|
|
100
101
|
@metrics.record_success(provider_name, duration)
|
|
101
|
-
|
|
102
|
+
# Only update shared health state for default-executor traffic;
|
|
103
|
+
# request-scoped executor successes must not heal the global provider.
|
|
104
|
+
@provider_manager.record_success(provider_name) unless executor
|
|
102
105
|
|
|
103
106
|
response
|
|
104
107
|
rescue AuthenticationError => e
|
|
@@ -113,17 +116,31 @@ module AgentHarness
|
|
|
113
116
|
@metrics.record_failure(provider_name, e)
|
|
114
117
|
raise
|
|
115
118
|
rescue RateLimitError => e
|
|
116
|
-
|
|
117
|
-
|
|
119
|
+
# Only update shared rate-limit state for default-executor traffic;
|
|
120
|
+
# request-scoped executor failures must not poison the global provider.
|
|
121
|
+
@provider_manager.mark_rate_limited(provider_name, reset_at: e.reset_time) unless executor
|
|
122
|
+
provider_name = handle_provider_failure(e, provider_name, :switch, executor: executor)
|
|
118
123
|
retry if should_retry?(retries += 1, max_retries)
|
|
119
124
|
raise
|
|
120
125
|
rescue CircuitOpenError => e
|
|
121
|
-
handle_provider_failure(e, provider_name, :switch)
|
|
126
|
+
provider_name = handle_provider_failure(e, provider_name, :switch, executor: executor)
|
|
122
127
|
retry if should_retry?(retries += 1, max_retries)
|
|
123
128
|
raise
|
|
124
|
-
rescue
|
|
129
|
+
rescue IdleTimeoutError => e
|
|
130
|
+
@metrics.record_failure(provider_name, e)
|
|
125
131
|
@provider_manager.record_failure(provider_name)
|
|
126
|
-
|
|
132
|
+
raise
|
|
133
|
+
rescue TimeoutError, ProviderError => e
|
|
134
|
+
# Only update shared health state for default-executor traffic;
|
|
135
|
+
# request-scoped executor failures must not poison the global provider.
|
|
136
|
+
@provider_manager.record_failure(provider_name) unless executor
|
|
137
|
+
# For executor-scoped requests we skip record_failure (above), so
|
|
138
|
+
# shared health/circuit state never degrades and select_provider
|
|
139
|
+
# would keep returning the same failing provider on retry. Use
|
|
140
|
+
# :switch instead of :retry so the request can still fall back to a
|
|
141
|
+
# healthy provider without poisoning global state.
|
|
142
|
+
strategy = executor ? :switch : :retry
|
|
143
|
+
provider_name = handle_provider_failure(e, provider_name, strategy, executor: executor)
|
|
127
144
|
retry if should_retry?(retries += 1, max_retries)
|
|
128
145
|
raise
|
|
129
146
|
rescue NoProvidersAvailableError
|
|
@@ -131,10 +148,12 @@ module AgentHarness
|
|
|
131
148
|
raise
|
|
132
149
|
rescue => e
|
|
133
150
|
@metrics.record_failure(provider_name, e)
|
|
134
|
-
|
|
151
|
+
# Only update shared health state for default-executor traffic;
|
|
152
|
+
# request-scoped executor failures must not poison the global provider.
|
|
153
|
+
@provider_manager.record_failure(provider_name) unless executor
|
|
135
154
|
|
|
136
155
|
# Try switching for unknown errors
|
|
137
|
-
handle_provider_failure(e, provider_name, :switch)
|
|
156
|
+
provider_name = handle_provider_failure(e, provider_name, :switch, executor: executor)
|
|
138
157
|
retry if should_retry?(retries += 1, max_retries)
|
|
139
158
|
raise ProviderError.new(e.message, original_error: e)
|
|
140
159
|
end
|
|
@@ -145,7 +164,7 @@ module AgentHarness
|
|
|
145
164
|
current_retries < max_retries
|
|
146
165
|
end
|
|
147
166
|
|
|
148
|
-
def handle_provider_failure(error, provider_name, strategy)
|
|
167
|
+
def handle_provider_failure(error, provider_name, strategy, executor: nil)
|
|
149
168
|
@metrics.record_failure(provider_name, error)
|
|
150
169
|
|
|
151
170
|
case strategy
|
|
@@ -153,8 +172,10 @@ module AgentHarness
|
|
|
153
172
|
if @config.orchestration_config.auto_switch_on_error
|
|
154
173
|
new_provider = begin
|
|
155
174
|
@provider_manager.switch_provider(
|
|
175
|
+
from: provider_name,
|
|
156
176
|
reason: error.class.name,
|
|
157
|
-
context: {error: error.message}
|
|
177
|
+
context: {error: error.message},
|
|
178
|
+
executor: executor
|
|
158
179
|
)
|
|
159
180
|
rescue NoProvidersAvailableError
|
|
160
181
|
nil
|
|
@@ -162,12 +183,15 @@ module AgentHarness
|
|
|
162
183
|
|
|
163
184
|
if new_provider
|
|
164
185
|
@metrics.record_switch(provider_name, new_provider.class.provider_name, error.class.name)
|
|
186
|
+
return new_provider.class.provider_name
|
|
165
187
|
end
|
|
166
188
|
end
|
|
167
189
|
when :retry
|
|
168
190
|
delay = calculate_retry_delay
|
|
169
191
|
sleep(delay) if delay > 0
|
|
170
192
|
end
|
|
193
|
+
|
|
194
|
+
provider_name
|
|
171
195
|
end
|
|
172
196
|
|
|
173
197
|
def calculate_retry_delay
|
|
@@ -30,49 +30,57 @@ module AgentHarness
|
|
|
30
30
|
# Select best available provider
|
|
31
31
|
#
|
|
32
32
|
# @param preferred [Symbol, nil] preferred provider name
|
|
33
|
+
# @param executor [CommandExecutor, nil] per-request executor override
|
|
33
34
|
# @return [Providers::Base] selected provider instance
|
|
34
35
|
# @raise [NoProvidersAvailableError] if no providers available
|
|
35
|
-
def select_provider(preferred = nil)
|
|
36
|
+
def select_provider(preferred = nil, executor: nil)
|
|
36
37
|
preferred ||= @current_provider
|
|
37
38
|
|
|
38
39
|
# Check circuit breaker
|
|
39
40
|
if circuit_open?(preferred)
|
|
40
|
-
return select_fallback(preferred, reason: :circuit_open)
|
|
41
|
+
return select_fallback(preferred, reason: :circuit_open, executor: executor)
|
|
41
42
|
end
|
|
42
43
|
|
|
43
44
|
# Check rate limit
|
|
44
45
|
if rate_limited?(preferred)
|
|
45
|
-
return select_fallback(preferred, reason: :rate_limited)
|
|
46
|
+
return select_fallback(preferred, reason: :rate_limited, executor: executor)
|
|
46
47
|
end
|
|
47
48
|
|
|
48
49
|
# Check health
|
|
49
50
|
unless healthy?(preferred)
|
|
50
|
-
return select_fallback(preferred, reason: :unhealthy)
|
|
51
|
+
return select_fallback(preferred, reason: :unhealthy, executor: executor)
|
|
51
52
|
end
|
|
52
53
|
|
|
53
|
-
get_provider(preferred)
|
|
54
|
+
get_provider(preferred, executor: executor)
|
|
54
55
|
end
|
|
55
56
|
|
|
56
57
|
# Get or create provider instance
|
|
57
58
|
#
|
|
58
59
|
# @param name [Symbol, String] the provider name
|
|
60
|
+
# @param executor [CommandExecutor, nil] per-request executor override
|
|
59
61
|
# @return [Providers::Base] the provider instance
|
|
60
|
-
def get_provider(name)
|
|
62
|
+
def get_provider(name, executor: nil)
|
|
61
63
|
name = name.to_sym
|
|
64
|
+
return create_provider(name, executor: executor) if executor
|
|
65
|
+
|
|
62
66
|
@provider_instances[name] ||= create_provider(name)
|
|
63
67
|
end
|
|
64
68
|
|
|
65
69
|
# Switch to next available provider
|
|
66
70
|
#
|
|
71
|
+
# @param from [Symbol, String] provider that failed and should be switched from
|
|
67
72
|
# @param reason [Symbol, String] reason for switch
|
|
68
73
|
# @param context [Hash] additional context
|
|
74
|
+
# @param executor [CommandExecutor, nil] per-request executor override
|
|
69
75
|
# @return [Providers::Base, nil] new provider or nil if none available
|
|
70
|
-
def switch_provider(reason:, context: {})
|
|
71
|
-
old_provider =
|
|
76
|
+
def switch_provider(reason:, context: {}, executor: nil, from: @current_provider)
|
|
77
|
+
old_provider = from.to_sym
|
|
72
78
|
|
|
73
|
-
fallback = select_fallback(
|
|
79
|
+
fallback = select_fallback(old_provider, reason: reason, executor: executor)
|
|
74
80
|
return nil unless fallback
|
|
75
81
|
|
|
82
|
+
return fallback if executor
|
|
83
|
+
|
|
76
84
|
@current_provider = fallback.class.provider_name
|
|
77
85
|
|
|
78
86
|
AgentHarness.logger&.info(
|
|
@@ -195,18 +203,18 @@ module AgentHarness
|
|
|
195
203
|
end
|
|
196
204
|
end
|
|
197
205
|
|
|
198
|
-
def create_provider(name)
|
|
206
|
+
def create_provider(name, executor: @config.command_executor)
|
|
199
207
|
klass = @registry.get(name)
|
|
200
208
|
config = @config.providers[name]
|
|
201
209
|
|
|
202
210
|
klass.new(
|
|
203
211
|
config: config,
|
|
204
|
-
executor:
|
|
212
|
+
executor: executor,
|
|
205
213
|
logger: AgentHarness.logger
|
|
206
214
|
)
|
|
207
215
|
end
|
|
208
216
|
|
|
209
|
-
def select_fallback(provider_name, reason:)
|
|
217
|
+
def select_fallback(provider_name, reason:, executor: nil)
|
|
210
218
|
chain = @fallback_chains[provider_name] || build_fallback_chain(provider_name)
|
|
211
219
|
|
|
212
220
|
chain.each do |fallback_name|
|
|
@@ -219,7 +227,7 @@ module AgentHarness
|
|
|
219
227
|
"[AgentHarness::ProviderManager] Falling back from #{provider_name} to #{fallback_name} (#{reason})"
|
|
220
228
|
)
|
|
221
229
|
|
|
222
|
-
return get_provider(fallback_name)
|
|
230
|
+
return get_provider(fallback_name, executor: executor)
|
|
223
231
|
end
|
|
224
232
|
|
|
225
233
|
# No fallback available
|