lemans 1.3.5 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +21 -0
- data/README.md +17 -2
- data/exe/lemans-remote +141 -3
- data/exe/lemans-viewer +962 -0
- data/lib/lemans/agent.rb +4 -1
- data/lib/lemans/agents/miniswen.rb +10 -5
- data/lib/lemans/agents/miniswen_installed.rb +23 -5
- data/lib/lemans/agents/nop.rb +1 -1
- data/lib/lemans/agents/oracle.rb +1 -1
- data/lib/lemans/agents.rb +4 -2
- data/lib/lemans/cli/templates/bench/bench.yml +7 -0
- data/lib/lemans/cli.rb +85 -31
- data/lib/lemans/config/verifier.rb +10 -2
- data/lib/lemans/environments/docker/proxy/Dockerfile +4 -0
- data/lib/lemans/environments/docker/proxy/README.md +49 -0
- data/lib/lemans/environments/docker/proxy/server.rb +79 -0
- data/lib/lemans/environments/docker.rb +79 -16
- data/lib/lemans/result.rb +52 -1
- data/lib/lemans/runner/task.rb +9 -5
- data/lib/lemans/runner.rb +45 -2
- data/lib/lemans/store.rb +6 -0
- data/lib/lemans/stores/fs.rb +16 -4
- data/lib/lemans/trial/patch.rb +21 -0
- data/lib/lemans/trial/setup.rb +1 -0
- data/lib/lemans/trial.rb +109 -41
- data/lib/lemans/version.rb +1 -1
- data/lib/lemans.rb +2 -0
- data/lib/miniswen/agent.rb +65 -18
- data/lib/miniswen/cli.rb +20 -4
- data/lib/miniswen/environment/docker.rb +1 -1
- data/lib/miniswen/jail/proxy.rb +62 -0
- data/lib/miniswen/jail.rb +36 -9
- data/lib/miniswen/local.rb +6 -7
- data/lib/miniswen/testing.rb +7 -1
- data/lib/miniswen/version.rb +1 -1
- metadata +49 -1
data/lib/lemans/trial.rb
CHANGED
|
@@ -12,13 +12,15 @@ module Lemans
|
|
|
12
12
|
class Trial
|
|
13
13
|
attr_reader :task, :config, :model, :agent_name, :environment, :result
|
|
14
14
|
|
|
15
|
-
private attr_reader :agent, :store, :snapshot, :patch, :current_step_index
|
|
15
|
+
private attr_reader :agent, :store, :restart_from, :restart_mode, :snapshot, :patch, :current_step_index
|
|
16
16
|
|
|
17
|
-
def initialize(task, model = nil, result: nil, store: nil, agent: nil, environment: nil)
|
|
17
|
+
def initialize(task, model = nil, result: nil, store: nil, agent: nil, environment: nil, restart_from: nil, restart_mode: nil)
|
|
18
18
|
@task = task
|
|
19
19
|
@config = task.config
|
|
20
20
|
@model = model || config.models.first
|
|
21
21
|
@store = store
|
|
22
|
+
@restart_from = restart_from
|
|
23
|
+
@restart_mode = restart_mode
|
|
22
24
|
|
|
23
25
|
@agent = agent.is_a?(Agent) ? agent : Agents.build(agent || config.agent_name, profile: config.agent, model: @model)
|
|
24
26
|
@agent_name = @agent.name
|
|
@@ -47,9 +49,12 @@ module Lemans
|
|
|
47
49
|
@snapshot = nil
|
|
48
50
|
@patch = nil
|
|
49
51
|
@current_step_index = nil
|
|
52
|
+
@recovered_since = nil
|
|
50
53
|
end
|
|
51
54
|
|
|
52
55
|
def run
|
|
56
|
+
restart! if restart_from
|
|
57
|
+
|
|
53
58
|
phase(:environment_setup) do
|
|
54
59
|
environment.start
|
|
55
60
|
|
|
@@ -64,45 +69,55 @@ module Lemans
|
|
|
64
69
|
# Seal the git state to collect the agent's patch later
|
|
65
70
|
@patch = Patch.new(task, environment)
|
|
66
71
|
patch.seal!
|
|
72
|
+
patch.replay!(settled_patches, (restart_artifact("agent.patch") if restart_mode)) if restart_from
|
|
67
73
|
|
|
68
|
-
agent
|
|
74
|
+
# A reverification of the last step runs no agent
|
|
75
|
+
agent.install(task, environment) unless restart_mode == :reverify && restart_step == task.steps
|
|
69
76
|
|
|
70
77
|
environment.switch_network_policy!(config.agent.environment.network)
|
|
71
78
|
end
|
|
72
79
|
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
80
|
+
adopt_phases! if restart_from
|
|
81
|
+
|
|
82
|
+
pending = restart_mode
|
|
83
|
+
each_step(from: restart_from ? restart_step : 1) do |step_task|
|
|
84
|
+
mode, pending = pending, nil
|
|
85
|
+
|
|
86
|
+
unless mode == :reverify
|
|
87
|
+
history = restart_artifact("agent.result.json") if mode == :recover
|
|
88
|
+
response =
|
|
89
|
+
phase(:agent, started_at: (@recovered_since if mode == :recover)) do
|
|
90
|
+
agent.run(step_task, environment, history:)
|
|
91
|
+
rescue InfrastructureError, ::Miniswen::InfrastructureError => e
|
|
92
|
+
# Mark the failure here, where the agent phase is still known
|
|
93
|
+
result.failed!(:agent_error, e.message)
|
|
94
|
+
collect_patch!
|
|
95
|
+
raise
|
|
96
|
+
rescue ::Miniswen::AccountingError
|
|
97
|
+
# Classified by the outer rescue; the work is still on disk
|
|
98
|
+
collect_patch!
|
|
99
|
+
raise
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# Whatever the agent brought back is evidence, a failed run's included
|
|
103
|
+
save_trajectory!(response.trajectory)
|
|
104
|
+
store&.save_artifact(result, response.raw_result, path: with_step_index("agent.result.json")) if response.raw_result
|
|
105
|
+
|
|
106
|
+
if response.error?
|
|
107
|
+
result.failed!(:agent_error, response.error)
|
|
84
108
|
collect_patch!
|
|
85
|
-
|
|
109
|
+
return result
|
|
86
110
|
end
|
|
87
111
|
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
result.failed!(:agent_error, response.error)
|
|
94
|
-
collect_patch!
|
|
95
|
-
return result
|
|
96
|
-
end
|
|
112
|
+
if task.multistep?
|
|
113
|
+
result.step_completed!(response.outcome, response.usage, duration: result.phases.last.duration)
|
|
114
|
+
else
|
|
115
|
+
result.completed!(response.outcome, response.usage)
|
|
116
|
+
end
|
|
97
117
|
|
|
98
|
-
|
|
99
|
-
result.step_completed!(response.outcome, response.usage, duration: result.phases.last.duration)
|
|
100
|
-
else
|
|
101
|
-
result.completed!(response.outcome, response.usage)
|
|
118
|
+
check_cost_limit!
|
|
102
119
|
end
|
|
103
120
|
|
|
104
|
-
check_cost_limit!
|
|
105
|
-
|
|
106
121
|
collect_patch!
|
|
107
122
|
if step_task.final_step?
|
|
108
123
|
patch.compile!(result, store) if task.multistep? && store
|
|
@@ -114,8 +129,8 @@ module Lemans
|
|
|
114
129
|
|
|
115
130
|
if result.scored? && step_task.verifiable?
|
|
116
131
|
phase(:verifier) do
|
|
117
|
-
# The sandbox is sealed before the tests arrive
|
|
118
|
-
environment.switch_network_policy!(
|
|
132
|
+
# The sandbox is sealed before the tests arrive, unless the task names hosts
|
|
133
|
+
environment.switch_network_policy!(step_task.verifier.environment.network)
|
|
119
134
|
|
|
120
135
|
verification = Verifier.new(step_task, environment, snapshot).verify! do |evidence, path|
|
|
121
136
|
store&.save_artifact(result, evidence, path: with_step_index(path))
|
|
@@ -166,31 +181,84 @@ module Lemans
|
|
|
166
181
|
store.save_artifact(result, JSON.pretty_generate(trajectory.to_atif), path:)
|
|
167
182
|
end
|
|
168
183
|
|
|
169
|
-
def each_step
|
|
184
|
+
def each_step(from: 1)
|
|
170
185
|
return yield task unless task.multistep?
|
|
171
186
|
|
|
172
187
|
catch(:halt) do
|
|
173
|
-
|
|
174
|
-
|
|
188
|
+
from.upto(task.steps) do |index|
|
|
189
|
+
# The first step of a restart finds a fresh tree, already replayed
|
|
190
|
+
resume_agent! if index > from
|
|
175
191
|
@current_step_index = index
|
|
176
192
|
yield task.for_step(index)
|
|
177
193
|
end
|
|
178
194
|
end
|
|
179
195
|
end
|
|
180
196
|
|
|
197
|
+
# The step a restart begins at: the one a reverification grades again, or
|
|
198
|
+
# the first one not settled (run afresh, or recovered from its history).
|
|
199
|
+
def restart_step
|
|
200
|
+
@restart_step ||= restart_mode == :reverify ? restart_from.verified_step : restart_from.settled_steps + 1
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
# The new run carries the settled steps' evidence, and the agent's part of
|
|
204
|
+
# a step it grades again: the patch is collected anew.
|
|
205
|
+
def restart!
|
|
206
|
+
result.restart!(restart_from, step: restart_step, mode: restart_mode)
|
|
207
|
+
carried = restart_mode == :reverify ? %w[trajectory.json agent.result.json].map { restart_path(it) } : []
|
|
208
|
+
|
|
209
|
+
store.artifacts(restart_from).each do |path|
|
|
210
|
+
next unless path[%r{\.(\d+)(?:\.[^./]+)?\z}, 1].to_i.between?(1, restart_step - 1) || carried.include?(path)
|
|
211
|
+
|
|
212
|
+
store.save_artifact(result, store.read_artifact(restart_from, path), path:)
|
|
213
|
+
end
|
|
214
|
+
end
|
|
215
|
+
|
|
216
|
+
def settled_patches
|
|
217
|
+
(1...restart_step).map do |index|
|
|
218
|
+
store.read_artifact(restart_from, "agent.#{index}.patch") ||
|
|
219
|
+
raise(InfrastructureError, "#{restart_from.id} has no agent.#{index}.patch to replay")
|
|
220
|
+
end
|
|
221
|
+
end
|
|
222
|
+
|
|
223
|
+
def restart_artifact(name)
|
|
224
|
+
path = restart_path(name)
|
|
225
|
+
store.read_artifact(restart_from, path) || raise(InfrastructureError, "#{restart_from.id} has no #{path} to restart from")
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
def restart_path(name) = task.multistep? ? with_step_index(name, restart_step) : name
|
|
229
|
+
|
|
230
|
+
# A recovered step's agent phase reaches back over the part the source
|
|
231
|
+
# run already spent, so the step lasts as long as both parts together.
|
|
232
|
+
def adopt_phases!
|
|
233
|
+
names = restart_from.phases.map(&:name).select { it.to_s[/\.(\d+)\z/, 1].to_i.between?(1, restart_step - 1) }
|
|
234
|
+
agent_phase = restart_path("agent").to_sym
|
|
235
|
+
now = Time.now.utc
|
|
236
|
+
|
|
237
|
+
case restart_mode
|
|
238
|
+
when :reverify
|
|
239
|
+
result.adopt_phases!(restart_from, names + [ agent_phase ], now:)
|
|
240
|
+
when :recover
|
|
241
|
+
result.adopt_phases!(restart_from, names, through: agent_phase, now:)
|
|
242
|
+
partial = restart_from.phases.find { it.name == agent_phase }
|
|
243
|
+
@recovered_since = now - (partial.finished_at - partial.started_at)
|
|
244
|
+
else
|
|
245
|
+
result.adopt_phases!(restart_from, names, now:)
|
|
246
|
+
end
|
|
247
|
+
end
|
|
248
|
+
|
|
181
249
|
def resume_agent!
|
|
182
250
|
patch.restore!
|
|
183
251
|
environment.exec!("rm -rf #{Verifier::TESTS_DIR} #{Shellwords.escape(task.verifier.logs_dir)}")
|
|
184
252
|
environment.switch_network_policy!(config.agent.environment.network)
|
|
185
253
|
end
|
|
186
254
|
|
|
187
|
-
def with_step_index(path)
|
|
188
|
-
return path unless
|
|
255
|
+
def with_step_index(path, index = current_step_index)
|
|
256
|
+
return path unless index
|
|
189
257
|
|
|
190
258
|
*pre, last = path.to_s.split(".")
|
|
191
|
-
return "#{last}.#{
|
|
259
|
+
return "#{last}.#{index}" if pre.empty?
|
|
192
260
|
|
|
193
|
-
[ *pre,
|
|
261
|
+
[ *pre, index, last ].join(".")
|
|
194
262
|
end
|
|
195
263
|
|
|
196
264
|
def sandbox_ttl
|
|
@@ -209,8 +277,8 @@ module Lemans
|
|
|
209
277
|
)
|
|
210
278
|
end
|
|
211
279
|
|
|
212
|
-
def phase(name)
|
|
213
|
-
result.phase_started(with_step_index(name).to_sym)
|
|
280
|
+
def phase(name, started_at: nil)
|
|
281
|
+
result.phase_started(with_step_index(name).to_sym, started_at)
|
|
214
282
|
yield
|
|
215
283
|
ensure
|
|
216
284
|
result.phase_finished(with_step_index(name).to_sym)
|
data/lib/lemans/version.rb
CHANGED
data/lib/lemans.rb
CHANGED
|
@@ -11,6 +11,8 @@ loader.ignore("#{__dir__}/miniswen.rb", "#{__dir__}/miniswen")
|
|
|
11
11
|
loader.ignore("#{__dir__}/lemans/trial/verifier/assets")
|
|
12
12
|
# templates/ holds the files `lemans init` scaffolds, not Ruby the harness loads.
|
|
13
13
|
loader.ignore("#{__dir__}/lemans/cli/templates")
|
|
14
|
+
# proxy/ holds the image the Docker backend runs for allowlists, not Ruby the harness loads.
|
|
15
|
+
loader.ignore("#{__dir__}/lemans/environments/docker/proxy")
|
|
14
16
|
loader.setup
|
|
15
17
|
|
|
16
18
|
require "miniswen"
|
data/lib/miniswen/agent.rb
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "json"
|
|
4
|
+
require "time"
|
|
4
5
|
require "miniswen/version"
|
|
5
6
|
require "miniswen/ruby_llm"
|
|
6
7
|
|
|
@@ -261,25 +262,17 @@ module Miniswen
|
|
|
261
262
|
@reporter = reporter
|
|
262
263
|
end
|
|
263
264
|
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
]
|
|
273
|
-
|
|
274
|
-
@steps = 0
|
|
275
|
-
@cost = 0.0
|
|
276
|
-
|
|
277
|
-
@totals = { input_tokens: 0, output_tokens: 0, cached_tokens: 0, thinking_tokens: 0 }
|
|
265
|
+
# With a `history` (the Result of an interrupted run), the session goes on
|
|
266
|
+
# where it stopped, under the budget the earlier part already spent.
|
|
267
|
+
def run(instruction = nil, history: nil)
|
|
268
|
+
if history
|
|
269
|
+
resume(history)
|
|
270
|
+
else
|
|
271
|
+
start(instruction)
|
|
272
|
+
end
|
|
278
273
|
|
|
279
|
-
@cost_known = true
|
|
280
274
|
@consecutive_format_errors = 0
|
|
281
275
|
@refused_turns = 0
|
|
282
|
-
@started_at = @clock.call
|
|
283
276
|
|
|
284
277
|
loop do
|
|
285
278
|
(status = limit_reached) and return finish(status)
|
|
@@ -294,6 +287,8 @@ module Miniswen
|
|
|
294
287
|
end
|
|
295
288
|
|
|
296
289
|
actions.each do |action|
|
|
290
|
+
return finish(:time_limit) if out_of_time?
|
|
291
|
+
|
|
297
292
|
reporter&.on_tool_call(action)
|
|
298
293
|
result = execute(action.fetch(:arguments).fetch("command"))
|
|
299
294
|
# The submit command's output is observed too, so the final tool
|
|
@@ -327,6 +322,49 @@ module Miniswen
|
|
|
327
322
|
|
|
328
323
|
private
|
|
329
324
|
|
|
325
|
+
def start(instruction)
|
|
326
|
+
uname = execute("uname -srvm").output.to_s.strip
|
|
327
|
+
@messages = [
|
|
328
|
+
{ role: "system", content: SYSTEM_TEMPLATE },
|
|
329
|
+
{ role: "user", content: format(INSTANCE_TEMPLATE,
|
|
330
|
+
instruction: instruction,
|
|
331
|
+
system_information: uname,
|
|
332
|
+
macos_sed_note: uname.start_with?("Darwin") ? "\n#{MACOS_SED_NOTE}" : "") }
|
|
333
|
+
]
|
|
334
|
+
|
|
335
|
+
@steps = 0
|
|
336
|
+
@cost = 0.0
|
|
337
|
+
@cost_known = true
|
|
338
|
+
@totals = { input_tokens: 0, output_tokens: 0, cached_tokens: 0, thinking_tokens: 0 }
|
|
339
|
+
@started_at = @clock.call
|
|
340
|
+
end
|
|
341
|
+
|
|
342
|
+
def resume(history)
|
|
343
|
+
@messages = answered_messages(history.messages)
|
|
344
|
+
|
|
345
|
+
@steps = history.steps.to_i
|
|
346
|
+
@cost = history.cost_usd.to_f
|
|
347
|
+
@cost_known = !history.cost_usd.nil?
|
|
348
|
+
@totals = { input_tokens: history.input_tokens.to_i, output_tokens: history.output_tokens.to_i,
|
|
349
|
+
cached_tokens: history.cached_tokens.to_i, thinking_tokens: history.thinking_tokens.to_i }
|
|
350
|
+
@started_at = @clock.call - elapsed(history.messages)
|
|
351
|
+
end
|
|
352
|
+
|
|
353
|
+
# A turn whose calls did not all get a result cannot go back to a provider:
|
|
354
|
+
# it is dropped, and the model decides again from the tree it left.
|
|
355
|
+
def answered_messages(messages)
|
|
356
|
+
turn = messages.rindex { it[:role] == "assistant" && it[:tool_calls] }
|
|
357
|
+
return messages.dup unless turn
|
|
358
|
+
|
|
359
|
+
answered = messages[(turn + 1)..].filter_map { it[:tool_call_id] if it[:role] == "tool" }
|
|
360
|
+
messages[turn][:tool_calls].all? { answered.include?(it[:id]) } ? messages.dup : messages[0...turn]
|
|
361
|
+
end
|
|
362
|
+
|
|
363
|
+
def elapsed(messages)
|
|
364
|
+
times = messages.filter_map { Time.parse(it[:timestamp]) if it[:role] == "assistant" && it[:timestamp] }
|
|
365
|
+
times.empty? ? 0.0 : times.max - times.min
|
|
366
|
+
end
|
|
367
|
+
|
|
330
368
|
def execute(command)
|
|
331
369
|
environment.exec(command, timeout: exec_timeout, env: EXEC_ENV)
|
|
332
370
|
end
|
|
@@ -334,12 +372,14 @@ module Miniswen
|
|
|
334
372
|
# Checked before the model is asked, so the tripping step is never paid for.
|
|
335
373
|
def limit_reached
|
|
336
374
|
return :step_limit if max_steps.positive? && @steps >= max_steps
|
|
337
|
-
return :time_limit if
|
|
375
|
+
return :time_limit if out_of_time?
|
|
338
376
|
return :cost_limit if max_cost && @cost_known && @cost >= max_cost
|
|
339
377
|
|
|
340
378
|
nil
|
|
341
379
|
end
|
|
342
380
|
|
|
381
|
+
def out_of_time? = max_time.positive? && (@clock.call - @started_at) >= max_time
|
|
382
|
+
|
|
343
383
|
# One model turn. Returns the actions to run, or nil after appending a
|
|
344
384
|
# format-error message the model gets to react to on its next turn.
|
|
345
385
|
def next_actions
|
|
@@ -568,6 +608,11 @@ module Miniswen
|
|
|
568
608
|
registry: nil)
|
|
569
609
|
end
|
|
570
610
|
return nil unless info
|
|
611
|
+
if free?
|
|
612
|
+
return CostSource.new(name: :free_model, model: @model,
|
|
613
|
+
priced_as: "#{info.provider}/#{info.id} ($0.00, free)",
|
|
614
|
+
registry: Miniswen.registry_revision)
|
|
615
|
+
end
|
|
571
616
|
|
|
572
617
|
CostSource.new(name: :model_registry, model: @model,
|
|
573
618
|
priced_as: "#{info.provider}/#{info.id}",
|
|
@@ -668,6 +713,8 @@ module Miniswen
|
|
|
668
713
|
|
|
669
714
|
def local? = LOCAL_PROVIDERS.include?((@provider || info&.provider)&.to_sym)
|
|
670
715
|
|
|
716
|
+
def free? = info&.provider == "openrouter" && info.input_price_per_million.nil? && info.output_price_per_million.nil?
|
|
717
|
+
|
|
671
718
|
def info
|
|
672
719
|
return @info if defined?(@info)
|
|
673
720
|
|
|
@@ -681,7 +728,7 @@ module Miniswen
|
|
|
681
728
|
end
|
|
682
729
|
|
|
683
730
|
def price(response)
|
|
684
|
-
return 0.0 if local?
|
|
731
|
+
return 0.0 if local? || free?
|
|
685
732
|
|
|
686
733
|
input = info&.input_price_per_million
|
|
687
734
|
output = info&.output_price_per_million
|
data/lib/miniswen/cli.rb
CHANGED
|
@@ -23,10 +23,14 @@ module Miniswen
|
|
|
23
23
|
@refresh_registry = false
|
|
24
24
|
@skip_registry_refresh = false
|
|
25
25
|
@jail = false
|
|
26
|
+
@allowed_hosts = nil
|
|
27
|
+
@workdir = nil
|
|
28
|
+
@history = nil
|
|
26
29
|
end
|
|
27
30
|
|
|
28
31
|
def run
|
|
29
32
|
parse_args!
|
|
33
|
+
Dir.chdir(@workdir) if @workdir
|
|
30
34
|
|
|
31
35
|
# Require the core library after parsing options,
|
|
32
36
|
# so env flags kick in
|
|
@@ -45,7 +49,7 @@ module Miniswen
|
|
|
45
49
|
Environment::Docker.new(@docker_id)
|
|
46
50
|
elsif @jail
|
|
47
51
|
require "miniswen/jail"
|
|
48
|
-
Jail.new.start
|
|
52
|
+
Jail.new(allowed_hosts: @allowed_hosts).start
|
|
49
53
|
else
|
|
50
54
|
Local.new
|
|
51
55
|
end
|
|
@@ -53,7 +57,7 @@ module Miniswen
|
|
|
53
57
|
agent = Agent.new(model:, reporter:, environment:, **options)
|
|
54
58
|
|
|
55
59
|
begin
|
|
56
|
-
result = agent.run(instruction)
|
|
60
|
+
result = agent.run(instruction, history: @history && Agent::Result.from_h(JSON.parse(@history)))
|
|
57
61
|
rescue StandardError => e
|
|
58
62
|
write_results(agent.partial_result(error_message(e)))
|
|
59
63
|
raise
|
|
@@ -102,7 +106,7 @@ module Miniswen
|
|
|
102
106
|
|
|
103
107
|
def parse_args!
|
|
104
108
|
parser = OptionParser.new do |opts|
|
|
105
|
-
opts.banner = "Usage: miniswen -m MODEL -p INSTRUCTION [...options]"
|
|
109
|
+
opts.banner = "Usage: miniswen -m MODEL (-p INSTRUCTION | --continue-from PATH) [...options]"
|
|
106
110
|
|
|
107
111
|
opts.on("-m MODEL", "--model=MODEL", String,
|
|
108
112
|
"LLM to use (litellm format, e.g.: openrouter/openai/gpt-5.6-luna") do |v|
|
|
@@ -113,6 +117,10 @@ module Miniswen
|
|
|
113
117
|
@instruction = File.file?(v) ? File.read(v) : v
|
|
114
118
|
end
|
|
115
119
|
|
|
120
|
+
opts.on("--continue-from=PATH", String, "Continue the session saved by --results-path at PATH (no -p needed)") do |v|
|
|
121
|
+
@history = File.read(v)
|
|
122
|
+
end
|
|
123
|
+
|
|
116
124
|
opts.on("--max-steps=STEPS", Integer, "Max steps count") do |v|
|
|
117
125
|
options[:max_steps] = v
|
|
118
126
|
end
|
|
@@ -161,6 +169,14 @@ module Miniswen
|
|
|
161
169
|
@jail = true
|
|
162
170
|
end
|
|
163
171
|
|
|
172
|
+
opts.on("--allow-hosts=HOSTS", String, "Let jailed commands reach these hosts through a proxy (comma-separated)") do |v|
|
|
173
|
+
@allowed_hosts = v.split(",").map(&:strip).reject(&:empty?)
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
opts.on("--workdir=DIR", String, "Run commands in DIR instead of the current directory") do |v|
|
|
177
|
+
@workdir = v
|
|
178
|
+
end
|
|
179
|
+
|
|
164
180
|
opts.on("--refresh-registry", "Refresh the model registry, persist it, and exit") do
|
|
165
181
|
@refresh_registry = true
|
|
166
182
|
end
|
|
@@ -186,7 +202,7 @@ module Miniswen
|
|
|
186
202
|
return if @refresh_registry
|
|
187
203
|
|
|
188
204
|
raise "Use -m to specify the model" unless @model
|
|
189
|
-
raise "Please, provide instructions via -p option" unless @instruction
|
|
205
|
+
raise "Please, provide instructions via -p option" unless @instruction || @history
|
|
190
206
|
end
|
|
191
207
|
end
|
|
192
208
|
end
|
|
@@ -20,7 +20,7 @@ module Miniswen
|
|
|
20
20
|
env&.each { |key, value| argv += [ "--env", "#{key}=#{value}" ] }
|
|
21
21
|
argv << id
|
|
22
22
|
argv += [ "timeout", timeout.ceil.to_s ] if timeout&.positive?
|
|
23
|
-
argv += [ "
|
|
23
|
+
argv += [ "bash", "-c", command ]
|
|
24
24
|
|
|
25
25
|
Open3.popen2e(*argv) do |stdin, pipe, wait|
|
|
26
26
|
stdin.close
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "fileutils"
|
|
4
|
+
require "socket"
|
|
5
|
+
|
|
6
|
+
module Miniswen
|
|
7
|
+
class Jail < Local
|
|
8
|
+
# An HTTP CONNECT proxy that lets jailed commands reach the allowed hosts only.
|
|
9
|
+
class Proxy
|
|
10
|
+
def self.forward(port, socket)
|
|
11
|
+
server = TCPServer.new("127.0.0.1", port)
|
|
12
|
+
exit!(0) if fork
|
|
13
|
+
serve(server) { pump(it, UNIXSocket.new(socket)) }
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def self.serve(server)
|
|
17
|
+
Thread.report_on_exception = false
|
|
18
|
+
loop { Thread.new(server.accept) { yield it } }
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def self.pump(client, upstream)
|
|
22
|
+
Thread.new do
|
|
23
|
+
IO.copy_stream(client, upstream)
|
|
24
|
+
upstream.close_write
|
|
25
|
+
end
|
|
26
|
+
IO.copy_stream(upstream, client)
|
|
27
|
+
ensure
|
|
28
|
+
client.close
|
|
29
|
+
upstream.close
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def initialize(hosts:, socket:)
|
|
33
|
+
@hosts = hosts
|
|
34
|
+
@socket = socket
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def start
|
|
38
|
+
FileUtils.rm_f(@socket)
|
|
39
|
+
server = UNIXServer.new(@socket)
|
|
40
|
+
@pid = fork { self.class.serve(server) { handle(it) } }
|
|
41
|
+
self
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def stop
|
|
45
|
+
Process.kill(:KILL, @pid)
|
|
46
|
+
Process.wait(@pid)
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def handle(client)
|
|
50
|
+
_, target = client.gets("\r\n\r\n").split(" ", 3)
|
|
51
|
+
host, port = target.split(":", 2)
|
|
52
|
+
return client.write("HTTP/1.1 403 Forbidden\r\n\r\n") unless @hosts.include?(host)
|
|
53
|
+
|
|
54
|
+
upstream = Socket.tcp(host, port.to_i, connect_timeout: 10)
|
|
55
|
+
client.write("HTTP/1.1 200 Connection Established\r\n\r\n")
|
|
56
|
+
self.class.pump(client, upstream)
|
|
57
|
+
ensure
|
|
58
|
+
client.close
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
data/lib/miniswen/jail.rb
CHANGED
|
@@ -1,30 +1,39 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "open3"
|
|
3
4
|
require "shellwords"
|
|
4
5
|
|
|
5
6
|
require "miniswen/local"
|
|
7
|
+
require "miniswen/jail/proxy"
|
|
6
8
|
|
|
7
9
|
module Miniswen
|
|
8
10
|
# Runs every command in its own namespaces: none of the harness's environment,
|
|
9
|
-
# no network, read-only system, none of its files.
|
|
11
|
+
# no network, read-only system, none of its files. A task that may reach some
|
|
12
|
+
# hosts gets an HTTP proxy on loopback that allows only those.
|
|
10
13
|
class Jail < Local
|
|
11
|
-
|
|
14
|
+
PROXY_PORT = 3128
|
|
15
|
+
PROXY_SOCKET = "/var/lib/miniswen/run/miniswen-proxy.sock"
|
|
16
|
+
|
|
17
|
+
def initialize(workdir: Dir.pwd, allowed_hosts: nil)
|
|
12
18
|
@workdir = workdir
|
|
19
|
+
@allowed_hosts = allowed_hosts
|
|
13
20
|
end
|
|
14
21
|
|
|
15
22
|
def start
|
|
16
23
|
_, @stdout, @stderr, @holder = Open3.popen3(
|
|
17
|
-
|
|
24
|
+
container_env,
|
|
18
25
|
"unshare", "--net", "--mount", "--pid", "--fork", "--kill-child", "--mount-proc", "sh", "-c", setup,
|
|
19
26
|
pgroup: true, unsetenv_others: true
|
|
20
27
|
)
|
|
21
|
-
|
|
28
|
+
raise InfrastructureError, "jail did not start: #{@stderr.read}" unless @stdout.gets == "ready\n"
|
|
22
29
|
|
|
23
|
-
|
|
30
|
+
@proxy = Proxy.new(hosts: @allowed_hosts, socket: PROXY_SOCKET).start if @allowed_hosts
|
|
31
|
+
self
|
|
24
32
|
end
|
|
25
33
|
|
|
26
34
|
def stop
|
|
27
|
-
Process.kill(:KILL, -@holder.pid) if @holder
|
|
35
|
+
Process.kill(:KILL, -@holder.pid) if @holder&.alive?
|
|
36
|
+
@proxy&.stop
|
|
28
37
|
end
|
|
29
38
|
|
|
30
39
|
private
|
|
@@ -37,22 +46,40 @@ module Miniswen
|
|
|
37
46
|
mount --bind #{Shellwords.escape(@workdir)} #{Shellwords.escape(@workdir)}
|
|
38
47
|
# Make private temp dirs their own mounts
|
|
39
48
|
for dir in /tmp /run /root; do mkdir -p "/var/lib/miniswen$dir" && mount --bind "/var/lib/miniswen$dir" "$dir"; done
|
|
49
|
+
#{forwarder_command if @allowed_hosts}
|
|
40
50
|
# Make everything without its own mount read-only
|
|
41
51
|
mount -o remount,bind,ro /
|
|
42
52
|
echo ready
|
|
43
53
|
exec sleep infinity
|
|
44
54
|
SH
|
|
45
55
|
|
|
56
|
+
def forwarder_command
|
|
57
|
+
forward = "Miniswen::Jail::Proxy.forward(#{PROXY_PORT}, #{"/run/#{File.basename(PROXY_SOCKET)}".inspect})"
|
|
58
|
+
[ RbConfig.ruby, "-I", File.expand_path("..", __dir__), "-rminiswen/jail", "-e", forward ].map { Shellwords.escape(it) }.join(" ")
|
|
59
|
+
end
|
|
60
|
+
|
|
46
61
|
def spawn_arguments(command, env)
|
|
47
|
-
[
|
|
62
|
+
[ command_env(env),
|
|
48
63
|
"nsenter", "--target", @holder.pid.to_s, "--net", "--mount", "--pid=/proc/#{@holder.pid}/ns/pid_for_children", "--wd=#{@workdir}",
|
|
49
|
-
"--", "setpriv", "--bounding-set=-all", "--inh-caps=-all", "--no-new-privs", "--", "
|
|
64
|
+
"--", "setpriv", "--bounding-set=-all", "--inh-caps=-all", "--no-new-privs", "--", "bash", "-c", command ]
|
|
50
65
|
end
|
|
51
66
|
|
|
52
67
|
def spawn_options = super.merge(unsetenv_others: true)
|
|
53
68
|
|
|
69
|
+
def command_env(env)
|
|
70
|
+
variables = container_env.merge(env.to_h).slice(*container_variables)
|
|
71
|
+
return variables unless @allowed_hosts
|
|
72
|
+
|
|
73
|
+
proxy = "http://127.0.0.1:#{PROXY_PORT}"
|
|
74
|
+
variables.merge("http_proxy" => proxy, "https_proxy" => proxy, "no_proxy" => "localhost,127.0.0.1")
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def container_env
|
|
78
|
+
@container_env ||= File.read("/proc/1/environ").split("\0").to_h { it.split("=", 2) }
|
|
79
|
+
end
|
|
80
|
+
|
|
54
81
|
def container_variables
|
|
55
|
-
@container_variables ||=
|
|
82
|
+
@container_variables ||= container_env.keys | Agent::EXEC_ENV.keys
|
|
56
83
|
end
|
|
57
84
|
end
|
|
58
85
|
end
|
data/lib/miniswen/local.rb
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require "
|
|
3
|
+
require "tempfile"
|
|
4
4
|
|
|
5
5
|
require "miniswen/environment"
|
|
6
6
|
|
|
@@ -12,24 +12,23 @@ module Miniswen
|
|
|
12
12
|
# Always through a shell: Ruby execs a metacharacter-free string directly,
|
|
13
13
|
# and a missing binary would then raise ENOENT here instead of exiting 127.
|
|
14
14
|
def exec(command, timeout: nil, env: nil)
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
reader = Thread.new { io.read }
|
|
15
|
+
Tempfile.create("miniswen") do |log|
|
|
16
|
+
wait_thr = Process.detach(Process.spawn(*spawn_arguments(command, env), in: File::NULL, %i[out err] => log, **spawn_options))
|
|
18
17
|
|
|
19
18
|
if timeout&.positive? && wait_thr.join(timeout).nil?
|
|
20
19
|
kill_group(wait_thr.pid)
|
|
21
20
|
wait_thr.join
|
|
22
|
-
output = "#{scrub(
|
|
21
|
+
output = "#{scrub(File.read(log))}\n<command timed out after #{timeout} seconds>"
|
|
23
22
|
return ExecResult.new(exit_code: TIMEOUT_EXIT_CODE, output:)
|
|
24
23
|
end
|
|
25
24
|
|
|
26
|
-
ExecResult.new(exit_code: exit_code(wait_thr.value), output: scrub(
|
|
25
|
+
ExecResult.new(exit_code: exit_code(wait_thr.value), output: scrub(File.read(log)))
|
|
27
26
|
end
|
|
28
27
|
end
|
|
29
28
|
|
|
30
29
|
private
|
|
31
30
|
|
|
32
|
-
def spawn_arguments(command, env) = [ env || {}, "
|
|
31
|
+
def spawn_arguments(command, env) = [ env || {}, "bash", "-c", command ]
|
|
33
32
|
|
|
34
33
|
def spawn_options = { pgroup: true }
|
|
35
34
|
|