lemans 1.5.0 → 1.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +14 -0
- data/README.md +86 -1
- data/exe/lemans-remote +64 -8
- data/exe/lemans-viewer +45 -9
- data/lib/lemans/agent.rb +5 -4
- data/lib/lemans/agents.rb +13 -5
- data/lib/lemans/cli/regrade.rb +29 -77
- data/lib/lemans/cli/report/aggregate.rb +61 -34
- data/lib/lemans/cli/report.rb +183 -29
- data/lib/lemans/cli.rb +53 -10
- data/lib/lemans/environments/daytona/retries.rb +2 -2
- data/lib/lemans/environments/daytona.rb +7 -1
- data/lib/lemans/environments/docker/proxy/README.md +1 -1
- data/lib/lemans/environments/docker/proxy/server.rb +2 -1
- data/lib/lemans/environments/docker.rb +5 -1
- data/lib/lemans/environments.rb +14 -4
- data/lib/lemans/result.rb +14 -6
- data/lib/lemans/stores/fs.rb +5 -4
- data/lib/lemans/trial/verifier/test_scanner.rb +136 -0
- data/lib/lemans/trial/verifier.rb +19 -10
- data/lib/lemans/trial.rb +2 -1
- data/lib/lemans/version.rb +1 -1
- data/lib/miniswen/agent.rb +20 -9
- data/lib/miniswen/ruby_llm.rb +13 -0
- data/lib/miniswen/version.rb +1 -1
- metadata +2 -1
data/lib/miniswen/agent.rb
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "json"
|
|
4
|
+
require "securerandom"
|
|
4
5
|
require "time"
|
|
5
6
|
require "miniswen/version"
|
|
6
7
|
require "miniswen/ruby_llm"
|
|
@@ -188,7 +189,9 @@ module Miniswen
|
|
|
188
189
|
|
|
189
190
|
Result = Data.define(:status, :submission, :messages, :steps, :cost_source,
|
|
190
191
|
:input_tokens, :output_tokens, :cached_tokens, :thinking_tokens, :cost_usd,
|
|
191
|
-
:error) do
|
|
192
|
+
:error, :cache_id) do
|
|
193
|
+
def initialize(cache_id: nil, **attributes) = super(cache_id:, **attributes)
|
|
194
|
+
|
|
192
195
|
def success? = status == :submitted
|
|
193
196
|
|
|
194
197
|
def to_h = super.merge(cost_source: cost_source&.to_h, version: Miniswen::VERSION)
|
|
@@ -206,7 +209,7 @@ module Miniswen
|
|
|
206
209
|
input_tokens: data[:input_tokens], output_tokens: data[:output_tokens],
|
|
207
210
|
cached_tokens: data[:cached_tokens], thinking_tokens: data[:thinking_tokens],
|
|
208
211
|
cost_usd: data[:cost_usd],
|
|
209
|
-
error: data[:error]
|
|
212
|
+
error: data[:error], cache_id: data[:cache_id]
|
|
210
213
|
)
|
|
211
214
|
end
|
|
212
215
|
|
|
@@ -231,7 +234,7 @@ module Miniswen
|
|
|
231
234
|
attr_reader :messages, :environment
|
|
232
235
|
|
|
233
236
|
private attr_reader :max_steps, :max_time, :max_cost, :exec_timeout, :max_output_tokens,
|
|
234
|
-
:clock, :reporter
|
|
237
|
+
:cache_id, :clock, :reporter
|
|
235
238
|
|
|
236
239
|
# `model` is a litellm-style name ("openrouter/z-ai/glm-5.2"), optionally
|
|
237
240
|
# suffixed with a reasoning effort ("openrouter/openai/gpt-5.6-luna#xhigh").
|
|
@@ -303,7 +306,7 @@ module Miniswen
|
|
|
303
306
|
Result.new(
|
|
304
307
|
status: :error, submission: nil, messages: @messages || [], steps: @steps.to_i,
|
|
305
308
|
cost_source: cost_source, cost_usd: @cost_known == false ? nil : @cost.to_f,
|
|
306
|
-
error: error,
|
|
309
|
+
error: error, cache_id:,
|
|
307
310
|
**(@totals || { input_tokens: 0, output_tokens: 0, cached_tokens: 0, thinking_tokens: 0 })
|
|
308
311
|
)
|
|
309
312
|
end
|
|
@@ -323,6 +326,7 @@ module Miniswen
|
|
|
323
326
|
private
|
|
324
327
|
|
|
325
328
|
def start(instruction)
|
|
329
|
+
@cache_id = SecureRandom.uuid
|
|
326
330
|
uname = execute("uname -srvm").output.to_s.strip
|
|
327
331
|
@messages = [
|
|
328
332
|
{ role: "system", content: SYSTEM_TEMPLATE },
|
|
@@ -340,6 +344,7 @@ module Miniswen
|
|
|
340
344
|
end
|
|
341
345
|
|
|
342
346
|
def resume(history)
|
|
347
|
+
@cache_id = history.cache_id || SecureRandom.uuid
|
|
343
348
|
@messages = answered_messages(history.messages)
|
|
344
349
|
|
|
345
350
|
@steps = history.steps.to_i
|
|
@@ -445,7 +450,9 @@ module Miniswen
|
|
|
445
450
|
error = +""
|
|
446
451
|
error << "Unknown tool '#{call[:name]}'." if call[:name] != "bash"
|
|
447
452
|
arguments = call[:arguments]
|
|
448
|
-
if
|
|
453
|
+
if arguments.is_a?(String)
|
|
454
|
+
error << "The bash tool call arguments are not valid JSON. Resend the call with a JSON object of the form {\"command\": \"...\"}."
|
|
455
|
+
elsif !arguments.is_a?(Hash) || !arguments["command"]
|
|
449
456
|
error << "Missing 'command' argument in bash tool call."
|
|
450
457
|
elsif arguments["command"].to_s.include?("\0")
|
|
451
458
|
# Process.spawn rejects strings with NUL bytes, so the command could
|
|
@@ -533,7 +540,7 @@ module Miniswen
|
|
|
533
540
|
def finish(status, submission: nil)
|
|
534
541
|
Result.new(
|
|
535
542
|
status: status, submission: submission, messages: @messages, steps: @steps,
|
|
536
|
-
cost_source: cost_source, cost_usd: @cost_known ? @cost : nil, error: nil, **@totals
|
|
543
|
+
cost_source: cost_source, cost_usd: @cost_known ? @cost : nil, error: nil, cache_id:, **@totals
|
|
537
544
|
)
|
|
538
545
|
end
|
|
539
546
|
|
|
@@ -577,10 +584,14 @@ module Miniswen
|
|
|
577
584
|
end
|
|
578
585
|
|
|
579
586
|
def routing_params
|
|
580
|
-
|
|
581
|
-
return {} unless order && @provider == "openrouter"
|
|
587
|
+
return {} unless @provider == "openrouter"
|
|
582
588
|
|
|
583
|
-
|
|
589
|
+
params = {}
|
|
590
|
+
params[:prompt_cache_key] = cache_id if @id.start_with?("meta/muse-")
|
|
591
|
+
if (order = provider_order)
|
|
592
|
+
params[:provider] = { order: order.split(",").map(&:strip), allow_fallbacks: false }
|
|
593
|
+
end
|
|
594
|
+
params
|
|
584
595
|
end
|
|
585
596
|
|
|
586
597
|
def provider_order = ENV["LEMANS_PROVIDER_ORDER"] || ENV["OPENROUTER_PROVIDER_ORDER"]
|
data/lib/miniswen/ruby_llm.rb
CHANGED
|
@@ -43,7 +43,20 @@ module Miniswen
|
|
|
43
43
|
module RetryTransientFailures
|
|
44
44
|
def retry_exceptions = super + [ Faraday::SSLError, Faraday::ParsingError ]
|
|
45
45
|
end
|
|
46
|
+
|
|
47
|
+
# ruby_llm parses tool-call arguments while it builds the message, so one
|
|
48
|
+
# malformed call would raise and take the whole completion with it. The raw
|
|
49
|
+
# string goes through instead and bounces as a format error.
|
|
50
|
+
module LenientToolArguments
|
|
51
|
+
def parse_tool_call_arguments(tool_call)
|
|
52
|
+
super
|
|
53
|
+
rescue JSON::ParserError
|
|
54
|
+
tool_call.dig("function", "arguments")
|
|
55
|
+
end
|
|
56
|
+
end
|
|
46
57
|
end
|
|
47
58
|
|
|
48
59
|
RubyLLM::Providers::OpenRouter.prepend(Miniswen::VerbatimReasoningDetails)
|
|
49
60
|
RubyLLM::Connection.prepend(Miniswen::RetryTransientFailures)
|
|
61
|
+
RubyLLM::Providers::OpenAI::Tools.prepend(Miniswen::LenientToolArguments)
|
|
62
|
+
RubyLLM::Providers::OpenAI::Tools.singleton_class.prepend(Miniswen::LenientToolArguments)
|
data/lib/miniswen/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: lemans
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.5.
|
|
4
|
+
version: 1.5.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Svyatoslav Kryukov
|
|
@@ -276,6 +276,7 @@ files:
|
|
|
276
276
|
- lib/lemans/trial/verifier.rb
|
|
277
277
|
- lib/lemans/trial/verifier/assets/eport-lemans.rb
|
|
278
278
|
- lib/lemans/trial/verifier/assets/lemans_minitest_reporter.rb
|
|
279
|
+
- lib/lemans/trial/verifier/test_scanner.rb
|
|
279
280
|
- lib/lemans/version.rb
|
|
280
281
|
- lib/miniswen.rb
|
|
281
282
|
- lib/miniswen/agent.rb
|