lemans 1.0.0.pre.1 → 1.0.0.rc.2

data/lib/lemans/trial/verifier.rb

@@ -64,8 +64,8 @@ module Lemans
       def upload_tests!
         environment.exec!("rm -rf #{TESTS_DIR} && mkdir -p #{TESTS_DIR}")
 
-        uploads = task.verifier.files.to_h { |local, remote| [remote, local] }
-                      .merge(task.test_files.to_h { |local, remote| [remote, local] })
+        uploads = task.verifier.files.to_h { |local, remote| [ remote, local ] }
+                      .merge(task.test_files.to_h { |local, remote| [ remote, local ] })
 
         uploads.each { |remote, local| environment.upload(local, "#{TESTS_DIR}/#{remote}") }
         ASSETS.glob("*.rb").each { |asset| environment.upload(asset, "#{TESTS_DIR}/#{asset.basename}") }
@@ -102,7 +102,7 @@ module Lemans
       end
 
       def verifier_script
-        [task.verifier.preverify, task.verifier.command].compact.map { "( #{it} )" }.join(" && ")
+        [ task.verifier.preverify, task.verifier.command ].compact.map { "( #{it} )" }.join(" && ")
       end
 
       def read_reward(command_result)

data/lib/lemans/trial.rb

@@ -118,7 +118,7 @@ module Lemans
       raise
     rescue StandardError => e
       # A harness bug must leave evidence.
-      result.failed!(:harness_crash, ["#{e.class}: #{e.message}", *Array(e.backtrace).first(5)].join("\n"))
+      result.failed!(:harness_crash, [ "#{e.class}: #{e.message}", *Array(e.backtrace).first(5) ].join("\n"))
     ensure
       environment&.stop
     end

data/lib/lemans/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Lemans
-  VERSION = "1.0.0.pre.1"
+  VERSION = "1.0.0.rc.2"
 end

data/lib/lemans.rb

@@ -9,6 +9,8 @@ loader.inflector.inflect("cli" => "CLI", "atif" => "ATIF", "fs" => "FS",
 loader.ignore("#{__dir__}/miniswen.rb", "#{__dir__}/miniswen")
 # assets/ holds files uploaded into sandboxes, not Ruby the harness loads.
 loader.ignore("#{__dir__}/lemans/trial/verifier/assets")
+# templates/ holds the files `lemans init` scaffolds, not Ruby the harness loads.
+loader.ignore("#{__dir__}/lemans/cli/templates")
 loader.setup
 
 require "miniswen"

data/lib/miniswen/agent.rb

@@ -215,9 +215,9 @@ module Miniswen
           value.to_h do |key, item|
             key = key.to_sym
             # Tool-call arguments and reasoning details keep their provider-style string keys.
-            [key, %i[arguments reasoning_details].include?(key) ? item : deep_symbolize(item)]
+            [ key, %i[arguments reasoning_details].include?(key) ? item : deep_symbolize(item) ]
           end
-        when Array then value.map { deep_symbolize(_1) }
+        when Array then value.map { deep_symbolize(it) }
         else value
         end
       end
@@ -313,7 +313,7 @@ module Miniswen
     # them back from ENV on boot (the option upcased).
     def provider_env
       _, provider = resolved
-      env = provider.configuration_requirements.to_h { [_1.to_s.upcase, RubyLLM.config.public_send(_1)] }.compact
+      env = provider.configuration_requirements.to_h { [ it.to_s.upcase, RubyLLM.config.public_send(it) ] }.compact
 
       order = ENV["LEMANS_PROVIDER_ORDER"]
       env["LEMANS_PROVIDER_ORDER"] = order if order
@@ -340,7 +340,7 @@ module Miniswen
     def next_actions
       response = complete(@messages)
       @steps += 1
-      @totals.each_key { @totals[_1] += response[_1].to_i }
+      @totals.each_key { @totals[it] += response[it].to_i }
       track_cost(response)
 
       entry = { role: "assistant", timestamp: Time.now.utc.iso8601, content: response[:content].to_s,
@@ -496,7 +496,7 @@ module Miniswen
     def complete(messages)
       model_info, provider = resolved
       response = provider.complete(
-        with_cache_breakpoints(messages.map { as_ruby_llm(_1) }),
+        with_cache_breakpoints(messages.map { as_ruby_llm(it) }),
         tools: { bash: @bash_tool },
         temperature: nil,
         model: model_info,
@@ -518,13 +518,13 @@ module Miniswen
     def with_cache_breakpoints(messages)
       return messages unless explicit_cache?
 
-      system = messages.find { _1.role == :system }
-      [system, messages.last].compact.uniq.each do |message|
+      system = messages.find { it.role == :system }
+      [ system, messages.last ].compact.uniq.each do |message|
         text = message.content
         next unless text.is_a?(String) && !text.empty?
 
         message.content = RubyLLM::Content::Raw.new(
-          [{ type: "text", text: text, cache_control: CACHE_CONTROL }]
+          [ { type: "text", text: text, cache_control: CACHE_CONTROL } ]
         )
       end
       messages
@@ -580,8 +580,8 @@ module Miniswen
       return nil if tool_calls.nil? || tool_calls.empty?
 
       tool_calls.to_h do |call|
-        [call[:id], RubyLLM::ToolCall.new(id: call[:id], name: call[:name], arguments: call[:arguments],
-                                          thought_signature: call[:thought_signature])]
+        [ call[:id], RubyLLM::ToolCall.new(id: call[:id], name: call[:name], arguments: call[:arguments],
+                                          thought_signature: call[:thought_signature]) ]
       end
     end
 
@@ -668,7 +668,7 @@ module Miniswen
       cache_write = info.cache_write_input_price_per_million || input
       # Providers usually fold thinking into output_tokens; max() bills the
       # larger count once and can never double-bill.
-      generated = [response.output_tokens.to_i, tokens&.thinking.to_i].max
+      generated = [ response.output_tokens.to_i, tokens&.thinking.to_i ].max
       ((response.input_tokens.to_i * input) +
         (tokens&.cached.to_i * cache_read) +
         (tokens&.cache_creation.to_i * cache_write) +

data/lib/miniswen/ruby_llm.rb

@@ -8,6 +8,12 @@ RubyLLM.configure do |config|
   config.logger = Logger.new(IO::NULL) unless ENV["MINISWEN_DEBUG"] == "1"
 end
 
+# Increase retry window to handle egress network issues
+RubyLLM.configure do |config|
+  config.max_retries = 3
+  config.retry_interval = 1
+end
+
 # ruby_llm reads no API keys from ENV on its own; the conventional variable is the provider's
 # config option upcased.
 RubyLLM.configure do |config|
@@ -34,7 +40,7 @@ module Miniswen
   # ConnectionFailed retries ruby_llm already does; it only lists SSL errors
   # as fatal.
   module RetryTransientSSL
-    def retry_exceptions = super + [Faraday::SSLError]
+    def retry_exceptions = super + [ Faraday::SSLError ]
   end
 end
 

data/lib/miniswen/testing.rb

@@ -60,14 +60,14 @@ module Miniswen
       end
 
       def on(command, output, exit_code: 0)
-        @canned[command] = [exit_code, output]
+        @canned[command] = [ exit_code, output ]
       end
 
       def exec(command, timeout: nil, env: nil) # rubocop:disable Lint/UnusedMethodArgument
         @commands << command
         return result(0, command.delete_prefix("echo ")) if command.start_with?("echo ")
 
-        exit_code, output = @canned.fetch(command, [0, ""])
+        exit_code, output = @canned.fetch(command, [ 0, "" ])
         result(exit_code, output)
       end
 
@@ -87,7 +87,7 @@ module Miniswen
     # and/or :content, plus optional overrides (:tool_calls, :finish_reason,
     # :thinking, token counts). Overrides given here apply to every answer.
     def stub_llm(*answers, **overrides)
-      RubyLLM::Test.stub_responses(*answers.map { llm_answer(_1, **overrides) })
+      RubyLLM::Test.stub_responses(*answers.map { llm_answer(it, **overrides) })
     end
 
     def llm_answer(answer, **overrides)
@@ -115,15 +115,15 @@ module Miniswen
     def raw_body_for(answer, calls)
       choice = { "finish_reason" => answer.fetch(:finish_reason) { calls.empty? ? "stop" : "tool_calls" } }
       choice["message"] = { "reasoning_details" => answer[:reasoning_details] } if answer[:reasoning_details]
-      { "choices" => [choice] }
+      { "choices" => [ choice ] }
     end
 
     def tool_calls_for(answer)
-      calls = answer[:tool_calls] || Array(answer[:cmd]).map { { name: "bash", arguments: { "command" => _1 } } }
+      calls = answer[:tool_calls] || Array(answer[:cmd]).map { { name: "bash", arguments: { "command" => it } } }
       calls.to_h do |call|
         id = call[:id] || "call_#{@llm_answer_ids = @llm_answer_ids.to_i + 1}"
-        [id, RubyLLM::ToolCall.new(id: id, name: call[:name], arguments: call[:arguments],
-                                   thought_signature: call[:thought_signature])]
+        [ id, RubyLLM::ToolCall.new(id: id, name: call[:name], arguments: call[:arguments],
+                                   thought_signature: call[:thought_signature]) ]
       end
     end
   end

data/lib/miniswen/trajectory.rb

@@ -83,7 +83,7 @@ module Miniswen
     end
 
     def tool_calls_for(calls)
-      calls.map { { tool_call_id: _1[:id], function_name: _1[:name], arguments: _1[:arguments] } }
+      calls.map { { tool_call_id: it[:id], function_name: it[:name], arguments: it[:arguments] } }
     end
 
     def metrics_for(metrics)

data/lib/miniswen/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miniswen
-  VERSION = "0.1.1"
+  VERSION = "1.0.0.rc.1"
 end

metadata

@@ -1,11 +1,12 @@
 --- !ruby/object:Gem::Specification
 name: lemans
 version: !ruby/object:Gem::Version
-  version: 1.0.0.pre.1
+  version: 1.0.0.rc.2
 platform: ruby
 authors:
 - Svyatoslav Kryukov
 - Artur Petrov
+- Vladimir Dementyev
 bindir: exe
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
@@ -140,6 +141,7 @@ description: Lemans runs tasks against a coding agent in a sandbox.
 email:
 - me@skryukov.dev
 - ardecvz@gmail.com
+- dementiev.vm@gmail.com
 executables:
 - lemans
 - lemans-remote
@@ -163,6 +165,17 @@ files:
 - lib/lemans/cli/progress_reporter.rb
 - lib/lemans/cli/report.rb
 - lib/lemans/cli/report/aggregate.rb
+- lib/lemans/cli/templates/bench/README.md
+- lib/lemans/cli/templates/bench/bench.yml
+- lib/lemans/cli/templates/bench/environment/Dockerfile
+- lib/lemans/cli/templates/bench/tasks/example-task/environment.patch
+- lib/lemans/cli/templates/bench/tasks/example-task/instruction.md
+- lib/lemans/cli/templates/bench/tasks/example-task/solution.patch
+- lib/lemans/cli/templates/bench/tasks/example-task/verification_test.rb
+- lib/lemans/cli/templates/bench/tasks/hello-world/environment.patch
+- lib/lemans/cli/templates/bench/tasks/hello-world/instruction.md
+- lib/lemans/cli/templates/bench/tasks/hello-world/solution.patch
+- lib/lemans/cli/templates/bench/tasks/hello-world/verification_test.rb
 - lib/lemans/clobber.rb
 - lib/lemans/config.rb
 - lib/lemans/config/agent.rb
@@ -186,6 +199,7 @@ files:
 - lib/lemans/runner.rb
 - lib/lemans/runner/executor.rb
 - lib/lemans/runner/task.rb
+- lib/lemans/secrets_filter.rb
 - lib/lemans/store.rb
 - lib/lemans/stores/fs.rb
 - lib/lemans/task_definition.rb