miniswen 1.4.0 → 1.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 49e7b2a4a062867c6b85476f94ebda50ce7d356add5500d40b2c5c513155f7ca
-  data.tar.gz: 4b5dfc61e161b9e2776b2547e8a10862e785fedcbd5fff1a1621054657ed21fb
+  metadata.gz: b72b4a7a4e82cc48a0a50e7a27c338f34685ee9e6fefdfc555213b8c5d8b2475
+  data.tar.gz: 3d309e32ffc77990653f49768ee1dd1e6c581b56a93eddebd61f37d8ac9739dc
 SHA512:
-  metadata.gz: b6e67519d94b5e32469173068f8121f8682dd5f16398c38c118f4516931d9e8b2e068e89f612bdc3d543df0a10cc9d8279bc425de404b1e63774085f77f8fab1
-  data.tar.gz: addc1fbee5c18257a2e0f115316a33a4f0f5e9233bb6a7f47ed43d244f1e49ec0a4fd912519e9442edb149a01c25d21cebe3f84cf8d4d6142d170bc30266c80b
+  metadata.gz: 2fd6ae2583048739c94ee8dc70e92c214f4163cb7a86e2201e450d14549abc3cb5d92aba5f1f3c6c1a934968f1b738f3d8561392a2523412393cf9f0c33e04d9
+  data.tar.gz: 89ae33237c77e4025e9fe4bf5a5c5036b3098f32f9e32f91a4bd2d0925cd61cc366bd58a3de3b182b0fa4f31f6e0027a674f5b1d78a072cd2905bc0d1d3d52f5

data/README.md

@@ -6,6 +6,7 @@ lemans is a harness for benchmarking coding agents, the Ruby way:
 - **Conventional**, aka _boilerplate-free_: an instruction, a Docker environment, and a test script — that's all you need to describe an eval
 - **Trustworthy**: a grade can't be gamed by the agent, an infrastructure failure can't masquerade as a model failure, and every result carries enough digests to prove what it actually measured
 - Powered by **miniswen**, a Ruby version of mini-swe-agent (powered by [RubyLLM](https://rubyllm.com/), so it works with any LLM) and [Daytona](https://www.daytona.io) sandboxes.
+- **Extensible**: load a custom agent or a sandboxing backend on-the-fly defined in a Ruby file or provided by a gem.
 
 > [!TIP]
 > Check [Rails AI Evals](https://github.com/rails/ai-evals) for a full-featured example.
@@ -15,6 +16,7 @@ lemans is a harness for benchmarking coding agents, the Ruby way:
 - Ruby 3.4+ is required to run `lemans`
 - Daytona account (API token) or Docker (for local sandboxes)
 - Some LLM provider/proxy credentials (e.g., OpenRouter)
+- Bash in the sandbox image for command execution
 - iproute2 in the sandbox image, so a jailed agent keeps loopback while cut off the internet
 
 ## Getting started
@@ -102,6 +104,8 @@ verifier:
   # restore: [test, bin]                            # [optional] paths restored from the pre-agent snapshot, so grading can't be tampered with
 ```
 
+On the Docker backend, `allowlist` sends traffic through a proxy container. Only tools that use `http_proxy`/`https_proxy` can reach the listed hosts. Host names must match exactly, and IP ranges are not supported. See [the proxy README](lib/lemans/environments/docker/proxy/README.md).
+
 A minimal task example—checking whether an agent can write "Hello, world" into a file:
 
 - `instruction.md`: the task itself (note the preamble about the training corpora and the frontmatter)
@@ -231,6 +235,17 @@ Each trial writes a flat `runs/<model>/<task>__<id>/` directory:
  
 Every `result.json` is stamped with the `lemans_version` that wrote it, and newer lemans keeps reading runs produced by older releases. `lemans run --resume` skips trials that already have a scored result for the same agent and model.
 
+`lemans restart <run>` continues a multistep run that failed for reasons outside the model, such as a provider outage. Give it the run directories or the trial ids: `lemans restart RUN [RUN...]` restarts every run under the same options, `-c` at once, and runs none if any is refused. The new run replays the agent patches of the settled steps in a fresh sandbox and starts at the next step. It copies the settled steps' usage, phases, and artifacts, and records the source in `restarted_from`. The failed run stays as it is. It restarts even if the task or the bench changed since; the new run records the current digests.
+
+Two modes restart inside a step:
+
+- `--recover` continues the failed step's agent session. The new run also replays the step's partial patch, and the agent goes on from the saved `agent.result.json`. The session keeps the steps, tokens, cost, and time it already spent. Only `miniswen` and `miniswen-installed` can recover.
+- `--reverify` runs the last verification again with the current tests. Use it after you change the grading or the harness. The new run replays the graded step's patch and does not run its agent. If the verification was an intermediate gate and it now passes, the trial goes on to the next step. A reverification accepts scored runs.
+
+A scored run restarts in the other modes only with `--allow-scored`.
+
+Only the files that git tracks come back: ignored files (databases, logs, installed dependencies) are not in the patches.
+
 You can also run `lemans report` with various flags to see aggregated results, e.g.:
 
 ```sh
@@ -249,6 +264,83 @@ gpt-5.6-luna  ar-archive-book-access  2/2    2m 23s  $0.0132  12.5   156905
 6 trials: 6 scored, 0 invalid, 6 solved (100%) · $0.0801 · pass@2 3/3 tasks (100%)
 ```
 
+### 6. Extensions
+
+You can bring **custom agents and backends** to lemans. For that, you should:
+
+- Write an agent or a backend implementation in a Ruby file/gem
+- Load it using the `--require` / `-r` option specified **before the command**:
+
+```sh
+lemans -r ./my-agent.rb run --agent my-agent ...
+```
+
+- **IMPORTANT:** The extension must register itself in the corresponding registry and follow the leman's interface:
+
+```ruby
+# my-agent.rb
+class MyAgent < Lemans::Agent
+  NAME = "my-agent"
+
+  def install(_task, environment)
+    environment.upload("/path/to/my-agent", "/usr/local/bin/my-agent")
+    environment.exec!("chmod +x /usr/local/bin/my-agent")
+  end
+
+  def run(task, environment)
+    finished = environment.exec("cd #{task.environment.workdir} && my-agent --model #{model}", timeout:)
+    outcome = finished.exit_code == 124 ? :agent_timeout : :completed
+
+    # The returned object is a Lemans::Agent::Response.
+    # IMPORTANT: `usage` is required for correct work
+    Response.new(outcome: Lemans::Result::Outcome.new(outcome), usage: Lemans::Result::Usage.zero)
+  end
+end
+
+Lemans::Agents.register(MyAgent::NAME, MyAgent)
+
+# my-backend.rb
+class MyBackend < Lemans::Environment
+  NAME = "my-backend"
+
+  # Called with image:, resources:, network:, labels:, build_timeout:, and ttl: (see Lemans::Environment)
+  def start
+    @sandbox = MyCloud.create(image: image.name, cpus: resources.cpus, memory: resources.memory)
+    switch_network_policy!(network)
+    self
+  end
+
+  def exec(command, timeout: nil, env: {})
+    run = @sandbox.run(command, timeout: timeout || DEFAULT_TIMEOUT, env:)
+    ExecResult.new(command:, exit_code: run.exit_code, output: run.output, duration: run.duration)
+  end
+
+  def upload(local_path, remote_path) = @sandbox.put(local_path, remote_path)
+
+  def download(remote_path, local_path) = @sandbox.get(remote_path, local_path)
+
+  # Phases narrow the network: setup, then the agent's policy, then none for the verifier.
+  # IMPORTANT: raise if the policy cannot be enforced
+  def switch_network_policy!(policy)
+    @sandbox.set_network(policy.mode, hosts: policy.hosts)
+    @network = policy
+  end
+
+  def stop = @sandbox&.destroy
+end
+
+Lemans::Environments.register(MyBackend::NAME, MyBackend)
+```
+
+```sh
+lemans -r ./my-backend.rb run --backend my-backend ...
+```
+
+See [examples](./examples) for real-world extensions:
+
+- [Apple Container backend](./examples/apple-container.rb)
+- [Claude Code agent](./examples/claude-code.rb)
+
 ## CLI
 
 | Command | What it does |
@@ -256,8 +348,9 @@ gpt-5.6-luna  ar-archive-book-access  2/2    2m 23s  $0.0132  12.5   156905
 | `lemans init` | Scaffold a new bench directory: an annotated `bench.yml` and two example tasks |
 | `lemans tasks` | List the tasks in a bench (`--tag` to filter) |
 | `lemans run` | Run tasks and grade them (`--task`, `--tag`, `--agent`, `--model`, `--max-output-tokens`, `-k`, `-c`, `--resume`) |
-| `lemans report` | Summarize `runs/` as a table or CSV (`--task`, `--tag`, `--metadata key:value` to filter, `-A [task-agent-model]` to aggregate, `-S <column>` to sort); repeated attempts add pass@k per model × task, fractional grading a `credit` column |
-| `lemans clobber` | Delete run results (`--task`, `--ttl 10m\|2h\|1d`, `--invalid`, `-f` to skip the confirmation) |
+| `lemans restart <run>...` | Continue failed multistep runs from their last settled step in new runs (`-c`, `--recover` to continue the failed step's session, `--reverify` to grade again, `--allow-scored`, `--backend`, `--max-output-tokens`) |
+| `lemans report [RUNS_DIR]` | Summarize `runs/` (or `RUNS_DIR`) as a table or CSV (`--task`, `--tag`, `--model`, `--metadata key:value` to filter, `--skip-invalid` to leave out invalid trials, `-A [task-agent-model]` to aggregate, `-S <columns>` to sort, e.g. `-S score-credit`; numbers high to low, names A-Z, `^column` reverses that column; `--hide-columns steps-tokens` for a narrower table); repeated attempts add pass@k per model × task, fractional grading a `credit` column, multistep tasks a `progress` column (steps completed / task steps) |
+| `lemans clobber [RUNS_DIR]` | Delete run results under `runs/` (or `RUNS_DIR`) (`--task`, `--ttl 10m\|2h\|1d`, `--invalid`, `-f` to skip the confirmation) |
 
 ## miniswen
 
@@ -273,6 +366,13 @@ $ miniswen --model 'openrouter/openai/gpt-5.6-luna' --prompt 'Write hello-world
 
 Currently, it's a one-shot agent that doesn't ask any questions. It's mostly useful for playing with eval ideas before encoding them as lemans tasks.
 
+## More tools: lemans-remote and lemans-viewer
+
+The gem also ships two convenient (and 100% AI-maintained) CLIs:
+
+- `lemans-remote`: scale `lemans run`-s horizontally using Daytona sandboxes; schedule tasks and pull results later
+- `lemans-viewer`: a web UI for browsing trial results (if you still need a pair of human eyes to look at them)
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies, `rake test` to run the tests, and `bin/console` for an interactive prompt.

data/lib/miniswen/agent.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require "json"
+require "securerandom"
+require "time"
 require "miniswen/version"
 require "miniswen/ruby_llm"
 
@@ -187,7 +189,9 @@ module Miniswen
 
     Result = Data.define(:status, :submission, :messages, :steps, :cost_source,
                          :input_tokens, :output_tokens, :cached_tokens, :thinking_tokens, :cost_usd,
-                         :error) do
+                         :error, :cache_id) do
+      def initialize(cache_id: nil, **attributes) = super(cache_id:, **attributes)
+
       def success? = status == :submitted
 
       def to_h = super.merge(cost_source: cost_source&.to_h, version: Miniswen::VERSION)
@@ -205,7 +209,7 @@ module Miniswen
           input_tokens: data[:input_tokens], output_tokens: data[:output_tokens],
           cached_tokens: data[:cached_tokens], thinking_tokens: data[:thinking_tokens],
           cost_usd: data[:cost_usd],
-          error: data[:error]
+          error: data[:error], cache_id: data[:cache_id]
         )
       end
 
@@ -230,7 +234,7 @@ module Miniswen
     attr_reader :messages, :environment
 
     private attr_reader :max_steps, :max_time, :max_cost, :exec_timeout, :max_output_tokens,
-                        :clock, :reporter
+                        :cache_id, :clock, :reporter
 
     # `model` is a litellm-style name ("openrouter/z-ai/glm-5.2"), optionally
     # suffixed with a reasoning effort ("openrouter/openai/gpt-5.6-luna#xhigh").
@@ -261,25 +265,17 @@ module Miniswen
       @reporter = reporter
     end
 
-    def run(instruction)
-      uname = execute("uname -srvm").output.to_s.strip
-      @messages = [
-        { role: "system", content: SYSTEM_TEMPLATE },
-        { role: "user", content: format(INSTANCE_TEMPLATE,
-                                        instruction: instruction,
-                                        system_information: uname,
-                                        macos_sed_note: uname.start_with?("Darwin") ? "\n#{MACOS_SED_NOTE}" : "") }
-      ]
-
-      @steps = 0
-      @cost = 0.0
-
-      @totals = { input_tokens: 0, output_tokens: 0, cached_tokens: 0, thinking_tokens: 0 }
+    # With a `history` (the Result of an interrupted run), the session goes on
+    # where it stopped, under the budget the earlier part already spent.
+    def run(instruction = nil, history: nil)
+      if history
+        resume(history)
+      else
+        start(instruction)
+      end
 
-      @cost_known = true
       @consecutive_format_errors = 0
       @refused_turns = 0
-      @started_at = @clock.call
 
       loop do
         (status = limit_reached) and return finish(status)
@@ -310,7 +306,7 @@ module Miniswen
       Result.new(
         status: :error, submission: nil, messages: @messages || [], steps: @steps.to_i,
         cost_source: cost_source, cost_usd: @cost_known == false ? nil : @cost.to_f,
-        error: error,
+        error: error, cache_id:,
         **(@totals || { input_tokens: 0, output_tokens: 0, cached_tokens: 0, thinking_tokens: 0 })
       )
     end
@@ -329,6 +325,51 @@ module Miniswen
 
     private
 
+    def start(instruction)
+      @cache_id = SecureRandom.uuid
+      uname = execute("uname -srvm").output.to_s.strip
+      @messages = [
+        { role: "system", content: SYSTEM_TEMPLATE },
+        { role: "user", content: format(INSTANCE_TEMPLATE,
+                                        instruction: instruction,
+                                        system_information: uname,
+                                        macos_sed_note: uname.start_with?("Darwin") ? "\n#{MACOS_SED_NOTE}" : "") }
+      ]
+
+      @steps = 0
+      @cost = 0.0
+      @cost_known = true
+      @totals = { input_tokens: 0, output_tokens: 0, cached_tokens: 0, thinking_tokens: 0 }
+      @started_at = @clock.call
+    end
+
+    def resume(history)
+      @cache_id = history.cache_id || SecureRandom.uuid
+      @messages = answered_messages(history.messages)
+
+      @steps = history.steps.to_i
+      @cost = history.cost_usd.to_f
+      @cost_known = !history.cost_usd.nil?
+      @totals = { input_tokens: history.input_tokens.to_i, output_tokens: history.output_tokens.to_i,
+                  cached_tokens: history.cached_tokens.to_i, thinking_tokens: history.thinking_tokens.to_i }
+      @started_at = @clock.call - elapsed(history.messages)
+    end
+
+    # A turn whose calls did not all get a result cannot go back to a provider:
+    # it is dropped, and the model decides again from the tree it left.
+    def answered_messages(messages)
+      turn = messages.rindex { it[:role] == "assistant" && it[:tool_calls] }
+      return messages.dup unless turn
+
+      answered = messages[(turn + 1)..].filter_map { it[:tool_call_id] if it[:role] == "tool" }
+      messages[turn][:tool_calls].all? { answered.include?(it[:id]) } ? messages.dup : messages[0...turn]
+    end
+
+    def elapsed(messages)
+      times = messages.filter_map { Time.parse(it[:timestamp]) if it[:role] == "assistant" && it[:timestamp] }
+      times.empty? ? 0.0 : times.max - times.min
+    end
+
     def execute(command)
       environment.exec(command, timeout: exec_timeout, env: EXEC_ENV)
     end
@@ -409,7 +450,9 @@ module Miniswen
         error = +""
         error << "Unknown tool '#{call[:name]}'." if call[:name] != "bash"
         arguments = call[:arguments]
-        if !arguments.is_a?(Hash) || !arguments["command"]
+        if arguments.is_a?(String)
+          error << "The bash tool call arguments are not valid JSON. Resend the call with a JSON object of the form {\"command\": \"...\"}."
+        elsif !arguments.is_a?(Hash) || !arguments["command"]
           error << "Missing 'command' argument in bash tool call."
         elsif arguments["command"].to_s.include?("\0")
           # Process.spawn rejects strings with NUL bytes, so the command could
@@ -497,7 +540,7 @@ module Miniswen
     def finish(status, submission: nil)
       Result.new(
         status: status, submission: submission, messages: @messages, steps: @steps,
-        cost_source: cost_source, cost_usd: @cost_known ? @cost : nil, error: nil, **@totals
+        cost_source: cost_source, cost_usd: @cost_known ? @cost : nil, error: nil, cache_id:, **@totals
       )
     end
 
@@ -541,10 +584,14 @@ module Miniswen
     end
 
     def routing_params
-      order = provider_order
-      return {} unless order && @provider == "openrouter"
+      return {} unless @provider == "openrouter"
 
-      { provider: { order: order.split(",").map(&:strip), allow_fallbacks: false } }
+      params = {}
+      params[:prompt_cache_key] = cache_id if @id.start_with?("meta/muse-")
+      if (order = provider_order)
+        params[:provider] = { order: order.split(",").map(&:strip), allow_fallbacks: false }
+      end
+      params
     end
 
     def provider_order = ENV["LEMANS_PROVIDER_ORDER"] || ENV["OPENROUTER_PROVIDER_ORDER"]

data/lib/miniswen/cli.rb

@@ -25,6 +25,7 @@ module Miniswen
       @jail = false
       @allowed_hosts = nil
       @workdir = nil
+      @history = nil
     end
 
     def run
@@ -56,7 +57,7 @@ module Miniswen
       agent = Agent.new(model:, reporter:, environment:, **options)
 
       begin
-        result = agent.run(instruction)
+        result = agent.run(instruction, history: @history && Agent::Result.from_h(JSON.parse(@history)))
       rescue StandardError => e
         write_results(agent.partial_result(error_message(e)))
         raise
@@ -105,7 +106,7 @@ module Miniswen
 
     def parse_args!
       parser = OptionParser.new do |opts|
-        opts.banner = "Usage: miniswen -m MODEL -p INSTRUCTION [...options]"
+        opts.banner = "Usage: miniswen -m MODEL (-p INSTRUCTION | --continue-from PATH) [...options]"
 
         opts.on("-m MODEL", "--model=MODEL", String,
                 "LLM to use (litellm format, e.g.: openrouter/openai/gpt-5.6-luna") do |v|
@@ -116,6 +117,10 @@ module Miniswen
           @instruction = File.file?(v) ? File.read(v) : v
         end
 
+        opts.on("--continue-from=PATH", String, "Continue the session saved by --results-path at PATH (no -p needed)") do |v|
+          @history = File.read(v)
+        end
+
         opts.on("--max-steps=STEPS", Integer, "Max steps count") do |v|
           options[:max_steps] = v
         end
@@ -197,7 +202,7 @@ module Miniswen
       return if @refresh_registry
 
       raise "Use -m to specify the model" unless @model
-      raise "Please, provide instructions via -p option" unless @instruction
+      raise "Please, provide instructions via -p option" unless @instruction || @history
     end
   end
 end

data/lib/miniswen/environment/docker.rb

@@ -20,7 +20,7 @@ module Miniswen
         env&.each { |key, value| argv += [ "--env", "#{key}=#{value}" ] }
         argv << id
         argv += [ "timeout", timeout.ceil.to_s ] if timeout&.positive?
-        argv += [ "sh", "-c", command ]
+        argv += [ "bash", "-c", command ]
 
         Open3.popen2e(*argv) do |stdin, pipe, wait|
           stdin.close

data/lib/miniswen/jail.rb

@@ -61,7 +61,7 @@ module Miniswen
     def spawn_arguments(command, env)
       [ command_env(env),
         "nsenter", "--target", @holder.pid.to_s, "--net", "--mount", "--pid=/proc/#{@holder.pid}/ns/pid_for_children", "--wd=#{@workdir}",
-        "--", "setpriv", "--bounding-set=-all", "--inh-caps=-all", "--no-new-privs", "--", "sh", "-c", command ]
+        "--", "setpriv", "--bounding-set=-all", "--inh-caps=-all", "--no-new-privs", "--", "bash", "-c", command ]
     end
 
     def spawn_options = super.merge(unsetenv_others: true)

data/lib/miniswen/local.rb

@@ -28,7 +28,7 @@ module Miniswen
 
     private
 
-    def spawn_arguments(command, env) = [ env || {}, "sh", "-c", command ]
+    def spawn_arguments(command, env) = [ env || {}, "bash", "-c", command ]
 
     def spawn_options = { pgroup: true }
 

data/lib/miniswen/ruby_llm.rb

@@ -43,7 +43,20 @@ module Miniswen
   module RetryTransientFailures
     def retry_exceptions = super + [ Faraday::SSLError, Faraday::ParsingError ]
   end
+
+  # ruby_llm parses tool-call arguments while it builds the message, so one
+  # malformed call would raise and take the whole completion with it. The raw
+  # string goes through instead and bounces as a format error.
+  module LenientToolArguments
+    def parse_tool_call_arguments(tool_call)
+      super
+    rescue JSON::ParserError
+      tool_call.dig("function", "arguments")
+    end
+  end
 end
 
 RubyLLM::Providers::OpenRouter.prepend(Miniswen::VerbatimReasoningDetails)
 RubyLLM::Connection.prepend(Miniswen::RetryTransientFailures)
+RubyLLM::Providers::OpenAI::Tools.prepend(Miniswen::LenientToolArguments)
+RubyLLM::Providers::OpenAI::Tools.singleton_class.prepend(Miniswen::LenientToolArguments)

data/lib/miniswen/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miniswen
-  VERSION = "1.4.0"
+  VERSION = "1.5.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: miniswen
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.5.2
 platform: ruby
 authors:
 - Svyatoslav Kryukov