miniswen 1.5.0 → 1.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2df0b59321d93fcea1ddb87d555323b3dd247eed780532bd645d1410e5a6a391
-  data.tar.gz: 07c36283c59e9a311af76aa59b46118b92ff705a5478dca847f5bd321c03c7de
+  metadata.gz: 318e13712f3e46a9ad03b1105b7f5c5c6892d5b75a6566d0c272d196de54d491
+  data.tar.gz: 4f0c7dbd37fe212152e66a1abca5ee39513136d564a0293098920947a1a0a02f
 SHA512:
-  metadata.gz: d481729c89f35e3aae9cbb02dc2c5db59327b292c49a4a30920e81a178480411d075e72bc59dc845bc0906e9194af7cd37d2833f2a092ae135a359b7460eb066
-  data.tar.gz: 1eb136297aac2c4702efc6b2c6bc4829a53bdca70a85c582bd39150cc17e0cd7851f62adcb0acee53cb05f82461339efb9d7a1d5057657cc47fc45300aee051a
+  metadata.gz: f0eb42558de2ad1e9db1a80b1c9f9fdf488dc7cdcc82e848a3086f8a517cc1a5ed5fe36a6901e86e81320d192ef8f218935346e343aea30ca6de8fbb7586f497
+  data.tar.gz: 82ad81a8753d0f5ba9c9415e4e6b4c198c8cb2692c539b05c47f9ed446752df96893f49922ca060ab5356820ad2e456bb5625087247b1d7040b63f55c035656c

data/README.md

@@ -6,6 +6,7 @@ lemans is a harness for benchmarking coding agents, the Ruby way:
 - **Conventional**, aka _boilerplate-free_: an instruction, a Docker environment, and a test script — that's all you need to describe an eval
 - **Trustworthy**: a grade can't be gamed by the agent, an infrastructure failure can't masquerade as a model failure, and every result carries enough digests to prove what it actually measured
 - Powered by **miniswen**, a Ruby version of mini-swe-agent (powered by [RubyLLM](https://rubyllm.com/), so it works with any LLM) and [Daytona](https://www.daytona.io) sandboxes.
+- **Extensible**: load a custom agent or a sandboxing backend on-the-fly defined in a Ruby file or provided by a gem.
 
 > [!TIP]
 > Check [Rails AI Evals](https://github.com/rails/ai-evals) for a full-featured example.
@@ -263,6 +264,84 @@ gpt-5.6-luna  ar-archive-book-access  2/2    2m 23s  $0.0132  12.5   156905
 6 trials: 6 scored, 0 invalid, 6 solved (100%) · $0.0801 · pass@2 3/3 tasks (100%)
 ```
 
+### 6. Extensions
+
+You can bring **custom agents and backends** to lemans. For that, you should:
+
+- Write an agent or a backend implementation in a Ruby file/gem
+- Load it using the `--require` / `-r` option specified **before the command**:
+
+```sh
+lemans -r ./my-agent.rb run --agent my-agent ...
+```
+
+- **IMPORTANT:** The extension must register itself in the corresponding registry and follow the leman's interface:
+
+```ruby
+# my-agent.rb
+class MyAgent < Lemans::Agent
+  NAME = "my-agent"
+
+  def install(_task, environment)
+    environment.upload("/path/to/my-agent", "/usr/local/bin/my-agent")
+    environment.exec!("chmod +x /usr/local/bin/my-agent")
+  end
+
+  def run(task, environment)
+    finished = environment.exec("cd #{task.environment.workdir} && my-agent --model #{model}", timeout:)
+    outcome = finished.exit_code == 124 ? :agent_timeout : :completed
+
+    # The returned object is a Lemans::Agent::Response.
+    # IMPORTANT: `usage` is required for correct work
+    Response.new(outcome: Lemans::Result::Outcome.new(outcome), usage: Lemans::Result::Usage.zero)
+  end
+end
+
+Lemans::Agents.register(MyAgent::NAME, MyAgent)
+
+# my-backend.rb
+class MyBackend < Lemans::Environment
+  NAME = "my-backend"
+
+  # Called with image:, resources:, network:, labels:, build_timeout:, and ttl: (see Lemans::Environment)
+  def start
+    @sandbox = MyCloud.create(image: image.name, cpus: resources.cpus, memory: resources.memory)
+    switch_network_policy!(network)
+    self
+  end
+
+  def exec(command, timeout: nil, env: {})
+    run = @sandbox.run(command, timeout: timeout || DEFAULT_TIMEOUT, env:)
+    ExecResult.new(command:, exit_code: run.exit_code, output: run.output, duration: run.duration)
+  end
+
+  def upload(local_path, remote_path) = @sandbox.put(local_path, remote_path)
+
+  def download(remote_path, local_path) = @sandbox.get(remote_path, local_path)
+
+  # Phases narrow the network: setup, then the agent's policy, then none for the verifier.
+  # IMPORTANT: raise if the policy cannot be enforced
+  def switch_network_policy!(policy)
+    @sandbox.set_network(policy.mode, hosts: policy.hosts)
+    @network = policy
+  end
+
+  def stop = @sandbox&.destroy
+end
+
+Lemans::Environments.register(MyBackend::NAME, MyBackend)
+```
+
+```sh
+lemans -r ./my-backend.rb run --backend my-backend ...
+```
+
+See [examples](./examples) for real-world extensions:
+
+- [Apple Container backend](./examples/apple-container.rb)
+- [Claude Code agent](./examples/claude-code.rb)
+- [Codex agent](./examples/codex/README.md)
+
 ## CLI
 
 | Command | What it does |
@@ -271,7 +350,7 @@ gpt-5.6-luna  ar-archive-book-access  2/2    2m 23s  $0.0132  12.5   156905
 | `lemans tasks` | List the tasks in a bench (`--tag` to filter) |
 | `lemans run` | Run tasks and grade them (`--task`, `--tag`, `--agent`, `--model`, `--max-output-tokens`, `-k`, `-c`, `--resume`) |
 | `lemans restart <run>...` | Continue failed multistep runs from their last settled step in new runs (`-c`, `--recover` to continue the failed step's session, `--reverify` to grade again, `--allow-scored`, `--backend`, `--max-output-tokens`) |
-| `lemans report [RUNS_DIR]` | Summarize `runs/` (or `RUNS_DIR`) as a table or CSV (`--task`, `--tag`, `--metadata key:value` to filter, `-A [task-agent-model]` to aggregate, `-S <column>` to sort); repeated attempts add pass@k per model × task, fractional grading a `credit` column |
+| `lemans report [RUNS_DIR]` | Summarize `runs/` (or `RUNS_DIR`) as a table or CSV (`--task`, `--tag`, `--model`, `--metadata key:value` to filter, `--skip-invalid` to leave out invalid trials, `-A [task-agent-model]` to aggregate, `-S <columns>` to sort, e.g. `-S score-credit`; numbers high to low, names A-Z, `^column` reverses that column; `--hide-columns steps-tokens` for a narrower table); repeated attempts add pass@k per model × task, fractional grading a `credit` column, multistep tasks a `progress` column (steps completed / task steps) |
 | `lemans clobber [RUNS_DIR]` | Delete run results under `runs/` (or `RUNS_DIR`) (`--task`, `--ttl 10m\|2h\|1d`, `--invalid`, `-f` to skip the confirmation) |
 
 ## miniswen
@@ -288,6 +367,13 @@ $ miniswen --model 'openrouter/openai/gpt-5.6-luna' --prompt 'Write hello-world
 
 Currently, it's a one-shot agent that doesn't ask any questions. It's mostly useful for playing with eval ideas before encoding them as lemans tasks.
 
+## More tools: lemans-remote and lemans-viewer
+
+The gem also ships two convenient (and 100% AI-maintained) CLIs:
+
+- `lemans-remote`: scale `lemans run`-s horizontally using Daytona sandboxes; schedule tasks and pull results later
+- `lemans-viewer`: a web UI for browsing trial results (if you still need a pair of human eyes to look at them)
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies, `rake test` to run the tests, and `bin/console` for an interactive prompt.

data/lib/miniswen/agent.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require "json"
+require "securerandom"
+require "shellwords"
 require "time"
 require "miniswen/version"
 require "miniswen/ruby_llm"
@@ -13,21 +15,6 @@ module Miniswen
     MAX_OBSERVATION_CHARS = 10_000
     MAX_CONSECUTIVE_FORMAT_ERRORS = 3
 
-    # Both finish_reason dialects accepted raw: OpenAI-shaped providers say
-    # "length"/"tool_calls", Anthropic says "max_tokens"/"tool_use".
-    TRUNCATION_FINISH_REASONS = %w[length max_tokens model_context_window_exceeded].freeze
-    CLAIMED_TOOL_FINISH_REASONS = %w[tool_calls tool_use].freeze
-    # A safety stop, which arrives looking exactly like a model that forgot
-    # to call the tool: no content, no tool call, and — since the provider
-    # bills nothing for a turn it refused — no tokens either. Only the
-    # finish reason tells the two apart, so the run is labelled by it. The
-    # retry is unchanged: the nudge still goes back, because matching
-    # mini-swe-agent turn for turn is what makes runs comparable.
-    REFUSAL_FINISH_REASONS = %w[content_filter refusal safety].freeze
-
-    # The breakpoint marker Anthropic reads, shaped the way OpenRouter forwards it.
-    CACHE_CONTROL = { type: "ephemeral" }.freeze
-
     EXEC_ENV = {
       "PAGER" => "cat",
       "MANPAGER" => "cat",
@@ -36,9 +23,6 @@ module Miniswen
       "TQDM_DISABLE" => "1"
     }.freeze
 
-    # Providers that serve local inference (and cost zero)
-    LOCAL_PROVIDERS = %i[ollama gpustack].freeze
-
     SYSTEM_TEMPLATE = <<~PROMPT
       You are a helpful assistant that can interact with a computer.
     PROMPT
@@ -167,20 +151,10 @@ module Miniswen
       without any other command.
     MESSAGE
 
-    # Thinking that also carries the provider's reasoning_details for verbatim replay.
-    class VerbatimThinking < RubyLLM::Thinking
-      attr_reader :details
-
-      def initialize(text: nil, signature: nil, details: nil)
-        super(text: text, signature: signature)
-        @details = details
-      end
-    end
-
     # Only ever rendered into the request — the completion executes nothing — so no execute body.
     class BashTool < RubyLLM::Tool
       description "Execute a bash command"
-      param :command, desc: "The bash command to execute"
+      parameter :command, description: "The bash command to execute"
 
       # ruby_llm would otherwise derive "lemans--miniswen--bash" from the class path.
       def name = "bash"
@@ -188,7 +162,9 @@ module Miniswen
 
     Result = Data.define(:status, :submission, :messages, :steps, :cost_source,
                          :input_tokens, :output_tokens, :cached_tokens, :thinking_tokens, :cost_usd,
-                         :error) do
+                         :error, :cache_id) do
+      def initialize(cache_id: nil, **attributes) = super(cache_id:, **attributes)
+
       def success? = status == :submitted
 
       def to_h = super.merge(cost_source: cost_source&.to_h, version: Miniswen::VERSION)
@@ -206,7 +182,7 @@ module Miniswen
           input_tokens: data[:input_tokens], output_tokens: data[:output_tokens],
           cached_tokens: data[:cached_tokens], thinking_tokens: data[:thinking_tokens],
           cost_usd: data[:cost_usd],
-          error: data[:error]
+          error: data[:error], cache_id: data[:cache_id]
         )
       end
 
@@ -230,13 +206,13 @@ module Miniswen
 
     attr_reader :messages, :environment
 
-    private attr_reader :max_steps, :max_time, :max_cost, :exec_timeout, :max_output_tokens,
-                        :clock, :reporter
+    private attr_reader :workdir, :max_steps, :max_time, :max_cost, :exec_timeout, :max_output_tokens,
+                        :cache_id, :clock, :reporter
 
     # `model` is a litellm-style name ("openrouter/z-ai/glm-5.2"), optionally
     # suffixed with a reasoning effort ("openrouter/openai/gpt-5.6-luna#xhigh").
     # Limits of 0 or nil are disabled.
-    def initialize(model:, environment:, max_steps: 0, max_time: 0, max_cost: nil,
+    def initialize(model:, environment:, workdir: nil, max_steps: 0, max_time: 0, max_cost: nil,
                    exec_timeout: 30, max_output_tokens: 0,
                    clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) },
                    reporter: nil)
@@ -249,6 +225,7 @@ module Miniswen
 
       @model = model
       @environment = environment
+      @workdir = workdir
 
       @bash_tool = BashTool.new
 
@@ -303,7 +280,7 @@ module Miniswen
       Result.new(
         status: :error, submission: nil, messages: @messages || [], steps: @steps.to_i,
         cost_source: cost_source, cost_usd: @cost_known == false ? nil : @cost.to_f,
-        error: error,
+        error: error, cache_id:,
         **(@totals || { input_tokens: 0, output_tokens: 0, cached_tokens: 0, thinking_tokens: 0 })
       )
     end
@@ -323,6 +300,7 @@ module Miniswen
     private
 
     def start(instruction)
+      @cache_id = SecureRandom.uuid
       uname = execute("uname -srvm").output.to_s.strip
       @messages = [
         { role: "system", content: SYSTEM_TEMPLATE },
@@ -335,16 +313,19 @@ module Miniswen
       @steps = 0
       @cost = 0.0
       @cost_known = true
+      @cost_reported = false
       @totals = { input_tokens: 0, output_tokens: 0, cached_tokens: 0, thinking_tokens: 0 }
       @started_at = @clock.call
     end
 
     def resume(history)
+      @cache_id = history.cache_id || SecureRandom.uuid
       @messages = answered_messages(history.messages)
 
       @steps = history.steps.to_i
       @cost = history.cost_usd.to_f
       @cost_known = !history.cost_usd.nil?
+      @cost_reported = history.cost_source&.name == :provider_reported
       @totals = { input_tokens: history.input_tokens.to_i, output_tokens: history.output_tokens.to_i,
                   cached_tokens: history.cached_tokens.to_i, thinking_tokens: history.thinking_tokens.to_i }
       @started_at = @clock.call - elapsed(history.messages)
@@ -366,6 +347,7 @@ module Miniswen
     end
 
     def execute(command)
+      command = "cd #{Shellwords.escape(workdir)}; #{command}" if workdir
       environment.exec(command, timeout: exec_timeout, env: EXEC_ENV)
     end
 
@@ -394,6 +376,7 @@ module Miniswen
       # content, no tool call and no billed tokens.
       entry[:finish_reason] = response[:finish_reason] if response[:finish_reason]
       entry[:thinking] = response[:thinking] if response[:thinking]
+      entry[:provider] = response[:provider] if response[:provider]
       # The provider's opaque handles on this turn's reasoning: replayed back
       # to the provider only, never into the trajectory.
       entry[:thinking_signature] = response[:thinking_signature] if response[:thinking_signature]
@@ -407,7 +390,7 @@ module Miniswen
         # message with unanswered tool calls is a request providers reject.
         entry[:invalid_tool_calls] = tool_calls if tool_calls.any?
         @consecutive_format_errors += 1
-        @refused_turns += 1 if refused?(response)
+        @refused_turns += 1 if response[:refused]
 
         observe_message({ role: "user", content: format_error_message(error, response, tool_calls) })
 
@@ -420,8 +403,6 @@ module Miniswen
       tool_calls
     end
 
-    def refused?(response) = REFUSAL_FINISH_REASONS.include?(response[:finish_reason].to_s)
-
     # An unpriced completion under a cost ceiling is fatal on the spot: only failing fast stops
     # the spend. Without a ceiling, unknown cost is just a fact to report.
     def track_cost(response)
@@ -436,6 +417,7 @@ module Miniswen
       else
         @cost += cost
       end
+      @cost_reported = true if response[:reported]
     end
 
     def actions_error(tool_calls)
@@ -445,7 +427,9 @@ module Miniswen
         error = +""
         error << "Unknown tool '#{call[:name]}'." if call[:name] != "bash"
         arguments = call[:arguments]
-        if !arguments.is_a?(Hash) || !arguments["command"]
+        if arguments.is_a?(String)
+          error << "The bash tool call arguments are not valid JSON. Resend the call with a JSON object of the form {\"command\": \"...\"}."
+        elsif !arguments.is_a?(Hash) || !arguments["command"]
           error << "Missing 'command' argument in bash tool call."
         elsif arguments["command"].to_s.include?("\0")
           # Process.spawn rejects strings with NUL bytes, so the command could
@@ -459,10 +443,8 @@ module Miniswen
     end
 
     def format_error_message(error, response, tool_calls)
-      finish_reason = response[:finish_reason].to_s
-      if TRUNCATION_FINISH_REASONS.include?(finish_reason) ||
-         (CLAIMED_TOOL_FINISH_REASONS.include?(finish_reason) && tool_calls.empty?)
-        format(TRUNCATION_ERROR_MESSAGE, finish_reason: finish_reason)
+      if response[:truncated] || (response[:finish_reason] == "tool_calls" && tool_calls.empty?)
+        format(TRUNCATION_ERROR_MESSAGE, finish_reason: response[:finish_reason])
       else
         format(TOOL_CALL_ERROR_MESSAGE, error: error)
       end
@@ -533,7 +515,7 @@ module Miniswen
     def finish(status, submission: nil)
       Result.new(
         status: status, submission: submission, messages: @messages, steps: @steps,
-        cost_source: cost_source, cost_usd: @cost_known ? @cost : nil, error: nil, **@totals
+        cost_source: cost_source, cost_usd: @cost_known ? @cost : nil, error: nil, cache_id:, **@totals
       )
     end
 
@@ -541,11 +523,13 @@ module Miniswen
     def complete(messages)
       model_info, provider = resolved
       response = provider.complete(
-        with_cache_breakpoints(messages.map { as_ruby_llm(it) }),
+        messages.map { as_ruby_llm(it) },
         tools: { bash: @bash_tool },
         temperature: nil,
         model: model_info,
-        params: routing_params.merge(output_cap_params(model_info)),
+        provider_options: routing_params,
+        max_output_tokens: output_cap,
+        caching: {},
         thinking: (RubyLLM::Thinking::Config.new(effort: @effort) if @effort)
       )
       payload(response)
@@ -557,48 +541,20 @@ module Miniswen
       raise InfrastructureError, "miniswen: the model call failed: #{e.class}: #{e.message}"
     end
 
-    # Anthropic bills every token fresh unless the request marks explicit cache
-    # breakpoints
-    def explicit_cache? = @provider == "openrouter" && @id.to_s.start_with?("anthropic/")
-
-    def with_cache_breakpoints(messages)
-      return messages unless explicit_cache?
-
-      system = messages.find { it.role == :system }
-      [ system, messages.last ].compact.uniq.each do |message|
-        text = message.content
-        next unless text.is_a?(String) && !text.empty?
-
-        message.content = RubyLLM::Content::Raw.new(
-          [ { type: "text", text: text, cache_control: CACHE_CONTROL } ]
-        )
-      end
-      messages
-    end
-
     def routing_params
-      order = provider_order
-      return {} unless order && @provider == "openrouter"
+      return {} unless @provider == "openrouter"
 
-      { provider: { order: order.split(",").map(&:strip), allow_fallbacks: false } }
+      params = { prompt_cache_key: cache_id }
+      if (order = provider_order)
+        params[:provider] = { order: order.split(",").map(&:strip), allow_fallbacks: false }
+      end
+      params
     end
 
     def provider_order = ENV["LEMANS_PROVIDER_ORDER"] || ENV["OPENROUTER_PROVIDER_ORDER"]
 
-    # OpenAI itself retired `max_tokens` for its reasoning models; the
-    # OpenAI-compatible providers and Anthropic still read it.
-    def output_cap_params(model_info)
-      return {} if max_output_tokens.zero?
-
-      cap = [ info&.max_tokens, max_output_tokens ].compact.min
-      provider_class = RubyLLM::Provider.providers[model_info.provider.to_sym]
-      if [ RubyLLM::Providers::OpenAI, RubyLLM::Providers::Azure ].include?(provider_class)
-        { max_completion_tokens: cap }
-      elsif provider_class <= RubyLLM::Providers::OpenAI || provider_class <= RubyLLM::Providers::Anthropic
-        { max_tokens: cap }
-      else
-        {}
-      end
+    def output_cap
+      [ info&.max_output_tokens, max_output_tokens ].compact.min unless max_output_tokens.zero?
     end
 
     def cost_source
@@ -607,12 +563,12 @@ module Miniswen
                               priced_as: "#{@provider || info&.provider}/#{@id} ($0.00, local)",
                               registry: nil)
       end
-      return nil unless info
-      if free?
-        return CostSource.new(name: :free_model, model: @model,
-                              priced_as: "#{info.provider}/#{info.id} ($0.00, free)",
+      if @cost_reported
+        return CostSource.new(name: :provider_reported, model: @model,
+                              priced_as: "#{@provider || info&.provider}/#{@id} (as billed, else the registry price)",
                               registry: Miniswen.registry_revision)
       end
+      return nil unless info
 
       CostSource.new(name: :model_registry, model: @model,
                      priced_as: "#{info.provider}/#{info.id}",
@@ -620,14 +576,16 @@ module Miniswen
     end
 
     def resolved
-      @resolved ||= RubyLLM::Models.resolve(@id, provider: @provider, assume_exists: !@provider.nil?)
+      @resolved ||= RubyLLM::Models.resolve(@id, provider: @provider, assume_model_exists: !@provider.nil?)
     end
 
     def as_ruby_llm(entry)
       case entry[:role]
       when "assistant"
-        RubyLLM::Message.new(role: :assistant, content: entry[:content],
-                             thinking: as_ruby_llm_thinking(entry),
+        thinking = RubyLLM::Thinking.build(text: entry[:thinking], signature: entry[:thinking_signature])
+        RubyLLM::Message.new(role: :assistant, content: entry[:content], thinking:,
+                             raw_reasoning: entry[:reasoning_details],
+                             usage_entries: [ producing_entry ],
                              tool_calls: as_ruby_llm_tool_calls(entry[:tool_calls]))
       when "tool"
         RubyLLM::Message.new(role: :tool, content: entry[:content], tool_call_id: entry[:tool_call_id])
@@ -636,13 +594,11 @@ module Miniswen
       end
     end
 
-    def as_ruby_llm_thinking(entry)
-      details = entry[:reasoning_details]
-      if details && !details.empty?
-        VerbatimThinking.new(text: entry[:thinking], signature: entry[:thinking_signature], details: details)
-      else
-        RubyLLM::Thinking.build(text: entry[:thinking], signature: entry[:thinking_signature])
-      end
+    # A rebuilt turn must name its provider, or ruby_llm drops its thinking signature.
+    def producing_entry
+      model_info, provider = resolved
+      RubyLLM::Accounting::Usage::Entry.new(operation: :chat, provider: provider.slug, model: model_info.id,
+                                            status: :succeeded)
     end
 
     def as_ruby_llm_tool_calls(tool_calls)
@@ -663,16 +619,20 @@ module Miniswen
         content: response.content.to_s,
         thinking: response.thinking&.text,
         thinking_signature: response.thinking&.signature,
-        reasoning_details: reasoning_details_from(response),
+        reasoning_details: response.raw_reasoning,
         tool_calls: tool_calls_from(response),
-        finish_reason: finish_reason_from(response),
+        finish_reason: response.finish_reason&.to_s,
+        truncated: response.max_tokens?,
+        refused: response.content_filtered?,
         # ruby_llm's input_tokens is the cache-miss remainder only; the
         # convention counts the whole prompt, cached and cache-write included.
-        input_tokens: response.input_tokens.to_i + tokens&.cached.to_i + tokens&.cache_creation.to_i,
-        output_tokens: response.output_tokens.to_i,
-        cached_tokens: tokens&.cached.to_i,
-        thinking_tokens: tokens&.thinking.to_i,
-        cost_usd: price(response)
+        input_tokens: tokens.input.to_i + tokens.cache_read.to_i + tokens.cache_write.to_i,
+        output_tokens: tokens.output.to_i,
+        cached_tokens: tokens.cache_read.to_i,
+        thinking_tokens: tokens.thinking.to_i,
+        cost_usd: price(response),
+        reported: !tokens.reported_cost.nil?,
+        provider: response.raw&.body&.dig("provider")
       }
     end
 
@@ -693,27 +653,7 @@ module Miniswen
       arguments
     end
 
-    def finish_reason_from(response)
-      body = raw_body(response)
-      body && (body.dig("choices", 0, "finish_reason") || body["stop_reason"])
-    end
-
-    def reasoning_details_from(response)
-      details = raw_body(response)&.dig("choices", 0, "message", "reasoning_details")
-      details.is_a?(Array) && !details.empty? ? details : nil
-    end
-
-    def raw_body(response)
-      body = response.raw&.body
-      body = JSON.parse(body) if body.is_a?(String)
-      body.is_a?(Hash) ? body : nil
-    rescue JSON::ParserError
-      nil
-    end
-
-    def local? = LOCAL_PROVIDERS.include?((@provider || info&.provider)&.to_sym)
-
-    def free? = info&.provider == "openrouter" && info.input_price_per_million.nil? && info.output_price_per_million.nil?
+    def local? = RubyLLM::Provider.providers[(@provider || info&.provider)&.to_sym]&.local? || false
 
     def info
       return @info if defined?(@info)
@@ -722,28 +662,15 @@ module Miniswen
     end
 
     def find_model
-      @provider ? RubyLLM.models.find(@id, @provider) : RubyLLM.models.find(@id)
+      RubyLLM.models.find(@id, provider: @provider)
     rescue RubyLLM::ModelNotFoundError
       nil
     end
 
     def price(response)
-      return 0.0 if local? || free?
+      return 0.0 if local? || (response.tokens.to_h.empty? && response.tokens.reported_cost.nil?)
 
-      input = info&.input_price_per_million
-      output = info&.output_price_per_million
-      return nil unless input.is_a?(Numeric) && output.is_a?(Numeric)
-
-      tokens = response.tokens
-      cache_read = info.cache_read_input_price_per_million || input
-      cache_write = info.cache_write_input_price_per_million || input
-      # Providers usually fold thinking into output_tokens; max() bills the
-      # larger count once and can never double-bill.
-      generated = [ response.output_tokens.to_i, tokens&.thinking.to_i ].max
-      ((response.input_tokens.to_i * input) +
-        (tokens&.cached.to_i * cache_read) +
-        (tokens&.cache_creation.to_i * cache_write) +
-        (generated * output)) / 1_000_000.0
+      info ? info.cost_for(response.tokens).total : response.tokens.reported_cost
     end
   end
 end

data/lib/miniswen/cli.rb

@@ -99,7 +99,7 @@ module Miniswen
     end
 
     def refresh_registry_and_exit
-      refreshed = Miniswen.refresh_registry!(persist: true)
+      refreshed = Miniswen.refresh_registry!
       $stdout.puts Miniswen.registry_revision if refreshed
       Kernel.exit(refreshed ? 0 : 1)
     end

data/lib/miniswen/jail/proxy.rb

@@ -37,7 +37,13 @@ module Miniswen
       def start
         FileUtils.rm_f(@socket)
         server = UNIXServer.new(@socket)
-        @pid = fork { self.class.serve(server) { handle(it) } }
+        check, @lifeline = IO.pipe
+        @pid = fork do
+          @lifeline.close
+          Thread.new { check.wait_readable; exit! }
+          self.class.serve(server) { handle(it) }
+        end
+        check.close
         self
       end
 

data/lib/miniswen/jail.rb

@@ -22,7 +22,8 @@ module Miniswen
     def start
       _, @stdout, @stderr, @holder = Open3.popen3(
         container_env,
-        "unshare", "--net", "--mount", "--pid", "--fork", "--kill-child", "--mount-proc", "sh", "-c", setup,
+        "setpriv", "--pdeathsig=KILL", "unshare", "--net", "--mount", "--pid", "--fork", "--kill-child", "--mount-proc",
+        "sh", "-c", setup,
         pgroup: true, unsetenv_others: true
       )
       raise InfrastructureError, "jail did not start: #{@stderr.read}" unless @stdout.gets == "ready\n"

data/lib/miniswen/ruby_llm.rb

@@ -4,8 +4,8 @@ require "ruby_llm"
 
 # Logging configuration
 RubyLLM.configure do |config|
-  config.log_level = ENV.fetch("RUBYLLM_LOG_LEVEL", "info").to_sym
-  config.logger = Logger.new(IO::NULL) unless ENV["MINISWEN_DEBUG"] == "1"
+  config.log_level = ENV["RUBYLLM_LOG_LEVEL"].to_sym if ENV["RUBYLLM_LOG_LEVEL"]
+  config.logger = Logger.new(IO::NULL) unless ENV["MINISWEN_DEBUG"] == "1" || ENV["RUBYLLM_DEBUG"] == "1"
 end
 
 # About ten minutes of retries (1, 2, 4, ... 256s plus jitter): provider
@@ -13,6 +13,7 @@ end
 RubyLLM.configure do |config|
   config.max_retries = 9
   config.retry_interval = 1
+  config.retry_max_interval = 256
 end
 
 # ruby_llm reads no API keys from ENV on its own; the conventional variable is the provider's
@@ -27,23 +28,37 @@ RubyLLM.configure do |config|
 end
 
 module Miniswen
-  # OpenRouter requires reasoning_details replayed exactly as received; ruby_llm
-  # rebuilds them from its collapsed text+signature pair, and providers that
-  # sign each block separately reject that as a corrupted thought signature.
-  module VerbatimReasoningDetails
-    def format_thinking(msg)
-      details = msg.thinking.respond_to?(:details) ? msg.thinking.details : nil
-      details && !details.empty? ? { reasoning_details: details } : super
-    end
+  # A 200 cut off mid-body is a dropped connection, but ruby_llm doesn't retry it.
+  module RetryTransientFailures
+    def retry_exceptions = super + [ Faraday::ParsingError ]
   end
 
-  # ruby_llm lists SSL errors as fatal, but a handshake reset never sent the
-  # request; a 200 with a truncated JSON body is the same dropped connection
-  # one step later.
-  module RetryTransientFailures
-    def retry_exceptions = super + [ Faraday::SSLError, Faraday::ParsingError ]
+  # ruby_llm parses tool-call arguments while it builds the message, so one
+  # malformed call would raise and take the whole completion with it. The raw
+  # string goes through instead and bounces as a format error.
+  module LenientToolArguments
+    def parse_tool_call_arguments(tool_call, **)
+      super
+    rescue RubyLLM::ToolCallParseError
+      tool_call.dig("function", "arguments")
+    end
+
+    def parse_function_call_arguments(arguments, **)
+      super
+    rescue RubyLLM::ToolCallParseError
+      arguments
+    end
+
+    def parse_arguments(arguments, *)
+      super
+    rescue RubyLLM::ToolCallParseError
+      arguments
+    end
   end
 end
 
-RubyLLM::Providers::OpenRouter.prepend(Miniswen::VerbatimReasoningDetails)
-RubyLLM::Connection.prepend(Miniswen::RetryTransientFailures)
+RubyLLM::Transport::Connection.singleton_class.prepend(Miniswen::RetryTransientFailures)
+[ RubyLLM::Protocols::ChatCompletions::Tools, RubyLLM::Protocols::Responses::Chat, RubyLLM::Protocols::Cohere::Tools ].each do |parser|
+  parser.prepend(Miniswen::LenientToolArguments)
+  parser.singleton_class.prepend(Miniswen::LenientToolArguments)
+end

data/lib/miniswen/testing.rb

@@ -21,22 +21,22 @@ module Miniswen
 
     # $20/M input (fresh and cached) and $400/M output price the default answer
     # (95 fresh + 5 cached input tokens, 20 output tokens) at exactly $0.01.
-    PRICED_MODEL = RubyLLM::Model::Info.new(
-      id: "test", name: "Test", provider: "test",
+    PRICED_MODEL = RubyLLM::Model.new(
+      id: "test", name: "Test", provider: "test", max_output_tokens: 4096,
       capabilities: %w[function_calling],
       modalities: { input: %w[text], output: %w[text] },
       pricing: { text_tokens: { standard: {
-        input_per_million: 20.0, cached_input_per_million: 20.0, output_per_million: 400.0
+        input_per_million: 20.0, cache_read_input_per_million: 20.0, output_per_million: 400.0
       } } }
     )
 
-    UNPRICED_MODEL = RubyLLM::Model::Info.new(
+    UNPRICED_MODEL = RubyLLM::Model.new(
       id: "test-unpriced", name: "Test unpriced", provider: "test",
       capabilities: %w[function_calling],
       modalities: { input: %w[text], output: %w[text] }
     )
 
-    FREE_MODEL = RubyLLM::Model::Info.new(
+    FREE_MODEL = RubyLLM::Model.new(
       id: "test-free", name: "Test free", provider: "openrouter",
       capabilities: %w[function_calling],
       modalities: { input: %w[text], output: %w[text] }
@@ -46,13 +46,11 @@ module Miniswen
 
     # Serves the test models from ruby_llm's registry lookups.
     module FindTestModels
-      def find(model_id, provider = nil)
+      def find(model_id, **)
         MODELS[model_id] || super
       end
     end
 
-    RawResponse = Data.define(:body)
-
     SUBMIT = { content: "Done.", cmd: "echo #{Agent::SUBMIT_MARKER}" }.freeze
 
     # A shell made of a Hash. `echo` behaves like echo, so the submit marker
@@ -71,6 +69,7 @@ module Miniswen
 
       def exec(command, timeout: nil, env: nil) # rubocop:disable Lint/UnusedMethodArgument
         @commands << command
+        command = command.sub(/\Acd .+?; /, "")
         return result(0, command.delete_prefix("echo ")) if command.start_with?("echo ")
 
         exit_code, output = @canned.fetch(command, [ 0, "" ])
@@ -91,7 +90,7 @@ module Miniswen
 
     # Each answer is a command string, or a hash with :cmd (one command or many)
     # and/or :content, plus optional overrides (:tool_calls, :finish_reason,
-    # :thinking, token counts). Overrides given here apply to every answer.
+    # :thinking, :raw, token counts). Overrides given here apply to every answer.
     def stub_llm(*answers, **overrides)
       RubyLLM::Test.stub_responses(*answers.map { llm_answer(it, **overrides) })
     end
@@ -110,20 +109,17 @@ module Miniswen
                                                              signature: answer[:thinking_signature]),
         input_tokens: answer.fetch(:input_tokens, 95),
         output_tokens: answer.fetch(:output_tokens, 20),
-        cached_tokens: answer.fetch(:cached_tokens, 5),
+        cache_read_tokens: answer.fetch(:cached_tokens, 5),
         thinking_tokens: answer[:thinking] && 40,
-        raw: RawResponse.new(body: raw_body_for(answer, calls))
+        raw_reasoning: answer[:reasoning_details],
+        reported_cost: answer[:reported_cost],
+        raw: answer[:raw],
+        finish_reason: answer.fetch(:finish_reason) { calls.empty? ? "stop" : "tool_calls" }
       )
     end
 
     private
 
-    def raw_body_for(answer, calls)
-      choice = { "finish_reason" => answer.fetch(:finish_reason) { calls.empty? ? "stop" : "tool_calls" } }
-      choice["message"] = { "reasoning_details" => answer[:reasoning_details] } if answer[:reasoning_details]
-      { "choices" => [ choice ] }
-    end
-
     def tool_calls_for(answer)
       calls = answer[:tool_calls] || Array(answer[:cmd]).map { { name: "bash", arguments: { "command" => it } } }
       calls.to_h do |call|

data/lib/miniswen/trajectory.rb

@@ -78,7 +78,8 @@ module Miniswen
         step[:llm_call_count] = 1
       end
       # ATIF has no field for how a turn ended
-      step[:extra] = { finish_reason: message[:finish_reason] } if message[:finish_reason]
+      extra = { finish_reason: message[:finish_reason], provider: message[:provider] }.compact
+      step[:extra] = extra unless extra.empty?
       step
     end
 

data/lib/miniswen/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miniswen
-  VERSION = "1.5.0"
+  VERSION = "1.6.0"
 end

data/lib/miniswen.rb

@@ -16,14 +16,10 @@ module Miniswen # :nodoc:
   class AccountingError < Error; end
 
   class << self
-    def refresh_registry!(persist: false)
+    def refresh_registry!
       return true if @ruby_llm_refreshed
 
-      RubyLLM.models.refresh!
-      # save_to_json writes to the registry file every boot loads from (the
-      # gem's own models.json by default), so a persisted refresh outlives
-      # this process — later runs in the same environment boot from it.
-      RubyLLM.models.save_to_json if persist
+      RubyLLM.models.refresh
       @ruby_llm_refreshed = true
     rescue StandardError => e
       warn "Failed to refresh RubyLLM registry: #{e.message}"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: miniswen
 version: !ruby/object:Gem::Version
-  version: 1.5.0
+  version: 1.6.0
 platform: ruby
 authors:
 - Svyatoslav Kryukov
@@ -31,14 +31,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: 2.1.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: 2.1.1
 description: A Ruby port of mini-swe-agent.
 email:
 - me@skryukov.dev