miniswen 1.0.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 85e11349e68be75bb5c676a1ad629639d663cf117fd1bc27555522c9b2b9e4c7
-  data.tar.gz: a1c34a07733af4791e348f856326847e01183cd4f7e78d03bca08c665c86bb04
+  metadata.gz: de70dca3df1d5d69b884c838142ecf6143714e4160df12cd70ffd9d82d9cd823
+  data.tar.gz: 9fc776f43acb75cd49b0b02a45dbaf2017927c2a5a9d0ac53589925b5be6cb99
 SHA512:
-  metadata.gz: 86f44e426ca5c3e66cfccf4defd18489c2019a54844505026e497ae0481dfe11fa68c32131fef6cc303ca968d1b423fe1e44ac34f4bc84c618e356f974033168
-  data.tar.gz: 33f3b1333f54a4a9c3f642e26ac09ac3717a8906b56e82248fcd1555e545bafca024793bd5eac05cd181baffd2f2adae5bc906a5443e13a892b4833f1a1420e3
+  metadata.gz: b65975d0c54ef1588a896917dc7ee86b1c87c7a6c750d0354e6d1d8681743e20c2561890ebeefbce3cae2d96981ac5b41c26a6411b2d350653bce026ca9e730c
+  data.tar.gz: c067f93043fd5f056a243200b08e5d92bd3c4d25356bc961ae8a377e2dc86ef91600ce9d468747f5b18daa746be2f50e87122f5ad9ea4de6827b0fb1e1727b84

data/README.md

@@ -46,7 +46,7 @@ my-bench/
     ├── hello-world/
     │   ├── instruction.md          # what the agent is asked to do; YAML frontmatter carries
     │   │                           # name, description, difficulty, tags, metadata — plus the
-    │   │                           # per-task overrides: setup, restore, verifier.setup
+    │   │                           # per-task overrides: setup, restore, verifier.setup, environment
     │   ├── environment/Dockerfile  # [optional] a task-specific sandbox image, instead of the shared one
     │   ├── environment.patch       # [optional] task setup patch: applied and resealed as a fresh git repo at setup
     │   ├── verification_test.rb    # grades the result
@@ -59,6 +59,9 @@ The `bench.yml` looks like this:
 ```yaml
 version: 1
 
+# inherit_from: ../bench.yml      # [optional] start from another bench.yml; the sections below are
+#                                 # deep-merged over it (lists replace; set a key to ~ to drop it)
+
 # setup:                          # [optional] sandbox preparation, run before the agent starts
 #   files: [fixtures/seed.sql]    # uploaded for the commands to consume, then wiped
 #   commands: [bin/sandbox-setup]
@@ -68,6 +71,11 @@ environment:
   # backend: "daytona" # or "docker"
   # dockerfile: environment/Dockerfile  # the default — or pin a published image instead:
   # image: ghcr.io/acme/my-bench@sha256:...
+  # profiles:                           # [optional] named alternatives to the shared image; a task
+  #   campfire:                         # picks one with `environment: campfire` in its frontmatter
+  #     dockerfile: docker/campfire/Dockerfile
+  #   fizzy:
+  #     image: ghcr.io/acme/fizzy@sha256:...
   resources: { cpus: 2, memory: 2GB, storage: 5GB }
   build_timeout: 10m
   network:
@@ -76,6 +84,7 @@ environment:
 agent:
   name: miniswen-installed
   model: openrouter/z-ai/glm-5.2 # any model RubyLLM can reach; a list benchmarks several in one run
+                                 # append #<effort> (e.g. openrouter/openai/gpt-5.6-luna#xhigh) to pin the reasoning effort
   timeout: 30m
   step_limit: 100
   cost_limit: 5.0
@@ -133,7 +142,7 @@ A minimal task example—checking whether an agent can write "Hello, world" into
   +Hello, world
   ```
 
-- `verification_test.rb`: a Ruby test that grades the solution (pass → 1, fail → 0; for partial credit, write a float in 0.0..1.0 to `$LOGS/reward.txt` instead). We use Minitest:
+- `verification_test.rb`: a Ruby test that grades the solution (pass → 1, fail → 0; for partial credit, write a float in 0.0..1.0 to `$LOGS/reward.txt` instead). A check worth recording but not grading goes into `allow_failure { ... }` (`include LemansReport::Assertions`): a failed assertion inside lands in `checks.json` as `fail (allowed)` with its message and leaves the reward alone, while errors and skips stay hard failures. We use Minitest:
 
   ```ruby
   require "minitest/autorun"
@@ -161,13 +170,16 @@ verifier:
 
 An `environment.patch` next to `instruction.md` is always applied, declared or not.
 
+A heavy task may need more than the bench-wide budgets: a `bench.yml` in the task directory is deep-merged over the bench's (as if it had `inherit_from` pointing at it) and applies to that task's trials only — say, `agent: { timeout: 2h, step_limit: 300 }`.
+
 ### 3. Set credentials
 
 ```bash
 export DAYTONA_API_KEY=...      # or DAYTONA_TOKEN (if using Daytona)
 export OPENROUTER_API_KEY=...   # or ANTHROPIC_API_KEY, OPENAI_API_KEY, ... — matching your model
 
-export LEMANS_PROVIDER_ORDER="Chutes"  # [optional] Pin an OpenRouter model to named backends
+export LEMANS_PROVIDER_ORDER="chutes"  # [optional] Pin an OpenRouter model to provider slugs (alias: OPENROUTER_PROVIDER_ORDER)
+export LEMANS_PROVIDER_ORDER="z-ai,chutes"  # ...or try them in order
 ```
 
 ### 4. Prove the bench before benchmarking anything

data/lib/miniswen/agent.rb

@@ -232,11 +232,14 @@ module Miniswen
     private attr_reader :max_steps, :max_time, :max_cost, :exec_timeout,
                         :clock, :reporter
 
-    # `model` is a litellm-style name ("openrouter/z-ai/glm-5.2"). Limits of 0 or nil are disabled.
+    # `model` is a litellm-style name ("openrouter/z-ai/glm-5.2"), optionally
+    # suffixed with a reasoning effort ("openrouter/openai/gpt-5.6-luna#xhigh").
+    # Limits of 0 or nil are disabled.
     def initialize(model:, environment:, max_steps: 0, max_time: 0, max_cost: nil,
                    exec_timeout: 30, clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) },
                    reporter: nil)
-      @provider, @id = model.split("/", 2)
+      name, @effort = model.split("#", 2)
+      @provider, @id = name.split("/", 2)
       unless @id
         @id = @provider
         @provider = nil
@@ -315,8 +318,8 @@ module Miniswen
       _, provider = resolved
       env = provider.configuration_requirements.to_h { [ it.to_s.upcase, RubyLLM.config.public_send(it) ] }.compact
 
-      order = ENV["LEMANS_PROVIDER_ORDER"]
-      env["LEMANS_PROVIDER_ORDER"] = order if order
+      order = provider_order
+      env["OPENROUTER_PROVIDER_ORDER"] = order if order
       env
     end
 
@@ -500,7 +503,8 @@ module Miniswen
         tools: { bash: @bash_tool },
         temperature: nil,
         model: model_info,
-        params: routing_params
+        params: routing_params,
+        thinking: (RubyLLM::Thinking::Config.new(effort: @effort) if @effort)
       )
       payload(response)
     rescue RubyLLM::Error => e
@@ -531,12 +535,14 @@ module Miniswen
     end
 
     def routing_params
-      order = ENV["LEMANS_PROVIDER_ORDER"]
+      order = provider_order
       return {} unless order && @provider == "openrouter"
 
       { provider: { order: order.split(",").map(&:strip), allow_fallbacks: false } }
     end
 
+    def provider_order = ENV["LEMANS_PROVIDER_ORDER"] || ENV["OPENROUTER_PROVIDER_ORDER"]
+
     def cost_source
       if local?
         return CostSource.new(name: :local_provider, model: @model,

data/lib/miniswen/cli/reporter.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+module Miniswen
+  class CLI
+    # Prints messages and tool calls in real-time. The renderer deliberately
+    # keeps the captured (non-TTY) version plain, which makes it useful in CI
+    # and when piping a run to a log file too.
+    class Reporter
+      private attr_reader :io
+
+      # Tool output can be extremely noisy (for example, a recursive grep or
+      # a test runner dumping a log). Keep the normal report useful while
+      # allowing -vv to retain the complete output for debugging.
+      MAX_TOOL_OUTPUT_CHARS = 1_000
+
+      def initialize(io = $stdout, verbose: false, tool_output: verbose, reasoning: true)
+        @io = io
+        @verbose = verbose
+        @tool_output = tool_output
+        @reasoning = reasoning
+      end
+
+      def on_message(message)
+        case message[:role].to_s
+        when "assistant"
+          write_assistant(message)
+        when "tool"
+          write_tool(message)
+        when "user"
+          write_block("!", message[:content], :warning)
+        else
+          write_block("·", message[:content], :muted)
+        end
+      end
+
+      # Tool calls are reported separately so the command is visible before
+      # its output arrives. It is not added to the trajectory sent to the LLM.
+      def on_tool_call(call)
+        command = call.dig(:arguments, "command") || call.dig(:arguments, :command)
+        return if command.to_s.empty?
+
+        line = style("$ #{command}", :command)
+        io.puts("  #{line}")
+      end
+
+      def print_summary(result)
+        write_assistant(result.messages.last)
+        write_block("↳", "steps=#{result.steps} · cost=$#{result.cost_usd}", :muted)
+      end
+
+      def print_failure(result)
+        write_block("!", "Miniswen failed: #{result.status}", :warning)
+      end
+
+      private
+
+      def write_tool(message)
+        return write_block("↳", message[:content], :tool) if @tool_output
+
+        exit_code = message.dig(:observation, :exit_code)
+        return if exit_code.nil?
+
+        if exit_code.zero?
+          write_block("↳", "ok", :muted)
+        else
+          output = truncate(message.dig(:observation, :output).to_s.strip)
+          write_block("!", "not ok: exit #{exit_code}\n#{output}", :warning)
+        end
+      end
+
+      def write_assistant(message)
+        content = message[:content].to_s.strip
+        write_block("∴", message[:thinking], :thinking) if @reasoning && (@verbose || content.empty?)
+        write_block("●", content, :assistant)
+      end
+
+      def write_block(marker, content, tone)
+        text = content.to_s.strip
+        return if text.empty?
+
+        text = truncate(text) if %i[tool thinking].include?(tone)
+        lines = text.lines(chomp: true)
+        io.puts("#{style(marker, tone)} #{style(lines.shift, tone)}")
+        lines.each { |line| io.puts("  #{style(line, tone)}") }
+      end
+
+      def truncate(text)
+        return text if @verbose || text.length <= MAX_TOOL_OUTPUT_CHARS
+
+        head = MAX_TOOL_OUTPUT_CHARS / 2
+        tail = MAX_TOOL_OUTPUT_CHARS - head
+        omitted = text.length - head - tail
+        "#{text[0, head]}\n... [#{omitted} characters omitted] ...\n#{text[-tail, tail]}"
+      end
+
+      def style(text, tone)
+        return text unless io.respond_to?(:tty?) && io.tty?
+
+        colors = { assistant: 36, tool: 32, warning: 33, command: 35, muted: 90, thinking: 90 }
+        "\e[#{colors.fetch(tone)}m#{text}\e[0m"
+      end
+    end
+  end
+end

data/lib/miniswen/cli.rb

@@ -1,88 +1,13 @@
 # frozen_string_literal: true
 
 require "optparse"
+require "fileutils"
 
 require "miniswen/version"
+require "miniswen/cli/reporter"
 
 module Miniswen
   class CLI # :nodoc:
-    # Prints messages and tool calls in real-time. The renderer deliberately
-    # keeps the captured (non-TTY) version plain, which makes it useful in CI
-    # and when piping a run to a log file too.
-    class Reporter
-      private attr_reader :io
-
-      # Tool output can be extremely noisy (for example, a recursive grep or
-      # a test runner dumping a log). Keep the normal report useful while
-      # allowing -vv to retain the complete output for debugging.
-      MAX_TOOL_OUTPUT_CHARS = 1_000
-
-      def initialize(io = $stdout, verbose: false)
-        @io = io
-        @verbose = verbose
-      end
-
-      def on_message(message)
-        case message[:role].to_s
-        when "assistant"
-          write_block("●", message[:content], :assistant)
-        when "tool"
-          write_block("↳", message[:content], :tool)
-        when "user"
-          write_block("!", message[:content], :warning)
-        else
-          write_block("·", message[:content], :muted)
-        end
-      end
-
-      # Tool calls are reported separately so the command is visible before
-      # its output arrives. It is not added to the trajectory sent to the LLM.
-      def on_tool_call(call)
-        command = call.dig(:arguments, "command") || call.dig(:arguments, :command)
-        return if command.to_s.empty?
-
-        line = style("$ #{command}", :command)
-        io.puts("  #{line}")
-      end
-
-      def print_summary(result)
-        write_block("●", result.messages.last[:content], :assistant)
-        write_block("↳", "steps=#{result.steps} · cost=$#{result.cost_usd}", :muted)
-      end
-
-      def print_failure(result)
-        write_block("!", "Miniswen failed: #{result.status}", :warning)
-      end
-
-      private
-
-      def write_block(marker, content, tone)
-        text = content.to_s.strip
-        return if text.empty?
-
-        text = truncate_tool_output(text) if tone == :tool
-        lines = text.lines(chomp: true)
-        io.puts("#{style(marker, tone)} #{style(lines.shift, tone)}")
-        lines.each { |line| io.puts("  #{style(line, tone)}") }
-      end
-
-      def truncate_tool_output(text)
-        return text if @verbose || text.length <= MAX_TOOL_OUTPUT_CHARS
-
-        head = MAX_TOOL_OUTPUT_CHARS / 2
-        tail = MAX_TOOL_OUTPUT_CHARS - head
-        omitted = text.length - head - tail
-        "#{text[0, head]}\n... [#{omitted} characters omitted] ...\n#{text[-tail, tail]}"
-      end
-
-      def style(text, tone)
-        return text unless io.respond_to?(:tty?) && io.tty?
-
-        colors = { assistant: 36, tool: 32, warning: 33, command: 35, muted: 90 }
-        "\e[#{colors.fetch(tone)}m#{text}\e[0m"
-      end
-    end
-
     attr_reader :instruction, :model, :options
 
     def initialize
@@ -90,6 +15,8 @@ module Miniswen
       @model = ENV.fetch("MINISWEN_MODEL", nil)
       @options = {}
       @verbose = false
+      @show_output = false
+      @reasoning = true
       @quiet = false
       @results_path = nil
       @atif_path = nil
@@ -110,8 +37,16 @@ module Miniswen
 
       require "miniswen/local"
 
-      reporter = @quiet ? nil : Reporter.new(verbose: @verbose)
-      agent = Agent.new(model:, reporter:, environment: Local.new, **options)
+      reporter = @quiet ? nil : Reporter.new(verbose: @verbose, tool_output: @verbose || @show_output, reasoning: @reasoning)
+      environment =
+        if @docker_id
+          require "miniswen/environment/docker"
+          Environment::Docker.new(@docker_id)
+        else
+          Local.new
+        end
+
+      agent = Agent.new(model:, reporter:, environment:, **options)
 
       begin
         result = agent.run(instruction)
@@ -133,8 +68,15 @@ module Miniswen
     private
 
     def write_results(result)
-      File.write(@results_path, JSON.generate(result.to_h)) if @results_path
-      write_atif(result) if @atif_path
+      if @results_path
+        FileUtils.mkdir_p(File.dirname(@results_path))
+        File.write(@results_path, JSON.generate(result.to_h))
+      end
+
+      if @atif_path
+        FileUtils.mkdir_p(File.dirname(@atif_path))
+        write_atif(result)
+      end
     end
 
     def error_message(error)
@@ -162,7 +104,7 @@ module Miniswen
         end
 
         opts.on("-p INSTRUCTION", "--prompt=INSTRUCTION", String, "Instruction prompt") do |v|
-          @instruction = v
+          @instruction = File.file?(v) ? File.read(v) : v
         end
 
         opts.on("--max-steps=STEPS", Integer, "Max steps count") do |v|
@@ -185,6 +127,14 @@ module Miniswen
           @quiet = true
         end
 
+        opts.on("--show-output", "Print tool output instead of just the exit status") do
+          @show_output = true
+        end
+
+        opts.on("--no-reasoning", "Hide the model's reasoning") do
+          @reasoning = false
+        end
+
         opts.on("--results-path=PATH", String, "Write the run result as JSON to PATH") do |v|
           @results_path = v
         end
@@ -193,6 +143,10 @@ module Miniswen
           @atif_path = v
         end
 
+        opts.on("--docker=ID", String, "Docker container ID to exec commands on") do |v|
+          @docker_id = v
+        end
+
         opts.on("--refresh-registry", "Refresh the model registry, persist it, and exit") do
           @refresh_registry = true
         end

data/lib/miniswen/environment/docker.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require "open3"
+
+require "miniswen/environment"
+
+module Miniswen
+  class Environment
+    class Docker < self
+      TIMEOUT_MARKED_EXIT_CODES = [ 124, 143 ].freeze
+
+      private attr_reader :id
+
+      def initialize(id)
+        @id = id
+      end
+
+      def exec(command, timeout: nil, env: nil)
+        argv = [ "docker", "exec" ]
+        env&.each { |key, value| argv += [ "--env", "#{key}=#{value}" ] }
+        argv << id
+        argv += [ "timeout", timeout.ceil.to_s ] if timeout&.positive?
+        argv += [ "sh", "-c", command ]
+
+        Open3.popen2e(*argv) do |stdin, pipe, wait|
+          stdin.close
+          deadline = (now + timeout + 10 if timeout&.positive?) # add some slack
+          output = +""
+          timed_out = false
+
+          loop do
+            remaining = deadline && deadline - now
+            if remaining && remaining <= 0
+              timed_out = true
+              kill(wait.pid)
+              break
+            end
+            next unless pipe.wait_readable(remaining)
+
+            chunk = pipe.read_nonblock(65_536, exception: false)
+            break if chunk.nil?
+            next if chunk == :wait_readable
+
+            output << chunk
+          end
+
+          status = wait.value
+          exit_code = timed_out ? 124 : (status.exitstatus || 1)
+          output = output.force_encoding(Encoding::UTF_8).scrub
+          if timeout&.positive? && (timed_out || TIMEOUT_MARKED_EXIT_CODES.include?(exit_code))
+            output = "#{output}\n<command timed out after #{timeout} seconds>"
+          end
+
+          ExecResult.new(exit_code:, output:)
+        end
+      end
+
+      private
+
+      def kill(pid)
+        Process.kill("KILL", pid)
+      rescue Errno::ESRCH
+        nil
+      end
+
+      def now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+  end
+end

data/lib/miniswen/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miniswen
-  VERSION = "1.0.0"
+  VERSION = "1.2.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: miniswen
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.2.0
 platform: ruby
 authors:
 - Svyatoslav Kryukov
@@ -55,7 +55,9 @@ files:
 - lib/miniswen.rb
 - lib/miniswen/agent.rb
 - lib/miniswen/cli.rb
+- lib/miniswen/cli/reporter.rb
 - lib/miniswen/environment.rb
+- lib/miniswen/environment/docker.rb
 - lib/miniswen/local.rb
 - lib/miniswen/ruby_llm.rb
 - lib/miniswen/testing.rb