lemans 1.4.0 → 1.5.1

data/lib/lemans/trial/verifier.rb

@@ -10,7 +10,7 @@ module Lemans
     # Verifies a trial in the sandbox the agent worked in, after Trial has closed
     # its network. The tests are uploaded fresh at verification time, never before.
     class Verifier
-      Verification = Data.define(:reward, :credit, :logs)
+      Verification = Data.define(:reward, :credit, :features, :logs)
 
       REWARD_RANGE = (0.0..1.0)
 
@@ -22,6 +22,13 @@ module Lemans
 
       VERIFY_BIN = "verify"
 
+      # A feature passes when all its checks passed; one with a check the run
+      # never reported is left out.
+      def self.features_from(statuses, features)
+        graded = features.to_h.select { |_, checks| checks.all? { statuses.key?(it) } }
+        graded.transform_values { |checks| checks.all? { statuses[it] == "pass" } }.sort.to_h unless graded.empty?
+      end
+
       private attr_reader :task, :environment, :snapshot, :timeout
 
       def initialize(task, environment, snapshot)
@@ -94,7 +101,9 @@ module Lemans
         result = environment.exec(command, timeout:, env:)
 
         reward = read_reward(result)
-        Verification.new(reward:, credit: read_credit(reward), logs: result.output.to_s)
+        checks = read_checks
+        features = (self.class.features_from(checks.fetch("checks", {}), TestScanner.for(task)&.features) if checks && task.final_step?)
+        Verification.new(reward:, credit: credit_from(checks, reward), features:, logs: result.output.to_s)
       end
 
       def verifier_script
@@ -120,20 +129,20 @@ module Lemans
         value
       end
 
-      def read_credit(reward)
+      def read_checks
         path = File.join(task.verifier.logs_dir, "checks.json")
-        return reward unless environment.exec("test -e #{Shellwords.escape(path)}").success?
+        return nil unless environment.exec("test -e #{Shellwords.escape(path)}").success?
 
         result = environment.exec("cat #{Shellwords.escape(path)}")
         raise VerifierError, "could not read #{path}: #{result.output.to_s[0, 500]}" unless result.success?
 
-        checks = begin
-          JSON.parse(result.output.to_s)
-        rescue JSON::ParserError => e
-          raise VerifierError, "#{path} is not JSON: #{e.message[0, 500]}"
-        end
+        JSON.parse(result.output.to_s)
+      rescue JSON::ParserError => e
+        raise VerifierError, "#{path} is not JSON: #{e.message[0, 500]}"
+      end
 
-        grading = checks["grading"]
+      def credit_from(checks, reward)
+        grading = checks&.dig("grading")
         return reward unless grading && (base_credit = grading["base_credit"])
         return 0.0 if reward.zero?
 

data/lib/lemans/trial.rb

@@ -12,13 +12,15 @@ module Lemans
   class Trial
     attr_reader :task, :config, :model, :agent_name, :environment, :result
 
-    private attr_reader :agent, :store, :snapshot, :patch, :current_step_index
+    private attr_reader :agent, :store, :restart_from, :restart_mode, :snapshot, :patch, :current_step_index
 
-    def initialize(task, model = nil, result: nil, store: nil, agent: nil, environment: nil)
+    def initialize(task, model = nil, result: nil, store: nil, agent: nil, environment: nil, restart_from: nil, restart_mode: nil)
       @task = task
       @config = task.config
       @model = model || config.models.first
       @store = store
+      @restart_from = restart_from
+      @restart_mode = restart_mode
 
       @agent = agent.is_a?(Agent) ? agent : Agents.build(agent || config.agent_name, profile: config.agent, model: @model)
       @agent_name = @agent.name
@@ -47,9 +49,12 @@ module Lemans
       @snapshot = nil
       @patch = nil
       @current_step_index = nil
+      @recovered_since = nil
     end
 
     def run
+      restart! if restart_from
+
       phase(:environment_setup) do
         environment.start
 
@@ -64,45 +69,55 @@ module Lemans
         # Seal the git state to collect the agent's patch later
         @patch = Patch.new(task, environment)
         patch.seal!
+        patch.replay!(settled_patches, (restart_artifact("agent.patch") if restart_mode)) if restart_from
 
-        agent.install(task, environment)
+        # A reverification of the last step runs no agent
+        agent.install(task, environment) unless restart_mode == :reverify && restart_step == task.steps
 
         environment.switch_network_policy!(config.agent.environment.network)
       end
 
-      each_step do |step_task|
-        response =
-          phase(:agent) do
-            agent.run(step_task, environment)
-          rescue InfrastructureError, ::Miniswen::InfrastructureError => e
-            # Mark the failure here, where the agent phase is still known
-            result.failed!(:agent_error, e.message)
-            collect_patch!
-            raise
-          rescue ::Miniswen::AccountingError
-            # Classified by the outer rescue; the work is still on disk
+      adopt_phases! if restart_from
+
+      pending = restart_mode
+      each_step(from: restart_from ? restart_step : 1) do |step_task|
+        mode, pending = pending, nil
+
+        unless mode == :reverify
+          history = restart_artifact("agent.result.json") if mode == :recover
+          response =
+            phase(:agent, started_at: (@recovered_since if mode == :recover)) do
+              agent.run(step_task, environment, history:)
+            rescue InfrastructureError, ::Miniswen::InfrastructureError => e
+              # Mark the failure here, where the agent phase is still known
+              result.failed!(:agent_error, e.message)
+              collect_patch!
+              raise
+            rescue ::Miniswen::AccountingError
+              # Classified by the outer rescue; the work is still on disk
+              collect_patch!
+              raise
+            end
+
+          # Whatever the agent brought back is evidence, a failed run's included
+          save_trajectory!(response.trajectory)
+          store&.save_artifact(result, response.raw_result, path: with_step_index("agent.result.json")) if response.raw_result
+
+          if response.error?
+            result.failed!(:agent_error, response.error)
             collect_patch!
-            raise
+            return result
           end
 
-        # Whatever the agent brought back is evidence, a failed run's included
-        save_trajectory!(response.trajectory)
-        store&.save_artifact(result, response.raw_result, path: with_step_index("agent.result.json")) if response.raw_result
-
-        if response.error?
-          result.failed!(:agent_error, response.error)
-          collect_patch!
-          return result
-        end
+          if task.multistep?
+            result.step_completed!(response.outcome, response.usage, duration: result.phases.last.duration)
+          else
+            result.completed!(response.outcome, response.usage)
+          end
 
-        if task.multistep?
-          result.step_completed!(response.outcome, response.usage, duration: result.phases.last.duration)
-        else
-          result.completed!(response.outcome, response.usage)
+          check_cost_limit!
         end
 
-        check_cost_limit!
-
         collect_patch!
         if step_task.final_step?
           patch.compile!(result, store) if task.multistep? && store
@@ -124,7 +139,7 @@ module Lemans
             store&.save_artifact(result, verification.logs, path: with_step_index("verifier.log"))
 
             if step_task.final_step?
-              result.graded!(verification.reward, credit: verification.credit)
+              result.graded!(verification.reward, credit: verification.credit, features: verification.features)
             elsif verification.reward.zero?
               result.graded!(0.0)
               throw :halt
@@ -166,31 +181,84 @@ module Lemans
       store.save_artifact(result, JSON.pretty_generate(trajectory.to_atif), path:)
     end
 
-    def each_step
+    def each_step(from: 1)
       return yield task unless task.multistep?
 
       catch(:halt) do
-        1.upto(task.steps) do |index|
-          resume_agent! if index > 1
+        from.upto(task.steps) do |index|
+          # The first step of a restart finds a fresh tree, already replayed
+          resume_agent! if index > from
           @current_step_index = index
           yield task.for_step(index)
         end
       end
     end
 
+    # The step a restart begins at: the one a reverification grades again, or
+    # the first one not settled (run afresh, or recovered from its history).
+    def restart_step
+      @restart_step ||= restart_mode == :reverify ? restart_from.verified_step : restart_from.settled_steps + 1
+    end
+
+    # The new run carries the settled steps' evidence, and the agent's part of
+    # a step it grades again: the patch is collected anew.
+    def restart!
+      result.restart!(restart_from, step: restart_step, mode: restart_mode)
+      carried = restart_mode == :reverify ? %w[trajectory.json agent.result.json].map { restart_path(it) } : []
+
+      store.artifacts(restart_from).each do |path|
+        next unless path[%r{\.(\d+)(?:\.[^./]+)?\z}, 1].to_i.between?(1, restart_step - 1) || carried.include?(path)
+
+        store.save_artifact(result, store.read_artifact(restart_from, path), path:)
+      end
+    end
+
+    def settled_patches
+      (1...restart_step).map do |index|
+        store.read_artifact(restart_from, "agent.#{index}.patch") ||
+          raise(InfrastructureError, "#{restart_from.id} has no agent.#{index}.patch to replay")
+      end
+    end
+
+    def restart_artifact(name)
+      path = restart_path(name)
+      store.read_artifact(restart_from, path) || raise(InfrastructureError, "#{restart_from.id} has no #{path} to restart from")
+    end
+
+    def restart_path(name) = task.multistep? ? with_step_index(name, restart_step) : name
+
+    # A recovered step's agent phase reaches back over the part the source
+    # run already spent, so the step lasts as long as both parts together.
+    def adopt_phases!
+      names = restart_from.phases.map(&:name).select { it.to_s[/\.(\d+)\z/, 1].to_i.between?(1, restart_step - 1) }
+      agent_phase = restart_path("agent").to_sym
+      now = Time.now.utc
+
+      case restart_mode
+      when :reverify
+        result.adopt_phases!(restart_from, names + [ agent_phase ], now:)
+      when :recover
+        result.adopt_phases!(restart_from, names, through: agent_phase, now:)
+        partial = restart_from.phases.find { it.name == agent_phase }
+        @recovered_since = now - (partial.finished_at - partial.started_at)
+      else
+        result.adopt_phases!(restart_from, names, now:)
+      end
+    end
+
     def resume_agent!
       patch.restore!
       environment.exec!("rm -rf #{Verifier::TESTS_DIR} #{Shellwords.escape(task.verifier.logs_dir)}")
       environment.switch_network_policy!(config.agent.environment.network)
     end
 
-    def with_step_index(path)
-      return path unless current_step_index
+    def with_step_index(path, index = current_step_index)
+      return path unless index
 
       *pre, last = path.to_s.split(".")
-      return "#{last}.#{current_step_index}" if pre.empty?
+      return "#{last}.#{index}" if pre.empty?
 
-      [ *pre, current_step_index, last ].join(".")
+      [ *pre, index, last ].join(".")
     end
 
     def sandbox_ttl
@@ -209,8 +277,8 @@ module Lemans
       )
     end
 
-    def phase(name)
-      result.phase_started(with_step_index(name).to_sym)
+    def phase(name, started_at: nil)
+      result.phase_started(with_step_index(name).to_sym, started_at)
       yield
     ensure
       result.phase_finished(with_step_index(name).to_sym)

data/lib/lemans/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Lemans
-  VERSION = "1.4.0"
+  VERSION = "1.5.1"
 end

data/lib/lemans.rb

@@ -11,6 +11,8 @@ loader.ignore("#{__dir__}/miniswen.rb", "#{__dir__}/miniswen")
 loader.ignore("#{__dir__}/lemans/trial/verifier/assets")
 # templates/ holds the files `lemans init` scaffolds, not Ruby the harness loads.
 loader.ignore("#{__dir__}/lemans/cli/templates")
+# proxy/ holds the image the Docker backend runs for allowlists, not Ruby the harness loads.
+loader.ignore("#{__dir__}/lemans/environments/docker/proxy")
 loader.setup
 
 require "miniswen"

data/lib/miniswen/agent.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "json"
+require "time"
 require "miniswen/version"
 require "miniswen/ruby_llm"
 
@@ -261,25 +262,17 @@ module Miniswen
       @reporter = reporter
     end
 
-    def run(instruction)
-      uname = execute("uname -srvm").output.to_s.strip
-      @messages = [
-        { role: "system", content: SYSTEM_TEMPLATE },
-        { role: "user", content: format(INSTANCE_TEMPLATE,
-                                        instruction: instruction,
-                                        system_information: uname,
-                                        macos_sed_note: uname.start_with?("Darwin") ? "\n#{MACOS_SED_NOTE}" : "") }
-      ]
-
-      @steps = 0
-      @cost = 0.0
-
-      @totals = { input_tokens: 0, output_tokens: 0, cached_tokens: 0, thinking_tokens: 0 }
+    # With a `history` (the Result of an interrupted run), the session goes on
+    # where it stopped, under the budget the earlier part already spent.
+    def run(instruction = nil, history: nil)
+      if history
+        resume(history)
+      else
+        start(instruction)
+      end
 
-      @cost_known = true
       @consecutive_format_errors = 0
       @refused_turns = 0
-      @started_at = @clock.call
 
       loop do
         (status = limit_reached) and return finish(status)
@@ -329,6 +322,49 @@ module Miniswen
 
     private
 
+    def start(instruction)
+      uname = execute("uname -srvm").output.to_s.strip
+      @messages = [
+        { role: "system", content: SYSTEM_TEMPLATE },
+        { role: "user", content: format(INSTANCE_TEMPLATE,
+                                        instruction: instruction,
+                                        system_information: uname,
+                                        macos_sed_note: uname.start_with?("Darwin") ? "\n#{MACOS_SED_NOTE}" : "") }
+      ]
+
+      @steps = 0
+      @cost = 0.0
+      @cost_known = true
+      @totals = { input_tokens: 0, output_tokens: 0, cached_tokens: 0, thinking_tokens: 0 }
+      @started_at = @clock.call
+    end
+
+    def resume(history)
+      @messages = answered_messages(history.messages)
+
+      @steps = history.steps.to_i
+      @cost = history.cost_usd.to_f
+      @cost_known = !history.cost_usd.nil?
+      @totals = { input_tokens: history.input_tokens.to_i, output_tokens: history.output_tokens.to_i,
+                  cached_tokens: history.cached_tokens.to_i, thinking_tokens: history.thinking_tokens.to_i }
+      @started_at = @clock.call - elapsed(history.messages)
+    end
+
+    # A turn whose calls did not all get a result cannot go back to a provider:
+    # it is dropped, and the model decides again from the tree it left.
+    def answered_messages(messages)
+      turn = messages.rindex { it[:role] == "assistant" && it[:tool_calls] }
+      return messages.dup unless turn
+
+      answered = messages[(turn + 1)..].filter_map { it[:tool_call_id] if it[:role] == "tool" }
+      messages[turn][:tool_calls].all? { answered.include?(it[:id]) } ? messages.dup : messages[0...turn]
+    end
+
+    def elapsed(messages)
+      times = messages.filter_map { Time.parse(it[:timestamp]) if it[:role] == "assistant" && it[:timestamp] }
+      times.empty? ? 0.0 : times.max - times.min
+    end
+
     def execute(command)
       environment.exec(command, timeout: exec_timeout, env: EXEC_ENV)
     end

data/lib/miniswen/cli.rb

@@ -25,6 +25,7 @@ module Miniswen
       @jail = false
       @allowed_hosts = nil
       @workdir = nil
+      @history = nil
     end
 
     def run
@@ -56,7 +57,7 @@ module Miniswen
       agent = Agent.new(model:, reporter:, environment:, **options)
 
       begin
-        result = agent.run(instruction)
+        result = agent.run(instruction, history: @history && Agent::Result.from_h(JSON.parse(@history)))
       rescue StandardError => e
         write_results(agent.partial_result(error_message(e)))
         raise
@@ -105,7 +106,7 @@ module Miniswen
 
     def parse_args!
       parser = OptionParser.new do |opts|
-        opts.banner = "Usage: miniswen -m MODEL -p INSTRUCTION [...options]"
+        opts.banner = "Usage: miniswen -m MODEL (-p INSTRUCTION | --continue-from PATH) [...options]"
 
         opts.on("-m MODEL", "--model=MODEL", String,
                 "LLM to use (litellm format, e.g.: openrouter/openai/gpt-5.6-luna") do |v|
@@ -116,6 +117,10 @@ module Miniswen
           @instruction = File.file?(v) ? File.read(v) : v
         end
 
+        opts.on("--continue-from=PATH", String, "Continue the session saved by --results-path at PATH (no -p needed)") do |v|
+          @history = File.read(v)
+        end
+
         opts.on("--max-steps=STEPS", Integer, "Max steps count") do |v|
           options[:max_steps] = v
         end
@@ -197,7 +202,7 @@ module Miniswen
       return if @refresh_registry
 
       raise "Use -m to specify the model" unless @model
-      raise "Please, provide instructions via -p option" unless @instruction
+      raise "Please, provide instructions via -p option" unless @instruction || @history
     end
   end
 end

data/lib/miniswen/environment/docker.rb

@@ -20,7 +20,7 @@ module Miniswen
         env&.each { |key, value| argv += [ "--env", "#{key}=#{value}" ] }
         argv << id
         argv += [ "timeout", timeout.ceil.to_s ] if timeout&.positive?
-        argv += [ "sh", "-c", command ]
+        argv += [ "bash", "-c", command ]
 
         Open3.popen2e(*argv) do |stdin, pipe, wait|
           stdin.close

data/lib/miniswen/jail.rb

@@ -61,7 +61,7 @@ module Miniswen
     def spawn_arguments(command, env)
       [ command_env(env),
         "nsenter", "--target", @holder.pid.to_s, "--net", "--mount", "--pid=/proc/#{@holder.pid}/ns/pid_for_children", "--wd=#{@workdir}",
-        "--", "setpriv", "--bounding-set=-all", "--inh-caps=-all", "--no-new-privs", "--", "sh", "-c", command ]
+        "--", "setpriv", "--bounding-set=-all", "--inh-caps=-all", "--no-new-privs", "--", "bash", "-c", command ]
     end
 
     def spawn_options = super.merge(unsetenv_others: true)

data/lib/miniswen/local.rb

@@ -28,7 +28,7 @@ module Miniswen
 
     private
 
-    def spawn_arguments(command, env) = [ env || {}, "sh", "-c", command ]
+    def spawn_arguments(command, env) = [ env || {}, "bash", "-c", command ]
 
     def spawn_options = { pgroup: true }
 

data/lib/miniswen/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miniswen
-  VERSION = "1.4.0"
+  VERSION = "1.5.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lemans
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.5.1
 platform: ruby
 authors:
 - Svyatoslav Kryukov
@@ -95,6 +95,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: rack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+- !ruby/object:Gem::Dependency
+  name: rackup
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
 - !ruby/object:Gem::Dependency
   name: rexml
   requirement: !ruby/object:Gem::Requirement
@@ -137,6 +165,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: webrick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
 - !ruby/object:Gem::Dependency
   name: zeitwerk
   requirement: !ruby/object:Gem::Requirement
@@ -159,6 +201,7 @@ email:
 executables:
 - lemans
 - lemans-remote
+- lemans-viewer
 extensions: []
 extra_rdoc_files: []
 files:
@@ -167,6 +210,7 @@ files:
 - README.md
 - exe/lemans
 - exe/lemans-remote
+- exe/lemans-viewer
 - lib/lemans.rb
 - lib/lemans/agent.rb
 - lib/lemans/agents.rb
@@ -213,6 +257,9 @@ files:
 - lib/lemans/environments/daytona/shell.rb
 - lib/lemans/environments/daytona/snapshot_store.rb
 - lib/lemans/environments/docker.rb
+- lib/lemans/environments/docker/proxy/Dockerfile
+- lib/lemans/environments/docker/proxy/README.md
+- lib/lemans/environments/docker/proxy/server.rb
 - lib/lemans/ext/deep_merge.rb
 - lib/lemans/result.rb
 - lib/lemans/runner.rb
@@ -229,6 +276,7 @@ files:
 - lib/lemans/trial/verifier.rb
 - lib/lemans/trial/verifier/assets/eport-lemans.rb
 - lib/lemans/trial/verifier/assets/lemans_minitest_reporter.rb
+- lib/lemans/trial/verifier/test_scanner.rb
 - lib/lemans/version.rb
 - lib/miniswen.rb
 - lib/miniswen/agent.rb