lemans 1.0.0 → 1.2.0

data/lib/lemans/config/verifier.rb

@@ -24,6 +24,7 @@ module Lemans
           conf.preverify = data["preverify"] if data["preverify"]
           conf.restore_paths = restore_paths!(data["restore"]) if data["restore"]
           conf.logs_dir = absolute_path!(data["logs_dir"]) if data["logs_dir"]
+          conf.verification = root.join(data["verification"]) if data["verification"]
 
           conf
         end
@@ -41,31 +42,41 @@ module Lemans
         end
       end
 
-      attr_accessor :root, :timeout, :setup, :command, :preverify, :restore_paths, :logs_dir
+      attr_accessor :timeout, :setup, :command, :preverify, :restore_paths, :logs_dir, :verification
 
       def initialize
-        @root = Pathname("./")
         @timeout = 10 * 60
         @setup = Setup.new
         @command = DEFAULT_COMMAND
         @preverify = nil
         @restore_paths = []
         @logs_dir = "/logs/verifier"
+        @verification = nil
       end
 
       def reward_path = "#{logs_dir.chomp("/")}/reward.txt"
 
+      def to_h
+        {
+          "timeout" => timeout,
+          "setup" => setup.to_h,
+          "command" => command,
+          "preverify" => preverify,
+          "restore" => restore_paths,
+          "logs_dir" => logs_dir,
+          "verification" => verification&.to_s
+        }.compact
+      end
+
       # [absolute, remote-relative] pairs. Shared verification files grade
       # every trial, so they ship alongside each task's own tests.
       def files
-        @files ||= begin
-          dir = root.join(VERIFICATION_DIR)
-          if dir.directory?
-            dir.glob("**/*", File::FNM_DOTMATCH).select(&:file?).map { [ it, it.relative_path_from(dir).to_s ] }
+        @files ||=
+          if verification&.directory?
+            verification.glob("**/*", File::FNM_DOTMATCH).select(&:file?).map { [ it, it.relative_path_from(verification).to_s ] }
           else
             []
           end
-        end
       end
     end
   end

data/lib/lemans/config.rb

@@ -7,18 +7,24 @@ require "pathname"
 module Lemans
   # Benchmark configuration and task definitions container.
   class Config
-    attr_reader :root, :config_path, :tasks_dir,
-                :tasks, :version,
+    attr_reader :root, :config_path, :parent, :tasks_dir, :version,
                 :backend, :concurrency, :attempts
 
     # Nested configs
     attr_reader :environment, :agent, :verifier, :setup
 
+    using Ext::DeepMerge
+
+    FILENAMES = %w[bench.yaml bench.yml].freeze
+
     class << self
-      def load_file(path)
+      # A task directory's bench.yml is loaded with the bench as its `parent`:
+      # inherit_from is implied.
+      def load_file(path, parent: nil)
+        path = File.dirname(path) if File.file?(path)
         raise ConfigError, "bench directory not found: #{path}" unless File.directory?(path)
 
-        config_name = %w[bench.yaml bench.yml].find { File.file?(File.join(path, it)) }
+        config_name = FILENAMES.find { File.file?(File.join(path, it)) }
 
         raise ConfigError, "bench.yml not found" unless config_name
 
@@ -29,6 +35,17 @@ module Lemans
 
         root = Pathname(path)
 
+        parent = load_file(root.join(contents["inherit_from"])) if contents["inherit_from"]
+
+        if parent
+          # Local conventions win over the inherited config (unless declared, even as ~)
+          environment = (contents["environment"] ||= {})
+          environment["dockerfile"] = root.join("environment/Dockerfile").to_s if root.join("environment/Dockerfile").file? && !environment.key?("dockerfile")
+          verifier = (contents["verifier"] ||= {})
+          verifier["verification"] = root.join(Verifier::VERIFICATION_DIR).to_s if root.join(Verifier::VERIFICATION_DIR).directory? && !verifier.key?("verification")
+          contents = parent.to_h.deep_merge(contents)
+        end
+
         sections = {}
         sections[:version] = contents["version"]
         sections[:tasks_dir] = contents["tasks"]
@@ -44,17 +61,18 @@ module Lemans
           sections[:setup].commands = Array(commands) + sections[:setup].commands
         end
 
-        new(root, config_path:, **sections)
+        new(root, config_path:, parent:, **sections)
       rescue Psych::Exception => e
         raise ConfigError, "#{path}: #{e.message}"
       end
     end
 
-    def initialize(root = Pathname("./"), config_path: Pathname("./bench.yml"), version: nil, tasks_dir: "tasks",
+    def initialize(root = Pathname("./"), config_path: Pathname("./bench.yml"), parent: nil, version: nil, tasks_dir: "tasks",
                    setup: nil, agent: Agent.new("miniswen", "openrouter/z-ai/glm-5.2"),
                    environment: Environment.new, verifier: Verifier.new)
       @root = root
       @config_path = config_path
+      @parent = parent
       @version = version
       @tasks_dir = root.join(tasks_dir)
       @setup = setup || Setup.new
@@ -62,26 +80,41 @@ module Lemans
       @environment = environment
       @environment.dockerfile = root.join(@environment.dockerfile) if @environment.dockerfile
       @environment.dockerfile ||= default_dockerfile unless @environment.image
+      @environment.profiles.each_value { it.dockerfile = root.join(it.dockerfile) if it.dockerfile }
       @verifier = verifier
-      @verifier.root = root
+      @verifier.verification ||= root.join(Verifier::VERIFICATION_DIR)
       @concurrency = 4
       @attempts = 1
       @backend = @environment.backend
-      @tasks = parse_tasks
     end
 
+    def tasks = @tasks ||= parse_tasks
+
     def load_options(agent: nil, model: nil, attempts: nil, concurrency: nil, backend: nil, **)
       @agent.name = agent if agent
       @agent.models = Array(model) if model
       @attempts = attempts if attempts
       @concurrency = concurrency if concurrency
       @backend = environment.backend = backend if backend
+      tasks.each { it.config.load_options(agent:, model:, attempts:, concurrency:, backend:) unless it.config.equal?(self) }
     end
 
     def agent_name = agent.name
 
     def models = agent.models
 
+    # The bench.yml this config reads as, paths resolved: what a bench
+    # inheriting from it merges over. Tasks are a bench's own.
+    def to_h
+      {
+        "version" => version,
+        "setup" => setup.to_h,
+        "agent" => agent.to_h,
+        "environment" => environment.to_h,
+        "verifier" => verifier.to_h
+      }.compact
+    end
+
     # Resolved once: an hours-long run reports the bench it started from, not
     # later tree drift.
     def revision
@@ -92,9 +125,11 @@ module Lemans
     def digest
       @digest ||= begin
         sha = Digest::SHA256.new
-        sha << TreeDigest.call(root.join(Verifier::VERIFICATION_DIR))
+        sha << TreeDigest.call(verifier.verification)
         sha << TreeDigest.call(environment.dockerfile.dirname) if environment.dockerfile
+        environment.profiles.each_value { sha << TreeDigest.call(it.dockerfile.dirname) if it.dockerfile }
         sha << Digest::SHA256.file(config_path.to_s).hexdigest if config_path.file?
+        sha << parent.digest if parent
         sha.hexdigest[0, 16]
       end
     end

data/lib/lemans/environments/docker.rb

@@ -152,7 +152,7 @@ module Lemans
           end
 
           status = wait.value
-          [ timed_out ? 124 : (status.exitstatus || 1), output.scrub ]
+          [ timed_out ? 124 : (status.exitstatus || 1), output.force_encoding(Encoding::UTF_8).scrub ]
         end
       rescue Errno::ENOENT
         raise ConfigError, "docker: CLI not found — install Docker or pick another backend"

data/lib/lemans/ext/deep_merge.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Lemans
+  module Ext
+    module DeepMerge
+      refine Hash do
+        def deep_merge(other)
+          merge(other) { |_, base, over| base.is_a?(Hash) && over.is_a?(Hash) ? base.deep_merge(over) : over }
+        end
+      end
+    end
+  end
+end

data/lib/lemans/result.rb

@@ -59,6 +59,18 @@ module Lemans
       :cost_usd, :cost_source
     ) do
       def as_json(**) = to_h.merge(cost_source: cost_source&.to_h).compact
+
+      # A multistep trial's totals; an unknown step cost makes the sum unknown.
+      def +(other)
+        self.class.new(
+          input_tokens: input_tokens + other.input_tokens,
+          output_tokens: output_tokens + other.output_tokens,
+          cached_tokens: cached_tokens + other.cached_tokens,
+          steps: steps + other.steps,
+          cost_usd: cost_usd && other.cost_usd && cost_usd + other.cost_usd,
+          cost_source: other.cost_source || cost_source
+        )
+      end
     end
 
     def Usage.zero
@@ -95,6 +107,8 @@ module Lemans
         @finished_at = time || Time.now.utc
       end
 
+      def duration = finished_at && (finished_at - started_at).round(1)
+
       def as_json(**)
         {
           name:,
@@ -117,13 +131,23 @@ module Lemans
       def as_json(**) = to_h
     end
 
+    Step = Data.define(:outcome, :usage, :duration) do
+      def as_json(**) = { outcome: outcome.as_json, usage: usage&.as_json, duration: }.compact
+
+      def self.from_json(data)
+        new(outcome: Outcome.from_json(data[:outcome]),
+            usage: data[:usage] && Usage.from_json(data[:usage]),
+            duration: data[:duration])
+      end
+    end
+
     # attributes that must be initialized/specified during construction
     attr_reader :id, :task, :agent, :model, :index,
                 :profile_digest, :task_digest, :revision
 
     attr_accessor :tags, :metadata
 
-    attr_reader :phases
+    attr_reader :phases, :steps
 
     # outcome-related attributes (we use setter-like methods, not accessors)
     attr_reader :reward, :outcome, :usage
@@ -141,6 +165,7 @@ module Lemans
       @tags = []
       @metadata = {}
       @phases = []
+      @steps = nil
 
       @id = id || "#{task}__#{SecureRandom.alphanumeric(7)}"
       @outcome = Outcome.new(:pending)
@@ -184,6 +209,14 @@ module Lemans
       self
     end
 
+    def step_completed!(outcome, usage = nil, duration: nil)
+      outcome = outcome.is_a?(Outcome) ? outcome : Outcome.new(outcome)
+      @steps ||= []
+      steps << Step.new(outcome:, usage:, duration:)
+      # aggregate right away (so we don't lose data on failure)
+      completed!(outcome, aggregate_usage)
+    end
+
     def graded!(reward)
       @reward = reward
       self
@@ -199,12 +232,15 @@ module Lemans
       self
     end
 
+    private def aggregate_usage = steps.filter_map(&:usage).reduce(:+)
+
     def as_json(**)
       {
         trial: id, task:, agent:, model:, index:,
         profile_digest:, task_digest:, revision: revision&.as_json,
         lemans_version: VERSION,
         tags:, metadata:, phases: phases.map(&:as_json),
+        steps: steps&.map(&:as_json),
         reward:, outcome: outcome.as_json, usage: usage&.as_json, duration:,
         started_at: started_at&.iso8601,
         finished_at: finished_at&.iso8601
@@ -222,6 +258,10 @@ module Lemans
         result.tags = data[:tags] || []
         result.metadata = data[:metadata] || {}
         phases_from(data).each { result.phases << it }
+        # Steps first: the stored outcome/usage below override the aggregates.
+        data[:steps]&.map { Step.from_json(it) }&.each do |step|
+          result.step_completed!(step.outcome, step.usage, duration: step.duration)
+        end
         if data[:outcome]
           result.completed!(
             Outcome.from_json(data[:outcome]),

data/lib/lemans/runner.rb

@@ -69,7 +69,7 @@ module Lemans
         run.task == task.name &&
           run.model == (model || config.agent.model) &&
           run.agent == config.agent_name &&
-          run.profile_digest == config.digest &&
+          run.profile_digest == task.config.digest &&
           run.task_digest == task.digest &&
           run.scored?
       end

data/lib/lemans/stores/fs.rb

@@ -128,7 +128,7 @@ module Lemans
 
       # result.json is stored at <root>/<model-short>/<result-id>
       def result_dir(result)
-        root.join((result.model || result.agent).to_s.split("/").last, result.id)
+        root.join((result.model || result.agent).to_s.split("/").last.tr("#", "-"), result.id)
       end
     end
   end

data/lib/lemans/task_definition.rb

@@ -19,17 +19,32 @@ module Lemans
     FLAT_SEED = "environment.patch"
 
     FRONTMATTER = /\A---\n(.*?)\n---\n/m
+    STEP_SEPARATOR = /^---[ \t]*\n/
+
+    STEP_TESTS = { "tests.%d" => :dir, "verification_test.%d.rb" => FLAT_TEST, "verify.%d" => "verify" }.freeze
+    # The same step files, kept inside the shared tests/ directory
+    SHARED_STEP_TESTS = { "verification_test.%d.rb" => FLAT_TEST, "verify.%d" => "verify" }.freeze
+    # What the final step alone runs (see Config::Verifier::DEFAULT_COMMAND): never shipped to an earlier step
+    FINAL_STEP_TESTS = [ FLAT_TEST, "verify", "test.sh" ].freeze
+    STEP_SOLUTIONS = { "solution.%d" => :dir, "solution.%d.patch" => FLAT_SOLUTION,
+                       "solve.%d" => "solve", "solve.%d.sh" => "solve.sh" }.freeze
+
+    STEP_FILE = /\A(?:tests\.(?<test>\d+)|verification_test\.(?<test>\d+)\.rb|verify\.(?<test>\d+)|
+                     solution\.(?<solution>\d+)(?:\.patch)?|solve\.(?<solution>\d+)(?:\.sh)?)\z/x
 
     class << self
       def load_from_directory(config, dir)
         dir = Pathname(dir)
         data = frontmatter(dir)
+        config = Config.load_file(dir, parent: config) if Config::FILENAMES.any? { dir.join(it).file? }
 
         task = new(config, data["name"] || dir.basename.to_s, dir:)
         task.description = data["description"].to_s if data["description"]
         task.difficulty = data["difficulty"].to_sym if data["difficulty"]
         task.tags = Array(data["tags"]).map(&:to_s) if data["tags"]
         task.metadata = data["metadata"] if data["metadata"]
+        task.environment_profile = data["environment"] if data["environment"]
+        task.multistep = data["multistep"] if data.key?("multistep")
 
         declared_setup = Config::Setup.from_config(data["setup"], root: dir)
         refuse_config_collisions!(task, declared_setup, config.setup)
@@ -79,8 +94,21 @@ module Lemans
           raise ConfigError, "#{task.dir}: a task may only override verifier.setup, not verifier.#{extras.first}"
         end
 
-        raise ConfigError, "#{task.dir}: #{ENVIRONMENT_DIR}/Dockerfile is required when the bench declares no shared image or dockerfile" unless
-          task.config.environment.image || task.config.environment.dockerfile || task.environment_dockerfile.file?
+        if (profile = task.environment_profile)
+          unless task.config.environment.profiles.key?(profile)
+            declared = task.config.environment.profiles.keys
+            listing = declared.any? ? "declares #{declared.join(", ")}" : "declares none"
+            raise ConfigError, "#{task.dir}: unknown environment #{profile} — bench.yml #{listing}"
+          end
+
+          raise ConfigError, "#{task.dir}: environment #{profile} and a local #{ENVIRONMENT_DIR}/Dockerfile are mutually exclusive" if
+            task.environment_dockerfile.file?
+        end
+
+        raise ConfigError, "#{task.dir}: #{ENVIRONMENT_DIR}/Dockerfile is required when the bench declares no shared image or dockerfile and the task names no environment" unless
+          task.environment_profile || task.config.environment.image || task.config.environment.dockerfile || task.environment_dockerfile.file?
+
+        validate_steps!(task)
 
         return unless task.test_files.empty?
 
@@ -88,6 +116,41 @@ module Lemans
                            "the verifier uploads it at verification time"
       end
 
+      def validate_steps!(task)
+        if task.multistep? && task.steps < 2
+          raise ConfigError, "#{task.dir}: multistep: true but #{INSTRUCTION} holds a single section — " \
+                             "separate step instructions with --- (the first section is the shared preamble)"
+        end
+
+        indexed_solutions = false
+        entries = task.dir.children
+        # Only step tests may carry an index inside tests/; a solution there is just a file.
+        entries += task.tests_dir.children.select { |entry| entry.file? && STEP_FILE.match(entry.basename.to_s)&.[](:test) } if task.tests_dir.directory?
+        entries.each do |entry|
+          match = STEP_FILE.match(entry.basename.to_s) or next
+          name = entry.relative_path_from(task.dir)
+
+          raise ConfigError, "#{task.dir}: #{name} is an indexed step file, but the task is not multistep: true" unless
+            task.multistep?
+
+          step = (match[:test] || match[:solution]).to_i
+          raise ConfigError, "#{task.dir}: #{name} names step #{step}, but the task has #{task.steps} steps" unless
+            step.between?(1, task.steps)
+
+          if match[:solution]
+            indexed_solutions = true
+          elsif step == task.steps
+            raise ConfigError, "#{task.dir}: #{name} indexes the final step — the final verification keeps " \
+                               "the unindexed name"
+          end
+        end
+
+        return unless indexed_solutions && task.solution_files.any?
+
+        raise ConfigError, "#{task.dir}: #{FLAT_SOLUTION} is the whole task's solution while indexed step solutions " \
+                           "chain step by step — ship one or the other, not both"
+      end
+
       # Collisions would be resolved by upload order, so a task never gets to
       # shadow the bench-wide copy.
       def refuse_config_collisions!(task, declared, base)
@@ -103,7 +166,9 @@ module Lemans
 
     attr_reader :config, :name, :dir
 
-    attr_accessor :difficulty, :tags, :description, :metadata
+    attr_accessor :difficulty, :tags, :description, :metadata, :environment_profile, :multistep
+
+    alias multistep? multistep
 
     def initialize(config, name, dir: nil)
       @config = config
@@ -113,13 +178,26 @@ module Lemans
       @tags = []
       @description = ""
       @metadata = {}
+      @environment_profile = nil
+      @multistep = false
 
+      @step = nil
       @dir = dir || config.tasks_dir.join(name)
     end
 
-    # The story alone: frontmatter is for the harness, never for the agent.
+    # A copy of the task definition for a particular step
+    # (so we can generated correct paths and instructions)
+    def for_step(index) = dup.tap { it.step = index }
+
+    def final_step? = !multistep? || step == steps
+
+    def steps = multistep? ? sections.size - 1 : 1
+
     def instruction
-      @instruction ||= dir.join(INSTRUCTION).read.sub(FRONTMATTER, "")
+      return body unless multistep?
+      raise ArgumentError, "#{name} is multistep — only a step projection (for_step) has an instruction" unless step
+
+      "#{sections[0].strip}\n\n#{sections.fetch(step).strip}\n"
     end
 
     def digest
@@ -147,14 +225,39 @@ module Lemans
 
     # [absolute, remote-relative] pairs. Tests stay on the harness side while
     # the agent works; uploaded into the sandbox only at verification.
+    #
+    # A multistep task keeps one tests/ directory: whatever in it carries no
+    # step index (helpers, fixtures) ships with every verified step, the
+    # step's own verification_test.N.rb ships as verification_test.rb, and
+    # the unindexed verification_test.rb stays with the final step.
     def test_files
-      tests_dir.directory? ? expand(tests_dir) : flat(FLAT_TEST)
+      return final_test_files unless step && !final_step?
+
+      indexed = step_files(STEP_TESTS) + step_files(SHARED_STEP_TESTS, root: tests_dir)
+      return [] if indexed.empty?
+
+      shared_test_files.reject { |_, remote| FINAL_STEP_TESTS.include?(remote) } + indexed
     end
 
+    def verifiable? = test_files.any?
+
+    # A lone whole-task solution is applied before the first step: intermediate
+    # gates must see it, and later steps have nothing left to add.
     def solution_files
+      if step
+        return step_files(STEP_SOLUTIONS) if indexed_solutions?
+        return [] if multistep? && step > 1
+      end
+
       solution_dir.directory? ? expand(solution_dir) : flat(FLAT_SOLUTION)
     end
 
+    # True for the later steps of a multistep task whose lone solution already
+    # shipped with step 1
+    def solution_applied_earlier?
+      !!(step && step > 1 && !indexed_solutions? && (solution_dir.directory? || dir.join(FLAT_SOLUTION).file?))
+    end
+
     def solution? = solution_files.any?
 
     def tests_dir = dir.join(TESTS_DIR)
@@ -164,17 +267,62 @@ module Lemans
     def environment_dockerfile = dir.join(ENVIRONMENT_DIR, "Dockerfile")
 
     def environment_image
-      if environment_dockerfile.file?
+      if (profile = environment.profiles[environment_profile])
+        if profile.image
+          Config::ImageSpec.registry(profile.image)
+        else
+          Config::ImageSpec.dockerfile(profile.dockerfile, slug: environment_profile)
+        end
+      elsif environment_dockerfile.file?
         Config::ImageSpec.dockerfile(environment_dockerfile, slug: name)
-      elsif environment.image
-        Config::ImageSpec.registry(environment.image)
-      else
+      elsif environment.dockerfile
         Config::ImageSpec.dockerfile(environment.dockerfile, slug: "shared")
+      else
+        Config::ImageSpec.registry(environment.image)
       end
     end
 
+    protected attr_writer :step
+
     private
 
+    attr_reader :step
+
+    def body
+      @body ||= dir.join(INSTRUCTION).read.sub(FRONTMATTER, "")
+    end
+
+    def sections
+      @sections ||= body.split(STEP_SEPARATOR)
+    end
+
+    def indexed_solutions? = (1..steps).any? { step_files(STEP_SOLUTIONS, it).any? }
+
+    # A task may keep its final test flat at the root and use tests/ for shared files
+    def final_test_files
+      return flat(FLAT_TEST) unless tests_dir.directory?
+
+      shared_test_files + flat(FLAT_TEST)
+    end
+
+    # Everything under tests/ that no step claims for itself
+    def shared_test_files
+      return [] unless tests_dir.directory?
+
+      expand(tests_dir).reject { |_, remote| STEP_FILE.match?(remote) }
+    end
+
+    def step_files(patterns, index = step, root: dir)
+      patterns.flat_map do |pattern, remote|
+        path = root.join(format(pattern, index))
+        if remote == :dir
+          path.directory? ? expand(path) : []
+        else
+          path.file? ? [ [ path, remote ] ] : []
+        end
+      end
+    end
+
     def own_setup_with_seed
       declared = @declared_setup || Config::Setup.new
       return declared unless seed? && declared.files.none? { |_, remote| remote == FLAT_SEED }

data/lib/lemans/trial/patch.rb

@@ -12,7 +12,7 @@ module Lemans
       REMOTE_PATCH = "/tmp/lemans-agent.patch"
       REMOTE_INDEX = "/tmp/lemans-patch.idx"
 
-      private attr_reader :task, :environment, :path, :workdir, :baseline, :timeout
+      private attr_reader :task, :environment, :path, :workdir, :baseline, :savepoint, :timeout
 
       def initialize(task, environment, timeout: 300, path: "agent.patch")
         @task = task
@@ -22,37 +22,75 @@ module Lemans
 
         @workdir = task.environment.workdir
         @baseline = nil
+        @savepoint = nil
       end
 
       def seal!
         @baseline = write_tree
+        @savepoint = @baseline
       end
 
       # Must run before the verifier restores the graded surfaces: a patch taken
-      # after would not show what the agent did to them
-      def collect!(result, store)
+      # after would not show what the agent did to them. Diffs from the last
+      # savepoint (the sealed baseline until a multistep trial moves it), so a
+      # step's patch shows that step's work alone.
+      def collect!(result, store, path: @path)
+        return unless savepoint
+
+        after = write_tree
+        return unless after
+
+        save_diff(result, store, savepoint, after, path)
+      end
+
+      # The compilation of every step: the whole run against the sealed baseline.
+      def compile!(result, store)
         return unless baseline
 
         after = write_tree
         return unless after
 
-        diffed = environment.exec("#{git} diff --binary #{baseline} #{after} > #{REMOTE_PATCH}", timeout:)
+        save_diff(result, store, baseline, after, @path)
+      end
+
+      # Marks the tree a finished step left: the next collect! diffs from here,
+      # and restore! comes back here. The mark is load-bearing, so failing to
+      # write it is an environment error, not a lost artifact.
+      def savepoint!
+        @savepoint = write_tree
+        raise InfrastructureError, "could not savepoint the tree the step left behind" unless savepoint
+      end
+
+      # Puts the workdir back to the savepoint exactly: the verifier restored
+      # the graded surfaces from the baseline and may have littered the tree,
+      # and the next step's agent must find neither.
+      def restore!
+        return unless savepoint
+
+        environment.exec!(
+          "#{git} read-tree #{savepoint} && #{git} checkout-index -f -a && #{git} clean -fd && #{git} reset -q",
+          timeout:
+        )
+      end
+
+      private
+
+      def save_diff(result, store, from, to, destination)
+        diffed = environment.exec("#{git} diff --binary #{from} #{to} > #{REMOTE_PATCH}", timeout:)
         return unless diffed.success?
 
         Tempfile.create(%w[agent .patch]) do |file|
           environment.download(REMOTE_PATCH, file.path)
-          store.save_artifact(result, Pathname(file.path), path:)
+          store.save_artifact(result, Pathname(file.path), path: destination)
         end
 
         environment.exec("rm -f #{REMOTE_PATCH} #{REMOTE_INDEX}", timeout:)
-        path
+        destination
       rescue InfrastructureError => e
         warn "lemans: could not collect the agent patch for #{result.id}: #{e.message}"
         nil
       end
 
-      private
-
       # `safe.directory` because the sandbox may run the tree as a different user
       # than built it, and git refuses to read a repo it thinks is someone else's.
       def git = "git -c safe.directory='*' -C #{Shellwords.escape(workdir)}"