lemans 1.3.5 → 1.5.0

data/lib/lemans/environments/docker.rb

@@ -12,6 +12,7 @@ module Lemans
       HOUSEKEEPING_TIMEOUT = 60
       MAX_OUTPUT_BYTES = 200_000
       EXEC_SLACK = 30
+      PROXY_PORT = 3128
 
       attr_reader :container
 
@@ -24,7 +25,8 @@ module Lemans
       end
 
       def start
-        build_image! if image.built?
+        build_image!(image) if image.built?
+        start_proxy!(network.hosts) if network.allowlist?
         docker!("run", *run_args, timeout: build_timeout)
         @container = @name
         self
@@ -37,9 +39,10 @@ module Lemans
         timeout ||= DEFAULT_TIMEOUT
         started = now
         argv = [ "exec" ]
+        env = proxy_env.merge(env) if network.allowlist?
         env.each { |key, value| argv += [ "--env", "#{key}=#{value}" ] }
         # The in-container timeout is what actually kills the process
-        argv += [ container, "timeout", timeout.to_i.to_s, "sh", "-c", command ]
+        argv += [ container, "timeout", timeout.to_i.to_s, "bash", "-c", command ]
 
         exit_code, output = capture("docker", *argv, timeout: timeout + EXEC_SLACK)
         ExecResult.new(command:, exit_code:, output:, duration: (now - started).round(3))
@@ -59,13 +62,13 @@ module Lemans
       def switch_network_policy!(policy)
         assert_policy_supported!(policy)
 
-        if policy.none?
-          connected_networks.each { docker!("network", "disconnect", it, container) }
-        else
-          networks = connected_networks
-          docker!("network", "disconnect", "none", container) if networks.include?("none")
-          docker!("network", "connect", "bridge", container) unless networks.include?("bridge")
-        end
+        wanted = ("bridge" if policy.public?) || (internal_network if policy.allowlist?)
+        networks = connected_networks
+        (networks - [ wanted ]).each { docker!("network", "disconnect", it, container) }
+
+        remove_proxy
+        start_proxy!(policy.hosts) if policy.allowlist?
+        docker!("network", "connect", wanted, container) if wanted && !networks.include?(wanted)
 
         @network = policy
       end
@@ -81,41 +84,101 @@ module Lemans
         end
       rescue StandardError => e
         warn "lemans: container #{container} may still be running — remove failed: #{e.class}: #{e.message}"
+      ensure
+        remove_proxy
+        remove_internal_network
       end
 
       private
 
       def assert_policy_supported!(policy)
-        return if policy.none? || policy.public?
+        return unless policy.allowlist? && policy.ip_targets.any?
 
-        raise ConfigError, "docker: #{policy.mode} is not supported (public and none only)"
+        raise ConfigError, "docker: an allowlist takes host names only (#{policy.ip_targets.join(", ")})"
       end
 
       # The tag is the content digest, so an existing image is the identical thing
-      def build_image!
-        exists, = capture("docker", "image", "inspect", image.name, timeout: HOUSEKEEPING_TIMEOUT)
+      def build_image!(spec)
+        exists, = capture("docker", "image", "inspect", spec.name, timeout: HOUSEKEEPING_TIMEOUT)
         return if exists.zero?
 
-        docker!("build", "--tag", image.name, image.context_dir.to_s, timeout: build_timeout, on_output: @logger)
+        docker!("build", "--tag", spec.name, spec.context_dir.to_s, timeout: build_timeout, on_output: @logger)
+      end
+
+      # The task container sits on an internal network with no route out; the
+      # proxy is on it and on bridge, so the hosts it allows are the only way out.
+      def start_proxy!(hosts)
+        image = Config::ImageSpec.dockerfile(Pathname(__dir__).join("docker/proxy/Dockerfile"), slug: "proxy")
+        build_image!(image)
+        create_internal_network!
+        docker!("run", "--detach", "--name", proxy_name, "--network", internal_network,
+                *label_args, image.name, *hosts)
+        docker!("network", "connect", "bridge", proxy_name)
+        wait_for_proxy!
+      end
+
+      def wait_for_proxy!(attempts: 50)
+        attempts.times do
+          _, output = capture("docker", "logs", proxy_name, timeout: HOUSEKEEPING_TIMEOUT)
+          return if output.include?("listening on")
+
+          sleep 0.2
+        end
+
+        raise InfrastructureError, "docker: the allowlist proxy did not start"
       end
 
+      def proxy_env
+        url = "http://#{proxy_name}:#{PROXY_PORT}"
+        no_proxy = "localhost,127.0.0.1"
+        { "http_proxy" => url, "https_proxy" => url, "HTTP_PROXY" => url, "HTTPS_PROXY" => url,
+          "no_proxy" => no_proxy, "NO_PROXY" => no_proxy }
+      end
+
+      def create_internal_network!
+        exists, = capture("docker", "network", "inspect", internal_network, timeout: HOUSEKEEPING_TIMEOUT)
+        return if exists.zero?
+
+        docker!("network", "create", "--internal", *label_args, internal_network)
+      end
+
+      def remove_proxy
+        capture("docker", "rm", "--force", proxy_name, timeout: HOUSEKEEPING_TIMEOUT)
+      rescue StandardError
+        nil
+      end
+
+      def remove_internal_network
+        capture("docker", "network", "rm", internal_network, timeout: HOUSEKEEPING_TIMEOUT)
+      rescue StandardError
+        nil
+      end
+
+      def proxy_name = "#{@name}-proxy"
+
+      def internal_network = "#{@name}-net"
+
       def run_args
         args = [ "--detach", "--init", "--name", @name,
                  "--cpus", resources.cpus.to_s, "--memory", "#{resources.memory}m",
                  "--cap-add", "SYS_ADMIN", "--cap-add", "NET_ADMIN", "--security-opt", "apparmor=unconfined",
                  "--entrypoint", "sh" ]
         args += [ "--network", "none" ] if network.none?
+        args += [ "--network", internal_network ] if network.allowlist?
         env.each { |key, value| args += [ "--env", "#{key}=#{value}" ] }
-        labels.each { |key, value| args += [ "--label", "#{key}=#{value}" ] }
-        args + [ image.name, "-c", "tail -f /dev/null" ]
+        args + label_args + [ image.name, "-c", "tail -f /dev/null" ]
       end
 
+      def label_args = labels.flat_map { |key, value| [ "--label", "#{key}=#{value}" ] }
+
       def connected_networks
         docker!("inspect", "--format", "{{range $name, $_ := .NetworkSettings.Networks}}{{$name}} {{end}}", container).split
       end
 
       def remove
         capture("docker", "rm", "--force", "--volumes", @name, timeout: HOUSEKEEPING_TIMEOUT)
+        remove_proxy
+        remove_internal_network
       rescue StandardError
         nil
       end

data/lib/lemans/result.rb

@@ -136,6 +136,12 @@ module Lemans
       def as_json(**) = to_h
     end
 
+    Restart = Data.define(:trial, :step, :mode) do
+      def initialize(trial:, step:, mode: nil) = super
+
+      def as_json(**) = to_h.compact
+    end
+
     Step = Data.define(:outcome, :usage, :duration) do
       def as_json(**) = { outcome: outcome.as_json, usage: usage&.as_json, duration: }.compact
 
@@ -150,7 +156,7 @@ module Lemans
     attr_reader :id, :task, :agent, :model, :index,
                 :profile_digest, :task_digest, :revision
 
-    attr_accessor :tags, :metadata
+    attr_accessor :tags, :metadata, :restarted_from
 
     attr_reader :phases, :steps
 
@@ -171,6 +177,7 @@ module Lemans
       @metadata = {}
       @phases = []
       @steps = nil
+      @restarted_from = nil
 
       @id = id || "#{task}__#{SecureRandom.alphanumeric(7)}"
       @outcome = Outcome.new(:pending)
@@ -239,12 +246,55 @@ module Lemans
       self
     end
 
+    # A step is settled once the next one started: it passed its gate and left a savepoint.
+    def settled_steps
+      names = phases.map(&:name)
+      (1..steps.to_a.size).count { names.include?(:"agent.#{it + 1}") }
+    end
+
+    # The step whose verification ran last, nil when none did
+    def verified_step
+      name = phases.reverse.find { it.name.to_s.match?(/\Averifier(\.\d+)?\z/) }&.name or return nil
+      name.to_s[/\.(\d+)\z/, 1]&.to_i || steps&.size || 1
+    end
+
+    # Carries over the steps before `step`; a reverification carries `step` itself too.
+    def restart!(source, step:, mode: nil)
+      @restarted_from = Restart.new(trial: source.id, step:, mode: mode&.to_s)
+      carried = mode == :reverify ? step : step - 1
+
+      if source.steps
+        source.steps.first(carried).each { step_completed!(it.outcome, it.usage, duration: it.duration) }
+      elsif carried.positive?
+        # A single-step run graded before keeps its agent outcome; a failed grading lost it.
+        completed!(source.scored? ? source.outcome : Outcome.new(:completed), source.usage)
+      end
+      self
+    end
+
+    # Splices in the named phases of the source run, shifted so that its
+    # `through` phase ends now, and moves this run's environment setup back in
+    # front of them: the timeline reads as one run.
+    def adopt_phases!(source, names, through: names.last, now: Time.now.utc)
+      delta = now - source.phases.find { it.name == through }.finished_at
+
+      setup = phases.first
+      shift = source.phases.find { it.name == setup.name }.finished_at + delta - setup.finished_at
+      phases[0] = Phase.new(setup.name, started_at: setup.started_at + shift, finished_at: setup.finished_at + shift)
+
+      source.phases.select { names.include?(it.name) }.each do |phase|
+        phases << Phase.new(phase.name, started_at: phase.started_at + delta, finished_at: phase.finished_at + delta)
+      end
+      self
+    end
+
     private def aggregate_usage = steps.filter_map(&:usage).reduce(:+)
 
     def as_json(**)
       {
         trial: id, task:, agent:, model:, index:,
         profile_digest:, task_digest:, revision: revision&.as_json,
+        restarted_from: restarted_from&.as_json,
         lemans_version: VERSION,
         tags:, metadata:, phases: phases.map(&:as_json),
         steps: steps&.map(&:as_json),
@@ -264,6 +314,7 @@ module Lemans
         )
         result.tags = data[:tags] || []
         result.metadata = data[:metadata] || {}
+        result.restarted_from = Restart.new(**data[:restarted_from]) if data[:restarted_from]
         phases_from(data).each { result.phases << it }
 
         # Steps first: the stored outcome/usage below override the aggregates.

data/lib/lemans/runner/task.rb

@@ -20,18 +20,22 @@ module Lemans
       def_delegators :definition, :name, :config
       def_delegators :result, :id
 
-      private attr_reader :definition, :store, :reporter
+      private attr_reader :definition, :store, :reporter, :restart_from, :restart_mode
 
-      def initialize(model, task_definition, index: 0, store: nil, reporter: nil)
+      def initialize(model, task_definition, index: 0, store: nil, reporter: nil, restart_from: nil, restart_mode: nil)
         @model = model
         @definition = task_definition
         @index = index
         @store = store
         @reporter = reporter
+        @restart_from = restart_from
+        @restart_mode = restart_mode
         @status = :pending
 
-        # prepare the result object: it's used by the actual execution down the stack
-        @result = Result.from_task(definition, index:, model: model || config.models.first)
+        # prepare the result object: it's used by the actual execution down the stack;
+        # a restart keeps the agent of the run it restarts
+        @result = Result.from_task(definition, index:, model: model || config.models.first,
+                                               **({ agent: restart_from.agent } if restart_from))
       end
 
       def with_reporter(reporter)
@@ -55,7 +59,7 @@ module Lemans
       private
 
       def execute!
-        Trial.new(definition, model, store:, result:).run
+        Trial.new(definition, model, store:, result:, agent: restart_from&.agent, restart_from:, restart_mode:).run
       end
     end
   end

data/lib/lemans/runner.rb

@@ -20,20 +20,26 @@ module Lemans
 
     attr_reader :config, :tasks, :store, :reporter
 
-    private attr_reader :resuming, :executor
+    private attr_reader :resuming, :restarts, :restart_mode, :allow_scored, :executor
 
-    def initialize(config, tasks, store: nil, reporter: nil, executor: nil, resume: false)
+    def initialize(config, tasks, store: nil, reporter: nil, executor: nil, resume: false,
+                   restarts: [], restart_mode: nil, allow_scored: false)
       @config = config
       @tasks = tasks
       @store = store
       @reporter = reporter
       @executor = executor || Executor.new(config.concurrency)
       @resuming = resume
+      @restarts = restarts
+      @restart_mode = restart_mode
+      @allow_scored = allow_scored
     end
 
     def resuming? = @resuming
 
     def attempts
+      return @attempts ||= restart_attempts if restarts.any?
+
       @attempts ||= config.agent.models.flat_map do |model|
         @tasks.flat_map do |task|
           completed = resuming? ? completed_attempts(task, model) : 0
@@ -64,6 +70,43 @@ module Lemans
 
     private
 
+    # Every run is checked before any starts: one refused run stops the batch.
+    def restart_attempts
+      refusals = []
+      attempts = restarts.filter_map do |restart|
+        restart_attempt(restart)
+      rescue ConfigError => e
+        refusals << e.message
+        nil
+      end
+      raise ConfigError, refusals.join("\n") if refusals.any?
+
+      attempts
+    end
+
+    # A run named for a restart is restarted on purpose: a changed task or
+    # bench is the operator's call, and a reverification may grade a scored run.
+    def restart_attempt(restart)
+      refuse = ->(reason) { raise ConfigError, "cannot restart #{restart.id}: #{reason}" }
+      task = tasks.find { it.name == restart.task }
+      refuse.("task #{restart.task} is not in the bench") unless task
+      refuse.("it is already scored (--allow-scored to restart it anyway)") if restart.scored? && restart_mode != :reverify && !allow_scored
+
+      case restart_mode
+      when :reverify
+        refuse.("it never reached a verification") unless restart.verified_step
+      when :recover
+        refuse.("#{restart.agent} cannot recover a session") unless Agents.lookup(restart.agent).recoverable?
+
+        history = task.multistep? ? "agent.result.#{restart.settled_steps + 1}.json" : "agent.result.json"
+        refuse.("it has no #{history} to recover from") unless store&.read_artifact(restart, history)
+      else
+        refuse.("no step was settled before it failed") if restart.settled_steps.zero?
+      end
+
+      Task.new(restart.model, task, store:, index: restart.index || 1, restart_from: restart, restart_mode:)
+    end
+
     def completed_attempts(task, model)
       completed_runs.count do |run|
         run.task == task.name &&

data/lib/lemans/store.rb

@@ -46,5 +46,11 @@ module Lemans
     def read_artifact(result, path)
       raise NotImplementedError
     end
+
+    # Returns the sorted paths of the artifacts the result stored (the result
+    # record is not one of them)
+    def artifacts(result)
+      raise NotImplementedError
+    end
   end
 end

data/lib/lemans/stores/fs.rb

@@ -52,8 +52,8 @@ module Lemans
       end
 
       def delete(result)
-        dir = root.glob("**/#{result.id}").find(&:directory?)
-        return unless dir
+        dir = result_dir(result)
+        return unless dir.directory?
 
         FileUtils.remove_entry(dir.to_s)
         prune_empty_parents(dir.parent)
@@ -90,6 +90,13 @@ module Lemans
         file.read if file.file?
       end
 
+      def artifacts(result)
+        dir = result_dir(result)
+        return [] unless dir.directory?
+
+        (dir.glob("**/*").select(&:file?).map { it.relative_path_from(dir).to_s } - [ FILENAME ]).sort
+      end
+
       private
 
       def filtered(text) = filterer ? filterer.filter(text) : text
@@ -132,9 +139,14 @@ module Lemans
         tmp&.delete if tmp&.exist?
       end
 
-      # result.json is stored at <root>/<model-short>/<result-id>
+      # result.json is stored at <root>/<model-short>/<result-id>; a tree
+      # arranged by hand (runs grouped under batch directories) is searched
+      # for the id instead.
       def result_dir(result)
-        root.join((result.model || result.agent).to_s.split("/").last.tr("#", "-"), result.id)
+        canonical = root.join((result.model || result.agent).to_s.split("/").last.to_s.tr("#", "-"), result.id)
+        return canonical if canonical.directory?
+
+        root.glob("**/#{result.id}").find(&:directory?) || canonical
       end
     end
   end

data/lib/lemans/trial/patch.rb

@@ -61,6 +61,15 @@ module Lemans
         raise InfrastructureError, "could not savepoint the tree the step left behind" unless savepoint
       end
 
+      # Brings a fresh tree to where an earlier run left it: the settled steps'
+      # patches applied in order and marked as the savepoint, then the work of
+      # the step that run stopped in, on top of it, so collect! sees it as this step's.
+      def replay!(settled, unsettled = nil)
+        settled.each { apply!(it) }
+        savepoint!
+        apply!(unsettled) if unsettled
+      end
+
       # Puts the workdir back to the savepoint exactly: the verifier restored
       # the graded surfaces from the baseline and may have littered the tree,
       # and the next step's agent must find neither.
@@ -75,6 +84,18 @@ module Lemans
 
       private
 
+      def apply!(contents)
+        return if contents.empty?
+
+        Tempfile.create(%w[agent .patch]) do |file|
+          file.write(contents)
+          file.flush
+          environment.upload(file.path, REMOTE_PATCH)
+        end
+
+        environment.exec!("#{git} apply --binary --whitespace=nowarn #{REMOTE_PATCH} && rm -f #{REMOTE_PATCH}", timeout:)
+      end
+
       def save_diff(result, store, from, to, destination)
         diffed = environment.exec("#{git} diff --binary #{from} #{to} > #{REMOTE_PATCH}", timeout:)
         return unless diffed.success?

data/lib/lemans/trial/setup.rb

@@ -52,6 +52,7 @@ module Lemans
         workdir = Shellwords.escape(task.environment.workdir)
         environment.exec!(
           "cd #{workdir} && git apply --binary --whitespace=nowarn #{Shellwords.escape(seed)} && " \
+          "find . -path ./.git -prune -o -exec touch -h {} + && " \
           "rm -rf .git && git init -q && git add -A && " \
           "git -c user.name=lemans -c user.email=lemans@localhost commit -qm 'Initial commit'",
           timeout: