lemans 1.4.0 → 1.5.1

data/lib/lemans/environments/docker.rb

@@ -12,6 +12,7 @@ module Lemans
       HOUSEKEEPING_TIMEOUT = 60
       MAX_OUTPUT_BYTES = 200_000
       EXEC_SLACK = 30
+      PROXY_PORT = 3128
 
       attr_reader :container
 
@@ -24,7 +25,8 @@ module Lemans
       end
 
       def start
-        build_image! if image.built?
+        build_image!(image) if image.built?
+        start_proxy!(network.hosts) if network.allowlist?
         docker!("run", *run_args, timeout: build_timeout)
         @container = @name
         self
@@ -37,9 +39,10 @@ module Lemans
         timeout ||= DEFAULT_TIMEOUT
         started = now
         argv = [ "exec" ]
+        env = proxy_env.merge(env) if network.allowlist?
         env.each { |key, value| argv += [ "--env", "#{key}=#{value}" ] }
         # The in-container timeout is what actually kills the process
-        argv += [ container, "timeout", timeout.to_i.to_s, "sh", "-c", command ]
+        argv += [ container, "timeout", timeout.to_i.to_s, "bash", "-c", command ]
 
         exit_code, output = capture("docker", *argv, timeout: timeout + EXEC_SLACK)
         ExecResult.new(command:, exit_code:, output:, duration: (now - started).round(3))
@@ -59,13 +62,13 @@ module Lemans
       def switch_network_policy!(policy)
         assert_policy_supported!(policy)
 
-        if policy.none?
-          connected_networks.each { docker!("network", "disconnect", it, container) }
-        else
-          networks = connected_networks
-          docker!("network", "disconnect", "none", container) if networks.include?("none")
-          docker!("network", "connect", "bridge", container) unless networks.include?("bridge")
-        end
+        wanted = ("bridge" if policy.public?) || (internal_network if policy.allowlist?)
+        networks = connected_networks
+        (networks - [ wanted ]).each { docker!("network", "disconnect", it, container) }
+
+        remove_proxy
+        start_proxy!(policy.hosts) if policy.allowlist?
+        docker!("network", "connect", wanted, container) if wanted && !networks.include?(wanted)
 
         @network = policy
       end
@@ -81,41 +84,101 @@ module Lemans
         end
       rescue StandardError => e
         warn "lemans: container #{container} may still be running — remove failed: #{e.class}: #{e.message}"
+      ensure
+        remove_proxy
+        remove_internal_network
       end
 
       private
 
       def assert_policy_supported!(policy)
-        return if policy.none? || policy.public?
+        return unless policy.allowlist? && policy.ip_targets.any?
 
-        raise ConfigError, "docker: #{policy.mode} is not supported (public and none only)"
+        raise ConfigError, "docker: an allowlist takes host names only (#{policy.ip_targets.join(", ")})"
       end
 
       # The tag is the content digest, so an existing image is the identical thing
-      def build_image!
-        exists, = capture("docker", "image", "inspect", image.name, timeout: HOUSEKEEPING_TIMEOUT)
+      def build_image!(spec)
+        exists, = capture("docker", "image", "inspect", spec.name, timeout: HOUSEKEEPING_TIMEOUT)
         return if exists.zero?
 
-        docker!("build", "--tag", image.name, image.context_dir.to_s, timeout: build_timeout, on_output: @logger)
+        docker!("build", "--tag", spec.name, spec.context_dir.to_s, timeout: build_timeout, on_output: @logger)
+      end
+
+      # The task container sits on an internal network with no route out; the
+      # proxy is on it and on bridge, so the hosts it allows are the only way out.
+      def start_proxy!(hosts)
+        image = Config::ImageSpec.dockerfile(Pathname(__dir__).join("docker/proxy/Dockerfile"), slug: "proxy")
+        build_image!(image)
+        create_internal_network!
+        docker!("run", "--detach", "--name", proxy_name, "--network", internal_network,
+                *label_args, image.name, *hosts)
+        docker!("network", "connect", "bridge", proxy_name)
+        wait_for_proxy!
+      end
+
+      def wait_for_proxy!(attempts: 50)
+        attempts.times do
+          _, output = capture("docker", "logs", proxy_name, timeout: HOUSEKEEPING_TIMEOUT)
+          return if output.include?("listening on")
+
+          sleep 0.2
+        end
+
+        raise InfrastructureError, "docker: the allowlist proxy did not start"
       end
 
+      def proxy_env
+        url = "http://#{proxy_name}:#{PROXY_PORT}"
+        no_proxy = "localhost,127.0.0.1"
+        { "http_proxy" => url, "https_proxy" => url, "HTTP_PROXY" => url, "HTTPS_PROXY" => url,
+          "no_proxy" => no_proxy, "NO_PROXY" => no_proxy }
+      end
+
+      def create_internal_network!
+        exists, = capture("docker", "network", "inspect", internal_network, timeout: HOUSEKEEPING_TIMEOUT)
+        return if exists.zero?
+
+        docker!("network", "create", "--internal", *label_args, internal_network)
+      end
+
+      def remove_proxy
+        capture("docker", "rm", "--force", proxy_name, timeout: HOUSEKEEPING_TIMEOUT)
+      rescue StandardError
+        nil
+      end
+
+      def remove_internal_network
+        capture("docker", "network", "rm", internal_network, timeout: HOUSEKEEPING_TIMEOUT)
+      rescue StandardError
+        nil
+      end
+
+      def proxy_name = "#{@name}-proxy"
+
+      def internal_network = "#{@name}-net"
+
       def run_args
         args = [ "--detach", "--init", "--name", @name,
                  "--cpus", resources.cpus.to_s, "--memory", "#{resources.memory}m",
                  "--cap-add", "SYS_ADMIN", "--cap-add", "NET_ADMIN", "--security-opt", "apparmor=unconfined",
                  "--entrypoint", "sh" ]
         args += [ "--network", "none" ] if network.none?
+        args += [ "--network", internal_network ] if network.allowlist?
         env.each { |key, value| args += [ "--env", "#{key}=#{value}" ] }
-        labels.each { |key, value| args += [ "--label", "#{key}=#{value}" ] }
-        args + [ image.name, "-c", "tail -f /dev/null" ]
+        args + label_args + [ image.name, "-c", "tail -f /dev/null" ]
       end
 
+      def label_args = labels.flat_map { |key, value| [ "--label", "#{key}=#{value}" ] }
+
       def connected_networks
         docker!("inspect", "--format", "{{range $name, $_ := .NetworkSettings.Networks}}{{$name}} {{end}}", container).split
       end
 
       def remove
         capture("docker", "rm", "--force", "--volumes", @name, timeout: HOUSEKEEPING_TIMEOUT)
+        remove_proxy
+        remove_internal_network
       rescue StandardError
         nil
       end

data/lib/lemans/result.rb

@@ -136,6 +136,12 @@ module Lemans
       def as_json(**) = to_h
     end
 
+    Restart = Data.define(:trial, :step, :mode) do
+      def initialize(trial:, step:, mode: nil) = super
+
+      def as_json(**) = to_h.compact
+    end
+
     Step = Data.define(:outcome, :usage, :duration) do
       def as_json(**) = { outcome: outcome.as_json, usage: usage&.as_json, duration: }.compact
 
@@ -150,12 +156,12 @@ module Lemans
     attr_reader :id, :task, :agent, :model, :index,
                 :profile_digest, :task_digest, :revision
 
-    attr_accessor :tags, :metadata
+    attr_accessor :tags, :metadata, :restarted_from, :total_steps
 
     attr_reader :phases, :steps
 
     # outcome-related attributes (we use setter-like methods, not accessors)
-    attr_reader :reward, :credit, :outcome, :usage
+    attr_reader :reward, :credit, :features, :outcome, :usage
 
     def initialize(task:, agent:, model:, id: nil, index: nil,
                    profile_digest: nil, task_digest: nil, revision: nil)
@@ -171,6 +177,8 @@ module Lemans
       @metadata = {}
       @phases = []
       @steps = nil
+      @total_steps = nil
+      @restarted_from = nil
 
       @id = id || "#{task}__#{SecureRandom.alphanumeric(7)}"
       @outcome = Outcome.new(:pending)
@@ -222,9 +230,10 @@ module Lemans
       completed!(outcome, aggregate_usage)
     end
 
-    def graded!(reward, credit: reward)
+    def graded!(reward, credit: reward, features: nil)
       @reward = reward
       @credit = credit
+      @features = features
       self
     end
 
@@ -236,6 +245,49 @@ module Lemans
       @outcome = Outcome.new(reason, detail)
       @reward = nil
       @credit = nil
+      @features = nil
+      self
+    end
+
+    # A step is settled once the next one started: it passed its gate and left a savepoint.
+    def settled_steps
+      names = phases.map(&:name)
+      (1..steps.to_a.size).count { names.include?(:"agent.#{it + 1}") }
+    end
+
+    # The step whose verification ran last, nil when none did
+    def verified_step
+      name = phases.reverse.find { it.name.to_s.match?(/\Averifier(\.\d+)?\z/) }&.name or return nil
+      name.to_s[/\.(\d+)\z/, 1]&.to_i || steps&.size || 1
+    end
+
+    # Carries over the steps before `step`; a reverification carries `step` itself too.
+    def restart!(source, step:, mode: nil)
+      @restarted_from = Restart.new(trial: source.id, step:, mode: mode&.to_s)
+      carried = mode == :reverify ? step : step - 1
+
+      if source.steps
+        source.steps.first(carried).each { step_completed!(it.outcome, it.usage, duration: it.duration) }
+      elsif carried.positive?
+        # A single-step run graded before keeps its agent outcome; a failed grading lost it.
+        completed!(source.scored? ? source.outcome : Outcome.new(:completed), source.usage)
+      end
+      self
+    end
+
+    # Splices in the named phases of the source run, shifted so that its
+    # `through` phase ends now, and moves this run's environment setup back in
+    # front of them: the timeline reads as one run.
+    def adopt_phases!(source, names, through: names.last, now: Time.now.utc)
+      delta = now - source.phases.find { it.name == through }.finished_at
+
+      setup = phases.first
+      shift = source.phases.find { it.name == setup.name }.finished_at + delta - setup.finished_at
+      phases[0] = Phase.new(setup.name, started_at: setup.started_at + shift, finished_at: setup.finished_at + shift)
+
+      source.phases.select { names.include?(it.name) }.each do |phase|
+        phases << Phase.new(phase.name, started_at: phase.started_at + delta, finished_at: phase.finished_at + delta)
+      end
       self
     end
 
@@ -245,10 +297,11 @@ module Lemans
       {
         trial: id, task:, agent:, model:, index:,
         profile_digest:, task_digest:, revision: revision&.as_json,
+        restarted_from: restarted_from&.as_json,
         lemans_version: VERSION,
         tags:, metadata:, phases: phases.map(&:as_json),
-        steps: steps&.map(&:as_json),
-        reward:, credit:, outcome: outcome.as_json, usage: usage&.as_json, duration:,
+        steps: steps&.map(&:as_json), total_steps:,
+        reward:, credit:, features:, outcome: outcome.as_json, usage: usage&.as_json, duration:,
         started_at: started_at&.iso8601,
         finished_at: finished_at&.iso8601
       }.compact
@@ -264,6 +317,8 @@ module Lemans
         )
         result.tags = data[:tags] || []
         result.metadata = data[:metadata] || {}
+        result.restarted_from = Restart.new(**data[:restarted_from]) if data[:restarted_from]
+        result.total_steps = data[:total_steps]
         phases_from(data).each { result.phases << it }
 
         # Steps first: the stored outcome/usage below override the aggregates.
@@ -281,7 +336,10 @@ module Lemans
           )
         end
 
-        result.graded!(data[:reward], credit: data[:credit] || data[:reward]) unless data[:reward].nil?
+        unless data[:reward].nil?
+          result.graded!(data[:reward], credit: data[:credit] || data[:reward],
+                                        features: data[:features]&.transform_keys(&:to_s))
+        end
         result
       end
 
@@ -298,6 +356,7 @@ module Lemans
         )
         result.tags = definition.tags
         result.metadata = definition.metadata
+        result.total_steps = definition.steps if definition.multistep?
         result
       end
 

data/lib/lemans/runner/task.rb

@@ -20,18 +20,22 @@ module Lemans
       def_delegators :definition, :name, :config
       def_delegators :result, :id
 
-      private attr_reader :definition, :store, :reporter
+      private attr_reader :definition, :store, :reporter, :restart_from, :restart_mode
 
-      def initialize(model, task_definition, index: 0, store: nil, reporter: nil)
+      def initialize(model, task_definition, index: 0, store: nil, reporter: nil, restart_from: nil, restart_mode: nil)
         @model = model
         @definition = task_definition
         @index = index
         @store = store
         @reporter = reporter
+        @restart_from = restart_from
+        @restart_mode = restart_mode
         @status = :pending
 
-        # prepare the result object: it's used by the actual execution down the stack
-        @result = Result.from_task(definition, index:, model: model || config.models.first)
+        # prepare the result object: it's used by the actual execution down the stack;
+        # a restart keeps the agent of the run it restarts
+        @result = Result.from_task(definition, index:, model: model || config.models.first,
+                                               **({ agent: restart_from.agent } if restart_from))
       end
 
       def with_reporter(reporter)
@@ -55,7 +59,7 @@ module Lemans
       private
 
       def execute!
-        Trial.new(definition, model, store:, result:).run
+        Trial.new(definition, model, store:, result:, agent: restart_from&.agent, restart_from:, restart_mode:).run
       end
     end
   end

data/lib/lemans/runner.rb

@@ -20,20 +20,26 @@ module Lemans
 
     attr_reader :config, :tasks, :store, :reporter
 
-    private attr_reader :resuming, :executor
+    private attr_reader :resuming, :restarts, :restart_mode, :allow_scored, :executor
 
-    def initialize(config, tasks, store: nil, reporter: nil, executor: nil, resume: false)
+    def initialize(config, tasks, store: nil, reporter: nil, executor: nil, resume: false,
+                   restarts: [], restart_mode: nil, allow_scored: false)
       @config = config
       @tasks = tasks
       @store = store
       @reporter = reporter
       @executor = executor || Executor.new(config.concurrency)
       @resuming = resume
+      @restarts = restarts
+      @restart_mode = restart_mode
+      @allow_scored = allow_scored
     end
 
     def resuming? = @resuming
 
     def attempts
+      return @attempts ||= restart_attempts if restarts.any?
+
       @attempts ||= config.agent.models.flat_map do |model|
         @tasks.flat_map do |task|
           completed = resuming? ? completed_attempts(task, model) : 0
@@ -64,6 +70,43 @@ module Lemans
 
     private
 
+    # Every run is checked before any starts: one refused run stops the batch.
+    def restart_attempts
+      refusals = []
+      attempts = restarts.filter_map do |restart|
+        restart_attempt(restart)
+      rescue ConfigError => e
+        refusals << e.message
+        nil
+      end
+      raise ConfigError, refusals.join("\n") if refusals.any?
+
+      attempts
+    end
+
+    # A run named for a restart is restarted on purpose: a changed task or
+    # bench is the operator's call, and a reverification may grade a scored run.
+    def restart_attempt(restart)
+      refuse = ->(reason) { raise ConfigError, "cannot restart #{restart.id}: #{reason}" }
+      task = tasks.find { it.name == restart.task }
+      refuse.("task #{restart.task} is not in the bench") unless task
+      refuse.("it is already scored (--allow-scored to restart it anyway)") if restart.scored? && restart_mode != :reverify && !allow_scored
+
+      case restart_mode
+      when :reverify
+        refuse.("it never reached a verification") unless restart.verified_step
+      when :recover
+        refuse.("#{restart.agent} cannot recover a session") unless Agents.lookup(restart.agent).recoverable?
+
+        history = task.multistep? ? "agent.result.#{restart.settled_steps + 1}.json" : "agent.result.json"
+        refuse.("it has no #{history} to recover from") unless store&.read_artifact(restart, history)
+      else
+        refuse.("no step was settled before it failed") if restart.settled_steps.zero?
+      end
+
+      Task.new(restart.model, task, store:, index: restart.index || 1, restart_from: restart, restart_mode:)
+    end
+
     def completed_attempts(task, model)
       completed_runs.count do |run|
         run.task == task.name &&

data/lib/lemans/store.rb

@@ -46,5 +46,11 @@ module Lemans
     def read_artifact(result, path)
       raise NotImplementedError
     end
+
+    # Returns the sorted paths of the artifacts the result stored (the result
+    # record is not one of them)
+    def artifacts(result)
+      raise NotImplementedError
+    end
   end
 end

data/lib/lemans/stores/fs.rb

@@ -52,8 +52,8 @@ module Lemans
       end
 
       def delete(result)
-        dir = root.glob("**/#{result.id}").find(&:directory?)
-        return unless dir
+        dir = result_dir(result)
+        return unless dir.directory?
 
         FileUtils.remove_entry(dir.to_s)
         prune_empty_parents(dir.parent)
@@ -90,6 +90,13 @@ module Lemans
         file.read if file.file?
       end
 
+      def artifacts(result)
+        dir = result_dir(result)
+        return [] unless dir.directory?
+
+        (dir.glob("**/*").select(&:file?).map { it.relative_path_from(dir).to_s } - [ FILENAME ]).sort
+      end
+
       private
 
       def filtered(text) = filterer ? filterer.filter(text) : text
@@ -132,9 +139,14 @@ module Lemans
         tmp&.delete if tmp&.exist?
       end
 
-      # result.json is stored at <root>/<model-short>/<result-id>
+      # result.json is stored at <root>/<model-short>/<result-id>; a tree
+      # arranged by hand (runs grouped under batch directories) is searched
+      # for the id instead.
       def result_dir(result)
-        root.join((result.model || result.agent).to_s.split("/").last.tr("#", "-"), result.id)
+        canonical = root.join((result.model || result.agent).to_s.split("/").last.to_s.tr("#", "-"), result.id)
+        return canonical if canonical.directory?
+
+        root.glob("**/#{result.id}").find(&:directory?) || canonical
       end
     end
   end

data/lib/lemans/trial/patch.rb

@@ -61,6 +61,15 @@ module Lemans
         raise InfrastructureError, "could not savepoint the tree the step left behind" unless savepoint
       end
 
+      # Brings a fresh tree to where an earlier run left it: the settled steps'
+      # patches applied in order and marked as the savepoint, then the work of
+      # the step that run stopped in, on top of it, so collect! sees it as this step's.
+      def replay!(settled, unsettled = nil)
+        settled.each { apply!(it) }
+        savepoint!
+        apply!(unsettled) if unsettled
+      end
+
       # Puts the workdir back to the savepoint exactly: the verifier restored
       # the graded surfaces from the baseline and may have littered the tree,
       # and the next step's agent must find neither.
@@ -75,6 +84,18 @@ module Lemans
 
       private
 
+      def apply!(contents)
+        return if contents.empty?
+
+        Tempfile.create(%w[agent .patch]) do |file|
+          file.write(contents)
+          file.flush
+          environment.upload(file.path, REMOTE_PATCH)
+        end
+
+        environment.exec!("#{git} apply --binary --whitespace=nowarn #{REMOTE_PATCH} && rm -f #{REMOTE_PATCH}", timeout:)
+      end
+
       def save_diff(result, store, from, to, destination)
         diffed = environment.exec("#{git} diff --binary #{from} #{to} > #{REMOTE_PATCH}", timeout:)
         return unless diffed.success?

data/lib/lemans/trial/verifier/test_scanner.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+require "pathname"
+require "prism"
+
+module Lemans
+  class Trial
+    class Verifier
+      # Reads the grading schema off the test files without running them: every
+      # `def test_*` and ActiveSupport `test "..."` is a check, an
+      # `allow_failure` call inside makes it an extra worth its `points:`, and
+      # a `# @feature <name>` comment right above a check or inside it tracks
+      # the check as that feature. Files required from the test file's
+      # directory are read too: suites often live apart.
+      class TestScanner < Prism::Visitor
+        ENTRY = "verification_test.rb"
+        FEATURE = /\A#\s*@feature\s+(\S+)/
+
+        Span = Data.define(:check, :file, :lines)
+
+        attr_reader :tests, :points, :features, :base_credit, :stray_features
+
+        # The task's verification_test.rb scanned, nil when it has none
+        def self.for(task)
+          local, = task.test_files.find { |_, remote| remote == ENTRY }
+          return unless local
+
+          new(Pathname(local).dirname).tap { it.scan(Pathname(local)) }
+        end
+
+        def initialize(root)
+          super()
+          @root = root
+          @scope = []
+          @tests = []
+          @points = {}
+          @features = {}
+          @stray_features = []
+          @spans = []
+          @current = nil
+          @scanned = []
+        end
+
+        def scan(path)
+          return if @scanned.include?(path)
+
+          @scanned << path
+          file = @file
+          begin
+            @file = path
+            parsed = Prism.parse_file(path.to_s)
+            parsed.value.accept(self)
+            attach_features(path, parsed.comments)
+          ensure
+            @file = file
+          end
+        end
+
+        def visit_module_node(node) = scoped(node) { super }
+
+        def visit_class_node(node) = scoped(node) { super }
+
+        def visit_def_node(node)
+          return super unless node.name.start_with?("test_")
+
+          within("#{@scope.join("::")}##{node.name}", node) { super }
+        end
+
+        def visit_call_node(node)
+          case node.name
+          when :test
+            title = node.arguments&.arguments&.first
+            if node.receiver.nil? && node.block && title.is_a?(Prism::StringNode)
+              return within("#{@scope.join("::")}#test_#{title.unescaped.gsub(/\s+/, "_")}", node) { super }
+            end
+          when :allow_failure
+            @points[@current] ||= points_of(node) if @current
+          when :require, :require_relative
+            required = node.arguments&.arguments&.first
+            scan_required(node.name, required.unescaped) if node.receiver.nil? && required.is_a?(Prism::StringNode)
+            return
+          when :base_credit=
+            @base_credit = node.arguments.arguments.first.value if node.receiver.is_a?(Prism::ConstantReadNode) && node.receiver.name == :LemansReport
+          end
+          super
+        end
+
+        private
+
+        def scan_required(how, name)
+          path = (how == :require_relative ? @file.dirname : @root).join("#{name.delete_suffix(".rb")}.rb")
+          scan(path) if path.file?
+        end
+
+        def scoped(node)
+          @scope.push(node.constant_path.full_name)
+          yield
+        ensure
+          @scope.pop
+        end
+
+        def within(check, node)
+          @tests << check
+          @spans << Span.new(check:, file: @file, lines: node.location.start_line..node.location.end_line)
+          @current = check
+          yield
+        ensure
+          @current = nil
+        end
+
+        def points_of(node)
+          keywords = node.arguments&.arguments&.grep(Prism::KeywordHashNode)&.first
+          pair = keywords&.elements&.find { it.is_a?(Prism::AssocNode) && it.key.is_a?(Prism::SymbolNode) && it.key.unescaped == "points" }
+          pair ? pair.value.value : 1
+        end
+
+        # A comment belongs to the check it sits in, or to the one its
+        # comment block opens onto.
+        def attach_features(path, comments)
+          own_line = path.readlines.each_with_index.filter_map { |text, index| index + 1 if text.lstrip.start_with?("#") }
+          spans = @spans.select { it.file == path }
+
+          comments.each do |comment|
+            name = comment.slice[FEATURE, 1] or next
+            line = comment.location.start_line
+            span = spans.find { it.lines.cover?(line) } ||
+                   spans.find { |s| s.lines.begin > line && (line...s.lines.begin).all? { own_line.include?(it) } }
+            next @stray_features << "#{path}:#{line}" unless span
+
+            (@features[name] ||= []) << span.check unless @features[name]&.include?(span.check)
+          end
+        end
+      end
+    end
+  end
+end