lemans 0.2.2 → 1.0.0.pre.1

data/lib/lemans/cli.rb

@@ -19,12 +19,14 @@ module Lemans
 
     desc "tasks", "List the tasks in a bench"
     option :bench, default: ".", desc: "Directory holding bench.yml"
-    option :tag, desc: "Only tasks carrying this tag"
+    option :tag, desc: "Only tasks carrying this tag", repeatable: true
     def tasks
-      bench = Bench.load(options[:bench])
+      config = Config.load_file(options[:bench])
+      tasks = filter_tasks(config.tasks, tags: options[:tag])
+
       print_table(
         [%w[task difficulty tags description]] +
-        select_tasks(bench).map { [_1.name, _1.difficulty, _1.tags.join(","), _1.description] }
+        tasks.map { [it.name, it.difficulty, it.tags.join(","), it.description] }
       )
     rescue ConfigError => e
       raise Thor::Error, "lemans: #{e.message}"
@@ -34,12 +36,12 @@ module Lemans
     desc "run", "Run tasks and verify them"
     option :bench, default: ".", desc: "Directory holding bench.yml"
     option :task, desc: "Run task(s) by name", repeatable: true
-    option :tag, desc: "Run every task carrying this tag"
-    option :agent, desc: "Override the agent from bench.yml (miniswen, oracle, nop)"
+    option :tag, desc: "Run every task carrying this tag(s)", repeatable: true
+    option :agent, desc: "Override the agent from bench.yml (miniswen, miniswen-installed, oracle, nop)"
     option :model, desc: "Override the model(s) from bench.yml", repeatable: true
     option :attempts, type: :numeric, default: 1, aliases: "-k", desc: "Trials per task"
     option :concurrency, type: :numeric, default: 4, aliases: "-c", desc: "Trials in flight at once"
-    option :runs_dir, default: "runs", desc: "Where to write run directories"
+    option :runs_dir, default: "./runs", desc: "Where to write run directories"
     option :backend, default: "daytona", enum: Environments::BACKENDS.keys, desc: "Sandbox backend"
     option :resume, type: :boolean, default: false, desc: "Skip trials that already have a result"
     def run_bench
@@ -47,74 +49,68 @@ module Lemans
       # front, so every trial prices completions against the same revision.
       Miniswen.refresh_registry!
 
-      bench = Bench.load(options[:bench])
-      tasks = select_tasks(bench)
-
-      run = Run.new(
-        bench: bench,
-        tasks: tasks,
-        agent_name: options[:agent] || bench.agent.name,
-        model: options[:model],
-        backend: options[:backend],
-        runs_dir: options[:runs_dir],
-        attempts: Integer(options[:attempts]),
-        concurrency: Integer(options[:concurrency]),
-        resume: options[:resume]
-      )
-      if run.total.zero?
+      config = Config.load_file(options[:bench])
+      config.load_options(**options.transform_keys(&:to_sym))
+
+      tasks = filter_tasks(config.tasks, tags: options[:tag], name: options[:task])
+
+      store = Stores::FS.new(options[:runs_dir])
+
+      runner = Runner.new(config, tasks, store:, resume: options[:resume])
+
+      if runner.resuming? && runner.attempts.empty?
         say_status :resume, "nothing to run — every task × model already has " \
-                            "#{options[:attempts]} scored attempt(s)", :green
+                            "#{config.attempts} scored attempt(s)", :green
+
+        return
       end
 
-      # A tty gets the live board; a pipe gets plain streamed lines.
-      progress =
+      reporter =
         if interactive?
-          models = options[:model] || (bench.agent.models.empty? ? [bench.agent.model] : bench.agent.models)
-          BoardReporter.new(tasks: tasks.map(&:name), models: models,
-                            attempts: Integer(options[:attempts]), total: run.total)
+          BoardReporter.new(tasks: tasks.map(&:name), models: config.models,
+                            attempts: config.attempts, total: runner.attempts.size)
         else
-          ProgressReporter.new(shell: shell, task_width: tasks.map { _1.name.length }.max)
+          ProgressReporter.new(shell:, tasks: tasks.map(&:name))
         end
-      progress.start
-      summary = run.call { |event, data| progress.record(event, data) }
-      progress.stop
+
+      reporter.start
+
+      summary = runner.run(reporter)
 
       say ""
       say_status :report, "collecting results from #{options[:runs_dir]}", :cyan
-      print_report Results::Report.load(options[:runs_dir])
-      exit 130 if summary[:interrupted]
-      exit 1 if summary[:invalid].positive?
+      print_report Report.load(store)
+
+      exit 130 if summary.status == :interrupted
+      exit 1 if summary.status == :invalid
     rescue ConfigError => e
       raise Thor::Error, "lemans: #{e.message}"
     rescue Interrupt
       say ""
       exit 130
     ensure
-      progress&.stop
+      reporter&.stop
     end
 
     desc "clobber", "Delete run results"
-    option :runs_dir, default: "runs", desc: "Directory holding run directories"
-    option :task, type: :array, desc: "Only these tasks' runs (space-separated)"
+    option :runs_dir, default: "./runs", desc: "Directory holding run directories"
+    option :task, desc: "Only these tasks' runs", repeatable: true
     option :ttl, desc: "Only runs older than this (10m, 2h, 1d)"
     option :invalid, type: :boolean, default: false, desc: "Only runs that measured nothing (invalid or unreadable)"
     option :force, type: :boolean, default: false, aliases: "-f", desc: "Delete without asking"
     def clobber
-      clobber = Clobber.new(
-        runs_dir: options[:runs_dir],
-        tasks: options[:task],
-        ttl_sec: Units.seconds(options[:ttl], field: "--ttl"),
-        invalid: options[:invalid]
-      )
+      store = Stores::FS.new(options[:runs_dir])
+      clobber = Clobber.new(store, tasks: options[:task], ttl: options[:ttl], invalid: options[:invalid])
+
       doomed = clobber.matches
       return say "lemans: nothing to clobber under #{options[:runs_dir]}" if doomed.empty?
 
       unless options[:force]
-        doomed.each { say _1.to_s }
+        doomed.each { say it.id }
         return say "lemans: nothing deleted" unless yes?("Delete #{doomed.size} run(s) under #{options[:runs_dir]}? [y/N]")
       end
 
-      removed = clobber.call
+      removed = clobber.execute!
       say "deleted #{removed.size} run(s)"
     rescue ConfigError => e
       raise Thor::Error, "lemans: #{e.message}"
@@ -122,19 +118,18 @@ module Lemans
 
     desc "report", "Summarize run results as a table or CSV"
     option :runs_dir, default: "runs", desc: "Directory holding run directories"
-    option :tag, desc: "Only runs whose result carries this tag"
+    option :tag, desc: "Only runs whose result carries this tag", repeatable: true
+    option :task, desc: "Only these tasks' runs", repeatable: true
     option :format, default: "table", enum: %w[table csv], desc: "Output format"
     option :aggregate, aliases: "-A", banner: "COLUMNS", lazy_default: "task-model",
                        desc: "Group results by 1-3 dash-joined columns (task, agent, model)"
     option :sort, aliases: "-S", banner: "COLUMN", desc: "Sort by a column"
     def report
-      results = Results::Report.load(options[:runs_dir], tag: options[:tag])
-      if results.empty?
-        tagged = options[:tag] ? " tagged #{options[:tag].inspect}" : ""
-        raise Thor::Error, "lemans: no results#{tagged} under #{options[:runs_dir]}"
-      end
+      store = Stores::FS.new(options[:runs_dir])
+      results = Report.load(store, tags: options[:tag], names: options[:task])
+      raise Thor::Error, "lemans: no matching results found" if results.empty?
 
-      results = Results::Aggregate.new(results, keys: Results::Aggregate.keys(options[:aggregate])) if options[:aggregate]
+      results = Report::Aggregate.new(results, keys: Report::Aggregate.keys(options[:aggregate])) if options[:aggregate]
       results.order_by!(options[:sort]) if options[:sort]
       options[:format] == "csv" ? say(results.to_csv) : print_report(results)
     rescue ConfigError => e
@@ -143,24 +138,24 @@ module Lemans
 
     private
 
-    # The one task filter for every command that walks a bench: an empty
-    # selection is an error, because running or listing nothing is never
-    # what a named task or tag meant.
-    def select_tasks(bench)
-      tasks = bench.tasks
-      tasks = tasks.select { options[:task].include?(_1.name) } if options[:task]
-      tasks = tasks.select { _1.tags.include?(options[:tag]) } if options[:tag]
+    def filter_tasks(tasks, tags: nil, name: nil)
+      tasks = tasks.dup
+
+      name = Array(name) if name
+      tags = Array(tags) if tags
+
+      tasks.select! { name.include?(it.name) } if name
+      tasks.select! { tags.intersect?(it.tags) } if tags
+
       return tasks unless tasks.empty?
 
-      wanted = [options[:task] && "task named #{options[:task].inspect}",
-                options[:tag] && "task tagged #{options[:tag].inspect}"].compact.join(" and no ")
-      raise Thor::Error, "lemans: no #{wanted.empty? ? "tasks in #{options[:bench]}" : wanted}"
+      raise Thor::Error, "lemans: no matching tasks"
     end
 
     def print_report(report)
       print_table report.to_rows
       color = report.summary[:invalid].positive? ? :red : nil
-      report.summary_lines.each { say _1, color }
+      report.summary_lines.each { say it, color }
     end
 
     def interactive?

data/lib/lemans/clobber.rb

@@ -1,18 +1,16 @@
 # frozen_string_literal: true
 
-require "fileutils"
-require "json"
-require "pathname"
-
 module Lemans
-  # Deletes run directories based on the provided filters.
+  # Deletes stored results based on the provided filters.
   class Clobber
-    TRIAL_DIR = /\A(?<task>.+)__[A-Za-z0-9]{7}\z/
+    include Config::Conversion
+
+    private attr_reader :store, :tasks, :ttl, :invalid
 
-    def initialize(runs_dir:, tasks: [], ttl_sec: nil, invalid: false)
-      @runs_dir = Pathname(runs_dir)
+    def initialize(store, tasks: [], ttl: nil, invalid: false)
+      @store = store
       @tasks = Array(tasks)
-      @ttl_sec = ttl_sec
+      @ttl = ttl && seconds!(ttl)
       @invalid = invalid
     end
 
@@ -20,60 +18,24 @@ module Lemans
       @matches ||= select_matches
     end
 
-    # Deletes everything it can, says what it could not, and returns what it
-    # actually removed — the caller's "deleted N" must not count survivors.
-    def call
-      deleted = matches.select do |entry|
-        FileUtils.remove_entry(entry.to_s)
-        true
-      rescue SystemCallError => e
-        warn "lemans: could not delete #{entry}: #{e.message}"
-        false
-      end
-      prune_emptied_parents(deleted)
-      deleted
+    def execute!
+      matches.select { store.delete(it) }
     end
 
     private
 
-    attr_reader :runs_dir, :tasks, :ttl_sec
-
     def select_matches
-      return [] unless runs_dir.directory?
-
-      runs_dir.glob("**/").map(&:cleanpath).sort.select do |entry|
-        task = task_name(entry)
-        next false if task.nil?
+      candidates = store.fetch
+      candidates += store.unreadable if invalid
 
-        (tasks.empty? || tasks.include?(task)) &&
-          (ttl_sec.nil? || age_sec(entry) > ttl_sec) &&
-          (!@invalid || invalid?(entry))
+      matched = candidates.select do |result|
+        (tasks.empty? || tasks.include?(result.task)) &&
+          (ttl.nil? || age(result) > ttl) &&
+          (!invalid || result.invalid?)
       end
+      matched.sort_by { it.id.to_s }
     end
 
-    def prune_emptied_parents(deleted)
-      root = runs_dir.cleanpath
-      deleted.each do |entry|
-        dir = entry.parent
-        while dir != root && dir.children.empty?
-          dir.rmdir
-          dir = dir.parent
-        end
-      rescue SystemCallError
-        next
-      end
-    end
-
-    def task_name(entry) = TRIAL_DIR.match(entry.basename.to_s)&.[](:task)
-
-    def age_sec(entry) = Time.now - entry.mtime
-
-    # A trial that measured nothing; an unreadable result counts — it will
-    # never be read as anything else.
-    def invalid?(entry)
-      JSON.parse(entry.join("result.json").read).dig("outcome", "scored") != true
-    rescue JSON::ParserError, SystemCallError, IOError
-      true
-    end
+    def age(result) = result.finished_at ? Time.now.utc - result.finished_at : Float::INFINITY
   end
 end

data/lib/lemans/config/agent.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Lemans
+  class Config
+    class Agent # :nodoc:
+      class << self
+        include Conversion
+
+        def from_config(data)
+          return if data.nil?
+
+          name = data.fetch("name")
+          model = data.fetch("model")
+
+          conf = new(name, model)
+
+          conf.step_limit = integer!(data["step_limit"]) if data["step_limit"]
+          conf.cost_limit = float!(data["cost_limit"]) if data["cost_limit"]
+          conf.timeout = seconds!(data["timeout"]) if data["timeout"]
+          conf.exec_timeout = seconds!(data["exec_timeout"]) if data["exec_timeout"]
+
+          if (network_data = data.dig("environment", "network"))
+            conf.environment = Environment.new(network: NetworkPolicy.from_config(network_data))
+          end
+
+          conf
+        rescue KeyError => e
+          raise ConfigError, "agent.#{e.key} must be provided"
+        end
+      end
+
+      attr_accessor :name, :models, :timeout,
+                    :step_limit, :cost_limit, :exec_timeout,
+                    :environment
+
+      def model = models.first
+
+      Environment = Struct.new(:network, keyword_init: true)
+
+      def initialize(name, model)
+        @name = name
+        @models = Array(model)
+        @step_limit = 100
+        @cost_limit = nil
+        @exec_timeout = 300
+        @timeout = 30 * 60
+        @environment = Environment.new(network: NetworkPolicy.new)
+      end
+    end
+  end
+end

data/lib/lemans/config/conversion.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require "pathname"
+
+module Lemans
+  class Config
+    # Helper methods to parse/convert config-provided values
+    module Conversion
+      DURATION = /\A(\d+(?:\.\d+)?)\s*(ms|s|m|h|d)?\z/
+      DURATION_FACTORS = { "ms" => 0.001, "s" => 1, "m" => 60, "h" => 3600, "d" => 86_400 }.freeze
+
+      SIZE = /\A(\d+(?:\.\d+)?)\s*(MB|GB|TB)?\z/i
+      SIZE_FACTORS = { "mb" => 1, "gb" => 1024, "tb" => 1024 * 1024 }.freeze
+
+      def integer!(value)
+        Integer(value)
+      rescue ArgumentError, TypeError
+        raise ConfigError, "cannot read #{value.inspect} as an integer"
+      end
+
+      def float!(value)
+        Float(value)
+      rescue ArgumentError, TypeError
+        raise ConfigError, "cannot read #{value.inspect} as a number"
+      end
+
+      def seconds!(value)
+        if value.is_a?(Numeric)
+          raise ConfigError, "duration must be non-negative, got: #{value}" if value.negative?
+
+          return value
+        end
+
+        match = DURATION.match(value.to_s.strip)
+
+        raise ConfigError, "cannot read #{value.inspect} as a duration (try 30m, 300s, 1h)" unless match
+
+        match[1].to_f * DURATION_FACTORS.fetch(match[2] || "s")
+      end
+
+      def megabytes!(value)
+        if value.is_a?(Numeric)
+          raise ConfigError, "size must be non-negative, got: #{value}" if value.negative?
+
+          return value.round
+        end
+
+        match = SIZE.match(value.to_s.strip)
+
+        raise ConfigError, "cannot read #{value.inspect} as a size (try 2GB, 512MB)" unless match
+
+        (match[1].to_f * SIZE_FACTORS.fetch((match[2] || "mb").downcase)).round
+      end
+
+      def absolute_path!(value)
+        raise ConfigError, "#{value.inspect} is not an absolute path" unless value.start_with?("/")
+
+        value
+      end
+    end
+  end
+end

data/lib/lemans/config/environment.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Lemans
+  class Config
+    class Environment # :nodoc:
+      Resources = Struct.new(:cpus, :memory, :storage, keyword_init: true)
+
+      class << self
+        include Conversion
+
+        def from_config(data)
+          return if data.nil?
+
+          conf = new
+          conf.image = data["image"] if data["image"]
+          conf.workdir = absolute_path!(data["workdir"]) if data["workdir"]
+          conf.build_timeout = seconds!(data["build_timeout"]) if data["build_timeout"]
+          conf.network = NetworkPolicy.from_config(data["network"]) if data["network"]
+
+          conf.resources.cpus = integer!(data.dig("resources", "cpus")) if data.dig("resources", "cpus")
+          conf.resources.memory = megabytes!(data.dig("resources", "memory")) if data.dig("resources", "memory")
+          conf.resources.storage = megabytes!(data.dig("resources", "storage")) if data.dig("resources", "storage")
+
+          conf
+        end
+      end
+
+      attr_accessor :image, :workdir, :resources, :build_timeout, :network
+
+      def initialize
+        @image = nil
+        @workdir = "/app"
+        @resources = Resources.new(cpus: 2, memory: 2048, storage: 5120)
+        @build_timeout = 10 * 60
+        @network = NetworkPolicy.new
+      end
+    end
+  end
+end

data/lib/lemans/config/image_spec.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require "digest"
+require "pathname"
+
+module Lemans
+  class Config
+    # An image, either already published or built from a task's Dockerfile.
+    # Built images are named by content digest, so reuse is only ever of the identical thing.
+    class ImageSpec
+      attr_reader :reference, :dockerfile_path, :slug, :digest
+
+      def self.registry(reference) = new(reference:)
+
+      def self.dockerfile(path, slug:)
+        path = Pathname(path)
+        raise ConfigError, "no Dockerfile at #{path}" unless path.file?
+
+        new(dockerfile_path: path, slug:)
+      end
+
+      def initialize(reference: nil, dockerfile_path: nil, slug: nil)
+        @reference = reference
+        @dockerfile_path = dockerfile_path
+        @slug = slug
+
+        # Hashing the reference gives backends one answer to "is this the same image" either way.
+        @digest = built? ? TreeDigest.call(context_dir) : Digest::SHA256.hexdigest(reference.to_s)
+      end
+
+      def built? = !dockerfile_path.nil?
+
+      def context_dir = dockerfile_path&.dirname
+
+      def name
+        built? ? "lemans-#{digest[0, 32]}" : reference
+      end
+
+      def to_s = name
+    end
+  end
+end

data/lib/lemans/config/network_policy.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require "ipaddr"
+
+module Lemans
+  class Config
+    class NetworkPolicy # :nodoc:
+      MODES = %w[none allowlist public].freeze
+
+      MODES.each do |name|
+        define_method(:"#{name}?") { mode == name.to_s }
+      end
+
+      class << self
+        def from_config(data)
+          return if data.nil?
+
+          mode = (data["mode"] || "public").to_s
+          raise ConfigError, "#{mode.inspect} is not a network mode (#{MODES.join(", ")})" unless MODES.include?(mode)
+
+          hosts = data["hosts"]
+          raise ConfigError, "network.hosts is only meaningful with mode: allowlist" if mode != "allowlist" && hosts
+          raise ConfigError, "network.hosts must be provided with mode: allowlist" if mode == "allowlist" && Array(hosts).empty?
+
+          new(mode, hosts)
+        end
+      end
+
+      attr_reader :mode, :hosts
+
+      def initialize(mode = "public", hosts = nil)
+        @mode = mode.to_s
+        @hosts = hosts
+      end
+
+      # Backends allowlist domains and IP ranges through separate APIs.
+      def domains = partitioned_hosts.last
+
+      def ip_targets = partitioned_hosts.first
+
+      private
+
+      def partitioned_hosts
+        @partitioned_hosts ||= Array(hosts).partition { ip_target?(it) }
+      end
+
+      def ip_target?(entry)
+        IPAddr.new(entry)
+        true
+      rescue IPAddr::Error
+        false
+      end
+    end
+  end
+end

data/lib/lemans/config/revision.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require "open3"
+
+module Lemans
+  class Config
+    # Which revision of a bench produced a score.
+    # Obtain either from git or from ENV["COMMIT_SHA"] (lemans-remote convention)
+    class Revision
+      class << self
+        def detect(dir)
+          commit, dirty = ENV["COMMIT_SHA"]&.split("~")
+          return new(commit, dirty == "!") if commit
+
+          commit = git("rev-parse", "HEAD", dir:)
+          return new if commit.nil?
+
+          status = git("status", "--porcelain", dir:)
+          new(commit, status.nil? ? nil : !status.empty?)
+        end
+
+        private
+
+        def git(*, dir:)
+          output, status = Open3.capture2e("git", "-C", dir.to_s, *)
+          status.success? ? output.strip : nil
+        rescue SystemCallError
+          nil
+        end
+      end
+
+      attr_reader :commit, :dirty
+
+      def initialize(commit = nil, dirty = nil)
+        @commit = commit
+        @dirty = dirty
+      end
+
+      def as_json(**) = { commit:, dirty: }
+    end
+  end
+end

data/lib/lemans/config/setup.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Lemans
+  class Config
+    # One phase's preparation: the files uploaded into the sandbox and the
+    # commands run against them. A bare list is shorthand for commands only.
+    class Setup
+      class << self
+        def from_config(data, root:)
+          return if data.nil?
+
+          conf = new
+          case data
+          when Hash
+            conf.commands = Array(data["commands"])
+            conf.files = Array(data["files"]).map { file!(it, root:) }
+          else
+            conf.commands = Array(data)
+          end
+
+          conf
+        end
+
+        private
+
+        # Files are confined to the directory that declared them and must
+        # exist at load time.
+        def file!(path, root:)
+          raise ConfigError, "setup file #{path.inspect} must be relative to #{root}" if path.start_with?("/")
+
+          relative = Pathname(path).cleanpath
+          raise ConfigError, "setup file #{path.inspect} must name a file inside #{root}" if relative.each_filename.include?("..") || relative.to_s == "."
+          raise ConfigError, "setup file #{path} is not a file" unless root.join(relative).file?
+
+          [root.join(relative), relative.to_s]
+        end
+      end
+
+      # Files are [local absolute, remote relative] pairs; commands run in order.
+      attr_accessor :files, :commands
+
+      def initialize
+        @files = []
+        @commands = []
+      end
+
+      def merge(other)
+        dup.tap do |merged|
+          merged.files = files + other.files
+          merged.commands = commands + other.commands
+        end
+      end
+
+      def empty? = files.empty? && commands.empty?
+    end
+  end
+end