lemans 1.4.0 → 1.5.1

data/lib/lemans/agent.rb

@@ -15,6 +15,9 @@ module Lemans
 
     attr_reader :profile, :model
 
+    # Whether #run can go on from the history (raw result) of an interrupted run
+    def self.recoverable? = false
+
     def initialize(profile:, model: nil)
       @profile = profile
 
@@ -28,7 +31,7 @@ module Lemans
     def install(_task, _environment) = nil
 
     # Run the task.
-    def run(task, environment)
+    def run(task, environment, history: nil)
       raise NotImplementedError
     end
 

data/lib/lemans/agents/miniswen.rb

@@ -21,9 +21,13 @@ module Lemans
         cost_limit: :cost_ceiling_reached
       }.freeze
 
-      def run(task, environment)
-        run_result = obtain_result(task, environment)
+      def self.recoverable? = true
+
+      def run(task, environment, history: nil)
+        history &&= ::Miniswen::Agent::Result.from_h(JSON.parse(history))
+        run_result = obtain_result(task, environment, history)
         trajectory = trajectory_for(run_result)
+        raw_result = raw_result_for(run_result)
 
         # A failed model call is still an answer: the trial saves the
         # trajectory as evidence before failing.
@@ -39,16 +43,17 @@ module Lemans
 
       private
 
-      def obtain_result(task, environment)
+      def obtain_result(task, environment, history)
         agent = agent_for(environment)
         begin
-          agent.run(task.instruction)
+          agent.run(task.instruction, history:)
         rescue InfrastructureError, ::Miniswen::InfrastructureError => e
           agent.partial_result(e.message)
         end
       end
 
-      def raw_result = nil
+      # The run as miniswen records it, signatures included: what a recovery continues from
+      def raw_result_for(run_result) = JSON.generate(run_result.to_h)
 
       def agent_for(environment)
         raise ConfigError, "miniswen needs a model to drive" if model.to_s.empty?

data/lib/lemans/agents/miniswen_installed.rb

@@ -13,6 +13,7 @@ module Lemans
     class MiniswenInstalled < Miniswen
       NAME = "miniswen-installed"
       RESULTS_PATH = "/tmp/lemans-miniswen.result.json"
+      HISTORY_PATH = "/tmp/lemans-miniswen.history.json"
       INSTALL_TIMEOUT_SEC = 300
       # The CLI enforces max-time itself, but between steps only: a command
       # started just before the deadline runs to its own exec timeout first,
@@ -31,8 +32,9 @@ module Lemans
 
       # An in-sandbox run self-reports: everything but the verifier's reward
       # comes from a file the sandbox wrote.
-      def obtain_result(task, environment)
-        run = environment.exec(command_for(task), timeout: outer_timeout, env: provider_env(environment))
+      def obtain_result(task, environment, history)
+        upload_history(environment, history) if history
+        run = environment.exec(command_for(task, history:), timeout: outer_timeout, env: provider_env(environment))
 
         begin
           Tempfile.create(%w[miniswen .result.json]) do |file|
@@ -47,7 +49,7 @@ module Lemans
         end
       end
 
-      attr_reader :raw_result
+      def raw_result_for(_run_result) = @raw_result
 
       def outer_timeout = profile.timeout + profile.exec_timeout + EXEC_SLACK_SEC
 
@@ -64,9 +66,18 @@ module Lemans
         policy.mode == "allowlist" ? policy.domains : []
       end
 
-      def command_for(task)
+      def upload_history(environment, history)
+        Tempfile.create(%w[miniswen .history.json]) do |file|
+          file.write(JSON.generate(history.to_h))
+          file.flush
+          environment.upload(file.path, HISTORY_PATH)
+        end
+      end
+
+      def command_for(task, history: nil)
         argv = [ "miniswen", "-q", "--no-refresh-registry", "--jail",
-                "-m", model.to_s, "-p", task.instruction,
+                "-m", model.to_s,
+                *(history ? [ "--continue-from", HISTORY_PATH ] : [ "-p", task.instruction ]),
                 "--results-path", RESULTS_PATH,
                 "--max-steps", profile.step_limit, "--max-time", profile.timeout.to_i,
                 "--exec-timeout", profile.exec_timeout.to_i,

data/lib/lemans/agents/nop.rb

@@ -7,7 +7,7 @@ module Lemans
     class Nop < Agent
       NAME = "nop"
 
-      def run(_task, _environment)
+      def run(_task, _environment, history: nil)
         Response.new(outcome: Result::Outcome.new(:completed), usage: Result::Usage.zero)
       end
     end

data/lib/lemans/agents/oracle.rb

@@ -13,7 +13,7 @@ module Lemans
       ENTRYPOINT = "solve.sh"
       PATCH = "solution.patch"
 
-      def run(task, environment)
+      def run(task, environment, history: nil)
         files = task.solution_files
         if files.empty?
           raise ConfigError, "#{task.name}: no solution/ to run — the oracle has nothing to prove" if

data/lib/lemans/agents.rb

@@ -11,11 +11,13 @@ module Lemans
       "miniswen-installed" => "MiniswenInstalled"
     }.freeze
 
-    def self.build(name, profile:, model: nil)
+    def self.build(name, profile:, model: nil) = lookup(name).new(profile: profile, model: model)
+
+    def self.lookup(name)
       constant = REGISTRY[name] or
         raise ConfigError, "unknown agent #{name.inspect} (known: #{REGISTRY.keys.join(", ")})"
 
-      const_get(constant).new(profile: profile, model: model)
+      const_get(constant)
     end
   end
 end

data/lib/lemans/cli/regrade.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "json"
-require "prism"
 
 module Lemans
   class CLI < Thor
@@ -12,13 +11,19 @@ module Lemans
       ALLOWED = "fail (allowed)"
       CHECKS = "checks.json"
 
-      Mapping = Struct.new(:checks, :base_credit, :points, keyword_init: true) do
+      # A checks.json-shaped file: `checks`, `grading`, and the features as
+      # `features: { name => [checks] }`.
+      Mapping = Struct.new(:checks, :base_credit, :points, :features, :stray_features, keyword_init: true) do
         def self.from_json(data)
           checks = data["checks"] or raise ConfigError, "a mapping needs a `checks` section"
           grading = data["grading"] || {}
           declared = grading["points"] || {}
           allowed = checks.select { |_, status| status == ALLOWED }.keys
-          new(checks:, base_credit: grading["base_credit"], points: allowed.to_h { [ it, declared.fetch(it, 1) ] })
+          features = data["features"] || {}
+          unknown = features.values.flatten - checks.keys
+          raise ConfigError, "the mapping's features name unknown checks: #{unknown.inspect}" if unknown.any?
+
+          new(checks:, base_credit: grading["base_credit"], points: allowed.to_h { [ it, declared.fetch(it, 1) ] }, features:)
         end
 
         def names = checks.keys.sort
@@ -28,82 +33,18 @@ module Lemans
         def grading = { base_credit:, points: }.compact
       end
 
-      Change = Struct.new(:result, :reward, :credit, keyword_init: true)
-
-      # Reads the grading schema off the test file without running it: every
-      # `def test_*` and ActiveSupport `test "..."` is a check, an
-      # `allow_failure` call inside makes it an extra worth its `points:`.
-      class TestScanner < Prism::Visitor
-        attr_reader :tests, :points, :base_credit
-
-        def initialize
-          super
-          @scope = []
-          @tests = []
-          @points = {}
-          @current = nil
-        end
-
-        def visit_module_node(node) = scoped(node) { super }
-
-        def visit_class_node(node) = scoped(node) { super }
-
-        def visit_def_node(node)
-          return super unless node.name.start_with?("test_")
-
-          within("#{@scope.join("::")}##{node.name}") { super }
-        end
-
-        def visit_call_node(node)
-          case node.name
-          when :test
-            title = node.arguments&.arguments&.first
-            if node.receiver.nil? && node.block && title.is_a?(Prism::StringNode)
-              return within("#{@scope.join("::")}#test_#{title.unescaped.gsub(/\s+/, "_")}") { super }
-            end
-          when :allow_failure
-            @points[@current] ||= points_of(node) if @current
-          when :base_credit=
-            @base_credit = node.arguments.arguments.first.value if node.receiver.is_a?(Prism::ConstantReadNode) && node.receiver.name == :LemansReport
-          end
-          super
-        end
-
-        private
-
-        def scoped(node)
-          @scope.push(node.constant_path.full_name)
-          yield
-        ensure
-          @scope.pop
-        end
-
-        def within(check)
-          @tests << check
-          @current = check
-          yield
-        ensure
-          @current = nil
-        end
-
-        def points_of(node)
-          keywords = node.arguments&.arguments&.grep(Prism::KeywordHashNode)&.first
-          pair = keywords&.elements&.find { it.is_a?(Prism::AssocNode) && it.key.is_a?(Prism::SymbolNode) && it.key.unescaped == "points" }
-          pair ? pair.value.value : 1
-        end
-      end
+      Change = Struct.new(:result, :reward, :credit, :features, keyword_init: true)
 
       class << self
         def mapping_from_file(path) = Mapping.from_json(JSON.parse(File.read(path)))
 
         def mapping_for(task)
-          local, = task.test_files.find { |_, remote| remote == "verification_test.rb" }
-          raise ConfigError, "#{task.name} has no verification_test.rb to read the grading from" unless local
+          scanner = Trial::Verifier::TestScanner.for(task)
+          raise ConfigError, "#{task.name} has no verification_test.rb to read the grading from" unless scanner
 
-          scanner = TestScanner.new
-          Prism.parse_file(local.to_s).value.accept(scanner)
           checks = scanner.tests.to_h { [ it, scanner.points.key?(it) ? ALLOWED : "fail" ] }
-          Mapping.new(checks:, base_credit: scanner.base_credit, points: scanner.points)
+          Mapping.new(checks:, base_credit: scanner.base_credit, points: scanner.points,
+                      features: scanner.features, stray_features: scanner.stray_features)
         end
       end
 
@@ -115,8 +56,14 @@ module Lemans
         @mapping = mapping
       end
 
-      def results
-        @results ||= store.query(task:).select(&:scored?).sort_by { it.id.to_s }
+      def results = runs.select(&:scored?)
+
+      # Older multistep results lack the task's step count; returns the runs that gained it.
+      def record_total_steps!(total)
+        runs.select { it.steps && it.total_steps != total }.each do |result|
+          result.total_steps = total
+          store.save(result)
+        end
       end
 
       # A statically read mapping is only trusted once a stored checks.json
@@ -149,6 +96,8 @@ module Lemans
 
       private
 
+      def runs = @runs ||= store.query(task:).sort_by { it.id.to_s }
+
       def checks_of(result)
         raw = store.read_artifact(result, CHECKS)
         raw && JSON.parse(raw)
@@ -164,11 +113,14 @@ module Lemans
         updated[:grading] = mapping.grading unless mapping.grading.empty?
         reward = failures.empty? ? 1.0 : 0.0
         credit = credit_of(statuses, reward)
-        return if reward == result.reward && credit == result.credit && JSON.parse(JSON.generate(updated)) == checks
+        features = Trial::Verifier.features_from(statuses, mapping.features)
+        return if reward == result.reward && credit == result.credit && features == result.features &&
+                  JSON.parse(JSON.generate(updated)) == checks
 
         store.save_artifact(result, "#{JSON.pretty_generate(updated)}\n", path: CHECKS, force: true)
-        change = Change.new(result:, reward: [ result.reward, reward ], credit: [ result.credit, credit ])
-        store.save(result.graded!(reward, credit:))
+        change = Change.new(result:, reward: [ result.reward, reward ], credit: [ result.credit, credit ],
+                            features: [ result.features, features ])
+        store.save(result.graded!(reward, credit:, features:))
         change
       end
 

data/lib/lemans/cli/report/aggregate.rb

@@ -10,7 +10,7 @@ module Lemans
       # task, agent, model — "task-model" reads as two columns.
       class Aggregate
         KEYS = %i[task agent model].freeze
-        METRICS = %i[score credit time cost steps tokens].freeze
+        METRICS = %i[score credit features time cost steps tokens].freeze
         METRIC_SOURCES = { credit: :credit, time: :duration, cost: :cost_usd, steps: :steps, tokens: :tokens }.freeze
 
         attr_reader :report, :keys
@@ -32,32 +32,36 @@ module Lemans
                           .sort_by { |group| keys.map { group[it].to_s } }
         end
 
-        def order_by!(column)
-          column = Report.sort_column(column, allowed: keys + METRICS)
-          @groups =
-            if keys.include?(column)
-              Report.sort_rows(@groups) { it[column].to_s }
-            elsif column == :score
-              Report.sort_rows(@groups, descending: true) { [ Rational(it[:solved], it[:attempts]), it[:attempts] ] }
-            else
-              Report.sort_rows(@groups, descending: true) { it[METRIC_SOURCES.fetch(column)] }
-            end
+        def order_by!(spec)
+          sort_keys = Report.sort_columns(spec, allowed: keys + METRICS + report.feature_columns).map do |column, reversed|
+            [ ->(group) { sort_value(group, column) }, !keys.include?(column) != reversed ]
+          end
+          @groups = Report.sort_rows(@groups, sort_keys)
           self
         end
 
         def to_rows
-          metrics = report.fractional? ? METRICS : METRICS - [ :credit ]
-          [ keys.map(&:to_s) + metrics.map(&:to_s) ] +
+          metrics = METRICS - [ (:credit unless report.fractional?), (:features unless report.features?) ].compact
+          columns = keys + metrics + report.shown_feature_columns
+          columns -= Report.hidden_columns(report.hide_columns, allowed: columns)
+          [ columns.map(&:to_s) ] +
             @groups.map do |group|
-              keys.map { |key| display_key(key, group[key]) } + metrics.map { cell(it, group) }
+              columns.map do |column|
+                if keys.include?(column) then display_key(column, group[column])
+                elsif metrics.include?(column) then cell(column, group)
+                else pass_rate(group[column])
+                end
+              end
             end
         end
 
         def to_csv
-          columns = keys + %i[solved attempts credit duration cost_usd steps tokens]
+          columns = keys + %i[solved attempts credit features_passed features_total duration cost_usd steps tokens]
           CSV.generate do |csv|
-            csv << columns
-            @groups.each { |group| csv << columns.map { group[it] } }
+            csv << columns + report.shown_feature_columns
+            @groups.each do |group|
+              csv << columns.map { group[it] } + report.shown_feature_columns.map { group[it] && pass_rate(group[it]) }
+            end
           end
         end
 
@@ -67,6 +71,16 @@ module Lemans
 
         private
 
+        def sort_value(group, column)
+          if column == :model then Report.short_model(group[:model])
+          elsif keys.include?(column) then group[column].to_s
+          elsif report.feature_columns.include?(column) then group[column] && Rational(*group[column])
+          elsif column == :features then group[:features_total] && Rational(group[:features_passed], group[:features_total])
+          elsif column == :score then [ Rational(group[:solved], group[:attempts]), group[:attempts] ]
+          else group[METRIC_SOURCES.fetch(column)]
+          end
+        end
+
         # Attempts count every run; means and the median skip runs that never
         # measured the value, so one invalid trial cannot zero out a cell.
         def build(values, group)
@@ -77,18 +91,37 @@ module Lemans
             duration: median(group.filter_map { it[:duration] }),
             cost_usd: mean(group.filter_map { it[:cost_usd] }),
             steps: mean(group.filter_map { it[:steps] }),
-            tokens: mean(group.filter_map { it[:tokens] })
+            tokens: mean(group.filter_map { it[:tokens] }),
+            **features_sum(group),
+            **report.feature_columns.to_h { [ it, feature_tally(group, it) ] }
           )
         end
 
+        # Passed out of the runs that graded the feature; nil when none did
+        def feature_tally(group, column)
+          graded = group.map { Report.feature_of(it, column) }.reject(&:nil?)
+          [ graded.count(true), graded.size ] unless graded.empty?
+        end
+
+        def pass_rate(tally) = tally ? tally.join("/") : "-"
+
+        # Features passed out of graded, summed over the group's runs
+        def features_sum(group)
+          tallies = group.filter_map { Report.features_tally(it) }
+          return { features_passed: nil, features_total: nil } if tallies.empty?
+
+          { features_passed: tallies.sum(&:first), features_total: tallies.sum(&:last) }
+        end
+
         def cell(metric, group)
           case metric
           when :score then "#{group[:solved]}/#{group[:attempts]}"
           when :credit then mean_display(group[:credit], 2)
-          when :time then time(group[:duration])
-          when :cost then cost(group[:cost_usd])
+          when :features then group[:features_total] ? "#{group[:features_passed]}/#{group[:features_total]}" : "-"
+          when :time then Report.duration_display(group[:duration])
+          when :cost then Report.cost_display(group[:cost_usd])
           when :steps then mean_display(group[:steps], 1)
-          when :tokens then mean_display(group[:tokens], 0)
+          when :tokens then Report.tokens_display(group[:tokens])
           end
         end
 
@@ -108,15 +141,6 @@ module Lemans
           key == :model ? Report.short_model(value) : value.to_s
         end
 
-        def time(sec)
-          return "-" if sec.nil?
-
-          minutes, seconds = sec.round.divmod(60)
-          minutes.positive? ? "#{minutes}m #{seconds}s" : "#{seconds}s"
-        end
-
-        def cost(value) = value.nil? ? "-" : "$#{format("%g", value.round(4))}"
-
         def mean_display(value, digits) = value.nil? ? "-" : format("%g", value.round(digits))
       end
     end