llm-experiment 0.1.0

data/lib/llm_experiment/cli/build_command.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module LLMExperiment
+  class CLI
+    class BuildCommand < Command
+      NAME = "build"
+      SUMMARY = "build the base image or a per-app image"
+
+      def add_options(parser)
+        parser.banner = "Usage: llmx build base [options]\n       llmx build app KEY [options]"
+        parser.on("--no-cache", "rebuild every layer") { @no_cache = true }
+        parser.on("--tag TAG", "image tag (base only)") { |t| @tag = t }
+      end
+
+      def run(argv)
+        case argv.shift
+        when "base"
+          pins = begin
+            experiment.pins
+          rescue ConfigError
+            Pins.resolve
+          end
+          ImageBuilder::Base.new(container: container)
+                            .build(pins: pins, tag: @tag || LLMExperiment.base_image, no_cache: !!@no_cache)
+        when "app"
+          key = argv.shift or raise Error, "usage: llmx build app KEY"
+          ImageBuilder::App.new(experiment: experiment, container: container)
+                           .build(app: experiment.app(key), no_cache: !!@no_cache)
+        else
+          raise Error, "usage: llmx build base | llmx build app KEY"
+        end
+      end
+    end
+
+    register BuildCommand::NAME, BuildCommand
+  end
+end

data/lib/llm_experiment/cli/clean_command.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module LLMExperiment
+  class CLI
+    class CleanCommand < Command
+      NAME = "clean"
+      SUMMARY = "reclaim disk: app images and the BuildKit cache, verified by observation"
+
+      def add_options(parser)
+        parser.banner = <<~BANNER
+          Usage: llmx clean [--images] [--builder] [--base] [--dry-run]
+
+          With no flags this reports and deletes nothing.
+
+          The report measures the unpacked snapshot on disk, because that is what
+          an image costs: `container image inspect` sums layers and reports about
+          a third of it.
+
+          `--builder` verifies afterwards with `container ls -a` and prints the
+          kill-the-VM recovery if the builder is still there, because `container
+          builder stop` can exit 0 and do nothing.
+
+        BANNER
+        parser.on("--images", "delete every llmx-app-* image") { @images = true }
+        parser.on("--base", "also delete llmx-base (implies --images; 20-40 min to rebuild)") { @base = true }
+        parser.on("--builder", "stop the BuildKit builder and delete its cache") { @builder = true }
+        parser.on("--dry-run", "print what would go, delete nothing") { @dry_run = true }
+      end
+
+      def run(_argv)
+        cleaner = Cleaner.new(container: container)
+        images = @images || @base
+        return cleaner.report unless images || @builder
+
+        cleaner.clean_images(dry_run: !@dry_run.nil?, base: !@base.nil?) if images
+        cleaner.clean_builder(dry_run: !!@dry_run) if @builder
+      end
+    end
+
+    register CleanCommand::NAME, CleanCommand
+  end
+end

data/lib/llm_experiment/cli/doctor_command.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module LLMExperiment
+  class CLI
+    class DoctorCommand < Command
+      NAME = "doctor"
+      SUMMARY = "preflight checks: CLI, services, disk, auth, images, prompts"
+
+      MARKS = { ok: "ok", warn: "warn", fail: "FAIL" }.freeze
+
+      def run(_argv)
+        exp = begin
+          experiment
+        rescue ConfigError
+          nil
+        end
+        checks = Doctor.new(container: container).checks(exp)
+        checks.each do |check|
+          line = format("  %-5s %-22s %s", MARKS[check.status], check.name, check.detail)
+          line += "  -> #{check.hint}" if check.hint && check.status != :ok
+          puts line
+        end
+        puts exp ? "\nexperiment: #{exp.name} (#{exp.root})" : "\n(no experiment.yml found here; machine checks only)"
+        raise Error, "doctor found failures" if checks.any? { |c| c.status == :fail }
+      end
+    end
+
+    register DoctorCommand::NAME, DoctorCommand
+  end
+end

data/lib/llm_experiment/cli/login_command.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module LLMExperiment
+  class CLI
+    class LoginCommand < Command
+      NAME = "login"
+      SUMMARY = "log the agent CLIs in to your plan subscriptions"
+
+      def add_options(parser)
+        parser.banner = "Usage: llmx login [--agent NAME] [--verify]"
+        parser.on("--agent NAME", "log in one agent only (#{Auth::AGENTS.join(", ")})") { |a| @agent = a }
+        parser.on("--verify", "report what is logged in, log nothing in") { @verify = true }
+      end
+
+      def run(_argv)
+        auth = Auth.new(container: container)
+        return auth.verify if @verify
+
+        auth.login(agents: @agent ? [@agent] : Auth::AGENTS)
+      end
+    end
+
+    register LoginCommand::NAME, LoginCommand
+  end
+end

data/lib/llm_experiment/cli/metrics_command.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module LLMExperiment
+  class CLI
+    class MetricsCommand < Command
+      NAME = "metrics"
+      SUMMARY = "medians and exact Mann-Whitney U, never pooled"
+
+      def add_options(parser)
+        parser.banner = <<~BANNER
+          Usage: llmx metrics [options]
+
+          Reads every metrics.json written by `llmx parse` and reports one cell
+          per app and agent. Nothing is pooled across those: two agents do not
+          count tool calls the same way, and two apps are different sizes of
+          haystack, so a number averaged over them measures nothing.
+
+          p-values are exact. With a handful of tasks per cell the normal
+          approximation to the Mann-Whitney U is not trustworthy, so every
+          split of the pooled values is enumerated instead.
+
+        BANNER
+      end
+
+      def run(_argv)
+        MetricsReport.new(experiment: experiment).render
+      end
+    end
+
+    register MetricsCommand::NAME, MetricsCommand
+  end
+end

data/lib/llm_experiment/cli/new_command.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module LLMExperiment
+  class CLI
+    class NewCommand < Command
+      NAME = "new"
+      SUMMARY = "scaffold a new experiment directory"
+
+      def add_options(parser)
+        parser.banner = "Usage: llmx new NAME"
+      end
+
+      def run(argv)
+        target = argv.shift or raise Error, "usage: llmx new NAME"
+        Scaffold.new(target).create
+      end
+    end
+
+    register NewCommand::NAME, NewCommand
+  end
+end

data/lib/llm_experiment/cli/parse_command.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module LLMExperiment
+  class CLI
+    class ParseCommand < Command
+      NAME = "parse"
+      SUMMARY = "transcripts to events.jsonl and metrics.json"
+
+      ROW = "%-18s %-8s %-6s %6s %8s %7s %7s %6s"
+
+      def add_options(parser)
+        parser.banner = <<~BANNER
+          Usage: llmx parse TRIAL_DIR [TRIAL_DIR...]
+                 llmx parse --all
+
+          Reads each trial's transcript.jsonl and writes events.jsonl plus
+          metrics.json beside it. Re-running is safe: both files are derived.
+
+          ctx1st is the context the very first model request carried, before any
+          tool could have fetched anything. It is blank for Codex, which reports
+          usage once per turn rather than per request.
+
+        BANNER
+        parser.on("--all", "parse every trial under results-raw/") { @all = true }
+        parser.on("--lenient", "report contaminated trials without failing") { @lenient = true }
+      end
+
+      def run(argv)
+        reader = Transcript::Parser.new(experiment: experiment)
+        dirs = @all ? reader.trial_dirs : argv
+        raise Error, "llmx parse needs a trial directory, or --all" if dirs.empty?
+
+        rows = dirs.filter_map { |dir| reader.parse(dir) }
+        print_table(rows)
+        flagged = report_contamination(rows)
+        return if flagged.empty? || @lenient
+
+        # Exit non-zero rather than warn and succeed. A contaminated trial that
+        # only prints a message gets pooled with clean ones by whatever runs
+        # next, and in a 90-cell batch nobody re-reads the scrollback. Parsing
+        # still wrote every metrics.json, so --lenient recovers the old
+        # behaviour once the trials have been looked at.
+        raise Error, "#{flagged.size} contaminated trial(s); re-run them, or pass --lenient"
+      end
+
+      private
+
+      def print_table(rows)
+        puts format(ROW, "task", "agent", "cond", "tools", "ctx1st", "out", "wall", "fixed")
+        rows.each do |r|
+          puts format(ROW, r["task_id"], r["agent"], r["condition"],
+                      r["total_tool_calls"], r["context_before_first_tool_call"],
+                      r["output_tokens"], r["wall_seconds"], r["fix_verified"])
+        end
+      end
+
+      # Only Claude reports what it was given at startup, so in practice only
+      # Claude rows can be checked. Every row is still offered to the check
+      # rather than filtered by agent, so a future CLI that starts reporting
+      # its startup state is covered without anyone remembering to come back.
+      def report_contamination(rows)
+        flagged = rows.map { |r| [r, Transcript::Hermeticity.contamination(r)] }
+                      .reject { |(_, reasons)| reasons.empty? }
+        return flagged if flagged.empty?
+
+        warn "\nWARNING: #{flagged.size} trial(s) ran with state they should not have had:"
+        flagged.each do |(row, reasons)|
+          warn "  #{row["trial_dir"]}"
+          reasons.each { |reason| warn "    - #{reason}" }
+        end
+        flagged
+      end
+    end
+
+    register ParseCommand::NAME, ParseCommand
+  end
+end

data/lib/llm_experiment/cli/run_command.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module LLMExperiment
+  class CLI
+    class RunCommand < Command
+      NAME = "run"
+      SUMMARY = "run one trial, or the whole grid with --all"
+
+      def add_options(parser)
+        parser.banner = <<~BANNER
+          Usage: llmx run --task ID --agent NAME --condition NAME [options]
+                 llmx run --all [--app KEY] [--task ID] [--agent NAME] [options]
+
+          One trial is one task, one agent, one condition, in a fresh container.
+          --all runs the grid: sequentially, blocked by task, and resumable —
+          a cell that already has results is skipped unless --redo is given.
+
+        BANNER
+        parser.on("--task ID", "task id from experiment.yml (a filter under --all)") { |v| @task = v }
+        parser.on("--agent NAME", "agent to run (a filter under --all)") { |v| @agent = v }
+        parser.on("--condition NAME", "condition to run") { |v| @condition = v }
+        parser.on("--all", "run every remaining cell of the grid") { @all = true }
+        parser.on("--app KEY", "with --all: only this app's tasks") { |v| @app = v }
+        parser.on("--redo", "with --all: re-run cells that already have results") { @redo = true }
+        parser.on("--timeout S", Integer, "wall-clock seconds for the agent") { |v| @timeout = v }
+        parser.on("--dry-run", "show what would run, run nothing") { @dry_run = true }
+      end
+
+      def run(_argv)
+        return run_grid if @all
+
+        unless @task && @agent && @condition
+          raise Error, "llmx run needs --task, --agent and --condition (or --all for the grid)"
+        end
+
+        Trial.new(experiment: experiment, task_id: @task, agent: @agent, condition: @condition,
+                  container: container, timeout: @timeout).run(dry_run: !!@dry_run)
+      end
+
+      private
+
+      def run_grid
+        grid = Grid.new(experiment: experiment, container: container)
+        filters = { app: @app, agent: @agent, task: @task }
+        planned = grid.plan(**filters).size
+        todo = @redo ? planned : grid.pending(**filters).size
+        puts "grid: #{planned} cell(s) - #{todo} to run, #{planned - todo} already have results"
+        puts
+
+        result = grid.run(**filters, redo_done: !@redo.nil?, dry_run: !@dry_run.nil?) do |index, total, cell|
+          puts format("[%d/%d] %s", index, total, grid.label(cell))
+        end
+        report(result)
+      end
+
+      def report(result)
+        if @dry_run
+          puts "\n(dry run: nothing was executed)"
+          return
+        end
+
+        puts
+        puts "ran #{result.ran}/#{result.total} cell(s)"
+        return if result.ok?
+
+        puts "\n#{result.failures.size} failed:"
+        result.failures.each { |cell| puts "  #{cell[:task]} / #{cell[:agent]} / #{cell[:condition]}" }
+        raise Error, "#{result.failures.size} cell(s) failed; re-run `llmx run --all` to retry just those"
+      end
+    end
+
+    register RunCommand::NAME, RunCommand
+  end
+end

data/lib/llm_experiment/cli/sanitize_command.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module LLMExperiment
+  class CLI
+    class SanitizeCommand < Command
+      NAME = "sanitize"
+      SUMMARY = "gate results-raw/ into the committed results/"
+
+      def add_options(parser)
+        parser.banner = <<~BANNER
+          Usage: llmx sanitize [--check]
+
+          Copies each trial from results-raw/ into results/. Derived measurements
+          always cross; raw transcripts cross only for an app declared with
+          `publish_transcripts: true`, because a transcript contains whatever
+          source the agent read.
+
+          Anything that looks like a host path or a secret is a hard failure, not
+          a redaction: something unexpected in a transcript should stop the commit
+          and be looked at, not be quietly rewritten into something that looks safe.
+
+        BANNER
+        parser.on("--check", "report what would be refused, copy nothing") { @check = true }
+      end
+
+      def run(_argv)
+        sanitizer = Sanitizer.new(experiment: experiment)
+        @check ? sanitizer.check : sanitizer.publish
+      end
+    end
+
+    register SanitizeCommand::NAME, SanitizeCommand
+  end
+end

data/lib/llm_experiment/cli/shell_command.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module LLMExperiment
+  class CLI
+    class ShellCommand < Command
+      NAME = "shell"
+      SUMMARY = "interactive shell in a trial-shaped container, for debugging"
+
+      def add_options(parser)
+        parser.banner = "Usage: llmx shell [APP] [options]\n" \
+                        "       APP  open the app's image at /workspace/app; " \
+                        "omit for the base image"
+      end
+
+      def run(argv)
+        app_key = argv.shift
+        image = app_key ? experiment.app(app_key).image : LLMExperiment.base_image
+        container.ensure_system!
+        raise Error, "image #{image} not found; build it first" unless container.image?(image)
+
+        exp = app_key ? experiment : nil
+        argv_out = container.run_argv(
+          image, "bash",
+          memory: exp ? exp.memory : LLMExperiment.default_memory,
+          cpus: exp ? exp.cpus : LLMExperiment.default_cpus,
+          interactive: true,
+          workdir: app_key ? "/workspace/app" : nil
+        )
+        Shell.interactive(*argv_out)
+      end
+    end
+
+    register ShellCommand::NAME, ShellCommand
+  end
+end

data/lib/llm_experiment/cli/status_command.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module LLMExperiment
+  class CLI
+    class StatusCommand < Command
+      NAME = "status"
+      SUMMARY = "grid completion: which cells have results"
+
+      def add_options(parser)
+        parser.banner = <<~BANNER
+          Usage: llmx status [options]
+
+          One row per task, one column per agent x condition. A cell reads `done`
+          once it has a run that got as far as writing meta.json.
+
+          Trials run one at a time, on purpose: two at once share the machine and
+          contaminate wall_seconds. Expect the grid to take as long as the sum of
+          its cells, and resume it with `llmx run --all` whenever it is interrupted.
+
+        BANNER
+        parser.on("--app KEY", "only this app's tasks") { |v| @app = v }
+        parser.on("--task ID", "only this task") { |v| @task = v }
+        parser.on("--agent NAME", "only this agent") { |v| @agent = v }
+      end
+
+      def run(_argv)
+        cells = Grid.new(experiment: experiment)
+                    .status(app: @app, task: @task, agent: @agent)
+        print_table(cells)
+        print_totals(cells)
+      end
+
+      private
+
+      # Columns come from experiment.yml, not from the plan: the plan rotates
+      # the conditions per task, and a header that rotated with it would be
+      # unreadable.
+      def print_table(cells)
+        agents = experiment.agents.keys & cells.map { |c| c[:agent] }
+        columns = agents.product(experiment.conditions)
+        tasks = cells.map { |c| c[:task] }.uniq
+        done = cells.to_h { |c| [[c[:task], c[:agent], c[:condition]], c[:done]] }
+
+        headers = columns.map { |agent, condition| "#{agent}/#{condition}" }
+        widths = headers.map { |header| [header.length, 4].max }
+        first = [tasks.map(&:length).max, 4].max
+
+        puts row(["task".ljust(first)], headers, widths)
+        tasks.each do |task|
+          marks = columns.map { |agent, condition| done[[task, agent, condition]] ? "done" : "-" }
+          puts row([task.ljust(first)], marks, widths)
+        end
+      end
+
+      def row(lead, values, widths)
+        (lead + values.each_with_index.map { |value, i| value.ljust(widths[i]) }).join("  ").rstrip
+      end
+
+      def print_totals(cells)
+        finished = cells.count { |c| c[:done] }
+        puts
+        puts "#{finished}/#{cells.size} cell(s) have results; #{cells.size - finished} still to run."
+        puts "Run them with: llmx run --all"
+      end
+    end
+
+    register StatusCommand::NAME, StatusCommand
+  end
+end

data/lib/llm_experiment/cli/version_command.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module LLMExperiment
+  class CLI
+    class VersionCommand < Command
+      NAME = "version"
+      SUMMARY = "gem version and default toolchain pins"
+
+      def run(_argv)
+        puts "llm-experiment #{VERSION}"
+        Pins::DEFAULTS.each do |tool, pin|
+          puts format("  %-14s %s", tool, Array(pin).join(" "))
+        end
+      end
+    end
+
+    register VersionCommand::NAME, VersionCommand
+  end
+end

data/lib/llm_experiment/cli.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+require "optparse"
+
+module LLMExperiment
+  class CLI
+    COMMANDS = {} # rubocop:disable Style/MutableConstant -- command files register themselves at load time
+
+    def self.register(name, klass)
+      COMMANDS[name] = klass
+    end
+
+    def self.run(argv)
+      argv = argv.dup
+      name = argv.shift
+      name = "version" if %w[-v --version].include?(name)
+
+      if name.nil? || %w[-h --help help].include?(name)
+        print_help
+        return 0
+      end
+
+      klass = COMMANDS[name]
+      unless klass
+        warn "llmx: unknown command #{name.inspect}"
+        warn ""
+        capture_help_to_stderr
+        return 1
+      end
+
+      klass.new.call(argv)
+      0
+    rescue Error => e
+      warn "llmx: #{e.message}"
+      1
+    rescue Interrupt
+      130
+    end
+
+    def self.print_help(io = $stdout)
+      io.puts "llmx — run controlled, reproducible experiments on coding agents"
+      io.puts
+      io.puts "Usage: llmx COMMAND [options]"
+      io.puts
+      width = COMMANDS.keys.map(&:length).max
+      COMMANDS.sort.each do |name, klass|
+        io.puts format("  %-#{width}s  %s", name, klass.summary)
+      end
+      io.puts
+      io.puts "Run `llmx COMMAND --help` for details."
+    end
+
+    def self.capture_help_to_stderr
+      print_help($stderr)
+    end
+
+    # Base class for commands. Subclasses set NAME and SUMMARY, implement
+    # #run(argv), and optionally #add_options(parser).
+    class Command
+      def self.summary = self::SUMMARY
+
+      def initialize(container: nil)
+        @container = container
+        @experiment_dir = nil
+        @help_shown = false
+      end
+
+      def call(argv)
+        parser.parse!(argv)
+        return if @help_shown
+
+        run(argv)
+      end
+
+      def run(_argv)
+        raise NotImplementedError
+      end
+
+      private
+
+      def container
+        @container ||= Container.new
+      end
+
+      def parser
+        @parser ||= OptionParser.new do |o|
+          o.banner = "Usage: llmx #{self.class::NAME} [options]"
+          add_options(o)
+          o.on("--experiment DIR", "experiment directory (default: walk up to experiment.yml)") do |d|
+            @experiment_dir = d
+          end
+          o.on("--verbose", "echo every container command") { LLMExperiment.verbose = true }
+          o.on("-h", "--help", "show this help") do
+            puts o
+            @help_shown = true
+          end
+        end
+      end
+
+      def add_options(_parser); end
+
+      def experiment
+        @experiment ||= @experiment_dir ? Experiment.load(@experiment_dir) : Experiment.find
+      end
+    end
+  end
+end
+
+require_relative "cli/version_command"
+require_relative "cli/new_command"
+require_relative "cli/doctor_command"
+require_relative "cli/build_command"
+require_relative "cli/login_command"
+require_relative "cli/shell_command"
+require_relative "cli/run_command"
+require_relative "cli/status_command"
+require_relative "cli/parse_command"
+require_relative "cli/metrics_command"
+require_relative "cli/sanitize_command"
+require_relative "cli/clean_command"