llm-experiment 0.1.0

data/lib/llm_experiment/trial.rb

@@ -0,0 +1,204 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "json"
+require "shellwords"
+
+module LLMExperiment
+  # One trial: one task, one agent, one condition, in a fresh container.
+  #
+  # On caps: the trial runs to completion under a wall-clock timeout rather
+  # than being killed at the first edit. Stopping at the first edit would
+  # measure the cost of locating the defect but throw away whether the fix was
+  # any good, and an analysis cap can always be applied afterwards from the
+  # event log, whereas a killed trial cannot be un-killed. The transcript
+  # records the index of the first defect-file read and of the first edit, so a
+  # "cost to locate" figure survives without destroying anything.
+  #
+  # Results land in results-raw/, which is gitignored. Nothing reaches
+  # results/ until the sanitize gate has looked at it.
+  class Trial
+    # Both agents read the mounted credential store, so this asks a dead login
+    # the same question a trial would.
+    LOGIN_CHECKS = {
+      "claude" => ["claude auth status 2>&1", '"loggedIn": false'],
+      "codex" => ["codex login status 2>&1", "Not logged in"]
+    }.freeze
+
+    attr_reader :experiment, :task_id, :agent, :condition
+
+    def initialize(experiment:, task_id:, agent:, condition:, container: nil,
+                   timeout: nil, shell: Shell)
+      @experiment = experiment
+      @task_id = task_id.to_s
+      @agent = agent.to_s
+      @condition = condition.to_s
+      @container = container
+      @timeout = timeout
+      @shell = shell
+    end
+
+    def task = @task ||= experiment.task(task_id)
+    def app = @app ||= experiment.app(task.app)
+    def timeout_seconds = (@timeout || experiment.timeout_seconds).to_i
+    def prompt_path = experiment.prompt_path(task, condition)
+
+    # Pure: reads configuration and the prompt file, touches nothing else. The
+    # container never sees anything that is not in here.
+    def env
+      validate!
+      values = {
+        "LLMX_TASK_ID" => task.id,
+        "LLMX_APP" => app.key,
+        "LLMX_AGENT" => agent,
+        "LLMX_CONDITION" => condition,
+        "LLMX_BRANCH" => task.branch,
+        "LLMX_TEST_FILE" => task.test_file,
+        "LLMX_IMPL_FILES" => task.impl_files.join(","),
+        "LLMX_DB_PREPARE" => app.db_prepare,
+        "LLMX_DATABASE" => app.database,
+        "LLMX_TEST_COMMAND" => app.test_command,
+        "LLMX_SUITE_RAN_PATTERN" => app.suite_ran_pattern,
+        "LLMX_TIMEOUT_SECONDS" => timeout_seconds.to_s,
+        # Base64 keeps the prompt out of shell quoting entirely: newlines,
+        # backticks and quotes in a prompt must not become shell syntax.
+        # pack("m0") is strict_encode64 without needing the base64 gem, which
+        # Ruby 3.4 unbundled -- and the runner decodes it with unpack1("m0").
+        "LLMX_PROMPT_B64" => [prompt].pack("m0")
+      }
+      model = experiment.agents.dig(agent, "model")
+      values["LLMX_MODEL"] = model if model
+      values
+    end
+
+    def run(dry_run: false)
+      environment = env
+      return dry_run_report(environment) if dry_run
+
+      preflight!
+      out_dir = prepare_results_dir
+      argv = run_argv(out_dir, environment)
+
+      puts "trial #{task_id} / #{agent} / #{condition} -> #{out_dir}"
+      started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      ok = @shell.sh(*argv, allow_failure: true,
+                            log_as: run_argv(out_dir, redacted(environment)).shelljoin)
+      elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - started).round(1)
+
+      report(record_meta(out_dir, elapsed, ok), elapsed)
+      out_dir
+    end
+
+    private
+
+    def container = @container ||= Container.new
+
+    # Read once. The copy saved next to the results has to be the bytes the
+    # agent actually got, not whatever the file says a second later.
+    def prompt
+      @prompt ||= begin
+        path = prompt_path
+        raise Error, "missing prompt file #{path}; render the prompts first" unless File.exist?(path)
+
+        File.read(path)
+      end
+    end
+
+    def validate!
+      unless experiment.agents.key?(agent)
+        raise Error, "unknown agent #{agent.inspect}; experiment.yml declares #{experiment.agents.keys.join(", ")}"
+      end
+      return if experiment.conditions.include?(condition)
+
+      raise Error, "unknown condition #{condition.inspect}; experiment.yml declares #{experiment.conditions.join(", ")}"
+    end
+
+    # The prompt is the one environment value worth hiding from a log line:
+    # it is the whole task, base64-encoded, and it is already saved verbatim
+    # as prompt.txt beside the results.
+    def redacted(environment)
+      environment.merge("LLMX_PROMPT_B64" => "<#{environment["LLMX_PROMPT_B64"].bytesize} bytes>")
+    end
+
+    def run_argv(out_dir, environment)
+      container.run_argv(app.image, "exec ruby /results/runner.rb",
+                         memory: experiment.memory, cpus: experiment.cpus,
+                         volumes: ["#{out_dir}:/results"],
+                         env: environment, workdir: "/workspace/app")
+    end
+
+    def preflight!
+      container.ensure_system!
+      raise Error, "image #{app.image} not found; run `llmx build app #{app.key}`" unless container.image?(app.image)
+
+      check_login!
+    end
+
+    # Checked once up front rather than discovered halfway through a grid: a
+    # dead login turns every remaining trial into a wasted container.
+    def check_login!
+      command, dead = LOGIN_CHECKS[agent]
+      return unless command
+
+      status, = @shell.try(*container.run_argv(LLMExperiment.base_image, command,
+                                               memory: LLMExperiment.default_memory,
+                                               cpus: LLMExperiment.default_cpus))
+      return unless status.include?(dead)
+
+      raise Error, <<~MSG
+        #{agent} is not logged in inside the container, so this trial would fail.
+
+        Run this once and complete the browser flow:
+          llmx login --agent #{agent}
+
+        Credentials persist in #{LLMExperiment.auth_dir} and are mounted into every trial.
+      MSG
+    end
+
+    def prepare_results_dir
+      stamp = Time.now.utc.strftime("%Y%m%dT%H%M%SZ")
+      dir = experiment.trial_dir(task_id, agent, condition, stamp)
+      FileUtils.mkdir_p(dir)
+      # The runner is copied in rather than baked into the image, so fixing the
+      # harness does not mean rebuilding every app image.
+      FileUtils.cp(LLMExperiment.template_path("runner.rb"), File.join(dir, "runner.rb"))
+      File.write(File.join(dir, "prompt.txt"), prompt)
+      dir
+    end
+
+    def record_meta(out_dir, elapsed, ok)
+      path = File.join(out_dir, "meta.json")
+      unless File.exist?(path)
+        raise Error, "trial produced no meta.json after #{elapsed}s (container exit #{ok.inspect}). " \
+                     "Inspect #{out_dir}, or open a shell with `llmx shell #{app.key}`"
+      end
+
+      meta = JSON.parse(File.read(path))
+      meta["host_wall_seconds"] = elapsed
+      meta["container_ok"] = ok
+      # No result should ever be unattributable to the harness that produced it.
+      meta["llm_experiment_version"] = LLMExperiment::VERSION
+      File.write(path, JSON.pretty_generate(meta))
+      meta
+    end
+
+    def report(meta, elapsed)
+      puts format("\n  reproduced before : %s", meta["task_reproduces"])
+      puts format("  agent exit        : %s (timed out: %s)", meta["agent_exit"], meta["timed_out"])
+      puts format("  changed files     : %s", meta["changed_files"].inspect)
+      puts format("  touched impl file : %s", meta["touched_impl_file"])
+      puts format("  fix verified      : %s", meta["fix_verified"])
+      puts format("  wall seconds      : %s (container), %s (host)", meta["wall_seconds"], elapsed)
+    end
+
+    # A dry run starts no container and writes no results directory: it is for
+    # reading the prompt and the command back, not for rehearsing them.
+    def dry_run_report(environment)
+      out_dir = experiment.trial_dir(task_id, agent, condition, "<stamp>")
+      puts "would run:"
+      puts "  #{run_argv(out_dir, redacted(environment)).join(" ")}"
+      puts "\nprompt (#{condition}):\n\n#{prompt}"
+      out_dir
+    end
+  end
+end

data/lib/llm_experiment/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module LLMExperiment
+  VERSION = "0.1.0"
+end

data/lib/llm_experiment.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require_relative "llm_experiment/version"
+
+module LLMExperiment
+  class Error < StandardError; end
+  class ConfigError < Error; end
+  class ContainerError < Error; end
+
+  class << self
+    attr_accessor :base_image, :auth_dir, :agent_user,
+                  :default_memory, :default_cpus,
+                  :builder_cpus, :builder_memory, :min_free_gb,
+                  :container_data_dir, :verbose
+  end
+
+  # Every default can be overridden by an LLMX_* variable, matching the source
+  # harness, so existing setups keep working.
+  self.base_image = ENV.fetch("LLMX_BASE_IMAGE", "llmx-base:latest")
+  self.auth_dir = File.expand_path(ENV.fetch("LLMX_AUTH_DIR", "~/.llmx/auth"))
+  self.agent_user = "agent"
+  self.default_memory = ENV.fetch("LLMX_MEMORY", "6g")
+  self.default_cpus = ENV.fetch("LLMX_CPUS", "4")
+  self.builder_cpus = ENV.fetch("LLMX_BUILDER_CPUS", "6")
+  self.builder_memory = ENV.fetch("LLMX_BUILDER_MEMORY", "8G")
+  # Measured, not guessed: a campfire-sized Rails app image drove free space
+  # from 34 GB to 1.8 GB. About 24 GB, and roughly 17 GB of that was the
+  # BuildKit builder VM rather than the image itself. The previous 12 GB floor
+  # green-lit exactly that build.
+  self.min_free_gb = ENV.fetch("LLMX_MIN_FREE_GB", "25").to_i
+  # Where the CLI unpacks images. What an image costs is what lives here, not
+  # what its manifest sums to, so `llmx clean` plans against `du` on this path.
+  self.container_data_dir = File.expand_path(
+    ENV.fetch("LLMX_CONTAINER_DATA_DIR", "~/Library/Application Support/com.apple.container")
+  )
+
+  # `--verbose` echoes every container command. Off by default: a grid runs
+  # hundreds of them and the interesting output belongs to the agent.
+  self.verbose = ENV.key?("LLMX_VERBOSE")
+
+  def self.template_path(name)
+    File.expand_path(File.join(__dir__, "..", "templates", name))
+  end
+end
+
+# Below the module body on purpose. These files reference Error, ConfigError
+# and the accessors above, and a file that touches one of them at load time
+# rather than at call time would fail with an inscrutable NameError if it were
+# required first.
+require_relative "llm_experiment/shell"
+require_relative "llm_experiment/pins"
+require_relative "llm_experiment/container"
+require_relative "llm_experiment/auth"
+require_relative "llm_experiment/experiment"
+require_relative "llm_experiment/scaffold"
+require_relative "llm_experiment/doctor"
+require_relative "llm_experiment/image_builder/base"
+require_relative "llm_experiment/image_builder/app"
+require_relative "llm_experiment/trial"
+require_relative "llm_experiment/grid"
+require_relative "llm_experiment/transcript"
+require_relative "llm_experiment/transcript/claude"
+require_relative "llm_experiment/transcript/codex"
+require_relative "llm_experiment/transcript/hermeticity"
+require_relative "llm_experiment/transcript/parser"
+require_relative "llm_experiment/stats"
+require_relative "llm_experiment/metrics_report"
+require_relative "llm_experiment/sanitizer"
+require_relative "llm_experiment/cleaner"
+require_relative "llm_experiment/cli"

data/templates/README.md.erb

@@ -0,0 +1,26 @@
+# <%= name %>
+
+**Status: designed, not yet run.**
+
+## The question
+
+## The design
+
+## What gets measured
+
+## Running it
+
+```sh
+export LLMX_APP_SAMPLE=/path/to/your/checkout   # never committed
+llmx doctor
+llmx build base        # once per machine
+llmx login             # once, interactive
+llmx build app sample
+llmx run --all --dry-run
+llmx run --all
+llmx parse --all
+llmx metrics
+llmx sanitize
+```
+
+## Caveats

data/templates/base.Containerfile

@@ -0,0 +1,120 @@
+# Base image for LLM agent experiments.
+#
+# Gives every trial a freshly installed agent CLI with no host state: no user
+# CLAUDE.md, no MCP servers, no login. Credentials arrive at run time from a
+# mounted auth volume, never from this image.
+#
+# Built by `llmx build base`, which owns the version pins.
+
+FROM ubuntu:24.04
+
+ARG RUBY_VERSIONS="3.4.5 4.0.1"
+ARG NODE_VERSION="22"
+ARG CLAUDE_CODE_VERSION
+ARG CODEX_VERSION
+ARG OPENCODE_VERSION
+ARG AGENT_USER=agent
+ARG AGENT_UID=1001
+
+ENV DEBIAN_FRONTEND=noninteractive \
+    LANG=C.UTF-8 \
+    LC_ALL=C.UTF-8 \
+    TZ=UTC
+
+# System packages: Ruby build dependencies, native media libraries, SQLite,
+# PostgreSQL, and git for the fixture clone.
+RUN apt-get update && apt-get install -y --no-install-recommends \
+      ca-certificates curl wget git xz-utils unzip \
+      build-essential autoconf bison patch pkg-config \
+      libssl-dev libyaml-dev libreadline-dev zlib1g-dev libncurses-dev \
+      libffi-dev libgdbm-dev libdb-dev uuid-dev libgmp-dev \
+      libsqlite3-dev sqlite3 \
+      libpq-dev postgresql postgresql-contrib \
+      libvips42 libvips-tools ffmpeg \
+      libjemalloc2 tzdata procps less ripgrep jq \
+    && rm -rf /var/lib/apt/lists/*
+
+# mise manages the rubies and node. Installed into /opt so every user sees the
+# same toolchain and the shims work in non-login shells.
+ENV MISE_DATA_DIR=/opt/mise \
+    MISE_CONFIG_DIR=/opt/mise/config \
+    MISE_CACHE_DIR=/opt/mise/cache \
+    MISE_STATE_DIR=/opt/mise/state \
+    MISE_INSTALL_PATH=/usr/local/bin/mise \
+    MISE_YES=1
+ENV PATH=/opt/mise/shims:$PATH
+
+RUN curl -fsSL https://mise.run | sh && mise --version
+
+# Rubies are compiled here once so trials never pay for it.
+RUN set -eux; \
+    for v in ${RUBY_VERSIONS}; do mise install "ruby@${v}"; done; \
+    mise install "node@${NODE_VERSION}"; \
+    first_ruby="$(echo ${RUBY_VERSIONS} | awk '{print $1}')"; \
+    mise use -g "ruby@${first_ruby}" "node@${NODE_VERSION}"; \
+    mise reshim
+
+# Agent CLIs. Pinned by build_base.rb so a rebuild is reproducible.
+#
+# opencode needs its platform binary named explicitly. opencode-ai resolves its
+# real executable through optional per-platform packages, and npm picks the musl
+# build for this glibc image, then fails in the postinstall with "Unsupported
+# platform ... wanted libc musl, current glibc". Naming the package settles the
+# choice instead of leaving it to detection, and the architecture is read at
+# build time so this works on Apple silicon and on x86 alike.
+RUN set -eux; \
+    case "$(uname -m)" in \
+      aarch64|arm64) opencode_platform="opencode-linux-arm64" ;; \
+      x86_64|amd64)  opencode_platform="opencode-linux-x64" ;; \
+      *) echo "no opencode build for $(uname -m)" >&2; exit 1 ;; \
+    esac; \
+    npm install -g \
+      "@anthropic-ai/claude-code@${CLAUDE_CODE_VERSION}" \
+      "@openai/codex@${CODEX_VERSION}" \
+      "opencode-ai@${OPENCODE_VERSION}" \
+      "${opencode_platform}@${OPENCODE_VERSION}"; \
+    mise reshim
+
+# Never let a trial silently run a different CLI build than the one recorded.
+ENV DISABLE_AUTOUPDATER=1 \
+    DISABLE_TELEMETRY=1 \
+    DISABLE_ERROR_REPORTING=1 \
+    CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1 \
+    npm_config_update_notifier=false
+
+# The toolchain is owned by the agent user, not root. `gem install` triggers a
+# mise reshim, and a root-owned shims directory makes that fail noisily on every
+# app image build.
+RUN useradd --create-home --uid ${AGENT_UID} --shell /bin/bash ${AGENT_USER} \
+    && chown -R ${AGENT_USER}:${AGENT_USER} /opt/mise \
+    && mkdir -p /workspace /results \
+    && chown ${AGENT_USER}:${AGENT_USER} /workspace /results
+
+# Gems live outside the checkout so `git status` in a trial stays clean and the
+# agent's edits are the only diff.
+ENV BUNDLE_PATH=/home/${AGENT_USER}/bundle \
+    BUNDLE_JOBS=4 \
+    BUNDLE_RETRY=3 \
+    GEM_HOME=/home/${AGENT_USER}/gems \
+    PATH=/home/${AGENT_USER}/gems/bin:/opt/mise/shims:$PATH
+
+# Record exactly what got installed. Trials copy this into their metadata so a
+# result can always be traced back to a toolchain.
+RUN set -eux; \
+    printf '{\n' > /etc/llmx-versions.json; \
+    printf '  "image_built_at": "%s",\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> /etc/llmx-versions.json; \
+    printf '  "ubuntu": "%s",\n' "$(. /etc/os-release; echo $VERSION_ID)" >> /etc/llmx-versions.json; \
+    printf '  "arch": "%s",\n' "$(uname -m)" >> /etc/llmx-versions.json; \
+    printf '  "mise": "%s",\n' "$(mise --version | head -1)" >> /etc/llmx-versions.json; \
+    printf '  "rubies": "%s",\n' "${RUBY_VERSIONS}" >> /etc/llmx-versions.json; \
+    printf '  "node": "%s",\n' "$(node --version)" >> /etc/llmx-versions.json; \
+    printf '  "claude_code": "%s",\n' "$(claude --version 2>&1 | head -1)" >> /etc/llmx-versions.json; \
+    printf '  "codex": "%s",\n' "$(codex --version 2>&1 | head -1)" >> /etc/llmx-versions.json; \
+    printf '  "opencode": "%s",\n' "$(opencode --version 2>&1 | head -1)" >> /etc/llmx-versions.json; \
+    printf '  "postgres": "%s"\n' "$(/usr/lib/postgresql/*/bin/postgres --version | head -1)" >> /etc/llmx-versions.json; \
+    printf '}\n' >> /etc/llmx-versions.json; \
+    cat /etc/llmx-versions.json
+
+USER ${AGENT_USER}
+WORKDIR /workspace
+CMD ["bash"]

data/templates/experiment.yml.erb

@@ -0,0 +1,30 @@
+name: <%= name %>
+question: "State the question this experiment answers."
+
+agents:
+  claude:
+    model: claude-opus-5
+  codex:
+    model: gpt-5.6-sol
+
+conditions: [control, treatment]
+
+trial:
+  timeout_seconds: 900
+
+apps:
+  sample:
+    ruby: "3.4.5"
+    bundler: default
+    database: sqlite3
+    test_command: bin/rails test
+    db_prepare: bin/rails db:test:prepare
+    publish_transcripts: false
+    branch_prefix: exp/<%= name %>
+    neutralize: [CLAUDE.md, AGENTS.md, .mcp.json, .claude, .codex]
+
+tasks:
+  - id: sample-01
+    app: sample
+    test_file: test/models/replace_me_test.rb
+    impl_files: [app/models/replace_me.rb]

data/templates/gitignore

@@ -0,0 +1,3 @@
+results-raw/
+.apps.local
+.DS_Store