llm-experiment 0.1.0

data/lib/llm_experiment/image_builder/app.rb

@@ -0,0 +1,267 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "shellwords"
+
+module LLMExperiment
+  module ImageBuilder
+    # Builds a per-app image on top of the base image, so a trial starts instantly
+    # instead of paying for `bundle install` and database setup every time.
+    #
+    # The app arrives as a git bundle containing only the experiment branches. That
+    # matters: a bundle carries committed history and nothing else, so untracked
+    # secrets, stale local bundler overrides and repository hooks on the host can
+    # never reach the image.
+    #
+    # The image also strips the repository of anything that would let an agent read
+    # the answer instead of finding it. See FLATTENING below.
+    #
+    # BRANCH NAMING
+    #
+    # The source branches are not created here. The experiment's own planting
+    # script owns them, and this builder expects exactly these names:
+    #
+    #   <branch_prefix>/base       the app as it ships
+    #   <branch_prefix>/<task id>  one branch per task in experiment.yml
+    #
+    # `branch_prefix` comes from the app's entry in experiment.yml. Nothing else
+    # in the name is read, and nothing else may be encoded in it: the harness this
+    # was ported from named branches `exp/path-hints/bug-campfire-01-nat64-recheck`
+    # and the trailing slug handed the agent the answer. A missing branch is an
+    # error, never a silently skipped task.
+    #
+    # Images holding private code are never pushed to a registry.
+    class App
+      BUILD_ROOT = ENV.fetch("LLMX_BUILD_ROOT", "/tmp/llmx/build")
+
+      def initialize(experiment:, container: Container.new, shell: Shell)
+        @experiment = experiment
+        @container = container
+        @shell = shell
+      end
+
+      def build(app:, no_cache: false)
+        repo = app.repo_path
+        if repo.nil? || repo.empty?
+          raise ConfigError,
+                "LLMX_APP_#{app.key.upcase} is not set; point it at the #{app.key} checkout on this machine"
+        end
+
+        @container.ensure_disk!
+        @container.ensure_builder!
+        unless @container.image?(LLMExperiment.base_image)
+          raise Error, "base image #{LLMExperiment.base_image} not found; run llmx build base first"
+        end
+
+        context = File.join(BUILD_ROOT, app.key)
+        FileUtils.rm_rf(context)
+        FileUtils.mkdir_p(context)
+
+        raise ConfigError, "#{app.key}: no tasks in experiment.yml use this app" if tasks_for(app).empty?
+
+        map = branch_map(app)
+        require_branches!(app, repo, map)
+
+        bundle_path = File.join(context, "app.bundle")
+        @shell.log "bundling #{map.size} branch(es) from #{app.key}"
+        @shell.sh("git", "-C", repo, "bundle", "create", bundle_path, *map.keys)
+        # -C names the repository. `git bundle verify` resolves the bundle's
+        # prerequisites against a repository, and without -C it uses the working
+        # directory, which for an experiment directory is usually not one.
+        @shell.sh("git", "-C", repo, "bundle", "verify", bundle_path)
+
+        containerfile = File.join(context, "Containerfile")
+        File.write(containerfile, containerfile_for(app, map))
+
+        @shell.log "building #{app.image}"
+        @container.build(tag: app.image, file: containerfile, context: context,
+                         build_args: { "APP_KEY" => app.key, "RUBY_VERSION" => app.ruby },
+                         no_cache: no_cache)
+        @shell.log "app image ready: #{app.image} (never push: it carries the subject app's code)"
+      end
+
+      # FLATTENING
+      #
+      # Each experiment branch becomes one orphan commit called "Import application
+      # source", renamed to an opaque `trial/<task id>`.
+      #
+      # Three leaks close here, all of which would let an agent read the answer rather
+      # than search for it:
+      #
+      #   the commit message  said "reintroduce the defect fixed in <sha>"
+      #   the commit diff     was the real fix, inverted
+      #   the branch name     carried the defect's slug, e.g. -nat64-recheck
+      #
+      # Leaks like these do not just add noise, they bias: an agent given no path is
+      # far likelier to go digging through history than one handed the path outright,
+      # so the shortcut would help exactly the condition the experiment expects to be
+      # slowest. The trial runner closes the fourth leak by deleting every branch
+      # except the one under test, so `git diff trial/base` cannot reveal it either.
+      def branch_map(app)
+        map = { "#{app.branch_prefix}/base" => "trial/base" }
+        tasks_for(app).each { |task| map["#{app.branch_prefix}/#{task.id}"] = task.branch }
+        map
+      end
+
+      # Pure: builds the Containerfile text from the app and its branch map, so the
+      # generated recipe can be read and tested without a container or a checkout.
+      def containerfile_for(app, branch_map)
+        # Cloning a bundle materialises only the checked-out branch locally; the rest
+        # arrive as origin/* refs, which disappear along with the remote. Create real
+        # local branches first, then drop the remote.
+        localise = branch_map.keys.map { |b| "git branch -f #{b} origin/#{b} 2>/dev/null || true;" }
+                             .join(" \\\n      ")
+
+        flatten = branch_map.map do |source, target|
+          "git checkout -q #{source}; git checkout -q --orphan #{target}; " \
+            "#{neutralize_fragment(app)}" \
+            "git add -A; git commit -q -m 'Import application source';"
+        end.join(" \\\n      ")
+
+        drop_originals = branch_map.keys.map { |b| "git branch -q -D #{b};" }.join(" ")
+
+        containerfile = +<<~DOCKER
+          FROM #{LLMExperiment.base_image}
+
+          ARG APP_KEY
+          ARG RUBY_VERSION
+
+          ENV LLMX_APP=${APP_KEY} \\
+              MISE_RUBY_VERSION=${RUBY_VERSION} \\
+              RAILS_ENV=test
+
+          USER agent
+          WORKDIR /workspace
+
+          COPY --chown=agent:agent app.bundle /home/agent/app.bundle
+
+          RUN set -eux; \\
+              git clone --branch #{app.branch_prefix}/base /home/agent/app.bundle /workspace/app; \\
+              cd /workspace/app; \\
+              #{localise} \\
+              git remote remove origin; \\
+              git config user.email dev@example.invalid; \\
+              git config user.name Developer; \\
+              #{flatten} \\
+              #{drop_originals} \\
+              git checkout -q trial/base; \\
+              rm -f /home/agent/app.bundle; \\
+              git reflog expire --expire=now --all; \\
+              git gc --prune=now --quiet; \\
+              git branch; \\
+              test -z "$(git log --all --oneline --format='%s' | grep -viE '^Import application source$' || true)"
+
+          WORKDIR /workspace/app
+        DOCKER
+
+        containerfile << postgres_block if app.database == "postgresql"
+        containerfile << bundler_block(app)
+        containerfile
+      end
+
+      private
+
+      def tasks_for(app)
+        @experiment.tasks.select { |task| task.app == app.key }
+      end
+
+      def require_branches!(app, repo, map)
+        branches = @shell.capture("git", "-C", repo, "branch", "--list", "#{app.branch_prefix}/*",
+                                  "--format=%(refname:short)").split("\n").map(&:strip).reject(&:empty?)
+        missing = map.keys - branches
+        return if missing.empty?
+
+        raise Error, "#{app.key}: missing branches #{missing.inspect} in #{repo}; " \
+                     "the experiment's planting script has to create them first"
+      end
+
+      # Coaching files - CLAUDE.md, .mcp.json, .claude and friends - must never
+      # reach an image, whatever the planting script did or forgot to do. Deleting
+      # them inside the flatten sequence, after the orphan checkout and before the
+      # single commit, means no branch and no history can carry them back in.
+      def neutralize_fragment(app)
+        return "" if app.neutralize.empty?
+
+        "git rm -r -f -q --ignore-unmatch #{app.neutralize.map { |p| Shellwords.escape(p) }.join(" ")}; "
+      end
+
+      # A private cluster owned by the agent user, not the packaged system cluster.
+      # The system one needs root to start, root is not available mid-trial, and
+      # installing sudo just to reach it would put a privilege-escalation path in a
+      # container that runs a model's shell commands. initdb as `agent` makes agent
+      # the superuser and pg_ctl needs no privileges at all.
+      #
+      # The socket directory has to move with it. Postgres creates a Unix socket at
+      # startup even when every client speaks TCP, and its default
+      # /var/run/postgresql belongs to the postgres system user, so an agent-owned
+      # cluster dies on the spot with "could not create lock file ... Permission
+      # denied" - the one privilege the design set out to avoid needing. /tmp is
+      # agent-writable, and nothing here uses the socket anyway: PGHOST is
+      # 127.0.0.1 and listen_addresses matches.
+      def postgres_block
+        <<~DOCKER
+
+          ENV PGDATA=/home/agent/pgdata \\
+              PGHOST=127.0.0.1 \\
+              PGPORT=5432 \\
+              PGUSER=agent
+          RUN set -eux; \\
+              PGBIN=$(dirname $(ls /usr/lib/postgresql/*/bin/initdb | head -1)); \\
+              echo "export PATH=$PGBIN:\\$PATH" >> /home/agent/.bashrc; \\
+              $PGBIN/initdb -D $PGDATA -U agent --auth=trust --encoding=UTF8; \\
+              echo "listen_addresses = '127.0.0.1'" >> $PGDATA/postgresql.conf; \\
+              echo "unix_socket_directories = '/tmp'" >> $PGDATA/postgresql.conf; \\
+              echo "fsync = off" >> $PGDATA/postgresql.conf; \\
+              $PGBIN/pg_ctl -D $PGDATA -o "-p 5432" -w start; \\
+              $PGBIN/createdb -U agent agent; \\
+              $PGBIN/pg_ctl -D $PGDATA -w stop
+        DOCKER
+      end
+
+      def bundler_block(app)
+        <<~DOCKER
+
+          # Gems install to BUNDLE_PATH from the base image, which is outside the
+          # checkout, so `git status` during a trial shows only the agent's own edits.
+          #
+          # Bundler rewrites Gemfile.lock when the lockfile has no entry for the
+          # container's platform: these images are aarch64-linux, and an app locked only
+          # on darwin and x86_64-linux gains an aarch64-linux line during install. That
+          # leaves the tree dirty before any agent has touched it, so every trial would
+          # start with a modified Gemfile.lock and carry it in the agent's own diff.
+          #
+          # The lockfile change is committed rather than reverted. Reverting invites
+          # bundler to redo it mid-trial, which would dirty the tree at a point that
+          # actually matters. It is amended onto each branch's single commit so the
+          # "one commit called Import application source" property survives, and the
+          # working tree carries the same change across every checkout because the
+          # planted bugs only ever touch implementation files.
+          RUN set -eux; \\
+              cd /workspace/app; \\
+              ruby --version; \\
+              gem install bundler --no-document #{app.bundler == "default" ? "" : "-v #{app.bundler}"}; \\
+              bundle install; \\
+              if ! git diff --quiet; then \\
+                git add -A; \\
+                git commit -q --amend --no-edit; \\
+                for b in $(git branch --format='%(refname:short)'); do \\
+                  if [ "$b" != trial/base ]; then \\
+                    git checkout -q "$b"; \\
+                    git checkout -q trial/base -- Gemfile.lock; \\
+                    git add -A; \\
+                    git commit -q --amend --no-edit; \\
+                    test -z "$(git status --porcelain)"; \\
+                  fi; \\
+                done; \\
+                git checkout -q trial/base; \\
+                git reflog expire --expire=now --all; \\
+                git gc --prune=now --quiet; \\
+              fi; \\
+              git status --porcelain; \\
+              test -z "$(git status --porcelain)"; \\
+              test -z "$(git log --all --oneline --format='%s' | grep -viE '^Import application source$' || true)"
+        DOCKER
+      end
+    end
+  end
+end

data/lib/llm_experiment/image_builder/base.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module LLMExperiment
+  module ImageBuilder
+    # Builds and smoke-checks the base image. Compiles the rubies from source,
+    # so the first build is 15-30 minutes and every later one is cached.
+    class Base
+      def initialize(container: Container.new, shell: Shell)
+        @container = container
+        @shell = shell
+      end
+
+      # Compiling two rubies from source measured about 8 GB. Less than an app
+      # image, but it is the longest build in the tool, so losing it to a full
+      # disk costs the most.
+      NEED_GB = 10
+
+      def build(pins:, tag: LLMExperiment.base_image, no_cache: false)
+        @container.ensure_disk!(need_gb: NEED_GB)
+        @container.ensure_builder!
+        @shell.log "building #{tag} (compiles #{pins["ruby_versions"].join(" ")} from source; expect 15-30 min cold)"
+        @container.build(
+          tag: tag,
+          file: LLMExperiment.template_path("base.Containerfile"),
+          context: File.dirname(LLMExperiment.template_path("base.Containerfile")),
+          build_args: {
+            "RUBY_VERSIONS" => pins["ruby_versions"].join(" "),
+            "NODE_VERSION" => pins["node"],
+            "CLAUDE_CODE_VERSION" => pins["claude_code"],
+            "CODEX_VERSION" => pins["codex"],
+            "OPENCODE_VERSION" => pins["opencode"]
+          },
+          no_cache: no_cache
+        )
+        smoke_check(tag, pins)
+        @shell.log "base image ready: #{tag}"
+      end
+
+      private
+
+      def smoke_check(tag, pins)
+        @shell.log "smoke-checking the toolchain inside #{tag}"
+        script = <<~SH
+          set -e
+          echo "ruby:   $(ruby --version)"
+          echo "node:   $(node --version)"
+          echo "claude: $(claude --version)"
+          echo "codex:  $(codex --version)"
+          echo "opencode: $(opencode --version)"
+          echo "vips:   $(vips --version)"
+          echo "psql:   $(psql --version)"
+          for v in #{pins["ruby_versions"].join(" ")}; do
+            echo "ruby $v: $(mise exec ruby@$v -- ruby -e 'print RUBY_VERSION')"
+          done
+          cat /etc/llmx-versions.json
+        SH
+        @container.run(tag, script,
+                       memory: LLMExperiment.default_memory,
+                       cpus: LLMExperiment.default_cpus,
+                       mount_auth: false)
+      end
+    end
+  end
+end

data/lib/llm_experiment/metrics_report.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+require "json"
+
+module LLMExperiment
+  # Aggregates trial metrics and compares the conditions pairwise.
+  #
+  # Never pool across agents or apps. Claude and Codex do not count tool calls
+  # the same way, and two subject apps are different sizes of haystack. A number
+  # averaged over those is not a measurement of anything. So the rows are
+  # grouped by [app, agent] and each cell is reported on its own.
+  class MetricsReport
+    METRICS = %w[
+      wall_seconds
+      output_tokens
+      input_tokens
+      cache_creation_input_tokens
+      total_tool_calls
+      context_before_first_tool_call
+      tool_calls_to_first_defect_read
+    ].freeze
+
+    def initialize(experiment:)
+      @experiment = experiment
+    end
+
+    # One tree, never both. results-raw/ holds every trial; results/ holds the
+    # published subset, and for a public app those are the same files copied. A
+    # glob of "results*" matched both, so every published trial was counted
+    # twice -- which does not merely inflate n, it duplicates each observation,
+    # halves the apparent variance and pushes the exact p-values toward
+    # significance. Prefer the raw tree, which is always complete, and fall back
+    # to the published one so a fresh clone without results-raw/ still computes.
+    def files
+      @files ||= begin
+        raw = Dir.glob(File.join(@experiment.results_raw_dir, "**", "metrics.json"))
+        raw.empty? ? Dir.glob(File.join(@experiment.results_dir, "**", "metrics.json")) : raw
+      end
+    end
+
+    # A trial whose task never reproduced measured nothing: the agent was asked
+    # to fix something that was not broken.
+    def rows
+      @rows ||= files.sort.map { |f| JSON.parse(File.read(f)) }
+                     .reject { |r| r["task_reproduces"] == false }
+    end
+
+    def conditions = @experiment.conditions
+
+    def condition_pairs = @experiment.conditions.combination(2).to_a
+
+    def render(io = $stdout)
+      raise Error, "no metrics.json found in #{@experiment.root}. Run trials, then `llmx parse --all`" if files.empty?
+
+      io.puts "#{rows.size} trial(s) from #{files.size} file(s)"
+      report_undeclared_conditions(io)
+      io.puts
+      cells.each { |(app, agent), cell| render_cell(io, app, agent, cell) }
+      render_fix_verified(io)
+    end
+
+    private
+
+    # Only declared conditions are tabulated, so results from a condition the
+    # experiment has since retired are read and then left out. Say so: a run
+    # that prints "102 trials" and reports on 90 otherwise reads as complete.
+    def report_undeclared_conditions(io)
+      found = rows.map { |r| r["condition"] }.compact.uniq
+      undeclared = found - conditions
+      return if undeclared.empty?
+
+      undeclared.sort.each do |cond|
+        n = rows.count { |r| r["condition"] == cond }
+        io.puts "  excluded: #{n} trial(s) in condition #{cond.inspect}, " \
+                "which experiment.yml does not declare"
+      end
+    end
+
+    def cells
+      rows.group_by { |r| [r["app"], r["agent"]] }
+          .sort_by { |(app, agent), _| [app.to_s, agent.to_s] }
+    end
+
+    def render_cell(io, app, agent, cell)
+      by_condition = cell.group_by { |r| r["condition"] }
+      io.puts "=" * 78
+      io.puts "#{app} / #{agent}   (n per condition: #{conditions.map do |c|
+        "#{c}=#{(by_condition[c] || []).size}"
+      end.join(" ")})"
+      io.puts "=" * 78
+
+      METRICS.each { |metric| render_metric(io, by_condition, metric) }
+      io.puts
+    end
+
+    def render_metric(io, by_condition, metric)
+      series = conditions.to_h do |cond|
+        [cond, (by_condition[cond] || []).filter_map { |r| r[metric] }.map(&:to_f)]
+      end
+      return if series.values.all?(&:empty?)
+
+      io.puts "\n  #{metric}"
+      series.each do |cond, values|
+        next if values.empty?
+
+        io.puts format("    %-5s n=%-3d median=%-12s values=%s",
+                       cond, values.size, Stats.median(values).round(2), values.map { |v| v.round(1) }.inspect)
+      end
+
+      condition_pairs.each { |(left, right)| render_pair(io, series, left, right) }
+    end
+
+    def render_pair(io, series, left, right)
+      a = series[left]
+      b = series[right]
+      return if a.empty? || b.empty?
+
+      result = Stats.exact_mann_whitney(a, b)
+      p_text = result[:p] ? format("p=%.4f", result[:p]) : "p=#{result[:note]}"
+      delta = Stats.median(b) - Stats.median(a)
+      io.puts format("    %-5s vs %-5s  U=%-7s %-16s median change %+.2f",
+                     left, right, result[:u], p_text, delta)
+    end
+
+    def render_fix_verified(io)
+      io.puts "\nfix_verified by condition (not pooled):"
+      cells.each do |(app, agent), cell|
+        parts = conditions.map do |cond|
+          subset = cell.select { |r| r["condition"] == cond }
+          next "#{cond}=-" if subset.empty?
+
+          "#{cond}=#{subset.count { |r| r["fix_verified"] }}/#{subset.size}"
+        end
+        io.puts format("  %-18s %-8s %s", app, agent, parts.join("  "))
+      end
+    end
+  end
+end

data/lib/llm_experiment/pins.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module LLMExperiment
+  # Default version pins. An experiment overrides them in experiment.yml under
+  # `pins:`; a result can always be traced back to a toolchain.
+  module Pins
+    DEFAULTS = {
+      "ruby_versions" => ["3.4.5", "4.0.1"],
+      "node" => "22",
+      "claude_code" => "2.1.233",
+      "codex" => "0.147.0",
+      "opencode" => "1.18.15"
+    }.freeze
+
+    def self.resolve(overrides = {})
+      unknown = overrides.keys - DEFAULTS.keys
+      raise ConfigError, "unknown pins: #{unknown.join(", ")}" if unknown.any?
+
+      DEFAULTS.merge(overrides)
+    end
+  end
+end

data/lib/llm_experiment/sanitizer.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "json"
+
+module LLMExperiment
+  # The gate between results-raw/ (gitignored) and results/ (committed).
+  #
+  # A subject app may be private, so its raw transcripts cannot be published: a
+  # transcript contains whatever source the agent read. For those apps only
+  # derived measurements cross. An app marked `publish_transcripts: true` in
+  # experiment.yml crosses whole.
+  #
+  # The check is deliberately a hard failure rather than a redaction. Something
+  # unexpected in a transcript should stop the commit and be looked at, not be
+  # quietly rewritten into something that looks safe.
+  class Sanitizer
+    # Anything matching these must never reach the committed tree, whatever the
+    # app's privacy setting. Host paths identify the machine; the rest are secrets.
+    FORBIDDEN = {
+      "host home path" => %r{/Users/[a-z]},
+      "AWS-style key" => /AKIA[0-9A-Z]{16}/,
+      "private key block" => /-----BEGIN [A-Z ]*PRIVATE KEY-----/,
+      "bearer token" => /\b(?:sk|pk)-[A-Za-z0-9_-]{20,}/,
+      "rails master key" => /\b[0-9a-f]{32}\b(?=.*master)/i,
+      "generic api secret" => /(?:api[_-]?key|secret|password|token)["'\s:=]+[A-Za-z0-9_-]{24,}/i
+    }.freeze
+
+    # Derived measurements always cross. Raw transcripts only for public apps.
+    DERIVED = %w[meta.json metrics.json prompt.txt].freeze
+    RAW = %w[transcript.jsonl events.jsonl agent.diff test_before.txt test_after.txt].freeze
+
+    PRIVATE_README = <<~MD
+      Private application. Raw transcripts and diffs stay in `results-raw/`,
+      which is gitignored; only the measurements derived from them are here.
+
+      - `meta.json` what the trial did and whether the fix held
+      - `metrics.json` the numbers `llmx metrics` reads
+      - `prompt.txt` the exact prompt, which contains no application source
+    MD
+
+    def initialize(experiment:)
+      @experiment = experiment
+    end
+
+    def public_apps
+      @public_apps ||= @experiment.apps.values.select(&:publish_transcripts).map(&:key)
+    end
+
+    # Report what would be refused, copy nothing.
+    def check = run(check_only: true)
+
+    # Copy what may cross, or refuse and leave nothing behind.
+    def publish = run(check_only: false)
+
+    private
+
+    def run(check_only:)
+      dirs = trial_dirs
+      raise Error, "nothing in #{@experiment.results_raw_dir} yet" if dirs.empty?
+
+      violations = []
+      copied = 0
+
+      dirs.each do |dir|
+        crossing = crossing_files(dir)
+        violations.concat(scan_all(dir, crossing))
+        next if check_only
+
+        copied += copy(dir, crossing)
+      end
+
+      refuse(violations, check_only: check_only) if violations.any?
+      report(dirs.size, copied, check_only: check_only)
+    end
+
+    def trial_dirs
+      Dir.glob(File.join(@experiment.results_raw_dir, "**", "meta.json")).map { |p| File.dirname(p) }.sort
+    end
+
+    def public?(dir)
+      meta = JSON.parse(File.read(File.join(dir, "meta.json")))
+      public_apps.include?(meta["app"])
+    end
+
+    def crossing_files(dir)
+      public?(dir) ? DERIVED + RAW : DERIVED
+    end
+
+    def relative(dir) = dir.sub("#{@experiment.results_raw_dir}/", "")
+
+    def scan_all(dir, crossing)
+      crossing.flat_map do |name|
+        src = File.join(dir, name)
+        next [] unless File.exist?(src)
+
+        scan(File.read(src), File.join(relative(dir), name))
+      end
+    end
+
+    def scan(text, label)
+      FORBIDDEN.filter_map do |name, pattern|
+        next unless (m = text[pattern])
+
+        { "file" => label, "kind" => name, "sample" => m[0, 40] }
+      end
+    end
+
+    def copy(dir, crossing)
+      target = File.join(@experiment.results_dir, relative(dir))
+      FileUtils.mkdir_p(target)
+      copied = 0
+      crossing.each do |name|
+        src = File.join(dir, name)
+        next unless File.exist?(src)
+
+        FileUtils.cp(src, File.join(target, name))
+        copied += 1
+      end
+      File.write(File.join(target, "README.md"), PRIVATE_README) unless public?(dir)
+      copied
+    end
+
+    def refuse(violations, check_only:)
+      warn "REFUSING to publish. #{violations.size} problem(s):\n"
+      violations.first(40).each { |v| warn format("  %-56s %-18s %s", v["file"], v["kind"], v["sample"].inspect) }
+      unless check_only
+        warn "\nNothing was copied."
+        FileUtils.rm_rf(@experiment.results_dir)
+      end
+
+      raise Error, "#{violations.size} problem(s) in what would be published; nothing was published"
+    end
+
+    def report(trials, copied, check_only:)
+      if check_only
+        puts "checked #{trials} trial(s): clean"
+      else
+        puts "published #{copied} file(s) from #{trials} trial(s) to #{@experiment.results_dir}"
+        puts "raw transcripts committed only for: #{public_apps.empty? ? "(no app)" : public_apps.join(", ")}"
+      end
+    end
+  end
+end

data/lib/llm_experiment/scaffold.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require "erb"
+require "fileutils"
+
+module LLMExperiment
+  # `llmx new NAME`. Scaffolds the directory convention every other command
+  # assumes. Prompts and task generation stay experiment-owned.
+  class Scaffold
+    def initialize(target)
+      @target = File.expand_path(target)
+      @name = File.basename(@target)
+    end
+
+    def create
+      raise Error, "#{@target} already exists" if File.exist?(@target)
+
+      FileUtils.mkdir_p(@target)
+      %w[prompts results-raw results].each { |d| FileUtils.mkdir_p(File.join(@target, d)) }
+      write_template("experiment.yml.erb", "experiment.yml")
+      write_template("README.md.erb", "README.md")
+      FileUtils.cp(LLMExperiment.template_path("gitignore"), File.join(@target, ".gitignore"))
+      File.write(File.join(@target, "results-raw", ".gitkeep"), "")
+
+      puts "created #{@name}/"
+      puts "next: edit #{@name}/experiment.yml, then run `llmx doctor`"
+    end
+
+    private
+
+    def write_template(template, out)
+      erb = ERB.new(File.read(LLMExperiment.template_path(template)), trim_mode: "-")
+      File.write(File.join(@target, out), erb.result(template_binding))
+    end
+
+    # The templates read `name` from this binding. It is a parameter rather
+    # than a local, because a local assigned only for ERB's benefit reads as
+    # dead code to the parser and warns on every test run.
+    def template_binding(name = @name)
+      binding
+    end
+  end
+end