llm-experiment 0.1.0

data/lib/llm_experiment/container.rb

@@ -0,0 +1,178 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+module LLMExperiment
+  # Wraps the Apple `container` CLI. Every invocation of the CLI in the gem
+  # goes through here, so its quirks are handled in exactly one place.
+  class Container
+    def initialize(shell: Shell)
+      @shell = shell
+    end
+
+    def cli_version
+      out = @shell.capture("container", "--version", allow_failure: true).strip
+      out.empty? ? nil : out
+    end
+
+    def require_cli!
+      cli_version or raise ContainerError,
+                           "Apple `container` CLI not found. Install it, then run `container system start`."
+    end
+
+    def system_running?
+      @shell.capture("container", "system", "status", allow_failure: true)
+            .include?("apiserver is running")
+    end
+
+    def ensure_system!
+      require_cli!
+      return true if system_running?
+
+      @shell.log "starting container system"
+      @shell.sh("container", "system", "start")
+      true
+    end
+
+    def builder_status
+      ensure_system!
+      @shell.capture("container", "builder", "status", allow_failure: true)
+    end
+
+    # The builder is a separate VM with its own envelope (2 CPUs / 2 GB by
+    # default), and a starved builder dies mid-build with no error at all.
+    # `container builder status` prints one row:
+    #
+    #   ID        IMAGE     STATE    ADDR          CPUS  MEMORY
+    #   buildkit  ...:0.6.1  running  192.168.64.3  6     8192 MB
+    #
+    # Memory is reported in MB and configured in G, so the two are compared as
+    # numbers rather than strings.
+    BUILDER_ROW = /^buildkit\s.*\brunning\s+\S+\s+(\d+)\s+(\d+\s*[KMGT]?B?)/i
+
+    # "8G", "8g", "8GB", "8192 MB" and "8192MB" all mean the same envelope.
+    # A bare number is read as MB, which is how the CLI reports it.
+    def self.megabytes(text)
+      value, unit = text.to_s.strip.match(/\A(\d+)\s*([KMGT]?)B?\z/i)&.captures
+      return 0 unless value
+
+      case unit.to_s.upcase
+      when "G" then value.to_i * 1024
+      when "T" then value.to_i * 1024 * 1024
+      when "K" then value.to_i / 1024
+      else value.to_i
+      end
+    end
+
+    # A starved builder dies mid-build with no error, so an existing builder is
+    # only reused when it meets the whole envelope. Checking CPUs alone accepted
+    # a builder with 6 CPUs and 2 GB.
+    def ensure_builder!
+      if (row = builder_status.match(BUILDER_ROW))
+        cpus = row[1].to_i
+        memory_mb = self.class.megabytes(row[2])
+        want_cpus = LLMExperiment.builder_cpus.to_i
+        want_mb = self.class.megabytes(LLMExperiment.builder_memory)
+        return true if cpus >= want_cpus && memory_mb >= want_mb
+
+        @shell.log "builder has #{cpus} CPUs and #{memory_mb} MB; " \
+                   "restarting with #{want_cpus} CPUs and #{LLMExperiment.builder_memory}"
+        @shell.sh("container", "builder", "stop", allow_failure: true)
+      end
+
+      @shell.log "starting image builder (#{LLMExperiment.builder_cpus} CPUs, #{LLMExperiment.builder_memory})"
+      @shell.sh("container", "builder", "start",
+                "--cpus", LLMExperiment.builder_cpus,
+                "--memory", LLMExperiment.builder_memory)
+      true
+    end
+
+    # The subcommand is `container image list`; `container images` fails
+    # confusingly, which is why nothing else in the gem calls it directly.
+    def images
+      @shell.capture("container", "image", "list", allow_failure: true)
+            .lines.drop(1)
+            .filter_map do |line|
+              repo, tag = line.split
+              "#{repo}:#{tag}" if repo && tag
+            end
+    end
+
+    def image?(name)
+      repo, tag = name.split(":", 2)
+      tag ||= "latest"
+      images.include?("#{repo}:#{tag}")
+    end
+
+    def build(tag:, file:, context:, build_args: {}, no_cache: false)
+      cmd = ["container", "build", "--tag", tag, "--file", file]
+      build_args.each { |key, value| cmd += ["--build-arg", "#{key}=#{value}"] }
+      cmd << "--no-cache" if no_cache
+      cmd << context
+      @shell.sh(*cmd)
+    end
+
+    def delete_image(name)
+      @shell.sh("container", "image", "delete", name, allow_failure: true)
+    end
+
+    def containers_all
+      @shell.capture("container", "ls", "-a", allow_failure: true)
+    end
+
+    def free_gb
+      line = @shell.capture("df", "-g", "/System/Volumes/Data", allow_failure: true).lines.last.to_s
+      line.split[3].to_i
+    end
+
+    # Running out of disk mid-build does not fail cleanly: the build dies with
+    # no message, and on a truly full disk even writing the error fails.
+    def ensure_disk!(need_gb: LLMExperiment.min_free_gb)
+      free = free_gb
+      return true if free >= need_gb
+
+      raise ContainerError, <<~MSG
+        Only #{free} GB free; this build needs about #{need_gb} GB.
+        Most of that is usually the BuildKit builder VM rather than any image:
+          llmx clean              report what is on disk, delete nothing
+          llmx clean --builder    drop the builder cache, which is regenerable
+      MSG
+    end
+
+    # Arguments common to every trial container: resource envelope plus the
+    # credential store mount. The store is a seed, not a home — runners copy
+    # credentials out of it and never write back.
+    def run_args(memory:, cpus:, mount_auth: true)
+      args = ["--memory", memory, "--cpus", cpus.to_s]
+      if mount_auth
+        %w[claude codex opencode].each do |agent|
+          FileUtils.mkdir_p(File.join(LLMExperiment.auth_dir, agent))
+        end
+        home = "/home/#{LLMExperiment.agent_user}"
+        args += ["--volume", "#{LLMExperiment.auth_dir}:#{home}/.agent-auth"]
+        args += ["--env", "CLAUDE_CONFIG_DIR=#{home}/.agent-auth/claude"]
+        args += ["--env", "CODEX_HOME=#{home}/.agent-auth/codex"]
+        args += ["--env", "LLMX_OPENCODE_AUTH=#{home}/.agent-auth/opencode"]
+      end
+      args
+    end
+
+    # The full argv for one containerized command. Always through a login
+    # shell: `container run` does not apply the image's ENV PATH, and the
+    # rubies are mise shims.
+    def run_argv(image, script, memory:, cpus:, mount_auth: true,
+                 volumes: [], env: {}, workdir: nil, interactive: false)
+      argv = ["container", "run", "--rm"]
+      argv += ["--interactive", "--tty"] if interactive
+      argv += run_args(memory: memory, cpus: cpus, mount_auth: mount_auth)
+      volumes.each { |volume| argv += ["--volume", volume] }
+      env.each { |key, value| argv += ["--env", "#{key}=#{value}"] }
+      argv += ["--workdir", workdir] if workdir
+      argv + [image, "bash", "-lc", script]
+    end
+
+    def run(image, script, **options)
+      @shell.sh(*run_argv(image, script, **options))
+    end
+  end
+end

data/lib/llm_experiment/doctor.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+module LLMExperiment
+  # Preflight. Every check returns what is wrong AND what to type next,
+  # because each one encodes a failure that used to cost an afternoon.
+  class Doctor
+    Check = Data.define(:name, :status, :detail, :hint)
+
+    def initialize(container: Container.new)
+      @container = container
+    end
+
+    def checks(experiment = nil)
+      list = [cli_check]
+      # Without the CLI nothing else is answerable.
+      return list if list.first.status == :fail
+
+      list += [system_check, builder_check, disk_check, auth_check]
+      list += experiment_checks(experiment) if experiment
+      list
+    end
+
+    def ok?(experiment = nil)
+      checks(experiment).none? { |c| c.status == :fail }
+    end
+
+    private
+
+    def cli_check
+      version = @container.cli_version
+      if version
+        Check.new(name: "container CLI", status: :ok, detail: version, hint: nil)
+      else
+        Check.new(name: "container CLI", status: :fail,
+                  detail: "Apple `container` CLI not found",
+                  hint: "install it, then run `container system start`")
+      end
+    end
+
+    def system_check
+      if @container.system_running?
+        Check.new(name: "apiserver", status: :ok, detail: "running", hint: nil)
+      else
+        Check.new(name: "apiserver", status: :fail, detail: "not running",
+                  hint: "container system start")
+      end
+    end
+
+    def builder_check
+      status = @container.builder_status
+      if status.match?(/^buildkit\s.*\brunning\b/)
+        cpus = status[/\brunning\s+\S+\s+(\d+)/, 1].to_i
+        if cpus >= LLMExperiment.builder_cpus.to_i
+          Check.new(name: "builder", status: :ok, detail: "running with #{cpus} CPUs", hint: nil)
+        else
+          Check.new(name: "builder", status: :warn,
+                    detail: "running with only #{cpus} CPUs; native gem builds will die silently",
+                    hint: "llmx build will restart it with #{LLMExperiment.builder_cpus} CPUs")
+        end
+      else
+        Check.new(name: "builder", status: :warn, detail: "not running",
+                  hint: "started automatically on first build")
+      end
+    end
+
+    def disk_check
+      free = @container.free_gb
+      if free >= LLMExperiment.min_free_gb
+        Check.new(name: "disk space", status: :ok, detail: "#{free} GB free", hint: nil)
+      else
+        Check.new(name: "disk space", status: :fail,
+                  detail: "#{free} GB free; app image builds need ~#{LLMExperiment.min_free_gb} GB and die without an error message when disk runs out",
+                  hint: "llmx clean --builder")
+      end
+    end
+
+    def auth_check
+      # Dir.children, not a glob: the credential files are dotfiles
+      # (.credentials.json, .claude.json) and a "*" glob misses them.
+      seeded = %w[claude codex opencode].select do |agent|
+        dir = File.join(LLMExperiment.auth_dir, agent)
+        Dir.exist?(dir) && Dir.children(dir).any?
+      end
+      if seeded.any?
+        Check.new(name: "auth store", status: :ok,
+                  detail: "credentials for: #{seeded.join(", ")}", hint: nil)
+      else
+        Check.new(name: "auth store", status: :warn,
+                  detail: "#{LLMExperiment.auth_dir} is empty", hint: "llmx login")
+      end
+    end
+
+    def experiment_checks(experiment)
+      checks = []
+      checks << image_check("base image", LLMExperiment.base_image, "llmx build base")
+      experiment.apps.each_value do |app|
+        checks << image_check("app image #{app.key}", app.image, "llmx build app #{app.key}")
+        next unless app.repo_path.to_s.empty?
+
+        checks << Check.new(name: "checkout #{app.key}", status: :warn,
+                            detail: "LLMX_APP_#{app.key.upcase} is not set",
+                            hint: "needed only to build the app image")
+      end
+      missing = experiment.tasks.flat_map do |task|
+        experiment.conditions.reject { |c| File.exist?(experiment.prompt_path(task, c)) }
+                  .map { |c| "#{task.id}/#{c}" }
+      end
+      checks << if missing.empty?
+                  Check.new(name: "prompts", status: :ok, detail: "all rendered", hint: nil)
+                else
+                  Check.new(name: "prompts", status: :warn,
+                            detail: "missing: #{missing.first(5).join(", ")}#{missing.size > 5 ? " (+#{missing.size - 5})" : ""}",
+                            hint: "render your prompts before `llmx run`")
+                end
+      checks
+    end
+
+    def image_check(name, image, hint)
+      if @container.image?(image)
+        Check.new(name: name, status: :ok, detail: image, hint: nil)
+      else
+        Check.new(name: name, status: :warn, detail: "#{image} not built", hint: hint)
+      end
+    end
+  end
+end

data/lib/llm_experiment/experiment.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+require "psych"
+
+module LLMExperiment
+  App = Data.define(:key, :ruby, :bundler, :database, :test_command, :db_prepare,
+                    :suite_ran_pattern, :publish_transcripts, :branch_prefix, :neutralize) do
+    # Host checkout paths never live in experiment.yml; they identify the
+    # machine and belong in the environment (.apps.local convention).
+    def repo_path
+      ENV["LLMX_APP_#{key.upcase}"]
+    end
+
+    def image
+      "llmx-app-#{key}:latest"
+    end
+  end
+
+  Task = Data.define(:id, :app, :test_file, :impl_files) do
+    def branch
+      "trial/#{id}"
+    end
+  end
+
+  # experiment.yml, loaded and validated. Unknown keys are errors: a typoed
+  # option that silently does nothing is how experiments measure the wrong thing.
+  class Experiment
+    TOP_KEYS = %w[name question pins agents conditions trial apps tasks].freeze
+    APP_KEYS = %w[ruby bundler database test_command db_prepare suite_ran_pattern
+                  publish_transcripts branch_prefix neutralize].freeze
+    TASK_KEYS = %w[id app test_file impl_files].freeze
+    TRIAL_KEYS = %w[timeout_seconds memory cpus].freeze
+    AGENT_KEYS = %w[model].freeze
+    KNOWN_AGENTS = %w[claude codex].freeze
+    DATABASES = %w[sqlite3 postgresql].freeze
+    DEFAULT_SUITE_RAN = '\d+ runs?, \d+ assertions?'
+
+    attr_reader :root, :name, :question, :pins, :agents, :conditions,
+                :apps, :tasks, :timeout_seconds, :memory, :cpus
+
+    def self.find(start = Dir.pwd)
+      dir = File.expand_path(start)
+      until File.exist?(File.join(dir, "experiment.yml"))
+        parent = File.dirname(dir)
+        raise ConfigError, "no experiment.yml found in #{start} or any parent" if parent == dir
+
+        dir = parent
+      end
+      load(dir)
+    end
+
+    def self.load(dir)
+      path = File.join(dir, "experiment.yml")
+      raise ConfigError, "missing #{path}" unless File.exist?(path)
+
+      new(Psych.safe_load(File.read(path)) || {}, root: File.expand_path(dir))
+    end
+
+    def initialize(data, root:)
+      @root = root
+      reject_unknown!("experiment.yml", data, TOP_KEYS)
+      @name = data["name"] or raise ConfigError, "experiment.yml needs a name"
+      @question = data["question"].to_s
+      @pins = Pins.resolve(data["pins"] || {})
+      @agents = parse_agents(data["agents"] || {})
+      @conditions = Array(data["conditions"]).map(&:to_s)
+      raise ConfigError, "conditions must not be empty" if @conditions.empty?
+
+      trial = data["trial"] || {}
+      reject_unknown!("trial", trial, TRIAL_KEYS)
+      @timeout_seconds = (trial["timeout_seconds"] || 900).to_i
+      @memory = trial["memory"] || LLMExperiment.default_memory
+      @cpus = (trial["cpus"] || LLMExperiment.default_cpus).to_s
+
+      @apps = parse_apps(data["apps"] || {})
+      @tasks = parse_tasks(data["tasks"] || [])
+    end
+
+    def app(key)
+      @apps.fetch(key) { raise ConfigError, "unknown app: #{key}" }
+    end
+
+    def task(id)
+      @tasks.find { |t| t.id == id } or raise ConfigError, "unknown task: #{id}"
+    end
+
+    def prompt_path(task, condition)
+      File.join(root, "prompts", task.app, task.id, "#{condition}.txt")
+    end
+
+    def results_raw_dir = File.join(root, "results-raw")
+    def results_dir = File.join(root, "results")
+
+    def trial_dir(task_id, agent, condition, stamp)
+      File.join(results_raw_dir, task_id, agent, condition, stamp)
+    end
+
+    private
+
+    def reject_unknown!(where, hash, known)
+      unknown = hash.keys - known
+      raise ConfigError, "unknown keys in #{where}: #{unknown.join(", ")}" if unknown.any?
+    end
+
+    def parse_agents(data)
+      raise ConfigError, "agents must not be empty" if data.empty?
+
+      unknown = data.keys - KNOWN_AGENTS
+      raise ConfigError, "unknown agents: #{unknown.join(", ")} (supported: #{KNOWN_AGENTS.join(", ")})" if unknown.any?
+
+      data.to_h do |key, value|
+        value ||= {}
+        reject_unknown!("agents.#{key}", value, AGENT_KEYS)
+        [key, { "model" => value["model"] }]
+      end
+    end
+
+    def parse_apps(data)
+      raise ConfigError, "apps must not be empty" if data.empty?
+
+      data.to_h do |key, value|
+        reject_unknown!("apps.#{key}", value, APP_KEYS)
+        unless DATABASES.include?(value["database"])
+          raise ConfigError,
+                "apps.#{key}: database must be one of #{DATABASES.join(", ")}"
+        end
+
+        [key, App.new(
+          key: key,
+          ruby: value.fetch("ruby") { raise ConfigError, "apps.#{key} needs ruby" }.to_s,
+          bundler: (value["bundler"] || "default").to_s,
+          database: value["database"],
+          test_command: value["test_command"] || "bin/rails test",
+          db_prepare: value["db_prepare"] || "bin/rails db:test:prepare",
+          suite_ran_pattern: value["suite_ran_pattern"] || DEFAULT_SUITE_RAN,
+          publish_transcripts: value["publish_transcripts"] == true,
+          branch_prefix: value.fetch("branch_prefix") { raise ConfigError, "apps.#{key} needs branch_prefix" },
+          neutralize: Array(value["neutralize"])
+        )]
+      end
+    end
+
+    def parse_tasks(data)
+      raise ConfigError, "tasks must not be empty" if data.empty?
+
+      ids = data.map { |t| t["id"] }
+      dupes = ids.tally.select { |_, n| n > 1 }.keys
+      raise ConfigError, "duplicate task ids: #{dupes.join(", ")}" if dupes.any?
+
+      data.map do |value|
+        reject_unknown!("task #{value["id"]}", value, TASK_KEYS)
+        raise ConfigError, "task #{value["id"]}: unknown app #{value["app"]}" unless @apps.key?(value["app"])
+
+        Task.new(
+          id: value.fetch("id"),
+          app: value.fetch("app"),
+          test_file: value.fetch("test_file") { raise ConfigError, "task #{value["id"]} needs test_file" },
+          impl_files: Array(value["impl_files"])
+        )
+      end
+    end
+  end
+end

data/lib/llm_experiment/grid.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+module LLMExperiment
+  # The grid of trials: every task x every agent x every condition.
+  #
+  # Order is load-bearing.
+  #
+  # Blocked by task, because the comparison is *within* a task: all of one
+  # task's cells run close together, so a slow patch of machine time lands on
+  # every condition of that task rather than on one of them.
+  #
+  # Condition order rotates with the task index, so no condition always runs
+  # first and none always runs last.
+  #
+  # Sequential on purpose. Two trials at once share the machine and contaminate
+  # wall_seconds — that happened in the source experiment and made a set of
+  # timings unusable. Never add parallelism here.
+  #
+  # Resumable, because a cell that already has a meta.json is done and is
+  # skipped, and because one bad cell must not cost the other 89: a failure is
+  # recorded and the run carries on.
+  class Grid
+    Result = Data.define(:total, :failures) do
+      def ran = total - failures.size
+      def ok? = failures.empty?
+    end
+
+    def initialize(experiment:, container: nil)
+      @experiment = experiment
+      @container = container
+    end
+
+    def plan(app: nil, agent: nil, task: nil)
+      tasks = filtered_tasks(app: app, task: task)
+      agents = filtered_agents(agent)
+      conditions = @experiment.conditions
+
+      tasks.each_with_index.flat_map do |subject, index|
+        ordered = conditions.rotate(index % conditions.size)
+        agents.flat_map do |name|
+          ordered.map { |condition| { task: subject.id, agent: name, condition: condition } }
+        end
+      end
+    end
+
+    # A cell is done once it has at least one run that got as far as writing
+    # meta.json. Anything earlier than that is an aborted attempt, not a result.
+    def done?(task:, agent:, condition:)
+      Dir.glob(File.join(@experiment.results_raw_dir, task, agent, condition, "*", "meta.json")).any?
+    end
+
+    def pending(**filters)
+      plan(**filters).reject { |cell| done?(**cell) }
+    end
+
+    def status(**filters)
+      plan(**filters).map { |cell| cell.merge(done: done?(**cell)) }
+    end
+
+    def run(app: nil, agent: nil, task: nil, redo_done: false, dry_run: false)
+      filters = { app: app, agent: agent, task: task }
+      cells = redo_done ? plan(**filters) : pending(**filters)
+      failures = []
+
+      # One trial at a time. See the note on this class.
+      cells.each_with_index do |cell, index|
+        yield(index + 1, cells.size, cell) if block_given?
+        # A grid dry run is about the order and the count. To inspect one
+        # command and its prompt, dry-run that single trial instead.
+        next if dry_run
+
+        begin
+          trial(cell).run
+        rescue StandardError => e
+          # Record and carry on: the grid is resumable, so a failed cell is
+          # simply re-run later, and aborting here would waste every cell after it.
+          failures << cell.merge(error: e.message)
+          Shell.log "FAILED: #{label(cell)}: #{e.message.lines.first.to_s.strip}"
+        end
+      end
+
+      Result.new(total: cells.size, failures: failures)
+    end
+
+    def label(cell) = format("%s / %s / %s", cell[:task], cell[:agent], cell[:condition])
+
+    private
+
+    def trial(cell)
+      Trial.new(experiment: @experiment, task_id: cell[:task], agent: cell[:agent],
+                condition: cell[:condition], container: @container)
+    end
+
+    def filtered_tasks(app:, task:)
+      tasks = @experiment.tasks
+      tasks = tasks.select { |t| t.app == app } if app
+      tasks = tasks.select { |t| t.id == task } if task
+      raise Error, "no tasks match #{{ app: app, task: task }.compact.inspect}" if tasks.empty?
+
+      tasks
+    end
+
+    def filtered_agents(agent)
+      known = @experiment.agents.keys
+      return known unless agent
+
+      wanted = Array(agent).map(&:to_s)
+      unknown = wanted - known
+      if unknown.any?
+        raise Error,
+              "unknown agent(s): #{unknown.join(", ")} (experiment.yml declares #{known.join(", ")})"
+      end
+
+      wanted
+    end
+  end
+end