llm-experiment 0.1.0

data/lib/llm_experiment/shell.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require "open3"
+require "shellwords"
+
+module LLMExperiment
+  # Runs commands. Kept as one small module so container-facing classes can
+  # take a `shell:` argument and tests can pass a recorder instead.
+  module Shell
+    module_function
+
+    def log(message)
+      warn "[llmx] #{message}"
+    end
+
+    # Command echo, as opposed to progress. Every container invocation goes
+    # through here, which is a wall of text during a grid, so it is off unless
+    # asked for. Progress messages use `log` and always print.
+    def log_command(message)
+      log(message) if LLMExperiment.verbose
+    end
+
+    # Run a command, stream its output, raise unless it succeeds.
+    #
+    # `log_as` replaces what gets echoed, for a command carrying an argument
+    # nobody wants in their scrollback: a trial passes its whole prompt as one
+    # base64 environment value, which is kilobytes of noise on every run.
+    def sh(*cmd, allow_failure: false, log_as: nil)
+      log_command(log_as || cmd.shelljoin)
+      ok = system(*cmd)
+      return ok if ok || allow_failure
+
+      raise Error, "command failed: #{log_as || cmd.shelljoin}"
+    end
+
+    # Hand the terminal to a command (login device flows need a real TTY).
+    def interactive(*cmd)
+      system(*cmd)
+    end
+
+    # Run a command and capture stdout. Raises unless it succeeds.
+    def capture(*cmd, allow_failure: false)
+      out, err, status = Open3.capture3(*cmd)
+      raise Error, "command failed: #{cmd.shelljoin}\n#{err}" unless status.success? || allow_failure
+
+      out
+    end
+
+    # Run a command and report whether it worked.
+    def try(*cmd)
+      out, err, status = Open3.capture3(*cmd)
+      [out + err, status.success?]
+    end
+
+    def try_input(input, *cmd)
+      out, err, status = Open3.capture3(*cmd, stdin_data: input)
+      [out + err, status.success?]
+    end
+  end
+end

data/lib/llm_experiment/stats.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module LLMExperiment
+  # Exact, not approximate. With five tasks per cell the normal approximation to
+  # the Mann-Whitney U is not trustworthy, and the whole permutation set is
+  # small enough to enumerate: C(10,5) is 252. So the p-values here are counted,
+  # not estimated.
+  module Stats
+    module_function
+
+    def median(values)
+      return nil if values.empty?
+
+      sorted = values.sort
+      mid = sorted.size / 2
+      sorted.size.odd? ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2.0
+    end
+
+    # Mid-ranks, so ties do not silently inflate the statistic.
+    def ranks(values)
+      indexed = values.each_with_index.sort_by { |v, _| v }
+      result = Array.new(values.size)
+      i = 0
+      while i < indexed.size
+        j = i
+        j += 1 while j + 1 < indexed.size && indexed[j + 1][0] == indexed[i][0]
+        mid = ((i + 1) + (j + 1)) / 2.0
+        (i..j).each { |k| result[indexed[k][1]] = mid }
+        i = j + 1
+      end
+      result
+    end
+
+    # U for sample a against sample b.
+    def u_statistic(a, b)
+      all = a + b
+      r = ranks(all)
+      rank_sum_a = r.first(a.size).sum
+      rank_sum_a - (a.size * (a.size + 1) / 2.0)
+    end
+
+    # Exact two-sided p: enumerate every way to split the pooled values into two
+    # samples of the observed sizes and count the splits at least as extreme.
+    def exact_mann_whitney(a, b)
+      n1 = a.size
+      n2 = b.size
+      return { u: nil, p: nil, note: "empty sample" } if n1.zero? || n2.zero?
+
+      total = n1 + n2
+      combos = (1..total).reduce(1, :*) / ((1..n1).reduce(1, :*) * (1..n2).reduce(1, :*))
+      return { u: u_statistic(a, b), p: nil, note: "#{combos} permutations; too many to enumerate" } if combos > 200_000
+
+      pooled = a + b
+      observed = u_statistic(a, b)
+      mean_u = n1 * n2 / 2.0
+      observed_deviation = (observed - mean_u).abs
+
+      at_least_as_extreme = 0
+      (0...total).to_a.combination(n1).each do |idx|
+        set = idx.to_h { |i| [i, true] }
+        left = idx.map { |i| pooled[i] }
+        right = (0...total).reject { |i| set[i] }.map { |i| pooled[i] }
+        at_least_as_extreme += 1 if (u_statistic(left, right) - mean_u).abs >= observed_deviation - 1e-9
+      end
+
+      { u: observed, p: at_least_as_extreme.to_f / combos, permutations: combos }
+    end
+  end
+end

data/lib/llm_experiment/transcript/claude.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+require "json"
+
+module LLMExperiment
+  module Transcript
+    # Claude Code emits stream-json: one system/init event, one assistant event
+    # per model request, and a final result event carrying the totals.
+    #
+    # Usage is reported per model request, so the context the very first request
+    # carried - the auto-attachment signature - is directly observable here.
+    module Claude
+      EDIT_TOOLS = %w[Edit Write NotebookEdit MultiEdit].freeze
+
+      module_function
+
+      def summarize(events, impl_files:, test_file: nil)
+        init = events.find { |e| e["subtype"] == "init" }
+        result = events.reverse.find { |e| e["type"] == "result" }
+
+        turns = []
+        tool_calls = []
+        events.each do |event|
+          next unless event["type"] == "assistant"
+
+          u = usage_of(event) || {}
+          turns << {
+            "input_tokens" => u["input_tokens"].to_i,
+            "output_tokens" => u["output_tokens"].to_i,
+            "cache_creation_input_tokens" => u["cache_creation_input_tokens"].to_i,
+            "cache_read_input_tokens" => u["cache_read_input_tokens"].to_i,
+            "context_tokens" => u["input_tokens"].to_i +
+                                u["cache_creation_input_tokens"].to_i +
+                                u["cache_read_input_tokens"].to_i,
+            "tool_calls" => tool_uses_in(event).map { |t| t["name"] }
+          }
+          tool_uses_in(event).each do |use|
+            tool_calls << { "name" => use["name"], "paths" => paths_in_tool_use(use),
+                            "blob" => (use["input"] || {}).to_json }
+          end
+        end
+
+        first_defect_read = tool_calls.index { |t| Transcript.references_impl?(t["blob"], impl_files) }
+        first_edit = tool_calls.index { |t| EDIT_TOOLS.include?(t["name"]) }
+
+        usage = result&.dig("usage") || {}
+
+        {
+          "agent" => "claude",
+          "model_reported" => init&.dig("model"),
+          "cli_version" => init&.dig("claude_code_version"),
+          # Hermeticity: a clean container should report none of these. If any is
+          # non-empty the trial ran with help it was not supposed to have.
+          "mcp_servers" => init&.dig("mcp_servers") || [],
+          "memory_paths" => init&.dig("memory_paths") || {},
+          "tools_available" => (init&.dig("tools") || []).size,
+          "num_turns" => result&.dig("num_turns"),
+          "duration_ms" => result&.dig("duration_ms"),
+          "duration_api_ms" => result&.dig("duration_api_ms"),
+          "total_cost_usd" => result&.dig("total_cost_usd"),
+          "stop_reason" => result&.dig("stop_reason"),
+          "is_error" => result&.dig("is_error"),
+          "input_tokens" => usage["input_tokens"],
+          "output_tokens" => usage["output_tokens"],
+          "cache_creation_input_tokens" => usage["cache_creation_input_tokens"],
+          "cache_read_input_tokens" => usage["cache_read_input_tokens"],
+          "thinking_tokens" => usage.dig("output_tokens_details", "thinking_tokens"),
+          "total_tool_calls" => tool_calls.size,
+          "tool_call_names" => tool_calls.map { |t| t["name"] },
+          "tool_calls_to_first_defect_read" => first_defect_read,
+          "tool_calls_to_first_edit" => first_edit,
+          "read_before_edit" => first_defect_read && first_edit ? first_defect_read < first_edit : nil,
+          # The auto-attachment signature: how much context the very first request
+          # carried, before any tool could have fetched anything.
+          "context_before_first_tool_call" => turns.first&.dig("context_tokens"),
+          "turns" => turns,
+          "test_file" => test_file
+        }
+      end
+
+      def usage_of(event)
+        event.dig("message", "usage") || event["usage"]
+      end
+
+      def tool_uses_in(event)
+        content = event.dig("message", "content")
+        return [] unless content.is_a?(Array)
+
+        content.select { |c| c["type"] == "tool_use" }
+      end
+
+      # Which file a tool call touched, when it is knowable. Reads, edits and
+      # writes name a path directly; a shell command has to be scanned for one.
+      def paths_in_tool_use(use)
+        input = use["input"] || {}
+        direct = [input["file_path"], input["path"], input["notebook_path"]].compact
+        return direct unless direct.empty?
+
+        text = [input["command"], input["pattern"], input["prompt"]].compact.join(" ")
+        text.scan(%r{[\w./-]+\.(?:rb|erb|yml|yaml)}).uniq
+      end
+    end
+  end
+end

data/lib/llm_experiment/transcript/codex.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+require "json"
+
+module LLMExperiment
+  module Transcript
+    # Codex emits `item.completed` envelopes and one `turn.completed` carrying
+    # usage. Item types seen in practice: command_execution (command,
+    # aggregated_output, exit_code, status), file_change, agent_message,
+    # reasoning, error.
+    #
+    # One asymmetry to be honest about: Codex reports tokens per TURN, and a
+    # turn contains all of its tool calls. So the context carried before the
+    # first tool call - the auto-attachment signature this kind of experiment is
+    # chasing - is directly observable for Claude and not for Codex. For Codex
+    # the same effect can only show up in the totals, which is weaker but not
+    # nothing. It is left nil rather than filled with a number that does not
+    # mean what the column says.
+    module Codex
+      EDIT_ITEMS = %w[file_change patch_apply].freeze
+      TOOL_ITEMS = (EDIT_ITEMS + %w[command_execution mcp_tool_call web_search]).freeze
+
+      module_function
+
+      def summarize(events, impl_files:)
+        tool_calls = []
+        turns = []
+        errors = []
+
+        events.each do |event|
+          case event["type"]
+          when "item.completed"
+            item = event["item"] || {}
+            errors << item["message"] if item["type"] == "error"
+            next unless TOOL_ITEMS.include?(item["type"])
+
+            command = Array(item["command"]).join(" ")
+            paths = command.scan(%r{[\w./-]+\.(?:rb|erb|yml|yaml)}).uniq
+            paths |= changed_paths(item["changes"])
+
+            tool_calls << { "name" => item["type"], "paths" => paths, "command" => command,
+                            "exit_code" => item["exit_code"], "blob" => item.to_json }
+          when "turn.completed"
+            u = event["usage"] || {}
+            turns << {
+              "input_tokens" => u["input_tokens"].to_i,
+              "output_tokens" => u["output_tokens"].to_i,
+              "cache_read_input_tokens" => u["cached_input_tokens"].to_i,
+              "cache_creation_input_tokens" => u["cache_write_input_tokens"].to_i,
+              "reasoning_output_tokens" => u["reasoning_output_tokens"].to_i,
+              "context_tokens" => u["input_tokens"].to_i
+            }
+          end
+        end
+
+        first_defect_read = tool_calls.index { |t| Transcript.references_impl?(t["blob"], impl_files) }
+        first_edit = tool_calls.index { |t| EDIT_ITEMS.include?(t["name"]) }
+
+        {
+          "agent" => "codex",
+          "input_tokens" => turns.sum { |t| t["input_tokens"] },
+          "output_tokens" => turns.sum { |t| t["output_tokens"] },
+          "cache_read_input_tokens" => turns.sum { |t| t["cache_read_input_tokens"] },
+          "cache_creation_input_tokens" => turns.sum { |t| t["cache_creation_input_tokens"] },
+          "thinking_tokens" => turns.sum { |t| t["reasoning_output_tokens"] },
+          "total_tool_calls" => tool_calls.size,
+          "tool_call_names" => tool_calls.map { |t| t["name"] },
+          "tool_calls_to_first_defect_read" => first_defect_read,
+          "tool_calls_to_first_edit" => first_edit,
+          "read_before_edit" => first_defect_read && first_edit ? first_defect_read < first_edit : nil,
+          # Only meaningful if Codex ever reports more than one turn; see the note above.
+          "context_before_first_tool_call" => turns.size > 1 ? turns.first["context_tokens"] : nil,
+          "codex_errors" => errors,
+          "turns" => turns
+        }
+      end
+
+      # A file_change item carries `changes` as a path => diff map.
+      #
+      # This diverges from the source harness, which wrote
+      # `Array(item["changes"]).flat_map { |c| c.is_a?(Hash) ? c.keys : [c.to_s] }`.
+      # Array() turns a Hash into [key, value] pairs before the is_a?(Hash)
+      # test can ever see one, so that branch was unreachable and every path
+      # came out as the string '["app/x.rb", "@@ -1 +1 @@"]' -- the diff text
+      # included. Nothing published reads this field today, which is exactly
+      # why it could stay wrong unnoticed.
+      def changed_paths(changes)
+        case changes
+        when Hash then changes.keys
+        when Array then changes.map(&:to_s)
+        else []
+        end
+      end
+    end
+  end
+end

data/lib/llm_experiment/transcript/hermeticity.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module LLMExperiment
+  module Transcript
+    # A trial is contaminated when it can reach state that outlives it or is
+    # shared with another trial. The CLI always reports an auto-memory path, so
+    # the question is not whether one exists -- it always does -- but where it
+    # points. Inside the per-trial config directory it dies with the container.
+    # Under the mounted credential store it is the same directory for every
+    # trial, and what one trial could leave there is the location of the defect.
+    # Testing for existence rather than location flags every clean run, which
+    # trains you to ignore the warning that matters.
+    module Hermeticity
+      SHARED_MOUNT = "/home/agent/.agent-auth"
+
+      module_function
+
+      def contamination(row)
+        reasons = []
+
+        servers = Array(row["mcp_servers"])
+        unless servers.empty?
+          names = servers.map { |s| s.is_a?(Hash) ? s["name"] : s }
+          reasons << "mcp servers attached: #{names.join(", ")}"
+        end
+
+        Hash(row["memory_paths"]).each do |kind, path|
+          reasons << "#{kind} memory lives in the shared mount: #{path}" if path.to_s.start_with?(SHARED_MOUNT)
+        end
+
+        reasons
+      end
+    end
+  end
+end

data/lib/llm_experiment/transcript/parser.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require "json"
+
+module LLMExperiment
+  module Transcript
+    # Reads one trial directory and writes back what can be measured from it:
+    # events.jsonl (the transcript with unparsable lines dropped) and
+    # metrics.json (the summary the stats read).
+    class Parser
+      # Claude emits stream-json; Codex emits its own JSONL. Detect rather than
+      # configure, so a transcript can be parsed without knowing who produced it.
+      def self.detect_agent(events)
+        return "claude" if events.any? { |e| e["type"] == "system" && e["subtype"] == "init" }
+        return "codex" if events.any? { |e| e["type"].to_s.start_with?("item.", "turn.", "thread.") }
+
+        "unknown"
+      end
+
+      def initialize(experiment:)
+        @experiment = experiment
+      end
+
+      # Every trial that got as far as writing a transcript.
+      def trial_dirs
+        Dir.glob(File.join(@experiment.results_raw_dir, "**", "transcript.jsonl"))
+           .map { |p| File.dirname(p) }.sort
+      end
+
+      # Returns the summary it wrote, or nil for a directory with no transcript.
+      def parse(dir)
+        transcript = File.join(dir, "transcript.jsonl")
+        unless File.exist?(transcript)
+          Shell.log "skip #{dir}: no transcript.jsonl"
+          return nil
+        end
+
+        meta = read_meta(dir)
+        bad = 0
+        events = File.readlines(transcript).filter_map do |line|
+          JSON.parse(line)
+        rescue JSON::ParserError
+          bad += 1
+          nil
+        end
+
+        summary = summarize(events, meta)
+        summary["unparsable_lines"] = bad
+        summary["raw_events"] = events.size
+        summary.merge!(meta_fields(meta))
+        summary["trial_dir"] = dir.sub("#{@experiment.root}/", "")
+
+        File.write(File.join(dir, "events.jsonl"), "#{events.map(&:to_json).join("\n")}\n")
+        File.write(File.join(dir, "metrics.json"), JSON.pretty_generate(summary))
+        summary
+      end
+
+      private
+
+      def summarize(events, meta)
+        impl_files, test_file = ground_truth(meta)
+
+        case meta["agent"] || self.class.detect_agent(events)
+        when "claude" then Claude.summarize(events, impl_files: impl_files, test_file: test_file)
+        when "codex" then Codex.summarize(events, impl_files: impl_files)
+        else { "agent" => "unknown", "events" => events.size }
+        end
+      end
+
+      # experiment.yml is the fallback for impl_files/test_file. A trial recorded
+      # before the runner persisted them has neither in meta.json, and matching
+      # against an empty list makes tool_calls_to_first_defect_read nil for every
+      # row.
+      def ground_truth(meta)
+        task = task_for(meta["task_id"])
+        [Array(meta["impl_files"] || task&.impl_files), meta["test_file"] || task&.test_file]
+      end
+
+      def task_for(id)
+        return nil unless id
+
+        @experiment.task(id)
+      rescue ConfigError
+        nil
+      end
+
+      def read_meta(dir)
+        path = File.join(dir, "meta.json")
+        File.exist?(path) ? JSON.parse(File.read(path)) : {}
+      end
+
+      def meta_fields(meta)
+        {
+          "task_id" => meta["task_id"],
+          "condition" => meta["condition"],
+          "app" => meta["app"],
+          "fix_verified" => meta["fix_verified"],
+          "wall_seconds" => meta["wall_seconds"],
+          "timed_out" => meta["timed_out"],
+          "task_reproduces" => meta["task_reproduces"]
+        }
+      end
+    end
+  end
+end

data/lib/llm_experiment/transcript.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module LLMExperiment
+  # Turning a trial's raw JSONL transcript into measurements.
+  #
+  # The headline number an experiment like this is after is context growth
+  # BEFORE the first tool call. If a mention makes a CLI attach the file itself,
+  # the first request carries the file's tokens and no read is needed to get
+  # them; if the mention is inert text, the first request looks like the bare
+  # one and the file arrives later through a normal read. Those two shapes are
+  # distinguishable only because the per-agent summarizers keep per-turn token
+  # counts rather than only the totals.
+  module Transcript
+    module_function
+
+    # Does this tool call refer to the defect file?
+    #
+    # Matching on the full relative path rather than the basename: a basename
+    # would also fire on an unrelated grep for a common word, and the point of
+    # the metric is how many calls it took to arrive at the file, not how many
+    # mentioned a similar-sounding name. The whole serialised input is searched,
+    # so a Read with a file_path and a shell `cat <path>` both count.
+    def references_impl?(blob, impl_files)
+      impl_files.any? { |f| blob.include?(f) }
+    end
+  end
+end