llm_conductor 1.7.1 → 1.8.0

data/lib/llm_conductor/eval/report_builder.rb

@@ -0,0 +1,258 @@
+# frozen_string_literal: true
+
+require 'csv'
+require_relative 'verdict'
+require_relative 'report'
+
+module LlmConductor
+  module Eval
+    # Pure aggregation: turns a run's per-(input, model) rows into a Report
+    # (CSV string + decision-aid markdown + per-model summary + needs-review
+    # list). Unlike the Rails prototype's ReportBuilder it writes no files;
+    # persistence is the caller's / Store's job.
+    #
+    # +rows+ is an Array of { model_result: Result, judge_verdict: Verdict }.
+    class ReportBuilder
+      BASE_CSV_COLUMNS = %w[
+        input_id input_label model vendor status
+        latency_ms input_tokens output_tokens total_tokens estimated_cost_usd
+        parsed_score parsed_bucket
+        judge_quality_score
+      ].freeze
+
+      JUDGE_TAIL_COLUMNS = %w[
+        judge_verdict_one_line judge_issues judge_error
+        self_judge needs_human_review review_reasons
+        raw_output_ref parsed_output_ref error
+      ].freeze
+
+      def initialize(rows:, run_id:, judge_model:, spec:)
+        @rows = rows
+        @run_id = run_id
+        @judge_model = judge_model
+        @spec = spec
+      end
+
+      def build
+        bucket_disagreement = compute_bucket_disagreement
+        summary = build_summary
+        Report.new(
+          rows: @rows,
+          summary:,
+          needs_review: build_needs_review(bucket_disagreement),
+          csv_string: build_csv(bucket_disagreement),
+          markdown_string: build_markdown(bucket_disagreement, summary)
+        )
+      end
+
+      private
+
+      attr_reader :rows, :run_id, :judge_model, :spec
+
+      def judge_dimension_columns
+        @judge_dimension_columns ||= spec.judge_dimensions.map { |d| "judge_#{d[:key]}" }
+      end
+
+      def extra_csv_keys
+        @extra_csv_keys ||= rows.flat_map { |r| (r[:model_result].extra_columns || {}).keys }.uniq
+      end
+
+      def csv_columns
+        BASE_CSV_COLUMNS + judge_dimension_columns + JUDGE_TAIL_COLUMNS + extra_csv_keys
+      end
+
+      def compute_bucket_disagreement
+        by_input = Hash.new { |h, k| h[k] = Set.new }
+        rows.each do |row|
+          mr = row[:model_result]
+          next unless mr.status == 'ok' && mr.parsed_bucket
+
+          by_input[mr.input_id] << mr.parsed_bucket
+        end
+        by_input.transform_values { |set| set.size > 1 ? set.sort : [] }
+      end
+
+      def build_csv(bucket_disagreement)
+        columns = csv_columns
+        CSV.generate(write_headers: true, headers: columns) do |csv|
+          rows.each { |row| csv << csv_row(row, bucket_disagreement, columns) }
+        end
+      end
+
+      def csv_row(row, bucket_disagreement, columns)
+        mr = row[:model_result]
+        jv = row[:judge_verdict]
+        base = base_csv_values(mr, jv, flag_reasons(mr, jv, bucket_disagreement))
+        spec.judge_dimensions.each { |d| base["judge_#{d[:key]}"] = jv&.dimensions&.dig(d[:key]) }
+        (mr.extra_columns || {}).each { |k, v| base[k] = v }
+        columns.map { |c| base[c] }
+      end
+
+      def base_csv_values(result, verdict, review_reasons)
+        {
+          'input_id' => result.input_id, 'input_label' => result.input_label, 'model' => result.model,
+          'vendor' => result.vendor, 'status' => result.status, 'latency_ms' => result.latency_ms,
+          'input_tokens' => result.input_tokens, 'output_tokens' => result.output_tokens,
+          'total_tokens' => result.total_tokens, 'estimated_cost_usd' => result.estimated_cost_usd,
+          'parsed_score' => result.parsed_score, 'parsed_bucket' => result.parsed_bucket,
+          'judge_quality_score' => verdict&.quality_score, 'judge_verdict_one_line' => verdict&.verdict_one_line,
+          'judge_issues' => Array(verdict&.issues).join(' | '), 'judge_error' => verdict&.judge_error,
+          'self_judge' => (result.model == judge_model), 'needs_human_review' => review_reasons.any?,
+          'review_reasons' => review_reasons.join('; '), 'raw_output_ref' => result.raw_output_ref,
+          'parsed_output_ref' => result.parsed_output_ref, 'error' => result.error
+        }
+      end
+
+      def build_needs_review(bucket_disagreement)
+        rows.filter_map do |row|
+          mr = row[:model_result]
+          reasons = flag_reasons(mr, row[:judge_verdict], bucket_disagreement)
+          next if reasons.empty?
+
+          { input_id: mr.input_id, input_label: mr.input_label, model: mr.model, reasons: }
+        end
+      end
+
+      def flag_reasons(result, verdict, bucket_disagreement)
+        reasons = []
+        disagree = bucket_disagreement[result.input_id]
+        reasons << "buckets_disagree(#{disagree.join(',')})" if disagree&.any?
+        reasons << 'judge_borderline' if verdict && Verdict.borderline?(verdict.quality_score)
+        reasons << 'parse_failed' if result.status == 'parse_error'
+        reasons << 'llm_error' if result.status == 'llm_error'
+        reasons
+      end
+
+      def build_summary
+        rows.group_by { |r| r[:model_result].model }
+            .map { |model, model_rows| summarize_model(model, model_rows) }
+            .sort_by { |s| -(s[:mean_quality] || -1) }
+      end
+
+      def summarize_model(model, model_rows)
+        ok = model_rows.count { |r| r[:model_result].status == 'ok' }
+        latencies = model_rows.filter_map { |r| r[:model_result].latency_ms }
+        costs = model_rows.filter_map { |r| r[:model_result].estimated_cost_usd }.map(&:to_f)
+        qualities = model_rows.filter_map { |r| r[:judge_verdict]&.quality_score }
+        {
+          model:, parse_ok_pct: pct(ok, model_rows.size), mean_quality: mean(qualities)&.round(1),
+          median_latency_ms: percentile(latencies, 50), p95_latency_ms: percentile(latencies, 95),
+          mean_cost: costs.empty? ? 0.0 : (costs.sum / costs.size).round(5), total_cost: costs.sum.round(4),
+          review_pct: pct(review_count(model_rows), model_rows.size)
+        }
+      end
+
+      def review_count(model_rows)
+        model_rows.count do |r|
+          jv = r[:judge_verdict]
+          r[:model_result].status != 'ok' || (jv && Verdict.borderline?(jv.quality_score))
+        end
+      end
+
+      def build_markdown(bucket_disagreement, summary)
+        <<~MD
+          # LLM Eval — #{run_id}
+
+          - Inputs in this run: **#{rows.map { |r| r[:model_result].input_id }.uniq.size}**
+          - Candidate models: **#{summary.size}**
+          - Judge model: `#{judge_model}` (rows where the judged model == judge model are flagged `self_judge`)
+
+          ## Per-model summary
+
+          | Model | Parse OK% | Mean Judge | Median latency | P95 latency | Mean cost | Total cost | % needs review |
+          |---|---:|---:|---:|---:|---:|---:|---:|
+          #{summary.map { |s| markdown_summary_row(s) }.join("\n")}
+
+          ## Recommendation buckets per input
+
+          #{bucket_table}
+
+          ## Bucket-disagreement cases (model choice matters here)
+
+          #{bucket_disagreement_section(bucket_disagreement)}
+
+          ## Pareto picks
+
+          #{pareto_section(summary)}
+        MD
+      end
+
+      def markdown_summary_row(summary)
+        "| `#{summary[:model]}` | #{summary[:parse_ok_pct]}% | #{summary[:mean_quality] || 'n/a'} | " \
+          "#{summary[:median_latency_ms] || 'n/a'}ms | #{summary[:p95_latency_ms] || 'n/a'}ms | " \
+          "$#{format('%.5f', summary[:mean_cost])} | $#{format('%.4f', summary[:total_cost])} | " \
+          "#{summary[:review_pct]}% |"
+      end
+
+      def bucket_table
+        lines = ['| Input | Model | Bucket | Score | Judge | Status |', '|---|---|---|---:|---:|---|']
+        rows.group_by { |r| r[:model_result].input_id }.each do |input_id, input_rows|
+          label = input_rows.first[:model_result].input_label
+          input_rows.each do |r|
+            mr = r[:model_result]
+            jv = r[:judge_verdict]
+            lines << "| #{input_id} #{label} | `#{mr.model}` | #{mr.parsed_bucket || '-'} | " \
+                     "#{mr.parsed_score || '-'} | #{jv&.quality_score || '-'} | #{mr.status} |"
+          end
+        end
+        lines.join("\n")
+      end
+
+      def bucket_disagreement_section(bucket_disagreement)
+        cases = bucket_disagreement.reject { |_, v| v.empty? }
+        return '_None — every input received the same bucket across all models._' if cases.empty?
+
+        cases.flat_map { |input_id, buckets| disagreement_block(input_id, buckets) }.join("\n")
+      end
+
+      def disagreement_block(input_id, buckets)
+        input_rows = rows.select { |r| r[:model_result].input_id == input_id && r[:model_result].status == 'ok' }
+        return [] if input_rows.empty?
+
+        label = input_rows.first[:model_result].input_label
+        lines = ["### #{input_id} — #{label}", '', "Buckets seen: **#{buckets.join(', ')}**", '',
+                 '| Model | Bucket | Score | Judge |', '|---|---|---:|---:|']
+        input_rows.each do |r|
+          mr = r[:model_result]
+          jv = r[:judge_verdict]
+          lines << "| `#{mr.model}` | #{mr.parsed_bucket || '-'} | #{mr.parsed_score || '-'} | #{jv&.quality_score || '-'} |"
+        end
+        lines << ''
+        lines
+      end
+
+      def pareto_section(summary)
+        return '_No successful runs to rank._' if summary.empty?
+
+        lines = ['Top 3 by mean judge quality:', '']
+        summary.first(3).each_with_index do |s, i|
+          lines << "#{i + 1}. **`#{s[:model]}`** — judge **#{s[:mean_quality] || 'n/a'}**, " \
+                   "median **#{s[:median_latency_ms]}ms**, mean cost **$#{format('%.5f', s[:mean_cost])}**, " \
+                   "parse OK **#{s[:parse_ok_pct]}%**."
+        end
+        lines << ''
+        lines << '_Rows flagged `needs_human_review=true` are the ones to sanity-check manually._'
+        lines.join("\n")
+      end
+
+      def pct(num, den)
+        return 0 if den.zero?
+
+        ((num.to_f / den) * 100).round(1)
+      end
+
+      def mean(values)
+        return nil if values.empty?
+
+        values.sum.to_f / values.size
+      end
+
+      def percentile(values, target_pct)
+        return nil if values.empty?
+
+        sorted = values.sort
+        sorted[((target_pct / 100.0) * (sorted.size - 1)).round]
+      end
+    end
+  end
+end

data/lib/llm_conductor/eval/result.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module LlmConductor
+  module Eval
+    # Outcome of running ONE (input, model) pair through the engine.
+    #
+    # Ported from the Rails prototype's ModelRunner::Result struct, with the
+    # +record_*+ fields renamed to +input_*+ and the on-disk +*_path+ fields
+    # generalized to +*_ref+ (a Store handle — a filesystem path for FileStore,
+    # an opaque key for InMemory).
+    #
+    # +status+ is one of: 'ok', 'parse_error', 'llm_error', 'exception'.
+    Result = Struct.new(
+      :input_id, :input_label, :model, :vendor, :status, :latency_ms,
+      :input_tokens, :output_tokens, :total_tokens, :estimated_cost_usd,
+      :parsed_score, :parsed_bucket, :extra_columns,
+      :raw_output_ref, :parsed_output_ref, :error,
+      keyword_init: true
+    ) do
+      # String-keyed hash for JSON manifest persistence.
+      def to_h
+        super.transform_keys(&:to_s)
+      end
+
+      def ok?
+        status == 'ok'
+      end
+    end
+  end
+end

data/lib/llm_conductor/eval/runner.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+require 'time'
+require_relative 'model_runner'
+require_relative 'judge'
+require_relative 'report_builder'
+require_relative 'result'
+require_relative 'verdict'
+
+module LlmConductor
+  module Eval
+    # Top-level orchestrator. For each input, builds the prompt data once, runs
+    # every candidate (input, model) pair through ModelRunner, judges it, and
+    # rewrites the manifest after each pair so the run stays resumable /
+    # reportable mid-flight.
+    #
+    # Unlike the Rails prototype it does NO data selection — the caller passes
+    # +inputs:+ directly. See LlmConductor::Eval.run for the public entrypoint.
+    class Runner
+      def initialize(spec:, inputs:, models:, judge:, store:, logger:, run_id:)
+        @spec = spec
+        @inputs = inputs.to_a
+        @models = models
+        @judge_config = self.class.normalize_judge(judge)
+        @store = store
+        @logger = logger
+        @run_id = run_id
+      end
+
+      def run
+        @logger.info("LLM eval run=#{@run_id} models=#{@models.map { |m| m[:model] }.join(',')} " \
+                     "judge=#{@judge_config[:model]}")
+        warn_self_judge
+        manifest = base_manifest
+        rows = run_all_pairs(manifest)
+        manifest[:finished_at] = Time.now.utc.iso8601
+        @store.write_manifest(@run_id, manifest)
+        build_report(rows)
+      end
+
+      # Rebuild a Report from a stored manifest without recalling models or judge.
+      def self.report_only(run_id:, spec:, store:)
+        manifest = store.read_manifest(run_id) or raise ArgumentError, "No manifest for run_id=#{run_id}"
+        rows = manifest['rows'].map { |raw| restore_row(raw) }
+        ReportBuilder.new(rows:, run_id:, judge_model: manifest['judge_model'], spec:).build
+      end
+
+      # Re-run the judge against stored candidate outputs (e.g. after changing
+      # the judge model). Fully self-contained: input data is read from the store.
+      def self.judge_only(run_id:, spec:, store:, judge:, logger:)
+        config = normalize_judge(judge)
+        manifest = store.read_manifest(run_id) or raise ArgumentError, "No manifest for run_id=#{run_id}"
+        judge_obj = Judge.new(spec:, store:, run_id:, logger:,
+                              judge_model: config[:model], judge_vendor: config[:vendor])
+        rows = manifest['rows'].map { |raw| rejudge_row(raw, judge_obj, store, run_id) }
+        manifest['judge_model'] = config[:model]
+        manifest['rejudged_at'] = Time.now.utc.iso8601
+        store.write_manifest(run_id, manifest)
+        ReportBuilder.new(rows:, run_id:, judge_model: config[:model], spec:).build
+      end
+
+      def self.normalize_judge(judge)
+        judge ||= {}
+        { model: judge[:model] || Judge::DEFAULT_MODEL,
+          vendor: (judge[:vendor] || Judge::DEFAULT_VENDOR).to_sym }
+      end
+
+      def self.restore_result(raw)
+        Result.new(**raw.transform_keys(&:to_sym))
+      end
+
+      def self.restore_verdict(raw)
+        raw ? Verdict.new(**raw.transform_keys(&:to_sym)) : nil
+      end
+
+      def self.restore_row(raw)
+        { model_result: restore_result(raw['model_result']),
+          judge_verdict: restore_verdict(raw['judge_verdict']) }
+      end
+
+      def self.rejudge_row(raw, judge_obj, store, run_id)
+        result = restore_result(raw['model_result'])
+        input_data = store.read_input_data(run_id, result.input_id)
+        verdict = judge_obj.judge(model_result: result, input_data:)
+        raw['judge_verdict'] = verdict&.to_h
+        { model_result: result, judge_verdict: verdict }
+      end
+
+      private
+
+      def run_all_pairs(manifest)
+        rows = []
+        @inputs.each_with_index do |input, idx|
+          input_id = @spec.input_id(input)
+          data = @spec.build_data(input)
+          @store.write_input_data(@run_id, input_id, data)
+          @models.each do |cand|
+            row = run_pair(input, data, cand)
+            rows << row
+            manifest[:rows] << serialize_row(row)
+            @store.write_manifest(@run_id, manifest)
+            log_pair(idx, cand, row)
+          end
+        end
+        rows
+      end
+
+      def run_pair(input, data, cand)
+        result = ModelRunner.new(input, model: cand[:model], vendor: cand[:vendor], spec: @spec,
+                                        store: @store, run_id: @run_id, logger: @logger, data:).run
+        verdict = build_judge.judge(model_result: result, input_data: data)
+        { model_result: result, judge_verdict: verdict }
+      end
+
+      def build_judge
+        Judge.new(spec: @spec, store: @store, run_id: @run_id, logger: @logger,
+                  judge_model: @judge_config[:model], judge_vendor: @judge_config[:vendor])
+      end
+
+      def base_manifest
+        { run_id: @run_id, started_at: Time.now.utc.iso8601, judge_model: @judge_config[:model],
+          models: @models, rows: [] }
+      end
+
+      def serialize_row(row)
+        { 'model_result' => row[:model_result].to_h, 'judge_verdict' => row[:judge_verdict]&.to_h }
+      end
+
+      def build_report(rows)
+        ReportBuilder.new(rows:, run_id: @run_id, judge_model: @judge_config[:model], spec: @spec).build
+      end
+
+      def warn_self_judge
+        return unless @models.any? { |m| m[:model] == @judge_config[:model] }
+
+        @logger.warn("[Eval] judge model #{@judge_config[:model]} also appears in candidates — " \
+                     'those rows will be flagged self_judge=true and should be discounted when ranking.')
+      end
+
+      def log_pair(idx, cand, row)
+        result = row[:model_result]
+        verdict = row[:judge_verdict]
+        @logger.info("  [#{idx + 1}/#{@inputs.size}] #{cand[:model]} -> status=#{result.status} " \
+                     "latency=#{result.latency_ms}ms judge=#{verdict&.quality_score}")
+      end
+    end
+  end
+end

data/lib/llm_conductor/eval/spec.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require_relative 'json_parser'
+
+module LlmConductor
+  module Eval
+    # The public extension seam. Subclass (or duck-type) this to describe how to
+    # evaluate one LLM-powered feature: how to turn a caller-supplied input into
+    # a prompt payload, how to parse the output, and what the judge should grade.
+    #
+    # The engine itself is generic and feature-agnostic; everything
+    # feature-specific lives here. Unlike the Rails prototype's Feature::Base,
+    # there is no +select_cases+ — selecting which inputs to evaluate is the
+    # caller's job, done before calling LlmConductor::Eval.run and passed via
+    # +inputs:+. The engine never queries a database.
+    class Spec
+      # Symbol passed to LlmConductor.generate as +type:+ (must match a
+      # registered prompt). Return nil if instead you build a full prompt
+      # string in #build_data, in which case the engine passes it as +prompt:+.
+      def prompt_type
+        raise NotImplementedError
+      end
+
+      # Stable id for an input (was record.id). Used for output grouping/paths.
+      def input_id(_input)
+        raise NotImplementedError
+      end
+
+      # Human label for an input (was record.name). Defaults to the id.
+      def input_label(input)
+        input_id(input).to_s
+      end
+
+      # Build the prompt payload for one input. When #prompt_type is set this is
+      # passed as +data:+; otherwise it must be a full prompt String passed as
+      # +prompt:+ (was build_data(record)).
+      def build_data(_input)
+        raise NotImplementedError
+      end
+
+      # Parse the LLM's raw text into a Hash, or nil on failure. Defaults to the
+      # gem's conservative JsonParser; override for tuned/feature-specific parsing.
+      def parse(raw)
+        JsonParser.parse(raw)
+      end
+
+      # Vendor-specific generation params (e.g. a deterministic Ollama seed).
+      # Return {} for vendors that don't expose one.
+      # rubocop:disable Lint/UnusedMethodArgument
+      def vendor_params(vendor:, input_id:)
+        {}
+      end
+      # rubocop:enable Lint/UnusedMethodArgument
+
+      # { score: Numeric|nil, bucket: String|nil } — powers CSV columns and the
+      # bucket-disagreement detection. +bucket+ may be any discrete label.
+      def output_summary(_parsed)
+        raise NotImplementedError
+      end
+
+      # Text inlined into the judge prompt describing the rubric the candidate
+      # was asked to follow.
+      def judge_rubric_excerpt
+        raise NotImplementedError
+      end
+
+      # [{ key:, description: }] — dimensions the judge scores 0-100 each.
+      def judge_dimensions
+        raise NotImplementedError
+      end
+
+      # Extra per-row CSV columns beyond the base set. Keys become headers.
+      def extra_columns(_parsed)
+        {}
+      end
+    end
+  end
+end

data/lib/llm_conductor/eval/store/base.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module LlmConductor
+  module Eval
+    module Store
+      # Pluggable persistence interface for an eval run. Replaces the prototype's
+      # hard-coded Rails.root.join('tmp', ...) + File.read/write calls.
+      #
+      # Two implementations ship with the gem: InMemory (default; nothing hits
+      # disk) and FileStore (resumable, reproduces the prototype's tmp/<run_id>/
+      # layout). Implement this interface to persist anywhere else.
+      #
+      # Write methods return an opaque "ref" (a filesystem path for FileStore, a
+      # key for InMemory) recorded on the Result for the report's path columns.
+      class Base
+        def write_raw(_run_id, _input_id, _model_slug, _text)
+          raise NotImplementedError
+        end
+
+        def read_raw(_run_id, _input_id, _model_slug)
+          raise NotImplementedError
+        end
+
+        def write_parsed(_run_id, _input_id, _model_slug, _hash)
+          raise NotImplementedError
+        end
+
+        # Returns the parsed Hash/Array (not the ref), or nil if absent.
+        def read_parsed(_run_id, _input_id, _model_slug)
+          raise NotImplementedError
+        end
+
+        def write_input_data(_run_id, _input_id, _hash)
+          raise NotImplementedError
+        end
+
+        # Enables self-contained re-judge / report without the original inputs.
+        def read_input_data(_run_id, _input_id)
+          raise NotImplementedError
+        end
+
+        def write_manifest(_run_id, _manifest_hash)
+          raise NotImplementedError
+        end
+
+        def read_manifest(_run_id)
+          raise NotImplementedError
+        end
+
+        # True when this (input, model) pair already has stored output — lets a
+        # future restart skip already-completed pairs.
+        def completed?(_run_id, _input_id, _model_slug)
+          raise NotImplementedError
+        end
+      end
+    end
+  end
+end

data/lib/llm_conductor/eval/store/file_store.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'fileutils'
+require_relative 'base'
+
+module LlmConductor
+  module Eval
+    module Store
+      # Resumable on-disk store. Reproduces the Rails prototype's layout:
+      #
+      #   <base_dir>/<run_id>/manifest.json
+      #   <base_dir>/<run_id>/<input_id>/_input.json
+      #   <base_dir>/<run_id>/<input_id>/<model_slug>.raw.txt
+      #   <base_dir>/<run_id>/<input_id>/<model_slug>.json
+      #
+      # The manifest is rewritten after every (input, model) pair, so a run is
+      # reportable / re-judgeable mid-flight (see Runner.report_only/judge_only).
+      class FileStore < Base
+        attr_reader :base_dir
+
+        def initialize(base_dir)
+          super()
+          @base_dir = base_dir.to_s
+        end
+
+        def write_raw(run_id, input_id, model_slug, text)
+          write_file(output_path(run_id, input_id, "#{model_slug}.raw.txt"), text.to_s)
+        end
+
+        def read_raw(run_id, input_id, model_slug)
+          read_file(output_path(run_id, input_id, "#{model_slug}.raw.txt"))
+        end
+
+        def write_parsed(run_id, input_id, model_slug, hash)
+          write_file(output_path(run_id, input_id, "#{model_slug}.json"), JSON.pretty_generate(hash))
+        end
+
+        def read_parsed(run_id, input_id, model_slug)
+          read_json(output_path(run_id, input_id, "#{model_slug}.json"))
+        end
+
+        def write_input_data(run_id, input_id, hash)
+          write_file(output_path(run_id, input_id, '_input.json'), JSON.pretty_generate(hash))
+        end
+
+        def read_input_data(run_id, input_id)
+          read_json(output_path(run_id, input_id, '_input.json'))
+        end
+
+        def write_manifest(run_id, manifest_hash)
+          write_file(manifest_path(run_id), JSON.pretty_generate(manifest_hash))
+        end
+
+        def read_manifest(run_id)
+          read_json(manifest_path(run_id))
+        end
+
+        def completed?(run_id, input_id, model_slug)
+          File.exist?(output_path(run_id, input_id, "#{model_slug}.json")) ||
+            File.exist?(output_path(run_id, input_id, "#{model_slug}.raw.txt"))
+        end
+
+        private
+
+        def output_path(run_id, input_id, name)
+          File.join(@base_dir, run_id.to_s, input_id.to_s, name)
+        end
+
+        def manifest_path(run_id)
+          File.join(@base_dir, run_id.to_s, 'manifest.json')
+        end
+
+        def write_file(path, content)
+          FileUtils.mkdir_p(File.dirname(path))
+          File.write(path, content)
+          path
+        end
+
+        def read_file(path)
+          File.exist?(path) ? File.read(path) : nil
+        end
+
+        def read_json(path)
+          return nil unless File.exist?(path)
+
+          JSON.parse(File.read(path))
+        rescue JSON::ParserError
+          nil
+        end
+      end
+    end
+  end
+end