llm_conductor 1.7.1 → 1.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2a6fce29f76d891a6c773b998ec21a2cc30b1c34f979640f86a71d20bfc6af8d
-  data.tar.gz: 07ead7a9e05e819a6a644fc741ed5a118123d9130ea96d8725da6b4be09573ca
+  metadata.gz: ac4b318b4227e3f089f42d471d51a1d7fefc9c54d080c30bbb2e748039ee4ae0
+  data.tar.gz: c21eb439aed4fc671fde4dc30c06c12f0853a5f99ba29d47c3bd3b389f5b27c0
 SHA512:
-  metadata.gz: 7cc94df2e2ed528d7fff9d688c9ac69cdad48282fc268da8aa78b8a0aa99db66bbffccd74661860b47f4df04a0f98826b60db27052eb55845282712bf514bfef
-  data.tar.gz: '032148a55e18c3402765695baa053f9fed3e07fcbb0dd7d550fa6199ba29a1ef1a63d7df644d6d839c502952614e6ae08585a24459345b7b34e96989fe60c350'
+  metadata.gz: 918613d702e3918ae5651ec8aefab34126a36a17c0be75549a2227fb9a3e6b7392c245a7d96a2d15df6e76996e871c66c1572cb1f061d5bef1c96fe1e31ec8e4
+  data.tar.gz: e4b356669ce32103a8a264c66574edf7f0819379240798d0d91b0cd29c623e512cd6776eb4d5bcf421d6fb8cb30bb2467cc188154910928a8aa6b16adc017d76

data/.rubocop.yml

@@ -31,6 +31,10 @@ Lint/ConstantDefinitionInBlock:
 
 Metrics/ClassLength:
   Max: 130
+  Exclude:
+    # Eval engine classes are faithful ports of the Rails prototype; their size
+    # mirrors the reference implementation.
+    - 'lib/llm_conductor/eval/**/*'
 
 Metrics/MethodLength:
   Max: 15
@@ -39,6 +43,7 @@ Metrics/MethodLength:
     - 'lib/llm_conductor/clients/openrouter_client.rb'
     - 'lib/llm_conductor/clients/zai_client.rb'
     - 'lib/llm_conductor/client_factory.rb'
+    - 'lib/llm_conductor/eval/**/*'
     - 'examples/*.rb'
 
 RSpec/ExampleLength:
@@ -98,12 +103,15 @@ Metrics/AbcSize:
     - 'lib/llm_conductor/prompts.rb'
     - 'lib/llm_conductor/clients/openrouter_client.rb'
     - 'lib/llm_conductor/clients/zai_client.rb'
+    - 'lib/llm_conductor/eval/**/*'
     - 'examples/*.rb'
 
 Metrics/ParameterLists:
   Exclude:
     - 'lib/llm_conductor.rb'
     - 'lib/llm_conductor/configuration.rb'
+    - 'lib/llm_conductor/eval.rb'
+    - 'lib/llm_conductor/eval/**/*'
 
 Metrics/CyclomaticComplexity:
   Exclude:
@@ -111,6 +119,7 @@ Metrics/CyclomaticComplexity:
     - 'lib/llm_conductor/prompts.rb'
     - 'lib/llm_conductor/clients/openrouter_client.rb'
     - 'lib/llm_conductor/clients/zai_client.rb'
+    - 'lib/llm_conductor/eval/**/*'
     - 'examples/*.rb'
 
 Metrics/PerceivedComplexity:
@@ -118,6 +127,7 @@ Metrics/PerceivedComplexity:
     - 'lib/llm_conductor/prompts.rb'
     - 'lib/llm_conductor/clients/openrouter_client.rb'
     - 'lib/llm_conductor/clients/zai_client.rb'
+    - 'lib/llm_conductor/eval/**/*'
 
 Layout/LineLength:
   Max: 125

data/README.md

@@ -252,6 +252,68 @@ else
 end
 ```
 
+## Model Evaluation (opt-in)
+
+Which model/vendor is best for *your* prompt? The eval harness runs the same
+prompt across N `(model, vendor)` pairs over M inputs and compares them on
+**cost, latency, tokens, and LLM-judged quality** — three of which `generate`
+already produces for free.
+
+It's behind a separate require so core users pay nothing:
+
+```ruby
+require 'llm_conductor/eval'
+
+# 1. Describe how to evaluate your feature (the only feature-specific code).
+class ArticleSummarySpec < LlmConductor::Eval::Spec
+  def prompt_type        = :summarize_text                 # a registered prompt type
+  def input_id(article)  = article[:id]
+  def input_label(article) = article[:title]
+  def build_data(article) = { content: article[:body] }    # payload for generate(type:, data:)
+
+  # { score:, bucket: } — bucket is any discrete label; powers disagreement detection.
+  def output_summary(parsed) = { score: parsed['rating'], bucket: parsed['verdict'] }
+
+  def judge_rubric_excerpt = 'A good summary is faithful, concise, and covers the key points.'
+  def judge_dimensions
+    [{ key: 'faithfulness', description: 'no hallucinations vs. the source' },
+     { key: 'coverage',     description: 'captures the key points' }]
+  end
+end
+
+# 2. Run it. `inputs` is ANY enumerable — selection is YOUR job, never the gem's.
+report = LlmConductor::Eval.run(
+  spec:   ArticleSummarySpec.new,
+  inputs: my_articles,
+  models: [                                       # caller-owned; no baked-in defaults
+    { model: 'phi4-mini',        vendor: :ollama },
+    { model: 'gemini-2.5-flash', vendor: :gemini },
+    { model: 'gpt-4o-mini',      vendor: :openai }
+  ],
+  judge:  { model: 'llama-3.3-70b-versatile', vendor: :groq },  # default; needs Groq creds
+  store:  LlmConductor::Eval::Store::FileStore.new('tmp/llm_eval') # or in-memory (default)
+)
+
+report.summary       # per-model aggregates: parse-OK%, mean quality, p50/p95 latency, cost
+report.to_markdown   # decision-aid report (you persist it)
+report.to_csv        # full per-row data
+report.needs_review  # rows flagged for a human (bucket disagreement / borderline / errors)
+```
+
+**Judge bias matters.** The judge defaults to Groq's `llama-3.3-70b-versatile`
+precisely because it sits *outside* the Gemini/OpenAI/Ollama candidate families —
+a model grading its own family scores it high. Any row where the judged model
+equals the judge model is flagged `self_judge=true` so you can discount it.
+
+Cheap re-runs reuse stored candidate outputs — no re-calling the candidates:
+
+```ruby
+LlmConductor::Eval.judge_only(run_id:, spec:, store:, judge: { model: 'gemini-2.5-pro', vendor: :gemini })
+LlmConductor::Eval.report_only(run_id:, spec:, store:)
+```
+
+See [`examples/model_eval_usage.rb`](examples/model_eval_usage.rb) for a complete runnable example.
+
 ## Documentation
 
 - **[Custom Parameters Guide](docs/custom-parameters.md)** - Temperature, top_p, and more
@@ -272,6 +334,7 @@ Check the [examples/](examples/) directory for comprehensive examples:
 - `data_builder_usage.rb` - Data builder patterns
 - `prompt_registration.rb` - Custom prompt classes
 - `rag_usage.rb` - Retrieval-Augmented Generation
+- `model_eval_usage.rb` - Model evaluation harness (cost/latency/quality comparison)
 
 Run any example:
 

data/examples/model_eval_usage.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+# Model Evaluation harness example.
+#
+# Runs the same prompt across several (model, vendor) pairs over a handful of
+# inputs and compares them on cost, latency, tokens, and LLM-judged quality.
+#
+# Run with:  ruby examples/model_eval_usage.rb
+#
+# Requires credentials for whichever vendors you list in CANDIDATES (and Groq
+# for the default judge). Configure them via ENV (OPENAI_API_KEY, GEMINI_API_KEY,
+# GROQ_API_KEY, OLLAMA_ADDRESS, ...) or LlmConductor.configure.
+
+require 'llm_conductor/eval'
+
+# 1. A Spec describes the ONE feature being evaluated. It is the only
+#    feature-specific code; the engine itself is generic.
+class SentimentSpec < LlmConductor::Eval::Spec
+  # We build a full prompt string ourselves, so prompt_type is nil and the
+  # engine passes build_data as `prompt:` (instead of `type:` + `data:`).
+  def prompt_type = nil
+
+  def input_id(review)    = review[:id]
+  def input_label(review) = review[:product]
+
+  def build_data(review)
+    <<~PROMPT
+      Classify the sentiment of this product review. Respond with ONLY a JSON
+      object: {"sentiment": "positive|neutral|negative", "confidence": 0-100}
+
+      Review: #{review[:text]}
+    PROMPT
+  end
+
+  # score + bucket drive the CSV and bucket-disagreement detection. The bucket
+  # here is the sentiment label — if models disagree on it, the row is flagged.
+  def output_summary(parsed)
+    { score: parsed['confidence'], bucket: parsed['sentiment'] }
+  end
+
+  def judge_rubric_excerpt
+    'A correct classification matches the review\'s actual sentiment and gives ' \
+      'a calibrated confidence (high only when the text is unambiguous).'
+  end
+
+  def judge_dimensions
+    [{ key: 'correctness', description: 'is the sentiment label correct' },
+     { key: 'calibration', description: 'is the confidence well-calibrated' }]
+  end
+end
+
+# 2. Inputs are ANY enumerable of opaque objects — selecting them is YOUR job.
+reviews = [
+  { id: 1, product: 'Widget',  text: 'Absolutely love it, works perfectly!' },
+  { id: 2, product: 'Gadget',  text: 'It broke after two days. Very disappointed.' },
+  { id: 3, product: 'Gizmo',   text: 'It is fine. Does what it says, nothing special.' }
+]
+
+# 3. Candidate (model, vendor) pairs are caller-owned — there is no baked-in
+#    default list (which models you have pulled / hold keys for is your concern).
+CANDIDATES = [
+  { model: 'gpt-4o-mini',      vendor: :openai },
+  { model: 'gemini-2.5-flash', vendor: :gemini }
+].freeze
+
+report = LlmConductor::Eval.run(
+  spec: SentimentSpec.new,
+  inputs: reviews,
+  models: CANDIDATES,
+  # Judge defaults to llama-3.3-70b-versatile on Groq (outside the candidate
+  # families → no self-judge bias). Override here if you have other quota.
+  judge: { model: 'llama-3.3-70b-versatile', vendor: :groq },
+  # InMemory store is the default; swap in FileStore to persist + enable
+  # report_only / judge_only re-runs:
+  store: LlmConductor::Eval::Store::FileStore.new('tmp/llm_eval')
+)
+
+puts report.to_markdown
+puts "\n--- Rows needing human review ---"
+report.needs_review.each do |row|
+  puts "input=#{row[:input_id]} model=#{row[:model]} reasons=#{row[:reasons].join(', ')}"
+end
+
+# Persist the CSV yourself — the engine returns data, it doesn't impose a layout.
+File.write('tmp/llm_eval_results.csv', report.to_csv)
+puts "\nWrote tmp/llm_eval_results.csv"

data/lib/llm_conductor/eval/json_parser.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module LlmConductor
+  module Eval
+    # Minimal, conservative JSON-from-LLM-text parser.
+    #
+    # Replaces the app-level LlmJsonCleaner the Rails prototype relied on. The
+    # guiding principle (from docs/llm_eval_framework.md) is: NEVER "repair"
+    # already-valid JSON — heavy cleaning corrupts numeric scores and the like.
+    # We only strip markdown fences, drop any preamble before the first brace,
+    # trim to the outermost balanced object/array, then parse once.
+    module JsonParser
+      module_function
+
+      # Parse +text+ into a Hash or Array, or return nil on any failure.
+      def parse(text)
+        prepared = prepare_text(text)
+        return nil if prepared.empty?
+
+        obj = begin
+          JSON.parse(prepared)
+        rescue JSON::ParserError
+          nil
+        end
+        obj.is_a?(Hash) || obj.is_a?(Array) ? obj : nil
+      end
+
+      # Strip ```json fences, drop preamble before the first [ or {, and trim
+      # to the matching closing brace/bracket. Returns '' when there is no
+      # JSON-looking content at all.
+      def prepare_text(text)
+        str = text.to_s.strip
+                  .gsub(/\A```(?:json)?\s*/i, '')
+                  .gsub(/```\s*\z/, '')
+                  .strip
+        start = str.index(/[\[{]/)
+        return '' if start.nil?
+
+        balance(str[start..])
+      end
+
+      # Given a string that starts with '{' or '[', return the substring up to
+      # and including its matching close. String contents (and escapes) are
+      # skipped so braces inside string literals don't throw off the depth.
+      def balance(str)
+        open = str[0]
+        close = open == '{' ? '}' : ']'
+        depth = 0
+        in_string = false
+        escape = false
+
+        str.each_char.with_index do |char, index|
+          if in_string
+            if escape then escape = false
+            elsif char == '\\' then escape = true
+            elsif char == '"' then in_string = false
+            end
+            next
+          end
+
+          case char
+          when '"' then in_string = true
+          when open then depth += 1
+          when close
+            depth -= 1
+            return str[0..index] if depth.zero?
+          end
+        end
+
+        str
+      end
+    end
+  end
+end

data/lib/llm_conductor/eval/judge.rb

@@ -0,0 +1,188 @@
+# frozen_string_literal: true
+
+require 'json'
+require_relative 'verdict'
+require_relative 'json_parser'
+require_relative 'model_runner'
+
+module LlmConductor
+  module Eval
+    # LLM-as-judge for one candidate (input, model) output.
+    #
+    # Sends the judge model the original input data, the spec's rubric excerpt,
+    # and the candidate's parsed output (or raw text on parse failure), and
+    # expects strict JSON back with a quality_score + per-dimension scores.
+    #
+    # Judge defaults to Groq's llama-3.3-70b-versatile: it sits OUTSIDE the
+    # Gemini/OpenAI/Ollama families that dominate most candidate lists (avoiding
+    # self-judge bias — Gemini grades its own output ~10pts high) and Groq's
+    # free tier offers far more throughput than Gemini Pro's ~2 RPM. Override
+    # via the +judge:+ config. It needs Groq credentials configured; rows where
+    # the judged model == the judge model are flagged +self_judge+ in the report.
+    class Judge
+      DEFAULT_MODEL = 'llama-3.3-70b-versatile'
+      DEFAULT_VENDOR = :groq
+
+      def self.borderline?(score)
+        Verdict.borderline?(score)
+      end
+
+      def initialize(spec:, store:, run_id:, logger:, judge_model: DEFAULT_MODEL,
+                     judge_vendor: DEFAULT_VENDOR, rate_limit_retries: 3,
+                     rate_limit_backoff_seconds: 20)
+        @spec = spec
+        @store = store
+        @run_id = run_id
+        @logger = logger
+        @judge_model = judge_model
+        @judge_vendor = judge_vendor.to_sym
+        @rate_limit_retries = rate_limit_retries
+        @rate_limit_backoff_seconds = rate_limit_backoff_seconds
+      end
+
+      # +model_result+ is an Eval::Result. +input_data+ is the spec's data Hash
+      # for the input being judged.
+      def judge(model_result:, input_data:)
+        prompt = build_prompt(model_result:, input_data:)
+        response, latency_ms = call_with_rate_limit_retry(prompt)
+
+        unless response&.success?
+          error = response&.metadata&.dig(:error) || 'judge LLM call failed'
+          return failure_verdict(latency_ms:, response:, error:)
+        end
+
+        parsed = JsonParser.parse(response.output)
+        if parsed.nil?
+          return failure_verdict(latency_ms:, response:,
+                                 error: "judge output not valid JSON: #{response.output.to_s[0, 200]}")
+        end
+
+        build_verdict(parsed:, latency_ms:, response:)
+      rescue StandardError => e
+        @logger.error("[Eval::Judge] #{@judge_model}: #{e.class}: #{e.message}")
+        Verdict.new(judge_model: @judge_model, judge_error: "#{e.class}: #{e.message}")
+      end
+
+      private
+
+      def build_verdict(parsed:, latency_ms:, response:)
+        Verdict.new(
+          quality_score: clamp_score(parsed['quality_score']),
+          dimensions: extract_dimensions(parsed['dimensions']),
+          issues: Array(parsed['issues']).map(&:to_s),
+          verdict_one_line: parsed['verdict_one_line'].to_s,
+          judge_model: @judge_model,
+          judge_latency_ms: latency_ms,
+          judge_input_tokens: response.input_tokens,
+          judge_output_tokens: response.output_tokens,
+          judge_estimated_cost_usd: response.estimated_cost
+        )
+      end
+
+      def call_with_rate_limit_retry(prompt)
+        attempt = 0
+        started_at = Time.now.utc
+        loop do
+          response = LlmConductor.generate(model: @judge_model, prompt:, vendor: @judge_vendor)
+          if !response&.success? && rate_limited?(response) && attempt < @rate_limit_retries
+            wait = @rate_limit_backoff_seconds * (2**attempt)
+            @logger.warn("[Eval::Judge] 429 from #{@judge_model}; sleeping #{wait}s then retrying " \
+                         "(attempt #{attempt + 1}/#{@rate_limit_retries})")
+            sleep(wait)
+            attempt += 1
+            next
+          end
+          return [response, ((Time.now.utc - started_at) * 1000).round]
+        end
+      end
+
+      def rate_limited?(response)
+        error = response&.metadata&.dig(:error).to_s
+        error.include?('429') || error.match?(/rate.limit/i)
+      end
+
+      def build_prompt(model_result:, input_data:)
+        <<~PROMPT
+          You are an impartial judge evaluating how well a candidate LLM performed a
+          task against its rubric. Score the candidate's output on a 0-100 quality
+          scale. Be strict but fair: a perfect rubric-adherent response grounded in
+          the provided evidence is 90-100; obvious hallucinations or rubric violations
+          should drop the score significantly.
+
+          <rubric_excerpt>
+          #{@spec.judge_rubric_excerpt}
+          </rubric_excerpt>
+
+          <original_input_data>
+          #{JSON.pretty_generate(input_data)}
+          </original_input_data>
+
+          <candidate_output>
+          #{candidate_block(model_result)}
+          </candidate_output>
+
+          <judging_dimensions>
+          #{judging_dimensions_block}
+          </judging_dimensions>
+
+          Return ONE JSON object with no markdown fences and no commentary:
+
+          {
+            "quality_score": 0-100,
+            "dimensions": {
+          #{dimensions_json_template}
+            },
+            "issues": ["concrete one-line problem", "..."],
+            "verdict_one_line": "one-line summary of overall judgment"
+          }
+        PROMPT
+      end
+
+      def candidate_block(model_result)
+        slug = ModelRunner.slug(model_result.model)
+        parsed = @store.read_parsed(@run_id, model_result.input_id, slug)
+        return parsed.is_a?(String) ? parsed : JSON.pretty_generate(parsed) if parsed
+
+        raw = @store.read_raw(@run_id, model_result.input_id, slug)
+        if raw && !raw.empty?
+          "PARSE FAILED. RAW OUTPUT:\n#{raw}"
+        else
+          "CANDIDATE PRODUCED NO USABLE OUTPUT. status=#{model_result.status} error=#{model_result.error}"
+        end
+      end
+
+      def judging_dimensions_block
+        @spec.judge_dimensions.map { |d| "  - #{d[:key]} (0-100): #{d[:description]}" }.join("\n")
+      end
+
+      def dimensions_json_template
+        @spec.judge_dimensions.map { |d| "    \"#{d[:key]}\": 0-100" }.join(",\n")
+      end
+
+      def extract_dimensions(raw)
+        return {} unless raw.is_a?(Hash)
+
+        @spec.judge_dimensions.each_with_object({}) do |d, acc|
+          acc[d[:key]] = clamp_score(raw[d[:key]] || raw[d[:key].to_s])
+        end
+      end
+
+      def clamp_score(raw)
+        return nil if raw.nil?
+
+        Integer(raw).clamp(0, 100)
+      rescue ArgumentError, TypeError
+        nil
+      end
+
+      def failure_verdict(latency_ms:, response:, error:)
+        Verdict.new(
+          judge_model: @judge_model, judge_latency_ms: latency_ms,
+          judge_input_tokens: response&.input_tokens, judge_output_tokens: response&.output_tokens,
+          judge_estimated_cost_usd: response&.estimated_cost, judge_error: error,
+          quality_score: 0, dimensions: {}, issues: [], verdict_one_line: ''
+        )
+      end
+    end
+  end
+end

data/lib/llm_conductor/eval/model_runner.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+require_relative 'result'
+
+module LlmConductor
+  module Eval
+    # Runs one (input, model) pair through LlmConductor.generate, capturing
+    # latency / tokens / cost / parse status and writing raw + parsed outputs
+    # through the Store. Side-effect free — never touches the caller's data.
+    #
+    # All feature-specific behavior (prompt type, payload, parsing,
+    # score/bucket extraction) is delegated to the Spec.
+    class ModelRunner
+      # Filesystem-safe slug for a model name (e.g. "gemini-2.5-flash").
+      def self.slug(model)
+        model.to_s.gsub(/[^A-Za-z0-9_.-]+/, '_')
+      end
+
+      def initialize(input, model:, vendor:, spec:, store:, run_id:, logger:, data: nil)
+        @input = input
+        @model = model
+        @vendor = vendor.to_sym
+        @spec = spec
+        @store = store
+        @run_id = run_id
+        @logger = logger
+        @data = data
+      end
+
+      def run
+        input_id = @spec.input_id(@input)
+        data = @data || @spec.build_data(@input)
+
+        started_at = Time.now.utc
+        response = LlmConductor.generate(**generate_args(data, input_id))
+        latency_ms = ((Time.now.utc - started_at) * 1000).round
+
+        raw_ref = @store.write_raw(@run_id, input_id, slug, response&.output.to_s)
+
+        if response.nil? || !response.success?
+          error = response&.metadata&.dig(:error) || 'LLM returned no response'
+          return build_result(input_id:, status: 'llm_error', latency_ms:, response:, raw_ref:, error:)
+        end
+
+        parsed = @spec.parse(response.output)
+        if parsed.nil?
+          return build_result(input_id:, status: 'parse_error', latency_ms:, response:, raw_ref:,
+                              error: 'LLM output not valid structured data')
+        end
+
+        parsed_ref = @store.write_parsed(@run_id, input_id, slug, parsed)
+        build_result(input_id:, status: 'ok', latency_ms:, response:, raw_ref:, parsed_ref:, parsed:)
+      rescue StandardError => e
+        @logger.error("[Eval::ModelRunner] #{@model}@#{@spec.input_id(@input)}: #{e.class}: #{e.message}")
+        Result.new(input_id: @spec.input_id(@input), input_label:, model: @model,
+                   vendor: @vendor, status: 'exception', error: "#{e.class}: #{e.message}")
+      end
+
+      def slug
+        self.class.slug(@model)
+      end
+
+      private
+
+      def generate_args(data, input_id)
+        args = { model: @model, vendor: @vendor }
+        if @spec.prompt_type
+          args[:type] = @spec.prompt_type
+          args[:data] = data
+        else
+          args[:prompt] = data
+        end
+        params = @spec.vendor_params(vendor: @vendor, input_id:)
+        args[:params] = params unless params.nil? || params.empty?
+        args
+      end
+
+      def build_result(input_id:, status:, latency_ms:, response:, raw_ref:, parsed_ref: nil, parsed: nil, error: nil)
+        summary = parsed ? @spec.output_summary(parsed) : { score: nil, bucket: nil }
+        Result.new(
+          input_id:, input_label:, model: @model, vendor: @vendor, status:, latency_ms:,
+          input_tokens: response&.input_tokens, output_tokens: response&.output_tokens,
+          total_tokens: response&.total_tokens, estimated_cost_usd: response&.estimated_cost,
+          parsed_score: summary[:score], parsed_bucket: summary[:bucket],
+          extra_columns: parsed ? @spec.extra_columns(parsed) : {},
+          raw_output_ref: raw_ref, parsed_output_ref: parsed_ref, error:
+        )
+      end
+
+      def input_label
+        @spec.input_label(@input)
+      end
+    end
+  end
+end

data/lib/llm_conductor/eval/report.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module LlmConductor
+  module Eval
+    # Value object returned by a run. Holds the aggregated results and renders
+    # CSV / markdown on demand. The caller decides whether to persist anything —
+    # the engine never forces a filesystem layout on consumers.
+    #
+    # - +rows+         : Array of { model_result: Result, judge_verdict: Verdict }
+    # - +summary+      : Array of per-model aggregate Hashes, best-quality first
+    # - +needs_review+ : Array of Hashes for rows flagged for human eyeball
+    Report = Struct.new(:rows, :summary, :needs_review, :csv_string, :markdown_string, keyword_init: true) do
+      def to_csv
+        csv_string
+      end
+
+      def to_markdown
+        markdown_string
+      end
+    end
+  end
+end