omniai-evals 0.1.0

data/lib/omniai/evals/assertions/abstention.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Evals
+    module Assertions
+      # Passes when the output declines to assert content for a region it cannot read.
+      # Two ways to pass: it carries an abstention marker for the region, or it names
+      # none of the forbidden terms. Hallucinated confidence is the failure this catches.
+      module Abstention
+        # "~~~~" is what the production model actually emits for an illegible region --
+        # a non-lexical marker, measured on a production baseline page. A run of 3 or
+        # more tildes counts, since the model is not consistent about the length.
+        #
+        # The lexical markers are the fallbacks. A criterion can replace this whole list
+        # via target["markers"] when a different target has its own convention.
+        MARKERS = ["[illegible]", "[unreadable]", "[not legible]", "cannot read", "unable to read"].freeze
+        NON_LEXICAL = /~{3,}/
+
+      module_function
+
+        def call(example:, output:, criterion:)
+          raw = Normalize.text(output)
+          text = Normalize.phrase(raw)
+          markers = Array(criterion.target["markers"]).then { |list| list.empty? ? MARKERS : list }
+          forbidden = Array(criterion.target["forbidden_terms"]).map { |term| Normalize.phrase(term) }
+
+          abstained = NON_LEXICAL.match?(raw) || markers.any? { |m| text.include?(Normalize.phrase(m)) }
+          asserted = forbidden.select { |term| text.include?(term) }
+          score = abstained || asserted.empty?
+
+          Result.new(key: criterion.id, score:, comment: comment(score, abstained, asserted, criterion))
+        end
+
+        def comment(score, abstained, asserted, criterion)
+          region = criterion.target["region"]
+          return "asserted #{asserted.join(', ')} for #{region}" unless score
+
+          abstained ? "abstained on #{region}" : "asserted no forbidden term for #{region}"
+        end
+      end
+    end
+  end
+end

data/lib/omniai/evals/assertions/circled.rb

@@ -0,0 +1,180 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Evals
+    module Assertions
+      # Which option on a form was circled -- the `structure` type's `{"circled": ...}` target.
+      #
+      # It lives apart from the rest of Structure because it is the only assertion whose job is
+      # to decide who a MARK belongs to, rather than to compare a value. Every rule here exists
+      # to stop a mark being credited to an option that did not carry it, in either direction:
+      # a correct answer scoring false, or an unmarked page scoring true.
+      module Circled
+        # A transcription marks a circled option one of two ways, and both count.
+        #
+        # LEXICAL: it says so -- "(circled)", "[x]", "circled", "selected".
+        MARKED = /\(circled\)|\[x\]|\bcircled\b|\bselected\b/i
+        # EMPHATIC: it renders the pen circle as emphasis, because production prompts tell it
+        # to "match any formatting as closely as possible" and a circle IS emphasis. Measured
+        # on a physician-completed exam form: four separate configs all transcribed the
+        # circled option in bold and not one of them used the word "circled".
+        #
+        # The single-delimiter forms require non-word boundaries. Without that, `_(.+?)_`
+        # matches the middle of a snake_case token and invents an emphasised span that was
+        # never emphasis -- "field_Normal_name" would mark "Normal" as circled.
+        EMPHASIS = /
+          \*\*(.+?)\*\*
+          | __(.+?)__
+          | (?<![\w*])\*(?!\*)([^*]+?)\*(?![\w*])
+          | (?<![\w_])_(?!_)([^_]+?)_(?![\w_])
+        /x
+        # A model transcribing a pen circle often leaves its ink as stray punctuation --
+        # "(2) Normal)" with an unbalanced paren appeared in three of those four configs --
+        # so text is compared with its punctuation edges trimmed.
+        EDGE_NOISE = /\A[^[:alnum:]]+|[^[:alnum:]]+\z/
+        # A leading option enumerator. Models bold the option text without it ("**Normal**"
+        # for "2) Normal"), so the enumerator is stripped from both sides before comparing.
+        ENUMERATOR = /\A(?:\(\d+\)|\d+[).]|[a-z][).])\s*/
+
+      module_function
+
+        def call(output:, criterion:)
+          target = criterion.target
+          option = target.fetch("circled")
+          siblings = Array(target["among"]).reject { |s| same?(s, option) }
+          field = field_text(Normalize.text(output), option, target)
+          return failure(criterion, "option not found in output") if field.nil?
+
+          rivals = siblings.select { |sibling| marked?(field, sibling, target) }
+          unless rivals.empty?
+            return failure(criterion,
+              "#{rivals.size + 1} of the options are marked; the mark is not discriminating")
+          end
+
+          return pass(criterion, "marked circled") if lexical?(field, option, target)
+          return pass(criterion, "marked by emphasis") if emphasised?(field, option)
+
+          failure(criterion, "option present but not marked")
+        end
+
+        # The lines that carry this field's options -- every line mentioning the target or any
+        # sibling -- rather than the whole page.
+        #
+        # Page scope is wrong in BOTH directions and a form makes both happen. Option strings
+        # repeat: Yes/No, Normal, None, Left/Right. A bolded "No" in an unrelated field three
+        # sections down refuses a correct mark (false negative), and a bolded target in some
+        # other field would accept an unmarked one (false positive). The enumeration is one
+        # line or a short run of adjacent ones, so that is the unit.
+        #
+        # @return [String, nil] nil when the option is not mentioned at all
+        def field_text(text, option, target)
+          lines = text.lines
+          index = lines.index { |line| mentions?(line, option) }
+          return nil if index.nil?
+
+          siblings = Array(target["among"]).reject { |sibling| same?(sibling, option) }
+          first = index
+          first -= 1 while first.positive? && mentions_any?(lines[first - 1], siblings)
+          last = index
+          last += 1 while last < lines.size - 1 && mentions_any?(lines[last + 1], siblings)
+          lines[first..last].join
+        end
+
+        # The run must be CONTIGUOUS, not every line mentioning a sibling. Option words repeat
+        # across a form -- a Yes/No field elsewhere on the page also mentions "No" -- so
+        # collecting every such line drags an unrelated field back in and its emphasis becomes
+        # a phantom rival.
+        def mentions_any?(line, options)
+          options.any? { |option| mentions?(line, option) }
+        end
+
+        # Emphasis is checked across the field; a lexical marker must be ADJACENT to the
+        # option, so it is checked in that option's own window.
+        def marked?(field, option, target)
+          lexical?(field, option, target) || emphasised?(field, option)
+        end
+
+        # A lexical marker counts only for the option it belongs to.
+        #
+        # Anywhere-on-the-line is not discriminating: "1) Slim (circled) 2) Normal 3) Muscular"
+        # would score "2) Normal" as circled, the same hole the emphasis branch already closed.
+        # Nor is a window bounded by the neighbours enough -- a marker sitting BETWEEN two
+        # options falls inside both windows, marks both, and refuses a correct page as
+        # "not discriminating".
+        #
+        # So each marker gets exactly one owner: the option it FOLLOWS, which is how these
+        # transcriptions read ("2) Normal (circled)"). A marker with nothing before it -- a
+        # checkbox opening the line -- belongs to the option it precedes instead.
+        def lexical?(field, option, target)
+          owners(field, target).any? { |owner| same?(owner, option) }
+        end
+
+        # @return [Array<String>] the options that own a lexical marker
+        def owners(field, target)
+          text = Normalize.phrase(field)
+          options = [target.fetch("circled"), *Array(target["among"])].uniq
+          placed = options.filter_map { |option| locate(text, option)&.then { |at| [option, at] } }
+
+          marks(text, target).filter_map { |mark| owner_of(mark, placed) }
+        end
+
+        # @return [Array<Integer>] where each lexical marker starts
+        def marks(text, target)
+          text.enum_for(:scan, markers(target)).map { Regexp.last_match.begin(0) }
+        end
+
+        def owner_of(mark, placed)
+          before = placed.select { |(_, at)| at.last <= mark }.max_by { |(_, at)| at.last }
+          after = placed.select { |(_, at)| at.first >= mark }.min_by { |(_, at)| at.first }
+          (before || after)&.first
+        end
+
+        # @return [Range, nil] where the option sits in the normalized text
+        def locate(text, option)
+          forms(option).filter_map do |form|
+            index = text.index(form)
+            index && (index..(index + form.length))
+          end.min_by(&:first)
+        end
+
+        def mentions?(line, option)
+          text = Normalize.phrase(line)
+          forms(option).any? { |form| text.include?(form) }
+        end
+
+        # True when the option IS the emphasised span -- not merely when its field contains
+        # emphasis somewhere. That distinction is the whole assertion: on a form where every
+        # line carries a bold label, "this contains bold" passes vacuously for every option,
+        # including the ones that were not circled.
+        #
+        # Equality after trimming, deliberately, rather than containment: a span wrapping
+        # several options ("**1) Slim 2) Normal 3) Muscular**") contains the option but marks
+        # none of them, and scoring that as circled is the same vacuous pass one level down.
+        def emphasised?(field, option)
+          wanted = forms(option)
+          field.scan(EMPHASIS).flatten.compact.any? { |span| forms(span).intersect?(wanted) }
+        end
+
+        # The comparable spellings of one option: punctuation edges trimmed, and the same
+        # again without a leading enumerator, because a model may bold "Normal" for "2) Normal".
+        def forms(option)
+          canonical = Normalize.phrase(option).gsub(EDGE_NOISE, "")
+          [canonical, canonical.sub(ENUMERATOR, "").gsub(EDGE_NOISE, "")].uniq.reject(&:empty?)
+        end
+
+        def same?(left, right) = forms(left).intersect?(forms(right))
+
+        # A criterion may supply its own lexical markers for a target with its own convention.
+        def markers(target)
+          custom = Array(target["markers"])
+          return MARKED if custom.empty?
+
+          Regexp.union(custom.map { |marker| Regexp.new(Regexp.escape(marker), Regexp::IGNORECASE) })
+        end
+
+        def pass(criterion, comment) = Result.new(key: criterion.id, score: true, comment:)
+        def failure(criterion, comment) = Result.new(key: criterion.id, score: false, comment:)
+      end
+    end
+  end
+end

data/lib/omniai/evals/assertions/identity.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Evals
+    module Assertions
+      # "A header name is transcribed AS-IS" (docs/contract.md).
+      #
+      # Two target shapes, selected by which key is present:
+      #
+      #   {"text": "Patient: JANE Q."}       text mode -- the output must contain this
+      #                                       string exactly, after whitespace collapse
+      #   {"field": "claim_number",           structured mode -- the output is a Hash and
+      #    "value": "ABC-123"}                that field equals this value
+      #
+      # Text mode is CASE-SENSITIVE and normalizes nothing but whitespace runs, unlike
+      # every other assertion here. That is the point of the type: a banded EHR table
+      # whose header names a DIFFERENT patient must be transcribed as-is, and an OCR pass
+      # that silently "fixes" the case or the spelling has produced the wrong-patient
+      # failure this criterion exists to detect. Case-folding here would hide it.
+      #
+      # Neither mode is fuzzy. A claim number that is nearly right is wrong.
+      module Identity
+      module_function
+
+        def call(example:, output:, criterion:)
+          criterion.target.key?("text") ? text(output, criterion) : field(output, criterion)
+        end
+
+        def text(output, criterion)
+          expected = Normalize.squish(criterion.target.fetch("text"))
+          actual = Normalize.squish(Normalize.text(output))
+          ok = actual.include?(expected)
+          Result.new(key: criterion.id, score: ok,
+            comment: ok ? "transcribed as-is" : "not transcribed as-is (exact, case-sensitive)")
+        end
+
+        def field(output, criterion)
+          name = criterion.target.fetch("field")
+          expected = Normalize.squish(criterion.target.fetch("value").to_s)
+          actual = output.is_a?(Hash) ? Normalize.squish(output[name].to_s) : nil
+
+          ok = !actual.nil? && actual == expected
+          Result.new(key: criterion.id, score: ok, comment: ok ? "#{name} matches" : "#{name} mismatch")
+        end
+      end
+    end
+  end
+end

data/lib/omniai/evals/assertions/presence.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Evals
+    module Assertions
+      # Passes when the key's date (compared parsed) or text (compared with case and
+      # whitespace runs normalized) appears in the output.
+      module Presence
+      module_function
+
+        def call(example:, output:, criterion:)
+          text = Normalize.text(output)
+          key = criterion.target
+
+          score, comment = key.key?("date") ? date(text, key) : text_match(text, key)
+          Result.new(key: criterion.id, score:, comment:)
+        end
+
+        def date(text, key)
+          wanted = Dates.parse(key.fetch("date"))
+          found = Dates.scan(text)
+          hit = found.include?(wanted)
+          [hit, hit ? "date #{wanted} found (parsed)" : "date #{wanted} not among #{found.size} parsed dates"]
+        end
+
+        def text_match(text, key)
+          hit = Normalize.phrase(text).include?(Normalize.phrase(key.fetch("text")))
+          [hit, hit ? "text found" : "text not found"]
+        end
+      end
+    end
+  end
+end

data/lib/omniai/evals/assertions/structure.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Evals
+    module Assertions
+      # "Row count of a banded table; which option is circled" (docs/contract.md) -- a
+      # different skill from transcription, so it gets its own type.
+      #
+      # Three target shapes, selected by which key is present:
+      #
+      #   {"table_rows": 12}                  text mode -- the output's markdown table has
+      #                                       this many BODY rows (header and the ---
+      #                                       separator excluded)
+      #   {"circled": "Option B"}             text mode -- the output marks this option as
+      #                                       circled/selected
+      #   {"path": ["exam", "date"],          structured mode -- the output is a Hash, the
+      #    "type": "string"}                  path resolves, the value has that JSON type
+      #
+      # Text mode exists because the first consumer's target emits markdown, not JSON.
+      # A Hash-only assertion silently fails every example of a text bench, which reads
+      # identically to a model that got everything wrong.
+      module Structure
+        TYPES = {
+          "string" => [String],
+          "number" => [Numeric],
+          "boolean" => [TrueClass, FalseClass],
+          "array" => [Array],
+          "object" => [Hash],
+        }.freeze
+
+        # A markdown separator row: | --- | :--: | and friends. Never a body row.
+        SEPARATOR = /\A\s*\|[\s:|-]+\|\s*\z/
+        ROW = /\A\s*\|/
+
+      module_function
+
+        def call(example:, output:, criterion:)
+          target = criterion.target
+          return table_rows(output, criterion) if target.key?("table_rows")
+          return Circled.call(output:, criterion:) if target.key?("circled")
+
+          path(output, criterion)
+        end
+
+        # Body rows only: the header row and the --- separator are table syntax, not data,
+        # and counting them would make every key off by two.
+        def table_rows(output, criterion)
+          rows = Normalize.text(output).lines.grep(ROW).grep_v(SEPARATOR)
+          body = [rows.size - 1, 0].max # the first remaining row is the header
+          expected = criterion.target.fetch("table_rows")
+          ok = body == expected
+          Result.new(key: criterion.id, score: ok, comment: "#{body} body rows, expected #{expected}")
+        end
+
+        def path(output, criterion)
+          return failure(criterion, "output is not a structure (#{output.class})") unless output.is_a?(Hash)
+
+          steps = criterion.target.fetch("path")
+          expected = criterion.target.fetch("type")
+          value = dig(output, steps)
+          return failure(criterion, "path #{steps.join('.')} missing") if value.nil?
+
+          if matches?(value, expected)
+            pass(criterion, "#{steps.join('.')} is #{expected}")
+          else
+            failure(criterion, "#{steps.join('.')} is #{value.class}")
+          end
+        end
+
+        # Walks the path, giving up (nil) the moment a step lands on a non-Hash.
+        def dig(output, steps)
+          steps.reduce(output) { |node, step| node.is_a?(Hash) ? node[step] : nil }
+        end
+
+        def matches?(value, expected)
+          klasses = TYPES.fetch(expected) { raise ArgumentError, %(unknown structure type "#{expected}") }
+          klasses.any? { |klass| value.is_a?(klass) }
+        end
+
+        def pass(criterion, comment) = Result.new(key: criterion.id, score: true, comment:)
+        def failure(criterion, comment) = Result.new(key: criterion.id, score: false, comment:)
+      end
+    end
+  end
+end

data/lib/omniai/evals/assertions.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require_relative "assertions/presence"
+require_relative "assertions/absence"
+require_relative "assertions/circled"
+require_relative "assertions/structure"
+require_relative "assertions/abstention"
+require_relative "assertions/identity"
+
+module OmniAI
+  module Evals
+    # The deterministic, free scorers -- one per criterion type in docs/contract.md 3.2.
+    # "faithful" is deliberately absent: it is a judge, not an assertion, and asking for
+    # it here should fail loudly rather than fall back to a string compare.
+    module Assertions
+      REGISTRY = {
+        "presence" => Presence,
+        "absence" => Absence,
+        "structure" => Structure,
+        "abstention" => Abstention,
+        "identity" => Identity,
+      }.freeze
+
+    module_function
+
+      # @raise [ArgumentError]
+      def for(type)
+        REGISTRY.fetch(type.to_s) { raise ArgumentError, %(no assertion for "#{type}") }
+      end
+
+      def registered = REGISTRY.keys
+    end
+  end
+end

data/lib/omniai/evals/compare.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Evals
+    # The comparability key -- (dataset sha, criteria sha, prompt_version) -- and the check
+    # that refuses to compare runs it does not match.
+    #
+    # Invariant 4: a mismatched key is NEVER silently compared. `force:` exists because
+    # sometimes you genuinely want to look anyway, but it returns a warning the caller
+    # must put in the report rather than quietly proceeding.
+    module Compare
+    module_function
+
+      def key(dataset_sha:, criteria_sha:, prompt_version:)
+        [dataset_sha, criteria_sha, prompt_version].join("|")
+      end
+
+      # @raise [ComparabilityError]
+      # @return [nil, String] nil when comparable; a warning when forced
+      def check!(left, right, force: false)
+        a = left["comparability_key"]
+        b = right["comparability_key"]
+
+        raise ComparabilityError, "missing comparability_key (#{a.inspect} vs #{b.inspect})" if a.nil? || b.nil?
+        return nil if a == b
+
+        message = "comparability keys differ:\n  #{a}\n  #{b}"
+        raise ComparabilityError, message unless force
+
+        "FORCED COMPARISON -- #{message}"
+      end
+    end
+  end
+end

data/lib/omniai/evals/contract.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Evals
+    # The version stamped into every file the gem reads or writes. Bump it in the same
+    # commit as any change to a format in docs/contract.md section 3 -- the fixtures under
+    # spec/fixtures/contract/ are pinned to it, so a silent format drift fails the suite.
+    CONTRACT_VERSION = 1
+
+    # Guards the "contract" field on every file we load.
+    module Contract
+    module_function
+
+      # @param payload [Hash] a parsed JSON document
+      # @param path [String] where it came from, for the error message
+      # @raise [ContractError]
+      # @return [Hash] the payload, unchanged
+      def check!(payload, path:)
+        version = payload["contract"]
+        raise ContractError, %(#{path}: missing "contract" version) if version.nil?
+        raise ContractError, "#{path}: contract #{version}, expected #{CONTRACT_VERSION}" unless
+          version == CONTRACT_VERSION
+
+        payload
+      end
+    end
+  end
+end

data/lib/omniai/evals/criteria.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require "digest"
+
+module OmniAI
+  module Evals
+    # Every criterion for a dataset, indexed by example id.
+    class Criteria
+      # @param dataset [Dataset]
+      # @return [Criteria]
+      def self.load(dataset:)
+        index = {}
+        digest = Digest::SHA256.new
+
+        # Manifest order, not directory order: the sha has to be reproducible, and
+        # Dir.glob's order is filesystem-dependent.
+        dataset.examples.each do |example|
+          path = File.join(dataset.dir, example.target)
+          digest << File.binread(path)
+          payload = Contract.check!(JSON.parse(File.read(path)), path:)
+          index[payload.fetch("example_id")] = payload.fetch("criteria").map do |row|
+            Criterion.new(
+              id: row.fetch("id"),
+              family: row.fetch("family"),
+              type: row.fetch("type"),
+              scorer: row.fetch("scorer"),
+              target: row.fetch("target", {}),
+              provenance: row.fetch("provenance", {})
+            )
+          end
+        end
+
+        new(index:, sha256: digest.hexdigest)
+      end
+
+      attr_reader :sha256
+
+      def initialize(index:, sha256:)
+        @index = index.freeze
+        @sha256 = sha256
+        freeze
+      end
+
+      # @return [Array<Criterion>]
+      def for(example_id) = @index.fetch(example_id, [])
+
+      # @return [Array<Criterion>]
+      def all = @index.values.flatten
+    end
+  end
+end

data/lib/omniai/evals/criterion.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Evals
+    # One atomic, binary check on one example. `type` selects the assertion (or the judge
+    # prompt, for "faithful"); `target` shape is per type and documented in docs/contract.md.
+    Criterion = Data.define(:id, :family, :type, :scorer, :target, :provenance)
+  end
+end

data/lib/omniai/evals/dataset.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "digest"
+
+module OmniAI
+  module Evals
+    # A frozen set of examples, identified by a content hash.
+    class Dataset
+      attr_reader :name, :examples, :dir
+
+      # @param path [String] the manifest.json to load
+      # @raise [ContractError]
+      # @return [Dataset]
+      def self.load(path)
+        payload = Contract.check!(JSON.parse(File.read(path)), path:)
+        examples = payload.fetch("examples", []).map do |row|
+          Example.new(
+            id: row.fetch("id"),
+            input: row.fetch("input"),
+            target: row.fetch("target"),
+            metadata: row.fetch("metadata", {})
+          )
+        end
+        new(name: payload.fetch("name"), examples:, dir: File.dirname(File.expand_path(path)),
+          manifest_bytes: File.binread(path))
+      end
+
+      def initialize(name:, examples:, dir:, manifest_bytes:)
+        @name = name
+        @examples = examples.freeze
+        @dir = dir
+        @manifest_bytes = manifest_bytes
+      end
+
+      # Absolute path to an example's input file.
+      #
+      # @raise [ContractError] for an input with no file behind it. "messages" inputs are
+      #   inline, so asking for their path is a caller bug, not a missing file -- saying so
+      #   beats returning a path that does not exist.
+      def input_path_for(example)
+        path = example.input["path"]
+        raise ContractError, %(example #{example.id}: input kind "#{example.input['kind']}" has no path) if path.nil?
+
+        File.join(@dir, path)
+      end
+
+      # Every example's input file, in manifest order. Inline inputs contribute nothing.
+      def input_paths
+        @examples.filter_map { |example| example.input["path"] && input_path_for(example) }
+      end
+
+      # SHA-256 over the manifest bytes plus every referenced input file, in manifest
+      # order. Hashing the file CONTENT rather than its path is what makes a re-render a
+      # different dataset -- the manifest is byte-identical when only the pixels changed.
+      def sha256
+        @sha256 ||= begin
+          digest = Digest::SHA256.new
+          digest << @manifest_bytes
+          input_paths.each { |path| digest << File.binread(path) }
+          digest.hexdigest
+        end
+      end
+    end
+  end
+end