omniai-evals 0.1.0

data/lib/omniai/evals/dates.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require "date"
+
+module OmniAI
+  module Evals
+    # Dates are compared PARSED, never as strings: "2019-03-14", "03/14/2019" and
+    # "March 14, 2019" are the same day, and a string compare scores three different
+    # transcriptions of one correct answer as two failures.
+    module Dates
+      # Numeric slash AND dash dates are read as US month-first (MM/DD/YYYY, MM-DD-YYYY),
+      # NOT by Date.parse.
+      #
+      # This is not a style preference, it is a correctness fix. Ruby's Date.parse reads
+      # a slash date as DD/MM/YYYY, so it returns nil for "03/14/2019" (there is no month
+      # 14) and -- far worse -- silently returns February 4th for "04/02/2019". A judge
+      # would then mark a correct April 2nd transcription as wrong, with no error anywhere
+      # to notice. Wrong-but-plausible is the failure mode this whole gem exists to catch,
+      # so it cannot live in the date comparison itself.
+      #
+      # STATED DEFAULT (docs/contract.md): every dataset this grades is US, so month-first
+      # is the rule. A non-US dataset needs a per-dataset locale field in the contract --
+      # it is not something to infer from the text, because "04/02/2019" is valid in both
+      # readings and guessing reintroduces exactly the silent error above. An unambiguous
+      # key should be written ISO in the criteria file regardless.
+      US_NUMERIC = %r{\A(\d{1,2})[/-](\d{1,2})[/-](\d{2,4})\z}
+
+      # ISO stays ISO: "2019-03-14" is unambiguous and must NOT hit the US rule.
+      ISO = /\A\d{4}-\d{2}-\d{2}\z/
+      # "2019/03/14" is year-first by shape, so it is unambiguous too.
+      ISO_SLASH = %r{\A(\d{4})/(\d{1,2})/(\d{1,2})\z}
+
+      MONTH = "January|February|March|April|May|June|July|August|September|October|November|December|" \
+        "Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Sept|Oct|Nov|Dec"
+
+      # Every date shape seen in the faxed medical records this grades. Order matters:
+      # the alternation is tried left to right, so longer textual forms come before the
+      # numeric ones that could match a prefix of them.
+      TOKEN = Regexp.union(
+        /(?:#{MONTH})\.?\s+\d{1,2},?\s+\d{4}/,     # March 14, 2019 / Mar. 14 2019 / Mar 14 2019
+        /\d{1,2}\s+(?:#{MONTH})\.?,?\s+\d{4}/,     # 14 March 2019
+        %r{\d{4}[-/]\d{1,2}[-/]\d{1,2}},           # 2019-03-14 / 2019/03/14
+        %r{\d{1,2}[/-]\d{1,2}[/-]\d{2,4}}          # 03/14/2019 / 03-14-2019 / 3/14/19
+      )
+
+    module_function
+
+      # @return [Date, nil] nil when the value is not a date
+      def parse(value)
+        return nil if value.nil?
+
+        text = value.to_s.strip
+        return iso(text) if ISO.match?(text)
+
+        slash = ISO_SLASH.match(text)
+        return ymd(slash[1], slash[2], slash[3]) if slash
+
+        us = US_NUMERIC.match(text)
+        return us_numeric(us) if us
+
+        textual(text)
+      end
+
+      # @return [Array<Date>] every date-shaped token in a blob of text, parsed. Used by
+      #   the presence assertion so a date key matches whatever format the model wrote.
+      def scan(text)
+        text.to_s.scan(TOKEN).filter_map { |token| parse(token) }
+      end
+
+      # A textual month ("Mar. 14, 2019", "14 March 2019") is unambiguous, so Date.parse
+      # is safe here -- the day-first trap only applies to all-numeric forms.
+      def textual(text)
+        Date.parse(text)
+      rescue Date::Error
+        nil
+      end
+
+      def iso(text)
+        Date.iso8601(text)
+      rescue Date::Error
+        nil
+      end
+
+      def us_numeric(match)
+        month, day, year = match.captures
+        year = expand_year(year)
+        ymd(year, month, day)
+      end
+
+      # A two-digit year in a medical record is this century unless that would be absurd;
+      # 70 is the conventional pivot and matches Ruby's own strptime behaviour.
+      def expand_year(year)
+        return year if year.length == 4
+
+        year.to_i < 70 ? "20#{year}" : "19#{year}"
+      end
+
+      def ymd(year, month, day)
+        Date.new(year.to_i, month.to_i, day.to_i)
+      rescue Date::Error
+        nil
+      end
+    end
+  end
+end

data/lib/omniai/evals/errors.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Evals
+    # Base for every error the gem raises.
+    class Error < StandardError; end
+
+    # A file is not the contract version this gem speaks, or is missing a required field.
+    class ContractError < Error; end
+
+    # Two runs were compared whose comparability keys do not match.
+    class ComparabilityError < Error; end
+
+    # An adapter was used without being required. The core loads on stdlib alone, so
+    # "default adapter" is a documentation claim, not an implicit require: naming the
+    # missing require is the whole point of this error.
+    class AdapterNotLoaded < Error; end
+  end
+end

data/lib/omniai/evals/example.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Evals
+    # One unit under test. `input` is the section 3.1 input object ("kind" is
+    # image/text/messages); `target` is the path to this example's criteria file,
+    # relative to the manifest.
+    Example = Data.define(:id, :input, :target, :metadata)
+  end
+end

data/lib/omniai/evals/judge.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Evals
+    # The shape of a judge criterion: one boolean field plus a short reason.
+    #
+    # `packet` takes key_text and output and NOTHING else. That is the enforcement of
+    # "the judge receives the key text and the output, never the example input" -- there
+    # is no parameter through which the input could arrive, so no caller can leak it.
+    #
+    # The schema is a plain JSON Schema Hash, not an omniai type: the core loads on
+    # stdlib alone (invariant 3), and both adapters want raw JSON Schema anyway --
+    # ClaudeCLI passes it to --json-schema, and Adapters::OmniAI deserializes it.
+    module Judge
+    module_function
+
+      def schema
+        {
+          type: "object",
+          properties: {
+            passed: { type: "boolean", description: "true when the output is faithful to the key" },
+            reason: { type: "string", description: "one sentence, under 200 characters" },
+          },
+          required: %i[passed reason],
+        }
+      end
+
+      def packet(key_text:, output:)
+        <<~PACKET
+          KEY (human-authored ground truth):
+          #{key_text}
+
+          OUTPUT (model under test):
+          #{output}
+        PACKET
+      end
+
+      # @raise [Error] when the verdict's score is not a boolean. A lax provider can
+      #   return 0.8 or "true"; the raw ArgumentError from Result names the class but not
+      #   the criterion, which is useless in a run over hundreds of them.
+      def result(verdict, criterion:)
+        Result.new(key: criterion.id, score: verdict["passed"], comment: verdict["reason"].to_s)
+      rescue ArgumentError => e
+        raise Error, "judge verdict for #{criterion.id}: #{e.message}"
+      end
+    end
+  end
+end

data/lib/omniai/evals/manifest.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require "open3"
+
+module OmniAI
+  module Evals
+    # The run manifest written beside the results.
+    #
+    # The first five fields match the first consumer's existing run manifest, in the
+    # same shape, so its port is a rename rather than a rewrite. Invariant 2 applies:
+    # nothing here may carry input or output text -- argv is sanitized on the way in.
+    module Manifest
+      REDACTED = "<redacted>"
+
+    module_function
+
+      def build(tool:, repo_root:, run_id:, dataset:, criteria:, target:, model:, epochs:,
+        argv: ARGV, now: Time.now.utc, params: {}, seed: "none", reducer: "all_epochs",
+        gems: {}, judge: nil)
+        {
+          "contract" => CONTRACT_VERSION,
+          "run_id" => run_id,
+          "tool" => tool,
+          "argv_sanitized" => sanitize_argv(argv),
+          "toolkit_sha" => toolkit_sha(repo_root),
+          "ruby_version" => RUBY_VERSION,
+          "ts" => now.strftime("%Y-%m-%dT%H:%M:%SZ"),
+          "dataset" => dataset,
+          "criteria" => criteria,
+          "target" => target,
+          "model" => model,
+          "params" => params,
+          "seed" => seed,
+          "epochs" => epochs,
+          "reducer" => reducer,
+          "gems" => gems,
+          "judge" => judge,
+          "comparability_key" => Compare.key(dataset_sha: dataset.fetch("sha256"),
+            criteria_sha: criteria.fetch("sha256"),
+            prompt_version: target.fetch("prompt_version")),
+        }.compact
+      end
+
+      # Redacts the value following --claimant (both "--claimant X" and "--claimant=X"
+      # forms) so the manifest never records a claimant name; all other args pass through.
+      def sanitize_argv(argv)
+        redact_next = false
+        argv.map do |arg|
+          if redact_next
+            redact_next = false
+            REDACTED
+          elsif arg == "--claimant"
+            redact_next = true
+            arg
+          elsif arg.start_with?("--claimant=")
+            "--claimant=#{REDACTED}"
+          else
+            arg
+          end
+        end
+      end
+
+      # The toolkit's short git SHA for provenance; "unknown" when repo_root isn't a repo
+      # or git is unavailable (never fails the run over provenance).
+      def toolkit_sha(repo_root)
+        stdout, _stderr, status = Open3.capture3("git", "-C", repo_root, "rev-parse", "--short", "HEAD")
+        status.success? ? stdout.strip : "unknown"
+      rescue Errno::ENOENT
+        "unknown"
+      end
+    end
+  end
+end

data/lib/omniai/evals/normalize.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Evals
+    # Text tidying shared by the assertions. Kept in one place so "does the output
+    # contain this" means the same thing in every assertion.
+    module Normalize
+    module_function
+
+      # Output may be a String or the Hash a structured target returned.
+      def text(output) = output.is_a?(String) ? output : JSON.generate(output)
+
+      # Collapse whitespace runs; keep the case.
+      def squish(value) = value.to_s.gsub(/\s+/, " ").strip
+
+      # Collapse whitespace runs AND case -- the comparison form for contains-checks.
+      def phrase(value) = squish(value).downcase
+    end
+  end
+end

data/lib/omniai/evals/plan.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Evals
+    # What a run WOULD do: examples x configs x epochs, estimated calls and cost, and
+    # zero model calls.
+    #
+    # `adapter:` is accepted and deliberately never invoked -- the spec passes one that
+    # raises, so invariant 5 is enforced by the signature rather than by convention.
+    # Prices come from the consumer: the gem knows nothing about any provider's rates.
+    #
+    # `configs:` is a PLANNING dimension only. A run is one target version, so each
+    # config is its own run with its own manifest and its own results file -- this
+    # exists so you can price "default vs effort:low" in one go BEFORE committing the
+    # spend, not so two configs share a results file. Results.write refuses to mix
+    # run_ids in one file, which is what actually prevents the merge.
+    module Plan
+    module_function
+
+      def build(dataset:, criteria:, configs:, epochs:, prices:, adapter: nil)
+        examples = dataset.examples.size
+        judge_criteria = criteria.all.count { |criterion| criterion.scorer == "judge" }
+        target_calls = examples * configs.size * epochs
+        judge_calls = judge_criteria * configs.size * epochs
+
+        {
+          "examples" => examples,
+          "configs" => configs.size,
+          "epochs" => epochs,
+          "judge_criteria" => judge_criteria,
+          "calls" => target_calls + judge_calls,
+          "target_calls" => target_calls,
+          "judge_calls" => judge_calls,
+          "target_cost_usd" => cost(target_calls, prices["target"]),
+          "judge_cost_usd" => cost(judge_calls, prices["judge"]),
+          "cost_usd" => cost(target_calls, prices["target"]) + cost(judge_calls, prices["judge"]),
+        }
+      end
+
+      # The consumer supplies the price table; an absent price is 0.0 rather than an
+      # error, because a bench of pure assertions has no metered call to price.
+      def cost(calls, price) = (calls * (price || 0.0)).round(4)
+
+      def render(plan)
+        format("%<examples>d examples x %<configs>d configs x %<epochs>d epochs\n" \
+          "%<target_calls>d target calls, estimated $%<target_cost>.2f\n" \
+          "%<judge_calls>d judge calls, estimated $%<judge_cost>.2f\n" \
+          "%<calls>d calls total, estimated $%<cost>.2f\n",
+          examples: plan.fetch("examples"), configs: plan.fetch("configs"),
+          epochs: plan.fetch("epochs"), calls: plan.fetch("calls"),
+          target_calls: plan.fetch("target_calls"), target_cost: plan.fetch("target_cost_usd"),
+          judge_calls: plan.fetch("judge_calls"), judge_cost: plan.fetch("judge_cost_usd"),
+          cost: plan.fetch("cost_usd"))
+      end
+    end
+  end
+end

data/lib/omniai/evals/record.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Evals
+    # One line of the results file: (example, criterion, epoch).
+    #
+    # `key`/`score`/`comment` are the LangSmith / Braintrust / openevals record and the
+    # only required fields; the rest are ours. THIS FILE IS PHI-BEARING -- a judge comment
+    # may quote output -- so the consumer stores it under its own protected path. The
+    # scorecard never carries comments.
+    module Record
+      # docs/contract.md section 2: a scorer is an assertion, a judge, or a label. A typo
+      # here would otherwise reach the scorecard as its own silent family.
+      SCORERS = %w[assertion judge label].freeze
+
+    module_function
+
+      def build(result:, run_id:, example_id:, family:, scorer:, epoch:, cost_usd: 0.0, latency_ms: 0)
+        raise ArgumentError, "unknown scorer #{scorer.inspect}" unless SCORERS.include?(scorer.to_s)
+
+        {
+          "contract" => CONTRACT_VERSION,
+          "run_id" => run_id,
+          "example_id" => example_id,
+          "key" => result.key,
+          "score" => result.score,
+          "comment" => result.comment,
+          "family" => family,
+          "scorer" => scorer,
+          "epoch" => epoch,
+          "cost_usd" => cost_usd,
+          "latency_ms" => latency_ms,
+        }
+      end
+    end
+  end
+end

data/lib/omniai/evals/reducer.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Evals
+    # How the epochs of one criterion combine into one verdict.
+    #
+    # `all_epochs` is the default and the only reducer in v1. FLAKY is a first-class
+    # verdict rather than a rounding decision: a criterion that passes 3 of 5 times is
+    # not "60% correct", it is unreliable, and the reader needs to see that.
+    module Reducer
+    module_function
+
+      # @param results [Array<Result>] one per epoch, for a single criterion
+      # @return [Symbol] :pass, :fail, or :flaky
+      def all_epochs(results)
+        raise ArgumentError, "no epochs to reduce" if results.empty?
+
+        scores = results.map(&:score)
+        return :pass if scores.all?
+        return :fail if scores.none?
+
+        :flaky
+      end
+    end
+  end
+end

data/lib/omniai/evals/result.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Evals
+    # One graded criterion. `key`/`score`/`comment` are the LangSmith / Braintrust /
+    # openevals record and the only required fields in a result line.
+    #
+    # The boolean check is here, at the single point every scorer funnels through, so
+    # invariant 1 ("no float ever reaches a result record") cannot be violated by a
+    # future scorer that returns a confidence. Data is frozen and setter-free, so a
+    # validated score is the score that gets written.
+    Result = Data.define(:key, :score, :comment) do
+      def initialize(key:, score:, comment:)
+        unless score.is_a?(TrueClass) || score.is_a?(FalseClass)
+          raise ArgumentError, "score must be true or false, got #{score.class}"
+        end
+
+        super(key: key.to_s, score:, comment: comment.to_s)
+      end
+    end
+  end
+end

data/lib/omniai/evals/results.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Evals
+    # The JSONL results file.
+    module Results
+    module_function
+
+      # Appends rather than truncates: a run writes epoch by epoch, and losing the
+      # earlier epochs to a reopen would silently change every reduced verdict.
+      #
+      # Appending is therefore also the hazard. Re-running a runner into a path that
+      # already holds another run doubles every epoch, and the reducer sees 2n epochs
+      # without anything raising -- a PASS built from two runs it was never asked to
+      # combine. So a file holds exactly one run_id, and a second one is refused.
+      #
+      # @param records [Array<Hash>]
+      # @raise [Error] when the records do not belong to the file's existing run
+      def write(path, records)
+        return if records.empty?
+
+        guard_run_id!(path, records)
+        File.open(path, "a") do |file|
+          records.each { |record| file.puts(JSON.generate(record)) }
+        end
+      end
+
+      def guard_run_id!(path, records)
+        incoming = records.map { |record| record.fetch("run_id") }.uniq
+        raise Error, "records span #{incoming.size} run_ids: #{incoming.join(', ')}" unless incoming.size == 1
+        return unless File.exist?(path) && !File.empty?(path)
+
+        existing = JSON.parse(File.foreach(path).first).fetch("run_id")
+        return if existing == incoming.first
+
+        raise Error, "#{path} holds run #{existing}; refusing to append run #{incoming.first}"
+      end
+
+      # @raise [ContractError]
+      # @return [Array<Hash>]
+      def read(path)
+        File.readlines(path).reject { |line| line.strip.empty? }.map.with_index(1) do |line, number|
+          Contract.check!(JSON.parse(line), path: "#{path}:#{number}")
+        end
+      end
+    end
+  end
+end

data/lib/omniai/evals/scorecard.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Evals
+    # The PHI-free aggregate: counts, all-pass on top, family rates beneath.
+    #
+    # Counts, not percentages -- small datasets make rates misleading and the reader can
+    # divide. Invariant 2 is enforced by construction: this builds a fresh Hash of
+    # integers from the records rather than filtering fields out of them, so there is no
+    # path by which a comment reaches the scorecard.
+    module Scorecard
+    module_function
+
+      def build(records:, run_id:, comparability_key:, cost_usd: nil, calls: nil)
+        by_key = records.group_by { |record| record.fetch("key") }
+        verdicts = by_key.transform_values { |rows| reduce(rows) }
+
+        {
+          "contract" => CONTRACT_VERSION,
+          "run_id" => run_id,
+          "comparability_key" => comparability_key,
+          "examples" => examples_for(by_key, verdicts),
+          "families" => families_for(by_key, verdicts),
+          "cost_usd" => cost_usd,
+          "calls" => calls,
+        }
+      end
+
+      # Comments are dropped here, not carried and filtered later: the Result rebuilt for
+      # the reducer takes an empty comment, so PHI has no route into this method's output.
+      def reduce(rows)
+        Reducer.all_epochs(rows.map { |row| Result.new(key: row.fetch("key"), score: row.fetch("score"), comment: "") })
+      end
+
+      # An example is only all_pass when every one of its criteria passed; a single FAIL
+      # makes it failed, otherwise a FLAKY makes it flaky.
+      def examples_for(by_key, verdicts)
+        by_example = Hash.new { |hash, key| hash[key] = [] }
+        by_key.each { |key, rows| by_example[rows.first.fetch("example_id")] << verdicts.fetch(key) }
+
+        counts = { "total" => by_example.size, "all_pass" => 0, "flaky" => 0, "failed" => 0 }
+        by_example.each_value { |list| counts[bucket(list)] += 1 }
+        counts
+      end
+
+      def bucket(verdicts)
+        return "failed" if verdicts.include?(:fail)
+        return "flaky" if verdicts.include?(:flaky)
+
+        "all_pass"
+      end
+
+      def families_for(by_key, verdicts)
+        by_key.each_with_object({}) do |(key, rows), acc|
+          family = rows.first.fetch("family")
+          acc[family] ||= { "criteria" => 0, "pass" => 0, "flaky" => 0, "fail" => 0 }
+          acc[family]["criteria"] += 1
+          acc[family][verdicts.fetch(key).to_s] += 1
+        end
+      end
+    end
+  end
+end

data/lib/omniai/evals/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Evals
+    VERSION = "0.1.0"
+  end
+end

data/lib/omniai/evals.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require "json"
+
+require_relative "evals/version"
+require_relative "evals/errors"
+require_relative "evals/contract"
+require_relative "evals/result"
+require_relative "evals/example"
+require_relative "evals/dataset"
+require_relative "evals/criterion"
+require_relative "evals/criteria"
+require_relative "evals/normalize"
+require_relative "evals/dates"
+require_relative "evals/assertions"
+require_relative "evals/reducer"
+require_relative "evals/record"
+require_relative "evals/results"
+require_relative "evals/compare"
+require_relative "evals/manifest"
+require_relative "evals/scorecard"
+require_relative "evals/plan"
+require_relative "evals/judge"
+
+# Structure and grading for LLM evals.
+#
+# The core loads with stdlib and `json` only. Judge adapters reach a model and are
+# therefore NOT required here -- a consumer requires the one it wants:
+#
+#   require "omniai/evals"
+#   require "omniai/evals/adapters/omniai"
+#
+module OmniAI
+  # Structure and grading for LLM evals: datasets, boolean criteria, assertions,
+  # judges, reducers, and PHI-free scorecards.
+  module Evals
+  end
+end

metadata

@@ -0,0 +1,91 @@
+--- !ruby/object:Gem::Specification
+name: omniai-evals
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Dean Singh
+bindir: exe
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: omniai
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.8'
+description: Datasets, boolean criteria, assertions, judges, reducers, and PHI-free
+  scorecards. A companion to the omniai gem family.
+email:
+- dean.singh@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- Gemfile
+- LICENSE
+- README.md
+- lib/omniai/evals.rb
+- lib/omniai/evals/adapters/call_record.rb
+- lib/omniai/evals/adapters/claude_cli.rb
+- lib/omniai/evals/adapters/omniai.rb
+- lib/omniai/evals/assertions.rb
+- lib/omniai/evals/assertions/absence.rb
+- lib/omniai/evals/assertions/abstention.rb
+- lib/omniai/evals/assertions/circled.rb
+- lib/omniai/evals/assertions/identity.rb
+- lib/omniai/evals/assertions/presence.rb
+- lib/omniai/evals/assertions/structure.rb
+- lib/omniai/evals/compare.rb
+- lib/omniai/evals/contract.rb
+- lib/omniai/evals/criteria.rb
+- lib/omniai/evals/criterion.rb
+- lib/omniai/evals/dataset.rb
+- lib/omniai/evals/dates.rb
+- lib/omniai/evals/errors.rb
+- lib/omniai/evals/example.rb
+- lib/omniai/evals/judge.rb
+- lib/omniai/evals/manifest.rb
+- lib/omniai/evals/normalize.rb
+- lib/omniai/evals/plan.rb
+- lib/omniai/evals/record.rb
+- lib/omniai/evals/reducer.rb
+- lib/omniai/evals/result.rb
+- lib/omniai/evals/results.rb
+- lib/omniai/evals/scorecard.rb
+- lib/omniai/evals/version.rb
+homepage: https://github.com/DeanSingh/omniai-evals
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/DeanSingh/omniai-evals
+  changelog_uri: https://github.com/DeanSingh/omniai-evals/blob/main/CHANGELOG.md
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.2.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.9
+specification_version: 4
+summary: Structure and grading for LLM evals
+test_files: []