ruby_llm-tribunal 0.1.0

data/lib/tasks/tribunal.rake

@@ -0,0 +1,269 @@
+# frozen_string_literal: true
+
+require 'ruby_llm/tribunal'
+
+namespace :tribunal do
+  desc 'Run LLM evaluations from dataset files'
+  task :eval, [:files] => :environment do |_t, args|
+    options = parse_options
+
+    files = if args[:files]
+              args[:files].split(',')
+            elsif options[:files]
+              options[:files]
+            else
+              find_default_files
+            end
+
+    if files.empty?
+      puts 'No eval files found. Create datasets in test/evals/ or spec/evals/'
+      exit 0
+    end
+
+    provider = parse_provider(options[:provider])
+    format = options[:format] || 'console'
+    output = options[:output]
+    threshold = options[:threshold]
+    strict = options[:strict] || false
+    concurrency = options[:concurrency] || 1
+
+    start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond)
+
+    results = files.flat_map { |file| load_and_run(file, provider, concurrency) }
+    aggregated = aggregate_results(results, start_time)
+
+    # Determine pass/fail based on threshold
+    passed = if strict
+               aggregated[:summary][:failed].zero?
+             elsif threshold
+               aggregated[:summary][:pass_rate] >= threshold
+             else
+               true
+             end
+
+    aggregated[:summary][:threshold_passed] = passed
+    aggregated[:summary][:threshold] = threshold
+    aggregated[:summary][:strict] = strict
+
+    formatted = RubyLLM::Tribunal::Reporter.format(aggregated, format)
+
+    if output
+      File.write(output, formatted)
+      puts "Results written to #{output}"
+    else
+      puts formatted
+    end
+
+    exit 1 unless passed
+  end
+
+  desc 'Initialize eval directory structure'
+  task :init do
+    base_dir = ENV.fetch('TRIBUNAL_BASE_DIR', '.')
+
+    create_dir(File.join(base_dir, 'test/evals'))
+    create_dir(File.join(base_dir, 'test/evals/datasets'))
+
+    create_file(File.join(base_dir, 'test/evals/datasets/example.json'), example_dataset_json)
+    create_file(File.join(base_dir, 'test/evals/datasets/example.yaml'), example_dataset_yaml)
+
+    puts <<~MSG
+
+      ✅ Created eval structure:
+
+          test/evals/
+          └── datasets/
+              ├── example.json
+              └── example.yaml
+
+      Run evals with: rake tribunal:eval
+    MSG
+  end
+
+  private
+
+  def parse_options
+    options = {}
+    ARGV.each do |arg|
+      case arg
+      when /^--format=(.+)$/
+        options[:format] = Regexp.last_match(1)
+      when /^--output=(.+)$/
+        options[:output] = Regexp.last_match(1)
+      when /^--provider=(.+)$/
+        options[:provider] = Regexp.last_match(1)
+      when /^--threshold=(.+)$/
+        options[:threshold] = Regexp.last_match(1).to_f
+      when '--strict'
+        options[:strict] = true
+      when /^--concurrency=(.+)$/
+        options[:concurrency] = Regexp.last_match(1).to_i
+      when /^--files=(.+)$/
+        options[:files] = Regexp.last_match(1).split(',')
+      end
+    end
+    options
+  end
+
+  def find_default_files
+    patterns = [
+      'test/evals/**/*.json',
+      'test/evals/**/*.yaml',
+      'test/evals/**/*.yml',
+      'spec/evals/**/*.json',
+      'spec/evals/**/*.yaml',
+      'spec/evals/**/*.yml'
+    ]
+    patterns.flat_map { |p| Dir.glob(p) }
+  end
+
+  def parse_provider(str)
+    return nil unless str
+
+    parts = str.split(':')
+    raise 'Invalid provider format. Use Module:method (e.g. MyApp::RAG:query)' unless parts.length == 2
+
+    mod = parts[0].split('::').reduce(Object) { |m, c| m.const_get(c) }
+    [mod, parts[1].to_sym]
+  end
+
+  def load_and_run(path, provider, concurrency)
+    puts "Loading #{path}..."
+
+    cases = RubyLLM::Tribunal::Dataset.load_with_assertions(path)
+
+    if concurrency > 1
+      require 'parallel'
+      Parallel.map(cases, in_threads: concurrency) do |test_case, assertions|
+        run_case(test_case, assertions, provider)
+      end
+    else
+      cases.map { |test_case, assertions| run_case(test_case, assertions, provider) }
+    end
+  end
+
+  def run_case(test_case, assertions, provider)
+    start = Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond)
+
+    if provider
+      mod, method = provider
+      output = mod.send(method, test_case)
+      test_case = test_case.with_output(output)
+    end
+
+    results = if test_case.actual_output
+                RubyLLM::Tribunal::Assertions.evaluate_all(assertions, test_case)
+              else
+                {}
+              end
+
+    duration = Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond) - start
+
+    failures = results
+               .select { |_type, result| result.first == :fail }
+               .map { |type, (_status, details)| [type, details[:reason]] }
+
+    {
+      input: test_case.input,
+      status: failures.empty? ? :passed : :failed,
+      failures:,
+      results:,
+      duration_ms: duration
+    }
+  end
+
+  def aggregate_results(cases, start_time)
+    duration = Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond) - start_time
+
+    passed = cases.count { |c| c[:status] == :passed }
+    failed = cases.count { |c| c[:status] == :failed }
+    total = cases.length
+
+    metrics = aggregate_metrics(cases)
+
+    {
+      summary: {
+        total:,
+        passed:,
+        failed:,
+        pass_rate: total.positive? ? passed.to_f / total : 0,
+        duration_ms: duration
+      },
+      metrics:,
+      cases:
+    }
+  end
+
+  def aggregate_metrics(cases)
+    cases
+      .flat_map { |c| c[:results].map { |type, result| [type, result.first == :pass] } }
+      .group_by(&:first)
+      .transform_values do |results|
+        {
+          passed: results.count { |_, passed| passed },
+          total: results.length
+        }
+      end
+  end
+
+  def create_dir(path)
+    FileUtils.mkdir_p(path)
+    puts "Created #{path}/"
+  end
+
+  def create_file(path, content)
+    return if File.exist?(path)
+
+    File.write(path, content)
+    puts "Created #{path}"
+  end
+
+  def example_dataset_json
+    <<~JSON
+      [
+        {
+          "input": "What is the return policy?",
+          "context": "Returns are accepted within 30 days of purchase with a valid receipt. Items must be in original condition.",
+          "expected": {
+            "contains": ["30 days", "receipt"],
+            "not_contains": ["no returns"]
+          }
+        },
+        {
+          "input": "Do you ship internationally?",
+          "context": "We currently ship to the United States and Canada only.",
+          "expected": {
+            "contains_any": ["United States", "US", "Canada"],
+            "not_contains": ["worldwide", "international"]
+          }
+        }
+      ]
+    JSON
+  end
+
+  def example_dataset_yaml
+    <<~YAML
+      - input: What is the return policy?
+        context: Returns are accepted within 30 days of purchase with a valid receipt.
+        expected:
+          contains:
+            - 30 days
+            - receipt
+
+      - input: What are the store hours?
+        context: We are open Monday through Friday, 9am to 5pm.
+        expected:
+          contains_any:
+            - "9am"
+            - "9:00"
+          regex: "\\\\d+[ap]m"
+    YAML
+  end
+end
+
+# Make tasks available without :environment for non-Rails apps
+unless Rake::Task.task_defined?(:environment)
+  task :environment do
+    # No-op for non-Rails apps
+  end
+end

metadata

@@ -0,0 +1,99 @@
+--- !ruby/object:Gem::Specification
+name: ruby_llm-tribunal
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Florian
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2026-01-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: ruby_llm
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+description: |-
+  Tribunal provides tools for evaluating LLM outputs, detecting hallucinations,
+  measuring response quality, and ensuring safety. Features deterministic assertions,
+  LLM-as-judge evaluations, red team attack generation, and multiple output formats.
+  A RubyLLM plugin inspired by the Elixir Tribunal library.
+email:
+- florian@alqemist.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE.txt
+- README.md
+- lib/ruby_llm-tribunal.rb
+- lib/ruby_llm/tribunal.rb
+- lib/ruby_llm/tribunal/assertions.rb
+- lib/ruby_llm/tribunal/assertions/deterministic.rb
+- lib/ruby_llm/tribunal/assertions/embedding.rb
+- lib/ruby_llm/tribunal/assertions/judge.rb
+- lib/ruby_llm/tribunal/configuration.rb
+- lib/ruby_llm/tribunal/dataset.rb
+- lib/ruby_llm/tribunal/eval_helpers.rb
+- lib/ruby_llm/tribunal/judge.rb
+- lib/ruby_llm/tribunal/judges/bias.rb
+- lib/ruby_llm/tribunal/judges/correctness.rb
+- lib/ruby_llm/tribunal/judges/faithful.rb
+- lib/ruby_llm/tribunal/judges/hallucination.rb
+- lib/ruby_llm/tribunal/judges/harmful.rb
+- lib/ruby_llm/tribunal/judges/jailbreak.rb
+- lib/ruby_llm/tribunal/judges/pii.rb
+- lib/ruby_llm/tribunal/judges/refusal.rb
+- lib/ruby_llm/tribunal/judges/relevant.rb
+- lib/ruby_llm/tribunal/judges/toxicity.rb
+- lib/ruby_llm/tribunal/red_team.rb
+- lib/ruby_llm/tribunal/reporter.rb
+- lib/ruby_llm/tribunal/reporters/console.rb
+- lib/ruby_llm/tribunal/reporters/github.rb
+- lib/ruby_llm/tribunal/reporters/html.rb
+- lib/ruby_llm/tribunal/reporters/json.rb
+- lib/ruby_llm/tribunal/reporters/junit.rb
+- lib/ruby_llm/tribunal/reporters/text.rb
+- lib/ruby_llm/tribunal/test_case.rb
+- lib/ruby_llm/tribunal/version.rb
+- lib/tasks/tribunal.rake
+homepage: https://github.com/Alqemist-labs/ruby_llm-tribunal
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/Alqemist-labs/ruby_llm-tribunal
+  source_code_uri: https://github.com/Alqemist-labs/ruby_llm-tribunal
+  changelog_uri: https://github.com/Alqemist-labs/ruby_llm-tribunal/blob/main/CHANGELOG.md
+  rubygems_mfa_required: 'true'
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.2.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3.1
+signing_key: 
+specification_version: 4
+summary: LLM evaluation framework for Ruby
+test_files: []