llm_bench 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: d90ed99b03730fd89c2fd93d62ec728c4b474f9cc6fefc4b4030f635fdf6effd
+  data.tar.gz: '054129a5c38f180e2bd46ba6372bc61474d41b8e5a74f9e2e21aa335ab35278a'
+SHA512:
+  metadata.gz: eabdee5f9298517c6b9617fbf6ccd346e49a64021a81f7d35150439262a5587fcfed197c6710049402d2e2a7908fbfbf5005cb71d54bf73e6466be84e540be19
+  data.tar.gz: 6631ef5c989762cdbe86baf86e29682baaf78ddc9b005db1bc9f4e4c37a8c07bffa5864b590f0340323b88d50dd885b3adcca2b5e130084d6bf252d86fc3b95e

data/.claude/settings.local.json

@@ -0,0 +1,10 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(gem build:*)",
+      "Bash(gem install:*)"
+    ],
+    "deny": [],
+    "ask": []
+  }
+}

data/.gitignore

@@ -0,0 +1,57 @@
+# Configuration files with API keys
+models.yaml
+
+# Benchmark results CSV files
+*.csv
+
+# Ruby cache files
+*.rbc
+*.gem
+*.rbo
+
+# Ruby bundler
+Gemfile.lock
+.bundle
+/.bundle
+
+# Ruby temporary files
+*.tmp
+*.temp
+
+# IDE files
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# OS generated files
+.DS_Store
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes
+ehthumbs.db
+Thumbs.db
+
+# Logs
+*.log
+
+# Coverage reports
+coverage/
+
+# Documentation
+/doc/
+/.yardoc/
+/_yardoc/
+/.rdoc/
+/rdoc/
+
+# Environment files
+.env
+.env.*
+!.env.example
+
+# Temporary test files
+test_*.rb
+verify_*.rb

data/README.md

@@ -0,0 +1,104 @@
+# LLMBench
+
+A Ruby gem for benchmarking and comparing the performance of different Large Language Model providers and APIs.
+
+## Features
+
+- Support for both OpenAI and Anthropic-compatible API formats
+- Parallel execution across multiple models and providers
+- Continuous tracking with CSV export functionality
+- Clean, modular architecture with proper gem structure
+- No external dependencies - uses only Ruby standard library
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'llm_bench'
+```
+
+And then execute:
+```bash
+bundle install
+```
+
+Or install it yourself as:
+```bash
+gem install llm_bench
+```
+
+## Usage
+
+### Configuration
+
+Create a configuration file named `models.yaml` in your current directory, or specify a custom path with the `--config` argument:
+
+```yaml
+prompt: "Explain the concept of machine learning in simple terms in exactly 300 words..."
+
+providers:
+  - name: "openai"
+    base_url: "https://api.openai.com/v1"
+    api_key: "your-api-key-here"
+    models:
+      - nickname: "gpt-4"
+        id: "gpt-4"
+        api_format: "openai"
+
+  - name: "anthropic"
+    base_url: "https://api.anthropic.com"
+    api_key: "your-api-key-here"
+    models:
+      - nickname: "claude"
+        id: "claude-3-sonnet-20240229"
+        api_format: "anthropic"
+```
+
+### Commands
+
+#### Benchmark a single model:
+```bash
+llm_bench --config ./my-config.yaml --provider openai --model gpt-4
+```
+
+#### Benchmark all configured models:
+```bash
+llm_bench --all
+```
+
+#### Benchmark all models with custom config:
+```bash
+llm_bench --config ./my-config.yaml --all
+```
+
+#### Enable continuous tracking:
+```bash
+llm_bench --config ./my-config.yaml --all --track
+```
+
+#### Print full responses:
+```bash
+llm_bench --config ./my-config.yaml --provider openai --model gpt-4 --print-result
+```
+
+**Note**: If no `--config` argument is provided, `llm_bench` will look for `models.yaml` in the current directory. If the configuration file is not found, an error will be displayed.
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To build and install the gem locally:
+
+```bash
+gem build llm_bench.gemspec
+gem install ./llm_bench-0.1.0.gem
+```
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/vito/llm-bench.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/exe/llm_bench

@@ -0,0 +1,106 @@
+#!/usr/bin/env ruby
+
+# Add the lib directory to the load path when running from source
+if __FILE__ == $PROGRAM_NAME
+  lib_path = File.expand_path('../../lib', __FILE__)
+  $LOAD_PATH.unshift(lib_path) if File.directory?(lib_path)
+end
+
+begin
+  require 'llm_bench'
+rescue LoadError
+  # If we can't load the gem, try to load from source
+  require_relative '../lib/llm_bench'
+end
+
+require 'yaml'
+require 'optparse'
+
+def parse_arguments
+  options = {}
+  OptionParser.new do |opts|
+    opts.banner = "Usage: llm_bench --config CONFIG --provider PROVIDER --model NICKNAME [--print-result]"
+    opts.banner += "\n       llm_bench --config CONFIG --all [--track] [--print-result]"
+
+    opts.on('--config CONFIG', 'Path to configuration file (default: models.yaml)') do |config|
+      options[:config] = config
+    end
+
+    opts.on('--provider PROVIDER', 'Provider name from config file') do |provider|
+      options[:provider] = provider
+    end
+
+    opts.on('--model NICKNAME', 'Model nickname from config file') do |model|
+      options[:model] = model
+    end
+
+    opts.on('--all', 'Run benchmark on all configured models') do
+      options[:all] = true
+    end
+
+    opts.on('--track', 'Enable continuous tracking with CSV output (requires --all)') do
+      options[:track] = true
+    end
+
+    opts.on('--print-result', 'Print the full message returned by each LLM') do
+      options[:print_result] = true
+    end
+
+    opts.on('--help', 'Display help') do
+      puts opts
+      exit
+    end
+  end.parse!
+
+  if options[:track] && !options[:all]
+    puts "Error: --track requires --all"
+    puts "Use --help for usage information"
+    exit 1
+  end
+
+  if options[:all]
+    options
+  elsif options[:provider] && options[:model]
+    options
+  else
+    puts "Error: Either --provider and --model, or --all is required"
+    puts "Use --help for usage information"
+    exit 1
+  end
+
+  options
+end
+
+def main
+  options = parse_arguments
+
+  # Determine config file path
+  config_path = options[:config] || './models.yaml'
+
+  # Validate config file exists
+  unless File.exist?(config_path)
+    puts "Error: Configuration file not found at #{config_path}"
+    exit 1
+  end
+
+  # Load configuration
+  config = YAML.load_file(config_path)
+
+  if options[:all]
+    if options[:track]
+      tracker = LLMBench::Tracker.new(config)
+      tracker.start_tracking
+    else
+      parallel_benchmark = LLMBench::ParallelBenchmark.new(config, options[:print_result])
+      parallel_benchmark.run_all
+    end
+  else
+    benchmark = LLMBench::Benchmark.new(options[:provider], options[:model], options[:print_result], config)
+    benchmark.run_benchmark
+  end
+rescue StandardError => e
+  puts "Error: #{e.message}"
+  exit 1
+end
+
+main

data/lib/llm_bench/benchmark.rb

@@ -0,0 +1,225 @@
+require 'yaml'
+require 'json'
+require 'net/http'
+require 'uri'
+require 'time'
+
+module LLMBench
+  class Benchmark
+  attr_reader :config, :provider, :model, :start_time, :end_time
+
+  def initialize(provider_name, model_nickname, print_result = false, config = nil)
+    @provider_name = provider_name
+    @model_nickname = model_nickname
+    @print_result = print_result
+    @config = config || load_config
+    validate_provider_and_model!
+  end
+
+  def load_config
+    config_path = File.join(__dir__, '..', 'models.yaml')
+    unless File.exist?(config_path)
+      raise "Configuration file models.yaml not found"
+    end
+
+    YAML.load_file(config_path)
+  end
+
+  def validate_provider_and_model!
+    provider_config = @config['providers'].find { |p| p['name'] == @provider_name }
+    unless provider_config
+      raise "Provider '#{@provider_name}' not found in configuration"
+    end
+
+    model_config = provider_config['models'].find { |m| m['nickname'] == @model_nickname }
+    unless model_config
+      raise "Model '#{@model_nickname}' not found for provider '#{@provider_name}'"
+    end
+
+    model_config['api_format'] ||= 'openai'
+
+    unless ['openai', 'anthropic'].include?(model_config['api_format'])
+      raise "Invalid API format '#{model_config['api_format']}' for model '#{@model_nickname}'. Must be 'openai' or 'anthropic'"
+    end
+
+    @provider = provider_config
+    @model = model_config
+  end
+
+  def run_benchmark
+    puts "=== LLM Benchmark ==="
+    puts "Provider: #{@provider_name}"
+    puts "Model: #{@model_nickname} (#{@model['id']})"
+    puts "Starting benchmark..."
+
+    @start_time = Time.now
+    puts "Start time: #{@start_time.strftime('%Y-%m-%d %H:%M:%S.%3N')}"
+
+    response = make_api_call
+
+    @end_time = Time.now
+    puts "End time: #{@end_time.strftime('%Y-%m-%d %H:%M:%S.%3N')}"
+
+    calculate_and_display_metrics(response)
+  end
+
+  def anthropic_format?
+    @model['api_format'] == 'anthropic'
+  end
+
+  def api_endpoint
+    anthropic_format? ? "#{@provider['base_url']}/v1/messages" : "#{@provider['base_url']}/chat/completions"
+  end
+
+  def build_request_headers
+    headers = { 'Content-Type' => 'application/json' }
+    if anthropic_format?
+      headers['x-api-key'] = @provider['api_key']
+      headers['anthropic-version'] = '2023-06-01'
+    else
+      headers['Authorization'] = "Bearer #{@provider['api_key']}"
+    end
+    headers
+  end
+
+  def build_request_body
+    base_body = {
+      model: @model['id'],
+      messages: [{ role: 'user', content: @config['prompt'] }]
+    }
+
+    if anthropic_format?
+      base_body.merge(max_tokens: 1000)
+    else
+      base_body.merge(max_tokens: 1000, temperature: 0.7)
+    end
+  end
+
+  def extract_response_content(response)
+    if anthropic_format?
+      extract_anthropic_content(response)
+    else
+      response.dig('choices', 0, 'message', 'content') || ''
+    end
+  end
+
+  def extract_token_counts(response, message_content)
+    if anthropic_format?
+      input_tokens = response.dig('usage', 'input_tokens') || estimate_tokens(@config['prompt'])
+      output_tokens = response.dig('usage', 'output_tokens') || estimate_tokens(message_content)
+    else
+      input_tokens = response.dig('usage', 'prompt_tokens') || estimate_tokens(@config['prompt'])
+      output_tokens = response.dig('usage', 'completion_tokens') || estimate_tokens(message_content)
+    end
+    [input_tokens, output_tokens]
+  end
+
+  def make_api_call
+    uri = URI.parse(api_endpoint)
+    request = Net::HTTP::Post.new(uri)
+    request['Content-Type'] = 'application/json'
+
+    build_request_headers.each { |key, value| request[key] = value }
+    request.body = build_request_body.to_json
+
+    http = Net::HTTP.new(uri.host, uri.port)
+    http.use_ssl = uri.scheme == 'https'
+
+    response = http.request(request)
+
+    handle_api_error(response) unless response.is_a?(Net::HTTPSuccess)
+
+    JSON.parse(response.body)
+  end
+
+  def handle_api_error(response)
+    error_response = JSON.parse(response.body)
+    error_msg = error_response['msg'] || error_response['message'] ||
+                 error_response.dig('error', 'message') || response.message
+    raise "API request failed: #{response.code} - #{error_msg}"
+  rescue JSON::ParserError
+    raise "API request failed: #{response.code} #{response.message}"
+  end
+
+  def calculate_metrics(response)
+    duration = @end_time - @start_time
+    message_content = extract_response_content(response)
+    input_tokens, output_tokens = extract_token_counts(response, message_content)
+
+    total_tokens = input_tokens + output_tokens
+    tokens_per_second = total_tokens / duration if duration.positive?
+
+    {
+      duration: duration,
+      input_tokens: input_tokens,
+      output_tokens: output_tokens,
+      total_tokens: total_tokens,
+      tokens_per_second: tokens_per_second,
+      message_content: message_content
+    }
+  end
+
+  def calculate_and_display_metrics(response)
+    metrics = calculate_metrics(response)
+
+    puts "\n=== Results ==="
+    puts "Duration: #{metrics[:duration].round(3)} seconds"
+    puts "Input tokens: #{metrics[:input_tokens]}"
+    puts "Output tokens: #{metrics[:output_tokens]}"
+    puts "Total tokens: #{metrics[:total_tokens]}"
+    puts "Tokens per second: #{metrics[:tokens_per_second].round(2)}"
+
+    puts "\n=== Message Content ==="
+    puts metrics[:message_content] if @print_result
+  end
+
+  def extract_anthropic_content(response)
+    if response.key?('code') && response.key?('msg') && response.key?('success')
+      return "Error: #{response['msg']}"
+    end
+
+    content_blocks = response.dig('content')
+
+    if content_blocks.is_a?(Array) && !content_blocks.empty?
+      text_block = content_blocks.find { |block| block.is_a?(Hash) && block['type'] == 'text' }
+      text_block ? text_block['text'] : nil
+    elsif response.dig('content', 0, 'text')
+      response.dig('content', 0, 'text')
+    else
+      nil
+    end
+  end
+
+  def estimate_tokens(text)
+    (text.length / 4.0).round
+  end
+
+  def run_benchmark_for_results
+    @start_time = Time.now
+    response = make_api_call
+    @end_time = Time.now
+
+    metrics = calculate_metrics(response)
+    {
+      provider: @provider_name,
+      model: @model_nickname,
+      total_tokens: metrics[:total_tokens],
+      tokens_per_second: metrics[:tokens_per_second].round(2),
+      duration: metrics[:duration].round(3),
+      success: true,
+      message_content: metrics[:message_content]
+    }
+  rescue StandardError => e
+    {
+      provider: @provider_name,
+      model: @model_nickname,
+      total_tokens: 0,
+      tokens_per_second: 0,
+      duration: 0,
+      success: false,
+      error: e.message,
+      message_content: ''
+    }
+  end
+  end
+end

data/lib/llm_bench/parallel_benchmark.rb

@@ -0,0 +1,130 @@
+module LLMBench
+  class ParallelBenchmark
+  def initialize(config, print_result = false)
+    @config = config
+    @print_result = print_result
+  end
+
+  def run_all
+    puts "=== LLM Benchmark ==="
+    puts "Running benchmarks on all configured models..."
+    puts "Starting at #{Time.now.strftime('%Y-%m-%d %H:%M:%S.%3N')}"
+    puts
+
+    benchmarks = create_benchmarks
+    results = run_parallel(benchmarks)
+
+    display_results_table(results)
+    display_summary(results)
+  end
+
+  def run_silent
+    benchmarks = create_benchmarks
+    run_parallel(benchmarks)
+  end
+
+  private
+
+  def create_benchmarks
+    benchmarks = []
+
+    @config['providers'].each do |provider|
+      provider['models'].each do |model|
+        benchmarks << Benchmark.new(provider['name'], model['nickname'], @print_result, @config)
+      end
+    end
+
+    benchmarks
+  end
+
+  def run_parallel(benchmarks)
+    results = []
+    mutex = Mutex.new
+
+    threads = benchmarks.map do |benchmark|
+      Thread.new do
+        result = benchmark.run_benchmark_for_results
+        mutex.synchronize { results << result }
+      end
+    end
+
+    threads.each(&:join)
+    results
+  end
+
+  def display_results_table(results)
+    sorted_results = results.sort_by { |r| -r[:tokens_per_second] }
+
+    provider_width = sorted_results.map { |r| r[:provider].length }.max
+    model_width = sorted_results.map { |r| r[:model].length }.max
+    tokens_width = 12
+    tps_width = 15
+
+    if @print_result
+      header = "| #{"Provider".ljust(provider_width)} | #{"Model".ljust(model_width)} | #{"Total Tokens".rjust(tokens_width)} | #{"Tokens/sec".rjust(tps_width)} | Message Content"
+      separator = "| #{'-' * provider_width} | #{'-' * model_width} | #{'-' * tokens_width} | #{'-' * tps_width} | #{'-' * 80}"
+    else
+      header = "| #{"Provider".ljust(provider_width)} | #{"Model".ljust(model_width)} | #{"Total Tokens".rjust(tokens_width)} | #{"Tokens/sec".rjust(tps_width)} |"
+      separator = "| #{'-' * provider_width} | #{'-' * model_width} | #{'-' * tokens_width} | #{'-' * tps_width} |"
+    end
+
+    puts header
+    puts separator
+
+    sorted_results.each do |result|
+      provider_col = result[:provider].ljust(provider_width)
+      model_col = result[:model].ljust(model_width)
+
+      if result[:success]
+        tokens_col = result[:total_tokens].to_s.rjust(tokens_width)
+        tps_col = result[:tokens_per_second].to_s.rjust(tps_width)
+
+        if @print_result
+          message_content = result[:message_content][0..79]
+          puts "| #{provider_col} | #{model_col} | #{tokens_col} | #{tps_col} | #{message_content}"
+        else
+          puts "| #{provider_col} | #{model_col} | #{tokens_col} | #{tps_col} |"
+        end
+      else
+        tokens_col = "ERROR".rjust(tokens_width)
+        tps_col = "FAILED".rjust(tps_width)
+
+        if @print_result
+          puts "| #{provider_col} | #{model_col} | #{tokens_col} | #{tps_col} | #{result[:error][0..79]}"
+        else
+          puts "| #{provider_col} | #{model_col} | #{tokens_col} | #{tps_col} |"
+        end
+      end
+    end
+
+    puts
+  end
+
+  def display_summary(results)
+    successful = results.select { |r| r[:success] }
+    failed = results.select { |r| !r[:success] }
+
+    puts "=== Summary ==="
+    puts "Total benchmarks: #{results.length}"
+    puts "Successful: #{successful.length}"
+    puts "Failed: #{failed.length}"
+
+    if successful.any?
+      avg_tps = successful.map { |r| r[:tokens_per_second] }.sum / successful.length
+      fastest = successful.max_by { |r| r[:tokens_per_second] }
+      slowest = successful.min_by { |r| r[:tokens_per_second] }
+
+      puts "Average tokens/sec: #{avg_tps.round(2)}"
+      puts "Fastest: #{fastest[:provider]}/#{fastest[:model]} (#{fastest[:tokens_per_second]} tokens/sec)"
+      puts "Slowest: #{slowest[:provider]}/#{slowest[:model]} (#{slowest[:tokens_per_second]} tokens/sec)"
+    end
+
+    return unless failed.any?
+
+    puts "\nFailed benchmarks:"
+    failed.each do |result|
+      puts "  #{result[:provider]}/#{result[:model]}: #{result[:error]}"
+    end
+  end
+  end
+end

data/lib/llm_bench/tracker.rb

@@ -0,0 +1,136 @@
+module LLMBench
+  class Tracker
+  def initialize(config)
+    @config = config
+    @csv_file = "llm_benchmark_results_#{Time.now.strftime('%Y%m%d_%H%M%S')}.csv"
+    @running = true
+    @next_run_time = Time.now
+    setup_signal_handlers
+  end
+
+  def start_tracking
+    puts "=== LLM Performance Tracker ==="
+    puts "Tracking all models every 60 seconds"
+    puts "Results will be saved to: #{@csv_file}"
+    puts "Press Ctrl+C to stop tracking"
+    puts
+
+    initialize_csv
+
+    run_tracking_cycle
+
+    while @running
+      time_until_next_run = @next_run_time - Time.now
+
+      if time_until_next_run.positive?
+        sleep_time = [time_until_next_run, 1.0].min
+        sleep(sleep_time)
+      else
+        run_tracking_cycle
+        @next_run_time = Time.now + 60
+      end
+    end
+
+    puts "\nTracking stopped by user"
+    puts "Results saved to: #{@csv_file}"
+  end
+
+  private
+
+  def setup_signal_handlers
+    Signal.trap('INT') do
+      @running = false
+      puts "\nStopping tracking..."
+    end
+
+    Signal.trap('TERM') do
+      @running = false
+      puts "\nStopping tracking..."
+    end
+  end
+
+  def initialize_csv
+    File.open(@csv_file, 'w') do |file|
+      file.write("timestamp,provider_model,tokens_per_second,total_tokens,duration_seconds\n")
+    end
+  end
+
+  def run_tracking_cycle
+    timestamp = Time.now
+    puts "[#{timestamp.strftime('%Y-%m-%d %H:%M:%S')}] Running benchmark cycle..."
+
+    parallel_benchmark = ParallelBenchmark.new(@config)
+    results = parallel_benchmark.run_silent
+
+    write_results_to_csv(timestamp, results)
+    display_cycle_summary(results)
+  end
+
+  def write_results_to_csv(timestamp, results)
+    File.open(@csv_file, 'a') do |file|
+      results.each do |result|
+        next unless result[:success]
+
+        provider_model = "#{result[:provider]}+#{result[:model]}"
+        csv_line = [
+          timestamp.strftime('%Y-%m-%d %H:%M:%S'),
+          provider_model,
+          result[:tokens_per_second],
+          result[:total_tokens],
+          result[:duration]
+        ].join(',') + "\n"
+        file.write(csv_line)
+      end
+    end
+  end
+
+  def display_cycle_summary(results)
+    successful = results.select { |r| r[:success] }
+    failed = results.select { |r| !r[:success] }
+
+    puts "  Completed: #{successful.length} successful, #{failed.length} failed"
+
+    if successful.any?
+      avg_tps = successful.map { |r| r[:tokens_per_second] }.sum / successful.length
+      puts "  Average tokens/sec: #{avg_tps.round(2)}"
+    end
+
+    if failed.any?
+      puts "  Failed: #{failed.map { |f| "#{f[:provider]}/#{f[:model]}" }.join(', ')}"
+    end
+
+    puts "\n  === Individual Model Results ==="
+
+    sorted_results = results.sort_by { |r| -r[:tokens_per_second] }
+
+    provider_width = sorted_results.map { |r| r[:provider].length }.max
+    model_width = sorted_results.map { |r| r[:model].length }.max
+    tokens_width = 12
+    tps_width = 15
+    duration_width = 12
+
+    header = "  | #{"Provider".ljust(provider_width)} | #{"Model".ljust(model_width)} | #{"Tokens/sec".rjust(tps_width)} | #{"Total Tokens".rjust(tokens_width)} | #{"Duration".rjust(duration_width)} |"
+    separator = "  | #{'-' * provider_width} | #{'-' * model_width} | #{'-' * tps_width} | #{'-' * tokens_width} | #{'-' * duration_width} |"
+
+    puts header
+    puts separator
+
+    sorted_results.each do |result|
+      provider_col = result[:provider].ljust(provider_width)
+      model_col = result[:model].ljust(model_width)
+
+      if result[:success]
+        tps_col = result[:tokens_per_second].to_s.rjust(tps_width)
+        tokens_col = result[:total_tokens].to_s.rjust(tokens_width)
+        duration_col = "#{result[:duration]}s".rjust(duration_width)
+        puts "  | #{provider_col} | #{model_col} | #{tps_col} | #{tokens_col} | #{duration_col} |"
+      else
+        tps_col = "FAILED".rjust(tps_width)
+        tokens_col = "ERROR".rjust(tokens_width)
+        duration_col = "N/A".rjust(duration_width)
+        puts "  | #{provider_col} | #{model_col} | #{tps_col} | #{tokens_col} | #{duration_col} |"
+      end
+    end
+  end
+  end
+end

data/lib/llm_bench/version.rb

@@ -0,0 +1,3 @@
+module LLMBench
+  VERSION = "0.1.0"
+end

data/lib/llm_bench.rb

@@ -0,0 +1,9 @@
+require_relative "llm_bench/version"
+require_relative "llm_bench/benchmark"
+require_relative "llm_bench/parallel_benchmark"
+require_relative "llm_bench/tracker"
+
+module LLMBench
+  class Error < StandardError; end
+  # Your code goes here...
+end

data/llm_bench.gemspec

@@ -0,0 +1,30 @@
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'llm_bench/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "llm_bench"
+  spec.version       = LLMBench::VERSION
+  spec.authors       = ["Vito"]
+  spec.email         = []
+
+  spec.summary       = "A tool for benchmarking LLM performance across different providers and models"
+  spec.description   = "LLM Bench is a Ruby gem that allows you to benchmark and compare the performance of different Large Language Model providers and APIs. It supports both OpenAI and Anthropic-compatible API formats, provides parallel execution, and includes continuous tracking capabilities with CSV export."
+  spec.homepage      = "https://github.com/vitobotta/llm-bench"
+  spec.license       = "MIT"
+  spec.required_ruby_version = ">= 2.7.0"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = spec.homepage
+  spec.metadata["changelog_uri"] = "#{spec.homepage}/blob/main/CHANGELOG.md"
+
+  spec.files = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
+  end
+
+  spec.bindir        = "exe"
+  spec.executables   = ["llm_bench"]
+  spec.require_paths = ["lib"]
+
+  # Standard library dependencies - no external gems required
+end

data/models.yaml.example

@@ -0,0 +1,20 @@
+# LLM Benchmark Configuration
+prompt: "Explain the concept of machine learning in simple terms in exactly 300 words or as close to it as possible, covering the main types of learning approaches and providing examples of real-world applications."
+
+providers:
+  - name: "z-ai"
+    base_url: "https://api.z.ai/api/coding/paas/v4"
+    api_key: "YOUR_API_KEY_HERE"
+    models:
+      - nickname: "glm"
+        id: "glm-4.5"
+        # api_format can be 'openai' (default) or 'anthropic'
+        api_format: "openai"
+  - name: "z-ai-anthropic"
+    base_url: "https://api.z.ai/api/anthropic"
+    api_key: "YOUR_API_KEY_HERE"
+    models:
+      - nickname: "glm-anthropic"
+        id: "glm-4.5"
+        # api_format can be 'openai' (default) or 'anthropic'
+        api_format: "anthropic"

metadata

@@ -0,0 +1,58 @@
+--- !ruby/object:Gem::Specification
+name: llm_bench
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Vito
+bindir: exe
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies: []
+description: LLM Bench is a Ruby gem that allows you to benchmark and compare the
+  performance of different Large Language Model providers and APIs. It supports both
+  OpenAI and Anthropic-compatible API formats, provides parallel execution, and includes
+  continuous tracking capabilities with CSV export.
+email: []
+executables:
+- llm_bench
+extensions: []
+extra_rdoc_files: []
+files:
+- ".claude/settings.local.json"
+- ".gitignore"
+- README.md
+- Rakefile
+- exe/llm_bench
+- lib/llm_bench.rb
+- lib/llm_bench/benchmark.rb
+- lib/llm_bench/parallel_benchmark.rb
+- lib/llm_bench/tracker.rb
+- lib/llm_bench/version.rb
+- llm_bench.gemspec
+- models.yaml.example
+homepage: https://github.com/vitobotta/llm-bench
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/vitobotta/llm-bench
+  source_code_uri: https://github.com/vitobotta/llm-bench
+  changelog_uri: https://github.com/vitobotta/llm-bench/blob/main/CHANGELOG.md
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.7.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.9
+specification_version: 4
+summary: A tool for benchmarking LLM performance across different providers and models
+test_files: []