fastembed 1.0.0 → 1.1.0

data/benchmark/compare_all.rb

@@ -0,0 +1,167 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Unified benchmark comparing Ruby fastembed with Python fastembed
+# Runs both implementations and reports side-by-side results
+
+require 'bundler/setup'
+require 'fastembed'
+require 'benchmark'
+require 'json'
+require 'open3'
+
+TEXTS = [
+  'Machine learning is a subset of artificial intelligence that enables systems to learn from data.',
+  'Ruby on Rails is a server-side web application framework written in Ruby under the MIT License.',
+  'Vector databases store embeddings and enable fast similarity search across millions of documents.',
+  'Natural language processing helps computers understand, interpret, and generate human language.',
+  'The quick brown fox jumps over the lazy dog. This is a classic pangram used in typing tests.'
+].freeze
+
+def run_ruby_benchmark
+  puts 'Running Ruby benchmark...'
+  results = {}
+
+  # Model loading
+  start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+  embedding = Fastembed::TextEmbedding.new(model_name: 'BAAI/bge-small-en-v1.5', show_progress: false)
+  results[:load_time] = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - start) * 1000).round(1)
+
+  # Warmup
+  embedding.embed(['warmup']).to_a
+
+  # Single document latency
+  times = []
+  10.times do
+    start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    embedding.embed([TEXTS.first]).to_a
+    times << (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start) * 1000
+  end
+  results[:single_latency] = times.min.round(2)
+
+  # Throughput tests
+  [100, 500, 1000].each do |count|
+    texts = TEXTS.cycle.take(count)
+    start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    embedding.embed(texts, batch_size: 64).to_a
+    elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
+    results[:"throughput_#{count}"] = (count / elapsed).round(1)
+  end
+
+  results
+end
+
+def run_python_benchmark
+  puts 'Running Python benchmark...'
+
+  python_script = <<~PYTHON
+    import json
+    import time
+    from fastembed import TextEmbedding
+
+    TEXTS = [
+        "Machine learning is a subset of artificial intelligence that enables systems to learn from data.",
+        "Ruby on Rails is a server-side web application framework written in Ruby under the MIT License.",
+        "Vector databases store embeddings and enable fast similarity search across millions of documents.",
+        "Natural language processing helps computers understand, interpret, and generate human language.",
+        "The quick brown fox jumps over the lazy dog. This is a classic pangram used in typing tests."
+    ]
+
+    results = {}
+
+    # Model loading
+    start = time.time()
+    embedding = TextEmbedding(model_name="BAAI/bge-small-en-v1.5")
+    results["load_time"] = round((time.time() - start) * 1000, 1)
+
+    # Warmup
+    list(embedding.embed(["warmup"]))
+
+    # Single document latency
+    times = []
+    for _ in range(10):
+        start = time.time()
+        list(embedding.embed([TEXTS[0]]))
+        times.append((time.time() - start) * 1000)
+    results["single_latency"] = round(min(times), 2)
+
+    # Throughput tests
+    for count in [100, 500, 1000]:
+        texts = (TEXTS * (count // len(TEXTS) + 1))[:count]
+        start = time.time()
+        list(embedding.embed(texts, batch_size=64))
+        elapsed = time.time() - start
+        results[f"throughput_{count}"] = round(count / elapsed, 1)
+
+    print(json.dumps(results))
+  PYTHON
+
+  stdout, status = Open3.capture2('python3', '-c', python_script)
+
+  if status.success?
+    JSON.parse(stdout)
+  else
+    puts 'Warning: Python benchmark failed. Is fastembed installed? (pip install fastembed)'
+    nil
+  end
+rescue Errno::ENOENT
+  puts 'Warning: Python not found'
+  nil
+end
+
+def print_comparison(ruby_results, python_results)
+  puts
+  puts '=' * 70
+  puts 'RUBY vs PYTHON FASTEMBED COMPARISON'
+  puts '=' * 70
+  puts
+
+  metrics = [
+    [:load_time, 'Model load time', 'ms', :lower_better],
+    [:single_latency, 'Single doc latency', 'ms', :lower_better],
+    [:throughput_100, '100 docs throughput', 'docs/sec', :higher_better],
+    [:throughput_500, '500 docs throughput', 'docs/sec', :higher_better],
+    [:throughput_1000, '1000 docs throughput', 'docs/sec', :higher_better]
+  ]
+
+  puts format('%-25s %15s %15s %10s', 'Metric', 'Ruby', 'Python', 'Winner')
+  puts '-' * 70
+
+  metrics.each do |key, label, unit, direction|
+    ruby_val = ruby_results[key]
+    python_val = python_results&.fetch(key.to_s, nil)
+
+    if python_val
+      if direction == :lower_better
+        winner = ruby_val < python_val ? 'Ruby' : 'Python'
+        ratio = python_val / ruby_val
+      else
+        winner = ruby_val > python_val ? 'Ruby' : 'Python'
+        ratio = ruby_val / python_val
+      end
+      ratio_str = winner == 'Ruby' ? "(#{ratio.round(1)}x)" : ''
+      winner_str = "#{winner} #{ratio_str}"
+    else
+      winner_str = 'N/A'
+    end
+
+    ruby_str = "#{ruby_val} #{unit}"
+    python_str = python_val ? "#{python_val} #{unit}" : 'N/A'
+
+    puts format('%-25s %15s %15s %10s', label, ruby_str, python_str, winner_str)
+  end
+
+  puts
+end
+
+# Run benchmarks
+ruby_results = run_ruby_benchmark
+python_results = run_python_benchmark
+
+print_comparison(ruby_results, python_results)
+
+puts 'Summary:'
+puts '- Both use the same ONNX Runtime and HuggingFace Tokenizers'
+puts '- Performance differences come from language overhead and batching'
+puts '- Ruby tends to win on latency, Python on large batch throughput'
+puts

data/benchmark/compare_python.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+"""Compare Python FastEmbed performance with Ruby FastEmbed."""
+
+import time
+from fastembed import TextEmbedding
+
+TEXTS = [
+    "Machine learning is a subset of artificial intelligence that enables systems to learn from data.",
+    "Ruby on Rails is a server-side web application framework written in Ruby under the MIT License.",
+    "Vector databases store embeddings and enable fast similarity search across millions of documents.",
+    "Natural language processing helps computers understand, interpret, and generate human language.",
+    "The quick brown fox jumps over the lazy dog. This is a classic pangram used in typing tests."
+]
+
+def benchmark_python():
+    print("=" * 60)
+    print("PYTHON FASTEMBED BENCHMARK")
+    print("=" * 60)
+    print()
+
+    # Model loading time
+    start = time.time()
+    embedding = TextEmbedding(model_name="BAAI/bge-small-en-v1.5")
+    load_time = time.time() - start
+    print(f"Model load time: {load_time * 1000:.1f}ms")
+
+    # Warmup
+    list(embedding.embed(["warmup"]))
+
+    # Single document latency
+    print()
+    print("Single document latency:")
+    for i, text in enumerate(TEXTS[:3]):
+        times = []
+        for _ in range(10):
+            start = time.time()
+            list(embedding.embed([text]))
+            times.append(time.time() - start)
+        avg = sum(times) / len(times)
+        min_time = min(times)
+        print(f"  Text {i+1} ({len(text)} chars): avg {avg*1000:.2f}ms, min {min_time*1000:.2f}ms")
+
+    # Throughput
+    print()
+    print("Throughput:")
+    for count in [100, 500, 1000]:
+        texts = (TEXTS * (count // len(TEXTS) + 1))[:count]
+
+        start = time.time()
+        list(embedding.embed(texts, batch_size=64))
+        elapsed = time.time() - start
+
+        rate = count / elapsed
+        print(f"  {count} docs: {rate:.1f} docs/sec ({elapsed*1000:.1f}ms)")
+
+    print()
+    print("=" * 60)
+
+if __name__ == "__main__":
+    benchmark_python()

data/benchmark/memory_profile.rb

@@ -0,0 +1,70 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'fastembed'
+
+def memory_mb
+  `ps -o rss= -p #{Process.pid}`.to_i / 1024.0
+end
+
+def print_memory(label)
+  puts "#{label}: #{memory_mb.round(1)} MB"
+end
+
+puts '=' * 60
+puts 'MEMORY PROFILING'
+puts '=' * 60
+puts
+
+print_memory('Initial')
+
+# Load model
+embedding = Fastembed::TextEmbedding.new
+print_memory('After model load')
+
+# Generate sample texts
+texts = Array.new(1000) { |i| "This is document number #{i} with some content for embedding." }
+
+# Process in batches
+GC.start
+print_memory('Before embedding 1000 docs')
+
+vectors = embedding.embed(texts, batch_size: 64).to_a
+print_memory('After embedding 1000 docs (holding results)')
+
+# Clear vectors
+vectors = nil
+GC.start
+sleep 0.1
+print_memory('After clearing vectors + GC')
+
+# Test lazy evaluation memory efficiency
+puts
+puts 'Testing lazy evaluation memory efficiency...'
+print_memory('Before lazy processing')
+
+count = 0
+embedding.embed(texts, batch_size: 64).each do |_vec|
+  count += 1
+  # Don't store vectors, just count them
+end
+puts "Processed #{count} vectors without storing"
+
+GC.start
+sleep 0.1
+print_memory('After lazy processing + GC')
+
+# Stress test - multiple rounds
+puts
+puts 'Stress test - 5 rounds of 1000 docs each...'
+5.times do |round|
+  embedding.embed(texts, batch_size: 64).to_a
+  GC.start
+  print_memory("After round #{round + 1}")
+end
+
+puts
+puts '=' * 60
+puts 'MEMORY PROFILE COMPLETE'
+puts '=' * 60

data/benchmark/profile.rb

@@ -0,0 +1,198 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'fastembed'
+require 'benchmark'
+
+# Sample texts of varying lengths
+SHORT_TEXTS = [
+  'Hello world',
+  'Ruby is great',
+  'Machine learning',
+  'Vector embeddings',
+  'Semantic search'
+].freeze
+
+MEDIUM_TEXTS = [
+  'The quick brown fox jumps over the lazy dog. This is a classic pangram used in typing tests.',
+  'Machine learning is a subset of artificial intelligence that enables systems to learn from data.',
+  'Ruby on Rails is a server-side web application framework written in Ruby under the MIT License.',
+  'Vector databases store embeddings and enable fast similarity search across millions of documents.',
+  'Natural language processing helps computers understand, interpret, and generate human language.'
+].freeze
+
+LONG_TEXTS = [
+  'Artificial intelligence (AI) is intelligence demonstrated by machines, as opposed to natural intelligence displayed by animals including humans. AI research has been defined as the field of study of intelligent agents, which refers to any system that perceives its environment and takes actions that maximize its chance of achieving its goals. The term artificial intelligence had previously been used to describe machines that mimic and display human cognitive skills that are associated with the human mind, such as learning and problem-solving.',
+  'Ruby is an interpreted, high-level, general-purpose programming language which supports multiple programming paradigms. It was designed with an emphasis on programming productivity and simplicity. In Ruby, everything is an object, including primitive data types. It was developed in the mid-1990s by Yukihiro Matsumoto in Japan. Ruby is dynamically typed and uses garbage collection and just-in-time compilation.',
+  'Text embeddings are dense vector representations of text that capture semantic meaning. They are produced by machine learning models trained on large corpora of text data. These embeddings enable semantic similarity calculations, clustering, and information retrieval tasks. Modern embedding models like BERT, Sentence Transformers, and OpenAI embeddings have revolutionized natural language processing applications.',
+  'Vector databases are specialized database systems designed to store and query high-dimensional vector data efficiently. They use approximate nearest neighbor algorithms like HNSW, IVF, and PQ to enable fast similarity search at scale. Popular vector databases include Pinecone, Weaviate, Qdrant, Milvus, and pgvector. They are essential infrastructure for semantic search, recommendation systems, and RAG applications.',
+  'The Transformer architecture, introduced in the paper "Attention Is All You Need", revolutionized natural language processing. It relies entirely on self-attention mechanisms, dispensing with recurrence and convolutions. This enables much more parallelization and has led to significant improvements in translation quality. Transformers are the foundation of modern language models like GPT, BERT, and T5.'
+].freeze
+
+def print_separator
+  puts '-' * 70
+end
+
+def format_rate(count, time)
+  rate = count / time
+  "#{rate.round(1)} docs/sec"
+end
+
+def profile_batch(embedding, texts, batch_size, iterations = 3)
+  times = []
+  iterations.times do
+    GC.start
+    time = Benchmark.realtime do
+      embedding.embed(texts, batch_size: batch_size).to_a
+    end
+    times << time
+  end
+  times.min # Return best time
+end
+
+puts '=' * 70
+puts 'FASTEMBED-RB PERFORMANCE PROFILE'
+puts '=' * 70
+puts
+puts "Ruby version: #{RUBY_VERSION}"
+puts "Platform: #{RUBY_PLATFORM}"
+puts "Fastembed version: #{Fastembed::VERSION}"
+puts
+
+# Model loading benchmark
+print_separator
+puts 'MODEL LOADING TIME'
+print_separator
+
+models = [
+  'BAAI/bge-small-en-v1.5',
+  'sentence-transformers/all-MiniLM-L6-v2'
+]
+
+models.each do |model_name|
+  # Ensure model is downloaded first
+  Fastembed::TextEmbedding.new(model_name: model_name)
+  GC.start
+
+  time = Benchmark.realtime do
+    Fastembed::TextEmbedding.new(model_name: model_name)
+  end
+  puts "#{model_name}: #{(time * 1000).round(1)}ms"
+end
+
+puts
+
+# Single document latency
+print_separator
+puts 'SINGLE DOCUMENT LATENCY (lower is better)'
+print_separator
+
+embedding = Fastembed::TextEmbedding.new
+warmup = embedding.embed(['warmup']).to_a # Warm up
+
+[SHORT_TEXTS.first, MEDIUM_TEXTS.first, LONG_TEXTS.first].each_with_index do |text, i|
+  label = %w[Short Medium Long][i]
+  times = []
+  10.times do
+    time = Benchmark.realtime { embedding.embed([text]).to_a }
+    times << time
+  end
+  avg = times.sum / times.length
+  min = times.min
+  puts "#{label} text (#{text.length} chars): avg #{(avg * 1000).round(2)}ms, min #{(min * 1000).round(2)}ms"
+end
+
+puts
+
+# Throughput benchmarks
+print_separator
+puts 'THROUGHPUT (higher is better)'
+print_separator
+
+[10, 100, 500, 1000].each do |count|
+  texts = MEDIUM_TEXTS.cycle.take(count)
+
+  [32, 64, 128, 256].each do |batch_size|
+    next if batch_size > count
+
+    time = profile_batch(embedding, texts, batch_size)
+    rate = format_rate(count, time)
+    puts "#{count} docs, batch #{batch_size}: #{rate} (#{(time * 1000).round(1)}ms total)"
+  end
+  puts
+end
+
+# Memory efficiency test
+print_separator
+puts 'LAZY EVALUATION TEST'
+print_separator
+
+texts = MEDIUM_TEXTS.cycle.take(1000)
+processed = 0
+
+time = Benchmark.realtime do
+  embedding.embed(texts, batch_size: 64).each do |_vec|
+    processed += 1
+    break if processed >= 100 # Only process first 100
+  end
+end
+
+puts "Processed #{processed}/1000 documents in #{(time * 1000).round(1)}ms"
+puts '(Lazy evaluation means we only computed embeddings for documents we needed)'
+
+puts
+
+# Embedding quality sanity check
+print_separator
+puts 'EMBEDDING QUALITY SANITY CHECK'
+print_separator
+
+test_pairs = [
+  ['dog', 'puppy', 'high'],
+  ['dog', 'cat', 'medium'],
+  ['dog', 'airplane', 'low'],
+  ['machine learning', 'artificial intelligence', 'high'],
+  ['machine learning', 'cooking recipes', 'low']
+]
+
+def cosine_similarity(a, b)
+  a.zip(b).sum { |x, y| x * y }
+end
+
+test_pairs.each do |text1, text2, expected|
+  vecs = embedding.embed([text1, text2]).to_a
+  sim = cosine_similarity(vecs[0], vecs[1])
+  status = case expected
+           when 'high' then sim > 0.7 ? 'PASS' : 'FAIL'
+           when 'medium' then sim > 0.4 && sim < 0.8 ? 'PASS' : 'FAIL'
+           when 'low' then sim < 0.5 ? 'PASS' : 'FAIL'
+           end
+  puts "#{status}: '#{text1}' vs '#{text2}' = #{sim.round(3)} (expected #{expected})"
+end
+
+puts
+
+# Compare with batch sizes
+print_separator
+puts 'OPTIMAL BATCH SIZE ANALYSIS'
+print_separator
+
+texts = MEDIUM_TEXTS.cycle.take(500)
+results = {}
+
+[1, 8, 16, 32, 64, 128, 256, 512].each do |batch_size|
+  time = profile_batch(embedding, texts, batch_size, 2)
+  rate = 500.0 / time
+  results[batch_size] = rate
+  puts "Batch #{batch_size.to_s.rjust(3)}: #{rate.round(1)} docs/sec"
+end
+
+optimal = results.max_by { |_, v| v }
+puts
+puts "Optimal batch size: #{optimal[0]} (#{optimal[1].round(1)} docs/sec)"
+
+puts
+puts '=' * 70
+puts 'PROFILE COMPLETE'
+puts '=' * 70

data/benchmark/reranker_benchmark.rb

@@ -0,0 +1,158 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Benchmark script for TextCrossEncoder (reranker) performance
+
+require 'bundler/setup'
+require 'fastembed'
+require 'benchmark'
+
+QUERY = 'What is machine learning?'
+
+DOCUMENTS = [
+  'Machine learning is a subset of artificial intelligence that enables systems to learn from data.',
+  'Ruby on Rails is a server-side web application framework written in Ruby under the MIT License.',
+  'Deep learning uses neural networks with many layers to model complex patterns in data.',
+  'Vector databases store embeddings and enable fast similarity search across millions of documents.',
+  'Supervised learning requires labeled training data to learn the mapping from inputs to outputs.',
+  'Natural language processing helps computers understand, interpret, and generate human language.',
+  'Random forests are ensemble learning methods that construct multiple decision trees.',
+  'The quick brown fox jumps over the lazy dog. This is a classic pangram used in typing tests.',
+  'Gradient descent is an optimization algorithm used to minimize the loss function in ML models.',
+  'Transformers use self-attention mechanisms to process sequential data in parallel.'
+].freeze
+
+def print_separator
+  puts '-' * 70
+end
+
+puts '=' * 70
+puts 'RERANKER (CROSS-ENCODER) PERFORMANCE BENCHMARK'
+puts '=' * 70
+puts
+puts "Ruby version: #{RUBY_VERSION}"
+puts "Fastembed version: #{Fastembed::VERSION}"
+puts
+
+# Model loading benchmark
+print_separator
+puts 'MODEL LOADING TIME'
+print_separator
+
+Fastembed::SUPPORTED_RERANKER_MODELS.each_key do |model_name|
+  # Ensure model is downloaded first
+  begin
+    Fastembed::TextCrossEncoder.new(model_name: model_name)
+  rescue StandardError
+    puts "#{model_name}: (skipped - not available)"
+    next
+  end
+  GC.start
+
+  time = Benchmark.realtime do
+    Fastembed::TextCrossEncoder.new(model_name: model_name)
+  end
+  puts "#{model_name}: #{(time * 1000).round(1)}ms"
+end
+
+puts
+
+# Use default model for latency tests
+reranker = Fastembed::TextCrossEncoder.new
+
+# Single query latency
+print_separator
+puts 'SINGLE QUERY LATENCY (reranking against 10 documents)'
+print_separator
+
+times = []
+20.times do
+  GC.start
+  time = Benchmark.realtime do
+    reranker.rerank(query: QUERY, documents: DOCUMENTS)
+  end
+  times << time
+end
+
+avg = times.sum / times.length
+min = times.min
+max = times.max
+puts "Average: #{(avg * 1000).round(2)}ms"
+puts "Min: #{(min * 1000).round(2)}ms"
+puts "Max: #{(max * 1000).round(2)}ms"
+puts
+
+# Throughput with varying document counts
+print_separator
+puts 'THROUGHPUT VS DOCUMENT COUNT'
+print_separator
+
+[10, 50, 100, 200].each do |doc_count|
+  docs = DOCUMENTS.cycle.take(doc_count)
+
+  times = []
+  3.times do
+    GC.start
+    time = Benchmark.realtime do
+      reranker.rerank(query: QUERY, documents: docs, batch_size: 64)
+    end
+    times << time
+  end
+
+  min_time = times.min
+  rate = doc_count / min_time
+  puts "#{doc_count} documents: #{rate.round(1)} docs/sec (#{(min_time * 1000).round(1)}ms)"
+end
+
+puts
+
+# Batch size optimization
+print_separator
+puts 'BATCH SIZE OPTIMIZATION (100 documents)'
+print_separator
+
+docs = DOCUMENTS.cycle.take(100)
+results = {}
+
+[8, 16, 32, 64, 128].each do |batch_size|
+  times = []
+  3.times do
+    GC.start
+    time = Benchmark.realtime do
+      reranker.rerank(query: QUERY, documents: docs, batch_size: batch_size)
+    end
+    times << time
+  end
+
+  min_time = times.min
+  rate = 100.0 / min_time
+  results[batch_size] = rate
+  puts "Batch #{batch_size.to_s.rjust(3)}: #{rate.round(1)} docs/sec"
+end
+
+optimal = results.max_by { |_, v| v }
+puts
+puts "Optimal batch size: #{optimal[0]} (#{optimal[1].round(1)} docs/sec)"
+
+puts
+
+# Quality check
+print_separator
+puts 'RERANKING QUALITY CHECK'
+print_separator
+
+results = reranker.rerank_with_scores(query: QUERY, documents: DOCUMENTS, top_k: 5)
+
+puts "Query: '#{QUERY}'"
+puts
+puts 'Top 5 results:'
+results.each_with_index do |result, i|
+  score = result[:score]
+  doc = result[:document][0, 60]
+  puts "#{i + 1}. (#{score.round(3)}) #{doc}..."
+end
+
+puts
+puts '=' * 70
+puts 'BENCHMARK COMPLETE'
+puts '=' * 70

data/exe/fastembed

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'fastembed/cli'
+
+Fastembed::CLI.new(ARGV).run

data/fastembed.gemspec

@@ -32,8 +32,11 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'onnxruntime', '~> 0.9'
   spec.add_dependency 'tokenizers', '~> 0.5'
 
+  spec.add_development_dependency 'mini_magick', '~> 4.0'
   spec.add_development_dependency 'rake', '~> 13.0'
   spec.add_development_dependency 'rspec', '~> 3.0'
   spec.add_development_dependency 'rubocop', '~> 1.0'
   spec.add_development_dependency 'rubocop-rspec', '~> 3.0'
+  spec.add_development_dependency 'webmock', '~> 3.0'
+  spec.add_development_dependency 'yard', '~> 0.9'
 end