rag-ruby 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2b029b1c946d1f57c25b9098864b13297520b56e8970e4882e54684a09f2bbc1
-  data.tar.gz: ff6367e701579b6e93bca9d9c4c2f58d686d9af400e974bd1627936cd80f95e5
+  metadata.gz: 415d46447fc599932fe8f23891c018c4d31fe99de8624efb9173a33fb2fff8c6
+  data.tar.gz: 46b603041b9a0078a22de67ff1d4f6c283c4b5a3809864932ac2e13b700d05ac
 SHA512:
-  metadata.gz: 266712e7a269d2de55c13cc64b47309501692c2b3476d757095708b1ba0bcbf107cb278e20cdab527e216ba97776ae957927827a2d9ea8e32dc9a6b6b84caa1d
-  data.tar.gz: 91e1297be5d3e35dd7d9b119391e38a1c4507727144cfd88f5ade6ba334a6d4103a2b70c4a6869e9262899aab066e6323c36ce7ea234a0486ff0f11b0fb2c857
+  metadata.gz: cda111927e7b402a0a3d66d1d7a5581333f21175421a3f738099f612ae289b6f64b41fc79b9c57c78eb0421a4f1c9b90b97798f07720bb3b4118ab71ea19d65b
+  data.tar.gz: c0d88360302da40b8dde125c3ae4df4894a402e6abd809f81ed0baabc98544f9c128b5c5e75e4deb1d2e77c56f123379b0c1b03755a932cfa5045373a7bd8a17

data/MILESTONES.md

@@ -0,0 +1,12 @@
+# Milestones
+
+## v0.1.1 (2026-03-10)
+
+### Changes
+- Nil response guards
+- error hierarchy
+- retry logic
+- rag_remove_record impl
+
+## v0.1.0 (Initial release)
+- Initial release

data/lib/rag_ruby/configuration.rb

@@ -12,7 +12,10 @@ module RagRuby
     EMBEDDER_REGISTRY = {
       openai: ->(opts) { Embedders::OpenAI.new(**opts) },
       onnx: ->(opts) { Embedders::Onnx.new(**opts) },
-      cohere: ->(opts) { Embedders::Cohere.new(**opts) }
+      cohere: ->(opts) { Embedders::Cohere.new(**opts) },
+      voyage: ->(opts) { Embedders::Voyage.new(**opts) },
+      ollama: ->(opts) { Embedders::Ollama.new(**opts) },
+      hugging_face: ->(opts) { Embedders::HuggingFace.new(**opts) }
     }.freeze
 
     STORE_REGISTRY = {
@@ -22,11 +25,30 @@ module RagRuby
 
     GENERATOR_REGISTRY = {
       openai: ->(opts) { Generators::OpenAI.new(**opts) },
-      ruby_llm: ->(opts) { Generators::RubyLLM.new(**opts) }
+      ruby_llm: ->(opts) { Generators::RubyLLM.new(**opts) },
+      anthropic: ->(opts) { Generators::Anthropic.new(**opts) },
+      gemini: ->(opts) { Generators::Gemini.new(**opts) },
+      ollama: ->(opts) { Generators::Ollama.new(**opts) }
+    }.freeze
+
+    # Maps API key env vars / prefixes to provider symbols for auto-detection
+    AUTO_DETECT_EMBEDDER = {
+      "VOYAGE_API_KEY" => :voyage,
+      "COHERE_API_KEY" => :cohere,
+      "HUGGINGFACE_API_KEY" => :hugging_face,
+      "OPENAI_API_KEY" => :openai
+    }.freeze
+
+    AUTO_DETECT_GENERATOR = {
+      "ANTHROPIC_API_KEY" => :anthropic,
+      "GEMINI_API_KEY" => :gemini,
+      "OPENAI_API_KEY" => :openai
     }.freeze
 
     attr_accessor :loader_instance, :embedder_instance, :store_instance, :generator_instance,
+                  :reranker_instance,
                   :chunk_size, :chunk_overlap, :chunk_strategy,
+                  :retrieval_strategy, :mmr_lambda, :mmr_fetch_k,
                   :http_timeout, :read_timeout
 
     def initialize
@@ -34,6 +56,9 @@ module RagRuby
       @chunk_size = 1000
       @chunk_overlap = 200
       @chunk_strategy = :recursive_character
+      @retrieval_strategy = :similarity
+      @mmr_lambda = 0.5
+      @mmr_fetch_k = 20
       @http_timeout = 30
       @read_timeout = 60
     end
@@ -82,6 +107,16 @@ module RagRuby
                             end
     end
 
+    def reranker(instance)
+      @reranker_instance = instance
+    end
+
+    def retrieval(strategy, lambda: nil, fetch_k: nil)
+      @retrieval_strategy = strategy
+      @mmr_lambda = lambda if lambda
+      @mmr_fetch_k = fetch_k if fetch_k
+    end
+
     def on(event, &block)
       @callbacks[event] << block
     end
@@ -89,5 +124,21 @@ module RagRuby
     def callbacks_for(event)
       @callbacks[event]
     end
+
+    # Auto-detect embedder from available API keys
+    def self.detect_embedder
+      AUTO_DETECT_EMBEDDER.each do |env_var, provider|
+        return provider if ENV[env_var] && !ENV[env_var].empty?
+      end
+      nil
+    end
+
+    # Auto-detect generator from available API keys
+    def self.detect_generator
+      AUTO_DETECT_GENERATOR.each do |env_var, provider|
+        return provider if ENV[env_var] && !ENV[env_var].empty?
+      end
+      nil
+    end
   end
 end

data/lib/rag_ruby/embedders/hugging_face.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "uri"
+require "json"
+
+module RagRuby
+  module Embedders
+    class HuggingFace < Base
+      ENDPOINT = "https://api-inference.huggingface.co/pipeline/feature-extraction"
+
+      def initialize(model: "sentence-transformers/all-MiniLM-L6-v2", api_key: nil)
+        @model = model
+        @api_key = api_key || ENV["HUGGINGFACE_API_KEY"]
+        raise ArgumentError, "HuggingFace API key is required (set HUGGINGFACE_API_KEY or pass api_key:)" unless @api_key
+      end
+
+      def embed(text)
+        embed_batch([text]).first
+      end
+
+      def embed_batch(texts)
+        uri = URI.parse("#{ENDPOINT}/#{@model}")
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = true
+        http.open_timeout = 30
+        http.read_timeout = 120
+
+        req = Net::HTTP::Post.new(uri)
+        req["Authorization"] = "Bearer #{@api_key}"
+        req["Content-Type"] = "application/json"
+        req.body = JSON.generate(inputs: texts)
+
+        response = http.request(req)
+        unless response.is_a?(Net::HTTPSuccess)
+          raise RagRuby::Error, "HuggingFace API error (#{response.code}): #{response.body}"
+        end
+
+        embeddings = JSON.parse(response.body)
+        # HF returns [[token_embeddings]] for each text — mean pool if needed
+        embeddings.map { |e| e.first.is_a?(Array) ? mean_pool(e) : e }
+      end
+
+      def dimension
+        384
+      end
+
+      private
+
+      def mean_pool(token_embeddings)
+        dim = token_embeddings.first.length
+        count = token_embeddings.length.to_f
+        sum = Array.new(dim, 0.0)
+        token_embeddings.each do |vec|
+          vec.each_with_index { |v, i| sum[i] += v }
+        end
+        sum.map { |v| v / count }
+      end
+    end
+  end
+end

data/lib/rag_ruby/embedders/ollama.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "uri"
+require "json"
+
+module RagRuby
+  module Embedders
+    class Ollama < Base
+      def initialize(model: "nomic-embed-text", base_url: nil)
+        @model = model
+        @base_url = base_url || ENV["OLLAMA_URL"] || "http://localhost:11434"
+      end
+
+      def embed(text)
+        uri = URI.parse("#{@base_url}/api/embeddings")
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = uri.scheme == "https"
+        http.open_timeout = 30
+        http.read_timeout = 120
+
+        req = Net::HTTP::Post.new(uri)
+        req["Content-Type"] = "application/json"
+        req.body = JSON.generate(model: @model, prompt: text)
+
+        response = http.request(req)
+        unless response.is_a?(Net::HTTPSuccess)
+          raise RagRuby::Error, "Ollama API error (#{response.code}): #{response.body}"
+        end
+
+        JSON.parse(response.body)["embedding"]
+      end
+
+      def dimension
+        768
+      end
+    end
+  end
+end

data/lib/rag_ruby/embedders/voyage.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "uri"
+require "json"
+
+module RagRuby
+  module Embedders
+    class Voyage < Base
+      ENDPOINT = "https://api.voyageai.com/v1/embeddings"
+
+      def initialize(model: "voyage-3", api_key: nil)
+        @model = model
+        @api_key = api_key || ENV["VOYAGE_API_KEY"]
+        raise ArgumentError, "Voyage API key is required (set VOYAGE_API_KEY or pass api_key:)" unless @api_key
+      end
+
+      def embed(text)
+        embed_batch([text]).first
+      end
+
+      def embed_batch(texts)
+        uri = URI.parse(ENDPOINT)
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = true
+        http.open_timeout = 30
+        http.read_timeout = 60
+
+        req = Net::HTTP::Post.new(uri)
+        req["Authorization"] = "Bearer #{@api_key}"
+        req["Content-Type"] = "application/json"
+        req.body = JSON.generate(model: @model, input: texts)
+
+        response = http.request(req)
+        unless response.is_a?(Net::HTTPSuccess)
+          raise RagRuby::Error, "Voyage API error (#{response.code}): #{response.body}"
+        end
+
+        parsed = JSON.parse(response.body)
+        parsed["data"].sort_by { |d| d["index"] }.map { |d| d["embedding"] }
+      end
+
+      def dimension
+        1024
+      end
+    end
+  end
+end

data/lib/rag_ruby/generators/anthropic.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "uri"
+require "json"
+
+module RagRuby
+  module Generators
+    class Anthropic < Base
+      ENDPOINT = "https://api.anthropic.com/v1/messages"
+
+      def initialize(model: "claude-sonnet-4-20250514", api_key: nil, max_tokens: 4096)
+        @model = model
+        @api_key = api_key || ENV["ANTHROPIC_API_KEY"]
+        @max_tokens = max_tokens
+        raise ArgumentError, "Anthropic API key is required (set ANTHROPIC_API_KEY or pass api_key:)" unless @api_key
+      end
+
+      def generate(prompt:, system_prompt: nil, temperature: 0.7)
+        body = {
+          model: @model,
+          max_tokens: @max_tokens,
+          messages: [{ role: "user", content: prompt }],
+          temperature: temperature
+        }
+        body[:system] = system_prompt if system_prompt
+
+        request_with_retry do
+          uri = URI.parse(ENDPOINT)
+          http = Net::HTTP.new(uri.host, uri.port)
+          http.use_ssl = true
+          http.open_timeout = 30
+          http.read_timeout = 120
+
+          req = Net::HTTP::Post.new(uri)
+          req["x-api-key"] = @api_key
+          req["anthropic-version"] = "2023-06-01"
+          req["Content-Type"] = "application/json"
+          req.body = JSON.generate(body)
+
+          response = http.request(req)
+
+          unless response.is_a?(Net::HTTPSuccess)
+            raise RagRuby::Error, "Anthropic API error (#{response.code}): #{response.body}"
+          end
+
+          data = JSON.parse(response.body)
+          text = data.dig("content", 0, "text")
+          raise RagRuby::Error, "Empty response from Anthropic" if text.nil?
+          usage = data["usage"] || {}
+
+          {
+            text: text,
+            tokens_used: {
+              prompt: usage["input_tokens"],
+              completion: usage["output_tokens"]
+            }
+          }
+        end
+      end
+
+      private
+
+      def request_with_retry(max_retries: 3)
+        retries = 0
+        begin
+          yield
+        rescue => e
+          retries += 1
+          if retries <= max_retries && retryable?(e)
+            sleep(2**(retries - 1))
+            retry
+          end
+          raise
+        end
+      end
+
+      def retryable?(e)
+        e.message.match?(/429|500|502|503|529/)
+      end
+    end
+  end
+end

data/lib/rag_ruby/generators/gemini.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "uri"
+require "json"
+
+module RagRuby
+  module Generators
+    class Gemini < Base
+      ENDPOINT = "https://generativelanguage.googleapis.com/v1beta/models"
+
+      def initialize(model: "gemini-2.0-flash", api_key: nil)
+        @model = model
+        @api_key = api_key || ENV["GEMINI_API_KEY"]
+        raise ArgumentError, "Gemini API key is required (set GEMINI_API_KEY or pass api_key:)" unless @api_key
+      end
+
+      def generate(prompt:, system_prompt: nil, temperature: 0.7)
+        body = {
+          contents: [{ parts: [{ text: prompt }] }],
+          generationConfig: { temperature: temperature }
+        }
+        if system_prompt
+          body[:systemInstruction] = { parts: [{ text: system_prompt }] }
+        end
+
+        request_with_retry do
+          uri = URI.parse("#{ENDPOINT}/#{@model}:generateContent?key=#{@api_key}")
+          http = Net::HTTP.new(uri.host, uri.port)
+          http.use_ssl = true
+          http.open_timeout = 30
+          http.read_timeout = 120
+
+          req = Net::HTTP::Post.new(uri)
+          req["Content-Type"] = "application/json"
+          req.body = JSON.generate(body)
+
+          response = http.request(req)
+
+          unless response.is_a?(Net::HTTPSuccess)
+            raise RagRuby::Error, "Gemini API error (#{response.code}): #{response.body}"
+          end
+
+          data = JSON.parse(response.body)
+          text = data.dig("candidates", 0, "content", "parts", 0, "text")
+          raise RagRuby::Error, "Empty response from Gemini" if text.nil?
+          usage = data["usageMetadata"] || {}
+
+          {
+            text: text,
+            tokens_used: {
+              prompt: usage["promptTokenCount"],
+              completion: usage["candidatesTokenCount"]
+            }
+          }
+        end
+      end
+
+      private
+
+      def request_with_retry(max_retries: 3)
+        retries = 0
+        begin
+          yield
+        rescue => e
+          retries += 1
+          if retries <= max_retries && retryable?(e)
+            sleep(2**(retries - 1))
+            retry
+          end
+          raise
+        end
+      end
+
+      def retryable?(e)
+        e.message.match?(/429|500|502|503/)
+      end
+    end
+  end
+end

data/lib/rag_ruby/generators/ollama.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "uri"
+require "json"
+
+module RagRuby
+  module Generators
+    class Ollama < Base
+      def initialize(model: "llama3.2", base_url: nil)
+        @model = model
+        @base_url = base_url || ENV["OLLAMA_URL"] || "http://localhost:11434"
+      end
+
+      def generate(prompt:, system_prompt: nil, temperature: 0.7)
+        uri = URI.parse("#{@base_url}/api/chat")
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = uri.scheme == "https"
+        http.open_timeout = 30
+        http.read_timeout = 300
+
+        messages = []
+        messages << { role: "system", content: system_prompt } if system_prompt
+        messages << { role: "user", content: prompt }
+
+        body = {
+          model: @model,
+          messages: messages,
+          stream: false,
+          options: { temperature: temperature }
+        }
+
+        req = Net::HTTP::Post.new(uri)
+        req["Content-Type"] = "application/json"
+        req.body = JSON.generate(body)
+
+        response = http.request(req)
+
+        unless response.is_a?(Net::HTTPSuccess)
+          raise RagRuby::Error, "Ollama API error (#{response.code}): #{response.body}"
+        end
+
+        data = JSON.parse(response.body)
+        text = data.dig("message", "content")
+        raise RagRuby::Error, "Empty response from Ollama" if text.nil?
+
+        {
+          text: text,
+          tokens_used: {
+            prompt: data["prompt_eval_count"],
+            completion: data["eval_count"]
+          }
+        }
+      end
+    end
+  end
+end

data/lib/rag_ruby/pipeline.rb

@@ -54,8 +54,20 @@ module RagRuby
       # Embed the question
       query_embedding = @config.embedder_instance.embed(question)
 
-      # Search the store
-      results = @config.store_instance.search(query_embedding, top_k: top_k, filter: filter)
+      # Search the store (with retrieval strategy)
+      search_opts = { top_k: top_k, filter: filter }
+      if @config.retrieval_strategy == :mmr
+        search_opts[:strategy] = :mmr
+        search_opts[:lambda] = @config.mmr_lambda
+        search_opts[:fetch_k] = @config.mmr_fetch_k
+      end
+
+      results = @config.store_instance.search(query_embedding, **search_opts)
+
+      # Rerank if configured
+      if @config.reranker_instance
+        results = rerank_results(question, results)
+      end
 
       # Build sources from results
       sources = results.map do |result|
@@ -161,6 +173,22 @@ module RagRuby
       context.strip
     end
 
+    def rerank_results(query, results)
+      documents = results.map do |r|
+        chunk = r[:chunk]
+        chunk.respond_to?(:text) ? chunk.text : chunk.to_s
+      end
+
+      reranked = @config.reranker_instance.rerank(query, documents)
+
+      reranked.map do |rr|
+        idx = rr.respond_to?(:index) ? rr.index : rr[:index]
+        score = rr.respond_to?(:score) ? rr.score : rr[:score]
+        original = results[idx]
+        original.merge(score: score)
+      end
+    end
+
     def fire(event, *args)
       @config.callbacks_for(event).each { |cb| cb.call(*args) }
     end

data/lib/rag_ruby/stores/memory.rb

@@ -14,7 +14,7 @@ module RagRuby
         @entries[id] = Entry.new(id: id, embedding: embedding, metadata: metadata, chunk: chunk)
       end
 
-      def search(embedding, top_k: 5, filter: nil)
+      def search(embedding, top_k: 5, filter: nil, strategy: :similarity, lambda: 0.5, fetch_k: 20)
         results = @entries.values
 
         if filter
@@ -23,11 +23,16 @@ module RagRuby
           end
         end
 
-        results
+        scored = results
           .map { |entry| [entry, cosine_similarity(embedding, entry.embedding)] }
           .sort_by { |_, score| -score }
-          .first(top_k)
-          .map { |entry, score| { id: entry.id, score: score, metadata: entry.metadata, chunk: entry.chunk } }
+
+        if strategy == :mmr
+          mmr_select(scored, embedding, top_k: top_k, lambda: lambda, fetch_k: fetch_k)
+        else
+          scored.first(top_k)
+            .map { |entry, score| { id: entry.id, score: score, metadata: entry.metadata, chunk: entry.chunk } }
+        end
       end
 
       def delete(id)
@@ -52,6 +57,43 @@ module RagRuby
 
         dot / (mag_a * mag_b)
       end
+
+      # Maximal Marginal Relevance: balances relevance and diversity
+      def mmr_select(scored, query_embedding, top_k:, lambda:, fetch_k:)
+        candidates = scored.first(fetch_k)
+        return [] if candidates.empty?
+
+        selected = []
+        remaining = candidates.dup
+
+        top_k.times do
+          break if remaining.empty?
+
+          best = nil
+          best_mmr = -Float::INFINITY
+
+          remaining.each do |entry, relevance|
+            if selected.empty?
+              diversity = 0.0
+            else
+              diversity = selected.map { |sel, _| cosine_similarity(entry.embedding, sel.embedding) }.max
+            end
+
+            mmr_score = lambda * relevance - (1.0 - lambda) * diversity
+
+            if mmr_score > best_mmr
+              best_mmr = mmr_score
+              best = [entry, relevance]
+            end
+          end
+
+          break unless best
+          selected << best
+          remaining.delete(best)
+        end
+
+        selected.map { |entry, score| { id: entry.id, score: score, metadata: entry.metadata, chunk: entry.chunk } }
+      end
     end
   end
 end

data/lib/rag_ruby/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RagRuby
-  VERSION = "0.1.1"
+  VERSION = "0.2.0"
 end

data/lib/rag_ruby.rb

@@ -15,12 +15,19 @@ require_relative "rag_ruby/loaders/active_record"
 
 require_relative "rag_ruby/embedders/base"
 require_relative "rag_ruby/embedders/openai"
+require_relative "rag_ruby/embedders/cohere"
+require_relative "rag_ruby/embedders/voyage"
+require_relative "rag_ruby/embedders/ollama"
+require_relative "rag_ruby/embedders/hugging_face"
 
 require_relative "rag_ruby/stores/base"
 require_relative "rag_ruby/stores/memory"
 
 require_relative "rag_ruby/generators/base"
 require_relative "rag_ruby/generators/openai"
+require_relative "rag_ruby/generators/anthropic"
+require_relative "rag_ruby/generators/gemini"
+require_relative "rag_ruby/generators/ollama"
 
 require_relative "rag_ruby/configuration"
 require_relative "rag_ruby/pipeline"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rag-ruby
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Johannes Dwi Cahyo
@@ -76,6 +76,7 @@ extra_rdoc_files: []
 files:
 - Gemfile
 - LICENSE
+- MILESTONES.md
 - README.md
 - Rakefile
 - lib/rag_ruby.rb
@@ -85,9 +86,15 @@ files:
 - lib/rag_ruby/document.rb
 - lib/rag_ruby/embedders/base.rb
 - lib/rag_ruby/embedders/cohere.rb
+- lib/rag_ruby/embedders/hugging_face.rb
+- lib/rag_ruby/embedders/ollama.rb
 - lib/rag_ruby/embedders/onnx.rb
 - lib/rag_ruby/embedders/openai.rb
+- lib/rag_ruby/embedders/voyage.rb
+- lib/rag_ruby/generators/anthropic.rb
 - lib/rag_ruby/generators/base.rb
+- lib/rag_ruby/generators/gemini.rb
+- lib/rag_ruby/generators/ollama.rb
 - lib/rag_ruby/generators/openai.rb
 - lib/rag_ruby/generators/ruby_llm.rb
 - lib/rag_ruby/loaders/active_record.rb