rag-ruby 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 18916baa3b84e6a947a1fa8411658475e4608ff4a127d681ca9b29332d669e81
+  data.tar.gz: f18c5e4d77640750e570ae4762122d0b687b73b2d12731e11add7b34d00b9d8c
+SHA512:
+  metadata.gz: f397468f410d2475d1d59557749e83092c6b65b6f844b4169fff2774dbaac436a7eca5d9e667a35f7750937722d4bf92f97ec550b1919fd0deadd1857a9b1531
+  data.tar.gz: 54d733930d3ba8c0a241240318fbe58c4e8dd1750e3e9f9dc5969c86b9280721386fbe24f9379c47f06a598871950e5d97e8e6b42a13c72d6f8d89da40d2f1d9

data/Gemfile

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gemspec
+
+gem "minitest", "~> 5.0"
+gem "rake", "~> 13.0"
+gem "webmock", "~> 3.0"

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Johannes Dwi Cahyo
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,288 @@
+# rag-ruby
+
+A batteries-included RAG (Retrieval-Augmented Generation) pipeline framework for Ruby and Rails.
+
+Orchestrates the full workflow: **document loading → chunking → embedding → storage → retrieval → generation**. Think LangChain for Ruby — simpler, more opinionated, and Rails-native.
+
+## Installation
+
+Add to your Gemfile:
+
+```ruby
+gem "rag-ruby"
+```
+
+Then run:
+
+```bash
+bundle install
+```
+
+## Quick Start
+
+```ruby
+require "rag_ruby"
+
+pipeline = RagRuby::Pipeline.new do |config|
+  config.loader :file
+  config.chunker :recursive_character, chunk_size: 1000, chunk_overlap: 200
+  config.embedder :openai, model: "text-embedding-3-small"
+  config.store :memory, dimension: 1536
+  config.generator :openai, model: "gpt-4o"
+end
+
+# Ingest documents
+pipeline.ingest("docs/manual.md")
+pipeline.ingest_directory("docs/", glob: "**/*.{md,txt}")
+
+# Query with RAG
+answer = pipeline.query("How do I reset my password?")
+answer.text       # => "To reset your password, go to Settings > Security..."
+answer.sources    # => [#<Source chunk="..." score=0.92>, ...]
+answer.tokens_used # => { prompt: 1200, completion: 150 }
+```
+
+## Components
+
+Every stage of the pipeline is pluggable. Mix and match providers to fit your stack.
+
+### Document Loaders
+
+| Loader | Description | Require |
+|--------|-------------|---------|
+| `:file` | Local files (.txt, .md) | Built-in |
+| `:directory` | Bulk load from directory | Built-in |
+| `:url` | Fetch from URLs | Built-in |
+| `:active_record` | Load from ActiveRecord models | Built-in |
+
+```ruby
+# Load a single file
+pipeline.ingest("path/to/document.md")
+
+# Load a directory
+pipeline.ingest_directory("documents/", glob: "**/*.{md,txt}")
+
+# Custom loader
+class SlackLoader < RagRuby::Loaders::Base
+  def load(channel_id)
+    messages = SlackAPI.history(channel_id)
+    messages.map do |msg|
+      RagRuby::Document.new(
+        content: msg.text,
+        metadata: { author: msg.user, channel: channel_id }
+      )
+    end
+  end
+end
+
+pipeline.ingest(channel_id, loader: SlackLoader.new)
+```
+
+### Embedders
+
+| Provider | Description | Require |
+|----------|-------------|---------|
+| `:openai` | OpenAI text-embedding-3-small/large | `OPENAI_API_KEY` env var |
+| `:cohere` | Cohere embed-english-v3.0 | `COHERE_API_KEY` env var |
+| `:onnx` | Local ONNX models (all-MiniLM-L6-v2) | `gem "onnx-ruby"` |
+
+```ruby
+# API-based
+config.embedder :openai, model: "text-embedding-3-small"
+config.embedder :cohere, model: "embed-english-v3.0"
+
+# Local (no API calls)
+config.embedder :onnx, model: "all-MiniLM-L6-v2"
+```
+
+### Vector Stores
+
+| Store | Description | Require |
+|-------|-------------|---------|
+| `:memory` | In-memory store (great for dev/test) | Built-in |
+| `:zvec` | Persistent file-based vector store | `gem "zvec-ruby"` |
+
+```ruby
+config.store :memory, dimension: 1536
+config.store :zvec, path: "./vectors", dimension: 1536
+```
+
+Custom stores are easy — implement `add`, `search`, `delete`, and `count`:
+
+```ruby
+class PineconeStore < RagRuby::Stores::Base
+  def add(id, embedding:, metadata: {}, chunk: nil) = ...
+  def search(embedding, top_k:, filter: nil) = ...
+  def delete(id) = ...
+  def count = ...
+end
+```
+
+### Generators
+
+| Provider | Description | Require |
+|----------|-------------|---------|
+| `:openai` | OpenAI chat completions | `OPENAI_API_KEY` env var |
+| `:ruby_llm` | Any model via ruby_llm | `gem "ruby_llm"` |
+
+```ruby
+config.generator :openai, model: "gpt-4o"
+config.generator :ruby_llm, model: "claude-sonnet-4-20250514"
+```
+
+## Query Options
+
+```ruby
+answer = pipeline.query("What changed in v2.0?",
+  top_k: 10,                    # number of chunks to retrieve
+  filter: { category: "changelog" }, # metadata filter
+  temperature: 0.0,             # generation temperature
+  system_prompt: "You are a technical docs assistant."
+)
+
+answer.text        # generated answer
+answer.sources     # retrieved chunks with scores
+answer.tokens_used # { prompt: ..., completion: ... }
+answer.duration    # query time in seconds
+answer.query       # original question
+```
+
+## Callbacks & Observability
+
+Hook into every stage of the pipeline for logging, metrics, or debugging:
+
+```ruby
+pipeline = RagRuby::Pipeline.new do |config|
+  # ... providers ...
+
+  config.on(:before_load)  { |src| puts "Loading: #{src}" }
+  config.on(:after_load)   { |docs| puts "Loaded #{docs.size} documents" }
+  config.on(:before_chunk) { |doc| puts "Chunking: #{doc.source}" }
+  config.on(:after_chunk)  { |chunks| puts "Created #{chunks.size} chunks" }
+  config.on(:before_embed) { |chunks| puts "Embedding #{chunks.size} chunks" }
+  config.on(:after_embed)  { |chunks| puts "Embedded #{chunks.size} chunks" }
+  config.on(:before_store) { |chunks| puts "Storing #{chunks.size} chunks" }
+  config.on(:after_store)  { |chunks| puts "Stored #{chunks.size} chunks" }
+  config.on(:before_query) { |q| Metrics.increment("rag.queries") }
+  config.on(:after_query)  { |q, answer| Metrics.timing("rag.latency", answer.duration) }
+end
+```
+
+## Rails Integration
+
+### Setup
+
+```bash
+rails generate rag:install
+```
+
+This creates:
+- `config/rag.yml` — environment-specific configuration
+- `config/initializers/rag_ruby.rb` — optional programmatic config
+
+### Configuration
+
+```yaml
+# config/rag.yml
+default: &default
+  chunker:
+    strategy: recursive_character
+    chunk_size: 1000
+    chunk_overlap: 200
+  embedder:
+    provider: openai
+    model: text-embedding-3-small
+  store:
+    provider: memory
+    dimension: 1536
+  generator:
+    provider: openai
+    model: gpt-4o
+
+development:
+  <<: *default
+
+production:
+  <<: *default
+  store:
+    provider: zvec
+    path: db/vectors
+    dimension: 1536
+```
+
+### Auto-Index Models
+
+```ruby
+class Article < ApplicationRecord
+  include RagRuby::Indexable
+
+  rag_index :content,
+    metadata: ->(article) { { category: article.category, author: article.author } },
+    on: [:create, :update]
+end
+
+# Articles are automatically indexed when saved
+Article.create!(title: "Guide", content: "# Getting Started\n...")
+```
+
+### Global API
+
+```ruby
+# Search for relevant chunks
+results = RagRuby.search("How to get started?", top_k: 5)
+
+# Full RAG: retrieve + generate
+answer = RagRuby.ask("How to get started?")
+```
+
+### Controller Usage
+
+```ruby
+class ChatController < ApplicationController
+  def ask
+    answer = RagRuby.ask(params[:question])
+    render json: {
+      answer: answer.text,
+      sources: answer.sources.map(&:to_h)
+    }
+  end
+end
+```
+
+## Architecture
+
+### Ingestion Flow
+
+```
+Document → Loader → [Document] → Chunker → [Chunk] → Embedder → [Chunk+Embedding] → Store
+```
+
+### Query Flow
+
+```
+Question → Embedder → Vector → Store.search → [Chunk] → build_context → Generator → Answer
+```
+
+Each stage is independent and swappable. The `Pipeline` class orchestrates the flow.
+
+## Dependencies
+
+| Gem | Purpose | Required? |
+|-----|---------|-----------|
+| `chunker-ruby` | Text chunking | Yes |
+| `zvec-ruby` | Persistent vector storage | Optional |
+| `onnx-ruby` | Local ONNX embeddings | Optional |
+| `ruby_llm` | Multi-provider LLM generation | Optional |
+
+## Development
+
+```bash
+git clone https://github.com/johannesdwicahyo/rag-ruby.git
+cd rag-ruby
+bundle install
+bundle exec rake test
+```
+
+## License
+
+MIT License. See [LICENSE](LICENSE) for details.

data/Rakefile

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/test_*.rb"]
+end
+
+task default: :test

data/lib/rag_ruby/answer.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module RagRuby
+  class Answer
+    attr_reader :text, :sources, :tokens_used, :duration, :query
+
+    def initialize(text:, sources: [], tokens_used: {}, duration: nil, query: nil)
+      @text = text
+      @sources = sources
+      @tokens_used = tokens_used
+      @duration = duration
+      @query = query
+    end
+
+    def to_s
+      text
+    end
+
+    def to_h
+      {
+        text: text,
+        sources: sources.map(&:to_h),
+        tokens_used: tokens_used,
+        duration: duration,
+        query: query
+      }
+    end
+  end
+end

data/lib/rag_ruby/chunk.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module RagRuby
+  class Chunk
+    attr_accessor :text, :embedding, :metadata, :document_source, :index
+
+    def initialize(text:, metadata: {}, document_source: nil, index: 0)
+      @text = text
+      @metadata = metadata
+      @document_source = document_source
+      @index = index
+      @embedding = nil
+    end
+
+    def embedded?
+      !@embedding.nil?
+    end
+
+    def to_s
+      text
+    end
+
+    def bytesize
+      text.bytesize
+    end
+  end
+end

data/lib/rag_ruby/configuration.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module RagRuby
+  class Configuration
+    LOADER_REGISTRY = {
+      file: -> { Loaders::File.new },
+      directory: -> { Loaders::Directory.new },
+      url: -> { Loaders::URL.new },
+      active_record: -> { Loaders::ActiveRecord.new }
+    }.freeze
+
+    EMBEDDER_REGISTRY = {
+      openai: ->(opts) { Embedders::OpenAI.new(**opts) },
+      onnx: ->(opts) { Embedders::Onnx.new(**opts) },
+      cohere: ->(opts) { Embedders::Cohere.new(**opts) }
+    }.freeze
+
+    STORE_REGISTRY = {
+      zvec: ->(opts) { Stores::Zvec.new(**opts) },
+      memory: ->(opts) { Stores::Memory.new(**opts) }
+    }.freeze
+
+    GENERATOR_REGISTRY = {
+      openai: ->(opts) { Generators::OpenAI.new(**opts) },
+      ruby_llm: ->(opts) { Generators::RubyLLM.new(**opts) }
+    }.freeze
+
+    attr_accessor :loader_instance, :embedder_instance, :store_instance, :generator_instance,
+                  :chunk_size, :chunk_overlap, :chunk_strategy
+
+    def initialize
+      @callbacks = Hash.new { |h, k| h[k] = [] }
+      @chunk_size = 1000
+      @chunk_overlap = 200
+      @chunk_strategy = :recursive_character
+    end
+
+    def loader(name, **opts)
+      @loader_instance = if LOADER_REGISTRY.key?(name)
+                           LOADER_REGISTRY[name].call
+                         else
+                           raise ArgumentError, "Unknown loader: #{name}"
+                         end
+    end
+
+    def chunker(strategy, chunk_size: 1000, chunk_overlap: 200)
+      @chunk_strategy = strategy
+      @chunk_size = chunk_size
+      @chunk_overlap = chunk_overlap
+    end
+
+    def embedder(name, **opts)
+      @embedder_instance = if EMBEDDER_REGISTRY.key?(name)
+                             EMBEDDER_REGISTRY[name].call(opts)
+                           elsif name.is_a?(Class) || name.respond_to?(:embed)
+                             name
+                           else
+                             raise ArgumentError, "Unknown embedder: #{name}"
+                           end
+    end
+
+    def store(name, **opts)
+      @store_instance = if STORE_REGISTRY.key?(name)
+                          STORE_REGISTRY[name].call(opts)
+                        elsif name.is_a?(Class) || name.respond_to?(:search)
+                          name
+                        else
+                          raise ArgumentError, "Unknown store: #{name}"
+                        end
+    end
+
+    def generator(name, **opts)
+      @generator_instance = if GENERATOR_REGISTRY.key?(name)
+                              GENERATOR_REGISTRY[name].call(opts)
+                            elsif name.is_a?(Class) || name.respond_to?(:generate)
+                              name
+                            else
+                              raise ArgumentError, "Unknown generator: #{name}"
+                            end
+    end
+
+    def on(event, &block)
+      @callbacks[event] << block
+    end
+
+    def callbacks_for(event)
+      @callbacks[event]
+    end
+  end
+end

data/lib/rag_ruby/document.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module RagRuby
+  class Document
+    attr_accessor :content, :metadata, :source
+
+    def initialize(content:, metadata: {}, source: nil)
+      @content = content
+      @metadata = metadata
+      @source = source
+    end
+
+    def to_s
+      content
+    end
+
+    def bytesize
+      content.bytesize
+    end
+
+    def empty?
+      content.nil? || content.strip.empty?
+    end
+  end
+end

data/lib/rag_ruby/embedders/base.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module RagRuby
+  module Embedders
+    class Base
+      def embed(text)
+        raise NotImplementedError, "#{self.class}#embed must be implemented"
+      end
+
+      def embed_batch(texts)
+        texts.map { |t| embed(t) }
+      end
+
+      def dimension
+        raise NotImplementedError, "#{self.class}#dimension must be implemented"
+      end
+    end
+  end
+end

data/lib/rag_ruby/embedders/cohere.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "uri"
+require "json"
+
+module RagRuby
+  module Embedders
+    class Cohere < Base
+      ENDPOINT = "https://api.cohere.ai/v1/embed"
+
+      def initialize(model: "embed-english-v3.0", api_key: nil)
+        @model = model
+        @api_key = api_key || ENV["COHERE_API_KEY"]
+        raise ArgumentError, "Cohere API key is required (set COHERE_API_KEY or pass api_key:)" unless @api_key
+      end
+
+      def embed(text)
+        embed_batch([text]).first
+      end
+
+      def embed_batch(texts)
+        uri = URI.parse(ENDPOINT)
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = true
+
+        req = Net::HTTP::Post.new(uri)
+        req["Authorization"] = "Bearer #{@api_key}"
+        req["Content-Type"] = "application/json"
+        req.body = JSON.generate(
+          model: @model,
+          texts: texts,
+          input_type: "search_document"
+        )
+
+        response = http.request(req)
+
+        unless response.is_a?(Net::HTTPSuccess)
+          raise "Cohere API error (#{response.code}): #{response.body}"
+        end
+
+        JSON.parse(response.body)["embeddings"]
+      end
+
+      def dimension
+        1024
+      end
+    end
+  end
+end

data/lib/rag_ruby/embedders/onnx.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module RagRuby
+  module Embedders
+    class Onnx < Base
+      def initialize(model: "all-MiniLM-L6-v2", model_path: nil)
+        @model = model
+        @model_path = model_path
+
+        begin
+          require "onnx_ruby"
+        rescue LoadError
+          raise LoadError, "onnx-ruby gem is required for ONNX embeddings. Add `gem 'onnx-ruby'` to your Gemfile."
+        end
+
+        @session = create_session
+      end
+
+      def embed(text)
+        @session.embed(text)
+      end
+
+      def embed_batch(texts)
+        texts.map { |t| embed(t) }
+      end
+
+      def dimension
+        384 # all-MiniLM-L6-v2 default
+      end
+
+      private
+
+      def create_session
+        if @model_path
+          OnnxRuby::Session.new(@model_path)
+        else
+          OnnxRuby::Session.from_pretrained(@model)
+        end
+      end
+    end
+  end
+end

data/lib/rag_ruby/embedders/openai.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "uri"
+require "json"
+
+module RagRuby
+  module Embedders
+    class OpenAI < Base
+      ENDPOINT = "https://api.openai.com/v1/embeddings"
+
+      DIMENSIONS = {
+        "text-embedding-3-small" => 1536,
+        "text-embedding-3-large" => 3072,
+        "text-embedding-ada-002" => 1536
+      }.freeze
+
+      def initialize(model: "text-embedding-3-small", api_key: nil)
+        @model = model
+        @api_key = api_key || ENV["OPENAI_API_KEY"]
+        raise ArgumentError, "OpenAI API key is required (set OPENAI_API_KEY or pass api_key:)" unless @api_key
+      end
+
+      def embed(text)
+        response = request([text])
+        response.dig("data", 0, "embedding")
+      end
+
+      def embed_batch(texts)
+        response = request(texts)
+        response["data"]
+          .sort_by { |d| d["index"] }
+          .map { |d| d["embedding"] }
+      end
+
+      def dimension
+        DIMENSIONS.fetch(@model) { 1536 }
+      end
+
+      private
+
+      def request(input)
+        uri = URI.parse(ENDPOINT)
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = true
+        http.open_timeout = 30
+        http.read_timeout = 60
+
+        req = Net::HTTP::Post.new(uri)
+        req["Authorization"] = "Bearer #{@api_key}"
+        req["Content-Type"] = "application/json"
+        req.body = JSON.generate(model: @model, input: input)
+
+        response = http.request(req)
+
+        unless response.is_a?(Net::HTTPSuccess)
+          raise "OpenAI API error (#{response.code}): #{response.body}"
+        end
+
+        JSON.parse(response.body)
+      end
+    end
+  end
+end

data/lib/rag_ruby/generators/base.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module RagRuby
+  module Generators
+    class Base
+      def generate(prompt:, system_prompt: nil, temperature: 0.7)
+        raise NotImplementedError, "#{self.class}#generate must be implemented"
+      end
+    end
+  end
+end