rag-ruby 0.1.0

data/lib/rag_ruby/generators/openai.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "uri"
+require "json"
+
+module RagRuby
+  module Generators
+    class OpenAI < Base
+      ENDPOINT = "https://api.openai.com/v1/chat/completions"
+
+      def initialize(model: "gpt-4o", api_key: nil)
+        @model = model
+        @api_key = api_key || ENV["OPENAI_API_KEY"]
+        raise ArgumentError, "OpenAI API key is required (set OPENAI_API_KEY or pass api_key:)" unless @api_key
+      end
+
+      def generate(prompt:, system_prompt: nil, temperature: 0.7)
+        messages = []
+        messages << { role: "system", content: system_prompt } if system_prompt
+        messages << { role: "user", content: prompt }
+
+        body = {
+          model: @model,
+          messages: messages,
+          temperature: temperature
+        }
+
+        uri = URI.parse(ENDPOINT)
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = true
+        http.open_timeout = 30
+        http.read_timeout = 120
+
+        req = Net::HTTP::Post.new(uri)
+        req["Authorization"] = "Bearer #{@api_key}"
+        req["Content-Type"] = "application/json"
+        req.body = JSON.generate(body)
+
+        response = http.request(req)
+
+        unless response.is_a?(Net::HTTPSuccess)
+          raise "OpenAI API error (#{response.code}): #{response.body}"
+        end
+
+        data = JSON.parse(response.body)
+        text = data.dig("choices", 0, "message", "content")
+        usage = data["usage"] || {}
+
+        {
+          text: text,
+          tokens_used: {
+            prompt: usage["prompt_tokens"],
+            completion: usage["completion_tokens"]
+          }
+        }
+      end
+    end
+  end
+end

data/lib/rag_ruby/generators/ruby_llm.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module RagRuby
+  module Generators
+    class RubyLLM < Base
+      def initialize(model: "gpt-4o", provider: nil)
+        begin
+          require "ruby_llm"
+        rescue LoadError
+          raise LoadError, "ruby_llm gem is required. Add `gem 'ruby_llm'` to your Gemfile."
+        end
+
+        @model = model
+        @provider = provider
+      end
+
+      def generate(prompt:, system_prompt: nil, temperature: 0.7)
+        chat = ::RubyLLM.chat(model: @model)
+        chat.with_temperature(temperature)
+        chat.with_instructions(system_prompt) if system_prompt
+
+        response = chat.ask(prompt)
+
+        {
+          text: response.content,
+          tokens_used: {
+            prompt: response.input_tokens,
+            completion: response.output_tokens
+          }
+        }
+      end
+    end
+  end
+end

data/lib/rag_ruby/loaders/active_record.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module RagRuby
+  module Loaders
+    class ActiveRecord < Base
+      def initialize(column: :content, metadata_columns: [])
+        @column = column
+        @metadata_columns = metadata_columns
+      end
+
+      def load(scope)
+        records = scope.respond_to?(:find_each) ? scope.to_a : [scope]
+
+        records.map do |record|
+          content = record.public_send(@column).to_s
+          metadata = build_metadata(record)
+
+          Document.new(
+            content: content,
+            metadata: metadata,
+            source: "#{record.class.name}##{record.id}"
+          )
+        end
+      end
+
+      private
+
+      def build_metadata(record)
+        meta = { model: record.class.name, id: record.id }
+        @metadata_columns.each do |col|
+          meta[col] = record.public_send(col) if record.respond_to?(col)
+        end
+        meta
+      end
+    end
+  end
+end

data/lib/rag_ruby/loaders/base.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module RagRuby
+  module Loaders
+    class Base
+      def load(source)
+        raise NotImplementedError, "#{self.class}#load must be implemented"
+      end
+    end
+  end
+end

data/lib/rag_ruby/loaders/directory.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module RagRuby
+  module Loaders
+    class Directory < Base
+      DEFAULT_GLOB = "**/*.{txt,md,markdown}"
+
+      def initialize(glob: DEFAULT_GLOB)
+        @glob = glob
+        @file_loader = File.new
+      end
+
+      def load(dir_path)
+        dir_path = ::File.expand_path(dir_path)
+        raise ArgumentError, "Directory not found: #{dir_path}" unless ::Dir.exist?(dir_path)
+
+        pattern = ::File.join(dir_path, @glob)
+        files = ::Dir.glob(pattern).sort
+
+        files.flat_map do |file_path|
+          @file_loader.load(file_path)
+        rescue ArgumentError
+          # Skip unsupported file types
+          []
+        end
+      end
+    end
+  end
+end

data/lib/rag_ruby/loaders/file.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module RagRuby
+  module Loaders
+    class File < Base
+      SUPPORTED_EXTENSIONS = %w[.txt .md .markdown].freeze
+
+      def load(path)
+        path = ::File.expand_path(path)
+        raise ArgumentError, "File not found: #{path}" unless ::File.exist?(path)
+
+        ext = ::File.extname(path).downcase
+        unless SUPPORTED_EXTENSIONS.include?(ext)
+          raise ArgumentError, "Unsupported file type: #{ext}. Supported: #{SUPPORTED_EXTENSIONS.join(', ')}"
+        end
+
+        content = ::File.read(path, encoding: "UTF-8")
+
+        [Document.new(
+          content: content,
+          metadata: {
+            source: path,
+            filename: ::File.basename(path),
+            extension: ext,
+            size: ::File.size(path)
+          },
+          source: path
+        )]
+      end
+    end
+  end
+end

data/lib/rag_ruby/loaders/url.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "uri"
+
+module RagRuby
+  module Loaders
+    class URL < Base
+      def initialize(timeout: 30)
+        @timeout = timeout
+      end
+
+      def load(url)
+        uri = URI.parse(url)
+        raise ArgumentError, "Invalid URL: #{url}" unless uri.is_a?(URI::HTTP) || uri.is_a?(URI::HTTPS)
+
+        response = fetch(uri)
+        content = response.body.force_encoding("UTF-8")
+
+        [Document.new(
+          content: content,
+          metadata: {
+            source: url,
+            content_type: response["content-type"],
+            status: response.code.to_i
+          },
+          source: url
+        )]
+      end
+
+      private
+
+      def fetch(uri, redirect_limit: 5)
+        raise "Too many redirects" if redirect_limit == 0
+
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = uri.scheme == "https"
+        http.open_timeout = @timeout
+        http.read_timeout = @timeout
+
+        request = Net::HTTP::Get.new(uri)
+        response = http.request(request)
+
+        case response
+        when Net::HTTPRedirection
+          fetch(URI.parse(response["location"]), redirect_limit: redirect_limit - 1)
+        when Net::HTTPSuccess
+          response
+        else
+          raise "HTTP #{response.code}: #{response.message}"
+        end
+      end
+    end
+  end
+end

data/lib/rag_ruby/pipeline.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+require "securerandom"
+
+module RagRuby
+  class Pipeline
+    attr_reader :config
+
+    def initialize(&block)
+      @config = Configuration.new
+      yield @config if block_given?
+      @prompt_template = PromptTemplate.new
+      @chunk_store = {} # id -> chunk mapping
+    end
+
+    def ingest(source, loader: nil)
+      loader ||= @config.loader_instance || Loaders::File.new
+
+      # Load documents
+      fire(:before_load, source)
+      documents = loader.load(source)
+      fire(:after_load, documents)
+
+      documents.each do |doc|
+        next if doc.empty?
+
+        # Chunk
+        fire(:before_chunk, doc)
+        chunks = chunk_document(doc)
+        fire(:after_chunk, chunks)
+
+        # Embed
+        fire(:before_embed, chunks)
+        embed_chunks(chunks)
+        fire(:after_embed, chunks)
+
+        # Store
+        fire(:before_store, chunks)
+        store_chunks(chunks)
+        fire(:after_store, chunks)
+      end
+    end
+
+    def ingest_directory(dir_path, glob: "**/*.{md,txt}")
+      loader = Loaders::Directory.new(glob: glob)
+      ingest(dir_path, loader: loader)
+    end
+
+    def query(question, top_k: 5, filter: nil, temperature: 0.7, system_prompt: nil)
+      start_time = Time.now
+
+      fire(:before_query, question)
+
+      # Embed the question
+      query_embedding = @config.embedder_instance.embed(question)
+
+      # Search the store
+      results = @config.store_instance.search(query_embedding, top_k: top_k, filter: filter)
+
+      # Build sources from results
+      sources = results.map do |result|
+        chunk = result[:chunk] || @chunk_store[result[:id]]
+        next unless chunk
+
+        Source.new(chunk: chunk, score: result[:score])
+      end.compact
+
+      # Build context from retrieved chunks
+      context = build_context(sources)
+
+      # Generate answer
+      prompt_text = @prompt_template.render(context: context, question: question)
+      sys_prompt = system_prompt || @prompt_template.system_prompt
+
+      gen_result = @config.generator_instance.generate(
+        prompt: prompt_text,
+        system_prompt: sys_prompt,
+        temperature: temperature
+      )
+
+      duration = Time.now - start_time
+
+      answer = Answer.new(
+        text: gen_result[:text],
+        sources: sources,
+        tokens_used: gen_result[:tokens_used],
+        duration: duration,
+        query: question
+      )
+
+      fire(:after_query, question, answer)
+
+      answer
+    end
+
+    private
+
+    def chunk_document(doc)
+      begin
+        require "chunker_ruby"
+        chunker = ChunkerRuby::RecursiveCharacter.new(
+          chunk_size: @config.chunk_size,
+          chunk_overlap: @config.chunk_overlap
+        )
+        texts = chunker.split(doc.content)
+      rescue LoadError
+        # Fallback: simple chunking without chunker-ruby
+        texts = simple_chunk(doc.content, @config.chunk_size, @config.chunk_overlap)
+      end
+
+      texts.each_with_index.map do |text, i|
+        chunk_text = text.respond_to?(:text) ? text.text : text.to_s
+        Chunk.new(
+          text: chunk_text,
+          metadata: doc.metadata.dup,
+          document_source: doc.source,
+          index: i
+        )
+      end
+    end
+
+    def simple_chunk(text, size, overlap)
+      chunks = []
+      start = 0
+      while start < text.length
+        chunk_end = [start + size, text.length].min
+        chunks << text[start...chunk_end]
+        start += size - overlap
+        break if start >= text.length
+      end
+      chunks
+    end
+
+    def embed_chunks(chunks)
+      texts = chunks.map(&:text)
+      embeddings = @config.embedder_instance.embed_batch(texts)
+      chunks.each_with_index do |chunk, i|
+        chunk.embedding = embeddings[i]
+      end
+    end
+
+    def store_chunks(chunks)
+      chunks.each do |chunk|
+        id = SecureRandom.uuid
+        @chunk_store[id] = chunk
+        @config.store_instance.add(id, embedding: chunk.embedding, metadata: chunk.metadata, chunk: chunk)
+      end
+    end
+
+    def build_context(sources, max_chars: 12000)
+      context = ""
+      sources.each do |source|
+        candidate = context + "\n---\n" + source.text
+        break if candidate.length > max_chars
+        context = candidate
+      end
+      context.strip
+    end
+
+    def fire(event, *args)
+      @config.callbacks_for(event).each { |cb| cb.call(*args) }
+    end
+  end
+end

data/lib/rag_ruby/prompt_template.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module RagRuby
+  class PromptTemplate
+    DEFAULT_SYSTEM = "You are a helpful assistant that answers questions based on the provided context."
+
+    DEFAULT_USER = <<~PROMPT
+      Answer the question based on the following context. If the context doesn't
+      contain enough information to answer, say so.
+
+      Context:
+      {{context}}
+
+      Question: {{question}}
+
+      Answer:
+    PROMPT
+
+    attr_reader :system_prompt, :user_template
+
+    def initialize(system_prompt: DEFAULT_SYSTEM, user_template: DEFAULT_USER)
+      @system_prompt = system_prompt
+      @user_template = user_template
+    end
+
+    def render(context:, question:)
+      user_template
+        .gsub("{{context}}", context)
+        .gsub("{{question}}", question)
+    end
+  end
+end

data/lib/rag_ruby/rails/generators/install_generator.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+
+module Rag
+  module Generators
+    class InstallGenerator < Rails::Generators::Base
+      source_root ::File.expand_path("templates", __dir__)
+
+      desc "Install RagRuby configuration"
+
+      def create_config_file
+        template "rag.yml", "config/rag.yml"
+      end
+
+      def create_initializer
+        template "initializer.rb", "config/initializers/rag_ruby.rb"
+      end
+
+      def show_post_install
+        say ""
+        say "RagRuby installed successfully!", :green
+        say ""
+        say "Next steps:"
+        say "  1. Edit config/rag.yml with your settings"
+        say "  2. Set OPENAI_API_KEY in your environment"
+        say "  3. Add `include RagRuby::Indexable` to your models"
+        say ""
+      end
+    end
+  end
+end

data/lib/rag_ruby/rails/generators/templates/initializer.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+# RagRuby configuration is loaded from config/rag.yml
+# You can also configure programmatically here:
+#
+# RagRuby.configure do |config|
+#   config.embedder :openai, model: "text-embedding-3-small"
+#   config.store :memory, dimension: 1536
+#   config.generator :openai, model: "gpt-4o"
+# end

data/lib/rag_ruby/rails/generators/templates/rag.yml

@@ -0,0 +1,30 @@
+default: &default
+  chunker:
+    strategy: recursive_character
+    chunk_size: 1000
+    chunk_overlap: 200
+  embedder:
+    provider: openai
+    model: text-embedding-3-small
+  store:
+    provider: memory
+    dimension: 1536
+  generator:
+    provider: openai
+    model: gpt-4o
+
+development:
+  <<: *default
+
+test:
+  <<: *default
+  store:
+    provider: memory
+    dimension: 1536
+
+production:
+  <<: *default
+  store:
+    provider: zvec
+    path: db/vectors
+    dimension: 1536

data/lib/rag_ruby/rails/indexable.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module RagRuby
+  module Indexable
+    def self.included(base)
+      base.extend ClassMethods
+    end
+
+    module ClassMethods
+      def rag_index(column, metadata: nil, chunk_strategy: :recursive_character, on: [:create, :update])
+        class_attribute :rag_column, default: column
+        class_attribute :rag_metadata_proc, default: metadata
+        class_attribute :rag_chunk_strategy, default: chunk_strategy
+
+        if on.include?(:create)
+          after_create :rag_index_record
+        end
+
+        if on.include?(:update)
+          after_update :rag_index_record, if: -> { saved_change_to_attribute?(column) }
+        end
+
+        if on.include?(:destroy) || on.include?(:delete)
+          after_destroy :rag_remove_record
+        end
+      end
+    end
+
+    private
+
+    def rag_index_record
+      content = public_send(self.class.rag_column).to_s
+      return if content.strip.empty?
+
+      metadata = {}
+      if self.class.rag_metadata_proc
+        metadata = self.class.rag_metadata_proc.call(self)
+      end
+
+      doc = Document.new(
+        content: content,
+        metadata: metadata.merge(model: self.class.name, record_id: id),
+        source: "#{self.class.name}##{id}"
+      )
+
+      RagRuby.pipeline.ingest(doc.source, loader: InlineLoader.new(doc))
+    end
+
+    def rag_remove_record
+      # Remove from store by metadata filter
+      # Implementation depends on store capabilities
+    end
+
+    class InlineLoader < Loaders::Base
+      def initialize(document)
+        @document = document
+      end
+
+      def load(_source)
+        [@document]
+      end
+    end
+  end
+end

data/lib/rag_ruby/rails/railtie.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module RagRuby
+  class Railtie < ::Rails::Railtie
+    initializer "rag_ruby.configure" do |app|
+      config_path = app.root.join("config", "rag.yml")
+
+      if config_path.exist?
+        require "yaml"
+        require "erb"
+
+        yaml = ERB.new(config_path.read).result
+        all_config = YAML.safe_load(yaml, aliases: true) || {}
+        env_config = all_config[Rails.env] || all_config["default"] || {}
+
+        RagRuby.configure_from_hash(env_config)
+      end
+    end
+  end
+end

data/lib/rag_ruby/source.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module RagRuby
+  class Source
+    attr_reader :chunk, :score, :document_source
+
+    def initialize(chunk:, score:)
+      @chunk = chunk
+      @score = score
+      @document_source = chunk.document_source
+    end
+
+    def text
+      chunk.text
+    end
+
+    def metadata
+      chunk.metadata
+    end
+
+    def to_h
+      {
+        text: text,
+        score: score,
+        document_source: document_source,
+        metadata: metadata
+      }
+    end
+  end
+end

data/lib/rag_ruby/stores/base.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module RagRuby
+  module Stores
+    class Base
+      def add(id, embedding:, metadata: {})
+        raise NotImplementedError, "#{self.class}#add must be implemented"
+      end
+
+      def search(embedding, top_k:, filter: nil)
+        raise NotImplementedError, "#{self.class}#search must be implemented"
+      end
+
+      def delete(id)
+        raise NotImplementedError, "#{self.class}#delete must be implemented"
+      end
+
+      def count
+        raise NotImplementedError, "#{self.class}#count must be implemented"
+      end
+    end
+  end
+end