leann 0.1.0

data/lib/leann/rails/active_record/index.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Leann
+  module Rails
+    # ActiveRecord model for storing LEANN indexes
+    #
+    # @example
+    #   index = Leann::Rails::Index.find_by(name: "products")
+    #   index.search("running shoes")
+    #
+    class Index < ::ActiveRecord::Base
+      self.table_name = "leann_indexes"
+
+      has_many :passages,
+               class_name: "Leann::Rails::Passage",
+               foreign_key: :leann_index_id,
+               dependent: :delete_all
+
+      validates :name, presence: true, uniqueness: true
+      validates :embedding_provider, presence: true
+      validates :dimensions, presence: true, numericality: { greater_than: 0 }
+
+      serialize :config, coder: JSON
+
+      # Search this index
+      #
+      # @param query [String] Search query
+      # @param limit [Integer] Maximum results
+      # @param threshold [Float] Minimum similarity score
+      # @param filters [Hash] Metadata filters
+      # @return [Leann::SearchResults]
+      def search(query, limit: 5, threshold: nil, filters: nil)
+        searcher = Searcher.new(self)
+        searcher.search(query, limit: limit, threshold: threshold, filters: filters)
+      end
+
+      # Get number of documents
+      # @return [Integer]
+      def document_count
+        passages.count
+      end
+
+      # Get embedding provider as symbol
+      # @return [Symbol]
+      def embedding_provider_sym
+        embedding_provider.to_sym
+      end
+
+      # Index info as string
+      # @return [String]
+      def to_s
+        lines = [
+          "Index: #{name}",
+          "  Documents: #{document_count}",
+          "  Embedding: #{embedding_provider}/#{embedding_model}",
+          "  Dimensions: #{dimensions}",
+          "  Backend: active_record",
+          "  Created: #{created_at&.strftime('%Y-%m-%d %H:%M:%S') || 'unknown'}"
+        ]
+        lines.join("\n")
+      end
+
+      # Detailed inspection
+      # @return [String]
+      def inspect
+        "#<Leann::Rails::Index id=#{id} name=#{name.inspect} documents=#{document_count}>"
+      end
+    end
+  end
+end

data/lib/leann/rails/active_record/passage.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Leann
+  module Rails
+    # ActiveRecord model for storing passages/documents within an index
+    #
+    # @example
+    #   passage = Leann::Rails::Passage.find(123)
+    #   passage.text      # => "Document content..."
+    #   passage.metadata  # => { category: "docs" }
+    #
+    class Passage < ::ActiveRecord::Base
+      self.table_name = "leann_passages"
+
+      belongs_to :index,
+                 class_name: "Leann::Rails::Index",
+                 foreign_key: :leann_index_id
+
+      validates :external_id, presence: true
+      validates :text, presence: true
+      validates :leann_index_id, presence: true
+
+      serialize :metadata, coder: JSON
+      serialize :neighbors, coder: JSON
+
+      # Get metadata with symbolized keys
+      # @return [Hash]
+      def metadata_sym
+        (metadata || {}).transform_keys(&:to_sym)
+      end
+
+      # Get neighbor IDs
+      # @return [Array<String>]
+      def neighbor_ids
+        neighbors || []
+      end
+
+      # Convert to hash for search results
+      # @return [Hash]
+      def to_h
+        {
+          id: external_id,
+          text: text,
+          metadata: metadata_sym
+        }
+      end
+
+      # Detailed inspection
+      # @return [String]
+      def inspect
+        text_preview = text.length > 50 ? "#{text[0..47]}..." : text
+        "#<Leann::Rails::Passage id=#{id} external_id=#{external_id.inspect} text=#{text_preview.inspect}>"
+      end
+    end
+  end
+end

data/lib/leann/rails/builder.rb

@@ -0,0 +1,205 @@
+# frozen_string_literal: true
+
+require "securerandom"
+
+module Leann
+  module Rails
+    # Builds a new LEANN index stored in the database
+    #
+    # @example DSL style
+    #   Leann::Rails.build("products") do
+    #     add "Red running shoes", category: "shoes"
+    #     add "Blue denim jeans", category: "pants"
+    #   end
+    #
+    # @example Programmatic style
+    #   builder = Leann::Rails::Builder.new("products")
+    #   builder.add("Red running shoes", category: "shoes")
+    #   builder.save
+    #
+    class Builder
+      # @return [String] Index name
+      attr_reader :name
+
+      # @return [Array<Hash>] Documents to be indexed
+      attr_reader :documents
+
+      # @param name [String] Index name (must be unique)
+      # @param embedding [Symbol] Embedding provider (:ruby_llm, :openai, :ollama, :fastembed)
+      # @param model [String, nil] Embedding model name
+      # @param force [Boolean] Overwrite existing index
+      def initialize(name, embedding: nil, model: nil, force: false)
+        @name = name
+        @embedding_provider = embedding || Leann.configuration.embedding_provider
+        @embedding_model = model || Leann.configuration.embedding_model_for(@embedding_provider)
+        @force = force
+        @documents = []
+
+        check_existing_index unless force
+      end
+
+      # Add a text document
+      #
+      # @param text [String] Document text
+      # @param metadata [Hash] Additional metadata
+      # @return [self]
+      def add(text, **metadata)
+        raise ArgumentError, "Text cannot be nil" if text.nil?
+        raise ArgumentError, "Text cannot be empty" if text.to_s.strip.empty?
+
+        doc = {
+          id: metadata.delete(:id) || SecureRandom.uuid,
+          text: text.to_s.strip,
+          metadata: metadata
+        }
+
+        @documents << doc
+        self
+      end
+
+      # Add document (alias for add)
+      alias << add
+
+      # Add content from a file
+      #
+      # @param file_path [String] Path to file
+      # @param metadata [Hash] Additional metadata
+      # @return [self]
+      def add_file(file_path, **metadata)
+        raise ArgumentError, "File not found: #{file_path}" unless File.exist?(file_path)
+
+        content = File.read(file_path)
+        file_metadata = {
+          source: file_path,
+          filename: File.basename(file_path),
+          extension: File.extname(file_path)
+        }.merge(metadata)
+
+        add(content, **file_metadata)
+      end
+
+      # Add all files from a directory
+      #
+      # @param directory [String] Directory path
+      # @param pattern [String] Glob pattern
+      # @param extensions [Array<String>, nil] Filter by extensions
+      # @param metadata [Hash] Additional metadata for all files
+      # @return [self]
+      def add_directory(directory, pattern: "**/*", extensions: nil, **metadata)
+        raise ArgumentError, "Directory not found: #{directory}" unless Dir.exist?(directory)
+
+        full_pattern = File.join(directory, pattern)
+        Dir.glob(full_pattern).each do |file_path|
+          next unless File.file?(file_path)
+          next if extensions && !extensions.include?(File.extname(file_path))
+
+          add_file(file_path, **metadata)
+        end
+
+        self
+      end
+
+      # Add multiple documents at once
+      #
+      # @param docs [Array<String>, Array<Hash>] Documents to add
+      # @return [self]
+      def add_all(docs)
+        docs.each do |doc|
+          case doc
+          when String
+            add(doc)
+          when Hash
+            text = doc.delete(:text) || doc.delete("text")
+            add(text, **doc.transform_keys(&:to_sym))
+          else
+            raise ArgumentError, "Invalid document type: #{doc.class}"
+          end
+        end
+
+        self
+      end
+
+      # Get number of documents added
+      # @return [Integer]
+      def count
+        @documents.size
+      end
+      alias size count
+
+      # Check if any documents have been added
+      # @return [Boolean]
+      def empty?
+        @documents.empty?
+      end
+
+      # Build and save the index to the database
+      # @return [Leann::Rails::Index] The built index record
+      def save
+        raise Leann::EmptyIndexError if empty?
+
+        puts "Building index '#{name}' with #{count} documents..."
+
+        # Delete existing if force mode
+        Index.find_by(name: name)&.destroy if @force
+
+        # Compute embeddings
+        embeddings = compute_embeddings
+
+        # Create index record
+        index = Index.create!(
+          name: name,
+          embedding_provider: @embedding_provider.to_s,
+          embedding_model: @embedding_model,
+          dimensions: embeddings.first&.size || 0,
+          config: {
+            hnsw_m: Leann.configuration.hnsw_m,
+            hnsw_ef_construction: Leann.configuration.hnsw_ef_construction
+          }
+        )
+
+        # Build and store graph
+        backend = ActiveRecordBackend.new(index)
+        backend.build(@documents, embeddings)
+
+        puts "Index '#{name}' created successfully!"
+
+        index
+      end
+      alias build save
+
+      private
+
+      def check_existing_index
+        raise Leann::IndexExistsError, name if Index.exists?(name: name)
+      end
+
+      def compute_embeddings
+        texts = @documents.map { |d| d[:text] }
+        embedding_provider.compute(texts)
+      end
+
+      def embedding_provider
+        @_embedding_provider ||= load_embedding_provider
+      end
+
+      def load_embedding_provider
+        case @embedding_provider
+        when :ruby_llm
+          require "leann/embedding/ruby_llm"
+          Leann::Embedding::RubyLLM.new(model: @embedding_model)
+        when :openai
+          require "leann/embedding/openai"
+          Leann::Embedding::OpenAI.new(model: @embedding_model)
+        when :ollama
+          require "leann/embedding/ollama"
+          Leann::Embedding::Ollama.new(model: @embedding_model)
+        when :fastembed
+          require "leann/embedding/fastembed"
+          Leann::Embedding::FastEmbed.new(model: @embedding_model)
+        else
+          raise Leann::ConfigurationError, "Unknown embedding provider: #{@embedding_provider}"
+        end
+      end
+    end
+  end
+end

data/lib/leann/rails/railtie.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Leann
+  module Rails
+    class Railtie < ::Rails::Railtie
+      initializer "leann.configure_rails" do
+        # Auto-configure based on Rails environment
+      end
+
+      # Expose generators
+      generators do
+        require "generators/leann/install/install_generator"
+      end
+    end
+  end
+end

data/lib/leann/rails/searcher.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+module Leann
+  module Rails
+    # Handles search operations on a database-stored index
+    class Searcher
+      # @return [Leann::Rails::Index]
+      attr_reader :index
+
+      # @param index [Leann::Rails::Index]
+      def initialize(index)
+        @index = index
+        @embedding_provider = nil
+      end
+
+      # Search the index
+      #
+      # @param query [String] Search query
+      # @param limit [Integer] Maximum results
+      # @param threshold [Float, nil] Minimum score threshold
+      # @param filters [Hash, nil] Metadata filters
+      # @return [Leann::SearchResults]
+      def search(query, limit: 5, threshold: nil, filters: nil)
+        start_time = Time.now
+
+        # Compute query embedding
+        query_embedding = embedding_provider.compute([query]).first
+
+        # Load all passages for embedding recomputation
+        passages = load_all_passages
+
+        # Search with on-the-fly embedding recomputation
+        backend = ActiveRecordBackend.new(index)
+        raw_results = backend.search(
+          query_embedding,
+          embedding_provider: embedding_provider,
+          passages: passages,
+          limit: limit * 2
+        )
+
+        # Build results
+        results = raw_results.map do |id, score|
+          passage = index.passages.find_by(external_id: id)
+          next unless passage
+
+          Leann::SearchResult.new(
+            id: id,
+            text: passage.text,
+            score: score,
+            metadata: passage.metadata_sym
+          )
+        end.compact
+
+        # Apply threshold filter
+        results = results.select { |r| r.score >= threshold } if threshold
+
+        # Apply metadata filters
+        results = apply_filters(results, filters) if filters
+
+        # Limit and sort results
+        results = results.sort.first(limit)
+
+        duration = Time.now - start_time
+
+        Leann::SearchResults.new(results, query: query, duration: duration)
+      end
+
+      private
+
+      def embedding_provider
+        @embedding_provider ||= load_embedding_provider
+      end
+
+      def load_embedding_provider
+        case index.embedding_provider_sym
+        when :ruby_llm
+          require "leann/embedding/ruby_llm"
+          Leann::Embedding::RubyLLM.new(model: index.embedding_model)
+        when :openai
+          require "leann/embedding/openai"
+          Leann::Embedding::OpenAI.new(model: index.embedding_model)
+        when :ollama
+          require "leann/embedding/ollama"
+          Leann::Embedding::Ollama.new(model: index.embedding_model)
+        when :fastembed
+          require "leann/embedding/fastembed"
+          Leann::Embedding::FastEmbed.new(model: index.embedding_model)
+        else
+          raise Leann::ConfigurationError, "Unknown embedding provider: #{index.embedding_provider}"
+        end
+      end
+
+      def load_all_passages
+        index.passages.pluck(:external_id, :text).to_h
+      end
+
+      def apply_filters(results, filters)
+        results.select do |result|
+          filters.all? do |key, value|
+            metadata_value = result.metadata[key.to_sym]
+
+            case value
+            when Range
+              value.cover?(metadata_value)
+            when Array
+              value.include?(metadata_value)
+            when Regexp
+              value.match?(metadata_value.to_s)
+            else
+              metadata_value == value
+            end
+          end
+        end
+      end
+    end
+  end
+end