leann 0.1.0

data/lib/leann/embedding/ollama.rb

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+require "json"
+require "net/http"
+require "uri"
+require_relative "base"
+
+module Leann
+  module Embedding
+    # Ollama Embeddings API provider
+    #
+    # Uses local Ollama server for computing embeddings.
+    # Requires Ollama to be running: https://ollama.com
+    #
+    # @example
+    #   provider = Leann::Embedding::Ollama.new(model: "nomic-embed-text")
+    #   embeddings = provider.compute(["Hello", "World"])
+    #
+    class Ollama < Base
+      DEFAULT_HOST = "http://localhost:11434"
+      EMBED_PATH = "/api/embed"
+      MAX_BATCH_SIZE = 32
+      TIMEOUT = 60
+
+      # Popular embedding models
+      POPULAR_MODELS = %w[
+        nomic-embed-text
+        mxbai-embed-large
+        bge-m3
+        all-minilm
+        snowflake-arctic-embed
+      ].freeze
+
+      # @param model [String] Ollama embedding model name
+      # @param host [String, nil] Ollama server URL
+      def initialize(model: "nomic-embed-text", host: nil)
+        super(model: model)
+
+        @host = host || Leann.configuration.ollama_host || ENV["OLLAMA_HOST"] || DEFAULT_HOST
+        @dimensions = nil
+
+        check_connection!
+      end
+
+      # Compute embeddings for texts
+      #
+      # @param texts [Array<String>]
+      # @return [Array<Array<Float>>]
+      def compute(texts)
+        return [] if texts.empty?
+
+        all_embeddings = []
+
+        in_batches(texts, MAX_BATCH_SIZE) do |batch|
+          batch_embeddings = compute_batch(batch)
+          all_embeddings.concat(batch_embeddings)
+          print "." # Progress indicator
+        end
+
+        puts " Done! (#{all_embeddings.size} embeddings)" unless texts.size < MAX_BATCH_SIZE
+
+        # Normalize embeddings (Ollama may not normalize by default)
+        all_embeddings.map { |emb| normalize(emb) }
+      end
+
+      private
+
+      def check_connection!
+        uri = URI.parse("#{@host}/api/version")
+
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = uri.scheme == "https"
+        http.open_timeout = 5
+        http.read_timeout = 5
+
+        response = http.get(uri.request_uri)
+
+        return if response.code.to_i == 200
+
+        raise EmbeddingError.new(
+          "Cannot connect to Ollama at #{@host}. Is Ollama running?",
+          provider: :ollama
+        )
+      rescue Errno::ECONNREFUSED, Errno::ETIMEDOUT, Net::OpenTimeout => e
+        raise EmbeddingError.new(
+          connection_error_message,
+          provider: :ollama,
+          original_error: e
+        )
+      end
+
+      def compute_batch(texts)
+        uri = URI.parse("#{@host}#{EMBED_PATH}")
+
+        body = {
+          model: model,
+          input: texts
+        }
+
+        response = make_request(uri, body)
+        parse_response(response, texts.size)
+      end
+
+      def make_request(uri, body)
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = uri.scheme == "https"
+        http.read_timeout = TIMEOUT
+        http.open_timeout = 10
+
+        request = Net::HTTP::Post.new(uri.request_uri)
+        request["Content-Type"] = "application/json"
+        request.body = JSON.generate(body)
+
+        response = http.request(request)
+
+        case response.code.to_i
+        when 200
+          response
+        when 404
+          raise EmbeddingError.new(
+            model_not_found_message,
+            provider: :ollama
+          )
+        else
+          error_message = parse_error(response)
+          raise EmbeddingError.new(
+            "Ollama API error: #{error_message}",
+            provider: :ollama
+          )
+        end
+      end
+
+      def parse_response(response, expected_count)
+        data = JSON.parse(response.body)
+        embeddings = data["embeddings"]
+
+        unless embeddings && embeddings.is_a?(Array)
+          raise EmbeddingError.new(
+            "Invalid response from Ollama: missing embeddings",
+            provider: :ollama
+          )
+        end
+
+        unless embeddings.size == expected_count
+          raise EmbeddingError.new(
+            "Ollama returned #{embeddings.size} embeddings, expected #{expected_count}",
+            provider: :ollama
+          )
+        end
+
+        embeddings
+      rescue JSON::ParserError => e
+        raise EmbeddingError.new(
+          "Failed to parse Ollama response: #{e.message}",
+          provider: :ollama,
+          original_error: e
+        )
+      end
+
+      def parse_error(response)
+        data = JSON.parse(response.body)
+        data["error"] || response.body
+      rescue JSON::ParserError
+        response.body
+      end
+
+      def connection_error_message
+        <<~MSG
+          Cannot connect to Ollama at #{@host}.
+
+          Please ensure Ollama is running:
+            macOS/Linux: ollama serve
+            Windows: Make sure Ollama is running in the system tray
+
+          Installation: https://ollama.com/download
+        MSG
+      end
+
+      def model_not_found_message
+        <<~MSG
+          Model '#{model}' not found in Ollama.
+
+          To install:
+            ollama pull #{model}
+
+          Popular embedding models:
+            #{POPULAR_MODELS.map { |m| "ollama pull #{m}" }.join("\n  ")}
+
+          Browse more: https://ollama.com/library
+        MSG
+      end
+    end
+  end
+end

data/lib/leann/embedding/openai.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+require "json"
+require "net/http"
+require "uri"
+require_relative "base"
+
+module Leann
+  module Embedding
+    # OpenAI Embeddings API provider
+    #
+    # @example
+    #   provider = Leann::Embedding::OpenAI.new(model: "text-embedding-3-small")
+    #   embeddings = provider.compute(["Hello", "World"])
+    #
+    class OpenAI < Base
+      BASE_URL = "https://api.openai.com/v1/embeddings"
+      MAX_BATCH_SIZE = 2048
+      MAX_RETRIES = 3
+      RETRY_DELAY = 1.0
+
+      # Model dimensions lookup
+      DIMENSIONS = {
+        "text-embedding-3-small" => 1536,
+        "text-embedding-3-large" => 3072,
+        "text-embedding-ada-002" => 1536
+      }.freeze
+
+      # @param model [String] OpenAI embedding model name
+      # @param api_key [String, nil] API key (defaults to ENV or config)
+      # @param base_url [String, nil] Custom base URL
+      def initialize(model: "text-embedding-3-small", api_key: nil, base_url: nil)
+        super(model: model)
+
+        @api_key = api_key || Leann.configuration.openai_api_key || ENV["OPENAI_API_KEY"]
+        @base_url = base_url || Leann.configuration.openai_base_url || BASE_URL
+        @dimensions = DIMENSIONS[model]
+
+        validate_configuration!
+      end
+
+      # Compute embeddings for texts
+      #
+      # @param texts [Array<String>]
+      # @return [Array<Array<Float>>]
+      def compute(texts)
+        return [] if texts.empty?
+
+        all_embeddings = []
+
+        in_batches(texts, MAX_BATCH_SIZE) do |batch|
+          batch_embeddings = compute_batch(batch)
+          all_embeddings.concat(batch_embeddings)
+          print "." # Progress indicator
+        end
+
+        puts " Done! (#{all_embeddings.size} embeddings)" unless texts.size < MAX_BATCH_SIZE
+
+        all_embeddings
+      end
+
+      private
+
+      def validate_configuration!
+        return if @api_key && !@api_key.empty?
+
+        raise ConfigurationError, <<~MSG
+          OpenAI API key is required.
+
+          Set it via:
+            - Environment: OPENAI_API_KEY=your-key
+            - Configuration: Leann.configure { |c| c.openai_api_key = "your-key" }
+            - Builder option: Leann.build("index", embedding: :openai, api_key: "your-key")
+        MSG
+      end
+
+      def compute_batch(texts)
+        uri = URI.parse(@base_url)
+
+        body = {
+          model: model,
+          input: texts,
+          encoding_format: "float"
+        }
+
+        response = make_request(uri, body)
+        parse_response(response)
+      end
+
+      def make_request(uri, body, retries = 0)
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = uri.scheme == "https"
+        http.read_timeout = 60
+        http.open_timeout = 10
+
+        request = Net::HTTP::Post.new(uri.request_uri)
+        request["Content-Type"] = "application/json"
+        request["Authorization"] = "Bearer #{@api_key}"
+        request.body = JSON.generate(body)
+
+        response = http.request(request)
+
+        case response.code.to_i
+        when 200
+          response
+        when 429, 500, 502, 503
+          # Rate limit or server error - retry
+          if retries < MAX_RETRIES
+            delay = RETRY_DELAY * (2**retries)
+            sleep(delay)
+            make_request(uri, body, retries + 1)
+          else
+            raise EmbeddingError.new(
+              "OpenAI API error after #{MAX_RETRIES} retries: #{response.code} #{response.body}",
+              provider: :openai
+            )
+          end
+        else
+          error_message = parse_error(response)
+          raise EmbeddingError.new(
+            "OpenAI API error: #{error_message}",
+            provider: :openai
+          )
+        end
+      end
+
+      def parse_response(response)
+        data = JSON.parse(response.body)
+
+        # Sort by index to ensure order matches input
+        embeddings = data["data"].sort_by { |e| e["index"] }
+        embeddings.map { |e| e["embedding"] }
+      rescue JSON::ParserError => e
+        raise EmbeddingError.new(
+          "Failed to parse OpenAI response: #{e.message}",
+          provider: :openai,
+          original_error: e
+        )
+      end
+
+      def parse_error(response)
+        data = JSON.parse(response.body)
+        data.dig("error", "message") || response.body
+      rescue JSON::ParserError
+        response.body
+      end
+    end
+  end
+end

data/lib/leann/embedding/ruby_llm.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module Leann
+  module Embedding
+    # RubyLLM embedding provider
+    #
+    # Uses RubyLLM's unified embedding API which supports multiple providers
+    # (OpenAI, Ollama, etc.) through a single interface.
+    #
+    # @example
+    #   provider = Leann::Embedding::RubyLLM.new
+    #   vectors = provider.compute(["Hello world", "Another text"])
+    #
+    class RubyLLM < Base
+      # @param model [String, nil] Embedding model (uses RubyLLM default if nil)
+      def initialize(model: nil)
+        super(model: model)
+
+        unless defined?(::RubyLLM)
+          raise ConfigurationError, "RubyLLM gem is required. Add 'ruby_llm' to your Gemfile."
+        end
+      end
+
+      # Compute embeddings for texts
+      # @param texts [Array<String>] Texts to embed
+      # @return [Array<Array<Float>>] Embedding vectors
+      def compute(texts)
+        texts = Array(texts)
+        return [] if texts.empty?
+
+        options = {}
+        options[:model] = @model if @model
+
+        result = ::RubyLLM.embed(texts, **options)
+        result.vectors
+      rescue ::RubyLLM::Error => e
+        raise EmbeddingError, "RubyLLM embedding failed: #{e.message}"
+      end
+
+      # @return [Integer] Embedding dimensions (model-dependent)
+      def dimensions
+        # Get dimensions by computing a test embedding
+        @dimensions ||= begin
+          test = compute(["test"])
+          test.first&.size || 1536
+        end
+      end
+
+      # @return [String] Provider name
+      def provider_name
+        :ruby_llm
+      end
+    end
+  end
+end

data/lib/leann/errors.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Leann
+  # Base error class for all Leann errors
+  class Error < StandardError; end
+
+  # Raised when configuration is invalid
+  class ConfigurationError < Error; end
+
+  # Raised when an index is not found
+  class IndexNotFoundError < Error
+    attr_reader :index_name
+
+    def initialize(index_name)
+      @index_name = index_name
+      super("Index not found: #{index_name}")
+    end
+  end
+
+  # Raised when an index already exists
+  class IndexExistsError < Error
+    attr_reader :index_name
+
+    def initialize(index_name)
+      @index_name = index_name
+      super("Index already exists: #{index_name}. Use force: true to overwrite.")
+    end
+  end
+
+  # Raised when embedding computation fails
+  class EmbeddingError < Error
+    attr_reader :provider, :original_error
+
+    def initialize(message, provider: nil, original_error: nil)
+      @provider = provider
+      @original_error = original_error
+      super(message)
+    end
+  end
+
+  # Raised when LLM request fails
+  class LLMError < Error
+    attr_reader :provider, :original_error
+
+    def initialize(message, provider: nil, original_error: nil)
+      @provider = provider
+      @original_error = original_error
+      super(message)
+    end
+  end
+
+  # Raised when index is corrupted or invalid
+  class CorruptedIndexError < Error
+    attr_reader :index_name, :reason
+
+    def initialize(index_name, reason = nil)
+      @index_name = index_name
+      @reason = reason
+      message = "Corrupted index: #{index_name}"
+      message += " (#{reason})" if reason
+      super(message)
+    end
+  end
+
+  # Raised when no documents are provided to builder
+  class EmptyIndexError < Error
+    def initialize
+      super("Cannot build an empty index. Add at least one document.")
+    end
+  end
+end

data/lib/leann/index.rb

@@ -0,0 +1,236 @@
+# frozen_string_literal: true
+
+require "json"
+require "fileutils"
+
+module Leann
+  # Represents a Leann index on disk
+  #
+  # @example Open and search
+  #   index = Leann::Index.open("my_index")
+  #   results = index.search("query")
+  #
+  # @example Get info
+  #   index = Leann::Index.open("my_index")
+  #   puts index.document_count
+  #   puts index.embedding_model
+  #
+  class Index
+    # @return [String] Index name
+    attr_reader :name
+
+    # @return [String] Index path
+    attr_reader :path
+
+    # @return [Hash] Index metadata
+    attr_reader :metadata
+
+    INDEX_EXTENSION = ".leann"
+    META_SUFFIX = ".meta.json"
+    PASSAGES_SUFFIX = ".passages.jsonl"
+    OFFSETS_SUFFIX = ".passages.offsets"
+    VECTORS_SUFFIX = ".vectors"
+    IDS_SUFFIX = ".ids"
+
+    class << self
+      # Open an existing index
+      # @param name [String] Index name or path
+      # @return [Index]
+      # @raise [IndexNotFoundError] if index doesn't exist
+      def open(name)
+        path = resolve_path(name)
+        raise IndexNotFoundError, name unless exists_at?(path)
+
+        new(path)
+      end
+
+      # Check if an index exists
+      # @param name [String] Index name or path
+      # @return [Boolean]
+      def exists?(name)
+        path = resolve_path(name)
+        exists_at?(path)
+      end
+
+      # List all indexes in a directory
+      # @param directory [String]
+      # @return [Array<String>]
+      def list(directory = ".")
+        pattern = File.join(directory, "**", "*#{META_SUFFIX}")
+        Dir.glob(pattern).map do |meta_file|
+          # Extract index name from path
+          File.basename(meta_file, META_SUFFIX).sub(/#{INDEX_EXTENSION}$/, "")
+        end.uniq.sort
+      end
+
+      # Delete an index
+      # @param name [String] Index name or path
+      # @return [Boolean]
+      def delete(name)
+        path = resolve_path(name)
+        return false unless exists_at?(path)
+
+        # Delete all index files
+        files_to_delete = [
+          "#{path}#{META_SUFFIX}",
+          "#{path}#{PASSAGES_SUFFIX}",
+          "#{path}#{OFFSETS_SUFFIX}",
+          "#{path}#{VECTORS_SUFFIX}",
+          "#{path}#{IDS_SUFFIX}",
+          "#{path}.graph.bin",     # LEANN graph file
+          "#{path}.graph.meta.json" # LEANN graph metadata
+        ]
+
+        files_to_delete.each do |file|
+          FileUtils.rm_f(file)
+        end
+
+        true
+      end
+
+      private
+
+      def resolve_path(name)
+        # If it's already a full path with extension, use it
+        return name if name.end_with?(INDEX_EXTENSION)
+
+        # Check in current directory
+        local_path = "#{name}#{INDEX_EXTENSION}"
+        return local_path if exists_at?(local_path)
+
+        # Check in configured index directory
+        config_dir = Leann.configuration.index_directory
+        if config_dir && Dir.exist?(config_dir)
+          configured_path = File.join(config_dir, "#{name}#{INDEX_EXTENSION}")
+          return configured_path if exists_at?(configured_path)
+        end
+
+        # Return local path as default
+        local_path
+      end
+
+      def exists_at?(path)
+        meta_file = "#{path}#{META_SUFFIX}"
+        File.exist?(meta_file)
+      end
+    end
+
+    # @param path [String] Full path to index
+    def initialize(path)
+      @path = path
+      @name = File.basename(path, INDEX_EXTENSION)
+      @metadata = load_metadata
+      @searcher = nil
+    end
+
+    # Search the index
+    # @param query [String] Search query
+    # @param limit [Integer] Maximum results
+    # @param threshold [Float, nil] Minimum score threshold
+    # @param filters [Hash, nil] Metadata filters
+    # @return [SearchResults]
+    def search(query, limit: 5, threshold: nil, filters: nil)
+      searcher.search(query, limit: limit, threshold: threshold, filters: filters)
+    end
+
+    # Get number of documents in the index
+    # @return [Integer]
+    def document_count
+      metadata["document_count"] || count_documents
+    end
+
+    # Get embedding model used
+    # @return [String]
+    def embedding_model
+      metadata["embedding_model"]
+    end
+
+    # Get embedding provider
+    # @return [Symbol]
+    def embedding_provider
+      (metadata["embedding_provider"] || "openai").to_sym
+    end
+
+    # Get embedding dimensions
+    # @return [Integer]
+    def dimensions
+      metadata["dimensions"]
+    end
+
+    # Get creation timestamp
+    # @return [Time, nil]
+    def created_at
+      return nil unless metadata["created_at"]
+
+      Time.parse(metadata["created_at"])
+    end
+
+    # Get backend type
+    # @return [Symbol]
+    def backend
+      (metadata["backend"] || "leann").to_sym
+    end
+
+    # Index info as string
+    # @return [String]
+    def to_s
+      lines = [
+        "Index: #{name}",
+        "  Documents: #{document_count}",
+        "  Embedding: #{embedding_provider}/#{embedding_model}",
+        "  Dimensions: #{dimensions}",
+        "  Backend: #{backend}",
+        "  Created: #{created_at&.strftime("%Y-%m-%d %H:%M:%S") || "unknown"}"
+      ]
+      lines.join("\n")
+    end
+
+    # Detailed inspection
+    # @return [String]
+    def inspect
+      "#<Leann::Index name=#{name.inspect} documents=#{document_count} model=#{embedding_model.inspect}>"
+    end
+
+    # Get all passages (lazy loaded)
+    # @return [Enumerator]
+    def each_passage
+      return enum_for(:each_passage) unless block_given?
+
+      passages_file = "#{path}#{PASSAGES_SUFFIX}"
+      return unless File.exist?(passages_file)
+
+      File.foreach(passages_file) do |line|
+        yield JSON.parse(line.strip, symbolize_names: true)
+      end
+    end
+
+    # Get passage by ID
+    # @param id [String]
+    # @return [Hash, nil]
+    def get_passage(id)
+      each_passage.find { |p| p[:id] == id }
+    end
+
+    private
+
+    def load_metadata
+      meta_file = "#{path}#{META_SUFFIX}"
+      JSON.parse(File.read(meta_file))
+    rescue JSON::ParserError => e
+      raise CorruptedIndexError.new(name, "Invalid metadata JSON: #{e.message}")
+    rescue Errno::ENOENT
+      raise IndexNotFoundError, name
+    end
+
+    def count_documents
+      passages_file = "#{path}#{PASSAGES_SUFFIX}"
+      return 0 unless File.exist?(passages_file)
+
+      File.foreach(passages_file).count
+    end
+
+    def searcher
+      @searcher ||= Searcher.new(self)
+    end
+  end
+end