leann 0.1.0

data/lib/leann/searcher.rb

@@ -0,0 +1,189 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Leann
+  # Handles search operations on an index
+  class Searcher
+    # @return [Index]
+    attr_reader :index
+
+    # @param index [Index]
+    def initialize(index)
+      @index = index
+      @backend = nil
+      @embedding_provider = nil
+      @offsets = nil
+      @passages_cache = nil
+    end
+
+    # Search the index
+    #
+    # @param query [String] Search query
+    # @param limit [Integer] Maximum results (default: 5)
+    # @param threshold [Float, nil] Minimum score threshold (0.0-1.0)
+    # @param filters [Hash, nil] Metadata filters
+    # @return [SearchResults]
+    #
+    # @example Basic search
+    #   results = searcher.search("machine learning")
+    #
+    # @example With options
+    #   results = searcher.search("auth", limit: 10, threshold: 0.7)
+    #
+    # @example With metadata filters
+    #   results = searcher.search("query", filters: { category: "docs" })
+    def search(query, limit: 5, threshold: nil, filters: nil)
+      start_time = Time.now
+
+      # Compute query embedding
+      query_embedding = embedding_provider.compute([query]).first
+
+      # Load all passages for on-the-fly embedding computation
+      passages = load_all_passages
+
+      # Search with on-the-fly embedding recomputation
+      raw_results = backend.search(
+        query_embedding,
+        embedding_provider: embedding_provider,
+        passages: passages,
+        limit: limit * 2
+      )
+
+      # Load passages and build results
+      results = raw_results.map do |id, score|
+        passage = load_passage(id)
+        next unless passage
+
+        SearchResult.new(
+          id: id,
+          text: passage[:text],
+          score: score,
+          metadata: passage[:metadata] || {}
+        )
+      end.compact
+
+      # Apply threshold filter
+      results = results.select { |r| r.score >= threshold } if threshold
+
+      # Apply metadata filters
+      results = apply_filters(results, filters) if filters
+
+      # Limit and sort results
+      results = results.sort.first(limit)
+
+      duration = Time.now - start_time
+
+      SearchResults.new(results, query: query, duration: duration)
+    end
+
+    private
+
+    def backend
+      @backend ||= load_backend
+    end
+
+    def load_backend
+      require_relative "backend/leann_graph"
+      Backend::LeannGraph.load(index.path)
+    end
+
+    def load_all_passages
+      return @passages_cache if @passages_cache
+
+      @passages_cache = {}
+      passages_file = "#{index.path}#{Index::PASSAGES_SUFFIX}"
+      return @passages_cache unless File.exist?(passages_file)
+
+      File.foreach(passages_file) do |line|
+        doc = JSON.parse(line, symbolize_names: true)
+        @passages_cache[doc[:id]] = doc[:text]
+      end
+
+      @passages_cache
+    end
+
+    def embedding_provider
+      @embedding_provider ||= load_embedding_provider
+    end
+
+    def load_embedding_provider
+      require_relative "embedding/base"
+
+      case index.embedding_provider
+      when :ruby_llm
+        require_relative "embedding/ruby_llm"
+        Embedding::RubyLLM.new(model: index.embedding_model)
+      when :openai
+        require_relative "embedding/openai"
+        Embedding::OpenAI.new(model: index.embedding_model)
+      when :ollama
+        require_relative "embedding/ollama"
+        Embedding::Ollama.new(model: index.embedding_model)
+      when :fastembed
+        require_relative "embedding/fastembed"
+        Embedding::FastEmbed.new(model: index.embedding_model)
+      else
+        raise ConfigurationError, "Unknown embedding provider: #{index.embedding_provider}"
+      end
+    end
+
+    def offsets
+      @offsets ||= load_offsets
+    end
+
+    def load_offsets
+      offsets_file = "#{index.path}#{Index::OFFSETS_SUFFIX}"
+      return {} unless File.exist?(offsets_file)
+
+      JSON.parse(File.read(offsets_file))
+    rescue JSON::ParserError
+      {}
+    end
+
+    def load_passage(id)
+      passages_file = "#{index.path}#{Index::PASSAGES_SUFFIX}"
+      return nil unless File.exist?(passages_file)
+
+      offset = offsets[id]
+
+      if offset
+        # Fast random access using offset
+        File.open(passages_file, "r") do |f|
+          f.seek(offset)
+          line = f.gets
+          return JSON.parse(line, symbolize_names: true) if line
+        end
+      else
+        # Fallback to linear scan (slower but works without offsets)
+        File.foreach(passages_file) do |line|
+          doc = JSON.parse(line, symbolize_names: true)
+          return doc if doc[:id] == id
+        end
+      end
+
+      nil
+    rescue JSON::ParserError
+      nil
+    end
+
+    def apply_filters(results, filters)
+      results.select do |result|
+        filters.all? do |key, value|
+          metadata_value = result.metadata[key.to_sym]
+
+          case value
+          when Range
+            value.cover?(metadata_value)
+          when Array
+            value.include?(metadata_value)
+          when Regexp
+            value.match?(metadata_value.to_s)
+          else
+            metadata_value == value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/leann/version.rb

@@ -0,0 +1,3 @@
+module Leann
+  VERSION = "0.1.0"
+end

data/lib/leann.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+require_relative "leann/version"
+require_relative "leann/configuration"
+require_relative "leann/errors"
+require_relative "leann/search_result"
+require_relative "leann/embedding/base"
+require_relative "leann/embedding/openai"
+require_relative "leann/embedding/ollama"
+require_relative "leann/backend/base"
+require_relative "leann/backend/leann_graph"
+require_relative "leann/index"
+require_relative "leann/builder"
+require_relative "leann/searcher"
+
+# LEANN - Lightweight Embedding-Aware Neural Neighbor search
+#
+# A Ruby gem for building and searching vector indexes with minimal storage.
+# Stores only the graph structure, achieving 85-96% storage savings by
+# recomputing embeddings on-the-fly during search.
+#
+# @example Quick start - build an index
+#   Leann.build("knowledge_base") do
+#     add "LEANN saves 85-96% storage compared to traditional vector databases."
+#     add "It uses graph-only storage with on-demand recomputation."
+#   end
+#
+# @example Search
+#   results = Leann.search("knowledge_base", "storage savings")
+#   results.each { |r| puts "#{r.score}: #{r.text}" }
+#
+module Leann
+  class << self
+    # Global configuration
+    # @return [Configuration]
+    def configuration
+      @configuration ||= Configuration.new
+    end
+
+    # Configure Leann globally
+    #
+    # @example
+    #   Leann.configure do |config|
+    #     config.embedding_provider = :openai
+    #     config.openai_api_key = ENV["OPENAI_API_KEY"]
+    #   end
+    #
+    # @yield [Configuration]
+    def configure
+      yield(configuration)
+    end
+
+    # Build a new index with a DSL block
+    #
+    # @param name [String] Index name (will be created in current directory or specified path)
+    # @param options [Hash] Options for building
+    # @option options [Symbol] :embedding (:openai) Embedding provider (:openai, :ollama, :ruby_llm)
+    # @option options [String] :model Embedding model name
+    # @option options [String] :path Custom path for index storage
+    #
+    # @example Simple usage
+    #   Leann.build("my_index") do
+    #     add "First document"
+    #     add "Second document"
+    #   end
+    #
+    # @example With metadata
+    #   Leann.build("docs", embedding: :ollama) do
+    #     add "Content here", source: "file.md", chapter: 1
+    #     add_file "README.md"
+    #     add_directory "docs/", pattern: "**/*.md"
+    #   end
+    #
+    # @return [Index] The built index
+    def build(name, **options, &block)
+      builder = Builder.new(name, **options)
+      builder.instance_eval(&block) if block_given?
+      builder.save
+    end
+
+    # Search an existing index
+    #
+    # @param name [String] Index name or path
+    # @param query [String] Search query
+    # @param limit [Integer] Maximum results (default: 5)
+    # @param threshold [Float] Minimum similarity score (0.0-1.0)
+    # @param filters [Hash] Metadata filters
+    #
+    # @example Basic search
+    #   results = Leann.search("my_index", "machine learning")
+    #
+    # @example With filters
+    #   results = Leann.search("docs", "auth", limit: 10, filters: { chapter: 1..5 })
+    #
+    # @return [Array<SearchResult>] Search results
+    def search(name, query, limit: 5, threshold: nil, filters: nil)
+      index = Index.open(name)
+      index.search(query, limit: limit, threshold: threshold, filters: filters)
+    end
+
+    # Open an existing index for advanced operations
+    #
+    # @param name [String] Index name or path
+    # @return [Index]
+    def open(name)
+      Index.open(name)
+    end
+
+    # List all indexes in a directory
+    #
+    # @param path [String] Directory to scan (default: current directory)
+    # @return [Array<String>] Index names
+    def list(path: ".")
+      Index.list(path)
+    end
+
+    # Check if an index exists
+    #
+    # @param name [String] Index name or path
+    # @return [Boolean]
+    def exists?(name)
+      Index.exists?(name)
+    end
+
+    # Delete an index
+    #
+    # @param name [String] Index name or path
+    # @return [Boolean] true if deleted
+    def delete(name)
+      Index.delete(name)
+    end
+  end
+end

metadata

@@ -0,0 +1,177 @@
+--- !ruby/object:Gem::Specification
+name: leann
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Chris Hasiński
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2026-05-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: hnswlib
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.17'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.17'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+description: |
+  LEANN (Lightweight Embedding-Aware Neural Neighbor) is a Ruby gem for
+  building and searching vector indexes with minimal storage. It provides
+  semantic search and RAG capabilities with a beautiful, simple API.
+  Supports multiple embedding providers: RubyLLM, OpenAI, Ollama, and FastEmbed.
+email:
+- krzysztof.hasinski@gmail.com
+executables:
+- leann
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE.txt
+- README.md
+- exe/leann
+- lib/generators/leann/install/install_generator.rb
+- lib/generators/leann/install/templates/migration.rb.erb
+- lib/leann.rb
+- lib/leann/backend/base.rb
+- lib/leann/backend/leann_graph.rb
+- lib/leann/builder.rb
+- lib/leann/configuration.rb
+- lib/leann/embedding/base.rb
+- lib/leann/embedding/fastembed.rb
+- lib/leann/embedding/ollama.rb
+- lib/leann/embedding/openai.rb
+- lib/leann/embedding/ruby_llm.rb
+- lib/leann/errors.rb
+- lib/leann/index.rb
+- lib/leann/rails.rb
+- lib/leann/rails/active_record/index.rb
+- lib/leann/rails/active_record/passage.rb
+- lib/leann/rails/builder.rb
+- lib/leann/rails/railtie.rb
+- lib/leann/rails/searcher.rb
+- lib/leann/rails/storage/active_record_backend.rb
+- lib/leann/ruby_llm/search.rb
+- lib/leann/search_result.rb
+- lib/leann/searcher.rb
+- lib/leann/version.rb
+homepage: https://github.com/khasinski/leann-rb
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/khasinski/leann-rb
+  source_code_uri: https://github.com/khasinski/leann-rb
+  changelog_uri: https://github.com/khasinski/leann-rb/blob/main/CHANGELOG.md
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.22
+signing_key:
+specification_version: 4
+summary: Lightweight vector search and RAG for Ruby
+test_files: []