phronomy 0.8.0 → 0.9.0

data/lib/phronomy/vector_store/loader/csv_loader.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module Phronomy
+  module VectorStore
+    module Loader
+      # Loads a CSV file, converting each row into a separate document.
+      #
+      # By default the first row is treated as a header and column names are
+      # available in the document metadata.  The full row is serialised to
+      # a human-readable "key: value" string for embedding.
+      #
+      # @example
+      #   loader = Phronomy::VectorStore::Loader::CsvLoader.new
+      #   docs   = loader.load("products.csv")
+      #   # => [
+      #   #   { text: "name: Widget\nprice: 9.99", metadata: { source: "...", row: 1, name: "Widget", price: "9.99" } },
+      #   #   ...
+      #   # ]
+      class CsvLoader < Base
+        # @param headers     [Boolean] treat the first row as headers (default: true)
+        # @param text_column [String, nil] if set, use only this column as the document text
+        # @api public
+        def initialize(headers: true, text_column: nil)
+          @headers = headers
+          @text_column = text_column
+        end
+
+        # @param source [String] path to a CSV file
+        # @return [Array<Hash>]
+        # @raise [Errno::ENOENT] if the file does not exist
+        # @api public
+        def load(source)
+          rows = CSV.read(source, headers: @headers, encoding: "UTF-8")
+
+          if @headers
+            rows.each_with_index.map do |row, idx|
+              row_hash = row.to_h
+              text = if @text_column
+                row_hash[@text_column].to_s
+              else
+                row_hash.map { |k, v| "#{k}: #{v}" }.join("\n")
+              end
+              metadata = row_hash.transform_keys(&:to_sym).merge(source: source, row: idx + 1)
+              {text: text, metadata: metadata}
+            end
+          else
+            rows.each_with_index.map do |row, idx|
+              text = row.join(", ")
+              {text: text, metadata: {source: source, row: idx + 1}}
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/phronomy/vector_store/loader/markdown_loader.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module VectorStore
+    module Loader
+      # Loads a Markdown file, optionally splitting on top-level headings.
+      #
+      # When +split_on_headings:+ is true (the default), each H1/H2 section
+      # becomes a separate document so that embeddings capture section semantics
+      # rather than the full file at once.
+      #
+      # @example Single document (heading split disabled)
+      #   loader = Phronomy::VectorStore::Loader::MarkdownLoader.new(split_on_headings: false)
+      #   docs   = loader.load("README.md")
+      #   # => [{ text: "# Title\n...", metadata: { source: "README.md" } }]
+      #
+      # @example Split per heading (default)
+      #   loader = Phronomy::VectorStore::Loader::MarkdownLoader.new
+      #   docs   = loader.load("guide.md")
+      #   # => [
+      #   #   { text: "# Section 1\n...", metadata: { source: "guide.md", section: "Section 1" } },
+      #   #   { text: "## Sub-section\n...", metadata: { source: "guide.md", section: "Sub-section" } },
+      #   # ]
+      class MarkdownLoader < Base
+        HEADING_RE = /^(\#{1,6})\s+(.+)$/
+
+        # @param split_on_headings [Boolean] split on H1–H6 boundaries (default: true)
+        # @api public
+        def initialize(split_on_headings: true)
+          @split_on_headings = split_on_headings
+        end
+
+        # @param source [String] path to a Markdown file
+        # @return [Array<Hash>]
+        # @raise [Errno::ENOENT] if the file does not exist
+        # @api public
+        def load(source)
+          content = File.read(source, encoding: "UTF-8")
+          return [{text: content, metadata: {source: source}}] unless @split_on_headings
+
+          split_by_headings(content, source)
+        end
+
+        private
+
+        def split_by_headings(content, source)
+          sections = []
+          current_lines = []
+          current_heading = nil
+
+          content.each_line do |line|
+            if (m = HEADING_RE.match(line.chomp))
+              flush_section(sections, current_lines, current_heading, source) if current_lines.any?
+              current_heading = m[2].strip
+              current_lines = [line]
+            else
+              current_lines << line
+            end
+          end
+
+          flush_section(sections, current_lines, current_heading, source) if current_lines.any?
+
+          # Fall back to single document if no headings were found
+          sections.empty? ? [{text: content, metadata: {source: source}}] : sections
+        end
+
+        def flush_section(sections, lines, heading, source)
+          text = lines.join
+          return if text.strip.empty?
+
+          metadata = {source: source}
+          metadata[:section] = heading if heading
+          sections << {text: text, metadata: metadata}
+        end
+      end
+    end
+  end
+end

data/lib/phronomy/vector_store/loader/plain_text_loader.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module VectorStore
+    module Loader
+      # Loads a plain-text file as a single document.
+      #
+      # @example
+      #   loader = Phronomy::VectorStore::Loader::PlainTextLoader.new
+      #   docs   = loader.load("/path/to/file.txt")
+      #   # => [{ text: "...", metadata: { source: "/path/to/file.txt" } }]
+      class PlainTextLoader < Base
+        # @param source [String] absolute or relative path to a text file
+        # @return [Array<Hash>] single-element array with the file contents
+        # @raise [Errno::ENOENT] if the file does not exist
+        # @api public
+        def load(source)
+          text = File.read(source, encoding: "UTF-8")
+          [{text: text, metadata: {source: source}}]
+        end
+      end
+    end
+  end
+end

data/lib/phronomy/vector_store/pgvector.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Phronomy
+  module VectorStore
+    # PostgreSQL-backed vector store using the pgvector extension.
+    #
+    # Requires:
+    #   - The +pgvector+ gem (add to your Gemfile)
+    #   - An ActiveRecord model class with the following columns:
+    #       id        (string / uuid)
+    #       embedding (vector — from the pgvector column type)
+    #       metadata  (text or jsonb — stores arbitrary metadata as JSON)
+    #
+    # @example Usage
+    #   store = Phronomy::VectorStore::Pgvector.new(model_class: VectorDocument)
+    #   store.add(id: "doc1", embedding: [0.1, 0.9], metadata: {text: "hello"})
+    #   results = store.search(query_embedding: [0.1, 0.8], k: 5)
+    class Pgvector < Base
+      # @param model_class [Class]        ActiveRecord model with id/embedding/metadata columns
+      # @param dimension   [Integer, nil] expected embedding dimension for Phronomy-side
+      #   pre-validation.  When nil, dimension enforcement is delegated to the
+      #   database schema; no pre-validation is performed by Phronomy.
+      # @api public
+      def initialize(model_class:, dimension: nil)
+        begin
+          require "pgvector"
+        rescue LoadError
+          raise LoadError,
+            "pgvector gem is required for Phronomy::VectorStore::Pgvector. " \
+            "Add `gem 'pgvector'` to your Gemfile."
+        end
+        @model_class = model_class
+        @dimension = dimension
+      end
+
+      # @param id                 [String]
+      # @param embedding          [Array<Float>]
+      # @param metadata           [Hash]
+      # @param cancellation_token [Phronomy::Concurrency::CancellationToken, nil]
+      # @api public
+      def add(id:, embedding:, metadata: {}, cancellation_token: nil)
+        cancellation_token&.raise_if_cancelled!
+        validate_embedding_dimension!(embedding, @dimension)
+        @model_class.upsert(
+          {id: id, embedding: safe_vector(embedding), metadata: metadata.to_json},
+          unique_by: :id
+        )
+        self
+      end
+
+      # @param query_embedding    [Array<Float>]
+      # @param k                  [Integer]
+      # @param cancellation_token [Phronomy::Concurrency::CancellationToken, nil]
+      # @return [Array<Hash>] sorted by descending similarity score
+      # @api public
+      def search(query_embedding:, k: 5, cancellation_token: nil)
+        cancellation_token&.raise_if_cancelled!
+        k_safe = validate_k!(k)
+        validate_embedding_dimension!(query_embedding, @dimension)
+        vec = safe_vector_literal(query_embedding)
+        conn = @model_class.connection
+        quoted_vec = "#{conn.quote(vec)}::vector"
+
+        @model_class
+          .select("id, metadata, 1 - (embedding <=> #{quoted_vec}) AS score")
+          .order("embedding <=> #{quoted_vec}")
+          .limit(k_safe)
+          .map do |r|
+            {
+              id: r.id.to_s,
+              score: r.score.to_f,
+              metadata: parse_metadata(r.metadata)
+            }
+          end
+      end
+
+      def remove(id:)
+        @model_class.where(id: id).delete_all
+        self
+      end
+
+      def clear
+        @model_class.delete_all
+        self
+      end
+
+      # Returns the number of documents in the backing table.
+      def size
+        @model_class.count
+      end
+
+      private
+
+      # Parses a metadata value returned by the pg driver.
+      # Handles NULL (nil), already-parsed Hash, and JSON string forms.
+      def parse_metadata(raw)
+        return {} if raw.nil?
+        return symbolize_hash_keys(raw) if raw.is_a?(Hash)
+
+        parsed = JSON.parse(raw.to_s, symbolize_names: true)
+        parsed.is_a?(Hash) ? parsed : {}
+      rescue JSON::ParserError
+        {}
+      end
+
+      # Recursively symbolizes keys for an already-parsed Hash.
+      def symbolize_hash_keys(hash)
+        hash.each_with_object({}) do |(k, v), h|
+          h[k.to_sym] = v.is_a?(Hash) ? symbolize_hash_keys(v) : v
+        end
+      end
+
+      # Validates that all elements are numeric and converts to a pgvector-
+      # compatible literal string (e.g. "[1.0,0.5,-0.3]").
+      def safe_vector_literal(embedding)
+        "[#{embedding.map { |v| Float(v) }.join(",")}]"
+      end
+
+      # Returns a validated vector for the upsert call.
+      def safe_vector(embedding)
+        safe_vector_literal(embedding)
+      end
+    end
+  end
+end

data/lib/phronomy/vector_store/redis_search.rb

@@ -0,0 +1,192 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Phronomy
+  module VectorStore
+    # Redis-backed vector store using the RediSearch module (FT.* commands).
+    #
+    # Requires:
+    #   - The +redis+ gem (add to your Gemfile)
+    #   - A Redis server with the RediSearch (RedisSearch) module enabled
+    #     (or Redis Stack which bundles RediSearch)
+    #
+    # Vectors are stored as FLOAT32 binary blobs in Redis Hash fields and
+    # searched using the KNN approximate-nearest-neighbour algorithm.
+    #
+    # @example Usage
+    #   redis = Redis.new(url: "redis://localhost:6379")
+    #   store = Phronomy::VectorStore::RedisSearch.new(redis: redis, dimension: 1536)
+    #   store.add(id: "doc1", embedding: [0.1, 0.9], metadata: {text: "hello"})
+    #   results = store.search(query_embedding: [0.1, 0.8], k: 5)
+    class RedisSearch < Base
+      DOC_PREFIX = "phronomy_doc:"
+      private_constant :DOC_PREFIX
+
+      # @param redis      [Redis]          configured Redis client
+      # @param index_name [String]         RediSearch index name
+      # @param dimension  [Integer, nil]   vector dimension; auto-detected on first add.
+      #   When connecting to an **existing** RediSearch index, you MUST pass
+      #   dimension: explicitly.  Without it, a freshly constructed instance
+      #   treats the index as uninitialized until #add is called, and #search
+      #   silently returns [] in the meantime.
+      # @api public
+      def initialize(redis:, index_name: "phronomy_vectors", dimension: nil)
+        begin
+          require "redis"
+        rescue LoadError
+          raise LoadError,
+            "redis gem is required for Phronomy::VectorStore::RedisSearch. " \
+            "Add `gem 'redis'` to your Gemfile."
+        end
+        @redis = redis
+        @index_name = index_name
+        @dimension = dimension
+        @index_created = false
+        @mutex = Mutex.new
+      end
+
+      # @param id                 [String]
+      # @param embedding          [Array<Float>]
+      # @param metadata           [Hash]
+      # @param cancellation_token [Phronomy::Concurrency::CancellationToken, nil]
+      # @api public
+      def add(id:, embedding:, metadata: {}, cancellation_token: nil)
+        cancellation_token&.raise_if_cancelled!
+        # Establish expected dimension on first add (not race-free for concurrent
+        # first adds), then validate, then create/reuse the index.
+        @dimension ||= embedding.size
+        validate_embedding_dimension!(embedding, @dimension)
+        ensure_index!(@dimension)
+        @redis.call(
+          "HSET", "#{DOC_PREFIX}#{id}",
+          "embedding", pack_vector(embedding),
+          "metadata", metadata.to_json
+        )
+        self
+      end
+
+      # @param query_embedding    [Array<Float>]
+      # @param k                  [Integer]
+      # @param cancellation_token [Phronomy::Concurrency::CancellationToken, nil]
+      # @return [Array<Hash>] sorted by descending similarity score
+      # @api public
+      def search(query_embedding:, k: 5, cancellation_token: nil)
+        cancellation_token&.raise_if_cancelled!
+        # search never establishes dimension.  If dimension is unknown and the
+        # index has not been created yet, there are no documents to return.
+        return [] if @dimension.nil? && !@index_created
+
+        validate_embedding_dimension!(query_embedding, @dimension)
+        ensure_index!(@dimension)
+        k_safe = validate_k!(k)
+        blob = pack_vector(query_embedding)
+
+        raw = @redis.call(
+          "FT.SEARCH", @index_name,
+          "*=>[KNN #{k_safe} @embedding $BLOB AS score]",
+          "PARAMS", 2, "BLOB", blob,
+          "SORTBY", "score",
+          "RETURN", 2, "score", "metadata",
+          "DIALECT", 2
+        )
+
+        parse_results(raw)
+      end
+
+      def remove(id:)
+        @redis.call("DEL", "#{DOC_PREFIX}#{id}")
+        self
+      end
+
+      # Returns the number of documents indexed.
+      # Queries FT.INFO when the index has been created; returns 0 otherwise.
+      def size
+        return 0 unless @index_created
+
+        raw = @redis.call("FT.INFO", @index_name)
+        return 0 unless raw.is_a?(Array)
+
+        idx = raw.index("num_docs")
+        idx ? raw[idx + 1].to_i : 0
+      rescue
+        0
+      end
+
+      def clear
+        @mutex.synchronize do
+          begin
+            @redis.call("FT.DROPINDEX", @index_name, "DD")
+          rescue => e
+            raise unless e.message.to_s.include?("Unknown Index name")
+          end
+          @index_created = false
+        end
+        self
+      end
+
+      private
+
+      def ensure_index!(dim)
+        @mutex.synchronize do
+          return if @index_created
+
+          @dimension ||= dim
+          begin
+            @redis.call(
+              "FT.CREATE", @index_name,
+              "ON", "HASH",
+              "PREFIX", 1, DOC_PREFIX,
+              "SCHEMA",
+              "embedding", "VECTOR", "FLAT", 6,
+              "TYPE", "FLOAT32",
+              "DIM", @dimension,
+              "DISTANCE_METRIC", "COSINE",
+              "metadata", "TEXT"
+            )
+          rescue => e
+            raise unless e.message.to_s.include?("Index already exists")
+          end
+          @index_created = true
+        end
+      end
+
+      # Pack a Float array as a FLOAT32 binary string for RediSearch.
+      def pack_vector(embedding)
+        embedding.map { |v| Float(v) }.pack("f*")
+      end
+
+      # Parse the raw FT.SEARCH response into the standard Hash format.
+      #
+      # Redis FT.SEARCH returns: [count, key1, [field, value, ...], key2, ...]
+      def parse_results(raw)
+        return [] if raw.nil? || !raw.is_a?(Array) || raw.size < 2
+
+        results = []
+        i = 1
+        while i < raw.size
+          key = raw[i]
+          fields = raw[i + 1]
+          i += 2
+
+          next unless fields.is_a?(Array)
+
+          field_hash = fields.each_slice(2).to_h
+          score_str = field_hash["score"]
+          metadata_str = field_hash["metadata"]
+
+          next if score_str.nil?
+
+          id = key.to_s.delete_prefix(DOC_PREFIX)
+          # RediSearch returns cosine distance (0=identical, 2=opposite);
+          # convert to cosine similarity for consistency with other backends.
+          score = 1.0 - score_str.to_f
+          metadata = metadata_str ? JSON.parse(metadata_str, symbolize_names: true) : {}
+
+          results << {id: id, score: score, metadata: metadata}
+        end
+        results
+      end
+    end
+  end
+end

data/lib/phronomy/vector_store/splitter/base.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module VectorStore
+    module Splitter
+      # Abstract base class for text splitters.
+      #
+      # A splitter takes a single document hash (or plain text) and returns an
+      # array of smaller chunk documents:
+      #
+      #   [{ text: String, metadata: Hash }, ...]
+      #
+      # Subclasses must implement {#split}.
+      class Base
+        # Split +document+ into an array of chunk documents.
+        #
+        # @param document [Hash, String]
+        #   Either a document hash (<tt>{ text: String, metadata: Hash }</tt>)
+        #   returned by a Loader, or a plain String.
+        # @return [Array<Hash>] array of <tt>{ text: String, metadata: Hash }</tt>
+        # @raise [NotImplementedError] when not overridden by a subclass
+        # @api public
+        def split(document)
+          raise NotImplementedError, "#{self.class}#split is not implemented"
+        end
+
+        # Convenience method: split an array of documents.
+        #
+        # @param documents [Array<Hash, String>]
+        # @return [Array<Hash>]
+        # @api public
+        def split_all(documents)
+          documents.flat_map { |doc| split(doc) }
+        end
+
+        private
+
+        # Normalise a document-or-string argument into {text:, metadata:}.
+        def normalise(document)
+          case document
+          when Hash then {text: document[:text].to_s, metadata: document.fetch(:metadata, {})}
+          when String then {text: document, metadata: {}}
+          else raise ArgumentError, "document must be a Hash or String, got #{document.class}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/phronomy/vector_store/splitter/fixed_size_splitter.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module VectorStore
+    module Splitter
+      # Splits text into fixed-size character chunks with optional overlap.
+      #
+      # @example
+      #   splitter = Phronomy::VectorStore::Splitter::FixedSizeSplitter.new(chunk_size: 200, chunk_overlap: 20)
+      #   chunks   = splitter.split({ text: long_text, metadata: { source: "doc.txt" } })
+      #   # => [
+      #   #   { text: "...(200 chars)...", metadata: { source: "doc.txt", chunk: 0 } },
+      #   #   { text: "...(200 chars, 20-char overlap)...", metadata: { source: "doc.txt", chunk: 1 } },
+      #   # ]
+      class FixedSizeSplitter < Base
+        # @param chunk_size    [Integer] maximum characters per chunk (default: 1000)
+        # @param chunk_overlap [Integer] characters to repeat at the start of each
+        #   subsequent chunk (default: 200); must be less than chunk_size
+        # @api public
+        def initialize(chunk_size: 1000, chunk_overlap: 200)
+          raise ArgumentError, "chunk_overlap must be less than chunk_size" if chunk_overlap >= chunk_size
+
+          @chunk_size = chunk_size
+          @chunk_overlap = chunk_overlap
+        end
+
+        # @param document [Hash, String]
+        # @return [Array<Hash>]
+        # @api public
+        def split(document)
+          doc = normalise(document)
+          text = doc[:text]
+          base_metadata = doc[:metadata]
+
+          chunks = []
+          start = 0
+          index = 0
+
+          while start < text.length
+            chunk_text = text[start, @chunk_size]
+            chunks << {text: chunk_text, metadata: base_metadata.merge(chunk: index)}
+            break if start + @chunk_size >= text.length
+
+            start += @chunk_size - @chunk_overlap
+            index += 1
+          end
+
+          chunks
+        end
+      end
+    end
+  end
+end

data/lib/phronomy/vector_store/splitter/recursive_splitter.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module VectorStore
+    module Splitter
+      # Splits text recursively using a prioritised list of separator strings.
+      #
+      # The splitter tries each separator in order.  When a separator produces
+      # chunks that are still larger than +chunk_size+, it recurses with the
+      # next separator in the list.  This mirrors LangChain's
+      # RecursiveCharacterTextSplitter behaviour.
+      #
+      # Default separators (in priority order):
+      #   1. "\n\n"  — paragraph breaks
+      #   2. "\n"    — line breaks
+      #   3. ". "    — sentence boundaries
+      #   4. " "     — word boundaries
+      #   5. ""      — character-level fallback
+      #
+      # @example
+      #   splitter = Phronomy::VectorStore::Splitter::RecursiveSplitter.new(chunk_size: 300, chunk_overlap: 30)
+      #   chunks   = splitter.split({ text: long_markdown, metadata: { source: "guide.md" } })
+      class RecursiveSplitter < Base
+        DEFAULT_SEPARATORS = ["\n\n", "\n", ". ", " ", ""].freeze
+
+        # @param chunk_size    [Integer] maximum characters per chunk (default: 1000)
+        # @param chunk_overlap [Integer] overlap characters (default: 200)
+        # @param separators    [Array<String>] separator list in priority order
+        # @api public
+        def initialize(chunk_size: 1000, chunk_overlap: 200, separators: DEFAULT_SEPARATORS)
+          raise ArgumentError, "chunk_overlap must be less than chunk_size" if chunk_overlap >= chunk_size
+
+          @chunk_size = chunk_size
+          @chunk_overlap = chunk_overlap
+          @separators = separators
+        end
+
+        # @param document [Hash, String]
+        # @return [Array<Hash>]
+        # @api public
+        def split(document)
+          doc = normalise(document)
+          texts = recursive_split(doc[:text], @separators)
+          merge_with_overlap(texts).each_with_index.map do |text, idx|
+            {text: text, metadata: doc[:metadata].merge(chunk: idx)}
+          end
+        end
+
+        private
+
+        # Split +text+ using the first separator that yields non-trivial pieces,
+        # then recurse on any piece that is still too large.
+        def recursive_split(text, separators)
+          return [text] if text.length <= @chunk_size || separators.empty?
+
+          sep, *rest_seps = separators
+
+          # Character-level fallback: just slice
+          if sep == ""
+            return FixedSizeSplitter
+                .new(chunk_size: @chunk_size, chunk_overlap: @chunk_overlap)
+                .split(text)
+                .map { |c| c[:text] }
+          end
+
+          parts = text.split(sep)
+
+          # If this separator doesn't split, try the next
+          return recursive_split(text, rest_seps) if parts.length <= 1
+
+          # Re-attach the separator to each part except the last so context is preserved
+          parts_with_sep = parts.each_with_index.map do |part, i|
+            (i < parts.length - 1) ? part + sep : part
+          end
+
+          parts_with_sep.flat_map do |part|
+            if part.length > @chunk_size
+              recursive_split(part, rest_seps)
+            else
+              [part]
+            end
+          end.reject { |t| t.strip.empty? }
+        end
+
+        # Merge small adjacent pieces and apply overlap between chunks.
+        def merge_with_overlap(texts)
+          merged = []
+          current = +""
+
+          texts.each do |text|
+            if current.length + text.length <= @chunk_size
+              current << text
+            else
+              merged << current.strip unless current.strip.empty?
+              # Start next chunk with overlap from the end of current
+              overlap_text = (current.length > @chunk_overlap) ? current[-@chunk_overlap..] : current
+              current = overlap_text + text
+            end
+          end
+
+          merged << current.strip unless current.strip.empty?
+          merged
+        end
+      end
+    end
+  end
+end