phronomy 0.7.1 → 0.8.0

data/lib/phronomy/vector_store/in_memory.rb

@@ -1,93 +0,0 @@
-# frozen_string_literal: true
-
-module Phronomy
-  module VectorStore
-    # Pure-Ruby in-memory vector store using cosine similarity.
-    #
-    # Intended for tests, short-lived agents, and Retrieval::Semantic scenarios where
-    # the message count is small enough that a linear scan is fast enough.
-    #
-    # @example
-    #   store = Phronomy::VectorStore::InMemory.new
-    #   store.add(id: "1", embedding: [0.1, 0.9], metadata: { message: msg })
-    #   results = store.search(query_embedding: [0.1, 0.8], k: 3)
-    class InMemory < Base
-      # @param dimension [Integer, nil] expected embedding dimension.
-      #   When nil, the dimension is inferred from the first call to #add.
-      #   For multi-threaded use, pass dimension: explicitly; concurrent first
-      #   adds are not guaranteed to be race-free.
-      # @api public
-      def initialize(dimension: nil)
-        @documents = {}
-        @expected_dimension = dimension
-      end
-
-      # @param id                 [String]
-      # @param embedding          [Array<Float>]
-      # @param metadata           [Hash]
-      # @param cancellation_token [Phronomy::CancellationToken, nil]
-      # @api public
-      def add(id:, embedding:, metadata: {}, cancellation_token: nil)
-        cancellation_token&.raise_if_cancelled!
-        # Establish expected dimension on first add, then validate.
-        @expected_dimension ||= embedding.size
-        validate_embedding_dimension!(embedding, @expected_dimension)
-        @documents[id] = {embedding: embedding, metadata: metadata}
-        self
-      end
-
-      # @param query_embedding    [Array<Float>]
-      # @param k                  [Integer]
-      # @param cancellation_token [Phronomy::CancellationToken, nil]
-      # @return [Array<Hash>] sorted by descending score
-      # @api public
-      def search(query_embedding:, k: 5, cancellation_token: nil)
-        cancellation_token&.raise_if_cancelled!
-        k = validate_k!(k)
-        # search never establishes dimension; validate only when dimension is known.
-        validate_embedding_dimension!(query_embedding, @expected_dimension)
-        # Take an atomic snapshot before iterating.  Hash#dup is a C-level
-        # call that completes without releasing the GVL, so it is atomic with
-        # respect to any other Ruby thread.  Iterating the copy instead of
-        # @documents directly prevents "can't add a new key into hash during
-        # iteration" when a concurrent thread calls #add.
-        snapshot = @documents.dup
-        results = snapshot.map do |id, doc|
-          score = cosine_similarity(query_embedding, doc[:embedding])
-          {id: id, score: score, metadata: doc[:metadata]}
-        end
-        results.sort_by { |r| -r[:score] }.first(k)
-      end
-
-      def remove(id:)
-        @documents.delete(id)
-        self
-      end
-
-      def clear
-        @documents.clear
-        self
-      end
-
-      # @return [Integer] number of documents stored
-      # @api public
-      def size
-        @documents.size
-      end
-
-      private
-
-      def cosine_similarity(a, b)
-        return 0.0 if a.empty? || b.empty?
-
-        dot = a.zip(b).sum { |x, y| x * y }
-        norm_a = Math.sqrt(a.sum { |x| x**2 })
-        norm_b = Math.sqrt(b.sum { |x| x**2 })
-
-        return 0.0 if norm_a.zero? || norm_b.zero?
-
-        dot / (norm_a * norm_b)
-      end
-    end
-  end
-end

data/lib/phronomy/vector_store/pgvector.rb

@@ -1,127 +0,0 @@
-# frozen_string_literal: true
-
-require "json"
-
-module Phronomy
-  module VectorStore
-    # PostgreSQL-backed vector store using the pgvector extension.
-    #
-    # Requires:
-    #   - The +pgvector+ gem (add to your Gemfile)
-    #   - An ActiveRecord model class with the following columns:
-    #       id        (string / uuid)
-    #       embedding (vector — from the pgvector column type)
-    #       metadata  (text or jsonb — stores arbitrary metadata as JSON)
-    #
-    # @example Usage
-    #   store = Phronomy::VectorStore::Pgvector.new(model_class: VectorDocument)
-    #   store.add(id: "doc1", embedding: [0.1, 0.9], metadata: {text: "hello"})
-    #   results = store.search(query_embedding: [0.1, 0.8], k: 5)
-    class Pgvector < Base
-      # @param model_class [Class]        ActiveRecord model with id/embedding/metadata columns
-      # @param dimension   [Integer, nil] expected embedding dimension for Phronomy-side
-      #   pre-validation.  When nil, dimension enforcement is delegated to the
-      #   database schema; no pre-validation is performed by Phronomy.
-      # @api public
-      def initialize(model_class:, dimension: nil)
-        begin
-          require "pgvector"
-        rescue LoadError
-          raise LoadError,
-            "pgvector gem is required for Phronomy::VectorStore::Pgvector. " \
-            "Add `gem 'pgvector'` to your Gemfile."
-        end
-        @model_class = model_class
-        @dimension = dimension
-      end
-
-      # @param id                 [String]
-      # @param embedding          [Array<Float>]
-      # @param metadata           [Hash]
-      # @param cancellation_token [Phronomy::CancellationToken, nil]
-      # @api public
-      def add(id:, embedding:, metadata: {}, cancellation_token: nil)
-        cancellation_token&.raise_if_cancelled!
-        validate_embedding_dimension!(embedding, @dimension)
-        @model_class.upsert(
-          {id: id, embedding: safe_vector(embedding), metadata: metadata.to_json},
-          unique_by: :id
-        )
-        self
-      end
-
-      # @param query_embedding    [Array<Float>]
-      # @param k                  [Integer]
-      # @param cancellation_token [Phronomy::CancellationToken, nil]
-      # @return [Array<Hash>] sorted by descending similarity score
-      # @api public
-      def search(query_embedding:, k: 5, cancellation_token: nil)
-        cancellation_token&.raise_if_cancelled!
-        k_safe = validate_k!(k)
-        validate_embedding_dimension!(query_embedding, @dimension)
-        vec = safe_vector_literal(query_embedding)
-        conn = @model_class.connection
-        quoted_vec = "#{conn.quote(vec)}::vector"
-
-        @model_class
-          .select("id, metadata, 1 - (embedding <=> #{quoted_vec}) AS score")
-          .order("embedding <=> #{quoted_vec}")
-          .limit(k_safe)
-          .map do |r|
-            {
-              id: r.id.to_s,
-              score: r.score.to_f,
-              metadata: parse_metadata(r.metadata)
-            }
-          end
-      end
-
-      def remove(id:)
-        @model_class.where(id: id).delete_all
-        self
-      end
-
-      def clear
-        @model_class.delete_all
-        self
-      end
-
-      # Returns the number of documents in the backing table.
-      def size
-        @model_class.count
-      end
-
-      private
-
-      # Parses a metadata value returned by the pg driver.
-      # Handles NULL (nil), already-parsed Hash, and JSON string forms.
-      def parse_metadata(raw)
-        return {} if raw.nil?
-        return symbolize_hash_keys(raw) if raw.is_a?(Hash)
-
-        parsed = JSON.parse(raw.to_s, symbolize_names: true)
-        parsed.is_a?(Hash) ? parsed : {}
-      rescue JSON::ParserError
-        {}
-      end
-
-      # Recursively symbolizes keys for an already-parsed Hash.
-      def symbolize_hash_keys(hash)
-        hash.each_with_object({}) do |(k, v), h|
-          h[k.to_sym] = v.is_a?(Hash) ? symbolize_hash_keys(v) : v
-        end
-      end
-
-      # Validates that all elements are numeric and converts to a pgvector-
-      # compatible literal string (e.g. "[1.0,0.5,-0.3]").
-      def safe_vector_literal(embedding)
-        "[#{embedding.map { |v| Float(v) }.join(",")}]"
-      end
-
-      # Returns a validated vector for the upsert call.
-      def safe_vector(embedding)
-        safe_vector_literal(embedding)
-      end
-    end
-  end
-end

data/lib/phronomy/vector_store/redis_search.rb

@@ -1,192 +0,0 @@
-# frozen_string_literal: true
-
-require "json"
-
-module Phronomy
-  module VectorStore
-    # Redis-backed vector store using the RediSearch module (FT.* commands).
-    #
-    # Requires:
-    #   - The +redis+ gem (add to your Gemfile)
-    #   - A Redis server with the RediSearch (RedisSearch) module enabled
-    #     (or Redis Stack which bundles RediSearch)
-    #
-    # Vectors are stored as FLOAT32 binary blobs in Redis Hash fields and
-    # searched using the KNN approximate-nearest-neighbour algorithm.
-    #
-    # @example Usage
-    #   redis = Redis.new(url: "redis://localhost:6379")
-    #   store = Phronomy::VectorStore::RedisSearch.new(redis: redis, dimension: 1536)
-    #   store.add(id: "doc1", embedding: [0.1, 0.9], metadata: {text: "hello"})
-    #   results = store.search(query_embedding: [0.1, 0.8], k: 5)
-    class RedisSearch < Base
-      DOC_PREFIX = "phronomy_doc:"
-      private_constant :DOC_PREFIX
-
-      # @param redis      [Redis]          configured Redis client
-      # @param index_name [String]         RediSearch index name
-      # @param dimension  [Integer, nil]   vector dimension; auto-detected on first add.
-      #   When connecting to an **existing** RediSearch index, you MUST pass
-      #   dimension: explicitly.  Without it, a freshly constructed instance
-      #   treats the index as uninitialized until #add is called, and #search
-      #   silently returns [] in the meantime.
-      # @api public
-      def initialize(redis:, index_name: "phronomy_vectors", dimension: nil)
-        begin
-          require "redis"
-        rescue LoadError
-          raise LoadError,
-            "redis gem is required for Phronomy::VectorStore::RedisSearch. " \
-            "Add `gem 'redis'` to your Gemfile."
-        end
-        @redis = redis
-        @index_name = index_name
-        @dimension = dimension
-        @index_created = false
-        @mutex = Mutex.new
-      end
-
-      # @param id                 [String]
-      # @param embedding          [Array<Float>]
-      # @param metadata           [Hash]
-      # @param cancellation_token [Phronomy::CancellationToken, nil]
-      # @api public
-      def add(id:, embedding:, metadata: {}, cancellation_token: nil)
-        cancellation_token&.raise_if_cancelled!
-        # Establish expected dimension on first add (not race-free for concurrent
-        # first adds), then validate, then create/reuse the index.
-        @dimension ||= embedding.size
-        validate_embedding_dimension!(embedding, @dimension)
-        ensure_index!(@dimension)
-        @redis.call(
-          "HSET", "#{DOC_PREFIX}#{id}",
-          "embedding", pack_vector(embedding),
-          "metadata", metadata.to_json
-        )
-        self
-      end
-
-      # @param query_embedding    [Array<Float>]
-      # @param k                  [Integer]
-      # @param cancellation_token [Phronomy::CancellationToken, nil]
-      # @return [Array<Hash>] sorted by descending similarity score
-      # @api public
-      def search(query_embedding:, k: 5, cancellation_token: nil)
-        cancellation_token&.raise_if_cancelled!
-        # search never establishes dimension.  If dimension is unknown and the
-        # index has not been created yet, there are no documents to return.
-        return [] if @dimension.nil? && !@index_created
-
-        validate_embedding_dimension!(query_embedding, @dimension)
-        ensure_index!(@dimension)
-        k_safe = validate_k!(k)
-        blob = pack_vector(query_embedding)
-
-        raw = @redis.call(
-          "FT.SEARCH", @index_name,
-          "*=>[KNN #{k_safe} @embedding $BLOB AS score]",
-          "PARAMS", 2, "BLOB", blob,
-          "SORTBY", "score",
-          "RETURN", 2, "score", "metadata",
-          "DIALECT", 2
-        )
-
-        parse_results(raw)
-      end
-
-      def remove(id:)
-        @redis.call("DEL", "#{DOC_PREFIX}#{id}")
-        self
-      end
-
-      # Returns the number of documents indexed.
-      # Queries FT.INFO when the index has been created; returns 0 otherwise.
-      def size
-        return 0 unless @index_created
-
-        raw = @redis.call("FT.INFO", @index_name)
-        return 0 unless raw.is_a?(Array)
-
-        idx = raw.index("num_docs")
-        idx ? raw[idx + 1].to_i : 0
-      rescue
-        0
-      end
-
-      def clear
-        @mutex.synchronize do
-          begin
-            @redis.call("FT.DROPINDEX", @index_name, "DD")
-          rescue => e
-            raise unless e.message.to_s.include?("Unknown Index name")
-          end
-          @index_created = false
-        end
-        self
-      end
-
-      private
-
-      def ensure_index!(dim)
-        @mutex.synchronize do
-          return if @index_created
-
-          @dimension ||= dim
-          begin
-            @redis.call(
-              "FT.CREATE", @index_name,
-              "ON", "HASH",
-              "PREFIX", 1, DOC_PREFIX,
-              "SCHEMA",
-              "embedding", "VECTOR", "FLAT", 6,
-              "TYPE", "FLOAT32",
-              "DIM", @dimension,
-              "DISTANCE_METRIC", "COSINE",
-              "metadata", "TEXT"
-            )
-          rescue => e
-            raise unless e.message.to_s.include?("Index already exists")
-          end
-          @index_created = true
-        end
-      end
-
-      # Pack a Float array as a FLOAT32 binary string for RediSearch.
-      def pack_vector(embedding)
-        embedding.map { |v| Float(v) }.pack("f*")
-      end
-
-      # Parse the raw FT.SEARCH response into the standard Hash format.
-      #
-      # Redis FT.SEARCH returns: [count, key1, [field, value, ...], key2, ...]
-      def parse_results(raw)
-        return [] if raw.nil? || !raw.is_a?(Array) || raw.size < 2
-
-        results = []
-        i = 1
-        while i < raw.size
-          key = raw[i]
-          fields = raw[i + 1]
-          i += 2
-
-          next unless fields.is_a?(Array)
-
-          field_hash = fields.each_slice(2).to_h
-          score_str = field_hash["score"]
-          metadata_str = field_hash["metadata"]
-
-          next if score_str.nil?
-
-          id = key.to_s.delete_prefix(DOC_PREFIX)
-          # RediSearch returns cosine distance (0=identical, 2=opposite);
-          # convert to cosine similarity for consistency with other backends.
-          score = 1.0 - score_str.to_f
-          metadata = metadata_str ? JSON.parse(metadata_str, symbolize_names: true) : {}
-
-          results << {id: id, score: score, metadata: metadata}
-        end
-        results
-      end
-    end
-  end
-end