phronomy 0.7.0 → 0.8.0

data/lib/phronomy/agent/context/knowledge/vector_store/async_backend.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Agent
+    module Context
+      module Knowledge
+        module VectorStore
+          # Mixin that defines the async interface for VectorStore backends.
+          #
+          # Mixing this module into a VectorStore class provides three choices:
+          #
+          # 1. **Do nothing** — inherits default implementations from {VectorStore::Base}
+          #    that route through {BlockingAdapterPool} (the previous behaviour).
+          #
+          # 2. **Override selectively** — override only the async methods where the
+          #    backend has a native async driver, while the remaining methods fall back
+          #    to the pool.
+          #
+          # 3. **Implement all natively** — override all async methods to avoid pool
+          #    allocation entirely.
+          #
+          # @example Native async search (no pool worker thread allocated)
+          #   class MyFastStore < Phronomy::Agent::Context::Knowledge::VectorStore::Base
+          #     include Phronomy::Agent::Context::Knowledge::VectorStore::AsyncBackend
+          #
+          #     def search_async(query_embedding:, k: 5, cancellation_token: nil, timeout: nil)
+          #       # Returns a PendingOperation backed by a native async driver.
+          #       native_async_search(query_embedding, k)
+          #     end
+          #   end
+          #
+          # @api public
+          module AsyncBackend
+            # Async variant of {VectorStore::Base#add}.
+            #
+            # Submits the add call to {BlockingAdapterPool} by default.
+            # Override to use a native async driver.
+            #
+            # @param id                 [String]
+            # @param embedding          [Array<Float>]
+            # @param metadata           [Hash]
+            # @param cancellation_token [Phronomy::Concurrency::CancellationToken, nil]
+            # @param timeout            [Numeric, nil]
+            # @return [BlockingAdapterPool::PendingOperation]
+            # @api public
+            def add_async(id:, embedding:, metadata: {}, cancellation_token: nil, timeout: nil)
+              Phronomy::Runtime.instance.blocking_io.submit(
+                timeout: timeout,
+                cancellation_token: cancellation_token
+              ) do
+                add(id: id, embedding: embedding, metadata: metadata, cancellation_token: cancellation_token)
+              end
+            end
+
+            # Async variant of {VectorStore::Base#search}.
+            #
+            # Submits the search call to {BlockingAdapterPool} by default.
+            # Override to use a native async driver.
+            #
+            # @param query_embedding    [Array<Float>]
+            # @param k                  [Integer]
+            # @param cancellation_token [Phronomy::Concurrency::CancellationToken, nil]
+            # @param timeout            [Numeric, nil]
+            # @return [BlockingAdapterPool::PendingOperation]
+            # @api public
+            def search_async(query_embedding:, k: 5, cancellation_token: nil, timeout: nil)
+              Phronomy::Runtime.instance.blocking_io.submit(
+                timeout: timeout,
+                cancellation_token: cancellation_token
+              ) do
+                search(query_embedding: query_embedding, k: k, cancellation_token: cancellation_token)
+              end
+            end
+
+            # Async variant of {VectorStore::Base#remove}.
+            #
+            # Submits the remove call to {BlockingAdapterPool} by default.
+            # Override to use a native async driver.
+            #
+            # @param id                 [String]
+            # @param cancellation_token [Phronomy::Concurrency::CancellationToken, nil]
+            # @param timeout            [Numeric, nil]
+            # @return [BlockingAdapterPool::PendingOperation]
+            # @api public
+            def remove_async(id:, cancellation_token: nil, timeout: nil)
+              Phronomy::Runtime.instance.blocking_io.submit(
+                timeout: timeout,
+                cancellation_token: cancellation_token
+              ) do
+                remove(id: id)
+              end
+            end
+
+            # Async variant of {VectorStore::Base#clear}.
+            #
+            # Submits the clear call to {BlockingAdapterPool} by default.
+            # Override to use a native async driver.
+            #
+            # @param cancellation_token [Phronomy::Concurrency::CancellationToken, nil]
+            # @param timeout            [Numeric, nil]
+            # @return [BlockingAdapterPool::PendingOperation]
+            # @api public
+            def clear_async(cancellation_token: nil, timeout: nil)
+              Phronomy::Runtime.instance.blocking_io.submit(
+                timeout: timeout,
+                cancellation_token: cancellation_token
+              ) do
+                clear
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/phronomy/agent/context/knowledge/vector_store/base.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Agent
+    module Context
+      module Knowledge
+        module VectorStore
+          # Abstract interface for vector stores.
+          #
+          # Implementations manage a collection of (embedding, metadata) pairs and
+          # support similarity search.
+          #
+          # Async methods (`search_async`, `add_async`, `remove_async`, `clear_async`)
+          # are provided by the {AsyncBackend} mixin which defaults to routing calls
+          # through {BlockingAdapterPool}.  Backends with native async drivers may
+          # override individual async methods without touching the pool at all.
+          class Base
+            include AsyncBackend
+
+            # Add a document with its vector embedding.
+            #
+            # @param id                 [String]                         unique document identifier
+            # @param embedding          [Array<Float>]                   vector embedding
+            # @param metadata           [Hash]                           arbitrary metadata (e.g. the original message object)
+            # @param cancellation_token [Phronomy::Concurrency::CancellationToken, nil] optional; raises CancellationError when cancelled
+            # @api public
+            def add(id:, embedding:, metadata: {}, cancellation_token: nil)
+              cancellation_token&.raise_if_cancelled!
+              raise NotImplementedError, "#{self.class}#add is not implemented"
+            end
+
+            # Return the k most similar documents to the query embedding.
+            #
+            # @param query_embedding    [Array<Float>]
+            # @param k                  [Integer]                        number of results
+            # @param cancellation_token [Phronomy::Concurrency::CancellationToken, nil] optional; raises CancellationError when cancelled
+            # @return [Array<Hash>] each element: { id:, score:, metadata: }
+            # @api public
+            def search(query_embedding:, k: 5, cancellation_token: nil)
+              cancellation_token&.raise_if_cancelled!
+              raise NotImplementedError, "#{self.class}#search is not implemented"
+            end
+
+            # Remove a single document by id.
+            #
+            # @param id [String] document identifier
+            # @api public
+            def remove(id:)
+              raise NotImplementedError, "#{self.class}#remove is not implemented"
+            end
+
+            # Remove all documents.
+            def clear
+              raise NotImplementedError, "#{self.class}#clear is not implemented"
+            end
+
+            # Return the number of documents stored.
+            #
+            # @return [Integer]
+            # @api public
+            def size
+              raise NotImplementedError, "#{self.class}#size is not implemented"
+            end
+
+            private
+
+            # Validates that embedding has the expected dimension.
+            # Raises ArgumentError if sizes differ.
+            # A nil expected_dimension is a no-op (dimension not yet established).
+            def validate_embedding_dimension!(embedding, expected_dimension)
+              return unless expected_dimension
+
+              actual = embedding.size
+              return if actual == expected_dimension
+
+              raise ArgumentError,
+                "Embedding dimension mismatch: expected #{expected_dimension}, got #{actual}"
+            end
+
+            # Validates that k is a positive integer.
+            # Accepts any value accepted by Integer() (e.g. "5"), but raises
+            # ArgumentError for non-integer strings, zero, and negative values.
+            def validate_k!(k)
+              int_k = Integer(k)
+              raise ArgumentError, "k must be a positive integer, got #{int_k}" unless int_k >= 1
+              int_k
+            rescue ArgumentError => e
+              raise ArgumentError, "k must be a positive integer: #{e.message}"
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/phronomy/agent/context/knowledge/vector_store/in_memory.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Agent
+    module Context
+      module Knowledge
+        module VectorStore
+          # Pure-Ruby in-memory vector store using cosine similarity.
+          #
+          # Intended for tests, short-lived agents, and Retrieval::Semantic scenarios where
+          # the message count is small enough that a linear scan is fast enough.
+          #
+          # @example
+          #   store = Phronomy::Agent::Context::Knowledge::VectorStore::InMemory.new
+          #   store.add(id: "1", embedding: [0.1, 0.9], metadata: { message: msg })
+          #   results = store.search(query_embedding: [0.1, 0.8], k: 3)
+          class InMemory < Base
+            # @param dimension [Integer, nil] expected embedding dimension.
+            #   When nil, the dimension is inferred from the first call to #add.
+            #   For multi-threaded use, pass dimension: explicitly; concurrent first
+            #   adds are not guaranteed to be race-free.
+            # @api public
+            def initialize(dimension: nil)
+              @documents = {}
+              @expected_dimension = dimension
+            end
+
+            # @param id                 [String]
+            # @param embedding          [Array<Float>]
+            # @param metadata           [Hash]
+            # @param cancellation_token [Phronomy::Concurrency::CancellationToken, nil]
+            # @api public
+            def add(id:, embedding:, metadata: {}, cancellation_token: nil)
+              cancellation_token&.raise_if_cancelled!
+              # Establish expected dimension on first add, then validate.
+              @expected_dimension ||= embedding.size
+              validate_embedding_dimension!(embedding, @expected_dimension)
+              @documents[id] = {embedding: embedding, metadata: metadata}
+              self
+            end
+
+            # @param query_embedding    [Array<Float>]
+            # @param k                  [Integer]
+            # @param cancellation_token [Phronomy::Concurrency::CancellationToken, nil]
+            # @return [Array<Hash>] sorted by descending score
+            # @api public
+            # mutant:disable - genuine equivalent mutations: doc.fetch(:embedding) vs doc[:embedding] (key
+            #   always present); {id:, score:, metadata: doc.fetch(:metadata)} shorthand+fetch vs []
+            #   (key always present); -r.fetch(:score) vs -r[:score] (key always present); snapshot = @documents
+            #   vs .dup is equivalent in single-threaded tests (GVL makes Hash#dup atomic, no behaviour
+            #   difference under test isolation)
+            def search(query_embedding:, k: 5, cancellation_token: nil)
+              cancellation_token&.raise_if_cancelled!
+              k = validate_k!(k)
+              # search never establishes dimension; validate only when dimension is known.
+              validate_embedding_dimension!(query_embedding, @expected_dimension)
+              # Take an atomic snapshot before iterating.  Hash#dup is a C-level
+              # call that completes without releasing the GVL, so it is atomic with
+              # respect to any other Ruby thread.  Iterating the copy instead of
+              # @documents directly prevents "can't add a new key into hash during
+              # iteration" when a concurrent thread calls #add.
+              snapshot = @documents.dup
+              results = snapshot.map do |id, doc|
+                score = cosine_similarity(query_embedding, doc[:embedding])
+                {id: id, score: score, metadata: doc[:metadata]}
+              end
+              results.sort_by { |r| -r[:score] }.first(k)
+            end
+
+            def remove(id:)
+              @documents.delete(id)
+              self
+            end
+
+            def clear
+              @documents.clear
+              self
+            end
+
+            # @return [Integer] number of documents stored
+            # @api public
+            def size
+              @documents.size
+            end
+
+            private
+
+            # mutant:disable - empty-vector early-return condition variants (if false, if nil, if a.empty?,
+            #   if b.empty?, if a.empty? && b.empty?, if a.empty? || false, if false || b.empty?,
+            #   if nil || b.empty?, if nil && b.empty?) are genuine equivalents: dimension validation in
+            #   #add and #search enforces same-size embeddings, so a.empty? iff b.empty?; when both are
+            #   empty norm_a = sqrt(0) = 0 so the norm_a.zero? guard returns 0.0 anyway
+            def cosine_similarity(a, b)
+              return 0.0 if a.empty? || b.empty?
+
+              dot = a.zip(b).sum { |x, y| x * y }
+              norm_a = Math.sqrt(a.sum { |x| x**2 })
+              norm_b = Math.sqrt(b.sum { |x| x**2 })
+
+              return 0.0 if norm_a.zero? || norm_b.zero?
+
+              dot / (norm_a * norm_b)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/phronomy/agent/context/knowledge/vector_store/pgvector.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Phronomy
+  module Agent
+    module Context
+      module Knowledge
+        module VectorStore
+          # PostgreSQL-backed vector store using the pgvector extension.
+          #
+          # Requires:
+          #   - The +pgvector+ gem (add to your Gemfile)
+          #   - An ActiveRecord model class with the following columns:
+          #       id        (string / uuid)
+          #       embedding (vector — from the pgvector column type)
+          #       metadata  (text or jsonb — stores arbitrary metadata as JSON)
+          #
+          # @example Usage
+          #   store = Phronomy::Agent::Context::Knowledge::VectorStore::Pgvector.new(model_class: VectorDocument)
+          #   store.add(id: "doc1", embedding: [0.1, 0.9], metadata: {text: "hello"})
+          #   results = store.search(query_embedding: [0.1, 0.8], k: 5)
+          class Pgvector < Base
+            # @param model_class [Class]        ActiveRecord model with id/embedding/metadata columns
+            # @param dimension   [Integer, nil] expected embedding dimension for Phronomy-side
+            #   pre-validation.  When nil, dimension enforcement is delegated to the
+            #   database schema; no pre-validation is performed by Phronomy.
+            # @api public
+            def initialize(model_class:, dimension: nil)
+              begin
+                require "pgvector"
+              rescue LoadError
+                raise LoadError,
+                  "pgvector gem is required for Phronomy::Agent::Context::Knowledge::VectorStore::Pgvector. " \
+                  "Add `gem 'pgvector'` to your Gemfile."
+              end
+              @model_class = model_class
+              @dimension = dimension
+            end
+
+            # @param id                 [String]
+            # @param embedding          [Array<Float>]
+            # @param metadata           [Hash]
+            # @param cancellation_token [Phronomy::Concurrency::CancellationToken, nil]
+            # @api public
+            def add(id:, embedding:, metadata: {}, cancellation_token: nil)
+              cancellation_token&.raise_if_cancelled!
+              validate_embedding_dimension!(embedding, @dimension)
+              @model_class.upsert(
+                {id: id, embedding: safe_vector(embedding), metadata: metadata.to_json},
+                unique_by: :id
+              )
+              self
+            end
+
+            # @param query_embedding    [Array<Float>]
+            # @param k                  [Integer]
+            # @param cancellation_token [Phronomy::Concurrency::CancellationToken, nil]
+            # @return [Array<Hash>] sorted by descending similarity score
+            # @api public
+            def search(query_embedding:, k: 5, cancellation_token: nil)
+              cancellation_token&.raise_if_cancelled!
+              k_safe = validate_k!(k)
+              validate_embedding_dimension!(query_embedding, @dimension)
+              vec = safe_vector_literal(query_embedding)
+              conn = @model_class.connection
+              quoted_vec = "#{conn.quote(vec)}::vector"
+
+              @model_class
+                .select("id, metadata, 1 - (embedding <=> #{quoted_vec}) AS score")
+                .order("embedding <=> #{quoted_vec}")
+                .limit(k_safe)
+                .map do |r|
+                  {
+                    id: r.id.to_s,
+                    score: r.score.to_f,
+                    metadata: parse_metadata(r.metadata)
+                  }
+                end
+            end
+
+            def remove(id:)
+              @model_class.where(id: id).delete_all
+              self
+            end
+
+            def clear
+              @model_class.delete_all
+              self
+            end
+
+            # Returns the number of documents in the backing table.
+            def size
+              @model_class.count
+            end
+
+            private
+
+            # Parses a metadata value returned by the pg driver.
+            # Handles NULL (nil), already-parsed Hash, and JSON string forms.
+            def parse_metadata(raw)
+              return {} if raw.nil?
+              return symbolize_hash_keys(raw) if raw.is_a?(Hash)
+
+              parsed = JSON.parse(raw.to_s, symbolize_names: true)
+              parsed.is_a?(Hash) ? parsed : {}
+            rescue JSON::ParserError
+              {}
+            end
+
+            # Recursively symbolizes keys for an already-parsed Hash.
+            def symbolize_hash_keys(hash)
+              hash.each_with_object({}) do |(k, v), h|
+                h[k.to_sym] = v.is_a?(Hash) ? symbolize_hash_keys(v) : v
+              end
+            end
+
+            # Validates that all elements are numeric and converts to a pgvector-
+            # compatible literal string (e.g. "[1.0,0.5,-0.3]").
+            def safe_vector_literal(embedding)
+              "[#{embedding.map { |v| Float(v) }.join(",")}]"
+            end
+
+            # Returns a validated vector for the upsert call.
+            def safe_vector(embedding)
+              safe_vector_literal(embedding)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/phronomy/agent/context/knowledge/vector_store/redis_search.rb

@@ -0,0 +1,198 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Phronomy
+  module Agent
+    module Context
+      module Knowledge
+        module VectorStore
+          # Redis-backed vector store using the RediSearch module (FT.* commands).
+          #
+          # Requires:
+          #   - The +redis+ gem (add to your Gemfile)
+          #   - A Redis server with the RediSearch (RedisSearch) module enabled
+          #     (or Redis Stack which bundles RediSearch)
+          #
+          # Vectors are stored as FLOAT32 binary blobs in Redis Hash fields and
+          # searched using the KNN approximate-nearest-neighbour algorithm.
+          #
+          # @example Usage
+          #   redis = Redis.new(url: "redis://localhost:6379")
+          #   store = Phronomy::Agent::Context::Knowledge::VectorStore::RedisSearch.new(redis: redis, dimension: 1536)
+          #   store.add(id: "doc1", embedding: [0.1, 0.9], metadata: {text: "hello"})
+          #   results = store.search(query_embedding: [0.1, 0.8], k: 5)
+          class RedisSearch < Base
+            DOC_PREFIX = "phronomy_doc:"
+            private_constant :DOC_PREFIX
+
+            # @param redis      [Redis]          configured Redis client
+            # @param index_name [String]         RediSearch index name
+            # @param dimension  [Integer, nil]   vector dimension; auto-detected on first add.
+            #   When connecting to an **existing** RediSearch index, you MUST pass
+            #   dimension: explicitly.  Without it, a freshly constructed instance
+            #   treats the index as uninitialized until #add is called, and #search
+            #   silently returns [] in the meantime.
+            # @api public
+            def initialize(redis:, index_name: "phronomy_vectors", dimension: nil)
+              begin
+                require "redis"
+              rescue LoadError
+                raise LoadError,
+                  "redis gem is required for Phronomy::Agent::Context::Knowledge::VectorStore::RedisSearch. " \
+                  "Add `gem 'redis'` to your Gemfile."
+              end
+              @redis = redis
+              @index_name = index_name
+              @dimension = dimension
+              @index_created = false
+              @mutex = Mutex.new
+            end
+
+            # @param id                 [String]
+            # @param embedding          [Array<Float>]
+            # @param metadata           [Hash]
+            # @param cancellation_token [Phronomy::Concurrency::CancellationToken, nil]
+            # @api public
+            def add(id:, embedding:, metadata: {}, cancellation_token: nil)
+              cancellation_token&.raise_if_cancelled!
+              # Establish expected dimension on first add (not race-free for concurrent
+              # first adds), then validate, then create/reuse the index.
+              @dimension ||= embedding.size
+              validate_embedding_dimension!(embedding, @dimension)
+              ensure_index!(@dimension)
+              @redis.call(
+                "HSET", "#{DOC_PREFIX}#{id}",
+                "embedding", pack_vector(embedding),
+                "metadata", metadata.to_json
+              )
+              self
+            end
+
+            # @param query_embedding    [Array<Float>]
+            # @param k                  [Integer]
+            # @param cancellation_token [Phronomy::Concurrency::CancellationToken, nil]
+            # @return [Array<Hash>] sorted by descending similarity score
+            # @api public
+            def search(query_embedding:, k: 5, cancellation_token: nil)
+              cancellation_token&.raise_if_cancelled!
+              # search never establishes dimension.  If dimension is unknown and the
+              # index has not been created yet, there are no documents to return.
+              return [] if @dimension.nil? && !@index_created
+
+              validate_embedding_dimension!(query_embedding, @dimension)
+              ensure_index!(@dimension)
+              k_safe = validate_k!(k)
+              blob = pack_vector(query_embedding)
+
+              raw = @redis.call(
+                "FT.SEARCH", @index_name,
+                "*=>[KNN #{k_safe} @embedding $BLOB AS score]",
+                "PARAMS", 2, "BLOB", blob,
+                "SORTBY", "score",
+                "RETURN", 2, "score", "metadata",
+                "DIALECT", 2
+              )
+
+              parse_results(raw)
+            end
+
+            def remove(id:)
+              @redis.call("DEL", "#{DOC_PREFIX}#{id}")
+              self
+            end
+
+            # Returns the number of documents indexed.
+            # Queries FT.INFO when the index has been created; returns 0 otherwise.
+            def size
+              return 0 unless @index_created
+
+              raw = @redis.call("FT.INFO", @index_name)
+              return 0 unless raw.is_a?(Array)
+
+              idx = raw.index("num_docs")
+              idx ? raw[idx + 1].to_i : 0
+            rescue
+              0
+            end
+
+            def clear
+              @mutex.synchronize do
+                begin
+                  @redis.call("FT.DROPINDEX", @index_name, "DD")
+                rescue => e
+                  raise unless e.message.to_s.include?("Unknown Index name")
+                end
+                @index_created = false
+              end
+              self
+            end
+
+            private
+
+            def ensure_index!(dim)
+              @mutex.synchronize do
+                return if @index_created
+
+                @dimension ||= dim
+                begin
+                  @redis.call(
+                    "FT.CREATE", @index_name,
+                    "ON", "HASH",
+                    "PREFIX", 1, DOC_PREFIX,
+                    "SCHEMA",
+                    "embedding", "VECTOR", "FLAT", 6,
+                    "TYPE", "FLOAT32",
+                    "DIM", @dimension,
+                    "DISTANCE_METRIC", "COSINE",
+                    "metadata", "TEXT"
+                  )
+                rescue => e
+                  raise unless e.message.to_s.include?("Index already exists")
+                end
+                @index_created = true
+              end
+            end
+
+            # Pack a Float array as a FLOAT32 binary string for RediSearch.
+            def pack_vector(embedding)
+              embedding.map { |v| Float(v) }.pack("f*")
+            end
+
+            # Parse the raw FT.SEARCH response into the standard Hash format.
+            #
+            # Redis FT.SEARCH returns: [count, key1, [field, value, ...], key2, ...]
+            def parse_results(raw)
+              return [] if raw.nil? || !raw.is_a?(Array) || raw.size < 2
+
+              results = []
+              i = 1
+              while i < raw.size
+                key = raw[i]
+                fields = raw[i + 1]
+                i += 2
+
+                next unless fields.is_a?(Array)
+
+                field_hash = fields.each_slice(2).to_h
+                score_str = field_hash["score"]
+                metadata_str = field_hash["metadata"]
+
+                next if score_str.nil?
+
+                id = key.to_s.delete_prefix(DOC_PREFIX)
+                # RediSearch returns cosine distance (0=identical, 2=opposite);
+                # convert to cosine similarity for consistency with other backends.
+                score = 1.0 - score_str.to_f
+                metadata = metadata_str ? JSON.parse(metadata_str, symbolize_names: true) : {}
+
+                results << {id: id, score: score, metadata: metadata}
+              end
+              results
+            end
+          end
+        end
+      end
+    end
+  end
+end