phronomy 0.5.3 → 0.5.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 49b20f3defaed56477f9f1ee375a450d26a770d004c052754cc5c045746587cc
-  data.tar.gz: 1d2fe811e467c7d04208b82cc9d9ca5fca9b17d0bf6061aa952a2fb862c23a53
+  metadata.gz: a820781cc6a8a1f5d2740f3eec3d7438e414755b6237e8617a3a38d6a3d3f7b1
+  data.tar.gz: 2de51efabc0fd0eff481aa2a58ee897e782b85bd78045feed9e6ba82917baeda
 SHA512:
-  metadata.gz: 763cf25297e0c8799ad76bcd362ecb5f1899a9ccd0d90791e119d2d0946c59f7c076f7a00d92e01e64735e90b45d7e1aa5462e41efceee9147daf45ac214551f
-  data.tar.gz: 3a30e9198008dd9e4e512c374324b4c1cfda40c2a41762ff160f90ffa8ac98c0669f700d810e899fa661bd68af3a14db0145a00afc4d0e97d560d7de27989db0
+  metadata.gz: 5e06a07c3cb2af23be1018afc0ad006b9b161fc02df38762fa8635b48cd46e8f84d82a14178129e64ab115846034a29177d55d0960af73f40d602fd56caae85c
+  data.tar.gz: 3e96270b7752a531fb5c2f432fbd80217e421fb16ddac47ac94224e3b783412cb5ea747d07a1603f2827c830aecf351a394c77449f99a9303135f56b94c53449

data/lib/phronomy/agent/orchestrator.rb

@@ -88,31 +88,112 @@ module Phronomy
       # threads. Each task is a Hash describing one agent invocation.
       #
       # Results are returned in the same order as the input +tasks+ array.
-      # If any thread raises an exception, the exception is re-raised in the
-      # calling thread after all threads have completed (via +Thread#value+).
+      # Concurrency is bounded by +max_concurrency+; when nil all tasks run at
+      # once (original behaviour).
       #
-      # @param tasks [Array<Hash>]
-      # @option task [Class]  :agent  agent class to invoke (required)
-      # @option task [String] :input  input string for the agent (required)
-      # @option task [Hash]   :config forwarded to +agent#invoke+ (default: +{}+)
-      # @return [Array<Hash>] agent results in the same order as +tasks+
-      def dispatch_parallel(*tasks)
-        threads = tasks.map do |task|
-          Thread.new do
-            task[:agent].new.invoke(task[:input], config: task.fetch(:config, {}))
-          end
+      # Error semantics are controlled by +on_error+:
+      # - +:raise+ (default) — every task runs to completion; the first
+      #   exception in input order is then re-raised in the calling thread.
+      # - +:skip+            — failed tasks return +nil+; no exception is raised.
+      #
+      # @param tasks           [Array<Hash>]
+      # @option task [Class]   :agent  agent class to invoke (required)
+      # @option task [String]  :input  input string for the agent (required)
+      # @option task [Hash]    :config forwarded to +agent#invoke+ (default: +{}+)
+      # @param max_concurrency [Integer, nil] maximum number of concurrent threads;
+      #   nil means no limit (all tasks run simultaneously)
+      # @param on_error        [Symbol] +:raise+ or +:skip+
+      # @return [Array<Hash, nil>] agent results in the same order as +tasks+
+      # @raise [ArgumentError] if +on_error+ is not +:raise+ or +:skip+
+      # @raise [ArgumentError] if +max_concurrency+ is not a positive Integer or nil
+      def dispatch_parallel(*tasks, max_concurrency: nil, on_error: :raise)
+        unless [:raise, :skip].include?(on_error)
+          raise ArgumentError, "unknown on_error: #{on_error.inspect}"
         end
-        threads.map(&:value)
+        if max_concurrency && !(max_concurrency.is_a?(Integer) && max_concurrency.positive?)
+          raise ArgumentError, "max_concurrency must be a positive Integer"
+        end
+
+        bounded_map(tasks, max_concurrency: max_concurrency, on_error: on_error)
       end
 
       # Runs the same agent against multiple inputs in parallel (fan-out pattern).
       #
-      # @param agent  [Class]         agent class to invoke for every input
-      # @param inputs [Array<String>] list of input strings
-      # @param config [Hash]          forwarded to every +agent#invoke+ call
-      # @return [Array<Hash>] results in the same order as +inputs+
-      def fan_out(agent:, inputs:, config: {})
-        dispatch_parallel(*inputs.map { |input| {agent: agent, input: input, config: config} })
+      # Accepts the same +max_concurrency:+ and +on_error:+ keyword arguments as
+      # {#dispatch_parallel} and forwards them unchanged.
+      #
+      # @param agent           [Class]         agent class to invoke for every input
+      # @param inputs          [Array<String>] list of input strings
+      # @param config          [Hash]          forwarded to every +agent#invoke+ call
+      # @param max_concurrency [Integer, nil]  forwarded to {#dispatch_parallel}
+      # @param on_error        [Symbol]        forwarded to {#dispatch_parallel}
+      # @return [Array<Hash, nil>] results in the same order as +inputs+
+      def fan_out(agent:, inputs:, config: {}, max_concurrency: nil, on_error: :raise)
+        dispatch_parallel(
+          *inputs.map { |input| {agent: agent, input: input, config: config} },
+          max_concurrency: max_concurrency,
+          on_error: on_error
+        )
+      end
+
+      private
+
+      # Worker-pool implementation shared by {#dispatch_parallel} and {#fan_out}.
+      #
+      # Uses a +Queue+ as a work-stealing mechanism: each worker thread pops a
+      # task, executes it, and loops until the queue is empty.  The number of
+      # workers is +min(max_concurrency, tasks.length)+, capped at the task count
+      # so we never spin up idle threads.
+      #
+      # +errors+ is indexed by task position so that the first error in *input*
+      # order is deterministically re-raised when +on_error: :raise+ is used.
+      # A +Mutex+ guards concurrent writes to +errors+ even though Array element
+      # assignment at different indices is safe in MRI; this keeps the code
+      # correct across alternative Ruby runtimes.
+      def bounded_map(tasks, max_concurrency:, on_error:)
+        return [] if tasks.empty?
+
+        results = Array.new(tasks.length)
+        errors = Array.new(tasks.length)
+        errors_mutex = Mutex.new
+
+        queue = Queue.new
+        tasks.each_with_index { |task, i| queue << [i, task] }
+
+        worker_count = [max_concurrency || tasks.length, tasks.length].min
+
+        workers = worker_count.times.map do
+          Thread.new do
+            loop do
+              i, task = begin
+                queue.pop(true)
+              rescue ThreadError
+                break # queue is empty; this worker is done
+              end
+
+              begin
+                results[i] = task[:agent].new.invoke(
+                  task[:input],
+                  config: task.fetch(:config, {})
+                )
+              rescue => e
+                case on_error
+                when :skip
+                  results[i] = nil
+                else
+                  errors_mutex.synchronize { errors[i] = e }
+                end
+              end
+            end
+          end
+        end
+
+        workers.each(&:join)
+
+        first_error = errors.compact.first
+        raise first_error if first_error
+
+        results
       end
     end
   end

data/lib/phronomy/vector_store/base.rb

@@ -36,6 +36,21 @@ module Phronomy
       def clear
         raise NotImplementedError, "#{self.class}#clear is not implemented"
       end
+
+      private
+
+      # Validates that embedding has the expected dimension.
+      # Raises ArgumentError if sizes differ.
+      # A nil expected_dimension is a no-op (dimension not yet established).
+      def validate_embedding_dimension!(embedding, expected_dimension)
+        return unless expected_dimension
+
+        actual = embedding.size
+        return if actual == expected_dimension
+
+        raise ArgumentError,
+          "Embedding dimension mismatch: expected #{expected_dimension}, got #{actual}"
+      end
     end
   end
 end

data/lib/phronomy/vector_store/in_memory.rb

@@ -12,14 +12,22 @@ module Phronomy
     #   store.add(id: "1", embedding: [0.1, 0.9], metadata: { message: msg })
     #   results = store.search(query_embedding: [0.1, 0.8], k: 3)
     class InMemory < Base
-      def initialize
+      # @param dimension [Integer, nil] expected embedding dimension.
+      #   When nil, the dimension is inferred from the first call to #add.
+      #   For multi-threaded use, pass dimension: explicitly; concurrent first
+      #   adds are not guaranteed to be race-free.
+      def initialize(dimension: nil)
         @documents = {}
+        @expected_dimension = dimension
       end
 
       # @param id        [String]
       # @param embedding [Array<Float>]
       # @param metadata  [Hash]
       def add(id:, embedding:, metadata: {})
+        # Establish expected dimension on first add, then validate.
+        @expected_dimension ||= embedding.size
+        validate_embedding_dimension!(embedding, @expected_dimension)
         @documents[id] = {embedding: embedding, metadata: metadata}
         self
       end
@@ -28,6 +36,8 @@ module Phronomy
       # @param k               [Integer]
       # @return [Array<Hash>] sorted by descending score
       def search(query_embedding:, k: 5)
+        # search never establishes dimension; validate only when dimension is known.
+        validate_embedding_dimension!(query_embedding, @expected_dimension)
         # Take an atomic snapshot before iterating.  Hash#dup is a C-level
         # call that completes without releasing the GVL, so it is atomic with
         # respect to any other Ruby thread.  Iterating the copy instead of

data/lib/phronomy/vector_store/pgvector.rb

@@ -18,8 +18,11 @@ module Phronomy
     #   store.add(id: "doc1", embedding: [0.1, 0.9], metadata: {text: "hello"})
     #   results = store.search(query_embedding: [0.1, 0.8], k: 5)
     class Pgvector < Base
-      # @param model_class [Class] ActiveRecord model with id/embedding/metadata columns
-      def initialize(model_class:)
+      # @param model_class [Class]        ActiveRecord model with id/embedding/metadata columns
+      # @param dimension   [Integer, nil] expected embedding dimension for Phronomy-side
+      #   pre-validation.  When nil, dimension enforcement is delegated to the
+      #   database schema; no pre-validation is performed by Phronomy.
+      def initialize(model_class:, dimension: nil)
         begin
           require "pgvector"
         rescue LoadError
@@ -28,12 +31,14 @@ module Phronomy
             "Add `gem 'pgvector'` to your Gemfile."
         end
         @model_class = model_class
+        @dimension = dimension
       end
 
       # @param id        [String]
       # @param embedding [Array<Float>]
       # @param metadata  [Hash]
       def add(id:, embedding:, metadata: {})
+        validate_embedding_dimension!(embedding, @dimension)
         @model_class.upsert(
           {id: id, embedding: safe_vector(embedding), metadata: metadata.to_json},
           unique_by: :id
@@ -45,6 +50,7 @@ module Phronomy
       # @param k               [Integer]
       # @return [Array<Hash>] sorted by descending similarity score
       def search(query_embedding:, k: 5)
+        validate_embedding_dimension!(query_embedding, @dimension)
         vec = safe_vector_literal(query_embedding)
         k_safe = Integer(k)
         conn = @model_class.connection

data/lib/phronomy/vector_store/redis_search.rb

@@ -25,7 +25,11 @@ module Phronomy
 
       # @param redis      [Redis]          configured Redis client
       # @param index_name [String]         RediSearch index name
-      # @param dimension  [Integer, nil]   vector dimension; auto-detected on first add
+      # @param dimension  [Integer, nil]   vector dimension; auto-detected on first add.
+      #   When connecting to an **existing** RediSearch index, you MUST pass
+      #   dimension: explicitly.  Without it, a freshly constructed instance
+      #   treats the index as uninitialized until #add is called, and #search
+      #   silently returns [] in the meantime.
       def initialize(redis:, index_name: "phronomy_vectors", dimension: nil)
         begin
           require "redis"
@@ -45,7 +49,11 @@ module Phronomy
       # @param embedding [Array<Float>]
       # @param metadata  [Hash]
       def add(id:, embedding:, metadata: {})
-        ensure_index!(embedding.length)
+        # Establish expected dimension on first add (not race-free for concurrent
+        # first adds), then validate, then create/reuse the index.
+        @dimension ||= embedding.size
+        validate_embedding_dimension!(embedding, @dimension)
+        ensure_index!(@dimension)
         @redis.call(
           "HSET", "#{DOC_PREFIX}#{id}",
           "embedding", pack_vector(embedding),
@@ -58,7 +66,12 @@ module Phronomy
       # @param k               [Integer]
       # @return [Array<Hash>] sorted by descending similarity score
       def search(query_embedding:, k: 5)
-        ensure_index!(query_embedding.length)
+        # search never establishes dimension.  If dimension is unknown and the
+        # index has not been created yet, there are no documents to return.
+        return [] if @dimension.nil? && !@index_created
+
+        validate_embedding_dimension!(query_embedding, @dimension)
+        ensure_index!(@dimension)
         k_safe = Integer(k)
         blob = pack_vector(query_embedding)
 

data/lib/phronomy/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Phronomy
-  VERSION = "0.5.3"
+  VERSION = "0.5.4"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: phronomy
 version: !ruby/object:Gem::Version
-  version: 0.5.3
+  version: 0.5.4
 platform: ruby
 authors:
 - Raizo T.C.S