bx_builder_chain 0.1.16 → 0.1.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aad98d1bac1b55ae34b9f092ca2e1611a3358c69a10b3b3b5af552f2619589ea
-  data.tar.gz: 32f1d91df4c276ad118adc687ea272f5dac5760ec04ad3d2970607f348abdd5c
+  metadata.gz: 022f97be7d06eac31ca9d9f70f541b9fc3636ae44f7fda16ee72f43e5b86aa69
+  data.tar.gz: dbe3501ebd87e3e3f1149b31e66a35bd26649c60c06e5195d87ba98a7c7c7913
 SHA512:
-  metadata.gz: 8c6ade2b2818b871b89bb770438d43c67a6889522ae00d7ff4c30b58b3f92adf66874e7966b842325a4b6d5a7e311560d5fdc4d7496b4ab96caa0c09314e1093
-  data.tar.gz: 4ad5f52faa63cb00f1bd3decd42ac176bf931cef497857e77d505224edd5d4dc44d48db0b5b6a9ddefb84f53a220a6239726982eaaef98752c7a2e8469f08763
+  metadata.gz: c10fbc01e9cedfa8ffb188e3d47c3522c931830002099a71b623b31574af2fb0b3df089760593d3b7e87fb38a1257eef54c4fde0b32eb600adea8dabdc234471
+  data.tar.gz: 95b0b54b0fbeba57225f6d7a07f2166dc3b28b4ea0a973d08cecdff7e3f870030f470df662a329dcc4b364b7c346e9357f9acfd693e67307e2c8cd05db4d8a92

data/lib/bx_builder_chain/vectorsearch/pgvector.rb

@@ -69,21 +69,26 @@ module BxBuilderChain::Vectorsearch
     # @return [Array<Integer>] The the ids of the added texts.
     def add_texts(texts:, ids: nil)
       if ids.nil? || ids.empty?
-        Async do |parent|
-          data = texts.each_slice(20).flat_map do |text_batch|  # Process in batches of 20
+        mutex = Mutex.new
+        texts.each_slice(10).flat_map do |text_batch|  # Process in batches of 10
+          data = []
+          Async do |parent|
             text_batch.map do |text|
               parent.async do |task|
                 begin
-                  {content: text, vectors: llm.embed(text: text).to_s, namespace: namespaces[0]}
+                  vectorised_text = {content: text, vectors: llm.embed(text: text).to_s, namespace: namespaces[0]}
+                  mutex.synchronize do
+                    data << vectorised_text
+                  end
                 rescue => e
                   puts "Error processing text: #{e.message}"
                   nil  # or some error indication
                 end
               end
-            end.map(&:wait)  # Ensure all tasks in the batch are completed
-          end
+            end # Ensure all tasks in the batch are completed
+          end.map(&:wait) 
+          @db[@table_name.to_sym].multi_insert(data, return: :primary_key)
         end        
-        @db[@table_name.to_sym].multi_insert(data, return: :primary_key)
       else
         upsert_texts(texts: texts, ids: ids)
       end

data/lib/bx_builder_chain/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module BxBuilderChain
-  VERSION = "0.1.16"
+  VERSION = "0.1.17"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bx_builder_chain
 version: !ruby/object:Gem::Version
-  version: 0.1.16
+  version: 0.1.17
 platform: ruby
 authors:
 - Paul Ketelle
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-02-02 00:00:00.000000000 Z
+date: 2024-02-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: zeitwerk