bx_builder_chain 0.1.16 → 0.1.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/bx_builder_chain/vectorsearch/pgvector.rb +11 -6
- data/lib/bx_builder_chain/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 237c376277c4f4330b156d083479d167f875bf7753713c557236300b80a837d0
|
|
4
|
+
data.tar.gz: 3635f1ee1fcbf7301b25ec305b8313462d6b8179fd7471fbe320526e0849ffa3
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: b87525970086538996e8d5b3831ae6639e6a741d95175045538314c99b3edc7958c2ca0fb0ebfb4f3224e63131df629842beb6d058aa38e658ce9dbde7798eb6
|
|
7
|
+
data.tar.gz: 4ff088e45098ad00f22b3c77f6f615a82b7592ab75460b2e21be5b0e1e079b78aedb9f45a1cd54288b703bece07c137d09eecddfd6280008fcb84b90cd9f653c
|
|
@@ -69,21 +69,26 @@ module BxBuilderChain::Vectorsearch
|
|
|
69
69
|
# @return [Array<Integer>] The the ids of the added texts.
|
|
70
70
|
def add_texts(texts:, ids: nil)
|
|
71
71
|
if ids.nil? || ids.empty?
|
|
72
|
-
|
|
73
|
-
|
|
72
|
+
mutex = Mutex.new
|
|
73
|
+
texts.each_slice(10).flat_map do |text_batch| # Process in batches of 10
|
|
74
|
+
data = []
|
|
75
|
+
Async do |parent|
|
|
74
76
|
text_batch.map do |text|
|
|
75
77
|
parent.async do |task|
|
|
76
78
|
begin
|
|
77
|
-
{content: text, vectors: llm.embed(text: text).to_s, namespace: namespaces[0]}
|
|
79
|
+
vectorised_text = {content: text, vectors: llm.embed(text: text).to_s, namespace: namespaces[0]}
|
|
80
|
+
mutex.synchronize do
|
|
81
|
+
data << vectorised_text
|
|
82
|
+
end
|
|
78
83
|
rescue => e
|
|
79
84
|
puts "Error processing text: #{e.message}"
|
|
80
85
|
nil # or some error indication
|
|
81
86
|
end
|
|
82
87
|
end
|
|
83
|
-
end
|
|
84
|
-
end
|
|
88
|
+
end # Ensure all tasks in the batch are completed
|
|
89
|
+
end.wait
|
|
90
|
+
@db[@table_name.to_sym].multi_insert(data, return: :primary_key)
|
|
85
91
|
end
|
|
86
|
-
@db[@table_name.to_sym].multi_insert(data, return: :primary_key)
|
|
87
92
|
else
|
|
88
93
|
upsert_texts(texts: texts, ids: ids)
|
|
89
94
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: bx_builder_chain
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.18
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Paul Ketelle
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2024-02-
|
|
11
|
+
date: 2024-02-19 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: zeitwerk
|