search-engine-for-typesense 30.1.8.24 → 30.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +22 -0
- data/README.md +76 -3
- data/lib/search_engine/base/relation_delegation.rb +1 -0
- data/lib/search_engine/client/http_adapter.rb +3 -11
- data/lib/search_engine/client/request_builder.rb +5 -7
- data/lib/search_engine/client/services/search.rb +6 -18
- data/lib/search_engine/indexer.rb +1 -1
- data/lib/search_engine/observability.rb +14 -5
- data/lib/search_engine/relation/compiler.rb +7 -15
- data/lib/search_engine/relation/dsl/vectors.rb +67 -21
- data/lib/search_engine/relation/dsl.rb +1 -2
- data/lib/search_engine/relation/dx/dry_run.rb +9 -4
- data/lib/search_engine/relation/dx.rb +3 -3
- data/lib/search_engine/result.rb +18 -0
- data/lib/search_engine/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: f8363aa6879873e4b2d897181f0e87494b454b5c1ccf51f19d29f4590e992b24
|
|
4
|
+
data.tar.gz: bd13ccd5af83d57a4499227629bbc6caa360dd08e15457e8eb80067d174fc20e
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: d8d3847c59ab120f22f076d3c76aeebf7ea0e6b74ccf29771b72f30896f8a7789f213c1d139c40a8565ed4e63612adebafafbe81ae25eb1dd5190a7dc6215e92
|
|
7
|
+
data.tar.gz: 03700343db8c49a61e5d722992cae22e23ff0dfd51506d98124b2039fd0e7302f0da495914b549e0082718832199778d3ffbedd3d62e89c53f056c56b58c0250
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,27 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 30.9.0
|
|
4
|
+
|
|
5
|
+
- Qualify geo/vector search and native caching against Typesense 30.2 while keeping the gem major
|
|
6
|
+
version aligned with the Typesense major version.
|
|
7
|
+
- Expose model-level geo/vector query methods, preserve external hybrid weights and historical query
|
|
8
|
+
boundaries, and sort the active vector query with Typesense's native distance token.
|
|
9
|
+
- Validate finite real vectors, declared field types/dimensions and integer options before execution.
|
|
10
|
+
Preserve backend vector distance on flat, grouped and multi-search hits.
|
|
11
|
+
- Redact scientific-notation vector components and forward single/multi-search cache options through
|
|
12
|
+
the Ruby SDK's request parameters.
|
|
13
|
+
- Preserve caller-owned history strings and IDs, reject invalid scalar conversions consistently, and
|
|
14
|
+
mask malformed vector values. Align single-search adapters, telemetry and shell-quoted cURL previews
|
|
15
|
+
with GET requests and URL cache controls.
|
|
16
|
+
- Keep stale-deletion estimates uncached so maintenance previews observe current documents.
|
|
17
|
+
- Add database-free query examples and isolated Typesense 30.2 integration checks, including scoped-key
|
|
18
|
+
cache isolation and opt-in model-backed search. Upgrade older servers before using scoped-key caching.
|
|
19
|
+
|
|
20
|
+
## 30.8.24
|
|
21
|
+
|
|
22
|
+
- Adopt `{typesense_major}.{gem_minor}.{gem_patch}` versioning, omitting the Typesense minor component.
|
|
23
|
+
- Include the same runtime fixes as 30.1.8.24; no additional runtime behavior changes.
|
|
24
|
+
|
|
3
25
|
## 30.1.8.24
|
|
4
26
|
|
|
5
27
|
- Preserve caller transactions during PostgreSQL streaming by rejecting active transactions before opening a cursor.
|
data/README.md
CHANGED
|
@@ -8,15 +8,18 @@ Mountless Rails::Engine for [Typesense](https://typesense.org). Expressive Relat
|
|
|
8
8
|
|
|
9
9
|
## Versioning
|
|
10
10
|
|
|
11
|
-
|
|
11
|
+
Versions use `{typesense_major}.{gem_minor}.{gem_patch}`. The first component identifies the targeted
|
|
12
|
+
Typesense server major version; the remaining components track gem enhancements and fixes independently
|
|
13
|
+
of Typesense minor releases.
|
|
12
14
|
|
|
13
|
-
|
|
15
|
+
Gem `30.9.0` targets Typesense **30.x** and is tested against **30.2**. Typesense minor versions are not
|
|
16
|
+
encoded in the gem version.
|
|
14
17
|
|
|
15
18
|
## Quickstart
|
|
16
19
|
|
|
17
20
|
```ruby
|
|
18
21
|
# Gemfile
|
|
19
|
-
gem "search-engine-for-typesense"
|
|
22
|
+
gem "search-engine-for-typesense", "~> 30.9"
|
|
20
23
|
```
|
|
21
24
|
|
|
22
25
|
```ruby
|
|
@@ -180,8 +183,56 @@ SearchEngine::Venue
|
|
|
180
183
|
# Access geo distance on results (present when order_geo is used)
|
|
181
184
|
result = SearchEngine::Venue.all.order_geo(:location, from: { lat: 54.69, lng: 25.28 }).execute
|
|
182
185
|
result.hits.first.geo_distance_meters # => { "location" => 1234 }
|
|
186
|
+
|
|
187
|
+
# External vectors: supply embeddings generated by your application
|
|
188
|
+
class SearchEngine::Article < SearchEngine::Base
|
|
189
|
+
collection :articles
|
|
190
|
+
identify_by :id
|
|
191
|
+
attribute :title, :string
|
|
192
|
+
embedding :embedding, suffix: false, num_dim: 3
|
|
193
|
+
query_by :title
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
result = SearchEngine::Article
|
|
197
|
+
.search('chair')
|
|
198
|
+
.vector_search(query: [1, 0, 0], alpha: 0.8, k: 20)
|
|
199
|
+
.order(vector_distance: :asc)
|
|
200
|
+
.execute
|
|
201
|
+
result.hits.first.vector_distance # Exact backend distance, including 0
|
|
202
|
+
SearchEngine::Article.find_similar('article-42', k: 20).to_a
|
|
183
203
|
```
|
|
184
204
|
|
|
205
|
+
`where_geo`, `order_geo`, `order_eval`, `vector_search` and `find_similar` work directly on models
|
|
206
|
+
and on relations. They build lazy queries. Vector distance ordering uses the active query's native
|
|
207
|
+
`_vector_distance` sort; it does not generate another embedding. The metadata reader is present when
|
|
208
|
+
the backend supplies a distance, including grouped and multi-search hits. It takes precedence over
|
|
209
|
+
a same-named document reader; the original document remains in `result.raw`.
|
|
210
|
+
|
|
211
|
+
Vectors must contain finite real numbers and match declared dimensions. Integer options (`k`, `ef`,
|
|
212
|
+
`flat_search_cutoff`) reject fractional values and accept integer strings. Historical queries preserve
|
|
213
|
+
commas, parentheses and brackets; colons, backticks, control characters and surrounding whitespace
|
|
214
|
+
raise `InvalidVectorQuery`. Similarity IDs reject commas, colons, parentheses, control characters and
|
|
215
|
+
surrounding whitespace. Weight sums keep the configurable wrapper tolerance (default `0.01`), but
|
|
216
|
+
Typesense 30.1/30.2 require a sum of 1.0; a server rejection propagates as `Errors::Api`.
|
|
217
|
+
|
|
218
|
+
Typesense **30.2** is the tested server baseline. Geo, external vectors, auto-embedded hybrid search,
|
|
219
|
+
historical queries and native caching are checked against that version. Earlier 30.1 qualification
|
|
220
|
+
exposed the scoped-cache isolation defect described below; upgrade before using scoped-key caching.
|
|
221
|
+
See the [vector guide](https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30/vector-search)
|
|
222
|
+
for auto-embedding declarations and query modes.
|
|
223
|
+
|
|
224
|
+
Run the database-free examples from the gem root:
|
|
225
|
+
|
|
226
|
+
```bash
|
|
227
|
+
bundle exec ruby examples/demo_shop/bin/query_smoke
|
|
228
|
+
```
|
|
229
|
+
|
|
230
|
+
This compiles geo/vector requests and checks metadata fixtures without HTTP requests or model downloads.
|
|
231
|
+
See [the demo instructions](examples/demo_shop/README.md#query-examples) for explicitly isolated live execution,
|
|
232
|
+
including highlighting through `Client#search` and `hit.snippet_for`.
|
|
233
|
+
CI uses a disposable `typesense/typesense:30.2` service for geo, vector, cache and live CLI checks,
|
|
234
|
+
including the built-in embedding model. The default unit suite runs separately without integration ENV.
|
|
235
|
+
|
|
185
236
|
`upsert_bulk` defaults to `on_failure: :raise`. Use the exact symbol `:return` when the caller needs to
|
|
186
237
|
handle valid Typesense row failures itself. Malformed responses and response/document count mismatches always
|
|
187
238
|
raise in either mode; the gem never guesses which submitted document a missing response row belongs to.
|
|
@@ -207,6 +258,28 @@ You can control this explicitly with:
|
|
|
207
258
|
|
|
208
259
|
If you set `SearchEngine.configure { |c| c.client = ... }`, the custom client is always used.
|
|
209
260
|
|
|
261
|
+
## Native search caching
|
|
262
|
+
|
|
263
|
+
The gem defaults to `use_cache: true` and `cache_ttl_s: 60`. Single-search cache options are sent as
|
|
264
|
+
GET parameters; multi-search options apply to the whole request URL, outside individual search bodies.
|
|
265
|
+
Use `Model.cache(false).count` or `.to_a` for fresh maintenance reads.
|
|
266
|
+
|
|
267
|
+
On Typesense 30.1/30.2, cached responses remained stale after document writes and alias swaps until
|
|
268
|
+
expiry or explicit clearing. A direct key's URL `cache_ttl` did not override the native default;
|
|
269
|
+
embed `cache_ttl` in a scoped search key when controlling TTL. Revoked keys fail authentication even
|
|
270
|
+
when a matching response is cached.
|
|
271
|
+
|
|
272
|
+
**Typesense 30.1 has a scoped-cache isolation defect:** identical single/multi-search requests with
|
|
273
|
+
different scoped filters can receive each other's cached results. The same checks pass on 30.2.
|
|
274
|
+
On 30.1, set `c.use_cache = false` or use `.cache(false)` for scoped authorization; upgrade to 30.2
|
|
275
|
+
before enabling it. The integration tests retain strict isolation assertions and expose this server
|
|
276
|
+
failure rather than skipping it. The gem's existing global cache default is unchanged.
|
|
277
|
+
|
|
278
|
+
`SearchEngine::Cache.clear` clears the contacted server's native search cache. For incremental outbox
|
|
279
|
+
writes, `postgres_outbox.clear_cache_after_write` remains opt-in: clearing occurs before database
|
|
280
|
+
acknowledgement, and clearing failures propagate. No Redis response-cache layer is required for
|
|
281
|
+
sharing cached searches across application processes reaching the same Typesense node.
|
|
282
|
+
|
|
210
283
|
## Async partition indexing
|
|
211
284
|
|
|
212
285
|
Async partition indexing is an advanced opt-in mode for partitioned full indexing. The default remains
|
|
@@ -19,6 +19,7 @@ module SearchEngine
|
|
|
19
19
|
# Delegate materializers and query dsl to `.all` so callers can do `Model.first` etc.
|
|
20
20
|
%i[
|
|
21
21
|
where rewhere merge order preset ranking prefix search
|
|
22
|
+
where_geo order_geo order_eval vector_search find_similar
|
|
22
23
|
pin hide curate clear_curation
|
|
23
24
|
facet_by facet_query group_by unscope
|
|
24
25
|
limit offset page per_page per options cache
|
|
@@ -31,7 +31,7 @@ module SearchEngine
|
|
|
31
31
|
# {Response#body}.
|
|
32
32
|
#
|
|
33
33
|
# Supported forms (current usage):
|
|
34
|
-
# -
|
|
34
|
+
# - GET `/collections/:collection/documents/search` (with URL search/cache parameters)
|
|
35
35
|
#
|
|
36
36
|
# @param request [SearchEngine::Client::RequestBuilder::Request] normalized request
|
|
37
37
|
# @return [Response] response wrapper with status, headers and raw body
|
|
@@ -40,22 +40,14 @@ module SearchEngine
|
|
|
40
40
|
def perform(request)
|
|
41
41
|
method = request.http_method.to_sym
|
|
42
42
|
path = request.path.to_s
|
|
43
|
-
if method == :
|
|
43
|
+
if method == :get && path.match?(%r{\A/collections/[^/]+/documents/search\z})
|
|
44
44
|
collection = path.split('/')[2]
|
|
45
45
|
docs = @typesense.collections[collection].documents
|
|
46
|
-
|
|
47
|
-
params = docs.method(:search).parameters
|
|
48
|
-
supports_common = params.any? { |(kind, _)| %i[key keyreq keyrest].include?(kind) }
|
|
49
|
-
rescue StandardError
|
|
50
|
-
supports_common = false
|
|
51
|
-
end
|
|
52
|
-
raw = supports_common ? docs.search(request.body, common_params: request.params) : docs.search(request.body)
|
|
46
|
+
raw = docs.search(request.params)
|
|
53
47
|
# The official client returns parsed JSON (Hash). No headers/status exposed here.
|
|
54
48
|
return Response.new(status: 200, headers: {}, body: raw)
|
|
55
49
|
end
|
|
56
50
|
|
|
57
|
-
# Fallback: attempt a generic call via low-level typesense endpoint access if available.
|
|
58
|
-
# This keeps adapter permissive for future endpoints without adding Faraday here.
|
|
59
51
|
raise ArgumentError, "Unsupported request path for adapter: #{request.http_method} #{request.path}"
|
|
60
52
|
end
|
|
61
53
|
end
|
|
@@ -63,17 +63,15 @@ module SearchEngine
|
|
|
63
63
|
# @see `https://typesense.org/docs/latest/api/documents.html#search-document`
|
|
64
64
|
def self.build_search(collection:, compiled_params:, url_opts: {})
|
|
65
65
|
name = collection.to_s
|
|
66
|
-
|
|
67
|
-
# Ensure deterministic ordering even after sanitization
|
|
68
|
-
body_json = SearchEngine::CompiledParams.from(body_hash).to_json
|
|
66
|
+
query_params = sanitize_body_params(compiled_params.to_h).merge(url_opts || {})
|
|
69
67
|
|
|
70
68
|
Request.new(
|
|
71
|
-
http_method: :
|
|
69
|
+
http_method: :get,
|
|
72
70
|
path: COLLECTIONS_PREFIX + name + DOCUMENTS_SEARCH_SUFFIX,
|
|
73
|
-
params:
|
|
71
|
+
params: query_params,
|
|
74
72
|
headers: DEFAULT_HEADERS_JSON,
|
|
75
|
-
body:
|
|
76
|
-
body_json:
|
|
73
|
+
body: nil,
|
|
74
|
+
body_json: nil
|
|
77
75
|
)
|
|
78
76
|
end
|
|
79
77
|
|
|
@@ -21,8 +21,8 @@ module SearchEngine
|
|
|
21
21
|
|
|
22
22
|
result = instrumented_search(collection, params_obj, cache_params, path, payload, start)
|
|
23
23
|
duration = current_monotonic_ms - start
|
|
24
|
-
instrument(:
|
|
25
|
-
log_success(:
|
|
24
|
+
instrument(:get, path, duration, cache_params, request_token: start)
|
|
25
|
+
log_success(:get, path, start, cache_params)
|
|
26
26
|
|
|
27
27
|
klass = begin
|
|
28
28
|
SearchEngine.collection_for(collection)
|
|
@@ -45,7 +45,7 @@ module SearchEngine
|
|
|
45
45
|
bodies = Array(searches).map { |s| sanitize_body_params(s) }
|
|
46
46
|
|
|
47
47
|
result = with_exception_mapping(:post, path, cache_params, start) do
|
|
48
|
-
typesense.multi_search.perform({ searches: bodies },
|
|
48
|
+
typesense.multi_search.perform({ searches: bodies }, cache_params)
|
|
49
49
|
end
|
|
50
50
|
|
|
51
51
|
instrument(:post, path, current_monotonic_ms - start, cache_params, request_token: start)
|
|
@@ -64,7 +64,7 @@ module SearchEngine
|
|
|
64
64
|
result = nil
|
|
65
65
|
SearchEngine::Instrumentation.instrument('search_engine.search', se_payload) do |ctx|
|
|
66
66
|
ctx[:params_preview] = SearchEngine::Instrumentation.redact(params_obj.to_h)
|
|
67
|
-
result = with_exception_mapping(:
|
|
67
|
+
result = with_exception_mapping(:get, path, cache_params, start) do
|
|
68
68
|
docs = typesense.collections[collection].documents
|
|
69
69
|
documents_search(docs, payload, cache_params)
|
|
70
70
|
end
|
|
@@ -80,7 +80,7 @@ module SearchEngine
|
|
|
80
80
|
end
|
|
81
81
|
result
|
|
82
82
|
else
|
|
83
|
-
with_exception_mapping(:
|
|
83
|
+
with_exception_mapping(:get, path, cache_params, start) do
|
|
84
84
|
docs = typesense.collections[collection].documents
|
|
85
85
|
documents_search(docs, payload, cache_params)
|
|
86
86
|
end
|
|
@@ -170,19 +170,7 @@ module SearchEngine
|
|
|
170
170
|
end
|
|
171
171
|
|
|
172
172
|
def documents_search(docs, payload, common_params)
|
|
173
|
-
|
|
174
|
-
docs.search(payload, common_params: common_params)
|
|
175
|
-
else
|
|
176
|
-
docs.search(payload)
|
|
177
|
-
end
|
|
178
|
-
end
|
|
179
|
-
|
|
180
|
-
def documents_search_supports_common_params?(docs)
|
|
181
|
-
m = docs.method(:search)
|
|
182
|
-
params = m.parameters
|
|
183
|
-
params.any? { |(kind, _)| %i[key keyreq keyrest].include?(kind) }
|
|
184
|
-
rescue StandardError
|
|
185
|
-
false
|
|
173
|
+
docs.search(payload.merge(common_params))
|
|
186
174
|
end
|
|
187
175
|
end
|
|
188
176
|
end
|
|
@@ -241,7 +241,7 @@ module SearchEngine
|
|
|
241
241
|
client = SearchEngine.client
|
|
242
242
|
payload = { q: '*', query_by: cfg.default_query_by, per_page: 0, filter_by: filter }
|
|
243
243
|
params = SearchEngine::CompiledParams.new(payload)
|
|
244
|
-
res = client.search(collection: into, params: params, url_opts: {})
|
|
244
|
+
res = client.search(collection: into, params: params, url_opts: { use_cache: false })
|
|
245
245
|
res&.found
|
|
246
246
|
rescue StandardError
|
|
247
247
|
nil
|
|
@@ -141,12 +141,21 @@ module SearchEngine
|
|
|
141
141
|
# @param vq [String]
|
|
142
142
|
# @return [String]
|
|
143
143
|
def self.redact_vector_query(vq)
|
|
144
|
-
return vq
|
|
145
|
-
return
|
|
144
|
+
return vq if vq.nil? || SearchEngine.config.observability.redact_vectors == false
|
|
145
|
+
return '[REDACTED]' unless vq.is_a?(String)
|
|
146
146
|
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
147
|
+
vector_array = /\A([^:]+:\(\s*)\[([^\[\]]*)\](?=\s*[,)])/
|
|
148
|
+
return '[REDACTED]' unless vq.match?(vector_array)
|
|
149
|
+
|
|
150
|
+
vq.sub(vector_array) do
|
|
151
|
+
prefix = Regexp.last_match(1)
|
|
152
|
+
values = Regexp.last_match(2)
|
|
153
|
+
next "#{prefix}[]" if values.strip.empty?
|
|
154
|
+
next "#{prefix}[#{values}]" if values.match?(/\A<\d+ dims>\z/)
|
|
155
|
+
|
|
156
|
+
numbers = values.split(',', -1).map(&:strip)
|
|
157
|
+
numeric = numbers.all? { |number| number.match?(/\A[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?\z/) }
|
|
158
|
+
"#{prefix}#{numeric ? "[<#{numbers.size} dims>]" : '[REDACTED]'}"
|
|
150
159
|
end
|
|
151
160
|
end
|
|
152
161
|
|
|
@@ -518,7 +518,6 @@ module SearchEngine
|
|
|
518
518
|
|
|
519
519
|
auto_include_embedding_in_query_by!(params, vq, field)
|
|
520
520
|
auto_exclude_embedding_from_response!(params, field)
|
|
521
|
-
resolve_vector_distance_sort!(params, field)
|
|
522
521
|
instrument_vector_compile(vq, params, field)
|
|
523
522
|
end
|
|
524
523
|
|
|
@@ -538,7 +537,10 @@ module SearchEngine
|
|
|
538
537
|
parts << "#{key}:#{vq[key]}" if vq.key?(key)
|
|
539
538
|
end
|
|
540
539
|
parts << "alpha:#{vq[:alpha]}" if vq.key?(:alpha) && hybrid_vector_mode?(params, vq)
|
|
541
|
-
|
|
540
|
+
if vq[:queries]
|
|
541
|
+
queries = vq[:queries].map { |query| "`#{query}`" }
|
|
542
|
+
parts << "queries:[#{queries.join(', ')}]"
|
|
543
|
+
end
|
|
542
544
|
parts << "query_weights:[#{vq[:weights].join(', ')}]" if vq[:weights]
|
|
543
545
|
|
|
544
546
|
inner = [vector_part, *parts].join(', ')
|
|
@@ -547,9 +549,10 @@ module SearchEngine
|
|
|
547
549
|
|
|
548
550
|
# Hybrid mode: auto-append the embedding field to `query_by` so that
|
|
549
551
|
# Typesense performs rank-fusion between keyword and vector results.
|
|
550
|
-
#
|
|
552
|
+
# Explicit vectors already supply the embedding and need no query_by addition.
|
|
551
553
|
def auto_include_embedding_in_query_by!(params, vq, field)
|
|
552
554
|
return unless hybrid_vector_mode?(params, vq)
|
|
555
|
+
return if vq.key?(:query)
|
|
553
556
|
|
|
554
557
|
current = params[:query_by].to_s
|
|
555
558
|
fields = current.split(',').map(&:strip)
|
|
@@ -571,19 +574,8 @@ module SearchEngine
|
|
|
571
574
|
params[:exclude_fields] = current.empty? ? field : "#{current},#{field}"
|
|
572
575
|
end
|
|
573
576
|
|
|
574
|
-
# Replace the `_vector_distance` sentinel produced by order normalization
|
|
575
|
-
# with the real Typesense sort token that references the embedding field.
|
|
576
|
-
def resolve_vector_distance_sort!(params, vq_field)
|
|
577
|
-
sort = params[:sort_by]
|
|
578
|
-
return unless sort&.include?(VECTOR_DISTANCE_SENTINEL)
|
|
579
|
-
|
|
580
|
-
params[:sort_by] = sort.gsub(/#{VECTOR_DISTANCE_SENTINEL}:(asc|desc)/) do
|
|
581
|
-
"_vector_query(#{vq_field}:([])):#{Regexp.last_match(1)}"
|
|
582
|
-
end
|
|
583
|
-
end
|
|
584
|
-
|
|
585
577
|
def hybrid_vector_mode?(params, vq)
|
|
586
|
-
params[:q].to_s != '*' && !vq.key?(:
|
|
578
|
+
params[:q].to_s != '*' && !vq.key?(:id) && !vq.key?(:queries)
|
|
587
579
|
end
|
|
588
580
|
|
|
589
581
|
def explicitly_selected_field?(field_name)
|
|
@@ -63,6 +63,9 @@ module SearchEngine
|
|
|
63
63
|
|
|
64
64
|
validate_vector_query_mode_exclusivity!(query, id, queries)
|
|
65
65
|
validate_vector_query_array!(query)
|
|
66
|
+
validate_vector_dimensions!(field_sym, query) if query
|
|
67
|
+
normalized_id = id.to_s unless id.nil?
|
|
68
|
+
validate_vector_id!(normalized_id)
|
|
66
69
|
validate_vector_queries!(queries)
|
|
67
70
|
validate_vector_weights!(weights, queries)
|
|
68
71
|
|
|
@@ -75,11 +78,11 @@ module SearchEngine
|
|
|
75
78
|
result = { field: field_sym }
|
|
76
79
|
result[:k] = coerced_k if coerced_k
|
|
77
80
|
result[:alpha] = Float(alpha) if alpha
|
|
78
|
-
result[:query] = query if query
|
|
79
|
-
result[:id] =
|
|
81
|
+
result[:query] = query.map(&:to_f) if query
|
|
82
|
+
result[:id] = normalized_id.dup unless normalized_id.nil?
|
|
80
83
|
result[:distance_threshold] = Float(distance_threshold) if distance_threshold
|
|
81
|
-
result[:queries] = queries if queries
|
|
82
|
-
result[:weights] = weights if weights
|
|
84
|
+
result[:queries] = queries.map(&:dup) if queries
|
|
85
|
+
result[:weights] = weights.map(&:to_f) if weights
|
|
83
86
|
result[:ef] = coerced_ef if coerced_ef
|
|
84
87
|
result[:flat_search_cutoff] = coerced_flat if coerced_flat
|
|
85
88
|
result
|
|
@@ -106,11 +109,19 @@ module SearchEngine
|
|
|
106
109
|
|
|
107
110
|
sym = field.to_sym
|
|
108
111
|
attrs = safe_attributes_map
|
|
109
|
-
return if attrs.nil?
|
|
112
|
+
return if attrs.nil?
|
|
110
113
|
|
|
111
114
|
embeddings = vector_embeddings_map
|
|
112
115
|
|
|
113
|
-
return if
|
|
116
|
+
return if embeddings.key?(sym) || [:vector, [:float], :'float[]', 'float[]'].include?(attrs[sym])
|
|
117
|
+
|
|
118
|
+
if attrs.key?(sym)
|
|
119
|
+
raise SearchEngine::Errors::InvalidVectorQuery.new(
|
|
120
|
+
"InvalidVectorQuery: field :#{sym} must be a vector or float array",
|
|
121
|
+
doc: VECTOR_SEARCH_DOC_URL,
|
|
122
|
+
details: { field: sym, type: attrs[sym] }
|
|
123
|
+
)
|
|
124
|
+
end
|
|
114
125
|
|
|
115
126
|
known = (attrs.keys + embeddings.keys).uniq
|
|
116
127
|
suggestions = suggest_fields(sym, known)
|
|
@@ -128,9 +139,9 @@ module SearchEngine
|
|
|
128
139
|
|
|
129
140
|
def validate_vector_query_mode_exclusivity!(query, id, queries)
|
|
130
141
|
modes = []
|
|
131
|
-
modes << :query
|
|
132
|
-
modes << :id
|
|
133
|
-
modes << :queries
|
|
142
|
+
modes << :query unless query.nil?
|
|
143
|
+
modes << :id unless id.nil?
|
|
144
|
+
modes << :queries unless queries.nil?
|
|
134
145
|
return if modes.length <= 1
|
|
135
146
|
|
|
136
147
|
raise SearchEngine::Errors::InvalidVectorQuery.new(
|
|
@@ -146,9 +157,9 @@ module SearchEngine
|
|
|
146
157
|
def validate_vector_query_array!(query)
|
|
147
158
|
return if query.nil?
|
|
148
159
|
|
|
149
|
-
unless query.is_a?(Array) && query.all? { |v|
|
|
160
|
+
unless query.is_a?(Array) && query.all? { |v| finite_vector_number?(v) }
|
|
150
161
|
raise SearchEngine::Errors::InvalidVectorQuery.new(
|
|
151
|
-
'InvalidVectorQuery: query: must be an Array of
|
|
162
|
+
'InvalidVectorQuery: query: must be an Array of finite real numbers',
|
|
152
163
|
hint: 'Pass a float array like [0.1, 0.2, ...]',
|
|
153
164
|
doc: VECTOR_SEARCH_DOC_URL,
|
|
154
165
|
details: { query_class: query.class.name }
|
|
@@ -164,6 +175,35 @@ module SearchEngine
|
|
|
164
175
|
)
|
|
165
176
|
end
|
|
166
177
|
|
|
178
|
+
def finite_vector_number?(value)
|
|
179
|
+
value.is_a?(Numeric) && value.real? && value.to_f.finite?
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
def validate_vector_dimensions!(field, query)
|
|
183
|
+
metadata = vector_embeddings_map[field] || {}
|
|
184
|
+
options = @klass.respond_to?(:attribute_options) ? @klass.attribute_options[field] || {} : {}
|
|
185
|
+
dimension = metadata[:num_dim] || options[:num_dim]
|
|
186
|
+
return if dimension.nil? || query.size == Integer(dimension)
|
|
187
|
+
|
|
188
|
+
raise SearchEngine::Errors::InvalidVectorQuery.new(
|
|
189
|
+
"InvalidVectorQuery: field :#{field} expects #{dimension} dimensions (got #{query.size})",
|
|
190
|
+
doc: VECTOR_SEARCH_DOC_URL,
|
|
191
|
+
details: { field: field, expected: dimension, actual: query.size }
|
|
192
|
+
)
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
def validate_vector_id!(id)
|
|
196
|
+
return if id.nil?
|
|
197
|
+
|
|
198
|
+
value = id.to_s
|
|
199
|
+
return if !value.empty? && value == value.strip && !value.match?(/[:,()[:cntrl:]]/)
|
|
200
|
+
|
|
201
|
+
raise SearchEngine::Errors::InvalidVectorQuery.new(
|
|
202
|
+
'InvalidVectorQuery: id: cannot be blank or contain vector-query delimiters or surrounding whitespace',
|
|
203
|
+
doc: VECTOR_SEARCH_DOC_URL
|
|
204
|
+
)
|
|
205
|
+
end
|
|
206
|
+
|
|
167
207
|
def validate_vector_queries!(queries)
|
|
168
208
|
return if queries.nil?
|
|
169
209
|
|
|
@@ -183,6 +223,14 @@ module SearchEngine
|
|
|
183
223
|
)
|
|
184
224
|
end
|
|
185
225
|
|
|
226
|
+
if queries.any? { |query| query != query.strip || query.match?(/[:`[:cntrl:]]/) }
|
|
227
|
+
raise SearchEngine::Errors::InvalidVectorQuery.new(
|
|
228
|
+
'InvalidVectorQuery: queries: cannot contain unsupported delimiters or surrounding whitespace',
|
|
229
|
+
hint: 'Colons, backticks and control characters cannot be represented in historical queries',
|
|
230
|
+
doc: VECTOR_SEARCH_DOC_URL
|
|
231
|
+
)
|
|
232
|
+
end
|
|
233
|
+
|
|
186
234
|
blank_indices = queries.each_index.select { |i| queries[i].strip.empty? }
|
|
187
235
|
return if blank_indices.empty?
|
|
188
236
|
|
|
@@ -197,9 +245,9 @@ module SearchEngine
|
|
|
197
245
|
def validate_vector_weights!(weights, queries)
|
|
198
246
|
return if weights.nil?
|
|
199
247
|
|
|
200
|
-
unless weights.is_a?(Array) && weights.all? { |v|
|
|
248
|
+
unless weights.is_a?(Array) && weights.all? { |v| finite_vector_number?(v) }
|
|
201
249
|
raise SearchEngine::Errors::InvalidVectorQuery.new(
|
|
202
|
-
'InvalidVectorQuery: weights: must be an Array of
|
|
250
|
+
'InvalidVectorQuery: weights: must be an Array of finite real numbers',
|
|
203
251
|
doc: VECTOR_SEARCH_DOC_URL,
|
|
204
252
|
details: { weights_class: weights.class.name }
|
|
205
253
|
)
|
|
@@ -221,7 +269,7 @@ module SearchEngine
|
|
|
221
269
|
)
|
|
222
270
|
end
|
|
223
271
|
|
|
224
|
-
sum = weights.sum
|
|
272
|
+
sum = weights.sum(&:to_f)
|
|
225
273
|
tolerance = SearchEngine.config.embedding.weights_sum_tolerance
|
|
226
274
|
return if (sum - 1.0).abs <= tolerance
|
|
227
275
|
|
|
@@ -239,7 +287,7 @@ module SearchEngine
|
|
|
239
287
|
|
|
240
288
|
begin
|
|
241
289
|
fv = Float(alpha)
|
|
242
|
-
rescue ArgumentError, TypeError
|
|
290
|
+
rescue ArgumentError, TypeError, RangeError
|
|
243
291
|
raise SearchEngine::Errors::InvalidVectorQuery.new(
|
|
244
292
|
"InvalidVectorQuery: alpha: must be a Numeric between 0.0 and 1.0 (got #{alpha.inspect})",
|
|
245
293
|
doc: VECTOR_SEARCH_DOC_URL,
|
|
@@ -262,7 +310,7 @@ module SearchEngine
|
|
|
262
310
|
|
|
263
311
|
begin
|
|
264
312
|
fv = Float(distance_threshold)
|
|
265
|
-
rescue ArgumentError, TypeError
|
|
313
|
+
rescue ArgumentError, TypeError, RangeError
|
|
266
314
|
raise SearchEngine::Errors::InvalidVectorQuery.new(
|
|
267
315
|
'InvalidVectorQuery: distance_threshold: must be a non-negative Numeric ' \
|
|
268
316
|
"(got #{distance_threshold.inspect})",
|
|
@@ -285,7 +333,7 @@ module SearchEngine
|
|
|
285
333
|
|
|
286
334
|
begin
|
|
287
335
|
iv = Integer(value)
|
|
288
|
-
rescue ArgumentError, TypeError
|
|
336
|
+
rescue ArgumentError, TypeError, RangeError
|
|
289
337
|
raise SearchEngine::Errors::InvalidVectorQuery.new(
|
|
290
338
|
"InvalidVectorQuery: #{name}: must be a positive Integer (got #{value.inspect})",
|
|
291
339
|
doc: VECTOR_SEARCH_DOC_URL,
|
|
@@ -293,10 +341,10 @@ module SearchEngine
|
|
|
293
341
|
)
|
|
294
342
|
end
|
|
295
343
|
|
|
296
|
-
return iv if iv.positive?
|
|
344
|
+
return iv if iv.positive? && (value.is_a?(String) || value == iv)
|
|
297
345
|
|
|
298
346
|
raise SearchEngine::Errors::InvalidVectorQuery.new(
|
|
299
|
-
"InvalidVectorQuery: #{name}: must be
|
|
347
|
+
"InvalidVectorQuery: #{name}: must be a positive Integer without truncation",
|
|
300
348
|
doc: VECTOR_SEARCH_DOC_URL,
|
|
301
349
|
details: { name => iv }
|
|
302
350
|
)
|
|
@@ -342,8 +390,6 @@ module SearchEngine
|
|
|
342
390
|
return {} unless @klass.respond_to?(:embeddings_config)
|
|
343
391
|
|
|
344
392
|
@klass.embeddings_config || {}
|
|
345
|
-
rescue StandardError
|
|
346
|
-
{}
|
|
347
393
|
end
|
|
348
394
|
|
|
349
395
|
def build_vector_field_suggestion(suggestions)
|
|
@@ -548,8 +548,7 @@ module SearchEngine
|
|
|
548
548
|
ORDER_ALIASES = { 'text_match' => '_text_match', 'vector_distance' => '_vector_distance' }.freeze
|
|
549
549
|
|
|
550
550
|
# Map DSL sort aliases to Typesense internal tokens.
|
|
551
|
-
# `_vector_distance`
|
|
552
|
-
# embedding field name is known.
|
|
551
|
+
# `_vector_distance` sorts the active vector query without re-embedding it.
|
|
553
552
|
def resolve_order_alias(field)
|
|
554
553
|
ORDER_ALIASES.fetch(field, field)
|
|
555
554
|
end
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require 'json'
|
|
4
|
+
require 'shellwords'
|
|
4
5
|
|
|
5
6
|
module SearchEngine
|
|
6
7
|
class Relation
|
|
@@ -36,13 +37,17 @@ module SearchEngine
|
|
|
36
37
|
end
|
|
37
38
|
end
|
|
38
39
|
|
|
39
|
-
# Build a
|
|
40
|
+
# Build a shell-quoted GET search command with redacted URL parameters.
|
|
40
41
|
# @param url [String]
|
|
41
42
|
# @param params [Hash]
|
|
43
|
+
# @param url_opts [Hash] request-level cache controls
|
|
42
44
|
# @return [String]
|
|
43
|
-
def self.curl(url, params)
|
|
44
|
-
|
|
45
|
-
|
|
45
|
+
def self.curl(url, params, url_opts: {})
|
|
46
|
+
query = redact_params(params).reject { |key, _value| key == :_hits }
|
|
47
|
+
query.merge!(SearchEngine::Observability.filtered_url_opts(url_opts))
|
|
48
|
+
arguments = ['curl', '--get', url, '-H', 'X-TYPESENSE-API-KEY: ***']
|
|
49
|
+
query.each { |key, value| arguments.concat(['--data-urlencode', "#{key}=#{value}"]) }
|
|
50
|
+
Shellwords.join(arguments)
|
|
46
51
|
end
|
|
47
52
|
|
|
48
53
|
# Return a structured dry-run payload with redacted JSON body and url options.
|
|
@@ -11,7 +11,7 @@ module SearchEngine
|
|
|
11
11
|
# DX helpers are mixed into `Relation` to offer redaction‑aware, zero‑I/O explain and preview utilities.
|
|
12
12
|
# These helpers are pure and never mutate relation state.
|
|
13
13
|
module Dx
|
|
14
|
-
# Return the
|
|
14
|
+
# Return the compiled search parameters as fully redacted JSON.
|
|
15
15
|
# @param pretty [Boolean] pretty-print with stable key ordering when true
|
|
16
16
|
# @return [String]
|
|
17
17
|
# @since M8
|
|
@@ -22,14 +22,14 @@ module SearchEngine
|
|
|
22
22
|
SearchEngine::Relation::Dx::DryRun.to_json(redacted, pretty: pretty)
|
|
23
23
|
end
|
|
24
24
|
|
|
25
|
-
# Return a
|
|
25
|
+
# Return a shell-quoted GET curl command with redacted API key and parameters.
|
|
26
26
|
# @return [String]
|
|
27
27
|
# @since M8
|
|
28
28
|
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/dx
|
|
29
29
|
def to_curl
|
|
30
30
|
url = compiled_url
|
|
31
31
|
params = SearchEngine::CompiledParams.from(to_typesense_params).to_h
|
|
32
|
-
SearchEngine::Relation::Dx::DryRun.curl(url, params)
|
|
32
|
+
SearchEngine::Relation::Dx::DryRun.curl(url, params, url_opts: compiled_url_opts)
|
|
33
33
|
end
|
|
34
34
|
|
|
35
35
|
# Compile and validate without performing network I/O.
|
data/lib/search_engine/result.rb
CHANGED
|
@@ -96,6 +96,7 @@ module SearchEngine
|
|
|
96
96
|
attach_curation!(obj, entry)
|
|
97
97
|
attach_highlighting!(obj, entry)
|
|
98
98
|
attach_geo_distance!(obj, entry)
|
|
99
|
+
attach_vector_distance!(obj, entry)
|
|
99
100
|
hydrated << obj
|
|
100
101
|
end
|
|
101
102
|
@hits = hydrated.freeze
|
|
@@ -296,6 +297,14 @@ module SearchEngine
|
|
|
296
297
|
end
|
|
297
298
|
end
|
|
298
299
|
|
|
300
|
+
# Backend vector distance metadata, distinct from document attributes.
|
|
301
|
+
module VectorDistance
|
|
302
|
+
# @return [Numeric, nil] the unmodified distance supplied by Typesense
|
|
303
|
+
def vector_distance
|
|
304
|
+
instance_variable_get(:@__se_vector_distance__)
|
|
305
|
+
end
|
|
306
|
+
end
|
|
307
|
+
|
|
299
308
|
# Per-hit curation mixin: added onto hydrated objects when Typesense
|
|
300
309
|
# returns curated metadata for the hit.
|
|
301
310
|
module HitCuration
|
|
@@ -414,6 +423,7 @@ module SearchEngine
|
|
|
414
423
|
attach_curation!(obj, sym_sub)
|
|
415
424
|
attach_highlighting!(obj, sym_sub)
|
|
416
425
|
attach_geo_distance!(obj, sym_sub)
|
|
426
|
+
attach_vector_distance!(obj, sym_sub)
|
|
417
427
|
hydrated << obj
|
|
418
428
|
end
|
|
419
429
|
|
|
@@ -569,6 +579,14 @@ module SearchEngine
|
|
|
569
579
|
obj
|
|
570
580
|
end
|
|
571
581
|
|
|
582
|
+
def attach_vector_distance!(obj, hit_entry)
|
|
583
|
+
return obj unless hit_entry.key?(:vector_distance)
|
|
584
|
+
|
|
585
|
+
obj.singleton_class.prepend(VectorDistance)
|
|
586
|
+
obj.instance_variable_set(:@__se_vector_distance__, hit_entry[:vector_distance])
|
|
587
|
+
obj
|
|
588
|
+
end
|
|
589
|
+
|
|
572
590
|
def attach_geo_distance!(obj, hit_entry)
|
|
573
591
|
raw_geo = hit_entry[:geo_distance_meters]
|
|
574
592
|
return obj unless raw_geo.is_a?(Hash) && !raw_geo.empty?
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: search-engine-for-typesense
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 30.
|
|
4
|
+
version: 30.9.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Nikita Shkoda
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-
|
|
11
|
+
date: 2026-11-02 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: concurrent-ruby
|