spree-typesense 6.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,901 @@
1
+ module Spree
2
+ module SearchProvider
3
+ class Typesense < Base
4
+ # Typesense rejected some documents of an import, which it reports per
5
+ # document rather than as an HTTP error.
6
+ class ImportError < StandardError
7
+ # The message names this many ids; #failures keeps them all.
8
+ MAX_IDS_IN_MESSAGE = 50
9
+
10
+ # @return [Array<Hash>] `{ id:, error: }` per rejected document
11
+ attr_reader :failures
12
+
13
+ # @param collection_name [String]
14
+ # @param failures [Array<Hash>] `{ id:, error: }` per rejected document
15
+ def initialize(collection_name, failures)
16
+ @failures = failures
17
+ ids = failures.map { |f| f[:id] }
18
+ shown = ids.first(MAX_IDS_IN_MESSAGE).join(', ')
19
+ more = ids.size > MAX_IDS_IN_MESSAGE ? " and #{ids.size - MAX_IDS_IN_MESSAGE} more" : ''
20
+ super("#{ids.size} #{ids.size == 1 ? 'document' : 'documents'} rejected by Typesense " \
21
+ "collection '#{collection_name}': #{shown}#{more}")
22
+ end
23
+ end
24
+
25
+ # A search Typesense answered with an error body instead of an HTTP
26
+ # error, as multi_search does per search.
27
+ class SearchError < ::Typesense::Error; end
28
+
29
+ # The network errors typesense-ruby 5.0's ApiCall retries on and, once
30
+ # its retries are spent, raises as they are.
31
+ NETWORK_ERRORS = [
32
+ Errno::EINVAL, Errno::ENETDOWN, Errno::ENETUNREACH, Errno::ENETRESET, Errno::ECONNABORTED,
33
+ Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::ECONNREFUSED, Errno::EHOSTDOWN, Errno::EHOSTUNREACH
34
+ ].freeze
35
+
36
+ # What the Typesense client raises when it cannot answer: its own errors
37
+ # (HTTP 4xx/5xx, timeouts, missing configuration, SearchError), Faraday's,
38
+ # the network errors above, and a response body that is not JSON. The
39
+ # public search entry points answer from the Database provider on any of
40
+ # these. The base class is ::Typesense::Error; there is no
41
+ # ::Typesense::Error::Error.
42
+ UNAVAILABLE_ERRORS = [::Typesense::Error, Faraday::Error, *NETWORK_ERRORS, JSON::ParserError].freeze
43
+
44
+ # The failures that pass by themselves: timeouts, 5xx, lost connections.
45
+ # These are reported as warnings. The rest (a rejected query from schema
46
+ # drift, a missing collection, a bad API key, missing configuration) will
47
+ # not pass without someone acting, and are reported as errors, so a broken
48
+ # index does not go unnoticed while the Database provider answers.
49
+ TRANSIENT_ERRORS = [
50
+ ::Typesense::Error::TimeoutError, ::Typesense::Error::ServerError, ::Typesense::Error::HTTPStatus0Error,
51
+ Faraday::Error, *NETWORK_ERRORS, JSON::ParserError
52
+ ].freeze
53
+
54
+ # Raised by #request_typesense in place of a failure in UNAVAILABLE_ERRORS,
55
+ # so the search entry points fall back on Typesense failures only, and
56
+ # never on an error raised around the request (the Database provider's,
57
+ # or the authorized scope's), whatever its class.
58
+ class Unavailable < StandardError
59
+ attr_reader :error
60
+
61
+ def initialize(error)
62
+ @error = error
63
+ super("#{error.class}: #{error.message}")
64
+ end
65
+ end
66
+ private_constant :Unavailable
67
+
68
+ # Reported once when the circuit breaker opens: Typesense has failed
69
+ # transiently several times in a row, and searches skip it for a while.
70
+ class CircuitOpened < StandardError; end
71
+
72
+ # Set once missing configuration has been logged; see #report_unavailable.
73
+ UNCONFIGURED_LOGGED = Concurrent::AtomicBoolean.new(false)
74
+
75
+ # 'manual' is the merchant's own order, as the Database provider offers
76
+ # it (Taxon/Collection::SORT_ORDERS). On Spree 6 a manually sorted
77
+ # category or collection listing sorts by the product's position in it
78
+ # (QueryBuilder.sort_by).
79
+ AVAILABLE_SORT_OPTIONS = %w[manual price -price name -name -available_on available_on best_selling].freeze
80
+
81
+ # Spree's Spree::Api::V3::ProductFilterSortOptionSerializer declares
82
+ # `attributes :id, :label` and raises KeyError when :label is absent.
83
+ #
84
+ # NOTE: core also appends custom-field sort options
85
+ # (`custom_field_schema.sort_options`). We do not yet surface those, so a
86
+ # store using custom-field sorts will not see them under this provider.
87
+ def sort_options_payload
88
+ AVAILABLE_SORT_OPTIONS.map { |id| { id: id, label: nil } }
89
+ end
90
+ DEFAULT_SORT = 'manual'.freeze
91
+
92
+ # Typesense defaults to 10 facet values per field and truncates silently.
93
+ # Needed by option/category facets; the price range reads stats instead.
94
+ MAX_FACET_VALUES = 100
95
+
96
+ # See #schema_client.
97
+ SCHEMA_CHANGE_TIMEOUT_SECONDS = 120
98
+
99
+ # See #search_client. TYPESENSE_SEARCH_TIMEOUT overrides it.
100
+ SEARCH_TIMEOUT_SECONDS = 2
101
+
102
+ def self.indexing_required?
103
+ true
104
+ end
105
+
106
+ # Brings the collection the live name serves from up to
107
+ # Schema.definition in place, adding the fields it lacks. With no index
108
+ # at all it does nothing: only #reindex creates one, so a store never
109
+ # serves an index that holds part of its catalogue without falling back.
110
+ def ensure_index_settings!
111
+ name = live_collection_name
112
+ update_collection_schema!(name) if name
113
+ nil
114
+ end
115
+
116
+ # Mirrors meilisearch.rb#reindex: default scope is store.products (not
117
+ # Spree::Product.where(store_id:)), batch size 500, deterministic order,
118
+ # lazy-preloaded associations. Returns the number of DOCUMENTS indexed
119
+ # (not products — one product can emit N locale x currency documents).
120
+ #
121
+ # Builds a fresh versioned collection from Schema.definition, marked
122
+ # `__building` while it fills so single-product writes reach it too,
123
+ # then points the live alias at it and drops what it replaced: the
124
+ # older versioned collections and a real collection holding the live
125
+ # name (0.2.0's, or one a rollback to 0.2.0 created). Searches answer
126
+ # from the previous collection until the swap, and documents of
127
+ # products that are gone or out of scope go with the old collection.
128
+ #
129
+ # The scope becomes the whole index: pass every product the store
130
+ # should find (the rake task passes store.products). To refresh some
131
+ # products, use #index for each.
132
+ #
133
+ # A batch with rejected documents does not stop the reindex, and those
134
+ # documents are not counted. Once every batch is sent, an ImportError
135
+ # names them all, so `rake spree:search:reindex` fails visibly; the new
136
+ # collection is then dropped and the live alias left where it was, so
137
+ # no product goes missing from search. Any other error does the same.
138
+ def reindex(scope = nil)
139
+ scope ||= store.products
140
+ layout = index_layout
141
+ building = layout.new_version_name(current_time_ms)
142
+ client.collections.create(Schema.definition(building))
143
+ layout.point!(layout.building_name, building)
144
+
145
+ begin
146
+ indexed, failures = index_scope(scope, into: building)
147
+ rescue StandardError
148
+ abandon_build(layout, building)
149
+ raise
150
+ end
151
+
152
+ Rails.logger.info do
153
+ "[spree_typesense] reindex complete: #{indexed} documents indexed, #{failures.size} failed"
154
+ end
155
+ if failures.any?
156
+ abandon_build(layout, building)
157
+ raise ImportError.new(building, failures)
158
+ end
159
+
160
+ swap_live_alias(layout, building)
161
+ indexed
162
+ end
163
+
164
+ # Upserts the product's documents, then deletes those of its documents
165
+ # it no longer produces: one product is a document per locale x currency,
166
+ # and a product that lost a currency's price, say, must not stay
167
+ # searchable in that market. A product that produces none loses them all.
168
+ #
169
+ # Every document is stamped `indexed_at` with the moment the product was
170
+ # read, and only documents stamped earlier are deleted, so when two jobs
171
+ # for one product overlap, the older one finishing last cannot drop what
172
+ # the newer one wrote (#9).
173
+ #
174
+ # Writes to the live collection and, while a reindex is filling one, to
175
+ # that one too, so the change survives the swap. A collection being
176
+ # built that has since gone (its reindex died) is skipped.
177
+ #
178
+ # Raises ImportError (after the deletion) when Typesense rejects a
179
+ # document, so the index job retries it.
180
+ def index(product)
181
+ read_at = current_time_ms
182
+ documents = Array(presenter_class.new(product, store).call)
183
+
184
+ results = nil
185
+ rejected = nil
186
+ each_write_target do |target, live|
187
+ begin
188
+ batch_results = index_batch(documents, into: target, indexed_at: read_at)
189
+ results = batch_results if live
190
+ rescue ImportError => e
191
+ rejected ||= e
192
+ end
193
+ remove_stale_documents(product.prefixed_id, documents, from: target, written_before: read_at)
194
+ end
195
+ raise rejected if rejected
196
+
197
+ results
198
+ end
199
+
200
+ # Typesense answers an import with HTTP 200 and one result per document,
201
+ # in order, so a rejected document raises nothing by itself. Each
202
+ # rejection is logged with its document id, and once the whole batch is
203
+ # sent an ImportError names them all.
204
+ #
205
+ # @param documents [Array<Hash>]
206
+ # @param into [String] the collection or alias to import into
207
+ # @param indexed_at [Integer] when the products were read, epoch ms
208
+ # @return [Array<Hash>, nil] Typesense's per-document results
209
+ # @raise [ImportError] when Typesense rejected any document
210
+ def index_batch(documents, into: collection_name, indexed_at: current_time_ms)
211
+ return if documents.blank?
212
+
213
+ documents = documents.map { |d| derive_fields(d, indexed_at) }
214
+ results = client.collections[into].documents.import(documents, action: 'upsert')
215
+
216
+ failures = import_failures(documents, results, into)
217
+ raise ImportError.new(into, failures) if failures.any?
218
+
219
+ results
220
+ end
221
+
222
+ def remove(product)
223
+ remove_by_id(product.prefixed_id)
224
+ end
225
+
226
+ # Public entry point. Answers from Typesense, or from the Database
227
+ # provider when Typesense cannot answer: unreachable, timed out, a 5xx,
228
+ # a rejected query (a 400 from a field the collection lacks, a missing
229
+ # collection), or not configured. Correct results, more slowly, beat
230
+ # an empty page. Only the Typesense request is covered: errors raised
231
+ # around it, by the Database provider or the authorized scope, still
232
+ # raise, as do errors that are not Typesense failing.
233
+ def search_and_filter(scope:, query: nil, filters: {}, sort: nil, page: 1, limit: 25)
234
+ fallback = lambda do
235
+ fallback_provider.search_and_filter(
236
+ scope: scope, query: query, filters: filters, sort: sort, page: page, limit: limit
237
+ )
238
+ end
239
+ answer_through_breaker(query, fallback) do
240
+ search_with_typesense(scope: scope, query: query, filters: filters, sort: sort, page: page, limit: limit)
241
+ end
242
+ end
243
+
244
+ # Raises Unavailable when Typesense cannot answer; #search_and_filter
245
+ # falls back. The intersection with the authorized scope runs outside
246
+ # #request_typesense, so its errors are never taken for Typesense's.
247
+ def search_with_typesense(scope:, query: nil, filters: {}, sort: nil, page: 1, limit: 25)
248
+ page = [page.to_i, 1].max
249
+ limit = limit.to_i.clamp(1, 100)
250
+
251
+ base, grouped = build_conditions(filters)
252
+ option_conditions, exact = QueryBuilder.option_conditions(grouped)
253
+ result = request_typesense do
254
+ execute_search(query: query, conditions: base + option_conditions, sort: sort, page: page, limit: limit,
255
+ filters: filters)
256
+ end
257
+
258
+ product_ids = result['hits'].map { |h| h['document']['product_id'] }.uniq
259
+ raw_ids = product_ids.filter_map { |pid| Spree::Product.decode_prefixed_id(pid) }
260
+
261
+ # Values of two or more option types must sit on ONE variant, as core's
262
+ # `Product.with_option_value_ids` decides. Documents carry combination
263
+ # tokens that answer it exactly. Where they cannot (more types than the
264
+ # tokens cover), the index only pre-filters and core's scope gives the
265
+ # exact answer, so a page can come back short of `limit` and the total
266
+ # over-count.
267
+ scope = scope.with_option_value_ids(grouped.values.flatten) unless exact
268
+
269
+ # SECURITY: intersect with the authorized scope, preserving engine order.
270
+ products = if raw_ids.any?
271
+ records = scope.where(id: raw_ids).reorder(nil).index_by(&:id)
272
+ raw_ids.filter_map { |id| records[id] }
273
+ else
274
+ scope.none
275
+ end
276
+
277
+ SearchResult.new(products: products, total_count: result['found'] || 0,
278
+ pagy: build_pagy(result['found'] || 0, page, limit))
279
+ end
280
+
281
+ # Facets from Typesense, or, when Typesense cannot answer, from the
282
+ # Database provider, which computes the same ones from the database.
283
+ # As in #search_and_filter, only the Typesense requests are covered.
284
+ def filters(scope:, query: nil, filters: {})
285
+ fallback = -> { fallback_provider.filters(scope: scope, query: query, filters: filters) }
286
+ answer_through_breaker(query, fallback) do
287
+ result, distribution, stats = execute_faceted_search(query: query, filters: filters)
288
+
289
+ FiltersResult.new(
290
+ filters: Facets.build(distribution, currency: currency, stats: stats),
291
+ sort_options: sort_options_payload,
292
+ default_sort: default_sort_for(filters),
293
+ total_count: result['found'] || 0
294
+ )
295
+ end
296
+ end
297
+
298
+ # Composite ids mean one product maps to N documents (locale × currency),
299
+ # so delete by filter on product_id rather than by document id. From the
300
+ # live collection and any collection a reindex is filling.
301
+ def remove_by_id(prefixed_id)
302
+ each_write_target do |target, _live|
303
+ delete_product_documents(prefixed_id, from: target)
304
+ end
305
+ nil
306
+ end
307
+
308
+ # The client indexing and removal go through: background jobs, which can
309
+ # afford to wait, and retry once on a node that fails.
310
+ def client
311
+ @client ||= build_client(connection_timeout_seconds: 5)
312
+ end
313
+
314
+ # The client searches and facets go through. A shopper waits on these,
315
+ # so a hung node must not hold the request: a short timeout and no retry
316
+ # (the indexing client's 5s and one retry cost about 10s), after which
317
+ # the search is answered by the Database provider. The timeout is
318
+ # TYPESENSE_SEARCH_TIMEOUT seconds when that is a positive number.
319
+ def search_client
320
+ @search_client ||= build_client(connection_timeout_seconds: search_timeout_seconds, num_retries: 0)
321
+ end
322
+
323
+ # The live name searches and writes go through: since 0.3.0 an alias
324
+ # over the versioned collection the last reindex built (see
325
+ # IndexLayout).
326
+ def collection_name
327
+ @collection_name ||= "products_#{store.code}"
328
+ end
329
+
330
+ private
331
+
332
+ # Where this store's collections and aliases live.
333
+ def index_layout
334
+ @index_layout ||= IndexLayout.new(client, store.code)
335
+ end
336
+
337
+ def current_time_ms = Process.clock_gettime(Process::CLOCK_REALTIME, :millisecond)
338
+
339
+ # The collection the live name serves from: a real collection holding
340
+ # the name, which takes requests ahead of an alias, else the alias's
341
+ # target. Nil when there is no index.
342
+ def live_collection_name
343
+ layout = index_layout
344
+ layout.legacy_collection? ? layout.live_name : layout.alias_target(layout.live_name)
345
+ end
346
+
347
+ # Yields each collection a single product's write goes to, and whether
348
+ # it is the live one. A collection being built that is gone is skipped:
349
+ # its reindex died, and the next one replaces it.
350
+ def each_write_target
351
+ index_layout.write_targets.each_with_index do |target, position|
352
+ live = position.zero?
353
+ begin
354
+ yield target, live
355
+ rescue ::Typesense::Error::ObjectNotFound
356
+ raise if live
357
+
358
+ Rails.logger.warn { "[spree_typesense] skipped '#{target}': the collection a reindex was building is gone" }
359
+ end
360
+ end
361
+ end
362
+
363
+ # Indexes the scope in batches into `into`.
364
+ # @return [Array(Integer, Array<Hash>)] documents accepted, and the rejected ones
365
+ def index_scope(scope, into:)
366
+ indexed = 0
367
+ failures = []
368
+ scope.reorder(id: :asc).preload_associations_lazily.find_in_batches(batch_size: 500) do |batch|
369
+ read_at = current_time_ms
370
+ documents = batch.flat_map { |product| presenter_class.new(product, store).call }
371
+ next if documents.empty?
372
+
373
+ accepted = documents.size
374
+ begin
375
+ index_batch(documents, into: into, indexed_at: read_at)
376
+ rescue ImportError => e
377
+ failures.concat(e.failures)
378
+ accepted -= e.failures.size
379
+ end
380
+ indexed += accepted
381
+ Rails.logger.info { "[spree_typesense] indexed #{accepted} (#{indexed} total) into #{into}" }
382
+ end
383
+ [indexed, failures]
384
+ end
385
+
386
+ # Forgets a reindex's collection: removes the marker and drops it,
387
+ # leaving the live alias as it was. Best effort, so the error that
388
+ # stopped the reindex is the one raised.
389
+ def abandon_build(layout, building)
390
+ layout.unpoint!(layout.building_name)
391
+ layout.drop!(building)
392
+ rescue StandardError => e
393
+ Rails.logger.error { "[spree_typesense] could not clean up '#{building}': #{e.class}: #{e.message}" }
394
+ end
395
+
396
+ # Points the live alias at the collection just built, then drops what
397
+ # it replaced. Typesense refuses an alias whose name a collection holds,
398
+ # so a real collection holding the live name is dropped first; searches
399
+ # in that moment get a 404 and answer from the Database provider.
400
+ # Pointing the alias is retried once: after that drop, failing it would
401
+ # leave the store with no index. Once it points at the new collection
402
+ # the reindex has succeeded, so the clean-up after it only logs what it
403
+ # could not do. Older versioned collections go, newer ones stay: a newer
404
+ # one belongs to a reindex still running, which will drop this one when
405
+ # it swaps.
406
+ def swap_live_alias(layout, building)
407
+ previous = layout.alias_target(layout.live_name)
408
+ legacy = layout.legacy_collection?
409
+ layout.drop!(layout.live_name) if legacy
410
+ point_live_alias(layout, building)
411
+
412
+ after_swap("remove the #{layout.building_name} alias") { layout.unpoint!(layout.building_name) }
413
+ replaced = after_swap('list the collections it replaced') do
414
+ layout.versioned_collections.select { |name| name < building }
415
+ end || []
416
+ dropped = replaced.count { |name| after_swap("drop #{name}") { layout.drop!(name) } }
417
+
418
+ Rails.logger.info do
419
+ was = [(layout.live_name if legacy), previous].compact.uniq.join(', ')
420
+ "[spree_typesense] #{layout.live_name} now serves #{building}#{" (was #{was})" unless was.empty?}; " \
421
+ "dropped #{dropped} older collection(s)#{' and the real collection holding the live name' if legacy}"
422
+ end
423
+ end
424
+
425
+ def point_live_alias(layout, building)
426
+ attempts = 0
427
+ begin
428
+ layout.point!(layout.live_name, building)
429
+ rescue StandardError => e
430
+ attempts += 1
431
+ raise if attempts > 1
432
+
433
+ Rails.logger.warn { "[spree_typesense] pointing #{layout.live_name} at #{building} failed (#{e.message}); retrying" }
434
+ retry
435
+ end
436
+ end
437
+
438
+ # Clean-up once the live alias points at the new collection: the
439
+ # reindex has succeeded, so a failure here is logged for the next
440
+ # reindex to finish, not raised. Returns the block's value (truthy for
441
+ # a drop), or nil when it failed.
442
+ def after_swap(what)
443
+ yield || true
444
+ rescue StandardError => e
445
+ Rails.logger.warn do
446
+ "[spree_typesense] the swap succeeded, but could not #{what} (#{e.class}: #{e.message}); " \
447
+ 'the next reindex will'
448
+ end
449
+ nil
450
+ end
451
+
452
+ # The rejected documents of an import, logged one by one. A result names
453
+ # its document only when the import asked for that, so fall back to the
454
+ # position: Typesense answers one result per document, in order.
455
+ def import_failures(documents, results, into = collection_name)
456
+ Array(results).each_with_index.filter_map do |result, index|
457
+ next if result['success']
458
+
459
+ failure = { id: result['id'] || documents.dig(index, 'id'), error: result['error'] }
460
+ Rails.logger.error do
461
+ "[spree_typesense] Typesense rejected document #{failure[:id]} in collection " \
462
+ "'#{into}': #{failure[:error]}"
463
+ end
464
+ failure
465
+ end
466
+ end
467
+
468
+ # Deletes the product's documents in `from` other than `documents`; all
469
+ # of them when `documents` is empty. Only documents written before
470
+ # `written_before` go: a later job's are its own business. Ids go into
471
+ # the filter quoted in backticks, Typesense's string literal, because
472
+ # locales such as `en-US` put a hyphen in them. An id that itself holds a
473
+ # backtick cannot be quoted, so nothing is deleted then rather than risk
474
+ # deleting the wrong documents.
475
+ def remove_stale_documents(prefixed_id, documents, from:, written_before:)
476
+ kept_ids = documents.map { |d| (d[:id] || d['id']).to_s }
477
+ return delete_product_documents(prefixed_id, from: from, written_before: written_before) if kept_ids.empty?
478
+
479
+ if kept_ids.any? { |id| id.include?('`') }
480
+ Rails.logger.warn { "[spree_typesense] not removing stale documents of #{prefixed_id}: an id holds a backtick" }
481
+ return
482
+ end
483
+
484
+ client.collections[from].documents.delete(
485
+ filter_by: "product_id:=#{sanitize_prefixed_id(prefixed_id)} && " \
486
+ "id:!=[#{kept_ids.map { |id| "`#{id}`" }.join(',')}] && indexed_at:<#{written_before.to_i}"
487
+ )
488
+ rescue ::Typesense::Error::ObjectNotFound
489
+ nil
490
+ end
491
+
492
+ # Nothing to remove from a collection that does not exist. With
493
+ # `written_before`, only documents stamped earlier.
494
+ def delete_product_documents(prefixed_id, from:, written_before: nil)
495
+ filter = "product_id:=#{sanitize_prefixed_id(prefixed_id)}"
496
+ filter += " && indexed_at:<#{written_before.to_i}" if written_before
497
+ client.collections[from].documents.delete(filter_by: filter)
498
+ rescue ::Typesense::Error::ObjectNotFound
499
+ nil
500
+ end
501
+
502
+ # The Typesense request; callers wrap it in #request_typesense. Sent as
503
+ # a one-search multi_search, a POST: a GET search carries its filter in
504
+ # the query string, which Typesense caps at 4000 characters, and a
505
+ # selection across option types can make a far longer filter.
506
+ def execute_search(query:, conditions:, sort:, page:, limit:, facet_by: nil, filters: {})
507
+ perform_searches(
508
+ [search_params(query: query, conditions: conditions, sort: sort, page: page, limit: limit,
509
+ facet_by: facet_by, filters: filters)]
510
+ ).first
511
+ end
512
+
513
+ # Faceted read path. Returns [main_result, distribution, stats], and
514
+ # raises Unavailable when Typesense cannot answer.
515
+ # `stats` always comes from the MAIN query — it covers numeric fields
516
+ # (price), which disjunctive relaxation never touches.
517
+ #
518
+ # DISJUNCTIVE FACETING: with option values selected, the counts for each
519
+ # selected option type must be computed with THAT type's own filter
520
+ # removed — otherwise picking "Red" reports every other colour as zero
521
+ # and the shopper can never widen their choice. That needs one extra
522
+ # query per selected type, so it goes through multi_search. With nothing
523
+ # selected there is nothing to relax, so we stay on the single-query path
524
+ # rather than pay for a multi_search on every request.
525
+ def execute_faceted_search(query:, filters:)
526
+ base, grouped = build_conditions(filters)
527
+ facet_by = Schema::FACET_FIELDS.join(',')
528
+
529
+ if grouped.empty?
530
+ result = request_typesense do
531
+ execute_search(query: query, conditions: base, sort: nil, page: 1, limit: 0, facet_by: facet_by)
532
+ end
533
+ return [result, Facets.normalize(result['facet_counts']), Facets.normalize_stats(result['facet_counts'])]
534
+ end
535
+
536
+ option_type_ids = grouped.keys
537
+ searches = [
538
+ search_params(query: query, conditions: base + QueryBuilder.option_conditions(grouped).first,
539
+ sort: nil, page: 1, limit: 0, facet_by: facet_by)
540
+ ]
541
+ option_type_ids.each do |option_type_id|
542
+ relaxed = QueryBuilder.option_conditions(grouped.except(option_type_id)).first
543
+ searches << search_params(query: query, conditions: base + relaxed,
544
+ sort: nil, page: 1, limit: 0, facet_by: 'option_value_ids')
545
+ end
546
+
547
+ results = request_typesense { execute_multi_search(searches) }
548
+ [results.first,
549
+ merge_disjunctive_facets(results.first, results.drop(1), option_type_ids),
550
+ Facets.normalize_stats(results.first['facet_counts'])]
551
+ end
552
+
553
+ # The disjunctive facet searches, in one request.
554
+ def execute_multi_search(searches) = perform_searches(searches)
555
+
556
+ # Typesense answers multi_search with HTTP 200 even when an individual
557
+ # search failed — the failure is a per-result {error:, code:} body, NOT an
558
+ # exception. Raise it here as the error a search would have raised for
559
+ # that status, so a broken filter cannot silently serve wrong results or
560
+ # counts, and a 5xx counts as transient.
561
+ def perform_searches(searches)
562
+ response = search_client.multi_search.perform(
563
+ { searches: searches.map { |s| s.merge(collection: collection_name) } }
564
+ )
565
+ results = response['results']
566
+
567
+ failed = Array(results).find { |r| r['error'] }
568
+ raise search_error_for(failed) if failed
569
+
570
+ results
571
+ end
572
+
573
+ PER_RESULT_ERRORS = {
574
+ 400 => ::Typesense::Error::RequestMalformed,
575
+ 401 => ::Typesense::Error::RequestUnauthorized,
576
+ 403 => ::Typesense::Error::RequestUnauthorized,
577
+ 404 => ::Typesense::Error::ObjectNotFound,
578
+ 422 => ::Typesense::Error::ObjectUnprocessable
579
+ }.freeze
580
+ private_constant :PER_RESULT_ERRORS
581
+
582
+ def search_error_for(failed)
583
+ code = failed['code'].to_i
584
+ klass = PER_RESULT_ERRORS.fetch(code) { (500..599).cover?(code) ? ::Typesense::Error::ServerError : SearchError }
585
+ klass.new("multi_search failed: #{failed['code']} #{failed['error']}")
586
+ end
587
+
588
+ # Start from the main (fully filtered) distribution, then for each option
589
+ # type overwrite the counts of the values BELONGING TO THAT TYPE with the
590
+ # counts from that type's own relaxed query. Values of other types keep
591
+ # the main query's counts. Mapping a value to its type needs the database,
592
+ # same lookup as +group_option_values_by_type+.
593
+ def merge_disjunctive_facets(main_result, relaxed_results, option_type_ids)
594
+ distribution = Facets.normalize(main_result['facet_counts'])
595
+ merged = (distribution['option_value_ids'] || {}).dup
596
+
597
+ per_type = {}
598
+ relaxed_results.each_with_index do |result, i|
599
+ per_type[option_type_ids[i]] = Facets.normalize(result['facet_counts'])['option_value_ids'] || {}
600
+ end
601
+
602
+ type_of = option_type_of(per_type.values.flat_map(&:keys).uniq)
603
+ per_type.each do |option_type_id, relaxed_distribution|
604
+ relaxed_distribution.each do |prefixed_id, count|
605
+ merged[prefixed_id] = count if type_of[prefixed_id] == option_type_id
606
+ end
607
+ end
608
+
609
+ distribution.merge('option_value_ids' => merged)
610
+ end
611
+
612
+ # { 'ov_red' => option_type_id, ... } in one query.
613
+ def option_type_of(prefixed_ids)
614
+ raw_ids = prefixed_ids.filter_map { |pid| Spree::OptionValue.decode_prefixed_id(pid) }
615
+ by_raw_id = Spree::OptionValue.where(id: raw_ids).pluck(:id, :option_type_id).to_h
616
+
617
+ prefixed_ids.each_with_object({}) do |prefixed_id, acc|
618
+ raw_id = Spree::OptionValue.decode_prefixed_id(prefixed_id)
619
+ acc[prefixed_id] = by_raw_id[raw_id] if raw_id
620
+ end
621
+ end
622
+
623
+ # Returns [base_conditions, grouped_option_values]. Kept separate because
624
+ # disjunctive faceting has to rebuild the option half per query while the
625
+ # base half (system scoping + user filters) stays fixed.
626
+ def build_conditions(filters)
627
+ f = normalize_filters(filters)
628
+ option_ids = f.delete('with_option_value_ids')
629
+
630
+ base = QueryBuilder.system_conditions(store: store, locale: locale, currency: currency,
631
+ channel_id: current_channel&.id)
632
+ base += QueryBuilder.user_conditions(f)
633
+
634
+ [base, group_option_values_by_type(Array(option_ids))]
635
+ end
636
+
637
+ # The channel the request is served on, read as core's storefront scope
638
+ # and the Meilisearch provider read it on every supported version. Nil
639
+ # outside a channel context, and then nothing is narrowed.
640
+ def current_channel
641
+ Spree::Current.channel
642
+ end
643
+
644
+ # A string-keyed copy of the filters, whichever shape they arrive in:
645
+ # Parameters, a symbol- or string-keyed Hash, or nil.
646
+ def normalize_filters(filters)
647
+ f = filters.respond_to?(:to_unsafe_h) ? filters.to_unsafe_h : (filters || {}).to_h
648
+ f.transform_keys(&:to_s)
649
+ end
650
+
651
+ # The sort a listing starts on, as core's FiltersAggregator derives it:
652
+ # the collection's own sort order (categories are manual only), given in
653
+ # the API's form ('price desc' -> '-price'), and 'manual' otherwise.
654
+ def default_sort_for(filters)
655
+ collection = normalize_filters(filters)['_collection']
656
+ sort_order = collection.sort_order.presence if collection.respond_to?(:sort_order)
657
+ field, direction = (sort_order || DEFAULT_SORT).to_s.split(' ', 2)
658
+ direction == 'desc' ? "-#{field}" : field
659
+ end
660
+
661
+ def search_params(query:, conditions:, sort:, page:, limit:, facet_by: nil, filters: {})
662
+ params = {
663
+ q: query.presence || '*',
664
+ query_by: Schema::SEARCHABLE_FIELDS.join(','),
665
+ filter_by: conditions.join(' && '),
666
+ page: page,
667
+ per_page: limit
668
+ }
669
+ sort_by = QueryBuilder.sort_by(sort, normalize_filters(filters))
670
+ params[:sort_by] = sort_by if sort_by
671
+ if facet_by
672
+ params[:facet_by] = facet_by
673
+ # Typesense caps facet values at 10 per field by default and truncates
674
+ # SILENTLY. The price range no longer depends on this (it reads
675
+ # stats.min/max, which are exact at any cardinality), but option and
676
+ # category facets DO — their counts only exist as returned values, and
677
+ # string facets get no stats block. Meilisearch's equivalent default is
678
+ # 100; match it. Residual risk: an option type with more than 100
679
+ # values still loses its tail.
680
+ params[:max_facet_values] = MAX_FACET_VALUES
681
+ end
682
+ params
683
+ end
684
+
685
+ # Answers through Typesense unless the circuit breaker says it is
686
+ # failing, in which case `fallback` (the Database provider) answers at
687
+ # once, with nothing logged. A transient failure counts towards opening
688
+ # it, and the one that opens it is announced once; anything else
689
+ # Typesense answered, even with an error, counts as it answering.
690
+ # Missing configuration made no request and counts as neither. Errors
691
+ # that are not Typesense's propagate, and free the probe if this was it.
692
+ def answer_through_breaker(query, fallback)
693
+ breaker = circuit_breaker
694
+ return fallback.call unless breaker.allow_request?
695
+
696
+ begin
697
+ answer = yield
698
+ rescue Unavailable => e
699
+ record_unavailable(breaker, e.error)
700
+ report_unavailable(e.error, query)
701
+ return fallback.call
702
+ rescue StandardError
703
+ breaker.release_probe
704
+ raise
705
+ end
706
+
707
+ Rails.logger.info { '[spree_typesense] Typesense is answering again' } if breaker.record_success == :closed
708
+ answer
709
+ end
710
+
711
+ def record_unavailable(breaker, error)
712
+ if error.is_a?(::Typesense::Error::MissingConfiguration)
713
+ breaker.release_probe
714
+ elsif TRANSIENT_ERRORS.any? { |klass| error.is_a?(klass) }
715
+ announce_circuit_opened(breaker, error) if breaker.record_failure == :opened
716
+ elsif breaker.record_success == :closed
717
+ Rails.logger.info { '[spree_typesense] Typesense is answering again' }
718
+ end
719
+ end
720
+
721
+ def announce_circuit_opened(breaker, error)
722
+ message = "Typesense failed #{breaker.threshold} times in a row (last: #{error.class}: #{error.message}); " \
723
+ "skipping it for #{breaker.open_seconds.to_s.delete_suffix('.0')} seconds and answering searches " \
724
+ 'from the Database provider'
725
+ Rails.logger.error { "[spree_typesense] #{message}" }
726
+ Rails.error.report(CircuitOpened.new(message), handled: true, severity: :warning,
727
+ context: { collection: collection_name })
728
+ end
729
+
730
+ def circuit_breaker = Spree::Typesense::CircuitBreaker.for(ENV.fetch('TYPESENSE_URL', ''))
731
+
732
+ # Runs a Typesense request, raising Unavailable in place of any failure
733
+ # in UNAVAILABLE_ERRORS. Wrap only the request itself: whatever the block
734
+ # raises in that list is taken for Typesense being unavailable.
735
+ def request_typesense
736
+ yield
737
+ rescue *UNAVAILABLE_ERRORS => e
738
+ raise Unavailable, e
739
+ end
740
+
741
+ # Logs and reports a failure the caller is about to answer from the
742
+ # Database provider: as a warning when it is transient, and as an error
743
+ # when it needs someone to act (see TRANSIENT_ERRORS). Missing
744
+ # configuration is logged and reported once per process: it fails every
745
+ # search the same way until someone sets it.
746
+ #
747
+ # A manual listing of a grouping no indexed document has a position in
748
+ # yet (before the first reindex that writes them, or a grouping with no
749
+ # indexed product) is refused by Typesense for want of the field. That
750
+ # is expected and needs nobody: it is logged at info, not reported, and
751
+ # the Database provider, which sorts by position too, answers it.
752
+ def report_unavailable(error, query)
753
+ if error.is_a?(::Typesense::Error::MissingConfiguration)
754
+ return unless UNCONFIGURED_LOGGED.make_true
755
+ end
756
+ if unindexed_position_sort?(error)
757
+ Rails.logger.info do
758
+ "[spree_typesense] no indexed document has a position here yet (#{error.message}); answering this " \
759
+ 'manual listing from the Database provider'
760
+ end
761
+ return
762
+ end
763
+
764
+ Rails.logger.error do
765
+ "[spree_typesense] Typesense could not answer (#{error.class}: #{error.message}); answering from the " \
766
+ 'Database provider. If the collection is missing or out of date, run `rake spree:search:reindex`.'
767
+ end
768
+ severity = TRANSIENT_ERRORS.any? { |klass| error.is_a?(klass) } ? :warning : :error
769
+ Rails.error.report(error, handled: true, severity: severity,
770
+ context: { collection: collection_name, query: query })
771
+ end
772
+
773
+ def unindexed_position_sort?(error)
774
+ error.is_a?(::Typesense::Error::ObjectNotFound) && error.message.to_s.include?('field named `pos_')
775
+ end
776
+
777
+ # Typesense alters a live collection in place: PATCH /collections/:name
778
+ # with the fields to add. Documents already stored keep their values and
779
+ # gain the new fields when next indexed. So a field a release adds
780
+ # reaches the collection without dropping it, and the store keeps
781
+ # serving searches throughout.
782
+ #
783
+ # What Typesense cannot do in place is change a field's type. Never drop
784
+ # the collection here, which would empty a live index: name the field
785
+ # and the remedy instead. A reindex builds a fresh collection, which has
786
+ # the declared type.
787
+ #
788
+ # A failed PATCH raises, so the caller stops rather than carrying on
789
+ # against a collection that still cannot serve the new filters.
790
+ #
791
+ # @param name [String] a real collection's name, not an alias
792
+ def update_collection_schema!(name)
793
+ live_fields = client.collections[name].retrieve['fields']
794
+
795
+ missing = Schema.missing_fields(live_fields)
796
+ if missing.any?
797
+ schema_client.collections[name].update(fields: missing)
798
+ Rails.logger.info do
799
+ "[spree_typesense] added #{missing.map { |f| f[:name] }.join(', ')} to collection '#{name}'"
800
+ end
801
+ end
802
+
803
+ Schema.type_mismatches(live_fields).each do |mismatch|
804
+ Rails.logger.warn do
805
+ "[spree_typesense] collection '#{name}' declares '#{mismatch[:name]}' as " \
806
+ "#{mismatch[:live_type]}, but this release declares it #{mismatch[:type]}. Typesense cannot change " \
807
+ "a field's type in place: reindex (`rake spree:search:reindex`), which builds a fresh collection."
808
+ end
809
+ end
810
+ end
811
+
812
+ # Schema changes block writes to the collection until Typesense finishes
813
+ # them, and the response comes only then, so they need a far longer
814
+ # timeout than indexing. No retries: Typesense runs one alter at a time,
815
+ # and would refuse a retry while the first is still running.
816
+ def schema_client
817
+ @schema_client ||= build_client(connection_timeout_seconds: SCHEMA_CHANGE_TIMEOUT_SECONDS, num_retries: 0)
818
+ end
819
+
820
+ # Raises Typesense::Error::MissingConfiguration, rather than KeyError,
821
+ # when TYPESENSE_URL or TYPESENSE_API_KEY is unset or the URL does not
822
+ # parse, so a search treats it as Typesense being unavailable.
823
+ def build_client(**options)
824
+ url = ENV.fetch('TYPESENSE_URL', nil).presence
825
+ api_key = ENV.fetch('TYPESENSE_API_KEY', nil).presence
826
+ unless url && api_key
827
+ missing = { 'TYPESENSE_URL' => url, 'TYPESENSE_API_KEY' => api_key }.select { |_, v| v.nil? }.keys
828
+ raise ::Typesense::Error::MissingConfiguration, "#{missing.join(' and ')} not set"
829
+ end
830
+
831
+ uri = URI.parse(url)
832
+ ::Typesense::Client.new(
833
+ nodes: [{ host: uri.host, port: uri.port, protocol: uri.scheme }],
834
+ api_key: api_key,
835
+ **options
836
+ )
837
+ rescue URI::InvalidURIError => e
838
+ raise ::Typesense::Error::MissingConfiguration, "TYPESENSE_URL is not a valid URL: #{e.message}"
839
+ end
840
+
841
+ def search_timeout_seconds
842
+ configured = Float(ENV.fetch('TYPESENSE_SEARCH_TIMEOUT', nil), exception: false)
843
+ configured&.positive? ? configured : SEARCH_TIMEOUT_SECONDS
844
+ end
845
+
846
+ # Mirrors meilisearch.rb#group_option_values_by_type. Hits the DB.
847
+ def group_option_values_by_type(prefixed_ids)
848
+ ids = prefixed_ids.flatten.compact.select { |id| QueryBuilder.valid_id?(id) }
849
+ return {} if ids.empty?
850
+
851
+ raw = ids.filter_map { |id| Spree::OptionValue.decode_prefixed_id(id) }
852
+ Spree::OptionValue.where(id: raw).group_by(&:option_type_id)
853
+ .transform_values { |ovs| ovs.map(&:prefixed_id) }
854
+ end
855
+
856
+ # The presenter emits Spree's canonical document; Typesense needs fields
857
+ # Spree does not provide. Derive them here rather than forking the presenter.
858
+ def derive_fields(document, indexed_at = nil)
859
+ doc = document.transform_keys(&:to_s)
860
+ doc['indexed_at'] = indexed_at.to_i if indexed_at
861
+ raw = doc['available_on']
862
+ doc['available_on_i'] = raw.present? ? Time.parse(raw.to_s).to_i : 0
863
+ doc
864
+ rescue ArgumentError
865
+ doc['available_on_i'] = 0
866
+ doc
867
+ end
868
+
869
+ def build_pagy(count, page, limit)
870
+ Pagy::Offset.new(count: count, page: page, limit: limit)
871
+ end
872
+
873
+ # The Database provider takes the same `audience:` as this one, and uses
874
+ # it for its query and ransack authorisation, so a back-office search
875
+ # (audience: nil) must not degrade into a storefront one.
876
+ def fallback_provider
877
+ Spree::SearchProvider::Database.new(store, audience: audience)
878
+ end
879
+
880
+ # The gem's own presenter, unless the application named a subclass of
881
+ # it. Core configures none (`search_product_presenter: nil`); the engine
882
+ # registers ours, which counts as overridden.
883
+ #
884
+ # The setting is shared by every installed search gem, so anything that is
885
+ # not ours is ignored: with spree_meilisearch installed it may hold
886
+ # Meilisearch's presenter, whose documents this schema does not describe.
887
+ def presenter_class
888
+ configured = Spree::Dependencies.search_product_presenter_class if Spree::Dependencies.overridden?(:search_product_presenter)
889
+ configured.is_a?(Class) && configured <= Spree::Typesense::ProductPresenter ? configured : Spree::Typesense::ProductPresenter
890
+ end
891
+
892
+ # `remove_by_id` is public and reached via Spree's RemoveJob (ActiveJob
893
+ # serialisation, not a guaranteed well-formed id), so strip anything
894
+ # that isn't part of a legitimate prefixed id before it lands in a
895
+ # Typesense filter expression — mirrors the Meilisearch provider's guard.
896
+ def sanitize_prefixed_id(value)
897
+ value.to_s.gsub(/[^a-zA-Z0-9_]/, '')
898
+ end
899
+ end
900
+ end
901
+ end