search-engine-for-typesense 30.9.1 → 30.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +19 -0
- data/README.md +51 -16
- data/lib/search_engine/active_record_syncable.rb +85 -197
- data/lib/search_engine/ast/node.rb +3 -6
- data/lib/search_engine/async_partition_coordinator.rb +5 -3
- data/lib/search_engine/client/services/search.rb +2 -0
- data/lib/search_engine/client.rb +9 -3
- data/lib/search_engine/compiler.rb +33 -128
- data/lib/search_engine/deletion.rb +12 -9
- data/lib/search_engine/dsl/parser.rb +51 -71
- data/lib/search_engine/engine.rb +25 -16
- data/lib/search_engine/filters/sanitizer.rb +162 -126
- data/lib/search_engine/hydration/materializers.rb +53 -118
- data/{app/search_engine → lib}/search_engine/index_partition_job.rb +1 -1
- data/lib/search_engine/indexing_run_store/rails_cache.rb +44 -33
- data/lib/search_engine/indexing_run_store.rb +10 -0
- data/lib/search_engine/joins/guard.rb +14 -8
- data/lib/search_engine/multi.rb +92 -6
- data/lib/search_engine/multi_result.rb +40 -23
- data/lib/search_engine/notifications/compact_logger.rb +2 -2
- data/lib/search_engine/observability.rb +6 -6
- data/lib/search_engine/postgres_outbox/database_queries.rb +17 -0
- data/lib/search_engine/postgres_outbox/drainer.rb +7 -12
- data/lib/search_engine/postgres_outbox/event.rb +12 -0
- data/lib/search_engine/postgres_outbox/materialization_cursor.rb +25 -11
- data/lib/search_engine/postgres_outbox/repository.rb +65 -115
- data/lib/search_engine/postgres_outbox.rb +2 -0
- data/lib/search_engine/registry.rb +27 -28
- data/lib/search_engine/relation/compiler.rb +18 -33
- data/lib/search_engine/relation/dsl/filters.rb +71 -175
- data/lib/search_engine/relation/dx.rb +3 -1
- data/lib/search_engine/relation/materializers.rb +6 -3
- data/lib/search_engine/relation/options.rb +5 -26
- data/lib/search_engine/relation/state.rb +3 -2
- data/lib/search_engine/relation.rb +5 -4
- data/lib/search_engine/result.rb +33 -2
- data/lib/search_engine/schema.rb +23 -24
- data/lib/search_engine/test/offline_client.rb +2 -2
- data/lib/search_engine/test/stub_client.rb +9 -16
- data/lib/search_engine/update.rb +1 -1
- data/lib/search_engine/version.rb +1 -1
- data/lib/search_engine.rb +11 -149
- data/lib/tasks/search_engine.rake +3 -5
- metadata +18 -3
- /data/{app/search_engine → lib}/search_engine/postgres_outbox/drain_job.rb +0 -0
|
@@ -7,7 +7,7 @@ module SearchEngine
|
|
|
7
7
|
module IndexingRunStore
|
|
8
8
|
# Rails.cache-backed implementation for async partition indexing run metadata.
|
|
9
9
|
class RailsCache
|
|
10
|
-
# @param cache [Object, nil] cache
|
|
10
|
+
# @param cache [Object, nil] Rails-compatible cache responding to read/write/delete
|
|
11
11
|
def initialize(cache: nil)
|
|
12
12
|
@cache = cache || resolve_rails_cache!
|
|
13
13
|
validate_cache!
|
|
@@ -29,17 +29,17 @@ module SearchEngine
|
|
|
29
29
|
created_at: SearchEngine::IndexingRun.iso8601_now,
|
|
30
30
|
updated_at: SearchEngine::IndexingRun.iso8601_now
|
|
31
31
|
}
|
|
32
|
-
|
|
32
|
+
write!(meta_cache_key(run_id), run, ttl_s: ttl_s)
|
|
33
33
|
entries.each do |partition_key, entry|
|
|
34
|
-
|
|
34
|
+
write!(partition_cache_key(run_id, partition_key), entry, ttl_s: ttl_s)
|
|
35
35
|
end
|
|
36
36
|
snapshot(run_id: run_id)
|
|
37
37
|
end
|
|
38
38
|
|
|
39
39
|
# Mark a partition as started.
|
|
40
|
-
# @return [Hash]
|
|
41
|
-
def mark_started(run_id:, partition_key:, job_id: nil)
|
|
42
|
-
update_partition(run_id, partition_key) do |entry|
|
|
40
|
+
# @return [Hash, nil] run snapshot when requested
|
|
41
|
+
def mark_started(run_id:, partition_key:, job_id: nil, return_snapshot: true)
|
|
42
|
+
update_partition(run_id, partition_key, return_snapshot:) do |entry|
|
|
43
43
|
ensure_non_terminal!(entry, 'start')
|
|
44
44
|
entry[:status] = 'running'
|
|
45
45
|
entry[:attempts] = entry[:attempts].to_i + 1
|
|
@@ -49,9 +49,9 @@ module SearchEngine
|
|
|
49
49
|
|
|
50
50
|
# Record one failed import attempt without terminalizing the partition.
|
|
51
51
|
# ActiveJob may safely start the same partition again on retry.
|
|
52
|
-
# @return [Hash]
|
|
53
|
-
def record_attempt(run_id:, partition_key:, summary:, error:)
|
|
54
|
-
update_partition(run_id, partition_key) do |entry|
|
|
52
|
+
# @return [Hash, nil] run snapshot when requested
|
|
53
|
+
def record_attempt(run_id:, partition_key:, summary:, error:, return_snapshot: true)
|
|
54
|
+
update_partition(run_id, partition_key, return_snapshot:) do |entry|
|
|
55
55
|
ensure_non_terminal!(entry, 'record an attempt for')
|
|
56
56
|
assign_summary!(entry, summary)
|
|
57
57
|
entry[:status] = 'running'
|
|
@@ -60,9 +60,9 @@ module SearchEngine
|
|
|
60
60
|
end
|
|
61
61
|
|
|
62
62
|
# Mark a partition as succeeded with import counters.
|
|
63
|
-
# @return [Hash]
|
|
64
|
-
def mark_succeeded(run_id:, partition_key:, summary:)
|
|
65
|
-
update_partition(run_id, partition_key) do |entry|
|
|
63
|
+
# @return [Hash, nil] run snapshot when requested
|
|
64
|
+
def mark_succeeded(run_id:, partition_key:, summary:, return_snapshot: true)
|
|
65
|
+
update_partition(run_id, partition_key, return_snapshot:) do |entry|
|
|
66
66
|
ensure_non_terminal!(entry, 'succeed')
|
|
67
67
|
assign_summary!(entry, summary)
|
|
68
68
|
entry[:status] = 'succeeded'
|
|
@@ -71,9 +71,9 @@ module SearchEngine
|
|
|
71
71
|
end
|
|
72
72
|
|
|
73
73
|
# Mark a partition as failed.
|
|
74
|
-
# @return [Hash]
|
|
75
|
-
def mark_failed(run_id:, partition_key:, error:)
|
|
76
|
-
update_partition(run_id, partition_key) do |entry|
|
|
74
|
+
# @return [Hash, nil] run snapshot when requested
|
|
75
|
+
def mark_failed(run_id:, partition_key:, error:, return_snapshot: true)
|
|
76
|
+
update_partition(run_id, partition_key, return_snapshot:) do |entry|
|
|
77
77
|
ensure_not_succeeded!(entry)
|
|
78
78
|
entry[:status] = 'failed'
|
|
79
79
|
entry[:sample_error] = error_message(error) if entry[:sample_error].to_s.strip.empty?
|
|
@@ -87,14 +87,13 @@ module SearchEngine
|
|
|
87
87
|
meta = symbolize_meta(@cache.read(meta_cache_key(run_id)))
|
|
88
88
|
return nil unless meta
|
|
89
89
|
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
end
|
|
90
|
+
cache_keys = Array(meta[:partition_keys]).to_h do |partition_key|
|
|
91
|
+
[partition_key.to_s, partition_cache_key(run_id, partition_key)]
|
|
92
|
+
end
|
|
93
|
+
entries = read_entries(cache_keys.values)
|
|
94
|
+
partitions = cache_keys.to_h do |partition_key, cache_key|
|
|
95
|
+
entry = entries[cache_key]
|
|
96
|
+
[partition_key, entry.is_a?(Hash) ? deep_symbolize(entry) : missing_partition_entry(partition_key)]
|
|
98
97
|
end
|
|
99
98
|
meta[:partitions] = partitions
|
|
100
99
|
meta[:status] = run_status(partitions)
|
|
@@ -150,30 +149,42 @@ module SearchEngine
|
|
|
150
149
|
end
|
|
151
150
|
end
|
|
152
151
|
|
|
153
|
-
def
|
|
154
|
-
|
|
152
|
+
def read_entries(keys)
|
|
153
|
+
return {} if keys.empty?
|
|
154
|
+
return @cache.read_multi(*keys) if @cache.respond_to?(:read_multi)
|
|
155
|
+
|
|
156
|
+
keys.to_h { |key| [key, @cache.read(key)] }
|
|
155
157
|
end
|
|
156
158
|
|
|
157
|
-
def
|
|
158
|
-
|
|
159
|
-
|
|
159
|
+
def write!(key, value, ttl_s:)
|
|
160
|
+
return if @cache.write(key, value, expires_in: ttl_s)
|
|
161
|
+
|
|
162
|
+
raise IOError, "failed to write indexing run cache entry #{key}"
|
|
163
|
+
end
|
|
160
164
|
|
|
165
|
+
def update_partition(run_id, partition_key, return_snapshot:)
|
|
161
166
|
partition_key = partition_key.to_s
|
|
162
|
-
|
|
167
|
+
cache_key = partition_cache_key(run_id, partition_key)
|
|
168
|
+
meta_key = meta_cache_key(run_id)
|
|
169
|
+
entries = read_entries([meta_key, cache_key])
|
|
170
|
+
meta = symbolize_meta(entries[meta_key])
|
|
171
|
+
raise StaleRun, "indexing run not found: #{run_id}" unless meta
|
|
172
|
+
|
|
173
|
+
unless Array(meta[:partition_keys]).any? { |key| key.to_s == partition_key }
|
|
163
174
|
raise StaleRun, "indexing run partition not found: #{partition_key}"
|
|
164
175
|
end
|
|
165
176
|
|
|
166
|
-
entry =
|
|
177
|
+
entry = entries[cache_key]
|
|
167
178
|
raise StaleRun, "indexing run partition not found: #{partition_key}" unless entry
|
|
168
179
|
|
|
169
180
|
entry = deep_symbolize(entry)
|
|
170
181
|
yield entry
|
|
171
182
|
now = SearchEngine::IndexingRun.iso8601_now
|
|
172
183
|
entry[:updated_at] = now
|
|
173
|
-
|
|
184
|
+
write!(cache_key, entry, ttl_s: meta[:ttl_s])
|
|
174
185
|
meta[:updated_at] = now
|
|
175
|
-
|
|
176
|
-
snapshot(run_id: run_id)
|
|
186
|
+
write!(meta_key, meta, ttl_s: meta[:ttl_s])
|
|
187
|
+
snapshot(run_id: run_id) if return_snapshot
|
|
177
188
|
end
|
|
178
189
|
|
|
179
190
|
def run_status(partitions)
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require 'active_support/core_ext/object/with_options'
|
|
4
|
+
|
|
3
5
|
module SearchEngine
|
|
4
6
|
# Resolver and interface contract for async partition indexing run stores.
|
|
5
7
|
module IndexingRunStore
|
|
@@ -20,6 +22,14 @@ module SearchEngine
|
|
|
20
22
|
default
|
|
21
23
|
end
|
|
22
24
|
|
|
25
|
+
# Worker and timeout transitions only need to persist the affected partition.
|
|
26
|
+
# Custom stores keep their configured transition signatures.
|
|
27
|
+
def partition_writer(store = resolve)
|
|
28
|
+
return store unless defined?(RailsCache) && store.instance_of?(RailsCache)
|
|
29
|
+
|
|
30
|
+
store.with_options(return_snapshot: false)
|
|
31
|
+
end
|
|
32
|
+
|
|
23
33
|
# Build the default run store.
|
|
24
34
|
# @return [SearchEngine::IndexingRunStore::RailsCache]
|
|
25
35
|
def default
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require 'search_engine/registry'
|
|
4
|
+
|
|
3
5
|
module SearchEngine
|
|
4
6
|
module Joins
|
|
5
7
|
# Stateless guard for validating join usage across Relation and Parser.
|
|
@@ -85,28 +87,32 @@ module SearchEngine
|
|
|
85
87
|
)
|
|
86
88
|
end
|
|
87
89
|
|
|
90
|
+
# Structured filters use association names; native expressions can also
|
|
91
|
+
# name a physical collection directly. A missing registration has no
|
|
92
|
+
# local schema. Errors loading an existing registration must propagate.
|
|
93
|
+
def target_model(source_klass, name)
|
|
94
|
+
association = source_klass.joins_config[name.to_sym] if source_klass.respond_to?(:joins_config)
|
|
95
|
+
SearchEngine.collection_for(association ? association.fetch(:collection) : name) { nil }
|
|
96
|
+
end
|
|
97
|
+
|
|
88
98
|
# Validate that a joined field exists on the target collection when the
|
|
89
99
|
# target model is registered and exposes attributes.
|
|
90
100
|
#
|
|
91
|
-
#
|
|
92
|
-
# is raised (prefer low-noise behavior on missing metadata).
|
|
101
|
+
# Collections without a local model leave validation to Typesense.
|
|
93
102
|
#
|
|
94
103
|
# @param assoc_cfg [Hash] normalized association config
|
|
95
104
|
# @param field [Symbol, String]
|
|
96
105
|
# @param source_klass [Class, nil] optional source model class for error messages
|
|
106
|
+
# @param target_klass [Class, nil] already resolved target model, if available
|
|
97
107
|
# @raise [SearchEngine::Errors::UnknownJoinField]
|
|
98
108
|
# @return [void]
|
|
99
|
-
def validate_joined_field!(assoc_cfg, field, source_klass: nil)
|
|
109
|
+
def validate_joined_field!(assoc_cfg, field, source_klass: nil, target_klass: nil)
|
|
100
110
|
return if assoc_cfg.nil?
|
|
101
111
|
|
|
102
112
|
collection = assoc_cfg[:collection]
|
|
103
113
|
return if blank?(collection)
|
|
104
114
|
|
|
105
|
-
target_klass
|
|
106
|
-
SearchEngine.collection_for(collection)
|
|
107
|
-
rescue StandardError
|
|
108
|
-
nil
|
|
109
|
-
end
|
|
115
|
+
target_klass ||= SearchEngine.collection_for(collection) { nil }
|
|
110
116
|
return unless target_klass.respond_to?(:attributes)
|
|
111
117
|
|
|
112
118
|
known = Array(target_klass.attributes).map { |k, _| k.to_s }
|
data/lib/search_engine/multi.rb
CHANGED
|
@@ -5,7 +5,7 @@ require 'set'
|
|
|
5
5
|
module SearchEngine
|
|
6
6
|
# Builder for assembling labeled Relations into a federated multi-search.
|
|
7
7
|
#
|
|
8
|
-
#
|
|
8
|
+
# Validates labels and relations, preserves insertion order,
|
|
9
9
|
# and produces per-search payload hashes ready for {SearchEngine::Client#multi_search}.
|
|
10
10
|
#
|
|
11
11
|
# Usage (via the module-level convenience):
|
|
@@ -21,7 +21,7 @@ module SearchEngine
|
|
|
21
21
|
URL_ONLY_KEYS = %i[use_cache cache_ttl].freeze
|
|
22
22
|
|
|
23
23
|
# Keys that must be present/retained for :only mode in multi-search payloads.
|
|
24
|
-
ESSENTIAL_MULTI_KEYS = %i[collection q page per_page preset].freeze
|
|
24
|
+
ESSENTIAL_MULTI_KEYS = %i[collection q page per_page offset limit preset].freeze
|
|
25
25
|
|
|
26
26
|
# Canonicalize a label into a case-insensitive Symbol key.
|
|
27
27
|
# @param label [String, Symbol]
|
|
@@ -90,7 +90,7 @@ module SearchEngine
|
|
|
90
90
|
# Presets:
|
|
91
91
|
# - Preset token lives inside each per-search payload under :preset (never top-level)
|
|
92
92
|
# - mode=:merge — pass through compiler output (includes :preset)
|
|
93
|
-
# - mode=:only — retain only ESSENTIAL_MULTI_KEYS
|
|
93
|
+
# - mode=:only — retain only ESSENTIAL_MULTI_KEYS after common merge
|
|
94
94
|
# - mode=:lock — defensively drop any keys present in Config.presets.locked_domains from the merged payload
|
|
95
95
|
# (conflict recording remains in single-search compile)
|
|
96
96
|
#
|
|
@@ -107,7 +107,7 @@ module SearchEngine
|
|
|
107
107
|
def to_payloads(common: {})
|
|
108
108
|
raise ArgumentError, 'common must be a Hash' unless common.is_a?(Hash)
|
|
109
109
|
|
|
110
|
-
filtered_common = filter_url_only_keys(common)
|
|
110
|
+
filtered_common = filter_url_only_keys(common).transform_keys(&:to_sym)
|
|
111
111
|
|
|
112
112
|
seen = Set.new
|
|
113
113
|
@entries.map do |e|
|
|
@@ -129,8 +129,14 @@ module SearchEngine
|
|
|
129
129
|
|
|
130
130
|
collection = collection_name_for_relation(e.relation)
|
|
131
131
|
|
|
132
|
-
|
|
133
|
-
|
|
132
|
+
common_params = if per_search.key?(:offset) || per_search.key?(:limit)
|
|
133
|
+
filtered_common.except(:page, :per_page)
|
|
134
|
+
elsif per_search.key?(:page) || per_search.key?(:per_page)
|
|
135
|
+
filtered_common.except(:offset, :limit)
|
|
136
|
+
else
|
|
137
|
+
filtered_common
|
|
138
|
+
end
|
|
139
|
+
merged = common_params.merge(per_search)
|
|
134
140
|
payload = { collection: collection }.merge(EssentialPruner.prune(merged, relation: e.relation))
|
|
135
141
|
prune_empty_values(payload)
|
|
136
142
|
end
|
|
@@ -142,6 +148,28 @@ module SearchEngine
|
|
|
142
148
|
@entries.map { |e| e.relation.klass }
|
|
143
149
|
end
|
|
144
150
|
|
|
151
|
+
# Execute the labeled searches together. Typed results retain each relation's context.
|
|
152
|
+
# Raw responses expose the complete backend envelope, including per-search errors.
|
|
153
|
+
def execute(common: {}, client: nil, url_opts: {}, raw: false)
|
|
154
|
+
keys = labels
|
|
155
|
+
enforce_multi_limit!(keys.size, SearchEngine.config.multi_search_limit)
|
|
156
|
+
searches = to_payloads(common: common)
|
|
157
|
+
client ||= SearchEngine.client
|
|
158
|
+
url_opts = search_url_options(client, url_opts)
|
|
159
|
+
payload = build_multi_event_payload(keys.size, keys, url_opts)
|
|
160
|
+
|
|
161
|
+
Instrumentation.instrument('search_engine.multi_search', payload) do |ctx|
|
|
162
|
+
begin
|
|
163
|
+
response = client.multi_search(searches: searches, url_opts: url_opts)
|
|
164
|
+
rescue Errors::Api => error
|
|
165
|
+
ctx[:http_status] = error.status
|
|
166
|
+
raise augment_multi_api_error(error, keys)
|
|
167
|
+
end
|
|
168
|
+
ctx[:http_status] = 200
|
|
169
|
+
raw ? response : to_result(response, keys)
|
|
170
|
+
end
|
|
171
|
+
end
|
|
172
|
+
|
|
145
173
|
# ResultSet maps labels to {SearchEngine::Result} while preserving order.
|
|
146
174
|
class ResultSet
|
|
147
175
|
# @param pairs [Array<Array(Symbol, SearchEngine::Result)>>] ordered (label, result) pairs
|
|
@@ -195,6 +223,64 @@ module SearchEngine
|
|
|
195
223
|
|
|
196
224
|
private
|
|
197
225
|
|
|
226
|
+
# Hydrate each result with the selection, facets and constraints of its relation.
|
|
227
|
+
def to_result(response, keys)
|
|
228
|
+
items = response[:results] || response['results'] if response.is_a?(Hash)
|
|
229
|
+
MultiResult.new(labels: keys, raw_results: items) do |raw, index|
|
|
230
|
+
relation = @entries.fetch(index).relation
|
|
231
|
+
if relation.is_a?(Relation)
|
|
232
|
+
relation.send(:enforce_hit_validator_if_needed!, raw[:found] || raw['found'])
|
|
233
|
+
Result.new(raw, **Hydration::Materializers.result_options(relation))
|
|
234
|
+
else
|
|
235
|
+
Result.new(raw, klass: relation.klass)
|
|
236
|
+
end
|
|
237
|
+
end
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
def enforce_multi_limit!(count, limit)
|
|
241
|
+
return unless count > limit
|
|
242
|
+
|
|
243
|
+
raise ArgumentError,
|
|
244
|
+
"multi_search: #{count} searches exceed limit (#{limit}). " \
|
|
245
|
+
'Increase `SearchEngine.config.multi_search_limit` if necessary.'
|
|
246
|
+
end
|
|
247
|
+
|
|
248
|
+
def search_url_options(client, overrides)
|
|
249
|
+
defaults = if client.respond_to?(:search_url_options)
|
|
250
|
+
client.search_url_options
|
|
251
|
+
else
|
|
252
|
+
ClientOptions.url_options_from_config
|
|
253
|
+
end
|
|
254
|
+
options = defaults.merge(overrides)
|
|
255
|
+
disabled = @entries.any? do |entry|
|
|
256
|
+
entry.relation.is_a?(Relation) && entry.relation.send(:build_url_opts)[:use_cache] == false
|
|
257
|
+
end
|
|
258
|
+
options[:use_cache] = false if disabled
|
|
259
|
+
options
|
|
260
|
+
end
|
|
261
|
+
|
|
262
|
+
def build_multi_event_payload(count, labels, url_opts)
|
|
263
|
+
{
|
|
264
|
+
searches_count: count,
|
|
265
|
+
labels: labels.map(&:to_s),
|
|
266
|
+
http_status: nil,
|
|
267
|
+
source: :multi,
|
|
268
|
+
url_opts: Observability.filtered_url_opts(url_opts)
|
|
269
|
+
}
|
|
270
|
+
end
|
|
271
|
+
|
|
272
|
+
def augment_multi_api_error(error, labels)
|
|
273
|
+
body = error.body
|
|
274
|
+
results = body[:results] || body['results'] if body.is_a?(Hash)
|
|
275
|
+
return error unless results.is_a?(Array)
|
|
276
|
+
|
|
277
|
+
labels.each_with_index do |label, index|
|
|
278
|
+
failure = MultiResult.entry_error(label, results[index])
|
|
279
|
+
return Errors::Api.new(failure.message, status: error.status, body: body) if failure
|
|
280
|
+
end
|
|
281
|
+
error
|
|
282
|
+
end
|
|
283
|
+
|
|
198
284
|
def validate_relation!(relation)
|
|
199
285
|
unless relation.respond_to?(:to_typesense_params) && relation.respond_to?(:klass)
|
|
200
286
|
raise ArgumentError, 'relation must respond to :to_typesense_params and :klass'
|
|
@@ -28,6 +28,21 @@ module SearchEngine
|
|
|
28
28
|
# mr.dig('brands').to_a
|
|
29
29
|
# mr.labels #=> [:products, :brands]
|
|
30
30
|
class MultiResult
|
|
31
|
+
attr_reader :result_set
|
|
32
|
+
|
|
33
|
+
# Typesense can report individual failures inside a successful HTTP response.
|
|
34
|
+
def self.entry_error(label, item)
|
|
35
|
+
return unless item.is_a?(Hash)
|
|
36
|
+
|
|
37
|
+
code = item[:code] || item['code'] || item[:status] || item['status']
|
|
38
|
+
status = Integer(code, exception: false)
|
|
39
|
+
return unless item[:error] || item['error'] || (status && !(200..299).cover?(status))
|
|
40
|
+
|
|
41
|
+
message = "Multi-search failed for label :#{label}"
|
|
42
|
+
message += " (status #{status})" if status
|
|
43
|
+
Errors::Api.new("#{message}.", status: status, body: item)
|
|
44
|
+
end
|
|
45
|
+
|
|
31
46
|
# Build a new MultiResult.
|
|
32
47
|
#
|
|
33
48
|
# @param labels [Array<String, Symbol>] ordered labels
|
|
@@ -36,36 +51,38 @@ module SearchEngine
|
|
|
36
51
|
# @raise [ArgumentError] when sizes mismatch, labels invalid/duplicate, or inputs malformed
|
|
37
52
|
# @see `https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/multi-search-guide`
|
|
38
53
|
def initialize(labels:, raw_results:, klasses: nil)
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
klass_by_index = build_klass_index(@labels, klasses)
|
|
48
|
-
|
|
49
|
-
@labels.each_with_index do |label, index|
|
|
50
|
-
raw = @raw_results[index]
|
|
51
|
-
kls = klass_by_index[index] || infer_klass_from_raw(raw)
|
|
52
|
-
@map[label] = SearchEngine::Result.new(raw, klass: kls)
|
|
54
|
+
keys = canonicalize_labels(labels)
|
|
55
|
+
validate_sizes!(keys, raw_results, klasses)
|
|
56
|
+
keys.each_with_index do |label, index|
|
|
57
|
+
raw = raw_results[index]
|
|
58
|
+
raise ArgumentError, "Expected a result Hash for label :#{label}" unless raw.is_a?(Hash)
|
|
59
|
+
|
|
60
|
+
failure = MultiResult.entry_error(label, raw)
|
|
61
|
+
raise failure if failure
|
|
53
62
|
end
|
|
54
63
|
|
|
64
|
+
klass_by_index = build_klass_index(keys, klasses)
|
|
65
|
+
pairs = keys.each_with_index.map do |label, index|
|
|
66
|
+
raw = raw_results[index]
|
|
67
|
+
result = if block_given?
|
|
68
|
+
yield(raw, index)
|
|
69
|
+
else
|
|
70
|
+
Result.new(raw, klass: klass_by_index[index] || infer_klass_from_raw(raw))
|
|
71
|
+
end
|
|
72
|
+
[label, result]
|
|
73
|
+
end
|
|
74
|
+
@result_set = Multi::ResultSet.new(pairs)
|
|
55
75
|
freeze
|
|
56
76
|
end
|
|
57
77
|
|
|
58
|
-
# Return a shallow copy of
|
|
59
|
-
# @return [Array<Symbol>]
|
|
78
|
+
# Return a shallow copy of the ordered labels.
|
|
60
79
|
def labels
|
|
61
|
-
@labels
|
|
80
|
+
@result_set.labels
|
|
62
81
|
end
|
|
63
82
|
|
|
64
|
-
# Fetch a
|
|
65
|
-
# @param label [String, Symbol]
|
|
66
|
-
# @return [SearchEngine::Result, nil]
|
|
83
|
+
# Fetch a result by its case-insensitive label.
|
|
67
84
|
def [](label)
|
|
68
|
-
@
|
|
85
|
+
@result_set[label]
|
|
69
86
|
rescue ArgumentError
|
|
70
87
|
nil
|
|
71
88
|
end
|
|
@@ -87,7 +104,7 @@ module SearchEngine
|
|
|
87
104
|
# Pure helper: returns a shallow copy and never performs HTTP.
|
|
88
105
|
# @return [Hash{Symbol=>SearchEngine::Result}]
|
|
89
106
|
def to_h
|
|
90
|
-
@
|
|
107
|
+
@result_set.to_h
|
|
91
108
|
end
|
|
92
109
|
|
|
93
110
|
# Iterate over (label, result) in insertion order.
|
|
@@ -98,7 +115,7 @@ module SearchEngine
|
|
|
98
115
|
def each_label
|
|
99
116
|
return enum_for(:each_label) unless block_given?
|
|
100
117
|
|
|
101
|
-
@
|
|
118
|
+
@result_set.each_pair { |label, result| yield([label, result]) }
|
|
102
119
|
end
|
|
103
120
|
|
|
104
121
|
# Convenience to map over (label, result) pairs.
|
|
@@ -335,7 +335,7 @@ module SearchEngine
|
|
|
335
335
|
{}
|
|
336
336
|
end
|
|
337
337
|
parts = []
|
|
338
|
-
%i[q query_by per_page page infix].each do |key|
|
|
338
|
+
%i[q query_by per_page page limit offset infix].each do |key|
|
|
339
339
|
next unless params_hash.key?(key)
|
|
340
340
|
|
|
341
341
|
parts << format_param(key, params_hash[key])
|
|
@@ -540,7 +540,7 @@ module SearchEngine
|
|
|
540
540
|
private_class_method :attach_grouping_fields!
|
|
541
541
|
|
|
542
542
|
def self.attach_selected_params!(h, params_hash)
|
|
543
|
-
%i[q query_by per_page page infix].each do |key|
|
|
543
|
+
%i[q query_by per_page page limit offset infix].each do |key|
|
|
544
544
|
h[key.to_s] = params_hash[key] if params_hash.key?(key)
|
|
545
545
|
end
|
|
546
546
|
h['filter_by'] = '***' if params_hash.key?(:filter_by)
|
|
@@ -12,7 +12,7 @@ module SearchEngine
|
|
|
12
12
|
|
|
13
13
|
# Whitelisted search parameter keys to include in payload excerpts.
|
|
14
14
|
PARAM_WHITELIST = %i[
|
|
15
|
-
q query_by include_fields exclude_fields per_page page infix filter_by sort_by
|
|
15
|
+
q query_by include_fields exclude_fields per_page page limit offset infix filter_by sort_by
|
|
16
16
|
group_by group_limit group_missing_values
|
|
17
17
|
facet_by max_facet_values facet_query
|
|
18
18
|
num_typos drop_tokens_threshold prioritize_exact_match query_by_weights
|
|
@@ -110,7 +110,7 @@ module SearchEngine
|
|
|
110
110
|
return str unless str.is_a?(String)
|
|
111
111
|
|
|
112
112
|
# Mask obvious quoted literals first
|
|
113
|
-
redacted = str.gsub(
|
|
113
|
+
redacted = str.gsub(/`[^`]*`|"[^"]*"|'[^']*'/, '***')
|
|
114
114
|
|
|
115
115
|
# Mask numeric literals (best-effort)
|
|
116
116
|
redacted.gsub(/\b\d+(?:\.\d+)?\b/, '***')
|
|
@@ -124,10 +124,10 @@ module SearchEngine
|
|
|
124
124
|
def self.redact_filter_by(filter)
|
|
125
125
|
return filter unless filter.is_a?(String)
|
|
126
126
|
|
|
127
|
-
#
|
|
128
|
-
#
|
|
129
|
-
masked = filter.gsub(
|
|
130
|
-
masked = masked.gsub(/
|
|
127
|
+
# Mask whole literals first: their contents may look like expressions.
|
|
128
|
+
# Once quoted values are gone, a list can safely be masked as a unit.
|
|
129
|
+
masked = filter.gsub(/`[^`]*`|"[^"]*"|'[^']*'/, '***').gsub(/\[[^\]]*\]/, '***')
|
|
130
|
+
masked = masked.gsub(/([!:><=]{1,2})\s*([^\s)&|]+)/, '\1***')
|
|
131
131
|
masked.gsub(/\b\d+(?:\.\d+)?\b/, '***')
|
|
132
132
|
end
|
|
133
133
|
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SearchEngine
|
|
4
|
+
module PostgresOutbox
|
|
5
|
+
# Operational reads must observe current delivery state and acquire row locks on each poll.
|
|
6
|
+
module DatabaseQueries
|
|
7
|
+
private
|
|
8
|
+
|
|
9
|
+
def select_rows(sql)
|
|
10
|
+
result = connection.respond_to?(:exec_query) ? connection.exec_query(sql) : connection.select_all(sql)
|
|
11
|
+
return result.to_a if result.respond_to?(:to_a)
|
|
12
|
+
|
|
13
|
+
Array(result)
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require 'socket'
|
|
4
|
-
require 'securerandom'
|
|
5
4
|
require 'set'
|
|
6
5
|
require 'search_engine/cache'
|
|
7
6
|
|
|
@@ -10,7 +9,6 @@ module SearchEngine
|
|
|
10
9
|
# Orchestrates one bounded PostgreSQL outbox drain pass.
|
|
11
10
|
class Drainer
|
|
12
11
|
BLOCKED_ERROR = 'Skipped because an earlier outbox collection group failed'
|
|
13
|
-
LEASE_OWNER_MAX_LENGTH = 255
|
|
14
12
|
|
|
15
13
|
# @param repository [SearchEngine::PostgresOutbox::Repository]
|
|
16
14
|
# @param processor [#call]
|
|
@@ -31,8 +29,8 @@ module SearchEngine
|
|
|
31
29
|
'search_engine.postgres_outbox.drain',
|
|
32
30
|
drain_payload(limit)
|
|
33
31
|
) do |payload|
|
|
34
|
-
lease_owner =
|
|
35
|
-
events = repository.claim_pending(limit: limit, worker_id: lease_owner)
|
|
32
|
+
lease_owner = Event.new_lease_owner(worker_id) unless repository_assigns_lease_owner?
|
|
33
|
+
events = repository.claim_pending(limit: limit, worker_id: lease_owner || worker_id)
|
|
36
34
|
summary = empty_summary(events)
|
|
37
35
|
next summary if events.empty?
|
|
38
36
|
|
|
@@ -43,7 +41,7 @@ module SearchEngine
|
|
|
43
41
|
record_acknowledgements(summary, :superseded, superseded_events, superseded_ids)
|
|
44
42
|
|
|
45
43
|
ordered = SearchEngine::DependencyPlanner.order_events(kept)
|
|
46
|
-
process_ordered_events(ordered, summary, lease_owner: lease_owner)
|
|
44
|
+
process_ordered_events(ordered, summary, lease_owner: lease_owner || events.first.event_lease_owner)
|
|
47
45
|
payload.merge!(summary)
|
|
48
46
|
summary
|
|
49
47
|
end
|
|
@@ -53,6 +51,10 @@ module SearchEngine
|
|
|
53
51
|
|
|
54
52
|
attr_reader :repository, :processor, :worker_id, :target_key
|
|
55
53
|
|
|
54
|
+
def repository_assigns_lease_owner?
|
|
55
|
+
repository.respond_to?(:assigns_event_lease_owner?) && repository.assigns_event_lease_owner?
|
|
56
|
+
end
|
|
57
|
+
|
|
56
58
|
def empty_summary(events)
|
|
57
59
|
summary = {
|
|
58
60
|
claimed: events.size,
|
|
@@ -250,13 +252,6 @@ module SearchEngine
|
|
|
250
252
|
"#{Socket.gethostname}:#{$PROCESS_ID}:#{Thread.current.object_id}"
|
|
251
253
|
end
|
|
252
254
|
|
|
253
|
-
def next_lease_owner
|
|
254
|
-
suffix = SecureRandom.uuid
|
|
255
|
-
prefix_limit = LEASE_OWNER_MAX_LENGTH - suffix.length - 1
|
|
256
|
-
prefix = worker_id.to_s.gsub(/[^a-zA-Z0-9_.:-]/, '_')[0, prefix_limit]
|
|
257
|
-
"#{prefix}:#{suffix}"
|
|
258
|
-
end
|
|
259
|
-
|
|
260
255
|
def processor_context(lease_owner)
|
|
261
256
|
context = { worker_id: worker_id, lease_owner: lease_owner }
|
|
262
257
|
context[:target_key] = target_key if target_key
|
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require 'securerandom'
|
|
4
|
+
|
|
3
5
|
module SearchEngine
|
|
4
6
|
module PostgresOutbox
|
|
5
7
|
# Immutable value object for one PostgreSQL outbox row.
|
|
6
8
|
class Event
|
|
9
|
+
LEASE_OWNER_MAX_LENGTH = 255
|
|
7
10
|
VALID_OPERATIONS = %i[upsert delete].freeze
|
|
8
11
|
|
|
9
12
|
attr_reader :id,
|
|
@@ -16,10 +19,18 @@ module SearchEngine
|
|
|
16
19
|
:attempts,
|
|
17
20
|
:payload,
|
|
18
21
|
:created_at,
|
|
22
|
+
:event_lease_owner,
|
|
19
23
|
:delivery_id,
|
|
20
24
|
:target_key,
|
|
21
25
|
:delivery_lease_owner
|
|
22
26
|
|
|
27
|
+
def self.new_lease_owner(worker_id)
|
|
28
|
+
suffix = SecureRandom.uuid
|
|
29
|
+
prefix_limit = LEASE_OWNER_MAX_LENGTH - suffix.length - 1
|
|
30
|
+
prefix = worker_id.to_s.gsub(/[^a-zA-Z0-9_.:-]/, '_')[0, prefix_limit]
|
|
31
|
+
"#{prefix}:#{suffix}"
|
|
32
|
+
end
|
|
33
|
+
|
|
23
34
|
# @param row [Hash] outbox row with string or symbol keys
|
|
24
35
|
# @raise [ArgumentError] when operation is not upsert/delete
|
|
25
36
|
def initialize(row)
|
|
@@ -33,6 +44,7 @@ module SearchEngine
|
|
|
33
44
|
@attempts = (value(row, :delivery_attempts) || value(row, :attempts)).to_i
|
|
34
45
|
@payload = value(row, :payload) || {}
|
|
35
46
|
@created_at = value(row, :created_at)
|
|
47
|
+
@event_lease_owner = string_value(row, :event_lease_owner)
|
|
36
48
|
@delivery_id = value(row, :delivery_id)
|
|
37
49
|
@target_key = string_value(row, :target_key)
|
|
38
50
|
@delivery_lease_owner = string_value(row, :delivery_lease_owner)
|