search-engine-for-typesense 30.9.1 → 30.9.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +19 -0
  3. data/README.md +51 -16
  4. data/lib/search_engine/active_record_syncable.rb +85 -197
  5. data/lib/search_engine/ast/node.rb +3 -6
  6. data/lib/search_engine/async_partition_coordinator.rb +5 -3
  7. data/lib/search_engine/client/services/search.rb +2 -0
  8. data/lib/search_engine/client.rb +9 -3
  9. data/lib/search_engine/compiler.rb +33 -128
  10. data/lib/search_engine/deletion.rb +12 -9
  11. data/lib/search_engine/dsl/parser.rb +51 -71
  12. data/lib/search_engine/engine.rb +25 -16
  13. data/lib/search_engine/filters/sanitizer.rb +162 -126
  14. data/lib/search_engine/hydration/materializers.rb +53 -118
  15. data/{app/search_engine → lib}/search_engine/index_partition_job.rb +1 -1
  16. data/lib/search_engine/indexing_run_store/rails_cache.rb +44 -33
  17. data/lib/search_engine/indexing_run_store.rb +10 -0
  18. data/lib/search_engine/joins/guard.rb +14 -8
  19. data/lib/search_engine/multi.rb +92 -6
  20. data/lib/search_engine/multi_result.rb +40 -23
  21. data/lib/search_engine/notifications/compact_logger.rb +2 -2
  22. data/lib/search_engine/observability.rb +6 -6
  23. data/lib/search_engine/postgres_outbox/database_queries.rb +17 -0
  24. data/lib/search_engine/postgres_outbox/drainer.rb +7 -12
  25. data/lib/search_engine/postgres_outbox/event.rb +12 -0
  26. data/lib/search_engine/postgres_outbox/materialization_cursor.rb +25 -11
  27. data/lib/search_engine/postgres_outbox/repository.rb +65 -115
  28. data/lib/search_engine/postgres_outbox.rb +2 -0
  29. data/lib/search_engine/registry.rb +27 -28
  30. data/lib/search_engine/relation/compiler.rb +18 -33
  31. data/lib/search_engine/relation/dsl/filters.rb +71 -175
  32. data/lib/search_engine/relation/dx.rb +3 -1
  33. data/lib/search_engine/relation/materializers.rb +6 -3
  34. data/lib/search_engine/relation/options.rb +5 -26
  35. data/lib/search_engine/relation/state.rb +3 -2
  36. data/lib/search_engine/relation.rb +5 -4
  37. data/lib/search_engine/result.rb +33 -2
  38. data/lib/search_engine/schema.rb +23 -24
  39. data/lib/search_engine/test/offline_client.rb +2 -2
  40. data/lib/search_engine/test/stub_client.rb +9 -16
  41. data/lib/search_engine/update.rb +1 -1
  42. data/lib/search_engine/version.rb +1 -1
  43. data/lib/search_engine.rb +11 -149
  44. data/lib/tasks/search_engine.rake +3 -5
  45. metadata +18 -3
  46. /data/{app/search_engine → lib}/search_engine/postgres_outbox/drain_job.rb +0 -0
@@ -7,7 +7,7 @@ module SearchEngine
7
7
  module IndexingRunStore
8
8
  # Rails.cache-backed implementation for async partition indexing run metadata.
9
9
  class RailsCache
10
- # @param cache [Object, nil] cache object responding to read/write/delete
10
+ # @param cache [Object, nil] Rails-compatible cache responding to read/write/delete
11
11
  def initialize(cache: nil)
12
12
  @cache = cache || resolve_rails_cache!
13
13
  validate_cache!
@@ -29,17 +29,17 @@ module SearchEngine
29
29
  created_at: SearchEngine::IndexingRun.iso8601_now,
30
30
  updated_at: SearchEngine::IndexingRun.iso8601_now
31
31
  }
32
- write_meta(run, ttl_s: ttl_s)
32
+ write!(meta_cache_key(run_id), run, ttl_s: ttl_s)
33
33
  entries.each do |partition_key, entry|
34
- @cache.write(partition_cache_key(run_id, partition_key), entry, expires_in: ttl_s)
34
+ write!(partition_cache_key(run_id, partition_key), entry, ttl_s: ttl_s)
35
35
  end
36
36
  snapshot(run_id: run_id)
37
37
  end
38
38
 
39
39
  # Mark a partition as started.
40
- # @return [Hash]
41
- def mark_started(run_id:, partition_key:, job_id: nil)
42
- update_partition(run_id, partition_key) do |entry|
40
+ # @return [Hash, nil] run snapshot when requested
41
+ def mark_started(run_id:, partition_key:, job_id: nil, return_snapshot: true)
42
+ update_partition(run_id, partition_key, return_snapshot:) do |entry|
43
43
  ensure_non_terminal!(entry, 'start')
44
44
  entry[:status] = 'running'
45
45
  entry[:attempts] = entry[:attempts].to_i + 1
@@ -49,9 +49,9 @@ module SearchEngine
49
49
 
50
50
  # Record one failed import attempt without terminalizing the partition.
51
51
  # ActiveJob may safely start the same partition again on retry.
52
- # @return [Hash]
53
- def record_attempt(run_id:, partition_key:, summary:, error:)
54
- update_partition(run_id, partition_key) do |entry|
52
+ # @return [Hash, nil] run snapshot when requested
53
+ def record_attempt(run_id:, partition_key:, summary:, error:, return_snapshot: true)
54
+ update_partition(run_id, partition_key, return_snapshot:) do |entry|
55
55
  ensure_non_terminal!(entry, 'record an attempt for')
56
56
  assign_summary!(entry, summary)
57
57
  entry[:status] = 'running'
@@ -60,9 +60,9 @@ module SearchEngine
60
60
  end
61
61
 
62
62
  # Mark a partition as succeeded with import counters.
63
- # @return [Hash]
64
- def mark_succeeded(run_id:, partition_key:, summary:)
65
- update_partition(run_id, partition_key) do |entry|
63
+ # @return [Hash, nil] run snapshot when requested
64
+ def mark_succeeded(run_id:, partition_key:, summary:, return_snapshot: true)
65
+ update_partition(run_id, partition_key, return_snapshot:) do |entry|
66
66
  ensure_non_terminal!(entry, 'succeed')
67
67
  assign_summary!(entry, summary)
68
68
  entry[:status] = 'succeeded'
@@ -71,9 +71,9 @@ module SearchEngine
71
71
  end
72
72
 
73
73
  # Mark a partition as failed.
74
- # @return [Hash]
75
- def mark_failed(run_id:, partition_key:, error:)
76
- update_partition(run_id, partition_key) do |entry|
74
+ # @return [Hash, nil] run snapshot when requested
75
+ def mark_failed(run_id:, partition_key:, error:, return_snapshot: true)
76
+ update_partition(run_id, partition_key, return_snapshot:) do |entry|
77
77
  ensure_not_succeeded!(entry)
78
78
  entry[:status] = 'failed'
79
79
  entry[:sample_error] = error_message(error) if entry[:sample_error].to_s.strip.empty?
@@ -87,14 +87,13 @@ module SearchEngine
87
87
  meta = symbolize_meta(@cache.read(meta_cache_key(run_id)))
88
88
  return nil unless meta
89
89
 
90
- partition_keys = Array(meta[:partition_keys]).map(&:to_s)
91
- partitions = partition_keys.each_with_object({}) do |partition_key, hash|
92
- entry = @cache.read(partition_cache_key(run_id, partition_key))
93
- hash[partition_key] = if entry.is_a?(Hash)
94
- deep_symbolize(entry)
95
- else
96
- missing_partition_entry(partition_key)
97
- end
90
+ cache_keys = Array(meta[:partition_keys]).to_h do |partition_key|
91
+ [partition_key.to_s, partition_cache_key(run_id, partition_key)]
92
+ end
93
+ entries = read_entries(cache_keys.values)
94
+ partitions = cache_keys.to_h do |partition_key, cache_key|
95
+ entry = entries[cache_key]
96
+ [partition_key, entry.is_a?(Hash) ? deep_symbolize(entry) : missing_partition_entry(partition_key)]
98
97
  end
99
98
  meta[:partitions] = partitions
100
99
  meta[:status] = run_status(partitions)
@@ -150,30 +149,42 @@ module SearchEngine
150
149
  end
151
150
  end
152
151
 
153
- def write_meta(run, ttl_s:)
154
- @cache.write(meta_cache_key(run[:run_id]), run, expires_in: ttl_s)
152
+ def read_entries(keys)
153
+ return {} if keys.empty?
154
+ return @cache.read_multi(*keys) if @cache.respond_to?(:read_multi)
155
+
156
+ keys.to_h { |key| [key, @cache.read(key)] }
155
157
  end
156
158
 
157
- def update_partition(run_id, partition_key)
158
- meta = symbolize_meta(@cache.read(meta_cache_key(run_id)))
159
- raise StaleRun, "indexing run not found: #{run_id}" unless meta
159
+ def write!(key, value, ttl_s:)
160
+ return if @cache.write(key, value, expires_in: ttl_s)
161
+
162
+ raise IOError, "failed to write indexing run cache entry #{key}"
163
+ end
160
164
 
165
+ def update_partition(run_id, partition_key, return_snapshot:)
161
166
  partition_key = partition_key.to_s
162
- unless Array(meta[:partition_keys]).map(&:to_s).include?(partition_key)
167
+ cache_key = partition_cache_key(run_id, partition_key)
168
+ meta_key = meta_cache_key(run_id)
169
+ entries = read_entries([meta_key, cache_key])
170
+ meta = symbolize_meta(entries[meta_key])
171
+ raise StaleRun, "indexing run not found: #{run_id}" unless meta
172
+
173
+ unless Array(meta[:partition_keys]).any? { |key| key.to_s == partition_key }
163
174
  raise StaleRun, "indexing run partition not found: #{partition_key}"
164
175
  end
165
176
 
166
- entry = @cache.read(partition_cache_key(run_id, partition_key))
177
+ entry = entries[cache_key]
167
178
  raise StaleRun, "indexing run partition not found: #{partition_key}" unless entry
168
179
 
169
180
  entry = deep_symbolize(entry)
170
181
  yield entry
171
182
  now = SearchEngine::IndexingRun.iso8601_now
172
183
  entry[:updated_at] = now
173
- @cache.write(partition_cache_key(run_id, partition_key), entry, expires_in: meta[:ttl_s])
184
+ write!(cache_key, entry, ttl_s: meta[:ttl_s])
174
185
  meta[:updated_at] = now
175
- write_meta(meta, ttl_s: meta[:ttl_s])
176
- snapshot(run_id: run_id)
186
+ write!(meta_key, meta, ttl_s: meta[:ttl_s])
187
+ snapshot(run_id: run_id) if return_snapshot
177
188
  end
178
189
 
179
190
  def run_status(partitions)
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'active_support/core_ext/object/with_options'
4
+
3
5
  module SearchEngine
4
6
  # Resolver and interface contract for async partition indexing run stores.
5
7
  module IndexingRunStore
@@ -20,6 +22,14 @@ module SearchEngine
20
22
  default
21
23
  end
22
24
 
25
+ # Worker and timeout transitions only need to persist the affected partition.
26
+ # Custom stores keep their configured transition signatures.
27
+ def partition_writer(store = resolve)
28
+ return store unless defined?(RailsCache) && store.instance_of?(RailsCache)
29
+
30
+ store.with_options(return_snapshot: false)
31
+ end
32
+
23
33
  # Build the default run store.
24
34
  # @return [SearchEngine::IndexingRunStore::RailsCache]
25
35
  def default
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'search_engine/registry'
4
+
3
5
  module SearchEngine
4
6
  module Joins
5
7
  # Stateless guard for validating join usage across Relation and Parser.
@@ -85,28 +87,32 @@ module SearchEngine
85
87
  )
86
88
  end
87
89
 
90
+ # Structured filters use association names; native expressions can also
91
+ # name a physical collection directly. A missing registration has no
92
+ # local schema. Errors loading an existing registration must propagate.
93
+ def target_model(source_klass, name)
94
+ association = source_klass.joins_config[name.to_sym] if source_klass.respond_to?(:joins_config)
95
+ SearchEngine.collection_for(association ? association.fetch(:collection) : name) { nil }
96
+ end
97
+
88
98
  # Validate that a joined field exists on the target collection when the
89
99
  # target model is registered and exposes attributes.
90
100
  #
91
- # Best-effort: when registry or attributes are unavailable, no exception
92
- # is raised (prefer low-noise behavior on missing metadata).
101
+ # Collections without a local model leave validation to Typesense.
93
102
  #
94
103
  # @param assoc_cfg [Hash] normalized association config
95
104
  # @param field [Symbol, String]
96
105
  # @param source_klass [Class, nil] optional source model class for error messages
106
+ # @param target_klass [Class, nil] already resolved target model, if available
97
107
  # @raise [SearchEngine::Errors::UnknownJoinField]
98
108
  # @return [void]
99
- def validate_joined_field!(assoc_cfg, field, source_klass: nil)
109
+ def validate_joined_field!(assoc_cfg, field, source_klass: nil, target_klass: nil)
100
110
  return if assoc_cfg.nil?
101
111
 
102
112
  collection = assoc_cfg[:collection]
103
113
  return if blank?(collection)
104
114
 
105
- target_klass = begin
106
- SearchEngine.collection_for(collection)
107
- rescue StandardError
108
- nil
109
- end
115
+ target_klass ||= SearchEngine.collection_for(collection) { nil }
110
116
  return unless target_klass.respond_to?(:attributes)
111
117
 
112
118
  known = Array(target_klass.attributes).map { |k, _| k.to_s }
@@ -5,7 +5,7 @@ require 'set'
5
5
  module SearchEngine
6
6
  # Builder for assembling labeled Relations into a federated multi-search.
7
7
  #
8
- # Pure collector: validates labels and relations, preserves insertion order,
8
+ # Validates labels and relations, preserves insertion order,
9
9
  # and produces per-search payload hashes ready for {SearchEngine::Client#multi_search}.
10
10
  #
11
11
  # Usage (via the module-level convenience):
@@ -21,7 +21,7 @@ module SearchEngine
21
21
  URL_ONLY_KEYS = %i[use_cache cache_ttl].freeze
22
22
 
23
23
  # Keys that must be present/retained for :only mode in multi-search payloads.
24
- ESSENTIAL_MULTI_KEYS = %i[collection q page per_page preset].freeze
24
+ ESSENTIAL_MULTI_KEYS = %i[collection q page per_page offset limit preset].freeze
25
25
 
26
26
  # Canonicalize a label into a case-insensitive Symbol key.
27
27
  # @param label [String, Symbol]
@@ -90,7 +90,7 @@ module SearchEngine
90
90
  # Presets:
91
91
  # - Preset token lives inside each per-search payload under :preset (never top-level)
92
92
  # - mode=:merge — pass through compiler output (includes :preset)
93
- # - mode=:only — retain only ESSENTIAL_MULTI_KEYS (collection,q,page,per_page,preset) after common merge
93
+ # - mode=:only — retain only ESSENTIAL_MULTI_KEYS after common merge
94
94
  # - mode=:lock — defensively drop any keys present in Config.presets.locked_domains from the merged payload
95
95
  # (conflict recording remains in single-search compile)
96
96
  #
@@ -107,7 +107,7 @@ module SearchEngine
107
107
  def to_payloads(common: {})
108
108
  raise ArgumentError, 'common must be a Hash' unless common.is_a?(Hash)
109
109
 
110
- filtered_common = filter_url_only_keys(common)
110
+ filtered_common = filter_url_only_keys(common).transform_keys(&:to_sym)
111
111
 
112
112
  seen = Set.new
113
113
  @entries.map do |e|
@@ -129,8 +129,14 @@ module SearchEngine
129
129
 
130
130
  collection = collection_name_for_relation(e.relation)
131
131
 
132
- # Shallow-merge with per-search winning
133
- merged = filtered_common.merge(per_search)
132
+ common_params = if per_search.key?(:offset) || per_search.key?(:limit)
133
+ filtered_common.except(:page, :per_page)
134
+ elsif per_search.key?(:page) || per_search.key?(:per_page)
135
+ filtered_common.except(:offset, :limit)
136
+ else
137
+ filtered_common
138
+ end
139
+ merged = common_params.merge(per_search)
134
140
  payload = { collection: collection }.merge(EssentialPruner.prune(merged, relation: e.relation))
135
141
  prune_empty_values(payload)
136
142
  end
@@ -142,6 +148,28 @@ module SearchEngine
142
148
  @entries.map { |e| e.relation.klass }
143
149
  end
144
150
 
151
+ # Execute the labeled searches together. Typed results retain each relation's context.
152
+ # Raw responses expose the complete backend envelope, including per-search errors.
153
+ def execute(common: {}, client: nil, url_opts: {}, raw: false)
154
+ keys = labels
155
+ enforce_multi_limit!(keys.size, SearchEngine.config.multi_search_limit)
156
+ searches = to_payloads(common: common)
157
+ client ||= SearchEngine.client
158
+ url_opts = search_url_options(client, url_opts)
159
+ payload = build_multi_event_payload(keys.size, keys, url_opts)
160
+
161
+ Instrumentation.instrument('search_engine.multi_search', payload) do |ctx|
162
+ begin
163
+ response = client.multi_search(searches: searches, url_opts: url_opts)
164
+ rescue Errors::Api => error
165
+ ctx[:http_status] = error.status
166
+ raise augment_multi_api_error(error, keys)
167
+ end
168
+ ctx[:http_status] = 200
169
+ raw ? response : to_result(response, keys)
170
+ end
171
+ end
172
+
145
173
  # ResultSet maps labels to {SearchEngine::Result} while preserving order.
146
174
  class ResultSet
147
175
  # @param pairs [Array<Array(Symbol, SearchEngine::Result)>>] ordered (label, result) pairs
@@ -195,6 +223,64 @@ module SearchEngine
195
223
 
196
224
  private
197
225
 
226
+ # Hydrate each result with the selection, facets and constraints of its relation.
227
+ def to_result(response, keys)
228
+ items = response[:results] || response['results'] if response.is_a?(Hash)
229
+ MultiResult.new(labels: keys, raw_results: items) do |raw, index|
230
+ relation = @entries.fetch(index).relation
231
+ if relation.is_a?(Relation)
232
+ relation.send(:enforce_hit_validator_if_needed!, raw[:found] || raw['found'])
233
+ Result.new(raw, **Hydration::Materializers.result_options(relation))
234
+ else
235
+ Result.new(raw, klass: relation.klass)
236
+ end
237
+ end
238
+ end
239
+
240
+ def enforce_multi_limit!(count, limit)
241
+ return unless count > limit
242
+
243
+ raise ArgumentError,
244
+ "multi_search: #{count} searches exceed limit (#{limit}). " \
245
+ 'Increase `SearchEngine.config.multi_search_limit` if necessary.'
246
+ end
247
+
248
+ def search_url_options(client, overrides)
249
+ defaults = if client.respond_to?(:search_url_options)
250
+ client.search_url_options
251
+ else
252
+ ClientOptions.url_options_from_config
253
+ end
254
+ options = defaults.merge(overrides)
255
+ disabled = @entries.any? do |entry|
256
+ entry.relation.is_a?(Relation) && entry.relation.send(:build_url_opts)[:use_cache] == false
257
+ end
258
+ options[:use_cache] = false if disabled
259
+ options
260
+ end
261
+
262
+ def build_multi_event_payload(count, labels, url_opts)
263
+ {
264
+ searches_count: count,
265
+ labels: labels.map(&:to_s),
266
+ http_status: nil,
267
+ source: :multi,
268
+ url_opts: Observability.filtered_url_opts(url_opts)
269
+ }
270
+ end
271
+
272
+ def augment_multi_api_error(error, labels)
273
+ body = error.body
274
+ results = body[:results] || body['results'] if body.is_a?(Hash)
275
+ return error unless results.is_a?(Array)
276
+
277
+ labels.each_with_index do |label, index|
278
+ failure = MultiResult.entry_error(label, results[index])
279
+ return Errors::Api.new(failure.message, status: error.status, body: body) if failure
280
+ end
281
+ error
282
+ end
283
+
198
284
  def validate_relation!(relation)
199
285
  unless relation.respond_to?(:to_typesense_params) && relation.respond_to?(:klass)
200
286
  raise ArgumentError, 'relation must respond to :to_typesense_params and :klass'
@@ -28,6 +28,21 @@ module SearchEngine
28
28
  # mr.dig('brands').to_a
29
29
  # mr.labels #=> [:products, :brands]
30
30
  class MultiResult
31
+ attr_reader :result_set
32
+
33
+ # Typesense can report individual failures inside a successful HTTP response.
34
+ def self.entry_error(label, item)
35
+ return unless item.is_a?(Hash)
36
+
37
+ code = item[:code] || item['code'] || item[:status] || item['status']
38
+ status = Integer(code, exception: false)
39
+ return unless item[:error] || item['error'] || (status && !(200..299).cover?(status))
40
+
41
+ message = "Multi-search failed for label :#{label}"
42
+ message += " (status #{status})" if status
43
+ Errors::Api.new("#{message}.", status: status, body: item)
44
+ end
45
+
31
46
  # Build a new MultiResult.
32
47
  #
33
48
  # @param labels [Array<String, Symbol>] ordered labels
@@ -36,36 +51,38 @@ module SearchEngine
36
51
  # @raise [ArgumentError] when sizes mismatch, labels invalid/duplicate, or inputs malformed
37
52
  # @see `https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/multi-search-guide`
38
53
  def initialize(labels:, raw_results:, klasses: nil)
39
- @labels = canonicalize_labels(labels)
40
- @map = {}
41
-
42
- # Store the raw array privately to document the one-time network call and hydration
43
- @raw_results = Array(raw_results).freeze
44
-
45
- validate_sizes!(@labels, @raw_results, klasses)
46
-
47
- klass_by_index = build_klass_index(@labels, klasses)
48
-
49
- @labels.each_with_index do |label, index|
50
- raw = @raw_results[index]
51
- kls = klass_by_index[index] || infer_klass_from_raw(raw)
52
- @map[label] = SearchEngine::Result.new(raw, klass: kls)
54
+ keys = canonicalize_labels(labels)
55
+ validate_sizes!(keys, raw_results, klasses)
56
+ keys.each_with_index do |label, index|
57
+ raw = raw_results[index]
58
+ raise ArgumentError, "Expected a result Hash for label :#{label}" unless raw.is_a?(Hash)
59
+
60
+ failure = MultiResult.entry_error(label, raw)
61
+ raise failure if failure
53
62
  end
54
63
 
64
+ klass_by_index = build_klass_index(keys, klasses)
65
+ pairs = keys.each_with_index.map do |label, index|
66
+ raw = raw_results[index]
67
+ result = if block_given?
68
+ yield(raw, index)
69
+ else
70
+ Result.new(raw, klass: klass_by_index[index] || infer_klass_from_raw(raw))
71
+ end
72
+ [label, result]
73
+ end
74
+ @result_set = Multi::ResultSet.new(pairs)
55
75
  freeze
56
76
  end
57
77
 
58
- # Return a shallow copy of labels to preserve immutability.
59
- # @return [Array<Symbol>]
78
+ # Return a shallow copy of the ordered labels.
60
79
  def labels
61
- @labels.dup
80
+ @result_set.labels
62
81
  end
63
82
 
64
- # Fetch a Result by label.
65
- # @param label [String, Symbol]
66
- # @return [SearchEngine::Result, nil]
83
+ # Fetch a result by its case-insensitive label.
67
84
  def [](label)
68
- @map[SearchEngine::Multi.canonicalize_label(label)]
85
+ @result_set[label]
69
86
  rescue ArgumentError
70
87
  nil
71
88
  end
@@ -87,7 +104,7 @@ module SearchEngine
87
104
  # Pure helper: returns a shallow copy and never performs HTTP.
88
105
  # @return [Hash{Symbol=>SearchEngine::Result}]
89
106
  def to_h
90
- @map.dup
107
+ @result_set.to_h
91
108
  end
92
109
 
93
110
  # Iterate over (label, result) in insertion order.
@@ -98,7 +115,7 @@ module SearchEngine
98
115
  def each_label
99
116
  return enum_for(:each_label) unless block_given?
100
117
 
101
- @labels.each { |l| yield([l, @map[l]]) }
118
+ @result_set.each_pair { |label, result| yield([label, result]) }
102
119
  end
103
120
 
104
121
  # Convenience to map over (label, result) pairs.
@@ -335,7 +335,7 @@ module SearchEngine
335
335
  {}
336
336
  end
337
337
  parts = []
338
- %i[q query_by per_page page infix].each do |key|
338
+ %i[q query_by per_page page limit offset infix].each do |key|
339
339
  next unless params_hash.key?(key)
340
340
 
341
341
  parts << format_param(key, params_hash[key])
@@ -540,7 +540,7 @@ module SearchEngine
540
540
  private_class_method :attach_grouping_fields!
541
541
 
542
542
  def self.attach_selected_params!(h, params_hash)
543
- %i[q query_by per_page page infix].each do |key|
543
+ %i[q query_by per_page page limit offset infix].each do |key|
544
544
  h[key.to_s] = params_hash[key] if params_hash.key?(key)
545
545
  end
546
546
  h['filter_by'] = '***' if params_hash.key?(:filter_by)
@@ -12,7 +12,7 @@ module SearchEngine
12
12
 
13
13
  # Whitelisted search parameter keys to include in payload excerpts.
14
14
  PARAM_WHITELIST = %i[
15
- q query_by include_fields exclude_fields per_page page infix filter_by sort_by
15
+ q query_by include_fields exclude_fields per_page page limit offset infix filter_by sort_by
16
16
  group_by group_limit group_missing_values
17
17
  facet_by max_facet_values facet_query
18
18
  num_typos drop_tokens_threshold prioritize_exact_match query_by_weights
@@ -110,7 +110,7 @@ module SearchEngine
110
110
  return str unless str.is_a?(String)
111
111
 
112
112
  # Mask obvious quoted literals first
113
- redacted = str.gsub(/"[^"]*"|'[^']*'/, '***')
113
+ redacted = str.gsub(/`[^`]*`|"[^"]*"|'[^']*'/, '***')
114
114
 
115
115
  # Mask numeric literals (best-effort)
116
116
  redacted.gsub(/\b\d+(?:\.\d+)?\b/, '***')
@@ -124,10 +124,10 @@ module SearchEngine
124
124
  def self.redact_filter_by(filter)
125
125
  return filter unless filter.is_a?(String)
126
126
 
127
- # Replace values that follow a comparator or a colon with *** until a
128
- # delimiter is reached. Also mask quoted strings and numbers.
129
- masked = filter.gsub(/([!:><=]{1,2})\s*([^\s)&|]+)/, '\1***')
130
- masked = masked.gsub(/"[^"]*"|'[^']*'/, '***')
127
+ # Mask whole literals first: their contents may look like expressions.
128
+ # Once quoted values are gone, a list can safely be masked as a unit.
129
+ masked = filter.gsub(/`[^`]*`|"[^"]*"|'[^']*'/, '***').gsub(/\[[^\]]*\]/, '***')
130
+ masked = masked.gsub(/([!:><=]{1,2})\s*([^\s)&|]+)/, '\1***')
131
131
  masked.gsub(/\b\d+(?:\.\d+)?\b/, '***')
132
132
  end
133
133
 
@@ -0,0 +1,17 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SearchEngine
4
+ module PostgresOutbox
5
+ # Operational reads must observe current delivery state and acquire row locks on each poll.
6
+ module DatabaseQueries
7
+ private
8
+
9
+ def select_rows(sql)
10
+ result = connection.respond_to?(:exec_query) ? connection.exec_query(sql) : connection.select_all(sql)
11
+ return result.to_a if result.respond_to?(:to_a)
12
+
13
+ Array(result)
14
+ end
15
+ end
16
+ end
17
+ end
@@ -1,7 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'socket'
4
- require 'securerandom'
5
4
  require 'set'
6
5
  require 'search_engine/cache'
7
6
 
@@ -10,7 +9,6 @@ module SearchEngine
10
9
  # Orchestrates one bounded PostgreSQL outbox drain pass.
11
10
  class Drainer
12
11
  BLOCKED_ERROR = 'Skipped because an earlier outbox collection group failed'
13
- LEASE_OWNER_MAX_LENGTH = 255
14
12
 
15
13
  # @param repository [SearchEngine::PostgresOutbox::Repository]
16
14
  # @param processor [#call]
@@ -31,8 +29,8 @@ module SearchEngine
31
29
  'search_engine.postgres_outbox.drain',
32
30
  drain_payload(limit)
33
31
  ) do |payload|
34
- lease_owner = next_lease_owner
35
- events = repository.claim_pending(limit: limit, worker_id: lease_owner)
32
+ lease_owner = Event.new_lease_owner(worker_id) unless repository_assigns_lease_owner?
33
+ events = repository.claim_pending(limit: limit, worker_id: lease_owner || worker_id)
36
34
  summary = empty_summary(events)
37
35
  next summary if events.empty?
38
36
 
@@ -43,7 +41,7 @@ module SearchEngine
43
41
  record_acknowledgements(summary, :superseded, superseded_events, superseded_ids)
44
42
 
45
43
  ordered = SearchEngine::DependencyPlanner.order_events(kept)
46
- process_ordered_events(ordered, summary, lease_owner: lease_owner)
44
+ process_ordered_events(ordered, summary, lease_owner: lease_owner || events.first.event_lease_owner)
47
45
  payload.merge!(summary)
48
46
  summary
49
47
  end
@@ -53,6 +51,10 @@ module SearchEngine
53
51
 
54
52
  attr_reader :repository, :processor, :worker_id, :target_key
55
53
 
54
+ def repository_assigns_lease_owner?
55
+ repository.respond_to?(:assigns_event_lease_owner?) && repository.assigns_event_lease_owner?
56
+ end
57
+
56
58
  def empty_summary(events)
57
59
  summary = {
58
60
  claimed: events.size,
@@ -250,13 +252,6 @@ module SearchEngine
250
252
  "#{Socket.gethostname}:#{$PROCESS_ID}:#{Thread.current.object_id}"
251
253
  end
252
254
 
253
- def next_lease_owner
254
- suffix = SecureRandom.uuid
255
- prefix_limit = LEASE_OWNER_MAX_LENGTH - suffix.length - 1
256
- prefix = worker_id.to_s.gsub(/[^a-zA-Z0-9_.:-]/, '_')[0, prefix_limit]
257
- "#{prefix}:#{suffix}"
258
- end
259
-
260
255
  def processor_context(lease_owner)
261
256
  context = { worker_id: worker_id, lease_owner: lease_owner }
262
257
  context[:target_key] = target_key if target_key
@@ -1,9 +1,12 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'securerandom'
4
+
3
5
  module SearchEngine
4
6
  module PostgresOutbox
5
7
  # Immutable value object for one PostgreSQL outbox row.
6
8
  class Event
9
+ LEASE_OWNER_MAX_LENGTH = 255
7
10
  VALID_OPERATIONS = %i[upsert delete].freeze
8
11
 
9
12
  attr_reader :id,
@@ -16,10 +19,18 @@ module SearchEngine
16
19
  :attempts,
17
20
  :payload,
18
21
  :created_at,
22
+ :event_lease_owner,
19
23
  :delivery_id,
20
24
  :target_key,
21
25
  :delivery_lease_owner
22
26
 
27
+ def self.new_lease_owner(worker_id)
28
+ suffix = SecureRandom.uuid
29
+ prefix_limit = LEASE_OWNER_MAX_LENGTH - suffix.length - 1
30
+ prefix = worker_id.to_s.gsub(/[^a-zA-Z0-9_.:-]/, '_')[0, prefix_limit]
31
+ "#{prefix}:#{suffix}"
32
+ end
33
+
23
34
  # @param row [Hash] outbox row with string or symbol keys
24
35
  # @raise [ArgumentError] when operation is not upsert/delete
25
36
  def initialize(row)
@@ -33,6 +44,7 @@ module SearchEngine
33
44
  @attempts = (value(row, :delivery_attempts) || value(row, :attempts)).to_i
34
45
  @payload = value(row, :payload) || {}
35
46
  @created_at = value(row, :created_at)
47
+ @event_lease_owner = string_value(row, :event_lease_owner)
36
48
  @delivery_id = value(row, :delivery_id)
37
49
  @target_key = string_value(row, :target_key)
38
50
  @delivery_lease_owner = string_value(row, :delivery_lease_owner)