search-engine-for-typesense 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +21 -0
  3. data/README.md +148 -0
  4. data/app/search_engine/search_engine/app_info.rb +11 -0
  5. data/app/search_engine/search_engine/index_partition_job.rb +170 -0
  6. data/lib/generators/search_engine/install/install_generator.rb +20 -0
  7. data/lib/generators/search_engine/install/templates/initializer.rb.tt +230 -0
  8. data/lib/generators/search_engine/model/model_generator.rb +86 -0
  9. data/lib/generators/search_engine/model/templates/model.rb.tt +12 -0
  10. data/lib/search-engine-for-typesense.rb +12 -0
  11. data/lib/search_engine/active_record_syncable.rb +247 -0
  12. data/lib/search_engine/admin/stopwords.rb +125 -0
  13. data/lib/search_engine/admin/synonyms.rb +125 -0
  14. data/lib/search_engine/admin.rb +12 -0
  15. data/lib/search_engine/ast/and.rb +52 -0
  16. data/lib/search_engine/ast/binary_op.rb +75 -0
  17. data/lib/search_engine/ast/eq.rb +19 -0
  18. data/lib/search_engine/ast/group.rb +18 -0
  19. data/lib/search_engine/ast/gt.rb +12 -0
  20. data/lib/search_engine/ast/gte.rb +12 -0
  21. data/lib/search_engine/ast/in.rb +28 -0
  22. data/lib/search_engine/ast/lt.rb +12 -0
  23. data/lib/search_engine/ast/lte.rb +12 -0
  24. data/lib/search_engine/ast/matches.rb +55 -0
  25. data/lib/search_engine/ast/node.rb +176 -0
  26. data/lib/search_engine/ast/not_eq.rb +13 -0
  27. data/lib/search_engine/ast/not_in.rb +24 -0
  28. data/lib/search_engine/ast/or.rb +52 -0
  29. data/lib/search_engine/ast/prefix.rb +51 -0
  30. data/lib/search_engine/ast/raw.rb +41 -0
  31. data/lib/search_engine/ast/unary_op.rb +43 -0
  32. data/lib/search_engine/ast.rb +101 -0
  33. data/lib/search_engine/base/creation.rb +727 -0
  34. data/lib/search_engine/base/deletion.rb +80 -0
  35. data/lib/search_engine/base/display_coercions.rb +36 -0
  36. data/lib/search_engine/base/hydration.rb +312 -0
  37. data/lib/search_engine/base/index_maintenance/cleanup.rb +202 -0
  38. data/lib/search_engine/base/index_maintenance/lifecycle.rb +251 -0
  39. data/lib/search_engine/base/index_maintenance/schema.rb +117 -0
  40. data/lib/search_engine/base/index_maintenance.rb +459 -0
  41. data/lib/search_engine/base/indexing_dsl.rb +255 -0
  42. data/lib/search_engine/base/joins.rb +479 -0
  43. data/lib/search_engine/base/model_dsl.rb +472 -0
  44. data/lib/search_engine/base/presets.rb +43 -0
  45. data/lib/search_engine/base/pretty_printer.rb +315 -0
  46. data/lib/search_engine/base/relation_delegation.rb +42 -0
  47. data/lib/search_engine/base/scopes.rb +113 -0
  48. data/lib/search_engine/base/updating.rb +92 -0
  49. data/lib/search_engine/base.rb +38 -0
  50. data/lib/search_engine/bulk.rb +284 -0
  51. data/lib/search_engine/cache.rb +33 -0
  52. data/lib/search_engine/cascade.rb +531 -0
  53. data/lib/search_engine/cli/doctor.rb +631 -0
  54. data/lib/search_engine/cli/support.rb +217 -0
  55. data/lib/search_engine/cli.rb +222 -0
  56. data/lib/search_engine/client/http_adapter.rb +63 -0
  57. data/lib/search_engine/client/request_builder.rb +92 -0
  58. data/lib/search_engine/client/services/base.rb +74 -0
  59. data/lib/search_engine/client/services/collections.rb +161 -0
  60. data/lib/search_engine/client/services/documents.rb +214 -0
  61. data/lib/search_engine/client/services/operations.rb +152 -0
  62. data/lib/search_engine/client/services/search.rb +190 -0
  63. data/lib/search_engine/client/services.rb +29 -0
  64. data/lib/search_engine/client.rb +765 -0
  65. data/lib/search_engine/client_options.rb +20 -0
  66. data/lib/search_engine/collection_resolver.rb +191 -0
  67. data/lib/search_engine/collections_graph.rb +330 -0
  68. data/lib/search_engine/compiled_params.rb +143 -0
  69. data/lib/search_engine/compiler.rb +383 -0
  70. data/lib/search_engine/config/observability.rb +27 -0
  71. data/lib/search_engine/config/presets.rb +92 -0
  72. data/lib/search_engine/config/selection.rb +16 -0
  73. data/lib/search_engine/config/typesense.rb +48 -0
  74. data/lib/search_engine/config/validators.rb +97 -0
  75. data/lib/search_engine/config.rb +917 -0
  76. data/lib/search_engine/console_helpers.rb +130 -0
  77. data/lib/search_engine/deletion.rb +103 -0
  78. data/lib/search_engine/dispatcher.rb +125 -0
  79. data/lib/search_engine/dsl/parser.rb +582 -0
  80. data/lib/search_engine/engine.rb +167 -0
  81. data/lib/search_engine/errors.rb +290 -0
  82. data/lib/search_engine/filters/sanitizer.rb +189 -0
  83. data/lib/search_engine/hydration/materializers.rb +808 -0
  84. data/lib/search_engine/hydration/selection_context.rb +96 -0
  85. data/lib/search_engine/indexer/batch_planner.rb +76 -0
  86. data/lib/search_engine/indexer/bulk_import.rb +626 -0
  87. data/lib/search_engine/indexer/import_dispatcher.rb +198 -0
  88. data/lib/search_engine/indexer/retry_policy.rb +103 -0
  89. data/lib/search_engine/indexer.rb +747 -0
  90. data/lib/search_engine/instrumentation.rb +308 -0
  91. data/lib/search_engine/joins/guard.rb +202 -0
  92. data/lib/search_engine/joins/resolver.rb +95 -0
  93. data/lib/search_engine/logging/color.rb +78 -0
  94. data/lib/search_engine/logging/format_helpers.rb +92 -0
  95. data/lib/search_engine/logging/partition_progress.rb +53 -0
  96. data/lib/search_engine/logging_subscriber.rb +388 -0
  97. data/lib/search_engine/mapper.rb +785 -0
  98. data/lib/search_engine/multi.rb +286 -0
  99. data/lib/search_engine/multi_result.rb +186 -0
  100. data/lib/search_engine/notifications/compact_logger.rb +675 -0
  101. data/lib/search_engine/observability.rb +162 -0
  102. data/lib/search_engine/operations.rb +58 -0
  103. data/lib/search_engine/otel.rb +227 -0
  104. data/lib/search_engine/partitioner.rb +128 -0
  105. data/lib/search_engine/ranking_plan.rb +118 -0
  106. data/lib/search_engine/registry.rb +158 -0
  107. data/lib/search_engine/relation/compiler.rb +711 -0
  108. data/lib/search_engine/relation/deletion.rb +37 -0
  109. data/lib/search_engine/relation/dsl/filters.rb +624 -0
  110. data/lib/search_engine/relation/dsl/selection.rb +240 -0
  111. data/lib/search_engine/relation/dsl.rb +903 -0
  112. data/lib/search_engine/relation/dx/dry_run.rb +59 -0
  113. data/lib/search_engine/relation/dx/friendly_where.rb +24 -0
  114. data/lib/search_engine/relation/dx.rb +231 -0
  115. data/lib/search_engine/relation/materializers.rb +118 -0
  116. data/lib/search_engine/relation/options.rb +138 -0
  117. data/lib/search_engine/relation/state.rb +274 -0
  118. data/lib/search_engine/relation/updating.rb +44 -0
  119. data/lib/search_engine/relation.rb +623 -0
  120. data/lib/search_engine/result.rb +664 -0
  121. data/lib/search_engine/schema.rb +1083 -0
  122. data/lib/search_engine/sources/active_record_source.rb +185 -0
  123. data/lib/search_engine/sources/base.rb +62 -0
  124. data/lib/search_engine/sources/lambda_source.rb +55 -0
  125. data/lib/search_engine/sources/sql_source.rb +196 -0
  126. data/lib/search_engine/sources.rb +71 -0
  127. data/lib/search_engine/stale_rules.rb +160 -0
  128. data/lib/search_engine/test/minitest_assertions.rb +57 -0
  129. data/lib/search_engine/test/offline_client.rb +134 -0
  130. data/lib/search_engine/test/rspec_matchers.rb +77 -0
  131. data/lib/search_engine/test/stub_client.rb +201 -0
  132. data/lib/search_engine/test.rb +66 -0
  133. data/lib/search_engine/test_autoload.rb +8 -0
  134. data/lib/search_engine/update.rb +35 -0
  135. data/lib/search_engine/version.rb +7 -0
  136. data/lib/search_engine.rb +332 -0
  137. data/lib/tasks/search_engine.rake +501 -0
  138. data/lib/tasks/search_engine_doctor.rake +16 -0
  139. metadata +225 -0
@@ -0,0 +1,675 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'logger'
4
+ require 'active_support/notifications'
5
+ require 'json'
6
+ require 'search_engine/logging/format_helpers'
7
+
8
+ module SearchEngine
9
+ module Notifications
10
+ # Opt-in compact logging subscriber for SearchEngine AS::Notifications events.
11
+ #
12
+ # Emits concise, single-line log entries with redacted parameters and
13
+ # stable keys. Designed to be lightweight and reloader-safe.
14
+ #
15
+ # Usage:
16
+ # SearchEngine::Notifications::CompactLogger.subscribe
17
+ # SearchEngine::Notifications::CompactLogger.unsubscribe
18
+ #
19
+ # @since M8
20
+ # @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/observability#logging
21
+ class CompactLogger
22
+ EVENT_SEARCH = 'search_engine.search'
23
+ EVENT_MULTI = 'search_engine.multi_search'
24
+ EVENT_SCHEMA_DIFF = 'search_engine.schema.diff'
25
+ EVENT_SCHEMA_APPLY = 'search_engine.schema.apply'
26
+ EVENT_JOINS_COMPILE = 'search_engine.joins.compile'
27
+ EVENT_PARTITION_START = 'search_engine.indexer.partition_start'
28
+ EVENT_PARTITION_FINISH = 'search_engine.indexer.partition_finish'
29
+ EVENT_BATCH_IMPORT = 'search_engine.indexer.batch_import'
30
+ EVENT_DELETE_STALE = 'search_engine.indexer.delete_stale'
31
+ # Legacy (emitted by current codebase): stale_deletes.*
32
+ LEGACY_STALE_STARTED = 'search_engine.stale_deletes.started'
33
+ LEGACY_STALE_FINISHED = 'search_engine.stale_deletes.finished'
34
+ LEGACY_STALE_ERROR = 'search_engine.stale_deletes.error'
35
+ LEGACY_STALE_SKIPPED = 'search_engine.stale_deletes.skipped'
36
+
37
+ # Subscribe to SearchEngine notifications.
38
+ #
39
+ # @param logger [#info,#warn,#error] Logger instance; defaults to config.logger or $stdout
40
+ # @param level [Symbol] one of :debug, :info, :warn, :error
41
+ # @param include_params [Boolean] when true, include whitelisted params for single-search
42
+ # @param format [Symbol, nil] :kv or :json; defaults to config.observability.log_format
43
+ # @return [Array<Object>] subscription handles that can be passed to {.unsubscribe}
44
+ # @since M8
45
+ # @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/observability#logging
46
+ def self.subscribe(logger: default_logger, level: :info, include_params: false, format: nil)
47
+ return [] unless defined?(ActiveSupport::Notifications)
48
+
49
+ severity = map_level(level)
50
+ log = logger || default_logger
51
+ fmt = (format || (SearchEngine.config.observability&.log_format || :kv)).to_sym
52
+
53
+ @last_handle = build_subscriptions(log, severity, include_params: include_params, fmt: fmt)
54
+ end
55
+
56
+ # Unsubscribe a previous subscription set.
57
+ # @param handle [Array<Object>, Object, nil] handles returned by {.subscribe}
58
+ # @return [Boolean] true when unsubscribed
59
+ # @since M8
60
+ # @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/observability#logging
61
+ def self.unsubscribe(handle = @last_handle)
62
+ return false unless handle
63
+
64
+ Array(handle).each do |sub|
65
+ ActiveSupport::Notifications.unsubscribe(sub)
66
+ end
67
+ @last_handle = nil
68
+ true
69
+ end
70
+
71
+ # Internal: Emit one compact line for a notification event.
72
+ def self.emit_line(logger, severity, event, include_params:, multi:)
73
+ return unless logger
74
+ return unless allow_log?(logger, severity)
75
+
76
+ p = event.payload
77
+ duration_ms = event.duration.round(1)
78
+
79
+ parts = []
80
+ if multi
81
+ parts.concat(multi_parts(p))
82
+ else
83
+ parts.concat(single_parts(p))
84
+ end
85
+
86
+ parts.concat(status_parts(p, duration_ms))
87
+ parts.concat(url_parts(p))
88
+ parts.concat(selection_parts(p)) unless multi
89
+ parts.concat(preset_parts(p)) unless multi
90
+ parts.concat(curation_parts(p)) unless multi
91
+
92
+ parts.concat(param_parts(p)) if include_params && !multi
93
+
94
+ line = parts.join(' ')
95
+ log_with_level(logger, severity, line)
96
+ rescue StandardError
97
+ nil
98
+ end
99
+
100
+ # Schema diff formatter
101
+ def self.emit_schema_diff(logger, severity, event, format: :kv)
102
+ return unless logger
103
+ return unless allow_log?(logger, severity)
104
+
105
+ p = event.payload
106
+ h = {
107
+ 'event' => 'schema.diff',
108
+ 'collection' => p[:collection] || p[:logical],
109
+ 'fields.changed' => p[:fields_changed_count],
110
+ 'fields.added' => p[:added_count],
111
+ 'fields.removed' => p[:removed_count],
112
+ 'in_sync' => p[:in_sync],
113
+ 'duration.ms' => event.duration.round(1)
114
+ }
115
+ emit(logger, severity, h, format)
116
+ rescue StandardError
117
+ nil
118
+ end
119
+
120
+ # Schema apply formatter
121
+ def self.emit_schema_apply(logger, severity, event, format: :kv)
122
+ return unless logger
123
+ return unless allow_log?(logger, severity)
124
+
125
+ p = event.payload
126
+ h = {
127
+ 'event' => 'schema.apply',
128
+ 'collection' => p[:collection] || p[:logical],
129
+ 'into' => p[:physical_new] || p[:new_physical],
130
+ 'alias_swapped' => p[:alias_swapped],
131
+ 'retention_deleted_count' => p[:retention_deleted_count] || p[:dropped_count],
132
+ 'status' => p[:status] || 'ok',
133
+ 'duration.ms' => event.duration.round(1)
134
+ }
135
+ emit(logger, severity, h, format)
136
+ rescue StandardError
137
+ nil
138
+ end
139
+
140
+ # JOINs compile formatter
141
+ # Emits a single-line summary of associations and their usage, with lengths only
142
+ # and without raw filter/include/sort strings.
143
+ def self.emit_joins_compile(logger, severity, event, format: :kv)
144
+ return unless logger
145
+ return unless allow_log?(logger, severity)
146
+
147
+ p = event.payload
148
+ used = p[:used_in] || {}
149
+ used_compact = []
150
+ %i[include filter sort].each do |k|
151
+ arr = Array(used[k]).map(&:to_s)
152
+ used_compact << "#{k}:#{arr.join(',')}" unless arr.empty?
153
+ end
154
+
155
+ h = {
156
+ 'event' => 'joins.compile',
157
+ 'collection' => p[:collection],
158
+ 'joins.assocs' => Array(p[:assocs]).join(','),
159
+ 'joins.count' => p[:join_count],
160
+ 'joins.used_in' => used_compact.join('|'),
161
+ 'joins.include.len' => p[:include_len],
162
+ 'joins.filter.len' => p[:filter_len],
163
+ 'joins.sort.len' => p[:sort_len],
164
+ 'has_joins' => p[:has_joins],
165
+ 'duration.ms' => p[:duration_ms] || event.duration.round(1)
166
+ }
167
+ emit(logger, severity, h.compact, format)
168
+ rescue StandardError
169
+ nil
170
+ end
171
+
172
+ def self.emit_partition_start(logger, severity, event, format: :kv)
173
+ return unless logger
174
+ return unless allow_log?(logger, severity)
175
+
176
+ p = event.payload
177
+ h = {
178
+ 'event' => 'indexer.partition_start',
179
+ 'collection' => p[:collection],
180
+ 'into' => p[:into],
181
+ 'partition' => p[:partition_hash] || p[:partition],
182
+ 'dispatch_mode' => p[:dispatch_mode],
183
+ 'job_id' => p[:job_id],
184
+ 'timestamp' => p[:timestamp]
185
+ }
186
+ emit(logger, severity, h, format)
187
+ rescue StandardError
188
+ nil
189
+ end
190
+
191
+ def self.emit_partition_finish(logger, severity, event, format: :kv)
192
+ return unless logger
193
+ return unless allow_log?(logger, severity)
194
+
195
+ p = event.payload
196
+ h = {
197
+ 'event' => 'indexer.partition_finish',
198
+ 'collection' => p[:collection],
199
+ 'into' => p[:into],
200
+ 'partition' => p[:partition_hash] || p[:partition],
201
+ 'batches.total' => p[:batches_total],
202
+ 'docs.total' => p[:docs_total],
203
+ 'success.total' => p[:success_total],
204
+ 'failed.total' => p[:failed_total],
205
+ 'status' => p[:status],
206
+ 'duration.ms' => event.duration.round(1)
207
+ }
208
+ emit(logger, severity, h, format)
209
+ rescue StandardError
210
+ nil
211
+ end
212
+
213
+ def self.emit_batch_import(logger, severity, event, format: :kv)
214
+ return unless logger
215
+ return unless allow_log?(logger, severity)
216
+
217
+ p = event.payload
218
+ h = {
219
+ 'event' => 'indexer.batch_import',
220
+ 'collection' => p[:collection],
221
+ 'into' => p[:into],
222
+ 'batch_index' => p[:batch_index],
223
+ 'docs.count' => p[:docs_count],
224
+ 'success.count' => p[:success_count],
225
+ 'failure.count' => p[:failure_count],
226
+ 'attempts' => p[:attempts],
227
+ 'http_status' => p[:http_status],
228
+ 'bytes.sent' => p[:bytes_sent],
229
+ 'duration.ms' => event.duration.round(1),
230
+ 'transient_retry' => p[:transient_retry]
231
+ }
232
+ if SearchEngine.config.observability.include_error_messages && p[:error_sample]
233
+ h['error.sample_count'] = Array(p[:error_sample]).size
234
+ h['message'] =
235
+ SearchEngine::Observability.truncate_message(Array(p[:error_sample]).first,
236
+ SearchEngine.config.observability.max_message_length
237
+ )
238
+ end
239
+ emit(logger, severity, h, format)
240
+ rescue StandardError
241
+ nil
242
+ end
243
+
244
+ def self.emit_delete_stale(logger, severity, event, format: :kv, legacy: nil)
245
+ return unless logger
246
+ return unless allow_log?(logger, severity)
247
+
248
+ p = normalize_stale_payload(event.payload, legacy: legacy)
249
+ h = {
250
+ 'event' => 'indexer.delete_stale',
251
+ 'collection' => p[:collection],
252
+ 'into' => p[:into],
253
+ 'partition' => p[:partition_hash] || p[:partition],
254
+ 'filter.hash' => p[:filter_hash],
255
+ 'deleted.count' => p[:deleted_count],
256
+ 'status' => p[:status] || 'ok',
257
+ 'reason' => p[:reason],
258
+ 'duration.ms' => p[:duration_ms] || event.duration.round(1)
259
+ }
260
+ emit(logger, severity, h, format)
261
+ rescue StandardError
262
+ nil
263
+ end
264
+
265
+ # Emit according to format
266
+ def self.emit(logger, severity, hash, format)
267
+ line =
268
+ if format == :json
269
+ JSON.generate(hash.compact)
270
+ else
271
+ SearchEngine::Logging::FormatHelpers.kv_compact(hash)
272
+ end
273
+ log_with_level(logger, severity, line)
274
+ end
275
+
276
+ def self.normalize_stale_payload(p, legacy: nil)
277
+ h = p.dup
278
+ case legacy
279
+ when :started
280
+ h[:status] = 'started'
281
+ when :finished
282
+ h[:status] = 'ok'
283
+ when :error
284
+ h[:status] = 'failed'
285
+ when :skipped
286
+ h[:status] = 'skipped'
287
+ end
288
+ h
289
+ end
290
+ private_class_method :normalize_stale_payload
291
+
292
+ def self.multi_parts(payload)
293
+ labels = Array(payload[:labels]).map(&:to_s)
294
+ labels = Array(payload[:collections]).map(&:to_s) if labels.empty? || labels.all?(&:empty?)
295
+ count = payload[:searches_count]
296
+ count ||= (payload[:params].is_a?(Array) ? payload[:params].size : nil)
297
+
298
+ parts = ['[se.multi]']
299
+ parts << "count=#{count}" if count
300
+ parts << "labels=#{labels.join(',')}" unless labels.empty?
301
+ parts
302
+ end
303
+ private_class_method :multi_parts
304
+
305
+ def self.single_parts(payload)
306
+ parts = ['[se.search]']
307
+ parts << "collection=#{payload[:collection]}" if payload[:collection]
308
+ parts
309
+ end
310
+ private_class_method :single_parts
311
+
312
+ def self.status_parts(payload, duration_ms)
313
+ status_val = payload.key?(:http_status) ? payload[:http_status] : (payload[:status] || 'ok')
314
+ ["status=#{status_val}", "duration=#{duration_ms}ms"]
315
+ end
316
+ private_class_method :status_parts
317
+
318
+ def self.url_parts(payload)
319
+ url = payload[:url_opts]
320
+ return [] unless url.is_a?(Hash)
321
+
322
+ results = []
323
+ results << "cache=#{url[:use_cache] ? true : false}" if url.key?(:use_cache)
324
+ results << "ttl=#{url[:cache_ttl]}" if url.key?(:cache_ttl)
325
+ results
326
+ end
327
+ private_class_method :url_parts
328
+
329
+ def self.param_parts(payload)
330
+ params_hash = if payload[:params_preview].is_a?(Hash)
331
+ payload[:params_preview]
332
+ elsif payload[:params].is_a?(Hash)
333
+ payload[:params]
334
+ else
335
+ {}
336
+ end
337
+ parts = []
338
+ %i[q query_by per_page page infix].each do |key|
339
+ next unless params_hash.key?(key)
340
+
341
+ parts << format_param(key, params_hash[key])
342
+ end
343
+ parts << 'filter_by=***' if params_hash.key?(:filter_by)
344
+ parts
345
+ end
346
+ private_class_method :param_parts
347
+
348
+ def self.selection_parts(payload)
349
+ inc = (payload[:selection_include_count] || 0).to_i
350
+ exc = (payload[:selection_exclude_count] || 0).to_i
351
+ nest = (payload[:selection_nested_assoc_count] || 0).to_i
352
+ ["sel=I:#{inc}|X:#{exc}|N:#{nest}"]
353
+ end
354
+ private_class_method :selection_parts
355
+
356
+ # Compact preset parts for text logs (single-line, allocation-light)
357
+ def self.preset_parts(payload)
358
+ name = payload[:preset_name] || (payload[:params].is_a?(Hash) ? payload[:params][:preset] : nil)
359
+ return [] unless name
360
+
361
+ mode = payload[:preset_mode]
362
+ pk_count = (payload[:preset_pruned_keys_count] || Array(payload[:preset_pruned_keys]).size).to_i
363
+ ld_count = payload[:preset_locked_domains_count]
364
+ if ld_count.nil?
365
+ begin
366
+ ld_count = SearchEngine.config.presets.locked_domains.size
367
+ rescue StandardError
368
+ ld_count = 0
369
+ end
370
+ end
371
+ ld_count = ld_count.to_i
372
+
373
+ # Token: pz=<name>|m=<mode>|pk=<count>|ld=<count>
374
+ # Truncate name conservatively to keep line short
375
+ shown_name = name.to_s
376
+ shown_name = shown_name.length > 64 ? "#{shown_name[0, 64]}…" : shown_name
377
+
378
+ parts = ["pz=#{shown_name}"]
379
+ parts << "m=#{mode}" if mode
380
+ parts << "pk=#{pk_count}" if pk_count.positive?
381
+ parts << "ld=#{ld_count}" if ld_count.positive?
382
+
383
+ # Optionally include small list of pruned keys when small (<=3)
384
+ keys = Array(payload[:preset_pruned_keys]).map { |k| k.respond_to?(:to_sym) ? k.to_sym : k }.grep(Symbol)
385
+ parts << "pk=[#{keys.map(&:to_s).join(',')}]" if keys.size.positive? && keys.size <= 3
386
+
387
+ [parts.join('|')]
388
+ rescue StandardError
389
+ []
390
+ end
391
+ private_class_method :preset_parts
392
+
393
+ # Compact curation parts for text logs
394
+ def self.curation_parts(payload)
395
+ pcount = (payload[:curation_pinned_count] || 0).to_i
396
+ hcount = (payload[:curation_hidden_count] || 0).to_i
397
+ fflag = if payload.key?(:curation_filter_flag)
398
+ payload[:curation_filter_flag].nil? ? '∅' : payload[:curation_filter_flag]
399
+ end
400
+ has_tags = payload[:curation_has_override_tags] ? true : false
401
+ seg = ["cu=p:#{pcount}|h:#{hcount}"]
402
+ seg << "f:#{fflag}" unless fflag.nil?
403
+ seg << "t:#{has_tags ? 1 : 0}"
404
+
405
+ parts = [seg.join('|')]
406
+ if payload[:curation_conflict_type]
407
+ type = payload[:curation_conflict_type].to_s
408
+ parts << "cf=#{type}"
409
+ end
410
+ parts
411
+ rescue StandardError
412
+ []
413
+ end
414
+ private_class_method :curation_parts
415
+
416
+ # Map a Symbol severity to Logger integer constant.
417
+ def self.map_level(level)
418
+ case level.to_s
419
+ when 'debug' then Logger::DEBUG
420
+ when 'info' then Logger::INFO
421
+ when 'warn' then Logger::WARN
422
+ when 'error' then Logger::ERROR
423
+ else Logger::INFO
424
+ end
425
+ end
426
+
427
+ # Minimal check for logger level
428
+ def self.allow_log?(logger, severity)
429
+ return true unless logger.respond_to?(:level)
430
+
431
+ logger.level <= severity
432
+ end
433
+ private_class_method :allow_log?
434
+
435
+ def self.format_param(key, value)
436
+ case key
437
+ when :q then %(q="#{SearchEngine::Observability.truncate_q(value)}")
438
+ else "#{key}=#{value}"
439
+ end
440
+ end
441
+ private_class_method :format_param
442
+
443
+ def self.default_logger
444
+ if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
445
+ Rails.logger
446
+ else
447
+ ::Logger.new($stdout)
448
+ end
449
+ end
450
+
451
+ def self.log_with_level(logger, severity, line)
452
+ case severity
453
+ when Logger::DEBUG then logger.debug(line)
454
+ when Logger::INFO then logger.info(line)
455
+ when Logger::WARN then logger.warn(line)
456
+ else logger.error(line)
457
+ end
458
+ end
459
+ private_class_method :log_with_level
460
+
461
+ # Build JSON object for search/multi events
462
+ def self.build_json_hash(payload, duration_ms:, multi:, include_params: false)
463
+ if multi
464
+ build_json_hash_multi(payload, duration_ms: duration_ms)
465
+ else
466
+ build_json_hash_single(payload, duration_ms: duration_ms, include_params: include_params)
467
+ end
468
+ end
469
+ private_class_method :build_json_hash
470
+
471
+ def self.build_json_hash_multi(payload, duration_ms:)
472
+ labels = Array(payload[:labels]).map(&:to_s)
473
+ labels = Array(payload[:collections]).map(&:to_s) if labels.empty? || labels.all?(&:empty?)
474
+ {
475
+ 'event' => 'multi',
476
+ 'labels' => (labels unless labels.empty?),
477
+ 'count' => payload[:searches_count] || (payload[:params].is_a?(Array) ? payload[:params].size : nil),
478
+ 'status' => payload[:http_status] || payload[:status] || 'ok',
479
+ 'duration.ms' => duration_ms,
480
+ 'cache' => extract_cache_flag(payload[:url_opts]),
481
+ 'ttl' => extract_ttl(payload[:url_opts])
482
+ }.compact
483
+ end
484
+ private_class_method :build_json_hash_multi
485
+
486
+ def self.build_json_hash_single(payload, duration_ms:, include_params:)
487
+ params_hash = extract_params_hash(payload)
488
+ h = base_single_json_hash(payload, duration_ms)
489
+ attach_grouping_fields!(h, params_hash)
490
+ attach_selected_params!(h, params_hash) if include_params
491
+ h.compact
492
+ end
493
+ private_class_method :build_json_hash_single
494
+
495
+ def self.extract_params_hash(payload)
496
+ if payload[:params_preview].is_a?(Hash)
497
+ payload[:params_preview]
498
+ elsif payload[:params].is_a?(Hash)
499
+ payload[:params]
500
+ else
501
+ {}
502
+ end
503
+ end
504
+ private_class_method :extract_params_hash
505
+
506
+ def self.base_single_json_hash(payload, duration_ms)
507
+ {
508
+ 'event' => 'search',
509
+ 'collection' => payload[:collection],
510
+ 'status' => payload[:http_status] || payload[:status] || 'ok',
511
+ 'duration.ms' => duration_ms,
512
+ 'cache' => extract_cache_flag(payload[:url_opts]),
513
+ 'ttl' => extract_ttl(payload[:url_opts]),
514
+ 'selection_include_count' => (payload[:selection_include_count] || 0).to_i,
515
+ 'selection_exclude_count' => (payload[:selection_exclude_count] || 0).to_i,
516
+ 'selection_nested_assoc_count' => (payload[:selection_nested_assoc_count] || 0).to_i,
517
+ 'preset_name' => payload[:preset_name],
518
+ 'preset_mode' => payload[:preset_mode],
519
+ 'preset_pruned_keys_count' => payload[:preset_pruned_keys_count],
520
+ 'preset_locked_domains_count' => payload[:preset_locked_domains_count],
521
+ 'preset_pruned_keys' => begin
522
+ arr = Array(payload[:preset_pruned_keys])
523
+ arr.empty? ? nil : arr.map(&:to_s)
524
+ end,
525
+ 'curation_pinned_count' => payload[:curation_pinned_count],
526
+ 'curation_hidden_count' => payload[:curation_hidden_count],
527
+ 'curation_has_override_tags' => payload[:curation_has_override_tags],
528
+ 'curation_filter_flag' => (payload.key?(:curation_filter_flag) ? payload[:curation_filter_flag] : nil),
529
+ 'curation_conflict_type' => payload[:curation_conflict_type],
530
+ 'curation_conflict_count' => payload[:curation_conflict_count]
531
+ }
532
+ end
533
+ private_class_method :base_single_json_hash
534
+
535
+ def self.attach_grouping_fields!(h, params_hash)
536
+ h['group_by'] = params_hash[:group_by] if params_hash.key?(:group_by)
537
+ h['group_limit'] = params_hash[:group_limit] if params_hash.key?(:group_limit)
538
+ h['group_missing_values'] = true if params_hash[:group_missing_values]
539
+ end
540
+ private_class_method :attach_grouping_fields!
541
+
542
+ def self.attach_selected_params!(h, params_hash)
543
+ %i[q query_by per_page page infix].each do |key|
544
+ h[key.to_s] = params_hash[key] if params_hash.key?(key)
545
+ end
546
+ h['filter_by'] = '***' if params_hash.key?(:filter_by)
547
+ end
548
+ private_class_method :attach_selected_params!
549
+
550
+ def self.extract_cache_flag(url_opts)
551
+ return nil unless url_opts.is_a?(Hash)
552
+
553
+ url_opts[:use_cache] ? true : false
554
+ end
555
+ private_class_method :extract_cache_flag
556
+
557
+ def self.extract_ttl(url_opts)
558
+ return nil unless url_opts.is_a?(Hash)
559
+
560
+ url_opts[:cache_ttl]
561
+ end
562
+ private_class_method :extract_ttl
563
+
564
+ def self.build_subscriptions(log, severity, include_params:, fmt:)
565
+ handles = []
566
+ handles << subscribe_search(log, severity, include_params)
567
+ handles << subscribe_multi(log, severity, include_params)
568
+ handles << subscribe_schema_diff(log, severity, fmt)
569
+ handles << subscribe_schema_apply(log, severity, fmt)
570
+ handles << subscribe_joins_compile(log, severity, fmt)
571
+ handles << subscribe_partition_start(log, severity, fmt)
572
+ handles << subscribe_partition_finish(log, severity, fmt)
573
+ handles << subscribe_batch_import(log, severity, fmt)
574
+ handles << subscribe_delete_stale(log, severity, fmt)
575
+ handles.concat(subscribe_legacy_stale(log, severity, fmt))
576
+ handles
577
+ end
578
+ private_class_method :build_subscriptions
579
+
580
+ def self.subscribe_search(log, severity, include_params)
581
+ ActiveSupport::Notifications.subscribe(EVENT_SEARCH) do |*args|
582
+ ev = ActiveSupport::Notifications::Event.new(*args)
583
+ emit_line(log, severity, ev, include_params: include_params, multi: false)
584
+ end
585
+ end
586
+ private_class_method :subscribe_search
587
+
588
+ def self.subscribe_multi(log, severity, include_params)
589
+ ActiveSupport::Notifications.subscribe(EVENT_MULTI) do |*args|
590
+ ev = ActiveSupport::Notifications::Event.new(*args)
591
+ emit_line(log, severity, ev, include_params: include_params, multi: true)
592
+ end
593
+ end
594
+ private_class_method :subscribe_multi
595
+
596
+ def self.subscribe_schema_diff(log, severity, fmt)
597
+ ActiveSupport::Notifications.subscribe(EVENT_SCHEMA_DIFF) do |*args|
598
+ ev = ActiveSupport::Notifications::Event.new(*args)
599
+ emit_schema_diff(log, severity, ev, format: fmt)
600
+ end
601
+ end
602
+ private_class_method :subscribe_schema_diff
603
+
604
+ def self.subscribe_schema_apply(log, severity, fmt)
605
+ ActiveSupport::Notifications.subscribe(EVENT_SCHEMA_APPLY) do |*args|
606
+ ev = ActiveSupport::Notifications::Event.new(*args)
607
+ emit_schema_apply(log, severity, ev, format: fmt)
608
+ end
609
+ end
610
+ private_class_method :subscribe_schema_apply
611
+
612
+ def self.subscribe_joins_compile(log, severity, fmt)
613
+ ActiveSupport::Notifications.subscribe(EVENT_JOINS_COMPILE) do |*args|
614
+ ev = ActiveSupport::Notifications::Event.new(*args)
615
+ emit_joins_compile(log, severity, ev, format: fmt)
616
+ end
617
+ end
618
+ private_class_method :subscribe_joins_compile
619
+
620
+ def self.subscribe_partition_start(log, severity, fmt)
621
+ ActiveSupport::Notifications.subscribe(EVENT_PARTITION_START) do |*args|
622
+ ev = ActiveSupport::Notifications::Event.new(*args)
623
+ emit_partition_start(log, severity, ev, format: fmt)
624
+ end
625
+ end
626
+ private_class_method :subscribe_partition_start
627
+
628
+ def self.subscribe_partition_finish(log, severity, fmt)
629
+ ActiveSupport::Notifications.subscribe(EVENT_PARTITION_FINISH) do |*args|
630
+ ev = ActiveSupport::Notifications::Event.new(*args)
631
+ emit_partition_finish(log, severity, ev, format: fmt)
632
+ end
633
+ end
634
+ private_class_method :subscribe_partition_finish
635
+
636
+ def self.subscribe_batch_import(log, severity, fmt)
637
+ ActiveSupport::Notifications.subscribe(EVENT_BATCH_IMPORT) do |*args|
638
+ ev = ActiveSupport::Notifications::Event.new(*args)
639
+ emit_batch_import(log, severity, ev, format: fmt)
640
+ end
641
+ end
642
+ private_class_method :subscribe_batch_import
643
+
644
+ def self.subscribe_delete_stale(log, severity, fmt)
645
+ ActiveSupport::Notifications.subscribe(EVENT_DELETE_STALE) do |*args|
646
+ ev = ActiveSupport::Notifications::Event.new(*args)
647
+ emit_delete_stale(log, severity, ev, format: fmt)
648
+ end
649
+ end
650
+ private_class_method :subscribe_delete_stale
651
+
652
+ def self.subscribe_legacy_stale(log, severity, fmt)
653
+ [
654
+ ActiveSupport::Notifications.subscribe(LEGACY_STALE_STARTED) do |*args|
655
+ ev = ActiveSupport::Notifications::Event.new(*args)
656
+ emit_delete_stale(log, severity, ev, format: fmt, legacy: :started)
657
+ end,
658
+ ActiveSupport::Notifications.subscribe(LEGACY_STALE_FINISHED) do |*args|
659
+ ev = ActiveSupport::Notifications::Event.new(*args)
660
+ emit_delete_stale(log, severity, ev, format: fmt, legacy: :finished)
661
+ end,
662
+ ActiveSupport::Notifications.subscribe(LEGACY_STALE_ERROR) do |*args|
663
+ ev = ActiveSupport::Notifications::Event.new(*args)
664
+ emit_delete_stale(log, severity, ev, format: fmt, legacy: :error)
665
+ end,
666
+ ActiveSupport::Notifications.subscribe(LEGACY_STALE_SKIPPED) do |*args|
667
+ ev = ActiveSupport::Notifications::Event.new(*args)
668
+ emit_delete_stale(log, severity, ev, format: fmt, legacy: :skipped)
669
+ end
670
+ ]
671
+ end
672
+ private_class_method :subscribe_legacy_stale
673
+ end
674
+ end
675
+ end