search-engine-for-typesense 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/README.md +148 -0
- data/app/search_engine/search_engine/app_info.rb +11 -0
- data/app/search_engine/search_engine/index_partition_job.rb +170 -0
- data/lib/generators/search_engine/install/install_generator.rb +20 -0
- data/lib/generators/search_engine/install/templates/initializer.rb.tt +230 -0
- data/lib/generators/search_engine/model/model_generator.rb +86 -0
- data/lib/generators/search_engine/model/templates/model.rb.tt +12 -0
- data/lib/search-engine-for-typesense.rb +12 -0
- data/lib/search_engine/active_record_syncable.rb +247 -0
- data/lib/search_engine/admin/stopwords.rb +125 -0
- data/lib/search_engine/admin/synonyms.rb +125 -0
- data/lib/search_engine/admin.rb +12 -0
- data/lib/search_engine/ast/and.rb +52 -0
- data/lib/search_engine/ast/binary_op.rb +75 -0
- data/lib/search_engine/ast/eq.rb +19 -0
- data/lib/search_engine/ast/group.rb +18 -0
- data/lib/search_engine/ast/gt.rb +12 -0
- data/lib/search_engine/ast/gte.rb +12 -0
- data/lib/search_engine/ast/in.rb +28 -0
- data/lib/search_engine/ast/lt.rb +12 -0
- data/lib/search_engine/ast/lte.rb +12 -0
- data/lib/search_engine/ast/matches.rb +55 -0
- data/lib/search_engine/ast/node.rb +176 -0
- data/lib/search_engine/ast/not_eq.rb +13 -0
- data/lib/search_engine/ast/not_in.rb +24 -0
- data/lib/search_engine/ast/or.rb +52 -0
- data/lib/search_engine/ast/prefix.rb +51 -0
- data/lib/search_engine/ast/raw.rb +41 -0
- data/lib/search_engine/ast/unary_op.rb +43 -0
- data/lib/search_engine/ast.rb +101 -0
- data/lib/search_engine/base/creation.rb +727 -0
- data/lib/search_engine/base/deletion.rb +80 -0
- data/lib/search_engine/base/display_coercions.rb +36 -0
- data/lib/search_engine/base/hydration.rb +312 -0
- data/lib/search_engine/base/index_maintenance/cleanup.rb +202 -0
- data/lib/search_engine/base/index_maintenance/lifecycle.rb +251 -0
- data/lib/search_engine/base/index_maintenance/schema.rb +117 -0
- data/lib/search_engine/base/index_maintenance.rb +459 -0
- data/lib/search_engine/base/indexing_dsl.rb +255 -0
- data/lib/search_engine/base/joins.rb +479 -0
- data/lib/search_engine/base/model_dsl.rb +472 -0
- data/lib/search_engine/base/presets.rb +43 -0
- data/lib/search_engine/base/pretty_printer.rb +315 -0
- data/lib/search_engine/base/relation_delegation.rb +42 -0
- data/lib/search_engine/base/scopes.rb +113 -0
- data/lib/search_engine/base/updating.rb +92 -0
- data/lib/search_engine/base.rb +38 -0
- data/lib/search_engine/bulk.rb +284 -0
- data/lib/search_engine/cache.rb +33 -0
- data/lib/search_engine/cascade.rb +531 -0
- data/lib/search_engine/cli/doctor.rb +631 -0
- data/lib/search_engine/cli/support.rb +217 -0
- data/lib/search_engine/cli.rb +222 -0
- data/lib/search_engine/client/http_adapter.rb +63 -0
- data/lib/search_engine/client/request_builder.rb +92 -0
- data/lib/search_engine/client/services/base.rb +74 -0
- data/lib/search_engine/client/services/collections.rb +161 -0
- data/lib/search_engine/client/services/documents.rb +214 -0
- data/lib/search_engine/client/services/operations.rb +152 -0
- data/lib/search_engine/client/services/search.rb +190 -0
- data/lib/search_engine/client/services.rb +29 -0
- data/lib/search_engine/client.rb +765 -0
- data/lib/search_engine/client_options.rb +20 -0
- data/lib/search_engine/collection_resolver.rb +191 -0
- data/lib/search_engine/collections_graph.rb +330 -0
- data/lib/search_engine/compiled_params.rb +143 -0
- data/lib/search_engine/compiler.rb +383 -0
- data/lib/search_engine/config/observability.rb +27 -0
- data/lib/search_engine/config/presets.rb +92 -0
- data/lib/search_engine/config/selection.rb +16 -0
- data/lib/search_engine/config/typesense.rb +48 -0
- data/lib/search_engine/config/validators.rb +97 -0
- data/lib/search_engine/config.rb +917 -0
- data/lib/search_engine/console_helpers.rb +130 -0
- data/lib/search_engine/deletion.rb +103 -0
- data/lib/search_engine/dispatcher.rb +125 -0
- data/lib/search_engine/dsl/parser.rb +582 -0
- data/lib/search_engine/engine.rb +167 -0
- data/lib/search_engine/errors.rb +290 -0
- data/lib/search_engine/filters/sanitizer.rb +189 -0
- data/lib/search_engine/hydration/materializers.rb +808 -0
- data/lib/search_engine/hydration/selection_context.rb +96 -0
- data/lib/search_engine/indexer/batch_planner.rb +76 -0
- data/lib/search_engine/indexer/bulk_import.rb +626 -0
- data/lib/search_engine/indexer/import_dispatcher.rb +198 -0
- data/lib/search_engine/indexer/retry_policy.rb +103 -0
- data/lib/search_engine/indexer.rb +747 -0
- data/lib/search_engine/instrumentation.rb +308 -0
- data/lib/search_engine/joins/guard.rb +202 -0
- data/lib/search_engine/joins/resolver.rb +95 -0
- data/lib/search_engine/logging/color.rb +78 -0
- data/lib/search_engine/logging/format_helpers.rb +92 -0
- data/lib/search_engine/logging/partition_progress.rb +53 -0
- data/lib/search_engine/logging_subscriber.rb +388 -0
- data/lib/search_engine/mapper.rb +785 -0
- data/lib/search_engine/multi.rb +286 -0
- data/lib/search_engine/multi_result.rb +186 -0
- data/lib/search_engine/notifications/compact_logger.rb +675 -0
- data/lib/search_engine/observability.rb +162 -0
- data/lib/search_engine/operations.rb +58 -0
- data/lib/search_engine/otel.rb +227 -0
- data/lib/search_engine/partitioner.rb +128 -0
- data/lib/search_engine/ranking_plan.rb +118 -0
- data/lib/search_engine/registry.rb +158 -0
- data/lib/search_engine/relation/compiler.rb +711 -0
- data/lib/search_engine/relation/deletion.rb +37 -0
- data/lib/search_engine/relation/dsl/filters.rb +624 -0
- data/lib/search_engine/relation/dsl/selection.rb +240 -0
- data/lib/search_engine/relation/dsl.rb +903 -0
- data/lib/search_engine/relation/dx/dry_run.rb +59 -0
- data/lib/search_engine/relation/dx/friendly_where.rb +24 -0
- data/lib/search_engine/relation/dx.rb +231 -0
- data/lib/search_engine/relation/materializers.rb +118 -0
- data/lib/search_engine/relation/options.rb +138 -0
- data/lib/search_engine/relation/state.rb +274 -0
- data/lib/search_engine/relation/updating.rb +44 -0
- data/lib/search_engine/relation.rb +623 -0
- data/lib/search_engine/result.rb +664 -0
- data/lib/search_engine/schema.rb +1083 -0
- data/lib/search_engine/sources/active_record_source.rb +185 -0
- data/lib/search_engine/sources/base.rb +62 -0
- data/lib/search_engine/sources/lambda_source.rb +55 -0
- data/lib/search_engine/sources/sql_source.rb +196 -0
- data/lib/search_engine/sources.rb +71 -0
- data/lib/search_engine/stale_rules.rb +160 -0
- data/lib/search_engine/test/minitest_assertions.rb +57 -0
- data/lib/search_engine/test/offline_client.rb +134 -0
- data/lib/search_engine/test/rspec_matchers.rb +77 -0
- data/lib/search_engine/test/stub_client.rb +201 -0
- data/lib/search_engine/test.rb +66 -0
- data/lib/search_engine/test_autoload.rb +8 -0
- data/lib/search_engine/update.rb +35 -0
- data/lib/search_engine/version.rb +7 -0
- data/lib/search_engine.rb +332 -0
- data/lib/tasks/search_engine.rake +501 -0
- data/lib/tasks/search_engine_doctor.rake +16 -0
- metadata +225 -0
|
@@ -0,0 +1,675 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'logger'
|
|
4
|
+
require 'active_support/notifications'
|
|
5
|
+
require 'json'
|
|
6
|
+
require 'search_engine/logging/format_helpers'
|
|
7
|
+
|
|
8
|
+
module SearchEngine
|
|
9
|
+
module Notifications
|
|
10
|
+
# Opt-in compact logging subscriber for SearchEngine AS::Notifications events.
|
|
11
|
+
#
|
|
12
|
+
# Emits concise, single-line log entries with redacted parameters and
|
|
13
|
+
# stable keys. Designed to be lightweight and reloader-safe.
|
|
14
|
+
#
|
|
15
|
+
# Usage:
|
|
16
|
+
# SearchEngine::Notifications::CompactLogger.subscribe
|
|
17
|
+
# SearchEngine::Notifications::CompactLogger.unsubscribe
|
|
18
|
+
#
|
|
19
|
+
# @since M8
|
|
20
|
+
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/observability#logging
|
|
21
|
+
class CompactLogger
|
|
22
|
+
EVENT_SEARCH = 'search_engine.search'
|
|
23
|
+
EVENT_MULTI = 'search_engine.multi_search'
|
|
24
|
+
EVENT_SCHEMA_DIFF = 'search_engine.schema.diff'
|
|
25
|
+
EVENT_SCHEMA_APPLY = 'search_engine.schema.apply'
|
|
26
|
+
EVENT_JOINS_COMPILE = 'search_engine.joins.compile'
|
|
27
|
+
EVENT_PARTITION_START = 'search_engine.indexer.partition_start'
|
|
28
|
+
EVENT_PARTITION_FINISH = 'search_engine.indexer.partition_finish'
|
|
29
|
+
EVENT_BATCH_IMPORT = 'search_engine.indexer.batch_import'
|
|
30
|
+
EVENT_DELETE_STALE = 'search_engine.indexer.delete_stale'
|
|
31
|
+
# Legacy (emitted by current codebase): stale_deletes.*
|
|
32
|
+
LEGACY_STALE_STARTED = 'search_engine.stale_deletes.started'
|
|
33
|
+
LEGACY_STALE_FINISHED = 'search_engine.stale_deletes.finished'
|
|
34
|
+
LEGACY_STALE_ERROR = 'search_engine.stale_deletes.error'
|
|
35
|
+
LEGACY_STALE_SKIPPED = 'search_engine.stale_deletes.skipped'
|
|
36
|
+
|
|
37
|
+
# Subscribe to SearchEngine notifications.
|
|
38
|
+
#
|
|
39
|
+
# @param logger [#info,#warn,#error] Logger instance; defaults to config.logger or $stdout
|
|
40
|
+
# @param level [Symbol] one of :debug, :info, :warn, :error
|
|
41
|
+
# @param include_params [Boolean] when true, include whitelisted params for single-search
|
|
42
|
+
# @param format [Symbol, nil] :kv or :json; defaults to config.observability.log_format
|
|
43
|
+
# @return [Array<Object>] subscription handles that can be passed to {.unsubscribe}
|
|
44
|
+
# @since M8
|
|
45
|
+
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/observability#logging
|
|
46
|
+
def self.subscribe(logger: default_logger, level: :info, include_params: false, format: nil)
|
|
47
|
+
return [] unless defined?(ActiveSupport::Notifications)
|
|
48
|
+
|
|
49
|
+
severity = map_level(level)
|
|
50
|
+
log = logger || default_logger
|
|
51
|
+
fmt = (format || (SearchEngine.config.observability&.log_format || :kv)).to_sym
|
|
52
|
+
|
|
53
|
+
@last_handle = build_subscriptions(log, severity, include_params: include_params, fmt: fmt)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# Unsubscribe a previous subscription set.
|
|
57
|
+
# @param handle [Array<Object>, Object, nil] handles returned by {.subscribe}
|
|
58
|
+
# @return [Boolean] true when unsubscribed
|
|
59
|
+
# @since M8
|
|
60
|
+
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/observability#logging
|
|
61
|
+
def self.unsubscribe(handle = @last_handle)
|
|
62
|
+
return false unless handle
|
|
63
|
+
|
|
64
|
+
Array(handle).each do |sub|
|
|
65
|
+
ActiveSupport::Notifications.unsubscribe(sub)
|
|
66
|
+
end
|
|
67
|
+
@last_handle = nil
|
|
68
|
+
true
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# Internal: Emit one compact line for a notification event.
|
|
72
|
+
def self.emit_line(logger, severity, event, include_params:, multi:)
|
|
73
|
+
return unless logger
|
|
74
|
+
return unless allow_log?(logger, severity)
|
|
75
|
+
|
|
76
|
+
p = event.payload
|
|
77
|
+
duration_ms = event.duration.round(1)
|
|
78
|
+
|
|
79
|
+
parts = []
|
|
80
|
+
if multi
|
|
81
|
+
parts.concat(multi_parts(p))
|
|
82
|
+
else
|
|
83
|
+
parts.concat(single_parts(p))
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
parts.concat(status_parts(p, duration_ms))
|
|
87
|
+
parts.concat(url_parts(p))
|
|
88
|
+
parts.concat(selection_parts(p)) unless multi
|
|
89
|
+
parts.concat(preset_parts(p)) unless multi
|
|
90
|
+
parts.concat(curation_parts(p)) unless multi
|
|
91
|
+
|
|
92
|
+
parts.concat(param_parts(p)) if include_params && !multi
|
|
93
|
+
|
|
94
|
+
line = parts.join(' ')
|
|
95
|
+
log_with_level(logger, severity, line)
|
|
96
|
+
rescue StandardError
|
|
97
|
+
nil
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
# Schema diff formatter
|
|
101
|
+
def self.emit_schema_diff(logger, severity, event, format: :kv)
|
|
102
|
+
return unless logger
|
|
103
|
+
return unless allow_log?(logger, severity)
|
|
104
|
+
|
|
105
|
+
p = event.payload
|
|
106
|
+
h = {
|
|
107
|
+
'event' => 'schema.diff',
|
|
108
|
+
'collection' => p[:collection] || p[:logical],
|
|
109
|
+
'fields.changed' => p[:fields_changed_count],
|
|
110
|
+
'fields.added' => p[:added_count],
|
|
111
|
+
'fields.removed' => p[:removed_count],
|
|
112
|
+
'in_sync' => p[:in_sync],
|
|
113
|
+
'duration.ms' => event.duration.round(1)
|
|
114
|
+
}
|
|
115
|
+
emit(logger, severity, h, format)
|
|
116
|
+
rescue StandardError
|
|
117
|
+
nil
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
# Schema apply formatter
|
|
121
|
+
def self.emit_schema_apply(logger, severity, event, format: :kv)
|
|
122
|
+
return unless logger
|
|
123
|
+
return unless allow_log?(logger, severity)
|
|
124
|
+
|
|
125
|
+
p = event.payload
|
|
126
|
+
h = {
|
|
127
|
+
'event' => 'schema.apply',
|
|
128
|
+
'collection' => p[:collection] || p[:logical],
|
|
129
|
+
'into' => p[:physical_new] || p[:new_physical],
|
|
130
|
+
'alias_swapped' => p[:alias_swapped],
|
|
131
|
+
'retention_deleted_count' => p[:retention_deleted_count] || p[:dropped_count],
|
|
132
|
+
'status' => p[:status] || 'ok',
|
|
133
|
+
'duration.ms' => event.duration.round(1)
|
|
134
|
+
}
|
|
135
|
+
emit(logger, severity, h, format)
|
|
136
|
+
rescue StandardError
|
|
137
|
+
nil
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
# JOINs compile formatter
|
|
141
|
+
# Emits a single-line summary of associations and their usage, with lengths only
|
|
142
|
+
# and without raw filter/include/sort strings.
|
|
143
|
+
def self.emit_joins_compile(logger, severity, event, format: :kv)
|
|
144
|
+
return unless logger
|
|
145
|
+
return unless allow_log?(logger, severity)
|
|
146
|
+
|
|
147
|
+
p = event.payload
|
|
148
|
+
used = p[:used_in] || {}
|
|
149
|
+
used_compact = []
|
|
150
|
+
%i[include filter sort].each do |k|
|
|
151
|
+
arr = Array(used[k]).map(&:to_s)
|
|
152
|
+
used_compact << "#{k}:#{arr.join(',')}" unless arr.empty?
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
h = {
|
|
156
|
+
'event' => 'joins.compile',
|
|
157
|
+
'collection' => p[:collection],
|
|
158
|
+
'joins.assocs' => Array(p[:assocs]).join(','),
|
|
159
|
+
'joins.count' => p[:join_count],
|
|
160
|
+
'joins.used_in' => used_compact.join('|'),
|
|
161
|
+
'joins.include.len' => p[:include_len],
|
|
162
|
+
'joins.filter.len' => p[:filter_len],
|
|
163
|
+
'joins.sort.len' => p[:sort_len],
|
|
164
|
+
'has_joins' => p[:has_joins],
|
|
165
|
+
'duration.ms' => p[:duration_ms] || event.duration.round(1)
|
|
166
|
+
}
|
|
167
|
+
emit(logger, severity, h.compact, format)
|
|
168
|
+
rescue StandardError
|
|
169
|
+
nil
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
def self.emit_partition_start(logger, severity, event, format: :kv)
|
|
173
|
+
return unless logger
|
|
174
|
+
return unless allow_log?(logger, severity)
|
|
175
|
+
|
|
176
|
+
p = event.payload
|
|
177
|
+
h = {
|
|
178
|
+
'event' => 'indexer.partition_start',
|
|
179
|
+
'collection' => p[:collection],
|
|
180
|
+
'into' => p[:into],
|
|
181
|
+
'partition' => p[:partition_hash] || p[:partition],
|
|
182
|
+
'dispatch_mode' => p[:dispatch_mode],
|
|
183
|
+
'job_id' => p[:job_id],
|
|
184
|
+
'timestamp' => p[:timestamp]
|
|
185
|
+
}
|
|
186
|
+
emit(logger, severity, h, format)
|
|
187
|
+
rescue StandardError
|
|
188
|
+
nil
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
def self.emit_partition_finish(logger, severity, event, format: :kv)
|
|
192
|
+
return unless logger
|
|
193
|
+
return unless allow_log?(logger, severity)
|
|
194
|
+
|
|
195
|
+
p = event.payload
|
|
196
|
+
h = {
|
|
197
|
+
'event' => 'indexer.partition_finish',
|
|
198
|
+
'collection' => p[:collection],
|
|
199
|
+
'into' => p[:into],
|
|
200
|
+
'partition' => p[:partition_hash] || p[:partition],
|
|
201
|
+
'batches.total' => p[:batches_total],
|
|
202
|
+
'docs.total' => p[:docs_total],
|
|
203
|
+
'success.total' => p[:success_total],
|
|
204
|
+
'failed.total' => p[:failed_total],
|
|
205
|
+
'status' => p[:status],
|
|
206
|
+
'duration.ms' => event.duration.round(1)
|
|
207
|
+
}
|
|
208
|
+
emit(logger, severity, h, format)
|
|
209
|
+
rescue StandardError
|
|
210
|
+
nil
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
def self.emit_batch_import(logger, severity, event, format: :kv)
|
|
214
|
+
return unless logger
|
|
215
|
+
return unless allow_log?(logger, severity)
|
|
216
|
+
|
|
217
|
+
p = event.payload
|
|
218
|
+
h = {
|
|
219
|
+
'event' => 'indexer.batch_import',
|
|
220
|
+
'collection' => p[:collection],
|
|
221
|
+
'into' => p[:into],
|
|
222
|
+
'batch_index' => p[:batch_index],
|
|
223
|
+
'docs.count' => p[:docs_count],
|
|
224
|
+
'success.count' => p[:success_count],
|
|
225
|
+
'failure.count' => p[:failure_count],
|
|
226
|
+
'attempts' => p[:attempts],
|
|
227
|
+
'http_status' => p[:http_status],
|
|
228
|
+
'bytes.sent' => p[:bytes_sent],
|
|
229
|
+
'duration.ms' => event.duration.round(1),
|
|
230
|
+
'transient_retry' => p[:transient_retry]
|
|
231
|
+
}
|
|
232
|
+
if SearchEngine.config.observability.include_error_messages && p[:error_sample]
|
|
233
|
+
h['error.sample_count'] = Array(p[:error_sample]).size
|
|
234
|
+
h['message'] =
|
|
235
|
+
SearchEngine::Observability.truncate_message(Array(p[:error_sample]).first,
|
|
236
|
+
SearchEngine.config.observability.max_message_length
|
|
237
|
+
)
|
|
238
|
+
end
|
|
239
|
+
emit(logger, severity, h, format)
|
|
240
|
+
rescue StandardError
|
|
241
|
+
nil
|
|
242
|
+
end
|
|
243
|
+
|
|
244
|
+
def self.emit_delete_stale(logger, severity, event, format: :kv, legacy: nil)
|
|
245
|
+
return unless logger
|
|
246
|
+
return unless allow_log?(logger, severity)
|
|
247
|
+
|
|
248
|
+
p = normalize_stale_payload(event.payload, legacy: legacy)
|
|
249
|
+
h = {
|
|
250
|
+
'event' => 'indexer.delete_stale',
|
|
251
|
+
'collection' => p[:collection],
|
|
252
|
+
'into' => p[:into],
|
|
253
|
+
'partition' => p[:partition_hash] || p[:partition],
|
|
254
|
+
'filter.hash' => p[:filter_hash],
|
|
255
|
+
'deleted.count' => p[:deleted_count],
|
|
256
|
+
'status' => p[:status] || 'ok',
|
|
257
|
+
'reason' => p[:reason],
|
|
258
|
+
'duration.ms' => p[:duration_ms] || event.duration.round(1)
|
|
259
|
+
}
|
|
260
|
+
emit(logger, severity, h, format)
|
|
261
|
+
rescue StandardError
|
|
262
|
+
nil
|
|
263
|
+
end
|
|
264
|
+
|
|
265
|
+
# Emit according to format
|
|
266
|
+
def self.emit(logger, severity, hash, format)
|
|
267
|
+
line =
|
|
268
|
+
if format == :json
|
|
269
|
+
JSON.generate(hash.compact)
|
|
270
|
+
else
|
|
271
|
+
SearchEngine::Logging::FormatHelpers.kv_compact(hash)
|
|
272
|
+
end
|
|
273
|
+
log_with_level(logger, severity, line)
|
|
274
|
+
end
|
|
275
|
+
|
|
276
|
+
def self.normalize_stale_payload(p, legacy: nil)
|
|
277
|
+
h = p.dup
|
|
278
|
+
case legacy
|
|
279
|
+
when :started
|
|
280
|
+
h[:status] = 'started'
|
|
281
|
+
when :finished
|
|
282
|
+
h[:status] = 'ok'
|
|
283
|
+
when :error
|
|
284
|
+
h[:status] = 'failed'
|
|
285
|
+
when :skipped
|
|
286
|
+
h[:status] = 'skipped'
|
|
287
|
+
end
|
|
288
|
+
h
|
|
289
|
+
end
|
|
290
|
+
private_class_method :normalize_stale_payload
|
|
291
|
+
|
|
292
|
+
def self.multi_parts(payload)
|
|
293
|
+
labels = Array(payload[:labels]).map(&:to_s)
|
|
294
|
+
labels = Array(payload[:collections]).map(&:to_s) if labels.empty? || labels.all?(&:empty?)
|
|
295
|
+
count = payload[:searches_count]
|
|
296
|
+
count ||= (payload[:params].is_a?(Array) ? payload[:params].size : nil)
|
|
297
|
+
|
|
298
|
+
parts = ['[se.multi]']
|
|
299
|
+
parts << "count=#{count}" if count
|
|
300
|
+
parts << "labels=#{labels.join(',')}" unless labels.empty?
|
|
301
|
+
parts
|
|
302
|
+
end
|
|
303
|
+
private_class_method :multi_parts
|
|
304
|
+
|
|
305
|
+
def self.single_parts(payload)
|
|
306
|
+
parts = ['[se.search]']
|
|
307
|
+
parts << "collection=#{payload[:collection]}" if payload[:collection]
|
|
308
|
+
parts
|
|
309
|
+
end
|
|
310
|
+
private_class_method :single_parts
|
|
311
|
+
|
|
312
|
+
def self.status_parts(payload, duration_ms)
|
|
313
|
+
status_val = payload.key?(:http_status) ? payload[:http_status] : (payload[:status] || 'ok')
|
|
314
|
+
["status=#{status_val}", "duration=#{duration_ms}ms"]
|
|
315
|
+
end
|
|
316
|
+
private_class_method :status_parts
|
|
317
|
+
|
|
318
|
+
def self.url_parts(payload)
|
|
319
|
+
url = payload[:url_opts]
|
|
320
|
+
return [] unless url.is_a?(Hash)
|
|
321
|
+
|
|
322
|
+
results = []
|
|
323
|
+
results << "cache=#{url[:use_cache] ? true : false}" if url.key?(:use_cache)
|
|
324
|
+
results << "ttl=#{url[:cache_ttl]}" if url.key?(:cache_ttl)
|
|
325
|
+
results
|
|
326
|
+
end
|
|
327
|
+
private_class_method :url_parts
|
|
328
|
+
|
|
329
|
+
def self.param_parts(payload)
|
|
330
|
+
params_hash = if payload[:params_preview].is_a?(Hash)
|
|
331
|
+
payload[:params_preview]
|
|
332
|
+
elsif payload[:params].is_a?(Hash)
|
|
333
|
+
payload[:params]
|
|
334
|
+
else
|
|
335
|
+
{}
|
|
336
|
+
end
|
|
337
|
+
parts = []
|
|
338
|
+
%i[q query_by per_page page infix].each do |key|
|
|
339
|
+
next unless params_hash.key?(key)
|
|
340
|
+
|
|
341
|
+
parts << format_param(key, params_hash[key])
|
|
342
|
+
end
|
|
343
|
+
parts << 'filter_by=***' if params_hash.key?(:filter_by)
|
|
344
|
+
parts
|
|
345
|
+
end
|
|
346
|
+
private_class_method :param_parts
|
|
347
|
+
|
|
348
|
+
def self.selection_parts(payload)
|
|
349
|
+
inc = (payload[:selection_include_count] || 0).to_i
|
|
350
|
+
exc = (payload[:selection_exclude_count] || 0).to_i
|
|
351
|
+
nest = (payload[:selection_nested_assoc_count] || 0).to_i
|
|
352
|
+
["sel=I:#{inc}|X:#{exc}|N:#{nest}"]
|
|
353
|
+
end
|
|
354
|
+
private_class_method :selection_parts
|
|
355
|
+
|
|
356
|
+
# Compact preset parts for text logs (single-line, allocation-light)
|
|
357
|
+
def self.preset_parts(payload)
|
|
358
|
+
name = payload[:preset_name] || (payload[:params].is_a?(Hash) ? payload[:params][:preset] : nil)
|
|
359
|
+
return [] unless name
|
|
360
|
+
|
|
361
|
+
mode = payload[:preset_mode]
|
|
362
|
+
pk_count = (payload[:preset_pruned_keys_count] || Array(payload[:preset_pruned_keys]).size).to_i
|
|
363
|
+
ld_count = payload[:preset_locked_domains_count]
|
|
364
|
+
if ld_count.nil?
|
|
365
|
+
begin
|
|
366
|
+
ld_count = SearchEngine.config.presets.locked_domains.size
|
|
367
|
+
rescue StandardError
|
|
368
|
+
ld_count = 0
|
|
369
|
+
end
|
|
370
|
+
end
|
|
371
|
+
ld_count = ld_count.to_i
|
|
372
|
+
|
|
373
|
+
# Token: pz=<name>|m=<mode>|pk=<count>|ld=<count>
|
|
374
|
+
# Truncate name conservatively to keep line short
|
|
375
|
+
shown_name = name.to_s
|
|
376
|
+
shown_name = shown_name.length > 64 ? "#{shown_name[0, 64]}…" : shown_name
|
|
377
|
+
|
|
378
|
+
parts = ["pz=#{shown_name}"]
|
|
379
|
+
parts << "m=#{mode}" if mode
|
|
380
|
+
parts << "pk=#{pk_count}" if pk_count.positive?
|
|
381
|
+
parts << "ld=#{ld_count}" if ld_count.positive?
|
|
382
|
+
|
|
383
|
+
# Optionally include small list of pruned keys when small (<=3)
|
|
384
|
+
keys = Array(payload[:preset_pruned_keys]).map { |k| k.respond_to?(:to_sym) ? k.to_sym : k }.grep(Symbol)
|
|
385
|
+
parts << "pk=[#{keys.map(&:to_s).join(',')}]" if keys.size.positive? && keys.size <= 3
|
|
386
|
+
|
|
387
|
+
[parts.join('|')]
|
|
388
|
+
rescue StandardError
|
|
389
|
+
[]
|
|
390
|
+
end
|
|
391
|
+
private_class_method :preset_parts
|
|
392
|
+
|
|
393
|
+
# Compact curation parts for text logs
|
|
394
|
+
def self.curation_parts(payload)
|
|
395
|
+
pcount = (payload[:curation_pinned_count] || 0).to_i
|
|
396
|
+
hcount = (payload[:curation_hidden_count] || 0).to_i
|
|
397
|
+
fflag = if payload.key?(:curation_filter_flag)
|
|
398
|
+
payload[:curation_filter_flag].nil? ? '∅' : payload[:curation_filter_flag]
|
|
399
|
+
end
|
|
400
|
+
has_tags = payload[:curation_has_override_tags] ? true : false
|
|
401
|
+
seg = ["cu=p:#{pcount}|h:#{hcount}"]
|
|
402
|
+
seg << "f:#{fflag}" unless fflag.nil?
|
|
403
|
+
seg << "t:#{has_tags ? 1 : 0}"
|
|
404
|
+
|
|
405
|
+
parts = [seg.join('|')]
|
|
406
|
+
if payload[:curation_conflict_type]
|
|
407
|
+
type = payload[:curation_conflict_type].to_s
|
|
408
|
+
parts << "cf=#{type}"
|
|
409
|
+
end
|
|
410
|
+
parts
|
|
411
|
+
rescue StandardError
|
|
412
|
+
[]
|
|
413
|
+
end
|
|
414
|
+
private_class_method :curation_parts
|
|
415
|
+
|
|
416
|
+
# Map a Symbol severity to Logger integer constant.
|
|
417
|
+
def self.map_level(level)
|
|
418
|
+
case level.to_s
|
|
419
|
+
when 'debug' then Logger::DEBUG
|
|
420
|
+
when 'info' then Logger::INFO
|
|
421
|
+
when 'warn' then Logger::WARN
|
|
422
|
+
when 'error' then Logger::ERROR
|
|
423
|
+
else Logger::INFO
|
|
424
|
+
end
|
|
425
|
+
end
|
|
426
|
+
|
|
427
|
+
# Minimal check for logger level
|
|
428
|
+
def self.allow_log?(logger, severity)
|
|
429
|
+
return true unless logger.respond_to?(:level)
|
|
430
|
+
|
|
431
|
+
logger.level <= severity
|
|
432
|
+
end
|
|
433
|
+
private_class_method :allow_log?
|
|
434
|
+
|
|
435
|
+
def self.format_param(key, value)
|
|
436
|
+
case key
|
|
437
|
+
when :q then %(q="#{SearchEngine::Observability.truncate_q(value)}")
|
|
438
|
+
else "#{key}=#{value}"
|
|
439
|
+
end
|
|
440
|
+
end
|
|
441
|
+
private_class_method :format_param
|
|
442
|
+
|
|
443
|
+
def self.default_logger
|
|
444
|
+
if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
|
|
445
|
+
Rails.logger
|
|
446
|
+
else
|
|
447
|
+
::Logger.new($stdout)
|
|
448
|
+
end
|
|
449
|
+
end
|
|
450
|
+
|
|
451
|
+
def self.log_with_level(logger, severity, line)
|
|
452
|
+
case severity
|
|
453
|
+
when Logger::DEBUG then logger.debug(line)
|
|
454
|
+
when Logger::INFO then logger.info(line)
|
|
455
|
+
when Logger::WARN then logger.warn(line)
|
|
456
|
+
else logger.error(line)
|
|
457
|
+
end
|
|
458
|
+
end
|
|
459
|
+
private_class_method :log_with_level
|
|
460
|
+
|
|
461
|
+
# Build JSON object for search/multi events
|
|
462
|
+
def self.build_json_hash(payload, duration_ms:, multi:, include_params: false)
|
|
463
|
+
if multi
|
|
464
|
+
build_json_hash_multi(payload, duration_ms: duration_ms)
|
|
465
|
+
else
|
|
466
|
+
build_json_hash_single(payload, duration_ms: duration_ms, include_params: include_params)
|
|
467
|
+
end
|
|
468
|
+
end
|
|
469
|
+
private_class_method :build_json_hash
|
|
470
|
+
|
|
471
|
+
def self.build_json_hash_multi(payload, duration_ms:)
|
|
472
|
+
labels = Array(payload[:labels]).map(&:to_s)
|
|
473
|
+
labels = Array(payload[:collections]).map(&:to_s) if labels.empty? || labels.all?(&:empty?)
|
|
474
|
+
{
|
|
475
|
+
'event' => 'multi',
|
|
476
|
+
'labels' => (labels unless labels.empty?),
|
|
477
|
+
'count' => payload[:searches_count] || (payload[:params].is_a?(Array) ? payload[:params].size : nil),
|
|
478
|
+
'status' => payload[:http_status] || payload[:status] || 'ok',
|
|
479
|
+
'duration.ms' => duration_ms,
|
|
480
|
+
'cache' => extract_cache_flag(payload[:url_opts]),
|
|
481
|
+
'ttl' => extract_ttl(payload[:url_opts])
|
|
482
|
+
}.compact
|
|
483
|
+
end
|
|
484
|
+
private_class_method :build_json_hash_multi
|
|
485
|
+
|
|
486
|
+
def self.build_json_hash_single(payload, duration_ms:, include_params:)
|
|
487
|
+
params_hash = extract_params_hash(payload)
|
|
488
|
+
h = base_single_json_hash(payload, duration_ms)
|
|
489
|
+
attach_grouping_fields!(h, params_hash)
|
|
490
|
+
attach_selected_params!(h, params_hash) if include_params
|
|
491
|
+
h.compact
|
|
492
|
+
end
|
|
493
|
+
private_class_method :build_json_hash_single
|
|
494
|
+
|
|
495
|
+
def self.extract_params_hash(payload)
|
|
496
|
+
if payload[:params_preview].is_a?(Hash)
|
|
497
|
+
payload[:params_preview]
|
|
498
|
+
elsif payload[:params].is_a?(Hash)
|
|
499
|
+
payload[:params]
|
|
500
|
+
else
|
|
501
|
+
{}
|
|
502
|
+
end
|
|
503
|
+
end
|
|
504
|
+
private_class_method :extract_params_hash
|
|
505
|
+
|
|
506
|
+
def self.base_single_json_hash(payload, duration_ms)
|
|
507
|
+
{
|
|
508
|
+
'event' => 'search',
|
|
509
|
+
'collection' => payload[:collection],
|
|
510
|
+
'status' => payload[:http_status] || payload[:status] || 'ok',
|
|
511
|
+
'duration.ms' => duration_ms,
|
|
512
|
+
'cache' => extract_cache_flag(payload[:url_opts]),
|
|
513
|
+
'ttl' => extract_ttl(payload[:url_opts]),
|
|
514
|
+
'selection_include_count' => (payload[:selection_include_count] || 0).to_i,
|
|
515
|
+
'selection_exclude_count' => (payload[:selection_exclude_count] || 0).to_i,
|
|
516
|
+
'selection_nested_assoc_count' => (payload[:selection_nested_assoc_count] || 0).to_i,
|
|
517
|
+
'preset_name' => payload[:preset_name],
|
|
518
|
+
'preset_mode' => payload[:preset_mode],
|
|
519
|
+
'preset_pruned_keys_count' => payload[:preset_pruned_keys_count],
|
|
520
|
+
'preset_locked_domains_count' => payload[:preset_locked_domains_count],
|
|
521
|
+
'preset_pruned_keys' => begin
|
|
522
|
+
arr = Array(payload[:preset_pruned_keys])
|
|
523
|
+
arr.empty? ? nil : arr.map(&:to_s)
|
|
524
|
+
end,
|
|
525
|
+
'curation_pinned_count' => payload[:curation_pinned_count],
|
|
526
|
+
'curation_hidden_count' => payload[:curation_hidden_count],
|
|
527
|
+
'curation_has_override_tags' => payload[:curation_has_override_tags],
|
|
528
|
+
'curation_filter_flag' => (payload.key?(:curation_filter_flag) ? payload[:curation_filter_flag] : nil),
|
|
529
|
+
'curation_conflict_type' => payload[:curation_conflict_type],
|
|
530
|
+
'curation_conflict_count' => payload[:curation_conflict_count]
|
|
531
|
+
}
|
|
532
|
+
end
|
|
533
|
+
private_class_method :base_single_json_hash
|
|
534
|
+
|
|
535
|
+
def self.attach_grouping_fields!(h, params_hash)
|
|
536
|
+
h['group_by'] = params_hash[:group_by] if params_hash.key?(:group_by)
|
|
537
|
+
h['group_limit'] = params_hash[:group_limit] if params_hash.key?(:group_limit)
|
|
538
|
+
h['group_missing_values'] = true if params_hash[:group_missing_values]
|
|
539
|
+
end
|
|
540
|
+
private_class_method :attach_grouping_fields!
|
|
541
|
+
|
|
542
|
+
def self.attach_selected_params!(h, params_hash)
|
|
543
|
+
%i[q query_by per_page page infix].each do |key|
|
|
544
|
+
h[key.to_s] = params_hash[key] if params_hash.key?(key)
|
|
545
|
+
end
|
|
546
|
+
h['filter_by'] = '***' if params_hash.key?(:filter_by)
|
|
547
|
+
end
|
|
548
|
+
private_class_method :attach_selected_params!
|
|
549
|
+
|
|
550
|
+
def self.extract_cache_flag(url_opts)
|
|
551
|
+
return nil unless url_opts.is_a?(Hash)
|
|
552
|
+
|
|
553
|
+
url_opts[:use_cache] ? true : false
|
|
554
|
+
end
|
|
555
|
+
private_class_method :extract_cache_flag
|
|
556
|
+
|
|
557
|
+
def self.extract_ttl(url_opts)
|
|
558
|
+
return nil unless url_opts.is_a?(Hash)
|
|
559
|
+
|
|
560
|
+
url_opts[:cache_ttl]
|
|
561
|
+
end
|
|
562
|
+
private_class_method :extract_ttl
|
|
563
|
+
|
|
564
|
+
def self.build_subscriptions(log, severity, include_params:, fmt:)
|
|
565
|
+
handles = []
|
|
566
|
+
handles << subscribe_search(log, severity, include_params)
|
|
567
|
+
handles << subscribe_multi(log, severity, include_params)
|
|
568
|
+
handles << subscribe_schema_diff(log, severity, fmt)
|
|
569
|
+
handles << subscribe_schema_apply(log, severity, fmt)
|
|
570
|
+
handles << subscribe_joins_compile(log, severity, fmt)
|
|
571
|
+
handles << subscribe_partition_start(log, severity, fmt)
|
|
572
|
+
handles << subscribe_partition_finish(log, severity, fmt)
|
|
573
|
+
handles << subscribe_batch_import(log, severity, fmt)
|
|
574
|
+
handles << subscribe_delete_stale(log, severity, fmt)
|
|
575
|
+
handles.concat(subscribe_legacy_stale(log, severity, fmt))
|
|
576
|
+
handles
|
|
577
|
+
end
|
|
578
|
+
private_class_method :build_subscriptions
|
|
579
|
+
|
|
580
|
+
def self.subscribe_search(log, severity, include_params)
|
|
581
|
+
ActiveSupport::Notifications.subscribe(EVENT_SEARCH) do |*args|
|
|
582
|
+
ev = ActiveSupport::Notifications::Event.new(*args)
|
|
583
|
+
emit_line(log, severity, ev, include_params: include_params, multi: false)
|
|
584
|
+
end
|
|
585
|
+
end
|
|
586
|
+
private_class_method :subscribe_search
|
|
587
|
+
|
|
588
|
+
def self.subscribe_multi(log, severity, include_params)
|
|
589
|
+
ActiveSupport::Notifications.subscribe(EVENT_MULTI) do |*args|
|
|
590
|
+
ev = ActiveSupport::Notifications::Event.new(*args)
|
|
591
|
+
emit_line(log, severity, ev, include_params: include_params, multi: true)
|
|
592
|
+
end
|
|
593
|
+
end
|
|
594
|
+
private_class_method :subscribe_multi
|
|
595
|
+
|
|
596
|
+
def self.subscribe_schema_diff(log, severity, fmt)
|
|
597
|
+
ActiveSupport::Notifications.subscribe(EVENT_SCHEMA_DIFF) do |*args|
|
|
598
|
+
ev = ActiveSupport::Notifications::Event.new(*args)
|
|
599
|
+
emit_schema_diff(log, severity, ev, format: fmt)
|
|
600
|
+
end
|
|
601
|
+
end
|
|
602
|
+
private_class_method :subscribe_schema_diff
|
|
603
|
+
|
|
604
|
+
def self.subscribe_schema_apply(log, severity, fmt)
|
|
605
|
+
ActiveSupport::Notifications.subscribe(EVENT_SCHEMA_APPLY) do |*args|
|
|
606
|
+
ev = ActiveSupport::Notifications::Event.new(*args)
|
|
607
|
+
emit_schema_apply(log, severity, ev, format: fmt)
|
|
608
|
+
end
|
|
609
|
+
end
|
|
610
|
+
private_class_method :subscribe_schema_apply
|
|
611
|
+
|
|
612
|
+
def self.subscribe_joins_compile(log, severity, fmt)
|
|
613
|
+
ActiveSupport::Notifications.subscribe(EVENT_JOINS_COMPILE) do |*args|
|
|
614
|
+
ev = ActiveSupport::Notifications::Event.new(*args)
|
|
615
|
+
emit_joins_compile(log, severity, ev, format: fmt)
|
|
616
|
+
end
|
|
617
|
+
end
|
|
618
|
+
private_class_method :subscribe_joins_compile
|
|
619
|
+
|
|
620
|
+
def self.subscribe_partition_start(log, severity, fmt)
|
|
621
|
+
ActiveSupport::Notifications.subscribe(EVENT_PARTITION_START) do |*args|
|
|
622
|
+
ev = ActiveSupport::Notifications::Event.new(*args)
|
|
623
|
+
emit_partition_start(log, severity, ev, format: fmt)
|
|
624
|
+
end
|
|
625
|
+
end
|
|
626
|
+
private_class_method :subscribe_partition_start
|
|
627
|
+
|
|
628
|
+
def self.subscribe_partition_finish(log, severity, fmt)
|
|
629
|
+
ActiveSupport::Notifications.subscribe(EVENT_PARTITION_FINISH) do |*args|
|
|
630
|
+
ev = ActiveSupport::Notifications::Event.new(*args)
|
|
631
|
+
emit_partition_finish(log, severity, ev, format: fmt)
|
|
632
|
+
end
|
|
633
|
+
end
|
|
634
|
+
private_class_method :subscribe_partition_finish
|
|
635
|
+
|
|
636
|
+
def self.subscribe_batch_import(log, severity, fmt)
|
|
637
|
+
ActiveSupport::Notifications.subscribe(EVENT_BATCH_IMPORT) do |*args|
|
|
638
|
+
ev = ActiveSupport::Notifications::Event.new(*args)
|
|
639
|
+
emit_batch_import(log, severity, ev, format: fmt)
|
|
640
|
+
end
|
|
641
|
+
end
|
|
642
|
+
private_class_method :subscribe_batch_import
|
|
643
|
+
|
|
644
|
+
def self.subscribe_delete_stale(log, severity, fmt)
|
|
645
|
+
ActiveSupport::Notifications.subscribe(EVENT_DELETE_STALE) do |*args|
|
|
646
|
+
ev = ActiveSupport::Notifications::Event.new(*args)
|
|
647
|
+
emit_delete_stale(log, severity, ev, format: fmt)
|
|
648
|
+
end
|
|
649
|
+
end
|
|
650
|
+
private_class_method :subscribe_delete_stale
|
|
651
|
+
|
|
652
|
+
def self.subscribe_legacy_stale(log, severity, fmt)
|
|
653
|
+
[
|
|
654
|
+
ActiveSupport::Notifications.subscribe(LEGACY_STALE_STARTED) do |*args|
|
|
655
|
+
ev = ActiveSupport::Notifications::Event.new(*args)
|
|
656
|
+
emit_delete_stale(log, severity, ev, format: fmt, legacy: :started)
|
|
657
|
+
end,
|
|
658
|
+
ActiveSupport::Notifications.subscribe(LEGACY_STALE_FINISHED) do |*args|
|
|
659
|
+
ev = ActiveSupport::Notifications::Event.new(*args)
|
|
660
|
+
emit_delete_stale(log, severity, ev, format: fmt, legacy: :finished)
|
|
661
|
+
end,
|
|
662
|
+
ActiveSupport::Notifications.subscribe(LEGACY_STALE_ERROR) do |*args|
|
|
663
|
+
ev = ActiveSupport::Notifications::Event.new(*args)
|
|
664
|
+
emit_delete_stale(log, severity, ev, format: fmt, legacy: :error)
|
|
665
|
+
end,
|
|
666
|
+
ActiveSupport::Notifications.subscribe(LEGACY_STALE_SKIPPED) do |*args|
|
|
667
|
+
ev = ActiveSupport::Notifications::Event.new(*args)
|
|
668
|
+
emit_delete_stale(log, severity, ev, format: fmt, legacy: :skipped)
|
|
669
|
+
end
|
|
670
|
+
]
|
|
671
|
+
end
|
|
672
|
+
private_class_method :subscribe_legacy_stale
|
|
673
|
+
end
|
|
674
|
+
end
|
|
675
|
+
end
|