search-engine-for-typesense 30.1.0 → 30.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +5 -5
- data/app/search_engine/search_engine/index_partition_job.rb +7 -26
- data/lib/generators/search_engine/install/install_generator.rb +1 -1
- data/lib/generators/search_engine/install/templates/initializer.rb.tt +11 -11
- data/lib/generators/search_engine/model/model_generator.rb +2 -2
- data/lib/generators/search_engine/model/templates/model.rb.tt +2 -2
- data/lib/search_engine/admin/stopwords.rb +1 -1
- data/lib/search_engine/admin/synonyms.rb +1 -1
- data/lib/search_engine/ast/node.rb +1 -1
- data/lib/search_engine/ast.rb +1 -1
- data/lib/search_engine/base/creation.rb +4 -4
- data/lib/search_engine/cli/doctor.rb +6 -6
- data/lib/search_engine/client/request_builder.rb +2 -2
- data/lib/search_engine/client.rb +19 -19
- data/lib/search_engine/collection_resolver.rb +7 -2
- data/lib/search_engine/config/presets.rb +7 -7
- data/lib/search_engine/config.rb +5 -5
- data/lib/search_engine/console_helpers.rb +4 -4
- data/lib/search_engine/dispatcher.rb +2 -2
- data/lib/search_engine/dsl/parser.rb +9 -9
- data/lib/search_engine/errors.rb +6 -6
- data/lib/search_engine/filters/sanitizer.rb +24 -44
- data/lib/search_engine/hydration/materializers.rb +13 -7
- data/lib/search_engine/indexer/batch_planner.rb +3 -8
- data/lib/search_engine/indexer/import_dispatcher.rb +1 -1
- data/lib/search_engine/indexer/retry_policy.rb +9 -6
- data/lib/search_engine/indexer.rb +3 -176
- data/lib/search_engine/instrumentation.rb +1 -1
- data/lib/search_engine/joins/guard.rb +4 -4
- data/lib/search_engine/joins/resolver.rb +2 -2
- data/lib/search_engine/logging_subscriber.rb +4 -4
- data/lib/search_engine/mapper.rb +13 -21
- data/lib/search_engine/multi.rb +2 -2
- data/lib/search_engine/multi_result.rb +1 -1
- data/lib/search_engine/notifications/compact_logger.rb +3 -3
- data/lib/search_engine/otel.rb +5 -5
- data/lib/search_engine/partitioner.rb +5 -5
- data/lib/search_engine/ranking_plan.rb +4 -4
- data/lib/search_engine/relation/compiler.rb +11 -6
- data/lib/search_engine/relation/dsl/filters.rb +9 -9
- data/lib/search_engine/relation/dsl/selection.rb +3 -3
- data/lib/search_engine/relation/dsl.rb +17 -17
- data/lib/search_engine/relation/dx.rb +4 -4
- data/lib/search_engine/relation/materializers.rb +1 -1
- data/lib/search_engine/relation/options.rb +1 -1
- data/lib/search_engine/relation.rb +1 -1
- data/lib/search_engine/result.rb +1 -1
- data/lib/search_engine/schema.rb +4 -4
- data/lib/search_engine/sources/active_record_source.rb +0 -1
- data/lib/search_engine/sources/lambda_source.rb +1 -1
- data/lib/search_engine/sources/sql_source.rb +1 -1
- data/lib/search_engine/sources.rb +3 -3
- data/lib/search_engine/test/stub_client.rb +8 -8
- data/lib/search_engine/test.rb +2 -2
- data/lib/search_engine/version.rb +1 -1
- metadata +2 -2
|
@@ -42,7 +42,7 @@ module SearchEngine
|
|
|
42
42
|
# @param into [String, nil] target collection; defaults to resolver or the logical collection alias
|
|
43
43
|
# @return [Summary]
|
|
44
44
|
# @raise [SearchEngine::Errors::InvalidParams]
|
|
45
|
-
# @see `https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/indexer#partitioning`
|
|
45
|
+
# @see `https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/indexer#partitioning`
|
|
46
46
|
def self.rebuild_partition!(klass, partition:, into: nil)
|
|
47
47
|
raise Errors::InvalidParams, 'klass must be a Class' unless klass.is_a?(Class)
|
|
48
48
|
unless klass.ancestors.include?(SearchEngine::Base)
|
|
@@ -102,7 +102,7 @@ module SearchEngine
|
|
|
102
102
|
# @param dry_run [Boolean]
|
|
103
103
|
# @return [Hash]
|
|
104
104
|
# @raise [SearchEngine::Errors::InvalidParams]
|
|
105
|
-
# @see `https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/indexer#stale-deletes`
|
|
105
|
+
# @see `https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/indexer#stale-deletes`
|
|
106
106
|
def self.delete_stale!(klass, partition: nil, into: nil, dry_run: false)
|
|
107
107
|
validate_stale_args!(klass)
|
|
108
108
|
|
|
@@ -164,7 +164,7 @@ module SearchEngine
|
|
|
164
164
|
# @param max_parallel [Integer] maximum parallel threads for batch processing (default: 1)
|
|
165
165
|
# @return [Summary]
|
|
166
166
|
# @raise [SearchEngine::Errors::InvalidParams]
|
|
167
|
-
# @see `https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/indexer`
|
|
167
|
+
# @see `https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/indexer`
|
|
168
168
|
# @see `https://typesense.org/docs/latest/api/documents.html#import-documents`
|
|
169
169
|
def self.import!(klass, into:, enum:, batch_size: nil, action: :upsert, log_batches: true, max_parallel: 1)
|
|
170
170
|
SearchEngine::Indexer::BulkImport.call(
|
|
@@ -357,179 +357,6 @@ module SearchEngine
|
|
|
357
357
|
end
|
|
358
358
|
end
|
|
359
359
|
|
|
360
|
-
def import_batch_with_handling(client, collection, docs, action, next_index)
|
|
361
|
-
buffer = +''
|
|
362
|
-
docs_count = encode_jsonl!(docs, buffer)
|
|
363
|
-
bytes_sent = buffer.bytesize
|
|
364
|
-
idx = next_index.call
|
|
365
|
-
|
|
366
|
-
begin
|
|
367
|
-
attempt_stats = with_retries do |attempt|
|
|
368
|
-
perform_attempt(client, collection, action, buffer, docs_count, bytes_sent, idx, attempt)
|
|
369
|
-
end
|
|
370
|
-
[attempt_stats]
|
|
371
|
-
rescue Errors::Api => error
|
|
372
|
-
if error.status.to_i == 413 && docs.size > 1
|
|
373
|
-
mid = docs.size / 2
|
|
374
|
-
left = docs[0...mid]
|
|
375
|
-
right = docs[mid..]
|
|
376
|
-
import_batch_with_handling(client, collection, left, action, next_index) +
|
|
377
|
-
import_batch_with_handling(client, collection, right, action, next_index)
|
|
378
|
-
else
|
|
379
|
-
[
|
|
380
|
-
{
|
|
381
|
-
index: idx,
|
|
382
|
-
docs_count: docs_count,
|
|
383
|
-
success_count: 0,
|
|
384
|
-
failure_count: docs_count,
|
|
385
|
-
attempts: 1,
|
|
386
|
-
http_status: error.status.to_i,
|
|
387
|
-
duration_ms: 0.0,
|
|
388
|
-
bytes_sent: bytes_sent,
|
|
389
|
-
errors_sample: [safe_error_excerpt(error)]
|
|
390
|
-
}
|
|
391
|
-
]
|
|
392
|
-
end
|
|
393
|
-
end
|
|
394
|
-
end
|
|
395
|
-
|
|
396
|
-
def perform_attempt(client, collection, action, jsonl, docs_count, bytes_sent, idx, attempt)
|
|
397
|
-
start = monotonic_ms
|
|
398
|
-
success_count = 0
|
|
399
|
-
failure_count = 0
|
|
400
|
-
http_status = 200
|
|
401
|
-
error_sample = []
|
|
402
|
-
|
|
403
|
-
if defined?(ActiveSupport::Notifications)
|
|
404
|
-
se_payload = {
|
|
405
|
-
collection: SearchEngine::Instrumentation.context[:collection] || collection,
|
|
406
|
-
into: collection,
|
|
407
|
-
batch_index: idx,
|
|
408
|
-
docs_count: docs_count,
|
|
409
|
-
success_count: nil,
|
|
410
|
-
failure_count: nil,
|
|
411
|
-
attempts: attempt,
|
|
412
|
-
http_status: nil,
|
|
413
|
-
bytes_sent: bytes_sent,
|
|
414
|
-
transient_retry: attempt > 1,
|
|
415
|
-
retry_after_s: nil,
|
|
416
|
-
error_sample: nil
|
|
417
|
-
}
|
|
418
|
-
SearchEngine::Instrumentation.instrument('search_engine.indexer.batch_import', se_payload) do |ctx|
|
|
419
|
-
raw = client.import_documents(collection: collection, jsonl: jsonl, action: action)
|
|
420
|
-
success_count, failure_count, error_sample = parse_import_response(raw)
|
|
421
|
-
http_status = 200
|
|
422
|
-
ctx[:success_count] = success_count
|
|
423
|
-
ctx[:failure_count] = failure_count
|
|
424
|
-
ctx[:http_status] = http_status
|
|
425
|
-
end
|
|
426
|
-
else
|
|
427
|
-
raw = client.import_documents(collection: collection, jsonl: jsonl, action: action)
|
|
428
|
-
success_count, failure_count, error_sample = parse_import_response(raw)
|
|
429
|
-
end
|
|
430
|
-
|
|
431
|
-
duration = monotonic_ms - start
|
|
432
|
-
{
|
|
433
|
-
index: idx,
|
|
434
|
-
docs_count: docs_count,
|
|
435
|
-
success_count: success_count,
|
|
436
|
-
failure_count: failure_count,
|
|
437
|
-
attempts: attempt,
|
|
438
|
-
http_status: http_status,
|
|
439
|
-
duration_ms: duration.round(1),
|
|
440
|
-
bytes_sent: bytes_sent,
|
|
441
|
-
errors_sample: error_sample
|
|
442
|
-
}
|
|
443
|
-
end
|
|
444
|
-
|
|
445
|
-
def with_retries
|
|
446
|
-
cfg = SearchEngine.config.indexer
|
|
447
|
-
attempts = cfg&.retries && cfg.retries[:attempts].to_i.positive? ? cfg.retries[:attempts].to_i : 3
|
|
448
|
-
base = cfg&.retries && cfg.retries[:base].to_f.positive? ? cfg.retries[:base].to_f : 0.5
|
|
449
|
-
max = cfg&.retries && cfg.retries[:max].to_f.positive? ? cfg.retries[:max].to_f : 5.0
|
|
450
|
-
jitter = cfg&.retries && cfg.retries[:jitter_fraction].to_f >= 0 ? cfg.retries[:jitter_fraction].to_f : 0.2
|
|
451
|
-
|
|
452
|
-
(1..attempts).each do |i|
|
|
453
|
-
return yield(i)
|
|
454
|
-
rescue Errors::Timeout, Errors::Connection
|
|
455
|
-
raise if i >= attempts
|
|
456
|
-
|
|
457
|
-
sleep_with_backoff(i, base: base, max: max, jitter_fraction: jitter)
|
|
458
|
-
rescue Errors::Api => error
|
|
459
|
-
code = error.status.to_i
|
|
460
|
-
raise unless transient_status?(code)
|
|
461
|
-
raise if i >= attempts
|
|
462
|
-
|
|
463
|
-
sleep_with_backoff(i, base: base, max: max, jitter_fraction: jitter)
|
|
464
|
-
end
|
|
465
|
-
end
|
|
466
|
-
|
|
467
|
-
def sleep_with_backoff(attempt, base:, max:, jitter_fraction:)
|
|
468
|
-
exp = [base * (2 ** (attempt - 1)), max].min
|
|
469
|
-
jitter = exp * jitter_fraction
|
|
470
|
-
delta = rand(-jitter..jitter)
|
|
471
|
-
sleep_time = exp + delta
|
|
472
|
-
sleep(sleep_time) if sleep_time.positive?
|
|
473
|
-
end
|
|
474
|
-
|
|
475
|
-
def transient_status?(code)
|
|
476
|
-
return true if code == 429
|
|
477
|
-
return true if code >= 500 && code <= 599
|
|
478
|
-
|
|
479
|
-
false
|
|
480
|
-
end
|
|
481
|
-
|
|
482
|
-
def to_array(batch)
|
|
483
|
-
return batch if batch.is_a?(Array)
|
|
484
|
-
|
|
485
|
-
batch.respond_to?(:to_a) ? batch.to_a : Array(batch)
|
|
486
|
-
end
|
|
487
|
-
|
|
488
|
-
def encode_jsonl!(docs, buffer)
|
|
489
|
-
count = 0
|
|
490
|
-
buffer.clear
|
|
491
|
-
docs.each do |raw|
|
|
492
|
-
doc = ensure_hash_document(raw)
|
|
493
|
-
ensure_id!(doc)
|
|
494
|
-
# Force system timestamp field just before serialization; developers cannot override.
|
|
495
|
-
now_i = if defined?(Time) && defined?(Time.zone) && Time.zone
|
|
496
|
-
Time.zone.now.to_i
|
|
497
|
-
else
|
|
498
|
-
Time.now.to_i
|
|
499
|
-
end
|
|
500
|
-
doc[:doc_updated_at] = now_i if doc.is_a?(Hash)
|
|
501
|
-
buffer << JSON.generate(doc)
|
|
502
|
-
buffer << "\n" if count < (docs.size - 1)
|
|
503
|
-
count += 1
|
|
504
|
-
end
|
|
505
|
-
count
|
|
506
|
-
end
|
|
507
|
-
|
|
508
|
-
def ensure_hash_document(obj)
|
|
509
|
-
if obj.is_a?(Hash)
|
|
510
|
-
obj
|
|
511
|
-
else
|
|
512
|
-
raise Errors::InvalidParams,
|
|
513
|
-
'Indexer requires batches of Hash-like documents with at least an :id key. ' \
|
|
514
|
-
'Mapping DSL is not available yet. See https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/indexer.'
|
|
515
|
-
end
|
|
516
|
-
end
|
|
517
|
-
|
|
518
|
-
def ensure_id!(doc)
|
|
519
|
-
has_id = doc.key?(:id) || doc.key?('id')
|
|
520
|
-
raise Errors::InvalidParams, 'document is missing required id' unless has_id
|
|
521
|
-
end
|
|
522
|
-
|
|
523
|
-
def parse_import_response(raw)
|
|
524
|
-
SearchEngine::Indexer::ImportResponseParser.parse(raw)
|
|
525
|
-
end
|
|
526
|
-
|
|
527
|
-
def safe_error_excerpt(error)
|
|
528
|
-
cls = error.class.name
|
|
529
|
-
msg = error.message.to_s
|
|
530
|
-
"#{cls}: #{msg[0, 200]}"
|
|
531
|
-
end
|
|
532
|
-
|
|
533
360
|
def monotonic_ms
|
|
534
361
|
SearchEngine::Instrumentation.monotonic_ms
|
|
535
362
|
end
|
|
@@ -202,7 +202,7 @@ module SearchEngine
|
|
|
202
202
|
# - :type [Symbol] one of :overlap, :limit_exceeded
|
|
203
203
|
# - :count [Integer]
|
|
204
204
|
# - :limit [Integer, optional] present when type==:limit_exceeded
|
|
205
|
-
# # See also: https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/presets#observability
|
|
205
|
+
# # See also: https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/presets#observability
|
|
206
206
|
# Measure a block and attach duration_ms to payload.
|
|
207
207
|
# @param event [String]
|
|
208
208
|
# @param base_payload [Hash]
|
|
@@ -28,7 +28,7 @@ module SearchEngine
|
|
|
28
28
|
raise SearchEngine::Errors::InvalidJoin.new(
|
|
29
29
|
msg,
|
|
30
30
|
hint: (suggestions&.any? ? "Did you mean #{suggestions.map { |s| ":#{s}" }.join(', ')}?" : nil),
|
|
31
|
-
doc: 'https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/joins#troubleshooting',
|
|
31
|
+
doc: 'https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/joins#troubleshooting',
|
|
32
32
|
details: { assoc: key, known: safe_joins_config(klass).keys }
|
|
33
33
|
)
|
|
34
34
|
end
|
|
@@ -61,7 +61,7 @@ module SearchEngine
|
|
|
61
61
|
raise SearchEngine::Errors::InvalidJoinConfig.new(
|
|
62
62
|
msg,
|
|
63
63
|
hint: 'Declare local_key and foreign_key in join config.',
|
|
64
|
-
doc: 'https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/joins#troubleshooting',
|
|
64
|
+
doc: 'https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/joins#troubleshooting',
|
|
65
65
|
details: { assoc: key, missing: missing }
|
|
66
66
|
)
|
|
67
67
|
end
|
|
@@ -80,7 +80,7 @@ module SearchEngine
|
|
|
80
80
|
|
|
81
81
|
raise SearchEngine::Errors::JoinNotApplied.new(
|
|
82
82
|
"Call .joins(:#{key}) before #{context} on #{key} fields",
|
|
83
|
-
doc: 'https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/joins#troubleshooting',
|
|
83
|
+
doc: 'https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/joins#troubleshooting',
|
|
84
84
|
details: { assoc: key, context: context }
|
|
85
85
|
)
|
|
86
86
|
end
|
|
@@ -141,7 +141,7 @@ module SearchEngine
|
|
|
141
141
|
|
|
142
142
|
raise SearchEngine::Errors::UnsupportedJoinNesting.new(
|
|
143
143
|
'Only one join hop is supported: `$assoc.field`. Use a separate pipeline step to denormalize deeper paths.',
|
|
144
|
-
doc: 'https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/joins#troubleshooting',
|
|
144
|
+
doc: 'https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/joins#troubleshooting',
|
|
145
145
|
details: { path: path }
|
|
146
146
|
)
|
|
147
147
|
end
|
|
@@ -52,7 +52,7 @@ module SearchEngine
|
|
|
52
52
|
model_name = klass.respond_to?(:name) && klass.name ? klass.name : klass.to_s
|
|
53
53
|
raise SearchEngine::Errors::InvalidJoin.new(
|
|
54
54
|
"Unknown #{side} key :#{key} for #{model_name}. Declare it via `attribute :#{key}, ...`.",
|
|
55
|
-
doc: 'https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/joins#troubleshooting',
|
|
55
|
+
doc: 'https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/joins#troubleshooting',
|
|
56
56
|
details: { side: side, key: key, model: model_name }
|
|
57
57
|
)
|
|
58
58
|
end
|
|
@@ -82,7 +82,7 @@ module SearchEngine
|
|
|
82
82
|
"Could not infer a unique shared key. Candidates: #{sugg}"
|
|
83
83
|
raise SearchEngine::Errors::InvalidJoinConfig.new(
|
|
84
84
|
msg,
|
|
85
|
-
doc: 'https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/joins#client-side-fallback',
|
|
85
|
+
doc: 'https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/joins#client-side-fallback',
|
|
86
86
|
details: { assoc: assoc_name, candidates: candidates }
|
|
87
87
|
)
|
|
88
88
|
end
|
|
@@ -20,8 +20,8 @@ module SearchEngine
|
|
|
20
20
|
# sample: 0.0 or mode: nil.
|
|
21
21
|
#
|
|
22
22
|
# @since M8
|
|
23
|
-
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/observability#logging
|
|
24
|
-
# @see `https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/observability`
|
|
23
|
+
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/observability#logging
|
|
24
|
+
# @see `https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/observability`
|
|
25
25
|
module LoggingSubscriber
|
|
26
26
|
class << self
|
|
27
27
|
# Install the subscriber in a reloader-safe and idempotent way.
|
|
@@ -29,7 +29,7 @@ module SearchEngine
|
|
|
29
29
|
# @param config [#mode,#level,#sample,#logger,nil]
|
|
30
30
|
# @return [Object, nil] subscription handle
|
|
31
31
|
# @since M8
|
|
32
|
-
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/observability#logging
|
|
32
|
+
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/observability#logging
|
|
33
33
|
def install!(config = nil)
|
|
34
34
|
uninstall!
|
|
35
35
|
|
|
@@ -60,7 +60,7 @@ module SearchEngine
|
|
|
60
60
|
# Uninstall previously installed subscriber.
|
|
61
61
|
# @return [Boolean]
|
|
62
62
|
# @since M8
|
|
63
|
-
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/observability#logging
|
|
63
|
+
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/observability#logging
|
|
64
64
|
def uninstall!
|
|
65
65
|
return false unless defined?(ActiveSupport::Notifications)
|
|
66
66
|
return false unless @handle
|
data/lib/search_engine/mapper.rb
CHANGED
|
@@ -15,7 +15,7 @@ module SearchEngine
|
|
|
15
15
|
# Describes where data is fetched from and how records are transformed into
|
|
16
16
|
# Typesense documents. Compiled by {SearchEngine::Mapper.for}.
|
|
17
17
|
#
|
|
18
|
-
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/indexer
|
|
18
|
+
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/indexer
|
|
19
19
|
class Dsl
|
|
20
20
|
# @return [Hash, nil] original source definition captured from DSL
|
|
21
21
|
attr_reader :source_def
|
|
@@ -42,7 +42,7 @@ module SearchEngine
|
|
|
42
42
|
# @yield for :lambda sources
|
|
43
43
|
# @return [void]
|
|
44
44
|
# @raise [ArgumentError] when type is nil/blank
|
|
45
|
-
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/indexer
|
|
45
|
+
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/indexer
|
|
46
46
|
def source(type, **options, &block)
|
|
47
47
|
@source_def = { type: type.to_sym, options: options, block: block }
|
|
48
48
|
nil
|
|
@@ -54,7 +54,7 @@ module SearchEngine
|
|
|
54
54
|
# @yieldreturn [Hash, #to_h, #as_json] a document-like object
|
|
55
55
|
# @return [void]
|
|
56
56
|
# @raise [ArgumentError] when no block is given
|
|
57
|
-
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/indexer
|
|
57
|
+
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/indexer
|
|
58
58
|
def map(&block)
|
|
59
59
|
raise ArgumentError, 'map requires a block' unless block
|
|
60
60
|
|
|
@@ -129,7 +129,7 @@ module SearchEngine
|
|
|
129
129
|
# @yieldreturn [Enumerable] a list/Enumerable of opaque partition keys
|
|
130
130
|
# @return [void]
|
|
131
131
|
# @raise [ArgumentError] when no block is given
|
|
132
|
-
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/indexer#partitioning
|
|
132
|
+
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/indexer#partitioning
|
|
133
133
|
def partitions(&block)
|
|
134
134
|
raise ArgumentError, 'partitions requires a block' unless block
|
|
135
135
|
|
|
@@ -173,7 +173,7 @@ module SearchEngine
|
|
|
173
173
|
# @yieldreturn [Enumerable<Array>] yields Arrays of records per batch
|
|
174
174
|
# @return [void]
|
|
175
175
|
# @raise [ArgumentError] when no block is given
|
|
176
|
-
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/indexer#partitioning
|
|
176
|
+
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/indexer#partitioning
|
|
177
177
|
def partition_fetch(&block)
|
|
178
178
|
raise ArgumentError, 'partition_fetch requires a block' unless block
|
|
179
179
|
|
|
@@ -186,7 +186,7 @@ module SearchEngine
|
|
|
186
186
|
# @yieldparam partition [Object]
|
|
187
187
|
# @return [void]
|
|
188
188
|
# @raise [ArgumentError] when no block is given
|
|
189
|
-
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/indexer#partitioning
|
|
189
|
+
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/indexer#partitioning
|
|
190
190
|
def before_partition(&block)
|
|
191
191
|
raise ArgumentError, 'before_partition requires a block' unless block
|
|
192
192
|
|
|
@@ -212,7 +212,7 @@ module SearchEngine
|
|
|
212
212
|
# @yieldparam partition [Object]
|
|
213
213
|
# @return [void]
|
|
214
214
|
# @raise [ArgumentError] when no block is given
|
|
215
|
-
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/indexer#partitioning
|
|
215
|
+
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/indexer#partitioning
|
|
216
216
|
def after_partition(&block)
|
|
217
217
|
raise ArgumentError, 'after_partition requires a block' unless block
|
|
218
218
|
|
|
@@ -234,7 +234,7 @@ module SearchEngine
|
|
|
234
234
|
|
|
235
235
|
# Freeze internal state for immutability and return a definition Hash.
|
|
236
236
|
# @return [Hash]
|
|
237
|
-
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/indexer
|
|
237
|
+
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/indexer
|
|
238
238
|
def to_definition
|
|
239
239
|
{
|
|
240
240
|
source: @source_def,
|
|
@@ -316,7 +316,7 @@ module SearchEngine
|
|
|
316
316
|
# Validates mapped documents against the compiled schema, sets hidden flags
|
|
317
317
|
# for array/optional fields and emits instrumentation.
|
|
318
318
|
#
|
|
319
|
-
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/indexer
|
|
319
|
+
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/indexer
|
|
320
320
|
class Compiled
|
|
321
321
|
attr_reader :klass
|
|
322
322
|
|
|
@@ -340,32 +340,24 @@ module SearchEngine
|
|
|
340
340
|
# @return [Array<Array<Hash>, Hash>] [documents, report]
|
|
341
341
|
# @raise [SearchEngine::Errors::InvalidParams] on missing required fields or invalid document shape
|
|
342
342
|
# @raise [SearchEngine::Errors::InvalidField] when strict_unknown_keys is enabled and extras are present
|
|
343
|
-
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/indexer#troubleshooting
|
|
343
|
+
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/indexer#troubleshooting
|
|
344
344
|
def map_batch!(rows, batch_index: nil)
|
|
345
345
|
start_ms = monotonic_ms
|
|
346
346
|
docs = []
|
|
347
347
|
stats = init_stats
|
|
348
|
+
now_i = defined?(Time.zone) && Time.zone ? Time.zone.now.to_i : Time.now.to_i
|
|
348
349
|
|
|
349
350
|
rows.each do |row|
|
|
350
351
|
hash = normalize_document(@map_proc.call(row))
|
|
351
|
-
# Ignore any provided id from map; always inject computed document id
|
|
352
352
|
hash.delete(:id)
|
|
353
353
|
hash.delete('id')
|
|
354
354
|
begin
|
|
355
355
|
computed_id = @klass.compute_document_id(row)
|
|
356
356
|
rescue NoMethodError
|
|
357
|
-
# Fallback for older compiled mappers if needed; derive from record.id
|
|
358
357
|
rid = row.respond_to?(:id) ? row.id : nil
|
|
359
358
|
computed_id = rid.is_a?(String) ? rid : rid.to_s
|
|
360
359
|
end
|
|
361
360
|
hash[:id] = computed_id
|
|
362
|
-
# Force system timestamp field on every document; developers cannot override.
|
|
363
|
-
now_i = if defined?(Time) && defined?(Time.zone) && Time.zone
|
|
364
|
-
Time.zone.now.to_i
|
|
365
|
-
else
|
|
366
|
-
Time.now.to_i
|
|
367
|
-
end
|
|
368
|
-
# Overwrite any provided value
|
|
369
361
|
hash[:doc_updated_at] = now_i
|
|
370
362
|
|
|
371
363
|
normalize_optional_blank_strings!(hash)
|
|
@@ -505,7 +497,7 @@ module SearchEngine
|
|
|
505
497
|
instrument_error(error_class: 'SearchEngine::Errors::InvalidParams', message: message)
|
|
506
498
|
raise SearchEngine::Errors::InvalidParams.new(
|
|
507
499
|
message,
|
|
508
|
-
doc: 'https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/indexer#troubleshooting',
|
|
500
|
+
doc: 'https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/indexer#troubleshooting',
|
|
509
501
|
details: { missing_required: stats[:missing_required].sort }
|
|
510
502
|
)
|
|
511
503
|
end
|
|
@@ -520,7 +512,7 @@ module SearchEngine
|
|
|
520
512
|
instrument_error(error_class: 'SearchEngine::Errors::InvalidField', message: message)
|
|
521
513
|
raise SearchEngine::Errors::InvalidField.new(
|
|
522
514
|
message,
|
|
523
|
-
doc: 'https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/indexer#troubleshooting',
|
|
515
|
+
doc: 'https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/indexer#troubleshooting',
|
|
524
516
|
details: { extras: stats[:extras_samples].sort }
|
|
525
517
|
)
|
|
526
518
|
end
|
data/lib/search_engine/multi.rb
CHANGED
|
@@ -51,7 +51,7 @@ module SearchEngine
|
|
|
51
51
|
# @return [self]
|
|
52
52
|
# @raise [ArgumentError] when label is duplicate/invalid, relation is invalid, or api_key is provided
|
|
53
53
|
# @note Per-search api_key is not supported by the underlying Typesense client and will raise.
|
|
54
|
-
# @see `https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/multi-search-guide`
|
|
54
|
+
# @see `https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/multi-search-guide`
|
|
55
55
|
def add(label, relation, api_key: nil)
|
|
56
56
|
key = Multi.canonicalize_label(label)
|
|
57
57
|
raise ArgumentError, "Multi#add: duplicate label #{label.inspect} (labels must be unique)." if @keys.include?(key)
|
|
@@ -103,7 +103,7 @@ module SearchEngine
|
|
|
103
103
|
# m.add(:products, Product.all.per(10))
|
|
104
104
|
# m.to_payloads(common: { query_by: SearchEngine.config.default_query_by })
|
|
105
105
|
# # => [{ collection: "products", q: "*", query_by: "name", per_page: 10 }]
|
|
106
|
-
# @see `https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/multi-search-guide`
|
|
106
|
+
# @see `https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/multi-search-guide`
|
|
107
107
|
def to_payloads(common: {})
|
|
108
108
|
raise ArgumentError, 'common must be a Hash' unless common.is_a?(Hash)
|
|
109
109
|
|
|
@@ -34,7 +34,7 @@ module SearchEngine
|
|
|
34
34
|
# @param raw_results [Array<Hash>] ordered raw result items (one per label)
|
|
35
35
|
# @param klasses [Array<Class>, Hash{(String,Symbol)=>Class}, nil] optional model classes
|
|
36
36
|
# @raise [ArgumentError] when sizes mismatch, labels invalid/duplicate, or inputs malformed
|
|
37
|
-
# @see `https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/multi-search-guide`
|
|
37
|
+
# @see `https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/multi-search-guide`
|
|
38
38
|
def initialize(labels:, raw_results:, klasses: nil)
|
|
39
39
|
@labels = canonicalize_labels(labels)
|
|
40
40
|
@map = {}
|
|
@@ -17,7 +17,7 @@ module SearchEngine
|
|
|
17
17
|
# SearchEngine::Notifications::CompactLogger.unsubscribe
|
|
18
18
|
#
|
|
19
19
|
# @since M8
|
|
20
|
-
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/observability#logging
|
|
20
|
+
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/observability#logging
|
|
21
21
|
class CompactLogger
|
|
22
22
|
EVENT_SEARCH = 'search_engine.search'
|
|
23
23
|
EVENT_MULTI = 'search_engine.multi_search'
|
|
@@ -42,7 +42,7 @@ module SearchEngine
|
|
|
42
42
|
# @param format [Symbol, nil] :kv or :json; defaults to config.observability.log_format
|
|
43
43
|
# @return [Array<Object>] subscription handles that can be passed to {.unsubscribe}
|
|
44
44
|
# @since M8
|
|
45
|
-
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/observability#logging
|
|
45
|
+
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/observability#logging
|
|
46
46
|
def self.subscribe(logger: default_logger, level: :info, include_params: false, format: nil)
|
|
47
47
|
return [] unless defined?(ActiveSupport::Notifications)
|
|
48
48
|
|
|
@@ -57,7 +57,7 @@ module SearchEngine
|
|
|
57
57
|
# @param handle [Array<Object>, Object, nil] handles returned by {.subscribe}
|
|
58
58
|
# @return [Boolean] true when unsubscribed
|
|
59
59
|
# @since M8
|
|
60
|
-
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/observability#logging
|
|
60
|
+
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/observability#logging
|
|
61
61
|
def self.unsubscribe(handle = @last_handle)
|
|
62
62
|
return false unless handle
|
|
63
63
|
|
data/lib/search_engine/otel.rb
CHANGED
|
@@ -6,7 +6,7 @@ module SearchEngine
|
|
|
6
6
|
# OpenTelemetry SDK and by `SearchEngine.config.opentelemetry.enabled`.
|
|
7
7
|
#
|
|
8
8
|
# @since M8
|
|
9
|
-
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/observability#opentelemetry
|
|
9
|
+
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/observability#opentelemetry
|
|
10
10
|
#
|
|
11
11
|
# Public API:
|
|
12
12
|
# - .installed? => Boolean
|
|
@@ -17,14 +17,14 @@ module SearchEngine
|
|
|
17
17
|
class << self
|
|
18
18
|
# @return [Boolean] whether the OpenTelemetry SDK is available
|
|
19
19
|
# @since M8
|
|
20
|
-
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/observability#opentelemetry
|
|
20
|
+
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/observability#opentelemetry
|
|
21
21
|
def installed?
|
|
22
22
|
defined?(::OpenTelemetry::SDK)
|
|
23
23
|
end
|
|
24
24
|
|
|
25
25
|
# @return [Boolean] whether the adapter should be active
|
|
26
26
|
# @since M8
|
|
27
|
-
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/observability#opentelemetry
|
|
27
|
+
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/observability#opentelemetry
|
|
28
28
|
def enabled?
|
|
29
29
|
installed? && SearchEngine.respond_to?(:config) && SearchEngine.config&.opentelemetry&.enabled
|
|
30
30
|
end
|
|
@@ -32,7 +32,7 @@ module SearchEngine
|
|
|
32
32
|
# Start the adapter (idempotent). No-ops when disabled or SDK unavailable.
|
|
33
33
|
# @return [Object, nil] subscription handle or nil when not installed/enabled
|
|
34
34
|
# @since M8
|
|
35
|
-
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/observability#opentelemetry
|
|
35
|
+
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/observability#opentelemetry
|
|
36
36
|
def start!
|
|
37
37
|
stop!
|
|
38
38
|
return nil unless enabled?
|
|
@@ -58,7 +58,7 @@ module SearchEngine
|
|
|
58
58
|
# Stop the adapter if previously started.
|
|
59
59
|
# @return [Boolean]
|
|
60
60
|
# @since M8
|
|
61
|
-
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/observability#opentelemetry
|
|
61
|
+
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/observability#opentelemetry
|
|
62
62
|
def stop!
|
|
63
63
|
return false unless defined?(ActiveSupport::Notifications)
|
|
64
64
|
return false unless @handle
|
|
@@ -34,7 +34,7 @@ module SearchEngine
|
|
|
34
34
|
# Enumerate partition keys. Validates the return value shape.
|
|
35
35
|
# @return [Enumerable] list/Enumerable of opaque partition tokens
|
|
36
36
|
# @raise [SearchEngine::Errors::InvalidParams] when the block does not return an Enumerable
|
|
37
|
-
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/indexer#partitioning
|
|
37
|
+
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/indexer#partitioning
|
|
38
38
|
def partitions
|
|
39
39
|
return [] unless @partitions_proc
|
|
40
40
|
|
|
@@ -42,7 +42,7 @@ module SearchEngine
|
|
|
42
42
|
unless res.respond_to?(:each)
|
|
43
43
|
raise SearchEngine::Errors::InvalidParams,
|
|
44
44
|
'partitions block must return an Enumerable of partition keys (Array acceptable). ' \
|
|
45
|
-
'See https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/indexer#partitioning.'
|
|
45
|
+
'See https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/indexer#partitioning.'
|
|
46
46
|
end
|
|
47
47
|
res
|
|
48
48
|
end
|
|
@@ -52,7 +52,7 @@ module SearchEngine
|
|
|
52
52
|
# @return [Enumerable<Array>] enumerator yielding Arrays of records
|
|
53
53
|
# @raise [ArgumentError] when partition_fetch is not defined
|
|
54
54
|
# @raise [SearchEngine::Errors::InvalidParams] when the block returns a non-enumerable or yields non-arrays
|
|
55
|
-
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/indexer#partitioning
|
|
55
|
+
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/indexer#partitioning
|
|
56
56
|
def partition_fetch_enum(partition)
|
|
57
57
|
raise ArgumentError, 'partition_fetch not defined' unless @partition_fetch_proc
|
|
58
58
|
|
|
@@ -60,7 +60,7 @@ module SearchEngine
|
|
|
60
60
|
unless enum.respond_to?(:each)
|
|
61
61
|
raise SearchEngine::Errors::InvalidParams,
|
|
62
62
|
'partition_fetch must return an Enumerable yielding Arrays of records. ' \
|
|
63
|
-
'See https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/indexer#partitioning.'
|
|
63
|
+
'See https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/indexer#partitioning.'
|
|
64
64
|
end
|
|
65
65
|
|
|
66
66
|
Enumerator.new do |y|
|
|
@@ -90,7 +90,7 @@ module SearchEngine
|
|
|
90
90
|
# Resolve a compiled partitioner for a model class, or nil if directives are absent.
|
|
91
91
|
# @param klass [Class]
|
|
92
92
|
# @return [SearchEngine::Partitioner::Compiled, nil]
|
|
93
|
-
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/indexer#partitioning
|
|
93
|
+
# @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/indexer#partitioning
|
|
94
94
|
def for(klass)
|
|
95
95
|
dsl = mapper_dsl_for(klass)
|
|
96
96
|
return nil unless dsl
|
|
@@ -47,7 +47,7 @@ module SearchEngine
|
|
|
47
47
|
raise SearchEngine::Errors::InvalidOption.new(
|
|
48
48
|
'InvalidOption: query_by is empty; cannot apply query_by_weights',
|
|
49
49
|
hint: 'Set SearchEngine.config.default_query_by or pass options(query_by: ...)',
|
|
50
|
-
doc: 'https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/ranking#weights'
|
|
50
|
+
doc: 'https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/ranking#weights'
|
|
51
51
|
)
|
|
52
52
|
end
|
|
53
53
|
|
|
@@ -55,7 +55,7 @@ module SearchEngine
|
|
|
55
55
|
if normalized_weights.all? { |w| w.to_i.zero? }
|
|
56
56
|
raise SearchEngine::Errors::InvalidOption.new(
|
|
57
57
|
'InvalidOption: at least one weighted field must have weight > 0',
|
|
58
|
-
doc: 'https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/ranking#weights'
|
|
58
|
+
doc: 'https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/ranking#weights'
|
|
59
59
|
)
|
|
60
60
|
end
|
|
61
61
|
out[:query_by_weights] = normalized_weights.join(',')
|
|
@@ -86,7 +86,7 @@ module SearchEngine
|
|
|
86
86
|
end
|
|
87
87
|
raise SearchEngine::Errors::InvalidOption.new(
|
|
88
88
|
"InvalidOption: weight specified for unknown field #{unknown.first.inspect}#{suffix}",
|
|
89
|
-
doc: 'https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/relation-reference#selection',
|
|
89
|
+
doc: 'https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/relation-reference#selection',
|
|
90
90
|
details: { unknown: unknown.first, allowed: known }
|
|
91
91
|
)
|
|
92
92
|
end
|
|
@@ -95,7 +95,7 @@ module SearchEngine
|
|
|
95
95
|
rescue ArgumentError, TypeError
|
|
96
96
|
raise SearchEngine::Errors::InvalidOption.new(
|
|
97
97
|
'InvalidOption: query_by_weights must compile to integers',
|
|
98
|
-
doc: 'https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/ranking#weights'
|
|
98
|
+
doc: 'https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/ranking#weights'
|
|
99
99
|
)
|
|
100
100
|
end
|
|
101
101
|
|