search-engine-for-typesense 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/README.md +148 -0
- data/app/search_engine/search_engine/app_info.rb +11 -0
- data/app/search_engine/search_engine/index_partition_job.rb +170 -0
- data/lib/generators/search_engine/install/install_generator.rb +20 -0
- data/lib/generators/search_engine/install/templates/initializer.rb.tt +230 -0
- data/lib/generators/search_engine/model/model_generator.rb +86 -0
- data/lib/generators/search_engine/model/templates/model.rb.tt +12 -0
- data/lib/search-engine-for-typesense.rb +12 -0
- data/lib/search_engine/active_record_syncable.rb +247 -0
- data/lib/search_engine/admin/stopwords.rb +125 -0
- data/lib/search_engine/admin/synonyms.rb +125 -0
- data/lib/search_engine/admin.rb +12 -0
- data/lib/search_engine/ast/and.rb +52 -0
- data/lib/search_engine/ast/binary_op.rb +75 -0
- data/lib/search_engine/ast/eq.rb +19 -0
- data/lib/search_engine/ast/group.rb +18 -0
- data/lib/search_engine/ast/gt.rb +12 -0
- data/lib/search_engine/ast/gte.rb +12 -0
- data/lib/search_engine/ast/in.rb +28 -0
- data/lib/search_engine/ast/lt.rb +12 -0
- data/lib/search_engine/ast/lte.rb +12 -0
- data/lib/search_engine/ast/matches.rb +55 -0
- data/lib/search_engine/ast/node.rb +176 -0
- data/lib/search_engine/ast/not_eq.rb +13 -0
- data/lib/search_engine/ast/not_in.rb +24 -0
- data/lib/search_engine/ast/or.rb +52 -0
- data/lib/search_engine/ast/prefix.rb +51 -0
- data/lib/search_engine/ast/raw.rb +41 -0
- data/lib/search_engine/ast/unary_op.rb +43 -0
- data/lib/search_engine/ast.rb +101 -0
- data/lib/search_engine/base/creation.rb +727 -0
- data/lib/search_engine/base/deletion.rb +80 -0
- data/lib/search_engine/base/display_coercions.rb +36 -0
- data/lib/search_engine/base/hydration.rb +312 -0
- data/lib/search_engine/base/index_maintenance/cleanup.rb +202 -0
- data/lib/search_engine/base/index_maintenance/lifecycle.rb +251 -0
- data/lib/search_engine/base/index_maintenance/schema.rb +117 -0
- data/lib/search_engine/base/index_maintenance.rb +459 -0
- data/lib/search_engine/base/indexing_dsl.rb +255 -0
- data/lib/search_engine/base/joins.rb +479 -0
- data/lib/search_engine/base/model_dsl.rb +472 -0
- data/lib/search_engine/base/presets.rb +43 -0
- data/lib/search_engine/base/pretty_printer.rb +315 -0
- data/lib/search_engine/base/relation_delegation.rb +42 -0
- data/lib/search_engine/base/scopes.rb +113 -0
- data/lib/search_engine/base/updating.rb +92 -0
- data/lib/search_engine/base.rb +38 -0
- data/lib/search_engine/bulk.rb +284 -0
- data/lib/search_engine/cache.rb +33 -0
- data/lib/search_engine/cascade.rb +531 -0
- data/lib/search_engine/cli/doctor.rb +631 -0
- data/lib/search_engine/cli/support.rb +217 -0
- data/lib/search_engine/cli.rb +222 -0
- data/lib/search_engine/client/http_adapter.rb +63 -0
- data/lib/search_engine/client/request_builder.rb +92 -0
- data/lib/search_engine/client/services/base.rb +74 -0
- data/lib/search_engine/client/services/collections.rb +161 -0
- data/lib/search_engine/client/services/documents.rb +214 -0
- data/lib/search_engine/client/services/operations.rb +152 -0
- data/lib/search_engine/client/services/search.rb +190 -0
- data/lib/search_engine/client/services.rb +29 -0
- data/lib/search_engine/client.rb +765 -0
- data/lib/search_engine/client_options.rb +20 -0
- data/lib/search_engine/collection_resolver.rb +191 -0
- data/lib/search_engine/collections_graph.rb +330 -0
- data/lib/search_engine/compiled_params.rb +143 -0
- data/lib/search_engine/compiler.rb +383 -0
- data/lib/search_engine/config/observability.rb +27 -0
- data/lib/search_engine/config/presets.rb +92 -0
- data/lib/search_engine/config/selection.rb +16 -0
- data/lib/search_engine/config/typesense.rb +48 -0
- data/lib/search_engine/config/validators.rb +97 -0
- data/lib/search_engine/config.rb +917 -0
- data/lib/search_engine/console_helpers.rb +130 -0
- data/lib/search_engine/deletion.rb +103 -0
- data/lib/search_engine/dispatcher.rb +125 -0
- data/lib/search_engine/dsl/parser.rb +582 -0
- data/lib/search_engine/engine.rb +167 -0
- data/lib/search_engine/errors.rb +290 -0
- data/lib/search_engine/filters/sanitizer.rb +189 -0
- data/lib/search_engine/hydration/materializers.rb +808 -0
- data/lib/search_engine/hydration/selection_context.rb +96 -0
- data/lib/search_engine/indexer/batch_planner.rb +76 -0
- data/lib/search_engine/indexer/bulk_import.rb +626 -0
- data/lib/search_engine/indexer/import_dispatcher.rb +198 -0
- data/lib/search_engine/indexer/retry_policy.rb +103 -0
- data/lib/search_engine/indexer.rb +747 -0
- data/lib/search_engine/instrumentation.rb +308 -0
- data/lib/search_engine/joins/guard.rb +202 -0
- data/lib/search_engine/joins/resolver.rb +95 -0
- data/lib/search_engine/logging/color.rb +78 -0
- data/lib/search_engine/logging/format_helpers.rb +92 -0
- data/lib/search_engine/logging/partition_progress.rb +53 -0
- data/lib/search_engine/logging_subscriber.rb +388 -0
- data/lib/search_engine/mapper.rb +785 -0
- data/lib/search_engine/multi.rb +286 -0
- data/lib/search_engine/multi_result.rb +186 -0
- data/lib/search_engine/notifications/compact_logger.rb +675 -0
- data/lib/search_engine/observability.rb +162 -0
- data/lib/search_engine/operations.rb +58 -0
- data/lib/search_engine/otel.rb +227 -0
- data/lib/search_engine/partitioner.rb +128 -0
- data/lib/search_engine/ranking_plan.rb +118 -0
- data/lib/search_engine/registry.rb +158 -0
- data/lib/search_engine/relation/compiler.rb +711 -0
- data/lib/search_engine/relation/deletion.rb +37 -0
- data/lib/search_engine/relation/dsl/filters.rb +624 -0
- data/lib/search_engine/relation/dsl/selection.rb +240 -0
- data/lib/search_engine/relation/dsl.rb +903 -0
- data/lib/search_engine/relation/dx/dry_run.rb +59 -0
- data/lib/search_engine/relation/dx/friendly_where.rb +24 -0
- data/lib/search_engine/relation/dx.rb +231 -0
- data/lib/search_engine/relation/materializers.rb +118 -0
- data/lib/search_engine/relation/options.rb +138 -0
- data/lib/search_engine/relation/state.rb +274 -0
- data/lib/search_engine/relation/updating.rb +44 -0
- data/lib/search_engine/relation.rb +623 -0
- data/lib/search_engine/result.rb +664 -0
- data/lib/search_engine/schema.rb +1083 -0
- data/lib/search_engine/sources/active_record_source.rb +185 -0
- data/lib/search_engine/sources/base.rb +62 -0
- data/lib/search_engine/sources/lambda_source.rb +55 -0
- data/lib/search_engine/sources/sql_source.rb +196 -0
- data/lib/search_engine/sources.rb +71 -0
- data/lib/search_engine/stale_rules.rb +160 -0
- data/lib/search_engine/test/minitest_assertions.rb +57 -0
- data/lib/search_engine/test/offline_client.rb +134 -0
- data/lib/search_engine/test/rspec_matchers.rb +77 -0
- data/lib/search_engine/test/stub_client.rb +201 -0
- data/lib/search_engine/test.rb +66 -0
- data/lib/search_engine/test_autoload.rb +8 -0
- data/lib/search_engine/update.rb +35 -0
- data/lib/search_engine/version.rb +7 -0
- data/lib/search_engine.rb +332 -0
- data/lib/tasks/search_engine.rake +501 -0
- data/lib/tasks/search_engine_doctor.rake +16 -0
- metadata +225 -0
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SearchEngine
|
|
4
|
+
class Client
|
|
5
|
+
module Services
|
|
6
|
+
# Handles single and multi search workflows while keeping observability hooks intact.
|
|
7
|
+
class Search < Base
|
|
8
|
+
# Execute a single search request.
|
|
9
|
+
# @param collection [String]
|
|
10
|
+
# @param params [Hash]
|
|
11
|
+
# @param url_opts [Hash]
|
|
12
|
+
# @return [SearchEngine::Result]
|
|
13
|
+
def call(collection:, params:, url_opts: {})
|
|
14
|
+
params_obj = SearchEngine::CompiledParams.from(params)
|
|
15
|
+
validate_single!(collection, params_obj.to_h)
|
|
16
|
+
|
|
17
|
+
cache_params = derive_cache_opts(url_opts)
|
|
18
|
+
start = current_monotonic_ms
|
|
19
|
+
payload = sanitize_body_params(params_obj.to_h)
|
|
20
|
+
path = [Client::RequestBuilder::COLLECTIONS_PREFIX, collection.to_s, Client::RequestBuilder::DOCUMENTS_SEARCH_SUFFIX].join
|
|
21
|
+
|
|
22
|
+
result = instrumented_search(collection, params_obj, cache_params, path, payload, start)
|
|
23
|
+
duration = current_monotonic_ms - start
|
|
24
|
+
instrument(:post, path, duration, cache_params)
|
|
25
|
+
log_success(:post, path, start, cache_params)
|
|
26
|
+
|
|
27
|
+
klass = begin
|
|
28
|
+
SearchEngine.collection_for(collection)
|
|
29
|
+
rescue ArgumentError
|
|
30
|
+
nil
|
|
31
|
+
end
|
|
32
|
+
SearchEngine::Result.new(result, klass: klass)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# Execute a multi search request.
|
|
36
|
+
# @param searches [Array<Hash>]
|
|
37
|
+
# @param url_opts [Hash]
|
|
38
|
+
# @return [Hash]
|
|
39
|
+
def multi(searches:, url_opts: {})
|
|
40
|
+
validate_multi!(searches)
|
|
41
|
+
|
|
42
|
+
cache_params = derive_cache_opts(url_opts)
|
|
43
|
+
start = current_monotonic_ms
|
|
44
|
+
path = '/multi_search'
|
|
45
|
+
bodies = Array(searches).map { |s| sanitize_body_params(s) }
|
|
46
|
+
|
|
47
|
+
result = with_exception_mapping(:post, path, cache_params, start) do
|
|
48
|
+
typesense.multi_search.perform({ searches: bodies }, common_params: cache_params)
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
instrument(:post, path, current_monotonic_ms - start, cache_params)
|
|
52
|
+
symbolize_keys_deep(result)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
private
|
|
56
|
+
|
|
57
|
+
def instrumented_search(collection, params_obj, cache_params, path, payload, start)
|
|
58
|
+
if defined?(ActiveSupport::Notifications)
|
|
59
|
+
se_payload = build_search_event_payload(
|
|
60
|
+
collection: collection,
|
|
61
|
+
params: params_obj.to_h,
|
|
62
|
+
cache_params: cache_params
|
|
63
|
+
)
|
|
64
|
+
result = nil
|
|
65
|
+
SearchEngine::Instrumentation.instrument('search_engine.search', se_payload) do |ctx|
|
|
66
|
+
ctx[:params_preview] = SearchEngine::Instrumentation.redact(params_obj.to_h)
|
|
67
|
+
result = with_exception_mapping(:post, path, cache_params, start) do
|
|
68
|
+
docs = typesense.collections[collection].documents
|
|
69
|
+
documents_search(docs, payload, cache_params)
|
|
70
|
+
end
|
|
71
|
+
ctx[:status] = :ok
|
|
72
|
+
rescue Errors::Api => error
|
|
73
|
+
ctx[:status] = error.status
|
|
74
|
+
ctx[:error_class] = error.class.name
|
|
75
|
+
raise
|
|
76
|
+
rescue Errors::Error => error
|
|
77
|
+
ctx[:status] = :error
|
|
78
|
+
ctx[:error_class] = error.class.name
|
|
79
|
+
raise
|
|
80
|
+
end
|
|
81
|
+
result
|
|
82
|
+
else
|
|
83
|
+
with_exception_mapping(:post, path, cache_params, start) do
|
|
84
|
+
docs = typesense.collections[collection].documents
|
|
85
|
+
documents_search(docs, payload, cache_params)
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def sanitize_body_params(params)
|
|
91
|
+
Client::RequestBuilder.send(:sanitize_body_params, params)
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def build_search_event_payload(collection:, params:, cache_params: {})
|
|
95
|
+
sel = params[:_selection].is_a?(Hash) ? params[:_selection] : {}
|
|
96
|
+
base = {
|
|
97
|
+
collection: collection,
|
|
98
|
+
params: Observability.redact(params),
|
|
99
|
+
url_opts: Observability.filtered_url_opts(cache_params),
|
|
100
|
+
status: nil,
|
|
101
|
+
error_class: nil,
|
|
102
|
+
retries: nil,
|
|
103
|
+
selection_include_count: sel[:include_count],
|
|
104
|
+
selection_exclude_count: sel[:exclude_count],
|
|
105
|
+
selection_nested_assoc_count: sel[:nested_assoc_count]
|
|
106
|
+
}
|
|
107
|
+
base.merge(build_preset_segment(params)).merge(build_curation_segment(params))
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def build_preset_segment(params)
|
|
111
|
+
preset_mode = params[:_preset_mode]
|
|
112
|
+
pruned = Array(params[:_preset_pruned_keys]).map { |k| k.respond_to?(:to_sym) ? k.to_sym : k }.grep(Symbol)
|
|
113
|
+
locked_count = begin
|
|
114
|
+
config.presets.locked_domains.size
|
|
115
|
+
rescue StandardError
|
|
116
|
+
nil
|
|
117
|
+
end
|
|
118
|
+
{
|
|
119
|
+
preset_name: params[:preset],
|
|
120
|
+
preset_mode: preset_mode,
|
|
121
|
+
preset_pruned_keys_count: pruned.empty? ? nil : pruned.size,
|
|
122
|
+
preset_locked_domains_count: locked_count,
|
|
123
|
+
preset_pruned_keys: pruned.empty? ? nil : pruned
|
|
124
|
+
}
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def build_curation_segment(params)
|
|
128
|
+
curation = params[:_curation].is_a?(Hash) ? params[:_curation] : {}
|
|
129
|
+
|
|
130
|
+
pinned = params[:pinned_hits]
|
|
131
|
+
hidden = params[:hidden_hits]
|
|
132
|
+
tags = params[:override_tags]
|
|
133
|
+
|
|
134
|
+
pinned_count =
|
|
135
|
+
case pinned
|
|
136
|
+
when String then pinned.split(',').count
|
|
137
|
+
when Array then pinned.size
|
|
138
|
+
else 0
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
hidden_count =
|
|
142
|
+
case hidden
|
|
143
|
+
when String then hidden.split(',').count
|
|
144
|
+
when Array then hidden.size
|
|
145
|
+
else 0
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
has_tags =
|
|
149
|
+
case tags
|
|
150
|
+
when String then !tags.to_s.strip.empty?
|
|
151
|
+
when Array then !tags.empty?
|
|
152
|
+
else false
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
out = {
|
|
156
|
+
curation_pinned_count: pinned_count,
|
|
157
|
+
curation_hidden_count: hidden_count,
|
|
158
|
+
curation_has_override_tags: has_tags
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
if params.key?(:filter_curated_hits) || curation.key?(:filter_curated_hits)
|
|
162
|
+
out[:curation_filter_flag] =
|
|
163
|
+
params.key?(:filter_curated_hits) ? params[:filter_curated_hits] : curation[:filter_curated_hits]
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
out[:curation_conflict_type] = curation[:_conflict_type] || params[:_curation_conflict_type]
|
|
167
|
+
out[:curation_conflict_count] = curation[:_conflict_count] || params[:_curation_conflict_count]
|
|
168
|
+
|
|
169
|
+
out.compact
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
def documents_search(docs, payload, common_params)
|
|
173
|
+
if documents_search_supports_common_params?(docs)
|
|
174
|
+
docs.search(payload, common_params: common_params)
|
|
175
|
+
else
|
|
176
|
+
docs.search(payload)
|
|
177
|
+
end
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
def documents_search_supports_common_params?(docs)
|
|
181
|
+
m = docs.method(:search)
|
|
182
|
+
params = m.parameters
|
|
183
|
+
params.any? { |(kind, _)| %i[key keyreq keyrest].include?(kind) }
|
|
184
|
+
rescue StandardError
|
|
185
|
+
false
|
|
186
|
+
end
|
|
187
|
+
end
|
|
188
|
+
end
|
|
189
|
+
end
|
|
190
|
+
end
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'search_engine/client/services/base'
|
|
4
|
+
require 'search_engine/client/services/search'
|
|
5
|
+
require 'search_engine/client/services/collections'
|
|
6
|
+
require 'search_engine/client/services/operations'
|
|
7
|
+
require 'search_engine/client/services/documents'
|
|
8
|
+
|
|
9
|
+
module SearchEngine
|
|
10
|
+
class Client
|
|
11
|
+
# Registry for all service objects used internally by the client.
|
|
12
|
+
module Services
|
|
13
|
+
SERVICES = {
|
|
14
|
+
search: Search,
|
|
15
|
+
collections: Collections,
|
|
16
|
+
operations: Operations,
|
|
17
|
+
documents: Documents
|
|
18
|
+
}.freeze
|
|
19
|
+
|
|
20
|
+
module_function
|
|
21
|
+
|
|
22
|
+
# @param client [SearchEngine::Client]
|
|
23
|
+
# @return [Hash<Symbol, SearchEngine::Client::Services::Base>]
|
|
24
|
+
def build(client)
|
|
25
|
+
SERVICES.transform_values { |klass| klass.new(client) }
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|