search-engine-for-typesense 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/README.md +148 -0
- data/app/search_engine/search_engine/app_info.rb +11 -0
- data/app/search_engine/search_engine/index_partition_job.rb +170 -0
- data/lib/generators/search_engine/install/install_generator.rb +20 -0
- data/lib/generators/search_engine/install/templates/initializer.rb.tt +230 -0
- data/lib/generators/search_engine/model/model_generator.rb +86 -0
- data/lib/generators/search_engine/model/templates/model.rb.tt +12 -0
- data/lib/search-engine-for-typesense.rb +12 -0
- data/lib/search_engine/active_record_syncable.rb +247 -0
- data/lib/search_engine/admin/stopwords.rb +125 -0
- data/lib/search_engine/admin/synonyms.rb +125 -0
- data/lib/search_engine/admin.rb +12 -0
- data/lib/search_engine/ast/and.rb +52 -0
- data/lib/search_engine/ast/binary_op.rb +75 -0
- data/lib/search_engine/ast/eq.rb +19 -0
- data/lib/search_engine/ast/group.rb +18 -0
- data/lib/search_engine/ast/gt.rb +12 -0
- data/lib/search_engine/ast/gte.rb +12 -0
- data/lib/search_engine/ast/in.rb +28 -0
- data/lib/search_engine/ast/lt.rb +12 -0
- data/lib/search_engine/ast/lte.rb +12 -0
- data/lib/search_engine/ast/matches.rb +55 -0
- data/lib/search_engine/ast/node.rb +176 -0
- data/lib/search_engine/ast/not_eq.rb +13 -0
- data/lib/search_engine/ast/not_in.rb +24 -0
- data/lib/search_engine/ast/or.rb +52 -0
- data/lib/search_engine/ast/prefix.rb +51 -0
- data/lib/search_engine/ast/raw.rb +41 -0
- data/lib/search_engine/ast/unary_op.rb +43 -0
- data/lib/search_engine/ast.rb +101 -0
- data/lib/search_engine/base/creation.rb +727 -0
- data/lib/search_engine/base/deletion.rb +80 -0
- data/lib/search_engine/base/display_coercions.rb +36 -0
- data/lib/search_engine/base/hydration.rb +312 -0
- data/lib/search_engine/base/index_maintenance/cleanup.rb +202 -0
- data/lib/search_engine/base/index_maintenance/lifecycle.rb +251 -0
- data/lib/search_engine/base/index_maintenance/schema.rb +117 -0
- data/lib/search_engine/base/index_maintenance.rb +459 -0
- data/lib/search_engine/base/indexing_dsl.rb +255 -0
- data/lib/search_engine/base/joins.rb +479 -0
- data/lib/search_engine/base/model_dsl.rb +472 -0
- data/lib/search_engine/base/presets.rb +43 -0
- data/lib/search_engine/base/pretty_printer.rb +315 -0
- data/lib/search_engine/base/relation_delegation.rb +42 -0
- data/lib/search_engine/base/scopes.rb +113 -0
- data/lib/search_engine/base/updating.rb +92 -0
- data/lib/search_engine/base.rb +38 -0
- data/lib/search_engine/bulk.rb +284 -0
- data/lib/search_engine/cache.rb +33 -0
- data/lib/search_engine/cascade.rb +531 -0
- data/lib/search_engine/cli/doctor.rb +631 -0
- data/lib/search_engine/cli/support.rb +217 -0
- data/lib/search_engine/cli.rb +222 -0
- data/lib/search_engine/client/http_adapter.rb +63 -0
- data/lib/search_engine/client/request_builder.rb +92 -0
- data/lib/search_engine/client/services/base.rb +74 -0
- data/lib/search_engine/client/services/collections.rb +161 -0
- data/lib/search_engine/client/services/documents.rb +214 -0
- data/lib/search_engine/client/services/operations.rb +152 -0
- data/lib/search_engine/client/services/search.rb +190 -0
- data/lib/search_engine/client/services.rb +29 -0
- data/lib/search_engine/client.rb +765 -0
- data/lib/search_engine/client_options.rb +20 -0
- data/lib/search_engine/collection_resolver.rb +191 -0
- data/lib/search_engine/collections_graph.rb +330 -0
- data/lib/search_engine/compiled_params.rb +143 -0
- data/lib/search_engine/compiler.rb +383 -0
- data/lib/search_engine/config/observability.rb +27 -0
- data/lib/search_engine/config/presets.rb +92 -0
- data/lib/search_engine/config/selection.rb +16 -0
- data/lib/search_engine/config/typesense.rb +48 -0
- data/lib/search_engine/config/validators.rb +97 -0
- data/lib/search_engine/config.rb +917 -0
- data/lib/search_engine/console_helpers.rb +130 -0
- data/lib/search_engine/deletion.rb +103 -0
- data/lib/search_engine/dispatcher.rb +125 -0
- data/lib/search_engine/dsl/parser.rb +582 -0
- data/lib/search_engine/engine.rb +167 -0
- data/lib/search_engine/errors.rb +290 -0
- data/lib/search_engine/filters/sanitizer.rb +189 -0
- data/lib/search_engine/hydration/materializers.rb +808 -0
- data/lib/search_engine/hydration/selection_context.rb +96 -0
- data/lib/search_engine/indexer/batch_planner.rb +76 -0
- data/lib/search_engine/indexer/bulk_import.rb +626 -0
- data/lib/search_engine/indexer/import_dispatcher.rb +198 -0
- data/lib/search_engine/indexer/retry_policy.rb +103 -0
- data/lib/search_engine/indexer.rb +747 -0
- data/lib/search_engine/instrumentation.rb +308 -0
- data/lib/search_engine/joins/guard.rb +202 -0
- data/lib/search_engine/joins/resolver.rb +95 -0
- data/lib/search_engine/logging/color.rb +78 -0
- data/lib/search_engine/logging/format_helpers.rb +92 -0
- data/lib/search_engine/logging/partition_progress.rb +53 -0
- data/lib/search_engine/logging_subscriber.rb +388 -0
- data/lib/search_engine/mapper.rb +785 -0
- data/lib/search_engine/multi.rb +286 -0
- data/lib/search_engine/multi_result.rb +186 -0
- data/lib/search_engine/notifications/compact_logger.rb +675 -0
- data/lib/search_engine/observability.rb +162 -0
- data/lib/search_engine/operations.rb +58 -0
- data/lib/search_engine/otel.rb +227 -0
- data/lib/search_engine/partitioner.rb +128 -0
- data/lib/search_engine/ranking_plan.rb +118 -0
- data/lib/search_engine/registry.rb +158 -0
- data/lib/search_engine/relation/compiler.rb +711 -0
- data/lib/search_engine/relation/deletion.rb +37 -0
- data/lib/search_engine/relation/dsl/filters.rb +624 -0
- data/lib/search_engine/relation/dsl/selection.rb +240 -0
- data/lib/search_engine/relation/dsl.rb +903 -0
- data/lib/search_engine/relation/dx/dry_run.rb +59 -0
- data/lib/search_engine/relation/dx/friendly_where.rb +24 -0
- data/lib/search_engine/relation/dx.rb +231 -0
- data/lib/search_engine/relation/materializers.rb +118 -0
- data/lib/search_engine/relation/options.rb +138 -0
- data/lib/search_engine/relation/state.rb +274 -0
- data/lib/search_engine/relation/updating.rb +44 -0
- data/lib/search_engine/relation.rb +623 -0
- data/lib/search_engine/result.rb +664 -0
- data/lib/search_engine/schema.rb +1083 -0
- data/lib/search_engine/sources/active_record_source.rb +185 -0
- data/lib/search_engine/sources/base.rb +62 -0
- data/lib/search_engine/sources/lambda_source.rb +55 -0
- data/lib/search_engine/sources/sql_source.rb +196 -0
- data/lib/search_engine/sources.rb +71 -0
- data/lib/search_engine/stale_rules.rb +160 -0
- data/lib/search_engine/test/minitest_assertions.rb +57 -0
- data/lib/search_engine/test/offline_client.rb +134 -0
- data/lib/search_engine/test/rspec_matchers.rb +77 -0
- data/lib/search_engine/test/stub_client.rb +201 -0
- data/lib/search_engine/test.rb +66 -0
- data/lib/search_engine/test_autoload.rb +8 -0
- data/lib/search_engine/update.rb +35 -0
- data/lib/search_engine/version.rb +7 -0
- data/lib/search_engine.rb +332 -0
- data/lib/tasks/search_engine.rake +501 -0
- data/lib/tasks/search_engine_doctor.rake +16 -0
- metadata +225 -0
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SearchEngine
|
|
4
|
+
class Relation
|
|
5
|
+
module DSL
|
|
6
|
+
# Selection-related chainers and normalizers.
|
|
7
|
+
# These methods are mixed into Relation's DSL and must preserve copy-on-write semantics.
|
|
8
|
+
module Selection
|
|
9
|
+
# Select a subset of fields for Typesense `include_fields`.
|
|
10
|
+
# @param fields [Array<Symbol,String,Hash,Array>]
|
|
11
|
+
# @return [SearchEngine::Relation]
|
|
12
|
+
# @see `https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/field-selection`
|
|
13
|
+
def select(*fields)
|
|
14
|
+
normalized = normalize_select_input(fields)
|
|
15
|
+
spawn do |s|
|
|
16
|
+
existing_base = Array(s[:select])
|
|
17
|
+
merged_base = (existing_base + normalized[:base]).each_with_object([]) do |f, acc|
|
|
18
|
+
acc << f unless acc.include?(f)
|
|
19
|
+
end
|
|
20
|
+
s[:select] = merged_base
|
|
21
|
+
|
|
22
|
+
existing_nested = s[:select_nested] || {}
|
|
23
|
+
existing_order = Array(s[:select_nested_order])
|
|
24
|
+
|
|
25
|
+
normalized[:nested_order].each do |assoc|
|
|
26
|
+
new_fields = Array(normalized[:nested][assoc])
|
|
27
|
+
next if new_fields.empty?
|
|
28
|
+
|
|
29
|
+
old_fields = Array(existing_nested[assoc])
|
|
30
|
+
merged_fields = (old_fields + new_fields).each_with_object([]) do |name, acc|
|
|
31
|
+
acc << name unless acc.include?(name)
|
|
32
|
+
end
|
|
33
|
+
existing_nested = existing_nested.merge(assoc => merged_fields)
|
|
34
|
+
existing_order << assoc unless existing_order.include?(assoc)
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
s[:select_nested] = existing_nested
|
|
38
|
+
s[:select_nested_order] = existing_order
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# Convenience alias for `select` supporting nested include_fields input.
|
|
43
|
+
# @return [SearchEngine::Relation]
|
|
44
|
+
alias_method :include_fields, :select
|
|
45
|
+
|
|
46
|
+
# Exclude a subset of fields from the final selection.
|
|
47
|
+
# @param fields [Array<Symbol,String,Hash,Array>]
|
|
48
|
+
# @return [SearchEngine::Relation]
|
|
49
|
+
# @see `https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/field-selection`
|
|
50
|
+
#
|
|
51
|
+
# When excluding, you may also pass a joined association name (after calling
|
|
52
|
+
# `joins(:assoc)`) to exclude the entire joined payload from the response.
|
|
53
|
+
# For example: `exclude(:authors)` compiles to Typesense `exclude_fields: "$authors(*,doc_updated_at)"`.
|
|
54
|
+
def exclude(*fields)
|
|
55
|
+
normalized = normalize_select_input(fields, context: 'excluding fields')
|
|
56
|
+
spawn do |s|
|
|
57
|
+
existing_base = Array(s[:exclude])
|
|
58
|
+
merged_base = (existing_base + normalized[:base]).each_with_object([]) do |f, acc|
|
|
59
|
+
acc << f unless acc.include?(f)
|
|
60
|
+
end
|
|
61
|
+
s[:exclude] = merged_base
|
|
62
|
+
|
|
63
|
+
existing_nested = s[:exclude_nested] || {}
|
|
64
|
+
existing_order = Array(s[:exclude_nested_order])
|
|
65
|
+
|
|
66
|
+
normalized[:nested_order].each do |assoc|
|
|
67
|
+
new_fields = Array(normalized[:nested][assoc])
|
|
68
|
+
next if new_fields.empty?
|
|
69
|
+
|
|
70
|
+
old_fields = Array(existing_nested[assoc])
|
|
71
|
+
merged_fields = (old_fields + new_fields).each_with_object([]) do |name, acc|
|
|
72
|
+
acc << name unless acc.include?(name)
|
|
73
|
+
end
|
|
74
|
+
existing_nested = existing_nested.merge(assoc => merged_fields)
|
|
75
|
+
existing_order << assoc unless existing_order.include?(assoc)
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
s[:exclude_nested] = existing_nested
|
|
79
|
+
s[:exclude_nested_order] = existing_order
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# Convenience alias for `exclude` supporting nested exclude_fields input.
|
|
84
|
+
# Mirrors Typesense param naming for API symmetry with `include_fields`.
|
|
85
|
+
# @return [SearchEngine::Relation]
|
|
86
|
+
alias_method :exclude_fields, :exclude
|
|
87
|
+
|
|
88
|
+
# Replace the selected fields list (Typesense `include_fields`).
|
|
89
|
+
# @param fields [Array<#to_sym,#to_s>]
|
|
90
|
+
# @return [SearchEngine::Relation]
|
|
91
|
+
# @see `https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/field-selection`
|
|
92
|
+
def reselect(*fields)
|
|
93
|
+
normalized = normalize_select_input(fields)
|
|
94
|
+
|
|
95
|
+
base_empty = Array(normalized[:base]).empty?
|
|
96
|
+
nested_empty = normalized[:nested_order].all? { |a| Array(normalized[:nested][a]).empty? }
|
|
97
|
+
raise ArgumentError, 'reselect: provide at least one non-blank field' if base_empty && nested_empty
|
|
98
|
+
|
|
99
|
+
spawn do |s|
|
|
100
|
+
s[:select] = normalized[:base]
|
|
101
|
+
s[:select_nested] = normalized[:nested]
|
|
102
|
+
s[:select_nested_order] = normalized[:nested_order]
|
|
103
|
+
s[:exclude] = []
|
|
104
|
+
s[:exclude_nested] = {}
|
|
105
|
+
s[:exclude_nested_order] = []
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
private
|
|
110
|
+
|
|
111
|
+
# Base-only normalization used internally and for legacy callers.
|
|
112
|
+
def normalize_select(fields)
|
|
113
|
+
list = Array(fields).flatten.compact
|
|
114
|
+
return [] if list.empty?
|
|
115
|
+
|
|
116
|
+
known_attrs = safe_attributes_map
|
|
117
|
+
known = known_attrs.keys.map(&:to_s)
|
|
118
|
+
|
|
119
|
+
ordered = []
|
|
120
|
+
list.each do |f|
|
|
121
|
+
name = f.to_s.strip
|
|
122
|
+
raise ArgumentError, 'select: field names must be non-empty' if name.empty?
|
|
123
|
+
|
|
124
|
+
if !known.empty? && !known.include?(name)
|
|
125
|
+
suggestions = suggest_fields(name.to_sym, known_attrs.keys.map(&:to_sym))
|
|
126
|
+
suggest = if suggestions.empty?
|
|
127
|
+
''
|
|
128
|
+
elsif suggestions.length == 1
|
|
129
|
+
" (did you mean :#{suggestions.first}?)"
|
|
130
|
+
else
|
|
131
|
+
last = suggestions.last
|
|
132
|
+
others = suggestions[0..-2].map { |s| ":#{s}" }.join(', ')
|
|
133
|
+
" (did you mean #{others}, or :#{last}?)"
|
|
134
|
+
end
|
|
135
|
+
raise SearchEngine::Errors::UnknownField,
|
|
136
|
+
"UnknownField: unknown field #{name.inspect} for #{klass_name_for_inspect}#{suggest}"
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
ordered << name unless ordered.include?(name)
|
|
140
|
+
end
|
|
141
|
+
ordered
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
# Extended normalization supporting nested association selections.
|
|
145
|
+
# Returns a Hash with keys: :base, :nested, :nested_order.
|
|
146
|
+
def normalize_select_input(fields, context: 'selecting fields')
|
|
147
|
+
list = Array(fields).flatten.compact
|
|
148
|
+
return { base: [], nested: {}, nested_order: [] } if list.empty?
|
|
149
|
+
|
|
150
|
+
base = []
|
|
151
|
+
nested = {}
|
|
152
|
+
nested_order = []
|
|
153
|
+
|
|
154
|
+
add_base = build_add_base_proc(context, base, nested, nested_order)
|
|
155
|
+
add_nested = build_add_nested_proc(context, nested, nested_order)
|
|
156
|
+
|
|
157
|
+
process_selection_list!(list, add_base, add_nested)
|
|
158
|
+
|
|
159
|
+
{ base: base.uniq, nested: nested, nested_order: nested_order }
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
# Base-field adder that, in excluding context, treats a bare joined assoc
|
|
163
|
+
# token as a request to exclude the entire association payload (sentinel :__all).
|
|
164
|
+
def build_add_base_proc(context, base, nested, nested_order)
|
|
165
|
+
lambda do |val|
|
|
166
|
+
# Only special-case in excluding context; otherwise fallback to base normalization.
|
|
167
|
+
if context.to_s.include?('exclud')
|
|
168
|
+
name = val.to_s.strip
|
|
169
|
+
unless name.empty?
|
|
170
|
+
key = name.to_sym
|
|
171
|
+
begin
|
|
172
|
+
# Validate join existence and that it was applied on this relation.
|
|
173
|
+
@klass.join_for(key)
|
|
174
|
+
SearchEngine::Joins::Guard.ensure_join_applied!(joins_list, key, context: context)
|
|
175
|
+
|
|
176
|
+
# Mark full association exclusion via sentinel; exclude wins later in compiler.
|
|
177
|
+
nested[key] = [:__all]
|
|
178
|
+
nested_order << key unless nested_order.include?(key)
|
|
179
|
+
return
|
|
180
|
+
rescue StandardError
|
|
181
|
+
# Not a known/applied join; fall through to base normalization
|
|
182
|
+
end
|
|
183
|
+
end
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
# Default behavior: treat as base field(s) and validate accordingly.
|
|
187
|
+
base_fields = normalize_select([val])
|
|
188
|
+
base.concat(base_fields)
|
|
189
|
+
end
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
def build_add_nested_proc(context, nested, nested_order)
|
|
193
|
+
lambda do |assoc, values|
|
|
194
|
+
key = assoc.to_sym
|
|
195
|
+
@klass.join_for(key)
|
|
196
|
+
SearchEngine::Joins::Guard.ensure_join_applied!(joins_list, key, context: context)
|
|
197
|
+
|
|
198
|
+
items = case values
|
|
199
|
+
when Array then values
|
|
200
|
+
when nil then []
|
|
201
|
+
else [values]
|
|
202
|
+
end
|
|
203
|
+
names = items.flatten.compact.map(&:to_s).map(&:strip).reject(&:empty?)
|
|
204
|
+
return if names.empty?
|
|
205
|
+
|
|
206
|
+
cfg = @klass.join_for(key)
|
|
207
|
+
names.each { |fname| SearchEngine::Joins::Guard.validate_joined_field!(cfg, fname, source_klass: @klass) }
|
|
208
|
+
|
|
209
|
+
existing = Array(nested[key])
|
|
210
|
+
merged = (existing + names).each_with_object([]) { |n, acc| acc << n unless acc.include?(n) }
|
|
211
|
+
nested[key] = merged
|
|
212
|
+
nested_order << key unless nested_order.include?(key)
|
|
213
|
+
end
|
|
214
|
+
end
|
|
215
|
+
|
|
216
|
+
def process_selection_list!(list, add_base, add_nested)
|
|
217
|
+
i = 0
|
|
218
|
+
while i < list.length
|
|
219
|
+
entry = list[i]
|
|
220
|
+
case entry
|
|
221
|
+
when Hash
|
|
222
|
+
entry.each { |k, v| add_nested.call(k, v) }
|
|
223
|
+
i += 1
|
|
224
|
+
when Symbol, String
|
|
225
|
+
add_base.call(entry)
|
|
226
|
+
i += 1
|
|
227
|
+
when Array
|
|
228
|
+
inner = Array(entry).flatten.compact
|
|
229
|
+
inner.each { |el| list << el }
|
|
230
|
+
i += 1
|
|
231
|
+
else
|
|
232
|
+
raise SearchEngine::Errors::ConflictingSelection,
|
|
233
|
+
"ConflictingSelection: unsupported input #{entry.class} in selection"
|
|
234
|
+
end
|
|
235
|
+
end
|
|
236
|
+
end
|
|
237
|
+
end
|
|
238
|
+
end
|
|
239
|
+
end
|
|
240
|
+
end
|