search-engine-for-typesense 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +21 -0
  3. data/README.md +148 -0
  4. data/app/search_engine/search_engine/app_info.rb +11 -0
  5. data/app/search_engine/search_engine/index_partition_job.rb +170 -0
  6. data/lib/generators/search_engine/install/install_generator.rb +20 -0
  7. data/lib/generators/search_engine/install/templates/initializer.rb.tt +230 -0
  8. data/lib/generators/search_engine/model/model_generator.rb +86 -0
  9. data/lib/generators/search_engine/model/templates/model.rb.tt +12 -0
  10. data/lib/search-engine-for-typesense.rb +12 -0
  11. data/lib/search_engine/active_record_syncable.rb +247 -0
  12. data/lib/search_engine/admin/stopwords.rb +125 -0
  13. data/lib/search_engine/admin/synonyms.rb +125 -0
  14. data/lib/search_engine/admin.rb +12 -0
  15. data/lib/search_engine/ast/and.rb +52 -0
  16. data/lib/search_engine/ast/binary_op.rb +75 -0
  17. data/lib/search_engine/ast/eq.rb +19 -0
  18. data/lib/search_engine/ast/group.rb +18 -0
  19. data/lib/search_engine/ast/gt.rb +12 -0
  20. data/lib/search_engine/ast/gte.rb +12 -0
  21. data/lib/search_engine/ast/in.rb +28 -0
  22. data/lib/search_engine/ast/lt.rb +12 -0
  23. data/lib/search_engine/ast/lte.rb +12 -0
  24. data/lib/search_engine/ast/matches.rb +55 -0
  25. data/lib/search_engine/ast/node.rb +176 -0
  26. data/lib/search_engine/ast/not_eq.rb +13 -0
  27. data/lib/search_engine/ast/not_in.rb +24 -0
  28. data/lib/search_engine/ast/or.rb +52 -0
  29. data/lib/search_engine/ast/prefix.rb +51 -0
  30. data/lib/search_engine/ast/raw.rb +41 -0
  31. data/lib/search_engine/ast/unary_op.rb +43 -0
  32. data/lib/search_engine/ast.rb +101 -0
  33. data/lib/search_engine/base/creation.rb +727 -0
  34. data/lib/search_engine/base/deletion.rb +80 -0
  35. data/lib/search_engine/base/display_coercions.rb +36 -0
  36. data/lib/search_engine/base/hydration.rb +312 -0
  37. data/lib/search_engine/base/index_maintenance/cleanup.rb +202 -0
  38. data/lib/search_engine/base/index_maintenance/lifecycle.rb +251 -0
  39. data/lib/search_engine/base/index_maintenance/schema.rb +117 -0
  40. data/lib/search_engine/base/index_maintenance.rb +459 -0
  41. data/lib/search_engine/base/indexing_dsl.rb +255 -0
  42. data/lib/search_engine/base/joins.rb +479 -0
  43. data/lib/search_engine/base/model_dsl.rb +472 -0
  44. data/lib/search_engine/base/presets.rb +43 -0
  45. data/lib/search_engine/base/pretty_printer.rb +315 -0
  46. data/lib/search_engine/base/relation_delegation.rb +42 -0
  47. data/lib/search_engine/base/scopes.rb +113 -0
  48. data/lib/search_engine/base/updating.rb +92 -0
  49. data/lib/search_engine/base.rb +38 -0
  50. data/lib/search_engine/bulk.rb +284 -0
  51. data/lib/search_engine/cache.rb +33 -0
  52. data/lib/search_engine/cascade.rb +531 -0
  53. data/lib/search_engine/cli/doctor.rb +631 -0
  54. data/lib/search_engine/cli/support.rb +217 -0
  55. data/lib/search_engine/cli.rb +222 -0
  56. data/lib/search_engine/client/http_adapter.rb +63 -0
  57. data/lib/search_engine/client/request_builder.rb +92 -0
  58. data/lib/search_engine/client/services/base.rb +74 -0
  59. data/lib/search_engine/client/services/collections.rb +161 -0
  60. data/lib/search_engine/client/services/documents.rb +214 -0
  61. data/lib/search_engine/client/services/operations.rb +152 -0
  62. data/lib/search_engine/client/services/search.rb +190 -0
  63. data/lib/search_engine/client/services.rb +29 -0
  64. data/lib/search_engine/client.rb +765 -0
  65. data/lib/search_engine/client_options.rb +20 -0
  66. data/lib/search_engine/collection_resolver.rb +191 -0
  67. data/lib/search_engine/collections_graph.rb +330 -0
  68. data/lib/search_engine/compiled_params.rb +143 -0
  69. data/lib/search_engine/compiler.rb +383 -0
  70. data/lib/search_engine/config/observability.rb +27 -0
  71. data/lib/search_engine/config/presets.rb +92 -0
  72. data/lib/search_engine/config/selection.rb +16 -0
  73. data/lib/search_engine/config/typesense.rb +48 -0
  74. data/lib/search_engine/config/validators.rb +97 -0
  75. data/lib/search_engine/config.rb +917 -0
  76. data/lib/search_engine/console_helpers.rb +130 -0
  77. data/lib/search_engine/deletion.rb +103 -0
  78. data/lib/search_engine/dispatcher.rb +125 -0
  79. data/lib/search_engine/dsl/parser.rb +582 -0
  80. data/lib/search_engine/engine.rb +167 -0
  81. data/lib/search_engine/errors.rb +290 -0
  82. data/lib/search_engine/filters/sanitizer.rb +189 -0
  83. data/lib/search_engine/hydration/materializers.rb +808 -0
  84. data/lib/search_engine/hydration/selection_context.rb +96 -0
  85. data/lib/search_engine/indexer/batch_planner.rb +76 -0
  86. data/lib/search_engine/indexer/bulk_import.rb +626 -0
  87. data/lib/search_engine/indexer/import_dispatcher.rb +198 -0
  88. data/lib/search_engine/indexer/retry_policy.rb +103 -0
  89. data/lib/search_engine/indexer.rb +747 -0
  90. data/lib/search_engine/instrumentation.rb +308 -0
  91. data/lib/search_engine/joins/guard.rb +202 -0
  92. data/lib/search_engine/joins/resolver.rb +95 -0
  93. data/lib/search_engine/logging/color.rb +78 -0
  94. data/lib/search_engine/logging/format_helpers.rb +92 -0
  95. data/lib/search_engine/logging/partition_progress.rb +53 -0
  96. data/lib/search_engine/logging_subscriber.rb +388 -0
  97. data/lib/search_engine/mapper.rb +785 -0
  98. data/lib/search_engine/multi.rb +286 -0
  99. data/lib/search_engine/multi_result.rb +186 -0
  100. data/lib/search_engine/notifications/compact_logger.rb +675 -0
  101. data/lib/search_engine/observability.rb +162 -0
  102. data/lib/search_engine/operations.rb +58 -0
  103. data/lib/search_engine/otel.rb +227 -0
  104. data/lib/search_engine/partitioner.rb +128 -0
  105. data/lib/search_engine/ranking_plan.rb +118 -0
  106. data/lib/search_engine/registry.rb +158 -0
  107. data/lib/search_engine/relation/compiler.rb +711 -0
  108. data/lib/search_engine/relation/deletion.rb +37 -0
  109. data/lib/search_engine/relation/dsl/filters.rb +624 -0
  110. data/lib/search_engine/relation/dsl/selection.rb +240 -0
  111. data/lib/search_engine/relation/dsl.rb +903 -0
  112. data/lib/search_engine/relation/dx/dry_run.rb +59 -0
  113. data/lib/search_engine/relation/dx/friendly_where.rb +24 -0
  114. data/lib/search_engine/relation/dx.rb +231 -0
  115. data/lib/search_engine/relation/materializers.rb +118 -0
  116. data/lib/search_engine/relation/options.rb +138 -0
  117. data/lib/search_engine/relation/state.rb +274 -0
  118. data/lib/search_engine/relation/updating.rb +44 -0
  119. data/lib/search_engine/relation.rb +623 -0
  120. data/lib/search_engine/result.rb +664 -0
  121. data/lib/search_engine/schema.rb +1083 -0
  122. data/lib/search_engine/sources/active_record_source.rb +185 -0
  123. data/lib/search_engine/sources/base.rb +62 -0
  124. data/lib/search_engine/sources/lambda_source.rb +55 -0
  125. data/lib/search_engine/sources/sql_source.rb +196 -0
  126. data/lib/search_engine/sources.rb +71 -0
  127. data/lib/search_engine/stale_rules.rb +160 -0
  128. data/lib/search_engine/test/minitest_assertions.rb +57 -0
  129. data/lib/search_engine/test/offline_client.rb +134 -0
  130. data/lib/search_engine/test/rspec_matchers.rb +77 -0
  131. data/lib/search_engine/test/stub_client.rb +201 -0
  132. data/lib/search_engine/test.rb +66 -0
  133. data/lib/search_engine/test_autoload.rb +8 -0
  134. data/lib/search_engine/update.rb +35 -0
  135. data/lib/search_engine/version.rb +7 -0
  136. data/lib/search_engine.rb +332 -0
  137. data/lib/tasks/search_engine.rake +501 -0
  138. data/lib/tasks/search_engine_doctor.rake +16 -0
  139. metadata +225 -0
@@ -0,0 +1,240 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SearchEngine
4
+ class Relation
5
+ module DSL
6
+ # Selection-related chainers and normalizers.
7
+ # These methods are mixed into Relation's DSL and must preserve copy-on-write semantics.
8
+ module Selection
9
+ # Select a subset of fields for Typesense `include_fields`.
10
+ # @param fields [Array<Symbol,String,Hash,Array>]
11
+ # @return [SearchEngine::Relation]
12
+ # @see `https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/field-selection`
13
+ def select(*fields)
14
+ normalized = normalize_select_input(fields)
15
+ spawn do |s|
16
+ existing_base = Array(s[:select])
17
+ merged_base = (existing_base + normalized[:base]).each_with_object([]) do |f, acc|
18
+ acc << f unless acc.include?(f)
19
+ end
20
+ s[:select] = merged_base
21
+
22
+ existing_nested = s[:select_nested] || {}
23
+ existing_order = Array(s[:select_nested_order])
24
+
25
+ normalized[:nested_order].each do |assoc|
26
+ new_fields = Array(normalized[:nested][assoc])
27
+ next if new_fields.empty?
28
+
29
+ old_fields = Array(existing_nested[assoc])
30
+ merged_fields = (old_fields + new_fields).each_with_object([]) do |name, acc|
31
+ acc << name unless acc.include?(name)
32
+ end
33
+ existing_nested = existing_nested.merge(assoc => merged_fields)
34
+ existing_order << assoc unless existing_order.include?(assoc)
35
+ end
36
+
37
+ s[:select_nested] = existing_nested
38
+ s[:select_nested_order] = existing_order
39
+ end
40
+ end
41
+
42
+ # Convenience alias for `select` supporting nested include_fields input.
43
+ # @return [SearchEngine::Relation]
44
+ alias_method :include_fields, :select
45
+
46
+ # Exclude a subset of fields from the final selection.
47
+ # @param fields [Array<Symbol,String,Hash,Array>]
48
+ # @return [SearchEngine::Relation]
49
+ # @see `https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/field-selection`
50
+ #
51
+ # When excluding, you may also pass a joined association name (after calling
52
+ # `joins(:assoc)`) to exclude the entire joined payload from the response.
53
+ # For example: `exclude(:authors)` compiles to Typesense `exclude_fields: "$authors(*,doc_updated_at)"`.
54
+ def exclude(*fields)
55
+ normalized = normalize_select_input(fields, context: 'excluding fields')
56
+ spawn do |s|
57
+ existing_base = Array(s[:exclude])
58
+ merged_base = (existing_base + normalized[:base]).each_with_object([]) do |f, acc|
59
+ acc << f unless acc.include?(f)
60
+ end
61
+ s[:exclude] = merged_base
62
+
63
+ existing_nested = s[:exclude_nested] || {}
64
+ existing_order = Array(s[:exclude_nested_order])
65
+
66
+ normalized[:nested_order].each do |assoc|
67
+ new_fields = Array(normalized[:nested][assoc])
68
+ next if new_fields.empty?
69
+
70
+ old_fields = Array(existing_nested[assoc])
71
+ merged_fields = (old_fields + new_fields).each_with_object([]) do |name, acc|
72
+ acc << name unless acc.include?(name)
73
+ end
74
+ existing_nested = existing_nested.merge(assoc => merged_fields)
75
+ existing_order << assoc unless existing_order.include?(assoc)
76
+ end
77
+
78
+ s[:exclude_nested] = existing_nested
79
+ s[:exclude_nested_order] = existing_order
80
+ end
81
+ end
82
+
83
+ # Convenience alias for `exclude` supporting nested exclude_fields input.
84
+ # Mirrors Typesense param naming for API symmetry with `include_fields`.
85
+ # @return [SearchEngine::Relation]
86
+ alias_method :exclude_fields, :exclude
87
+
88
+ # Replace the selected fields list (Typesense `include_fields`).
89
+ # @param fields [Array<#to_sym,#to_s>]
90
+ # @return [SearchEngine::Relation]
91
+ # @see `https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/field-selection`
92
+ def reselect(*fields)
93
+ normalized = normalize_select_input(fields)
94
+
95
+ base_empty = Array(normalized[:base]).empty?
96
+ nested_empty = normalized[:nested_order].all? { |a| Array(normalized[:nested][a]).empty? }
97
+ raise ArgumentError, 'reselect: provide at least one non-blank field' if base_empty && nested_empty
98
+
99
+ spawn do |s|
100
+ s[:select] = normalized[:base]
101
+ s[:select_nested] = normalized[:nested]
102
+ s[:select_nested_order] = normalized[:nested_order]
103
+ s[:exclude] = []
104
+ s[:exclude_nested] = {}
105
+ s[:exclude_nested_order] = []
106
+ end
107
+ end
108
+
109
+ private
110
+
111
+ # Base-only normalization used internally and for legacy callers.
112
+ def normalize_select(fields)
113
+ list = Array(fields).flatten.compact
114
+ return [] if list.empty?
115
+
116
+ known_attrs = safe_attributes_map
117
+ known = known_attrs.keys.map(&:to_s)
118
+
119
+ ordered = []
120
+ list.each do |f|
121
+ name = f.to_s.strip
122
+ raise ArgumentError, 'select: field names must be non-empty' if name.empty?
123
+
124
+ if !known.empty? && !known.include?(name)
125
+ suggestions = suggest_fields(name.to_sym, known_attrs.keys.map(&:to_sym))
126
+ suggest = if suggestions.empty?
127
+ ''
128
+ elsif suggestions.length == 1
129
+ " (did you mean :#{suggestions.first}?)"
130
+ else
131
+ last = suggestions.last
132
+ others = suggestions[0..-2].map { |s| ":#{s}" }.join(', ')
133
+ " (did you mean #{others}, or :#{last}?)"
134
+ end
135
+ raise SearchEngine::Errors::UnknownField,
136
+ "UnknownField: unknown field #{name.inspect} for #{klass_name_for_inspect}#{suggest}"
137
+ end
138
+
139
+ ordered << name unless ordered.include?(name)
140
+ end
141
+ ordered
142
+ end
143
+
144
+ # Extended normalization supporting nested association selections.
145
+ # Returns a Hash with keys: :base, :nested, :nested_order.
146
+ def normalize_select_input(fields, context: 'selecting fields')
147
+ list = Array(fields).flatten.compact
148
+ return { base: [], nested: {}, nested_order: [] } if list.empty?
149
+
150
+ base = []
151
+ nested = {}
152
+ nested_order = []
153
+
154
+ add_base = build_add_base_proc(context, base, nested, nested_order)
155
+ add_nested = build_add_nested_proc(context, nested, nested_order)
156
+
157
+ process_selection_list!(list, add_base, add_nested)
158
+
159
+ { base: base.uniq, nested: nested, nested_order: nested_order }
160
+ end
161
+
162
+ # Base-field adder that, in excluding context, treats a bare joined assoc
163
+ # token as a request to exclude the entire association payload (sentinel :__all).
164
+ def build_add_base_proc(context, base, nested, nested_order)
165
+ lambda do |val|
166
+ # Only special-case in excluding context; otherwise fallback to base normalization.
167
+ if context.to_s.include?('exclud')
168
+ name = val.to_s.strip
169
+ unless name.empty?
170
+ key = name.to_sym
171
+ begin
172
+ # Validate join existence and that it was applied on this relation.
173
+ @klass.join_for(key)
174
+ SearchEngine::Joins::Guard.ensure_join_applied!(joins_list, key, context: context)
175
+
176
+ # Mark full association exclusion via sentinel; exclude wins later in compiler.
177
+ nested[key] = [:__all]
178
+ nested_order << key unless nested_order.include?(key)
179
+ return
180
+ rescue StandardError
181
+ # Not a known/applied join; fall through to base normalization
182
+ end
183
+ end
184
+ end
185
+
186
+ # Default behavior: treat as base field(s) and validate accordingly.
187
+ base_fields = normalize_select([val])
188
+ base.concat(base_fields)
189
+ end
190
+ end
191
+
192
+ def build_add_nested_proc(context, nested, nested_order)
193
+ lambda do |assoc, values|
194
+ key = assoc.to_sym
195
+ @klass.join_for(key)
196
+ SearchEngine::Joins::Guard.ensure_join_applied!(joins_list, key, context: context)
197
+
198
+ items = case values
199
+ when Array then values
200
+ when nil then []
201
+ else [values]
202
+ end
203
+ names = items.flatten.compact.map(&:to_s).map(&:strip).reject(&:empty?)
204
+ return if names.empty?
205
+
206
+ cfg = @klass.join_for(key)
207
+ names.each { |fname| SearchEngine::Joins::Guard.validate_joined_field!(cfg, fname, source_klass: @klass) }
208
+
209
+ existing = Array(nested[key])
210
+ merged = (existing + names).each_with_object([]) { |n, acc| acc << n unless acc.include?(n) }
211
+ nested[key] = merged
212
+ nested_order << key unless nested_order.include?(key)
213
+ end
214
+ end
215
+
216
+ def process_selection_list!(list, add_base, add_nested)
217
+ i = 0
218
+ while i < list.length
219
+ entry = list[i]
220
+ case entry
221
+ when Hash
222
+ entry.each { |k, v| add_nested.call(k, v) }
223
+ i += 1
224
+ when Symbol, String
225
+ add_base.call(entry)
226
+ i += 1
227
+ when Array
228
+ inner = Array(entry).flatten.compact
229
+ inner.each { |el| list << el }
230
+ i += 1
231
+ else
232
+ raise SearchEngine::Errors::ConflictingSelection,
233
+ "ConflictingSelection: unsupported input #{entry.class} in selection"
234
+ end
235
+ end
236
+ end
237
+ end
238
+ end
239
+ end
240
+ end