search-engine-for-typesense 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +21 -0
  3. data/README.md +148 -0
  4. data/app/search_engine/search_engine/app_info.rb +11 -0
  5. data/app/search_engine/search_engine/index_partition_job.rb +170 -0
  6. data/lib/generators/search_engine/install/install_generator.rb +20 -0
  7. data/lib/generators/search_engine/install/templates/initializer.rb.tt +230 -0
  8. data/lib/generators/search_engine/model/model_generator.rb +86 -0
  9. data/lib/generators/search_engine/model/templates/model.rb.tt +12 -0
  10. data/lib/search-engine-for-typesense.rb +12 -0
  11. data/lib/search_engine/active_record_syncable.rb +247 -0
  12. data/lib/search_engine/admin/stopwords.rb +125 -0
  13. data/lib/search_engine/admin/synonyms.rb +125 -0
  14. data/lib/search_engine/admin.rb +12 -0
  15. data/lib/search_engine/ast/and.rb +52 -0
  16. data/lib/search_engine/ast/binary_op.rb +75 -0
  17. data/lib/search_engine/ast/eq.rb +19 -0
  18. data/lib/search_engine/ast/group.rb +18 -0
  19. data/lib/search_engine/ast/gt.rb +12 -0
  20. data/lib/search_engine/ast/gte.rb +12 -0
  21. data/lib/search_engine/ast/in.rb +28 -0
  22. data/lib/search_engine/ast/lt.rb +12 -0
  23. data/lib/search_engine/ast/lte.rb +12 -0
  24. data/lib/search_engine/ast/matches.rb +55 -0
  25. data/lib/search_engine/ast/node.rb +176 -0
  26. data/lib/search_engine/ast/not_eq.rb +13 -0
  27. data/lib/search_engine/ast/not_in.rb +24 -0
  28. data/lib/search_engine/ast/or.rb +52 -0
  29. data/lib/search_engine/ast/prefix.rb +51 -0
  30. data/lib/search_engine/ast/raw.rb +41 -0
  31. data/lib/search_engine/ast/unary_op.rb +43 -0
  32. data/lib/search_engine/ast.rb +101 -0
  33. data/lib/search_engine/base/creation.rb +727 -0
  34. data/lib/search_engine/base/deletion.rb +80 -0
  35. data/lib/search_engine/base/display_coercions.rb +36 -0
  36. data/lib/search_engine/base/hydration.rb +312 -0
  37. data/lib/search_engine/base/index_maintenance/cleanup.rb +202 -0
  38. data/lib/search_engine/base/index_maintenance/lifecycle.rb +251 -0
  39. data/lib/search_engine/base/index_maintenance/schema.rb +117 -0
  40. data/lib/search_engine/base/index_maintenance.rb +459 -0
  41. data/lib/search_engine/base/indexing_dsl.rb +255 -0
  42. data/lib/search_engine/base/joins.rb +479 -0
  43. data/lib/search_engine/base/model_dsl.rb +472 -0
  44. data/lib/search_engine/base/presets.rb +43 -0
  45. data/lib/search_engine/base/pretty_printer.rb +315 -0
  46. data/lib/search_engine/base/relation_delegation.rb +42 -0
  47. data/lib/search_engine/base/scopes.rb +113 -0
  48. data/lib/search_engine/base/updating.rb +92 -0
  49. data/lib/search_engine/base.rb +38 -0
  50. data/lib/search_engine/bulk.rb +284 -0
  51. data/lib/search_engine/cache.rb +33 -0
  52. data/lib/search_engine/cascade.rb +531 -0
  53. data/lib/search_engine/cli/doctor.rb +631 -0
  54. data/lib/search_engine/cli/support.rb +217 -0
  55. data/lib/search_engine/cli.rb +222 -0
  56. data/lib/search_engine/client/http_adapter.rb +63 -0
  57. data/lib/search_engine/client/request_builder.rb +92 -0
  58. data/lib/search_engine/client/services/base.rb +74 -0
  59. data/lib/search_engine/client/services/collections.rb +161 -0
  60. data/lib/search_engine/client/services/documents.rb +214 -0
  61. data/lib/search_engine/client/services/operations.rb +152 -0
  62. data/lib/search_engine/client/services/search.rb +190 -0
  63. data/lib/search_engine/client/services.rb +29 -0
  64. data/lib/search_engine/client.rb +765 -0
  65. data/lib/search_engine/client_options.rb +20 -0
  66. data/lib/search_engine/collection_resolver.rb +191 -0
  67. data/lib/search_engine/collections_graph.rb +330 -0
  68. data/lib/search_engine/compiled_params.rb +143 -0
  69. data/lib/search_engine/compiler.rb +383 -0
  70. data/lib/search_engine/config/observability.rb +27 -0
  71. data/lib/search_engine/config/presets.rb +92 -0
  72. data/lib/search_engine/config/selection.rb +16 -0
  73. data/lib/search_engine/config/typesense.rb +48 -0
  74. data/lib/search_engine/config/validators.rb +97 -0
  75. data/lib/search_engine/config.rb +917 -0
  76. data/lib/search_engine/console_helpers.rb +130 -0
  77. data/lib/search_engine/deletion.rb +103 -0
  78. data/lib/search_engine/dispatcher.rb +125 -0
  79. data/lib/search_engine/dsl/parser.rb +582 -0
  80. data/lib/search_engine/engine.rb +167 -0
  81. data/lib/search_engine/errors.rb +290 -0
  82. data/lib/search_engine/filters/sanitizer.rb +189 -0
  83. data/lib/search_engine/hydration/materializers.rb +808 -0
  84. data/lib/search_engine/hydration/selection_context.rb +96 -0
  85. data/lib/search_engine/indexer/batch_planner.rb +76 -0
  86. data/lib/search_engine/indexer/bulk_import.rb +626 -0
  87. data/lib/search_engine/indexer/import_dispatcher.rb +198 -0
  88. data/lib/search_engine/indexer/retry_policy.rb +103 -0
  89. data/lib/search_engine/indexer.rb +747 -0
  90. data/lib/search_engine/instrumentation.rb +308 -0
  91. data/lib/search_engine/joins/guard.rb +202 -0
  92. data/lib/search_engine/joins/resolver.rb +95 -0
  93. data/lib/search_engine/logging/color.rb +78 -0
  94. data/lib/search_engine/logging/format_helpers.rb +92 -0
  95. data/lib/search_engine/logging/partition_progress.rb +53 -0
  96. data/lib/search_engine/logging_subscriber.rb +388 -0
  97. data/lib/search_engine/mapper.rb +785 -0
  98. data/lib/search_engine/multi.rb +286 -0
  99. data/lib/search_engine/multi_result.rb +186 -0
  100. data/lib/search_engine/notifications/compact_logger.rb +675 -0
  101. data/lib/search_engine/observability.rb +162 -0
  102. data/lib/search_engine/operations.rb +58 -0
  103. data/lib/search_engine/otel.rb +227 -0
  104. data/lib/search_engine/partitioner.rb +128 -0
  105. data/lib/search_engine/ranking_plan.rb +118 -0
  106. data/lib/search_engine/registry.rb +158 -0
  107. data/lib/search_engine/relation/compiler.rb +711 -0
  108. data/lib/search_engine/relation/deletion.rb +37 -0
  109. data/lib/search_engine/relation/dsl/filters.rb +624 -0
  110. data/lib/search_engine/relation/dsl/selection.rb +240 -0
  111. data/lib/search_engine/relation/dsl.rb +903 -0
  112. data/lib/search_engine/relation/dx/dry_run.rb +59 -0
  113. data/lib/search_engine/relation/dx/friendly_where.rb +24 -0
  114. data/lib/search_engine/relation/dx.rb +231 -0
  115. data/lib/search_engine/relation/materializers.rb +118 -0
  116. data/lib/search_engine/relation/options.rb +138 -0
  117. data/lib/search_engine/relation/state.rb +274 -0
  118. data/lib/search_engine/relation/updating.rb +44 -0
  119. data/lib/search_engine/relation.rb +623 -0
  120. data/lib/search_engine/result.rb +664 -0
  121. data/lib/search_engine/schema.rb +1083 -0
  122. data/lib/search_engine/sources/active_record_source.rb +185 -0
  123. data/lib/search_engine/sources/base.rb +62 -0
  124. data/lib/search_engine/sources/lambda_source.rb +55 -0
  125. data/lib/search_engine/sources/sql_source.rb +196 -0
  126. data/lib/search_engine/sources.rb +71 -0
  127. data/lib/search_engine/stale_rules.rb +160 -0
  128. data/lib/search_engine/test/minitest_assertions.rb +57 -0
  129. data/lib/search_engine/test/offline_client.rb +134 -0
  130. data/lib/search_engine/test/rspec_matchers.rb +77 -0
  131. data/lib/search_engine/test/stub_client.rb +201 -0
  132. data/lib/search_engine/test.rb +66 -0
  133. data/lib/search_engine/test_autoload.rb +8 -0
  134. data/lib/search_engine/update.rb +35 -0
  135. data/lib/search_engine/version.rb +7 -0
  136. data/lib/search_engine.rb +332 -0
  137. data/lib/tasks/search_engine.rake +501 -0
  138. data/lib/tasks/search_engine_doctor.rake +16 -0
  139. metadata +225 -0
@@ -0,0 +1,190 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SearchEngine
4
+ class Client
5
+ module Services
6
+ # Handles single and multi search workflows while keeping observability hooks intact.
7
+ class Search < Base
8
+ # Execute a single search request.
9
+ # @param collection [String]
10
+ # @param params [Hash]
11
+ # @param url_opts [Hash]
12
+ # @return [SearchEngine::Result]
13
+ def call(collection:, params:, url_opts: {})
14
+ params_obj = SearchEngine::CompiledParams.from(params)
15
+ validate_single!(collection, params_obj.to_h)
16
+
17
+ cache_params = derive_cache_opts(url_opts)
18
+ start = current_monotonic_ms
19
+ payload = sanitize_body_params(params_obj.to_h)
20
+ path = [Client::RequestBuilder::COLLECTIONS_PREFIX, collection.to_s, Client::RequestBuilder::DOCUMENTS_SEARCH_SUFFIX].join
21
+
22
+ result = instrumented_search(collection, params_obj, cache_params, path, payload, start)
23
+ duration = current_monotonic_ms - start
24
+ instrument(:post, path, duration, cache_params)
25
+ log_success(:post, path, start, cache_params)
26
+
27
+ klass = begin
28
+ SearchEngine.collection_for(collection)
29
+ rescue ArgumentError
30
+ nil
31
+ end
32
+ SearchEngine::Result.new(result, klass: klass)
33
+ end
34
+
35
+ # Execute a multi search request.
36
+ # @param searches [Array<Hash>]
37
+ # @param url_opts [Hash]
38
+ # @return [Hash]
39
+ def multi(searches:, url_opts: {})
40
+ validate_multi!(searches)
41
+
42
+ cache_params = derive_cache_opts(url_opts)
43
+ start = current_monotonic_ms
44
+ path = '/multi_search'
45
+ bodies = Array(searches).map { |s| sanitize_body_params(s) }
46
+
47
+ result = with_exception_mapping(:post, path, cache_params, start) do
48
+ typesense.multi_search.perform({ searches: bodies }, common_params: cache_params)
49
+ end
50
+
51
+ instrument(:post, path, current_monotonic_ms - start, cache_params)
52
+ symbolize_keys_deep(result)
53
+ end
54
+
55
+ private
56
+
57
+ def instrumented_search(collection, params_obj, cache_params, path, payload, start)
58
+ if defined?(ActiveSupport::Notifications)
59
+ se_payload = build_search_event_payload(
60
+ collection: collection,
61
+ params: params_obj.to_h,
62
+ cache_params: cache_params
63
+ )
64
+ result = nil
65
+ SearchEngine::Instrumentation.instrument('search_engine.search', se_payload) do |ctx|
66
+ ctx[:params_preview] = SearchEngine::Instrumentation.redact(params_obj.to_h)
67
+ result = with_exception_mapping(:post, path, cache_params, start) do
68
+ docs = typesense.collections[collection].documents
69
+ documents_search(docs, payload, cache_params)
70
+ end
71
+ ctx[:status] = :ok
72
+ rescue Errors::Api => error
73
+ ctx[:status] = error.status
74
+ ctx[:error_class] = error.class.name
75
+ raise
76
+ rescue Errors::Error => error
77
+ ctx[:status] = :error
78
+ ctx[:error_class] = error.class.name
79
+ raise
80
+ end
81
+ result
82
+ else
83
+ with_exception_mapping(:post, path, cache_params, start) do
84
+ docs = typesense.collections[collection].documents
85
+ documents_search(docs, payload, cache_params)
86
+ end
87
+ end
88
+ end
89
+
90
+ def sanitize_body_params(params)
91
+ Client::RequestBuilder.send(:sanitize_body_params, params)
92
+ end
93
+
94
+ def build_search_event_payload(collection:, params:, cache_params: {})
95
+ sel = params[:_selection].is_a?(Hash) ? params[:_selection] : {}
96
+ base = {
97
+ collection: collection,
98
+ params: Observability.redact(params),
99
+ url_opts: Observability.filtered_url_opts(cache_params),
100
+ status: nil,
101
+ error_class: nil,
102
+ retries: nil,
103
+ selection_include_count: sel[:include_count],
104
+ selection_exclude_count: sel[:exclude_count],
105
+ selection_nested_assoc_count: sel[:nested_assoc_count]
106
+ }
107
+ base.merge(build_preset_segment(params)).merge(build_curation_segment(params))
108
+ end
109
+
110
+ def build_preset_segment(params)
111
+ preset_mode = params[:_preset_mode]
112
+ pruned = Array(params[:_preset_pruned_keys]).map { |k| k.respond_to?(:to_sym) ? k.to_sym : k }.grep(Symbol)
113
+ locked_count = begin
114
+ config.presets.locked_domains.size
115
+ rescue StandardError
116
+ nil
117
+ end
118
+ {
119
+ preset_name: params[:preset],
120
+ preset_mode: preset_mode,
121
+ preset_pruned_keys_count: pruned.empty? ? nil : pruned.size,
122
+ preset_locked_domains_count: locked_count,
123
+ preset_pruned_keys: pruned.empty? ? nil : pruned
124
+ }
125
+ end
126
+
127
+ def build_curation_segment(params)
128
+ curation = params[:_curation].is_a?(Hash) ? params[:_curation] : {}
129
+
130
+ pinned = params[:pinned_hits]
131
+ hidden = params[:hidden_hits]
132
+ tags = params[:override_tags]
133
+
134
+ pinned_count =
135
+ case pinned
136
+ when String then pinned.split(',').count
137
+ when Array then pinned.size
138
+ else 0
139
+ end
140
+
141
+ hidden_count =
142
+ case hidden
143
+ when String then hidden.split(',').count
144
+ when Array then hidden.size
145
+ else 0
146
+ end
147
+
148
+ has_tags =
149
+ case tags
150
+ when String then !tags.to_s.strip.empty?
151
+ when Array then !tags.empty?
152
+ else false
153
+ end
154
+
155
+ out = {
156
+ curation_pinned_count: pinned_count,
157
+ curation_hidden_count: hidden_count,
158
+ curation_has_override_tags: has_tags
159
+ }
160
+
161
+ if params.key?(:filter_curated_hits) || curation.key?(:filter_curated_hits)
162
+ out[:curation_filter_flag] =
163
+ params.key?(:filter_curated_hits) ? params[:filter_curated_hits] : curation[:filter_curated_hits]
164
+ end
165
+
166
+ out[:curation_conflict_type] = curation[:_conflict_type] || params[:_curation_conflict_type]
167
+ out[:curation_conflict_count] = curation[:_conflict_count] || params[:_curation_conflict_count]
168
+
169
+ out.compact
170
+ end
171
+
172
+ def documents_search(docs, payload, common_params)
173
+ if documents_search_supports_common_params?(docs)
174
+ docs.search(payload, common_params: common_params)
175
+ else
176
+ docs.search(payload)
177
+ end
178
+ end
179
+
180
+ def documents_search_supports_common_params?(docs)
181
+ m = docs.method(:search)
182
+ params = m.parameters
183
+ params.any? { |(kind, _)| %i[key keyreq keyrest].include?(kind) }
184
+ rescue StandardError
185
+ false
186
+ end
187
+ end
188
+ end
189
+ end
190
+ end
@@ -0,0 +1,29 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'search_engine/client/services/base'
4
+ require 'search_engine/client/services/search'
5
+ require 'search_engine/client/services/collections'
6
+ require 'search_engine/client/services/operations'
7
+ require 'search_engine/client/services/documents'
8
+
9
+ module SearchEngine
10
+ class Client
11
+ # Registry for all service objects used internally by the client.
12
+ module Services
13
+ SERVICES = {
14
+ search: Search,
15
+ collections: Collections,
16
+ operations: Operations,
17
+ documents: Documents
18
+ }.freeze
19
+
20
+ module_function
21
+
22
+ # @param client [SearchEngine::Client]
23
+ # @return [Hash<Symbol, SearchEngine::Client::Services::Base>]
24
+ def build(client)
25
+ SERVICES.transform_values { |klass| klass.new(client) }
26
+ end
27
+ end
28
+ end
29
+ end