noiseless 0.0.0 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/LICENSE.txt +28 -0
- data/README.md +214 -0
- data/lib/application_search.rb +15 -0
- data/lib/noiseless/adapter.rb +313 -0
- data/lib/noiseless/adapters/elasticsearch.rb +70 -0
- data/lib/noiseless/adapters/execution_modules/elasticsearch_execution.rb +188 -0
- data/lib/noiseless/adapters/execution_modules/opensearch_execution.rb +377 -0
- data/lib/noiseless/adapters/execution_modules/pgvector_support.rb +219 -0
- data/lib/noiseless/adapters/execution_modules/postgresql_execution.rb +461 -0
- data/lib/noiseless/adapters/execution_modules/typesense_execution.rb +472 -0
- data/lib/noiseless/adapters/open_search.rb +208 -0
- data/lib/noiseless/adapters/postgresql.rb +171 -0
- data/lib/noiseless/adapters/typesense.rb +70 -0
- data/lib/noiseless/adapters.rb +14 -0
- data/lib/noiseless/ast/aggregation.rb +56 -0
- data/lib/noiseless/ast/bool.rb +16 -0
- data/lib/noiseless/ast/bulk.rb +18 -0
- data/lib/noiseless/ast/collapse.rb +16 -0
- data/lib/noiseless/ast/combined_fields.rb +33 -0
- data/lib/noiseless/ast/conversation.rb +29 -0
- data/lib/noiseless/ast/filter.rb +15 -0
- data/lib/noiseless/ast/hybrid.rb +35 -0
- data/lib/noiseless/ast/image_query.rb +29 -0
- data/lib/noiseless/ast/join.rb +31 -0
- data/lib/noiseless/ast/match.rb +15 -0
- data/lib/noiseless/ast/multi_match.rb +24 -0
- data/lib/noiseless/ast/paginate.rb +15 -0
- data/lib/noiseless/ast/prefix.rb +15 -0
- data/lib/noiseless/ast/range.rb +18 -0
- data/lib/noiseless/ast/root.rb +69 -0
- data/lib/noiseless/ast/search_after.rb +14 -0
- data/lib/noiseless/ast/sort.rb +15 -0
- data/lib/noiseless/ast/vector.rb +27 -0
- data/lib/noiseless/ast/wildcard.rb +15 -0
- data/lib/noiseless/ast.rb +30 -0
- data/lib/noiseless/bulk_importer.rb +195 -0
- data/lib/noiseless/callbacks.rb +138 -0
- data/lib/noiseless/connection_manager.rb +26 -0
- data/lib/noiseless/document_manager.rb +137 -0
- data/lib/noiseless/dsl.rb +107 -0
- data/lib/noiseless/generators/application_search_generator.rb +24 -0
- data/lib/noiseless/instrumentation.rb +174 -0
- data/lib/noiseless/introspection/console.rb +228 -0
- data/lib/noiseless/introspection/query_visualizer.rb +533 -0
- data/lib/noiseless/introspection.rb +221 -0
- data/lib/noiseless/mapping.rb +253 -0
- data/lib/noiseless/mapping_definition_processor.rb +231 -0
- data/lib/noiseless/model.rb +111 -0
- data/lib/noiseless/model_registry.rb +77 -0
- data/lib/noiseless/multi_search.rb +244 -0
- data/lib/noiseless/pagination.rb +375 -0
- data/lib/noiseless/query_builder.rb +284 -0
- data/lib/noiseless/railtie.rb +35 -0
- data/lib/noiseless/response/aggregations.rb +46 -0
- data/lib/noiseless/response/empty.rb +20 -0
- data/lib/noiseless/response/records.rb +94 -0
- data/lib/noiseless/response/results.rb +110 -0
- data/lib/noiseless/response/suggestions.rb +55 -0
- data/lib/noiseless/response.rb +98 -0
- data/lib/noiseless/response_factory.rb +32 -0
- data/lib/noiseless/runtime_reset_middleware.rb +15 -0
- data/lib/noiseless/search_index_update_job.rb +84 -0
- data/lib/noiseless/test_case.rb +230 -0
- data/lib/noiseless/test_helper.rb +295 -0
- data/lib/noiseless/version.rb +2 -2
- data/lib/noiseless.rb +130 -2
- data/lib/tasks/benchmark.rake +35 -0
- data/lib/tasks/release.rake +22 -0
- data/lib/tasks/test.rake +11 -0
- metadata +260 -14
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Noiseless
|
|
4
|
+
class Model
|
|
5
|
+
extend DSL::ClassMethods
|
|
6
|
+
|
|
7
|
+
def initialize
|
|
8
|
+
@builder = QueryBuilder.new(self.class)
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def self.search(indexes: nil, connection: nil, response_type: nil)
|
|
12
|
+
client = Noiseless.connections.client(connection || self.connection)
|
|
13
|
+
builder = QueryBuilder.new(self)
|
|
14
|
+
builder.indexes(indexes) if indexes
|
|
15
|
+
yield(builder)
|
|
16
|
+
ast = builder.to_ast
|
|
17
|
+
client.search(ast, model_class: self, response_type: response_type)
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def self.search_sync(indexes: nil, connection: nil, response_type: nil, &)
|
|
21
|
+
Sync do
|
|
22
|
+
search(indexes: indexes, connection: connection, response_type: response_type, &).wait
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# Instance methods that delegate to the query builder
|
|
27
|
+
def match(field, value, **)
|
|
28
|
+
@builder.match(field, value, **)
|
|
29
|
+
self
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def multi_match(query, fields, **)
|
|
33
|
+
@builder.multi_match(query, fields, **)
|
|
34
|
+
self
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def filter(field, value, **)
|
|
38
|
+
@builder.filter(field, value, **)
|
|
39
|
+
self
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def range(field, gte: nil, lte: nil, gt: nil, lt: nil)
|
|
43
|
+
@builder.range(field, gte: gte, lte: lte, gt: gt, lt: lt)
|
|
44
|
+
self
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def sort(field, direction = :asc, **)
|
|
48
|
+
@builder.sort(field, direction, **)
|
|
49
|
+
self
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def limit(size)
|
|
53
|
+
@builder.limit(size)
|
|
54
|
+
self
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def offset(from)
|
|
58
|
+
@builder.offset(from)
|
|
59
|
+
self
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def paginate(page: nil, per_page: nil)
|
|
63
|
+
@builder.paginate(page: page, per_page: per_page)
|
|
64
|
+
self
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def aggregation(name, type, **)
|
|
68
|
+
@builder.aggregation(name, type, **)
|
|
69
|
+
self
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def geo_distance(field, lat:, lon:, distance:, **)
|
|
73
|
+
@builder.geo_distance(field, lat: lat, lon: lon, distance: distance, **)
|
|
74
|
+
self
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def indexes(names)
|
|
78
|
+
@builder.indexes(names)
|
|
79
|
+
self
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def remove_duplicates(value: true)
|
|
83
|
+
@builder.remove_duplicates(value: value)
|
|
84
|
+
self
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def facet_sample_slope(value)
|
|
88
|
+
@builder.facet_sample_slope(value)
|
|
89
|
+
self
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def pinned_hits(value)
|
|
93
|
+
@builder.pinned_hits(value)
|
|
94
|
+
self
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def execute(connection: nil, response_type: nil)
|
|
98
|
+
client = Noiseless.connections.client(connection || self.class.connection)
|
|
99
|
+
ast = @builder.to_ast
|
|
100
|
+
client.search(ast, model_class: self.class, response_type: response_type)
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def execute_sync(connection: nil, response_type: nil)
|
|
104
|
+
Sync do
|
|
105
|
+
execute(connection: connection, response_type: response_type).wait
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
delegate :to_ast, to: :@builder
|
|
110
|
+
end
|
|
111
|
+
end
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Noiseless
|
|
4
|
+
class ModelRegistry
|
|
5
|
+
include Singleton
|
|
6
|
+
|
|
7
|
+
def initialize
|
|
8
|
+
@models = {}
|
|
9
|
+
@models_by_index = {}
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def register(model_class, options = {})
|
|
13
|
+
model_name = model_class.name.to_sym
|
|
14
|
+
@models[model_name] = {
|
|
15
|
+
class: model_class,
|
|
16
|
+
options: options
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
# Index models by their search index names
|
|
20
|
+
index_names = Array(model_class.search_index || default_index_name(model_class))
|
|
21
|
+
index_names.each do |index_name|
|
|
22
|
+
@models_by_index[index_name.to_sym] ||= []
|
|
23
|
+
@models_by_index[index_name.to_sym] << model_class
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def unregister(model_class)
|
|
28
|
+
model_name = model_class.name.to_sym
|
|
29
|
+
@models.delete(model_name)
|
|
30
|
+
|
|
31
|
+
# Remove from index mapping
|
|
32
|
+
@models_by_index.each do |index_name, models|
|
|
33
|
+
models.delete(model_class)
|
|
34
|
+
@models_by_index.delete(index_name) if models.empty?
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def all_models
|
|
39
|
+
@models.values.map { |entry| entry[:class] }
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def find_model(name_or_class)
|
|
43
|
+
case name_or_class
|
|
44
|
+
when String, Symbol
|
|
45
|
+
entry = @models[name_or_class.to_sym]
|
|
46
|
+
entry ? entry[:class] : nil
|
|
47
|
+
when Class
|
|
48
|
+
@models.find { |_, entry| entry[:class] == name_or_class }&.last&.fetch(:class)
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def models_for_index(index_name)
|
|
53
|
+
@models_by_index[index_name.to_sym] || []
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def all_indexes
|
|
57
|
+
@models_by_index.keys.map(&:to_s)
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def searchable_models
|
|
61
|
+
@models.reject { |_, entry| entry[:options][:searchable] == false }
|
|
62
|
+
.values
|
|
63
|
+
.map { |entry| entry[:class] }
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def clear!
|
|
67
|
+
@models.clear
|
|
68
|
+
@models_by_index.clear
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
private
|
|
72
|
+
|
|
73
|
+
def default_index_name(model_class)
|
|
74
|
+
model_class.name.demodulize.underscore.pluralize
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
end
|
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Noiseless
|
|
4
|
+
class MultiSearch
|
|
5
|
+
def initialize(models: nil, indexes: nil, connection: nil)
|
|
6
|
+
@models = resolve_models(models)
|
|
7
|
+
@indexes = resolve_indexes(indexes)
|
|
8
|
+
@connection = connection || Noiseless.config.default_connection
|
|
9
|
+
@builder = QueryBuilder.new(nil)
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def search(&block)
|
|
13
|
+
yield(@builder) if block
|
|
14
|
+
|
|
15
|
+
client = Noiseless.connections.client(@connection)
|
|
16
|
+
ast = @builder.to_ast
|
|
17
|
+
|
|
18
|
+
# Override indexes in AST with our multi-model indexes
|
|
19
|
+
ast_with_indexes = AST::Root.new(
|
|
20
|
+
indexes: @indexes,
|
|
21
|
+
bool: ast.bool,
|
|
22
|
+
sort: ast.sort,
|
|
23
|
+
paginate: ast.paginate,
|
|
24
|
+
vector: ast.vector,
|
|
25
|
+
collapse: ast.collapse,
|
|
26
|
+
search_after: ast.search_after,
|
|
27
|
+
aggregations: ast.aggregations,
|
|
28
|
+
hybrid: ast.hybrid,
|
|
29
|
+
pipeline: ast.pipeline,
|
|
30
|
+
image_query: ast.image_query,
|
|
31
|
+
conversation: ast.conversation,
|
|
32
|
+
joins: ast.joins,
|
|
33
|
+
remove_duplicates: ast.remove_duplicates,
|
|
34
|
+
facet_sample_slope: ast.facet_sample_slope,
|
|
35
|
+
pinned_hits: ast.pinned_hits
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
raw_response = client.search(ast_with_indexes)
|
|
39
|
+
MultiSearchResponse.new(raw_response, @models, @indexes)
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# Delegate query building methods to the internal builder
|
|
43
|
+
def match(field, value, **)
|
|
44
|
+
@builder.match(field, value, **)
|
|
45
|
+
self
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def multi_match(query, fields, **)
|
|
49
|
+
@builder.multi_match(query, fields, **)
|
|
50
|
+
self
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def filter(field, value, **)
|
|
54
|
+
@builder.filter(field, value, **)
|
|
55
|
+
self
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def sort(field, direction = :asc, **)
|
|
59
|
+
@builder.sort(field, direction, **)
|
|
60
|
+
self
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def limit(size)
|
|
64
|
+
@builder.limit(size)
|
|
65
|
+
self
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def offset(from)
|
|
69
|
+
@builder.offset(from)
|
|
70
|
+
self
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def paginate(page, per_page)
|
|
74
|
+
@builder.paginate(page: page, per_page: per_page)
|
|
75
|
+
self
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def aggregation(name, type, **)
|
|
79
|
+
@builder.aggregation(name, type, **)
|
|
80
|
+
self
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def geo_distance(field, lat:, lon:, distance:, **)
|
|
84
|
+
@builder.geo_distance(field, lat: lat, lon: lon, distance: distance, **)
|
|
85
|
+
self
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def remove_duplicates(value: true)
|
|
89
|
+
@builder.remove_duplicates(value: value)
|
|
90
|
+
self
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def facet_sample_slope(value)
|
|
94
|
+
@builder.facet_sample_slope(value)
|
|
95
|
+
self
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def pinned_hits(value)
|
|
99
|
+
@builder.pinned_hits(value)
|
|
100
|
+
self
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
private
|
|
104
|
+
|
|
105
|
+
def resolve_models(models)
|
|
106
|
+
case models
|
|
107
|
+
when nil
|
|
108
|
+
Noiseless.searchable_models
|
|
109
|
+
when Array
|
|
110
|
+
models.filter_map { |m| resolve_single_model(m) }
|
|
111
|
+
else
|
|
112
|
+
[resolve_single_model(models)].compact
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def resolve_single_model(model)
|
|
117
|
+
case model
|
|
118
|
+
when String, Symbol
|
|
119
|
+
Noiseless.registry.find_model(model)
|
|
120
|
+
when Class
|
|
121
|
+
model
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def resolve_indexes(indexes)
|
|
126
|
+
case indexes
|
|
127
|
+
when nil
|
|
128
|
+
@models.flat_map do |model|
|
|
129
|
+
Array(model.search_index || default_index_name(model))
|
|
130
|
+
end.uniq
|
|
131
|
+
when Array
|
|
132
|
+
indexes.map(&:to_s)
|
|
133
|
+
when String, Symbol
|
|
134
|
+
[indexes.to_s]
|
|
135
|
+
else
|
|
136
|
+
[]
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def default_index_name(model_class)
|
|
141
|
+
model_class.name.demodulize.underscore.pluralize
|
|
142
|
+
end
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
# Multi-search response class
|
|
146
|
+
class MultiSearchResponse < Response::Base
|
|
147
|
+
def initialize(raw_response, models, indexes)
|
|
148
|
+
super(raw_response)
|
|
149
|
+
@models = models
|
|
150
|
+
@indexes = indexes
|
|
151
|
+
@results_by_model = nil
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
def each(&)
|
|
155
|
+
return enum_for(__method__) unless block_given?
|
|
156
|
+
|
|
157
|
+
hits.each(&)
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
def results_by_model
|
|
161
|
+
@results_by_model ||= group_results_by_model
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
def results_for_model(model_class)
|
|
165
|
+
model_key = model_class.name.to_sym
|
|
166
|
+
results_by_model[model_key] || []
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
def each_model_result(&)
|
|
170
|
+
return enum_for(__method__) unless block_given?
|
|
171
|
+
|
|
172
|
+
results_by_model.each(&)
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
def model_counts
|
|
176
|
+
results_by_model.transform_values(&:size)
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
def records_by_model
|
|
180
|
+
@records_by_model ||= load_records_by_model
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
def records_for_model(model_class)
|
|
184
|
+
model_key = model_class.name.to_sym
|
|
185
|
+
records_by_model[model_key] || []
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
private
|
|
189
|
+
|
|
190
|
+
def group_results_by_model
|
|
191
|
+
results = {}
|
|
192
|
+
|
|
193
|
+
hits.each do |hit|
|
|
194
|
+
model_class = determine_model_class(hit)
|
|
195
|
+
next unless model_class
|
|
196
|
+
|
|
197
|
+
model_key = model_class.name.to_sym
|
|
198
|
+
results[model_key] ||= []
|
|
199
|
+
results[model_key] << hit
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
results
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
def determine_model_class(hit)
|
|
206
|
+
index_name = hit["_index"]
|
|
207
|
+
|
|
208
|
+
# First, try to find models registered for this specific index
|
|
209
|
+
models_for_index = Noiseless.registry.models_for_index(index_name)
|
|
210
|
+
return models_for_index.first if models_for_index.size == 1
|
|
211
|
+
|
|
212
|
+
# If multiple models or none found, try to infer from index name
|
|
213
|
+
@models.find do |model|
|
|
214
|
+
model_indexes = Array(model.search_index || default_index_name(model))
|
|
215
|
+
model_indexes.include?(index_name)
|
|
216
|
+
end
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
def load_records_by_model
|
|
220
|
+
records = {}
|
|
221
|
+
|
|
222
|
+
results_by_model.each do |model_key, model_hits|
|
|
223
|
+
model_class = @models.find { |m| m.name.to_sym == model_key }
|
|
224
|
+
next unless model_class.respond_to?(:where)
|
|
225
|
+
|
|
226
|
+
ids = model_hits.map { |hit| hit["_id"] }
|
|
227
|
+
loaded_records = model_class.where(id: ids).to_a
|
|
228
|
+
|
|
229
|
+
# Sort by search relevance
|
|
230
|
+
sorted_records = model_hits.filter_map do |hit|
|
|
231
|
+
loaded_records.find { |record| record.id.to_s == hit["_id"].to_s }
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
records[model_key] = sorted_records
|
|
235
|
+
end
|
|
236
|
+
|
|
237
|
+
records
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
def default_index_name(model_class)
|
|
241
|
+
model_class.name.demodulize.underscore.pluralize
|
|
242
|
+
end
|
|
243
|
+
end
|
|
244
|
+
end
|