chewy 5.1.0 → 7.2.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/CODEOWNERS +1 -0
- data/.github/ISSUE_TEMPLATE/bug_report.md +39 -0
- data/.github/ISSUE_TEMPLATE/feature_request.md +20 -0
- data/.github/PULL_REQUEST_TEMPLATE.md +16 -0
- data/.github/workflows/ruby.yml +73 -0
- data/.rubocop.yml +13 -8
- data/.rubocop_todo.yml +110 -22
- data/CHANGELOG.md +449 -347
- data/CODE_OF_CONDUCT.md +14 -0
- data/CONTRIBUTING.md +63 -0
- data/Gemfile +3 -7
- data/Guardfile +3 -1
- data/LICENSE.txt +1 -1
- data/README.md +423 -311
- data/chewy.gemspec +8 -10
- data/gemfiles/rails.5.2.activerecord.gemfile +9 -14
- data/gemfiles/rails.6.0.activerecord.gemfile +11 -0
- data/gemfiles/rails.6.1.activerecord.gemfile +13 -0
- data/gemfiles/rails.7.0.activerecord.gemfile +13 -0
- data/lib/chewy/config.rb +42 -60
- data/lib/chewy/errors.rb +4 -10
- data/lib/chewy/fields/base.rb +80 -20
- data/lib/chewy/fields/root.rb +7 -17
- data/lib/chewy/index/actions.rb +62 -35
- data/lib/chewy/{type → index}/adapter/active_record.rb +18 -4
- data/lib/chewy/{type → index}/adapter/base.rb +2 -3
- data/lib/chewy/{type → index}/adapter/object.rb +28 -32
- data/lib/chewy/{type → index}/adapter/orm.rb +26 -24
- data/lib/chewy/index/aliases.rb +14 -5
- data/lib/chewy/{type → index}/crutch.rb +5 -5
- data/lib/chewy/index/import/bulk_builder.rb +311 -0
- data/lib/chewy/{type → index}/import/bulk_request.rb +6 -7
- data/lib/chewy/{type → index}/import/journal_builder.rb +11 -12
- data/lib/chewy/{type → index}/import/routine.rb +17 -16
- data/lib/chewy/{type → index}/import.rb +51 -33
- data/lib/chewy/{type → index}/mapping.rb +32 -37
- data/lib/chewy/index/observe/active_record_methods.rb +87 -0
- data/lib/chewy/index/observe/callback.rb +34 -0
- data/lib/chewy/index/observe.rb +17 -0
- data/lib/chewy/index/specification.rb +1 -0
- data/lib/chewy/{type → index}/syncer.rb +61 -62
- data/lib/chewy/{type → index}/witchcraft.rb +15 -9
- data/lib/chewy/{type → index}/wrapper.rb +13 -3
- data/lib/chewy/index.rb +46 -96
- data/lib/chewy/journal.rb +25 -14
- data/lib/chewy/minitest/helpers.rb +86 -13
- data/lib/chewy/minitest/search_index_receiver.rb +22 -26
- data/lib/chewy/multi_search.rb +62 -0
- data/lib/chewy/railtie.rb +6 -20
- data/lib/chewy/rake_helper.rb +136 -108
- data/lib/chewy/rspec/build_query.rb +12 -0
- data/lib/chewy/rspec/helpers.rb +55 -0
- data/lib/chewy/rspec/update_index.rb +55 -44
- data/lib/chewy/rspec.rb +2 -0
- data/lib/chewy/runtime.rb +1 -1
- data/lib/chewy/search/loader.rb +19 -41
- data/lib/chewy/search/parameters/collapse.rb +16 -0
- data/lib/chewy/search/parameters/concerns/query_storage.rb +2 -2
- data/lib/chewy/search/parameters/ignore_unavailable.rb +27 -0
- data/lib/chewy/search/parameters/indices.rb +12 -57
- data/lib/chewy/search/parameters/none.rb +1 -3
- data/lib/chewy/search/parameters/order.rb +6 -19
- data/lib/chewy/search/parameters/source.rb +5 -1
- data/lib/chewy/search/parameters/track_total_hits.rb +16 -0
- data/lib/chewy/search/parameters.rb +7 -4
- data/lib/chewy/search/query_proxy.rb +9 -2
- data/lib/chewy/search/request.rb +180 -154
- data/lib/chewy/search/response.rb +5 -5
- data/lib/chewy/search/scoping.rb +7 -8
- data/lib/chewy/search/scrolling.rb +16 -13
- data/lib/chewy/search.rb +7 -22
- data/lib/chewy/stash.rb +19 -30
- data/lib/chewy/strategy/active_job.rb +2 -2
- data/lib/chewy/strategy/atomic_no_refresh.rb +18 -0
- data/lib/chewy/strategy/base.rb +10 -0
- data/lib/chewy/strategy/lazy_sidekiq.rb +64 -0
- data/lib/chewy/strategy/sidekiq.rb +3 -2
- data/lib/chewy/strategy.rb +5 -19
- data/lib/chewy/version.rb +1 -1
- data/lib/chewy.rb +36 -80
- data/lib/generators/chewy/install_generator.rb +1 -1
- data/lib/tasks/chewy.rake +26 -32
- data/migration_guide.md +56 -0
- data/spec/chewy/config_spec.rb +15 -61
- data/spec/chewy/fields/base_spec.rb +432 -145
- data/spec/chewy/fields/root_spec.rb +20 -28
- data/spec/chewy/fields/time_fields_spec.rb +5 -5
- data/spec/chewy/index/actions_spec.rb +388 -55
- data/spec/chewy/{type → index}/adapter/active_record_spec.rb +110 -44
- data/spec/chewy/{type → index}/adapter/object_spec.rb +21 -6
- data/spec/chewy/index/aliases_spec.rb +3 -3
- data/spec/chewy/index/import/bulk_builder_spec.rb +494 -0
- data/spec/chewy/{type → index}/import/bulk_request_spec.rb +5 -12
- data/spec/chewy/{type → index}/import/journal_builder_spec.rb +14 -22
- data/spec/chewy/{type → index}/import/routine_spec.rb +19 -19
- data/spec/chewy/{type → index}/import_spec.rb +149 -96
- data/spec/chewy/index/mapping_spec.rb +135 -0
- data/spec/chewy/index/observe/active_record_methods_spec.rb +68 -0
- data/spec/chewy/index/observe/callback_spec.rb +139 -0
- data/spec/chewy/index/observe_spec.rb +143 -0
- data/spec/chewy/index/settings_spec.rb +3 -1
- data/spec/chewy/index/specification_spec.rb +20 -30
- data/spec/chewy/{type → index}/syncer_spec.rb +14 -19
- data/spec/chewy/{type → index}/witchcraft_spec.rb +34 -21
- data/spec/chewy/index/wrapper_spec.rb +100 -0
- data/spec/chewy/index_spec.rb +69 -137
- data/spec/chewy/journal_spec.rb +46 -91
- data/spec/chewy/minitest/helpers_spec.rb +122 -14
- data/spec/chewy/minitest/search_index_receiver_spec.rb +24 -26
- data/spec/chewy/multi_search_spec.rb +84 -0
- data/spec/chewy/rake_helper_spec.rb +293 -101
- data/spec/chewy/rspec/build_query_spec.rb +34 -0
- data/spec/chewy/rspec/helpers_spec.rb +61 -0
- data/spec/chewy/rspec/update_index_spec.rb +106 -102
- data/spec/chewy/runtime_spec.rb +2 -2
- data/spec/chewy/search/loader_spec.rb +19 -53
- data/spec/chewy/search/pagination/kaminari_examples.rb +3 -5
- data/spec/chewy/search/pagination/kaminari_spec.rb +1 -1
- data/spec/chewy/search/parameters/collapse_spec.rb +5 -0
- data/spec/chewy/search/parameters/ignore_unavailable_spec.rb +67 -0
- data/spec/chewy/search/parameters/indices_spec.rb +26 -118
- data/spec/chewy/search/parameters/none_spec.rb +1 -1
- data/spec/chewy/search/parameters/order_spec.rb +18 -11
- data/spec/chewy/search/parameters/query_storage_examples.rb +67 -21
- data/spec/chewy/search/parameters/search_after_spec.rb +4 -1
- data/spec/chewy/search/parameters/source_spec.rb +8 -2
- data/spec/chewy/search/parameters/track_total_hits_spec.rb +5 -0
- data/spec/chewy/search/parameters_spec.rb +23 -7
- data/spec/chewy/search/query_proxy_spec.rb +68 -17
- data/spec/chewy/search/request_spec.rb +344 -149
- data/spec/chewy/search/response_spec.rb +35 -25
- data/spec/chewy/search/scrolling_spec.rb +28 -26
- data/spec/chewy/search_spec.rb +69 -59
- data/spec/chewy/stash_spec.rb +16 -26
- data/spec/chewy/strategy/active_job_spec.rb +23 -10
- data/spec/chewy/strategy/atomic_no_refresh_spec.rb +60 -0
- data/spec/chewy/strategy/atomic_spec.rb +9 -10
- data/spec/chewy/strategy/lazy_sidekiq_spec.rb +214 -0
- data/spec/chewy/strategy/sidekiq_spec.rb +14 -10
- data/spec/chewy/strategy_spec.rb +19 -15
- data/spec/chewy_spec.rb +17 -110
- data/spec/spec_helper.rb +6 -29
- data/spec/support/active_record.rb +43 -5
- metadata +102 -198
- data/.travis.yml +0 -45
- data/Appraisals +0 -81
- data/LEGACY_DSL.md +0 -497
- data/gemfiles/rails.4.0.activerecord.gemfile +0 -15
- data/gemfiles/rails.4.1.activerecord.gemfile +0 -15
- data/gemfiles/rails.4.2.activerecord.gemfile +0 -16
- data/gemfiles/rails.4.2.mongoid.5.2.gemfile +0 -16
- data/gemfiles/rails.5.0.activerecord.gemfile +0 -16
- data/gemfiles/rails.5.0.mongoid.6.1.gemfile +0 -16
- data/gemfiles/rails.5.1.activerecord.gemfile +0 -16
- data/gemfiles/rails.5.1.mongoid.6.3.gemfile +0 -16
- data/gemfiles/sequel.4.45.gemfile +0 -11
- data/lib/chewy/backports/deep_dup.rb +0 -46
- data/lib/chewy/backports/duplicable.rb +0 -91
- data/lib/chewy/query/compose.rb +0 -68
- data/lib/chewy/query/criteria.rb +0 -191
- data/lib/chewy/query/filters.rb +0 -244
- data/lib/chewy/query/loading.rb +0 -110
- data/lib/chewy/query/nodes/and.rb +0 -25
- data/lib/chewy/query/nodes/base.rb +0 -17
- data/lib/chewy/query/nodes/bool.rb +0 -34
- data/lib/chewy/query/nodes/equal.rb +0 -34
- data/lib/chewy/query/nodes/exists.rb +0 -20
- data/lib/chewy/query/nodes/expr.rb +0 -28
- data/lib/chewy/query/nodes/field.rb +0 -110
- data/lib/chewy/query/nodes/has_child.rb +0 -15
- data/lib/chewy/query/nodes/has_parent.rb +0 -15
- data/lib/chewy/query/nodes/has_relation.rb +0 -59
- data/lib/chewy/query/nodes/match_all.rb +0 -11
- data/lib/chewy/query/nodes/missing.rb +0 -20
- data/lib/chewy/query/nodes/not.rb +0 -25
- data/lib/chewy/query/nodes/or.rb +0 -25
- data/lib/chewy/query/nodes/prefix.rb +0 -19
- data/lib/chewy/query/nodes/query.rb +0 -20
- data/lib/chewy/query/nodes/range.rb +0 -63
- data/lib/chewy/query/nodes/raw.rb +0 -15
- data/lib/chewy/query/nodes/regexp.rb +0 -35
- data/lib/chewy/query/nodes/script.rb +0 -20
- data/lib/chewy/query/pagination.rb +0 -25
- data/lib/chewy/query.rb +0 -1142
- data/lib/chewy/search/pagination/will_paginate.rb +0 -43
- data/lib/chewy/search/parameters/types.rb +0 -20
- data/lib/chewy/strategy/resque.rb +0 -27
- data/lib/chewy/strategy/shoryuken.rb +0 -40
- data/lib/chewy/type/actions.rb +0 -43
- data/lib/chewy/type/adapter/mongoid.rb +0 -67
- data/lib/chewy/type/adapter/sequel.rb +0 -93
- data/lib/chewy/type/import/bulk_builder.rb +0 -122
- data/lib/chewy/type/observe.rb +0 -82
- data/lib/chewy/type.rb +0 -117
- data/lib/sequel/plugins/chewy_observe.rb +0 -63
- data/spec/chewy/query/criteria_spec.rb +0 -700
- data/spec/chewy/query/filters_spec.rb +0 -201
- data/spec/chewy/query/loading_spec.rb +0 -124
- data/spec/chewy/query/nodes/and_spec.rb +0 -12
- data/spec/chewy/query/nodes/bool_spec.rb +0 -14
- data/spec/chewy/query/nodes/equal_spec.rb +0 -32
- data/spec/chewy/query/nodes/exists_spec.rb +0 -18
- data/spec/chewy/query/nodes/has_child_spec.rb +0 -59
- data/spec/chewy/query/nodes/has_parent_spec.rb +0 -59
- data/spec/chewy/query/nodes/match_all_spec.rb +0 -11
- data/spec/chewy/query/nodes/missing_spec.rb +0 -16
- data/spec/chewy/query/nodes/not_spec.rb +0 -14
- data/spec/chewy/query/nodes/or_spec.rb +0 -12
- data/spec/chewy/query/nodes/prefix_spec.rb +0 -16
- data/spec/chewy/query/nodes/query_spec.rb +0 -12
- data/spec/chewy/query/nodes/range_spec.rb +0 -32
- data/spec/chewy/query/nodes/raw_spec.rb +0 -11
- data/spec/chewy/query/nodes/regexp_spec.rb +0 -43
- data/spec/chewy/query/nodes/script_spec.rb +0 -15
- data/spec/chewy/query/pagination/kaminari_spec.rb +0 -5
- data/spec/chewy/query/pagination/will_paginate_spec.rb +0 -5
- data/spec/chewy/query/pagination_spec.rb +0 -39
- data/spec/chewy/query_spec.rb +0 -637
- data/spec/chewy/search/pagination/will_paginate_examples.rb +0 -63
- data/spec/chewy/search/pagination/will_paginate_spec.rb +0 -23
- data/spec/chewy/search/parameters/types_spec.rb +0 -5
- data/spec/chewy/strategy/resque_spec.rb +0 -46
- data/spec/chewy/strategy/shoryuken_spec.rb +0 -66
- data/spec/chewy/type/actions_spec.rb +0 -50
- data/spec/chewy/type/adapter/mongoid_spec.rb +0 -372
- data/spec/chewy/type/adapter/sequel_spec.rb +0 -472
- data/spec/chewy/type/import/bulk_builder_spec.rb +0 -279
- data/spec/chewy/type/mapping_spec.rb +0 -173
- data/spec/chewy/type/observe_spec.rb +0 -137
- data/spec/chewy/type/wrapper_spec.rb +0 -98
- data/spec/chewy/type_spec.rb +0 -55
- data/spec/support/mongoid.rb +0 -93
- data/spec/support/sequel.rb +0 -80
data/lib/chewy/index.rb
CHANGED
@@ -1,23 +1,48 @@
|
|
1
1
|
require 'chewy/search'
|
2
2
|
require 'chewy/index/actions'
|
3
|
+
require 'chewy/index/adapter/active_record'
|
4
|
+
require 'chewy/index/adapter/object'
|
3
5
|
require 'chewy/index/aliases'
|
6
|
+
require 'chewy/index/crutch'
|
7
|
+
require 'chewy/index/import'
|
8
|
+
require 'chewy/index/mapping'
|
9
|
+
require 'chewy/index/observe'
|
4
10
|
require 'chewy/index/settings'
|
5
11
|
require 'chewy/index/specification'
|
12
|
+
require 'chewy/index/syncer'
|
13
|
+
require 'chewy/index/witchcraft'
|
14
|
+
require 'chewy/index/wrapper'
|
6
15
|
|
7
16
|
module Chewy
|
8
17
|
class Index
|
18
|
+
IMPORT_OPTIONS_KEYS = %i[
|
19
|
+
batch_size bulk_size consistency direct_import journal
|
20
|
+
pipeline raw_import refresh replication
|
21
|
+
].freeze
|
22
|
+
|
9
23
|
include Search
|
10
24
|
include Actions
|
11
25
|
include Aliases
|
26
|
+
include Import
|
27
|
+
include Mapping
|
28
|
+
include Observe
|
29
|
+
include Crutch
|
30
|
+
include Witchcraft
|
31
|
+
include Wrapper
|
12
32
|
|
13
33
|
singleton_class.delegate :client, to: 'Chewy'
|
14
34
|
|
15
|
-
class_attribute :
|
16
|
-
self.
|
35
|
+
class_attribute :adapter
|
36
|
+
self.adapter = Chewy::Index::Adapter::Object.new(:default)
|
37
|
+
|
38
|
+
class_attribute :index_scope_defined
|
17
39
|
|
18
40
|
class_attribute :_settings
|
19
41
|
self._settings = Chewy::Index::Settings.new
|
20
42
|
|
43
|
+
class_attribute :_default_import_options
|
44
|
+
self._default_import_options = {}
|
45
|
+
|
21
46
|
class << self
|
22
47
|
# @overload index_name(suggest)
|
23
48
|
# If suggested name is passed, it is set up as the new base name for
|
@@ -47,7 +72,8 @@ module Chewy
|
|
47
72
|
# UsersIndex.index_name(prefix: '', suffix: '2017') # => 'users_2017'
|
48
73
|
#
|
49
74
|
# @param prefix [String] index name prefix, uses {.prefix} method by default
|
50
|
-
# @param suffix [String] index name suffix, used for creating several indexes for the same
|
75
|
+
# @param suffix [String] index name suffix, used for creating several indexes for the same
|
76
|
+
# alias during the zero-downtime reset
|
51
77
|
# @raise [UndefinedIndex] if the base name is blank
|
52
78
|
# @return [String] result index name
|
53
79
|
def index_name(suggest = nil, prefix: nil, suffix: nil)
|
@@ -55,7 +81,7 @@ module Chewy
|
|
55
81
|
@base_name = suggest.to_s.presence
|
56
82
|
else
|
57
83
|
[
|
58
|
-
prefix ||
|
84
|
+
prefix || self.prefix,
|
59
85
|
base_name,
|
60
86
|
suffix
|
61
87
|
].reject(&:blank?).join('_')
|
@@ -77,6 +103,7 @@ module Chewy
|
|
77
103
|
def base_name
|
78
104
|
@base_name ||= name.sub(/Index\z/, '').demodulize.underscore if name
|
79
105
|
raise UndefinedIndex if @base_name.blank?
|
106
|
+
|
80
107
|
@base_name
|
81
108
|
end
|
82
109
|
|
@@ -111,78 +138,30 @@ module Chewy
|
|
111
138
|
Chewy.configuration[:prefix]
|
112
139
|
end
|
113
140
|
|
114
|
-
# Defines
|
141
|
+
# Defines scope and options for the index. Arguments depends on adapter used. For
|
115
142
|
# ActiveRecord you can pass model or scope and options
|
116
143
|
#
|
117
144
|
# class CarsIndex < Chewy::Index
|
118
|
-
#
|
119
|
-
#
|
120
|
-
# end # defines VehiclesIndex::Car type
|
145
|
+
# index_scope Car
|
146
|
+
# ...
|
121
147
|
# end
|
122
148
|
#
|
123
|
-
#
|
124
|
-
#
|
125
|
-
# class VehiclesIndex < Chewy::Index
|
126
|
-
# define_type Vehicle.cars.includes(:manufacturer), name: 'cars' do
|
127
|
-
# ...
|
128
|
-
# end # defines VehiclesIndex::Cars type
|
129
|
-
#
|
130
|
-
# define_type Vehicle.motocycles.includes(:manufacturer), name: 'motocycles' do
|
131
|
-
# ...
|
132
|
-
# end # defines VehiclesIndex::Motocycles type
|
133
|
-
# end
|
134
|
-
#
|
135
|
-
# For plain objects:
|
149
|
+
# For plain objects you can completely omit this directive, unless you need to specify some options:
|
136
150
|
#
|
137
151
|
# class PlanesIndex < Chewy::Index
|
138
|
-
#
|
139
|
-
# ...
|
140
|
-
# end # defines PlanesIndex::Plane type
|
152
|
+
# ...
|
141
153
|
# end
|
142
154
|
#
|
143
155
|
# The main difference between using plain objects or ActiveRecord models for indexing
|
144
|
-
# is import. If you will call `CarsIndex
|
145
|
-
# automatically, while `PlanesIndex
|
156
|
+
# is import. If you will call `CarsIndex.import` - it will import all the cars
|
157
|
+
# automatically, while `PlanesIndex.import(my_planes)` requires import data to be
|
146
158
|
# passed.
|
147
159
|
#
|
148
|
-
def
|
149
|
-
|
150
|
-
self.type_hash = type_hash.merge(type_class.type_name => type_class)
|
151
|
-
end
|
160
|
+
def index_scope(target, options = {})
|
161
|
+
raise 'Index scope is already defined' if index_scope_defined?
|
152
162
|
|
153
|
-
|
154
|
-
|
155
|
-
#
|
156
|
-
# UsersIndex.types # => [UsersIndex::Admin, UsersIndex::Manager, UsersIndex::User]
|
157
|
-
#
|
158
|
-
# If arguments are passed it treats like a part of chainable query DSL and
|
159
|
-
# adds types array for index to select.
|
160
|
-
#
|
161
|
-
# UsersIndex.filters { name =~ 'ro' }.types(:admin, :manager)
|
162
|
-
# UsersIndex.types(:admin, :manager).filters { name =~ 'ro' } # the same as the first example
|
163
|
-
#
|
164
|
-
def types(*args)
|
165
|
-
if args.present?
|
166
|
-
all.types(*args)
|
167
|
-
else
|
168
|
-
type_hash.values
|
169
|
-
end
|
170
|
-
end
|
171
|
-
|
172
|
-
# Returns defined types names:
|
173
|
-
#
|
174
|
-
# UsersIndex.type_names # => ['admin', 'manager', 'user']
|
175
|
-
#
|
176
|
-
def type_names
|
177
|
-
type_hash.keys
|
178
|
-
end
|
179
|
-
|
180
|
-
# Returns named type:
|
181
|
-
#
|
182
|
-
# UserIndex.type('admin') # => UsersIndex::Admin
|
183
|
-
#
|
184
|
-
def type(type_name)
|
185
|
-
type_hash.fetch(type_name) { raise UndefinedType, "Unknown type in #{name}: #{type_name}" }
|
163
|
+
self.adapter = Chewy.adapters.find { |klass| klass.accepts?(target) }.new(target, **options)
|
164
|
+
self.index_scope_defined = true
|
186
165
|
end
|
187
166
|
|
188
167
|
# Used as a part of index definition DSL. Defines settings:
|
@@ -219,7 +198,7 @@ module Chewy
|
|
219
198
|
end
|
220
199
|
|
221
200
|
def mappings_hash
|
222
|
-
mappings =
|
201
|
+
mappings = root.mappings_hash
|
223
202
|
mappings.present? ? {mappings: mappings} : {}
|
224
203
|
end
|
225
204
|
|
@@ -232,44 +211,15 @@ module Chewy
|
|
232
211
|
[settings_hash, mappings_hash].inject(:merge)
|
233
212
|
end
|
234
213
|
|
235
|
-
def index_params
|
236
|
-
ActiveSupport::Deprecation.warn '`Chewy::Index.index_params` is deprecated and will be removed soon, use `Chewy::Index.specification_hash`'
|
237
|
-
specification_hash
|
238
|
-
end
|
239
|
-
|
240
214
|
# @see Chewy::Index::Specification
|
241
215
|
# @return [Chewy::Index::Specification] a specification object instance for this particular index
|
242
216
|
def specification
|
243
217
|
@specification ||= Specification.new(self)
|
244
218
|
end
|
245
219
|
|
246
|
-
def
|
247
|
-
|
248
|
-
|
249
|
-
end
|
250
|
-
|
251
|
-
# Handling old default_prefix if it is not defined.
|
252
|
-
def method_missing(name, *args, &block) # rubocop:disable Style/MethodMissing
|
253
|
-
if name == :default_prefix
|
254
|
-
ActiveSupport::Deprecation.warn '`Chewy::Index.default_prefix` is deprecated and will be removed soon, use `Chewy::Index.prefix` instead'
|
255
|
-
prefix
|
256
|
-
else
|
257
|
-
super
|
258
|
-
end
|
259
|
-
end
|
260
|
-
|
261
|
-
def prefix_with_deprecation
|
262
|
-
if respond_to?(:default_prefix)
|
263
|
-
ActiveSupport::Deprecation.warn '`Chewy::Index.default_prefix` is deprecated and will be removed soon, define `Chewy::Index.prefix` method instead'
|
264
|
-
default_prefix
|
265
|
-
else
|
266
|
-
prefix
|
267
|
-
end
|
268
|
-
end
|
269
|
-
|
270
|
-
def build_index_name(*args)
|
271
|
-
ActiveSupport::Deprecation.warn '`Chewy::Index.build_index_name` is deprecated and will be removed soon, use `Chewy::Index.index_name` instead'
|
272
|
-
index_name(args.extract_options!)
|
220
|
+
def default_import_options(params)
|
221
|
+
params.assert_valid_keys(IMPORT_OPTIONS_KEYS)
|
222
|
+
self._default_import_options = _default_import_options.merge(params)
|
273
223
|
end
|
274
224
|
end
|
275
225
|
end
|
data/lib/chewy/journal.rb
CHANGED
@@ -2,33 +2,36 @@ module Chewy
|
|
2
2
|
# A class to perform journal-related actions for the specified indexes/types.
|
3
3
|
#
|
4
4
|
# @example
|
5
|
-
# journal = Chewy::Journal.new('places
|
5
|
+
# journal = Chewy::Journal.new('places', UsersIndex)
|
6
6
|
# journal.apply(20.minutes.ago)
|
7
7
|
# journal.clean
|
8
8
|
#
|
9
9
|
class Journal
|
10
|
-
# @param only [Array<String, Chewy::Index
|
10
|
+
# @param only [Array<String, Chewy::Index>] indexes or string references to perform actions on
|
11
11
|
def initialize(*only)
|
12
12
|
@only = only
|
13
13
|
end
|
14
14
|
|
15
15
|
# Applies all changes that were done since the specified time to the
|
16
|
-
# specified indexes
|
16
|
+
# specified indexes.
|
17
17
|
#
|
18
18
|
# @param since_time [Time, DateTime] timestamp from which changes will be applied
|
19
|
-
# @param
|
19
|
+
# @param fetch_limit [Int] amount of entries to be fetched on each cycle
|
20
20
|
# @return [Integer] the amount of journal entries found
|
21
|
-
def apply(since_time,
|
21
|
+
def apply(since_time, fetch_limit: 10, **import_options)
|
22
22
|
stage = 1
|
23
23
|
since_time -= 1
|
24
24
|
count = 0
|
25
|
-
|
26
|
-
|
25
|
+
|
26
|
+
total_count = entries(since_time, fetch_limit).total_count
|
27
|
+
|
28
|
+
while count < total_count
|
29
|
+
entries = entries(since_time, fetch_limit).to_a.presence or break
|
27
30
|
count += entries.size
|
28
31
|
groups = reference_groups(entries)
|
29
32
|
ActiveSupport::Notifications.instrument 'apply_journal.chewy', stage: stage, groups: groups
|
30
|
-
groups.each do |
|
31
|
-
|
33
|
+
groups.each do |index, references|
|
34
|
+
index.import(references, import_options.merge(journal: false))
|
32
35
|
end
|
33
36
|
stage += 1
|
34
37
|
since_time = entries.map(&:created_at).max
|
@@ -40,16 +43,24 @@ module Chewy
|
|
40
43
|
#
|
41
44
|
# @param until_time [Time, DateTime] time to clean up until it
|
42
45
|
# @return [Hash] delete_by_query ES API call result
|
43
|
-
def clean(until_time = nil)
|
44
|
-
Chewy::Stash::Journal.clean(
|
46
|
+
def clean(until_time = nil, delete_by_query_options: {})
|
47
|
+
Chewy::Stash::Journal.clean(
|
48
|
+
until_time,
|
49
|
+
only: @only,
|
50
|
+
delete_by_query_options: delete_by_query_options.merge(refresh: false)
|
51
|
+
)
|
45
52
|
end
|
46
53
|
|
47
54
|
private
|
48
55
|
|
56
|
+
def entries(since_time, fetch_limit)
|
57
|
+
Chewy::Stash::Journal.entries(since_time, only: @only).order(:created_at).limit(fetch_limit)
|
58
|
+
end
|
59
|
+
|
49
60
|
def reference_groups(entries)
|
50
|
-
entries.group_by(&:
|
51
|
-
|
52
|
-
|
61
|
+
entries.group_by(&:index_name)
|
62
|
+
.transform_keys { |index_name| Chewy.derive_name(index_name) }
|
63
|
+
.transform_values { |grouped_entries| grouped_entries.map(&:references).inject(:|) }
|
53
64
|
end
|
54
65
|
end
|
55
66
|
end
|
@@ -6,7 +6,7 @@ module Chewy
|
|
6
6
|
extend ActiveSupport::Concern
|
7
7
|
|
8
8
|
# Assert that an index *changes* during a block.
|
9
|
-
# @param index [Chewy::
|
9
|
+
# @param index [Chewy::Index] the index to watch, eg EntitiesIndex.
|
10
10
|
# @param strategy [Symbol] the Chewy strategy to use around the block. See Chewy docs.
|
11
11
|
# @param bypass_actual_index [true, false]
|
12
12
|
# True to preempt the http call to Elastic, false otherwise.
|
@@ -14,11 +14,11 @@ module Chewy
|
|
14
14
|
#
|
15
15
|
# @return [SearchIndexReceiver] for optional further assertions on the nature of the index changes.
|
16
16
|
#
|
17
|
-
def assert_indexes(index, strategy: :atomic, bypass_actual_index: true)
|
18
|
-
|
17
|
+
def assert_indexes(index, strategy: :atomic, bypass_actual_index: true, &block)
|
18
|
+
index_class = Chewy.derive_name index
|
19
19
|
receiver = SearchIndexReceiver.new
|
20
20
|
|
21
|
-
bulk_method =
|
21
|
+
bulk_method = index_class.method :bulk
|
22
22
|
# Manually mocking #bulk because we need to properly capture `self`
|
23
23
|
bulk_mock = lambda do |*bulk_args|
|
24
24
|
receiver.catch bulk_args, self
|
@@ -28,13 +28,11 @@ module Chewy
|
|
28
28
|
{}
|
29
29
|
end
|
30
30
|
|
31
|
-
|
31
|
+
index_class.define_singleton_method :bulk, bulk_mock
|
32
32
|
|
33
|
-
Chewy.strategy(strategy)
|
34
|
-
yield
|
35
|
-
end
|
33
|
+
Chewy.strategy(strategy, &block)
|
36
34
|
|
37
|
-
|
35
|
+
index_class.define_singleton_method :bulk, bulk_method
|
38
36
|
|
39
37
|
assert_includes receiver.updated_indexes, index, "Expected #{index} to be updated but it wasn't"
|
40
38
|
|
@@ -44,10 +42,85 @@ module Chewy
|
|
44
42
|
# Run indexing for the database changes during the block provided.
|
45
43
|
# By default, indexing is run at the end of the block.
|
46
44
|
# @param strategy [Symbol] the Chewy index update strategy see Chewy docs.
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
45
|
+
#
|
46
|
+
def run_indexing(strategy: :atomic, &block)
|
47
|
+
Chewy.strategy strategy, &block
|
48
|
+
end
|
49
|
+
|
50
|
+
# Mock Elasticsearch response
|
51
|
+
# Simple usage - just pass index, expected raw response
|
52
|
+
# and block with the query.
|
53
|
+
#
|
54
|
+
# @param index [Chewy::Index] the index to watch, eg EntitiesIndex.
|
55
|
+
# @param raw_response [Hash] hash with response.
|
56
|
+
#
|
57
|
+
def mock_elasticsearch_response(index, raw_response)
|
58
|
+
mocked_request = Chewy::Search::Request.new(index)
|
59
|
+
|
60
|
+
original_new = Chewy::Search::Request.method(:new)
|
61
|
+
|
62
|
+
Chewy::Search::Request.define_singleton_method(:new) { |*_args| mocked_request }
|
63
|
+
|
64
|
+
original_perform = mocked_request.method(:perform)
|
65
|
+
mocked_request.define_singleton_method(:perform) { raw_response }
|
66
|
+
|
67
|
+
yield
|
68
|
+
ensure
|
69
|
+
mocked_request.define_singleton_method(:perform, original_perform)
|
70
|
+
Chewy::Search::Request.define_singleton_method(:new, original_new)
|
71
|
+
end
|
72
|
+
|
73
|
+
# Mock Elasticsearch response with defined sources
|
74
|
+
# Simple usage - just pass index, expected sources
|
75
|
+
# and block with the query.
|
76
|
+
#
|
77
|
+
# @param index [Chewy::Index] the index to watch, eg EntitiesIndex.
|
78
|
+
# @param hits [Hash] hash with sources.
|
79
|
+
#
|
80
|
+
def mock_elasticsearch_response_sources(index, hits, &block)
|
81
|
+
raw_response = {
|
82
|
+
'took' => 4,
|
83
|
+
'timed_out' => false,
|
84
|
+
'_shards' => {
|
85
|
+
'total' => 1,
|
86
|
+
'successful' => 1,
|
87
|
+
'skipped' => 0,
|
88
|
+
'failed' => 0
|
89
|
+
},
|
90
|
+
'hits' => {
|
91
|
+
'total' => {
|
92
|
+
'value' => hits.count,
|
93
|
+
'relation' => 'eq'
|
94
|
+
},
|
95
|
+
'max_score' => 1.0,
|
96
|
+
'hits' => hits.each_with_index.map do |hit, i|
|
97
|
+
{
|
98
|
+
'_index' => index.index_name,
|
99
|
+
'_type' => '_doc',
|
100
|
+
'_id' => (i + 1).to_s,
|
101
|
+
'_score' => 3.14,
|
102
|
+
'_source' => hit
|
103
|
+
}
|
104
|
+
end
|
105
|
+
}
|
106
|
+
}
|
107
|
+
|
108
|
+
mock_elasticsearch_response(index, raw_response, &block)
|
109
|
+
end
|
110
|
+
|
111
|
+
# Check the assertion that actual Elasticsearch query is rendered
|
112
|
+
# to the expected query
|
113
|
+
#
|
114
|
+
# @param query [::Query] the actual Elasticsearch query.
|
115
|
+
# @param expected_query [Hash] expected query.
|
116
|
+
#
|
117
|
+
# @return [Boolean]
|
118
|
+
# True - in the case when actual Elasticsearch query is rendered to the expected query.
|
119
|
+
# False - in the opposite case.
|
120
|
+
#
|
121
|
+
def assert_elasticsearch_query(query, expected_query)
|
122
|
+
actual_query = query.render
|
123
|
+
assert_equal expected_query, actual_query, "got #{actual_query.inspect} instead of expected query."
|
51
124
|
end
|
52
125
|
|
53
126
|
module ClassMethods
|
@@ -1,80 +1,76 @@
|
|
1
1
|
# Test helper class to provide minitest hooks for Chewy::Index testing.
|
2
2
|
#
|
3
3
|
# @note Intended to be used in conjunction with a test helper which mocks over the #bulk
|
4
|
-
# method on a {Chewy::
|
4
|
+
# method on a {Chewy::Index} class. (See {Chewy::Minitest::Helpers})
|
5
5
|
#
|
6
|
-
# The class will capture the data from the *param on the Chewy::
|
6
|
+
# The class will capture the data from the *param on the Chewy::Index.bulk method and
|
7
7
|
# aggregate the data for test analysis.
|
8
8
|
class SearchIndexReceiver
|
9
9
|
def initialize
|
10
10
|
@mutations = {}
|
11
11
|
end
|
12
12
|
|
13
|
-
# @param bulk_params [Hash] the bulk_params that should be sent to the Chewy::
|
14
|
-
# @param
|
15
|
-
def catch(bulk_params,
|
13
|
+
# @param bulk_params [Hash] the bulk_params that should be sent to the Chewy::Index.bulk method.
|
14
|
+
# @param index [Chewy::Index] the index executing this query.
|
15
|
+
def catch(bulk_params, index)
|
16
16
|
Array.wrap(bulk_params).map { |y| y[:body] }.flatten.each do |update|
|
17
17
|
if update[:delete]
|
18
|
-
mutation_for(
|
18
|
+
mutation_for(index).deletes << update[:delete][:_id]
|
19
19
|
elsif update[:index]
|
20
|
-
mutation_for(
|
20
|
+
mutation_for(index).indexes << update[:index]
|
21
21
|
end
|
22
22
|
end
|
23
23
|
end
|
24
24
|
|
25
|
-
# @param index [Chewy::Index] return only index requests to the specified {Chewy::
|
25
|
+
# @param index [Chewy::Index] return only index requests to the specified {Chewy::Index} index.
|
26
26
|
# @return [Hash] the index changes captured by the mock.
|
27
27
|
def indexes_for(index = nil)
|
28
28
|
if index
|
29
29
|
mutation_for(index).indexes
|
30
30
|
else
|
31
|
-
|
32
|
-
@mutations.map { |a, b| [a, b.indexes] }
|
33
|
-
]
|
31
|
+
@mutations.transform_values(&:indexes)
|
34
32
|
end
|
35
33
|
end
|
36
34
|
alias_method :indexes, :indexes_for
|
37
35
|
|
38
|
-
# @param index [Chewy::Index] return only delete requests to the specified {Chewy::
|
36
|
+
# @param index [Chewy::Index] return only delete requests to the specified {Chewy::Index} index.
|
39
37
|
# @return [Hash] the index deletes captured by the mock.
|
40
38
|
def deletes_for(index = nil)
|
41
39
|
if index
|
42
40
|
mutation_for(index).deletes
|
43
41
|
else
|
44
|
-
|
45
|
-
@mutations.map { |a, b| [a, b.deletes] }
|
46
|
-
]
|
42
|
+
@mutations.transform_values(&:deletes)
|
47
43
|
end
|
48
44
|
end
|
49
45
|
alias_method :deletes, :deletes_for
|
50
46
|
|
51
47
|
# Check to see if a given object has been indexed.
|
52
48
|
# @param obj [#id] obj the object to look for.
|
53
|
-
# @param
|
49
|
+
# @param index [Chewy::Index] what index the object should be indexed in.
|
54
50
|
# @return [true, false] if the object was indexed.
|
55
|
-
def indexed?(obj,
|
56
|
-
indexes_for(
|
51
|
+
def indexed?(obj, index)
|
52
|
+
indexes_for(index).map { |i| i[:_id] }.include? obj.id
|
57
53
|
end
|
58
54
|
|
59
55
|
# Check to see if a given object has been deleted.
|
60
56
|
# @param obj [#id] obj the object to look for.
|
61
|
-
# @param
|
57
|
+
# @param index [Chewy::Index] what index the object should have been deleted from.
|
62
58
|
# @return [true, false] if the object was deleted.
|
63
|
-
def deleted?(obj,
|
64
|
-
deletes_for(
|
59
|
+
def deleted?(obj, index)
|
60
|
+
deletes_for(index).include? obj.id
|
65
61
|
end
|
66
62
|
|
67
|
-
# @return [Array<Chewy::
|
63
|
+
# @return [Array<Chewy::Index>] a list of indexes changed.
|
68
64
|
def updated_indexes
|
69
65
|
@mutations.keys
|
70
66
|
end
|
71
67
|
|
72
68
|
private
|
73
69
|
|
74
|
-
# Get the mutation object for a given
|
75
|
-
# @param
|
70
|
+
# Get the mutation object for a given index.
|
71
|
+
# @param index [Chewy::Index] the index to fetch.
|
76
72
|
# @return [#indexes, #deletes] an object with a list of indexes and a list of deletes.
|
77
|
-
def mutation_for(
|
78
|
-
@mutations[
|
73
|
+
def mutation_for(index)
|
74
|
+
@mutations[index] ||= OpenStruct.new(indexes: [], deletes: [])
|
79
75
|
end
|
80
76
|
end
|
@@ -0,0 +1,62 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Chewy
|
4
|
+
# `Chewy::MultiSearch` provides an interface for executing multiple
|
5
|
+
# queries via the Elasticsearch Multi Search API. When a MultiSearch
|
6
|
+
# is performed it wraps the responses from Elasticsearch and assigns
|
7
|
+
# them to the appropriate queries.
|
8
|
+
class MultiSearch
|
9
|
+
attr_reader :queries
|
10
|
+
|
11
|
+
# Instantiate a new MultiSearch instance.
|
12
|
+
#
|
13
|
+
# @param queries [Array<Chewy::Search::Request>]
|
14
|
+
# @option [Elasticsearch::Transport::Client] :client (Chewy.client)
|
15
|
+
# The Elasticsearch client that should be used for issuing requests.
|
16
|
+
def initialize(queries, client: Chewy.client)
|
17
|
+
@client = client
|
18
|
+
@queries = Array(queries)
|
19
|
+
end
|
20
|
+
|
21
|
+
# Adds a query to be performed by the MultiSearch
|
22
|
+
#
|
23
|
+
# @param query [Chewy::Search::Request]
|
24
|
+
def add_query(query)
|
25
|
+
@queries << query
|
26
|
+
end
|
27
|
+
|
28
|
+
# Performs any unperformed queries and returns the responses for all queries.
|
29
|
+
#
|
30
|
+
# @return [Array<Chewy::Search::Response>]
|
31
|
+
def responses
|
32
|
+
perform
|
33
|
+
queries.map(&:response)
|
34
|
+
end
|
35
|
+
|
36
|
+
# Performs any unperformed queries.
|
37
|
+
def perform
|
38
|
+
unperformed_queries = queries.reject(&:performed?)
|
39
|
+
return if unperformed_queries.empty?
|
40
|
+
|
41
|
+
responses = msearch(unperformed_queries)['responses']
|
42
|
+
unperformed_queries.zip(responses).map { |query, response| query.response = response }
|
43
|
+
end
|
44
|
+
|
45
|
+
private
|
46
|
+
|
47
|
+
attr_reader :client
|
48
|
+
|
49
|
+
def msearch(queries_to_search)
|
50
|
+
body = queries_to_search.flat_map do |query|
|
51
|
+
rendered = query.render
|
52
|
+
[rendered.except(:body), rendered[:body]]
|
53
|
+
end
|
54
|
+
|
55
|
+
client.msearch(body: body)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
def self.msearch(queries)
|
60
|
+
Chewy::MultiSearch.new(queries)
|
61
|
+
end
|
62
|
+
end
|
data/lib/chewy/railtie.rb
CHANGED
@@ -14,7 +14,9 @@ module Chewy
|
|
14
14
|
if Rails.application.config.respond_to?(:assets) && env['PATH_INFO'].start_with?(Rails.application.config.assets.prefix)
|
15
15
|
@app.call(env)
|
16
16
|
else
|
17
|
-
|
17
|
+
if Chewy.logger && @request_strategy != Chewy.request_strategy
|
18
|
+
Chewy.logger.info("Chewy request strategy is `#{Chewy.request_strategy}`")
|
19
|
+
end
|
18
20
|
@request_strategy = Chewy.request_strategy
|
19
21
|
Chewy.strategy(Chewy.request_strategy) { @app.call(env) }
|
20
22
|
end
|
@@ -22,17 +24,6 @@ module Chewy
|
|
22
24
|
end
|
23
25
|
|
24
26
|
module MigrationStrategy
|
25
|
-
extend ActiveSupport::Concern
|
26
|
-
included do
|
27
|
-
alias_method_chain :migrate, :chewy
|
28
|
-
end
|
29
|
-
|
30
|
-
def migrate_with_chewy(*args)
|
31
|
-
Chewy.strategy(:bypass) { migrate_without_chewy(*args) }
|
32
|
-
end
|
33
|
-
end
|
34
|
-
|
35
|
-
module Rails5MigrationStrategy
|
36
27
|
def migrate(*args)
|
37
28
|
Chewy.strategy(:bypass) { super }
|
38
29
|
end
|
@@ -46,7 +37,7 @@ module Chewy
|
|
46
37
|
if app.sandbox?
|
47
38
|
Chewy.strategy(:bypass)
|
48
39
|
else
|
49
|
-
Chewy.strategy(
|
40
|
+
Chewy.strategy(Chewy.console_strategy)
|
50
41
|
end
|
51
42
|
puts "Chewy console strategy is `#{Chewy.strategy.current.name}`"
|
52
43
|
end
|
@@ -57,13 +48,8 @@ module Chewy
|
|
57
48
|
|
58
49
|
initializer 'chewy.migration_strategy' do
|
59
50
|
ActiveSupport.on_load(:active_record) do
|
60
|
-
|
61
|
-
|
62
|
-
ActiveRecord::Migrator.prepend(Rails5MigrationStrategy) if defined? ActiveRecord::Migrator
|
63
|
-
else
|
64
|
-
ActiveRecord::Migration.send(:include, MigrationStrategy)
|
65
|
-
ActiveRecord::Migrator.send(:include, MigrationStrategy) if defined? ActiveRecord::Migrator
|
66
|
-
end
|
51
|
+
ActiveRecord::Migration.prepend(MigrationStrategy)
|
52
|
+
ActiveRecord::Migrator.prepend(MigrationStrategy) if defined? ActiveRecord::Migrator
|
67
53
|
end
|
68
54
|
end
|
69
55
|
|