chewy 7.1.0 → 7.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ruby.yml +58 -0
- data/.rubocop.yml +13 -8
- data/.rubocop_todo.yml +110 -22
- data/CHANGELOG.md +53 -0
- data/Gemfile +0 -7
- data/Guardfile +3 -1
- data/README.md +282 -245
- data/chewy.gemspec +3 -5
- data/gemfiles/rails.5.2.activerecord.gemfile +8 -14
- data/gemfiles/rails.6.0.activerecord.gemfile +8 -14
- data/gemfiles/rails.6.1.activerecord.gemfile +8 -14
- data/lib/chewy.rb +21 -75
- data/lib/chewy/config.rb +40 -40
- data/lib/chewy/errors.rb +0 -12
- data/lib/chewy/fields/base.rb +11 -1
- data/lib/chewy/fields/root.rb +3 -4
- data/lib/chewy/index.rb +46 -87
- data/lib/chewy/index/actions.rb +51 -32
- data/lib/chewy/{type → index}/adapter/active_record.rb +12 -3
- data/lib/chewy/{type → index}/adapter/base.rb +2 -3
- data/lib/chewy/{type → index}/adapter/object.rb +27 -31
- data/lib/chewy/{type → index}/adapter/orm.rb +11 -14
- data/lib/chewy/{type → index}/crutch.rb +5 -5
- data/lib/chewy/{type → index}/import.rb +36 -27
- data/lib/chewy/{type → index}/import/bulk_builder.rb +15 -13
- data/lib/chewy/{type → index}/import/bulk_request.rb +6 -6
- data/lib/chewy/{type → index}/import/journal_builder.rb +10 -10
- data/lib/chewy/{type → index}/import/routine.rb +15 -14
- data/lib/chewy/{type → index}/mapping.rb +26 -31
- data/lib/chewy/{type → index}/observe.rb +9 -19
- data/lib/chewy/index/specification.rb +1 -0
- data/lib/chewy/{type → index}/syncer.rb +60 -57
- data/lib/chewy/{type → index}/witchcraft.rb +11 -7
- data/lib/chewy/{type → index}/wrapper.rb +2 -2
- data/lib/chewy/journal.rb +8 -8
- data/lib/chewy/minitest/helpers.rb +9 -13
- data/lib/chewy/minitest/search_index_receiver.rb +22 -26
- data/lib/chewy/railtie.rb +4 -2
- data/lib/chewy/rake_helper.rb +82 -107
- data/lib/chewy/rspec/update_index.rb +47 -43
- data/lib/chewy/search.rb +4 -17
- data/lib/chewy/search/loader.rb +18 -30
- data/lib/chewy/search/parameters.rb +4 -2
- data/lib/chewy/search/parameters/concerns/query_storage.rb +2 -2
- data/lib/chewy/search/parameters/source.rb +5 -1
- data/lib/chewy/search/query_proxy.rb +9 -2
- data/lib/chewy/search/request.rb +82 -86
- data/lib/chewy/search/response.rb +4 -4
- data/lib/chewy/search/scoping.rb +6 -7
- data/lib/chewy/search/scrolling.rb +11 -11
- data/lib/chewy/stash.rb +14 -22
- data/lib/chewy/strategy.rb +3 -19
- data/lib/chewy/strategy/sidekiq.rb +1 -0
- data/lib/chewy/version.rb +1 -1
- data/lib/generators/chewy/install_generator.rb +1 -1
- data/lib/tasks/chewy.rake +10 -22
- data/migration_guide.md +14 -0
- data/spec/chewy/config_spec.rb +14 -39
- data/spec/chewy/fields/base_spec.rb +412 -148
- data/spec/chewy/fields/root_spec.rb +16 -24
- data/spec/chewy/fields/time_fields_spec.rb +5 -5
- data/spec/chewy/index/actions_spec.rb +270 -24
- data/spec/chewy/{type → index}/adapter/active_record_spec.rb +68 -40
- data/spec/chewy/{type → index}/adapter/object_spec.rb +21 -6
- data/spec/chewy/{type → index}/import/bulk_builder_spec.rb +23 -31
- data/spec/chewy/{type → index}/import/bulk_request_spec.rb +5 -6
- data/spec/chewy/{type → index}/import/journal_builder_spec.rb +9 -15
- data/spec/chewy/{type → index}/import/routine_spec.rb +16 -16
- data/spec/chewy/{type → index}/import_spec.rb +102 -98
- data/spec/chewy/{type → index}/mapping_spec.rb +46 -54
- data/spec/chewy/index/observe_spec.rb +116 -0
- data/spec/chewy/index/settings_spec.rb +3 -1
- data/spec/chewy/index/specification_spec.rb +7 -17
- data/spec/chewy/{type → index}/syncer_spec.rb +14 -19
- data/spec/chewy/{type → index}/witchcraft_spec.rb +20 -22
- data/spec/chewy/index/wrapper_spec.rb +100 -0
- data/spec/chewy/index_spec.rb +59 -102
- data/spec/chewy/journal_spec.rb +9 -22
- data/spec/chewy/minitest/helpers_spec.rb +13 -15
- data/spec/chewy/minitest/search_index_receiver_spec.rb +22 -26
- data/spec/chewy/multi_search_spec.rb +4 -5
- data/spec/chewy/rake_helper_spec.rb +145 -55
- data/spec/chewy/rspec/update_index_spec.rb +74 -71
- data/spec/chewy/search/loader_spec.rb +19 -37
- data/spec/chewy/search/pagination/kaminari_examples.rb +3 -5
- data/spec/chewy/search/pagination/kaminari_spec.rb +1 -1
- data/spec/chewy/search/parameters/indices_spec.rb +2 -8
- data/spec/chewy/search/parameters/order_spec.rb +1 -1
- data/spec/chewy/search/parameters/query_storage_examples.rb +67 -21
- data/spec/chewy/search/parameters/search_after_spec.rb +4 -1
- data/spec/chewy/search/parameters/source_spec.rb +8 -2
- data/spec/chewy/search/parameters_spec.rb +12 -3
- data/spec/chewy/search/query_proxy_spec.rb +68 -17
- data/spec/chewy/search/request_spec.rb +222 -74
- data/spec/chewy/search/response_spec.rb +12 -12
- data/spec/chewy/search/scrolling_spec.rb +7 -9
- data/spec/chewy/search_spec.rb +32 -35
- data/spec/chewy/stash_spec.rb +9 -21
- data/spec/chewy/strategy/active_job_spec.rb +8 -8
- data/spec/chewy/strategy/atomic_spec.rb +9 -10
- data/spec/chewy/strategy/sidekiq_spec.rb +8 -8
- data/spec/chewy/strategy_spec.rb +19 -15
- data/spec/chewy_spec.rb +14 -100
- data/spec/spec_helper.rb +2 -21
- data/spec/support/active_record.rb +15 -5
- metadata +44 -103
- data/.circleci/config.yml +0 -214
- data/Appraisals +0 -81
- data/gemfiles/rails.5.2.mongoid.6.4.gemfile +0 -17
- data/gemfiles/sequel.4.45.gemfile +0 -11
- data/lib/chewy/search/pagination/will_paginate.rb +0 -43
- data/lib/chewy/strategy/resque.rb +0 -27
- data/lib/chewy/strategy/shoryuken.rb +0 -40
- data/lib/chewy/type.rb +0 -120
- data/lib/chewy/type/actions.rb +0 -43
- data/lib/chewy/type/adapter/mongoid.rb +0 -67
- data/lib/chewy/type/adapter/sequel.rb +0 -93
- data/lib/sequel/plugins/chewy_observe.rb +0 -63
- data/spec/chewy/search/pagination/will_paginate_examples.rb +0 -63
- data/spec/chewy/search/pagination/will_paginate_spec.rb +0 -23
- data/spec/chewy/strategy/resque_spec.rb +0 -46
- data/spec/chewy/strategy/shoryuken_spec.rb +0 -70
- data/spec/chewy/type/actions_spec.rb +0 -50
- data/spec/chewy/type/adapter/mongoid_spec.rb +0 -372
- data/spec/chewy/type/adapter/sequel_spec.rb +0 -472
- data/spec/chewy/type/observe_spec.rb +0 -137
- data/spec/chewy/type/wrapper_spec.rb +0 -100
- data/spec/chewy/type_spec.rb +0 -55
- data/spec/support/mongoid.rb +0 -93
- data/spec/support/sequel.rb +0 -80
data/lib/chewy/index.rb
CHANGED
@@ -1,23 +1,48 @@
|
|
1
1
|
require 'chewy/search'
|
2
2
|
require 'chewy/index/actions'
|
3
|
+
require 'chewy/index/adapter/active_record'
|
4
|
+
require 'chewy/index/adapter/object'
|
3
5
|
require 'chewy/index/aliases'
|
6
|
+
require 'chewy/index/crutch'
|
7
|
+
require 'chewy/index/import'
|
8
|
+
require 'chewy/index/mapping'
|
9
|
+
require 'chewy/index/observe'
|
4
10
|
require 'chewy/index/settings'
|
5
11
|
require 'chewy/index/specification'
|
12
|
+
require 'chewy/index/syncer'
|
13
|
+
require 'chewy/index/witchcraft'
|
14
|
+
require 'chewy/index/wrapper'
|
6
15
|
|
7
16
|
module Chewy
|
8
17
|
class Index
|
18
|
+
IMPORT_OPTIONS_KEYS = %i[
|
19
|
+
batch_size bulk_size consistency direct_import journal
|
20
|
+
pipeline raw_import refresh replication
|
21
|
+
].freeze
|
22
|
+
|
9
23
|
include Search
|
10
24
|
include Actions
|
11
25
|
include Aliases
|
26
|
+
include Import
|
27
|
+
include Mapping
|
28
|
+
include Observe
|
29
|
+
include Crutch
|
30
|
+
include Witchcraft
|
31
|
+
include Wrapper
|
12
32
|
|
13
33
|
singleton_class.delegate :client, to: 'Chewy'
|
14
34
|
|
15
|
-
class_attribute :
|
16
|
-
self.
|
35
|
+
class_attribute :adapter
|
36
|
+
self.adapter = Chewy::Index::Adapter::Object.new(:default)
|
37
|
+
|
38
|
+
class_attribute :index_scope_defined
|
17
39
|
|
18
40
|
class_attribute :_settings
|
19
41
|
self._settings = Chewy::Index::Settings.new
|
20
42
|
|
43
|
+
class_attribute :_default_import_options
|
44
|
+
self._default_import_options = {}
|
45
|
+
|
21
46
|
class << self
|
22
47
|
# @overload index_name(suggest)
|
23
48
|
# If suggested name is passed, it is set up as the new base name for
|
@@ -47,7 +72,8 @@ module Chewy
|
|
47
72
|
# UsersIndex.index_name(prefix: '', suffix: '2017') # => 'users_2017'
|
48
73
|
#
|
49
74
|
# @param prefix [String] index name prefix, uses {.prefix} method by default
|
50
|
-
# @param suffix [String] index name suffix, used for creating several indexes for the same
|
75
|
+
# @param suffix [String] index name suffix, used for creating several indexes for the same
|
76
|
+
# alias during the zero-downtime reset
|
51
77
|
# @raise [UndefinedIndex] if the base name is blank
|
52
78
|
# @return [String] result index name
|
53
79
|
def index_name(suggest = nil, prefix: nil, suffix: nil)
|
@@ -55,7 +81,7 @@ module Chewy
|
|
55
81
|
@base_name = suggest.to_s.presence
|
56
82
|
else
|
57
83
|
[
|
58
|
-
prefix ||
|
84
|
+
prefix || self.prefix,
|
59
85
|
base_name,
|
60
86
|
suffix
|
61
87
|
].reject(&:blank?).join('_')
|
@@ -77,6 +103,7 @@ module Chewy
|
|
77
103
|
def base_name
|
78
104
|
@base_name ||= name.sub(/Index\z/, '').demodulize.underscore if name
|
79
105
|
raise UndefinedIndex if @base_name.blank?
|
106
|
+
|
80
107
|
@base_name
|
81
108
|
end
|
82
109
|
|
@@ -111,69 +138,30 @@ module Chewy
|
|
111
138
|
Chewy.configuration[:prefix]
|
112
139
|
end
|
113
140
|
|
114
|
-
# Defines
|
141
|
+
# Defines scope and options for the index. Arguments depends on adapter used. For
|
115
142
|
# ActiveRecord you can pass model or scope and options
|
116
143
|
#
|
117
144
|
# class CarsIndex < Chewy::Index
|
118
|
-
#
|
119
|
-
#
|
120
|
-
# end # defines VehiclesIndex::Car type
|
121
|
-
# end
|
122
|
-
#
|
123
|
-
# Type name might be passed in complicated cases:
|
124
|
-
#
|
125
|
-
# class VehiclesIndex < Chewy::Index
|
126
|
-
# define_type Vehicle.cars.includes(:manufacturer), name: 'cars' do
|
127
|
-
# ...
|
128
|
-
# end # defines VehiclesIndex::Cars type
|
129
|
-
#
|
130
|
-
# define_type Vehicle.motocycles.includes(:manufacturer), name: 'motocycles' do
|
131
|
-
# ...
|
132
|
-
# end # defines VehiclesIndex::Motocycles type
|
145
|
+
# index_scope Car
|
146
|
+
# ...
|
133
147
|
# end
|
134
148
|
#
|
135
|
-
# For plain objects:
|
149
|
+
# For plain objects you can completely omit this directive, unless you need to specify some options:
|
136
150
|
#
|
137
151
|
# class PlanesIndex < Chewy::Index
|
138
|
-
#
|
139
|
-
# ...
|
140
|
-
# end # defines PlanesIndex::Plane type
|
152
|
+
# ...
|
141
153
|
# end
|
142
154
|
#
|
143
155
|
# The main difference between using plain objects or ActiveRecord models for indexing
|
144
|
-
# is import. If you will call `CarsIndex
|
145
|
-
# automatically, while `PlanesIndex
|
156
|
+
# is import. If you will call `CarsIndex.import` - it will import all the cars
|
157
|
+
# automatically, while `PlanesIndex.import(my_planes)` requires import data to be
|
146
158
|
# passed.
|
147
159
|
#
|
148
|
-
def
|
149
|
-
raise '
|
150
|
-
|
151
|
-
type_class = Chewy.create_type(self, target, options, &block)
|
152
|
-
self.type_hash = type_hash.merge(type_class.type_name => type_class)
|
153
|
-
end
|
154
|
-
|
155
|
-
# Returns defined type:
|
156
|
-
#
|
157
|
-
# UsersIndex.types # => [UsersIndex::User]
|
158
|
-
#
|
159
|
-
def types
|
160
|
-
type_hash.values
|
161
|
-
end
|
162
|
-
|
163
|
-
# Returns defined types names:
|
164
|
-
#
|
165
|
-
# UsersIndex.type_names # => ['admin', 'manager', 'user']
|
166
|
-
#
|
167
|
-
def type_names
|
168
|
-
type_hash.keys
|
169
|
-
end
|
160
|
+
def index_scope(target, options = {})
|
161
|
+
raise 'Index scope is already defined' if index_scope_defined?
|
170
162
|
|
171
|
-
|
172
|
-
|
173
|
-
# UserIndex.type('admin') # => UsersIndex::Admin
|
174
|
-
#
|
175
|
-
def type(type_name)
|
176
|
-
type_hash.fetch(type_name) { raise UndefinedType, "Unknown type in #{name}: #{type_name}" }
|
163
|
+
self.adapter = Chewy.adapters.find { |klass| klass.accepts?(target) }.new(target, **options)
|
164
|
+
self.index_scope_defined = true
|
177
165
|
end
|
178
166
|
|
179
167
|
# Used as a part of index definition DSL. Defines settings:
|
@@ -210,7 +198,7 @@ module Chewy
|
|
210
198
|
end
|
211
199
|
|
212
200
|
def mappings_hash
|
213
|
-
mappings =
|
201
|
+
mappings = root.mappings_hash
|
214
202
|
mappings.present? ? {mappings: mappings} : {}
|
215
203
|
end
|
216
204
|
|
@@ -223,44 +211,15 @@ module Chewy
|
|
223
211
|
[settings_hash, mappings_hash].inject(:merge)
|
224
212
|
end
|
225
213
|
|
226
|
-
def index_params
|
227
|
-
ActiveSupport::Deprecation.warn '`Chewy::Index.index_params` is deprecated and will be removed soon, use `Chewy::Index.specification_hash`'
|
228
|
-
specification_hash
|
229
|
-
end
|
230
|
-
|
231
214
|
# @see Chewy::Index::Specification
|
232
215
|
# @return [Chewy::Index::Specification] a specification object instance for this particular index
|
233
216
|
def specification
|
234
217
|
@specification ||= Specification.new(self)
|
235
218
|
end
|
236
219
|
|
237
|
-
def
|
238
|
-
|
239
|
-
|
240
|
-
end
|
241
|
-
|
242
|
-
# Handling old default_prefix if it is not defined.
|
243
|
-
def method_missing(name, *args, &block) # rubocop:disable Style/MethodMissing
|
244
|
-
if name == :default_prefix
|
245
|
-
ActiveSupport::Deprecation.warn '`Chewy::Index.default_prefix` is deprecated and will be removed soon, use `Chewy::Index.prefix` instead'
|
246
|
-
prefix
|
247
|
-
else
|
248
|
-
super
|
249
|
-
end
|
250
|
-
end
|
251
|
-
|
252
|
-
def prefix_with_deprecation
|
253
|
-
if respond_to?(:default_prefix)
|
254
|
-
ActiveSupport::Deprecation.warn '`Chewy::Index.default_prefix` is deprecated and will be removed soon, define `Chewy::Index.prefix` method instead'
|
255
|
-
default_prefix
|
256
|
-
else
|
257
|
-
prefix
|
258
|
-
end
|
259
|
-
end
|
260
|
-
|
261
|
-
def build_index_name(*args)
|
262
|
-
ActiveSupport::Deprecation.warn '`Chewy::Index.build_index_name` is deprecated and will be removed soon, use `Chewy::Index.index_name` instead'
|
263
|
-
index_name(args.extract_options!)
|
220
|
+
def default_import_options(params)
|
221
|
+
params.assert_valid_keys(IMPORT_OPTIONS_KEYS)
|
222
|
+
self._default_import_options = _default_import_options.merge(params)
|
264
223
|
end
|
265
224
|
end
|
266
225
|
end
|
data/lib/chewy/index/actions.rb
CHANGED
@@ -129,37 +129,6 @@ module Chewy
|
|
129
129
|
create! suffix
|
130
130
|
end
|
131
131
|
|
132
|
-
# Perform import operation for every defined type
|
133
|
-
#
|
134
|
-
# UsersIndex.import # imports default data for every index type
|
135
|
-
# UsersIndex.import user: User.active # imports specified objects for user type and default data for other types
|
136
|
-
# UsersIndex.import refresh: false # to disable index refreshing after import
|
137
|
-
# UsersIndex.import suffix: Time.now.to_i # imports data to index with specified suffix if such is exists
|
138
|
-
# UsersIndex.import batch_size: 300 # import batch size
|
139
|
-
#
|
140
|
-
# See [import.rb](lib/chewy/type/import.rb) for more details.
|
141
|
-
#
|
142
|
-
%i[import import!].each do |method|
|
143
|
-
class_eval <<-METHOD, __FILE__, __LINE__ + 1
|
144
|
-
def #{method}(*args)
|
145
|
-
return true if args.first.blank? && !args.first.nil?
|
146
|
-
|
147
|
-
options = args.extract_options!
|
148
|
-
if args.one? && type_names.one?
|
149
|
-
objects = {type_names.first.to_sym => args.first}
|
150
|
-
elsif args.one?
|
151
|
-
fail ArgumentError, "Please pass objects for `#{method}` as a hash with type names"
|
152
|
-
else
|
153
|
-
objects = options.reject { |k, v| !type_names.map(&:to_sym).include?(k) }
|
154
|
-
end
|
155
|
-
types.map do |type|
|
156
|
-
args = [objects[type.type_name.to_sym], options.dup].reject(&:blank?)
|
157
|
-
type.#{method} *args
|
158
|
-
end.all?
|
159
|
-
end
|
160
|
-
METHOD
|
161
|
-
end
|
162
|
-
|
163
132
|
# Deletes, creates and imports data to the index. Returns the
|
164
133
|
# import result. If index name suffix is passed as the first
|
165
134
|
# argument - performs zero-downtime index resetting.
|
@@ -187,7 +156,11 @@ module Chewy
|
|
187
156
|
suffixed_name = index_name(suffix: suffix)
|
188
157
|
|
189
158
|
optimize_index_settings suffixed_name
|
190
|
-
result = import import_options.merge(
|
159
|
+
result = import import_options.merge(
|
160
|
+
suffix: suffix,
|
161
|
+
journal: journal,
|
162
|
+
refresh: !Chewy.reset_disable_refresh_interval
|
163
|
+
)
|
191
164
|
original_index_settings suffixed_name
|
192
165
|
|
193
166
|
delete if indexes.blank?
|
@@ -209,6 +182,7 @@ module Chewy
|
|
209
182
|
specification.lock!
|
210
183
|
result
|
211
184
|
end
|
185
|
+
alias_method :reset, :reset!
|
212
186
|
|
213
187
|
# A {Chewy::Journal} instance for the particular index
|
214
188
|
#
|
@@ -217,6 +191,50 @@ module Chewy
|
|
217
191
|
@journal ||= Chewy::Journal.new(self)
|
218
192
|
end
|
219
193
|
|
194
|
+
def clear_cache(args = {index: index_name})
|
195
|
+
client.indices.clear_cache(args)
|
196
|
+
end
|
197
|
+
|
198
|
+
def reindex(source: index_name, dest: index_name)
|
199
|
+
client.reindex(
|
200
|
+
{
|
201
|
+
body:
|
202
|
+
{
|
203
|
+
source: {index: source},
|
204
|
+
dest: {index: dest}
|
205
|
+
}
|
206
|
+
}
|
207
|
+
)
|
208
|
+
end
|
209
|
+
|
210
|
+
# Adds new fields to an existing data stream or index.
|
211
|
+
# Change the search settings of existing fields.
|
212
|
+
#
|
213
|
+
# @example
|
214
|
+
# Chewy.client.update_mapping('cities', {properties: {new_field: {type: :text}}})
|
215
|
+
#
|
216
|
+
def update_mapping(name = index_name, body = root.mappings_hash)
|
217
|
+
client.indices.put_mapping(
|
218
|
+
index: name,
|
219
|
+
body: body
|
220
|
+
)['acknowledged']
|
221
|
+
end
|
222
|
+
|
223
|
+
# Performs missing and outdated objects synchronization for the current index.
|
224
|
+
#
|
225
|
+
# @example
|
226
|
+
# UsersIndex.sync
|
227
|
+
#
|
228
|
+
# @see Chewy::Index::Syncer
|
229
|
+
# @param parallel [true, Integer, Hash] options for parallel execution or the number of processes
|
230
|
+
# @return [Hash{Symbol, Object}, nil] a number of missing and outdated documents re-indexed and their ids,
|
231
|
+
# nil in case of errors
|
232
|
+
def sync(parallel: nil)
|
233
|
+
syncer = Syncer.new(self, parallel: parallel)
|
234
|
+
count = syncer.perform
|
235
|
+
{count: count, missing: syncer.missing_ids, outdated: syncer.outdated_ids} if count
|
236
|
+
end
|
237
|
+
|
220
238
|
private
|
221
239
|
|
222
240
|
def optimize_index_settings(index_name)
|
@@ -242,6 +260,7 @@ module Chewy
|
|
242
260
|
|
243
261
|
def index_settings(setting_name)
|
244
262
|
return {} unless settings_hash.key?(:settings) && settings_hash[:settings].key?(:index)
|
263
|
+
|
245
264
|
settings_hash[:settings][:index].slice(setting_name)
|
246
265
|
end
|
247
266
|
end
|
@@ -1,7 +1,7 @@
|
|
1
|
-
require 'chewy/
|
1
|
+
require 'chewy/index/adapter/orm'
|
2
2
|
|
3
3
|
module Chewy
|
4
|
-
class
|
4
|
+
class Index
|
5
5
|
module Adapter
|
6
6
|
class ActiveRecord < Orm
|
7
7
|
def self.accepts?(target)
|
@@ -60,7 +60,15 @@ module Chewy
|
|
60
60
|
end
|
61
61
|
|
62
62
|
def pluck_in_batches(scope, fields: [], batch_size: nil, typecast: true)
|
63
|
-
|
63
|
+
unless block_given?
|
64
|
+
return enum_for(
|
65
|
+
:pluck_in_batches,
|
66
|
+
scope,
|
67
|
+
fields: fields,
|
68
|
+
batch_size: batch_size,
|
69
|
+
typecast: typecast
|
70
|
+
)
|
71
|
+
end
|
64
72
|
|
65
73
|
scope = scope.reorder(target_id.asc).limit(batch_size)
|
66
74
|
ids = pluck(scope, fields: fields, typecast: typecast)
|
@@ -69,6 +77,7 @@ module Chewy
|
|
69
77
|
while ids.present?
|
70
78
|
yield ids
|
71
79
|
break if ids.size < batch_size
|
80
|
+
|
72
81
|
last_id = ids.last.is_a?(Array) ? ids.last.first : ids.last
|
73
82
|
ids = pluck(scope.where(target_id.gt(last_id)), fields: fields, typecast: typecast)
|
74
83
|
end
|
@@ -1,5 +1,5 @@
|
|
1
1
|
module Chewy
|
2
|
-
class
|
2
|
+
class Index
|
3
3
|
module Adapter
|
4
4
|
# Basic adapter class. Contains interface, need to implement to add any classes support
|
5
5
|
class Base
|
@@ -13,8 +13,7 @@ module Chewy
|
|
13
13
|
true
|
14
14
|
end
|
15
15
|
|
16
|
-
# Camelcased name
|
17
|
-
# For returned value 'Product' will be generated class name `ProductsIndex::Product`
|
16
|
+
# Camelcased name.
|
18
17
|
#
|
19
18
|
def name
|
20
19
|
raise NotImplementedError
|
@@ -1,7 +1,7 @@
|
|
1
|
-
require 'chewy/
|
1
|
+
require 'chewy/index/adapter/base'
|
2
2
|
|
3
3
|
module Chewy
|
4
|
-
class
|
4
|
+
class Index
|
5
5
|
module Adapter
|
6
6
|
# This adapter provides an ability to import documents from any
|
7
7
|
# source. You can actually use any class or even a symbol as
|
@@ -14,15 +14,15 @@ module Chewy
|
|
14
14
|
# @see #import
|
15
15
|
# @see #load
|
16
16
|
class Object < Base
|
17
|
-
# The signature of the
|
17
|
+
# The signature of the index scope definition.
|
18
18
|
#
|
19
19
|
# @example
|
20
|
-
#
|
21
|
-
#
|
22
|
-
#
|
20
|
+
# index_scope :geoname
|
21
|
+
# index_scope Geoname
|
22
|
+
# index_scope -> { Geoname.all_the_places }, name: 'geoname'
|
23
23
|
#
|
24
24
|
# @param target [Class, Symbol, String, Proc] a source of data and everything
|
25
|
-
# @option options [String, Symbol] :name redefines the inferred
|
25
|
+
# @option options [String, Symbol] :name redefines the inferred name if necessary
|
26
26
|
# @option options [String, Symbol] :import_all_method redefines import method name
|
27
27
|
# @option options [String, Symbol] :load_all_method redefines batch load method name
|
28
28
|
# @option options [String, Symbol] :load_one_method redefines per-object load method name
|
@@ -31,14 +31,13 @@ module Chewy
|
|
31
31
|
@options = options
|
32
32
|
end
|
33
33
|
|
34
|
-
#
|
35
|
-
# by default if possible.
|
34
|
+
# Inferred from the target by default if possible.
|
36
35
|
#
|
37
36
|
# @example
|
38
|
-
# # defines
|
39
|
-
#
|
40
|
-
# # still defines
|
41
|
-
#
|
37
|
+
# # defines name = Geoname
|
38
|
+
# index_scope :geoname
|
39
|
+
# # still defines name = Geoname
|
40
|
+
# index_scope -> { Geoname.all_the_places }, name: 'geoname'
|
42
41
|
#
|
43
42
|
# @return [String]
|
44
43
|
def name
|
@@ -54,14 +53,14 @@ module Chewy
|
|
54
53
|
Array.wrap(collection)
|
55
54
|
end
|
56
55
|
|
57
|
-
# This method is used internally by `Chewy::
|
56
|
+
# This method is used internally by `Chewy::Index.import`.
|
58
57
|
#
|
59
58
|
# The idea is that any object can be imported to ES if
|
60
59
|
# it responds to `#to_json` method.
|
61
60
|
#
|
62
61
|
# If method `destroyed?` is defined for object (or, in case of hash object,
|
63
62
|
# it has `:_destroyed` or `'_destroyed'` key) and returns `true` or object
|
64
|
-
# satisfy `delete_if`
|
63
|
+
# satisfy `delete_if` option then object will be deleted from index.
|
65
64
|
# But in order to be destroyable, objects need to respond to `id` method
|
66
65
|
# or have an `id` key so ElasticSearch could know which one to delete.
|
67
66
|
#
|
@@ -78,10 +77,10 @@ module Chewy
|
|
78
77
|
# end
|
79
78
|
# end
|
80
79
|
#
|
81
|
-
# # All the
|
82
|
-
#
|
83
|
-
#
|
84
|
-
#
|
80
|
+
# # All the following variants will work:
|
81
|
+
# index_scope Geoname
|
82
|
+
# index_scope Geoname, import_all_method: 'import_all'
|
83
|
+
# index_scope -> { FancyGeoAPI.all_points_collection }, name: 'geoname'
|
85
84
|
#
|
86
85
|
# @param args [Array<#to_json>]
|
87
86
|
# @option options [Integer] :batch_size import processing batch size
|
@@ -113,16 +112,15 @@ module Chewy
|
|
113
112
|
# end
|
114
113
|
# end
|
115
114
|
#
|
116
|
-
# @see Chewy::
|
117
|
-
def import_fields(*args)
|
115
|
+
# @see Chewy::Index::Adapter::Base#import_fields
|
116
|
+
def import_fields(*args, &block)
|
118
117
|
return enum_for(:import_fields, *args) unless block_given?
|
118
|
+
|
119
119
|
options = args.extract_options!
|
120
120
|
options[:batch_size] ||= BATCH_SIZE
|
121
121
|
|
122
122
|
if args.empty? && @target.respond_to?(pluck_method)
|
123
|
-
@target.send(pluck_method, :id, *options[:fields]).each_slice(options[:batch_size])
|
124
|
-
yield batch
|
125
|
-
end
|
123
|
+
@target.send(pluck_method, :id, *options[:fields]).each_slice(options[:batch_size], &block)
|
126
124
|
elsif options[:fields].blank?
|
127
125
|
import_references(*args, options) do |batch|
|
128
126
|
yield batch.map { |object| object_field(object, :id) || object }
|
@@ -140,14 +138,12 @@ module Chewy
|
|
140
138
|
|
141
139
|
# For the Object adapter returns the objects themselves in batches.
|
142
140
|
#
|
143
|
-
# @see Chewy::
|
144
|
-
def import_references(*args)
|
141
|
+
# @see Chewy::Index::Adapter::Base#import_references
|
142
|
+
def import_references(*args, &block)
|
145
143
|
return enum_for(:import_references, *args) unless block_given?
|
146
144
|
|
147
145
|
collection, options = import_args(*args)
|
148
|
-
collection.each_slice(options[:batch_size])
|
149
|
-
yield batch
|
150
|
-
end
|
146
|
+
collection.each_slice(options[:batch_size], &block)
|
151
147
|
end
|
152
148
|
|
153
149
|
# This method is used internally by the request DSL when the
|
@@ -157,7 +153,7 @@ module Chewy
|
|
157
153
|
#
|
158
154
|
# If none of the `load_all_method` or `load_one_method` is implemented
|
159
155
|
# for the target - the method will return nil. This means that the
|
160
|
-
# loader will return an array `Chewy::
|
156
|
+
# loader will return an array `Chewy::Index` objects that actually was passed.
|
161
157
|
#
|
162
158
|
# To use loading for objects it is obviously required to provide
|
163
159
|
# some meaningful ids for ES documents.
|
@@ -175,7 +171,7 @@ module Chewy
|
|
175
171
|
# end
|
176
172
|
# end
|
177
173
|
#
|
178
|
-
# MyIndex
|
174
|
+
# MyIndex.load(additional_data: true).objects
|
179
175
|
#
|
180
176
|
# @param ids [Array<Hash>] an array of ids from ES hits
|
181
177
|
# @param options [Hash] any options passed here with the request DSL `load` method.
|