chewy 5.1.0 → 7.2.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/CODEOWNERS +1 -0
- data/.github/ISSUE_TEMPLATE/bug_report.md +39 -0
- data/.github/ISSUE_TEMPLATE/feature_request.md +20 -0
- data/.github/PULL_REQUEST_TEMPLATE.md +16 -0
- data/.github/workflows/ruby.yml +73 -0
- data/.rubocop.yml +13 -8
- data/.rubocop_todo.yml +110 -22
- data/CHANGELOG.md +449 -347
- data/CODE_OF_CONDUCT.md +14 -0
- data/CONTRIBUTING.md +63 -0
- data/Gemfile +3 -7
- data/Guardfile +3 -1
- data/LICENSE.txt +1 -1
- data/README.md +423 -311
- data/chewy.gemspec +8 -10
- data/gemfiles/rails.5.2.activerecord.gemfile +9 -14
- data/gemfiles/rails.6.0.activerecord.gemfile +11 -0
- data/gemfiles/rails.6.1.activerecord.gemfile +13 -0
- data/gemfiles/rails.7.0.activerecord.gemfile +13 -0
- data/lib/chewy/config.rb +42 -60
- data/lib/chewy/errors.rb +4 -10
- data/lib/chewy/fields/base.rb +80 -20
- data/lib/chewy/fields/root.rb +7 -17
- data/lib/chewy/index/actions.rb +62 -35
- data/lib/chewy/{type → index}/adapter/active_record.rb +18 -4
- data/lib/chewy/{type → index}/adapter/base.rb +2 -3
- data/lib/chewy/{type → index}/adapter/object.rb +28 -32
- data/lib/chewy/{type → index}/adapter/orm.rb +26 -24
- data/lib/chewy/index/aliases.rb +14 -5
- data/lib/chewy/{type → index}/crutch.rb +5 -5
- data/lib/chewy/index/import/bulk_builder.rb +311 -0
- data/lib/chewy/{type → index}/import/bulk_request.rb +6 -7
- data/lib/chewy/{type → index}/import/journal_builder.rb +11 -12
- data/lib/chewy/{type → index}/import/routine.rb +17 -16
- data/lib/chewy/{type → index}/import.rb +51 -33
- data/lib/chewy/{type → index}/mapping.rb +32 -37
- data/lib/chewy/index/observe/active_record_methods.rb +87 -0
- data/lib/chewy/index/observe/callback.rb +34 -0
- data/lib/chewy/index/observe.rb +17 -0
- data/lib/chewy/index/specification.rb +1 -0
- data/lib/chewy/{type → index}/syncer.rb +61 -62
- data/lib/chewy/{type → index}/witchcraft.rb +15 -9
- data/lib/chewy/{type → index}/wrapper.rb +13 -3
- data/lib/chewy/index.rb +46 -96
- data/lib/chewy/journal.rb +25 -14
- data/lib/chewy/minitest/helpers.rb +86 -13
- data/lib/chewy/minitest/search_index_receiver.rb +22 -26
- data/lib/chewy/multi_search.rb +62 -0
- data/lib/chewy/railtie.rb +6 -20
- data/lib/chewy/rake_helper.rb +136 -108
- data/lib/chewy/rspec/build_query.rb +12 -0
- data/lib/chewy/rspec/helpers.rb +55 -0
- data/lib/chewy/rspec/update_index.rb +55 -44
- data/lib/chewy/rspec.rb +2 -0
- data/lib/chewy/runtime.rb +1 -1
- data/lib/chewy/search/loader.rb +19 -41
- data/lib/chewy/search/parameters/collapse.rb +16 -0
- data/lib/chewy/search/parameters/concerns/query_storage.rb +2 -2
- data/lib/chewy/search/parameters/ignore_unavailable.rb +27 -0
- data/lib/chewy/search/parameters/indices.rb +12 -57
- data/lib/chewy/search/parameters/none.rb +1 -3
- data/lib/chewy/search/parameters/order.rb +6 -19
- data/lib/chewy/search/parameters/source.rb +5 -1
- data/lib/chewy/search/parameters/track_total_hits.rb +16 -0
- data/lib/chewy/search/parameters.rb +7 -4
- data/lib/chewy/search/query_proxy.rb +9 -2
- data/lib/chewy/search/request.rb +180 -154
- data/lib/chewy/search/response.rb +5 -5
- data/lib/chewy/search/scoping.rb +7 -8
- data/lib/chewy/search/scrolling.rb +16 -13
- data/lib/chewy/search.rb +7 -22
- data/lib/chewy/stash.rb +19 -30
- data/lib/chewy/strategy/active_job.rb +2 -2
- data/lib/chewy/strategy/atomic_no_refresh.rb +18 -0
- data/lib/chewy/strategy/base.rb +10 -0
- data/lib/chewy/strategy/lazy_sidekiq.rb +64 -0
- data/lib/chewy/strategy/sidekiq.rb +3 -2
- data/lib/chewy/strategy.rb +5 -19
- data/lib/chewy/version.rb +1 -1
- data/lib/chewy.rb +36 -80
- data/lib/generators/chewy/install_generator.rb +1 -1
- data/lib/tasks/chewy.rake +26 -32
- data/migration_guide.md +56 -0
- data/spec/chewy/config_spec.rb +15 -61
- data/spec/chewy/fields/base_spec.rb +432 -145
- data/spec/chewy/fields/root_spec.rb +20 -28
- data/spec/chewy/fields/time_fields_spec.rb +5 -5
- data/spec/chewy/index/actions_spec.rb +388 -55
- data/spec/chewy/{type → index}/adapter/active_record_spec.rb +110 -44
- data/spec/chewy/{type → index}/adapter/object_spec.rb +21 -6
- data/spec/chewy/index/aliases_spec.rb +3 -3
- data/spec/chewy/index/import/bulk_builder_spec.rb +494 -0
- data/spec/chewy/{type → index}/import/bulk_request_spec.rb +5 -12
- data/spec/chewy/{type → index}/import/journal_builder_spec.rb +14 -22
- data/spec/chewy/{type → index}/import/routine_spec.rb +19 -19
- data/spec/chewy/{type → index}/import_spec.rb +149 -96
- data/spec/chewy/index/mapping_spec.rb +135 -0
- data/spec/chewy/index/observe/active_record_methods_spec.rb +68 -0
- data/spec/chewy/index/observe/callback_spec.rb +139 -0
- data/spec/chewy/index/observe_spec.rb +143 -0
- data/spec/chewy/index/settings_spec.rb +3 -1
- data/spec/chewy/index/specification_spec.rb +20 -30
- data/spec/chewy/{type → index}/syncer_spec.rb +14 -19
- data/spec/chewy/{type → index}/witchcraft_spec.rb +34 -21
- data/spec/chewy/index/wrapper_spec.rb +100 -0
- data/spec/chewy/index_spec.rb +69 -137
- data/spec/chewy/journal_spec.rb +46 -91
- data/spec/chewy/minitest/helpers_spec.rb +122 -14
- data/spec/chewy/minitest/search_index_receiver_spec.rb +24 -26
- data/spec/chewy/multi_search_spec.rb +84 -0
- data/spec/chewy/rake_helper_spec.rb +293 -101
- data/spec/chewy/rspec/build_query_spec.rb +34 -0
- data/spec/chewy/rspec/helpers_spec.rb +61 -0
- data/spec/chewy/rspec/update_index_spec.rb +106 -102
- data/spec/chewy/runtime_spec.rb +2 -2
- data/spec/chewy/search/loader_spec.rb +19 -53
- data/spec/chewy/search/pagination/kaminari_examples.rb +3 -5
- data/spec/chewy/search/pagination/kaminari_spec.rb +1 -1
- data/spec/chewy/search/parameters/collapse_spec.rb +5 -0
- data/spec/chewy/search/parameters/ignore_unavailable_spec.rb +67 -0
- data/spec/chewy/search/parameters/indices_spec.rb +26 -118
- data/spec/chewy/search/parameters/none_spec.rb +1 -1
- data/spec/chewy/search/parameters/order_spec.rb +18 -11
- data/spec/chewy/search/parameters/query_storage_examples.rb +67 -21
- data/spec/chewy/search/parameters/search_after_spec.rb +4 -1
- data/spec/chewy/search/parameters/source_spec.rb +8 -2
- data/spec/chewy/search/parameters/track_total_hits_spec.rb +5 -0
- data/spec/chewy/search/parameters_spec.rb +23 -7
- data/spec/chewy/search/query_proxy_spec.rb +68 -17
- data/spec/chewy/search/request_spec.rb +344 -149
- data/spec/chewy/search/response_spec.rb +35 -25
- data/spec/chewy/search/scrolling_spec.rb +28 -26
- data/spec/chewy/search_spec.rb +69 -59
- data/spec/chewy/stash_spec.rb +16 -26
- data/spec/chewy/strategy/active_job_spec.rb +23 -10
- data/spec/chewy/strategy/atomic_no_refresh_spec.rb +60 -0
- data/spec/chewy/strategy/atomic_spec.rb +9 -10
- data/spec/chewy/strategy/lazy_sidekiq_spec.rb +214 -0
- data/spec/chewy/strategy/sidekiq_spec.rb +14 -10
- data/spec/chewy/strategy_spec.rb +19 -15
- data/spec/chewy_spec.rb +17 -110
- data/spec/spec_helper.rb +6 -29
- data/spec/support/active_record.rb +43 -5
- metadata +102 -198
- data/.travis.yml +0 -45
- data/Appraisals +0 -81
- data/LEGACY_DSL.md +0 -497
- data/gemfiles/rails.4.0.activerecord.gemfile +0 -15
- data/gemfiles/rails.4.1.activerecord.gemfile +0 -15
- data/gemfiles/rails.4.2.activerecord.gemfile +0 -16
- data/gemfiles/rails.4.2.mongoid.5.2.gemfile +0 -16
- data/gemfiles/rails.5.0.activerecord.gemfile +0 -16
- data/gemfiles/rails.5.0.mongoid.6.1.gemfile +0 -16
- data/gemfiles/rails.5.1.activerecord.gemfile +0 -16
- data/gemfiles/rails.5.1.mongoid.6.3.gemfile +0 -16
- data/gemfiles/sequel.4.45.gemfile +0 -11
- data/lib/chewy/backports/deep_dup.rb +0 -46
- data/lib/chewy/backports/duplicable.rb +0 -91
- data/lib/chewy/query/compose.rb +0 -68
- data/lib/chewy/query/criteria.rb +0 -191
- data/lib/chewy/query/filters.rb +0 -244
- data/lib/chewy/query/loading.rb +0 -110
- data/lib/chewy/query/nodes/and.rb +0 -25
- data/lib/chewy/query/nodes/base.rb +0 -17
- data/lib/chewy/query/nodes/bool.rb +0 -34
- data/lib/chewy/query/nodes/equal.rb +0 -34
- data/lib/chewy/query/nodes/exists.rb +0 -20
- data/lib/chewy/query/nodes/expr.rb +0 -28
- data/lib/chewy/query/nodes/field.rb +0 -110
- data/lib/chewy/query/nodes/has_child.rb +0 -15
- data/lib/chewy/query/nodes/has_parent.rb +0 -15
- data/lib/chewy/query/nodes/has_relation.rb +0 -59
- data/lib/chewy/query/nodes/match_all.rb +0 -11
- data/lib/chewy/query/nodes/missing.rb +0 -20
- data/lib/chewy/query/nodes/not.rb +0 -25
- data/lib/chewy/query/nodes/or.rb +0 -25
- data/lib/chewy/query/nodes/prefix.rb +0 -19
- data/lib/chewy/query/nodes/query.rb +0 -20
- data/lib/chewy/query/nodes/range.rb +0 -63
- data/lib/chewy/query/nodes/raw.rb +0 -15
- data/lib/chewy/query/nodes/regexp.rb +0 -35
- data/lib/chewy/query/nodes/script.rb +0 -20
- data/lib/chewy/query/pagination.rb +0 -25
- data/lib/chewy/query.rb +0 -1142
- data/lib/chewy/search/pagination/will_paginate.rb +0 -43
- data/lib/chewy/search/parameters/types.rb +0 -20
- data/lib/chewy/strategy/resque.rb +0 -27
- data/lib/chewy/strategy/shoryuken.rb +0 -40
- data/lib/chewy/type/actions.rb +0 -43
- data/lib/chewy/type/adapter/mongoid.rb +0 -67
- data/lib/chewy/type/adapter/sequel.rb +0 -93
- data/lib/chewy/type/import/bulk_builder.rb +0 -122
- data/lib/chewy/type/observe.rb +0 -82
- data/lib/chewy/type.rb +0 -117
- data/lib/sequel/plugins/chewy_observe.rb +0 -63
- data/spec/chewy/query/criteria_spec.rb +0 -700
- data/spec/chewy/query/filters_spec.rb +0 -201
- data/spec/chewy/query/loading_spec.rb +0 -124
- data/spec/chewy/query/nodes/and_spec.rb +0 -12
- data/spec/chewy/query/nodes/bool_spec.rb +0 -14
- data/spec/chewy/query/nodes/equal_spec.rb +0 -32
- data/spec/chewy/query/nodes/exists_spec.rb +0 -18
- data/spec/chewy/query/nodes/has_child_spec.rb +0 -59
- data/spec/chewy/query/nodes/has_parent_spec.rb +0 -59
- data/spec/chewy/query/nodes/match_all_spec.rb +0 -11
- data/spec/chewy/query/nodes/missing_spec.rb +0 -16
- data/spec/chewy/query/nodes/not_spec.rb +0 -14
- data/spec/chewy/query/nodes/or_spec.rb +0 -12
- data/spec/chewy/query/nodes/prefix_spec.rb +0 -16
- data/spec/chewy/query/nodes/query_spec.rb +0 -12
- data/spec/chewy/query/nodes/range_spec.rb +0 -32
- data/spec/chewy/query/nodes/raw_spec.rb +0 -11
- data/spec/chewy/query/nodes/regexp_spec.rb +0 -43
- data/spec/chewy/query/nodes/script_spec.rb +0 -15
- data/spec/chewy/query/pagination/kaminari_spec.rb +0 -5
- data/spec/chewy/query/pagination/will_paginate_spec.rb +0 -5
- data/spec/chewy/query/pagination_spec.rb +0 -39
- data/spec/chewy/query_spec.rb +0 -637
- data/spec/chewy/search/pagination/will_paginate_examples.rb +0 -63
- data/spec/chewy/search/pagination/will_paginate_spec.rb +0 -23
- data/spec/chewy/search/parameters/types_spec.rb +0 -5
- data/spec/chewy/strategy/resque_spec.rb +0 -46
- data/spec/chewy/strategy/shoryuken_spec.rb +0 -66
- data/spec/chewy/type/actions_spec.rb +0 -50
- data/spec/chewy/type/adapter/mongoid_spec.rb +0 -372
- data/spec/chewy/type/adapter/sequel_spec.rb +0 -472
- data/spec/chewy/type/import/bulk_builder_spec.rb +0 -279
- data/spec/chewy/type/mapping_spec.rb +0 -173
- data/spec/chewy/type/observe_spec.rb +0 -137
- data/spec/chewy/type/wrapper_spec.rb +0 -98
- data/spec/chewy/type_spec.rb +0 -55
- data/spec/support/mongoid.rb +0 -93
- data/spec/support/sequel.rb +0 -80
@@ -1,5 +1,5 @@
|
|
1
1
|
module Chewy
|
2
|
-
class
|
2
|
+
class Index
|
3
3
|
module Mapping
|
4
4
|
extend ActiveSupport::Concern
|
5
5
|
|
@@ -13,42 +13,39 @@ module Chewy
|
|
13
13
|
end
|
14
14
|
|
15
15
|
module ClassMethods
|
16
|
-
# Defines root object for mapping and is optional for
|
16
|
+
# Defines root object for mapping and is optional for index
|
17
17
|
# definition. Use it only if you need to pass options for root
|
18
18
|
# object mapping, such as `date_detection` or `dynamic_date_formats`
|
19
19
|
#
|
20
20
|
# @example
|
21
21
|
# class UsersIndex < Chewy::Index
|
22
|
-
#
|
23
|
-
#
|
24
|
-
#
|
25
|
-
# end
|
22
|
+
# index_scope User
|
23
|
+
# # root object defined implicitly and optionless for current type
|
24
|
+
# field :full_name, type: 'keyword'
|
26
25
|
# end
|
27
26
|
#
|
28
27
|
# class CarsIndex < Chewy::Index
|
29
|
-
#
|
30
|
-
#
|
31
|
-
#
|
32
|
-
#
|
33
|
-
# end
|
28
|
+
# index_scope Car
|
29
|
+
# # explicit root definition with additional options
|
30
|
+
# root dynamic_date_formats: ['yyyy-MM-dd'] do
|
31
|
+
# field :model_name, type: 'keyword'
|
34
32
|
# end
|
35
33
|
# end
|
36
34
|
#
|
37
35
|
def root(**options)
|
38
|
-
self.root_object ||= Chewy::Fields::Root.new(
|
39
|
-
root_object.update_options!(options)
|
36
|
+
self.root_object ||= Chewy::Fields::Root.new(:root, **Chewy.default_root_options.merge(options))
|
37
|
+
root_object.update_options!(**options)
|
40
38
|
yield if block_given?
|
41
39
|
root_object
|
42
40
|
end
|
43
41
|
|
44
|
-
# Defines mapping field for
|
42
|
+
# Defines mapping field for index
|
45
43
|
#
|
46
44
|
# @example
|
47
45
|
# class UsersIndex < Chewy::Index
|
48
|
-
#
|
49
|
-
#
|
50
|
-
#
|
51
|
-
# end
|
46
|
+
# index_scope User
|
47
|
+
# # passing all the options to field definition:
|
48
|
+
# field :full_name, analyzer: 'special'
|
52
49
|
# end
|
53
50
|
#
|
54
51
|
# The `type` is optional and defaults to `string` if not defined:
|
@@ -126,9 +123,9 @@ module Chewy
|
|
126
123
|
#
|
127
124
|
def field(*args, **options, &block)
|
128
125
|
if args.size > 1
|
129
|
-
args.map { |name| field(name, options) }
|
126
|
+
args.map { |name| field(name, **options) }
|
130
127
|
else
|
131
|
-
expand_nested(Chewy::Fields::Base.new(args.first, options), &block)
|
128
|
+
expand_nested(Chewy::Fields::Base.new(args.first, **options), &block)
|
132
129
|
end
|
133
130
|
end
|
134
131
|
|
@@ -138,14 +135,13 @@ module Chewy
|
|
138
135
|
# # Suppose that a user has posts and each post has ratings
|
139
136
|
# # avg_post_rating is the mean of all ratings
|
140
137
|
# class UsersIndex < Chewy::Index
|
141
|
-
#
|
142
|
-
#
|
143
|
-
#
|
144
|
-
#
|
145
|
-
#
|
146
|
-
#
|
147
|
-
#
|
148
|
-
# end
|
138
|
+
# index_scope User
|
139
|
+
# field :posts do
|
140
|
+
# field :rating
|
141
|
+
# end
|
142
|
+
#
|
143
|
+
# agg :avg_rating do
|
144
|
+
# { avg: { field: 'posts.rating' } }
|
149
145
|
# end
|
150
146
|
# end
|
151
147
|
def agg(name, &block)
|
@@ -157,11 +153,10 @@ module Chewy
|
|
157
153
|
#
|
158
154
|
# @example
|
159
155
|
# class CarsIndex < Chewy::Index
|
160
|
-
#
|
161
|
-
#
|
162
|
-
#
|
163
|
-
#
|
164
|
-
# end
|
156
|
+
# index_scope Car
|
157
|
+
# template 'model.*', type: 'text', analyzer: 'special'
|
158
|
+
# field 'model', type: 'object' # here we can put { de: 'Der Mercedes', en: 'Mercedes' }
|
159
|
+
# # and template will be applied to this field
|
165
160
|
# end
|
166
161
|
#
|
167
162
|
# Name for each template is generated with the following
|
@@ -172,18 +167,18 @@ module Chewy
|
|
172
167
|
# template 'title.*', mapping_hash # dot in template causes "path_match" using
|
173
168
|
# template /tit.+/, mapping_hash # using "match_pattern": "regexp"
|
174
169
|
# template /title\..+/, mapping_hash # "\." - escaped dot causes "path_match" using
|
175
|
-
# template /tit.+/, type: 'text', mapping_hash # "match_mapping_type" as
|
170
|
+
# template /tit.+/, type: 'text', mapping_hash # "match_mapping_type" as an optional second argument
|
176
171
|
# template template42: {match: 'hello*', mapping: {type: 'object'}} # or even pass a template as is
|
177
172
|
#
|
178
|
-
def template(*args)
|
179
|
-
root.dynamic_template(*args)
|
173
|
+
def template(*args, **options)
|
174
|
+
root.dynamic_template(*args, **options)
|
180
175
|
end
|
181
176
|
alias_method :dynamic_template, :template
|
182
177
|
|
183
178
|
# Returns compiled mappings hash for current type
|
184
179
|
#
|
185
180
|
def mappings_hash
|
186
|
-
root.mappings_hash
|
181
|
+
root.mappings_hash
|
187
182
|
end
|
188
183
|
|
189
184
|
# Check whether the type has outdated_sync_field defined with a simple value.
|
@@ -0,0 +1,87 @@
|
|
1
|
+
module Chewy
|
2
|
+
class Index
|
3
|
+
module Observe
|
4
|
+
module Helpers
|
5
|
+
def update_proc(index_name, *args, &block)
|
6
|
+
options = args.extract_options!
|
7
|
+
method = args.first
|
8
|
+
|
9
|
+
proc do
|
10
|
+
reference = if index_name.is_a?(Proc)
|
11
|
+
if index_name.arity.zero?
|
12
|
+
instance_exec(&index_name)
|
13
|
+
else
|
14
|
+
index_name.call(self)
|
15
|
+
end
|
16
|
+
else
|
17
|
+
index_name
|
18
|
+
end
|
19
|
+
|
20
|
+
index = Chewy.derive_name(reference)
|
21
|
+
|
22
|
+
next if Chewy.strategy.current.name == :bypass
|
23
|
+
|
24
|
+
backreference = if method && method.to_s == 'self'
|
25
|
+
self
|
26
|
+
elsif method
|
27
|
+
send(method)
|
28
|
+
else
|
29
|
+
instance_eval(&block)
|
30
|
+
end
|
31
|
+
|
32
|
+
index.update_index(backreference, options)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def extract_callback_options!(args)
|
37
|
+
options = args.extract_options!
|
38
|
+
result = options.each_key.with_object({}) do |key, hash|
|
39
|
+
hash[key] = options.delete(key) if %i[if unless].include?(key)
|
40
|
+
end
|
41
|
+
args.push(options) unless options.empty?
|
42
|
+
result
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
extend Helpers
|
47
|
+
|
48
|
+
module ActiveRecordMethods
|
49
|
+
extend ActiveSupport::Concern
|
50
|
+
|
51
|
+
def run_chewy_callbacks
|
52
|
+
chewy_callbacks.each { |callback| callback.call(self) }
|
53
|
+
end
|
54
|
+
|
55
|
+
def update_chewy_indices
|
56
|
+
Chewy.strategy.current.update_chewy_indices(self)
|
57
|
+
end
|
58
|
+
|
59
|
+
included do
|
60
|
+
class_attribute :chewy_callbacks, default: []
|
61
|
+
end
|
62
|
+
|
63
|
+
class_methods do
|
64
|
+
def initialize_chewy_callbacks
|
65
|
+
if Chewy.use_after_commit_callbacks
|
66
|
+
after_commit :update_chewy_indices, on: %i[create update]
|
67
|
+
after_commit :run_chewy_callbacks, on: :destroy
|
68
|
+
else
|
69
|
+
after_save :update_chewy_indices
|
70
|
+
after_destroy :run_chewy_callbacks
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
ruby2_keywords def update_index(type_name, *args, &block)
|
75
|
+
callback_options = Observe.extract_callback_options!(args)
|
76
|
+
update_proc = Observe.update_proc(type_name, *args, &block)
|
77
|
+
callback = Chewy::Index::Observe::Callback.new(update_proc, callback_options)
|
78
|
+
|
79
|
+
initialize_chewy_callbacks if chewy_callbacks.empty?
|
80
|
+
|
81
|
+
self.chewy_callbacks = chewy_callbacks.dup << callback
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
module Chewy
|
2
|
+
class Index
|
3
|
+
module Observe
|
4
|
+
class Callback
|
5
|
+
def initialize(executable, filters = {})
|
6
|
+
@executable = executable
|
7
|
+
@if_filter = filters[:if]
|
8
|
+
@unless_filter = filters[:unless]
|
9
|
+
end
|
10
|
+
|
11
|
+
def call(context)
|
12
|
+
return if !@if_filter.nil? && !eval_filter(@if_filter, context)
|
13
|
+
return if !@unless_filter.nil? && eval_filter(@unless_filter, context)
|
14
|
+
|
15
|
+
eval_proc(@executable, context)
|
16
|
+
end
|
17
|
+
|
18
|
+
private
|
19
|
+
|
20
|
+
def eval_filter(filter, context)
|
21
|
+
case filter
|
22
|
+
when Symbol then context.send(filter)
|
23
|
+
when Proc then eval_proc(filter, context)
|
24
|
+
else filter
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def eval_proc(executable, context)
|
29
|
+
executable.arity.zero? ? context.instance_exec(&executable) : executable.call(context)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require 'chewy/index/observe/callback'
|
2
|
+
require 'chewy/index/observe/active_record_methods'
|
3
|
+
|
4
|
+
module Chewy
|
5
|
+
class Index
|
6
|
+
module Observe
|
7
|
+
extend ActiveSupport::Concern
|
8
|
+
|
9
|
+
module ClassMethods
|
10
|
+
def update_index(objects, options = {})
|
11
|
+
Chewy.strategy.current.update(self, objects, options)
|
12
|
+
true
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -1,18 +1,18 @@
|
|
1
1
|
module Chewy
|
2
|
-
class
|
2
|
+
class Index
|
3
3
|
# This class is able to find missing and outdated documents in the ES
|
4
4
|
# comparing ids from the data source and the ES index. Also, if `outdated_sync_field`
|
5
|
-
#
|
5
|
+
# exists in the index definition, it performs comparison of this field
|
6
6
|
# values for each source object and corresponding ES document. Usually,
|
7
7
|
# this field is `updated_at` and if its value in the source is not equal
|
8
8
|
# to the value in the index - this means that this document outdated and
|
9
9
|
# should be reindexed.
|
10
10
|
#
|
11
11
|
# To fetch necessary data from the source it uses adapter method
|
12
|
-
# {Chewy::
|
12
|
+
# {Chewy::Index::Adapter::Base#import_fields}, in case when the Object
|
13
13
|
# adapter is used it makes sense to read corresponding documentation.
|
14
14
|
#
|
15
|
-
# If `parallel` option is passed to the initializer - it will fetch
|
15
|
+
# If `parallel` option is passed to the initializer - it will fetch source and
|
16
16
|
# index data in parallel and then perform outdated objects calculation in
|
17
17
|
# parallel processes. Also, further import (if required) will be performed
|
18
18
|
# in parallel as well.
|
@@ -24,17 +24,17 @@ module Chewy
|
|
24
24
|
# ATTENTION: synchronization may be slow in case when synchronized tables
|
25
25
|
# are missing compound index on primary key and `outdated_sync_field`.
|
26
26
|
#
|
27
|
-
# @see Chewy::
|
27
|
+
# @see Chewy::Index::Actions::ClassMethods#sync
|
28
28
|
class Syncer
|
29
29
|
DEFAULT_SYNC_BATCH_SIZE = 20_000
|
30
|
-
ISO_DATETIME = /\A(\d{4})-(\d\d)-(\d\d) (\d\d):(\d\d):(\d\d)(\.\d+)?\z
|
31
|
-
OUTDATED_IDS_WORKER = lambda do |outdated_sync_field_type, source_data_hash,
|
32
|
-
::Process.setproctitle("chewy [#{
|
30
|
+
ISO_DATETIME = /\A(\d{4})-(\d\d)-(\d\d) (\d\d):(\d\d):(\d\d)(\.\d+)?\z/.freeze
|
31
|
+
OUTDATED_IDS_WORKER = lambda do |outdated_sync_field_type, source_data_hash, index, total, index_data|
|
32
|
+
::Process.setproctitle("chewy [#{index}]: sync outdated calculation (#{::Parallel.worker_number + 1}/#{total})") if index
|
33
33
|
index_data.each_with_object([]) do |(id, index_sync_value), result|
|
34
34
|
next unless source_data_hash[id]
|
35
35
|
|
36
36
|
outdated = if outdated_sync_field_type == 'date'
|
37
|
-
!Chewy::
|
37
|
+
!Chewy::Index::Syncer.dates_equal(typecast_date(source_data_hash[id]), Time.iso8601(index_sync_value))
|
38
38
|
else
|
39
39
|
source_data_hash[id] != index_sync_value
|
40
40
|
end
|
@@ -42,8 +42,8 @@ module Chewy
|
|
42
42
|
result.push(id) if outdated
|
43
43
|
end
|
44
44
|
end
|
45
|
-
SOURCE_OR_INDEX_DATA_WORKER = lambda do |syncer,
|
46
|
-
::Process.setproctitle("chewy [#{
|
45
|
+
SOURCE_OR_INDEX_DATA_WORKER = lambda do |syncer, index, kind|
|
46
|
+
::Process.setproctitle("chewy [#{index}]: sync fetching data (#{kind})")
|
47
47
|
result = case kind
|
48
48
|
when :source
|
49
49
|
syncer.send(:fetch_source_data)
|
@@ -56,7 +56,10 @@ module Chewy
|
|
56
56
|
def self.typecast_date(string)
|
57
57
|
if string.is_a?(String) && (match = ISO_DATETIME.match(string))
|
58
58
|
microsec = (match[7].to_r * 1_000_000).to_i
|
59
|
-
|
59
|
+
day = "#{match[1]}-#{match[2]}-#{match[3]}"
|
60
|
+
time_with_seconds = "#{match[4]}:#{match[5]}:#{match[6]}"
|
61
|
+
microseconds = format('%06d', microsec)
|
62
|
+
date = "#{day}T#{time_with_seconds}.#{microseconds}+00:00"
|
60
63
|
Time.iso8601(date)
|
61
64
|
else
|
62
65
|
string
|
@@ -68,18 +71,10 @@ module Chewy
|
|
68
71
|
[one.to_i, one.usec / 1000] == [two.to_i, two.usec / 1000]
|
69
72
|
end
|
70
73
|
|
71
|
-
#
|
72
|
-
# milliseconds, so ES stored time with the seconds precision.
|
73
|
-
if ActiveSupport::VERSION::STRING < '4.1.0'
|
74
|
-
def self.dates_equal(one, two)
|
75
|
-
one.to_i == two.to_i
|
76
|
-
end
|
77
|
-
end
|
78
|
-
|
79
|
-
# @param type [Chewy::Type] chewy type
|
74
|
+
# @param index [Chewy::Index] chewy index
|
80
75
|
# @param parallel [true, Integer, Hash] options for parallel execution or the number of processes
|
81
|
-
def initialize(
|
82
|
-
@
|
76
|
+
def initialize(index, parallel: nil)
|
77
|
+
@index = index
|
83
78
|
@parallel = if !parallel || parallel.is_a?(Hash)
|
84
79
|
parallel
|
85
80
|
elsif parallel.is_a?(Integer)
|
@@ -95,7 +90,8 @@ module Chewy
|
|
95
90
|
def perform
|
96
91
|
ids = missing_ids | outdated_ids
|
97
92
|
return 0 if ids.blank?
|
98
|
-
|
93
|
+
|
94
|
+
@index.import(ids, parallel: @parallel) && ids.count
|
99
95
|
end
|
100
96
|
|
101
97
|
# Finds ids of all the objects that are not indexed yet or deleted
|
@@ -113,21 +109,19 @@ module Chewy
|
|
113
109
|
end
|
114
110
|
end
|
115
111
|
|
116
|
-
# If
|
117
|
-
#
|
118
|
-
# and index and returns the ids of entities which
|
119
|
-
# different values there.
|
112
|
+
# If index supports outdated sync, it compares the values of the
|
113
|
+
# `outdated_sync_field` for each object and document in the source
|
114
|
+
# and index and returns the ids of entities which differ.
|
120
115
|
#
|
121
|
-
# @see Chewy::
|
116
|
+
# @see Chewy::Index::Mapping::ClassMethods#supports_outdated_sync?
|
122
117
|
# @return [Array<String>] an array of outdated ids
|
123
118
|
def outdated_ids
|
124
|
-
return [] if source_data.blank? || index_data.blank? || !@
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
end
|
119
|
+
return [] if source_data.blank? || index_data.blank? || !@index.supports_outdated_sync?
|
120
|
+
|
121
|
+
@outdated_ids ||= if @parallel
|
122
|
+
parallel_outdated_ids
|
123
|
+
else
|
124
|
+
linear_outdated_ids
|
131
125
|
end
|
132
126
|
end
|
133
127
|
|
@@ -142,44 +136,48 @@ module Chewy
|
|
142
136
|
end
|
143
137
|
|
144
138
|
def source_and_index_data
|
145
|
-
@source_and_index_data ||=
|
146
|
-
if
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
[result.first.values.first, result.second.values.first]
|
152
|
-
else
|
153
|
-
[result.second.values.first, result.first.values.first]
|
154
|
-
end
|
139
|
+
@source_and_index_data ||= if @parallel
|
140
|
+
::ActiveRecord::Base.connection.close if defined?(::ActiveRecord::Base)
|
141
|
+
result = ::Parallel.map(%i[source index], @parallel, &SOURCE_OR_INDEX_DATA_WORKER.curry[self, @index])
|
142
|
+
::ActiveRecord::Base.connection.reconnect! if defined?(::ActiveRecord::Base)
|
143
|
+
if result.first.keys.first == :source
|
144
|
+
[result.first.values.first, result.second.values.first]
|
155
145
|
else
|
156
|
-
[
|
146
|
+
[result.second.values.first, result.first.values.first]
|
157
147
|
end
|
148
|
+
else
|
149
|
+
[fetch_source_data, fetch_index_data]
|
158
150
|
end
|
159
151
|
end
|
160
152
|
|
161
153
|
def fetch_source_data
|
162
|
-
if @
|
163
|
-
|
154
|
+
if @index.supports_outdated_sync?
|
155
|
+
import_fields_args = {
|
156
|
+
fields: [@index.outdated_sync_field],
|
157
|
+
batch_size: DEFAULT_SYNC_BATCH_SIZE,
|
158
|
+
typecast: false
|
159
|
+
}
|
160
|
+
@index.adapter.import_fields(import_fields_args).to_a.flatten(1).each do |data|
|
164
161
|
data[0] = data[0].to_s
|
165
162
|
end
|
166
163
|
else
|
167
|
-
@
|
164
|
+
@index.adapter.import_fields(batch_size: DEFAULT_SYNC_BATCH_SIZE, typecast: false).to_a.flatten(1).map(&:to_s)
|
168
165
|
end
|
169
166
|
end
|
170
167
|
|
171
168
|
def fetch_index_data
|
172
|
-
if @
|
173
|
-
@
|
169
|
+
if @index.supports_outdated_sync?
|
170
|
+
@index.pluck(:_id, @index.outdated_sync_field).each do |data|
|
174
171
|
data[0] = data[0].to_s
|
175
172
|
end
|
176
173
|
else
|
177
|
-
@
|
174
|
+
@index.pluck(:_id).map(&:to_s)
|
178
175
|
end
|
179
176
|
end
|
180
177
|
|
181
178
|
def data_ids(data)
|
182
|
-
return data unless @
|
179
|
+
return data unless @index.supports_outdated_sync?
|
180
|
+
|
183
181
|
data.map(&:first)
|
184
182
|
end
|
185
183
|
|
@@ -192,7 +190,12 @@ module Chewy
|
|
192
190
|
batches = index_data.each_slice(size)
|
193
191
|
|
194
192
|
::ActiveRecord::Base.connection.close if defined?(::ActiveRecord::Base)
|
195
|
-
|
193
|
+
curried_outdated_ids_worker = OUTDATED_IDS_WORKER.curry[outdated_sync_field_type, source_data.to_h, @index, batches.size]
|
194
|
+
result = ::Parallel.map(
|
195
|
+
batches,
|
196
|
+
@parallel,
|
197
|
+
&curried_outdated_ids_worker
|
198
|
+
).flatten(1)
|
196
199
|
::ActiveRecord::Base.connection.reconnect! if defined?(::ActiveRecord::Base)
|
197
200
|
result
|
198
201
|
end
|
@@ -203,17 +206,13 @@ module Chewy
|
|
203
206
|
|
204
207
|
def outdated_sync_field_type
|
205
208
|
return @outdated_sync_field_type if instance_variable_defined?(:@outdated_sync_field_type)
|
206
|
-
return unless @
|
209
|
+
return unless @index.outdated_sync_field
|
207
210
|
|
208
|
-
mappings = @
|
209
|
-
index: @type.index_name,
|
210
|
-
type: @type.type_name
|
211
|
-
).values.first.fetch('mappings', {})
|
211
|
+
mappings = @index.client.indices.get_mapping(index: @index.index_name).values.first.fetch('mappings', {})
|
212
212
|
|
213
213
|
@outdated_sync_field_type = mappings
|
214
|
-
.fetch(@type.type_name, {})
|
215
214
|
.fetch('properties', {})
|
216
|
-
.fetch(@
|
215
|
+
.fetch(@index.outdated_sync_field.to_s, {})['type']
|
217
216
|
rescue Elasticsearch::Transport::Transport::Errors::NotFound
|
218
217
|
nil
|
219
218
|
end
|
@@ -7,7 +7,7 @@ rescue LoadError
|
|
7
7
|
end
|
8
8
|
|
9
9
|
module Chewy
|
10
|
-
class
|
10
|
+
class Index
|
11
11
|
module Witchcraft
|
12
12
|
extend ActiveSupport::Concern
|
13
13
|
|
@@ -43,10 +43,10 @@ module Chewy
|
|
43
43
|
class Cauldron
|
44
44
|
attr_reader :locals
|
45
45
|
|
46
|
-
# @param
|
46
|
+
# @param index [Chewy::Index] index for composition
|
47
47
|
# @param fields [Array<Symbol>] restricts the fields for composition
|
48
|
-
def initialize(
|
49
|
-
@
|
48
|
+
def initialize(index, fields: [])
|
49
|
+
@index = index
|
50
50
|
@locals = []
|
51
51
|
@fields = fields
|
52
52
|
end
|
@@ -58,9 +58,9 @@ module Chewy
|
|
58
58
|
private
|
59
59
|
|
60
60
|
def alicorn
|
61
|
-
@alicorn ||= class_eval <<-RUBY, __FILE__, __LINE__ + 1
|
61
|
+
@alicorn ||= singleton_class.class_eval <<-RUBY, __FILE__, __LINE__ + 1
|
62
62
|
-> (locals, object0, crutches) do
|
63
|
-
#{composed_values(@
|
63
|
+
#{composed_values(@index.root, 0)}
|
64
64
|
end
|
65
65
|
RUBY
|
66
66
|
end
|
@@ -79,7 +79,9 @@ module Chewy
|
|
79
79
|
if field.children.present? && !field.multi_field?
|
80
80
|
<<-RUBY
|
81
81
|
(result#{nesting} = #{fetcher}
|
82
|
-
if result#{nesting}.
|
82
|
+
if result#{nesting}.nil?
|
83
|
+
nil
|
84
|
+
elsif result#{nesting}.respond_to?(:to_ary)
|
83
85
|
result#{nesting}.map do |object#{nesting}|
|
84
86
|
#{composed_values(field, nesting)}
|
85
87
|
end
|
@@ -139,6 +141,7 @@ module Chewy
|
|
139
141
|
|
140
142
|
def non_proc_fields_for(parent, nesting)
|
141
143
|
return [] unless parent
|
144
|
+
|
142
145
|
fields = (parent.children || []).reject { |field| field.value.is_a?(Proc) }
|
143
146
|
|
144
147
|
if nesting.zero? && @fields.present?
|
@@ -150,6 +153,7 @@ module Chewy
|
|
150
153
|
|
151
154
|
def proc_fields_for(parent, nesting)
|
152
155
|
return [] unless parent
|
156
|
+
|
153
157
|
fields = (parent.children || []).select { |field| field.value.is_a?(Proc) }
|
154
158
|
|
155
159
|
if nesting.zero? && @fields.present?
|
@@ -171,7 +175,7 @@ module Chewy
|
|
171
175
|
if proc.arity.zero?
|
172
176
|
source = replace_self(source, :"object#{nesting}")
|
173
177
|
source = replace_send(source, :"object#{nesting}")
|
174
|
-
elsif proc.arity
|
178
|
+
elsif proc.arity.negative?
|
175
179
|
raise "Splat arguments are unsupported by witchcraft:\n`#{proc.source}"
|
176
180
|
else
|
177
181
|
(nesting + 1).times do |n|
|
@@ -190,6 +194,7 @@ module Chewy
|
|
190
194
|
|
191
195
|
def exctract_lambdas(node)
|
192
196
|
return unless node.is_a?(Parser::AST::Node)
|
197
|
+
|
193
198
|
if node.type == :block && node.children[0].type == :send && node.children[0].to_a == [nil, :lambda]
|
194
199
|
[node.children[2]]
|
195
200
|
else
|
@@ -212,7 +217,7 @@ module Chewy
|
|
212
217
|
def replace_send(node, variable)
|
213
218
|
if node.is_a?(Parser::AST::Node)
|
214
219
|
if node.type == :send && node.children[0].nil?
|
215
|
-
node.updated(nil, [Parser::AST::Node.new(:lvar, [variable]), *node.children[1
|
220
|
+
node.updated(nil, [Parser::AST::Node.new(:lvar, [variable]), *node.children[1..]])
|
216
221
|
else
|
217
222
|
node.updated(nil, node.children.map { |child| replace_send(child, variable) })
|
218
223
|
end
|
@@ -251,6 +256,7 @@ module Chewy
|
|
251
256
|
|
252
257
|
def binding_variable_list(node)
|
253
258
|
return unless node.is_a?(Parser::AST::Node)
|
259
|
+
|
254
260
|
if node.type == :send && node.children[0].nil?
|
255
261
|
node.children[1]
|
256
262
|
else
|
@@ -1,5 +1,5 @@
|
|
1
1
|
module Chewy
|
2
|
-
class
|
2
|
+
class Index
|
3
3
|
module Wrapper
|
4
4
|
extend ActiveSupport::Concern
|
5
5
|
|
@@ -28,7 +28,7 @@ module Chewy
|
|
28
28
|
def ==(other)
|
29
29
|
return true if super
|
30
30
|
|
31
|
-
if other.is_a?(Chewy::
|
31
|
+
if other.is_a?(Chewy::Index)
|
32
32
|
self.class == other.class && (respond_to?(:id) ? id == other.id : attributes == other.attributes)
|
33
33
|
elsif other.respond_to?(:id)
|
34
34
|
self.class.adapter.target.is_a?(Class) &&
|
@@ -49,6 +49,8 @@ module Chewy
|
|
49
49
|
m = method.to_s
|
50
50
|
if (name = highlight_name(m))
|
51
51
|
highlight(name)
|
52
|
+
elsif (name = highlight_names(m))
|
53
|
+
highlights(name)
|
52
54
|
elsif @attributes.key?(m)
|
53
55
|
@attributes[m]
|
54
56
|
elsif attribute_defined?(m)
|
@@ -60,7 +62,7 @@ module Chewy
|
|
60
62
|
|
61
63
|
def respond_to_missing?(method, include_private = false)
|
62
64
|
m = method.to_s
|
63
|
-
highlight_name(m) || @attributes.key?(m) || attribute_defined?(m) || super
|
65
|
+
highlight_name(m) || highlight_names(m) || @attributes.key?(m) || attribute_defined?(m) || super
|
64
66
|
end
|
65
67
|
|
66
68
|
private
|
@@ -69,6 +71,10 @@ module Chewy
|
|
69
71
|
method.sub(/_highlight\z/, '') if method.end_with?('_highlight')
|
70
72
|
end
|
71
73
|
|
74
|
+
def highlight_names(method)
|
75
|
+
method.sub(/_highlights\z/, '') if method.end_with?('_highlights')
|
76
|
+
end
|
77
|
+
|
72
78
|
def attribute_defined?(attribute)
|
73
79
|
self.class.root && self.class.root.children.find { |a| a.name.to_s == attribute }.present?
|
74
80
|
end
|
@@ -77,6 +83,10 @@ module Chewy
|
|
77
83
|
_data['highlight'][attribute].first if highlight?(attribute)
|
78
84
|
end
|
79
85
|
|
86
|
+
def highlights(attribute)
|
87
|
+
_data['highlight'][attribute] if highlight?(attribute)
|
88
|
+
end
|
89
|
+
|
80
90
|
def highlight?(attribute)
|
81
91
|
_data.key?('highlight') && _data['highlight'].key?(attribute)
|
82
92
|
end
|