chewy 6.0.0 → 7.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/CODEOWNERS +1 -0
- data/.github/ISSUE_TEMPLATE/bug_report.md +39 -0
- data/.github/ISSUE_TEMPLATE/feature_request.md +20 -0
- data/.github/PULL_REQUEST_TEMPLATE.md +16 -0
- data/.github/dependabot.yml +42 -0
- data/.github/workflows/ruby.yml +48 -0
- data/.rubocop.yml +16 -8
- data/.rubocop_todo.yml +110 -22
- data/CHANGELOG.md +385 -105
- data/CODE_OF_CONDUCT.md +14 -0
- data/CONTRIBUTING.md +63 -0
- data/Gemfile +4 -10
- data/Guardfile +3 -1
- data/README.md +494 -275
- data/chewy.gemspec +5 -20
- data/gemfiles/base.gemfile +12 -0
- data/gemfiles/rails.6.1.activerecord.gemfile +10 -15
- data/gemfiles/rails.7.0.activerecord.gemfile +14 -0
- data/gemfiles/rails.7.1.activerecord.gemfile +14 -0
- data/lib/chewy/config.rb +58 -50
- data/lib/chewy/elastic_client.rb +31 -0
- data/lib/chewy/errors.rb +7 -10
- data/lib/chewy/fields/base.rb +79 -13
- data/lib/chewy/fields/root.rb +4 -14
- data/lib/chewy/index/actions.rb +54 -37
- data/lib/chewy/{type → index}/adapter/active_record.rb +30 -6
- data/lib/chewy/{type → index}/adapter/base.rb +2 -3
- data/lib/chewy/{type → index}/adapter/object.rb +27 -31
- data/lib/chewy/{type → index}/adapter/orm.rb +17 -18
- data/lib/chewy/index/aliases.rb +14 -5
- data/lib/chewy/index/crutch.rb +40 -0
- data/lib/chewy/index/import/bulk_builder.rb +311 -0
- data/lib/chewy/{type → index}/import/bulk_request.rb +6 -7
- data/lib/chewy/{type → index}/import/journal_builder.rb +11 -12
- data/lib/chewy/{type → index}/import/routine.rb +18 -17
- data/lib/chewy/{type → index}/import.rb +76 -32
- data/lib/chewy/{type → index}/mapping.rb +29 -34
- data/lib/chewy/index/observe/active_record_methods.rb +87 -0
- data/lib/chewy/index/observe/callback.rb +34 -0
- data/lib/chewy/index/observe.rb +17 -0
- data/lib/chewy/index/specification.rb +1 -0
- data/lib/chewy/{type → index}/syncer.rb +59 -59
- data/lib/chewy/{type → index}/witchcraft.rb +11 -7
- data/lib/chewy/{type → index}/wrapper.rb +2 -2
- data/lib/chewy/index.rb +67 -94
- data/lib/chewy/journal.rb +25 -14
- data/lib/chewy/log_subscriber.rb +5 -1
- data/lib/chewy/minitest/helpers.rb +86 -13
- data/lib/chewy/minitest/search_index_receiver.rb +24 -26
- data/lib/chewy/railtie.rb +6 -20
- data/lib/chewy/rake_helper.rb +169 -113
- data/lib/chewy/rspec/build_query.rb +12 -0
- data/lib/chewy/rspec/helpers.rb +55 -0
- data/lib/chewy/rspec/update_index.rb +55 -44
- data/lib/chewy/rspec.rb +2 -0
- data/lib/chewy/runtime/version.rb +1 -1
- data/lib/chewy/runtime.rb +1 -1
- data/lib/chewy/search/loader.rb +19 -41
- data/lib/chewy/search/parameters/collapse.rb +16 -0
- data/lib/chewy/search/parameters/concerns/query_storage.rb +2 -2
- data/lib/chewy/search/parameters/ignore_unavailable.rb +27 -0
- data/lib/chewy/search/parameters/indices.rb +13 -58
- data/lib/chewy/search/parameters/knn.rb +16 -0
- data/lib/chewy/search/parameters/order.rb +6 -19
- data/lib/chewy/search/parameters/source.rb +5 -1
- data/lib/chewy/search/parameters/storage.rb +1 -1
- data/lib/chewy/search/parameters/track_total_hits.rb +16 -0
- data/lib/chewy/search/parameters.rb +6 -4
- data/lib/chewy/search/query_proxy.rb +9 -2
- data/lib/chewy/search/request.rb +169 -134
- data/lib/chewy/search/response.rb +5 -5
- data/lib/chewy/search/scoping.rb +7 -8
- data/lib/chewy/search/scrolling.rb +13 -13
- data/lib/chewy/search.rb +9 -19
- data/lib/chewy/stash.rb +19 -30
- data/lib/chewy/strategy/active_job.rb +1 -1
- data/lib/chewy/strategy/atomic_no_refresh.rb +18 -0
- data/lib/chewy/strategy/base.rb +10 -0
- data/lib/chewy/strategy/delayed_sidekiq/scheduler.rb +151 -0
- data/lib/chewy/strategy/delayed_sidekiq/worker.rb +52 -0
- data/lib/chewy/strategy/delayed_sidekiq.rb +30 -0
- data/lib/chewy/strategy/lazy_sidekiq.rb +64 -0
- data/lib/chewy/strategy/sidekiq.rb +2 -1
- data/lib/chewy/strategy.rb +6 -19
- data/lib/chewy/version.rb +1 -1
- data/lib/chewy.rb +39 -86
- data/lib/generators/chewy/install_generator.rb +1 -1
- data/lib/tasks/chewy.rake +36 -32
- data/migration_guide.md +46 -8
- data/spec/chewy/config_spec.rb +14 -39
- data/spec/chewy/elastic_client_spec.rb +26 -0
- data/spec/chewy/fields/base_spec.rb +432 -147
- data/spec/chewy/fields/root_spec.rb +20 -28
- data/spec/chewy/fields/time_fields_spec.rb +5 -5
- data/spec/chewy/index/actions_spec.rb +368 -59
- data/spec/chewy/{type → index}/adapter/active_record_spec.rb +156 -40
- data/spec/chewy/{type → index}/adapter/object_spec.rb +21 -6
- data/spec/chewy/index/aliases_spec.rb +3 -3
- data/spec/chewy/index/import/bulk_builder_spec.rb +494 -0
- data/spec/chewy/{type → index}/import/bulk_request_spec.rb +5 -12
- data/spec/chewy/{type → index}/import/journal_builder_spec.rb +9 -19
- data/spec/chewy/{type → index}/import/routine_spec.rb +19 -19
- data/spec/chewy/{type → index}/import_spec.rb +164 -98
- data/spec/chewy/index/mapping_spec.rb +135 -0
- data/spec/chewy/index/observe/active_record_methods_spec.rb +68 -0
- data/spec/chewy/index/observe/callback_spec.rb +139 -0
- data/spec/chewy/index/observe_spec.rb +143 -0
- data/spec/chewy/index/settings_spec.rb +3 -1
- data/spec/chewy/index/specification_spec.rb +20 -30
- data/spec/chewy/{type → index}/syncer_spec.rb +14 -19
- data/spec/chewy/{type → index}/witchcraft_spec.rb +20 -22
- data/spec/chewy/index/wrapper_spec.rb +100 -0
- data/spec/chewy/index_spec.rb +60 -105
- data/spec/chewy/journal_spec.rb +25 -74
- data/spec/chewy/minitest/helpers_spec.rb +123 -15
- data/spec/chewy/minitest/search_index_receiver_spec.rb +28 -30
- data/spec/chewy/multi_search_spec.rb +4 -5
- data/spec/chewy/rake_helper_spec.rb +315 -55
- data/spec/chewy/rspec/build_query_spec.rb +34 -0
- data/spec/chewy/rspec/helpers_spec.rb +61 -0
- data/spec/chewy/rspec/update_index_spec.rb +74 -71
- data/spec/chewy/runtime_spec.rb +2 -2
- data/spec/chewy/search/loader_spec.rb +19 -53
- data/spec/chewy/search/pagination/kaminari_examples.rb +4 -6
- data/spec/chewy/search/pagination/kaminari_spec.rb +2 -2
- data/spec/chewy/search/parameters/collapse_spec.rb +5 -0
- data/spec/chewy/search/parameters/ignore_unavailable_spec.rb +67 -0
- data/spec/chewy/search/parameters/indices_spec.rb +26 -117
- data/spec/chewy/search/parameters/knn_spec.rb +5 -0
- data/spec/chewy/search/parameters/order_spec.rb +18 -11
- data/spec/chewy/search/parameters/query_storage_examples.rb +67 -21
- data/spec/chewy/search/parameters/search_after_spec.rb +4 -1
- data/spec/chewy/search/parameters/source_spec.rb +8 -2
- data/spec/chewy/search/parameters/track_total_hits_spec.rb +5 -0
- data/spec/chewy/search/parameters_spec.rb +18 -4
- data/spec/chewy/search/query_proxy_spec.rb +68 -17
- data/spec/chewy/search/request_spec.rb +292 -110
- data/spec/chewy/search/response_spec.rb +12 -12
- data/spec/chewy/search/scrolling_spec.rb +10 -17
- data/spec/chewy/search_spec.rb +40 -34
- data/spec/chewy/stash_spec.rb +9 -21
- data/spec/chewy/strategy/active_job_spec.rb +16 -16
- data/spec/chewy/strategy/atomic_no_refresh_spec.rb +60 -0
- data/spec/chewy/strategy/atomic_spec.rb +9 -10
- data/spec/chewy/strategy/delayed_sidekiq_spec.rb +202 -0
- data/spec/chewy/strategy/lazy_sidekiq_spec.rb +214 -0
- data/spec/chewy/strategy/sidekiq_spec.rb +12 -12
- data/spec/chewy/strategy_spec.rb +19 -15
- data/spec/chewy_spec.rb +24 -107
- data/spec/spec_helper.rb +3 -22
- data/spec/support/active_record.rb +25 -7
- metadata +78 -339
- data/.circleci/config.yml +0 -240
- data/Appraisals +0 -81
- data/gemfiles/rails.5.2.activerecord.gemfile +0 -17
- data/gemfiles/rails.5.2.mongoid.6.4.gemfile +0 -17
- data/gemfiles/rails.6.0.activerecord.gemfile +0 -17
- data/gemfiles/sequel.4.45.gemfile +0 -11
- data/lib/chewy/backports/deep_dup.rb +0 -46
- data/lib/chewy/backports/duplicable.rb +0 -91
- data/lib/chewy/search/pagination/will_paginate.rb +0 -43
- data/lib/chewy/search/parameters/types.rb +0 -20
- data/lib/chewy/strategy/resque.rb +0 -27
- data/lib/chewy/strategy/shoryuken.rb +0 -40
- data/lib/chewy/type/actions.rb +0 -43
- data/lib/chewy/type/adapter/mongoid.rb +0 -67
- data/lib/chewy/type/adapter/sequel.rb +0 -93
- data/lib/chewy/type/crutch.rb +0 -32
- data/lib/chewy/type/import/bulk_builder.rb +0 -122
- data/lib/chewy/type/observe.rb +0 -82
- data/lib/chewy/type.rb +0 -120
- data/lib/sequel/plugins/chewy_observe.rb +0 -63
- data/spec/chewy/search/pagination/will_paginate_examples.rb +0 -63
- data/spec/chewy/search/pagination/will_paginate_spec.rb +0 -23
- data/spec/chewy/search/parameters/types_spec.rb +0 -5
- data/spec/chewy/strategy/resque_spec.rb +0 -46
- data/spec/chewy/strategy/shoryuken_spec.rb +0 -70
- data/spec/chewy/type/actions_spec.rb +0 -50
- data/spec/chewy/type/adapter/mongoid_spec.rb +0 -372
- data/spec/chewy/type/adapter/sequel_spec.rb +0 -472
- data/spec/chewy/type/import/bulk_builder_spec.rb +0 -194
- data/spec/chewy/type/mapping_spec.rb +0 -175
- data/spec/chewy/type/observe_spec.rb +0 -137
- data/spec/chewy/type/wrapper_spec.rb +0 -100
- data/spec/chewy/type_spec.rb +0 -55
- data/spec/support/mongoid.rb +0 -93
- data/spec/support/sequel.rb +0 -80
@@ -23,7 +23,7 @@ module Chewy
|
|
23
23
|
#
|
24
24
|
# @return [Integer]
|
25
25
|
def total
|
26
|
-
@total ||= hits_root
|
26
|
+
@total ||= hits_root.fetch('total', {}).fetch('value', 0)
|
27
27
|
end
|
28
28
|
|
29
29
|
# Response `max_score` field.
|
@@ -64,12 +64,12 @@ module Chewy
|
|
64
64
|
end
|
65
65
|
alias_method :aggregations, :aggs
|
66
66
|
|
67
|
-
# {Chewy::
|
67
|
+
# {Chewy::Index} wrappers collection instantiated on top of hits.
|
68
68
|
#
|
69
|
-
# @return [Array<Chewy::
|
69
|
+
# @return [Array<Chewy::Index>]
|
70
70
|
def wrappers
|
71
71
|
@wrappers ||= hits.map do |hit|
|
72
|
-
@loader.
|
72
|
+
@loader.derive_index(hit['_index']).build(hit)
|
73
73
|
end
|
74
74
|
end
|
75
75
|
|
@@ -102,7 +102,7 @@ module Chewy
|
|
102
102
|
# end
|
103
103
|
# @see #wrappers
|
104
104
|
# @see #objects
|
105
|
-
# @return [{Chewy::
|
105
|
+
# @return [{Chewy::Index => Object}] a hash with wrappers as keys and ORM/ODM objects as values
|
106
106
|
def object_hash
|
107
107
|
@object_hash ||= wrappers.zip(objects).to_h
|
108
108
|
end
|
data/lib/chewy/search/scoping.rb
CHANGED
@@ -9,17 +9,16 @@ module Chewy
|
|
9
9
|
# query(match: {name: name})
|
10
10
|
# end
|
11
11
|
#
|
12
|
-
#
|
13
|
-
#
|
14
|
-
#
|
15
|
-
# end
|
12
|
+
#
|
13
|
+
# def self.by_age(age)
|
14
|
+
# filter(term: {age: age})
|
16
15
|
# end
|
17
16
|
# end
|
18
17
|
#
|
19
18
|
# UsersIndex.limit(10).by_name('Martin')
|
20
19
|
# # => <UsersIndex::Query {..., :body=>{:size=>10, :query=>{:match=>{:name=>"Martin"}}}}>
|
21
|
-
# UsersIndex
|
22
|
-
# # => <UsersIndex::
|
20
|
+
# UsersIndex.limit(10).by_name('Martin').by_age(42)
|
21
|
+
# # => <UsersIndex::Query {..., :body=>{:size=>10, :query=>{:bool=>{
|
23
22
|
# # :must=>{:match=>{:name=>"Martin"}},
|
24
23
|
# # :filter=>{:term=>{:age=>42}}}}}}>
|
25
24
|
module Scoping
|
@@ -28,9 +27,9 @@ module Chewy
|
|
28
27
|
module ClassMethods
|
29
28
|
# The scopes stack.
|
30
29
|
#
|
31
|
-
# @return [Array<Chewy::Search::
|
30
|
+
# @return [Array<Chewy::Search::Request>] array of scopes
|
32
31
|
def scopes
|
33
|
-
|
32
|
+
Chewy.current[:chewy_scopes] ||= []
|
34
33
|
end
|
35
34
|
end
|
36
35
|
|
@@ -28,7 +28,7 @@ module Chewy
|
|
28
28
|
return enum_for(:scroll_batches, batch_size: batch_size, scroll: scroll) unless block_given?
|
29
29
|
|
30
30
|
result = perform(size: batch_size, scroll: scroll)
|
31
|
-
total = [raw_limit_value, result.fetch('hits', {}).fetch('total', 0)].compact.min
|
31
|
+
total = [raw_limit_value, result.fetch('hits', {}).fetch('total', {}).fetch('value', 0)].compact.min
|
32
32
|
last_batch_size = total % batch_size
|
33
33
|
fetched = 0
|
34
34
|
scroll_id = nil
|
@@ -40,10 +40,11 @@ module Chewy
|
|
40
40
|
yield(hits) if hits.present?
|
41
41
|
scroll_id = result['_scroll_id']
|
42
42
|
break if fetched >= total
|
43
|
+
|
43
44
|
result = perform_scroll(scroll: scroll, scroll_id: scroll_id)
|
44
45
|
end
|
45
46
|
ensure
|
46
|
-
Chewy.client.clear_scroll(scroll_id: scroll_id) if scroll_id
|
47
|
+
Chewy.client.clear_scroll(body: {scroll_id: scroll_id}) if scroll_id
|
47
48
|
end
|
48
49
|
|
49
50
|
# @!method scroll_hits(batch_size: 1000, scroll: '1m')
|
@@ -61,17 +62,17 @@ module Chewy
|
|
61
62
|
# @example
|
62
63
|
# PlaceIndex.scroll_hits.map { |hit| hit['_id'] }
|
63
64
|
# @return [Enumerator] a standard ruby Enumerator
|
64
|
-
def scroll_hits(**options)
|
65
|
+
def scroll_hits(**options, &block)
|
65
66
|
return enum_for(:scroll_hits, **options) unless block_given?
|
66
67
|
|
67
68
|
scroll_batches(**options).each do |batch|
|
68
|
-
batch.each
|
69
|
+
batch.each(&block)
|
69
70
|
end
|
70
71
|
end
|
71
72
|
|
72
73
|
# @!method scroll_wrappers(batch_size: 1000, scroll: '1m')
|
73
74
|
# Iterates through the documents of the scope in batches. Yields
|
74
|
-
# each hit wrapped with {Chewy::
|
75
|
+
# each hit wrapped with {Chewy::Index}.
|
75
76
|
#
|
76
77
|
# @param batch_size [Integer] batch size obviously, replaces `size` query parameter
|
77
78
|
# @param scroll [String] cursor expiration time
|
@@ -79,7 +80,7 @@ module Chewy
|
|
79
80
|
# @overload scroll_wrappers(batch_size: 1000, scroll: '1m')
|
80
81
|
# @example
|
81
82
|
# PlaceIndex.scroll_wrappers { |object| p object.id }
|
82
|
-
# @yieldparam object [Chewy::
|
83
|
+
# @yieldparam object [Chewy::Index] block is executed for each hit object
|
83
84
|
#
|
84
85
|
# @overload scroll_wrappers(batch_size: 1000, scroll: '1m')
|
85
86
|
# @example
|
@@ -89,7 +90,7 @@ module Chewy
|
|
89
90
|
return enum_for(:scroll_wrappers, **options) unless block_given?
|
90
91
|
|
91
92
|
scroll_hits(**options).each do |hit|
|
92
|
-
yield loader.
|
93
|
+
yield loader.derive_index(hit['_index']).build(hit)
|
93
94
|
end
|
94
95
|
end
|
95
96
|
|
@@ -113,12 +114,12 @@ module Chewy
|
|
113
114
|
# @example
|
114
115
|
# PlaceIndex.scroll_objects.map { |record| record.id }
|
115
116
|
# @return [Enumerator] a standard ruby Enumerator
|
116
|
-
def scroll_objects(**options)
|
117
|
+
def scroll_objects(**options, &block)
|
117
118
|
return enum_for(:scroll_objects, **options) unless block_given?
|
118
119
|
|
119
120
|
except(:source, :stored_fields, :script_fields, :docvalue_fields)
|
120
121
|
.source(false).scroll_batches(**options).each do |batch|
|
121
|
-
loader.load(batch).each
|
122
|
+
loader.load(batch).each(&block)
|
122
123
|
end
|
123
124
|
end
|
124
125
|
alias_method :scroll_records, :scroll_objects
|
@@ -127,10 +128,9 @@ module Chewy
|
|
127
128
|
private
|
128
129
|
|
129
130
|
def perform_scroll(body)
|
130
|
-
ActiveSupport::Notifications.instrument 'search_query.chewy',
|
131
|
-
|
132
|
-
|
133
|
-
end
|
131
|
+
ActiveSupport::Notifications.instrument 'search_query.chewy', notification_payload(request: body) do
|
132
|
+
Chewy.client.scroll(body)
|
133
|
+
end
|
134
134
|
end
|
135
135
|
end
|
136
136
|
end
|
data/lib/chewy/search.rb
CHANGED
@@ -6,29 +6,22 @@ require 'chewy/search/response'
|
|
6
6
|
require 'chewy/search/loader'
|
7
7
|
require 'chewy/search/request'
|
8
8
|
require 'chewy/search/pagination/kaminari'
|
9
|
-
require 'chewy/search/pagination/will_paginate'
|
10
9
|
|
11
10
|
module Chewy
|
12
11
|
# This module being included to any provides an interface to the
|
13
|
-
# request DSL. By default it is included to {Chewy::Index}
|
14
|
-
# {Chewy::Type}.
|
12
|
+
# request DSL. By default it is included to {Chewy::Index}.
|
15
13
|
#
|
16
14
|
# The class used as a request DSL provider is
|
17
15
|
# inherited from {Chewy::Search::Request}
|
18
16
|
#
|
19
|
-
# Also, the search class is refined with
|
20
|
-
# providing modules: {Chewy::Search::Pagination::Kaminari} or
|
21
|
-
# {Chewy::Search::Pagination::WillPaginate}.
|
17
|
+
# Also, the search class is refined with the pagination module {Chewy::Search::Pagination::Kaminari}.
|
22
18
|
#
|
23
19
|
# @example
|
24
20
|
# PlacesIndex.query(match: {name: 'Moscow'})
|
25
|
-
# PlacesIndex::City.query(match: {name: 'Moscow'})
|
26
21
|
# @see Chewy::Index
|
27
|
-
# @see Chewy::Type
|
28
22
|
# @see Chewy::Search::Request
|
29
23
|
# @see Chewy::Search::ClassMethods
|
30
24
|
# @see Chewy::Search::Pagination::Kaminari
|
31
|
-
# @see Chewy::Search::Pagination::WillPaginate
|
32
25
|
module Search
|
33
26
|
extend ActiveSupport::Concern
|
34
27
|
|
@@ -55,15 +48,14 @@ module Chewy
|
|
55
48
|
# @see https://www.elastic.co/guide/en/elasticsearch/reference/current/search-uri-request.html
|
56
49
|
# @return [Hash] the request result
|
57
50
|
def search_string(query, options = {})
|
58
|
-
options = options.merge(all.render.slice(:index
|
51
|
+
options = options.merge(all.render.slice(:index).merge(q: query))
|
59
52
|
Chewy.client.search(options)
|
60
53
|
end
|
61
54
|
|
62
|
-
# Delegates methods from the request class to the index
|
55
|
+
# Delegates methods from the request class to the index class
|
63
56
|
#
|
64
57
|
# @example
|
65
58
|
# PlacesIndex.query(match: {name: 'Moscow'})
|
66
|
-
# PlacesIndex::City.query(match: {name: 'Moscow'})
|
67
59
|
def method_missing(name, *args, &block)
|
68
60
|
if search_class::DELEGATED_METHODS.include?(name)
|
69
61
|
all.send(name, *args, &block)
|
@@ -85,11 +77,6 @@ module Chewy
|
|
85
77
|
def build_search_class(base)
|
86
78
|
search_class = Class.new(base)
|
87
79
|
|
88
|
-
if self < Chewy::Type
|
89
|
-
index_scopes = index.scopes - scopes
|
90
|
-
delegate_scoped index, search_class, index_scopes
|
91
|
-
end
|
92
|
-
|
93
80
|
delegate_scoped self, search_class, scopes
|
94
81
|
const_set('Query', search_class)
|
95
82
|
end
|
@@ -97,9 +84,12 @@ module Chewy
|
|
97
84
|
def delegate_scoped(source, destination, methods)
|
98
85
|
methods.each do |method|
|
99
86
|
destination.class_eval do
|
100
|
-
define_method method do |*args, &block|
|
101
|
-
scoping
|
87
|
+
define_method method do |*args, **kwargs, &block|
|
88
|
+
scoping do
|
89
|
+
source.public_send(method, *args, **kwargs, &block)
|
90
|
+
end
|
102
91
|
end
|
92
|
+
method
|
103
93
|
end
|
104
94
|
end
|
105
95
|
end
|
data/lib/chewy/stash.rb
CHANGED
@@ -9,11 +9,9 @@ module Chewy
|
|
9
9
|
class Specification < Chewy::Index
|
10
10
|
index_name 'chewy_specifications'
|
11
11
|
|
12
|
-
|
13
|
-
default_import_options journal: false
|
12
|
+
default_import_options journal: false
|
14
13
|
|
15
|
-
|
16
|
-
end
|
14
|
+
field :specification, type: 'binary'
|
17
15
|
end
|
18
16
|
|
19
17
|
class Journal < Chewy::Index
|
@@ -24,18 +22,18 @@ module Chewy
|
|
24
22
|
# @param since_time [Time, DateTime] a timestamp from which we load a journal
|
25
23
|
# @param only [Chewy::Index, Array<Chewy::Index>] journal entries related to these indices will be loaded only
|
26
24
|
def self.entries(since_time, only: [])
|
27
|
-
self.for(only).filter(range: {created_at: {gt: since_time}})
|
25
|
+
self.for(only).filter(range: {created_at: {gt: since_time}}).filter.minimum_should_match(1)
|
28
26
|
end
|
29
27
|
|
30
28
|
# Cleans up all the journal entries until the specified time. If nothing is
|
31
29
|
# specified - cleans up everything.
|
32
30
|
#
|
33
|
-
# @param
|
31
|
+
# @param until_time [Time, DateTime] Clean everything before that date
|
34
32
|
# @param only [Chewy::Index, Array<Chewy::Index>] indexes to clean up journal entries for
|
35
|
-
def self.clean(until_time = nil, only: [])
|
33
|
+
def self.clean(until_time = nil, only: [], delete_by_query_options: {})
|
36
34
|
scope = self.for(only)
|
37
35
|
scope = scope.filter(range: {created_at: {lte: until_time}}) if until_time
|
38
|
-
scope.delete_all
|
36
|
+
scope.delete_all(**delete_by_query_options)
|
39
37
|
end
|
40
38
|
|
41
39
|
# Selects all the journal entries for the specified indices.
|
@@ -43,35 +41,26 @@ module Chewy
|
|
43
41
|
# @param indices [Chewy::Index, Array<Chewy::Index>]
|
44
42
|
def self.for(*something)
|
45
43
|
something = something.flatten.compact
|
46
|
-
|
47
|
-
return none if something.present? &&
|
44
|
+
indexes = something.flat_map { |s| Chewy.derive_name(s) }
|
45
|
+
return none if something.present? && indexes.blank?
|
46
|
+
|
48
47
|
scope = all
|
49
|
-
|
50
|
-
scope = scope.or(
|
51
|
-
filter(term: {index_name: index.derivable_name})
|
52
|
-
.filter(terms: {type_name: index_types.map(&:type_name)})
|
53
|
-
)
|
48
|
+
indexes.each do |index|
|
49
|
+
scope = scope.or(filter(term: {index_name: index.derivable_name}))
|
54
50
|
end
|
55
51
|
scope
|
56
52
|
end
|
57
53
|
|
58
|
-
|
59
|
-
default_import_options journal: false
|
54
|
+
default_import_options journal: false
|
60
55
|
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
field :created_at, type: 'date'
|
66
|
-
|
67
|
-
def type
|
68
|
-
@type ||= Chewy.derive_type("#{index_name}##{type_name}")
|
69
|
-
end
|
56
|
+
field :index_name, type: 'keyword'
|
57
|
+
field :action, type: 'keyword'
|
58
|
+
field :references, type: 'binary'
|
59
|
+
field :created_at, type: 'date'
|
70
60
|
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
end
|
61
|
+
def references
|
62
|
+
@references ||= Array.wrap(@attributes['references']).map do |item|
|
63
|
+
JSON.load(Base64.decode64(item)) # rubocop:disable Security/JSONLoad
|
75
64
|
end
|
76
65
|
end
|
77
66
|
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
module Chewy
|
2
|
+
class Strategy
|
3
|
+
# This strategy works like atomic but import objects with `refresh=false` parameter.
|
4
|
+
#
|
5
|
+
# Chewy.strategy(:atomic_no_refresh) do
|
6
|
+
# User.all.map(&:save) # Does nothing here
|
7
|
+
# Post.all.map(&:save) # And here
|
8
|
+
# # It imports all the changed users and posts right here
|
9
|
+
# # before block leaving with bulk ES API, kinda optimization
|
10
|
+
# end
|
11
|
+
#
|
12
|
+
class AtomicNoRefresh < Atomic
|
13
|
+
def leave
|
14
|
+
@stash.all? { |type, ids| type.import!(ids, refresh: false) }
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
data/lib/chewy/strategy/base.rb
CHANGED
@@ -22,6 +22,16 @@ module Chewy
|
|
22
22
|
# strategies stack
|
23
23
|
#
|
24
24
|
def leave; end
|
25
|
+
|
26
|
+
# This method called when some model record is created or updated.
|
27
|
+
# Normally it will just evaluate all the Chewy callbacks and pass results
|
28
|
+
# to current strategy's update method.
|
29
|
+
# However it's possible to override it to achieve delayed evaluation of
|
30
|
+
# callbacks, e.g. using sidekiq.
|
31
|
+
#
|
32
|
+
def update_chewy_indices(object)
|
33
|
+
object.run_chewy_callbacks
|
34
|
+
end
|
25
35
|
end
|
26
36
|
end
|
27
37
|
end
|
@@ -0,0 +1,151 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative '../../index'
|
4
|
+
|
5
|
+
# The class is responsible for accumulating in redis [type, ids]
|
6
|
+
# that were requested to be reindexed during `latency` seconds.
|
7
|
+
# The reindex job is going to be scheduled after a `latency` seconds.
|
8
|
+
# that job is going to read accumulated [type, ids] from the redis
|
9
|
+
# and reindex all them at once.
|
10
|
+
module Chewy
|
11
|
+
class Strategy
|
12
|
+
class DelayedSidekiq
|
13
|
+
require_relative 'worker'
|
14
|
+
|
15
|
+
class Scheduler
|
16
|
+
DEFAULT_TTL = 60 * 60 * 24 # in seconds
|
17
|
+
DEFAULT_LATENCY = 10
|
18
|
+
DEFAULT_MARGIN = 2
|
19
|
+
DEFAULT_QUEUE = 'chewy'
|
20
|
+
KEY_PREFIX = 'chewy:delayed_sidekiq'
|
21
|
+
ALL_SETS_KEY = "#{KEY_PREFIX}:all_sets".freeze
|
22
|
+
FALLBACK_FIELDS = 'all'
|
23
|
+
FIELDS_IDS_SEPARATOR = ';'
|
24
|
+
IDS_SEPARATOR = ','
|
25
|
+
|
26
|
+
def initialize(type, ids, options = {})
|
27
|
+
@type = type
|
28
|
+
@ids = ids
|
29
|
+
@options = options
|
30
|
+
end
|
31
|
+
|
32
|
+
# the diagram:
|
33
|
+
#
|
34
|
+
# inputs:
|
35
|
+
# latency == 2
|
36
|
+
# reindex_time = Time.current
|
37
|
+
#
|
38
|
+
# Parallel OR Sequential triggers of reindex: | What is going on in reindex store (Redis):
|
39
|
+
# --------------------------------------------------------------------------------------------------
|
40
|
+
# |
|
41
|
+
# process 1 (reindex_time): | chewy:delayed_sidekiq:CitiesIndex:1679347866 = [1]
|
42
|
+
# Schedule.new(CitiesIndex, [1]).postpone | chewy:delayed_sidekiq:timechunks = [{ score: 1679347866, "chewy:delayed_sidekiq:CitiesIndex:1679347866"}]
|
43
|
+
# | & schedule a DelayedSidekiq::Worker at 1679347869 (at + 3)
|
44
|
+
# | it will zpop chewy:delayed_sidekiq:timechunks up to 1679347866 score and reindex all ids with zpoped keys
|
45
|
+
# | chewy:delayed_sidekiq:CitiesIndex:1679347866
|
46
|
+
# |
|
47
|
+
# |
|
48
|
+
# process 2 (reindex_time): | chewy:delayed_sidekiq:CitiesIndex:1679347866 = [1, 2]
|
49
|
+
# Schedule.new(CitiesIndex, [2]).postpone | chewy:delayed_sidekiq:timechunks = [{ score: 1679347866, "chewy:delayed_sidekiq:CitiesIndex:1679347866"}]
|
50
|
+
# | & do not schedule a new worker
|
51
|
+
# |
|
52
|
+
# |
|
53
|
+
# process 1 (reindex_time + (latency - 1).seconds): | chewy:delayed_sidekiq:CitiesIndex:1679347866 = [1, 2, 3]
|
54
|
+
# Schedule.new(CitiesIndex, [3]).postpone | chewy:delayed_sidekiq:timechunks = [{ score: 1679347866, "chewy:delayed_sidekiq:CitiesIndex:1679347866"}]
|
55
|
+
# | & do not schedule a new worker
|
56
|
+
# |
|
57
|
+
# |
|
58
|
+
# process 2 (reindex_time + (latency + 1).seconds): | chewy:delayed_sidekiq:CitiesIndex:1679347866 = [1, 2, 3]
|
59
|
+
# Schedule.new(CitiesIndex, [4]).postpone | chewy:delayed_sidekiq:CitiesIndex:1679347868 = [4]
|
60
|
+
# | chewy:delayed_sidekiq:timechunks = [
|
61
|
+
# | { score: 1679347866, "chewy:delayed_sidekiq:CitiesIndex:1679347866"}
|
62
|
+
# | { score: 1679347868, "chewy:delayed_sidekiq:CitiesIndex:1679347868"}
|
63
|
+
# | ]
|
64
|
+
# | & schedule a DelayedSidekiq::Worker at 1679347871 (at + 3)
|
65
|
+
# | it will zpop chewy:delayed_sidekiq:timechunks up to 1679347868 score and reindex all ids with zpoped keys
|
66
|
+
# | chewy:delayed_sidekiq:CitiesIndex:1679347866 (in case of failed previous reindex),
|
67
|
+
# | chewy:delayed_sidekiq:CitiesIndex:1679347868
|
68
|
+
def postpone
|
69
|
+
::Sidekiq.redis do |redis|
|
70
|
+
# warning: Redis#sadd will always return an Integer in Redis 5.0.0. Use Redis#sadd? instead
|
71
|
+
if redis.respond_to?(:sadd?)
|
72
|
+
redis.sadd?(ALL_SETS_KEY, timechunks_key)
|
73
|
+
redis.sadd?(timechunk_key, serialize_data)
|
74
|
+
else
|
75
|
+
redis.sadd(ALL_SETS_KEY, timechunks_key)
|
76
|
+
redis.sadd(timechunk_key, serialize_data)
|
77
|
+
end
|
78
|
+
|
79
|
+
redis.expire(timechunk_key, ttl)
|
80
|
+
|
81
|
+
unless redis.zrank(timechunks_key, timechunk_key)
|
82
|
+
redis.zadd(timechunks_key, at, timechunk_key)
|
83
|
+
redis.expire(timechunks_key, ttl)
|
84
|
+
|
85
|
+
::Sidekiq::Client.push(
|
86
|
+
'queue' => sidekiq_queue,
|
87
|
+
'at' => at + margin,
|
88
|
+
'class' => Chewy::Strategy::DelayedSidekiq::Worker,
|
89
|
+
'args' => [type_name, at]
|
90
|
+
)
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
private
|
96
|
+
|
97
|
+
attr_reader :type, :ids, :options
|
98
|
+
|
99
|
+
# this method returns predictable value that jumps by latency value
|
100
|
+
# another words each latency seconds it return the same value
|
101
|
+
def at
|
102
|
+
@at ||= begin
|
103
|
+
schedule_at = latency.seconds.from_now.to_f
|
104
|
+
|
105
|
+
(schedule_at - (schedule_at % latency)).to_i
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
def fields
|
110
|
+
options[:update_fields].presence || [FALLBACK_FIELDS]
|
111
|
+
end
|
112
|
+
|
113
|
+
def timechunks_key
|
114
|
+
"#{KEY_PREFIX}:#{type_name}:timechunks"
|
115
|
+
end
|
116
|
+
|
117
|
+
def timechunk_key
|
118
|
+
"#{KEY_PREFIX}:#{type_name}:#{at}"
|
119
|
+
end
|
120
|
+
|
121
|
+
def serialize_data
|
122
|
+
[ids.join(IDS_SEPARATOR), fields.join(IDS_SEPARATOR)].join(FIELDS_IDS_SEPARATOR)
|
123
|
+
end
|
124
|
+
|
125
|
+
def type_name
|
126
|
+
type.name
|
127
|
+
end
|
128
|
+
|
129
|
+
def latency
|
130
|
+
strategy_config.latency || DEFAULT_LATENCY
|
131
|
+
end
|
132
|
+
|
133
|
+
def margin
|
134
|
+
strategy_config.margin || DEFAULT_MARGIN
|
135
|
+
end
|
136
|
+
|
137
|
+
def ttl
|
138
|
+
strategy_config.ttl || DEFAULT_TTL
|
139
|
+
end
|
140
|
+
|
141
|
+
def sidekiq_queue
|
142
|
+
Chewy.settings.dig(:sidekiq, :queue) || DEFAULT_QUEUE
|
143
|
+
end
|
144
|
+
|
145
|
+
def strategy_config
|
146
|
+
type.strategy_config.delayed_sidekiq
|
147
|
+
end
|
148
|
+
end
|
149
|
+
end
|
150
|
+
end
|
151
|
+
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Chewy
|
4
|
+
class Strategy
|
5
|
+
class DelayedSidekiq
|
6
|
+
class Worker
|
7
|
+
include ::Sidekiq::Worker
|
8
|
+
|
9
|
+
def perform(type, score, options = {})
|
10
|
+
options[:refresh] = !Chewy.disable_refresh_async if Chewy.disable_refresh_async
|
11
|
+
|
12
|
+
::Sidekiq.redis do |redis|
|
13
|
+
timechunks_key = "#{Scheduler::KEY_PREFIX}:#{type}:timechunks"
|
14
|
+
timechunk_keys = redis.zrangebyscore(timechunks_key, -1, score)
|
15
|
+
members = timechunk_keys.flat_map { |timechunk_key| redis.smembers(timechunk_key) }.compact
|
16
|
+
|
17
|
+
# extract ids and fields & do the reset of records
|
18
|
+
ids, fields = extract_ids_and_fields(members)
|
19
|
+
options[:update_fields] = fields if fields
|
20
|
+
|
21
|
+
index = type.constantize
|
22
|
+
index.strategy_config.delayed_sidekiq.reindex_wrapper.call do
|
23
|
+
options.any? ? index.import!(ids, **options) : index.import!(ids)
|
24
|
+
end
|
25
|
+
|
26
|
+
redis.del(timechunk_keys)
|
27
|
+
redis.zremrangebyscore(timechunks_key, -1, score)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
def extract_ids_and_fields(members)
|
34
|
+
ids = []
|
35
|
+
fields = []
|
36
|
+
|
37
|
+
members.each do |member|
|
38
|
+
member_ids, member_fields = member.split(Scheduler::FIELDS_IDS_SEPARATOR).map do |v|
|
39
|
+
v.split(Scheduler::IDS_SEPARATOR)
|
40
|
+
end
|
41
|
+
ids |= member_ids
|
42
|
+
fields |= member_fields
|
43
|
+
end
|
44
|
+
|
45
|
+
fields = nil if fields.include?(Scheduler::FALLBACK_FIELDS)
|
46
|
+
|
47
|
+
[ids.map(&:to_i), fields]
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Chewy
|
4
|
+
class Strategy
|
5
|
+
class DelayedSidekiq < Sidekiq
|
6
|
+
require_relative 'delayed_sidekiq/scheduler'
|
7
|
+
|
8
|
+
# cleanup the redis sets used internally. Useful mainly in tests to avoid
|
9
|
+
# leak and potential flaky tests.
|
10
|
+
def self.clear_timechunks!
|
11
|
+
::Sidekiq.redis do |redis|
|
12
|
+
timechunk_sets = redis.smembers(Chewy::Strategy::DelayedSidekiq::Scheduler::ALL_SETS_KEY)
|
13
|
+
break if timechunk_sets.empty?
|
14
|
+
|
15
|
+
redis.pipelined do |pipeline|
|
16
|
+
timechunk_sets.each { |set| pipeline.del(set) }
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def leave
|
22
|
+
@stash.each do |type, ids|
|
23
|
+
next if ids.empty?
|
24
|
+
|
25
|
+
DelayedSidekiq::Scheduler.new(type, ids).postpone
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,64 @@
|
|
1
|
+
module Chewy
|
2
|
+
class Strategy
|
3
|
+
# The strategy works the same way as sidekiq, but performs
|
4
|
+
# async evaluation of all index callbacks on model create and update
|
5
|
+
# driven by sidekiq
|
6
|
+
#
|
7
|
+
# Chewy.strategy(:lazy_sidekiq) do
|
8
|
+
# User.all.map(&:save) # Does nothing here
|
9
|
+
# Post.all.map(&:save) # And here
|
10
|
+
# # It schedules import of all the changed users and posts right here
|
11
|
+
# end
|
12
|
+
#
|
13
|
+
class LazySidekiq < Sidekiq
|
14
|
+
class IndicesUpdateWorker
|
15
|
+
include ::Sidekiq::Worker
|
16
|
+
|
17
|
+
def perform(models)
|
18
|
+
Chewy.strategy(strategy) do
|
19
|
+
models.each do |model_type, model_ids|
|
20
|
+
model_type.constantize.where(id: model_ids).each(&:run_chewy_callbacks)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
private
|
26
|
+
|
27
|
+
def strategy
|
28
|
+
Chewy.disable_refresh_async ? :atomic_no_refresh : :atomic
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def initialize
|
33
|
+
# Use parent's @stash to store destroyed records, since callbacks for them have to
|
34
|
+
# be run immediately on the strategy block end because we won't be able to fetch
|
35
|
+
# records further in IndicesUpdateWorker. This will be done by avoiding of
|
36
|
+
# LazySidekiq#update_chewy_indices call and calling LazySidekiq#update instead.
|
37
|
+
super
|
38
|
+
|
39
|
+
# @lazy_stash is used to store all the lazy evaluated callbacks with call of
|
40
|
+
# strategy's #update_chewy_indices.
|
41
|
+
@lazy_stash = {}
|
42
|
+
end
|
43
|
+
|
44
|
+
def leave
|
45
|
+
# Fallback to Sidekiq#leave implementation for destroyed records stored in @stash.
|
46
|
+
super
|
47
|
+
|
48
|
+
# Proceed with other records stored in @lazy_stash
|
49
|
+
return if @lazy_stash.empty?
|
50
|
+
|
51
|
+
::Sidekiq::Client.push(
|
52
|
+
'queue' => sidekiq_queue,
|
53
|
+
'class' => Chewy::Strategy::LazySidekiq::IndicesUpdateWorker,
|
54
|
+
'args' => [@lazy_stash]
|
55
|
+
)
|
56
|
+
end
|
57
|
+
|
58
|
+
def update_chewy_indices(object)
|
59
|
+
@lazy_stash[object.class.name] ||= []
|
60
|
+
@lazy_stash[object.class.name] |= Array.wrap(object.id)
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|