chewy 6.0.0 → 7.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (188) hide show
  1. checksums.yaml +4 -4
  2. data/.github/CODEOWNERS +1 -0
  3. data/.github/ISSUE_TEMPLATE/bug_report.md +39 -0
  4. data/.github/ISSUE_TEMPLATE/feature_request.md +20 -0
  5. data/.github/PULL_REQUEST_TEMPLATE.md +16 -0
  6. data/.github/dependabot.yml +42 -0
  7. data/.github/workflows/ruby.yml +48 -0
  8. data/.rubocop.yml +16 -8
  9. data/.rubocop_todo.yml +110 -22
  10. data/CHANGELOG.md +385 -105
  11. data/CODE_OF_CONDUCT.md +14 -0
  12. data/CONTRIBUTING.md +63 -0
  13. data/Gemfile +4 -10
  14. data/Guardfile +3 -1
  15. data/README.md +494 -275
  16. data/chewy.gemspec +5 -20
  17. data/gemfiles/base.gemfile +12 -0
  18. data/gemfiles/rails.6.1.activerecord.gemfile +10 -15
  19. data/gemfiles/rails.7.0.activerecord.gemfile +14 -0
  20. data/gemfiles/rails.7.1.activerecord.gemfile +14 -0
  21. data/lib/chewy/config.rb +58 -50
  22. data/lib/chewy/elastic_client.rb +31 -0
  23. data/lib/chewy/errors.rb +7 -10
  24. data/lib/chewy/fields/base.rb +79 -13
  25. data/lib/chewy/fields/root.rb +4 -14
  26. data/lib/chewy/index/actions.rb +54 -37
  27. data/lib/chewy/{type → index}/adapter/active_record.rb +30 -6
  28. data/lib/chewy/{type → index}/adapter/base.rb +2 -3
  29. data/lib/chewy/{type → index}/adapter/object.rb +27 -31
  30. data/lib/chewy/{type → index}/adapter/orm.rb +17 -18
  31. data/lib/chewy/index/aliases.rb +14 -5
  32. data/lib/chewy/index/crutch.rb +40 -0
  33. data/lib/chewy/index/import/bulk_builder.rb +311 -0
  34. data/lib/chewy/{type → index}/import/bulk_request.rb +6 -7
  35. data/lib/chewy/{type → index}/import/journal_builder.rb +11 -12
  36. data/lib/chewy/{type → index}/import/routine.rb +18 -17
  37. data/lib/chewy/{type → index}/import.rb +76 -32
  38. data/lib/chewy/{type → index}/mapping.rb +29 -34
  39. data/lib/chewy/index/observe/active_record_methods.rb +87 -0
  40. data/lib/chewy/index/observe/callback.rb +34 -0
  41. data/lib/chewy/index/observe.rb +17 -0
  42. data/lib/chewy/index/specification.rb +1 -0
  43. data/lib/chewy/{type → index}/syncer.rb +59 -59
  44. data/lib/chewy/{type → index}/witchcraft.rb +11 -7
  45. data/lib/chewy/{type → index}/wrapper.rb +2 -2
  46. data/lib/chewy/index.rb +67 -94
  47. data/lib/chewy/journal.rb +25 -14
  48. data/lib/chewy/log_subscriber.rb +5 -1
  49. data/lib/chewy/minitest/helpers.rb +86 -13
  50. data/lib/chewy/minitest/search_index_receiver.rb +24 -26
  51. data/lib/chewy/railtie.rb +6 -20
  52. data/lib/chewy/rake_helper.rb +169 -113
  53. data/lib/chewy/rspec/build_query.rb +12 -0
  54. data/lib/chewy/rspec/helpers.rb +55 -0
  55. data/lib/chewy/rspec/update_index.rb +55 -44
  56. data/lib/chewy/rspec.rb +2 -0
  57. data/lib/chewy/runtime/version.rb +1 -1
  58. data/lib/chewy/runtime.rb +1 -1
  59. data/lib/chewy/search/loader.rb +19 -41
  60. data/lib/chewy/search/parameters/collapse.rb +16 -0
  61. data/lib/chewy/search/parameters/concerns/query_storage.rb +2 -2
  62. data/lib/chewy/search/parameters/ignore_unavailable.rb +27 -0
  63. data/lib/chewy/search/parameters/indices.rb +13 -58
  64. data/lib/chewy/search/parameters/knn.rb +16 -0
  65. data/lib/chewy/search/parameters/order.rb +6 -19
  66. data/lib/chewy/search/parameters/source.rb +5 -1
  67. data/lib/chewy/search/parameters/storage.rb +1 -1
  68. data/lib/chewy/search/parameters/track_total_hits.rb +16 -0
  69. data/lib/chewy/search/parameters.rb +6 -4
  70. data/lib/chewy/search/query_proxy.rb +9 -2
  71. data/lib/chewy/search/request.rb +169 -134
  72. data/lib/chewy/search/response.rb +5 -5
  73. data/lib/chewy/search/scoping.rb +7 -8
  74. data/lib/chewy/search/scrolling.rb +13 -13
  75. data/lib/chewy/search.rb +9 -19
  76. data/lib/chewy/stash.rb +19 -30
  77. data/lib/chewy/strategy/active_job.rb +1 -1
  78. data/lib/chewy/strategy/atomic_no_refresh.rb +18 -0
  79. data/lib/chewy/strategy/base.rb +10 -0
  80. data/lib/chewy/strategy/delayed_sidekiq/scheduler.rb +151 -0
  81. data/lib/chewy/strategy/delayed_sidekiq/worker.rb +52 -0
  82. data/lib/chewy/strategy/delayed_sidekiq.rb +30 -0
  83. data/lib/chewy/strategy/lazy_sidekiq.rb +64 -0
  84. data/lib/chewy/strategy/sidekiq.rb +2 -1
  85. data/lib/chewy/strategy.rb +6 -19
  86. data/lib/chewy/version.rb +1 -1
  87. data/lib/chewy.rb +39 -86
  88. data/lib/generators/chewy/install_generator.rb +1 -1
  89. data/lib/tasks/chewy.rake +36 -32
  90. data/migration_guide.md +46 -8
  91. data/spec/chewy/config_spec.rb +14 -39
  92. data/spec/chewy/elastic_client_spec.rb +26 -0
  93. data/spec/chewy/fields/base_spec.rb +432 -147
  94. data/spec/chewy/fields/root_spec.rb +20 -28
  95. data/spec/chewy/fields/time_fields_spec.rb +5 -5
  96. data/spec/chewy/index/actions_spec.rb +368 -59
  97. data/spec/chewy/{type → index}/adapter/active_record_spec.rb +156 -40
  98. data/spec/chewy/{type → index}/adapter/object_spec.rb +21 -6
  99. data/spec/chewy/index/aliases_spec.rb +3 -3
  100. data/spec/chewy/index/import/bulk_builder_spec.rb +494 -0
  101. data/spec/chewy/{type → index}/import/bulk_request_spec.rb +5 -12
  102. data/spec/chewy/{type → index}/import/journal_builder_spec.rb +9 -19
  103. data/spec/chewy/{type → index}/import/routine_spec.rb +19 -19
  104. data/spec/chewy/{type → index}/import_spec.rb +164 -98
  105. data/spec/chewy/index/mapping_spec.rb +135 -0
  106. data/spec/chewy/index/observe/active_record_methods_spec.rb +68 -0
  107. data/spec/chewy/index/observe/callback_spec.rb +139 -0
  108. data/spec/chewy/index/observe_spec.rb +143 -0
  109. data/spec/chewy/index/settings_spec.rb +3 -1
  110. data/spec/chewy/index/specification_spec.rb +20 -30
  111. data/spec/chewy/{type → index}/syncer_spec.rb +14 -19
  112. data/spec/chewy/{type → index}/witchcraft_spec.rb +20 -22
  113. data/spec/chewy/index/wrapper_spec.rb +100 -0
  114. data/spec/chewy/index_spec.rb +60 -105
  115. data/spec/chewy/journal_spec.rb +25 -74
  116. data/spec/chewy/minitest/helpers_spec.rb +123 -15
  117. data/spec/chewy/minitest/search_index_receiver_spec.rb +28 -30
  118. data/spec/chewy/multi_search_spec.rb +4 -5
  119. data/spec/chewy/rake_helper_spec.rb +315 -55
  120. data/spec/chewy/rspec/build_query_spec.rb +34 -0
  121. data/spec/chewy/rspec/helpers_spec.rb +61 -0
  122. data/spec/chewy/rspec/update_index_spec.rb +74 -71
  123. data/spec/chewy/runtime_spec.rb +2 -2
  124. data/spec/chewy/search/loader_spec.rb +19 -53
  125. data/spec/chewy/search/pagination/kaminari_examples.rb +4 -6
  126. data/spec/chewy/search/pagination/kaminari_spec.rb +2 -2
  127. data/spec/chewy/search/parameters/collapse_spec.rb +5 -0
  128. data/spec/chewy/search/parameters/ignore_unavailable_spec.rb +67 -0
  129. data/spec/chewy/search/parameters/indices_spec.rb +26 -117
  130. data/spec/chewy/search/parameters/knn_spec.rb +5 -0
  131. data/spec/chewy/search/parameters/order_spec.rb +18 -11
  132. data/spec/chewy/search/parameters/query_storage_examples.rb +67 -21
  133. data/spec/chewy/search/parameters/search_after_spec.rb +4 -1
  134. data/spec/chewy/search/parameters/source_spec.rb +8 -2
  135. data/spec/chewy/search/parameters/track_total_hits_spec.rb +5 -0
  136. data/spec/chewy/search/parameters_spec.rb +18 -4
  137. data/spec/chewy/search/query_proxy_spec.rb +68 -17
  138. data/spec/chewy/search/request_spec.rb +292 -110
  139. data/spec/chewy/search/response_spec.rb +12 -12
  140. data/spec/chewy/search/scrolling_spec.rb +10 -17
  141. data/spec/chewy/search_spec.rb +40 -34
  142. data/spec/chewy/stash_spec.rb +9 -21
  143. data/spec/chewy/strategy/active_job_spec.rb +16 -16
  144. data/spec/chewy/strategy/atomic_no_refresh_spec.rb +60 -0
  145. data/spec/chewy/strategy/atomic_spec.rb +9 -10
  146. data/spec/chewy/strategy/delayed_sidekiq_spec.rb +202 -0
  147. data/spec/chewy/strategy/lazy_sidekiq_spec.rb +214 -0
  148. data/spec/chewy/strategy/sidekiq_spec.rb +12 -12
  149. data/spec/chewy/strategy_spec.rb +19 -15
  150. data/spec/chewy_spec.rb +24 -107
  151. data/spec/spec_helper.rb +3 -22
  152. data/spec/support/active_record.rb +25 -7
  153. metadata +78 -339
  154. data/.circleci/config.yml +0 -240
  155. data/Appraisals +0 -81
  156. data/gemfiles/rails.5.2.activerecord.gemfile +0 -17
  157. data/gemfiles/rails.5.2.mongoid.6.4.gemfile +0 -17
  158. data/gemfiles/rails.6.0.activerecord.gemfile +0 -17
  159. data/gemfiles/sequel.4.45.gemfile +0 -11
  160. data/lib/chewy/backports/deep_dup.rb +0 -46
  161. data/lib/chewy/backports/duplicable.rb +0 -91
  162. data/lib/chewy/search/pagination/will_paginate.rb +0 -43
  163. data/lib/chewy/search/parameters/types.rb +0 -20
  164. data/lib/chewy/strategy/resque.rb +0 -27
  165. data/lib/chewy/strategy/shoryuken.rb +0 -40
  166. data/lib/chewy/type/actions.rb +0 -43
  167. data/lib/chewy/type/adapter/mongoid.rb +0 -67
  168. data/lib/chewy/type/adapter/sequel.rb +0 -93
  169. data/lib/chewy/type/crutch.rb +0 -32
  170. data/lib/chewy/type/import/bulk_builder.rb +0 -122
  171. data/lib/chewy/type/observe.rb +0 -82
  172. data/lib/chewy/type.rb +0 -120
  173. data/lib/sequel/plugins/chewy_observe.rb +0 -63
  174. data/spec/chewy/search/pagination/will_paginate_examples.rb +0 -63
  175. data/spec/chewy/search/pagination/will_paginate_spec.rb +0 -23
  176. data/spec/chewy/search/parameters/types_spec.rb +0 -5
  177. data/spec/chewy/strategy/resque_spec.rb +0 -46
  178. data/spec/chewy/strategy/shoryuken_spec.rb +0 -70
  179. data/spec/chewy/type/actions_spec.rb +0 -50
  180. data/spec/chewy/type/adapter/mongoid_spec.rb +0 -372
  181. data/spec/chewy/type/adapter/sequel_spec.rb +0 -472
  182. data/spec/chewy/type/import/bulk_builder_spec.rb +0 -194
  183. data/spec/chewy/type/mapping_spec.rb +0 -175
  184. data/spec/chewy/type/observe_spec.rb +0 -137
  185. data/spec/chewy/type/wrapper_spec.rb +0 -100
  186. data/spec/chewy/type_spec.rb +0 -55
  187. data/spec/support/mongoid.rb +0 -93
  188. data/spec/support/sequel.rb +0 -80
@@ -23,7 +23,7 @@ module Chewy
23
23
  #
24
24
  # @return [Integer]
25
25
  def total
26
- @total ||= hits_root['total'] || 0
26
+ @total ||= hits_root.fetch('total', {}).fetch('value', 0)
27
27
  end
28
28
 
29
29
  # Response `max_score` field.
@@ -64,12 +64,12 @@ module Chewy
64
64
  end
65
65
  alias_method :aggregations, :aggs
66
66
 
67
- # {Chewy::Type} wrappers collection instantiated on top of hits.
67
+ # {Chewy::Index} wrappers collection instantiated on top of hits.
68
68
  #
69
- # @return [Array<Chewy::Type>]
69
+ # @return [Array<Chewy::Index>]
70
70
  def wrappers
71
71
  @wrappers ||= hits.map do |hit|
72
- @loader.derive_type(hit['_index'], hit['_type']).build(hit)
72
+ @loader.derive_index(hit['_index']).build(hit)
73
73
  end
74
74
  end
75
75
 
@@ -102,7 +102,7 @@ module Chewy
102
102
  # end
103
103
  # @see #wrappers
104
104
  # @see #objects
105
- # @return [{Chewy::Type => Object}] a hash with wrappers as keys and ORM/ODM objects as values
105
+ # @return [{Chewy::Index => Object}] a hash with wrappers as keys and ORM/ODM objects as values
106
106
  def object_hash
107
107
  @object_hash ||= wrappers.zip(objects).to_h
108
108
  end
@@ -9,17 +9,16 @@ module Chewy
9
9
  # query(match: {name: name})
10
10
  # end
11
11
  #
12
- # define_type :user do
13
- # def self.by_age(age)
14
- # filter(term: {age: age})
15
- # end
12
+ #
13
+ # def self.by_age(age)
14
+ # filter(term: {age: age})
16
15
  # end
17
16
  # end
18
17
  #
19
18
  # UsersIndex.limit(10).by_name('Martin')
20
19
  # # => <UsersIndex::Query {..., :body=>{:size=>10, :query=>{:match=>{:name=>"Martin"}}}}>
21
- # UsersIndex::User.limit(10).by_name('Martin').by_age(42)
22
- # # => <UsersIndex::User::Query {..., :body=>{:size=>10, :query=>{:bool=>{
20
+ # UsersIndex.limit(10).by_name('Martin').by_age(42)
21
+ # # => <UsersIndex::Query {..., :body=>{:size=>10, :query=>{:bool=>{
23
22
  # # :must=>{:match=>{:name=>"Martin"}},
24
23
  # # :filter=>{:term=>{:age=>42}}}}}}>
25
24
  module Scoping
@@ -28,9 +27,9 @@ module Chewy
28
27
  module ClassMethods
29
28
  # The scopes stack.
30
29
  #
31
- # @return [Array<Chewy::Search::Reques>] array of scopes
30
+ # @return [Array<Chewy::Search::Request>] array of scopes
32
31
  def scopes
33
- Thread.current[:chewy_scopes] ||= []
32
+ Chewy.current[:chewy_scopes] ||= []
34
33
  end
35
34
  end
36
35
 
@@ -28,7 +28,7 @@ module Chewy
28
28
  return enum_for(:scroll_batches, batch_size: batch_size, scroll: scroll) unless block_given?
29
29
 
30
30
  result = perform(size: batch_size, scroll: scroll)
31
- total = [raw_limit_value, result.fetch('hits', {}).fetch('total', 0)].compact.min
31
+ total = [raw_limit_value, result.fetch('hits', {}).fetch('total', {}).fetch('value', 0)].compact.min
32
32
  last_batch_size = total % batch_size
33
33
  fetched = 0
34
34
  scroll_id = nil
@@ -40,10 +40,11 @@ module Chewy
40
40
  yield(hits) if hits.present?
41
41
  scroll_id = result['_scroll_id']
42
42
  break if fetched >= total
43
+
43
44
  result = perform_scroll(scroll: scroll, scroll_id: scroll_id)
44
45
  end
45
46
  ensure
46
- Chewy.client.clear_scroll(scroll_id: scroll_id) if scroll_id
47
+ Chewy.client.clear_scroll(body: {scroll_id: scroll_id}) if scroll_id
47
48
  end
48
49
 
49
50
  # @!method scroll_hits(batch_size: 1000, scroll: '1m')
@@ -61,17 +62,17 @@ module Chewy
61
62
  # @example
62
63
  # PlaceIndex.scroll_hits.map { |hit| hit['_id'] }
63
64
  # @return [Enumerator] a standard ruby Enumerator
64
- def scroll_hits(**options)
65
+ def scroll_hits(**options, &block)
65
66
  return enum_for(:scroll_hits, **options) unless block_given?
66
67
 
67
68
  scroll_batches(**options).each do |batch|
68
- batch.each { |hit| yield hit }
69
+ batch.each(&block)
69
70
  end
70
71
  end
71
72
 
72
73
  # @!method scroll_wrappers(batch_size: 1000, scroll: '1m')
73
74
  # Iterates through the documents of the scope in batches. Yields
74
- # each hit wrapped with {Chewy::Type}.
75
+ # each hit wrapped with {Chewy::Index}.
75
76
  #
76
77
  # @param batch_size [Integer] batch size obviously, replaces `size` query parameter
77
78
  # @param scroll [String] cursor expiration time
@@ -79,7 +80,7 @@ module Chewy
79
80
  # @overload scroll_wrappers(batch_size: 1000, scroll: '1m')
80
81
  # @example
81
82
  # PlaceIndex.scroll_wrappers { |object| p object.id }
82
- # @yieldparam object [Chewy::Type] block is executed for each hit object
83
+ # @yieldparam object [Chewy::Index] block is executed for each hit object
83
84
  #
84
85
  # @overload scroll_wrappers(batch_size: 1000, scroll: '1m')
85
86
  # @example
@@ -89,7 +90,7 @@ module Chewy
89
90
  return enum_for(:scroll_wrappers, **options) unless block_given?
90
91
 
91
92
  scroll_hits(**options).each do |hit|
92
- yield loader.derive_type(hit['_index'], hit['_type']).build(hit)
93
+ yield loader.derive_index(hit['_index']).build(hit)
93
94
  end
94
95
  end
95
96
 
@@ -113,12 +114,12 @@ module Chewy
113
114
  # @example
114
115
  # PlaceIndex.scroll_objects.map { |record| record.id }
115
116
  # @return [Enumerator] a standard ruby Enumerator
116
- def scroll_objects(**options)
117
+ def scroll_objects(**options, &block)
117
118
  return enum_for(:scroll_objects, **options) unless block_given?
118
119
 
119
120
  except(:source, :stored_fields, :script_fields, :docvalue_fields)
120
121
  .source(false).scroll_batches(**options).each do |batch|
121
- loader.load(batch).each { |object| yield object }
122
+ loader.load(batch).each(&block)
122
123
  end
123
124
  end
124
125
  alias_method :scroll_records, :scroll_objects
@@ -127,10 +128,9 @@ module Chewy
127
128
  private
128
129
 
129
130
  def perform_scroll(body)
130
- ActiveSupport::Notifications.instrument 'search_query.chewy',
131
- notification_payload(request: body) do
132
- Chewy.client.scroll(body)
133
- end
131
+ ActiveSupport::Notifications.instrument 'search_query.chewy', notification_payload(request: body) do
132
+ Chewy.client.scroll(body)
133
+ end
134
134
  end
135
135
  end
136
136
  end
data/lib/chewy/search.rb CHANGED
@@ -6,29 +6,22 @@ require 'chewy/search/response'
6
6
  require 'chewy/search/loader'
7
7
  require 'chewy/search/request'
8
8
  require 'chewy/search/pagination/kaminari'
9
- require 'chewy/search/pagination/will_paginate'
10
9
 
11
10
  module Chewy
12
11
  # This module being included to any provides an interface to the
13
- # request DSL. By default it is included to {Chewy::Index} and
14
- # {Chewy::Type}.
12
+ # request DSL. By default it is included to {Chewy::Index}.
15
13
  #
16
14
  # The class used as a request DSL provider is
17
15
  # inherited from {Chewy::Search::Request}
18
16
  #
19
- # Also, the search class is refined with one of the pagination-
20
- # providing modules: {Chewy::Search::Pagination::Kaminari} or
21
- # {Chewy::Search::Pagination::WillPaginate}.
17
+ # Also, the search class is refined with the pagination module {Chewy::Search::Pagination::Kaminari}.
22
18
  #
23
19
  # @example
24
20
  # PlacesIndex.query(match: {name: 'Moscow'})
25
- # PlacesIndex::City.query(match: {name: 'Moscow'})
26
21
  # @see Chewy::Index
27
- # @see Chewy::Type
28
22
  # @see Chewy::Search::Request
29
23
  # @see Chewy::Search::ClassMethods
30
24
  # @see Chewy::Search::Pagination::Kaminari
31
- # @see Chewy::Search::Pagination::WillPaginate
32
25
  module Search
33
26
  extend ActiveSupport::Concern
34
27
 
@@ -55,15 +48,14 @@ module Chewy
55
48
  # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/search-uri-request.html
56
49
  # @return [Hash] the request result
57
50
  def search_string(query, options = {})
58
- options = options.merge(all.render.slice(:index, :type).merge(q: query))
51
+ options = options.merge(all.render.slice(:index).merge(q: query))
59
52
  Chewy.client.search(options)
60
53
  end
61
54
 
62
- # Delegates methods from the request class to the index or type class
55
+ # Delegates methods from the request class to the index class
63
56
  #
64
57
  # @example
65
58
  # PlacesIndex.query(match: {name: 'Moscow'})
66
- # PlacesIndex::City.query(match: {name: 'Moscow'})
67
59
  def method_missing(name, *args, &block)
68
60
  if search_class::DELEGATED_METHODS.include?(name)
69
61
  all.send(name, *args, &block)
@@ -85,11 +77,6 @@ module Chewy
85
77
  def build_search_class(base)
86
78
  search_class = Class.new(base)
87
79
 
88
- if self < Chewy::Type
89
- index_scopes = index.scopes - scopes
90
- delegate_scoped index, search_class, index_scopes
91
- end
92
-
93
80
  delegate_scoped self, search_class, scopes
94
81
  const_set('Query', search_class)
95
82
  end
@@ -97,9 +84,12 @@ module Chewy
97
84
  def delegate_scoped(source, destination, methods)
98
85
  methods.each do |method|
99
86
  destination.class_eval do
100
- define_method method do |*args, &block|
101
- scoping { source.public_send(method, *args, &block) }
87
+ define_method method do |*args, **kwargs, &block|
88
+ scoping do
89
+ source.public_send(method, *args, **kwargs, &block)
90
+ end
102
91
  end
92
+ method
103
93
  end
104
94
  end
105
95
  end
data/lib/chewy/stash.rb CHANGED
@@ -9,11 +9,9 @@ module Chewy
9
9
  class Specification < Chewy::Index
10
10
  index_name 'chewy_specifications'
11
11
 
12
- define_type :specification do
13
- default_import_options journal: false
12
+ default_import_options journal: false
14
13
 
15
- field :specification, type: 'binary'
16
- end
14
+ field :specification, type: 'binary'
17
15
  end
18
16
 
19
17
  class Journal < Chewy::Index
@@ -24,18 +22,18 @@ module Chewy
24
22
  # @param since_time [Time, DateTime] a timestamp from which we load a journal
25
23
  # @param only [Chewy::Index, Array<Chewy::Index>] journal entries related to these indices will be loaded only
26
24
  def self.entries(since_time, only: [])
27
- self.for(only).filter(range: {created_at: {gt: since_time}})
25
+ self.for(only).filter(range: {created_at: {gt: since_time}}).filter.minimum_should_match(1)
28
26
  end
29
27
 
30
28
  # Cleans up all the journal entries until the specified time. If nothing is
31
29
  # specified - cleans up everything.
32
30
  #
33
- # @param since_time [Time, DateTime] the time top boundary
31
+ # @param until_time [Time, DateTime] Clean everything before that date
34
32
  # @param only [Chewy::Index, Array<Chewy::Index>] indexes to clean up journal entries for
35
- def self.clean(until_time = nil, only: [])
33
+ def self.clean(until_time = nil, only: [], delete_by_query_options: {})
36
34
  scope = self.for(only)
37
35
  scope = scope.filter(range: {created_at: {lte: until_time}}) if until_time
38
- scope.delete_all
36
+ scope.delete_all(**delete_by_query_options)
39
37
  end
40
38
 
41
39
  # Selects all the journal entries for the specified indices.
@@ -43,35 +41,26 @@ module Chewy
43
41
  # @param indices [Chewy::Index, Array<Chewy::Index>]
44
42
  def self.for(*something)
45
43
  something = something.flatten.compact
46
- types = something.flat_map { |s| Chewy.derive_types(s) }
47
- return none if something.present? && types.blank?
44
+ indexes = something.flat_map { |s| Chewy.derive_name(s) }
45
+ return none if something.present? && indexes.blank?
46
+
48
47
  scope = all
49
- types.group_by(&:index).each do |index, index_types|
50
- scope = scope.or(
51
- filter(term: {index_name: index.derivable_name})
52
- .filter(terms: {type_name: index_types.map(&:type_name)})
53
- )
48
+ indexes.each do |index|
49
+ scope = scope.or(filter(term: {index_name: index.derivable_name}))
54
50
  end
55
51
  scope
56
52
  end
57
53
 
58
- define_type :journal do
59
- default_import_options journal: false
54
+ default_import_options journal: false
60
55
 
61
- field :index_name, type: 'keyword'
62
- field :type_name, type: 'keyword'
63
- field :action, type: 'keyword'
64
- field :references, type: 'binary'
65
- field :created_at, type: 'date'
66
-
67
- def type
68
- @type ||= Chewy.derive_type("#{index_name}##{type_name}")
69
- end
56
+ field :index_name, type: 'keyword'
57
+ field :action, type: 'keyword'
58
+ field :references, type: 'binary'
59
+ field :created_at, type: 'date'
70
60
 
71
- def references
72
- @references ||= Array.wrap(@attributes['references']).map do |item|
73
- JSON.load(Base64.decode64(item)) # rubocop:disable Security/JSONLoad
74
- end
61
+ def references
62
+ @references ||= Array.wrap(@attributes['references']).map do |item|
63
+ JSON.load(Base64.decode64(item)) # rubocop:disable Security/JSONLoad
75
64
  end
76
65
  end
77
66
  end
@@ -15,7 +15,7 @@ module Chewy
15
15
 
16
16
  def perform(type, ids, options = {})
17
17
  options[:refresh] = !Chewy.disable_refresh_async if Chewy.disable_refresh_async
18
- type.constantize.import!(ids, options)
18
+ type.constantize.import!(ids, **options)
19
19
  end
20
20
  end
21
21
 
@@ -0,0 +1,18 @@
1
+ module Chewy
2
+ class Strategy
3
+ # This strategy works like atomic but import objects with `refresh=false` parameter.
4
+ #
5
+ # Chewy.strategy(:atomic_no_refresh) do
6
+ # User.all.map(&:save) # Does nothing here
7
+ # Post.all.map(&:save) # And here
8
+ # # It imports all the changed users and posts right here
9
+ # # before block leaving with bulk ES API, kinda optimization
10
+ # end
11
+ #
12
+ class AtomicNoRefresh < Atomic
13
+ def leave
14
+ @stash.all? { |type, ids| type.import!(ids, refresh: false) }
15
+ end
16
+ end
17
+ end
18
+ end
@@ -22,6 +22,16 @@ module Chewy
22
22
  # strategies stack
23
23
  #
24
24
  def leave; end
25
+
26
+ # This method called when some model record is created or updated.
27
+ # Normally it will just evaluate all the Chewy callbacks and pass results
28
+ # to current strategy's update method.
29
+ # However it's possible to override it to achieve delayed evaluation of
30
+ # callbacks, e.g. using sidekiq.
31
+ #
32
+ def update_chewy_indices(object)
33
+ object.run_chewy_callbacks
34
+ end
25
35
  end
26
36
  end
27
37
  end
@@ -0,0 +1,151 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative '../../index'
4
+
5
+ # The class is responsible for accumulating in redis [type, ids]
6
+ # that were requested to be reindexed during `latency` seconds.
7
+ # The reindex job is going to be scheduled after a `latency` seconds.
8
+ # that job is going to read accumulated [type, ids] from the redis
9
+ # and reindex all them at once.
10
+ module Chewy
11
+ class Strategy
12
+ class DelayedSidekiq
13
+ require_relative 'worker'
14
+
15
+ class Scheduler
16
+ DEFAULT_TTL = 60 * 60 * 24 # in seconds
17
+ DEFAULT_LATENCY = 10
18
+ DEFAULT_MARGIN = 2
19
+ DEFAULT_QUEUE = 'chewy'
20
+ KEY_PREFIX = 'chewy:delayed_sidekiq'
21
+ ALL_SETS_KEY = "#{KEY_PREFIX}:all_sets".freeze
22
+ FALLBACK_FIELDS = 'all'
23
+ FIELDS_IDS_SEPARATOR = ';'
24
+ IDS_SEPARATOR = ','
25
+
26
+ def initialize(type, ids, options = {})
27
+ @type = type
28
+ @ids = ids
29
+ @options = options
30
+ end
31
+
32
+ # the diagram:
33
+ #
34
+ # inputs:
35
+ # latency == 2
36
+ # reindex_time = Time.current
37
+ #
38
+ # Parallel OR Sequential triggers of reindex: | What is going on in reindex store (Redis):
39
+ # --------------------------------------------------------------------------------------------------
40
+ # |
41
+ # process 1 (reindex_time): | chewy:delayed_sidekiq:CitiesIndex:1679347866 = [1]
42
+ # Schedule.new(CitiesIndex, [1]).postpone | chewy:delayed_sidekiq:timechunks = [{ score: 1679347866, "chewy:delayed_sidekiq:CitiesIndex:1679347866"}]
43
+ # | & schedule a DelayedSidekiq::Worker at 1679347869 (at + 3)
44
+ # | it will zpop chewy:delayed_sidekiq:timechunks up to 1679347866 score and reindex all ids with zpoped keys
45
+ # | chewy:delayed_sidekiq:CitiesIndex:1679347866
46
+ # |
47
+ # |
48
+ # process 2 (reindex_time): | chewy:delayed_sidekiq:CitiesIndex:1679347866 = [1, 2]
49
+ # Schedule.new(CitiesIndex, [2]).postpone | chewy:delayed_sidekiq:timechunks = [{ score: 1679347866, "chewy:delayed_sidekiq:CitiesIndex:1679347866"}]
50
+ # | & do not schedule a new worker
51
+ # |
52
+ # |
53
+ # process 1 (reindex_time + (latency - 1).seconds): | chewy:delayed_sidekiq:CitiesIndex:1679347866 = [1, 2, 3]
54
+ # Schedule.new(CitiesIndex, [3]).postpone | chewy:delayed_sidekiq:timechunks = [{ score: 1679347866, "chewy:delayed_sidekiq:CitiesIndex:1679347866"}]
55
+ # | & do not schedule a new worker
56
+ # |
57
+ # |
58
+ # process 2 (reindex_time + (latency + 1).seconds): | chewy:delayed_sidekiq:CitiesIndex:1679347866 = [1, 2, 3]
59
+ # Schedule.new(CitiesIndex, [4]).postpone | chewy:delayed_sidekiq:CitiesIndex:1679347868 = [4]
60
+ # | chewy:delayed_sidekiq:timechunks = [
61
+ # | { score: 1679347866, "chewy:delayed_sidekiq:CitiesIndex:1679347866"}
62
+ # | { score: 1679347868, "chewy:delayed_sidekiq:CitiesIndex:1679347868"}
63
+ # | ]
64
+ # | & schedule a DelayedSidekiq::Worker at 1679347871 (at + 3)
65
+ # | it will zpop chewy:delayed_sidekiq:timechunks up to 1679347868 score and reindex all ids with zpoped keys
66
+ # | chewy:delayed_sidekiq:CitiesIndex:1679347866 (in case of failed previous reindex),
67
+ # | chewy:delayed_sidekiq:CitiesIndex:1679347868
68
+ def postpone
69
+ ::Sidekiq.redis do |redis|
70
+ # warning: Redis#sadd will always return an Integer in Redis 5.0.0. Use Redis#sadd? instead
71
+ if redis.respond_to?(:sadd?)
72
+ redis.sadd?(ALL_SETS_KEY, timechunks_key)
73
+ redis.sadd?(timechunk_key, serialize_data)
74
+ else
75
+ redis.sadd(ALL_SETS_KEY, timechunks_key)
76
+ redis.sadd(timechunk_key, serialize_data)
77
+ end
78
+
79
+ redis.expire(timechunk_key, ttl)
80
+
81
+ unless redis.zrank(timechunks_key, timechunk_key)
82
+ redis.zadd(timechunks_key, at, timechunk_key)
83
+ redis.expire(timechunks_key, ttl)
84
+
85
+ ::Sidekiq::Client.push(
86
+ 'queue' => sidekiq_queue,
87
+ 'at' => at + margin,
88
+ 'class' => Chewy::Strategy::DelayedSidekiq::Worker,
89
+ 'args' => [type_name, at]
90
+ )
91
+ end
92
+ end
93
+ end
94
+
95
+ private
96
+
97
+ attr_reader :type, :ids, :options
98
+
99
+ # this method returns predictable value that jumps by latency value
100
+ # another words each latency seconds it return the same value
101
+ def at
102
+ @at ||= begin
103
+ schedule_at = latency.seconds.from_now.to_f
104
+
105
+ (schedule_at - (schedule_at % latency)).to_i
106
+ end
107
+ end
108
+
109
+ def fields
110
+ options[:update_fields].presence || [FALLBACK_FIELDS]
111
+ end
112
+
113
+ def timechunks_key
114
+ "#{KEY_PREFIX}:#{type_name}:timechunks"
115
+ end
116
+
117
+ def timechunk_key
118
+ "#{KEY_PREFIX}:#{type_name}:#{at}"
119
+ end
120
+
121
+ def serialize_data
122
+ [ids.join(IDS_SEPARATOR), fields.join(IDS_SEPARATOR)].join(FIELDS_IDS_SEPARATOR)
123
+ end
124
+
125
+ def type_name
126
+ type.name
127
+ end
128
+
129
+ def latency
130
+ strategy_config.latency || DEFAULT_LATENCY
131
+ end
132
+
133
+ def margin
134
+ strategy_config.margin || DEFAULT_MARGIN
135
+ end
136
+
137
+ def ttl
138
+ strategy_config.ttl || DEFAULT_TTL
139
+ end
140
+
141
+ def sidekiq_queue
142
+ Chewy.settings.dig(:sidekiq, :queue) || DEFAULT_QUEUE
143
+ end
144
+
145
+ def strategy_config
146
+ type.strategy_config.delayed_sidekiq
147
+ end
148
+ end
149
+ end
150
+ end
151
+ end
@@ -0,0 +1,52 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Chewy
4
+ class Strategy
5
+ class DelayedSidekiq
6
+ class Worker
7
+ include ::Sidekiq::Worker
8
+
9
+ def perform(type, score, options = {})
10
+ options[:refresh] = !Chewy.disable_refresh_async if Chewy.disable_refresh_async
11
+
12
+ ::Sidekiq.redis do |redis|
13
+ timechunks_key = "#{Scheduler::KEY_PREFIX}:#{type}:timechunks"
14
+ timechunk_keys = redis.zrangebyscore(timechunks_key, -1, score)
15
+ members = timechunk_keys.flat_map { |timechunk_key| redis.smembers(timechunk_key) }.compact
16
+
17
+ # extract ids and fields & do the reset of records
18
+ ids, fields = extract_ids_and_fields(members)
19
+ options[:update_fields] = fields if fields
20
+
21
+ index = type.constantize
22
+ index.strategy_config.delayed_sidekiq.reindex_wrapper.call do
23
+ options.any? ? index.import!(ids, **options) : index.import!(ids)
24
+ end
25
+
26
+ redis.del(timechunk_keys)
27
+ redis.zremrangebyscore(timechunks_key, -1, score)
28
+ end
29
+ end
30
+
31
+ private
32
+
33
+ def extract_ids_and_fields(members)
34
+ ids = []
35
+ fields = []
36
+
37
+ members.each do |member|
38
+ member_ids, member_fields = member.split(Scheduler::FIELDS_IDS_SEPARATOR).map do |v|
39
+ v.split(Scheduler::IDS_SEPARATOR)
40
+ end
41
+ ids |= member_ids
42
+ fields |= member_fields
43
+ end
44
+
45
+ fields = nil if fields.include?(Scheduler::FALLBACK_FIELDS)
46
+
47
+ [ids.map(&:to_i), fields]
48
+ end
49
+ end
50
+ end
51
+ end
52
+ end
@@ -0,0 +1,30 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Chewy
4
+ class Strategy
5
+ class DelayedSidekiq < Sidekiq
6
+ require_relative 'delayed_sidekiq/scheduler'
7
+
8
+ # cleanup the redis sets used internally. Useful mainly in tests to avoid
9
+ # leak and potential flaky tests.
10
+ def self.clear_timechunks!
11
+ ::Sidekiq.redis do |redis|
12
+ timechunk_sets = redis.smembers(Chewy::Strategy::DelayedSidekiq::Scheduler::ALL_SETS_KEY)
13
+ break if timechunk_sets.empty?
14
+
15
+ redis.pipelined do |pipeline|
16
+ timechunk_sets.each { |set| pipeline.del(set) }
17
+ end
18
+ end
19
+ end
20
+
21
+ def leave
22
+ @stash.each do |type, ids|
23
+ next if ids.empty?
24
+
25
+ DelayedSidekiq::Scheduler.new(type, ids).postpone
26
+ end
27
+ end
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,64 @@
1
+ module Chewy
2
+ class Strategy
3
+ # The strategy works the same way as sidekiq, but performs
4
+ # async evaluation of all index callbacks on model create and update
5
+ # driven by sidekiq
6
+ #
7
+ # Chewy.strategy(:lazy_sidekiq) do
8
+ # User.all.map(&:save) # Does nothing here
9
+ # Post.all.map(&:save) # And here
10
+ # # It schedules import of all the changed users and posts right here
11
+ # end
12
+ #
13
+ class LazySidekiq < Sidekiq
14
+ class IndicesUpdateWorker
15
+ include ::Sidekiq::Worker
16
+
17
+ def perform(models)
18
+ Chewy.strategy(strategy) do
19
+ models.each do |model_type, model_ids|
20
+ model_type.constantize.where(id: model_ids).each(&:run_chewy_callbacks)
21
+ end
22
+ end
23
+ end
24
+
25
+ private
26
+
27
+ def strategy
28
+ Chewy.disable_refresh_async ? :atomic_no_refresh : :atomic
29
+ end
30
+ end
31
+
32
+ def initialize
33
+ # Use parent's @stash to store destroyed records, since callbacks for them have to
34
+ # be run immediately on the strategy block end because we won't be able to fetch
35
+ # records further in IndicesUpdateWorker. This will be done by avoiding of
36
+ # LazySidekiq#update_chewy_indices call and calling LazySidekiq#update instead.
37
+ super
38
+
39
+ # @lazy_stash is used to store all the lazy evaluated callbacks with call of
40
+ # strategy's #update_chewy_indices.
41
+ @lazy_stash = {}
42
+ end
43
+
44
+ def leave
45
+ # Fallback to Sidekiq#leave implementation for destroyed records stored in @stash.
46
+ super
47
+
48
+ # Proceed with other records stored in @lazy_stash
49
+ return if @lazy_stash.empty?
50
+
51
+ ::Sidekiq::Client.push(
52
+ 'queue' => sidekiq_queue,
53
+ 'class' => Chewy::Strategy::LazySidekiq::IndicesUpdateWorker,
54
+ 'args' => [@lazy_stash]
55
+ )
56
+ end
57
+
58
+ def update_chewy_indices(object)
59
+ @lazy_stash[object.class.name] ||= []
60
+ @lazy_stash[object.class.name] |= Array.wrap(object.id)
61
+ end
62
+ end
63
+ end
64
+ end