chewy 0.10.1 → 7.3.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (243) hide show
  1. checksums.yaml +5 -5
  2. data/.github/CODEOWNERS +1 -0
  3. data/.github/ISSUE_TEMPLATE/bug_report.md +39 -0
  4. data/.github/ISSUE_TEMPLATE/feature_request.md +20 -0
  5. data/.github/PULL_REQUEST_TEMPLATE.md +16 -0
  6. data/.github/workflows/ruby.yml +74 -0
  7. data/.rubocop.yml +28 -23
  8. data/.rubocop_todo.yml +110 -22
  9. data/CHANGELOG.md +480 -298
  10. data/CODE_OF_CONDUCT.md +14 -0
  11. data/CONTRIBUTING.md +63 -0
  12. data/Gemfile +3 -5
  13. data/Guardfile +3 -1
  14. data/LICENSE.txt +1 -1
  15. data/README.md +571 -333
  16. data/chewy.gemspec +12 -15
  17. data/gemfiles/rails.5.2.activerecord.gemfile +11 -0
  18. data/gemfiles/rails.6.0.activerecord.gemfile +11 -0
  19. data/gemfiles/rails.6.1.activerecord.gemfile +13 -0
  20. data/gemfiles/rails.7.0.activerecord.gemfile +13 -0
  21. data/lib/chewy/config.rb +48 -77
  22. data/lib/chewy/errors.rb +4 -10
  23. data/lib/chewy/fields/base.rb +88 -16
  24. data/lib/chewy/fields/root.rb +15 -21
  25. data/lib/chewy/index/actions.rb +67 -38
  26. data/lib/chewy/{type → index}/adapter/active_record.rb +18 -4
  27. data/lib/chewy/{type → index}/adapter/base.rb +11 -12
  28. data/lib/chewy/{type → index}/adapter/object.rb +28 -32
  29. data/lib/chewy/{type → index}/adapter/orm.rb +26 -24
  30. data/lib/chewy/index/aliases.rb +14 -5
  31. data/lib/chewy/index/crutch.rb +40 -0
  32. data/lib/chewy/index/import/bulk_builder.rb +311 -0
  33. data/lib/chewy/{type → index}/import/bulk_request.rb +10 -9
  34. data/lib/chewy/{type → index}/import/journal_builder.rb +11 -12
  35. data/lib/chewy/{type → index}/import/routine.rb +19 -18
  36. data/lib/chewy/{type → index}/import.rb +82 -36
  37. data/lib/chewy/{type → index}/mapping.rb +63 -62
  38. data/lib/chewy/index/observe/active_record_methods.rb +87 -0
  39. data/lib/chewy/index/observe/callback.rb +34 -0
  40. data/lib/chewy/index/observe.rb +17 -0
  41. data/lib/chewy/index/settings.rb +2 -0
  42. data/lib/chewy/index/specification.rb +13 -10
  43. data/lib/chewy/{type → index}/syncer.rb +62 -63
  44. data/lib/chewy/{type → index}/witchcraft.rb +15 -9
  45. data/lib/chewy/{type → index}/wrapper.rb +16 -6
  46. data/lib/chewy/index.rb +68 -93
  47. data/lib/chewy/journal.rb +25 -14
  48. data/lib/chewy/minitest/helpers.rb +91 -18
  49. data/lib/chewy/minitest/search_index_receiver.rb +29 -33
  50. data/lib/chewy/multi_search.rb +62 -0
  51. data/lib/chewy/railtie.rb +8 -24
  52. data/lib/chewy/rake_helper.rb +141 -112
  53. data/lib/chewy/rspec/build_query.rb +12 -0
  54. data/lib/chewy/rspec/helpers.rb +55 -0
  55. data/lib/chewy/rspec/update_index.rb +58 -49
  56. data/lib/chewy/rspec.rb +2 -0
  57. data/lib/chewy/runtime.rb +1 -1
  58. data/lib/chewy/search/loader.rb +19 -41
  59. data/lib/chewy/search/parameters/allow_partial_search_results.rb +27 -0
  60. data/lib/chewy/search/parameters/collapse.rb +16 -0
  61. data/lib/chewy/search/parameters/concerns/query_storage.rb +6 -5
  62. data/lib/chewy/search/parameters/ignore_unavailable.rb +27 -0
  63. data/lib/chewy/search/parameters/indices.rb +78 -0
  64. data/lib/chewy/search/parameters/none.rb +1 -3
  65. data/lib/chewy/search/parameters/order.rb +6 -19
  66. data/lib/chewy/search/parameters/source.rb +5 -1
  67. data/lib/chewy/search/parameters/track_total_hits.rb +16 -0
  68. data/lib/chewy/search/parameters.rb +28 -8
  69. data/lib/chewy/search/query_proxy.rb +9 -2
  70. data/lib/chewy/search/request.rb +207 -157
  71. data/lib/chewy/search/response.rb +5 -5
  72. data/lib/chewy/search/scoping.rb +7 -8
  73. data/lib/chewy/search/scrolling.rb +14 -13
  74. data/lib/chewy/search.rb +7 -26
  75. data/lib/chewy/stash.rb +27 -29
  76. data/lib/chewy/strategy/active_job.rb +2 -2
  77. data/lib/chewy/strategy/atomic.rb +1 -1
  78. data/lib/chewy/strategy/atomic_no_refresh.rb +18 -0
  79. data/lib/chewy/strategy/base.rb +10 -0
  80. data/lib/chewy/strategy/delayed_sidekiq/scheduler.rb +148 -0
  81. data/lib/chewy/strategy/delayed_sidekiq/worker.rb +52 -0
  82. data/lib/chewy/strategy/delayed_sidekiq.rb +17 -0
  83. data/lib/chewy/strategy/lazy_sidekiq.rb +64 -0
  84. data/lib/chewy/strategy/sidekiq.rb +3 -2
  85. data/lib/chewy/strategy.rb +6 -19
  86. data/lib/chewy/version.rb +1 -1
  87. data/lib/chewy.rb +37 -80
  88. data/lib/generators/chewy/install_generator.rb +1 -1
  89. data/lib/tasks/chewy.rake +26 -32
  90. data/migration_guide.md +56 -0
  91. data/spec/chewy/config_spec.rb +27 -57
  92. data/spec/chewy/fields/base_spec.rb +457 -174
  93. data/spec/chewy/fields/root_spec.rb +24 -32
  94. data/spec/chewy/fields/time_fields_spec.rb +5 -5
  95. data/spec/chewy/index/actions_spec.rb +425 -60
  96. data/spec/chewy/{type → index}/adapter/active_record_spec.rb +110 -44
  97. data/spec/chewy/{type → index}/adapter/object_spec.rb +21 -6
  98. data/spec/chewy/index/aliases_spec.rb +3 -3
  99. data/spec/chewy/index/import/bulk_builder_spec.rb +494 -0
  100. data/spec/chewy/{type → index}/import/bulk_request_spec.rb +5 -12
  101. data/spec/chewy/{type → index}/import/journal_builder_spec.rb +22 -30
  102. data/spec/chewy/{type → index}/import/routine_spec.rb +19 -19
  103. data/spec/chewy/{type → index}/import_spec.rb +154 -95
  104. data/spec/chewy/index/mapping_spec.rb +135 -0
  105. data/spec/chewy/index/observe/active_record_methods_spec.rb +68 -0
  106. data/spec/chewy/index/observe/callback_spec.rb +139 -0
  107. data/spec/chewy/index/observe_spec.rb +143 -0
  108. data/spec/chewy/index/settings_spec.rb +3 -1
  109. data/spec/chewy/index/specification_spec.rb +32 -33
  110. data/spec/chewy/{type → index}/syncer_spec.rb +14 -19
  111. data/spec/chewy/{type → index}/witchcraft_spec.rb +34 -21
  112. data/spec/chewy/index/wrapper_spec.rb +100 -0
  113. data/spec/chewy/index_spec.rb +99 -114
  114. data/spec/chewy/journal_spec.rb +56 -101
  115. data/spec/chewy/minitest/helpers_spec.rb +122 -14
  116. data/spec/chewy/minitest/search_index_receiver_spec.rb +24 -26
  117. data/spec/chewy/multi_search_spec.rb +84 -0
  118. data/spec/chewy/rake_helper_spec.rb +325 -101
  119. data/spec/chewy/rspec/build_query_spec.rb +34 -0
  120. data/spec/chewy/rspec/helpers_spec.rb +61 -0
  121. data/spec/chewy/rspec/update_index_spec.rb +106 -102
  122. data/spec/chewy/runtime_spec.rb +2 -2
  123. data/spec/chewy/search/loader_spec.rb +19 -53
  124. data/spec/chewy/search/pagination/kaminari_examples.rb +3 -5
  125. data/spec/chewy/search/pagination/kaminari_spec.rb +1 -1
  126. data/spec/chewy/search/parameters/collapse_spec.rb +5 -0
  127. data/spec/chewy/search/parameters/ignore_unavailable_spec.rb +67 -0
  128. data/spec/chewy/search/parameters/indices_spec.rb +99 -0
  129. data/spec/chewy/search/parameters/none_spec.rb +1 -1
  130. data/spec/chewy/search/parameters/order_spec.rb +18 -11
  131. data/spec/chewy/search/parameters/query_storage_examples.rb +67 -21
  132. data/spec/chewy/search/parameters/search_after_spec.rb +4 -1
  133. data/spec/chewy/search/parameters/source_spec.rb +8 -2
  134. data/spec/chewy/search/parameters/track_total_hits_spec.rb +5 -0
  135. data/spec/chewy/search/parameters_spec.rb +39 -8
  136. data/spec/chewy/search/query_proxy_spec.rb +68 -17
  137. data/spec/chewy/search/request_spec.rb +360 -149
  138. data/spec/chewy/search/response_spec.rb +35 -25
  139. data/spec/chewy/search/scrolling_spec.rb +28 -26
  140. data/spec/chewy/search_spec.rb +73 -53
  141. data/spec/chewy/stash_spec.rb +16 -26
  142. data/spec/chewy/strategy/active_job_spec.rb +23 -10
  143. data/spec/chewy/strategy/atomic_no_refresh_spec.rb +60 -0
  144. data/spec/chewy/strategy/atomic_spec.rb +9 -10
  145. data/spec/chewy/strategy/delayed_sidekiq_spec.rb +190 -0
  146. data/spec/chewy/strategy/lazy_sidekiq_spec.rb +214 -0
  147. data/spec/chewy/strategy/sidekiq_spec.rb +14 -10
  148. data/spec/chewy/strategy_spec.rb +19 -15
  149. data/spec/chewy_spec.rb +17 -110
  150. data/spec/spec_helper.rb +7 -22
  151. data/spec/support/active_record.rb +43 -5
  152. metadata +123 -198
  153. data/.travis.yml +0 -53
  154. data/Appraisals +0 -79
  155. data/LEGACY_DSL.md +0 -497
  156. data/gemfiles/rails.4.0.activerecord.gemfile +0 -14
  157. data/gemfiles/rails.4.1.activerecord.gemfile +0 -14
  158. data/gemfiles/rails.4.2.activerecord.gemfile +0 -15
  159. data/gemfiles/rails.4.2.mongoid.5.1.gemfile +0 -15
  160. data/gemfiles/rails.5.0.activerecord.gemfile +0 -15
  161. data/gemfiles/rails.5.0.mongoid.6.0.gemfile +0 -15
  162. data/gemfiles/rails.5.1.activerecord.gemfile +0 -15
  163. data/gemfiles/rails.5.1.mongoid.6.1.gemfile +0 -15
  164. data/gemfiles/sequel.4.45.gemfile +0 -11
  165. data/lib/chewy/backports/deep_dup.rb +0 -46
  166. data/lib/chewy/backports/duplicable.rb +0 -91
  167. data/lib/chewy/query/compose.rb +0 -68
  168. data/lib/chewy/query/criteria.rb +0 -191
  169. data/lib/chewy/query/filters.rb +0 -227
  170. data/lib/chewy/query/loading.rb +0 -111
  171. data/lib/chewy/query/nodes/and.rb +0 -25
  172. data/lib/chewy/query/nodes/base.rb +0 -17
  173. data/lib/chewy/query/nodes/bool.rb +0 -34
  174. data/lib/chewy/query/nodes/equal.rb +0 -34
  175. data/lib/chewy/query/nodes/exists.rb +0 -20
  176. data/lib/chewy/query/nodes/expr.rb +0 -28
  177. data/lib/chewy/query/nodes/field.rb +0 -110
  178. data/lib/chewy/query/nodes/has_child.rb +0 -15
  179. data/lib/chewy/query/nodes/has_parent.rb +0 -15
  180. data/lib/chewy/query/nodes/has_relation.rb +0 -59
  181. data/lib/chewy/query/nodes/match_all.rb +0 -11
  182. data/lib/chewy/query/nodes/missing.rb +0 -20
  183. data/lib/chewy/query/nodes/not.rb +0 -25
  184. data/lib/chewy/query/nodes/or.rb +0 -25
  185. data/lib/chewy/query/nodes/prefix.rb +0 -19
  186. data/lib/chewy/query/nodes/query.rb +0 -20
  187. data/lib/chewy/query/nodes/range.rb +0 -63
  188. data/lib/chewy/query/nodes/raw.rb +0 -15
  189. data/lib/chewy/query/nodes/regexp.rb +0 -35
  190. data/lib/chewy/query/nodes/script.rb +0 -20
  191. data/lib/chewy/query/pagination.rb +0 -25
  192. data/lib/chewy/query.rb +0 -1098
  193. data/lib/chewy/search/pagination/will_paginate.rb +0 -43
  194. data/lib/chewy/search/parameters/types.rb +0 -20
  195. data/lib/chewy/strategy/resque.rb +0 -27
  196. data/lib/chewy/strategy/shoryuken.rb +0 -40
  197. data/lib/chewy/type/actions.rb +0 -43
  198. data/lib/chewy/type/adapter/mongoid.rb +0 -69
  199. data/lib/chewy/type/adapter/sequel.rb +0 -95
  200. data/lib/chewy/type/crutch.rb +0 -32
  201. data/lib/chewy/type/import/bulk_builder.rb +0 -122
  202. data/lib/chewy/type/observe.rb +0 -78
  203. data/lib/chewy/type.rb +0 -117
  204. data/lib/sequel/plugins/chewy_observe.rb +0 -78
  205. data/spec/chewy/query/criteria_spec.rb +0 -700
  206. data/spec/chewy/query/filters_spec.rb +0 -201
  207. data/spec/chewy/query/loading_spec.rb +0 -124
  208. data/spec/chewy/query/nodes/and_spec.rb +0 -12
  209. data/spec/chewy/query/nodes/bool_spec.rb +0 -14
  210. data/spec/chewy/query/nodes/equal_spec.rb +0 -32
  211. data/spec/chewy/query/nodes/exists_spec.rb +0 -18
  212. data/spec/chewy/query/nodes/has_child_spec.rb +0 -59
  213. data/spec/chewy/query/nodes/has_parent_spec.rb +0 -59
  214. data/spec/chewy/query/nodes/match_all_spec.rb +0 -11
  215. data/spec/chewy/query/nodes/missing_spec.rb +0 -16
  216. data/spec/chewy/query/nodes/not_spec.rb +0 -13
  217. data/spec/chewy/query/nodes/or_spec.rb +0 -12
  218. data/spec/chewy/query/nodes/prefix_spec.rb +0 -16
  219. data/spec/chewy/query/nodes/query_spec.rb +0 -12
  220. data/spec/chewy/query/nodes/range_spec.rb +0 -32
  221. data/spec/chewy/query/nodes/raw_spec.rb +0 -11
  222. data/spec/chewy/query/nodes/regexp_spec.rb +0 -43
  223. data/spec/chewy/query/nodes/script_spec.rb +0 -15
  224. data/spec/chewy/query/pagination/kaminari_spec.rb +0 -5
  225. data/spec/chewy/query/pagination/will_paginate_spec.rb +0 -5
  226. data/spec/chewy/query/pagination_spec.rb +0 -39
  227. data/spec/chewy/query_spec.rb +0 -636
  228. data/spec/chewy/search/pagination/will_paginate_examples.rb +0 -63
  229. data/spec/chewy/search/pagination/will_paginate_spec.rb +0 -23
  230. data/spec/chewy/search/parameters/indices_boost_spec.rb +0 -83
  231. data/spec/chewy/search/parameters/types_spec.rb +0 -5
  232. data/spec/chewy/strategy/resque_spec.rb +0 -46
  233. data/spec/chewy/strategy/shoryuken_spec.rb +0 -64
  234. data/spec/chewy/type/actions_spec.rb +0 -50
  235. data/spec/chewy/type/adapter/mongoid_spec.rb +0 -372
  236. data/spec/chewy/type/adapter/sequel_spec.rb +0 -472
  237. data/spec/chewy/type/import/bulk_builder_spec.rb +0 -279
  238. data/spec/chewy/type/mapping_spec.rb +0 -142
  239. data/spec/chewy/type/observe_spec.rb +0 -137
  240. data/spec/chewy/type/wrapper_spec.rb +0 -98
  241. data/spec/chewy/type_spec.rb +0 -55
  242. data/spec/support/mongoid.rb +0 -93
  243. data/spec/support/sequel.rb +0 -80
@@ -0,0 +1,64 @@
1
+ module Chewy
2
+ class Strategy
3
+ # The strategy works the same way as sidekiq, but performs
4
+ # async evaluation of all index callbacks on model create and update
5
+ # driven by sidekiq
6
+ #
7
+ # Chewy.strategy(:lazy_sidekiq) do
8
+ # User.all.map(&:save) # Does nothing here
9
+ # Post.all.map(&:save) # And here
10
+ # # It schedules import of all the changed users and posts right here
11
+ # end
12
+ #
13
+ class LazySidekiq < Sidekiq
14
+ class IndicesUpdateWorker
15
+ include ::Sidekiq::Worker
16
+
17
+ def perform(models)
18
+ Chewy.strategy(strategy) do
19
+ models.each do |model_type, model_ids|
20
+ model_type.constantize.where(id: model_ids).each(&:run_chewy_callbacks)
21
+ end
22
+ end
23
+ end
24
+
25
+ private
26
+
27
+ def strategy
28
+ Chewy.disable_refresh_async ? :atomic_no_refresh : :atomic
29
+ end
30
+ end
31
+
32
+ def initialize
33
+ # Use parent's @stash to store destroyed records, since callbacks for them have to
34
+ # be run immediately on the strategy block end because we won't be able to fetch
35
+ # records further in IndicesUpdateWorker. This will be done by avoiding of
36
+ # LazySidekiq#update_chewy_indices call and calling LazySidekiq#update instead.
37
+ super
38
+
39
+ # @lazy_stash is used to store all the lazy evaluated callbacks with call of
40
+ # strategy's #update_chewy_indices.
41
+ @lazy_stash = {}
42
+ end
43
+
44
+ def leave
45
+ # Fallback to Sidekiq#leave implementation for destroyed records stored in @stash.
46
+ super
47
+
48
+ # Proceed with other records stored in @lazy_stash
49
+ return if @lazy_stash.empty?
50
+
51
+ ::Sidekiq::Client.push(
52
+ 'queue' => sidekiq_queue,
53
+ 'class' => Chewy::Strategy::LazySidekiq::IndicesUpdateWorker,
54
+ 'args' => [@lazy_stash]
55
+ )
56
+ end
57
+
58
+ def update_chewy_indices(object)
59
+ @lazy_stash[object.class.name] ||= []
60
+ @lazy_stash[object.class.name] |= Array.wrap(object.id)
61
+ end
62
+ end
63
+ end
64
+ end
@@ -15,13 +15,14 @@ module Chewy
15
15
 
16
16
  def perform(type, ids, options = {})
17
17
  options[:refresh] = !Chewy.disable_refresh_async if Chewy.disable_refresh_async
18
- type.constantize.import!(ids, options)
18
+ type.constantize.import!(ids, **options)
19
19
  end
20
20
  end
21
21
 
22
22
  def leave
23
23
  @stash.each do |type, ids|
24
24
  next if ids.empty?
25
+
25
26
  ::Sidekiq::Client.push(
26
27
  'queue' => sidekiq_queue,
27
28
  'class' => Chewy::Strategy::Sidekiq::Worker,
@@ -33,7 +34,7 @@ module Chewy
33
34
  private
34
35
 
35
36
  def sidekiq_queue
36
- Chewy.settings.fetch(:sidekiq, {})[:queue] || 'chewy'
37
+ Chewy.settings.dig(:sidekiq, :queue) || 'chewy'
37
38
  end
38
39
  end
39
40
  end
@@ -2,24 +2,13 @@ require 'chewy/strategy/base'
2
2
  require 'chewy/strategy/bypass'
3
3
  require 'chewy/strategy/urgent'
4
4
  require 'chewy/strategy/atomic'
5
-
6
- begin
7
- require 'resque'
8
- require 'chewy/strategy/resque'
9
- rescue LoadError
10
- nil
11
- end
5
+ require 'chewy/strategy/atomic_no_refresh'
12
6
 
13
7
  begin
14
8
  require 'sidekiq'
15
9
  require 'chewy/strategy/sidekiq'
16
- rescue LoadError
17
- nil
18
- end
19
-
20
- begin
21
- require 'shoryuken'
22
- require 'chewy/strategy/shoryuken'
10
+ require 'chewy/strategy/lazy_sidekiq'
11
+ require 'chewy/strategy/delayed_sidekiq'
23
12
  rescue LoadError
24
13
  nil
25
14
  end
@@ -60,6 +49,7 @@ module Chewy
60
49
 
61
50
  def pop
62
51
  raise "Can't pop root strategy" if @stack.one?
52
+
63
53
  result = @stack.pop.tap(&:leave)
64
54
  debug "[#{@stack.size}] -> #{result.name}, now #{current.name}" if @stack.size > 1
65
55
  result
@@ -75,17 +65,14 @@ module Chewy
75
65
  private
76
66
 
77
67
  def debug(string)
78
- return unless Chewy.logger && Chewy.logger.debug?
68
+ return unless Chewy.logger&.debug?
69
+
79
70
  line = caller.detect { |l| l !~ %r{lib/chewy/strategy.rb:|lib/chewy.rb:} }
80
71
  Chewy.logger.debug(["Chewy strategies stack: #{string}", line.sub(/:in\s.+$/, '')].join(' @ '))
81
72
  end
82
73
 
83
74
  def resolve(name)
84
75
  "Chewy::Strategy::#{name.to_s.camelize}".safe_constantize or raise "Can't find update strategy `#{name}`"
85
- rescue NameError => ex
86
- # WORKAROUND: Strange behavior of `safe_constantize` with mongoid gem
87
- raise "Can't find update strategy `#{name}`" if ex.name.to_s.demodulize == name.to_s.camelize
88
- raise
89
76
  end
90
77
  end
91
78
  end
data/lib/chewy/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Chewy
2
- VERSION = '0.10.1'.freeze
2
+ VERSION = '7.3.2'.freeze
3
3
  end
data/lib/chewy.rb CHANGED
@@ -4,19 +4,20 @@ require 'active_support/deprecation'
4
4
  require 'active_support/json'
5
5
  require 'active_support/log_subscriber'
6
6
 
7
+ require 'active_support/isolated_execution_state' if ActiveSupport::VERSION::MAJOR >= 7
7
8
  require 'active_support/core_ext/array/access'
8
9
  require 'active_support/core_ext/array/wrap'
9
10
  require 'active_support/core_ext/enumerable'
10
11
  require 'active_support/core_ext/hash/reverse_merge'
12
+ require 'active_support/core_ext/hash/keys'
11
13
  require 'active_support/core_ext/numeric/time'
12
14
  require 'active_support/core_ext/numeric/bytes'
13
15
  require 'active_support/core_ext/object/blank'
14
16
  require 'active_support/core_ext/object/inclusion'
15
17
  require 'active_support/core_ext/string/inflections'
16
18
 
17
- require 'i18n/core_ext/hash'
18
- require 'chewy/backports/deep_dup' unless Object.respond_to?(:deep_dup)
19
19
  require 'singleton'
20
+ require 'base64'
20
21
 
21
22
  require 'elasticsearch'
22
23
 
@@ -28,20 +29,12 @@ end
28
29
 
29
30
  try_require 'kaminari'
30
31
  try_require 'kaminari/core'
31
- try_require 'will_paginate'
32
- try_require 'will_paginate/collection'
33
32
  try_require 'parallel'
34
33
 
35
34
  ActiveSupport.on_load(:active_record) do
36
- try_require 'will_paginate/active_record'
37
35
  try_require 'kaminari/activerecord'
38
36
  end
39
37
 
40
- ActiveSupport.on_load(:mongoid) do
41
- try_require 'will_paginate/mongoid'
42
- try_require 'kaminari/mongoid'
43
- end
44
-
45
38
  require 'chewy/version'
46
39
  require 'chewy/errors'
47
40
  require 'chewy/config'
@@ -51,100 +44,59 @@ require 'chewy/runtime'
51
44
  require 'chewy/log_subscriber'
52
45
  require 'chewy/strategy'
53
46
  require 'chewy/index'
54
- require 'chewy/type'
55
47
  require 'chewy/fields/base'
56
48
  require 'chewy/fields/root'
57
49
  require 'chewy/journal'
58
50
  require 'chewy/railtie' if defined?(::Rails::Railtie)
59
51
 
60
52
  ActiveSupport.on_load(:active_record) do
61
- extend Chewy::Type::Observe::ActiveRecordMethods
62
- end
63
-
64
- ActiveSupport.on_load(:mongoid) do
65
- module Mongoid
66
- module Document
67
- module ClassMethods
68
- include Chewy::Type::Observe::MongoidMethods
69
- end
70
- end
71
- end
53
+ include Chewy::Index::Observe::ActiveRecordMethods
72
54
  end
73
55
 
74
56
  module Chewy
75
57
  @adapters = [
76
- Chewy::Type::Adapter::ActiveRecord,
77
- Chewy::Type::Adapter::Mongoid,
78
- Chewy::Type::Adapter::Sequel,
79
- Chewy::Type::Adapter::Object
58
+ Chewy::Index::Adapter::ActiveRecord,
59
+ Chewy::Index::Adapter::Object
80
60
  ]
81
61
 
82
62
  class << self
83
63
  attr_accessor :adapters
84
64
 
85
- # Derives a single type for the passed string identifier if possible.
86
- #
87
- # @example
88
- # Chewy.derive_types(UsersIndex::User) # => UsersIndex::User
89
- # Chewy.derive_types('namespace/users') # => Namespace::UsersIndex::User
90
- # Chewy.derive_types('places') # => raises Chewy::UnderivableType
91
- # Chewy.derive_types('places#city') # => PlacesIndex::City
92
- #
93
- # @param name [String, Chewy::Type] string type identifier
94
- # @raise [Chewy::UnderivableType] in cases when it is impossble to find index or type or more than one type found
95
- # @return [Chewy::Type] an array of derived types
96
- def derive_type(name)
97
- return name if name.is_a?(Class) && name < Chewy::Type
98
-
99
- types = derive_types(name)
100
- raise Chewy::UnderivableType, "Index `#{types.first.index}` has more than one type, please specify type via `#{types.first.index.derivable_name}#type_name`" unless types.one?
101
- types.first
65
+ # A thread-local variables accessor
66
+ # @return [Hash]
67
+ def current
68
+ unless Thread.current.thread_variable?(:chewy)
69
+ Thread.current.thread_variable_set(:chewy, {})
70
+ end
71
+
72
+ Thread.current.thread_variable_get(:chewy)
102
73
  end
103
74
 
104
- # Derives all the types for the passed string identifier if possible.
75
+ # Derives an index for the passed string identifier if possible.
105
76
  #
106
77
  # @example
107
- # Chewy.derive_types('namespace/users') # => [Namespace::UsersIndex::User]
108
- # Chewy.derive_types('places') # => [PlacesIndex::City, PlacesIndex::Country]
109
- # Chewy.derive_types('places#city') # => [PlacesIndex::City]
78
+ # Chewy.derive_name(UsersIndex) # => UsersIndex
79
+ # Chewy.derive_name('namespace/users') # => Namespace::UsersIndex
80
+ # Chewy.derive_name('missing') # => raises Chewy::UndefinedIndex
110
81
  #
111
- # @param from [String] string type identifier
112
- # @raise [Chewy::UnderivableType] in cases when it is impossible to find index or type
113
- # @return [Array<Chewy::Type>] an array of derived types
114
- def derive_types(from)
115
- return from.types if from.is_a?(Class) && (from < Chewy::Index || from < Chewy::Type)
82
+ # @param index_name [String, Chewy::Index] index identifier or class
83
+ # @raise [Chewy::UndefinedIndex] in cases when it is impossible to find index
84
+ # @return [Chewy::Index]
85
+ def derive_name(index_name)
86
+ return index_name if index_name.is_a?(Class) && index_name < Chewy::Index
116
87
 
117
- index_name, type_name = from.split('#', 2)
118
88
  class_name = "#{index_name.camelize.gsub(/Index\z/, '')}Index"
119
89
  index = class_name.safe_constantize
120
- raise Chewy::UnderivableType, "Can not find index named `#{class_name}`" unless index && index < Chewy::Index
121
- if type_name.present?
122
- type = index.type_hash[type_name] or raise Chewy::UnderivableType, "Index `#{class_name}` doesn`t have type named `#{type_name}`"
123
- [type]
124
- else
125
- index.types
126
- end
127
- end
128
90
 
129
- # Creates Chewy::Type ancestor defining index and adapter methods.
130
- #
131
- def create_type(index, target, options = {}, &block)
132
- type = Class.new(Chewy::Type)
133
-
134
- adapter = adapters.find { |klass| klass.accepts?(target) }.new(target, options)
135
-
136
- index.const_set(adapter.name, type)
137
- type.send(:define_singleton_method, :index) { index }
138
- type.send(:define_singleton_method, :adapter) { adapter }
91
+ return index if index && index < Chewy::Index
139
92
 
140
- type.class_eval(&block) if block
141
- type
93
+ raise Chewy::UndefinedIndex, "Can not find index named `#{class_name}`"
142
94
  end
143
95
 
144
96
  # Main elasticsearch-ruby client instance
145
97
  #
146
98
  def client
147
- Thread.current[:chewy_client] ||= begin
99
+ Chewy.current[:chewy_client] ||= begin
148
100
  client_configuration = configuration.deep_dup
149
101
  client_configuration.delete(:prefix) # used by Chewy, not relevant to Elasticsearch::Client
150
102
  block = client_configuration[:transport_options].try(:delete, :proc)
@@ -158,7 +110,9 @@ module Chewy
158
110
  # Does nothing in case of config `wait_for_status` is undefined.
159
111
  #
160
112
  def wait_for_status
161
- client.cluster.health wait_for_status: Chewy.configuration[:wait_for_status] if Chewy.configuration[:wait_for_status].present?
113
+ if Chewy.configuration[:wait_for_status].present?
114
+ client.cluster.health wait_for_status: Chewy.configuration[:wait_for_status]
115
+ end
162
116
  end
163
117
 
164
118
  # Deletes all corresponding indexes with current prefix from ElasticSearch.
@@ -194,15 +148,15 @@ module Chewy
194
148
  # city3.do_update! # index updated again
195
149
  #
196
150
  def strategy(name = nil, &block)
197
- Thread.current[:chewy_strategy] ||= Chewy::Strategy.new
151
+ Chewy.current[:chewy_strategy] ||= Chewy::Strategy.new
198
152
  if name
199
153
  if block
200
- Thread.current[:chewy_strategy].wrap name, &block
154
+ Chewy.current[:chewy_strategy].wrap name, &block
201
155
  else
202
- Thread.current[:chewy_strategy].push name
156
+ Chewy.current[:chewy_strategy].push name
203
157
  end
204
158
  else
205
- Thread.current[:chewy_strategy]
159
+ Chewy.current[:chewy_strategy]
206
160
  end
207
161
  end
208
162
 
@@ -226,7 +180,10 @@ module Chewy
226
180
 
227
181
  def eager_load!
228
182
  return unless defined?(Chewy::Railtie)
229
- dirs = Chewy::Railtie.all_engines.map { |engine| engine.paths[Chewy.configuration[:indices_path]] }.compact.map(&:existent).flatten.uniq
183
+
184
+ dirs = Chewy::Railtie.all_engines.map do |engine|
185
+ engine.paths[Chewy.configuration[:indices_path]]
186
+ end.compact.map(&:existent).flatten.uniq
230
187
 
231
188
  dirs.each do |dir|
232
189
  Dir.glob(File.join(dir, '**/*.rb')).each do |file|
@@ -1,7 +1,7 @@
1
1
  module Chewy
2
2
  module Generators
3
3
  class InstallGenerator < Rails::Generators::Base
4
- source_root File.expand_path('../../templates', __FILE__)
4
+ source_root File.expand_path('../templates', __dir__)
5
5
 
6
6
  def copy_configuration
7
7
  template 'chewy.yml', 'config/chewy.yml'
data/lib/tasks/chewy.rake CHANGED
@@ -23,22 +23,22 @@ end
23
23
  namespace :chewy do
24
24
  desc 'Destroys, recreates and imports data for the specified indexes or all of them'
25
25
  task reset: :environment do |_task, args|
26
- Chewy::RakeHelper.reset(parse_classes(args.extras))
26
+ Chewy::RakeHelper.reset(**parse_classes(args.extras))
27
27
  end
28
28
 
29
29
  desc 'Resets data for the specified indexes or all of them only if the index specification is changed'
30
30
  task upgrade: :environment do |_task, args|
31
- Chewy::RakeHelper.upgrade(parse_classes(args.extras))
31
+ Chewy::RakeHelper.upgrade(**parse_classes(args.extras))
32
32
  end
33
33
 
34
34
  desc 'Updates data for the specified indexes/types or all of them'
35
35
  task update: :environment do |_task, args|
36
- Chewy::RakeHelper.update(parse_classes(args.extras))
36
+ Chewy::RakeHelper.update(**parse_classes(args.extras))
37
37
  end
38
38
 
39
39
  desc 'Synchronizes data for the specified indexes/types or all of them'
40
40
  task sync: :environment do |_task, args|
41
- Chewy::RakeHelper.sync(parse_classes(args.extras))
41
+ Chewy::RakeHelper.sync(**parse_classes(args.extras))
42
42
  end
43
43
 
44
44
  desc 'Resets all the indexes with the specification changed and synchronizes the rest of them'
@@ -47,25 +47,35 @@ namespace :chewy do
47
47
  Chewy::RakeHelper.sync(except: processed)
48
48
  end
49
49
 
50
+ desc 'Reindex data from source index to destination index'
51
+ task :reindex, %i[source dest] => :environment do |_task, args|
52
+ Chewy::RakeHelper.reindex(source: args[:source], dest: args[:dest])
53
+ end
54
+
55
+ desc 'Update mapping of exising index with body hash'
56
+ task :update_mapping, %i[index_name] => :environment do |_task, args|
57
+ Chewy::RakeHelper.update_mapping(name: args[:index_name])
58
+ end
59
+
50
60
  namespace :parallel do
51
61
  desc 'Parallel version of `rake chewy:reset`'
52
62
  task reset: :environment do |_task, args|
53
- Chewy::RakeHelper.reset(parse_parallel_args(args.extras))
63
+ Chewy::RakeHelper.reset(**parse_parallel_args(args.extras))
54
64
  end
55
65
 
56
66
  desc 'Parallel version of `rake chewy:upgrade`'
57
67
  task upgrade: :environment do |_task, args|
58
- Chewy::RakeHelper.upgrade(parse_parallel_args(args.extras))
68
+ Chewy::RakeHelper.upgrade(**parse_parallel_args(args.extras))
59
69
  end
60
70
 
61
71
  desc 'Parallel version of `rake chewy:update`'
62
72
  task update: :environment do |_task, args|
63
- Chewy::RakeHelper.update(parse_parallel_args(args.extras))
73
+ Chewy::RakeHelper.update(**parse_parallel_args(args.extras))
64
74
  end
65
75
 
66
76
  desc 'Parallel version of `rake chewy:sync`'
67
77
  task sync: :environment do |_task, args|
68
- Chewy::RakeHelper.sync(parse_parallel_args(args.extras))
78
+ Chewy::RakeHelper.sync(**parse_parallel_args(args.extras))
69
79
  end
70
80
 
71
81
  desc 'Parallel version of `rake chewy:deploy`'
@@ -79,34 +89,18 @@ namespace :chewy do
79
89
  namespace :journal do
80
90
  desc 'Applies changes that were done after the specified time for the specified indexes/types or all of them'
81
91
  task apply: :environment do |_task, args|
82
- Chewy::RakeHelper.journal_apply(parse_journal_args(args.extras))
92
+ Chewy::RakeHelper.journal_apply(**parse_journal_args(args.extras))
83
93
  end
84
94
 
85
95
  desc 'Removes journal records created before the specified timestamp for the specified indexes/types or all of them'
86
96
  task clean: :environment do |_task, args|
87
- Chewy::RakeHelper.journal_clean(parse_journal_args(args.extras))
88
- end
89
- end
90
-
91
- task apply_changes_from: :environment do |_task, args|
92
- ActiveSupport::Deprecation.warn '`rake chewy:apply_changes_from` is deprecated and will be removed soon, use `rake chewy:journal:apply` instead'
93
-
94
- Chewy::RakeHelper.subscribed_task_stats do
95
- params = args.extras
96
-
97
- if params.empty?
98
- puts 'Please specify a timestamp like chewy:apply_changes_from[1469528705]'
99
- else
100
- timestamp, retries = params
101
- time = Time.at(timestamp.to_i)
102
- Chewy::Journal.new.apply(time, retries: (retries.to_i if retries))
103
- end
97
+ delete_options = Chewy::RakeHelper.delete_by_query_options_from_env(ENV)
98
+ Chewy::RakeHelper.journal_clean(
99
+ **[
100
+ parse_journal_args(args.extras),
101
+ {delete_by_query_options: delete_options}
102
+ ].reduce({}, :merge)
103
+ )
104
104
  end
105
105
  end
106
-
107
- task clean_journal: :environment do |_task, args|
108
- ActiveSupport::Deprecation.warn '`rake chewy:clean_journal` is deprecated and will be removed soon, use `rake chewy:journal:clean` instead'
109
-
110
- Chewy::Journal.new.clean(args.extras.first)
111
- end
112
106
  end
@@ -0,0 +1,56 @@
1
+ # Migration guide
2
+
3
+ This document outlines the steps you need to take when migrating between major versions of
4
+ Chewy and Elasticsearch. For simplicity's sake the guide will assume that you're using
5
+ Chewy alongside a matching Elasticsearch version.
6
+
7
+ ## Chewy 6/Elasticsearch 6 to Chewy 7/Elasticsearch 7
8
+
9
+ In order to upgrade Chewy 6/Elasticsearch 6 to Chewy 7/Elasticsearch 7 in the most seamless manner you have to:
10
+
11
+ * Upgrade to the latest 6.x stable releases, namely Chewy 6.0, Elasticsearch 6.8
12
+ * Study carefully [Breaking changes in 7.0](https://www.elastic.co/guide/en/elasticsearch/reference/7.17/breaking-changes-7.0.html), make sure your application conforms.
13
+ * Run your test suite on Chewy 7.0 / Elasticsearch 7
14
+ * Run manual tests on Chewy 7.0 / Elasticsearch 7
15
+ * Upgrade to Chewy 7.0
16
+ * The “total hits” counter is an integer for ES versions < 7 and an object (hash) for the versions starting from 7.0.0. Elasticsearch added a special option, `rest_total_hits_as_int`, to ease the upgrade, that could be appended to any request and results in the old “total hits” format. Unfortunately, this option is not recognized by ES versions prior to 7.0.0, which means that we have to check the version to decide if we need this option.
17
+ Normally Chewy does memoization of the current ES version, but this might be inappropriate for the upgrade, as the version changes live.
18
+ To handle that we have 2 versions of Chewy for this stage of the upgrade: 7.0.0 and 7.0.1. Version 7.0.0 does the memoization and version 7.0.1 requests the current version on every search request.
19
+ * You can use the 7.0.0 version if it's fine for you to have an application restart immediately after ES cluster upgrade.
20
+ * If you're using the 7.0.1 version you might be interested in keeping the timeframe between this step and updating to Chewy 7.1 as small as possible, as version 7.0.1 skips ES version memoization for search requests to help dynamically detect ES version. This leads to an extra version request on each search request, i.e. could affect the overall performance/latency of the search and a load of ES cluster.
21
+ * Perform a [rolling upgrade](https://www.elastic.co/guide/en/elasticsearch/reference//rolling-upgrades.html) of Elasticsearch
22
+ * Run your test suite on Chewy 7.1 / Elasticsearch 7
23
+ * Run manual tests on Chewy 7.1 / Elasticsearch 7
24
+ * Upgrade to Chewy 7.1
25
+ * Upgrade to Chewy 7.2:
26
+ * Remove all the the `Chewy::Type` class usages, e.g. remove `CitiesIndex::City` / `CitiesIndex.city`
27
+ * `CitiesIndex::City.import! ...` becomes `CitiesIndex.import! ...`
28
+ * Update indexes with simplified DSL:
29
+ * `define_type` block -> `index_scope` clause
30
+ * it can be omitted completely, if you don't need to specify the scope or options, e.g. `name`
31
+ * Remove type names from string representations:
32
+ * in `update_index` ActiveRecord helper and RSpec matcher, e.g.
33
+ * `update_index('cities#city')` -> `update_index('cities')`
34
+ * `update_index(UsersIndex::User)` -> `update_index(UsersIndex)`
35
+ * in rake tasks (e.g. `rake chewy:update[cities#city]` -> `rake chewy:update[cities]`)
36
+ * rake tasks output is also changed (e.g. `Imported CitiesIndex::City in 1s, stats: index 3` -> `Imported CitiesIndex in 1s, stats: index 3`)
37
+ * Use index name instead of type name in loader additional scope
38
+ * e.g. `CitiesIndex.filter(...).load(city: {scope: City.where(...)})` -> `CitiesIndex.filter(...).load(cities: {scope: City.where(...)})`
39
+
40
+ ## Chewy 5/Elasticsearch 5 to Chewy 6/Elasticsearch 6
41
+
42
+ In order to upgrade Chewy 5/Elasticsearch 5 to Chewy 6/Elasticsearch 6 in the most seamless manner you have to:
43
+
44
+ * Upgrade to the latest 5.x stable releases, namely Chewy 5.2, Elasticsearch 5.6
45
+ * [Migrate any multi-typed indexes into single-typed](https://www.elastic.co/guide/en/elasticsearch/reference/6.8/removal-of-types.html)
46
+ * Using [multi-index queries](https://github.com/toptal/chewy/pull/657) could be helpful
47
+ * Parent/Child [relationship is deprecated](https://www.elastic.co/guide/en/elasticsearch/reference/6.8/removal-of-types.html#parent-child-mapping-types) in favor of the [join field](https://www.elastic.co/guide/en/elasticsearch/reference/6.8/parent-join.html)
48
+ * Handle deprecation of `string` type & `not_analyzed` value for the `index` mapping parameter:
49
+ * replace fields with `{ type: 'string', index: 'not_analyzed'}` by `{type: 'keyword'}`
50
+ * replace fields with `{ type: 'string', index: 'analyzed'}` by `{type: 'text'}`
51
+ * `PathHierarchy` tokenizer' param `delimiter` now accepts only one argument, [others should be replaced by character filter ](https://discuss.elastic.co/t/multichar-delimiter-in-path-hierarchy-tokenizer/16203)
52
+ * Make sure you don't use any other of the [deprecated Elasticsearch 5 features](https://www.elastic.co/guide/en/elasticsearch/reference/6.8/breaking-changes-6.0.html)
53
+ * Run your test suite on Chewy 6 / Elasticsearch 6
54
+ * Run manual tests on Chewy 6 / Elasticsearch 6
55
+ * Upgrade to Chewy 6
56
+ * Perform a [rolling upgrade](https://www.elastic.co/guide/en/elasticsearch/reference/6.8/rolling-upgrades.html) of Elasticsearch
@@ -6,11 +6,9 @@ describe Chewy::Config do
6
6
  its(:logger) { should be_nil }
7
7
  its(:transport_logger) { should be_nil }
8
8
  its(:transport_logger) { should be_nil }
9
- its(:query_mode) { should == :must }
10
- its(:filter_mode) { should == :and }
11
- its(:post_filter_mode) { should be_nil }
12
9
  its(:root_strategy) { should == :base }
13
10
  its(:request_strategy) { should == :atomic }
11
+ its(:console_strategy) { should == :urgent }
14
12
  its(:use_after_commit_callbacks) { should == true }
15
13
  its(:indices_path) { should == 'app/chewy' }
16
14
  its(:reset_disable_refresh_interval) { should == false }
@@ -54,88 +52,60 @@ describe Chewy::Config do
54
52
  end
55
53
  end
56
54
 
57
- describe '#search_class=' do
58
- specify do
59
- expect { subject.search_class = Chewy::Query }
60
- .to change { subject.search_class }
61
- .from(be < Chewy::Search::Request)
62
- .to(be < Chewy::Query)
63
- end
64
-
65
- context do
66
- before { hide_const('Kaminari') }
67
-
68
- specify do
69
- expect(subject.search_class.included_modules)
70
- .to include(Chewy::Search::Pagination::WillPaginate)
71
- end
72
- end
73
- end
74
-
75
55
  describe '#search_class' do
76
56
  context 'nothing is defined' do
77
57
  before do
78
58
  hide_const('Kaminari')
79
- hide_const('WillPaginate')
80
59
  end
81
60
 
82
61
  specify do
83
62
  expect(subject.search_class.included_modules)
84
63
  .not_to include(Chewy::Search::Pagination::Kaminari)
85
64
  end
86
-
87
- specify do
88
- expect(subject.search_class.included_modules)
89
- .not_to include(Chewy::Search::Pagination::WillPaginate)
90
- end
91
65
  end
92
66
 
93
67
  context 'kaminari' do
94
- before { hide_const('WillPaginate') }
95
-
96
68
  specify do
97
69
  expect(subject.search_class.included_modules)
98
70
  .to include(Chewy::Search::Pagination::Kaminari)
99
71
  end
100
-
101
- specify do
102
- expect(subject.search_class.included_modules)
103
- .not_to include(Chewy::Search::Pagination::WillPaginate)
104
- end
105
72
  end
73
+ end
106
74
 
107
- context 'will_paginate' do
108
- before { hide_const('Kaminari') }
109
-
110
- specify do
111
- expect(subject.search_class.included_modules)
112
- .not_to include(Chewy::Search::Pagination::Kaminari)
113
- end
75
+ describe '#configuration' do
76
+ before { subject.settings = {indices_path: 'app/custom_indices_path'} }
114
77
 
115
- specify do
116
- expect(subject.search_class.included_modules)
117
- .to include(Chewy::Search::Pagination::WillPaginate)
118
- end
78
+ specify do
79
+ expect(subject.configuration).to include(indices_path: 'app/custom_indices_path')
119
80
  end
120
81
 
121
- context 'both are defined' do
122
- specify do
123
- expect(subject.search_class.included_modules)
124
- .to include(Chewy::Search::Pagination::Kaminari)
125
- end
82
+ context 'when Rails::VERSION constant is defined' do
83
+ it 'looks for configuration in "config/chewy.yml"' do
84
+ module Rails
85
+ VERSION = '5.1.1'.freeze
126
86
 
127
- specify do
128
- expect(subject.search_class.included_modules)
129
- .not_to include(Chewy::Search::Pagination::WillPaginate)
87
+ def self.root
88
+ Pathname.new(__dir__)
89
+ end
90
+ end
91
+
92
+ expect(File).to receive(:exist?)
93
+ .with(Pathname.new(__dir__).join('config', 'chewy.yml'))
94
+ subject.configuration
130
95
  end
131
96
  end
132
97
  end
133
98
 
134
- describe '#configuration' do
135
- before { subject.settings = {indices_path: 'app/custom_indices_path'} }
99
+ describe '.console_strategy' do
100
+ context 'sets .console_strategy' do
101
+ let(:default_strategy) { subject.console_strategy }
102
+ let(:new_strategy) { :atomic }
103
+ after { subject.console_strategy = default_strategy }
136
104
 
137
- specify do
138
- expect(subject.configuration).to include(indices_path: 'app/custom_indices_path')
105
+ specify do
106
+ expect { subject.console_strategy = new_strategy }
107
+ .to change { subject.console_strategy }.to(new_strategy)
108
+ end
139
109
  end
140
110
  end
141
111
  end