chewy 6.0.0 → 7.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (188) hide show
  1. checksums.yaml +4 -4
  2. data/.github/CODEOWNERS +1 -0
  3. data/.github/ISSUE_TEMPLATE/bug_report.md +39 -0
  4. data/.github/ISSUE_TEMPLATE/feature_request.md +20 -0
  5. data/.github/PULL_REQUEST_TEMPLATE.md +16 -0
  6. data/.github/dependabot.yml +42 -0
  7. data/.github/workflows/ruby.yml +60 -0
  8. data/.rubocop.yml +16 -8
  9. data/.rubocop_todo.yml +110 -22
  10. data/CHANGELOG.md +396 -105
  11. data/CODE_OF_CONDUCT.md +14 -0
  12. data/CONTRIBUTING.md +63 -0
  13. data/Gemfile +4 -10
  14. data/Guardfile +3 -1
  15. data/README.md +497 -275
  16. data/chewy.gemspec +5 -20
  17. data/gemfiles/base.gemfile +12 -0
  18. data/gemfiles/rails.6.1.activerecord.gemfile +10 -15
  19. data/gemfiles/rails.7.0.activerecord.gemfile +14 -0
  20. data/gemfiles/rails.7.1.activerecord.gemfile +14 -0
  21. data/lib/chewy/config.rb +60 -52
  22. data/lib/chewy/elastic_client.rb +31 -0
  23. data/lib/chewy/errors.rb +7 -10
  24. data/lib/chewy/fields/base.rb +79 -13
  25. data/lib/chewy/fields/root.rb +4 -14
  26. data/lib/chewy/index/actions.rb +54 -37
  27. data/lib/chewy/{type → index}/adapter/active_record.rb +30 -6
  28. data/lib/chewy/{type → index}/adapter/base.rb +2 -3
  29. data/lib/chewy/{type → index}/adapter/object.rb +27 -31
  30. data/lib/chewy/{type → index}/adapter/orm.rb +17 -18
  31. data/lib/chewy/index/aliases.rb +14 -5
  32. data/lib/chewy/index/crutch.rb +40 -0
  33. data/lib/chewy/index/import/bulk_builder.rb +311 -0
  34. data/lib/chewy/{type → index}/import/bulk_request.rb +6 -7
  35. data/lib/chewy/{type → index}/import/journal_builder.rb +11 -12
  36. data/lib/chewy/{type → index}/import/routine.rb +18 -17
  37. data/lib/chewy/{type → index}/import.rb +76 -32
  38. data/lib/chewy/{type → index}/mapping.rb +29 -34
  39. data/lib/chewy/index/observe/active_record_methods.rb +87 -0
  40. data/lib/chewy/index/observe/callback.rb +34 -0
  41. data/lib/chewy/index/observe.rb +17 -0
  42. data/lib/chewy/index/specification.rb +1 -0
  43. data/lib/chewy/{type → index}/syncer.rb +59 -59
  44. data/lib/chewy/{type → index}/witchcraft.rb +11 -7
  45. data/lib/chewy/{type → index}/wrapper.rb +2 -2
  46. data/lib/chewy/index.rb +67 -94
  47. data/lib/chewy/journal.rb +25 -14
  48. data/lib/chewy/log_subscriber.rb +5 -1
  49. data/lib/chewy/minitest/helpers.rb +86 -13
  50. data/lib/chewy/minitest/search_index_receiver.rb +24 -26
  51. data/lib/chewy/railtie.rb +6 -20
  52. data/lib/chewy/rake_helper.rb +169 -113
  53. data/lib/chewy/rspec/build_query.rb +12 -0
  54. data/lib/chewy/rspec/helpers.rb +55 -0
  55. data/lib/chewy/rspec/update_index.rb +55 -44
  56. data/lib/chewy/rspec.rb +2 -0
  57. data/lib/chewy/runtime/version.rb +1 -1
  58. data/lib/chewy/runtime.rb +1 -1
  59. data/lib/chewy/search/loader.rb +19 -41
  60. data/lib/chewy/search/parameters/collapse.rb +16 -0
  61. data/lib/chewy/search/parameters/concerns/query_storage.rb +2 -2
  62. data/lib/chewy/search/parameters/ignore_unavailable.rb +27 -0
  63. data/lib/chewy/search/parameters/indices.rb +13 -58
  64. data/lib/chewy/search/parameters/knn.rb +16 -0
  65. data/lib/chewy/search/parameters/order.rb +6 -19
  66. data/lib/chewy/search/parameters/source.rb +5 -1
  67. data/lib/chewy/search/parameters/storage.rb +1 -1
  68. data/lib/chewy/search/parameters/track_total_hits.rb +16 -0
  69. data/lib/chewy/search/parameters.rb +6 -4
  70. data/lib/chewy/search/query_proxy.rb +9 -2
  71. data/lib/chewy/search/request.rb +169 -134
  72. data/lib/chewy/search/response.rb +5 -5
  73. data/lib/chewy/search/scoping.rb +7 -8
  74. data/lib/chewy/search/scrolling.rb +13 -13
  75. data/lib/chewy/search.rb +9 -19
  76. data/lib/chewy/stash.rb +19 -30
  77. data/lib/chewy/strategy/active_job.rb +1 -1
  78. data/lib/chewy/strategy/atomic_no_refresh.rb +18 -0
  79. data/lib/chewy/strategy/base.rb +10 -0
  80. data/lib/chewy/strategy/delayed_sidekiq/scheduler.rb +168 -0
  81. data/lib/chewy/strategy/delayed_sidekiq/worker.rb +76 -0
  82. data/lib/chewy/strategy/delayed_sidekiq.rb +30 -0
  83. data/lib/chewy/strategy/lazy_sidekiq.rb +64 -0
  84. data/lib/chewy/strategy/sidekiq.rb +2 -1
  85. data/lib/chewy/strategy.rb +6 -19
  86. data/lib/chewy/version.rb +1 -1
  87. data/lib/chewy.rb +39 -86
  88. data/lib/generators/chewy/install_generator.rb +1 -1
  89. data/lib/tasks/chewy.rake +36 -32
  90. data/migration_guide.md +46 -8
  91. data/spec/chewy/config_spec.rb +16 -41
  92. data/spec/chewy/elastic_client_spec.rb +26 -0
  93. data/spec/chewy/fields/base_spec.rb +432 -147
  94. data/spec/chewy/fields/root_spec.rb +20 -28
  95. data/spec/chewy/fields/time_fields_spec.rb +5 -5
  96. data/spec/chewy/index/actions_spec.rb +368 -59
  97. data/spec/chewy/{type → index}/adapter/active_record_spec.rb +156 -40
  98. data/spec/chewy/{type → index}/adapter/object_spec.rb +21 -6
  99. data/spec/chewy/index/aliases_spec.rb +3 -3
  100. data/spec/chewy/index/import/bulk_builder_spec.rb +494 -0
  101. data/spec/chewy/{type → index}/import/bulk_request_spec.rb +5 -12
  102. data/spec/chewy/{type → index}/import/journal_builder_spec.rb +9 -19
  103. data/spec/chewy/{type → index}/import/routine_spec.rb +19 -19
  104. data/spec/chewy/{type → index}/import_spec.rb +164 -98
  105. data/spec/chewy/index/mapping_spec.rb +135 -0
  106. data/spec/chewy/index/observe/active_record_methods_spec.rb +68 -0
  107. data/spec/chewy/index/observe/callback_spec.rb +139 -0
  108. data/spec/chewy/index/observe_spec.rb +143 -0
  109. data/spec/chewy/index/settings_spec.rb +3 -1
  110. data/spec/chewy/index/specification_spec.rb +20 -30
  111. data/spec/chewy/{type → index}/syncer_spec.rb +14 -19
  112. data/spec/chewy/{type → index}/witchcraft_spec.rb +20 -22
  113. data/spec/chewy/index/wrapper_spec.rb +100 -0
  114. data/spec/chewy/index_spec.rb +60 -105
  115. data/spec/chewy/journal_spec.rb +25 -74
  116. data/spec/chewy/minitest/helpers_spec.rb +123 -15
  117. data/spec/chewy/minitest/search_index_receiver_spec.rb +28 -30
  118. data/spec/chewy/multi_search_spec.rb +4 -5
  119. data/spec/chewy/rake_helper_spec.rb +315 -55
  120. data/spec/chewy/rspec/build_query_spec.rb +34 -0
  121. data/spec/chewy/rspec/helpers_spec.rb +61 -0
  122. data/spec/chewy/rspec/update_index_spec.rb +74 -71
  123. data/spec/chewy/runtime_spec.rb +2 -2
  124. data/spec/chewy/search/loader_spec.rb +19 -53
  125. data/spec/chewy/search/pagination/kaminari_examples.rb +4 -6
  126. data/spec/chewy/search/pagination/kaminari_spec.rb +2 -2
  127. data/spec/chewy/search/parameters/collapse_spec.rb +5 -0
  128. data/spec/chewy/search/parameters/ignore_unavailable_spec.rb +67 -0
  129. data/spec/chewy/search/parameters/indices_spec.rb +26 -117
  130. data/spec/chewy/search/parameters/knn_spec.rb +5 -0
  131. data/spec/chewy/search/parameters/order_spec.rb +18 -11
  132. data/spec/chewy/search/parameters/query_storage_examples.rb +67 -21
  133. data/spec/chewy/search/parameters/search_after_spec.rb +4 -1
  134. data/spec/chewy/search/parameters/source_spec.rb +8 -2
  135. data/spec/chewy/search/parameters/track_total_hits_spec.rb +5 -0
  136. data/spec/chewy/search/parameters_spec.rb +18 -4
  137. data/spec/chewy/search/query_proxy_spec.rb +68 -17
  138. data/spec/chewy/search/request_spec.rb +292 -110
  139. data/spec/chewy/search/response_spec.rb +12 -12
  140. data/spec/chewy/search/scrolling_spec.rb +10 -17
  141. data/spec/chewy/search_spec.rb +40 -34
  142. data/spec/chewy/stash_spec.rb +9 -21
  143. data/spec/chewy/strategy/active_job_spec.rb +16 -16
  144. data/spec/chewy/strategy/atomic_no_refresh_spec.rb +60 -0
  145. data/spec/chewy/strategy/atomic_spec.rb +9 -10
  146. data/spec/chewy/strategy/delayed_sidekiq_spec.rb +208 -0
  147. data/spec/chewy/strategy/lazy_sidekiq_spec.rb +214 -0
  148. data/spec/chewy/strategy/sidekiq_spec.rb +12 -12
  149. data/spec/chewy/strategy_spec.rb +19 -15
  150. data/spec/chewy_spec.rb +24 -107
  151. data/spec/spec_helper.rb +3 -22
  152. data/spec/support/active_record.rb +25 -7
  153. metadata +78 -339
  154. data/.circleci/config.yml +0 -240
  155. data/Appraisals +0 -81
  156. data/gemfiles/rails.5.2.activerecord.gemfile +0 -17
  157. data/gemfiles/rails.5.2.mongoid.6.4.gemfile +0 -17
  158. data/gemfiles/rails.6.0.activerecord.gemfile +0 -17
  159. data/gemfiles/sequel.4.45.gemfile +0 -11
  160. data/lib/chewy/backports/deep_dup.rb +0 -46
  161. data/lib/chewy/backports/duplicable.rb +0 -91
  162. data/lib/chewy/search/pagination/will_paginate.rb +0 -43
  163. data/lib/chewy/search/parameters/types.rb +0 -20
  164. data/lib/chewy/strategy/resque.rb +0 -27
  165. data/lib/chewy/strategy/shoryuken.rb +0 -40
  166. data/lib/chewy/type/actions.rb +0 -43
  167. data/lib/chewy/type/adapter/mongoid.rb +0 -67
  168. data/lib/chewy/type/adapter/sequel.rb +0 -93
  169. data/lib/chewy/type/crutch.rb +0 -32
  170. data/lib/chewy/type/import/bulk_builder.rb +0 -122
  171. data/lib/chewy/type/observe.rb +0 -82
  172. data/lib/chewy/type.rb +0 -120
  173. data/lib/sequel/plugins/chewy_observe.rb +0 -63
  174. data/spec/chewy/search/pagination/will_paginate_examples.rb +0 -63
  175. data/spec/chewy/search/pagination/will_paginate_spec.rb +0 -23
  176. data/spec/chewy/search/parameters/types_spec.rb +0 -5
  177. data/spec/chewy/strategy/resque_spec.rb +0 -46
  178. data/spec/chewy/strategy/shoryuken_spec.rb +0 -70
  179. data/spec/chewy/type/actions_spec.rb +0 -50
  180. data/spec/chewy/type/adapter/mongoid_spec.rb +0 -372
  181. data/spec/chewy/type/adapter/sequel_spec.rb +0 -472
  182. data/spec/chewy/type/import/bulk_builder_spec.rb +0 -194
  183. data/spec/chewy/type/mapping_spec.rb +0 -175
  184. data/spec/chewy/type/observe_spec.rb +0 -137
  185. data/spec/chewy/type/wrapper_spec.rb +0 -100
  186. data/spec/chewy/type_spec.rb +0 -55
  187. data/spec/support/mongoid.rb +0 -93
  188. data/spec/support/sequel.rb +0 -80
@@ -1,5 +1,5 @@
1
1
  module Chewy
2
- class Type
2
+ class Index
3
3
  module Import
4
4
  # Adds additional features to elasticsearch-api bulk method:
5
5
  # * supports Chewy index suffix if necessary;
@@ -10,12 +10,12 @@ module Chewy
10
10
  #
11
11
  # @see https://github.com/elastic/elasticsearch-ruby/blob/master/elasticsearch-api/lib/elasticsearch/api/actions/bulk.rb
12
12
  class BulkRequest
13
- # @param type [Chewy::Type] a type for the request
13
+ # @param index [Chewy::Index] an index for the request
14
14
  # @param suffix [String] an index name optional suffix
15
15
  # @param bulk_size [Integer] bulk size in bytes
16
16
  # @param bulk_options [Hash] options passed to the elasticsearch-api bulk method
17
- def initialize(type, suffix: nil, bulk_size: nil, **bulk_options)
18
- @type = type
17
+ def initialize(index, suffix: nil, bulk_size: nil, **bulk_options)
18
+ @index = index
19
19
  @suffix = suffix
20
20
  @bulk_size = bulk_size - 1.kilobyte if bulk_size # 1 kilobyte for request header and newlines
21
21
  @bulk_options = bulk_options
@@ -33,7 +33,7 @@ module Chewy
33
33
  return [] if body.blank?
34
34
 
35
35
  request_bodies(body).each_with_object([]) do |request_body, results|
36
- response = @type.client.bulk request_base.merge(body: request_body) if request_body.present?
36
+ response = @index.client.bulk(**request_base.merge(body: request_body)) if request_body.present?
37
37
 
38
38
  next unless response.try(:[], 'errors')
39
39
 
@@ -47,8 +47,7 @@ module Chewy
47
47
 
48
48
  def request_base
49
49
  @request_base ||= {
50
- index: @type.index_name(suffix: @suffix),
51
- type: @type.type_name
50
+ index: @index.index_name(suffix: @suffix)
52
51
  }.merge!(@bulk_options)
53
52
  end
54
53
 
@@ -1,24 +1,23 @@
1
1
  module Chewy
2
- class Type
2
+ class Index
3
3
  module Import
4
4
  class JournalBuilder
5
- def initialize(type, index: [], delete: [])
6
- @type = type
5
+ def initialize(index, to_index: [], delete: [])
7
6
  @index = index
7
+ @to_index = to_index
8
8
  @delete = delete
9
9
  end
10
10
 
11
11
  def bulk_body
12
- Chewy::Type::Import::BulkBuilder.new(
13
- Chewy::Stash::Journal::Journal,
14
- index: [
15
- entries(:index, @index),
12
+ Chewy::Index::Import::BulkBuilder.new(
13
+ Chewy::Stash::Journal,
14
+ to_index: [
15
+ entries(:index, @to_index),
16
16
  entries(:delete, @delete)
17
17
  ].compact
18
18
  ).bulk_body.each do |item|
19
19
  item.values.first.merge!(
20
- _index: Chewy::Stash::Journal.index_name,
21
- _type: Chewy::Stash::Journal::Journal.type_name
20
+ _index: Chewy::Stash::Journal.index_name
22
21
  )
23
22
  end
24
23
  end
@@ -27,9 +26,9 @@ module Chewy
27
26
 
28
27
  def entries(action, objects)
29
28
  return unless objects.present?
29
+
30
30
  {
31
- index_name: @type.index.derivable_name,
32
- type_name: @type.type_name,
31
+ index_name: @index.derivable_name,
33
32
  action: action,
34
33
  references: identify(objects).map { |item| Base64.encode64(::Elasticsearch::API.serializer.dump(item)) },
35
34
  created_at: Time.now.utc
@@ -37,7 +36,7 @@ module Chewy
37
36
  end
38
37
 
39
38
  def identify(objects)
40
- @type.adapter.identify(objects)
39
+ @index.adapter.identify(objects)
41
40
  end
42
41
  end
43
42
  end
@@ -1,5 +1,5 @@
1
1
  module Chewy
2
- class Type
2
+ class Index
3
3
  module Import
4
4
  # This class performs the import routine for the options and objects given.
5
5
  #
@@ -20,7 +20,7 @@ module Chewy
20
20
  # when the document doesn't exist only if `update_failover` option is true. In order to
21
21
  # restore, it indexes such an objects completely on the next iteration.
22
22
  #
23
- # @see Chewy::Type::Import::ClassMethods#import
23
+ # @see Chewy::Index::Import::ClassMethods#import
24
24
  class Routine
25
25
  BULK_OPTIONS = %i[
26
26
  suffix bulk_size
@@ -33,18 +33,18 @@ module Chewy
33
33
  refresh: true,
34
34
  update_fields: [],
35
35
  update_failover: true,
36
- batch_size: Chewy::Type::Adapter::Base::BATCH_SIZE
36
+ batch_size: Chewy::Index::Adapter::Base::BATCH_SIZE
37
37
  }.freeze
38
38
 
39
39
  attr_reader :options, :parallel_options, :errors, :stats, :leftovers
40
40
 
41
41
  # Basically, processes passed options, extracting bulk request specific options.
42
- # @param type [Chewy::Type] chewy type
43
- # @param options [Hash] import options, see {Chewy::Type::Import::ClassMethods#import}
44
- def initialize(type, **options)
45
- @type = type
42
+ # @param index [Chewy::Index] chewy index
43
+ # @param options [Hash] import options, see {Chewy::Index::Import::ClassMethods#import}
44
+ def initialize(index, **options)
45
+ @index = index
46
46
  @options = options
47
- @options.reverse_merge!(@type._default_import_options)
47
+ @options.reverse_merge!(@index._default_import_options)
48
48
  @options.reverse_merge!(journal: Chewy.configuration[:journal])
49
49
  @options.reverse_merge!(DEFAULT_OPTIONS)
50
50
  @bulk_options = @options.slice(*BULK_OPTIONS)
@@ -61,27 +61,28 @@ module Chewy
61
61
  @leftovers = []
62
62
  end
63
63
 
64
- # Creates the journal index and the type corresponding index if necessary.
64
+ # Creates the journal index and the corresponding index if necessary.
65
65
  # @return [Object] whatever
66
66
  def create_indexes!
67
- Chewy::Stash::Journal.create if @options[:journal]
67
+ Chewy::Stash::Journal.create if @options[:journal] && !Chewy.configuration[:skip_journal_creation_on_import]
68
68
  return if Chewy.configuration[:skip_index_creation_on_import]
69
- @type.index.create!(**@bulk_options.slice(:suffix)) unless @type.index.exists?
69
+
70
+ @index.create!(**@bulk_options.slice(:suffix)) unless @index.exists?
70
71
  end
71
72
 
72
- # The main process method. Converts passed objects to thr bulk request body,
73
- # appends journal entires, performs this request and handles errors performing
73
+ # The main process method. Converts passed objects to the bulk request body,
74
+ # appends journal entries, performs this request and handles errors performing
74
75
  # failover procedures if applicable.
75
76
  #
76
77
  # @param index [Array<Object>] any acceptable objects for indexing
77
78
  # @param delete [Array<Object>] any acceptable objects for deleting
78
79
  # @return [true, false] the result of the request, true if no errors
79
80
  def process(index: [], delete: [])
80
- bulk_builder = BulkBuilder.new(@type, index: index, delete: delete, fields: @options[:update_fields])
81
+ bulk_builder = BulkBuilder.new(@index, to_index: index, delete: delete, fields: @options[:update_fields])
81
82
  bulk_body = bulk_builder.bulk_body
82
83
 
83
84
  if @options[:journal]
84
- journal_builder = JournalBuilder.new(@type, index: index, delete: delete)
85
+ journal_builder = JournalBuilder.new(@index, to_index: index, delete: delete)
85
86
  bulk_body.concat(journal_builder.bulk_body)
86
87
  end
87
88
 
@@ -126,11 +127,11 @@ module Chewy
126
127
  errors_to_cleanup.each { |error| errors.delete(error) }
127
128
 
128
129
  failed_objects = index_objects_by_id.values_at(*failed_ids_for_reimport)
129
- BulkBuilder.new(@type, index: failed_objects).bulk_body
130
+ BulkBuilder.new(@index, to_index: failed_objects).bulk_body
130
131
  end
131
132
 
132
133
  def bulk
133
- @bulk ||= BulkRequest.new(@type, **@bulk_options)
134
+ @bulk ||= BulkRequest.new(@index, **@bulk_options)
134
135
  end
135
136
  end
136
137
  end
@@ -1,43 +1,42 @@
1
- require 'chewy/type/import/journal_builder'
2
- require 'chewy/type/import/bulk_builder'
3
- require 'chewy/type/import/bulk_request'
4
- require 'chewy/type/import/routine'
1
+ require 'chewy/index/import/journal_builder'
2
+ require 'chewy/index/import/bulk_builder'
3
+ require 'chewy/index/import/bulk_request'
4
+ require 'chewy/index/import/routine'
5
5
 
6
6
  module Chewy
7
- class Type
7
+ class Index
8
8
  module Import
9
9
  extend ActiveSupport::Concern
10
10
 
11
- IMPORT_WORKER = lambda do |type, options, total, ids, index|
12
- ::Process.setproctitle("chewy [#{type}]: import data (#{index + 1}/#{total})")
13
- routine = Routine.new(type, **options)
14
- type.adapter.import(*ids, routine.options) do |action_objects|
11
+ IMPORT_WORKER = lambda do |index, options, total, ids, iteration|
12
+ ::Process.setproctitle("chewy [#{index}]: import data (#{iteration + 1}/#{total})")
13
+ routine = Routine.new(index, **options)
14
+ index.adapter.import(*ids, routine.options) do |action_objects|
15
15
  routine.process(**action_objects)
16
16
  end
17
17
  {errors: routine.errors, import: routine.stats, leftovers: routine.leftovers}
18
18
  end
19
19
 
20
- LEFTOVERS_WORKER = lambda do |type, options, total, body, index|
21
- ::Process.setproctitle("chewy [#{type}]: import leftovers (#{index + 1}/#{total})")
22
- routine = Routine.new(type, **options)
20
+ LEFTOVERS_WORKER = lambda do |index, options, total, body, iteration|
21
+ ::Process.setproctitle("chewy [#{index}]: import leftovers (#{iteration + 1}/#{total})")
22
+ routine = Routine.new(index, **options)
23
23
  routine.perform_bulk(body)
24
24
  routine.errors
25
25
  end
26
26
 
27
27
  module ClassMethods
28
28
  # @!method import(*collection, **options)
29
- # Basically, one of the main methods for type. Performs any objects import
30
- # to the index for a specified type. Does all the objects handling routines.
29
+ # Basically, one of the main methods for an index. Performs any objects import
30
+ # to the index. Does all the objects handling routines.
31
31
  # Performs document import by utilizing bulk API. Bulk size and objects batch
32
32
  # size are controlled by the corresponding options.
33
33
  #
34
34
  # It accepts ORM/ODM objects, PORO, hashes, ids which are used by adapter to
35
- # fetch objects from the source depenting on the used adapter. It destroys
36
- # passed objects from the index if they are not in the default type scope
35
+ # fetch objects from the source depending on the used adapter. It destroys
36
+ # passed objects from the index if they are not in the default scope
37
37
  # or marked for destruction.
38
38
  #
39
- # It handles parent-child relationships: if the object parent_id has been
40
- # changed it destroys the object and recreates it from scratch.
39
+ # It handles parent-child relationships with a join field reindexing children when the parent is reindexed.
41
40
  #
42
41
  # Performs journaling if enabled: it stores all the ids of the imported
43
42
  # objects to a specialized index. It is possible to replay particular import
@@ -51,15 +50,15 @@ module Chewy
51
50
  #
52
51
  # Utilizes `ActiveSupport::Notifications`, so it is possible to get imported
53
52
  # objects later by listening to the `import_objects.chewy` queue. It is also
54
- # possible to get the list of occured errors from the payload if something
53
+ # possible to get the list of occurred errors from the payload if something
55
54
  # went wrong.
56
55
  #
57
56
  # Import can also be run in parallel using the Parallel gem functionality.
58
57
  #
59
58
  # @example
60
- # UsersIndex::User.import(parallel: true) # imports everything in parallel with automatic workers number
61
- # UsersIndex::User.import(parallel: 3) # using 3 workers
62
- # UsersIndex::User.import(parallel: {in_threads: 10}) # in 10 threads
59
+ # UsersIndex.import(parallel: true) # imports everything in parallel with automatic workers number
60
+ # UsersIndex.import(parallel: 3) # using 3 workers
61
+ # UsersIndex.import(parallel: {in_threads: 10}) # in 10 threads
63
62
  #
64
63
  # @see https://github.com/elastic/elasticsearch-ruby/blob/master/elasticsearch-api/lib/elasticsearch/api/actions/bulk.rb
65
64
  # @param collection [Array<Object>] and array or anything to import
@@ -74,7 +73,7 @@ module Chewy
74
73
  # @option options [true, Integer, Hash] parallel enables parallel import processing with the Parallel gem, accepts the number of workers or any Parallel gem acceptable options
75
74
  # @return [true, false] false in case of errors
76
75
  def import(*args)
77
- import_routine(*args).blank?
76
+ intercept_import_using_strategy(*args).blank?
78
77
  end
79
78
 
80
79
  # @!method import!(*collection, **options)
@@ -85,8 +84,10 @@ module Chewy
85
84
  #
86
85
  # @raise [Chewy::ImportFailed] in case of errors
87
86
  def import!(*args)
88
- errors = import_routine(*args)
87
+ errors = intercept_import_using_strategy(*args)
88
+
89
89
  raise Chewy::ImportFailed.new(self, errors) if errors.present?
90
+
90
91
  true
91
92
  end
92
93
 
@@ -94,7 +95,7 @@ module Chewy
94
95
  # `bulk_size` and `suffix`.
95
96
  #
96
97
  # @see https://github.com/elastic/elasticsearch-ruby/blob/master/elasticsearch-api/lib/elasticsearch/api/actions/bulk.rb
97
- # @see Chewy::Type::Import::Bulk
98
+ # @see Chewy::Index::Import::BulkRequest
98
99
  # @param options [Hash{Symbol => Object}] besides specific import options, it accepts all the options suitable for the bulk API call like `refresh` or `timeout`
99
100
  # @option options [String] suffix bulk API chunk size in bytes; if passed, the request is performed several times for each chunk, empty by default
100
101
  # @option options [Integer] bulk_size bulk API chunk size in bytes; if passed, the request is performed several times for each chunk, empty by default
@@ -111,11 +112,11 @@ module Chewy
111
112
  # or normal composing under the hood.
112
113
  #
113
114
  # @param object [Object] a data source object
114
- # @param crutches [Object] optional crutches object; if ommited - a crutch for the single passed object is created as a fallback
115
+ # @param crutches [Object] optional crutches object; if omitted - a crutch for the single passed object is created as a fallback
115
116
  # @param fields [Array<Symbol>] and array of fields to restrict the generated document
116
117
  # @return [Hash] a JSON-ready hash
117
118
  def compose(object, crutches = nil, fields: [])
118
- crutches ||= Chewy::Type::Crutch::Crutches.new self, [object]
119
+ crutches ||= Chewy::Index::Crutch::Crutches.new self, [object]
119
120
 
120
121
  if witchcraft? && root.children.present?
121
122
  cauldron(fields: fields).brew(object, crutches)
@@ -126,8 +127,35 @@ module Chewy
126
127
 
127
128
  private
128
129
 
130
+ def intercept_import_using_strategy(*args)
131
+ args_clone = args.deep_dup
132
+ options = args_clone.extract_options!
133
+ strategy = options.delete(:strategy)
134
+
135
+ return import_routine(*args) if strategy.blank?
136
+
137
+ ids = args_clone.flatten
138
+ return {} if ids.blank?
139
+ return {argument: {"#{strategy} supports ids only!" => ids}} unless ids.all? do |id|
140
+ id.respond_to?(:to_i)
141
+ end
142
+
143
+ case strategy
144
+ when :delayed_sidekiq
145
+ begin
146
+ Chewy::Strategy::DelayedSidekiq::Scheduler.new(self, ids, options).postpone
147
+ {} # success. errors handling convention
148
+ rescue StandardError => e
149
+ {scheduler: {e.message => ids}}
150
+ end
151
+ else
152
+ {argument: {"unsupported strategy: '#{strategy}'" => ids}}
153
+ end
154
+ end
155
+
129
156
  def import_routine(*args)
130
- return if args.first.blank? && !args.first.nil?
157
+ return if !args.first.nil? && empty_objects_or_scope?(args.first)
158
+
131
159
  routine = Routine.new(self, **args.extract_options!)
132
160
  routine.create_indexes!
133
161
 
@@ -138,8 +166,16 @@ module Chewy
138
166
  end
139
167
  end
140
168
 
169
+ def empty_objects_or_scope?(objects_or_scope)
170
+ if objects_or_scope.respond_to?(:empty?)
171
+ objects_or_scope.empty?
172
+ else
173
+ objects_or_scope.blank?
174
+ end
175
+ end
176
+
141
177
  def import_linear(objects, routine)
142
- ActiveSupport::Notifications.instrument 'import_objects.chewy', type: self do |payload|
178
+ ActiveSupport::Notifications.instrument 'import_objects.chewy', index: self do |payload|
143
179
  adapter.import(*objects, routine.options) do |action_objects|
144
180
  routine.process(**action_objects)
145
181
  end
@@ -153,17 +189,25 @@ module Chewy
153
189
  def import_parallel(objects, routine)
154
190
  raise "The `parallel` gem is required for parallel import, please add `gem 'parallel'` to your Gemfile" unless '::Parallel'.safe_constantize
155
191
 
156
- ActiveSupport::Notifications.instrument 'import_objects.chewy', type: self do |payload|
192
+ ActiveSupport::Notifications.instrument 'import_objects.chewy', index: self do |payload|
157
193
  batches = adapter.import_references(*objects, routine.options.slice(:batch_size)).to_a
158
194
 
159
195
  ::ActiveRecord::Base.connection.close if defined?(::ActiveRecord::Base)
160
- results = ::Parallel.map_with_index(batches, routine.parallel_options, &IMPORT_WORKER.curry[self, routine.options, batches.size])
196
+ results = ::Parallel.map_with_index(
197
+ batches,
198
+ routine.parallel_options,
199
+ &IMPORT_WORKER.curry[self, routine.options, batches.size]
200
+ )
161
201
  ::ActiveRecord::Base.connection.reconnect! if defined?(::ActiveRecord::Base)
162
202
  errors, import, leftovers = process_parallel_import_results(results)
163
203
 
164
204
  if leftovers.present?
165
205
  batches = leftovers.each_slice(routine.options[:batch_size])
166
- results = ::Parallel.map_with_index(batches, routine.parallel_options, &LEFTOVERS_WORKER.curry[self, routine.options, batches.size])
206
+ results = ::Parallel.map_with_index(
207
+ batches,
208
+ routine.parallel_options,
209
+ &LEFTOVERS_WORKER.curry[self, routine.options, batches.size]
210
+ )
167
211
  errors.concat(results.flatten(1))
168
212
  end
169
213
 
@@ -1,5 +1,5 @@
1
1
  module Chewy
2
- class Type
2
+ class Index
3
3
  module Mapping
4
4
  extend ActiveSupport::Concern
5
5
 
@@ -13,42 +13,39 @@ module Chewy
13
13
  end
14
14
 
15
15
  module ClassMethods
16
- # Defines root object for mapping and is optional for type
16
+ # Defines root object for mapping and is optional for index
17
17
  # definition. Use it only if you need to pass options for root
18
18
  # object mapping, such as `date_detection` or `dynamic_date_formats`
19
19
  #
20
20
  # @example
21
21
  # class UsersIndex < Chewy::Index
22
- # define_type User do
23
- # # root object defined implicitly and optionless for current type
24
- # field :full_name, type: 'keyword'
25
- # end
22
+ # index_scope User
23
+ # # root object defined implicitly and optionless for current type
24
+ # field :full_name, type: 'keyword'
26
25
  # end
27
26
  #
28
27
  # class CarsIndex < Chewy::Index
29
- # define_type Car do
30
- # # explicit root definition with additional options
31
- # root dynamic_date_formats: ['yyyy-MM-dd'] do
32
- # field :model_name, type: 'keyword'
33
- # end
28
+ # index_scope Car
29
+ # # explicit root definition with additional options
30
+ # root dynamic_date_formats: ['yyyy-MM-dd'] do
31
+ # field :model_name, type: 'keyword'
34
32
  # end
35
33
  # end
36
34
  #
37
35
  def root(**options)
38
- self.root_object ||= Chewy::Fields::Root.new(type_name, **Chewy.default_root_options.merge(options))
36
+ self.root_object ||= Chewy::Fields::Root.new(:root, **Chewy.default_root_options.merge(options))
39
37
  root_object.update_options!(**options)
40
38
  yield if block_given?
41
39
  root_object
42
40
  end
43
41
 
44
- # Defines mapping field for current type
42
+ # Defines mapping field for index
45
43
  #
46
44
  # @example
47
45
  # class UsersIndex < Chewy::Index
48
- # define_type User do
49
- # # passing all the options to field definition:
50
- # field :full_name, analyzer: 'special'
51
- # end
46
+ # index_scope User
47
+ # # passing all the options to field definition:
48
+ # field :full_name, analyzer: 'special'
52
49
  # end
53
50
  #
54
51
  # The `type` is optional and defaults to `string` if not defined:
@@ -138,14 +135,13 @@ module Chewy
138
135
  # # Suppose that a user has posts and each post has ratings
139
136
  # # avg_post_rating is the mean of all ratings
140
137
  # class UsersIndex < Chewy::Index
141
- # define_type User do
142
- # field :posts do
143
- # field :rating
144
- # end
145
- #
146
- # agg :avg_rating do
147
- # { avg: { field: 'posts.rating' } }
148
- # end
138
+ # index_scope User
139
+ # field :posts do
140
+ # field :rating
141
+ # end
142
+ #
143
+ # agg :avg_rating do
144
+ # { avg: { field: 'posts.rating' } }
149
145
  # end
150
146
  # end
151
147
  def agg(name, &block)
@@ -157,11 +153,10 @@ module Chewy
157
153
  #
158
154
  # @example
159
155
  # class CarsIndex < Chewy::Index
160
- # define_type Car do
161
- # template 'model.*', type: 'text', analyzer: 'special'
162
- # field 'model', type: 'object' # here we can put { de: 'Der Mercedes', en: 'Mercedes' }
163
- # # and template will be applyed to this field
164
- # end
156
+ # index_scope Car
157
+ # template 'model.*', type: 'text', analyzer: 'special'
158
+ # field 'model', type: 'object' # here we can put { de: 'Der Mercedes', en: 'Mercedes' }
159
+ # # and template will be applied to this field
165
160
  # end
166
161
  #
167
162
  # Name for each template is generated with the following
@@ -172,18 +167,18 @@ module Chewy
172
167
  # template 'title.*', mapping_hash # dot in template causes "path_match" using
173
168
  # template /tit.+/, mapping_hash # using "match_pattern": "regexp"
174
169
  # template /title\..+/, mapping_hash # "\." - escaped dot causes "path_match" using
175
- # template /tit.+/, type: 'text', mapping_hash # "match_mapping_type" as the optionsl second argument
170
+ # template /tit.+/, type: 'text', mapping_hash # "match_mapping_type" as an optional second argument
176
171
  # template template42: {match: 'hello*', mapping: {type: 'object'}} # or even pass a template as is
177
172
  #
178
- def template(*args)
179
- root.dynamic_template(*args)
173
+ def template(*args, **options)
174
+ root.dynamic_template(*args, **options)
180
175
  end
181
176
  alias_method :dynamic_template, :template
182
177
 
183
178
  # Returns compiled mappings hash for current type
184
179
  #
185
180
  def mappings_hash
186
- root.mappings_hash[type_name.to_sym].present? ? root.mappings_hash : {}
181
+ root.mappings_hash
187
182
  end
188
183
 
189
184
  # Check whether the type has outdated_sync_field defined with a simple value.
@@ -0,0 +1,87 @@
1
+ module Chewy
2
+ class Index
3
+ module Observe
4
+ module Helpers
5
+ def update_proc(index_name, *args, &block)
6
+ options = args.extract_options!
7
+ method = args.first
8
+
9
+ proc do
10
+ reference = if index_name.is_a?(Proc)
11
+ if index_name.arity.zero?
12
+ instance_exec(&index_name)
13
+ else
14
+ index_name.call(self)
15
+ end
16
+ else
17
+ index_name
18
+ end
19
+
20
+ index = Chewy.derive_name(reference)
21
+
22
+ next if Chewy.strategy.current.name == :bypass
23
+
24
+ backreference = if method && method.to_s == 'self'
25
+ self
26
+ elsif method
27
+ send(method)
28
+ else
29
+ instance_eval(&block)
30
+ end
31
+
32
+ index.update_index(backreference, options)
33
+ end
34
+ end
35
+
36
+ def extract_callback_options!(args)
37
+ options = args.extract_options!
38
+ result = options.each_key.with_object({}) do |key, hash|
39
+ hash[key] = options.delete(key) if %i[if unless].include?(key)
40
+ end
41
+ args.push(options) unless options.empty?
42
+ result
43
+ end
44
+ end
45
+
46
+ extend Helpers
47
+
48
+ module ActiveRecordMethods
49
+ extend ActiveSupport::Concern
50
+
51
+ def run_chewy_callbacks
52
+ chewy_callbacks.each { |callback| callback.call(self) }
53
+ end
54
+
55
+ def update_chewy_indices
56
+ Chewy.strategy.current.update_chewy_indices(self)
57
+ end
58
+
59
+ included do
60
+ class_attribute :chewy_callbacks, default: []
61
+ end
62
+
63
+ class_methods do
64
+ def initialize_chewy_callbacks
65
+ if Chewy.use_after_commit_callbacks
66
+ after_commit :update_chewy_indices, on: %i[create update]
67
+ after_commit :run_chewy_callbacks, on: :destroy
68
+ else
69
+ after_save :update_chewy_indices
70
+ after_destroy :run_chewy_callbacks
71
+ end
72
+ end
73
+
74
+ def update_index(type_name, *args, &block)
75
+ callback_options = Observe.extract_callback_options!(args)
76
+ update_proc = Observe.update_proc(type_name, *args, &block)
77
+ callback = Chewy::Index::Observe::Callback.new(update_proc, callback_options)
78
+
79
+ initialize_chewy_callbacks if chewy_callbacks.empty?
80
+
81
+ self.chewy_callbacks = chewy_callbacks.dup << callback
82
+ end
83
+ end
84
+ end
85
+ end
86
+ end
87
+ end
@@ -0,0 +1,34 @@
1
+ module Chewy
2
+ class Index
3
+ module Observe
4
+ class Callback
5
+ def initialize(executable, filters = {})
6
+ @executable = executable
7
+ @if_filter = filters[:if]
8
+ @unless_filter = filters[:unless]
9
+ end
10
+
11
+ def call(context)
12
+ return if !@if_filter.nil? && !eval_filter(@if_filter, context)
13
+ return if !@unless_filter.nil? && eval_filter(@unless_filter, context)
14
+
15
+ eval_proc(@executable, context)
16
+ end
17
+
18
+ private
19
+
20
+ def eval_filter(filter, context)
21
+ case filter
22
+ when Symbol then context.send(filter)
23
+ when Proc then eval_proc(filter, context)
24
+ else filter
25
+ end
26
+ end
27
+
28
+ def eval_proc(executable, context)
29
+ executable.arity.zero? ? context.instance_exec(&executable) : executable.call(context)
30
+ end
31
+ end
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,17 @@
1
+ require 'chewy/index/observe/callback'
2
+ require 'chewy/index/observe/active_record_methods'
3
+
4
+ module Chewy
5
+ class Index
6
+ module Observe
7
+ extend ActiveSupport::Concern
8
+
9
+ module ClassMethods
10
+ def update_index(objects, options = {})
11
+ Chewy.strategy.current.update(self, objects, options)
12
+ true
13
+ end
14
+ end
15
+ end
16
+ end
17
+ end
@@ -36,6 +36,7 @@ module Chewy
36
36
  filter = {ids: {values: [@index.derivable_name]}}
37
37
  document = Chewy::Stash::Specification.filter(filter).first
38
38
  return {} unless document
39
+
39
40
  JSON.load(Base64.decode64(document.specification)) # rubocop:disable Security/JSONLoad
40
41
  end
41
42