chewy 6.0.0 → 7.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/CODEOWNERS +1 -0
- data/.github/ISSUE_TEMPLATE/bug_report.md +39 -0
- data/.github/ISSUE_TEMPLATE/feature_request.md +20 -0
- data/.github/PULL_REQUEST_TEMPLATE.md +16 -0
- data/.github/dependabot.yml +42 -0
- data/.github/workflows/ruby.yml +60 -0
- data/.rubocop.yml +16 -8
- data/.rubocop_todo.yml +110 -22
- data/CHANGELOG.md +396 -105
- data/CODE_OF_CONDUCT.md +14 -0
- data/CONTRIBUTING.md +63 -0
- data/Gemfile +4 -10
- data/Guardfile +3 -1
- data/README.md +497 -275
- data/chewy.gemspec +5 -20
- data/gemfiles/base.gemfile +12 -0
- data/gemfiles/rails.6.1.activerecord.gemfile +10 -15
- data/gemfiles/rails.7.0.activerecord.gemfile +14 -0
- data/gemfiles/rails.7.1.activerecord.gemfile +14 -0
- data/lib/chewy/config.rb +60 -52
- data/lib/chewy/elastic_client.rb +31 -0
- data/lib/chewy/errors.rb +7 -10
- data/lib/chewy/fields/base.rb +79 -13
- data/lib/chewy/fields/root.rb +4 -14
- data/lib/chewy/index/actions.rb +54 -37
- data/lib/chewy/{type → index}/adapter/active_record.rb +30 -6
- data/lib/chewy/{type → index}/adapter/base.rb +2 -3
- data/lib/chewy/{type → index}/adapter/object.rb +27 -31
- data/lib/chewy/{type → index}/adapter/orm.rb +17 -18
- data/lib/chewy/index/aliases.rb +14 -5
- data/lib/chewy/index/crutch.rb +40 -0
- data/lib/chewy/index/import/bulk_builder.rb +311 -0
- data/lib/chewy/{type → index}/import/bulk_request.rb +6 -7
- data/lib/chewy/{type → index}/import/journal_builder.rb +11 -12
- data/lib/chewy/{type → index}/import/routine.rb +18 -17
- data/lib/chewy/{type → index}/import.rb +76 -32
- data/lib/chewy/{type → index}/mapping.rb +29 -34
- data/lib/chewy/index/observe/active_record_methods.rb +87 -0
- data/lib/chewy/index/observe/callback.rb +34 -0
- data/lib/chewy/index/observe.rb +17 -0
- data/lib/chewy/index/specification.rb +1 -0
- data/lib/chewy/{type → index}/syncer.rb +59 -59
- data/lib/chewy/{type → index}/witchcraft.rb +11 -7
- data/lib/chewy/{type → index}/wrapper.rb +2 -2
- data/lib/chewy/index.rb +67 -94
- data/lib/chewy/journal.rb +25 -14
- data/lib/chewy/log_subscriber.rb +5 -1
- data/lib/chewy/minitest/helpers.rb +86 -13
- data/lib/chewy/minitest/search_index_receiver.rb +24 -26
- data/lib/chewy/railtie.rb +6 -20
- data/lib/chewy/rake_helper.rb +169 -113
- data/lib/chewy/rspec/build_query.rb +12 -0
- data/lib/chewy/rspec/helpers.rb +55 -0
- data/lib/chewy/rspec/update_index.rb +55 -44
- data/lib/chewy/rspec.rb +2 -0
- data/lib/chewy/runtime/version.rb +1 -1
- data/lib/chewy/runtime.rb +1 -1
- data/lib/chewy/search/loader.rb +19 -41
- data/lib/chewy/search/parameters/collapse.rb +16 -0
- data/lib/chewy/search/parameters/concerns/query_storage.rb +2 -2
- data/lib/chewy/search/parameters/ignore_unavailable.rb +27 -0
- data/lib/chewy/search/parameters/indices.rb +13 -58
- data/lib/chewy/search/parameters/knn.rb +16 -0
- data/lib/chewy/search/parameters/order.rb +6 -19
- data/lib/chewy/search/parameters/source.rb +5 -1
- data/lib/chewy/search/parameters/storage.rb +1 -1
- data/lib/chewy/search/parameters/track_total_hits.rb +16 -0
- data/lib/chewy/search/parameters.rb +6 -4
- data/lib/chewy/search/query_proxy.rb +9 -2
- data/lib/chewy/search/request.rb +169 -134
- data/lib/chewy/search/response.rb +5 -5
- data/lib/chewy/search/scoping.rb +7 -8
- data/lib/chewy/search/scrolling.rb +13 -13
- data/lib/chewy/search.rb +9 -19
- data/lib/chewy/stash.rb +19 -30
- data/lib/chewy/strategy/active_job.rb +1 -1
- data/lib/chewy/strategy/atomic_no_refresh.rb +18 -0
- data/lib/chewy/strategy/base.rb +10 -0
- data/lib/chewy/strategy/delayed_sidekiq/scheduler.rb +168 -0
- data/lib/chewy/strategy/delayed_sidekiq/worker.rb +76 -0
- data/lib/chewy/strategy/delayed_sidekiq.rb +30 -0
- data/lib/chewy/strategy/lazy_sidekiq.rb +64 -0
- data/lib/chewy/strategy/sidekiq.rb +2 -1
- data/lib/chewy/strategy.rb +6 -19
- data/lib/chewy/version.rb +1 -1
- data/lib/chewy.rb +39 -86
- data/lib/generators/chewy/install_generator.rb +1 -1
- data/lib/tasks/chewy.rake +36 -32
- data/migration_guide.md +46 -8
- data/spec/chewy/config_spec.rb +16 -41
- data/spec/chewy/elastic_client_spec.rb +26 -0
- data/spec/chewy/fields/base_spec.rb +432 -147
- data/spec/chewy/fields/root_spec.rb +20 -28
- data/spec/chewy/fields/time_fields_spec.rb +5 -5
- data/spec/chewy/index/actions_spec.rb +368 -59
- data/spec/chewy/{type → index}/adapter/active_record_spec.rb +156 -40
- data/spec/chewy/{type → index}/adapter/object_spec.rb +21 -6
- data/spec/chewy/index/aliases_spec.rb +3 -3
- data/spec/chewy/index/import/bulk_builder_spec.rb +494 -0
- data/spec/chewy/{type → index}/import/bulk_request_spec.rb +5 -12
- data/spec/chewy/{type → index}/import/journal_builder_spec.rb +9 -19
- data/spec/chewy/{type → index}/import/routine_spec.rb +19 -19
- data/spec/chewy/{type → index}/import_spec.rb +164 -98
- data/spec/chewy/index/mapping_spec.rb +135 -0
- data/spec/chewy/index/observe/active_record_methods_spec.rb +68 -0
- data/spec/chewy/index/observe/callback_spec.rb +139 -0
- data/spec/chewy/index/observe_spec.rb +143 -0
- data/spec/chewy/index/settings_spec.rb +3 -1
- data/spec/chewy/index/specification_spec.rb +20 -30
- data/spec/chewy/{type → index}/syncer_spec.rb +14 -19
- data/spec/chewy/{type → index}/witchcraft_spec.rb +20 -22
- data/spec/chewy/index/wrapper_spec.rb +100 -0
- data/spec/chewy/index_spec.rb +60 -105
- data/spec/chewy/journal_spec.rb +25 -74
- data/spec/chewy/minitest/helpers_spec.rb +123 -15
- data/spec/chewy/minitest/search_index_receiver_spec.rb +28 -30
- data/spec/chewy/multi_search_spec.rb +4 -5
- data/spec/chewy/rake_helper_spec.rb +315 -55
- data/spec/chewy/rspec/build_query_spec.rb +34 -0
- data/spec/chewy/rspec/helpers_spec.rb +61 -0
- data/spec/chewy/rspec/update_index_spec.rb +74 -71
- data/spec/chewy/runtime_spec.rb +2 -2
- data/spec/chewy/search/loader_spec.rb +19 -53
- data/spec/chewy/search/pagination/kaminari_examples.rb +4 -6
- data/spec/chewy/search/pagination/kaminari_spec.rb +2 -2
- data/spec/chewy/search/parameters/collapse_spec.rb +5 -0
- data/spec/chewy/search/parameters/ignore_unavailable_spec.rb +67 -0
- data/spec/chewy/search/parameters/indices_spec.rb +26 -117
- data/spec/chewy/search/parameters/knn_spec.rb +5 -0
- data/spec/chewy/search/parameters/order_spec.rb +18 -11
- data/spec/chewy/search/parameters/query_storage_examples.rb +67 -21
- data/spec/chewy/search/parameters/search_after_spec.rb +4 -1
- data/spec/chewy/search/parameters/source_spec.rb +8 -2
- data/spec/chewy/search/parameters/track_total_hits_spec.rb +5 -0
- data/spec/chewy/search/parameters_spec.rb +18 -4
- data/spec/chewy/search/query_proxy_spec.rb +68 -17
- data/spec/chewy/search/request_spec.rb +292 -110
- data/spec/chewy/search/response_spec.rb +12 -12
- data/spec/chewy/search/scrolling_spec.rb +10 -17
- data/spec/chewy/search_spec.rb +40 -34
- data/spec/chewy/stash_spec.rb +9 -21
- data/spec/chewy/strategy/active_job_spec.rb +16 -16
- data/spec/chewy/strategy/atomic_no_refresh_spec.rb +60 -0
- data/spec/chewy/strategy/atomic_spec.rb +9 -10
- data/spec/chewy/strategy/delayed_sidekiq_spec.rb +208 -0
- data/spec/chewy/strategy/lazy_sidekiq_spec.rb +214 -0
- data/spec/chewy/strategy/sidekiq_spec.rb +12 -12
- data/spec/chewy/strategy_spec.rb +19 -15
- data/spec/chewy_spec.rb +24 -107
- data/spec/spec_helper.rb +3 -22
- data/spec/support/active_record.rb +25 -7
- metadata +78 -339
- data/.circleci/config.yml +0 -240
- data/Appraisals +0 -81
- data/gemfiles/rails.5.2.activerecord.gemfile +0 -17
- data/gemfiles/rails.5.2.mongoid.6.4.gemfile +0 -17
- data/gemfiles/rails.6.0.activerecord.gemfile +0 -17
- data/gemfiles/sequel.4.45.gemfile +0 -11
- data/lib/chewy/backports/deep_dup.rb +0 -46
- data/lib/chewy/backports/duplicable.rb +0 -91
- data/lib/chewy/search/pagination/will_paginate.rb +0 -43
- data/lib/chewy/search/parameters/types.rb +0 -20
- data/lib/chewy/strategy/resque.rb +0 -27
- data/lib/chewy/strategy/shoryuken.rb +0 -40
- data/lib/chewy/type/actions.rb +0 -43
- data/lib/chewy/type/adapter/mongoid.rb +0 -67
- data/lib/chewy/type/adapter/sequel.rb +0 -93
- data/lib/chewy/type/crutch.rb +0 -32
- data/lib/chewy/type/import/bulk_builder.rb +0 -122
- data/lib/chewy/type/observe.rb +0 -82
- data/lib/chewy/type.rb +0 -120
- data/lib/sequel/plugins/chewy_observe.rb +0 -63
- data/spec/chewy/search/pagination/will_paginate_examples.rb +0 -63
- data/spec/chewy/search/pagination/will_paginate_spec.rb +0 -23
- data/spec/chewy/search/parameters/types_spec.rb +0 -5
- data/spec/chewy/strategy/resque_spec.rb +0 -46
- data/spec/chewy/strategy/shoryuken_spec.rb +0 -70
- data/spec/chewy/type/actions_spec.rb +0 -50
- data/spec/chewy/type/adapter/mongoid_spec.rb +0 -372
- data/spec/chewy/type/adapter/sequel_spec.rb +0 -472
- data/spec/chewy/type/import/bulk_builder_spec.rb +0 -194
- data/spec/chewy/type/mapping_spec.rb +0 -175
- data/spec/chewy/type/observe_spec.rb +0 -137
- data/spec/chewy/type/wrapper_spec.rb +0 -100
- data/spec/chewy/type_spec.rb +0 -55
- data/spec/support/mongoid.rb +0 -93
- data/spec/support/sequel.rb +0 -80
@@ -1,5 +1,5 @@
|
|
1
1
|
module Chewy
|
2
|
-
class
|
2
|
+
class Index
|
3
3
|
module Import
|
4
4
|
# Adds additional features to elasticsearch-api bulk method:
|
5
5
|
# * supports Chewy index suffix if necessary;
|
@@ -10,12 +10,12 @@ module Chewy
|
|
10
10
|
#
|
11
11
|
# @see https://github.com/elastic/elasticsearch-ruby/blob/master/elasticsearch-api/lib/elasticsearch/api/actions/bulk.rb
|
12
12
|
class BulkRequest
|
13
|
-
# @param
|
13
|
+
# @param index [Chewy::Index] an index for the request
|
14
14
|
# @param suffix [String] an index name optional suffix
|
15
15
|
# @param bulk_size [Integer] bulk size in bytes
|
16
16
|
# @param bulk_options [Hash] options passed to the elasticsearch-api bulk method
|
17
|
-
def initialize(
|
18
|
-
@
|
17
|
+
def initialize(index, suffix: nil, bulk_size: nil, **bulk_options)
|
18
|
+
@index = index
|
19
19
|
@suffix = suffix
|
20
20
|
@bulk_size = bulk_size - 1.kilobyte if bulk_size # 1 kilobyte for request header and newlines
|
21
21
|
@bulk_options = bulk_options
|
@@ -33,7 +33,7 @@ module Chewy
|
|
33
33
|
return [] if body.blank?
|
34
34
|
|
35
35
|
request_bodies(body).each_with_object([]) do |request_body, results|
|
36
|
-
response = @
|
36
|
+
response = @index.client.bulk(**request_base.merge(body: request_body)) if request_body.present?
|
37
37
|
|
38
38
|
next unless response.try(:[], 'errors')
|
39
39
|
|
@@ -47,8 +47,7 @@ module Chewy
|
|
47
47
|
|
48
48
|
def request_base
|
49
49
|
@request_base ||= {
|
50
|
-
index: @
|
51
|
-
type: @type.type_name
|
50
|
+
index: @index.index_name(suffix: @suffix)
|
52
51
|
}.merge!(@bulk_options)
|
53
52
|
end
|
54
53
|
|
@@ -1,24 +1,23 @@
|
|
1
1
|
module Chewy
|
2
|
-
class
|
2
|
+
class Index
|
3
3
|
module Import
|
4
4
|
class JournalBuilder
|
5
|
-
def initialize(
|
6
|
-
@type = type
|
5
|
+
def initialize(index, to_index: [], delete: [])
|
7
6
|
@index = index
|
7
|
+
@to_index = to_index
|
8
8
|
@delete = delete
|
9
9
|
end
|
10
10
|
|
11
11
|
def bulk_body
|
12
|
-
Chewy::
|
13
|
-
Chewy::Stash::Journal
|
14
|
-
|
15
|
-
entries(:index, @
|
12
|
+
Chewy::Index::Import::BulkBuilder.new(
|
13
|
+
Chewy::Stash::Journal,
|
14
|
+
to_index: [
|
15
|
+
entries(:index, @to_index),
|
16
16
|
entries(:delete, @delete)
|
17
17
|
].compact
|
18
18
|
).bulk_body.each do |item|
|
19
19
|
item.values.first.merge!(
|
20
|
-
_index: Chewy::Stash::Journal.index_name
|
21
|
-
_type: Chewy::Stash::Journal::Journal.type_name
|
20
|
+
_index: Chewy::Stash::Journal.index_name
|
22
21
|
)
|
23
22
|
end
|
24
23
|
end
|
@@ -27,9 +26,9 @@ module Chewy
|
|
27
26
|
|
28
27
|
def entries(action, objects)
|
29
28
|
return unless objects.present?
|
29
|
+
|
30
30
|
{
|
31
|
-
index_name: @
|
32
|
-
type_name: @type.type_name,
|
31
|
+
index_name: @index.derivable_name,
|
33
32
|
action: action,
|
34
33
|
references: identify(objects).map { |item| Base64.encode64(::Elasticsearch::API.serializer.dump(item)) },
|
35
34
|
created_at: Time.now.utc
|
@@ -37,7 +36,7 @@ module Chewy
|
|
37
36
|
end
|
38
37
|
|
39
38
|
def identify(objects)
|
40
|
-
@
|
39
|
+
@index.adapter.identify(objects)
|
41
40
|
end
|
42
41
|
end
|
43
42
|
end
|
@@ -1,5 +1,5 @@
|
|
1
1
|
module Chewy
|
2
|
-
class
|
2
|
+
class Index
|
3
3
|
module Import
|
4
4
|
# This class performs the import routine for the options and objects given.
|
5
5
|
#
|
@@ -20,7 +20,7 @@ module Chewy
|
|
20
20
|
# when the document doesn't exist only if `update_failover` option is true. In order to
|
21
21
|
# restore, it indexes such an objects completely on the next iteration.
|
22
22
|
#
|
23
|
-
# @see Chewy::
|
23
|
+
# @see Chewy::Index::Import::ClassMethods#import
|
24
24
|
class Routine
|
25
25
|
BULK_OPTIONS = %i[
|
26
26
|
suffix bulk_size
|
@@ -33,18 +33,18 @@ module Chewy
|
|
33
33
|
refresh: true,
|
34
34
|
update_fields: [],
|
35
35
|
update_failover: true,
|
36
|
-
batch_size: Chewy::
|
36
|
+
batch_size: Chewy::Index::Adapter::Base::BATCH_SIZE
|
37
37
|
}.freeze
|
38
38
|
|
39
39
|
attr_reader :options, :parallel_options, :errors, :stats, :leftovers
|
40
40
|
|
41
41
|
# Basically, processes passed options, extracting bulk request specific options.
|
42
|
-
# @param
|
43
|
-
# @param options [Hash] import options, see {Chewy::
|
44
|
-
def initialize(
|
45
|
-
@
|
42
|
+
# @param index [Chewy::Index] chewy index
|
43
|
+
# @param options [Hash] import options, see {Chewy::Index::Import::ClassMethods#import}
|
44
|
+
def initialize(index, **options)
|
45
|
+
@index = index
|
46
46
|
@options = options
|
47
|
-
@options.reverse_merge!(@
|
47
|
+
@options.reverse_merge!(@index._default_import_options)
|
48
48
|
@options.reverse_merge!(journal: Chewy.configuration[:journal])
|
49
49
|
@options.reverse_merge!(DEFAULT_OPTIONS)
|
50
50
|
@bulk_options = @options.slice(*BULK_OPTIONS)
|
@@ -61,27 +61,28 @@ module Chewy
|
|
61
61
|
@leftovers = []
|
62
62
|
end
|
63
63
|
|
64
|
-
# Creates the journal index and the
|
64
|
+
# Creates the journal index and the corresponding index if necessary.
|
65
65
|
# @return [Object] whatever
|
66
66
|
def create_indexes!
|
67
|
-
Chewy::Stash::Journal.create if @options[:journal]
|
67
|
+
Chewy::Stash::Journal.create if @options[:journal] && !Chewy.configuration[:skip_journal_creation_on_import]
|
68
68
|
return if Chewy.configuration[:skip_index_creation_on_import]
|
69
|
-
|
69
|
+
|
70
|
+
@index.create!(**@bulk_options.slice(:suffix)) unless @index.exists?
|
70
71
|
end
|
71
72
|
|
72
|
-
# The main process method. Converts passed objects to
|
73
|
-
# appends journal
|
73
|
+
# The main process method. Converts passed objects to the bulk request body,
|
74
|
+
# appends journal entries, performs this request and handles errors performing
|
74
75
|
# failover procedures if applicable.
|
75
76
|
#
|
76
77
|
# @param index [Array<Object>] any acceptable objects for indexing
|
77
78
|
# @param delete [Array<Object>] any acceptable objects for deleting
|
78
79
|
# @return [true, false] the result of the request, true if no errors
|
79
80
|
def process(index: [], delete: [])
|
80
|
-
bulk_builder = BulkBuilder.new(@
|
81
|
+
bulk_builder = BulkBuilder.new(@index, to_index: index, delete: delete, fields: @options[:update_fields])
|
81
82
|
bulk_body = bulk_builder.bulk_body
|
82
83
|
|
83
84
|
if @options[:journal]
|
84
|
-
journal_builder = JournalBuilder.new(@
|
85
|
+
journal_builder = JournalBuilder.new(@index, to_index: index, delete: delete)
|
85
86
|
bulk_body.concat(journal_builder.bulk_body)
|
86
87
|
end
|
87
88
|
|
@@ -126,11 +127,11 @@ module Chewy
|
|
126
127
|
errors_to_cleanup.each { |error| errors.delete(error) }
|
127
128
|
|
128
129
|
failed_objects = index_objects_by_id.values_at(*failed_ids_for_reimport)
|
129
|
-
BulkBuilder.new(@
|
130
|
+
BulkBuilder.new(@index, to_index: failed_objects).bulk_body
|
130
131
|
end
|
131
132
|
|
132
133
|
def bulk
|
133
|
-
@bulk ||= BulkRequest.new(@
|
134
|
+
@bulk ||= BulkRequest.new(@index, **@bulk_options)
|
134
135
|
end
|
135
136
|
end
|
136
137
|
end
|
@@ -1,43 +1,42 @@
|
|
1
|
-
require 'chewy/
|
2
|
-
require 'chewy/
|
3
|
-
require 'chewy/
|
4
|
-
require 'chewy/
|
1
|
+
require 'chewy/index/import/journal_builder'
|
2
|
+
require 'chewy/index/import/bulk_builder'
|
3
|
+
require 'chewy/index/import/bulk_request'
|
4
|
+
require 'chewy/index/import/routine'
|
5
5
|
|
6
6
|
module Chewy
|
7
|
-
class
|
7
|
+
class Index
|
8
8
|
module Import
|
9
9
|
extend ActiveSupport::Concern
|
10
10
|
|
11
|
-
IMPORT_WORKER = lambda do |
|
12
|
-
::Process.setproctitle("chewy [#{
|
13
|
-
routine = Routine.new(
|
14
|
-
|
11
|
+
IMPORT_WORKER = lambda do |index, options, total, ids, iteration|
|
12
|
+
::Process.setproctitle("chewy [#{index}]: import data (#{iteration + 1}/#{total})")
|
13
|
+
routine = Routine.new(index, **options)
|
14
|
+
index.adapter.import(*ids, routine.options) do |action_objects|
|
15
15
|
routine.process(**action_objects)
|
16
16
|
end
|
17
17
|
{errors: routine.errors, import: routine.stats, leftovers: routine.leftovers}
|
18
18
|
end
|
19
19
|
|
20
|
-
LEFTOVERS_WORKER = lambda do |
|
21
|
-
::Process.setproctitle("chewy [#{
|
22
|
-
routine = Routine.new(
|
20
|
+
LEFTOVERS_WORKER = lambda do |index, options, total, body, iteration|
|
21
|
+
::Process.setproctitle("chewy [#{index}]: import leftovers (#{iteration + 1}/#{total})")
|
22
|
+
routine = Routine.new(index, **options)
|
23
23
|
routine.perform_bulk(body)
|
24
24
|
routine.errors
|
25
25
|
end
|
26
26
|
|
27
27
|
module ClassMethods
|
28
28
|
# @!method import(*collection, **options)
|
29
|
-
# Basically, one of the main methods for
|
30
|
-
# to the index
|
29
|
+
# Basically, one of the main methods for an index. Performs any objects import
|
30
|
+
# to the index. Does all the objects handling routines.
|
31
31
|
# Performs document import by utilizing bulk API. Bulk size and objects batch
|
32
32
|
# size are controlled by the corresponding options.
|
33
33
|
#
|
34
34
|
# It accepts ORM/ODM objects, PORO, hashes, ids which are used by adapter to
|
35
|
-
# fetch objects from the source
|
36
|
-
# passed objects from the index if they are not in the default
|
35
|
+
# fetch objects from the source depending on the used adapter. It destroys
|
36
|
+
# passed objects from the index if they are not in the default scope
|
37
37
|
# or marked for destruction.
|
38
38
|
#
|
39
|
-
# It handles parent-child relationships
|
40
|
-
# changed it destroys the object and recreates it from scratch.
|
39
|
+
# It handles parent-child relationships with a join field reindexing children when the parent is reindexed.
|
41
40
|
#
|
42
41
|
# Performs journaling if enabled: it stores all the ids of the imported
|
43
42
|
# objects to a specialized index. It is possible to replay particular import
|
@@ -51,15 +50,15 @@ module Chewy
|
|
51
50
|
#
|
52
51
|
# Utilizes `ActiveSupport::Notifications`, so it is possible to get imported
|
53
52
|
# objects later by listening to the `import_objects.chewy` queue. It is also
|
54
|
-
# possible to get the list of
|
53
|
+
# possible to get the list of occurred errors from the payload if something
|
55
54
|
# went wrong.
|
56
55
|
#
|
57
56
|
# Import can also be run in parallel using the Parallel gem functionality.
|
58
57
|
#
|
59
58
|
# @example
|
60
|
-
# UsersIndex
|
61
|
-
# UsersIndex
|
62
|
-
# UsersIndex
|
59
|
+
# UsersIndex.import(parallel: true) # imports everything in parallel with automatic workers number
|
60
|
+
# UsersIndex.import(parallel: 3) # using 3 workers
|
61
|
+
# UsersIndex.import(parallel: {in_threads: 10}) # in 10 threads
|
63
62
|
#
|
64
63
|
# @see https://github.com/elastic/elasticsearch-ruby/blob/master/elasticsearch-api/lib/elasticsearch/api/actions/bulk.rb
|
65
64
|
# @param collection [Array<Object>] and array or anything to import
|
@@ -74,7 +73,7 @@ module Chewy
|
|
74
73
|
# @option options [true, Integer, Hash] parallel enables parallel import processing with the Parallel gem, accepts the number of workers or any Parallel gem acceptable options
|
75
74
|
# @return [true, false] false in case of errors
|
76
75
|
def import(*args)
|
77
|
-
|
76
|
+
intercept_import_using_strategy(*args).blank?
|
78
77
|
end
|
79
78
|
|
80
79
|
# @!method import!(*collection, **options)
|
@@ -85,8 +84,10 @@ module Chewy
|
|
85
84
|
#
|
86
85
|
# @raise [Chewy::ImportFailed] in case of errors
|
87
86
|
def import!(*args)
|
88
|
-
errors =
|
87
|
+
errors = intercept_import_using_strategy(*args)
|
88
|
+
|
89
89
|
raise Chewy::ImportFailed.new(self, errors) if errors.present?
|
90
|
+
|
90
91
|
true
|
91
92
|
end
|
92
93
|
|
@@ -94,7 +95,7 @@ module Chewy
|
|
94
95
|
# `bulk_size` and `suffix`.
|
95
96
|
#
|
96
97
|
# @see https://github.com/elastic/elasticsearch-ruby/blob/master/elasticsearch-api/lib/elasticsearch/api/actions/bulk.rb
|
97
|
-
# @see Chewy::
|
98
|
+
# @see Chewy::Index::Import::BulkRequest
|
98
99
|
# @param options [Hash{Symbol => Object}] besides specific import options, it accepts all the options suitable for the bulk API call like `refresh` or `timeout`
|
99
100
|
# @option options [String] suffix bulk API chunk size in bytes; if passed, the request is performed several times for each chunk, empty by default
|
100
101
|
# @option options [Integer] bulk_size bulk API chunk size in bytes; if passed, the request is performed several times for each chunk, empty by default
|
@@ -111,11 +112,11 @@ module Chewy
|
|
111
112
|
# or normal composing under the hood.
|
112
113
|
#
|
113
114
|
# @param object [Object] a data source object
|
114
|
-
# @param crutches [Object] optional crutches object; if
|
115
|
+
# @param crutches [Object] optional crutches object; if omitted - a crutch for the single passed object is created as a fallback
|
115
116
|
# @param fields [Array<Symbol>] and array of fields to restrict the generated document
|
116
117
|
# @return [Hash] a JSON-ready hash
|
117
118
|
def compose(object, crutches = nil, fields: [])
|
118
|
-
crutches ||= Chewy::
|
119
|
+
crutches ||= Chewy::Index::Crutch::Crutches.new self, [object]
|
119
120
|
|
120
121
|
if witchcraft? && root.children.present?
|
121
122
|
cauldron(fields: fields).brew(object, crutches)
|
@@ -126,8 +127,35 @@ module Chewy
|
|
126
127
|
|
127
128
|
private
|
128
129
|
|
130
|
+
def intercept_import_using_strategy(*args)
|
131
|
+
args_clone = args.deep_dup
|
132
|
+
options = args_clone.extract_options!
|
133
|
+
strategy = options.delete(:strategy)
|
134
|
+
|
135
|
+
return import_routine(*args) if strategy.blank?
|
136
|
+
|
137
|
+
ids = args_clone.flatten
|
138
|
+
return {} if ids.blank?
|
139
|
+
return {argument: {"#{strategy} supports ids only!" => ids}} unless ids.all? do |id|
|
140
|
+
id.respond_to?(:to_i)
|
141
|
+
end
|
142
|
+
|
143
|
+
case strategy
|
144
|
+
when :delayed_sidekiq
|
145
|
+
begin
|
146
|
+
Chewy::Strategy::DelayedSidekiq::Scheduler.new(self, ids, options).postpone
|
147
|
+
{} # success. errors handling convention
|
148
|
+
rescue StandardError => e
|
149
|
+
{scheduler: {e.message => ids}}
|
150
|
+
end
|
151
|
+
else
|
152
|
+
{argument: {"unsupported strategy: '#{strategy}'" => ids}}
|
153
|
+
end
|
154
|
+
end
|
155
|
+
|
129
156
|
def import_routine(*args)
|
130
|
-
return if args.first.
|
157
|
+
return if !args.first.nil? && empty_objects_or_scope?(args.first)
|
158
|
+
|
131
159
|
routine = Routine.new(self, **args.extract_options!)
|
132
160
|
routine.create_indexes!
|
133
161
|
|
@@ -138,8 +166,16 @@ module Chewy
|
|
138
166
|
end
|
139
167
|
end
|
140
168
|
|
169
|
+
def empty_objects_or_scope?(objects_or_scope)
|
170
|
+
if objects_or_scope.respond_to?(:empty?)
|
171
|
+
objects_or_scope.empty?
|
172
|
+
else
|
173
|
+
objects_or_scope.blank?
|
174
|
+
end
|
175
|
+
end
|
176
|
+
|
141
177
|
def import_linear(objects, routine)
|
142
|
-
ActiveSupport::Notifications.instrument 'import_objects.chewy',
|
178
|
+
ActiveSupport::Notifications.instrument 'import_objects.chewy', index: self do |payload|
|
143
179
|
adapter.import(*objects, routine.options) do |action_objects|
|
144
180
|
routine.process(**action_objects)
|
145
181
|
end
|
@@ -153,17 +189,25 @@ module Chewy
|
|
153
189
|
def import_parallel(objects, routine)
|
154
190
|
raise "The `parallel` gem is required for parallel import, please add `gem 'parallel'` to your Gemfile" unless '::Parallel'.safe_constantize
|
155
191
|
|
156
|
-
ActiveSupport::Notifications.instrument 'import_objects.chewy',
|
192
|
+
ActiveSupport::Notifications.instrument 'import_objects.chewy', index: self do |payload|
|
157
193
|
batches = adapter.import_references(*objects, routine.options.slice(:batch_size)).to_a
|
158
194
|
|
159
195
|
::ActiveRecord::Base.connection.close if defined?(::ActiveRecord::Base)
|
160
|
-
results = ::Parallel.map_with_index(
|
196
|
+
results = ::Parallel.map_with_index(
|
197
|
+
batches,
|
198
|
+
routine.parallel_options,
|
199
|
+
&IMPORT_WORKER.curry[self, routine.options, batches.size]
|
200
|
+
)
|
161
201
|
::ActiveRecord::Base.connection.reconnect! if defined?(::ActiveRecord::Base)
|
162
202
|
errors, import, leftovers = process_parallel_import_results(results)
|
163
203
|
|
164
204
|
if leftovers.present?
|
165
205
|
batches = leftovers.each_slice(routine.options[:batch_size])
|
166
|
-
results = ::Parallel.map_with_index(
|
206
|
+
results = ::Parallel.map_with_index(
|
207
|
+
batches,
|
208
|
+
routine.parallel_options,
|
209
|
+
&LEFTOVERS_WORKER.curry[self, routine.options, batches.size]
|
210
|
+
)
|
167
211
|
errors.concat(results.flatten(1))
|
168
212
|
end
|
169
213
|
|
@@ -1,5 +1,5 @@
|
|
1
1
|
module Chewy
|
2
|
-
class
|
2
|
+
class Index
|
3
3
|
module Mapping
|
4
4
|
extend ActiveSupport::Concern
|
5
5
|
|
@@ -13,42 +13,39 @@ module Chewy
|
|
13
13
|
end
|
14
14
|
|
15
15
|
module ClassMethods
|
16
|
-
# Defines root object for mapping and is optional for
|
16
|
+
# Defines root object for mapping and is optional for index
|
17
17
|
# definition. Use it only if you need to pass options for root
|
18
18
|
# object mapping, such as `date_detection` or `dynamic_date_formats`
|
19
19
|
#
|
20
20
|
# @example
|
21
21
|
# class UsersIndex < Chewy::Index
|
22
|
-
#
|
23
|
-
#
|
24
|
-
#
|
25
|
-
# end
|
22
|
+
# index_scope User
|
23
|
+
# # root object defined implicitly and optionless for current type
|
24
|
+
# field :full_name, type: 'keyword'
|
26
25
|
# end
|
27
26
|
#
|
28
27
|
# class CarsIndex < Chewy::Index
|
29
|
-
#
|
30
|
-
#
|
31
|
-
#
|
32
|
-
#
|
33
|
-
# end
|
28
|
+
# index_scope Car
|
29
|
+
# # explicit root definition with additional options
|
30
|
+
# root dynamic_date_formats: ['yyyy-MM-dd'] do
|
31
|
+
# field :model_name, type: 'keyword'
|
34
32
|
# end
|
35
33
|
# end
|
36
34
|
#
|
37
35
|
def root(**options)
|
38
|
-
self.root_object ||= Chewy::Fields::Root.new(
|
36
|
+
self.root_object ||= Chewy::Fields::Root.new(:root, **Chewy.default_root_options.merge(options))
|
39
37
|
root_object.update_options!(**options)
|
40
38
|
yield if block_given?
|
41
39
|
root_object
|
42
40
|
end
|
43
41
|
|
44
|
-
# Defines mapping field for
|
42
|
+
# Defines mapping field for index
|
45
43
|
#
|
46
44
|
# @example
|
47
45
|
# class UsersIndex < Chewy::Index
|
48
|
-
#
|
49
|
-
#
|
50
|
-
#
|
51
|
-
# end
|
46
|
+
# index_scope User
|
47
|
+
# # passing all the options to field definition:
|
48
|
+
# field :full_name, analyzer: 'special'
|
52
49
|
# end
|
53
50
|
#
|
54
51
|
# The `type` is optional and defaults to `string` if not defined:
|
@@ -138,14 +135,13 @@ module Chewy
|
|
138
135
|
# # Suppose that a user has posts and each post has ratings
|
139
136
|
# # avg_post_rating is the mean of all ratings
|
140
137
|
# class UsersIndex < Chewy::Index
|
141
|
-
#
|
142
|
-
#
|
143
|
-
#
|
144
|
-
#
|
145
|
-
#
|
146
|
-
#
|
147
|
-
#
|
148
|
-
# end
|
138
|
+
# index_scope User
|
139
|
+
# field :posts do
|
140
|
+
# field :rating
|
141
|
+
# end
|
142
|
+
#
|
143
|
+
# agg :avg_rating do
|
144
|
+
# { avg: { field: 'posts.rating' } }
|
149
145
|
# end
|
150
146
|
# end
|
151
147
|
def agg(name, &block)
|
@@ -157,11 +153,10 @@ module Chewy
|
|
157
153
|
#
|
158
154
|
# @example
|
159
155
|
# class CarsIndex < Chewy::Index
|
160
|
-
#
|
161
|
-
#
|
162
|
-
#
|
163
|
-
#
|
164
|
-
# end
|
156
|
+
# index_scope Car
|
157
|
+
# template 'model.*', type: 'text', analyzer: 'special'
|
158
|
+
# field 'model', type: 'object' # here we can put { de: 'Der Mercedes', en: 'Mercedes' }
|
159
|
+
# # and template will be applied to this field
|
165
160
|
# end
|
166
161
|
#
|
167
162
|
# Name for each template is generated with the following
|
@@ -172,18 +167,18 @@ module Chewy
|
|
172
167
|
# template 'title.*', mapping_hash # dot in template causes "path_match" using
|
173
168
|
# template /tit.+/, mapping_hash # using "match_pattern": "regexp"
|
174
169
|
# template /title\..+/, mapping_hash # "\." - escaped dot causes "path_match" using
|
175
|
-
# template /tit.+/, type: 'text', mapping_hash # "match_mapping_type" as
|
170
|
+
# template /tit.+/, type: 'text', mapping_hash # "match_mapping_type" as an optional second argument
|
176
171
|
# template template42: {match: 'hello*', mapping: {type: 'object'}} # or even pass a template as is
|
177
172
|
#
|
178
|
-
def template(*args)
|
179
|
-
root.dynamic_template(*args)
|
173
|
+
def template(*args, **options)
|
174
|
+
root.dynamic_template(*args, **options)
|
180
175
|
end
|
181
176
|
alias_method :dynamic_template, :template
|
182
177
|
|
183
178
|
# Returns compiled mappings hash for current type
|
184
179
|
#
|
185
180
|
def mappings_hash
|
186
|
-
root.mappings_hash
|
181
|
+
root.mappings_hash
|
187
182
|
end
|
188
183
|
|
189
184
|
# Check whether the type has outdated_sync_field defined with a simple value.
|
@@ -0,0 +1,87 @@
|
|
1
|
+
module Chewy
|
2
|
+
class Index
|
3
|
+
module Observe
|
4
|
+
module Helpers
|
5
|
+
def update_proc(index_name, *args, &block)
|
6
|
+
options = args.extract_options!
|
7
|
+
method = args.first
|
8
|
+
|
9
|
+
proc do
|
10
|
+
reference = if index_name.is_a?(Proc)
|
11
|
+
if index_name.arity.zero?
|
12
|
+
instance_exec(&index_name)
|
13
|
+
else
|
14
|
+
index_name.call(self)
|
15
|
+
end
|
16
|
+
else
|
17
|
+
index_name
|
18
|
+
end
|
19
|
+
|
20
|
+
index = Chewy.derive_name(reference)
|
21
|
+
|
22
|
+
next if Chewy.strategy.current.name == :bypass
|
23
|
+
|
24
|
+
backreference = if method && method.to_s == 'self'
|
25
|
+
self
|
26
|
+
elsif method
|
27
|
+
send(method)
|
28
|
+
else
|
29
|
+
instance_eval(&block)
|
30
|
+
end
|
31
|
+
|
32
|
+
index.update_index(backreference, options)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def extract_callback_options!(args)
|
37
|
+
options = args.extract_options!
|
38
|
+
result = options.each_key.with_object({}) do |key, hash|
|
39
|
+
hash[key] = options.delete(key) if %i[if unless].include?(key)
|
40
|
+
end
|
41
|
+
args.push(options) unless options.empty?
|
42
|
+
result
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
extend Helpers
|
47
|
+
|
48
|
+
module ActiveRecordMethods
|
49
|
+
extend ActiveSupport::Concern
|
50
|
+
|
51
|
+
def run_chewy_callbacks
|
52
|
+
chewy_callbacks.each { |callback| callback.call(self) }
|
53
|
+
end
|
54
|
+
|
55
|
+
def update_chewy_indices
|
56
|
+
Chewy.strategy.current.update_chewy_indices(self)
|
57
|
+
end
|
58
|
+
|
59
|
+
included do
|
60
|
+
class_attribute :chewy_callbacks, default: []
|
61
|
+
end
|
62
|
+
|
63
|
+
class_methods do
|
64
|
+
def initialize_chewy_callbacks
|
65
|
+
if Chewy.use_after_commit_callbacks
|
66
|
+
after_commit :update_chewy_indices, on: %i[create update]
|
67
|
+
after_commit :run_chewy_callbacks, on: :destroy
|
68
|
+
else
|
69
|
+
after_save :update_chewy_indices
|
70
|
+
after_destroy :run_chewy_callbacks
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
def update_index(type_name, *args, &block)
|
75
|
+
callback_options = Observe.extract_callback_options!(args)
|
76
|
+
update_proc = Observe.update_proc(type_name, *args, &block)
|
77
|
+
callback = Chewy::Index::Observe::Callback.new(update_proc, callback_options)
|
78
|
+
|
79
|
+
initialize_chewy_callbacks if chewy_callbacks.empty?
|
80
|
+
|
81
|
+
self.chewy_callbacks = chewy_callbacks.dup << callback
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
module Chewy
|
2
|
+
class Index
|
3
|
+
module Observe
|
4
|
+
class Callback
|
5
|
+
def initialize(executable, filters = {})
|
6
|
+
@executable = executable
|
7
|
+
@if_filter = filters[:if]
|
8
|
+
@unless_filter = filters[:unless]
|
9
|
+
end
|
10
|
+
|
11
|
+
def call(context)
|
12
|
+
return if !@if_filter.nil? && !eval_filter(@if_filter, context)
|
13
|
+
return if !@unless_filter.nil? && eval_filter(@unless_filter, context)
|
14
|
+
|
15
|
+
eval_proc(@executable, context)
|
16
|
+
end
|
17
|
+
|
18
|
+
private
|
19
|
+
|
20
|
+
def eval_filter(filter, context)
|
21
|
+
case filter
|
22
|
+
when Symbol then context.send(filter)
|
23
|
+
when Proc then eval_proc(filter, context)
|
24
|
+
else filter
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def eval_proc(executable, context)
|
29
|
+
executable.arity.zero? ? context.instance_exec(&executable) : executable.call(context)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require 'chewy/index/observe/callback'
|
2
|
+
require 'chewy/index/observe/active_record_methods'
|
3
|
+
|
4
|
+
module Chewy
|
5
|
+
class Index
|
6
|
+
module Observe
|
7
|
+
extend ActiveSupport::Concern
|
8
|
+
|
9
|
+
module ClassMethods
|
10
|
+
def update_index(objects, options = {})
|
11
|
+
Chewy.strategy.current.update(self, objects, options)
|
12
|
+
true
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|