chewy 7.1.0 → 7.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ruby.yml +58 -0
- data/.rubocop.yml +13 -8
- data/.rubocop_todo.yml +110 -22
- data/CHANGELOG.md +53 -0
- data/Gemfile +0 -7
- data/Guardfile +3 -1
- data/README.md +282 -245
- data/chewy.gemspec +3 -5
- data/gemfiles/rails.5.2.activerecord.gemfile +8 -14
- data/gemfiles/rails.6.0.activerecord.gemfile +8 -14
- data/gemfiles/rails.6.1.activerecord.gemfile +8 -14
- data/lib/chewy.rb +21 -75
- data/lib/chewy/config.rb +40 -40
- data/lib/chewy/errors.rb +0 -12
- data/lib/chewy/fields/base.rb +11 -1
- data/lib/chewy/fields/root.rb +3 -4
- data/lib/chewy/index.rb +46 -87
- data/lib/chewy/index/actions.rb +51 -32
- data/lib/chewy/{type → index}/adapter/active_record.rb +12 -3
- data/lib/chewy/{type → index}/adapter/base.rb +2 -3
- data/lib/chewy/{type → index}/adapter/object.rb +27 -31
- data/lib/chewy/{type → index}/adapter/orm.rb +11 -14
- data/lib/chewy/{type → index}/crutch.rb +5 -5
- data/lib/chewy/{type → index}/import.rb +36 -27
- data/lib/chewy/{type → index}/import/bulk_builder.rb +15 -13
- data/lib/chewy/{type → index}/import/bulk_request.rb +6 -6
- data/lib/chewy/{type → index}/import/journal_builder.rb +10 -10
- data/lib/chewy/{type → index}/import/routine.rb +15 -14
- data/lib/chewy/{type → index}/mapping.rb +26 -31
- data/lib/chewy/{type → index}/observe.rb +9 -19
- data/lib/chewy/index/specification.rb +1 -0
- data/lib/chewy/{type → index}/syncer.rb +60 -57
- data/lib/chewy/{type → index}/witchcraft.rb +11 -7
- data/lib/chewy/{type → index}/wrapper.rb +2 -2
- data/lib/chewy/journal.rb +8 -8
- data/lib/chewy/minitest/helpers.rb +9 -13
- data/lib/chewy/minitest/search_index_receiver.rb +22 -26
- data/lib/chewy/railtie.rb +4 -2
- data/lib/chewy/rake_helper.rb +82 -107
- data/lib/chewy/rspec/update_index.rb +47 -43
- data/lib/chewy/search.rb +4 -17
- data/lib/chewy/search/loader.rb +18 -30
- data/lib/chewy/search/parameters.rb +4 -2
- data/lib/chewy/search/parameters/concerns/query_storage.rb +2 -2
- data/lib/chewy/search/parameters/source.rb +5 -1
- data/lib/chewy/search/query_proxy.rb +9 -2
- data/lib/chewy/search/request.rb +82 -86
- data/lib/chewy/search/response.rb +4 -4
- data/lib/chewy/search/scoping.rb +6 -7
- data/lib/chewy/search/scrolling.rb +11 -11
- data/lib/chewy/stash.rb +14 -22
- data/lib/chewy/strategy.rb +3 -19
- data/lib/chewy/strategy/sidekiq.rb +1 -0
- data/lib/chewy/version.rb +1 -1
- data/lib/generators/chewy/install_generator.rb +1 -1
- data/lib/tasks/chewy.rake +10 -22
- data/migration_guide.md +14 -0
- data/spec/chewy/config_spec.rb +14 -39
- data/spec/chewy/fields/base_spec.rb +412 -148
- data/spec/chewy/fields/root_spec.rb +16 -24
- data/spec/chewy/fields/time_fields_spec.rb +5 -5
- data/spec/chewy/index/actions_spec.rb +270 -24
- data/spec/chewy/{type → index}/adapter/active_record_spec.rb +68 -40
- data/spec/chewy/{type → index}/adapter/object_spec.rb +21 -6
- data/spec/chewy/{type → index}/import/bulk_builder_spec.rb +23 -31
- data/spec/chewy/{type → index}/import/bulk_request_spec.rb +5 -6
- data/spec/chewy/{type → index}/import/journal_builder_spec.rb +9 -15
- data/spec/chewy/{type → index}/import/routine_spec.rb +16 -16
- data/spec/chewy/{type → index}/import_spec.rb +102 -98
- data/spec/chewy/{type → index}/mapping_spec.rb +46 -54
- data/spec/chewy/index/observe_spec.rb +116 -0
- data/spec/chewy/index/settings_spec.rb +3 -1
- data/spec/chewy/index/specification_spec.rb +7 -17
- data/spec/chewy/{type → index}/syncer_spec.rb +14 -19
- data/spec/chewy/{type → index}/witchcraft_spec.rb +20 -22
- data/spec/chewy/index/wrapper_spec.rb +100 -0
- data/spec/chewy/index_spec.rb +59 -102
- data/spec/chewy/journal_spec.rb +9 -22
- data/spec/chewy/minitest/helpers_spec.rb +13 -15
- data/spec/chewy/minitest/search_index_receiver_spec.rb +22 -26
- data/spec/chewy/multi_search_spec.rb +4 -5
- data/spec/chewy/rake_helper_spec.rb +145 -55
- data/spec/chewy/rspec/update_index_spec.rb +74 -71
- data/spec/chewy/search/loader_spec.rb +19 -37
- data/spec/chewy/search/pagination/kaminari_examples.rb +3 -5
- data/spec/chewy/search/pagination/kaminari_spec.rb +1 -1
- data/spec/chewy/search/parameters/indices_spec.rb +2 -8
- data/spec/chewy/search/parameters/order_spec.rb +1 -1
- data/spec/chewy/search/parameters/query_storage_examples.rb +67 -21
- data/spec/chewy/search/parameters/search_after_spec.rb +4 -1
- data/spec/chewy/search/parameters/source_spec.rb +8 -2
- data/spec/chewy/search/parameters_spec.rb +12 -3
- data/spec/chewy/search/query_proxy_spec.rb +68 -17
- data/spec/chewy/search/request_spec.rb +222 -74
- data/spec/chewy/search/response_spec.rb +12 -12
- data/spec/chewy/search/scrolling_spec.rb +7 -9
- data/spec/chewy/search_spec.rb +32 -35
- data/spec/chewy/stash_spec.rb +9 -21
- data/spec/chewy/strategy/active_job_spec.rb +8 -8
- data/spec/chewy/strategy/atomic_spec.rb +9 -10
- data/spec/chewy/strategy/sidekiq_spec.rb +8 -8
- data/spec/chewy/strategy_spec.rb +19 -15
- data/spec/chewy_spec.rb +14 -100
- data/spec/spec_helper.rb +2 -21
- data/spec/support/active_record.rb +15 -5
- metadata +44 -103
- data/.circleci/config.yml +0 -214
- data/Appraisals +0 -81
- data/gemfiles/rails.5.2.mongoid.6.4.gemfile +0 -17
- data/gemfiles/sequel.4.45.gemfile +0 -11
- data/lib/chewy/search/pagination/will_paginate.rb +0 -43
- data/lib/chewy/strategy/resque.rb +0 -27
- data/lib/chewy/strategy/shoryuken.rb +0 -40
- data/lib/chewy/type.rb +0 -120
- data/lib/chewy/type/actions.rb +0 -43
- data/lib/chewy/type/adapter/mongoid.rb +0 -67
- data/lib/chewy/type/adapter/sequel.rb +0 -93
- data/lib/sequel/plugins/chewy_observe.rb +0 -63
- data/spec/chewy/search/pagination/will_paginate_examples.rb +0 -63
- data/spec/chewy/search/pagination/will_paginate_spec.rb +0 -23
- data/spec/chewy/strategy/resque_spec.rb +0 -46
- data/spec/chewy/strategy/shoryuken_spec.rb +0 -70
- data/spec/chewy/type/actions_spec.rb +0 -50
- data/spec/chewy/type/adapter/mongoid_spec.rb +0 -372
- data/spec/chewy/type/adapter/sequel_spec.rb +0 -472
- data/spec/chewy/type/observe_spec.rb +0 -137
- data/spec/chewy/type/wrapper_spec.rb +0 -100
- data/spec/chewy/type_spec.rb +0 -55
- data/spec/support/mongoid.rb +0 -93
- data/spec/support/sequel.rb +0 -80
@@ -1,5 +1,5 @@
|
|
1
1
|
module Chewy
|
2
|
-
class
|
2
|
+
class Index
|
3
3
|
module Wrapper
|
4
4
|
extend ActiveSupport::Concern
|
5
5
|
|
@@ -28,7 +28,7 @@ module Chewy
|
|
28
28
|
def ==(other)
|
29
29
|
return true if super
|
30
30
|
|
31
|
-
if other.is_a?(Chewy::
|
31
|
+
if other.is_a?(Chewy::Index)
|
32
32
|
self.class == other.class && (respond_to?(:id) ? id == other.id : attributes == other.attributes)
|
33
33
|
elsif other.respond_to?(:id)
|
34
34
|
self.class.adapter.target.is_a?(Class) &&
|
data/lib/chewy/journal.rb
CHANGED
@@ -2,18 +2,18 @@ module Chewy
|
|
2
2
|
# A class to perform journal-related actions for the specified indexes/types.
|
3
3
|
#
|
4
4
|
# @example
|
5
|
-
# journal = Chewy::Journal.new('places
|
5
|
+
# journal = Chewy::Journal.new('places', UsersIndex)
|
6
6
|
# journal.apply(20.minutes.ago)
|
7
7
|
# journal.clean
|
8
8
|
#
|
9
9
|
class Journal
|
10
|
-
# @param only [Array<String, Chewy::Index
|
10
|
+
# @param only [Array<String, Chewy::Index>] indexes or string references to perform actions on
|
11
11
|
def initialize(*only)
|
12
12
|
@only = only
|
13
13
|
end
|
14
14
|
|
15
15
|
# Applies all changes that were done since the specified time to the
|
16
|
-
# specified indexes
|
16
|
+
# specified indexes.
|
17
17
|
#
|
18
18
|
# @param since_time [Time, DateTime] timestamp from which changes will be applied
|
19
19
|
# @param retries [Integer] maximum number of attempts to make journal empty, 10 by default
|
@@ -27,8 +27,8 @@ module Chewy
|
|
27
27
|
count += entries.size
|
28
28
|
groups = reference_groups(entries)
|
29
29
|
ActiveSupport::Notifications.instrument 'apply_journal.chewy', stage: stage, groups: groups
|
30
|
-
groups.each do |
|
31
|
-
|
30
|
+
groups.each do |index, references|
|
31
|
+
index.import(references, import_options.merge(journal: false))
|
32
32
|
end
|
33
33
|
stage += 1
|
34
34
|
since_time = entries.map(&:created_at).max
|
@@ -47,9 +47,9 @@ module Chewy
|
|
47
47
|
private
|
48
48
|
|
49
49
|
def reference_groups(entries)
|
50
|
-
entries.group_by(&:
|
51
|
-
|
52
|
-
|
50
|
+
entries.group_by(&:index_name)
|
51
|
+
.transform_keys { |index_name| Chewy.derive_name(index_name) }
|
52
|
+
.transform_values { |grouped_entries| grouped_entries.map(&:references).inject(:|) }
|
53
53
|
end
|
54
54
|
end
|
55
55
|
end
|
@@ -6,7 +6,7 @@ module Chewy
|
|
6
6
|
extend ActiveSupport::Concern
|
7
7
|
|
8
8
|
# Assert that an index *changes* during a block.
|
9
|
-
# @param index [Chewy::
|
9
|
+
# @param index [Chewy::Index] the index to watch, eg EntitiesIndex.
|
10
10
|
# @param strategy [Symbol] the Chewy strategy to use around the block. See Chewy docs.
|
11
11
|
# @param bypass_actual_index [true, false]
|
12
12
|
# True to preempt the http call to Elastic, false otherwise.
|
@@ -14,11 +14,11 @@ module Chewy
|
|
14
14
|
#
|
15
15
|
# @return [SearchIndexReceiver] for optional further assertions on the nature of the index changes.
|
16
16
|
#
|
17
|
-
def assert_indexes(index, strategy: :atomic, bypass_actual_index: true)
|
18
|
-
|
17
|
+
def assert_indexes(index, strategy: :atomic, bypass_actual_index: true, &block)
|
18
|
+
index_class = Chewy.derive_name index
|
19
19
|
receiver = SearchIndexReceiver.new
|
20
20
|
|
21
|
-
bulk_method =
|
21
|
+
bulk_method = index_class.method :bulk
|
22
22
|
# Manually mocking #bulk because we need to properly capture `self`
|
23
23
|
bulk_mock = lambda do |*bulk_args|
|
24
24
|
receiver.catch bulk_args, self
|
@@ -28,13 +28,11 @@ module Chewy
|
|
28
28
|
{}
|
29
29
|
end
|
30
30
|
|
31
|
-
|
31
|
+
index_class.define_singleton_method :bulk, bulk_mock
|
32
32
|
|
33
|
-
Chewy.strategy(strategy)
|
34
|
-
yield
|
35
|
-
end
|
33
|
+
Chewy.strategy(strategy, &block)
|
36
34
|
|
37
|
-
|
35
|
+
index_class.define_singleton_method :bulk, bulk_method
|
38
36
|
|
39
37
|
assert_includes receiver.updated_indexes, index, "Expected #{index} to be updated but it wasn't"
|
40
38
|
|
@@ -44,10 +42,8 @@ module Chewy
|
|
44
42
|
# Run indexing for the database changes during the block provided.
|
45
43
|
# By default, indexing is run at the end of the block.
|
46
44
|
# @param strategy [Symbol] the Chewy index update strategy see Chewy docs.
|
47
|
-
def run_indexing(strategy: :atomic)
|
48
|
-
Chewy.strategy strategy
|
49
|
-
yield
|
50
|
-
end
|
45
|
+
def run_indexing(strategy: :atomic, &block)
|
46
|
+
Chewy.strategy strategy, &block
|
51
47
|
end
|
52
48
|
|
53
49
|
module ClassMethods
|
@@ -1,80 +1,76 @@
|
|
1
1
|
# Test helper class to provide minitest hooks for Chewy::Index testing.
|
2
2
|
#
|
3
3
|
# @note Intended to be used in conjunction with a test helper which mocks over the #bulk
|
4
|
-
# method on a {Chewy::
|
4
|
+
# method on a {Chewy::Index} class. (See {Chewy::Minitest::Helpers})
|
5
5
|
#
|
6
|
-
# The class will capture the data from the *param on the Chewy::
|
6
|
+
# The class will capture the data from the *param on the Chewy::Index.bulk method and
|
7
7
|
# aggregate the data for test analysis.
|
8
8
|
class SearchIndexReceiver
|
9
9
|
def initialize
|
10
10
|
@mutations = {}
|
11
11
|
end
|
12
12
|
|
13
|
-
# @param bulk_params [Hash] the bulk_params that should be sent to the Chewy::
|
14
|
-
# @param
|
15
|
-
def catch(bulk_params,
|
13
|
+
# @param bulk_params [Hash] the bulk_params that should be sent to the Chewy::Index.bulk method.
|
14
|
+
# @param index [Chewy::Index] the index executing this query.
|
15
|
+
def catch(bulk_params, index)
|
16
16
|
Array.wrap(bulk_params).map { |y| y[:body] }.flatten.each do |update|
|
17
17
|
if update[:delete]
|
18
|
-
mutation_for(
|
18
|
+
mutation_for(index).deletes << update[:delete][:_id]
|
19
19
|
elsif update[:index]
|
20
|
-
mutation_for(
|
20
|
+
mutation_for(index).indexes << update[:index]
|
21
21
|
end
|
22
22
|
end
|
23
23
|
end
|
24
24
|
|
25
|
-
# @param index [Chewy::Index] return only index requests to the specified {Chewy::
|
25
|
+
# @param index [Chewy::Index] return only index requests to the specified {Chewy::Index} index.
|
26
26
|
# @return [Hash] the index changes captured by the mock.
|
27
27
|
def indexes_for(index = nil)
|
28
28
|
if index
|
29
29
|
mutation_for(index).indexes
|
30
30
|
else
|
31
|
-
|
32
|
-
@mutations.map { |a, b| [a, b.indexes] }
|
33
|
-
]
|
31
|
+
@mutations.transform_values(&:indexes)
|
34
32
|
end
|
35
33
|
end
|
36
34
|
alias_method :indexes, :indexes_for
|
37
35
|
|
38
|
-
# @param index [Chewy::Index] return only delete requests to the specified {Chewy::
|
36
|
+
# @param index [Chewy::Index] return only delete requests to the specified {Chewy::Index} index.
|
39
37
|
# @return [Hash] the index deletes captured by the mock.
|
40
38
|
def deletes_for(index = nil)
|
41
39
|
if index
|
42
40
|
mutation_for(index).deletes
|
43
41
|
else
|
44
|
-
|
45
|
-
@mutations.map { |a, b| [a, b.deletes] }
|
46
|
-
]
|
42
|
+
@mutations.transform_values(&:deletes)
|
47
43
|
end
|
48
44
|
end
|
49
45
|
alias_method :deletes, :deletes_for
|
50
46
|
|
51
47
|
# Check to see if a given object has been indexed.
|
52
48
|
# @param obj [#id] obj the object to look for.
|
53
|
-
# @param
|
49
|
+
# @param index [Chewy::Index] what index the object should be indexed in.
|
54
50
|
# @return [true, false] if the object was indexed.
|
55
|
-
def indexed?(obj,
|
56
|
-
indexes_for(
|
51
|
+
def indexed?(obj, index)
|
52
|
+
indexes_for(index).map { |i| i[:_id] }.include? obj.id
|
57
53
|
end
|
58
54
|
|
59
55
|
# Check to see if a given object has been deleted.
|
60
56
|
# @param obj [#id] obj the object to look for.
|
61
|
-
# @param
|
57
|
+
# @param index [Chewy::Index] what index the object should have been deleted from.
|
62
58
|
# @return [true, false] if the object was deleted.
|
63
|
-
def deleted?(obj,
|
64
|
-
deletes_for(
|
59
|
+
def deleted?(obj, index)
|
60
|
+
deletes_for(index).include? obj.id
|
65
61
|
end
|
66
62
|
|
67
|
-
# @return [Array<Chewy::
|
63
|
+
# @return [Array<Chewy::Index>] a list of indexes changed.
|
68
64
|
def updated_indexes
|
69
65
|
@mutations.keys
|
70
66
|
end
|
71
67
|
|
72
68
|
private
|
73
69
|
|
74
|
-
# Get the mutation object for a given
|
75
|
-
# @param
|
70
|
+
# Get the mutation object for a given index.
|
71
|
+
# @param index [Chewy::Index] the index to fetch.
|
76
72
|
# @return [#indexes, #deletes] an object with a list of indexes and a list of deletes.
|
77
|
-
def mutation_for(
|
78
|
-
@mutations[
|
73
|
+
def mutation_for(index)
|
74
|
+
@mutations[index] ||= OpenStruct.new(indexes: [], deletes: [])
|
79
75
|
end
|
80
76
|
end
|
data/lib/chewy/railtie.rb
CHANGED
@@ -14,7 +14,9 @@ module Chewy
|
|
14
14
|
if Rails.application.config.respond_to?(:assets) && env['PATH_INFO'].start_with?(Rails.application.config.assets.prefix)
|
15
15
|
@app.call(env)
|
16
16
|
else
|
17
|
-
|
17
|
+
if Chewy.logger && @request_strategy != Chewy.request_strategy
|
18
|
+
Chewy.logger.info("Chewy request strategy is `#{Chewy.request_strategy}`")
|
19
|
+
end
|
18
20
|
@request_strategy = Chewy.request_strategy
|
19
21
|
Chewy.strategy(Chewy.request_strategy) { @app.call(env) }
|
20
22
|
end
|
@@ -35,7 +37,7 @@ module Chewy
|
|
35
37
|
if app.sandbox?
|
36
38
|
Chewy.strategy(:bypass)
|
37
39
|
else
|
38
|
-
Chewy.strategy(
|
40
|
+
Chewy.strategy(Chewy.console_strategy)
|
39
41
|
end
|
40
42
|
puts "Chewy console strategy is `#{Chewy.strategy.current.name}`"
|
41
43
|
end
|
data/lib/chewy/rake_helper.rb
CHANGED
@@ -3,14 +3,12 @@ module Chewy
|
|
3
3
|
IMPORT_CALLBACK = lambda do |output, _name, start, finish, _id, payload|
|
4
4
|
duration = (finish - start).ceil
|
5
5
|
stats = payload.fetch(:import, {}).map { |key, count| "#{key} #{count}" }.join(', ')
|
6
|
-
output.puts " Imported #{payload[:
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
output.puts " on #{documents.count} documents: #{documents}"
|
13
|
-
end
|
6
|
+
output.puts " Imported #{payload[:index]} in #{human_duration(duration)}, stats: #{stats}"
|
7
|
+
payload[:errors]&.each do |action, errors|
|
8
|
+
output.puts " #{action.to_s.humanize} errors:"
|
9
|
+
errors.each do |error, documents|
|
10
|
+
output.puts " `#{error}`"
|
11
|
+
output.puts " on #{documents.count} documents: #{documents}"
|
14
12
|
end
|
15
13
|
end
|
16
14
|
end
|
@@ -36,7 +34,7 @@ module Chewy
|
|
36
34
|
# @param parallel [true, Integer, Hash] any acceptable parallel options for import
|
37
35
|
# @param output [IO] output io for logging
|
38
36
|
# @return [Array<Chewy::Index>] indexes that were reset
|
39
|
-
def reset(only: nil, except: nil, parallel: nil, output:
|
37
|
+
def reset(only: nil, except: nil, parallel: nil, output: $stdout)
|
40
38
|
subscribed_task_stats(output) do
|
41
39
|
indexes_from(only: only, except: except).each do |index|
|
42
40
|
reset_one(index, output, parallel: parallel)
|
@@ -59,7 +57,7 @@ module Chewy
|
|
59
57
|
# @param parallel [true, Integer, Hash] any acceptable parallel options for import
|
60
58
|
# @param output [IO] output io for logging
|
61
59
|
# @return [Array<Chewy::Index>] indexes that were actually reset
|
62
|
-
def upgrade(only: nil, except: nil, parallel: nil, output:
|
60
|
+
def upgrade(only: nil, except: nil, parallel: nil, output: $stdout)
|
63
61
|
subscribed_task_stats(output) do
|
64
62
|
indexes = indexes_from(only: only, except: except)
|
65
63
|
|
@@ -87,23 +85,21 @@ module Chewy
|
|
87
85
|
#
|
88
86
|
# @example
|
89
87
|
# Chewy::RakeHelper.update # updates everything
|
90
|
-
# Chewy::RakeHelper.update(only: 'places') # updates only PlacesIndex
|
91
|
-
# Chewy::RakeHelper.update(
|
92
|
-
# Chewy::RakeHelper.update(except: PlacesIndex::Country) # updates everything, but PlacesIndex::Country
|
93
|
-
# Chewy::RakeHelper.update(only: 'places', except: 'places#country') # updates PlacesIndex::City only
|
88
|
+
# Chewy::RakeHelper.update(only: 'places') # updates only PlacesIndex
|
89
|
+
# Chewy::RakeHelper.update(except: PlacesIndex) # updates everything, but PlacesIndex
|
94
90
|
#
|
95
|
-
# @param only [Array<Chewy::Index,
|
96
|
-
# @param except [Array<Chewy::Index,
|
91
|
+
# @param only [Array<Chewy::Index, String>, Chewy::Index, String] indexes to update; if nothing is passed - uses all the indexes defined in the app
|
92
|
+
# @param except [Array<Chewy::Index, String>, Chewy::Index, String] indexes to exclude from processing
|
97
93
|
# @param parallel [true, Integer, Hash] any acceptable parallel options for import
|
98
94
|
# @param output [IO] output io for logging
|
99
|
-
# @return [Array<Chewy::
|
100
|
-
def update(only: nil, except: nil, parallel: nil, output:
|
95
|
+
# @return [Array<Chewy::Index>] indexes that were actually updated
|
96
|
+
def update(only: nil, except: nil, parallel: nil, output: $stdout)
|
101
97
|
subscribed_task_stats(output) do
|
102
|
-
|
98
|
+
indexes_from(only: only, except: except).each_with_object([]) do |index, updated_indexes|
|
103
99
|
if index.exists?
|
104
100
|
output.puts "Updating #{index}"
|
105
|
-
|
106
|
-
|
101
|
+
index.import(parallel: parallel)
|
102
|
+
updated_indexes.push(index)
|
107
103
|
else
|
108
104
|
output.puts "Skipping #{index}, it does not exists (use rake chewy:reset[#{index.derivable_name}] to create and update it)"
|
109
105
|
end
|
@@ -115,31 +111,29 @@ module Chewy
|
|
115
111
|
#
|
116
112
|
# @example
|
117
113
|
# Chewy::RakeHelper.sync # synchronizes everything
|
118
|
-
# Chewy::RakeHelper.sync(only: 'places') # synchronizes only PlacesIndex
|
119
|
-
# Chewy::RakeHelper.sync(
|
120
|
-
# Chewy::RakeHelper.sync(except: PlacesIndex::Country) # synchronizes everything, but PlacesIndex::Country
|
121
|
-
# Chewy::RakeHelper.sync(only: 'places', except: 'places#country') # synchronizes PlacesIndex::City only
|
114
|
+
# Chewy::RakeHelper.sync(only: 'places') # synchronizes only PlacesIndex
|
115
|
+
# Chewy::RakeHelper.sync(except: PlacesIndex) # synchronizes everything, but PlacesIndex
|
122
116
|
#
|
123
|
-
# @param only [Array<Chewy::Index,
|
124
|
-
# @param except [Array<Chewy::Index,
|
117
|
+
# @param only [Array<Chewy::Index, String>, Chewy::Index, String] indexes to synchronize; if nothing is passed - uses all the indexes defined in the app
|
118
|
+
# @param except [Array<Chewy::Index, String>, Chewy::Index, String] indexes to exclude from processing
|
125
119
|
# @param parallel [true, Integer, Hash] any acceptable parallel options for sync
|
126
120
|
# @param output [IO] output io for logging
|
127
|
-
# @return [Array<Chewy::
|
128
|
-
def sync(only: nil, except: nil, parallel: nil, output:
|
121
|
+
# @return [Array<Chewy::Index>] indexes that were actually updated
|
122
|
+
def sync(only: nil, except: nil, parallel: nil, output: $stdout)
|
129
123
|
subscribed_task_stats(output) do
|
130
|
-
|
131
|
-
output.puts "Synchronizing #{
|
132
|
-
output.puts " #{
|
124
|
+
indexes_from(only: only, except: except).each_with_object([]) do |index, synced_indexes|
|
125
|
+
output.puts "Synchronizing #{index}"
|
126
|
+
output.puts " #{index} doesn't support outdated synchronization" unless index.supports_outdated_sync?
|
133
127
|
time = Time.now
|
134
|
-
sync_result =
|
128
|
+
sync_result = index.sync(parallel: parallel)
|
135
129
|
if !sync_result
|
136
|
-
output.puts " Something went wrong with the #{
|
137
|
-
elsif sync_result[:count]
|
130
|
+
output.puts " Something went wrong with the #{index} synchronization"
|
131
|
+
elsif (sync_result[:count]).positive?
|
138
132
|
output.puts " Missing documents: #{sync_result[:missing]}" if sync_result[:missing].present?
|
139
133
|
output.puts " Outdated documents: #{sync_result[:outdated]}" if sync_result[:outdated].present?
|
140
|
-
|
134
|
+
synced_indexes.push(index)
|
141
135
|
else
|
142
|
-
output.puts " Skipping #{
|
136
|
+
output.puts " Skipping #{index}, up to date"
|
143
137
|
end
|
144
138
|
output.puts " Took #{human_duration(Time.now - time)}"
|
145
139
|
end
|
@@ -147,49 +141,46 @@ module Chewy
|
|
147
141
|
end
|
148
142
|
|
149
143
|
# Applies changes that were done after the specified time for the
|
150
|
-
# specified indexes
|
144
|
+
# specified indexes or all of them.
|
151
145
|
#
|
152
146
|
# @example
|
153
147
|
# Chewy::RakeHelper.journal_apply(time: 1.minute.ago) # applies entries created for the last minute
|
154
|
-
# Chewy::RakeHelper.journal_apply(time: 1.minute.ago, only: 'places') # applies only PlacesIndex
|
155
|
-
# Chewy::RakeHelper.journal_apply(time: 1.minute.ago,
|
156
|
-
# Chewy::RakeHelper.journal_apply(time: 1.minute.ago, except: PlacesIndex::Country) # applies everything, but PlacesIndex::Country entries reated for the last minute
|
157
|
-
# Chewy::RakeHelper.journal_apply(time: 1.minute.ago, only: 'places', except: 'places#country') # applies PlacesIndex::City entries reated for the last minute only
|
148
|
+
# Chewy::RakeHelper.journal_apply(time: 1.minute.ago, only: 'places') # applies only PlacesIndex entries created for the last minute
|
149
|
+
# Chewy::RakeHelper.journal_apply(time: 1.minute.ago, except: PlacesIndex) # applies everything, but PlacesIndex, entries created for the last minute
|
158
150
|
#
|
159
151
|
# @param time [Time, DateTime] use only journal entries created after this time
|
160
|
-
# @param only [Array<Chewy::Index,
|
161
|
-
# @param except [Array<Chewy::Index,
|
152
|
+
# @param only [Array<Chewy::Index, String>, Chewy::Index, String] indexes to synchronize; if nothing is passed - uses all the indexes defined in the app
|
153
|
+
# @param except [Array<Chewy::Index, String>, Chewy::Index, String] indexes to exclude from processing
|
162
154
|
# @param output [IO] output io for logging
|
163
|
-
# @return [Array<Chewy::
|
164
|
-
def journal_apply(time: nil, only: nil, except: nil, output:
|
155
|
+
# @return [Array<Chewy::Index>] indexes that were actually updated
|
156
|
+
def journal_apply(time: nil, only: nil, except: nil, output: $stdout)
|
165
157
|
raise ArgumentError, 'Please specify the time to start with' unless time
|
158
|
+
|
166
159
|
subscribed_task_stats(output) do
|
167
160
|
output.puts "Applying journal entries created after #{time}"
|
168
|
-
count = Chewy::Journal.new(
|
161
|
+
count = Chewy::Journal.new(indexes_from(only: only, except: except)).apply(time)
|
169
162
|
output.puts 'No journal entries were created after the specified time' if count.zero?
|
170
163
|
end
|
171
164
|
end
|
172
165
|
|
173
166
|
# Removes journal records created before the specified timestamp for
|
174
|
-
# the specified indexes
|
167
|
+
# the specified indexes or all of them.
|
175
168
|
#
|
176
169
|
# @example
|
177
170
|
# Chewy::RakeHelper.journal_clean # cleans everything
|
178
171
|
# Chewy::RakeHelper.journal_clean(time: 1.minute.ago) # leaves only entries created for the last minute
|
179
|
-
# Chewy::RakeHelper.journal_clean(only: 'places') # cleans only PlacesIndex
|
180
|
-
# Chewy::RakeHelper.journal_clean(
|
181
|
-
# Chewy::RakeHelper.journal_clean(except: PlacesIndex::Country) # cleans everything, but PlacesIndex::Country entries
|
182
|
-
# Chewy::RakeHelper.journal_clean(only: 'places', except: 'places#country') # cleans PlacesIndex::City entries only
|
172
|
+
# Chewy::RakeHelper.journal_clean(only: 'places') # cleans only PlacesIndex entries
|
173
|
+
# Chewy::RakeHelper.journal_clean(except: PlacesIndex) # cleans everything, but PlacesIndex entries
|
183
174
|
#
|
184
175
|
# @param time [Time, DateTime] clean all the journal entries created before this time
|
185
|
-
# @param only [Array<Chewy::Index,
|
186
|
-
# @param except [Array<Chewy::Index,
|
176
|
+
# @param only [Array<Chewy::Index, String>, Chewy::Index, String] indexes to synchronize; if nothing is passed - uses all the indexes defined in the app
|
177
|
+
# @param except [Array<Chewy::Index, String>, Chewy::Index, String] indexes to exclude from processing
|
187
178
|
# @param output [IO] output io for logging
|
188
|
-
# @return [Array<Chewy::
|
189
|
-
def journal_clean(time: nil, only: nil, except: nil, output:
|
179
|
+
# @return [Array<Chewy::Index>] indexes that were actually updated
|
180
|
+
def journal_clean(time: nil, only: nil, except: nil, output: $stdout)
|
190
181
|
subscribed_task_stats(output) do
|
191
182
|
output.puts "Cleaning journal entries created before #{time}" if time
|
192
|
-
response = Chewy::Journal.new(
|
183
|
+
response = Chewy::Journal.new(indexes_from(only: only, except: except)).clean(time)
|
193
184
|
count = response['deleted'] || response['_indices']['_all']['deleted']
|
194
185
|
output.puts "Cleaned up #{count} journal entries"
|
195
186
|
end
|
@@ -204,45 +195,53 @@ module Chewy
|
|
204
195
|
Chewy::Index.descendants - [Chewy::Stash::Journal, Chewy::Stash::Specification]
|
205
196
|
end
|
206
197
|
|
198
|
+
# Reindex data from source index to destination index
|
199
|
+
#
|
200
|
+
# @example
|
201
|
+
# Chewy::RakeHelper.reindex(source: 'users_index', dest: 'cities_index') reindex data from 'users_index' index to 'cities_index'
|
202
|
+
#
|
203
|
+
# @param source [String], dest [String] indexes to reindex
|
204
|
+
def reindex(source:, dest:, output: $stdout)
|
205
|
+
subscribed_task_stats(output) do
|
206
|
+
output.puts "Source index is #{source}\nDestination index is #{dest}"
|
207
|
+
Chewy::Index.reindex(source: source, dest: dest)
|
208
|
+
output.puts "#{source} index successfully reindexed with #{dest} index data"
|
209
|
+
end
|
210
|
+
end
|
211
|
+
|
212
|
+
# Adds new fields to an existing data stream or index.
|
213
|
+
# Change the search settings of existing fields.
|
214
|
+
#
|
215
|
+
# @example
|
216
|
+
# Chewy::RakeHelper.update_mapping('cities', {properties: {new_field: {type: :text}}}) update 'cities' index with new_field of text type
|
217
|
+
#
|
218
|
+
# @param name [String], body_hash [Hash] index name and body hash to update
|
219
|
+
def update_mapping(name:, output: $stdout)
|
220
|
+
subscribed_task_stats(output) do
|
221
|
+
output.puts "Index name is #{name}"
|
222
|
+
normalize_index(name).update_mapping
|
223
|
+
output.puts "#{name} index successfully updated"
|
224
|
+
end
|
225
|
+
end
|
226
|
+
|
207
227
|
def normalize_indexes(*identifiers)
|
208
228
|
identifiers.flatten(1).map { |identifier| normalize_index(identifier) }
|
209
229
|
end
|
210
230
|
|
211
231
|
def normalize_index(identifier)
|
212
232
|
return identifier if identifier.is_a?(Class) && identifier < Chewy::Index
|
213
|
-
|
233
|
+
|
234
|
+
"#{identifier.to_s.camelize}Index".constantize
|
214
235
|
end
|
215
236
|
|
216
|
-
def subscribed_task_stats(output =
|
237
|
+
def subscribed_task_stats(output = $stdout, &block)
|
217
238
|
start = Time.now
|
218
239
|
ActiveSupport::Notifications.subscribed(JOURNAL_CALLBACK.curry[output], 'apply_journal.chewy') do
|
219
|
-
ActiveSupport::Notifications.subscribed(IMPORT_CALLBACK.curry[output], 'import_objects.chewy')
|
220
|
-
yield
|
221
|
-
end
|
240
|
+
ActiveSupport::Notifications.subscribed(IMPORT_CALLBACK.curry[output], 'import_objects.chewy', &block)
|
222
241
|
end
|
223
242
|
output.puts "Total: #{human_duration(Time.now - start)}"
|
224
243
|
end
|
225
244
|
|
226
|
-
def reset_index(*indexes)
|
227
|
-
ActiveSupport::Deprecation.warn '`Chewy::RakeHelper.reset_index` is deprecated and will be removed soon, use `Chewy::RakeHelper.reset` instead'
|
228
|
-
reset(only: indexes)
|
229
|
-
end
|
230
|
-
|
231
|
-
def reset_all(*except)
|
232
|
-
ActiveSupport::Deprecation.warn '`Chewy::RakeHelper.reset_all` is deprecated and will be removed soon, use `Chewy::RakeHelper.reset` instead'
|
233
|
-
reset(except: except)
|
234
|
-
end
|
235
|
-
|
236
|
-
def update_index(*indexes)
|
237
|
-
ActiveSupport::Deprecation.warn '`Chewy::RakeHelper.update_index` is deprecated and will be removed soon, use `Chewy::RakeHelper.update` instead'
|
238
|
-
update(only: indexes)
|
239
|
-
end
|
240
|
-
|
241
|
-
def update_all(*except)
|
242
|
-
ActiveSupport::Deprecation.warn '`Chewy::RakeHelper.update_all` is deprecated and will be removed soon, use `Chewy::RakeHelper.update` instead'
|
243
|
-
update(except: except)
|
244
|
-
end
|
245
|
-
|
246
245
|
private
|
247
246
|
|
248
247
|
def indexes_from(only: nil, except: nil)
|
@@ -261,33 +260,9 @@ module Chewy
|
|
261
260
|
indexes.sort_by(&:derivable_name)
|
262
261
|
end
|
263
262
|
|
264
|
-
def types_from(only: nil, except: nil)
|
265
|
-
types = if only.present?
|
266
|
-
normalize_types(Array.wrap(only))
|
267
|
-
else
|
268
|
-
all_indexes.flat_map(&:types)
|
269
|
-
end
|
270
|
-
|
271
|
-
types = if except.present?
|
272
|
-
types - normalize_types(Array.wrap(except))
|
273
|
-
else
|
274
|
-
types
|
275
|
-
end
|
276
|
-
|
277
|
-
types.sort_by(&:derivable_name)
|
278
|
-
end
|
279
|
-
|
280
|
-
def normalize_types(*identifiers)
|
281
|
-
identifiers.flatten(1).flat_map { |identifier| normalize_type(identifier) }
|
282
|
-
end
|
283
|
-
|
284
|
-
def normalize_type(identifier)
|
285
|
-
Chewy.derive_types(identifier)
|
286
|
-
end
|
287
|
-
|
288
263
|
def human_duration(seconds)
|
289
264
|
[[60, :s], [60, :m], [24, :h]].map do |amount, unit|
|
290
|
-
if seconds
|
265
|
+
if seconds.positive?
|
291
266
|
seconds, n = seconds.divmod(amount)
|
292
267
|
"#{n.to_i}#{unit}"
|
293
268
|
end
|