chewy 7.1.0 → 7.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ruby.yml +58 -0
- data/.rubocop.yml +13 -8
- data/.rubocop_todo.yml +110 -22
- data/CHANGELOG.md +53 -0
- data/Gemfile +0 -7
- data/Guardfile +3 -1
- data/README.md +282 -245
- data/chewy.gemspec +3 -5
- data/gemfiles/rails.5.2.activerecord.gemfile +8 -14
- data/gemfiles/rails.6.0.activerecord.gemfile +8 -14
- data/gemfiles/rails.6.1.activerecord.gemfile +8 -14
- data/lib/chewy.rb +21 -75
- data/lib/chewy/config.rb +40 -40
- data/lib/chewy/errors.rb +0 -12
- data/lib/chewy/fields/base.rb +11 -1
- data/lib/chewy/fields/root.rb +3 -4
- data/lib/chewy/index.rb +46 -87
- data/lib/chewy/index/actions.rb +51 -32
- data/lib/chewy/{type → index}/adapter/active_record.rb +12 -3
- data/lib/chewy/{type → index}/adapter/base.rb +2 -3
- data/lib/chewy/{type → index}/adapter/object.rb +27 -31
- data/lib/chewy/{type → index}/adapter/orm.rb +11 -14
- data/lib/chewy/{type → index}/crutch.rb +5 -5
- data/lib/chewy/{type → index}/import.rb +36 -27
- data/lib/chewy/{type → index}/import/bulk_builder.rb +15 -13
- data/lib/chewy/{type → index}/import/bulk_request.rb +6 -6
- data/lib/chewy/{type → index}/import/journal_builder.rb +10 -10
- data/lib/chewy/{type → index}/import/routine.rb +15 -14
- data/lib/chewy/{type → index}/mapping.rb +26 -31
- data/lib/chewy/{type → index}/observe.rb +9 -19
- data/lib/chewy/index/specification.rb +1 -0
- data/lib/chewy/{type → index}/syncer.rb +60 -57
- data/lib/chewy/{type → index}/witchcraft.rb +11 -7
- data/lib/chewy/{type → index}/wrapper.rb +2 -2
- data/lib/chewy/journal.rb +8 -8
- data/lib/chewy/minitest/helpers.rb +9 -13
- data/lib/chewy/minitest/search_index_receiver.rb +22 -26
- data/lib/chewy/railtie.rb +4 -2
- data/lib/chewy/rake_helper.rb +82 -107
- data/lib/chewy/rspec/update_index.rb +47 -43
- data/lib/chewy/search.rb +4 -17
- data/lib/chewy/search/loader.rb +18 -30
- data/lib/chewy/search/parameters.rb +4 -2
- data/lib/chewy/search/parameters/concerns/query_storage.rb +2 -2
- data/lib/chewy/search/parameters/source.rb +5 -1
- data/lib/chewy/search/query_proxy.rb +9 -2
- data/lib/chewy/search/request.rb +82 -86
- data/lib/chewy/search/response.rb +4 -4
- data/lib/chewy/search/scoping.rb +6 -7
- data/lib/chewy/search/scrolling.rb +11 -11
- data/lib/chewy/stash.rb +14 -22
- data/lib/chewy/strategy.rb +3 -19
- data/lib/chewy/strategy/sidekiq.rb +1 -0
- data/lib/chewy/version.rb +1 -1
- data/lib/generators/chewy/install_generator.rb +1 -1
- data/lib/tasks/chewy.rake +10 -22
- data/migration_guide.md +14 -0
- data/spec/chewy/config_spec.rb +14 -39
- data/spec/chewy/fields/base_spec.rb +412 -148
- data/spec/chewy/fields/root_spec.rb +16 -24
- data/spec/chewy/fields/time_fields_spec.rb +5 -5
- data/spec/chewy/index/actions_spec.rb +270 -24
- data/spec/chewy/{type → index}/adapter/active_record_spec.rb +68 -40
- data/spec/chewy/{type → index}/adapter/object_spec.rb +21 -6
- data/spec/chewy/{type → index}/import/bulk_builder_spec.rb +23 -31
- data/spec/chewy/{type → index}/import/bulk_request_spec.rb +5 -6
- data/spec/chewy/{type → index}/import/journal_builder_spec.rb +9 -15
- data/spec/chewy/{type → index}/import/routine_spec.rb +16 -16
- data/spec/chewy/{type → index}/import_spec.rb +102 -98
- data/spec/chewy/{type → index}/mapping_spec.rb +46 -54
- data/spec/chewy/index/observe_spec.rb +116 -0
- data/spec/chewy/index/settings_spec.rb +3 -1
- data/spec/chewy/index/specification_spec.rb +7 -17
- data/spec/chewy/{type → index}/syncer_spec.rb +14 -19
- data/spec/chewy/{type → index}/witchcraft_spec.rb +20 -22
- data/spec/chewy/index/wrapper_spec.rb +100 -0
- data/spec/chewy/index_spec.rb +59 -102
- data/spec/chewy/journal_spec.rb +9 -22
- data/spec/chewy/minitest/helpers_spec.rb +13 -15
- data/spec/chewy/minitest/search_index_receiver_spec.rb +22 -26
- data/spec/chewy/multi_search_spec.rb +4 -5
- data/spec/chewy/rake_helper_spec.rb +145 -55
- data/spec/chewy/rspec/update_index_spec.rb +74 -71
- data/spec/chewy/search/loader_spec.rb +19 -37
- data/spec/chewy/search/pagination/kaminari_examples.rb +3 -5
- data/spec/chewy/search/pagination/kaminari_spec.rb +1 -1
- data/spec/chewy/search/parameters/indices_spec.rb +2 -8
- data/spec/chewy/search/parameters/order_spec.rb +1 -1
- data/spec/chewy/search/parameters/query_storage_examples.rb +67 -21
- data/spec/chewy/search/parameters/search_after_spec.rb +4 -1
- data/spec/chewy/search/parameters/source_spec.rb +8 -2
- data/spec/chewy/search/parameters_spec.rb +12 -3
- data/spec/chewy/search/query_proxy_spec.rb +68 -17
- data/spec/chewy/search/request_spec.rb +222 -74
- data/spec/chewy/search/response_spec.rb +12 -12
- data/spec/chewy/search/scrolling_spec.rb +7 -9
- data/spec/chewy/search_spec.rb +32 -35
- data/spec/chewy/stash_spec.rb +9 -21
- data/spec/chewy/strategy/active_job_spec.rb +8 -8
- data/spec/chewy/strategy/atomic_spec.rb +9 -10
- data/spec/chewy/strategy/sidekiq_spec.rb +8 -8
- data/spec/chewy/strategy_spec.rb +19 -15
- data/spec/chewy_spec.rb +14 -100
- data/spec/spec_helper.rb +2 -21
- data/spec/support/active_record.rb +15 -5
- metadata +44 -103
- data/.circleci/config.yml +0 -214
- data/Appraisals +0 -81
- data/gemfiles/rails.5.2.mongoid.6.4.gemfile +0 -17
- data/gemfiles/sequel.4.45.gemfile +0 -11
- data/lib/chewy/search/pagination/will_paginate.rb +0 -43
- data/lib/chewy/strategy/resque.rb +0 -27
- data/lib/chewy/strategy/shoryuken.rb +0 -40
- data/lib/chewy/type.rb +0 -120
- data/lib/chewy/type/actions.rb +0 -43
- data/lib/chewy/type/adapter/mongoid.rb +0 -67
- data/lib/chewy/type/adapter/sequel.rb +0 -93
- data/lib/sequel/plugins/chewy_observe.rb +0 -63
- data/spec/chewy/search/pagination/will_paginate_examples.rb +0 -63
- data/spec/chewy/search/pagination/will_paginate_spec.rb +0 -23
- data/spec/chewy/strategy/resque_spec.rb +0 -46
- data/spec/chewy/strategy/shoryuken_spec.rb +0 -70
- data/spec/chewy/type/actions_spec.rb +0 -50
- data/spec/chewy/type/adapter/mongoid_spec.rb +0 -372
- data/spec/chewy/type/adapter/sequel_spec.rb +0 -472
- data/spec/chewy/type/observe_spec.rb +0 -137
- data/spec/chewy/type/wrapper_spec.rb +0 -100
- data/spec/chewy/type_spec.rb +0 -55
- data/spec/support/mongoid.rb +0 -93
- data/spec/support/sequel.rb +0 -80
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
module Chewy
|
|
2
|
-
class
|
|
2
|
+
class Index
|
|
3
3
|
module Wrapper
|
|
4
4
|
extend ActiveSupport::Concern
|
|
5
5
|
|
|
@@ -28,7 +28,7 @@ module Chewy
|
|
|
28
28
|
def ==(other)
|
|
29
29
|
return true if super
|
|
30
30
|
|
|
31
|
-
if other.is_a?(Chewy::
|
|
31
|
+
if other.is_a?(Chewy::Index)
|
|
32
32
|
self.class == other.class && (respond_to?(:id) ? id == other.id : attributes == other.attributes)
|
|
33
33
|
elsif other.respond_to?(:id)
|
|
34
34
|
self.class.adapter.target.is_a?(Class) &&
|
data/lib/chewy/journal.rb
CHANGED
|
@@ -2,18 +2,18 @@ module Chewy
|
|
|
2
2
|
# A class to perform journal-related actions for the specified indexes/types.
|
|
3
3
|
#
|
|
4
4
|
# @example
|
|
5
|
-
# journal = Chewy::Journal.new('places
|
|
5
|
+
# journal = Chewy::Journal.new('places', UsersIndex)
|
|
6
6
|
# journal.apply(20.minutes.ago)
|
|
7
7
|
# journal.clean
|
|
8
8
|
#
|
|
9
9
|
class Journal
|
|
10
|
-
# @param only [Array<String, Chewy::Index
|
|
10
|
+
# @param only [Array<String, Chewy::Index>] indexes or string references to perform actions on
|
|
11
11
|
def initialize(*only)
|
|
12
12
|
@only = only
|
|
13
13
|
end
|
|
14
14
|
|
|
15
15
|
# Applies all changes that were done since the specified time to the
|
|
16
|
-
# specified indexes
|
|
16
|
+
# specified indexes.
|
|
17
17
|
#
|
|
18
18
|
# @param since_time [Time, DateTime] timestamp from which changes will be applied
|
|
19
19
|
# @param retries [Integer] maximum number of attempts to make journal empty, 10 by default
|
|
@@ -27,8 +27,8 @@ module Chewy
|
|
|
27
27
|
count += entries.size
|
|
28
28
|
groups = reference_groups(entries)
|
|
29
29
|
ActiveSupport::Notifications.instrument 'apply_journal.chewy', stage: stage, groups: groups
|
|
30
|
-
groups.each do |
|
|
31
|
-
|
|
30
|
+
groups.each do |index, references|
|
|
31
|
+
index.import(references, import_options.merge(journal: false))
|
|
32
32
|
end
|
|
33
33
|
stage += 1
|
|
34
34
|
since_time = entries.map(&:created_at).max
|
|
@@ -47,9 +47,9 @@ module Chewy
|
|
|
47
47
|
private
|
|
48
48
|
|
|
49
49
|
def reference_groups(entries)
|
|
50
|
-
entries.group_by(&:
|
|
51
|
-
|
|
52
|
-
|
|
50
|
+
entries.group_by(&:index_name)
|
|
51
|
+
.transform_keys { |index_name| Chewy.derive_name(index_name) }
|
|
52
|
+
.transform_values { |grouped_entries| grouped_entries.map(&:references).inject(:|) }
|
|
53
53
|
end
|
|
54
54
|
end
|
|
55
55
|
end
|
|
@@ -6,7 +6,7 @@ module Chewy
|
|
|
6
6
|
extend ActiveSupport::Concern
|
|
7
7
|
|
|
8
8
|
# Assert that an index *changes* during a block.
|
|
9
|
-
# @param index [Chewy::
|
|
9
|
+
# @param index [Chewy::Index] the index to watch, eg EntitiesIndex.
|
|
10
10
|
# @param strategy [Symbol] the Chewy strategy to use around the block. See Chewy docs.
|
|
11
11
|
# @param bypass_actual_index [true, false]
|
|
12
12
|
# True to preempt the http call to Elastic, false otherwise.
|
|
@@ -14,11 +14,11 @@ module Chewy
|
|
|
14
14
|
#
|
|
15
15
|
# @return [SearchIndexReceiver] for optional further assertions on the nature of the index changes.
|
|
16
16
|
#
|
|
17
|
-
def assert_indexes(index, strategy: :atomic, bypass_actual_index: true)
|
|
18
|
-
|
|
17
|
+
def assert_indexes(index, strategy: :atomic, bypass_actual_index: true, &block)
|
|
18
|
+
index_class = Chewy.derive_name index
|
|
19
19
|
receiver = SearchIndexReceiver.new
|
|
20
20
|
|
|
21
|
-
bulk_method =
|
|
21
|
+
bulk_method = index_class.method :bulk
|
|
22
22
|
# Manually mocking #bulk because we need to properly capture `self`
|
|
23
23
|
bulk_mock = lambda do |*bulk_args|
|
|
24
24
|
receiver.catch bulk_args, self
|
|
@@ -28,13 +28,11 @@ module Chewy
|
|
|
28
28
|
{}
|
|
29
29
|
end
|
|
30
30
|
|
|
31
|
-
|
|
31
|
+
index_class.define_singleton_method :bulk, bulk_mock
|
|
32
32
|
|
|
33
|
-
Chewy.strategy(strategy)
|
|
34
|
-
yield
|
|
35
|
-
end
|
|
33
|
+
Chewy.strategy(strategy, &block)
|
|
36
34
|
|
|
37
|
-
|
|
35
|
+
index_class.define_singleton_method :bulk, bulk_method
|
|
38
36
|
|
|
39
37
|
assert_includes receiver.updated_indexes, index, "Expected #{index} to be updated but it wasn't"
|
|
40
38
|
|
|
@@ -44,10 +42,8 @@ module Chewy
|
|
|
44
42
|
# Run indexing for the database changes during the block provided.
|
|
45
43
|
# By default, indexing is run at the end of the block.
|
|
46
44
|
# @param strategy [Symbol] the Chewy index update strategy see Chewy docs.
|
|
47
|
-
def run_indexing(strategy: :atomic)
|
|
48
|
-
Chewy.strategy strategy
|
|
49
|
-
yield
|
|
50
|
-
end
|
|
45
|
+
def run_indexing(strategy: :atomic, &block)
|
|
46
|
+
Chewy.strategy strategy, &block
|
|
51
47
|
end
|
|
52
48
|
|
|
53
49
|
module ClassMethods
|
|
@@ -1,80 +1,76 @@
|
|
|
1
1
|
# Test helper class to provide minitest hooks for Chewy::Index testing.
|
|
2
2
|
#
|
|
3
3
|
# @note Intended to be used in conjunction with a test helper which mocks over the #bulk
|
|
4
|
-
# method on a {Chewy::
|
|
4
|
+
# method on a {Chewy::Index} class. (See {Chewy::Minitest::Helpers})
|
|
5
5
|
#
|
|
6
|
-
# The class will capture the data from the *param on the Chewy::
|
|
6
|
+
# The class will capture the data from the *param on the Chewy::Index.bulk method and
|
|
7
7
|
# aggregate the data for test analysis.
|
|
8
8
|
class SearchIndexReceiver
|
|
9
9
|
def initialize
|
|
10
10
|
@mutations = {}
|
|
11
11
|
end
|
|
12
12
|
|
|
13
|
-
# @param bulk_params [Hash] the bulk_params that should be sent to the Chewy::
|
|
14
|
-
# @param
|
|
15
|
-
def catch(bulk_params,
|
|
13
|
+
# @param bulk_params [Hash] the bulk_params that should be sent to the Chewy::Index.bulk method.
|
|
14
|
+
# @param index [Chewy::Index] the index executing this query.
|
|
15
|
+
def catch(bulk_params, index)
|
|
16
16
|
Array.wrap(bulk_params).map { |y| y[:body] }.flatten.each do |update|
|
|
17
17
|
if update[:delete]
|
|
18
|
-
mutation_for(
|
|
18
|
+
mutation_for(index).deletes << update[:delete][:_id]
|
|
19
19
|
elsif update[:index]
|
|
20
|
-
mutation_for(
|
|
20
|
+
mutation_for(index).indexes << update[:index]
|
|
21
21
|
end
|
|
22
22
|
end
|
|
23
23
|
end
|
|
24
24
|
|
|
25
|
-
# @param index [Chewy::Index] return only index requests to the specified {Chewy::
|
|
25
|
+
# @param index [Chewy::Index] return only index requests to the specified {Chewy::Index} index.
|
|
26
26
|
# @return [Hash] the index changes captured by the mock.
|
|
27
27
|
def indexes_for(index = nil)
|
|
28
28
|
if index
|
|
29
29
|
mutation_for(index).indexes
|
|
30
30
|
else
|
|
31
|
-
|
|
32
|
-
@mutations.map { |a, b| [a, b.indexes] }
|
|
33
|
-
]
|
|
31
|
+
@mutations.transform_values(&:indexes)
|
|
34
32
|
end
|
|
35
33
|
end
|
|
36
34
|
alias_method :indexes, :indexes_for
|
|
37
35
|
|
|
38
|
-
# @param index [Chewy::Index] return only delete requests to the specified {Chewy::
|
|
36
|
+
# @param index [Chewy::Index] return only delete requests to the specified {Chewy::Index} index.
|
|
39
37
|
# @return [Hash] the index deletes captured by the mock.
|
|
40
38
|
def deletes_for(index = nil)
|
|
41
39
|
if index
|
|
42
40
|
mutation_for(index).deletes
|
|
43
41
|
else
|
|
44
|
-
|
|
45
|
-
@mutations.map { |a, b| [a, b.deletes] }
|
|
46
|
-
]
|
|
42
|
+
@mutations.transform_values(&:deletes)
|
|
47
43
|
end
|
|
48
44
|
end
|
|
49
45
|
alias_method :deletes, :deletes_for
|
|
50
46
|
|
|
51
47
|
# Check to see if a given object has been indexed.
|
|
52
48
|
# @param obj [#id] obj the object to look for.
|
|
53
|
-
# @param
|
|
49
|
+
# @param index [Chewy::Index] what index the object should be indexed in.
|
|
54
50
|
# @return [true, false] if the object was indexed.
|
|
55
|
-
def indexed?(obj,
|
|
56
|
-
indexes_for(
|
|
51
|
+
def indexed?(obj, index)
|
|
52
|
+
indexes_for(index).map { |i| i[:_id] }.include? obj.id
|
|
57
53
|
end
|
|
58
54
|
|
|
59
55
|
# Check to see if a given object has been deleted.
|
|
60
56
|
# @param obj [#id] obj the object to look for.
|
|
61
|
-
# @param
|
|
57
|
+
# @param index [Chewy::Index] what index the object should have been deleted from.
|
|
62
58
|
# @return [true, false] if the object was deleted.
|
|
63
|
-
def deleted?(obj,
|
|
64
|
-
deletes_for(
|
|
59
|
+
def deleted?(obj, index)
|
|
60
|
+
deletes_for(index).include? obj.id
|
|
65
61
|
end
|
|
66
62
|
|
|
67
|
-
# @return [Array<Chewy::
|
|
63
|
+
# @return [Array<Chewy::Index>] a list of indexes changed.
|
|
68
64
|
def updated_indexes
|
|
69
65
|
@mutations.keys
|
|
70
66
|
end
|
|
71
67
|
|
|
72
68
|
private
|
|
73
69
|
|
|
74
|
-
# Get the mutation object for a given
|
|
75
|
-
# @param
|
|
70
|
+
# Get the mutation object for a given index.
|
|
71
|
+
# @param index [Chewy::Index] the index to fetch.
|
|
76
72
|
# @return [#indexes, #deletes] an object with a list of indexes and a list of deletes.
|
|
77
|
-
def mutation_for(
|
|
78
|
-
@mutations[
|
|
73
|
+
def mutation_for(index)
|
|
74
|
+
@mutations[index] ||= OpenStruct.new(indexes: [], deletes: [])
|
|
79
75
|
end
|
|
80
76
|
end
|
data/lib/chewy/railtie.rb
CHANGED
|
@@ -14,7 +14,9 @@ module Chewy
|
|
|
14
14
|
if Rails.application.config.respond_to?(:assets) && env['PATH_INFO'].start_with?(Rails.application.config.assets.prefix)
|
|
15
15
|
@app.call(env)
|
|
16
16
|
else
|
|
17
|
-
|
|
17
|
+
if Chewy.logger && @request_strategy != Chewy.request_strategy
|
|
18
|
+
Chewy.logger.info("Chewy request strategy is `#{Chewy.request_strategy}`")
|
|
19
|
+
end
|
|
18
20
|
@request_strategy = Chewy.request_strategy
|
|
19
21
|
Chewy.strategy(Chewy.request_strategy) { @app.call(env) }
|
|
20
22
|
end
|
|
@@ -35,7 +37,7 @@ module Chewy
|
|
|
35
37
|
if app.sandbox?
|
|
36
38
|
Chewy.strategy(:bypass)
|
|
37
39
|
else
|
|
38
|
-
Chewy.strategy(
|
|
40
|
+
Chewy.strategy(Chewy.console_strategy)
|
|
39
41
|
end
|
|
40
42
|
puts "Chewy console strategy is `#{Chewy.strategy.current.name}`"
|
|
41
43
|
end
|
data/lib/chewy/rake_helper.rb
CHANGED
|
@@ -3,14 +3,12 @@ module Chewy
|
|
|
3
3
|
IMPORT_CALLBACK = lambda do |output, _name, start, finish, _id, payload|
|
|
4
4
|
duration = (finish - start).ceil
|
|
5
5
|
stats = payload.fetch(:import, {}).map { |key, count| "#{key} #{count}" }.join(', ')
|
|
6
|
-
output.puts " Imported #{payload[:
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
output.puts " on #{documents.count} documents: #{documents}"
|
|
13
|
-
end
|
|
6
|
+
output.puts " Imported #{payload[:index]} in #{human_duration(duration)}, stats: #{stats}"
|
|
7
|
+
payload[:errors]&.each do |action, errors|
|
|
8
|
+
output.puts " #{action.to_s.humanize} errors:"
|
|
9
|
+
errors.each do |error, documents|
|
|
10
|
+
output.puts " `#{error}`"
|
|
11
|
+
output.puts " on #{documents.count} documents: #{documents}"
|
|
14
12
|
end
|
|
15
13
|
end
|
|
16
14
|
end
|
|
@@ -36,7 +34,7 @@ module Chewy
|
|
|
36
34
|
# @param parallel [true, Integer, Hash] any acceptable parallel options for import
|
|
37
35
|
# @param output [IO] output io for logging
|
|
38
36
|
# @return [Array<Chewy::Index>] indexes that were reset
|
|
39
|
-
def reset(only: nil, except: nil, parallel: nil, output:
|
|
37
|
+
def reset(only: nil, except: nil, parallel: nil, output: $stdout)
|
|
40
38
|
subscribed_task_stats(output) do
|
|
41
39
|
indexes_from(only: only, except: except).each do |index|
|
|
42
40
|
reset_one(index, output, parallel: parallel)
|
|
@@ -59,7 +57,7 @@ module Chewy
|
|
|
59
57
|
# @param parallel [true, Integer, Hash] any acceptable parallel options for import
|
|
60
58
|
# @param output [IO] output io for logging
|
|
61
59
|
# @return [Array<Chewy::Index>] indexes that were actually reset
|
|
62
|
-
def upgrade(only: nil, except: nil, parallel: nil, output:
|
|
60
|
+
def upgrade(only: nil, except: nil, parallel: nil, output: $stdout)
|
|
63
61
|
subscribed_task_stats(output) do
|
|
64
62
|
indexes = indexes_from(only: only, except: except)
|
|
65
63
|
|
|
@@ -87,23 +85,21 @@ module Chewy
|
|
|
87
85
|
#
|
|
88
86
|
# @example
|
|
89
87
|
# Chewy::RakeHelper.update # updates everything
|
|
90
|
-
# Chewy::RakeHelper.update(only: 'places') # updates only PlacesIndex
|
|
91
|
-
# Chewy::RakeHelper.update(
|
|
92
|
-
# Chewy::RakeHelper.update(except: PlacesIndex::Country) # updates everything, but PlacesIndex::Country
|
|
93
|
-
# Chewy::RakeHelper.update(only: 'places', except: 'places#country') # updates PlacesIndex::City only
|
|
88
|
+
# Chewy::RakeHelper.update(only: 'places') # updates only PlacesIndex
|
|
89
|
+
# Chewy::RakeHelper.update(except: PlacesIndex) # updates everything, but PlacesIndex
|
|
94
90
|
#
|
|
95
|
-
# @param only [Array<Chewy::Index,
|
|
96
|
-
# @param except [Array<Chewy::Index,
|
|
91
|
+
# @param only [Array<Chewy::Index, String>, Chewy::Index, String] indexes to update; if nothing is passed - uses all the indexes defined in the app
|
|
92
|
+
# @param except [Array<Chewy::Index, String>, Chewy::Index, String] indexes to exclude from processing
|
|
97
93
|
# @param parallel [true, Integer, Hash] any acceptable parallel options for import
|
|
98
94
|
# @param output [IO] output io for logging
|
|
99
|
-
# @return [Array<Chewy::
|
|
100
|
-
def update(only: nil, except: nil, parallel: nil, output:
|
|
95
|
+
# @return [Array<Chewy::Index>] indexes that were actually updated
|
|
96
|
+
def update(only: nil, except: nil, parallel: nil, output: $stdout)
|
|
101
97
|
subscribed_task_stats(output) do
|
|
102
|
-
|
|
98
|
+
indexes_from(only: only, except: except).each_with_object([]) do |index, updated_indexes|
|
|
103
99
|
if index.exists?
|
|
104
100
|
output.puts "Updating #{index}"
|
|
105
|
-
|
|
106
|
-
|
|
101
|
+
index.import(parallel: parallel)
|
|
102
|
+
updated_indexes.push(index)
|
|
107
103
|
else
|
|
108
104
|
output.puts "Skipping #{index}, it does not exists (use rake chewy:reset[#{index.derivable_name}] to create and update it)"
|
|
109
105
|
end
|
|
@@ -115,31 +111,29 @@ module Chewy
|
|
|
115
111
|
#
|
|
116
112
|
# @example
|
|
117
113
|
# Chewy::RakeHelper.sync # synchronizes everything
|
|
118
|
-
# Chewy::RakeHelper.sync(only: 'places') # synchronizes only PlacesIndex
|
|
119
|
-
# Chewy::RakeHelper.sync(
|
|
120
|
-
# Chewy::RakeHelper.sync(except: PlacesIndex::Country) # synchronizes everything, but PlacesIndex::Country
|
|
121
|
-
# Chewy::RakeHelper.sync(only: 'places', except: 'places#country') # synchronizes PlacesIndex::City only
|
|
114
|
+
# Chewy::RakeHelper.sync(only: 'places') # synchronizes only PlacesIndex
|
|
115
|
+
# Chewy::RakeHelper.sync(except: PlacesIndex) # synchronizes everything, but PlacesIndex
|
|
122
116
|
#
|
|
123
|
-
# @param only [Array<Chewy::Index,
|
|
124
|
-
# @param except [Array<Chewy::Index,
|
|
117
|
+
# @param only [Array<Chewy::Index, String>, Chewy::Index, String] indexes to synchronize; if nothing is passed - uses all the indexes defined in the app
|
|
118
|
+
# @param except [Array<Chewy::Index, String>, Chewy::Index, String] indexes to exclude from processing
|
|
125
119
|
# @param parallel [true, Integer, Hash] any acceptable parallel options for sync
|
|
126
120
|
# @param output [IO] output io for logging
|
|
127
|
-
# @return [Array<Chewy::
|
|
128
|
-
def sync(only: nil, except: nil, parallel: nil, output:
|
|
121
|
+
# @return [Array<Chewy::Index>] indexes that were actually updated
|
|
122
|
+
def sync(only: nil, except: nil, parallel: nil, output: $stdout)
|
|
129
123
|
subscribed_task_stats(output) do
|
|
130
|
-
|
|
131
|
-
output.puts "Synchronizing #{
|
|
132
|
-
output.puts " #{
|
|
124
|
+
indexes_from(only: only, except: except).each_with_object([]) do |index, synced_indexes|
|
|
125
|
+
output.puts "Synchronizing #{index}"
|
|
126
|
+
output.puts " #{index} doesn't support outdated synchronization" unless index.supports_outdated_sync?
|
|
133
127
|
time = Time.now
|
|
134
|
-
sync_result =
|
|
128
|
+
sync_result = index.sync(parallel: parallel)
|
|
135
129
|
if !sync_result
|
|
136
|
-
output.puts " Something went wrong with the #{
|
|
137
|
-
elsif sync_result[:count]
|
|
130
|
+
output.puts " Something went wrong with the #{index} synchronization"
|
|
131
|
+
elsif (sync_result[:count]).positive?
|
|
138
132
|
output.puts " Missing documents: #{sync_result[:missing]}" if sync_result[:missing].present?
|
|
139
133
|
output.puts " Outdated documents: #{sync_result[:outdated]}" if sync_result[:outdated].present?
|
|
140
|
-
|
|
134
|
+
synced_indexes.push(index)
|
|
141
135
|
else
|
|
142
|
-
output.puts " Skipping #{
|
|
136
|
+
output.puts " Skipping #{index}, up to date"
|
|
143
137
|
end
|
|
144
138
|
output.puts " Took #{human_duration(Time.now - time)}"
|
|
145
139
|
end
|
|
@@ -147,49 +141,46 @@ module Chewy
|
|
|
147
141
|
end
|
|
148
142
|
|
|
149
143
|
# Applies changes that were done after the specified time for the
|
|
150
|
-
# specified indexes
|
|
144
|
+
# specified indexes or all of them.
|
|
151
145
|
#
|
|
152
146
|
# @example
|
|
153
147
|
# Chewy::RakeHelper.journal_apply(time: 1.minute.ago) # applies entries created for the last minute
|
|
154
|
-
# Chewy::RakeHelper.journal_apply(time: 1.minute.ago, only: 'places') # applies only PlacesIndex
|
|
155
|
-
# Chewy::RakeHelper.journal_apply(time: 1.minute.ago,
|
|
156
|
-
# Chewy::RakeHelper.journal_apply(time: 1.minute.ago, except: PlacesIndex::Country) # applies everything, but PlacesIndex::Country entries reated for the last minute
|
|
157
|
-
# Chewy::RakeHelper.journal_apply(time: 1.minute.ago, only: 'places', except: 'places#country') # applies PlacesIndex::City entries reated for the last minute only
|
|
148
|
+
# Chewy::RakeHelper.journal_apply(time: 1.minute.ago, only: 'places') # applies only PlacesIndex entries created for the last minute
|
|
149
|
+
# Chewy::RakeHelper.journal_apply(time: 1.minute.ago, except: PlacesIndex) # applies everything, but PlacesIndex, entries created for the last minute
|
|
158
150
|
#
|
|
159
151
|
# @param time [Time, DateTime] use only journal entries created after this time
|
|
160
|
-
# @param only [Array<Chewy::Index,
|
|
161
|
-
# @param except [Array<Chewy::Index,
|
|
152
|
+
# @param only [Array<Chewy::Index, String>, Chewy::Index, String] indexes to synchronize; if nothing is passed - uses all the indexes defined in the app
|
|
153
|
+
# @param except [Array<Chewy::Index, String>, Chewy::Index, String] indexes to exclude from processing
|
|
162
154
|
# @param output [IO] output io for logging
|
|
163
|
-
# @return [Array<Chewy::
|
|
164
|
-
def journal_apply(time: nil, only: nil, except: nil, output:
|
|
155
|
+
# @return [Array<Chewy::Index>] indexes that were actually updated
|
|
156
|
+
def journal_apply(time: nil, only: nil, except: nil, output: $stdout)
|
|
165
157
|
raise ArgumentError, 'Please specify the time to start with' unless time
|
|
158
|
+
|
|
166
159
|
subscribed_task_stats(output) do
|
|
167
160
|
output.puts "Applying journal entries created after #{time}"
|
|
168
|
-
count = Chewy::Journal.new(
|
|
161
|
+
count = Chewy::Journal.new(indexes_from(only: only, except: except)).apply(time)
|
|
169
162
|
output.puts 'No journal entries were created after the specified time' if count.zero?
|
|
170
163
|
end
|
|
171
164
|
end
|
|
172
165
|
|
|
173
166
|
# Removes journal records created before the specified timestamp for
|
|
174
|
-
# the specified indexes
|
|
167
|
+
# the specified indexes or all of them.
|
|
175
168
|
#
|
|
176
169
|
# @example
|
|
177
170
|
# Chewy::RakeHelper.journal_clean # cleans everything
|
|
178
171
|
# Chewy::RakeHelper.journal_clean(time: 1.minute.ago) # leaves only entries created for the last minute
|
|
179
|
-
# Chewy::RakeHelper.journal_clean(only: 'places') # cleans only PlacesIndex
|
|
180
|
-
# Chewy::RakeHelper.journal_clean(
|
|
181
|
-
# Chewy::RakeHelper.journal_clean(except: PlacesIndex::Country) # cleans everything, but PlacesIndex::Country entries
|
|
182
|
-
# Chewy::RakeHelper.journal_clean(only: 'places', except: 'places#country') # cleans PlacesIndex::City entries only
|
|
172
|
+
# Chewy::RakeHelper.journal_clean(only: 'places') # cleans only PlacesIndex entries
|
|
173
|
+
# Chewy::RakeHelper.journal_clean(except: PlacesIndex) # cleans everything, but PlacesIndex entries
|
|
183
174
|
#
|
|
184
175
|
# @param time [Time, DateTime] clean all the journal entries created before this time
|
|
185
|
-
# @param only [Array<Chewy::Index,
|
|
186
|
-
# @param except [Array<Chewy::Index,
|
|
176
|
+
# @param only [Array<Chewy::Index, String>, Chewy::Index, String] indexes to synchronize; if nothing is passed - uses all the indexes defined in the app
|
|
177
|
+
# @param except [Array<Chewy::Index, String>, Chewy::Index, String] indexes to exclude from processing
|
|
187
178
|
# @param output [IO] output io for logging
|
|
188
|
-
# @return [Array<Chewy::
|
|
189
|
-
def journal_clean(time: nil, only: nil, except: nil, output:
|
|
179
|
+
# @return [Array<Chewy::Index>] indexes that were actually updated
|
|
180
|
+
def journal_clean(time: nil, only: nil, except: nil, output: $stdout)
|
|
190
181
|
subscribed_task_stats(output) do
|
|
191
182
|
output.puts "Cleaning journal entries created before #{time}" if time
|
|
192
|
-
response = Chewy::Journal.new(
|
|
183
|
+
response = Chewy::Journal.new(indexes_from(only: only, except: except)).clean(time)
|
|
193
184
|
count = response['deleted'] || response['_indices']['_all']['deleted']
|
|
194
185
|
output.puts "Cleaned up #{count} journal entries"
|
|
195
186
|
end
|
|
@@ -204,45 +195,53 @@ module Chewy
|
|
|
204
195
|
Chewy::Index.descendants - [Chewy::Stash::Journal, Chewy::Stash::Specification]
|
|
205
196
|
end
|
|
206
197
|
|
|
198
|
+
# Reindex data from source index to destination index
|
|
199
|
+
#
|
|
200
|
+
# @example
|
|
201
|
+
# Chewy::RakeHelper.reindex(source: 'users_index', dest: 'cities_index') reindex data from 'users_index' index to 'cities_index'
|
|
202
|
+
#
|
|
203
|
+
# @param source [String], dest [String] indexes to reindex
|
|
204
|
+
def reindex(source:, dest:, output: $stdout)
|
|
205
|
+
subscribed_task_stats(output) do
|
|
206
|
+
output.puts "Source index is #{source}\nDestination index is #{dest}"
|
|
207
|
+
Chewy::Index.reindex(source: source, dest: dest)
|
|
208
|
+
output.puts "#{source} index successfully reindexed with #{dest} index data"
|
|
209
|
+
end
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
# Adds new fields to an existing data stream or index.
|
|
213
|
+
# Change the search settings of existing fields.
|
|
214
|
+
#
|
|
215
|
+
# @example
|
|
216
|
+
# Chewy::RakeHelper.update_mapping('cities', {properties: {new_field: {type: :text}}}) update 'cities' index with new_field of text type
|
|
217
|
+
#
|
|
218
|
+
# @param name [String], body_hash [Hash] index name and body hash to update
|
|
219
|
+
def update_mapping(name:, output: $stdout)
|
|
220
|
+
subscribed_task_stats(output) do
|
|
221
|
+
output.puts "Index name is #{name}"
|
|
222
|
+
normalize_index(name).update_mapping
|
|
223
|
+
output.puts "#{name} index successfully updated"
|
|
224
|
+
end
|
|
225
|
+
end
|
|
226
|
+
|
|
207
227
|
def normalize_indexes(*identifiers)
|
|
208
228
|
identifiers.flatten(1).map { |identifier| normalize_index(identifier) }
|
|
209
229
|
end
|
|
210
230
|
|
|
211
231
|
def normalize_index(identifier)
|
|
212
232
|
return identifier if identifier.is_a?(Class) && identifier < Chewy::Index
|
|
213
|
-
|
|
233
|
+
|
|
234
|
+
"#{identifier.to_s.camelize}Index".constantize
|
|
214
235
|
end
|
|
215
236
|
|
|
216
|
-
def subscribed_task_stats(output =
|
|
237
|
+
def subscribed_task_stats(output = $stdout, &block)
|
|
217
238
|
start = Time.now
|
|
218
239
|
ActiveSupport::Notifications.subscribed(JOURNAL_CALLBACK.curry[output], 'apply_journal.chewy') do
|
|
219
|
-
ActiveSupport::Notifications.subscribed(IMPORT_CALLBACK.curry[output], 'import_objects.chewy')
|
|
220
|
-
yield
|
|
221
|
-
end
|
|
240
|
+
ActiveSupport::Notifications.subscribed(IMPORT_CALLBACK.curry[output], 'import_objects.chewy', &block)
|
|
222
241
|
end
|
|
223
242
|
output.puts "Total: #{human_duration(Time.now - start)}"
|
|
224
243
|
end
|
|
225
244
|
|
|
226
|
-
def reset_index(*indexes)
|
|
227
|
-
ActiveSupport::Deprecation.warn '`Chewy::RakeHelper.reset_index` is deprecated and will be removed soon, use `Chewy::RakeHelper.reset` instead'
|
|
228
|
-
reset(only: indexes)
|
|
229
|
-
end
|
|
230
|
-
|
|
231
|
-
def reset_all(*except)
|
|
232
|
-
ActiveSupport::Deprecation.warn '`Chewy::RakeHelper.reset_all` is deprecated and will be removed soon, use `Chewy::RakeHelper.reset` instead'
|
|
233
|
-
reset(except: except)
|
|
234
|
-
end
|
|
235
|
-
|
|
236
|
-
def update_index(*indexes)
|
|
237
|
-
ActiveSupport::Deprecation.warn '`Chewy::RakeHelper.update_index` is deprecated and will be removed soon, use `Chewy::RakeHelper.update` instead'
|
|
238
|
-
update(only: indexes)
|
|
239
|
-
end
|
|
240
|
-
|
|
241
|
-
def update_all(*except)
|
|
242
|
-
ActiveSupport::Deprecation.warn '`Chewy::RakeHelper.update_all` is deprecated and will be removed soon, use `Chewy::RakeHelper.update` instead'
|
|
243
|
-
update(except: except)
|
|
244
|
-
end
|
|
245
|
-
|
|
246
245
|
private
|
|
247
246
|
|
|
248
247
|
def indexes_from(only: nil, except: nil)
|
|
@@ -261,33 +260,9 @@ module Chewy
|
|
|
261
260
|
indexes.sort_by(&:derivable_name)
|
|
262
261
|
end
|
|
263
262
|
|
|
264
|
-
def types_from(only: nil, except: nil)
|
|
265
|
-
types = if only.present?
|
|
266
|
-
normalize_types(Array.wrap(only))
|
|
267
|
-
else
|
|
268
|
-
all_indexes.flat_map(&:types)
|
|
269
|
-
end
|
|
270
|
-
|
|
271
|
-
types = if except.present?
|
|
272
|
-
types - normalize_types(Array.wrap(except))
|
|
273
|
-
else
|
|
274
|
-
types
|
|
275
|
-
end
|
|
276
|
-
|
|
277
|
-
types.sort_by(&:derivable_name)
|
|
278
|
-
end
|
|
279
|
-
|
|
280
|
-
def normalize_types(*identifiers)
|
|
281
|
-
identifiers.flatten(1).flat_map { |identifier| normalize_type(identifier) }
|
|
282
|
-
end
|
|
283
|
-
|
|
284
|
-
def normalize_type(identifier)
|
|
285
|
-
Chewy.derive_types(identifier)
|
|
286
|
-
end
|
|
287
|
-
|
|
288
263
|
def human_duration(seconds)
|
|
289
264
|
[[60, :s], [60, :m], [24, :h]].map do |amount, unit|
|
|
290
|
-
if seconds
|
|
265
|
+
if seconds.positive?
|
|
291
266
|
seconds, n = seconds.divmod(amount)
|
|
292
267
|
"#{n.to_i}#{unit}"
|
|
293
268
|
end
|