chewy 0.8.4 → 7.3.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.github/CODEOWNERS +1 -0
- data/.github/ISSUE_TEMPLATE/bug_report.md +39 -0
- data/.github/ISSUE_TEMPLATE/feature_request.md +20 -0
- data/.github/PULL_REQUEST_TEMPLATE.md +16 -0
- data/.github/workflows/ruby.yml +74 -0
- data/.gitignore +1 -0
- data/.rubocop.yml +61 -0
- data/.rubocop_todo.yml +132 -0
- data/.yardopts +5 -0
- data/CHANGELOG.md +554 -245
- data/CODE_OF_CONDUCT.md +14 -0
- data/CONTRIBUTING.md +63 -0
- data/Gemfile +14 -11
- data/Guardfile +8 -6
- data/LICENSE.txt +1 -1
- data/README.md +748 -623
- data/Rakefile +11 -1
- data/chewy.gemspec +15 -19
- data/gemfiles/rails.5.2.activerecord.gemfile +11 -0
- data/gemfiles/rails.6.0.activerecord.gemfile +11 -0
- data/gemfiles/rails.6.1.activerecord.gemfile +13 -0
- data/gemfiles/rails.7.0.activerecord.gemfile +13 -0
- data/lib/chewy/config.rb +64 -50
- data/lib/chewy/errors.rb +10 -16
- data/lib/chewy/fields/base.rb +122 -32
- data/lib/chewy/fields/root.rb +48 -23
- data/lib/chewy/index/actions.rb +140 -54
- data/lib/chewy/index/adapter/active_record.rb +112 -0
- data/lib/chewy/{type → index}/adapter/base.rb +31 -12
- data/lib/chewy/index/adapter/object.rb +249 -0
- data/lib/chewy/index/adapter/orm.rb +194 -0
- data/lib/chewy/index/aliases.rb +14 -4
- data/lib/chewy/index/crutch.rb +40 -0
- data/lib/chewy/index/import/bulk_builder.rb +311 -0
- data/lib/chewy/index/import/bulk_request.rb +77 -0
- data/lib/chewy/index/import/journal_builder.rb +44 -0
- data/lib/chewy/index/import/routine.rb +139 -0
- data/lib/chewy/index/import.rb +243 -0
- data/lib/chewy/{type → index}/mapping.rb +79 -68
- data/lib/chewy/index/observe/active_record_methods.rb +87 -0
- data/lib/chewy/index/observe/callback.rb +34 -0
- data/lib/chewy/index/observe.rb +17 -0
- data/lib/chewy/index/settings.rb +10 -5
- data/lib/chewy/index/specification.rb +61 -0
- data/lib/chewy/index/syncer.rb +221 -0
- data/lib/chewy/{type → index}/witchcraft.rb +100 -39
- data/lib/chewy/index/wrapper.rb +95 -0
- data/lib/chewy/index.rb +216 -140
- data/lib/chewy/journal.rb +66 -0
- data/lib/chewy/log_subscriber.rb +8 -8
- data/lib/chewy/minitest/helpers.rb +150 -0
- data/lib/chewy/minitest/search_index_receiver.rb +76 -0
- data/lib/chewy/minitest.rb +1 -0
- data/lib/chewy/multi_search.rb +62 -0
- data/lib/chewy/railtie.rb +12 -25
- data/lib/chewy/rake_helper.rb +335 -37
- data/lib/chewy/repository.rb +2 -2
- data/lib/chewy/rspec/build_query.rb +12 -0
- data/lib/chewy/rspec/helpers.rb +55 -0
- data/lib/chewy/rspec/update_index.rb +106 -90
- data/lib/chewy/rspec.rb +3 -1
- data/lib/chewy/runtime/version.rb +4 -4
- data/lib/chewy/runtime.rb +1 -1
- data/lib/chewy/search/loader.rb +61 -0
- data/lib/chewy/{query → search}/pagination/kaminari.rb +13 -5
- data/lib/chewy/search/parameters/aggs.rb +16 -0
- data/lib/chewy/search/parameters/allow_partial_search_results.rb +27 -0
- data/lib/chewy/search/parameters/collapse.rb +16 -0
- data/lib/chewy/search/parameters/concerns/bool_storage.rb +24 -0
- data/lib/chewy/search/parameters/concerns/hash_storage.rb +23 -0
- data/lib/chewy/search/parameters/concerns/integer_storage.rb +14 -0
- data/lib/chewy/search/parameters/concerns/query_storage.rb +238 -0
- data/lib/chewy/search/parameters/concerns/string_array_storage.rb +23 -0
- data/lib/chewy/search/parameters/concerns/string_storage.rb +14 -0
- data/lib/chewy/search/parameters/docvalue_fields.rb +12 -0
- data/lib/chewy/search/parameters/explain.rb +16 -0
- data/lib/chewy/search/parameters/filter.rb +47 -0
- data/lib/chewy/search/parameters/highlight.rb +16 -0
- data/lib/chewy/search/parameters/ignore_unavailable.rb +27 -0
- data/lib/chewy/search/parameters/indices.rb +78 -0
- data/lib/chewy/search/parameters/indices_boost.rb +52 -0
- data/lib/chewy/search/parameters/limit.rb +17 -0
- data/lib/chewy/search/parameters/load.rb +32 -0
- data/lib/chewy/search/parameters/min_score.rb +16 -0
- data/lib/chewy/search/parameters/none.rb +25 -0
- data/lib/chewy/search/parameters/offset.rb +17 -0
- data/lib/chewy/search/parameters/order.rb +51 -0
- data/lib/chewy/search/parameters/post_filter.rb +19 -0
- data/lib/chewy/search/parameters/preference.rb +16 -0
- data/lib/chewy/search/parameters/profile.rb +16 -0
- data/lib/chewy/search/parameters/query.rb +19 -0
- data/lib/chewy/search/parameters/request_cache.rb +27 -0
- data/lib/chewy/search/parameters/rescore.rb +29 -0
- data/lib/chewy/search/parameters/script_fields.rb +16 -0
- data/lib/chewy/search/parameters/search_after.rb +20 -0
- data/lib/chewy/search/parameters/search_type.rb +16 -0
- data/lib/chewy/search/parameters/source.rb +77 -0
- data/lib/chewy/search/parameters/storage.rb +95 -0
- data/lib/chewy/search/parameters/stored_fields.rb +63 -0
- data/lib/chewy/search/parameters/suggest.rb +16 -0
- data/lib/chewy/search/parameters/terminate_after.rb +16 -0
- data/lib/chewy/search/parameters/timeout.rb +16 -0
- data/lib/chewy/search/parameters/track_scores.rb +16 -0
- data/lib/chewy/search/parameters/track_total_hits.rb +16 -0
- data/lib/chewy/search/parameters/version.rb +16 -0
- data/lib/chewy/search/parameters.rb +170 -0
- data/lib/chewy/search/query_proxy.rb +264 -0
- data/lib/chewy/search/request.rb +1071 -0
- data/lib/chewy/search/response.rb +119 -0
- data/lib/chewy/search/scoping.rb +49 -0
- data/lib/chewy/search/scrolling.rb +137 -0
- data/lib/chewy/search.rb +68 -28
- data/lib/chewy/stash.rb +68 -0
- data/lib/chewy/strategy/active_job.rb +3 -2
- data/lib/chewy/strategy/atomic.rb +2 -4
- data/lib/chewy/strategy/atomic_no_refresh.rb +18 -0
- data/lib/chewy/strategy/base.rb +13 -3
- data/lib/chewy/strategy/bypass.rb +1 -2
- data/lib/chewy/strategy/delayed_sidekiq/scheduler.rb +148 -0
- data/lib/chewy/strategy/delayed_sidekiq/worker.rb +52 -0
- data/lib/chewy/strategy/delayed_sidekiq.rb +17 -0
- data/lib/chewy/strategy/lazy_sidekiq.rb +64 -0
- data/lib/chewy/strategy/sidekiq.rb +15 -2
- data/lib/chewy/strategy/urgent.rb +1 -1
- data/lib/chewy/strategy.rb +16 -20
- data/lib/chewy/version.rb +1 -1
- data/lib/chewy.rb +81 -82
- data/lib/generators/chewy/install_generator.rb +3 -3
- data/lib/tasks/chewy.rake +99 -32
- data/migration_guide.md +56 -0
- data/spec/chewy/config_spec.rb +87 -15
- data/spec/chewy/fields/base_spec.rb +542 -233
- data/spec/chewy/fields/root_spec.rb +115 -17
- data/spec/chewy/fields/time_fields_spec.rb +13 -12
- data/spec/chewy/index/actions_spec.rb +595 -77
- data/spec/chewy/index/adapter/active_record_spec.rb +601 -0
- data/spec/chewy/index/adapter/object_spec.rb +243 -0
- data/spec/chewy/index/aliases_spec.rb +5 -5
- data/spec/chewy/index/import/bulk_builder_spec.rb +494 -0
- data/spec/chewy/index/import/bulk_request_spec.rb +95 -0
- data/spec/chewy/index/import/journal_builder_spec.rb +87 -0
- data/spec/chewy/index/import/routine_spec.rb +110 -0
- data/spec/chewy/index/import_spec.rb +615 -0
- data/spec/chewy/index/mapping_spec.rb +135 -0
- data/spec/chewy/index/observe/active_record_methods_spec.rb +68 -0
- data/spec/chewy/index/observe/callback_spec.rb +139 -0
- data/spec/chewy/index/observe_spec.rb +143 -0
- data/spec/chewy/index/settings_spec.rb +103 -50
- data/spec/chewy/index/specification_spec.rb +159 -0
- data/spec/chewy/index/syncer_spec.rb +118 -0
- data/spec/chewy/index/witchcraft_spec.rb +245 -0
- data/spec/chewy/index/wrapper_spec.rb +100 -0
- data/spec/chewy/index_spec.rb +149 -121
- data/spec/chewy/journal_spec.rb +223 -0
- data/spec/chewy/minitest/helpers_spec.rb +198 -0
- data/spec/chewy/minitest/search_index_receiver_spec.rb +118 -0
- data/spec/chewy/multi_search_spec.rb +84 -0
- data/spec/chewy/rake_helper_spec.rb +656 -0
- data/spec/chewy/repository_spec.rb +8 -8
- data/spec/chewy/rspec/build_query_spec.rb +34 -0
- data/spec/chewy/rspec/helpers_spec.rb +61 -0
- data/spec/chewy/rspec/update_index_spec.rb +220 -114
- data/spec/chewy/runtime_spec.rb +2 -2
- data/spec/chewy/search/loader_spec.rb +83 -0
- data/spec/chewy/search/pagination/kaminari_examples.rb +69 -0
- data/spec/chewy/search/pagination/kaminari_spec.rb +21 -0
- data/spec/chewy/search/parameters/aggs_spec.rb +5 -0
- data/spec/chewy/search/parameters/bool_storage_examples.rb +53 -0
- data/spec/chewy/search/parameters/collapse_spec.rb +5 -0
- data/spec/chewy/search/parameters/docvalue_fields_spec.rb +5 -0
- data/spec/chewy/search/parameters/explain_spec.rb +5 -0
- data/spec/chewy/search/parameters/filter_spec.rb +5 -0
- data/spec/chewy/search/parameters/hash_storage_examples.rb +59 -0
- data/spec/chewy/search/parameters/highlight_spec.rb +5 -0
- data/spec/chewy/search/parameters/ignore_unavailable_spec.rb +67 -0
- data/spec/chewy/search/parameters/indices_spec.rb +99 -0
- data/spec/chewy/search/parameters/integer_storage_examples.rb +32 -0
- data/spec/chewy/search/parameters/limit_spec.rb +5 -0
- data/spec/chewy/search/parameters/load_spec.rb +60 -0
- data/spec/chewy/search/parameters/min_score_spec.rb +32 -0
- data/spec/chewy/search/parameters/none_spec.rb +5 -0
- data/spec/chewy/search/parameters/offset_spec.rb +5 -0
- data/spec/chewy/search/parameters/order_spec.rb +72 -0
- data/spec/chewy/search/parameters/post_filter_spec.rb +5 -0
- data/spec/chewy/search/parameters/preference_spec.rb +5 -0
- data/spec/chewy/search/parameters/profile_spec.rb +5 -0
- data/spec/chewy/search/parameters/query_spec.rb +5 -0
- data/spec/chewy/search/parameters/query_storage_examples.rb +434 -0
- data/spec/chewy/search/parameters/request_cache_spec.rb +67 -0
- data/spec/chewy/search/parameters/rescore_spec.rb +62 -0
- data/spec/chewy/search/parameters/script_fields_spec.rb +5 -0
- data/spec/chewy/search/parameters/search_after_spec.rb +35 -0
- data/spec/chewy/search/parameters/search_type_spec.rb +5 -0
- data/spec/chewy/search/parameters/source_spec.rb +162 -0
- data/spec/chewy/search/parameters/storage_spec.rb +60 -0
- data/spec/chewy/search/parameters/stored_fields_spec.rb +126 -0
- data/spec/chewy/search/parameters/string_array_storage_examples.rb +63 -0
- data/spec/chewy/search/parameters/string_storage_examples.rb +32 -0
- data/spec/chewy/search/parameters/suggest_spec.rb +5 -0
- data/spec/chewy/search/parameters/terminate_after_spec.rb +5 -0
- data/spec/chewy/search/parameters/timeout_spec.rb +5 -0
- data/spec/chewy/search/parameters/track_scores_spec.rb +5 -0
- data/spec/chewy/search/parameters/track_total_hits_spec.rb +5 -0
- data/spec/chewy/search/parameters/version_spec.rb +5 -0
- data/spec/chewy/search/parameters_spec.rb +161 -0
- data/spec/chewy/search/query_proxy_spec.rb +119 -0
- data/spec/chewy/search/request_spec.rb +880 -0
- data/spec/chewy/search/response_spec.rb +202 -0
- data/spec/chewy/search/scrolling_spec.rb +171 -0
- data/spec/chewy/search_spec.rb +82 -55
- data/spec/chewy/stash_spec.rb +85 -0
- data/spec/chewy/strategy/active_job_spec.rb +27 -8
- data/spec/chewy/strategy/atomic_no_refresh_spec.rb +60 -0
- data/spec/chewy/strategy/atomic_spec.rb +13 -11
- data/spec/chewy/strategy/delayed_sidekiq_spec.rb +190 -0
- data/spec/chewy/strategy/lazy_sidekiq_spec.rb +214 -0
- data/spec/chewy/strategy/sidekiq_spec.rb +19 -7
- data/spec/chewy/strategy_spec.rb +19 -15
- data/spec/chewy_spec.rb +65 -88
- data/spec/spec_helper.rb +11 -20
- data/spec/support/active_record.rb +48 -6
- data/spec/support/class_helpers.rb +4 -19
- metadata +299 -183
- data/.travis.yml +0 -76
- data/Appraisals +0 -76
- data/gemfiles/rails.3.2.activerecord.gemfile +0 -15
- data/gemfiles/rails.3.2.activerecord.kaminari.gemfile +0 -14
- data/gemfiles/rails.3.2.activerecord.will_paginate.gemfile +0 -14
- data/gemfiles/rails.4.0.activerecord.gemfile +0 -15
- data/gemfiles/rails.4.0.activerecord.kaminari.gemfile +0 -14
- data/gemfiles/rails.4.0.activerecord.will_paginate.gemfile +0 -14
- data/gemfiles/rails.4.0.mongoid.4.0.0.gemfile +0 -15
- data/gemfiles/rails.4.0.mongoid.4.0.0.kaminari.gemfile +0 -14
- data/gemfiles/rails.4.0.mongoid.4.0.0.will_paginate.gemfile +0 -14
- data/gemfiles/rails.4.0.mongoid.5.1.0.gemfile +0 -15
- data/gemfiles/rails.4.0.mongoid.5.1.0.kaminari.gemfile +0 -14
- data/gemfiles/rails.4.0.mongoid.5.1.0.will_paginate.gemfile +0 -14
- data/gemfiles/rails.4.1.activerecord.gemfile +0 -15
- data/gemfiles/rails.4.1.activerecord.kaminari.gemfile +0 -14
- data/gemfiles/rails.4.1.activerecord.will_paginate.gemfile +0 -14
- data/gemfiles/rails.4.1.mongoid.4.0.0.gemfile +0 -15
- data/gemfiles/rails.4.1.mongoid.4.0.0.kaminari.gemfile +0 -14
- data/gemfiles/rails.4.1.mongoid.4.0.0.will_paginate.gemfile +0 -14
- data/gemfiles/rails.4.1.mongoid.5.1.0.gemfile +0 -15
- data/gemfiles/rails.4.1.mongoid.5.1.0.kaminari.gemfile +0 -14
- data/gemfiles/rails.4.1.mongoid.5.1.0.will_paginate.gemfile +0 -14
- data/gemfiles/rails.4.2.activerecord.gemfile +0 -16
- data/gemfiles/rails.4.2.activerecord.kaminari.gemfile +0 -15
- data/gemfiles/rails.4.2.activerecord.will_paginate.gemfile +0 -15
- data/gemfiles/rails.4.2.mongoid.4.0.0.gemfile +0 -15
- data/gemfiles/rails.4.2.mongoid.4.0.0.kaminari.gemfile +0 -14
- data/gemfiles/rails.4.2.mongoid.4.0.0.will_paginate.gemfile +0 -14
- data/gemfiles/rails.4.2.mongoid.5.1.0.gemfile +0 -15
- data/gemfiles/rails.4.2.mongoid.5.1.0.kaminari.gemfile +0 -14
- data/gemfiles/rails.4.2.mongoid.5.1.0.will_paginate.gemfile +0 -14
- data/gemfiles/rails.5.0.0.beta3.activerecord.gemfile +0 -16
- data/gemfiles/rails.5.0.0.beta3.activerecord.kaminari.gemfile +0 -16
- data/gemfiles/rails.5.0.0.beta3.activerecord.will_paginate.gemfile +0 -15
- data/gemfiles/sequel.4.31.gemfile +0 -13
- data/lib/chewy/backports/deep_dup.rb +0 -46
- data/lib/chewy/backports/duplicable.rb +0 -90
- data/lib/chewy/query/compose.rb +0 -69
- data/lib/chewy/query/criteria.rb +0 -181
- data/lib/chewy/query/filters.rb +0 -227
- data/lib/chewy/query/loading.rb +0 -111
- data/lib/chewy/query/nodes/and.rb +0 -25
- data/lib/chewy/query/nodes/base.rb +0 -17
- data/lib/chewy/query/nodes/bool.rb +0 -32
- data/lib/chewy/query/nodes/equal.rb +0 -34
- data/lib/chewy/query/nodes/exists.rb +0 -20
- data/lib/chewy/query/nodes/expr.rb +0 -28
- data/lib/chewy/query/nodes/field.rb +0 -106
- data/lib/chewy/query/nodes/has_child.rb +0 -14
- data/lib/chewy/query/nodes/has_parent.rb +0 -14
- data/lib/chewy/query/nodes/has_relation.rb +0 -61
- data/lib/chewy/query/nodes/match_all.rb +0 -11
- data/lib/chewy/query/nodes/missing.rb +0 -20
- data/lib/chewy/query/nodes/not.rb +0 -25
- data/lib/chewy/query/nodes/or.rb +0 -25
- data/lib/chewy/query/nodes/prefix.rb +0 -18
- data/lib/chewy/query/nodes/query.rb +0 -20
- data/lib/chewy/query/nodes/range.rb +0 -63
- data/lib/chewy/query/nodes/raw.rb +0 -15
- data/lib/chewy/query/nodes/regexp.rb +0 -31
- data/lib/chewy/query/nodes/script.rb +0 -20
- data/lib/chewy/query/pagination/will_paginate.rb +0 -27
- data/lib/chewy/query/pagination.rb +0 -16
- data/lib/chewy/query/scoping.rb +0 -20
- data/lib/chewy/query.rb +0 -1026
- data/lib/chewy/strategy/resque.rb +0 -26
- data/lib/chewy/type/actions.rb +0 -19
- data/lib/chewy/type/adapter/active_record.rb +0 -72
- data/lib/chewy/type/adapter/mongoid.rb +0 -58
- data/lib/chewy/type/adapter/object.rb +0 -89
- data/lib/chewy/type/adapter/orm.rb +0 -156
- data/lib/chewy/type/adapter/sequel.rb +0 -75
- data/lib/chewy/type/crutch.rb +0 -31
- data/lib/chewy/type/import.rb +0 -224
- data/lib/chewy/type/observe.rb +0 -76
- data/lib/chewy/type/wrapper.rb +0 -53
- data/lib/chewy/type.rb +0 -89
- data/lib/sequel/plugins/chewy_observe.rb +0 -78
- data/spec/chewy/query/criteria_spec.rb +0 -433
- data/spec/chewy/query/filters_spec.rb +0 -173
- data/spec/chewy/query/loading_spec.rb +0 -86
- data/spec/chewy/query/nodes/and_spec.rb +0 -16
- data/spec/chewy/query/nodes/bool_spec.rb +0 -22
- data/spec/chewy/query/nodes/equal_spec.rb +0 -32
- data/spec/chewy/query/nodes/exists_spec.rb +0 -18
- data/spec/chewy/query/nodes/has_child_spec.rb +0 -40
- data/spec/chewy/query/nodes/has_parent_spec.rb +0 -40
- data/spec/chewy/query/nodes/match_all_spec.rb +0 -11
- data/spec/chewy/query/nodes/missing_spec.rb +0 -15
- data/spec/chewy/query/nodes/not_spec.rb +0 -16
- data/spec/chewy/query/nodes/or_spec.rb +0 -16
- data/spec/chewy/query/nodes/prefix_spec.rb +0 -16
- data/spec/chewy/query/nodes/query_spec.rb +0 -12
- data/spec/chewy/query/nodes/range_spec.rb +0 -32
- data/spec/chewy/query/nodes/raw_spec.rb +0 -11
- data/spec/chewy/query/nodes/regexp_spec.rb +0 -31
- data/spec/chewy/query/nodes/script_spec.rb +0 -15
- data/spec/chewy/query/pagination/kaminari_spec.rb +0 -57
- data/spec/chewy/query/pagination/will_paginage_spec.rb +0 -60
- data/spec/chewy/query/pagination_spec.rb +0 -36
- data/spec/chewy/query_spec.rb +0 -632
- data/spec/chewy/strategy/resque_spec.rb +0 -40
- data/spec/chewy/type/actions_spec.rb +0 -31
- data/spec/chewy/type/adapter/active_record_spec.rb +0 -317
- data/spec/chewy/type/adapter/mongoid_spec.rb +0 -253
- data/spec/chewy/type/adapter/object_spec.rb +0 -139
- data/spec/chewy/type/adapter/sequel_spec.rb +0 -320
- data/spec/chewy/type/import_spec.rb +0 -433
- data/spec/chewy/type/mapping_spec.rb +0 -106
- data/spec/chewy/type/observe_spec.rb +0 -127
- data/spec/chewy/type/witchcraft_spec.rb +0 -154
- data/spec/chewy/type/wrapper_spec.rb +0 -58
- data/spec/chewy/type_spec.rb +0 -33
- data/spec/support/mongoid.rb +0 -81
- data/spec/support/sequel.rb +0 -75
@@ -0,0 +1,76 @@
|
|
1
|
+
# Test helper class to provide minitest hooks for Chewy::Index testing.
|
2
|
+
#
|
3
|
+
# @note Intended to be used in conjunction with a test helper which mocks over the #bulk
|
4
|
+
# method on a {Chewy::Index} class. (See {Chewy::Minitest::Helpers})
|
5
|
+
#
|
6
|
+
# The class will capture the data from the *param on the Chewy::Index.bulk method and
|
7
|
+
# aggregate the data for test analysis.
|
8
|
+
class SearchIndexReceiver
|
9
|
+
def initialize
|
10
|
+
@mutations = {}
|
11
|
+
end
|
12
|
+
|
13
|
+
# @param bulk_params [Hash] the bulk_params that should be sent to the Chewy::Index.bulk method.
|
14
|
+
# @param index [Chewy::Index] the index executing this query.
|
15
|
+
def catch(bulk_params, index)
|
16
|
+
Array.wrap(bulk_params).map { |y| y[:body] }.flatten.each do |update|
|
17
|
+
if update[:delete]
|
18
|
+
mutation_for(index).deletes << update[:delete][:_id]
|
19
|
+
elsif update[:index]
|
20
|
+
mutation_for(index).indexes << update[:index]
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
# @param index [Chewy::Index] return only index requests to the specified {Chewy::Index} index.
|
26
|
+
# @return [Hash] the index changes captured by the mock.
|
27
|
+
def indexes_for(index = nil)
|
28
|
+
if index
|
29
|
+
mutation_for(index).indexes
|
30
|
+
else
|
31
|
+
@mutations.transform_values(&:indexes)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
alias_method :indexes, :indexes_for
|
35
|
+
|
36
|
+
# @param index [Chewy::Index] return only delete requests to the specified {Chewy::Index} index.
|
37
|
+
# @return [Hash] the index deletes captured by the mock.
|
38
|
+
def deletes_for(index = nil)
|
39
|
+
if index
|
40
|
+
mutation_for(index).deletes
|
41
|
+
else
|
42
|
+
@mutations.transform_values(&:deletes)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
alias_method :deletes, :deletes_for
|
46
|
+
|
47
|
+
# Check to see if a given object has been indexed.
|
48
|
+
# @param obj [#id] obj the object to look for.
|
49
|
+
# @param index [Chewy::Index] what index the object should be indexed in.
|
50
|
+
# @return [true, false] if the object was indexed.
|
51
|
+
def indexed?(obj, index)
|
52
|
+
indexes_for(index).map { |i| i[:_id] }.include? obj.id
|
53
|
+
end
|
54
|
+
|
55
|
+
# Check to see if a given object has been deleted.
|
56
|
+
# @param obj [#id] obj the object to look for.
|
57
|
+
# @param index [Chewy::Index] what index the object should have been deleted from.
|
58
|
+
# @return [true, false] if the object was deleted.
|
59
|
+
def deleted?(obj, index)
|
60
|
+
deletes_for(index).include? obj.id
|
61
|
+
end
|
62
|
+
|
63
|
+
# @return [Array<Chewy::Index>] a list of indexes changed.
|
64
|
+
def updated_indexes
|
65
|
+
@mutations.keys
|
66
|
+
end
|
67
|
+
|
68
|
+
private
|
69
|
+
|
70
|
+
# Get the mutation object for a given index.
|
71
|
+
# @param index [Chewy::Index] the index to fetch.
|
72
|
+
# @return [#indexes, #deletes] an object with a list of indexes and a list of deletes.
|
73
|
+
def mutation_for(index)
|
74
|
+
@mutations[index] ||= OpenStruct.new(indexes: [], deletes: [])
|
75
|
+
end
|
76
|
+
end
|
@@ -0,0 +1 @@
|
|
1
|
+
require 'chewy/minitest/helpers'
|
@@ -0,0 +1,62 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Chewy
|
4
|
+
# `Chewy::MultiSearch` provides an interface for executing multiple
|
5
|
+
# queries via the Elasticsearch Multi Search API. When a MultiSearch
|
6
|
+
# is performed it wraps the responses from Elasticsearch and assigns
|
7
|
+
# them to the appropriate queries.
|
8
|
+
class MultiSearch
|
9
|
+
attr_reader :queries
|
10
|
+
|
11
|
+
# Instantiate a new MultiSearch instance.
|
12
|
+
#
|
13
|
+
# @param queries [Array<Chewy::Search::Request>]
|
14
|
+
# @option [Elasticsearch::Transport::Client] :client (Chewy.client)
|
15
|
+
# The Elasticsearch client that should be used for issuing requests.
|
16
|
+
def initialize(queries, client: Chewy.client)
|
17
|
+
@client = client
|
18
|
+
@queries = Array(queries)
|
19
|
+
end
|
20
|
+
|
21
|
+
# Adds a query to be performed by the MultiSearch
|
22
|
+
#
|
23
|
+
# @param query [Chewy::Search::Request]
|
24
|
+
def add_query(query)
|
25
|
+
@queries << query
|
26
|
+
end
|
27
|
+
|
28
|
+
# Performs any unperformed queries and returns the responses for all queries.
|
29
|
+
#
|
30
|
+
# @return [Array<Chewy::Search::Response>]
|
31
|
+
def responses
|
32
|
+
perform
|
33
|
+
queries.map(&:response)
|
34
|
+
end
|
35
|
+
|
36
|
+
# Performs any unperformed queries.
|
37
|
+
def perform
|
38
|
+
unperformed_queries = queries.reject(&:performed?)
|
39
|
+
return if unperformed_queries.empty?
|
40
|
+
|
41
|
+
responses = msearch(unperformed_queries)['responses']
|
42
|
+
unperformed_queries.zip(responses).map { |query, response| query.response = response }
|
43
|
+
end
|
44
|
+
|
45
|
+
private
|
46
|
+
|
47
|
+
attr_reader :client
|
48
|
+
|
49
|
+
def msearch(queries_to_search)
|
50
|
+
body = queries_to_search.flat_map do |query|
|
51
|
+
rendered = query.render
|
52
|
+
[rendered.except(:body), rendered[:body]]
|
53
|
+
end
|
54
|
+
|
55
|
+
client.msearch(body: body)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
def self.msearch(queries)
|
60
|
+
Chewy::MultiSearch.new(queries)
|
61
|
+
end
|
62
|
+
end
|
data/lib/chewy/railtie.rb
CHANGED
@@ -14,23 +14,16 @@ module Chewy
|
|
14
14
|
if Rails.application.config.respond_to?(:assets) && env['PATH_INFO'].start_with?(Rails.application.config.assets.prefix)
|
15
15
|
@app.call(env)
|
16
16
|
else
|
17
|
+
if Chewy.logger && @request_strategy != Chewy.request_strategy
|
18
|
+
Chewy.logger.info("Chewy request strategy is `#{Chewy.request_strategy}`")
|
19
|
+
end
|
20
|
+
@request_strategy = Chewy.request_strategy
|
17
21
|
Chewy.strategy(Chewy.request_strategy) { @app.call(env) }
|
18
22
|
end
|
19
23
|
end
|
20
24
|
end
|
21
25
|
|
22
26
|
module MigrationStrategy
|
23
|
-
extend ActiveSupport::Concern
|
24
|
-
included do
|
25
|
-
alias_method_chain :migrate, :chewy
|
26
|
-
end
|
27
|
-
|
28
|
-
def migrate_with_chewy(*args)
|
29
|
-
Chewy.strategy(:bypass) { migrate_without_chewy(*args) }
|
30
|
-
end
|
31
|
-
end
|
32
|
-
|
33
|
-
module Rails5MigrationStrategy
|
34
27
|
def migrate(*args)
|
35
28
|
Chewy.strategy(:bypass) { super }
|
36
29
|
end
|
@@ -41,38 +34,32 @@ module Chewy
|
|
41
34
|
end
|
42
35
|
|
43
36
|
console do |app|
|
44
|
-
Chewy.logger = ActiveRecord::Base.logger if defined?(ActiveRecord)
|
45
|
-
|
46
37
|
if app.sandbox?
|
47
38
|
Chewy.strategy(:bypass)
|
48
39
|
else
|
49
|
-
Chewy.strategy(
|
40
|
+
Chewy.strategy(Chewy.console_strategy)
|
50
41
|
end
|
42
|
+
puts "Chewy console strategy is `#{Chewy.strategy.current.name}`"
|
51
43
|
end
|
52
44
|
|
53
45
|
initializer 'chewy.logger', after: 'active_record.logger' do
|
54
|
-
ActiveSupport.on_load(:active_record)
|
46
|
+
ActiveSupport.on_load(:active_record) { Chewy.logger ||= ActiveRecord::Base.logger }
|
55
47
|
end
|
56
48
|
|
57
49
|
initializer 'chewy.migration_strategy' do
|
58
50
|
ActiveSupport.on_load(:active_record) do
|
59
|
-
|
60
|
-
|
61
|
-
ActiveRecord::Migrator.prepend(Rails5MigrationStrategy) if defined? ActiveRecord::Migrator
|
62
|
-
else
|
63
|
-
ActiveRecord::Migration.send(:include, MigrationStrategy)
|
64
|
-
ActiveRecord::Migrator.send(:include, MigrationStrategy) if defined? ActiveRecord::Migrator
|
65
|
-
end
|
51
|
+
ActiveRecord::Migration.prepend(MigrationStrategy)
|
52
|
+
ActiveRecord::Migrator.prepend(MigrationStrategy) if defined? ActiveRecord::Migrator
|
66
53
|
end
|
67
54
|
end
|
68
55
|
|
69
56
|
initializer 'chewy.request_strategy' do |app|
|
70
|
-
app.config.middleware.
|
57
|
+
app.config.middleware.insert_before(ActionDispatch::ShowExceptions, RequestStrategy)
|
71
58
|
end
|
72
59
|
|
73
|
-
initializer 'chewy.
|
60
|
+
initializer 'chewy.add_indices_path' do |_app|
|
74
61
|
Chewy::Railtie.all_engines.each do |engine|
|
75
|
-
engine.paths.add
|
62
|
+
engine.paths.add Chewy.configuration[:indices_path]
|
76
63
|
end
|
77
64
|
end
|
78
65
|
end
|
data/lib/chewy/rake_helper.rb
CHANGED
@@ -1,64 +1,362 @@
|
|
1
1
|
module Chewy
|
2
2
|
module RakeHelper
|
3
|
+
IMPORT_CALLBACK = lambda do |output, _name, start, finish, _id, payload|
|
4
|
+
duration = (finish - start).ceil
|
5
|
+
stats = payload.fetch(:import, {}).map { |key, count| "#{key} #{count}" }.join(', ')
|
6
|
+
output.puts " Imported #{payload[:index]} in #{human_duration(duration)}, stats: #{stats}"
|
7
|
+
payload[:errors]&.each do |action, errors|
|
8
|
+
output.puts " #{action.to_s.humanize} errors:"
|
9
|
+
errors.each do |error, documents|
|
10
|
+
output.puts " `#{error}`"
|
11
|
+
output.puts " on #{documents.count} documents: #{documents}"
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
JOURNAL_CALLBACK = lambda do |output, _, _, _, _, payload|
|
17
|
+
count = payload[:groups].values.map(&:size).sum
|
18
|
+
targets = payload[:groups].keys.sort_by(&:derivable_name)
|
19
|
+
output.puts " Applying journal to #{targets}, #{count} entries, stage #{payload[:stage]}"
|
20
|
+
end
|
21
|
+
|
22
|
+
DELETE_BY_QUERY_OPTIONS = %w[WAIT_FOR_COMPLETION REQUESTS_PER_SECOND SCROLL_SIZE].freeze
|
23
|
+
FALSE_VALUES = %w[0 f false off].freeze
|
24
|
+
|
3
25
|
class << self
|
26
|
+
# Performs zero-downtime reindexing of all documents for the specified indexes
|
27
|
+
#
|
28
|
+
# @example
|
29
|
+
# Chewy::RakeHelper.reset # resets everything
|
30
|
+
# Chewy::RakeHelper.reset(only: 'cities') # resets only CitiesIndex
|
31
|
+
# Chewy::RakeHelper.reset(only: ['cities', CountriesIndex]) # resets CitiesIndex and CountriesIndex
|
32
|
+
# Chewy::RakeHelper.reset(except: CitiesIndex) # resets everything, but CitiesIndex
|
33
|
+
# Chewy::RakeHelper.reset(only: ['cities', 'countries'], except: CitiesIndex) # resets only CountriesIndex
|
34
|
+
#
|
35
|
+
# @param only [Array<Chewy::Index, String>, Chewy::Index, String] index or indexes to reset; if nothing is passed - uses all the indexes defined in the app
|
36
|
+
# @param except [Array<Chewy::Index, String>, Chewy::Index, String] index or indexes to exclude from processing
|
37
|
+
# @param parallel [true, Integer, Hash] any acceptable parallel options for import
|
38
|
+
# @param output [IO] output io for logging
|
39
|
+
# @return [Array<Chewy::Index>] indexes that were reset
|
40
|
+
def reset(only: nil, except: nil, parallel: nil, output: $stdout)
|
41
|
+
warn_missing_index(output)
|
42
|
+
|
43
|
+
subscribed_task_stats(output) do
|
44
|
+
indexes_from(only: only, except: except).each do |index|
|
45
|
+
reset_one(index, output, parallel: parallel)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
# Performs zero-downtime reindexing of all documents for the specified
|
51
|
+
# indexes only if a particular index specification was changed.
|
52
|
+
#
|
53
|
+
# @example
|
54
|
+
# Chewy::RakeHelper.upgrade # resets everything
|
55
|
+
# Chewy::RakeHelper.upgrade(only: 'cities') # resets only CitiesIndex
|
56
|
+
# Chewy::RakeHelper.upgrade(only: ['cities', CountriesIndex]) # resets CitiesIndex and CountriesIndex
|
57
|
+
# Chewy::RakeHelper.upgrade(except: CitiesIndex) # resets everything, but CitiesIndex
|
58
|
+
# Chewy::RakeHelper.upgrade(only: ['cities', 'countries'], except: CitiesIndex) # resets only CountriesIndex
|
59
|
+
#
|
60
|
+
# @param only [Array<Chewy::Index, String>, Chewy::Index, String] index or indexes to reset; if nothing is passed - uses all the indexes defined in the app
|
61
|
+
# @param except [Array<Chewy::Index, String>, Chewy::Index, String] index or indexes to exclude from processing
|
62
|
+
# @param parallel [true, Integer, Hash] any acceptable parallel options for import
|
63
|
+
# @param output [IO] output io for logging
|
64
|
+
# @return [Array<Chewy::Index>] indexes that were actually reset
|
65
|
+
def upgrade(only: nil, except: nil, parallel: nil, output: $stdout)
|
66
|
+
warn_missing_index(output)
|
4
67
|
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
68
|
+
subscribed_task_stats(output) do
|
69
|
+
indexes = indexes_from(only: only, except: except)
|
70
|
+
|
71
|
+
changed_indexes = indexes.select do |index|
|
72
|
+
index.specification.changed?
|
73
|
+
end
|
74
|
+
|
75
|
+
if changed_indexes.present?
|
76
|
+
indexes.each do |index|
|
77
|
+
if changed_indexes.include?(index)
|
78
|
+
reset_one(index, output, parallel: parallel)
|
79
|
+
else
|
80
|
+
output.puts "Skipping #{index}, the specification didn't change"
|
81
|
+
end
|
14
82
|
end
|
15
|
-
|
83
|
+
else
|
84
|
+
output.puts 'No index specification was changed'
|
85
|
+
end
|
86
|
+
|
87
|
+
changed_indexes
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
# Performs full update for each passed type if the corresponding index exists.
|
92
|
+
#
|
93
|
+
# @example
|
94
|
+
# Chewy::RakeHelper.update # updates everything
|
95
|
+
# Chewy::RakeHelper.update(only: 'places') # updates only PlacesIndex
|
96
|
+
# Chewy::RakeHelper.update(except: PlacesIndex) # updates everything, but PlacesIndex
|
97
|
+
#
|
98
|
+
# @param only [Array<Chewy::Index, String>, Chewy::Index, String] indexes to update; if nothing is passed - uses all the indexes defined in the app
|
99
|
+
# @param except [Array<Chewy::Index, String>, Chewy::Index, String] indexes to exclude from processing
|
100
|
+
# @param parallel [true, Integer, Hash] any acceptable parallel options for import
|
101
|
+
# @param output [IO] output io for logging
|
102
|
+
# @return [Array<Chewy::Index>] indexes that were actually updated
|
103
|
+
def update(only: nil, except: nil, parallel: nil, output: $stdout)
|
104
|
+
subscribed_task_stats(output) do
|
105
|
+
indexes_from(only: only, except: except).each_with_object([]) do |index, updated_indexes|
|
106
|
+
if index.exists?
|
107
|
+
output.puts "Updating #{index}"
|
108
|
+
index.import(parallel: parallel)
|
109
|
+
updated_indexes.push(index)
|
110
|
+
else
|
111
|
+
output.puts "Skipping #{index}, it does not exists (use rake chewy:reset[#{index.derivable_name}] to create and update it)"
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
# Performs synchronization for each passed index if it exists.
|
118
|
+
#
|
119
|
+
# @example
|
120
|
+
# Chewy::RakeHelper.sync # synchronizes everything
|
121
|
+
# Chewy::RakeHelper.sync(only: 'places') # synchronizes only PlacesIndex
|
122
|
+
# Chewy::RakeHelper.sync(except: PlacesIndex) # synchronizes everything, but PlacesIndex
|
123
|
+
#
|
124
|
+
# @param only [Array<Chewy::Index, String>, Chewy::Index, String] indexes to synchronize; if nothing is passed - uses all the indexes defined in the app
|
125
|
+
# @param except [Array<Chewy::Index, String>, Chewy::Index, String] indexes to exclude from processing
|
126
|
+
# @param parallel [true, Integer, Hash] any acceptable parallel options for sync
|
127
|
+
# @param output [IO] output io for logging
|
128
|
+
# @return [Array<Chewy::Index>] indexes that were actually updated
|
129
|
+
def sync(only: nil, except: nil, parallel: nil, output: $stdout)
|
130
|
+
subscribed_task_stats(output) do
|
131
|
+
indexes_from(only: only, except: except).each_with_object([]) do |index, synced_indexes|
|
132
|
+
output.puts "Synchronizing #{index}"
|
133
|
+
output.puts " #{index} doesn't support outdated synchronization" unless index.supports_outdated_sync?
|
134
|
+
time = Time.now
|
135
|
+
sync_result = index.sync(parallel: parallel)
|
136
|
+
if !sync_result
|
137
|
+
output.puts " Something went wrong with the #{index} synchronization"
|
138
|
+
elsif (sync_result[:count]).positive?
|
139
|
+
output.puts " Missing documents: #{sync_result[:missing]}" if sync_result[:missing].present?
|
140
|
+
output.puts " Outdated documents: #{sync_result[:outdated]}" if sync_result[:outdated].present?
|
141
|
+
synced_indexes.push(index)
|
142
|
+
else
|
143
|
+
output.puts " Skipping #{index}, up to date"
|
144
|
+
end
|
145
|
+
output.puts " Took #{human_duration(Time.now - time)}"
|
146
|
+
end
|
147
|
+
end
|
148
|
+
end
|
149
|
+
|
150
|
+
# Applies changes that were done after the specified time for the
|
151
|
+
# specified indexes or all of them.
|
152
|
+
#
|
153
|
+
# @example
|
154
|
+
# Chewy::RakeHelper.journal_apply(time: 1.minute.ago) # applies entries created for the last minute
|
155
|
+
# Chewy::RakeHelper.journal_apply(time: 1.minute.ago, only: 'places') # applies only PlacesIndex entries created for the last minute
|
156
|
+
# Chewy::RakeHelper.journal_apply(time: 1.minute.ago, except: PlacesIndex) # applies everything, but PlacesIndex, entries created for the last minute
|
157
|
+
#
|
158
|
+
# @param time [Time, DateTime] use only journal entries created after this time
|
159
|
+
# @param only [Array<Chewy::Index, String>, Chewy::Index, String] indexes to synchronize; if nothing is passed - uses all the indexes defined in the app
|
160
|
+
# @param except [Array<Chewy::Index, String>, Chewy::Index, String] indexes to exclude from processing
|
161
|
+
# @param output [IO] output io for logging
|
162
|
+
# @return [Array<Chewy::Index>] indexes that were actually updated
|
163
|
+
def journal_apply(time: nil, only: nil, except: nil, output: $stdout)
|
164
|
+
raise ArgumentError, 'Please specify the time to start with' unless time
|
165
|
+
|
166
|
+
subscribed_task_stats(output) do
|
167
|
+
output.puts "Applying journal entries created after #{time}"
|
168
|
+
count = Chewy::Journal.new(journal_indexes_from(only: only, except: except)).apply(time)
|
169
|
+
output.puts 'No journal entries were created after the specified time' if count.zero?
|
170
|
+
end
|
171
|
+
end
|
172
|
+
|
173
|
+
# Removes journal records created before the specified timestamp for
|
174
|
+
# the specified indexes or all of them.
|
175
|
+
#
|
176
|
+
# @example
|
177
|
+
# Chewy::RakeHelper.journal_clean # cleans everything
|
178
|
+
# Chewy::RakeHelper.journal_clean(time: 1.minute.ago) # leaves only entries created for the last minute
|
179
|
+
# Chewy::RakeHelper.journal_clean(only: 'places') # cleans only PlacesIndex entries
|
180
|
+
# Chewy::RakeHelper.journal_clean(except: PlacesIndex) # cleans everything, but PlacesIndex entries
|
181
|
+
#
|
182
|
+
# @param time [Time, DateTime] clean all the journal entries created before this time
|
183
|
+
# @param only [Array<Chewy::Index, String>, Chewy::Index, String] indexes to synchronize; if nothing is passed - uses all the indexes defined in the app
|
184
|
+
# @param except [Array<Chewy::Index, String>, Chewy::Index, String] indexes to exclude from processing
|
185
|
+
# @param output [IO] output io for logging
|
186
|
+
# @return [Array<Chewy::Index>] indexes that were actually updated
|
187
|
+
def journal_clean(time: nil, only: nil, except: nil, delete_by_query_options: {}, output: $stdout)
|
188
|
+
subscribed_task_stats(output) do
|
189
|
+
output.puts "Cleaning journal entries created before #{time}" if time
|
190
|
+
response = Chewy::Journal.new(journal_indexes_from(only: only, except: except)).clean(time, delete_by_query_options: delete_by_query_options)
|
191
|
+
if response.key?('task')
|
192
|
+
output.puts "Task to cleanup the journal has been created, #{response['task']}"
|
193
|
+
else
|
194
|
+
count = response['deleted'] || response['_indices']['_all']['deleted']
|
195
|
+
output.puts "Cleaned up #{count} journal entries"
|
196
|
+
end
|
16
197
|
end
|
17
|
-
|
18
|
-
|
198
|
+
end
|
199
|
+
|
200
|
+
# Creates journal index.
|
201
|
+
#
|
202
|
+
# @example
|
203
|
+
# Chewy::RakeHelper.journal_create # creates journal
|
204
|
+
#
|
205
|
+
# @param output [IO] output io for logging
|
206
|
+
# @return Chewy::Index Returns instance of chewy index
|
207
|
+
def journal_create(output: $stdout)
|
208
|
+
subscribed_task_stats(output) do
|
209
|
+
Chewy::Stash::Journal.create!
|
210
|
+
end
|
211
|
+
end
|
212
|
+
|
213
|
+
# Eager loads and returns all the indexes defined in the application
|
214
|
+
# except Chewy::Stash::Specification and Chewy::Stash::Journal.
|
215
|
+
#
|
216
|
+
# @return [Array<Chewy::Index>] indexes found
|
217
|
+
def all_indexes
|
218
|
+
Chewy.eager_load!
|
219
|
+
Chewy::Index.descendants - [Chewy::Stash::Journal, Chewy::Stash::Specification]
|
220
|
+
end
|
221
|
+
|
222
|
+
# Reindex data from source index to destination index
|
223
|
+
#
|
224
|
+
# @example
|
225
|
+
# Chewy::RakeHelper.reindex(source: 'users_index', dest: 'cities_index') reindex data from 'users_index' index to 'cities_index'
|
226
|
+
#
|
227
|
+
# @param source [String], dest [String] indexes to reindex
|
228
|
+
def reindex(source:, dest:, output: $stdout)
|
229
|
+
subscribed_task_stats(output) do
|
230
|
+
output.puts "Source index is #{source}\nDestination index is #{dest}"
|
231
|
+
Chewy::Index.reindex(source: source, dest: dest)
|
232
|
+
output.puts "#{source} index successfully reindexed with #{dest} index data"
|
233
|
+
end
|
234
|
+
end
|
235
|
+
|
236
|
+
# Adds new fields to an existing data stream or index.
|
237
|
+
# Change the search settings of existing fields.
|
238
|
+
#
|
239
|
+
# @example
|
240
|
+
# Chewy::RakeHelper.update_mapping('cities', {properties: {new_field: {type: :text}}}) update 'cities' index with new_field of text type
|
241
|
+
#
|
242
|
+
# @param name [String], body_hash [Hash] index name and body hash to update
|
243
|
+
def update_mapping(name:, output: $stdout)
|
244
|
+
subscribed_task_stats(output) do
|
245
|
+
output.puts "Index name is #{name}"
|
246
|
+
normalize_index(name).update_mapping
|
247
|
+
output.puts "#{name} index successfully updated"
|
19
248
|
end
|
20
249
|
end
|
21
250
|
|
22
|
-
|
23
|
-
|
251
|
+
# Reads options that are required to run journal cleanup asynchronously from ENV hash
|
252
|
+
# @see https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-delete-by-query.html
|
253
|
+
#
|
254
|
+
# @example
|
255
|
+
# Chewy::RakeHelper.delete_by_query_options_from_env({'WAIT_FOR_COMPLETION' => 'false','REQUESTS_PER_SECOND' => '10','SCROLL_SIZE' => '5000'})
|
256
|
+
# # => { wait_for_completion: false, requests_per_second: 10.0, scroll_size: 5000 }
|
257
|
+
#
|
258
|
+
def delete_by_query_options_from_env(env)
|
259
|
+
env
|
260
|
+
.slice(*DELETE_BY_QUERY_OPTIONS)
|
261
|
+
.transform_keys { |k| k.downcase.to_sym }
|
262
|
+
.to_h do |key, value|
|
263
|
+
case key
|
264
|
+
when :wait_for_completion then [key, !FALSE_VALUES.include?(value.downcase)]
|
265
|
+
when :requests_per_second then [key, value.to_f]
|
266
|
+
when :scroll_size then [key, value.to_i]
|
267
|
+
end
|
268
|
+
end
|
269
|
+
end
|
24
270
|
|
25
|
-
|
26
|
-
|
27
|
-
|
271
|
+
def create_missing_indexes!(output: $stdout, env: ENV)
|
272
|
+
subscribed_task_stats(output) do
|
273
|
+
Chewy.eager_load!
|
274
|
+
all_indexes = Chewy::Index.descendants
|
275
|
+
all_indexes -= [Chewy::Stash::Journal] unless Chewy.configuration[:journal]
|
276
|
+
all_indexes.each do |index|
|
277
|
+
if index.exists?
|
278
|
+
output.puts "#{index.name} already exists, skipping" if env['VERBOSE']
|
279
|
+
next
|
280
|
+
end
|
281
|
+
|
282
|
+
index.create!
|
283
|
+
|
284
|
+
output.puts "#{index.name} index successfully created"
|
28
285
|
end
|
29
286
|
end
|
30
287
|
end
|
31
288
|
|
32
|
-
def
|
33
|
-
|
289
|
+
def normalize_indexes(*identifiers)
|
290
|
+
identifiers.flatten(1).map { |identifier| normalize_index(identifier) }
|
34
291
|
end
|
35
292
|
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
index.reset! (Time.now.to_f * 1000).round
|
293
|
+
def normalize_index(identifier)
|
294
|
+
return identifier if identifier.is_a?(Class) && identifier < Chewy::Index
|
295
|
+
|
296
|
+
"#{identifier.to_s.camelize}Index".constantize
|
41
297
|
end
|
42
298
|
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
299
|
+
def subscribed_task_stats(output = $stdout, &block)
|
300
|
+
start = Time.now
|
301
|
+
ActiveSupport::Notifications.subscribed(JOURNAL_CALLBACK.curry[output], 'apply_journal.chewy') do
|
302
|
+
ActiveSupport::Notifications.subscribed(IMPORT_CALLBACK.curry[output], 'import_objects.chewy', &block)
|
303
|
+
end
|
304
|
+
ensure
|
305
|
+
output.puts "Total: #{human_duration(Time.now - start)}"
|
47
306
|
end
|
48
307
|
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
if
|
53
|
-
|
308
|
+
private
|
309
|
+
|
310
|
+
def journal_indexes_from(only: nil, except: nil)
|
311
|
+
return if Array.wrap(only).empty? && Array.wrap(except).empty?
|
312
|
+
|
313
|
+
indexes_from(only: only, except: except)
|
314
|
+
end
|
315
|
+
|
316
|
+
def indexes_from(only: nil, except: nil)
|
317
|
+
indexes = if only.present?
|
318
|
+
normalize_indexes(Array.wrap(only))
|
54
319
|
else
|
55
|
-
|
320
|
+
all_indexes
|
56
321
|
end
|
322
|
+
|
323
|
+
indexes = if except.present?
|
324
|
+
indexes - normalize_indexes(Array.wrap(except))
|
325
|
+
else
|
326
|
+
indexes
|
327
|
+
end
|
328
|
+
|
329
|
+
indexes.sort_by(&:derivable_name)
|
57
330
|
end
|
58
331
|
|
59
|
-
def
|
60
|
-
|
61
|
-
|
332
|
+
def human_duration(seconds)
|
333
|
+
[[60, :s], [60, :m], [24, :h]].map do |amount, unit|
|
334
|
+
if seconds.positive?
|
335
|
+
seconds, n = seconds.divmod(amount)
|
336
|
+
"#{n.to_i}#{unit}"
|
337
|
+
end
|
338
|
+
end.compact.reverse.join(' ')
|
339
|
+
end
|
340
|
+
|
341
|
+
def reset_one(index, output, parallel: false)
|
342
|
+
output.puts "Resetting #{index}"
|
343
|
+
index.reset!((Time.now.to_f * 1000).round, parallel: parallel, apply_journal: journal_exists?)
|
344
|
+
end
|
345
|
+
|
346
|
+
def warn_missing_index(output)
|
347
|
+
return if journal_exists?
|
348
|
+
|
349
|
+
output.puts "############################################################\n" \
|
350
|
+
"WARN: You are risking to lose some changes during the reset.\n" \
|
351
|
+
" Please consider enabling journaling.\n" \
|
352
|
+
" See https://github.com/toptal/chewy#journaling\n" \
|
353
|
+
'############################################################'
|
354
|
+
end
|
355
|
+
|
356
|
+
def journal_exists?
|
357
|
+
@journal_exists = Chewy::Stash::Journal.exists? if @journal_exists.nil?
|
358
|
+
|
359
|
+
@journal_exists
|
62
360
|
end
|
63
361
|
end
|
64
362
|
end
|
data/lib/chewy/repository.rb
CHANGED
@@ -5,10 +5,10 @@ module Chewy
|
|
5
5
|
attr_reader :analyzers, :tokenizers, :filters, :char_filters
|
6
6
|
|
7
7
|
def self.delegated
|
8
|
-
public_instance_methods -
|
8
|
+
public_instance_methods - superclass.public_instance_methods - Singleton.public_instance_methods
|
9
9
|
end
|
10
10
|
|
11
|
-
def self.repository
|
11
|
+
def self.repository(name)
|
12
12
|
plural_name = name.to_s.pluralize
|
13
13
|
|
14
14
|
class_eval <<-METHOD, __FILE__, __LINE__ + 1
|
@@ -0,0 +1,12 @@
|
|
1
|
+
# Rspec helper to compare request and expected query
|
2
|
+
# To use it - add `require 'chewy/rspec/build_query'` to the `spec_helper.rb`
|
3
|
+
# Simple usage - just pass expected response as argument
|
4
|
+
# and then call needed query.
|
5
|
+
#
|
6
|
+
# expect { method1.method2...methodN }.to build_query(expected_query)
|
7
|
+
#
|
8
|
+
RSpec::Matchers.define :build_query do |expected_query = {}|
|
9
|
+
match do |request|
|
10
|
+
request.render == expected_query
|
11
|
+
end
|
12
|
+
end
|