chewy 5.1.0 → 7.2.7

data/lib/chewy/rake_helper.rb

@@ -3,14 +3,12 @@ module Chewy
     IMPORT_CALLBACK = lambda do |output, _name, start, finish, _id, payload|
       duration = (finish - start).ceil
       stats = payload.fetch(:import, {}).map { |key, count| "#{key} #{count}" }.join(', ')
-      output.puts "  Imported #{payload[:type]} in #{human_duration(duration)}, stats: #{stats}"
-      if payload[:errors]
-        payload[:errors].each do |action, errors|
-          output.puts "    #{action.to_s.humanize} errors:"
-          errors.each do |error, documents|
-            output.puts "      `#{error}`"
-            output.puts "        on #{documents.count} documents: #{documents}"
-          end
+      output.puts "  Imported #{payload[:index]} in #{human_duration(duration)}, stats: #{stats}"
+      payload[:errors]&.each do |action, errors|
+        output.puts "    #{action.to_s.humanize} errors:"
+        errors.each do |error, documents|
+          output.puts "      `#{error}`"
+          output.puts "        on #{documents.count} documents: #{documents}"
         end
       end
     end
@@ -21,6 +19,9 @@ module Chewy
       output.puts "  Applying journal to #{targets}, #{count} entries, stage #{payload[:stage]}"
     end
 
+    DELETE_BY_QUERY_OPTIONS = %w[WAIT_FOR_COMPLETION REQUESTS_PER_SECOND SCROLL_SIZE].freeze
+    FALSE_VALUES = %w[0 f false off].freeze
+
     class << self
       # Performs zero-downtime reindexing of all documents for the specified indexes
       #
@@ -36,7 +37,9 @@ module Chewy
       # @param parallel [true, Integer, Hash] any acceptable parallel options for import
       # @param output [IO] output io for logging
       # @return [Array<Chewy::Index>] indexes that were reset
-      def reset(only: nil, except: nil, parallel: nil, output: STDOUT)
+      def reset(only: nil, except: nil, parallel: nil, output: $stdout)
+        warn_missing_index(output)
+
         subscribed_task_stats(output) do
           indexes_from(only: only, except: except).each do |index|
             reset_one(index, output, parallel: parallel)
@@ -59,7 +62,9 @@ module Chewy
       # @param parallel [true, Integer, Hash] any acceptable parallel options for import
       # @param output [IO] output io for logging
       # @return [Array<Chewy::Index>] indexes that were actually reset
-      def upgrade(only: nil, except: nil, parallel: nil, output: STDOUT)
+      def upgrade(only: nil, except: nil, parallel: nil, output: $stdout)
+        warn_missing_index(output)
+
         subscribed_task_stats(output) do
           indexes = indexes_from(only: only, except: except)
 
@@ -87,23 +92,21 @@ module Chewy
       #
       # @example
       #   Chewy::RakeHelper.update # updates everything
-      #   Chewy::RakeHelper.update(only: 'places') # updates only PlacesIndex::City and PlacesIndex::Country
-      #   Chewy::RakeHelper.update(only: 'places#city') # updates PlacesIndex::City only
-      #   Chewy::RakeHelper.update(except: PlacesIndex::Country) # updates everything, but PlacesIndex::Country
-      #   Chewy::RakeHelper.update(only: 'places', except: 'places#country') # updates PlacesIndex::City only
+      #   Chewy::RakeHelper.update(only: 'places') # updates only PlacesIndex
+      #   Chewy::RakeHelper.update(except: PlacesIndex) # updates everything, but PlacesIndex
       #
-      # @param only [Array<Chewy::Index, Chewy::Type, String>, Chewy::Index, Chewy::Type, String] indexes or types to update; if nothing is passed - uses all the types defined in the app
-      # @param except [Array<Chewy::Index, Chewy::Type, String>, Chewy::Index, Chewy::Type, String] indexes or types to exclude from processing
+      # @param only [Array<Chewy::Index, String>, Chewy::Index, String] indexes to update; if nothing is passed - uses all the indexes defined in the app
+      # @param except [Array<Chewy::Index, String>, Chewy::Index, String] indexes to exclude from processing
       # @param parallel [true, Integer, Hash] any acceptable parallel options for import
       # @param output [IO] output io for logging
-      # @return [Array<Chewy::Type>] types that were actually updated
-      def update(only: nil, except: nil, parallel: nil, output: STDOUT)
+      # @return [Array<Chewy::Index>] indexes that were actually updated
+      def update(only: nil, except: nil, parallel: nil, output: $stdout)
         subscribed_task_stats(output) do
-          types_from(only: only, except: except).group_by(&:index).each_with_object([]) do |(index, types), update_types|
+          indexes_from(only: only, except: except).each_with_object([]) do |index, updated_indexes|
             if index.exists?
               output.puts "Updating #{index}"
-              types.each { |type| type.import(parallel: parallel) }
-              update_types.concat(types)
+              index.import(parallel: parallel)
+              updated_indexes.push(index)
             else
               output.puts "Skipping #{index}, it does not exists (use rake chewy:reset[#{index.derivable_name}] to create and update it)"
             end
@@ -115,31 +118,29 @@ module Chewy
       #
       # @example
       #   Chewy::RakeHelper.sync # synchronizes everything
-      #   Chewy::RakeHelper.sync(only: 'places') # synchronizes only PlacesIndex::City and PlacesIndex::Country
-      #   Chewy::RakeHelper.sync(only: 'places#city') # synchronizes PlacesIndex::City only
-      #   Chewy::RakeHelper.sync(except: PlacesIndex::Country) # synchronizes everything, but PlacesIndex::Country
-      #   Chewy::RakeHelper.sync(only: 'places', except: 'places#country') # synchronizes PlacesIndex::City only
+      #   Chewy::RakeHelper.sync(only: 'places') # synchronizes only PlacesIndex
+      #   Chewy::RakeHelper.sync(except: PlacesIndex) # synchronizes everything, but PlacesIndex
       #
-      # @param only [Array<Chewy::Index, Chewy::Type, String>, Chewy::Index, Chewy::Type, String] indexes or types to synchronize; if nothing is passed - uses all the types defined in the app
-      # @param except [Array<Chewy::Index, Chewy::Type, String>, Chewy::Index, Chewy::Type, String] indexes or types to exclude from processing
+      # @param only [Array<Chewy::Index, String>, Chewy::Index, String] indexes to synchronize; if nothing is passed - uses all the indexes defined in the app
+      # @param except [Array<Chewy::Index, String>, Chewy::Index, String] indexes to exclude from processing
       # @param parallel [true, Integer, Hash] any acceptable parallel options for sync
       # @param output [IO] output io for logging
-      # @return [Array<Chewy::Type>] types that were actually updated
-      def sync(only: nil, except: nil, parallel: nil, output: STDOUT)
+      # @return [Array<Chewy::Index>] indexes that were actually updated
+      def sync(only: nil, except: nil, parallel: nil, output: $stdout)
         subscribed_task_stats(output) do
-          types_from(only: only, except: except).each_with_object([]) do |type, synced_types|
-            output.puts "Synchronizing #{type}"
-            output.puts "  #{type} doesn't support outdated synchronization" unless type.supports_outdated_sync?
+          indexes_from(only: only, except: except).each_with_object([]) do |index, synced_indexes|
+            output.puts "Synchronizing #{index}"
+            output.puts "  #{index} doesn't support outdated synchronization" unless index.supports_outdated_sync?
             time = Time.now
-            sync_result = type.sync(parallel: parallel)
+            sync_result = index.sync(parallel: parallel)
             if !sync_result
-              output.puts "  Something went wrong with the #{type} synchronization"
-            elsif sync_result[:count] > 0
+              output.puts "  Something went wrong with the #{index} synchronization"
+            elsif (sync_result[:count]).positive?
               output.puts "  Missing documents: #{sync_result[:missing]}" if sync_result[:missing].present?
               output.puts "  Outdated documents: #{sync_result[:outdated]}" if sync_result[:outdated].present?
-              synced_types.push(type)
+              synced_indexes.push(index)
             else
-              output.puts "  Skipping #{type}, up to date"
+              output.puts "  Skipping #{index}, up to date"
             end
             output.puts "  Took #{human_duration(Time.now - time)}"
           end
@@ -147,51 +148,52 @@ module Chewy
       end
 
       # Applies changes that were done after the specified time for the
-      # specified indexes/types or all of them.
+      # specified indexes or all of them.
       #
       # @example
       #   Chewy::RakeHelper.journal_apply(time: 1.minute.ago) # applies entries created for the last minute
-      #   Chewy::RakeHelper.journal_apply(time: 1.minute.ago, only: 'places') # applies only PlacesIndex::City and PlacesIndex::Country entries reated for the last minute
-      #   Chewy::RakeHelper.journal_apply(time: 1.minute.ago, only: 'places#city') # applies PlacesIndex::City entries reated for the last minute only
-      #   Chewy::RakeHelper.journal_apply(time: 1.minute.ago, except: PlacesIndex::Country) # applies everything, but PlacesIndex::Country entries reated for the last minute
-      #   Chewy::RakeHelper.journal_apply(time: 1.minute.ago, only: 'places', except: 'places#country') # applies PlacesIndex::City entries reated for the last minute only
+      #   Chewy::RakeHelper.journal_apply(time: 1.minute.ago, only: 'places') # applies only PlacesIndex entries created for the last minute
+      #   Chewy::RakeHelper.journal_apply(time: 1.minute.ago, except: PlacesIndex) # applies everything, but PlacesIndex, entries created for the last minute
       #
       # @param time [Time, DateTime] use only journal entries created after this time
-      # @param only [Array<Chewy::Index, Chewy::Type, String>, Chewy::Index, Chewy::Type, String] indexes or types to synchronize; if nothing is passed - uses all the types defined in the app
-      # @param except [Array<Chewy::Index, Chewy::Type, String>, Chewy::Index, Chewy::Type, String] indexes or types to exclude from processing
+      # @param only [Array<Chewy::Index, String>, Chewy::Index, String] indexes to synchronize; if nothing is passed - uses all the indexes defined in the app
+      # @param except [Array<Chewy::Index, String>, Chewy::Index, String] indexes to exclude from processing
       # @param output [IO] output io for logging
-      # @return [Array<Chewy::Type>] types that were actually updated
-      def journal_apply(time: nil, only: nil, except: nil, output: STDOUT)
+      # @return [Array<Chewy::Index>] indexes that were actually updated
+      def journal_apply(time: nil, only: nil, except: nil, output: $stdout)
         raise ArgumentError, 'Please specify the time to start with' unless time
+
         subscribed_task_stats(output) do
           output.puts "Applying journal entries created after #{time}"
-          count = Chewy::Journal.new(types_from(only: only, except: except)).apply(time)
+          count = Chewy::Journal.new(journal_indexes_from(only: only, except: except)).apply(time)
           output.puts 'No journal entries were created after the specified time' if count.zero?
         end
       end
 
       # Removes journal records created before the specified timestamp for
-      # the specified indexes/types or all of them.
+      # the specified indexes or all of them.
       #
       # @example
       #   Chewy::RakeHelper.journal_clean # cleans everything
       #   Chewy::RakeHelper.journal_clean(time: 1.minute.ago) # leaves only entries created for the last minute
-      #   Chewy::RakeHelper.journal_clean(only: 'places') # cleans only PlacesIndex::City and PlacesIndex::Country entries
-      #   Chewy::RakeHelper.journal_clean(only: 'places#city') # cleans PlacesIndex::City entries only
-      #   Chewy::RakeHelper.journal_clean(except: PlacesIndex::Country) # cleans everything, but PlacesIndex::Country entries
-      #   Chewy::RakeHelper.journal_clean(only: 'places', except: 'places#country') # cleans PlacesIndex::City entries only
+      #   Chewy::RakeHelper.journal_clean(only: 'places') # cleans only PlacesIndex entries
+      #   Chewy::RakeHelper.journal_clean(except: PlacesIndex) # cleans everything, but PlacesIndex entries
       #
       # @param time [Time, DateTime] clean all the journal entries created before this time
-      # @param only [Array<Chewy::Index, Chewy::Type, String>, Chewy::Index, Chewy::Type, String] indexes or types to synchronize; if nothing is passed - uses all the types defined in the app
-      # @param except [Array<Chewy::Index, Chewy::Type, String>, Chewy::Index, Chewy::Type, String] indexes or types to exclude from processing
+      # @param only [Array<Chewy::Index, String>, Chewy::Index, String] indexes to synchronize; if nothing is passed - uses all the indexes defined in the app
+      # @param except [Array<Chewy::Index, String>, Chewy::Index, String] indexes to exclude from processing
       # @param output [IO] output io for logging
-      # @return [Array<Chewy::Type>] types that were actually updated
-      def journal_clean(time: nil, only: nil, except: nil, output: STDOUT)
+      # @return [Array<Chewy::Index>] indexes that were actually updated
+      def journal_clean(time: nil, only: nil, except: nil, delete_by_query_options: {}, output: $stdout)
         subscribed_task_stats(output) do
           output.puts "Cleaning journal entries created before #{time}" if time
-          response = Chewy::Journal.new(types_from(only: only, except: except)).clean(time)
-          count = response['deleted'] || response['_indices']['_all']['deleted']
-          output.puts "Cleaned up #{count} journal entries"
+          response = Chewy::Journal.new(journal_indexes_from(only: only, except: except)).clean(time, delete_by_query_options: delete_by_query_options)
+          if response.key?('task')
+            output.puts "Task to cleanup the journal has been created, #{response['task']}"
+          else
+            count = response['deleted'] || response['_indices']['_all']['deleted']
+            output.puts "Cleaned up #{count} journal entries"
+          end
         end
       end
 
@@ -204,47 +206,81 @@ module Chewy
         Chewy::Index.descendants - [Chewy::Stash::Journal, Chewy::Stash::Specification]
       end
 
+      # Reindex data from source index to destination index
+      #
+      # @example
+      #   Chewy::RakeHelper.reindex(source: 'users_index', dest: 'cities_index') reindex data from 'users_index' index to 'cities_index'
+      #
+      # @param source [String], dest [String] indexes to reindex
+      def reindex(source:, dest:, output: $stdout)
+        subscribed_task_stats(output) do
+          output.puts "Source index is #{source}\nDestination index is #{dest}"
+          Chewy::Index.reindex(source: source, dest: dest)
+          output.puts "#{source} index successfully reindexed with #{dest} index data"
+        end
+      end
+
+      # Adds new fields to an existing data stream or index.
+      # Change the search settings of existing fields.
+      #
+      # @example
+      #   Chewy::RakeHelper.update_mapping('cities', {properties: {new_field: {type: :text}}}) update 'cities' index with new_field of text type
+      #
+      # @param name [String], body_hash [Hash] index name and body hash to update
+      def update_mapping(name:, output: $stdout)
+        subscribed_task_stats(output) do
+          output.puts "Index name is #{name}"
+          normalize_index(name).update_mapping
+          output.puts "#{name} index successfully updated"
+        end
+      end
+
+      # Reads options that are required to run journal cleanup asynchronously from ENV hash
+      # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-delete-by-query.html
+      #
+      # @example
+      #   Chewy::RakeHelper.delete_by_query_options_from_env({'WAIT_FOR_COMPLETION' => 'false','REQUESTS_PER_SECOND' => '10','SCROLL_SIZE' => '5000'})
+      #   # => { wait_for_completion: false, requests_per_second: 10.0, scroll_size: 5000 }
+      #
+      def delete_by_query_options_from_env(env)
+        env
+          .slice(*DELETE_BY_QUERY_OPTIONS)
+          .transform_keys { |k| k.downcase.to_sym }
+          .to_h do |key, value|
+            case key
+            when :wait_for_completion then [key, !FALSE_VALUES.include?(value.downcase)]
+            when :requests_per_second then [key, value.to_f]
+            when :scroll_size then [key, value.to_i]
+            end
+          end
+      end
+
       def normalize_indexes(*identifiers)
         identifiers.flatten(1).map { |identifier| normalize_index(identifier) }
       end
 
       def normalize_index(identifier)
         return identifier if identifier.is_a?(Class) && identifier < Chewy::Index
-        "#{identifier.to_s.gsub(/identifier\z/i, '').camelize}Index".constantize
+
+        "#{identifier.to_s.camelize}Index".constantize
       end
 
-      def subscribed_task_stats(output = STDOUT)
+      def subscribed_task_stats(output = $stdout, &block)
         start = Time.now
         ActiveSupport::Notifications.subscribed(JOURNAL_CALLBACK.curry[output], 'apply_journal.chewy') do
-          ActiveSupport::Notifications.subscribed(IMPORT_CALLBACK.curry[output], 'import_objects.chewy') do
-            yield
-          end
+          ActiveSupport::Notifications.subscribed(IMPORT_CALLBACK.curry[output], 'import_objects.chewy', &block)
         end
         output.puts "Total: #{human_duration(Time.now - start)}"
       end
 
-      def reset_index(*indexes)
-        ActiveSupport::Deprecation.warn '`Chewy::RakeHelper.reset_index` is deprecated and will be removed soon, use `Chewy::RakeHelper.reset` instead'
-        reset(only: indexes)
-      end
-
-      def reset_all(*except)
-        ActiveSupport::Deprecation.warn '`Chewy::RakeHelper.reset_all` is deprecated and will be removed soon, use `Chewy::RakeHelper.reset` instead'
-        reset(except: except)
-      end
+    private
 
-      def update_index(*indexes)
-        ActiveSupport::Deprecation.warn '`Chewy::RakeHelper.update_index` is deprecated and will be removed soon, use `Chewy::RakeHelper.update` instead'
-        update(only: indexes)
-      end
+      def journal_indexes_from(only: nil, except: nil)
+        return if Array.wrap(only).empty? && Array.wrap(except).empty?
 
-      def update_all(*except)
-        ActiveSupport::Deprecation.warn '`Chewy::RakeHelper.update_all` is deprecated and will be removed soon, use `Chewy::RakeHelper.update` instead'
-        update(except: except)
+        indexes_from(only: only, except: except)
       end
 
-    private
-
       def indexes_from(only: nil, except: nil)
         indexes = if only.present?
           normalize_indexes(Array.wrap(only))
@@ -261,33 +297,9 @@ module Chewy
         indexes.sort_by(&:derivable_name)
       end
 
-      def types_from(only: nil, except: nil)
-        types = if only.present?
-          normalize_types(Array.wrap(only))
-        else
-          all_indexes.flat_map(&:types)
-        end
-
-        types = if except.present?
-          types - normalize_types(Array.wrap(except))
-        else
-          types
-        end
-
-        types.sort_by(&:derivable_name)
-      end
-
-      def normalize_types(*identifiers)
-        identifiers.flatten(1).flat_map { |identifier| normalize_type(identifier) }
-      end
-
-      def normalize_type(identifier)
-        Chewy.derive_types(identifier)
-      end
-
       def human_duration(seconds)
         [[60, :s], [60, :m], [24, :h]].map do |amount, unit|
-          if seconds > 0
+          if seconds.positive?
             seconds, n = seconds.divmod(amount)
             "#{n.to_i}#{unit}"
           end
@@ -296,7 +308,23 @@ module Chewy
 
       def reset_one(index, output, parallel: false)
         output.puts "Resetting #{index}"
-        index.reset!((Time.now.to_f * 1000).round, parallel: parallel)
+        index.reset!((Time.now.to_f * 1000).round, parallel: parallel, apply_journal: journal_exists?)
+      end
+
+      def warn_missing_index(output)
+        return if journal_exists?
+
+        output.puts "############################################################\n" \
+                    "WARN: You are risking to lose some changes during the reset.\n" \
+                    "      Please consider enabling journaling.\n" \
+                    "      See https://github.com/toptal/chewy#journaling\n" \
+                    '############################################################'
+      end
+
+      def journal_exists?
+        @journal_exists = Chewy::Stash::Journal.exists? if @journal_exists.nil?
+
+        @journal_exists
       end
     end
   end

data/lib/chewy/rspec/build_query.rb

@@ -0,0 +1,12 @@
+# Rspec helper to compare request and expected query
+# To use it - add `require 'chewy/rspec/build_query'` to the `spec_helper.rb`
+# Simple usage - just pass expected response as argument
+# and then call needed query.
+#
+#   expect { method1.method2...methodN }.to build_query(expected_query)
+#
+RSpec::Matchers.define :build_query do |expected_query = {}|
+  match do |request|
+    request.render == expected_query
+  end
+end

data/lib/chewy/rspec/helpers.rb

@@ -0,0 +1,55 @@
+module Chewy
+  module Rspec
+    module Helpers
+      extend ActiveSupport::Concern
+      # Rspec helper to mock elasticsearch response
+      # To use it - add `require 'chewy/rspec'` to the `spec_helper.rb`
+      #
+      #   mock_elasticsearch_response(CitiesIndex, raw_response)
+      #   expect(CitiesIndex.query({}).hits).to eq(hits)
+      #
+      def mock_elasticsearch_response(index, raw_response)
+        mocked_request = Chewy::Search::Request.new(index)
+        allow(Chewy::Search::Request).to receive(:new).and_return(mocked_request)
+        allow(mocked_request).to receive(:perform).and_return(raw_response)
+      end
+
+      # Rspec helper to mock Elasticsearch response source
+      # To use it - add `require 'chewy/rspec'` to the `spec_helper.rb`
+      #
+      #  mock_elasticsearch_response_sources(CitiesIndex, sources)
+      #  expect(CitiesIndex.query({}).hits).to eq(hits)
+      #
+      def mock_elasticsearch_response_sources(index, hits)
+        raw_response = {
+          'took' => 4,
+          'timed_out' => false,
+          '_shards' => {
+            'total' => 1,
+            'successful' => 1,
+            'skipped' => 0,
+            'failed' => 0
+          },
+          'hits' => {
+            'total' => {
+              'value' => hits.count,
+              'relation' => 'eq'
+            },
+            'max_score' => 1.0,
+            'hits' => hits.each_with_index.map do |hit, i|
+              {
+                '_index' => index.index_name,
+                '_type' => '_doc',
+                '_id' => (i + 1).to_s,
+                '_score' => 3.14,
+                '_source' => hit
+              }
+            end
+          }
+        }
+
+        mock_elasticsearch_response(index, raw_response)
+      end
+    end
+  end
+end