chewy 0.10.1 → 7.3.2

data/lib/chewy/railtie.rb

@@ -14,7 +14,9 @@ module Chewy
         if Rails.application.config.respond_to?(:assets) && env['PATH_INFO'].start_with?(Rails.application.config.assets.prefix)
           @app.call(env)
         else
-          Chewy.logger.info("Chewy request strategy is `#{Chewy.request_strategy}`") if @request_strategy != Chewy.request_strategy
+          if Chewy.logger && @request_strategy != Chewy.request_strategy
+            Chewy.logger.info("Chewy request strategy is `#{Chewy.request_strategy}`")
+          end
           @request_strategy = Chewy.request_strategy
           Chewy.strategy(Chewy.request_strategy) { @app.call(env) }
         end
@@ -22,17 +24,6 @@ module Chewy
     end
 
     module MigrationStrategy
-      extend ActiveSupport::Concern
-      included do
-        alias_method_chain :migrate, :chewy
-      end
-
-      def migrate_with_chewy(*args)
-        Chewy.strategy(:bypass) { migrate_without_chewy(*args) }
-      end
-    end
-
-    module Rails5MigrationStrategy
       def migrate(*args)
         Chewy.strategy(:bypass) { super }
       end
@@ -43,14 +34,12 @@ module Chewy
     end
 
     console do |app|
-      Chewy.logger = ActiveRecord::Base.logger if defined?(ActiveRecord)
-
       if app.sandbox?
         Chewy.strategy(:bypass)
       else
-        Chewy.strategy(:urgent)
+        Chewy.strategy(Chewy.console_strategy)
       end
-      Chewy.logger.info("Chewy console strategy is `#{Chewy.strategy.current.name}`")
+      puts "Chewy console strategy is `#{Chewy.strategy.current.name}`"
     end
 
     initializer 'chewy.logger', after: 'active_record.logger' do
@@ -59,18 +48,13 @@ module Chewy
 
     initializer 'chewy.migration_strategy' do
       ActiveSupport.on_load(:active_record) do
-        if Rails::VERSION::MAJOR >= 5
-          ActiveRecord::Migration.prepend(Rails5MigrationStrategy)
-          ActiveRecord::Migrator.prepend(Rails5MigrationStrategy) if defined? ActiveRecord::Migrator
-        else
-          ActiveRecord::Migration.send(:include, MigrationStrategy)
-          ActiveRecord::Migrator.send(:include, MigrationStrategy) if defined? ActiveRecord::Migrator
-        end
+        ActiveRecord::Migration.prepend(MigrationStrategy)
+        ActiveRecord::Migrator.prepend(MigrationStrategy) if defined? ActiveRecord::Migrator
       end
     end
 
     initializer 'chewy.request_strategy' do |app|
-      app.config.middleware.insert_after(Rails::Rack::Logger, RequestStrategy)
+      app.config.middleware.insert_before(ActionDispatch::ShowExceptions, RequestStrategy)
     end
 
     initializer 'chewy.add_indices_path' do |_app|

data/lib/chewy/rake_helper.rb

@@ -1,26 +1,27 @@
 module Chewy
   module RakeHelper
-    IMPORT_CALLBACK = lambda do |output, _name, start, finish, _id, payload| # rubocop:disable Metrics/ParameterLists
+    IMPORT_CALLBACK = lambda do |output, _name, start, finish, _id, payload|
       duration = (finish - start).ceil
       stats = payload.fetch(:import, {}).map { |key, count| "#{key} #{count}" }.join(', ')
-      output.puts "  Imported #{payload[:type]} for #{human_duration(duration)}, stats: #{stats}"
-      if payload[:errors]
-        payload[:errors].each do |action, errors|
-          output.puts "    #{action.to_s.humanize} errors:"
-          errors.each do |error, documents|
-            output.puts "      `#{error}`"
-            output.puts "        on #{documents.count} documents: #{documents}"
-          end
+      output.puts "  Imported #{payload[:index]} in #{human_duration(duration)}, stats: #{stats}"
+      payload[:errors]&.each do |action, errors|
+        output.puts "    #{action.to_s.humanize} errors:"
+        errors.each do |error, documents|
+          output.puts "      `#{error}`"
+          output.puts "        on #{documents.count} documents: #{documents}"
         end
       end
     end
 
-    JOURNAL_CALLBACK = lambda do |output, _, _, _, _, payload| # rubocop:disable Metrics/ParameterLists
+    JOURNAL_CALLBACK = lambda do |output, _, _, _, _, payload|
       count = payload[:groups].values.map(&:size).sum
       targets = payload[:groups].keys.sort_by(&:derivable_name)
       output.puts "  Applying journal to #{targets}, #{count} entries, stage #{payload[:stage]}"
     end
 
+    DELETE_BY_QUERY_OPTIONS = %w[WAIT_FOR_COMPLETION REQUESTS_PER_SECOND SCROLL_SIZE].freeze
+    FALSE_VALUES = %w[0 f false off].freeze
+
     class << self
       # Performs zero-downtime reindexing of all documents for the specified indexes
       #
@@ -36,7 +37,9 @@ module Chewy
       # @param parallel [true, Integer, Hash] any acceptable parallel options for import
       # @param output [IO] output io for logging
       # @return [Array<Chewy::Index>] indexes that were reset
-      def reset(only: nil, except: nil, parallel: nil, output: STDOUT)
+      def reset(only: nil, except: nil, parallel: nil, output: $stdout)
+        warn_missing_index(output)
+
         subscribed_task_stats(output) do
           indexes_from(only: only, except: except).each do |index|
             reset_one(index, output, parallel: parallel)
@@ -59,7 +62,9 @@ module Chewy
       # @param parallel [true, Integer, Hash] any acceptable parallel options for import
       # @param output [IO] output io for logging
       # @return [Array<Chewy::Index>] indexes that were actually reset
-      def upgrade(only: nil, except: nil, parallel: nil, output: STDOUT)
+      def upgrade(only: nil, except: nil, parallel: nil, output: $stdout)
+        warn_missing_index(output)
+
         subscribed_task_stats(output) do
           indexes = indexes_from(only: only, except: except)
 
@@ -87,23 +92,21 @@ module Chewy
       #
       # @example
       #   Chewy::RakeHelper.update # updates everything
-      #   Chewy::RakeHelper.update(only: 'places') # updates only PlacesIndex::City and PlacesIndex::Country
-      #   Chewy::RakeHelper.update(only: 'places#city') # updates PlacesIndex::City only
-      #   Chewy::RakeHelper.update(except: PlacesIndex::Country) # updates everything, but PlacesIndex::Country
-      #   Chewy::RakeHelper.update(only: 'places', except: 'places#country') # updates PlacesIndex::City only
+      #   Chewy::RakeHelper.update(only: 'places') # updates only PlacesIndex
+      #   Chewy::RakeHelper.update(except: PlacesIndex) # updates everything, but PlacesIndex
       #
-      # @param only [Array<Chewy::Index, Chewy::Type, String>, Chewy::Index, Chewy::Type, String] indexes or types to update; if nothing is passed - uses all the types defined in the app
-      # @param except [Array<Chewy::Index, Chewy::Type, String>, Chewy::Index, Chewy::Type, String] indexes or types to exclude from processing
+      # @param only [Array<Chewy::Index, String>, Chewy::Index, String] indexes to update; if nothing is passed - uses all the indexes defined in the app
+      # @param except [Array<Chewy::Index, String>, Chewy::Index, String] indexes to exclude from processing
       # @param parallel [true, Integer, Hash] any acceptable parallel options for import
       # @param output [IO] output io for logging
-      # @return [Array<Chewy::Type>] types that were actually updated
-      def update(only: nil, except: nil, parallel: nil, output: STDOUT)
+      # @return [Array<Chewy::Index>] indexes that were actually updated
+      def update(only: nil, except: nil, parallel: nil, output: $stdout)
         subscribed_task_stats(output) do
-          types_from(only: only, except: except).group_by(&:index).each_with_object([]) do |(index, types), update_types|
+          indexes_from(only: only, except: except).each_with_object([]) do |index, updated_indexes|
             if index.exists?
               output.puts "Updating #{index}"
-              types.each { |type| type.import(parallel: parallel) }
-              update_types.concat(types)
+              index.import(parallel: parallel)
+              updated_indexes.push(index)
             else
               output.puts "Skipping #{index}, it does not exists (use rake chewy:reset[#{index.derivable_name}] to create and update it)"
             end
@@ -115,31 +118,29 @@ module Chewy
       #
       # @example
       #   Chewy::RakeHelper.sync # synchronizes everything
-      #   Chewy::RakeHelper.sync(only: 'places') # synchronizes only PlacesIndex::City and PlacesIndex::Country
-      #   Chewy::RakeHelper.sync(only: 'places#city') # synchronizes PlacesIndex::City only
-      #   Chewy::RakeHelper.sync(except: PlacesIndex::Country) # synchronizes everything, but PlacesIndex::Country
-      #   Chewy::RakeHelper.sync(only: 'places', except: 'places#country') # synchronizes PlacesIndex::City only
+      #   Chewy::RakeHelper.sync(only: 'places') # synchronizes only PlacesIndex
+      #   Chewy::RakeHelper.sync(except: PlacesIndex) # synchronizes everything, but PlacesIndex
       #
-      # @param only [Array<Chewy::Index, Chewy::Type, String>, Chewy::Index, Chewy::Type, String] indexes or types to synchronize; if nothing is passed - uses all the types defined in the app
-      # @param except [Array<Chewy::Index, Chewy::Type, String>, Chewy::Index, Chewy::Type, String] indexes or types to exclude from processing
+      # @param only [Array<Chewy::Index, String>, Chewy::Index, String] indexes to synchronize; if nothing is passed - uses all the indexes defined in the app
+      # @param except [Array<Chewy::Index, String>, Chewy::Index, String] indexes to exclude from processing
       # @param parallel [true, Integer, Hash] any acceptable parallel options for sync
       # @param output [IO] output io for logging
-      # @return [Array<Chewy::Type>] types that were actually updated
-      def sync(only: nil, except: nil, parallel: nil, output: STDOUT)
+      # @return [Array<Chewy::Index>] indexes that were actually updated
+      def sync(only: nil, except: nil, parallel: nil, output: $stdout)
         subscribed_task_stats(output) do
-          types_from(only: only, except: except).each_with_object([]) do |type, synced_types|
-            output.puts "Synchronizing #{type}"
-            output.puts "  #{type} doesn't support outdated synchronization" unless type.supports_outdated_sync?
+          indexes_from(only: only, except: except).each_with_object([]) do |index, synced_indexes|
+            output.puts "Synchronizing #{index}"
+            output.puts "  #{index} doesn't support outdated synchronization" unless index.supports_outdated_sync?
             time = Time.now
-            sync_result = type.sync(parallel: parallel)
+            sync_result = index.sync(parallel: parallel)
             if !sync_result
-              output.puts "  Something went wrong with the #{type} synchronization"
-            elsif sync_result[:count] > 0
+              output.puts "  Something went wrong with the #{index} synchronization"
+            elsif (sync_result[:count]).positive?
               output.puts "  Missing documents: #{sync_result[:missing]}" if sync_result[:missing].present?
               output.puts "  Outdated documents: #{sync_result[:outdated]}" if sync_result[:outdated].present?
-              synced_types.push(type)
+              synced_indexes.push(index)
             else
-              output.puts "  Skipping #{type}, up to date"
+              output.puts "  Skipping #{index}, up to date"
             end
             output.puts "  Took #{human_duration(Time.now - time)}"
           end
@@ -147,61 +148,111 @@ module Chewy
       end
 
       # Applies changes that were done after the specified time for the
-      # specified indexes/types or all of them.
+      # specified indexes or all of them.
       #
       # @example
       #   Chewy::RakeHelper.journal_apply(time: 1.minute.ago) # applies entries created for the last minute
-      #   Chewy::RakeHelper.journal_apply(time: 1.minute.ago, only: 'places') # applies only PlacesIndex::City and PlacesIndex::Country entries reated for the last minute
-      #   Chewy::RakeHelper.journal_apply(time: 1.minute.ago, only: 'places#city') # applies PlacesIndex::City entries reated for the last minute only
-      #   Chewy::RakeHelper.journal_apply(time: 1.minute.ago, except: PlacesIndex::Country) # applies everything, but PlacesIndex::Country entries reated for the last minute
-      #   Chewy::RakeHelper.journal_apply(time: 1.minute.ago, only: 'places', except: 'places#country') # applies PlacesIndex::City entries reated for the last minute only
+      #   Chewy::RakeHelper.journal_apply(time: 1.minute.ago, only: 'places') # applies only PlacesIndex entries created for the last minute
+      #   Chewy::RakeHelper.journal_apply(time: 1.minute.ago, except: PlacesIndex) # applies everything, but PlacesIndex, entries created for the last minute
       #
       # @param time [Time, DateTime] use only journal entries created after this time
-      # @param only [Array<Chewy::Index, Chewy::Type, String>, Chewy::Index, Chewy::Type, String] indexes or types to synchronize; if nothing is passed - uses all the types defined in the app
-      # @param except [Array<Chewy::Index, Chewy::Type, String>, Chewy::Index, Chewy::Type, String] indexes or types to exclude from processing
+      # @param only [Array<Chewy::Index, String>, Chewy::Index, String] indexes to synchronize; if nothing is passed - uses all the indexes defined in the app
+      # @param except [Array<Chewy::Index, String>, Chewy::Index, String] indexes to exclude from processing
       # @param output [IO] output io for logging
-      # @return [Array<Chewy::Type>] types that were actually updated
-      def journal_apply(time: nil, only: nil, except: nil, output: STDOUT)
+      # @return [Array<Chewy::Index>] indexes that were actually updated
+      def journal_apply(time: nil, only: nil, except: nil, output: $stdout)
         raise ArgumentError, 'Please specify the time to start with' unless time
+
         subscribed_task_stats(output) do
           output.puts "Applying journal entries created after #{time}"
-          count = Chewy::Journal.new(types_from(only: only, except: except)).apply(time)
+          count = Chewy::Journal.new(journal_indexes_from(only: only, except: except)).apply(time)
           output.puts 'No journal entries were created after the specified time' if count.zero?
         end
       end
 
       # Removes journal records created before the specified timestamp for
-      # the specified indexes/types or all of them.
+      # the specified indexes or all of them.
       #
       # @example
       #   Chewy::RakeHelper.journal_clean # cleans everything
       #   Chewy::RakeHelper.journal_clean(time: 1.minute.ago) # leaves only entries created for the last minute
-      #   Chewy::RakeHelper.journal_clean(only: 'places') # cleans only PlacesIndex::City and PlacesIndex::Country entries
-      #   Chewy::RakeHelper.journal_clean(only: 'places#city') # cleans PlacesIndex::City entries only
-      #   Chewy::RakeHelper.journal_clean(except: PlacesIndex::Country) # cleans everything, but PlacesIndex::Country entries
-      #   Chewy::RakeHelper.journal_clean(only: 'places', except: 'places#country') # cleans PlacesIndex::City entries only
+      #   Chewy::RakeHelper.journal_clean(only: 'places') # cleans only PlacesIndex entries
+      #   Chewy::RakeHelper.journal_clean(except: PlacesIndex) # cleans everything, but PlacesIndex entries
       #
       # @param time [Time, DateTime] clean all the journal entries created before this time
-      # @param only [Array<Chewy::Index, Chewy::Type, String>, Chewy::Index, Chewy::Type, String] indexes or types to synchronize; if nothing is passed - uses all the types defined in the app
-      # @param except [Array<Chewy::Index, Chewy::Type, String>, Chewy::Index, Chewy::Type, String] indexes or types to exclude from processing
+      # @param only [Array<Chewy::Index, String>, Chewy::Index, String] indexes to synchronize; if nothing is passed - uses all the indexes defined in the app
+      # @param except [Array<Chewy::Index, String>, Chewy::Index, String] indexes to exclude from processing
       # @param output [IO] output io for logging
-      # @return [Array<Chewy::Type>] types that were actually updated
-      def journal_clean(time: nil, only: nil, except: nil, output: STDOUT)
+      # @return [Array<Chewy::Index>] indexes that were actually updated
+      def journal_clean(time: nil, only: nil, except: nil, delete_by_query_options: {}, output: $stdout)
         subscribed_task_stats(output) do
           output.puts "Cleaning journal entries created before #{time}" if time
-          response = Chewy::Journal.new(types_from(only: only, except: except)).clean(time)
-          count = response['deleted'] || response['_indices']['_all']['deleted']
-          output.puts "Cleaned up #{count} journal entries"
+          response = Chewy::Journal.new(journal_indexes_from(only: only, except: except)).clean(time, delete_by_query_options: delete_by_query_options)
+          if response.key?('task')
+            output.puts "Task to cleanup the journal has been created, #{response['task']}"
+          else
+            count = response['deleted'] || response['_indices']['_all']['deleted']
+            output.puts "Cleaned up #{count} journal entries"
+          end
         end
       end
 
       # Eager loads and returns all the indexes defined in the application
-      # except the Chewy::Stash.
+      # except Chewy::Stash::Specification and Chewy::Stash::Journal.
       #
       # @return [Array<Chewy::Index>] indexes found
       def all_indexes
         Chewy.eager_load!
-        Chewy::Index.descendants - [Chewy::Stash]
+        Chewy::Index.descendants - [Chewy::Stash::Journal, Chewy::Stash::Specification]
+      end
+
+      # Reindex data from source index to destination index
+      #
+      # @example
+      #   Chewy::RakeHelper.reindex(source: 'users_index', dest: 'cities_index') reindex data from 'users_index' index to 'cities_index'
+      #
+      # @param source [String], dest [String] indexes to reindex
+      def reindex(source:, dest:, output: $stdout)
+        subscribed_task_stats(output) do
+          output.puts "Source index is #{source}\nDestination index is #{dest}"
+          Chewy::Index.reindex(source: source, dest: dest)
+          output.puts "#{source} index successfully reindexed with #{dest} index data"
+        end
+      end
+
+      # Adds new fields to an existing data stream or index.
+      # Change the search settings of existing fields.
+      #
+      # @example
+      #   Chewy::RakeHelper.update_mapping('cities', {properties: {new_field: {type: :text}}}) update 'cities' index with new_field of text type
+      #
+      # @param name [String], body_hash [Hash] index name and body hash to update
+      def update_mapping(name:, output: $stdout)
+        subscribed_task_stats(output) do
+          output.puts "Index name is #{name}"
+          normalize_index(name).update_mapping
+          output.puts "#{name} index successfully updated"
+        end
+      end
+
+      # Reads options that are required to run journal cleanup asynchronously from ENV hash
+      # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-delete-by-query.html
+      #
+      # @example
+      #   Chewy::RakeHelper.delete_by_query_options_from_env({'WAIT_FOR_COMPLETION' => 'false','REQUESTS_PER_SECOND' => '10','SCROLL_SIZE' => '5000'})
+      #   # => { wait_for_completion: false, requests_per_second: 10.0, scroll_size: 5000 }
+      #
+      def delete_by_query_options_from_env(env)
+        env
+          .slice(*DELETE_BY_QUERY_OPTIONS)
+          .transform_keys { |k| k.downcase.to_sym }
+          .to_h do |key, value|
+            case key
+            when :wait_for_completion then [key, !FALSE_VALUES.include?(value.downcase)]
+            when :requests_per_second then [key, value.to_f]
+            when :scroll_size then [key, value.to_i]
+            end
+          end
       end
 
       def normalize_indexes(*identifiers)
@@ -210,41 +261,27 @@ module Chewy
 
       def normalize_index(identifier)
         return identifier if identifier.is_a?(Class) && identifier < Chewy::Index
-        "#{identifier.to_s.gsub(/identifier\z/i, '').camelize}Index".constantize
+
+        "#{identifier.to_s.camelize}Index".constantize
       end
 
-      def subscribed_task_stats(output = STDOUT)
+      def subscribed_task_stats(output = $stdout, &block)
         start = Time.now
         ActiveSupport::Notifications.subscribed(JOURNAL_CALLBACK.curry[output], 'apply_journal.chewy') do
-          ActiveSupport::Notifications.subscribed(IMPORT_CALLBACK.curry[output], 'import_objects.chewy') do
-            yield
-          end
+          ActiveSupport::Notifications.subscribed(IMPORT_CALLBACK.curry[output], 'import_objects.chewy', &block)
         end
+      ensure
         output.puts "Total: #{human_duration(Time.now - start)}"
       end
 
-      def reset_index(*indexes)
-        ActiveSupport::Deprecation.warn '`Chewy::RakeHelper.reset_index` is deprecated and will be removed soon, use `Chewy::RakeHelper.reset` instead'
-        reset(only: indexes)
-      end
-
-      def reset_all(*except)
-        ActiveSupport::Deprecation.warn '`Chewy::RakeHelper.reset_all` is deprecated and will be removed soon, use `Chewy::RakeHelper.reset` instead'
-        reset(except: except)
-      end
+    private
 
-      def update_index(*indexes)
-        ActiveSupport::Deprecation.warn '`Chewy::RakeHelper.update_index` is deprecated and will be removed soon, use `Chewy::RakeHelper.update` instead'
-        update(only: indexes)
-      end
+      def journal_indexes_from(only: nil, except: nil)
+        return if Array.wrap(only).empty? && Array.wrap(except).empty?
 
-      def update_all(*except)
-        ActiveSupport::Deprecation.warn '`Chewy::RakeHelper.update_all` is deprecated and will be removed soon, use `Chewy::RakeHelper.update` instead'
-        update(except: except)
+        indexes_from(only: only, except: except)
       end
 
-    private
-
       def indexes_from(only: nil, except: nil)
         indexes = if only.present?
           normalize_indexes(Array.wrap(only))
@@ -261,33 +298,9 @@ module Chewy
         indexes.sort_by(&:derivable_name)
       end
 
-      def types_from(only: nil, except: nil)
-        types = if only.present?
-          normalize_types(Array.wrap(only))
-        else
-          all_indexes.flat_map(&:types)
-        end
-
-        types = if except.present?
-          types - normalize_types(Array.wrap(except))
-        else
-          types
-        end
-
-        types.sort_by(&:derivable_name)
-      end
-
-      def normalize_types(*identifiers)
-        identifiers.flatten(1).flat_map { |identifier| normalize_type(identifier) }
-      end
-
-      def normalize_type(identifier)
-        Chewy.derive_types(identifier)
-      end
-
       def human_duration(seconds)
         [[60, :s], [60, :m], [24, :h]].map do |amount, unit|
-          if seconds > 0
+          if seconds.positive?
             seconds, n = seconds.divmod(amount)
             "#{n.to_i}#{unit}"
           end
@@ -296,7 +309,23 @@ module Chewy
 
       def reset_one(index, output, parallel: false)
         output.puts "Resetting #{index}"
-        index.reset!((Time.now.to_f * 1000).round, parallel: parallel)
+        index.reset!((Time.now.to_f * 1000).round, parallel: parallel, apply_journal: journal_exists?)
+      end
+
+      def warn_missing_index(output)
+        return if journal_exists?
+
+        output.puts "############################################################\n" \
+                    "WARN: You are risking to lose some changes during the reset.\n" \
+                    "      Please consider enabling journaling.\n" \
+                    "      See https://github.com/toptal/chewy#journaling\n" \
+                    '############################################################'
+      end
+
+      def journal_exists?
+        @journal_exists = Chewy::Stash::Journal.exists? if @journal_exists.nil?
+
+        @journal_exists
       end
     end
   end

data/lib/chewy/rspec/build_query.rb

@@ -0,0 +1,12 @@
+# Rspec helper to compare request and expected query
+# To use it - add `require 'chewy/rspec/build_query'` to the `spec_helper.rb`
+# Simple usage - just pass expected response as argument
+# and then call needed query.
+#
+#   expect { method1.method2...methodN }.to build_query(expected_query)
+#
+RSpec::Matchers.define :build_query do |expected_query = {}|
+  match do |request|
+    request.render == expected_query
+  end
+end

data/lib/chewy/rspec/helpers.rb

@@ -0,0 +1,55 @@
+module Chewy
+  module Rspec
+    module Helpers
+      extend ActiveSupport::Concern
+      # Rspec helper to mock elasticsearch response
+      # To use it - add `require 'chewy/rspec'` to the `spec_helper.rb`
+      #
+      #   mock_elasticsearch_response(CitiesIndex, raw_response)
+      #   expect(CitiesIndex.query({}).hits).to eq(hits)
+      #
+      def mock_elasticsearch_response(index, raw_response)
+        mocked_request = Chewy::Search::Request.new(index)
+        allow(Chewy::Search::Request).to receive(:new).and_return(mocked_request)
+        allow(mocked_request).to receive(:perform).and_return(raw_response)
+      end
+
+      # Rspec helper to mock Elasticsearch response source
+      # To use it - add `require 'chewy/rspec'` to the `spec_helper.rb`
+      #
+      #  mock_elasticsearch_response_sources(CitiesIndex, sources)
+      #  expect(CitiesIndex.query({}).hits).to eq(hits)
+      #
+      def mock_elasticsearch_response_sources(index, hits)
+        raw_response = {
+          'took' => 4,
+          'timed_out' => false,
+          '_shards' => {
+            'total' => 1,
+            'successful' => 1,
+            'skipped' => 0,
+            'failed' => 0
+          },
+          'hits' => {
+            'total' => {
+              'value' => hits.count,
+              'relation' => 'eq'
+            },
+            'max_score' => 1.0,
+            'hits' => hits.each_with_index.map do |hit, i|
+              {
+                '_index' => index.index_name,
+                '_type' => '_doc',
+                '_id' => (i + 1).to_s,
+                '_score' => 3.14,
+                '_source' => hit
+              }
+            end
+          }
+        }
+
+        mock_elasticsearch_response(index, raw_response)
+      end
+    end
+  end
+end