chewy 0.10.1 → 7.3.2

data/lib/chewy/index.rb

@@ -1,23 +1,52 @@
 require 'chewy/search'
 require 'chewy/index/actions'
+require 'chewy/index/adapter/active_record'
+require 'chewy/index/adapter/object'
 require 'chewy/index/aliases'
+require 'chewy/index/crutch'
+require 'chewy/index/import'
+require 'chewy/index/mapping'
+require 'chewy/index/observe'
 require 'chewy/index/settings'
 require 'chewy/index/specification'
+require 'chewy/index/syncer'
+require 'chewy/index/witchcraft'
+require 'chewy/index/wrapper'
 
 module Chewy
   class Index
+    IMPORT_OPTIONS_KEYS = %i[
+      batch_size bulk_size consistency direct_import journal
+      pipeline raw_import refresh replication
+    ].freeze
+
+    STRATEGY_OPTIONS = {
+      delayed_sidekiq: %i[latency margin ttl reindex_wrapper]
+    }.freeze
+
     include Search
     include Actions
     include Aliases
+    include Import
+    include Mapping
+    include Observe
+    include Crutch
+    include Witchcraft
+    include Wrapper
 
     singleton_class.delegate :client, to: 'Chewy'
 
-    class_attribute :type_hash
-    self.type_hash = {}
+    class_attribute :adapter
+    self.adapter = Chewy::Index::Adapter::Object.new(:default)
+
+    class_attribute :index_scope_defined
 
     class_attribute :_settings
     self._settings = Chewy::Index::Settings.new
 
+    class_attribute :_default_import_options
+    self._default_import_options = {}
+
     class << self
       # @overload index_name(suggest)
       #   If suggested name is passed, it is set up as the new base name for
@@ -47,15 +76,16 @@ module Chewy
       #     UsersIndex.index_name(prefix: '', suffix: '2017') # => 'users_2017'
       #
       #   @param prefix [String] index name prefix, uses {.prefix} method by default
-      #   @param suffix [String] index name suffix, used for creating several indexes for the same alias during the zero-downtime reset
+      #   @param suffix [String] index name suffix, used for creating several indexes for the same
+      #     alias during the zero-downtime reset
       #   @raise [UndefinedIndex] if the base name is blank
       #   @return [String] result index name
       def index_name(suggest = nil, prefix: nil, suffix: nil)
         if suggest
-          @base_name = suggest.to_s.underscore.presence
+          @base_name = suggest.to_s.presence
         else
           [
-            prefix || prefix_with_deprecation,
+            prefix || self.prefix,
             base_name,
             suffix
           ].reject(&:blank?).join('_')
@@ -77,6 +107,7 @@ module Chewy
       def base_name
         @base_name ||= name.sub(/Index\z/, '').demodulize.underscore if name
         raise UndefinedIndex if @base_name.blank?
+
         @base_name
       end
 
@@ -111,78 +142,30 @@ module Chewy
         Chewy.configuration[:prefix]
       end
 
-      # Defines type for the index. Arguments depends on adapter used. For
+      # Defines scope and options for the index. Arguments depends on adapter used. For
       # ActiveRecord you can pass model or scope and options
       #
       #   class CarsIndex < Chewy::Index
-      #     define_type Car do
-      #       ...
-      #     end # defines VehiclesIndex::Car type
-      #   end
-      #
-      # Type name might be passed in complicated cases:
-      #
-      #   class VehiclesIndex < Chewy::Index
-      #     define_type Vehicle.cars.includes(:manufacturer), name: 'cars' do
-      #        ...
-      #     end # defines VehiclesIndex::Cars type
-      #
-      #     define_type Vehicle.motocycles.includes(:manufacturer), name: 'motocycles' do
-      #        ...
-      #     end # defines VehiclesIndex::Motocycles type
+      #     index_scope Car
+      #     ...
       #   end
       #
-      # For plain objects:
+      # For plain objects you can completely omit this directive, unless you need to specify some options:
       #
       #   class PlanesIndex < Chewy::Index
-      #     define_type :plane do
-      #       ...
-      #     end # defines PlanesIndex::Plane type
+      #     ...
       #   end
       #
       # The main difference between using plain objects or ActiveRecord models for indexing
-      # is import. If you will call `CarsIndex::Car.import` - it will import all the cars
-      # automatically, while `PlanesIndex::Plane.import(my_planes)` requires import data to be
+      # is import. If you will call `CarsIndex.import` - it will import all the cars
+      # automatically, while `PlanesIndex.import(my_planes)` requires import data to be
       # passed.
       #
-      def define_type(target, options = {}, &block)
-        type_class = Chewy.create_type(self, target, options, &block)
-        self.type_hash = type_hash.merge(type_class.type_name => type_class)
-      end
+      def index_scope(target, options = {})
+        raise 'Index scope is already defined' if index_scope_defined?
 
-      # Types method has double usage.
-      # If no arguments are passed - it returns array of defined types:
-      #
-      #   UsersIndex.types # => [UsersIndex::Admin, UsersIndex::Manager, UsersIndex::User]
-      #
-      # If arguments are passed it treats like a part of chainable query DSL and
-      # adds types array for index to select.
-      #
-      #   UsersIndex.filters { name =~ 'ro' }.types(:admin, :manager)
-      #   UsersIndex.types(:admin, :manager).filters { name =~ 'ro' } # the same as the first example
-      #
-      def types(*args)
-        if args.present?
-          all.types(*args)
-        else
-          type_hash.values
-        end
-      end
-
-      # Returns defined types names:
-      #
-      #   UsersIndex.type_names # => ['admin', 'manager', 'user']
-      #
-      def type_names
-        type_hash.keys
-      end
-
-      # Returns named type:
-      #
-      #    UserIndex.type('admin') # => UsersIndex::Admin
-      #
-      def type(type_name)
-        type_hash.fetch(type_name) { raise UndefinedType, "Unknown type in #{name}: #{type_name}" }
+        self.adapter = Chewy.adapters.find { |klass| klass.accepts?(target) }.new(target, **options)
+        self.index_scope_defined = true
       end
 
       # Used as a part of index definition DSL. Defines settings:
@@ -219,7 +202,7 @@ module Chewy
       end
 
       def mappings_hash
-        mappings = types.map(&:mappings_hash).inject(:merge)
+        mappings = root.mappings_hash
         mappings.present? ? {mappings: mappings} : {}
       end
 
@@ -232,44 +215,36 @@ module Chewy
         [settings_hash, mappings_hash].inject(:merge)
       end
 
-      def index_params
-        ActiveSupport::Deprecation.warn '`Chewy::Index.index_params` is deprecated and will be removed soon, use `Chewy::Index.specification_hash`'
-        specification_hash
-      end
-
       # @see Chewy::Index::Specification
       # @return [Chewy::Index::Specification] a specification object instance for this particular index
       def specification
         @specification ||= Specification.new(self)
       end
 
-      def derivable_index_name
-        ActiveSupport::Deprecation.warn '`Chewy::Index.derivable_index_name` is deprecated and will be removed soon, use `Chewy::Index.derivable_name` instead'
-        derivable_name
+      def default_import_options(params)
+        params.assert_valid_keys(IMPORT_OPTIONS_KEYS)
+        self._default_import_options = _default_import_options.merge(params)
       end
 
-      # Handling old default_prefix if it is not defined.
-      def method_missing(name, *args, &block) # rubocop:disable Style/MethodMissing
-        if name == :default_prefix
-          ActiveSupport::Deprecation.warn '`Chewy::Index.default_prefix` is deprecated and will be removed soon, use `Chewy::Index.prefix` instead'
-          prefix
-        else
-          super
-        end
-      end
+      def strategy_config(params = {})
+        @strategy_config ||= begin
+          config_struct = Struct.new(*STRATEGY_OPTIONS.keys).new
 
-      def prefix_with_deprecation
-        if respond_to?(:default_prefix)
-          ActiveSupport::Deprecation.warn '`Chewy::Index.default_prefix` is deprecated and will be removed soon, define `Chewy::Index.prefix` method instead'
-          default_prefix
-        else
-          prefix
-        end
-      end
+          STRATEGY_OPTIONS.each_with_object(config_struct) do |(strategy, options), res|
+            res[strategy] = case strategy
+            when :delayed_sidekiq
+              Struct.new(*STRATEGY_OPTIONS[strategy]).new.tap do |config|
+                options.each do |option|
+                  config[option] = params.dig(strategy, option) || Chewy.configuration.dig(:strategy_config, strategy, option)
+                end
 
-      def build_index_name(*args)
-        ActiveSupport::Deprecation.warn '`Chewy::Index.build_index_name` is deprecated and will be removed soon, use `Chewy::Index.index_name` instead'
-        index_name(args.extract_options!)
+                config[:reindex_wrapper] ||= ->(&reindex) { reindex.call } # default wrapper
+              end
+            else
+              raise NotImplementedError, "Unsupported strategy: '#{strategy}'"
+            end
+          end
+        end
       end
     end
   end

data/lib/chewy/journal.rb

@@ -2,33 +2,36 @@ module Chewy
   # A class to perform journal-related actions for the specified indexes/types.
   #
   # @example
-  #   journal = Chewy::Journal.new('places#city', UsersIndex)
+  #   journal = Chewy::Journal.new('places', UsersIndex)
   #   journal.apply(20.minutes.ago)
   #   journal.clean
   #
   class Journal
-    # @param only [Array<String, Chewy::Index, Chewy::Type>] indexes/types or even string references to perform actions on
+    # @param only [Array<String, Chewy::Index>] indexes or string references to perform actions on
     def initialize(*only)
       @only = only
     end
 
     # Applies all changes that were done since the specified time to the
-    # specified indexes/types.
+    # specified indexes.
     #
     # @param since_time [Time, DateTime] timestamp from which changes will be applied
-    # @param retries [Integer] maximum number of attempts to make journal empty, 10 by default
+    # @param fetch_limit [Int] amount of entries to be fetched on each cycle
     # @return [Integer] the amount of journal entries found
-    def apply(since_time, retries: 10, **import_options)
+    def apply(since_time, fetch_limit: 10, **import_options)
       stage = 1
       since_time -= 1
       count = 0
-      while stage <= retries
-        entries = Chewy::Stash::Journal.entries(since_time, only: @only).to_a.presence or break
+
+      total_count = entries(since_time, fetch_limit).total_count
+
+      while count < total_count
+        entries = entries(since_time, fetch_limit).to_a.presence or break
         count += entries.size
         groups = reference_groups(entries)
         ActiveSupport::Notifications.instrument 'apply_journal.chewy', stage: stage, groups: groups
-        groups.each do |type, references|
-          type.import(references, import_options.merge(journal: false))
+        groups.each do |index, references|
+          index.import(references, import_options.merge(journal: false))
         end
         stage += 1
         since_time = entries.map(&:created_at).max
@@ -40,16 +43,24 @@ module Chewy
     #
     # @param until_time [Time, DateTime] time to clean up until it
     # @return [Hash] delete_by_query ES API call result
-    def clean(until_time = nil)
-      Chewy::Stash::Journal.clean(until_time, only: @only)
+    def clean(until_time = nil, delete_by_query_options: {})
+      Chewy::Stash::Journal.clean(
+        until_time,
+        only: @only,
+        delete_by_query_options: delete_by_query_options.merge(refresh: false)
+      )
     end
 
   private
 
+    def entries(since_time, fetch_limit)
+      Chewy::Stash::Journal.entries(since_time, only: @only).order(:created_at).limit(fetch_limit)
+    end
+
     def reference_groups(entries)
-      entries.group_by(&:type).map do |type, grouped_entries|
-        [type, grouped_entries.map(&:references).inject(:|)]
-      end.to_h
+      entries.group_by(&:index_name)
+        .transform_keys { |index_name| Chewy.derive_name(index_name) }
+        .transform_values { |grouped_entries| grouped_entries.map(&:references).inject(:|) }
     end
   end
 end

data/lib/chewy/minitest/helpers.rb

@@ -6,19 +6,19 @@ module Chewy
       extend ActiveSupport::Concern
 
       # Assert that an index *changes* during a block.
-      # @param (Chewy::Type) index the index / type to watch, eg EntitiesIndex::Entity.
-      # @param (Symbol) strategy the Chewy strategy to use around the block. See Chewy docs.
-      # @param (boolean) assert the index changes
-      # @param (boolean) bypass_actual_index
+      # @param index [Chewy::Index] the index to watch, eg EntitiesIndex.
+      # @param strategy [Symbol] the Chewy strategy to use around the block. See Chewy docs.
+      # @param bypass_actual_index [true, false]
       #   True to preempt the http call to Elastic, false otherwise.
       #   Should be set to true unless actually testing search functionality.
       #
-      # @return (SearchIndexReceiver) for optional further assertions on the nature of the index changes.
-      def assert_indexes(index, strategy: :atomic, bypass_actual_index: true)
-        type = Chewy.derive_type index
+      # @return [SearchIndexReceiver] for optional further assertions on the nature of the index changes.
+      #
+      def assert_indexes(index, strategy: :atomic, bypass_actual_index: true, &block)
+        index_class = Chewy.derive_name index
         receiver = SearchIndexReceiver.new
 
-        bulk_method = type.method :bulk
+        bulk_method = index_class.method :bulk
         # Manually mocking #bulk because we need to properly capture `self`
         bulk_mock = lambda do |*bulk_args|
           receiver.catch bulk_args, self
@@ -28,13 +28,11 @@ module Chewy
           {}
         end
 
-        type.define_singleton_method :bulk, bulk_mock
+        index_class.define_singleton_method :bulk, bulk_mock
 
-        Chewy.strategy(strategy) do
-          yield
-        end
+        Chewy.strategy(strategy, &block)
 
-        type.define_singleton_method :bulk, bulk_method
+        index_class.define_singleton_method :bulk, bulk_method
 
         assert_includes receiver.updated_indexes, index, "Expected #{index} to be updated but it wasn't"
 
@@ -43,11 +41,86 @@ module Chewy
 
       # Run indexing for the database changes during the block provided.
       # By default, indexing is run at the end of the block.
-      # @param (Symbol) strategy the Chewy index update strategy see Chewy docs.
-      def run_indexing(strategy: :atomic)
-        Chewy.strategy strategy do
-          yield
-        end
+      # @param strategy [Symbol] the Chewy index update strategy see Chewy docs.
+      #
+      def run_indexing(strategy: :atomic, &block)
+        Chewy.strategy strategy, &block
+      end
+
+      # Mock Elasticsearch response
+      # Simple usage - just pass index, expected raw response
+      # and block with the query.
+      #
+      # @param index [Chewy::Index] the index to watch, eg EntitiesIndex.
+      # @param raw_response [Hash] hash with response.
+      #
+      def mock_elasticsearch_response(index, raw_response)
+        mocked_request = Chewy::Search::Request.new(index)
+
+        original_new = Chewy::Search::Request.method(:new)
+
+        Chewy::Search::Request.define_singleton_method(:new) { |*_args| mocked_request }
+
+        original_perform = mocked_request.method(:perform)
+        mocked_request.define_singleton_method(:perform) { raw_response }
+
+        yield
+      ensure
+        mocked_request.define_singleton_method(:perform, original_perform)
+        Chewy::Search::Request.define_singleton_method(:new, original_new)
+      end
+
+      # Mock Elasticsearch response with defined sources
+      # Simple usage - just pass index, expected sources
+      # and block with the query.
+      #
+      # @param index [Chewy::Index] the index to watch, eg EntitiesIndex.
+      # @param hits [Hash] hash with sources.
+      #
+      def mock_elasticsearch_response_sources(index, hits, &block)
+        raw_response = {
+          'took' => 4,
+          'timed_out' => false,
+          '_shards' => {
+            'total' => 1,
+            'successful' => 1,
+            'skipped' => 0,
+            'failed' => 0
+          },
+          'hits' => {
+            'total' => {
+              'value' => hits.count,
+              'relation' => 'eq'
+            },
+            'max_score' => 1.0,
+            'hits' => hits.each_with_index.map do |hit, i|
+              {
+                '_index' => index.index_name,
+                '_type' => '_doc',
+                '_id' => hit[:id] || (i + 1).to_s,
+                '_score' => 3.14,
+                '_source' => hit
+              }
+            end
+          }
+        }
+
+        mock_elasticsearch_response(index, raw_response, &block)
+      end
+
+      # Check the assertion that actual Elasticsearch query is rendered
+      # to the expected query
+      #
+      # @param query [::Query] the actual Elasticsearch query.
+      # @param expected_query [Hash] expected query.
+      #
+      # @return [Boolean]
+      #   True - in the case when actual Elasticsearch query is rendered to the expected query.
+      #   False - in the opposite case.
+      #
+      def assert_elasticsearch_query(query, expected_query)
+        actual_query = query.render
+        assert_equal expected_query, actual_query, "got #{actual_query.inspect} instead of expected query."
       end
 
       module ClassMethods

data/lib/chewy/minitest/search_index_receiver.rb

@@ -1,80 +1,76 @@
 # Test helper class to provide minitest hooks for Chewy::Index testing.
 #
 # @note Intended to be used in conjunction with a test helper which mocks over the #bulk
-#   method on a Chewy::Type class. (See SearchTestHelper)
+#   method on a {Chewy::Index} class. (See {Chewy::Minitest::Helpers})
 #
-# The class will capture the data from the *param on the Chewy::Type#bulk method and
+# The class will capture the data from the *param on the Chewy::Index.bulk method and
 # aggregate the data for test analysis.
 class SearchIndexReceiver
   def initialize
     @mutations = {}
   end
 
-  # @param bulk_params the bulk_params that should be sent to the Chewy::Type#bulk method.
-  # @param (Chewy::Type) type the Index::Type executing this query.
-  def catch(bulk_params, type)
+  # @param bulk_params [Hash] the bulk_params that should be sent to the Chewy::Index.bulk method.
+  # @param index [Chewy::Index] the index executing this query.
+  def catch(bulk_params, index)
     Array.wrap(bulk_params).map { |y| y[:body] }.flatten.each do |update|
       if update[:delete]
-        mutation_for(type).deletes << update[:delete][:_id]
+        mutation_for(index).deletes << update[:delete][:_id]
       elsif update[:index]
-        mutation_for(type).indexes << update[:index]
+        mutation_for(index).indexes << update[:index]
       end
     end
   end
 
-  # @param index return only index requests to the specified Chewy::Type index.
-  # @return the index changes captured by the mock.
+  # @param index [Chewy::Index] return only index requests to the specified {Chewy::Index} index.
+  # @return [Hash] the index changes captured by the mock.
   def indexes_for(index = nil)
     if index
       mutation_for(index).indexes
     else
-      Hash[
-        @mutations.map { |a, b| [a, b.indexes] }
-      ]
+      @mutations.transform_values(&:indexes)
     end
   end
   alias_method :indexes, :indexes_for
 
-  # @param index return only delete requests to the specified Chewy::Type index.
-  # @return the index deletes captured by the mock.
+  # @param index [Chewy::Index] return only delete requests to the specified {Chewy::Index} index.
+  # @return [Hash] the index deletes captured by the mock.
   def deletes_for(index = nil)
     if index
       mutation_for(index).deletes
     else
-      Hash[
-        @mutations.map { |a, b| [a, b.deletes] }
-      ]
+      @mutations.transform_values(&:deletes)
     end
   end
   alias_method :deletes, :deletes_for
 
   # Check to see if a given object has been indexed.
-  # @param (#id) obj the object to look for.
-  # @param Chewy::Type what type the object should be indexed as.
-  # @return bool if the object was indexed.
-  def indexed?(obj, type)
-    indexes_for(type).map { |i| i[:_id] }.include? obj.id
+  # @param obj [#id] obj the object to look for.
+  # @param index [Chewy::Index] what index the object should be indexed in.
+  # @return [true, false] if the object was indexed.
+  def indexed?(obj, index)
+    indexes_for(index).map { |i| i[:_id] }.include? obj.id
   end
 
   # Check to see if a given object has been deleted.
-  # @param (#id) obj the object to look for.
-  # @param Chewy::Type what type the object should have been deleted from.
-  # @return bool if the object was deleted.
-  def deleted?(obj, type)
-    deletes_for(type).include? obj.id
+  # @param obj [#id] obj the object to look for.
+  # @param index [Chewy::Index] what index the object should have been deleted from.
+  # @return [true, false] if the object was deleted.
+  def deleted?(obj, index)
+    deletes_for(index).include? obj.id
   end
 
-  # @return a list of Chewy::Type indexes changed.
+  # @return [Array<Chewy::Index>] a list of indexes changed.
   def updated_indexes
     @mutations.keys
   end
 
 private
 
-  # Get the mutation object for a given type.
-  # @param (Chewy::Type) type the index type to fetch.
-  # @return (#indexes, #deletes) an object with a list of indexes and a list of deletes.
-  def mutation_for(type)
-    @mutations[type] ||= OpenStruct.new(indexes: [], deletes: [])
+  # Get the mutation object for a given index.
+  # @param index [Chewy::Index] the index to fetch.
+  # @return [#indexes, #deletes] an object with a list of indexes and a list of deletes.
+  def mutation_for(index)
+    @mutations[index] ||= OpenStruct.new(indexes: [], deletes: [])
   end
 end

data/lib/chewy/multi_search.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Chewy
+  # `Chewy::MultiSearch` provides an interface for executing multiple
+  # queries via the Elasticsearch Multi Search API. When a MultiSearch
+  # is performed it wraps the responses from Elasticsearch and assigns
+  # them to the appropriate queries.
+  class MultiSearch
+    attr_reader :queries
+
+    # Instantiate a new MultiSearch instance.
+    #
+    # @param queries [Array<Chewy::Search::Request>]
+    # @option [Elasticsearch::Transport::Client] :client (Chewy.client)
+    #   The Elasticsearch client that should be used for issuing requests.
+    def initialize(queries, client: Chewy.client)
+      @client = client
+      @queries = Array(queries)
+    end
+
+    # Adds a query to be performed by the MultiSearch
+    #
+    # @param query [Chewy::Search::Request]
+    def add_query(query)
+      @queries << query
+    end
+
+    # Performs any unperformed queries and returns the responses for all queries.
+    #
+    # @return [Array<Chewy::Search::Response>]
+    def responses
+      perform
+      queries.map(&:response)
+    end
+
+    # Performs any unperformed queries.
+    def perform
+      unperformed_queries = queries.reject(&:performed?)
+      return if unperformed_queries.empty?
+
+      responses = msearch(unperformed_queries)['responses']
+      unperformed_queries.zip(responses).map { |query, response| query.response = response }
+    end
+
+  private
+
+    attr_reader :client
+
+    def msearch(queries_to_search)
+      body = queries_to_search.flat_map do |query|
+        rendered = query.render
+        [rendered.except(:body), rendered[:body]]
+      end
+
+      client.msearch(body: body)
+    end
+  end
+
+  def self.msearch(queries)
+    Chewy::MultiSearch.new(queries)
+  end
+end