chewy 0.9.0 → 5.2.0

data/lib/chewy/rspec/update_index.rb

@@ -83,9 +83,7 @@ RSpec::Matchers.define :update_index do |type_name, options = {}| # rubocop:disa
   #     .to update_index(UsersIndex.user).and_reindex(user1).only }
   #
   chain(:only) do |*_args|
-    if @reindex.blank? && @delete.blank?
-      raise 'Use `only` in conjunction with `and_reindex` or `and_delete`'
-    end
+    raise 'Use `only` in conjunction with `and_reindex` or `and_delete`' if @reindex.blank? && @delete.blank?
 
     @only = true
   end
@@ -99,22 +97,19 @@ RSpec::Matchers.define :update_index do |type_name, options = {}| # rubocop:disa
     @delete ||= {}
     @missed_reindex = []
     @missed_delete = []
-    @updated = []
 
-    instance_eval <<-RUBY, __FILE__, __LINE__ + 1
-      type = Chewy.derive_type(type_name)
-      #{agnostic_stub} do |bulk_options|
-        @updated += bulk_options[:body].map do |updated_document|
-          updated_document.deep_symbolize_keys
-        end
-        {}
-      end
-    RUBY
+    type = Chewy.derive_type(type_name)
+    if defined?(Mocha) && RSpec.configuration.mock_framework.to_s == 'RSpec::Core::MockingAdapters::Mocha'
+      Chewy::Type::Import::BulkRequest.stubs(:new).with(type, any_parameters).returns(mock_bulk_request)
+    else
+      mocked_already = ::RSpec::Mocks.space.proxy_for(Chewy::Type::Import::BulkRequest).method_double_if_exists_for_message(:new)
+      allow(Chewy::Type::Import::BulkRequest).to receive(:new).and_call_original unless mocked_already
+      allow(Chewy::Type::Import::BulkRequest).to receive(:new).with(type, any_args).and_return(mock_bulk_request)
+    end
 
-    ActiveSupport::Deprecation.warn('`atomic: false` option is removed and not effective anymore, use `strategy: :atomic` option instead') if options.key?(:atomic)
     Chewy.strategy(options[:strategy] || :atomic) { block.call }
 
-    @updated.each do |updated_document|
+    mock_bulk_request.updates.each do |updated_document|
       if (body = updated_document[:index])
         if (document = @reindex[body[:_id].to_s])
           document[:real_count] += 1
@@ -131,18 +126,18 @@ RSpec::Matchers.define :update_index do |type_name, options = {}| # rubocop:disa
       end
     end
 
-    @reindex.each do |_, document|
+    @reindex.each_value do |document|
       document[:match_count] = (!document[:expected_count] && document[:real_count] > 0) ||
         (document[:expected_count] && document[:expected_count] == document[:real_count])
       document[:match_attributes] = document[:expected_attributes].blank? ||
         compare_attributes(document[:expected_attributes], document[:real_attributes])
     end
-    @delete.each do |_, document|
+    @delete.each_value do |document|
       document[:match_count] = (!document[:expected_count] && document[:real_count] > 0) ||
         (document[:expected_count] && document[:expected_count] == document[:real_count])
     end
 
-    @updated.present? && @missed_reindex.none? && @missed_delete.none? &&
+    mock_bulk_request.updates.present? && @missed_reindex.none? && @missed_delete.none? &&
       @reindex.all? { |_, document| document[:match_count] && document[:match_attributes] } &&
       @delete.all? { |_, document| document[:match_count] }
   end
@@ -150,7 +145,7 @@ RSpec::Matchers.define :update_index do |type_name, options = {}| # rubocop:disa
   failure_message do # rubocop:disable BlockLength
     output = ''
 
-    if @updated.none?
+    if mock_bulk_request.updates.none?
       output << "Expected index `#{type_name}` to be updated, but it was not\n"
     elsif @missed_reindex.present? || @missed_delete.present?
       message = "Expected index `#{type_name}` "
@@ -197,19 +192,15 @@ RSpec::Matchers.define :update_index do |type_name, options = {}| # rubocop:disa
   end
 
   failure_message_when_negated do
-    if @updated.present?
-      "Expected index `#{type_name}` not to be updated, but it was with #{@updated.map(&:values).flatten.group_by { |documents| documents[:_id] }.map do |id, documents|
+    if mock_bulk_request.updates.present?
+      "Expected index `#{type_name}` not to be updated, but it was with #{mock_bulk_request.updates.map(&:values).flatten.group_by { |documents| documents[:_id] }.map do |id, documents|
                                                                             "\n  document id `#{id}` (#{documents.count} times)"
                                                                           end.join}\n"
     end
   end
 
-  def agnostic_stub
-    if defined?(Mocha) && RSpec.configuration.mock_framework.to_s == 'RSpec::Core::MockingAdapters::Mocha'
-      'type.stubs(:bulk).with'
-    else
-      'allow(type).to receive(:bulk)'
-    end
+  def mock_bulk_request
+    @mock_bulk_request ||= MockBulkRequest.new
   end
 
   def extract_documents(*args)
@@ -251,4 +242,19 @@ RSpec::Matchers.define :update_index do |type_name, options = {}| # rubocop:disa
     end
     difference.none?
   end
+
+  # Collects request bodies coming through the perform method for
+  # the further analysis.
+  class MockBulkRequest
+    attr_reader :updates
+
+    def initialize
+      @updates = []
+    end
+
+    def perform(body)
+      @updates.concat(body.map(&:deep_symbolize_keys))
+      []
+    end
+  end
 end

data/lib/chewy/search.rb

@@ -1,47 +1,100 @@
+require 'chewy/search/scoping'
 require 'chewy/query'
+require 'chewy/search/scrolling'
+require 'chewy/search/query_proxy'
+require 'chewy/search/parameters'
+require 'chewy/search/response'
+require 'chewy/search/loader'
+require 'chewy/search/request'
+require 'chewy/search/pagination/kaminari'
+require 'chewy/search/pagination/will_paginate'
 
 module Chewy
+  # This module being included to any provides an interface to the
+  # request DSL. By default it is included to {Chewy::Index} and
+  # {Chewy::Type}.
+  #
+  # The class used as a request DSL provider is
+  # inherited from {Chewy::Search::Request} by default, but if you
+  # need ES < 2.0 DSL support - you can switch it to {Chewy::Query}
+  # using {Chewy::Config#search_class}
+  #
+  # Also, the search class is refined with one of the pagination-
+  # providing modules: {Chewy::Search::Pagination::Kaminari} or
+  # {Chewy::Search::Pagination::WillPaginate}.
+  #
+  # @example
+  #   PlacesIndex.query(match: {name: 'Moscow'})
+  #   PlacesIndex::City.query(match: {name: 'Moscow'})
+  # @see Chewy::Index
+  # @see Chewy::Type
+  # @see Chewy::Search::Request
+  # @see Chewy::Search::ClassMethods
+  # @see Chewy::Search::Pagination::Kaminari
+  # @see Chewy::Search::Pagination::WillPaginate
   module Search
     extend ActiveSupport::Concern
 
-    included do
-      singleton_class.delegate :explain, :query_mode, :filter_mode, :post_filter_mode,
-        :timeout, :limit, :offset, :highlight, :min_score, :rescore, :facets, :script_score,
-        :boost_factor, :weight, :random_score, :field_value_factor, :decay, :aggregations,
-        :suggest, :none, :strategy, :query, :filter, :post_filter, :boost_mode,
-        :score_mode, :order, :reorder, :only, :types, :delete_all, :find, :total,
-        :total_count, :total_entries, :unlimited, :script_fields, :track_scores, :preference,
-        to: :all
-    end
-
     module ClassMethods
+      # This is the entry point for the request composition, however,
+      # most of the {Chewy::Search::Request} methods are delegated
+      # directly as well.
+      #
+      # This method also provides an ability to use names scopes.
+      #
+      # @example
+      #   PlacesIndex.all.limit(10)
+      #   # is basically the same as:
+      #   PlacesIndex.limit(10)
+      # @see Chewy::Search::Request
+      # @see Chewy::Search::Scoping
+      # @return [Chewy::Search::Request] request instance
       def all
-        query_class.scopes.last || query_class.new(self)
+        search_class.scopes.last || search_class.new(self)
       end
 
+      # A simple way to execute search string query.
+      #
+      # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/search-uri-request.html
+      # @return [Hash] the request result
       def search_string(query, options = {})
-        options = options.merge(
-          index: all._indexes.map(&:index_name),
-          type: all._types.map(&:type_name),
-          q: query
-        )
+        options = options.merge(all.render.slice(:index, :type).merge(q: query))
         Chewy.client.search(options)
       end
 
+      # Delegates methods from the request class to the index or type class
+      #
+      # @example
+      #   PlacesIndex.query(match: {name: 'Moscow'})
+      #   PlacesIndex::City.query(match: {name: 'Moscow'})
+      def method_missing(name, *args, &block)
+        if search_class::DELEGATED_METHODS.include?(name)
+          all.send(name, *args, &block)
+        else
+          super
+        end
+      end
+
+      def respond_to_missing?(name, _)
+        search_class::DELEGATED_METHODS.include?(name) || super
+      end
+
     private
 
-      def query_class
-        @query_class ||= begin
-          query_class = Class.new(Chewy::Query)
-          if self < Chewy::Type
-            index_scopes = index.scopes - scopes
+      def search_class
+        @search_class ||= build_search_class(Chewy.search_class)
+      end
 
-            delegate_scoped index, query_class, index_scopes
-            delegate_scoped index, self, index_scopes
-          end
-          delegate_scoped self, query_class, scopes
-          const_set('Query', query_class)
+      def build_search_class(base)
+        search_class = Class.new(base)
+
+        if self < Chewy::Type
+          index_scopes = index.scopes - scopes
+          delegate_scoped index, search_class, index_scopes
         end
+
+        delegate_scoped self, search_class, scopes
+        const_set('Query', search_class)
       end
 
       def delegate_scoped(source, destination, methods)

data/lib/chewy/search/loader.rb

@@ -0,0 +1,83 @@
+module Chewy
+  module Search
+    # This class is used for two different purposes: load ORM/ODM
+    # source objects.
+    #
+    # @see Chewy::Type::Import
+    # @see Chewy::Search::Request#load
+    # @see Chewy::Search::Response#objects
+    # @see Chewy::Search::Scrolling#scroll_objects
+    class Loader
+      # @param indexes [Array<Chewy::Index>] list of indexes to lookup types
+      # @param only [Array<String, Symbol>] list of selected type names to load
+      # @param except [Array<String, Symbol>] list of type names which will not be loaded
+      # @param options [Hash] adapter-specific load options
+      # @see Chewy::Type::Adapter::Base#load
+      def initialize(indexes: [], only: [], except: [], **options)
+        @indexes = indexes
+        @only = Array.wrap(only).map(&:to_s)
+        @except = Array.wrap(except).map(&:to_s)
+        @options = options
+      end
+
+      # Returns a {Chewy::Type} object for index name and type name passed. Caches
+      # the result for each pair to make lookup faster.
+      #
+      # @param index [String] index name
+      # @param type [String] type name
+      # @return [Chewy::Type]
+      # @raise [Chewy::UnderivableType] when index or hash were not found
+      def derive_type(index, type)
+        (@derive_type ||= {})[[index, type]] ||= begin
+          index_class = derive_index(index)
+          raise Chewy::UnderivableType, "Can not find index named `#{index}`" unless index_class
+          index_class.type_hash[type] or raise Chewy::UnderivableType, "Index `#{index}` doesn`t have type named `#{type}`"
+        end
+      end
+
+      # For each passed hit this method loads an ORM/ORD source object
+      # using `hit['_id']`. The returned array is exactly in the same order
+      # as hits were. If source object was not found for some hit, `nil`
+      # will be returned at the corresponding position in array.
+      #
+      # Records/documents are loaded in an efficient manner, performing
+      # a single query for each type present.
+      #
+      # @param hits [Array<Hash>] ES hits array
+      # @return [Array<Object, nil>] the array of corresponding ORM/ODM objects
+      def load(hits)
+        hit_groups = hits.group_by { |hit| [hit['_index'], hit['_type']] }
+        loaded_objects = hit_groups.each_with_object({}) do |((index_name, type_name), hit_group), result|
+          next if skip_type?(type_name)
+
+          type = derive_type(index_name, type_name)
+          ids = hit_group.map { |hit| hit['_id'] }
+          loaded = type.adapter.load(ids, **@options.merge(_type: type))
+          loaded ||= hit_group.map { |hit| type.build(hit) }
+
+          result.merge!(hit_group.zip(loaded).to_h)
+        end
+
+        hits.map { |hit| loaded_objects[hit] }
+      end
+
+    private
+
+      def derive_index(index_name)
+        (@derive_index ||= {})[index_name] ||= indexes_hash[index_name] ||
+          indexes_hash[indexes_hash.keys.sort_by(&:length)
+            .reverse.detect do |name|
+              index_name.match(/#{name}(_.+|\z)/)
+            end]
+      end
+
+      def indexes_hash
+        @indexes_hash ||= @indexes.index_by(&:index_name)
+      end
+
+      def skip_type?(type_name)
+        @except.include?(type_name) || @only.present? && !@only.include?(type_name)
+      end
+    end
+  end
+end

data/lib/chewy/{query → search}/pagination/kaminari.rb

@@ -1,6 +1,12 @@
 module Chewy
-  class Query
+  module Search
     module Pagination
+      # This module provides `Kaminari` support for {Chewy::Search::Request}
+      # It is included automatically if `Kaminari` is available.
+      #
+      # @example
+      #   PlacesIndex.all.page(3).per(10).order(:name)
+      #   # => <PlacesIndex::Query {..., :body=>{:size=>10, :from=>20, :sort=>["name"]}}>
       module Kaminari
         extend ActiveSupport::Concern
 
@@ -17,11 +23,11 @@ module Chewy
         end
 
         def limit_value
-          (criteria.request_options[:size].presence || default_per_page).to_i
+          (raw_limit_value || default_per_page).to_i
         end
 
         def offset_value
-          criteria.request_options[:from].to_i
+          raw_offset_value.to_i
         end
 
       private
@@ -29,9 +35,11 @@ module Chewy
         def _kaminari_config
           ::Kaminari.config
         end
+
+        def paginated_collection(collection)
+          ::Kaminari.paginate_array(collection, limit: limit_value, offset: offset_value, total_count: total_count)
+        end
       end
     end
   end
 end
-
-Chewy::Query.send :include, Chewy::Query::Pagination::Kaminari

data/lib/chewy/search/pagination/will_paginate.rb

@@ -0,0 +1,43 @@
+module Chewy
+  module Search
+    module Pagination
+      # This module provides `WillPaginate` support for {Chewy::Search::Request}
+      # It is included automatically if `WillPaginate` is available.
+      #
+      # @example
+      #   PlacesIndex.all.paginate(page: 3, per_page: 10).order(:name)
+      #   # => <PlacesIndex::Query {..., :body=>{:size=>10, :from=>20, :sort=>["name"]}}>
+      module WillPaginate
+        extend ActiveSupport::Concern
+
+        included do
+          include ::WillPaginate::CollectionMethods
+          attr_reader :current_page, :per_page
+        end
+
+        def paginate(options = {})
+          @current_page = ::WillPaginate::PageNumber(options[:page] || @current_page || 1)
+          @page_multiplier = @current_page - 1
+          @per_page = (options[:per_page] || @per_page || ::WillPaginate.per_page).to_i
+
+          # call Chewy::Query methods to limit results
+          limit(@per_page).offset(@page_multiplier * @per_page)
+        end
+
+        def page(page)
+          paginate(page: page)
+        end
+
+      private
+
+        def paginated_collection(collection)
+          page = current_page || 1
+          per = per_page || ::WillPaginate.per_page
+          ::WillPaginate::Collection.create(page, per, total_entries) do |pager|
+            pager.replace collection
+          end
+        end
+      end
+    end
+  end
+end

data/lib/chewy/search/parameters.rb

@@ -0,0 +1,168 @@
+Dir.glob(File.join(File.dirname(__FILE__), 'parameters', 'concerns', '*.rb')) { |f| require f }
+Dir.glob(File.join(File.dirname(__FILE__), 'parameters', '*.rb')) { |f| require f }
+
+module Chewy
+  module Search
+    # This class is basically a compound storage of the request
+    # parameter storages. It encapsulates some storage-collection-handling
+    # logic.
+    #
+    # @see Chewy::Search::Request#parameters
+    # @see Chewy::Search::Parameters::Storage
+    class Parameters
+      QUERY_STRING_STORAGES = %i[indices search_type request_cache allow_partial_search_results].freeze
+
+      # Default storage classes warehouse. It is probably possible to
+      # add your own classes here if necessary, but I'm not sure it will work.
+      #
+      # @return [{Symbol => Chewy::Search::Parameters::Storage}]
+      def self.storages
+        @storages ||= Hash.new do |hash, name|
+          hash[name] = "Chewy::Search::Parameters::#{name.to_s.camelize}".constantize
+        end
+      end
+
+      # @return [{Symbol => Chewy::Search::Parameters::Storage}]
+      attr_accessor :storages
+      delegate :[], :[]=, to: :storages
+
+      # Accepts an initial hash as basic values or parameter storages.
+      #
+      # @example
+      #   Chewy::Search::Parameters.new(limit: 10, offset 10)
+      #   Chewy::Search::Parameters.new(
+      #     limit: Chewy::Search::Parameters::Limit.new(10),
+      #     limit: Chewy::Search::Parameters::Offset.new(10)
+      #   )
+      # @param initial [{Symbol => Object, Chewy::Search::Parameters::Storage}]
+      def initialize(initial = {}, **kinitial)
+        @storages = Hash.new do |hash, name|
+          hash[name] = self.class.storages[name].new
+        end
+        initial = initial.deep_dup.merge(kinitial)
+        initial.each_with_object(@storages) do |(name, value), result|
+          storage_class = self.class.storages[name]
+          storage = value.is_a?(storage_class) ? value : storage_class.new(value)
+          result[name] = storage
+        end
+      end
+
+      # Compares storages by their values.
+      #
+      # @param other [Object] any object
+      # @return [true, false]
+      def ==(other)
+        super || other.is_a?(self.class) && compare_storages(other)
+      end
+
+      # Clones the specified storage, performs the operation
+      # defined by block on the clone.
+      #
+      # @param name [Symbol] parameter name
+      # @yield the block is executed in the cloned storage instance binding
+      # @return [Chewy::Search::Parameters::Storage]
+      def modify!(name, &block)
+        @storages[name] = @storages[name].clone.tap do |s|
+          s.instance_exec(&block)
+        end
+      end
+
+      # Removes specified storages from the storages hash.
+      #
+      # @param names [Array<String, Symbol>]
+      # @return [{Symbol => Chewy::Search::Parameters::Storage}] removed storages hash
+      def only!(names)
+        @storages.slice!(*assert_storages(names))
+      end
+
+      # Keeps only specified storages removing everything else.
+      #
+      # @param names [Array<String, Symbol>]
+      # @return [{Symbol => Chewy::Search::Parameters::Storage}] kept storages hash
+      def except!(names)
+        @storages.except!(*assert_storages(names))
+      end
+
+      # Takes all the storages and merges them one by one using
+      # {Chewy::Search::Parameters::Storage#merge!} method. Merging
+      # is implemented in different ways for different storages: for
+      # limit, offset and other single-value classes it is a simple
+      # value replacement, for boolean storages (explain, none) it uses
+      # a disjunction result, for compound values - merging and
+      # concatenation, for query, filter, post_filter - it is the
+      # "and" operation.
+      #
+      # @see Chewy::Search::Parameters::Storage#merge!
+      # @return [{Symbol => Chewy::Search::Parameters::Storage}] storages from other parameters
+      def merge!(other)
+        other.storages.each do |name, storage|
+          modify!(name) { merge!(storage) }
+        end
+      end
+
+      # Renders and merges all the parameter storages into a single hash.
+      #
+      # @return [Hash] request body
+      def render
+        render_query_string_params.merge(render_body)
+      end
+
+    protected
+
+      def initialize_clone(origin)
+        @storages = origin.storages.clone
+      end
+
+      def compare_storages(other)
+        keys = (@storages.keys | other.storages.keys)
+        @storages.values_at(*keys) == other.storages.values_at(*keys)
+      end
+
+      def assert_storages(names)
+        raise ArgumentError, 'No storage names were specified' if names.empty?
+        names = names.map(&:to_sym)
+        self.class.storages.values_at(*names)
+        names
+      end
+
+      def render_query_string_params
+        query_string_storages = @storages.select do |storage_name, _|
+          QUERY_STRING_STORAGES.include?(storage_name)
+        end
+
+        query_string_storages.values.inject({}) do |result, storage|
+          result.merge!(storage.render || {})
+        end
+      end
+
+      def render_body
+        exceptions = %i[filter query none] + QUERY_STRING_STORAGES
+        body = @storages.except(*exceptions).values.inject({}) do |result, storage|
+          result.merge!(storage.render || {})
+        end
+        body.merge!(render_query || {})
+        {body: body}
+      end
+
+      def render_query
+        none = @storages[:none].render
+
+        return none if none
+
+        filter = @storages[:filter].render
+        query = @storages[:query].render
+
+        return query unless filter
+
+        if query && query[:query][:bool]
+          query[:query][:bool].merge!(filter)
+          query
+        elsif query
+          {query: {bool: {must: query[:query]}.merge!(filter)}}
+        else
+          {query: {bool: filter}}
+        end
+      end
+    end
+  end
+end