chewy 0.8.3 → 0.8.4

data/lib/chewy/query/compose.rb

@@ -30,7 +30,7 @@ module Chewy
       def _queries_join queries, logic
         queries = queries.compact
 
-        if queries.many? || (queries.any? && logic == :must_not)
+        if queries.many? || (queries.present? && logic == :must_not)
           case logic
           when :dis_max
             { dis_max: { queries: queries } }
@@ -51,7 +51,7 @@ module Chewy
       def _filters_join filters, logic
         filters = filters.compact
 
-        if filters.many? || (filters.any? && logic == :must_not)
+        if filters.many? || (filters.present? && logic == :must_not)
           case logic
           when :and, :or
             { logic => filters }

data/lib/chewy/query/criteria.rb

@@ -32,7 +32,7 @@ module Chewy
 
       STORAGES.each do |storage|
         define_method "#{storage}?" do
-          send(storage).any?
+          send(storage).present?
         end
       end
 
@@ -69,7 +69,7 @@ module Chewy
       end
 
       [:filters, :queries, :post_filters].each do |storage|
-        class_eval <<-RUBY
+        class_eval <<-RUBY, __FILE__, __LINE__ + 1
           def update_#{storage}(modifier)
             @#{storage} = #{storage} + Array.wrap(modifier).reject(&:blank?)
           end

data/lib/chewy/query/loading.rb

@@ -93,7 +93,7 @@ module Chewy
 
         loaded_objects = Hash[_results.group_by(&:class).map do |type, objects|
           next if except.include?(type.type_name)
-          next if only.any? && !only.include?(type.type_name)
+          next if only.present? && !only.include?(type.type_name)
 
           loaded = type.adapter.load(objects, options.merge(_type: type))
           [type, loaded.index_by.with_index do |loaded, i|

data/lib/chewy/query/nodes/bool.rb

@@ -20,7 +20,7 @@ module Chewy
           bool = {
             bool: Hash[METHODS.map do |method|
               value = instance_variable_get("@#{method}")
-              [method.to_sym, value.map(&:__render__)] if value.any?
+              [method.to_sym, value.map(&:__render__)] if value.present?
             end.compact]
           }
           bool[:bool][:_cache] = !!@options[:cache] if @options.key?(:cache)

data/lib/chewy/query/nodes/regexp.rb

@@ -8,8 +8,8 @@ module Chewy
           @name = name.to_s
           @regexp = regexp.respond_to?(:source) ? regexp.source : regexp.to_s
           @options = args.extract_options!
-          if args.any? || @options[:flags].present?
-            @options[:flags] = FLAGS & (args.any? ? args.flatten : @options[:flags]).map(&:to_s).map(&:downcase)
+          if args.present? || @options[:flags].present?
+            @options[:flags] = FLAGS & (args.present? ? args.flatten : @options[:flags]).map(&:to_s).map(&:downcase)
           end
         end
 

data/lib/chewy/railtie.rb

@@ -10,7 +10,8 @@ module Chewy
       end
 
       def call(env)
-        if env['PATH_INFO'].start_with?(Rails.application.config.assets.prefix)
+        # For Rails applications in `api_only` mode, the `assets` config isn't present
+        if Rails.application.config.respond_to?(:assets) && env['PATH_INFO'].start_with?(Rails.application.config.assets.prefix)
           @app.call(env)
         else
           Chewy.strategy(Chewy.request_strategy) { @app.call(env) }
@@ -29,6 +30,12 @@ module Chewy
       end
     end
 
+    module Rails5MigrationStrategy
+      def migrate(*args)
+        Chewy.strategy(:bypass) { super }
+      end
+    end
+
     rake_tasks do
       load 'tasks/chewy.rake'
     end
@@ -49,8 +56,13 @@ module Chewy
 
     initializer 'chewy.migration_strategy' do
       ActiveSupport.on_load(:active_record) do
-        ActiveRecord::Migration.send(:include, MigrationStrategy)
-        ActiveRecord::Migrator.send(:include, MigrationStrategy) if defined? ActiveRecord::Migrator
+        if Rails::VERSION::MAJOR >= 5
+          ActiveRecord::Migration.prepend(Rails5MigrationStrategy)
+          ActiveRecord::Migrator.prepend(Rails5MigrationStrategy) if defined? ActiveRecord::Migrator
+        else
+          ActiveRecord::Migration.send(:include, MigrationStrategy)
+          ActiveRecord::Migrator.send(:include, MigrationStrategy) if defined? ActiveRecord::Migrator
+        end
       end
     end
 

data/lib/chewy/rake_helper.rb

@@ -2,8 +2,8 @@ module Chewy
   module RakeHelper
     class << self
 
-      def subscribe_task_stats!
-        ActiveSupport::Notifications.subscribe('import_objects.chewy') do |name, start, finish, id, payload|
+      def subscribed_task_stats(&block)
+        callback = ->(name, start, finish, id, payload) do
           duration = (finish - start).round(2)
           puts "  Imported #{payload[:type]} for #{duration}s, documents total: #{payload[:import].try(:[], :index).to_i}"
           payload[:errors].each do |action, errors|
@@ -14,6 +14,9 @@ module Chewy
             end
           end if payload[:errors]
         end
+        ActiveSupport::Notifications.subscribed(callback, 'import_objects.chewy') do
+          yield
+        end
       end
 
       def eager_load_chewy!

data/lib/chewy/rspec/update_index.rb

@@ -100,7 +100,7 @@ RSpec::Matchers.define :update_index do |type_name, options = {}|
 
     type = Chewy.derive_type(type_name)
 
-    instance_eval <<-RUBY
+    instance_eval <<-RUBY, __FILE__, __LINE__ + 1
        #{agnostic_stub} do |bulk_options|
         @updated += bulk_options[:body].map do |updated_document|
           updated_document.deep_symbolize_keys
@@ -140,7 +140,7 @@ RSpec::Matchers.define :update_index do |type_name, options = {}|
         (document[:expected_count] && document[:expected_count] == document[:real_count])
     end
 
-    @updated.any? && @missed_reindex.none? && @missed_delete.none? &&
+    @updated.present? && @missed_reindex.none? && @missed_delete.none? &&
     @reindex.all? { |_, document| document[:match_count] && document[:match_attributes] } &&
     @delete.all? { |_, document| document[:match_count] }
   end
@@ -150,9 +150,20 @@ RSpec::Matchers.define :update_index do |type_name, options = {}|
 
     if @updated.none?
       output << "Expected index `#{type_name}` to be updated, but it was not\n"
-    else
-      output << "Expected index `#{type_name}` to update documents #{@reindex.keys} only, but #{@missed_reindex} was updated also\n" if @missed_reindex.any?
-      output << "Expected index `#{type_name}` to delete documents #{@delete.keys} only, but #{@missed_delete} was deleted also\n" if @missed_delete.any?
+    elsif @missed_reindex.present? || @missed_delete.present?
+      message = "Expected index `#{type_name}` "
+      message << [
+        ("to update documents #{@reindex.keys}" if @reindex.present?),
+        ("to delete documents #{@delete.keys}" if @delete.present?)
+      ].compact.join(' and ')
+      message << ' only, but '
+      message << [
+        ("#{@missed_reindex} was updated" if @missed_reindex.present?),
+        ("#{@missed_delete} was deleted" if @missed_delete.present?)
+      ].compact.join(' and ')
+      message << ' also.'
+
+      output << message
     end
 
     output << @reindex.each.with_object('') do |(id, document), output|
@@ -184,7 +195,7 @@ RSpec::Matchers.define :update_index do |type_name, options = {}|
   end
 
   failure_message_when_negated do
-    if @updated.any?
+    if @updated.present?
       "Expected index `#{type_name}` not to be updated, but it was with #{
         @updated.map(&:values).flatten.group_by { |documents| documents[:_id] }.map do |id, documents|
           "\n  document id `#{id}` (#{documents.count} times)"

data/lib/chewy/strategy.rb

@@ -56,10 +56,10 @@ module Chewy
     end
 
     def wrap name
-      push name
+      stack = push(name)
       yield
     ensure
-      pop
+      pop if stack
     end
 
   private
@@ -72,7 +72,11 @@ module Chewy
     end
 
     def resolve name
-      "Chewy::Strategy::#{name.to_s.camelize}".constantize or raise "Can't find update strategy `#{name}`"
+      "Chewy::Strategy::#{name.to_s.camelize}".safe_constantize or raise "Can't find update strategy `#{name}`"
+    rescue NameError => ex
+      # WORKAROUND: Strange behavior of `safe_constantize` with mongoid gem
+      raise "Can't find update strategy `#{name}`" if ex.name.to_s.demodulize == name.to_s.camelize
+      raise
     end
   end
 end

data/lib/chewy/strategy/active_job.rb

@@ -13,8 +13,8 @@ module Chewy
       class Worker < ::ActiveJob::Base
         queue_as :chewy
 
-        def perform(type, ids)
-          type.constantize.import!(ids)
+        def perform(type, ids, options = {})
+          type.constantize.import!(ids, options)
         end
       end
 

data/lib/chewy/strategy/resque.rb

@@ -13,8 +13,8 @@ module Chewy
       class Worker
         @queue = :chewy
 
-        def self.perform(type, ids)
-          type.constantize.import!(ids)
+        def self.perform(type, ids, options = {})
+          type.constantize.import!(ids, options)
         end
       end
 

data/lib/chewy/strategy/sidekiq.rb

@@ -13,8 +13,8 @@ module Chewy
       class Worker
         include ::Sidekiq::Worker
 
-        def perform(type, ids)
-          type.constantize.import!(ids)
+        def perform(type, ids, options = {})
+          type.constantize.import!(ids, options)
         end
       end
 

data/lib/chewy/type.rb

@@ -5,6 +5,7 @@ require 'chewy/type/observe'
 require 'chewy/type/actions'
 require 'chewy/type/crutch'
 require 'chewy/type/import'
+require 'chewy/type/witchcraft'
 require 'chewy/type/adapter/object'
 require 'chewy/type/adapter/active_record'
 require 'chewy/type/adapter/mongoid'
@@ -12,16 +13,22 @@ require 'chewy/type/adapter/sequel'
 
 module Chewy
   class Type
+    IMPORT_OPTIONS_KEYS = [:batch_size, :bulk_size, :refresh, :consistency, :replication]
+
     include Search
     include Mapping
     include Wrapper
     include Observe
     include Actions
     include Crutch
+    include Witchcraft
     include Import
 
     singleton_class.delegate :index_name, :client, to: :index
 
+    class_attribute :_default_import_options
+    self._default_import_options = {}
+
     # Chewy index current type belongs to. Defined inside `Chewy.create_type`
     #
     def self.index
@@ -47,12 +54,19 @@ module Chewy
       public_methods - Chewy::Type.public_methods
     end
 
+    def self.default_import_options(params)
+      params.assert_valid_keys(IMPORT_OPTIONS_KEYS)
+      self._default_import_options = _default_import_options.merge(params)
+    end
+
     def self.method_missing(method, *args, &block)
       if index.scopes.include?(method)
         define_singleton_method method do |*args, &block|
           all.scoping { index.public_send(method, *args, &block) }
         end
         send(method, *args, &block)
+      else
+        super
       end
     end
 

data/lib/chewy/type/adapter/active_record.rb

@@ -28,7 +28,7 @@ module Chewy
           ids = pluck_ids(scope)
           result = true
 
-          while ids.any?
+          while ids.present?
             result &= yield grouped_objects(default_scope_where_ids_in(ids))
             break if ids.size < batch_size
             ids = pluck_ids(scope.where(target_id.gt(ids.last)))
@@ -57,6 +57,16 @@ module Chewy
           ::ActiveRecord::Base
         end
       end
+
+      module Rails5
+        def pluck_ids(scope)
+          scope.except(:includes).distinct.pluck(target.primary_key.to_sym)
+        end
+      end
+
+      if defined?(::ActiveRecord) && ::ActiveRecord::VERSION::MAJOR >= 5
+        Chewy::Type::Adapter::ActiveRecord.prepend(Rails5)
+      end
     end
   end
 end

data/lib/chewy/type/adapter/orm.rb

@@ -94,13 +94,7 @@ module Chewy
           additional_scope = load_options[load_options[:_type].type_name.to_sym].try(:[], :scope) || load_options[:scope]
 
           scope = all_scope_where_ids_in(objects.map(&primary_key))
-          loaded_objects = if additional_scope.is_a?(Proc)
-            scope.instance_exec(&additional_scope)
-          elsif additional_scope.is_a?(relation_class) && scope.respond_to?(:merge)
-            scope.merge(additional_scope)
-          else
-            scope
-          end.index_by { |object| object.public_send(primary_key).to_s }
+          loaded_objects = load_scope_objects(scope, additional_scope).index_by { |object| object.public_send(primary_key).to_s }
 
           objects.map { |object| loaded_objects[object.public_send(primary_key).to_s] }
         end
@@ -108,16 +102,14 @@ module Chewy
       private
 
         def import_objects(collection, batch_size)
-          hash = collection.index_by do |entity|
-            entity.is_a?(object_class) ? entity.public_send(primary_key) : entity
-          end
+          hash = Hash[identify(collection).zip(collection)]
 
           indexed = hash.keys.each_slice(batch_size).map do |ids|
             batch = default_scope_where_ids_in(ids)
             if batch.empty?
               true
             else
-              batch.each { |object| hash.delete(object.public_send(primary_key)) }
+              identify(batch).each { |id| hash.delete(id) }
               yield grouped_objects(batch)
             end
           end.all?
@@ -148,6 +140,16 @@ module Chewy
         def model_of_relation relation
           relation.klass
         end
+
+        def load_scope_objects(scope, additional_scope = nil)
+          if additional_scope.is_a?(Proc)
+            scope.instance_exec(&additional_scope)
+          elsif additional_scope.is_a?(relation_class) && scope.respond_to?(:merge)
+            scope.merge(additional_scope)
+          else
+            scope
+          end
+        end
       end
     end
   end

data/lib/chewy/type/adapter/sequel.rb

@@ -13,16 +13,6 @@ module Chewy
             target.is_a?(::Sequel::Dataset))
         end
 
-        def identify collection
-          if collection.is_a?(relation_class)
-            pluck_ids(collection)
-          else
-            Array.wrap(collection).map do |entity|
-              entity.is_a?(object_class) ? entity.public_send(primary_key) : entity
-            end
-          end
-        end
-
       private
 
         def cleanup_default_scope!
@@ -39,8 +29,8 @@ module Chewy
           ids = pluck_ids(scope)
           result = true
 
-          while ids.any?
-            result &= yield grouped_objects(default_scope_where_ids_in(ids))
+          while ids.present?
+            result &= yield grouped_objects(default_scope_where_ids_in(ids).all)
             break if ids.size < batch_size
             ids = pluck_ids(scope.where { |o| o.__send__(primary_key) > ids.last })
           end
@@ -52,6 +42,10 @@ module Chewy
           target.primary_key
         end
 
+        def all_scope
+          target.dataset
+        end
+
         def pluck_ids(scope)
           scope.distinct.select_map(primary_key)
         end
@@ -71,6 +65,10 @@ module Chewy
         def object_class
           ::Sequel::Model
         end
+
+        def load_scope_objects(*args)
+          super.all
+        end
       end
     end
   end

data/lib/chewy/type/import.rb

@@ -3,6 +3,8 @@ module Chewy
     module Import
       extend ActiveSupport::Concern
 
+      BULK_OPTIONS = [:suffix, :bulk_size, :refresh, :consistency, :replication]
+
       module ClassMethods
         # Perform import operation for specified documents.
         # Returns true or false depending on success.
@@ -11,15 +13,19 @@ module Chewy
         #   UsersIndex::User.import User.active              # imports active users
         #   UsersIndex::User.import [1, 2, 3]                # imports users with specified ids
         #   UsersIndex::User.import users                    # imports users collection
-        #   UsersIndex::User.import refresh: false           # to disable index refreshing after import
         #   UsersIndex::User.import suffix: Time.now.to_i    # imports data to index with specified suffix if such is exists
+        #   UsersIndex::User.import refresh: false           # to disable index refreshing after import
         #   UsersIndex::User.import batch_size: 300          # import batch size
+        #   UsersIndex::User.import bulk_size: 10.megabytes  # import ElasticSearch bulk size in bytes
+        #   UsersIndex::User.import consistency: :quorum     # explicit write consistency setting for the operation (one, quorum, all)
+        #   UsersIndex::User.import replication: :async      # explicitly set the replication type (sync, async)
         #
         # See adapters documentation for more details.
         #
         def import *args
           import_options = args.extract_options!
-          bulk_options = import_options.reject { |k, v| ![:refresh, :suffix].include?(k) }.reverse_merge!(refresh: true)
+          import_options.reverse_merge! _default_import_options
+          bulk_options = import_options.reject { |k, _| !BULK_OPTIONS.include?(k) }.reverse_merge!(refresh: true)
 
           index.create!(bulk_options.slice(:suffix)) unless index.exists?
 
@@ -28,7 +34,7 @@ module Chewy
               indexed_objects = build_root.parent_id && fetch_indexed_objects(action_objects.values.flatten)
               body = bulk_body(action_objects, indexed_objects)
 
-              errors = bulk(bulk_options.merge(body: body)) if body.any?
+              errors = bulk(bulk_options.merge(body: body)) if body.present?
 
               fill_payload_import payload, action_objects
               fill_payload_errors payload, errors if errors.present?
@@ -39,15 +45,7 @@ module Chewy
 
         # Perform import operation for specified documents.
         # Raises Chewy::ImportFailed exception in case of import errors.
-        #
-        #   UsersIndex::User.import!                          # imports default data set
-        #   UsersIndex::User.import! User.active              # imports active users
-        #   UsersIndex::User.import! [1, 2, 3]                # imports users with specified ids
-        #   UsersIndex::User.import! users                    # imports users collection
-        #   UsersIndex::User.import! refresh: false           # to disable index refreshing after import
-        #   UsersIndex::User.import! suffix: Time.now.to_i    # imports data to index with specified suffix if such is exists
-        #   UsersIndex::User.import! batch_size: 300          # import batch size
-        #
+        # Options are completely the same as for `import` method
         # See adapters documentation for more details.
         #
         def import! *args
@@ -66,20 +64,46 @@ module Chewy
         # Adds `:suffix` option to bulk import to index with specified suffix.
         def bulk options = {}
           suffix = options.delete(:suffix)
+          bulk_size = options.delete(:bulk_size)
+          body = options.delete(:body)
+          header = { index: index.build_index_name(suffix: suffix), type: type_name }
+
+          bodies = if bulk_size
+            bulk_size -= 1.kilobyte # 1 kilobyte for request header and newlines
+            raise ArgumentError.new('Import `:bulk_size` can\'t be less then 1 kilobyte') if bulk_size <= 0
+
+            body.each_with_object(['']) do |entry, result|
+              operation, meta = entry.to_a.first
+              data = meta.delete(:data)
+              entry = [{ operation => meta }, data].compact.map(&:to_json).join("\n")
+              if entry.bytesize > bulk_size
+                raise ArgumentError.new('Import `:bulk_size` seems to be less then entry size')
+              elsif result.last.bytesize + entry.bytesize > bulk_size
+                result.push(entry)
+              else
+                result[-1] = [result[-1], entry].delete_if(&:blank?).join("\n")
+              end
+            end
+          else
+            [body]
+          end
 
-          result = client.bulk options.merge(index: index.build_index_name(suffix: suffix), type: type_name)
+          items = bodies.map do |body|
+            result = client.bulk options.merge(header).merge(body: body)
+            result.try(:[], 'items') || []
+          end.flatten
           Chewy.wait_for_status
 
-          extract_errors result
+          extract_errors items
         end
 
       private
 
         def bulk_body(action_objects, indexed_objects = nil)
-          action_objects.inject([]) do |result, (action, objects)|
+          action_objects.flat_map do |action, objects|
             method = "#{action}_bulk_entry"
             crutches = Chewy::Type::Crutch::Crutches.new self, objects
-            result.concat(objects.map { |object| send(method, object, indexed_objects, crutches) }.flatten)
+            objects.flat_map { |object| send(method, object, indexed_objects, crutches) }
           end
         end
 
@@ -150,15 +174,21 @@ module Chewy
         end
 
         def object_data object, crutches = nil
-          build_root.compose(object, crutches)[type_name.to_sym]
+          if witchcraft?
+            cauldron.brew(object, crutches)
+          else
+            build_root.compose(object, crutches)[type_name.to_sym]
+          end
         end
 
-        def extract_errors result
-          result && result['items'].map do |item|
+        def extract_errors items
+          items.each.with_object({}) do |item, memo|
             action = item.keys.first.to_sym
             data = item.values.first
-            {action: action, id: data['_id'], error: data['error']} if data['error']
-          end.compact.group_by { |item| item[:action] }.map do |action, items|
+            if data['error']
+              (memo[action] ||= []).push(action: action, id: data['_id'], error: data['error'])
+            end
+          end.map do |action, items|
             errors = items.group_by { |item| item[:error] }.map do |error, items|
               {error => items.map { |item| item[:id] }}
             end.reduce(&:merge)
@@ -181,7 +211,8 @@ module Chewy
             break if result['hits']['hits'].empty?
 
             result['hits']['hits'].map do |hit|
-              indexed_objects[hit['_id']] = { parent: hit['fields']['_parent'] }
+              parent = hit.has_key?('_parent') ? hit['_parent'] : hit['fields']['_parent']
+              indexed_objects[hit['_id']] = { parent: parent }
             end
           end