chewy 0.8.4 → 0.9.0

data/lib/chewy/type/adapter/active_record.rb

@@ -4,7 +4,6 @@ module Chewy
   class Type
     module Adapter
       class ActiveRecord < Orm
-
         def self.accepts?(target)
           defined?(::ActiveRecord::Base) && (
             target.is_a?(Class) && target < ::ActiveRecord::Base ||
@@ -22,15 +21,21 @@ module Chewy
           @default_scope = @default_scope.reorder(nil).limit(nil).offset(nil)
         end
 
-        def import_scope(scope, batch_size)
-          scope = scope.reorder(target_id.asc).limit(batch_size)
+        def import_scope(scope, options)
+          scope = scope.reorder(target_id.asc).limit(options[:batch_size])
 
           ids = pluck_ids(scope)
           result = true
 
           while ids.present?
-            result &= yield grouped_objects(default_scope_where_ids_in(ids))
-            break if ids.size < batch_size
+            objects =
+              if options[:raw_import]
+                raw_default_scope_where_ids_in(ids, options[:raw_import])
+              else
+                default_scope_where_ids_in(ids)
+              end
+            result &= yield grouped_objects(objects)
+            break if ids.size < options[:batch_size]
             ids = pluck_ids(scope.where(target_id.gt(ids.last)))
           end
 
@@ -49,6 +54,11 @@ module Chewy
           scope.where(target_id.in(Array.wrap(ids)))
         end
 
+        def raw_default_scope_where_ids_in(ids, converter)
+          sql = default_scope_where_ids_in(ids).to_sql
+          object_class.connection.execute(sql).map(&converter)
+        end
+
         def relation_class
           ::ActiveRecord::Relation
         end
@@ -58,15 +68,16 @@ module Chewy
         end
       end
 
-      module Rails5
-        def pluck_ids(scope)
-          scope.except(:includes).distinct.pluck(target.primary_key.to_sym)
+      ActiveSupport.on_load(:active_record) do
+        if ::ActiveRecord::VERSION::MAJOR >= 5
+          module Rails5
+            def pluck_ids(scope)
+              scope.except(:includes).distinct.pluck(target.primary_key.to_sym)
+            end
+          end
+          Chewy::Type::Adapter::ActiveRecord.prepend(Rails5)
         end
       end
-
-      if defined?(::ActiveRecord) && ::ActiveRecord::VERSION::MAJOR >= 5
-        Chewy::Type::Adapter::ActiveRecord.prepend(Rails5)
-      end
     end
   end
 end

data/lib/chewy/type/adapter/base.rb

@@ -9,7 +9,7 @@ module Chewy
 
         # Returns `true` if this adapter is applicable for the given target.
         #
-        def self.accepts? target
+        def self.accepts?(_target)
           true
         end
 
@@ -32,7 +32,7 @@ module Chewy
         # For ORM/ODM it will be an array of ids for simple objects -
         # just objects themselves
         #
-        def identify collection
+        def identify(_collection)
           raise NotImplementedError
         end
 
@@ -43,7 +43,7 @@ module Chewy
         #
         # Returns true id all the block call returns true and false otherwise
         #
-        def import *args, &block
+        def import(*_args)
           raise NotImplementedError
         end
 
@@ -53,7 +53,7 @@ module Chewy
         #   load(double(id: 1), double(id: 2), double(id: 3)) #=>
         #     # [<Product id: 1>, nil, <Product id: 3>], assuming, #2 was not found
         #
-        def load *args
+        def load(*_args)
           raise NotImplementedError
         end
 

data/lib/chewy/type/adapter/mongoid.rb

@@ -4,14 +4,13 @@ module Chewy
   class Type
     module Adapter
       class Mongoid < Orm
-
         def self.accepts?(target)
           defined?(::Mongoid::Document) && (
             target.is_a?(Class) && target.ancestors.include?(::Mongoid::Document) ||
             target.is_a?(::Mongoid::Criteria))
         end
 
-        def identify collection
+        def identify(collection)
           super(collection).map { |id| id.is_a?(BSON::ObjectId) ? id.to_s : id }
         end
 
@@ -27,10 +26,11 @@ module Chewy
           @default_scope = @default_scope.reorder(nil)
         end
 
-        def import_scope(scope, batch_size)
-          scope.batch_size(batch_size).no_timeout.pluck(:_id).each_slice(batch_size).map do |ids|
-            yield grouped_objects(default_scope_where_ids_in(ids))
-          end.all?
+        def import_scope(scope, options)
+          scope.batch_size(options[:batch_size]).no_timeout.pluck(:_id)
+            .each_slice(options[:batch_size]).map do |ids|
+              yield grouped_objects(default_scope_where_ids_in(ids))
+            end.all?
         end
 
         def pluck_ids(scope)

data/lib/chewy/type/adapter/object.rb

@@ -4,7 +4,7 @@ module Chewy
   class Type
     module Adapter
       class Object < Base
-        def initialize *args
+        def initialize(*args)
           @options = args.extract_options!
           @target = args.first
         end
@@ -13,7 +13,7 @@ module Chewy
           @name ||= (options[:name] || @target).to_s.camelize.demodulize
         end
 
-        def identify collection
+        def identify(collection)
           Array.wrap(collection)
         end
 
@@ -32,18 +32,21 @@ module Chewy
         # But to be destroyed objects need to respond to `id` method as well, so
         # ElasticSearch could know which one to delete.
         #
-        def import *args, &block
-          import_options = args.extract_options!
-          batch_size = import_options.delete(:batch_size) || BATCH_SIZE
+        def import(*args, &block)
+          options = args.extract_options!
+          options[:batch_size] ||= BATCH_SIZE
 
-          objects = args.empty? && @target.respond_to?(import_all_method) ?
-            @target.send(import_all_method) : args.flatten.compact
+          objects = if args.empty? && @target.respond_to?(import_all_method)
+            @target.send(import_all_method)
+          else
+            args.flatten.compact
+          end
 
-          import_objects(objects, batch_size, &block)
+          import_objects(objects, options, &block)
         end
 
-        def load *args
-          load_options = args.extract_options!
+        def load(*args)
+          args.extract_options!
           objects = args.flatten
           if target.respond_to?(load_all_method)
             target.send(load_all_method, objects)
@@ -59,8 +62,8 @@ module Chewy
 
       private
 
-        def import_objects(objects, batch_size)
-          objects.each_slice(batch_size).map do |group|
+        def import_objects(objects, options)
+          objects.each_slice(options[:batch_size]).map do |group|
             yield grouped_objects(group)
           end.all?
         end

data/lib/chewy/type/adapter/orm.rb

@@ -6,7 +6,7 @@ module Chewy
       class Orm < Base
         attr_reader :default_scope
 
-        def initialize *args
+        def initialize(*args)
           @options = args.extract_options!
           class_or_relation = args.first
           if class_or_relation.is_a?(relation_class)
@@ -23,7 +23,7 @@ module Chewy
           @name ||= (options[:name].presence || target.name).to_s.camelize.demodulize
         end
 
-        def identify collection
+        def identify(collection)
           if collection.is_a?(relation_class)
             pluck_ids(collection)
           else
@@ -73,21 +73,26 @@ module Chewy
         #   # or
         #   UsersIndex::User.import users.map(&:id) # user ids will be deleted from index
         #
-        def import *args, &block
-          import_options = args.extract_options!
-          batch_size = import_options[:batch_size] || BATCH_SIZE
-
-          collection = args.empty? ? default_scope :
-            (args.one? && args.first.is_a?(relation_class) ? args.first : args.flatten.compact)
+        def import(*args, &block)
+          options = args.extract_options!
+          options[:batch_size] ||= BATCH_SIZE
+
+          collection = if args.empty?
+            default_scope
+          elsif args.one? && args.first.is_a?(relation_class)
+            args.first
+          else
+            args.flatten.compact
+          end
 
           if collection.is_a?(relation_class)
-            import_scope(collection, batch_size, &block)
+            import_scope(collection, options, &block)
           else
-            import_objects(collection, batch_size, &block)
+            import_objects(collection, options, &block)
           end
         end
 
-        def load *args
+        def load(*args)
           load_options = args.extract_options!
           objects = args.flatten
 
@@ -101,10 +106,10 @@ module Chewy
 
       private
 
-        def import_objects(collection, batch_size)
+        def import_objects(collection, options)
           hash = Hash[identify(collection).zip(collection)]
 
-          indexed = hash.keys.each_slice(batch_size).map do |ids|
+          indexed = hash.keys.each_slice(options[:batch_size]).map do |ids|
             batch = default_scope_where_ids_in(ids)
             if batch.empty?
               true
@@ -114,7 +119,7 @@ module Chewy
             end
           end.all?
 
-          deleted = hash.keys.each_slice(batch_size).map do |group|
+          deleted = hash.keys.each_slice(options[:batch_size]).map do |group|
             yield delete: hash.values_at(*group)
           end.all?
 
@@ -137,7 +142,7 @@ module Chewy
           target.where(nil)
         end
 
-        def model_of_relation relation
+        def model_of_relation(relation)
           relation.klass
         end
 
@@ -150,6 +155,10 @@ module Chewy
             scope
           end
         end
+
+        def grouped_objects(objects)
+          options[:delete_if] ? super : { index: objects.to_a }
+        end
       end
     end
   end

data/lib/chewy/type/adapter/sequel.rb

@@ -23,16 +23,16 @@ module Chewy
           @default_scope = @default_scope.unordered.unlimited
         end
 
-        def import_scope(scope, batch_size)
-          scope = scope.unordered.order(::Sequel.asc(primary_key)).limit(batch_size)
+        def import_scope(scope, options)
+          scope = scope.unordered.order(::Sequel.asc(primary_key_with_table_name)).limit(options[:batch_size])
 
           ids = pluck_ids(scope)
           result = true
 
           while ids.present?
             result &= yield grouped_objects(default_scope_where_ids_in(ids).all)
-            break if ids.size < batch_size
-            ids = pluck_ids(scope.where { |o| o.__send__(primary_key) > ids.last })
+            break if ids.size < options[:batch_size]
+            ids = pluck_ids(scope.where { |o| o.__send__(primary_key_with_table_name) > ids.last })
           end
 
           result
@@ -42,19 +42,23 @@ module Chewy
           target.primary_key
         end
 
+        def primary_key_with_table_name
+          "#{target.table_name}__#{primary_key}".to_sym
+        end
+
         def all_scope
           target.dataset
         end
 
         def pluck_ids(scope)
-          scope.distinct.select_map(primary_key)
+          scope.distinct.select_map(primary_key_with_table_name)
         end
 
         def scope_where_ids_in(scope, ids)
-          scope.where(primary_key => Array.wrap(ids))
+          scope.where(primary_key_with_table_name => Array.wrap(ids))
         end
 
-        def model_of_relation relation
+        def model_of_relation(relation)
           relation.model
         end
 

data/lib/chewy/type/crutch.rb

@@ -9,8 +9,9 @@ module Chewy
       end
 
       class Crutches
-        def initialize type, collection
-          @type, @collection = type, collection
+        def initialize(type, collection)
+          @type = type
+          @collection = collection
           @type._crutches.keys.each do |name|
             singleton_class.class_eval <<-METHOD, __FILE__, __LINE__ + 1
               def #{name}
@@ -22,7 +23,7 @@ module Chewy
       end
 
       module ClassMethods
-        def crutch name, &block
+        def crutch(name, &block)
           self._crutches = _crutches.merge(name.to_sym => block)
         end
       end

data/lib/chewy/type/import.rb

@@ -3,7 +3,7 @@ module Chewy
     module Import
       extend ActiveSupport::Concern
 
-      BULK_OPTIONS = [:suffix, :bulk_size, :refresh, :consistency, :replication]
+      BULK_OPTIONS = [:suffix, :bulk_size, :refresh, :consistency, :replication].freeze
 
       module ClassMethods
         # Perform import operation for specified documents.
@@ -13,8 +13,9 @@ module Chewy
         #   UsersIndex::User.import User.active              # imports active users
         #   UsersIndex::User.import [1, 2, 3]                # imports users with specified ids
         #   UsersIndex::User.import users                    # imports users collection
-        #   UsersIndex::User.import suffix: Time.now.to_i    # imports data to index with specified suffix if such is exists
+        #   UsersIndex::User.import suffix: Time.now.to_i    # imports data to index with specified suffix if such exists
         #   UsersIndex::User.import refresh: false           # to disable index refreshing after import
+        #   UsersIndex::User.import journal: true            # import will record all the actions into special journal index
         #   UsersIndex::User.import batch_size: 300          # import batch size
         #   UsersIndex::User.import bulk_size: 10.megabytes  # import ElasticSearch bulk size in bytes
         #   UsersIndex::User.import consistency: :quorum     # explicit write consistency setting for the operation (one, quorum, all)
@@ -22,7 +23,7 @@ module Chewy
         #
         # See adapters documentation for more details.
         #
-        def import *args
+        def import(*args)
           import_options = args.extract_options!
           import_options.reverse_merge! _default_import_options
           bulk_options = import_options.reject { |k, _| !BULK_OPTIONS.include?(k) }.reverse_merge!(refresh: true)
@@ -31,10 +32,13 @@ module Chewy
 
           ActiveSupport::Notifications.instrument 'import_objects.chewy', type: self do |payload|
             adapter.import(*args, import_options) do |action_objects|
+              journal = Chewy::Journal.new(self)
+              journal.add(action_objects) if import_options.fetch(:journal) { journal? }
+
               indexed_objects = build_root.parent_id && fetch_indexed_objects(action_objects.values.flatten)
               body = bulk_body(action_objects, indexed_objects)
 
-              errors = bulk(bulk_options.merge(body: body)) if body.present?
+              errors = bulk(bulk_options.merge(body: body, journal: journal)) if body.present?
 
               fill_payload_import payload, action_objects
               fill_payload_errors payload, errors if errors.present?
@@ -48,12 +52,12 @@ module Chewy
         # Options are completely the same as for `import` method
         # See adapters documentation for more details.
         #
-        def import! *args
+        def import!(*args)
           errors = nil
-          subscriber = ActiveSupport::Notifications.subscribe('import_objects.chewy') do |*args|
-            errors = args.last[:errors]
+          subscriber = ActiveSupport::Notifications.subscribe('import_objects.chewy') do |*notification_args|
+            errors = notification_args.last[:errors]
           end
-          import *args
+          import(*args)
           raise Chewy::ImportFailed.new(self, errors) if errors.present?
           true
         ensure
@@ -62,34 +66,42 @@ module Chewy
 
         # Wraps elasticsearch-ruby client indices bulk method.
         # Adds `:suffix` option to bulk import to index with specified suffix.
-        def bulk options = {}
+        def bulk(options = {})
           suffix = options.delete(:suffix)
           bulk_size = options.delete(:bulk_size)
           body = options.delete(:body)
+          journal = options.delete(:journal)
           header = { index: index.build_index_name(suffix: suffix), type: type_name }
 
           bodies = if bulk_size
             bulk_size -= 1.kilobyte # 1 kilobyte for request header and newlines
-            raise ArgumentError.new('Import `:bulk_size` can\'t be less then 1 kilobyte') if bulk_size <= 0
+            raise ArgumentError, 'Import `:bulk_size` can\'t be less than 1 kilobyte' if bulk_size <= 0
 
-            body.each_with_object(['']) do |entry, result|
+            entries = body.each_with_object(['']) do |entry, result|
               operation, meta = entry.to_a.first
               data = meta.delete(:data)
               entry = [{ operation => meta }, data].compact.map(&:to_json).join("\n")
-              if entry.bytesize > bulk_size
-                raise ArgumentError.new('Import `:bulk_size` seems to be less then entry size')
-              elsif result.last.bytesize + entry.bytesize > bulk_size
+
+              raise ArgumentError, 'Import `:bulk_size` seems to be less than entry size' if entry.bytesize > bulk_size
+
+              if result.last.bytesize + entry.bytesize > bulk_size
                 result.push(entry)
               else
                 result[-1] = [result[-1], entry].delete_if(&:blank?).join("\n")
               end
             end
+            entries.map { |entry| entry + "\n" }
           else
             [body]
           end
 
-          items = bodies.map do |body|
-            result = client.bulk options.merge(header).merge(body: body)
+          if journal.any_records?
+            Chewy::Journal.create
+            bodies += [journal.bulk_body]
+          end
+
+          items = bodies.map do |item_body|
+            result = client.bulk options.merge(header).merge(body: item_body)
             result.try(:[], 'items') || []
           end.flatten
           Chewy.wait_for_status
@@ -97,6 +109,10 @@ module Chewy
           extract_errors items
         end
 
+        def journal?
+          _default_import_options.fetch(:journal) { Chewy.configuration[:journal] }
+        end
+
       private
 
         def bulk_body(action_objects, indexed_objects = nil)
@@ -107,7 +123,7 @@ module Chewy
           end
         end
 
-        def delete_bulk_entry(object, indexed_objects = nil, crutches = nil)
+        def delete_bulk_entry(object, indexed_objects = nil, _crutches = nil)
           entry = {}
 
           if root_object.id
@@ -121,7 +137,8 @@ module Chewy
 
           if root_object.parent_id
             existing_object = entry[:_id].present? && indexed_objects && indexed_objects[entry[:_id].to_s]
-            entry.merge!(parent: existing_object[:parent]) if existing_object
+            return [] unless existing_object
+            entry[:parent] = existing_object[:parent]
           end
 
           [{ delete: entry }]
@@ -153,7 +170,7 @@ module Chewy
           end
         end
 
-        def fill_payload_import payload, action_objects
+        def fill_payload_import(payload, action_objects)
           imported = Hash[action_objects.map { |action, objects| [action, objects.count] }]
           imported.each do |action, count|
             payload[:import] ||= {}
@@ -162,9 +179,9 @@ module Chewy
           end
         end
 
-        def fill_payload_errors payload, errors
-          errors.each do |action, errors|
-            errors.each do |error, documents|
+        def fill_payload_errors(payload, import_errors)
+          import_errors.each do |action, action_errors|
+            action_errors.each do |error, documents|
               payload[:errors] ||= {}
               payload[:errors][action] ||= {}
               payload[:errors][action][error] ||= []
@@ -173,26 +190,28 @@ module Chewy
           end
         end
 
-        def object_data object, crutches = nil
+        def object_data(object, crutches = nil)
           if witchcraft?
             cauldron.brew(object, crutches)
           else
-            build_root.compose(object, crutches)[type_name.to_sym]
+            build_root.compose(object, crutches)[type_name.to_s]
           end
         end
 
-        def extract_errors items
-          items.each.with_object({}) do |item, memo|
+        def extract_errors(items)
+          items = items.each.with_object({}) do |item, memo|
             action = item.keys.first.to_sym
             data = item.values.first
             if data['error']
               (memo[action] ||= []).push(action: action, id: data['_id'], error: data['error'])
             end
-          end.map do |action, items|
-            errors = items.group_by { |item| item[:error] }.map do |error, items|
-              {error => items.map { |item| item[:id] }}
+          end
+
+          items.map do |action, action_items|
+            errors = action_items.group_by { |item| item[:error] }.map do |error, error_items|
+              { error => error_items.map { |item| item[:id] } }
             end.reduce(&:merge)
-            {action => errors}
+            { action => errors }
           end.reduce(&:merge) || {}
         end
 
@@ -207,11 +226,11 @@ module Chewy
 
           indexed_objects = {}
 
-          while result = client.scroll(scroll_id: result['_scroll_id'], scroll: '1m') do
+          while (result = client.scroll(scroll_id: result['_scroll_id'], scroll: '1m'))
             break if result['hits']['hits'].empty?
 
             result['hits']['hits'].map do |hit|
-              parent = hit.has_key?('_parent') ? hit['_parent'] : hit['fields']['_parent']
+              parent = hit.key?('_parent') ? hit['_parent'] : hit['fields']['_parent']
               indexed_objects[hit['_id']] = { parent: parent }
             end
           end