chewy 0.8.4 → 5.1.0

data/lib/chewy/strategy.rb

@@ -7,18 +7,28 @@ begin
   require 'resque'
   require 'chewy/strategy/resque'
 rescue LoadError
+  nil
 end
 
 begin
   require 'sidekiq'
   require 'chewy/strategy/sidekiq'
 rescue LoadError
+  nil
+end
+
+begin
+  require 'shoryuken'
+  require 'chewy/strategy/shoryuken'
+rescue LoadError
+  nil
 end
 
 begin
   require 'active_job'
   require 'chewy/strategy/active_job'
 rescue LoadError
+  nil
 end
 
 module Chewy
@@ -42,20 +52,20 @@ module Chewy
       @stack.last
     end
 
-    def push name
+    def push(name)
       result = @stack.push resolve(name).new
-      debug "[#{@stack.size}] <- #{current.name}"
+      debug "[#{@stack.size - 1}] <- #{current.name}" if @stack.size > 2
       result
     end
 
     def pop
       raise "Can't pop root strategy" if @stack.one?
-      debug "[#{@stack.size}] -> #{current.name}"
       result = @stack.pop.tap(&:leave)
+      debug "[#{@stack.size}] -> #{result.name}, now #{current.name}" if @stack.size > 1
       result
     end
 
-    def wrap name
+    def wrap(name)
       stack = push(name)
       yield
     ensure
@@ -64,14 +74,13 @@ module Chewy
 
   private
 
-    def debug string
-      if Chewy.logger && Chewy.logger.debug?
-        line = caller.detect { |line| line !~ %r{lib/chewy/strategy.rb:|lib/chewy.rb:} }
-        Chewy.logger.debug(["DEBUG: Chewy strategies stack: #{string}", line.sub(/:in\s.+$/, '')].join(' @ '))
-      end
+    def debug(string)
+      return unless Chewy.logger && Chewy.logger.debug?
+      line = caller.detect { |l| l !~ %r{lib/chewy/strategy.rb:|lib/chewy.rb:} }
+      Chewy.logger.debug(["Chewy strategies stack: #{string}", line.sub(/:in\s.+$/, '')].join(' @ '))
     end
 
-    def resolve name
+    def resolve(name)
       "Chewy::Strategy::#{name.to_s.camelize}".safe_constantize or raise "Can't find update strategy `#{name}`"
     rescue NameError => ex
       # WORKAROUND: Strange behavior of `safe_constantize` with mongoid gem

data/lib/chewy/type/actions.rb

@@ -4,15 +4,39 @@ module Chewy
       extend ActiveSupport::Concern
 
       module ClassMethods
-        # Delete all documents of a type and reimport them
-        # Returns true or false depending on success.
+        # Deletes all documents of a type and reimports them
         #
+        # @example
         #   UsersIndex::User.reset
         #
+        # @see Chewy::Type::Import::ClassMethods#import
+        # @see Chewy::Type::Import::ClassMethods#import
+        # @return [true, false] the result of import
         def reset
           delete_all
           import
         end
+
+        # Performs missing and outdated objects synchronization for the current type.
+        #
+        # @example
+        #   UsersIndex::User.sync
+        #
+        # @see Chewy::Type::Syncer
+        # @param parallel [true, Integer, Hash] options for parallel execution or the number of processes
+        # @return [Hash{Symbol, Object}, nil] a number of missing and outdated documents reindexed and their ids, nil in case of errors
+        def sync(parallel: nil)
+          syncer = Syncer.new(self, parallel: parallel)
+          count = syncer.perform
+          {count: count, missing: syncer.missing_ids, outdated: syncer.outdated_ids} if count
+        end
+
+        # A {Chewy::Journal} instance for the particular type
+        #
+        # @return [Chewy::Journal] journal instance
+        def journal
+          @journal ||= Chewy::Journal.new(self)
+        end
       end
     end
   end

data/lib/chewy/type/adapter/active_record.rb

@@ -4,7 +4,6 @@ module Chewy
   class Type
     module Adapter
       class ActiveRecord < Orm
-
         def self.accepts?(target)
           defined?(::ActiveRecord::Base) && (
             target.is_a?(Class) && target < ::ActiveRecord::Base ||
@@ -22,33 +21,70 @@ module Chewy
           @default_scope = @default_scope.reorder(nil).limit(nil).offset(nil)
         end
 
-        def import_scope(scope, batch_size)
-          scope = scope.reorder(target_id.asc).limit(batch_size)
-
-          ids = pluck_ids(scope)
-          result = true
+        def import_scope(scope, options)
+          pluck_in_batches(scope, options.slice(:batch_size)).inject(true) do |result, ids|
+            objects = if options[:raw_import]
+              raw_default_scope_where_ids_in(ids, options[:raw_import])
+            else
+              default_scope_where_ids_in(ids)
+            end
 
-          while ids.present?
-            result &= yield grouped_objects(default_scope_where_ids_in(ids))
-            break if ids.size < batch_size
-            ids = pluck_ids(scope.where(target_id.gt(ids.last)))
+            result & yield(grouped_objects(objects))
           end
+        end
 
-          result
+        def primary_key
+          @primary_key ||= target.primary_key.to_sym
         end
 
         def target_id
-          target.arel_table[target.primary_key]
+          target.arel_table[primary_key.to_s]
         end
 
-        def pluck_ids(scope)
-          scope.except(:includes).uniq.pluck(target.primary_key.to_sym)
+        def pluck(scope, fields: [], typecast: true)
+          if typecast
+            scope.except(:includes).distinct.pluck(primary_key, *fields)
+          else
+            scope = scope.except(:includes).distinct
+            scope.select_values = [primary_key, *fields].map do |column|
+              target.columns_hash.key?(column) ? target.arel_table[column] : column
+            end
+            sql = scope.to_sql
+
+            if fields.present?
+              target.connection.select_rows(sql)
+            else
+              target.connection.select_values(sql)
+            end
+          end
+        end
+
+        def pluck_in_batches(scope, fields: [], batch_size: nil, typecast: true)
+          return enum_for(:pluck_in_batches, scope, fields: fields, batch_size: batch_size, typecast: typecast) unless block_given?
+
+          scope = scope.reorder(target_id.asc).limit(batch_size)
+          ids = pluck(scope, fields: fields, typecast: typecast)
+          count = 0
+
+          while ids.present?
+            yield ids
+            break if ids.size < batch_size
+            last_id = ids.last.is_a?(Array) ? ids.last.first : ids.last
+            ids = pluck(scope.where(target_id.gt(last_id)), fields: fields, typecast: typecast)
+          end
+
+          count
         end
 
         def scope_where_ids_in(scope, ids)
           scope.where(target_id.in(Array.wrap(ids)))
         end
 
+        def raw_default_scope_where_ids_in(ids, converter)
+          sql = default_scope_where_ids_in(ids).to_sql
+          object_class.connection.execute(sql).map(&converter)
+        end
+
         def relation_class
           ::ActiveRecord::Relation
         end
@@ -57,16 +93,6 @@ module Chewy
           ::ActiveRecord::Base
         end
       end
-
-      module Rails5
-        def pluck_ids(scope)
-          scope.except(:includes).distinct.pluck(target.primary_key.to_sym)
-        end
-      end
-
-      if defined?(::ActiveRecord) && ::ActiveRecord::VERSION::MAJOR >= 5
-        Chewy::Type::Adapter::ActiveRecord.prepend(Rails5)
-      end
     end
   end
 end

data/lib/chewy/type/adapter/base.rb

@@ -9,7 +9,7 @@ module Chewy
 
         # Returns `true` if this adapter is applicable for the given target.
         #
-        def self.accepts? target
+        def self.accepts?(_target)
           true
         end
 
@@ -32,28 +32,48 @@ module Chewy
         # For ORM/ODM it will be an array of ids for simple objects -
         # just objects themselves
         #
-        def identify collection
+        def identify(_collection)
           raise NotImplementedError
         end
 
         # Splits passed objects to groups according to `:batch_size` options.
-        # For every group crates hash with action keys. Example:
+        # For every group creates hash with action keys. Example:
         #
-        #   { delete: [object1, object2], index: [object3, object4, object5] }
+        #   { delete: [object_or_id1, object_or_id2], index: [object3, object4, object5] }
         #
-        # Returns true id all the block call returns true and false otherwise
+        # @yieldparam _batch [Array<Object>] each batch of objects
+        # @return [true, false] returns true if all the block call returns true and false otherwise
+        def import(_batch, &_block)
+          raise NotImplementedError
+        end
+
+        # Unlike {#import} fetches only ids (references) to the imported objects,
+        # using the same procedures as {#import}.
+        #
+        # @param _fields [Array<Symbol>] additional fields to fetch
+        # @param _batch_size [Integer] batch size, defaults to 1000
+        # @yieldparam batch [Array<Object>] each batch of objects
+        def import_fields(_fields, _batch_size, &_block)
+          raise NotImplementedError
+        end
+
+        # Uses the same strategy as import for the passed arguments, and returns
+        # an array of references to the passed objects. Returns ids if possible.
+        # Otherwise - and array of objects themselves.
         #
-        def import *args, &block
+        # @param _batch_size [Integer] batch size, defaults to 1000
+        # @yieldparam batch [Array<Object>] each batch of objects
+        def import_references(_batch_size, &_block)
           raise NotImplementedError
         end
 
-        # Returns array of loaded objects for passed objects array. If some object
+        # Returns array of loaded objects for passed ids array. If some object
         # was not loaded, it returns `nil` in the place of this object
         #
-        #   load(double(id: 1), double(id: 2), double(id: 3)) #=>
+        #   load([1, 2, 3]) #=>
         #     # [<Product id: 1>, nil, <Product id: 3>], assuming, #2 was not found
         #
-        def load *args
+        def load(_ids, **_options)
           raise NotImplementedError
         end
 

data/lib/chewy/type/adapter/mongoid.rb

@@ -4,41 +4,50 @@ module Chewy
   class Type
     module Adapter
       class Mongoid < Orm
-
         def self.accepts?(target)
           defined?(::Mongoid::Document) && (
             target.is_a?(Class) && target.ancestors.include?(::Mongoid::Document) ||
             target.is_a?(::Mongoid::Criteria))
         end
 
-        def identify collection
+        def identify(collection)
           super(collection).map { |id| id.is_a?(BSON::ObjectId) ? id.to_s : id }
         end
 
       private
 
         def cleanup_default_scope!
-          if Chewy.logger && @default_scope.options.values_at(:sort, :limit, :skip).compact.present?
-            Chewy.logger.warn('Default type scope order, limit and offset are ignored and will be nullified')
-          end
+          Chewy.logger.warn('Default type scope order, limit and offset are ignored and will be nullified') if Chewy.logger && @default_scope.options.values_at(:sort, :limit, :skip).compact.present?
 
           @default_scope.options.delete(:limit)
           @default_scope.options.delete(:skip)
           @default_scope = @default_scope.reorder(nil)
         end
 
-        def import_scope(scope, batch_size)
-          scope.batch_size(batch_size).no_timeout.pluck(:_id).each_slice(batch_size).map do |ids|
+        def import_scope(scope, options)
+          pluck_in_batches(scope, options.slice(:batch_size)).map do |ids|
             yield grouped_objects(default_scope_where_ids_in(ids))
           end.all?
         end
 
-        def pluck_ids(scope)
-          scope.pluck(:_id)
+        def primary_key
+          :_id
+        end
+
+        def pluck(scope, fields: [])
+          scope.pluck(primary_key, *fields)
+        end
+
+        def pluck_in_batches(scope, fields: [], batch_size: nil, **options)
+          return enum_for(:pluck_in_batches, scope, fields: fields, batch_size: batch_size, **options) unless block_given?
+
+          scope.batch_size(batch_size).no_timeout.pluck(primary_key, *fields).each_slice(batch_size) do |batch|
+            yield batch
+          end
         end
 
         def scope_where_ids_in(scope, ids)
-          scope.where(:_id.in => ids)
+          scope.where(primary_key.in => ids)
         end
 
         def all_scope

data/lib/chewy/type/adapter/object.rb

@@ -3,64 +3,203 @@ require 'chewy/type/adapter/base'
 module Chewy
   class Type
     module Adapter
+      # This adapter provides an ability to import documents from any
+      # source. You can actually use any class or even a symbol as
+      # a target.
+      #
+      # In case if a class is used - some of the additional features
+      # are available: it is possible to provide the default import
+      # data (used on reset) and source objects loading logic.
+      #
+      # @see #import
+      # @see #load
       class Object < Base
-        def initialize *args
-          @options = args.extract_options!
-          @target = args.first
+        # The signature of the type definition.
+        #
+        # @example
+        #   define_type :geoname
+        #   define_type Geoname
+        #   define_type -> { Geoname.all_the_places }, name: 'geoname'
+        #
+        # @param target [Class, Symbol, String, Proc] a source of data and everything
+        # @option options [String, Symbol] :name redefines the inferred type name if necessary
+        # @option options [String, Symbol] :import_all_method redefines import method name
+        # @option options [String, Symbol] :load_all_method redefines batch load method name
+        # @option options [String, Symbol] :load_one_method redefines per-object load method name
+        def initialize(target, **options)
+          @target = target
+          @options = options
         end
 
+        # Name is used for the type class creation. Inferred from the target
+        # by default if possible.
+        #
+        # @example
+        #   # defines MyIndex::Geoname
+        #   define_type :geoname
+        #   # still defines MyIndex::Geoname
+        #   define_type -> { Geoname.all_the_places }, name: 'geoname'
+        #
+        # @return [String]
         def name
           @name ||= (options[:name] || @target).to_s.camelize.demodulize
         end
 
-        def identify collection
+        # While for ORM adapters it returns an array of ids for the passed
+        # collection, for the object adapter it returns the collection itself.
+        #
+        # @param collection [Array<Object>, Object] a collection or an object
+        # @return [Array<Object>]
+        def identify(collection)
           Array.wrap(collection)
         end
 
-        # Imports passed data with options
+        # This method is used internally by `Chewy::Type.import`.
         #
-        # Import data types:
+        # The idea is that any object can be imported to ES if
+        # it responds to `#to_json` method.
         #
-        #   * Array ob objects
+        # If method `destroyed?` is defined for object (or, in case of hash object,
+        # it has `:_destroyed` or `'_destroyed'` key) and returns `true` or object
+        # satisfy `delete_if` type option then object will be deleted from index.
+        # But in order to be destroyable, objects need to respond to `id` method
+        # or have an `id` key so ElasticSearch could know which one to delete.
         #
-        # Import options:
+        # If nothing is passed the method tries to call `import_all_method`,
+        # which is `call` by default, on target to get the default objects batch.
         #
-        #   <tt>:batch_size</tt> - import batch size, 1000 objects by default
+        # @example
+        #   class Geoname
+        #     self < class
+        #       def self.call
+        #         FancyGeoAPI.all_points_collection
+        #       end
+        #       alias_method :import_all, :call
+        #     end
+        #   end
         #
-        # If method `destroyed?` is defined for object and returns true or object
-        # satisfy `delete_if` type option then object will be deleted from index.
-        # But to be destroyed objects need to respond to `id` method as well, so
-        # ElasticSearch could know which one to delete.
+        #   # All the folloving variants will work:
+        #   define_type Geoname
+        #   define_type Geoname, import_all_method: 'import_all'
+        #   define_type -> { FancyGeoAPI.all_points_collection }, name: 'geoname'
+        #
+        # @param args [Array<#to_json>]
+        # @option options [Integer] :batch_size import processing batch size
+        # @return [true, false]
+        def import(*args, &block)
+          collection, options = import_args(*args)
+          import_objects(collection, options, &block)
+        end
+
+        # For the object adapter this method tries to fetch :id and requested
+        # fields from the passed collection or the target's `import_all_method`
+        # when defined. Otherwise it tries to call the target `pluck_method`,
+        # which is configurable and `pluck` by default. The `pluck_method` have
+        # to act exactly the same way as the AR one. It returns an empty array
+        # when none of the methods are found.
+        #
+        # @example
+        #   class Geoname
+        #     self < class
+        #       def self.pluck(*fields)
+        #         if fields.one?
+        #           whatever_source.map { |object| object.send(fields.first) }
+        #         else
+        #           whatever_source.map do |object|
+        #             fields.map { |field| object.send(field) }
+        #           end
+        #         end
+        #       end
+        #     end
+        #   end
         #
-        def import *args, &block
-          import_options = args.extract_options!
-          batch_size = import_options.delete(:batch_size) || BATCH_SIZE
+        # @see Chewy::Type::Adapter::Base#import_fields
+        def import_fields(*args)
+          return enum_for(:import_fields, *args) unless block_given?
+          options = args.extract_options!
+          options[:batch_size] ||= BATCH_SIZE
 
-          objects = args.empty? && @target.respond_to?(import_all_method) ?
-            @target.send(import_all_method) : args.flatten.compact
+          if args.empty? && @target.respond_to?(pluck_method)
+            @target.send(pluck_method, :id, *options[:fields]).each_slice(options[:batch_size]) do |batch|
+              yield batch
+            end
+          elsif options[:fields].blank?
+            import_references(*args, options) do |batch|
+              yield batch.map { |object| object_field(object, :id) || object }
+            end
+          else
+            import_references(*args, options) do |batch|
+              batch = batch.map do |object|
+                options[:fields].map { |field| object_field(object, field) }
+                  .unshift(object_field(object, :id) || object)
+              end
+              yield batch
+            end
+          end
+        end
 
-          import_objects(objects, batch_size, &block)
+        # For the Object adapter returns the objects themselves in batches.
+        #
+        # @see Chewy::Type::Adapter::Base#import_references
+        def import_references(*args)
+          return enum_for(:import_references, *args) unless block_given?
+
+          collection, options = import_args(*args)
+          collection.each_slice(options[:batch_size]) do |batch|
+            yield batch
+          end
         end
 
-        def load *args
-          load_options = args.extract_options!
-          objects = args.flatten
+        # This method is used internally by the request DSL when the
+        # collection of ORM/ODM objects is requested.
+        #
+        # Options usage is implemented by `load_all_method` and `load_one_method`.
+        #
+        # If none of the `load_all_method` or `load_one_method` is implemented
+        # for the target - the method will return nil. This means that the
+        # loader will return an array `Chewy::Type` objects that actually was passed.
+        #
+        # To use loading for objects it is obviously required to provide
+        # some meaningful ids for ES documents.
+        #
+        # @example
+        #   class Geoname
+        #     def self.load_all(wrappers, options)
+        #       if options[:additional_data]
+        #         wrappers.map do |wrapper|
+        #           FancyGeoAPI.point_by_name(wrapper.name)
+        #         end
+        #       else
+        #         wrappers
+        #       end
+        #     end
+        #   end
+        #
+        #   MyIndex::Geoname.load(additional_data: true).objects
+        #
+        # @param ids [Array<Hash>] an array of ids from ES hits
+        # @param options [Hash] any options passed here with the request DSL `load` method.
+        # @return [Array<Object>, nil]
+        def load(ids, **options)
           if target.respond_to?(load_all_method)
-            target.send(load_all_method, objects)
+            if target.method(load_all_method).arity == 1
+              target.send(load_all_method, ids)
+            else
+              target.send(load_all_method, ids, options)
+            end
           elsif target.respond_to?(load_one_method)
-            objects.map { |object| target.send(load_one_method, object) }
-          elsif target.respond_to?(:wrap)
-            ActiveSupport::Deprecation.warn('Loading with `wrap` method is deprecated. Rename it to `load_one` or pass `load_one_method: :my_load_method` option to `define_type`')
-            objects.map { |object| target.wrap(object) }
-          else
-            objects
+            if target.method(load_one_method).arity == 1
+              ids.map { |hit| target.send(load_one_method, hit) }
+            else
+              ids.map { |hit| target.send(load_one_method, hit, options) }
+            end
           end
         end
 
       private
 
-        def import_objects(objects, batch_size)
-          objects.each_slice(batch_size).map do |group|
+        def import_objects(objects, options)
+          objects.each_slice(options[:batch_size]).map do |group|
             yield grouped_objects(group)
           end.all?
         end
@@ -72,10 +211,22 @@ module Chewy
           !!delete
         end
 
+        def object_field(object, name)
+          if object.respond_to?(name)
+            object.send(name)
+          elsif object.is_a?(Hash)
+            object[name.to_sym] || object[name.to_s]
+          end
+        end
+
         def import_all_method
           @import_all_method ||= options[:import_all_method] || :call
         end
 
+        def pluck_method
+          @pluck_method ||= options[:pluck_method] || :pluck
+        end
+
         def load_all_method
           @load_all_method ||= options[:load_all_method] || :load_all
         end
@@ -83,6 +234,19 @@ module Chewy
         def load_one_method
           @load_one_method ||= options[:load_one_method] || :load_one
         end
+
+        def import_args(*args)
+          options = args.extract_options!
+          options[:batch_size] ||= BATCH_SIZE
+
+          collection = if args.empty? && @target.respond_to?(import_all_method)
+            @target.send(import_all_method)
+          else
+            args.flatten(1).compact
+          end
+
+          [collection, options]
+        end
       end
     end
   end