chewy 0.9.0 → 0.10.0

data/lib/chewy/strategy/resque.rb

@@ -14,6 +14,7 @@ module Chewy
         @queue = :chewy
 
         def self.perform(type, ids, options = {})
+          options[:refresh] = !Chewy.disable_refresh_async if Chewy.disable_refresh_async
           type.constantize.import!(ids, options)
         end
       end

data/lib/chewy/strategy/shoryuken.rb

@@ -0,0 +1,40 @@
+module Chewy
+  class Strategy
+    # The strategy works the same way as atomic, but performs
+    # async index update driven by shoryuken
+    #
+    #   Chewy.strategy(:shoryuken) do
+    #     User.all.map(&:save) # Does nothing here
+    #     Post.all.map(&:save) # And here
+    #     # It imports all the changed users and posts right here
+    #   end
+    #
+    class Shoryuken < Atomic
+      class Worker
+        include ::Shoryuken::Worker
+
+        shoryuken_options auto_delete: true,
+                          body_parser: :json
+
+        def perform(_sqs_msg, body)
+          options = body['options'] || {}
+          options[:refresh] = !Chewy.disable_refresh_async if Chewy.disable_refresh_async
+          body['type'].constantize.import!(body['ids'], options.deep_symbolize_keys!)
+        end
+      end
+
+      def leave
+        @stash.each do |type, ids|
+          next if ids.empty?
+          Shoryuken::Worker.perform_async({type: type.name, ids: ids}, queue: shoryuken_queue)
+        end
+      end
+
+    private
+
+      def shoryuken_queue
+        Chewy.settings.fetch(:shoryuken, {})[:queue] || 'chewy'
+      end
+    end
+  end
+end

data/lib/chewy/strategy/sidekiq.rb

@@ -13,18 +13,28 @@ module Chewy
       class Worker
         include ::Sidekiq::Worker
 
-        sidekiq_options queue: :chewy
-
         def perform(type, ids, options = {})
+          options[:refresh] = !Chewy.disable_refresh_async if Chewy.disable_refresh_async
           type.constantize.import!(ids, options)
         end
       end
 
       def leave
         @stash.each do |type, ids|
-          Chewy::Strategy::Sidekiq::Worker.perform_async(type.name, ids) unless ids.empty?
+          next if ids.empty?
+          ::Sidekiq::Client.push(
+            'queue' => sidekiq_queue,
+            'class' => Chewy::Strategy::Sidekiq::Worker,
+            'args'  => [type.name, ids]
+          )
         end
       end
+
+    private
+
+      def sidekiq_queue
+        Chewy.settings.fetch(:sidekiq, {})[:queue] || 'chewy'
+      end
     end
   end
 end

data/lib/chewy/type.rb

@@ -1,19 +1,20 @@
 require 'chewy/search'
+require 'chewy/type/adapter/object'
+require 'chewy/type/adapter/active_record'
+require 'chewy/type/adapter/mongoid'
+require 'chewy/type/adapter/sequel'
 require 'chewy/type/mapping'
 require 'chewy/type/wrapper'
 require 'chewy/type/observe'
 require 'chewy/type/actions'
+require 'chewy/type/syncer'
 require 'chewy/type/crutch'
 require 'chewy/type/import'
 require 'chewy/type/witchcraft'
-require 'chewy/type/adapter/object'
-require 'chewy/type/adapter/active_record'
-require 'chewy/type/adapter/mongoid'
-require 'chewy/type/adapter/sequel'
 
 module Chewy
   class Type
-    IMPORT_OPTIONS_KEYS = [:batch_size, :bulk_size, :refresh, :consistency, :replication, :raw_import, :journal].freeze
+    IMPORT_OPTIONS_KEYS = %i[batch_size bulk_size refresh consistency replication raw_import journal].freeze
 
     include Search
     include Mapping
@@ -24,7 +25,7 @@ module Chewy
     include Witchcraft
     include Import
 
-    singleton_class.delegate :index_name, :_index_name, :derivable_index_name, :client, to: :index
+    singleton_class.delegate :index_name, :derivable_index_name, :client, to: :index
 
     class_attribute :_default_import_options
     self._default_import_options = {}
@@ -33,7 +34,7 @@ module Chewy
       # Chewy index current type belongs to. Defined inside `Chewy.create_type`
       #
       def index
-        raise NotImplementedError
+        raise NotImplementedError, 'Looks like this type ws defined outside the index scope and `.index` method is undefined for it'
       end
 
       # Current type adapter. Defined inside `Chewy.create_type`, derived from
@@ -49,6 +50,27 @@ module Chewy
         adapter.type_name
       end
 
+      # Appends type name to {Chewy::Index.derivable_name}
+      #
+      # @example
+      #   class Namespace::UsersIndex < Chewy::Index
+      #     define_type User
+      #   end
+      #   UsersIndex::User.derivable_name # => 'namespace/users#user'
+      #
+      # @see Chewy::Index.derivable_name
+      # @return [String, nil] derivable name or nil when it is impossible to calculate
+      def derivable_name
+        @derivable_name ||= [index.derivable_name, type_name].join('#') if index && index.derivable_name
+      end
+
+      # This method is an API shared with {Chewy::Index}, added for convenience.
+      #
+      # @return [Chewy::Type] array containing itself
+      def types
+        [self]
+      end
+
       # Returns list of public class methods defined in current type
       #
       def scopes

data/lib/chewy/type/actions.rb

@@ -4,15 +4,39 @@ module Chewy
       extend ActiveSupport::Concern
 
       module ClassMethods
-        # Delete all documents of a type and reimport them
-        # Returns true or false depending on success.
+        # Deletes all documents of a type and reimports them
         #
+        # @example
         #   UsersIndex::User.reset
         #
+        # @see Chewy::Type::Import::ClassMethods#import
+        # @see Chewy::Type::Import::ClassMethods#import
+        # @return [true, false] the result of import
         def reset
           delete_all
           import
         end
+
+        # Performs missing and outdated objects synchronization for the current type.
+        #
+        # @example
+        #   UsersIndex::User.sync
+        #
+        # @see Chewy::Type::Syncer
+        # @param parallel [true, Integer, Hash] options for parallel execution or the number of processes
+        # @return [Hash{Symbol, Object}, nil] a number of missing and outdated documents reindexed and their ids, nil in case of errors
+        def sync(parallel: nil)
+          syncer = Syncer.new(self, parallel: parallel)
+          count = syncer.perform
+          {count: count, missing: syncer.missing_ids, outdated: syncer.outdated_ids} if count
+        end
+
+        # A {Chewy::Journal} instance for the particular type
+        #
+        # @return [Chewy::Journal] journal instance
+        def journal
+          @journal ||= Chewy::Journal.new(self)
+        end
       end
     end
   end

data/lib/chewy/type/adapter/active_record.rb

@@ -22,32 +22,58 @@ module Chewy
         end
 
         def import_scope(scope, options)
-          scope = scope.reorder(target_id.asc).limit(options[:batch_size])
-
-          ids = pluck_ids(scope)
-          result = true
+          pluck_in_batches(scope, options.slice(:batch_size)).inject(true) do |result, ids|
+            objects = if options[:raw_import]
+              raw_default_scope_where_ids_in(ids, options[:raw_import])
+            else
+              default_scope_where_ids_in(ids)
+            end
 
-          while ids.present?
-            objects =
-              if options[:raw_import]
-                raw_default_scope_where_ids_in(ids, options[:raw_import])
-              else
-                default_scope_where_ids_in(ids)
-              end
-            result &= yield grouped_objects(objects)
-            break if ids.size < options[:batch_size]
-            ids = pluck_ids(scope.where(target_id.gt(ids.last)))
+            result & yield(grouped_objects(objects))
           end
+        end
 
-          result
+        def primary_key
+          @primary_key ||= target.primary_key.to_sym
         end
 
         def target_id
-          target.arel_table[target.primary_key]
+          target.arel_table[primary_key.to_s]
+        end
+
+        def pluck(scope, fields: [], typecast: true)
+          if typecast
+            scope.except(:includes).distinct.pluck(primary_key, *fields)
+          else
+            scope = scope.except(:includes).distinct
+            scope.select_values = [primary_key, *fields].map do |column|
+              target.columns_hash.key?(column) ? target.arel_table[column] : column
+            end
+            sql = scope.to_sql
+
+            if fields.present?
+              target.connection.select_rows(sql)
+            else
+              target.connection.select_values(sql)
+            end
+          end
         end
 
-        def pluck_ids(scope)
-          scope.except(:includes).uniq.pluck(target.primary_key.to_sym)
+        def pluck_in_batches(scope, fields: [], batch_size: nil, typecast: true)
+          return enum_for(:pluck_in_batches, scope, fields: fields, batch_size: batch_size, typecast: typecast) unless block_given?
+
+          scope = scope.reorder(target_id.asc).limit(batch_size)
+          ids = pluck(scope, fields: fields, typecast: typecast)
+          count = 0
+
+          while ids.present?
+            yield ids
+            break if ids.size < batch_size
+            last_id = ids.last.is_a?(Array) ? ids.last.first : ids.last
+            ids = pluck(scope.where(target_id.gt(last_id)), fields: fields, typecast: typecast)
+          end
+
+          count
         end
 
         def scope_where_ids_in(scope, ids)
@@ -67,17 +93,6 @@ module Chewy
           ::ActiveRecord::Base
         end
       end
-
-      ActiveSupport.on_load(:active_record) do
-        if ::ActiveRecord::VERSION::MAJOR >= 5
-          module Rails5
-            def pluck_ids(scope)
-              scope.except(:includes).distinct.pluck(target.primary_key.to_sym)
-            end
-          end
-          Chewy::Type::Adapter::ActiveRecord.prepend(Rails5)
-        end
-      end
     end
   end
 end

data/lib/chewy/type/adapter/base.rb

@@ -37,23 +37,43 @@ module Chewy
         end
 
         # Splits passed objects to groups according to `:batch_size` options.
-        # For every group crates hash with action keys. Example:
+        # For every group creates hash with action keys. Example:
         #
-        #   { delete: [object1, object2], index: [object3, object4, object5] }
+        #   { delete: [object_or_id1, object_or_id2], index: [object3, object4, object5] }
         #
-        # Returns true id all the block call returns true and false otherwise
+        # @yield batch [Array<Object>] each batch of objects
+        # @return [true, false] returns true if all the block call returns true and false otherwise
+        def import(*_args, &_block)
+          raise NotImplementedError
+        end
+
+        # Unlike {#import} fetches only ids (references) to the imported objects,
+        # using the same procedures as {#import}.
+        #
+        # @param fields [Array<Symbol>] additional fields to fetch
+        # @param batch_size [Integer] batch size, defaults to 1000
+        # @yield batch [Array<Object>] each batch of objects
+        def import_fields(*_args, &_block)
+          raise NotImplementedError
+        end
+
+        # Uses the same strategy as import for the passed arguments, and returns
+        # an array of references to the passed objects. Returns ids if possible.
+        # Otherwise - and array of objects themselves.
         #
-        def import(*_args)
+        # @param batch_size [Integer] batch size, defaults to 1000
+        # @yield batch [Array<Object>] each batch of objects
+        def import_references(*_args, &_block)
           raise NotImplementedError
         end
 
-        # Returns array of loaded objects for passed objects array. If some object
+        # Returns array of loaded objects for passed ids array. If some object
         # was not loaded, it returns `nil` in the place of this object
         #
-        #   load(double(id: 1), double(id: 2), double(id: 3)) #=>
+        #   load([1, 2, 3]) #=>
         #     # [<Product id: 1>, nil, <Product id: 3>], assuming, #2 was not found
         #
-        def load(*_args)
+        def load(_ids, **_options)
           raise NotImplementedError
         end
 

data/lib/chewy/type/adapter/mongoid.rb

@@ -27,18 +27,29 @@ module Chewy
         end
 
         def import_scope(scope, options)
-          scope.batch_size(options[:batch_size]).no_timeout.pluck(:_id)
-            .each_slice(options[:batch_size]).map do |ids|
-              yield grouped_objects(default_scope_where_ids_in(ids))
-            end.all?
+          pluck_in_batches(scope, options.slice(:batch_size)).map do |ids|
+            yield grouped_objects(default_scope_where_ids_in(ids))
+          end.all?
         end
 
-        def pluck_ids(scope)
-          scope.pluck(:_id)
+        def primary_key
+          :_id
+        end
+
+        def pluck(scope, fields: [])
+          scope.pluck(primary_key, *fields)
+        end
+
+        def pluck_in_batches(scope, fields: [], batch_size: nil, **options)
+          return enum_for(:pluck_in_batches, scope, fields: fields, batch_size: batch_size, **options) unless block_given?
+
+          scope.batch_size(batch_size).no_timeout.pluck(primary_key, *fields).each_slice(batch_size) do |batch|
+            yield batch
+          end
         end
 
         def scope_where_ids_in(scope, ids)
-          scope.where(:_id.in => ids)
+          scope.where(primary_key.in => ids)
         end
 
         def all_scope

data/lib/chewy/type/adapter/object.rb

@@ -3,60 +3,196 @@ require 'chewy/type/adapter/base'
 module Chewy
   class Type
     module Adapter
+      # This adapter provides an ability to import documents from any
+      # source. You can actually use any class or even a symbol as
+      # a target.
+      #
+      # In case if a class is used - some of the additional features
+      # are available: it is possible to provide the default import
+      # data (used on reset) and source objects loading logic.
+      #
+      # @see #import
+      # @see #load
       class Object < Base
-        def initialize(*args)
-          @options = args.extract_options!
-          @target = args.first
+        # The signature of the type definition.
+        #
+        # @example
+        #   define_type :geoname
+        #   define_type Geoname
+        #   define_type -> { Geoname.all_the_places }, name: 'geoname'
+        #
+        # @param target [Class, Symbol, String, Proc] a source of data and everything
+        # @option options [String, Symbol] :name redefines the inferred type name if necessary
+        # @option options [String, Symbol] :import_all_method redefines import method name
+        # @option options [String, Symbol] :load_all_method redefines batch load method name
+        # @option options [String, Symbol] :load_one_method redefines per-object load method name
+        def initialize(target, **options)
+          @target = target
+          @options = options
         end
 
+        # Name is used for the type class creation. Inferred from the target
+        # by default if possible.
+        #
+        # @example
+        #   # defines MyIndex::Geoname
+        #   define_type :geoname
+        #   # still defines MyIndex::Geoname
+        #   define_type -> { Geoname.all_the_places }, name: 'geoname'
+        #
+        # @return [String]
         def name
           @name ||= (options[:name] || @target).to_s.camelize.demodulize
         end
 
+        # While for ORM adapters it returns an array of ids for the passed
+        # collection, for the object adapter it returns the collection itself.
+        #
+        # @param collection [Array<Object>, Object] a collection or an object
+        # @return [Array<Object>]
         def identify(collection)
           Array.wrap(collection)
         end
 
-        # Imports passed data with options
+        # This method is used internally by `Chewy::Type.import`.
         #
-        # Import data types:
+        # The idea is that any object can be imported to ES if
+        # it responds to `#to_json` method.
         #
-        #   * Array ob objects
+        # If method `destroyed?` is defined for object (or, in case of hash object,
+        # it has `:_destroyed` or `'_destroyed'` key) and returns `true` or object
+        # satisfy `delete_if` type option then object will be deleted from index.
+        # But in order to be destroyable, objects need to respond to `id` method
+        # or have an `id` key so ElasticSearch could know which one to delete.
         #
-        # Import options:
+        # If nothing is passed the method tries to call `import_all_method`,
+        # which is `call` by default, on target to get the default objects batch.
         #
-        #   <tt>:batch_size</tt> - import batch size, 1000 objects by default
+        # @example
+        #   class Geoname
+        #     self < class
+        #       def self.call
+        #         FancyGeoAPI.all_points_collection
+        #       end
+        #       alias_method :import_all, :call
+        #     end
+        #   end
         #
-        # If method `destroyed?` is defined for object and returns true or object
-        # satisfy `delete_if` type option then object will be deleted from index.
-        # But to be destroyed objects need to respond to `id` method as well, so
-        # ElasticSearch could know which one to delete.
+        #   # All the folloving variants will work:
+        #   define_type Geoname
+        #   define_type Geoname, import_all_method: 'import_all'
+        #   define_type -> { FancyGeoAPI.all_points_collection }, name: 'geoname'
         #
+        # @param args [Array<#to_json>]
+        # @option options [Integer] :batch_size import processing batch size
+        # @return [true, false]
         def import(*args, &block)
+          collection, options = import_args(*args)
+          import_objects(collection, options, &block)
+        end
+
+        # For the object adapter this method tries to fetch :id and requested
+        # fields from the passed collection or the target's `import_all_method`
+        # when defined. Otherwise it tries to call the target `pluck_method`,
+        # which is configurable and `pluck` by default. The `pluck_method` have
+        # to act exactly the same way as the AR one. It returns an empty array
+        # when none of the methods are found.
+        #
+        # @example
+        #   class Geoname
+        #     self < class
+        #       def self.pluck(*fields)
+        #         if fields.one?
+        #           whatever_source.map { |object| object.send(fields.first) }
+        #         else
+        #           whatever_source.map do |object|
+        #             fields.map { |field| object.send(field) }
+        #           end
+        #         end
+        #       end
+        #     end
+        #   end
+        #
+        # @see Chewy::Type::Adapter::Base#import_fields
+        def import_fields(*args)
+          return enum_for(:import_fields, *args) unless block_given?
           options = args.extract_options!
           options[:batch_size] ||= BATCH_SIZE
 
-          objects = if args.empty? && @target.respond_to?(import_all_method)
-            @target.send(import_all_method)
+          if args.empty? && @target.respond_to?(pluck_method)
+            @target.send(pluck_method, :id, *options[:fields]).each_slice(options[:batch_size]) do |batch|
+              yield batch
+            end
+          elsif options[:fields].blank?
+            import_references(*args, options) do |batch|
+              yield batch.map { |object| object_field(object, :id) || object }
+            end
           else
-            args.flatten.compact
+            import_references(*args, options) do |batch|
+              batch = batch.map do |object|
+                options[:fields].map { |field| object_field(object, field) }
+                  .unshift(object_field(object, :id) || object)
+              end
+              yield batch
+            end
           end
+        end
+
+        # For the Object adapter returns the objects themselves in batches.
+        #
+        # @see Chewy::Type::Adapter::Base#import_references
+        def import_references(*args)
+          return enum_for(:import_references, *args) unless block_given?
 
-          import_objects(objects, options, &block)
+          collection, options = import_args(*args)
+          collection.each_slice(options[:batch_size]) do |batch|
+            yield batch
+          end
         end
 
-        def load(*args)
-          args.extract_options!
-          objects = args.flatten
+        # This method is used internally by the request DSL when the
+        # collection of ORM/ODM objects is requested.
+        #
+        # Options usage is implemented by `load_all_method` and `load_one_method`.
+        #
+        # If none of the `load_all_method` or `load_one_method` is implemented
+        # for the target - the method will return nil. This means that the
+        # loader will return an array `Chewy::Type` objects that actually was passed.
+        #
+        # To use loading for objects it is obviously required to provide
+        # some meaningful ids for ES documents.
+        #
+        # @example
+        #   class Geoname
+        #     def self.load_all(wrappers, options)
+        #       if options[:additional_data]
+        #         wrappers.map do |wrapper|
+        #           FancyGeoAPI.point_by_name(wrapper.name)
+        #         end
+        #       else
+        #         wrappers
+        #       end
+        #     end
+        #   end
+        #
+        #   MyIndex::Geoname.load(additional_data: true).objects
+        #
+        # @param ids [Array<Hash>] an array of ids from ES hits
+        # @param options [Hash] any options passed here with the request DSL `load` method.
+        # @return [Array<Object>, nil]
+        def load(ids, **options)
           if target.respond_to?(load_all_method)
-            target.send(load_all_method, objects)
+            if target.method(load_all_method).arity == 1
+              target.send(load_all_method, ids)
+            else
+              target.send(load_all_method, ids, options)
+            end
           elsif target.respond_to?(load_one_method)
-            objects.map { |object| target.send(load_one_method, object) }
-          elsif target.respond_to?(:wrap)
-            ActiveSupport::Deprecation.warn('Loading with `wrap` method is deprecated. Rename it to `load_one` or pass `load_one_method: :my_load_method` option to `define_type`')
-            objects.map { |object| target.wrap(object) }
-          else
-            objects
+            if target.method(load_one_method).arity == 1
+              ids.map { |hit| target.send(load_one_method, hit) }
+            else
+              ids.map { |hit| target.send(load_one_method, hit, options) }
+            end
           end
         end
 
@@ -75,10 +211,22 @@ module Chewy
           !!delete
         end
 
+        def object_field(object, name)
+          if object.respond_to?(name)
+            object.send(name)
+          elsif object.is_a?(Hash)
+            object[name.to_sym] || object[name.to_s]
+          end
+        end
+
         def import_all_method
           @import_all_method ||= options[:import_all_method] || :call
         end
 
+        def pluck_method
+          @pluck_method ||= options[:pluck_method] || :pluck
+        end
+
         def load_all_method
           @load_all_method ||= options[:load_all_method] || :load_all
         end
@@ -86,6 +234,19 @@ module Chewy
         def load_one_method
           @load_one_method ||= options[:load_one_method] || :load_one
         end
+
+        def import_args(*args)
+          options = args.extract_options!
+          options[:batch_size] ||= BATCH_SIZE
+
+          collection = if args.empty? && @target.respond_to?(import_all_method)
+            @target.send(import_all_method)
+          else
+            args.flatten(1).compact
+          end
+
+          [collection, options]
+        end
       end
     end
   end