chewy 0.10.1 → 7.3.2

data/lib/chewy/index/actions.rb

@@ -30,8 +30,8 @@ module Chewy
         # Suffixed index names might be used for zero-downtime mapping change, for example.
         # Description: (http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime/).
         #
-        def create(*args)
-          create!(*args)
+        def create(*args, **kwargs)
+          create!(*args, **kwargs)
         rescue Elasticsearch::Transport::Transport::Errors::BadRequest
           false
         end
@@ -57,14 +57,9 @@ module Chewy
           general_name = index_name
           suffixed_name = index_name(suffix: suffix)
 
-          if Chewy::Runtime.version >= 1.1
-            body = specification_hash
-            body[:aliases] = {general_name => {}} if options[:alias] && suffixed_name != general_name
-            result = client.indices.create(index: suffixed_name, body: body)
-          else
-            result = client.indices.create(index: suffixed_name, body: specification_hash)
-            result &&= client.indices.put_alias(index: suffixed_name, name: general_name) if options[:alias] && name != index_name
-          end
+          body = specification_hash
+          body[:aliases] = {general_name => {}} if options[:alias] && suffixed_name != general_name
+          result = client.indices.create(index: suffixed_name, body: body)
 
           Chewy.wait_for_status if result
           result
@@ -79,7 +74,13 @@ module Chewy
         #   UsersIndex.delete '01-2014' # deletes `users_01-2014` index
         #
         def delete(suffix = nil)
-          result = client.indices.delete index: index_name(suffix: suffix)
+          # Verify that the index_name is really the index_name and not an alias.
+          #
+          #   "The index parameter in the delete index API no longer accepts alias names.
+          #   Instead, it accepts only index names (or wildcards which will expand to matching indices)."
+          #   https://www.elastic.co/guide/en/elasticsearch/reference/6.8/breaking-changes-6.0.html#_delete_index_api_resolves_indices_expressions_only_against_indices
+          index_names = client.indices.get_alias(index: index_name(suffix: suffix)).keys
+          result = client.indices.delete index: index_names.join(',')
           Chewy.wait_for_status if result
           result
           # es-ruby >= 1.0.10 handles Elasticsearch::Transport::Transport::Errors::NotFound
@@ -128,28 +129,6 @@ module Chewy
           create! suffix
         end
 
-        # Perform import operation for every defined type
-        #
-        #   UsersIndex.import                           # imports default data for every index type
-        #   UsersIndex.import user: User.active         # imports specified objects for user type and default data for other types
-        #   UsersIndex.import refresh: false            # to disable index refreshing after import
-        #   UsersIndex.import suffix: Time.now.to_i     # imports data to index with specified suffix if such is exists
-        #   UsersIndex.import batch_size: 300           # import batch size
-        #
-        # See [import.rb](lib/chewy/type/import.rb) for more details.
-        #
-        %i[import import!].each do |method|
-          class_eval <<-METHOD, __FILE__, __LINE__ + 1
-            def #{method} options = {}
-              objects = options.reject { |k, v| !type_names.map(&:to_sym).include?(k) }
-              types.map do |type|
-                args = [objects[type.type_name.to_sym], options.dup].reject(&:blank?)
-                type.#{method} *args
-              end.all?
-            end
-          METHOD
-        end
-
         # Deletes, creates and imports data to the index. Returns the
         # import result. If index name suffix is passed as the first
         # argument - performs zero-downtime index resetting.
@@ -164,20 +143,24 @@ module Chewy
         # @see http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime
         # @param suffix [String] a suffix for the newly created index
         # @param apply_journal [true, false] if true, journal is applied after the import is completed
-        # @param journal [true, false] journalig is switched off for import during reset by default
+        # @param journal [true, false] journaling is switched off for import during reset by default
         # @param import_options [Hash] options, passed to the import call
         # @return [true, false] false in case of errors
         def reset!(suffix = nil, apply_journal: true, journal: false, **import_options)
           result = if suffix.present?
             start_time = Time.now
-            indexes = self.indexes
+            indexes = self.indexes - [index_name]
             create! suffix, alias: false
 
             general_name = index_name
             suffixed_name = index_name(suffix: suffix)
 
             optimize_index_settings suffixed_name
-            result = import import_options.merge(suffix: suffix, journal: journal, refresh: !Chewy.reset_disable_refresh_interval)
+            result = import(**import_options.merge(
+              suffix: suffix,
+              journal: journal,
+              refresh: !Chewy.reset_disable_refresh_interval
+            ))
             original_index_settings suffixed_name
 
             delete if indexes.blank?
@@ -193,12 +176,13 @@ module Chewy
             result
           else
             purge!
-            import import_options.merge(journal: journal)
+            import(**import_options.merge(journal: journal))
           end
 
           specification.lock!
           result
         end
+        alias_method :reset, :reset!
 
         # A {Chewy::Journal} instance for the particular index
         #
@@ -207,6 +191,50 @@ module Chewy
           @journal ||= Chewy::Journal.new(self)
         end
 
+        def clear_cache(args = {index: index_name})
+          client.indices.clear_cache(args)
+        end
+
+        def reindex(source: index_name, dest: index_name)
+          client.reindex(
+            {
+              body:
+                {
+                  source: {index: source},
+                  dest: {index: dest}
+                }
+            }
+          )
+        end
+
+        # Adds new fields to an existing data stream or index.
+        # Change the search settings of existing fields.
+        #
+        # @example
+        #   Chewy.client.update_mapping('cities', {properties: {new_field: {type: :text}}})
+        #
+        def update_mapping(name = index_name, body = root.mappings_hash)
+          client.indices.put_mapping(
+            index: name,
+            body: body
+          )['acknowledged']
+        end
+
+        # Performs missing and outdated objects synchronization for the current index.
+        #
+        # @example
+        #   UsersIndex.sync
+        #
+        # @see Chewy::Index::Syncer
+        # @param parallel [true, Integer, Hash] options for parallel execution or the number of processes
+        # @return [Hash{Symbol, Object}, nil] a number of missing and outdated documents re-indexed and their ids,
+        #   nil in case of errors
+        def sync(parallel: nil)
+          syncer = Syncer.new(self, parallel: parallel)
+          count = syncer.perform
+          {count: count, missing: syncer.missing_ids, outdated: syncer.outdated_ids} if count
+        end
+
       private
 
         def optimize_index_settings(index_name)
@@ -231,7 +259,8 @@ module Chewy
         end
 
         def index_settings(setting_name)
-          return {} unless settings_hash.key?(:settings) || settings_hash[:settings].key?(:index)
+          return {} unless settings_hash.key?(:settings) && settings_hash[:settings].key?(:index)
+
           settings_hash[:settings][:index].slice(setting_name)
         end
       end

data/lib/chewy/{type → index}/adapter/active_record.rb

@@ -1,7 +1,7 @@
-require 'chewy/type/adapter/orm'
+require 'chewy/index/adapter/orm'
 
 module Chewy
-  class Type
+  class Index
     module Adapter
       class ActiveRecord < Orm
         def self.accepts?(target)
@@ -22,7 +22,7 @@ module Chewy
         end
 
         def import_scope(scope, options)
-          pluck_in_batches(scope, options.slice(:batch_size)).inject(true) do |result, ids|
+          pluck_in_batches(scope, **options.slice(:batch_size)).inject(true) do |result, ids|
             objects = if options[:raw_import]
               raw_default_scope_where_ids_in(ids, options[:raw_import])
             else
@@ -60,7 +60,15 @@ module Chewy
         end
 
         def pluck_in_batches(scope, fields: [], batch_size: nil, typecast: true)
-          return enum_for(:pluck_in_batches, scope, fields: fields, batch_size: batch_size, typecast: typecast) unless block_given?
+          unless block_given?
+            return enum_for(
+              :pluck_in_batches,
+              scope,
+              fields: fields,
+              batch_size: batch_size,
+              typecast: typecast
+            )
+          end
 
           scope = scope.reorder(target_id.asc).limit(batch_size)
           ids = pluck(scope, fields: fields, typecast: typecast)
@@ -69,6 +77,7 @@ module Chewy
           while ids.present?
             yield ids
             break if ids.size < batch_size
+
             last_id = ids.last.is_a?(Array) ? ids.last.first : ids.last
             ids = pluck(scope.where(target_id.gt(last_id)), fields: fields, typecast: typecast)
           end
@@ -85,6 +94,11 @@ module Chewy
           object_class.connection.execute(sql).map(&converter)
         end
 
+        def raw(scope, converter)
+          sql = scope.to_sql
+          object_class.connection.execute(sql).map(&converter)
+        end
+
         def relation_class
           ::ActiveRecord::Relation
         end

data/lib/chewy/{type → index}/adapter/base.rb

@@ -1,5 +1,5 @@
 module Chewy
-  class Type
+  class Index
     module Adapter
       # Basic adapter class. Contains interface, need to implement to add any classes support
       class Base
@@ -13,8 +13,7 @@ module Chewy
           true
         end
 
-        # Camelcased name, used as type class constant name.
-        # For returned value 'Product' will be generated class name `ProductsIndex::Product`
+        # Camelcased name.
         #
         def name
           raise NotImplementedError
@@ -41,19 +40,19 @@ module Chewy
         #
         #   { delete: [object_or_id1, object_or_id2], index: [object3, object4, object5] }
         #
-        # @yield batch [Array<Object>] each batch of objects
+        # @yieldparam _batch [Array<Object>] each batch of objects
         # @return [true, false] returns true if all the block call returns true and false otherwise
-        def import(*_args, &_block)
+        def import(_batch, &_block)
           raise NotImplementedError
         end
 
         # Unlike {#import} fetches only ids (references) to the imported objects,
         # using the same procedures as {#import}.
         #
-        # @param fields [Array<Symbol>] additional fields to fetch
-        # @param batch_size [Integer] batch size, defaults to 1000
-        # @yield batch [Array<Object>] each batch of objects
-        def import_fields(*_args, &_block)
+        # @param _fields [Array<Symbol>] additional fields to fetch
+        # @param _batch_size [Integer] batch size, defaults to 1000
+        # @yieldparam batch [Array<Object>] each batch of objects
+        def import_fields(_fields, _batch_size, &_block)
           raise NotImplementedError
         end
 
@@ -61,9 +60,9 @@ module Chewy
         # an array of references to the passed objects. Returns ids if possible.
         # Otherwise - and array of objects themselves.
         #
-        # @param batch_size [Integer] batch size, defaults to 1000
-        # @yield batch [Array<Object>] each batch of objects
-        def import_references(*_args, &_block)
+        # @param _batch_size [Integer] batch size, defaults to 1000
+        # @yieldparam batch [Array<Object>] each batch of objects
+        def import_references(_batch_size, &_block)
           raise NotImplementedError
         end
 

data/lib/chewy/{type → index}/adapter/object.rb

@@ -1,7 +1,7 @@
-require 'chewy/type/adapter/base'
+require 'chewy/index/adapter/base'
 
 module Chewy
-  class Type
+  class Index
     module Adapter
       # This adapter provides an ability to import documents from any
       # source. You can actually use any class or even a symbol as
@@ -14,15 +14,15 @@ module Chewy
       # @see #import
       # @see #load
       class Object < Base
-        # The signature of the type definition.
+        # The signature of the index scope definition.
         #
         # @example
-        #   define_type :geoname
-        #   define_type Geoname
-        #   define_type -> { Geoname.all_the_places }, name: 'geoname'
+        #   index_scope :geoname
+        #   index_scope Geoname
+        #   index_scope -> { Geoname.all_the_places }, name: 'geoname'
         #
         # @param target [Class, Symbol, String, Proc] a source of data and everything
-        # @option options [String, Symbol] :name redefines the inferred type name if necessary
+        # @option options [String, Symbol] :name redefines the inferred name if necessary
         # @option options [String, Symbol] :import_all_method redefines import method name
         # @option options [String, Symbol] :load_all_method redefines batch load method name
         # @option options [String, Symbol] :load_one_method redefines per-object load method name
@@ -31,14 +31,13 @@ module Chewy
           @options = options
         end
 
-        # Name is used for the type class creation. Inferred from the target
-        # by default if possible.
+        # Inferred from the target by default if possible.
         #
         # @example
-        #   # defines MyIndex::Geoname
-        #   define_type :geoname
-        #   # still defines MyIndex::Geoname
-        #   define_type -> { Geoname.all_the_places }, name: 'geoname'
+        #   # defines name = Geoname
+        #   index_scope :geoname
+        #   # still defines name = Geoname
+        #   index_scope -> { Geoname.all_the_places }, name: 'geoname'
         #
         # @return [String]
         def name
@@ -54,14 +53,14 @@ module Chewy
           Array.wrap(collection)
         end
 
-        # This method is used internally by `Chewy::Type.import`.
+        # This method is used internally by `Chewy::Index.import`.
         #
         # The idea is that any object can be imported to ES if
         # it responds to `#to_json` method.
         #
         # If method `destroyed?` is defined for object (or, in case of hash object,
         # it has `:_destroyed` or `'_destroyed'` key) and returns `true` or object
-        # satisfy `delete_if` type option then object will be deleted from index.
+        # satisfy `delete_if` option then object will be deleted from index.
         # But in order to be destroyable, objects need to respond to `id` method
         # or have an `id` key so ElasticSearch could know which one to delete.
         #
@@ -78,15 +77,15 @@ module Chewy
         #     end
         #   end
         #
-        #   # All the folloving variants will work:
-        #   define_type Geoname
-        #   define_type Geoname, import_all_method: 'import_all'
-        #   define_type -> { FancyGeoAPI.all_points_collection }, name: 'geoname'
+        #   # All the following variants will work:
+        #   index_scope Geoname
+        #   index_scope Geoname, import_all_method: 'import_all'
+        #   index_scope -> { FancyGeoAPI.all_points_collection }, name: 'geoname'
         #
         # @param args [Array<#to_json>]
         # @option options [Integer] :batch_size import processing batch size
         # @return [true, false]
-        def import(*args, &block)
+        ruby2_keywords def import(*args, &block)
           collection, options = import_args(*args)
           import_objects(collection, options, &block)
         end
@@ -113,16 +112,15 @@ module Chewy
         #     end
         #   end
         #
-        # @see Chewy::Type::Adapter::Base#import_fields
-        def import_fields(*args)
+        # @see Chewy::Index::Adapter::Base#import_fields
+        ruby2_keywords def import_fields(*args, &block)
           return enum_for(:import_fields, *args) unless block_given?
+
           options = args.extract_options!
           options[:batch_size] ||= BATCH_SIZE
 
           if args.empty? && @target.respond_to?(pluck_method)
-            @target.send(pluck_method, :id, *options[:fields]).each_slice(options[:batch_size]) do |batch|
-              yield batch
-            end
+            @target.send(pluck_method, :id, *options[:fields]).each_slice(options[:batch_size], &block)
           elsif options[:fields].blank?
             import_references(*args, options) do |batch|
               yield batch.map { |object| object_field(object, :id) || object }
@@ -140,14 +138,12 @@ module Chewy
 
         # For the Object adapter returns the objects themselves in batches.
         #
-        # @see Chewy::Type::Adapter::Base#import_references
-        def import_references(*args)
+        # @see Chewy::Index::Adapter::Base#import_references
+        ruby2_keywords def import_references(*args, &block)
           return enum_for(:import_references, *args) unless block_given?
 
           collection, options = import_args(*args)
-          collection.each_slice(options[:batch_size]) do |batch|
-            yield batch
-          end
+          collection.each_slice(options[:batch_size], &block)
         end
 
         # This method is used internally by the request DSL when the
@@ -157,7 +153,7 @@ module Chewy
         #
         # If none of the `load_all_method` or `load_one_method` is implemented
         # for the target - the method will return nil. This means that the
-        # loader will return an array `Chewy::Type` objects that actually was passed.
+        # loader will return an array `Chewy::Index` objects that actually was passed.
         #
         # To use loading for objects it is obviously required to provide
         # some meaningful ids for ES documents.
@@ -175,7 +171,7 @@ module Chewy
         #     end
         #   end
         #
-        #   MyIndex::Geoname.load(additional_data: true).objects
+        #   MyIndex.load(additional_data: true).objects
         #
         # @param ids [Array<Hash>] an array of ids from ES hits
         # @param options [Hash] any options passed here with the request DSL `load` method.

data/lib/chewy/{type → index}/adapter/orm.rb

@@ -1,7 +1,7 @@
-require 'chewy/type/adapter/base'
+require 'chewy/index/adapter/base'
 
 module Chewy
-  class Type
+  class Index
     module Adapter
       class Orm < Base
         attr_reader :default_scope
@@ -45,6 +45,7 @@ module Chewy
         # Import options:
         #
         #   <tt>:batch_size</tt> - import batch size, 1000 objects by default
+        #   <tt>:direct_import</tt> - import objects without reloading
         #
         # Method handles destroyed objects as well. In case of objects ORM scope
         # or array passed, objects, responding with true to `destroyed?` method will be deleted
@@ -53,72 +54,73 @@ module Chewy
         #
         #   users = User.all
         #   users.each { |user| user.destroy if user.inactive? }
-        #   UsersIndex::User.import users # inactive users will be deleted from index
+        #   UsersIndex.import users # inactive users will be deleted from index
         #   # or
-        #   UsersIndex::User.import users.map(&:id) # deleted user ids will be deleted from index
+        #   UsersIndex.import users.map(&:id) # deleted user ids will be deleted from index
         #
         # Also there is custom type option `delete_if`. It it returns `true`
         # object will be deleted from index. Note that if this option is defined and
         # return `false` Chewy will still check `destroyed?` method. This is useful
         # for paranoid objects deleting implementation.
         #
-        #   define_type User, delete_if: ->{ deleted_at } do
-        #     ...
-        #   end
+        #   index_scope User, delete_if: ->{ deleted_at }
+        #   ...
         #
         #   users = User.all
         #   users.each { |user| user.deleted_at = Time.now }
-        #   UsersIndex::User.import users # paranoid deleted users will be deleted from index
+        #   UsersIndex.import users # paranoid deleted users will be deleted from index
         #   # or
-        #   UsersIndex::User.import users.map(&:id) # user ids will be deleted from index
+        #   UsersIndex.import users.map(&:id) # user ids will be deleted from index
         #
-        def import(*args, &block)
+        ruby2_keywords def import(*args, &block)
           collection, options = import_args(*args)
 
-          if collection.is_a?(relation_class)
-            import_scope(collection, options, &block)
-          else
+          if !collection.is_a?(relation_class) || options[:direct_import]
             import_objects(collection, options, &block)
+          else
+            import_scope(collection, options, &block)
           end
         end
 
-        def import_fields(*args, &block)
+        ruby2_keywords def import_fields(*args, &block)
           return enum_for(:import_fields, *args) unless block_given?
 
           collection, options = import_args(*args)
 
           if options[:fields].present? || collection.is_a?(relation_class)
             collection = all_scope_where_ids_in(identify(collection)) unless collection.is_a?(relation_class)
-            pluck_in_batches(collection, options.slice(:fields, :batch_size, :typecast), &block)
+            pluck_in_batches(collection, **options.slice(:fields, :batch_size, :typecast), &block)
           else
-            identify(collection).each_slice(options[:batch_size]) do |batch|
-              yield batch
-            end
+            identify(collection).each_slice(options[:batch_size], &block)
           end
         end
         alias_method :import_references, :import_fields
 
         def load(ids, **options)
           scope = all_scope_where_ids_in(ids)
-          additional_scope = options[options[:_type].type_name.to_sym].try(:[], :scope) || options[:scope]
+          additional_scope = options[options[:_index].to_sym].try(:[], :scope) || options[:scope]
 
           loaded_objects = load_scope_objects(scope, additional_scope)
-            .index_by do |object|
-              object.public_send(primary_key).to_s
-            end
+          loaded_objects = raw(loaded_objects, options[:raw_import]) if options[:raw_import]
+
+          indexed_objects = loaded_objects.index_by do |object|
+            object.public_send(primary_key).to_s
+          end
 
-          ids.map { |id| loaded_objects[id.to_s] }
+          ids.map { |id| indexed_objects[id.to_s] }
         end
 
       private
 
         def import_objects(collection, options)
           collection_ids = identify(collection)
-          hash = Hash[collection_ids.map(&:to_s).zip(collection)]
+          hash = collection_ids.map(&:to_s).zip(collection).to_h
 
           indexed = collection_ids.each_slice(options[:batch_size]).map do |ids|
             batch = if options[:raw_import]
               raw_default_scope_where_ids_in(ids, options[:raw_import])
+            elsif options[:direct_import]
+              hash.values_at(*ids.map(&:to_s))
             else
               default_scope_where_ids_in(ids)
             end

data/lib/chewy/index/aliases.rb

@@ -5,14 +5,23 @@ module Chewy
 
       module ClassMethods
         def indexes
-          client.indices.get_alias(name: index_name).keys
-        rescue Elasticsearch::Transport::Transport::Errors::NotFound
-          []
+          indexes = empty_if_not_found { client.indices.get(index: index_name).keys }
+          indexes += empty_if_not_found { client.indices.get_alias(name: index_name).keys }
+          indexes.compact.uniq
         end
 
         def aliases
-          name = index_name
-          client.indices.get_alias(index: name, name: '*')[name].try(:[], 'aliases').try(:keys) || []
+          empty_if_not_found do
+            client.indices.get_alias(index: index_name, name: '*').values.flat_map do |aliases|
+              aliases['aliases'].keys
+            end
+          end.compact.uniq
+        end
+
+      private
+
+        def empty_if_not_found
+          yield
         rescue Elasticsearch::Transport::Transport::Errors::NotFound
           []
         end

data/lib/chewy/index/crutch.rb

@@ -0,0 +1,40 @@
+module Chewy
+  class Index
+    module Crutch
+      extend ActiveSupport::Concern
+
+      included do
+        class_attribute :_crutches
+        self._crutches = {}
+      end
+
+      class Crutches
+        def initialize(index, collection)
+          @index = index
+          @collection = collection
+          @crutches_instances = {}
+        end
+
+        def method_missing(name, *, **)
+          return self[name] if @index._crutches.key?(name)
+
+          super
+        end
+
+        def respond_to_missing?(name, include_private = false)
+          @index._crutches.key?(name) || super
+        end
+
+        def [](name)
+          @crutches_instances[name] ||= @index._crutches[:"#{name}"].call(@collection)
+        end
+      end
+
+      module ClassMethods
+        def crutch(name, &block)
+          self._crutches = _crutches.merge(name.to_sym => block)
+        end
+      end
+    end
+  end
+end