chewy 0.8.4 → 7.3.4

data/lib/chewy/index/actions.rb

@@ -9,7 +9,7 @@ module Chewy
       module ClassMethods
         # Checks index existance. Returns true or false
         #
-        #   UsersIndex.exist? #=> true
+        #   UsersIndex.exists? #=> true
         #
         def exists?
           client.indices.exists(index: index_name)
@@ -30,8 +30,8 @@ module Chewy
         # Suffixed index names might be used for zero-downtime mapping change, for example.
         # Description: (http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime/).
         #
-        def create *args
-          create! *args
+        def create(*args, **kwargs)
+          create!(*args, **kwargs)
         rescue Elasticsearch::Transport::Transport::Errors::BadRequest
           false
         end
@@ -52,18 +52,14 @@ module Chewy
         # Suffixed index names might be used for zero-downtime mapping change, for example.
         # Description: (http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime/).
         #
-        def create! *args
-          options = args.extract_options!.reverse_merge!(alias: true)
-          name = build_index_name(suffix: args.first)
+        def create!(suffix = nil, **options)
+          options.reverse_merge!(alias: true)
+          general_name = index_name
+          suffixed_name = index_name(suffix: suffix)
 
-          if Chewy::Runtime.version >= 1.1
-            body = index_params
-            body.merge!(aliases: {index_name => {}}) if options[:alias] && name != index_name
-            result = client.indices.create(index: name, body: body)
-          else
-            result = client.indices.create(index: name, body: index_params)
-            result &&= client.indices.put_alias(index: name, name: index_name) if options[:alias] && name != index_name
-          end
+          body = specification_hash
+          body[:aliases] = {general_name => {}} if options[:alias] && suffixed_name != general_name
+          result = client.indices.create(index: suffixed_name, body: body)
 
           Chewy.wait_for_status if result
           result
@@ -77,8 +73,14 @@ module Chewy
         #
         #   UsersIndex.delete '01-2014' # deletes `users_01-2014` index
         #
-        def delete suffix = nil
-          result = client.indices.delete index: build_index_name(suffix: suffix)
+        def delete(suffix = nil)
+          # Verify that the index_name is really the index_name and not an alias.
+          #
+          #   "The index parameter in the delete index API no longer accepts alias names.
+          #   Instead, it accepts only index names (or wildcards which will expand to matching indices)."
+          #   https://www.elastic.co/guide/en/elasticsearch/reference/6.8/breaking-changes-6.0.html#_delete_index_api_resolves_indices_expressions_only_against_indices
+          index_names = client.indices.get_alias(index: index_name(suffix: suffix)).keys
+          result = client.indices.delete index: index_names.join(',')
           Chewy.wait_for_status if result
           result
           # es-ruby >= 1.0.10 handles Elasticsearch::Transport::Transport::Errors::NotFound
@@ -96,7 +98,7 @@ module Chewy
         #
         #   UsersIndex.delete '01-2014' # deletes `users_01-2014` index
         #
-        def delete! suffix = nil
+        def delete!(suffix = nil)
           # es-ruby >= 1.0.10 handles Elasticsearch::Transport::Transport::Errors::NotFound
           # by itself, so it is raised here
           delete(suffix) or raise Elasticsearch::Transport::Transport::Errors::NotFound
@@ -108,7 +110,7 @@ module Chewy
         #   UsersIndex.purge # deletes and creates `users` index
         #   UsersIndex.purge '01-2014' # deletes `users` and `users_01-2014` indexes, creates `users_01-2014`
         #
-        def purge suffix = nil
+        def purge(suffix = nil)
           delete if suffix.present?
           delete suffix
           create suffix
@@ -121,61 +123,145 @@ module Chewy
         #   UsersIndex.purge! # deletes and creates `users` index
         #   UsersIndex.purge! '01-2014' # deletes `users` and `users_01-2014` indexes, creates `users_01-2014`
         #
-        def purge! suffix = nil
+        def purge!(suffix = nil)
           delete if suffix.present? && exists?
           delete suffix
           create! suffix
         end
 
-        # Perform import operation for every defined type
-        #
-        #   UsersIndex.import                           # imports default data for every index type
-        #   UsersIndex.import user: User.active         # imports specified objects for user type and default data for other types
-        #   UsersIndex.import refresh: false            # to disable index refreshing after import
-        #   UsersIndex.import suffix: Time.now.to_i     # imports data to index with specified suffix if such is exists
-        #   UsersIndex.import batch_size: 300           # import batch size
+        # Deletes, creates and imports data to the index. Returns the
+        # import result. If index name suffix is passed as the first
+        # argument - performs zero-downtime index resetting.
         #
-        # See [import.rb](lib/chewy/type/import.rb) for more details.
-        #
-        [:import, :import!].each do |method|
-          class_eval <<-METHOD, __FILE__, __LINE__ + 1
-            def #{method} options = {}
-              objects = options.reject { |k, v| !type_names.map(&:to_sym).include?(k) }
-              types.map do |type|
-                args = [objects[type.type_name.to_sym], options.dup].reject(&:blank?)
-                type.#{method} *args
-              end.all?
-            end
-          METHOD
-        end
-
-        # Deletes, creates and imports data to the index.
-        # Returns import result
+        # It also applies journal if anything was journaled during the
+        # reset.
         #
+        # @example
         #   UsersIndex.reset!
-        #
-        # If index name suffix passed as the first argument - performs
-        # zero-downtime index resetting (described here:
-        # http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime/).
-        #
         #   UsersIndex.reset! Time.now.to_i
         #
-        def reset! suffix = nil
-          if suffix.present? && (indexes = self.indexes).present?
+        # @see http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime
+        # @param suffix [String] a suffix for the newly created index
+        # @param apply_journal [true, false] if true, journal is applied after the import is completed
+        # @param journal [true, false] journaling is switched off for import during reset by default
+        # @param import_options [Hash] options, passed to the import call
+        # @return [true, false] false in case of errors
+        def reset!(suffix = nil, apply_journal: true, journal: false, **import_options)
+          result = if suffix.present?
+            start_time = Time.now
+            indexes = self.indexes - [index_name]
             create! suffix, alias: false
-            result = import suffix: suffix
+
+            general_name = index_name
+            suffixed_name = index_name(suffix: suffix)
+
+            optimize_index_settings suffixed_name
+            result = import(**import_options.merge(
+              suffix: suffix,
+              journal: journal,
+              refresh: !Chewy.reset_disable_refresh_interval
+            ))
+            original_index_settings suffixed_name
+
+            delete if indexes.blank?
             client.indices.update_aliases body: {actions: [
               *indexes.map do |index|
-                {remove: {index: index, alias: index_name}}
+                {remove: {index: index, alias: general_name}}
               end,
-              {add: {index: build_index_name(suffix: suffix), alias: index_name}}
+              {add: {index: suffixed_name, alias: general_name}}
             ]}
             client.indices.delete index: indexes if indexes.present?
+
+            self.journal.apply(start_time, **import_options) if apply_journal
             result
           else
-            purge! suffix
-            import
+            purge!
+            import(**import_options.merge(journal: journal))
+          end
+
+          specification.lock!
+          result
+        end
+        alias_method :reset, :reset!
+
+        # A {Chewy::Journal} instance for the particular index
+        #
+        # @return [Chewy::Journal] journal instance
+        def journal
+          @journal ||= Chewy::Journal.new(self)
+        end
+
+        def clear_cache(args = {index: index_name})
+          client.indices.clear_cache(args)
+        end
+
+        def reindex(source: index_name, dest: index_name)
+          client.reindex(
+            {
+              body:
+                {
+                  source: {index: source},
+                  dest: {index: dest}
+                }
+            }
+          )
+        end
+
+        # Adds new fields to an existing data stream or index.
+        # Change the search settings of existing fields.
+        #
+        # @example
+        #   Chewy.client.update_mapping('cities', {properties: {new_field: {type: :text}}})
+        #
+        def update_mapping(name = index_name, body = root.mappings_hash)
+          client.indices.put_mapping(
+            index: name,
+            body: body
+          )['acknowledged']
+        end
+
+        # Performs missing and outdated objects synchronization for the current index.
+        #
+        # @example
+        #   UsersIndex.sync
+        #
+        # @see Chewy::Index::Syncer
+        # @param parallel [true, Integer, Hash] options for parallel execution or the number of processes
+        # @return [Hash{Symbol, Object}, nil] a number of missing and outdated documents re-indexed and their ids,
+        #   nil in case of errors
+        def sync(parallel: nil)
+          syncer = Syncer.new(self, parallel: parallel)
+          count = syncer.perform
+          {count: count, missing: syncer.missing_ids, outdated: syncer.outdated_ids} if count
+        end
+
+      private
+
+        def optimize_index_settings(index_name)
+          settings = {}
+          settings[:refresh_interval] = -1 if Chewy.reset_disable_refresh_interval
+          settings[:number_of_replicas] = 0 if Chewy.reset_no_replicas
+          update_settings index_name, settings: settings if settings.any?
+        end
+
+        def original_index_settings(index_name)
+          settings = {}
+          if Chewy.reset_disable_refresh_interval
+            settings.merge! index_settings(:refresh_interval)
+            settings[:refresh_interval] = '1s' if settings.empty?
           end
+          settings.merge! index_settings(:number_of_replicas) if Chewy.reset_no_replicas
+          update_settings index_name, settings: settings if settings.any?
+        end
+
+        def update_settings(index_name, **options)
+          client.indices.put_settings index: index_name, body: {index: options[:settings]}
+        end
+
+        def index_settings(setting_name)
+          return {} unless settings_hash.key?(:settings) && settings_hash[:settings].key?(:index)
+
+          settings_hash[:settings][:index].slice(setting_name)
         end
       end
     end

data/lib/chewy/index/adapter/active_record.rb

@@ -0,0 +1,112 @@
+require 'chewy/index/adapter/orm'
+
+module Chewy
+  class Index
+    module Adapter
+      class ActiveRecord < Orm
+        def self.accepts?(target)
+          defined?(::ActiveRecord::Base) && (
+            target.is_a?(Class) && target < ::ActiveRecord::Base ||
+            target.is_a?(::ActiveRecord::Relation))
+        end
+
+      private
+
+        def cleanup_default_scope!
+          if Chewy.logger && (@default_scope.arel.orders.present? ||
+             @default_scope.arel.limit.present? || @default_scope.arel.offset.present?)
+            Chewy.logger.warn('Default type scope order, limit and offset are ignored and will be nullified')
+          end
+
+          @default_scope = @default_scope.reorder(nil).limit(nil).offset(nil)
+        end
+
+        def import_scope(scope, options)
+          pluck_in_batches(scope, **options.slice(:batch_size)).inject(true) do |result, ids|
+            objects = if options[:raw_import]
+              raw_default_scope_where_ids_in(ids, options[:raw_import])
+            else
+              default_scope_where_ids_in(ids)
+            end
+
+            result & yield(grouped_objects(objects))
+          end
+        end
+
+        def primary_key
+          @primary_key ||= target.primary_key.to_sym
+        end
+
+        def target_id
+          target.arel_table[primary_key.to_s]
+        end
+
+        def pluck(scope, fields: [], typecast: true)
+          if typecast
+            scope.except(:includes).distinct.pluck(primary_key, *fields)
+          else
+            scope = scope.except(:includes).distinct
+            scope.select_values = [primary_key, *fields].map do |column|
+              target.columns_hash.key?(column) ? target.arel_table[column] : column
+            end
+            sql = scope.to_sql
+
+            if fields.present?
+              target.connection.select_rows(sql)
+            else
+              target.connection.select_values(sql)
+            end
+          end
+        end
+
+        def pluck_in_batches(scope, fields: [], batch_size: nil, typecast: true)
+          unless block_given?
+            return enum_for(
+              :pluck_in_batches,
+              scope,
+              fields: fields,
+              batch_size: batch_size,
+              typecast: typecast
+            )
+          end
+
+          scope = scope.reorder(target_id.asc).limit(batch_size)
+          ids = pluck(scope, fields: fields, typecast: typecast)
+          count = 0
+
+          while ids.present?
+            yield ids
+            break if ids.size < batch_size
+
+            last_id = ids.last.is_a?(Array) ? ids.last.first : ids.last
+            ids = pluck(scope.where(target_id.gt(last_id)), fields: fields, typecast: typecast)
+          end
+
+          count
+        end
+
+        def scope_where_ids_in(scope, ids)
+          scope.where(target_id.in(Array.wrap(ids)))
+        end
+
+        def raw_default_scope_where_ids_in(ids, converter)
+          sql = default_scope_where_ids_in(ids).to_sql
+          object_class.connection.execute(sql).map(&converter)
+        end
+
+        def raw(scope, converter)
+          sql = scope.to_sql
+          object_class.connection.execute(sql).map(&converter)
+        end
+
+        def relation_class
+          ::ActiveRecord::Relation
+        end
+
+        def object_class
+          ::ActiveRecord::Base
+        end
+      end
+    end
+  end
+end

data/lib/chewy/{type → index}/adapter/base.rb

@@ -1,5 +1,5 @@
 module Chewy
-  class Type
+  class Index
     module Adapter
       # Basic adapter class. Contains interface, need to implement to add any classes support
       class Base
@@ -9,12 +9,11 @@ module Chewy
 
         # Returns `true` if this adapter is applicable for the given target.
         #
-        def self.accepts? target
+        def self.accepts?(_target)
           true
         end
 
-        # Camelcased name, used as type class constant name.
-        # For returned value 'Product' will be generated class name `ProductsIndex::Product`
+        # Camelcased name.
         #
         def name
           raise NotImplementedError
@@ -32,28 +31,48 @@ module Chewy
         # For ORM/ODM it will be an array of ids for simple objects -
         # just objects themselves
         #
-        def identify collection
+        def identify(_collection)
           raise NotImplementedError
         end
 
         # Splits passed objects to groups according to `:batch_size` options.
-        # For every group crates hash with action keys. Example:
+        # For every group creates hash with action keys. Example:
         #
-        #   { delete: [object1, object2], index: [object3, object4, object5] }
+        #   { delete: [object_or_id1, object_or_id2], index: [object3, object4, object5] }
         #
-        # Returns true id all the block call returns true and false otherwise
+        # @yieldparam _batch [Array<Object>] each batch of objects
+        # @return [true, false] returns true if all the block call returns true and false otherwise
+        def import(_batch, &_block)
+          raise NotImplementedError
+        end
+
+        # Unlike {#import} fetches only ids (references) to the imported objects,
+        # using the same procedures as {#import}.
+        #
+        # @param _fields [Array<Symbol>] additional fields to fetch
+        # @param _batch_size [Integer] batch size, defaults to 1000
+        # @yieldparam batch [Array<Object>] each batch of objects
+        def import_fields(_fields, _batch_size, &_block)
+          raise NotImplementedError
+        end
+
+        # Uses the same strategy as import for the passed arguments, and returns
+        # an array of references to the passed objects. Returns ids if possible.
+        # Otherwise - and array of objects themselves.
         #
-        def import *args, &block
+        # @param _batch_size [Integer] batch size, defaults to 1000
+        # @yieldparam batch [Array<Object>] each batch of objects
+        def import_references(_batch_size, &_block)
           raise NotImplementedError
         end
 
-        # Returns array of loaded objects for passed objects array. If some object
+        # Returns array of loaded objects for passed ids array. If some object
         # was not loaded, it returns `nil` in the place of this object
         #
-        #   load(double(id: 1), double(id: 2), double(id: 3)) #=>
+        #   load([1, 2, 3]) #=>
         #     # [<Product id: 1>, nil, <Product id: 3>], assuming, #2 was not found
         #
-        def load *args
+        def load(_ids, **_options)
           raise NotImplementedError
         end