chewy 0.8.4 → 5.1.0

data/lib/chewy/type/import.rb

@@ -1,222 +1,196 @@
+require 'chewy/type/import/journal_builder'
+require 'chewy/type/import/bulk_builder'
+require 'chewy/type/import/bulk_request'
+require 'chewy/type/import/routine'
+
 module Chewy
   class Type
     module Import
       extend ActiveSupport::Concern
 
-      BULK_OPTIONS = [:suffix, :bulk_size, :refresh, :consistency, :replication]
+      IMPORT_WORKER = lambda do |type, options, total, ids, index|
+        ::Process.setproctitle("chewy [#{type}]: import data (#{index + 1}/#{total})")
+        routine = Routine.new(type, options)
+        type.adapter.import(*ids, routine.options) do |action_objects|
+          routine.process(**action_objects)
+        end
+        {errors: routine.errors, import: routine.stats, leftovers: routine.leftovers}
+      end
+
+      LEFTOVERS_WORKER = lambda do |type, options, total, body, index|
+        ::Process.setproctitle("chewy [#{type}]: import leftovers (#{index + 1}/#{total})")
+        routine = Routine.new(type, options)
+        routine.perform_bulk(body)
+        routine.errors
+      end
 
       module ClassMethods
-        # Perform import operation for specified documents.
-        # Returns true or false depending on success.
+        # @!method import(*collection, **options)
+        # Basically, one of the main methods for type. Performs any objects import
+        # to the index for a specified type. Does all the objects handling routines.
+        # Performs document import by utilizing bulk API. Bulk size and objects batch
+        # size are controlled by the corresponding options.
         #
-        #   UsersIndex::User.import                          # imports default data set
-        #   UsersIndex::User.import User.active              # imports active users
-        #   UsersIndex::User.import [1, 2, 3]                # imports users with specified ids
-        #   UsersIndex::User.import users                    # imports users collection
-        #   UsersIndex::User.import suffix: Time.now.to_i    # imports data to index with specified suffix if such is exists
-        #   UsersIndex::User.import refresh: false           # to disable index refreshing after import
-        #   UsersIndex::User.import batch_size: 300          # import batch size
-        #   UsersIndex::User.import bulk_size: 10.megabytes  # import ElasticSearch bulk size in bytes
-        #   UsersIndex::User.import consistency: :quorum     # explicit write consistency setting for the operation (one, quorum, all)
-        #   UsersIndex::User.import replication: :async      # explicitly set the replication type (sync, async)
+        # It accepts ORM/ODM objects, PORO, hashes, ids which are used by adapter to
+        # fetch objects from the source depenting on the used adapter. It destroys
+        # passed objects from the index if they are not in the default type scope
+        # or marked for destruction.
         #
-        # See adapters documentation for more details.
+        # It handles parent-child relationships: if the object parent_id has been
+        # changed it destroys the object and recreates it from scratch.
         #
-        def import *args
-          import_options = args.extract_options!
-          import_options.reverse_merge! _default_import_options
-          bulk_options = import_options.reject { |k, _| !BULK_OPTIONS.include?(k) }.reverse_merge!(refresh: true)
-
-          index.create!(bulk_options.slice(:suffix)) unless index.exists?
-
-          ActiveSupport::Notifications.instrument 'import_objects.chewy', type: self do |payload|
-            adapter.import(*args, import_options) do |action_objects|
-              indexed_objects = build_root.parent_id && fetch_indexed_objects(action_objects.values.flatten)
-              body = bulk_body(action_objects, indexed_objects)
-
-              errors = bulk(bulk_options.merge(body: body)) if body.present?
-
-              fill_payload_import payload, action_objects
-              fill_payload_errors payload, errors if errors.present?
-              !errors.present?
-            end
-          end
+        # Performs journaling if enabled: it stores all the ids of the imported
+        # objects to a specialized index. It is possible to replay particular import
+        # later to restore the data consistency.
+        #
+        # Performs partial index update using `update` bulk action if any `fields` are
+        # specified. Note that if document doesn't exist yet, an error will be raised
+        # by ES, but import catches this an errors and performs full indexing
+        # for the corresponding documents. This feature can be disabled by setting
+        # `update_failover` to `false`.
+        #
+        # Utilizes `ActiveSupport::Notifications`, so it is possible to get imported
+        # objects later by listening to the `import_objects.chewy` queue. It is also
+        # possible to get the list of occured errors from the payload if something
+        # went wrong.
+        #
+        # Import can also be run in parallel using the Parallel gem functionality.
+        #
+        # @example
+        #   UsersIndex::User.import(parallel: true) # imports everything in parallel with automatic workers number
+        #   UsersIndex::User.import(parallel: 3) # using 3 workers
+        #   UsersIndex::User.import(parallel: {in_threads: 10}) # in 10 threads
+        #
+        # @see https://github.com/elastic/elasticsearch-ruby/blob/master/elasticsearch-api/lib/elasticsearch/api/actions/bulk.rb
+        # @param collection [Array<Object>] and array or anything to import
+        # @param options [Hash{Symbol => Object}] besides specific import options, it accepts all the options suitable for the bulk API call like `refresh` or `timeout`
+        # @option options [String] suffix an index name suffix, used for zero-downtime reset mostly, no suffix by default
+        # @option options [Integer] bulk_size bulk API chunk size in bytes; if passed, the request is performed several times for each chunk, empty by default
+        # @option options [Integer] batch_size passed to the adapter import method, used to split imported objects in chunks, 1000 by default
+        # @option options [true, false] journal enables imported objects journaling, false by default
+        # @option options [Array<Symbol, String>] update_fields list of fields for the partial import, empty by default
+        # @option options [true, false] update_failover enables full objects reimport in cases of partial update errors, `true` by default
+        # @option options [true, Integer, Hash] parallel enables parallel import processing with the Parallel gem, accepts the number of workers or any Parallel gem acceptable options
+        # @return [true, false] false in case of errors
+        def import(*args)
+          import_routine(*args).blank?
         end
 
-        # Perform import operation for specified documents.
-        # Raises Chewy::ImportFailed exception in case of import errors.
-        # Options are completely the same as for `import` method
-        # See adapters documentation for more details.
+        # @!method import!(*collection, **options)
+        # (see #import)
         #
-        def import! *args
-          errors = nil
-          subscriber = ActiveSupport::Notifications.subscribe('import_objects.chewy') do |*args|
-            errors = args.last[:errors]
-          end
-          import *args
+        # The only difference from {#import} is that it raises an exception
+        # in case of any import errors.
+        #
+        # @raise [Chewy::ImportFailed] in case of errors
+        def import!(*args)
+          errors = import_routine(*args)
           raise Chewy::ImportFailed.new(self, errors) if errors.present?
           true
-        ensure
-          ActiveSupport::Notifications.unsubscribe(subscriber) if subscriber
         end
 
-        # Wraps elasticsearch-ruby client indices bulk method.
-        # Adds `:suffix` option to bulk import to index with specified suffix.
-        def bulk options = {}
-          suffix = options.delete(:suffix)
-          bulk_size = options.delete(:bulk_size)
-          body = options.delete(:body)
-          header = { index: index.build_index_name(suffix: suffix), type: type_name }
-
-          bodies = if bulk_size
-            bulk_size -= 1.kilobyte # 1 kilobyte for request header and newlines
-            raise ArgumentError.new('Import `:bulk_size` can\'t be less then 1 kilobyte') if bulk_size <= 0
-
-            body.each_with_object(['']) do |entry, result|
-              operation, meta = entry.to_a.first
-              data = meta.delete(:data)
-              entry = [{ operation => meta }, data].compact.map(&:to_json).join("\n")
-              if entry.bytesize > bulk_size
-                raise ArgumentError.new('Import `:bulk_size` seems to be less then entry size')
-              elsif result.last.bytesize + entry.bytesize > bulk_size
-                result.push(entry)
-              else
-                result[-1] = [result[-1], entry].delete_if(&:blank?).join("\n")
-              end
-            end
-          else
-            [body]
-          end
-
-          items = bodies.map do |body|
-            result = client.bulk options.merge(header).merge(body: body)
-            result.try(:[], 'items') || []
-          end.flatten
+        # Wraps elasticsearch API bulk method, adds additional features like
+        # `bulk_size` and `suffix`.
+        #
+        # @see https://github.com/elastic/elasticsearch-ruby/blob/master/elasticsearch-api/lib/elasticsearch/api/actions/bulk.rb
+        # @see Chewy::Type::Import::Bulk
+        # @param options [Hash{Symbol => Object}] besides specific import options, it accepts all the options suitable for the bulk API call like `refresh` or `timeout`
+        # @option options [String] suffix bulk API chunk size in bytes; if passed, the request is performed several times for each chunk, empty by default
+        # @option options [Integer] bulk_size bulk API chunk size in bytes; if passed, the request is performed several times for each chunk, empty by default
+        # @option options [Array<Hash>] body elasticsearch API bulk method body
+        # @return [Hash] tricky transposed errors hash, empty if everything is fine
+        def bulk(**options)
+          error_items = BulkRequest.new(self, **options).perform(options[:body])
           Chewy.wait_for_status
 
-          extract_errors items
+          payload_errors(error_items)
         end
 
-      private
-
-        def bulk_body(action_objects, indexed_objects = nil)
-          action_objects.flat_map do |action, objects|
-            method = "#{action}_bulk_entry"
-            crutches = Chewy::Type::Crutch::Crutches.new self, objects
-            objects.flat_map { |object| send(method, object, indexed_objects, crutches) }
-          end
-        end
-
-        def delete_bulk_entry(object, indexed_objects = nil, crutches = nil)
-          entry = {}
-
-          if root_object.id
-            entry[:_id] = root_object.compose_id(object)
+        # Composes a single document from the passed object. Uses either witchcraft
+        # or normal composing under the hood.
+        #
+        # @param object [Object] a data source object
+        # @param crutches [Object] optional crutches object; if ommited - a crutch for the single passed object is created as a fallback
+        # @param fields [Array<Symbol>] and array of fields to restrict the generated document
+        # @return [Hash] a JSON-ready hash
+        def compose(object, crutches = nil, fields: [])
+          crutches ||= Chewy::Type::Crutch::Crutches.new self, [object]
+
+          if witchcraft? && root.children.present?
+            cauldron(fields: fields).brew(object, crutches)
           else
-            entry[:_id] = object.id if object.respond_to?(:id)
-            entry[:_id] ||= object[:id] || object['id'] if object.is_a?(Hash)
-            entry[:_id] ||= object
-            entry[:_id] = entry[:_id].to_s if defined?(BSON) && entry[:_id].is_a?(BSON::ObjectId)
+            root.compose(object, crutches, fields: fields)
           end
-
-          if root_object.parent_id
-            existing_object = entry[:_id].present? && indexed_objects && indexed_objects[entry[:_id].to_s]
-            entry.merge!(parent: existing_object[:parent]) if existing_object
-          end
-
-          [{ delete: entry }]
         end
 
-        def index_bulk_entry(object, indexed_objects = nil, crutches = nil)
-          entry = {}
-
-          if root_object.id
-            entry[:_id] = root_object.compose_id(object)
-          else
-            entry[:_id] = object.id if object.respond_to?(:id)
-            entry[:_id] ||= object[:id] || object['id'] if object.is_a?(Hash)
-            entry[:_id] = entry[:_id].to_s if defined?(BSON) && entry[:_id].is_a?(BSON::ObjectId)
-          end
-          entry.delete(:_id) if entry[:_id].blank?
-
-          if root_object.parent_id
-            entry[:parent] = root_object.compose_parent(object)
-            existing_object = entry[:_id].present? && indexed_objects && indexed_objects[entry[:_id].to_s]
-          end
+      private
 
-          entry[:data] = object_data(object, crutches)
+        def import_routine(*args)
+          return if args.first.blank? && !args.first.nil?
+          routine = Routine.new(self, args.extract_options!)
+          routine.create_indexes!
 
-          if existing_object && entry[:parent].to_s != existing_object[:parent]
-            [{ delete: entry.except(:data).merge(parent: existing_object[:parent]) }, { index: entry }]
+          if routine.parallel_options
+            import_parallel(args, routine)
           else
-            [{ index: entry }]
+            import_linear(args, routine)
           end
         end
 
-        def fill_payload_import payload, action_objects
-          imported = Hash[action_objects.map { |action, objects| [action, objects.count] }]
-          imported.each do |action, count|
-            payload[:import] ||= {}
-            payload[:import][action] ||= 0
-            payload[:import][action] += count
+        def import_linear(objects, routine)
+          ActiveSupport::Notifications.instrument 'import_objects.chewy', type: self do |payload|
+            adapter.import(*objects, routine.options) do |action_objects|
+              routine.process(**action_objects)
+            end
+            routine.perform_bulk(routine.leftovers)
+            payload[:import] = routine.stats
+            payload[:errors] = payload_errors(routine.errors) if routine.errors.present?
+            payload[:errors]
           end
         end
 
-        def fill_payload_errors payload, errors
-          errors.each do |action, errors|
-            errors.each do |error, documents|
-              payload[:errors] ||= {}
-              payload[:errors][action] ||= {}
-              payload[:errors][action][error] ||= []
-              payload[:errors][action][error] |= documents
+        def import_parallel(objects, routine)
+          raise "The `parallel` gem is required for parallel import, please add `gem 'parallel'` to your Gemfile" unless '::Parallel'.safe_constantize
+
+          ActiveSupport::Notifications.instrument 'import_objects.chewy', type: self do |payload|
+            batches = adapter.import_references(*objects, routine.options.slice(:batch_size)).to_a
+
+            ::ActiveRecord::Base.connection.close if defined?(::ActiveRecord::Base)
+            results = ::Parallel.map_with_index(batches, routine.parallel_options, &IMPORT_WORKER.curry[self, routine.options, batches.size])
+            ::ActiveRecord::Base.connection.reconnect! if defined?(::ActiveRecord::Base)
+            errors, import, leftovers = process_parallel_import_results(results)
+
+            if leftovers.present?
+              batches = leftovers.each_slice(routine.options[:batch_size])
+              results = ::Parallel.map_with_index(batches, routine.parallel_options, &LEFTOVERS_WORKER.curry[self, routine.options, batches.size])
+              errors.concat(results.flatten(1))
             end
-          end
-        end
 
-        def object_data object, crutches = nil
-          if witchcraft?
-            cauldron.brew(object, crutches)
-          else
-            build_root.compose(object, crutches)[type_name.to_sym]
+            payload[:import] = import
+            payload[:errors] = payload_errors(errors) if errors.present?
+            payload[:errors]
           end
         end
 
-        def extract_errors items
-          items.each.with_object({}) do |item, memo|
-            action = item.keys.first.to_sym
-            data = item.values.first
-            if data['error']
-              (memo[action] ||= []).push(action: action, id: data['_id'], error: data['error'])
-            end
-          end.map do |action, items|
-            errors = items.group_by { |item| item[:error] }.map do |error, items|
-              {error => items.map { |item| item[:id] }}
-            end.reduce(&:merge)
-            {action => errors}
-          end.reduce(&:merge) || {}
+        def process_parallel_import_results(results)
+          results.each_with_object([[], {}, []]) do |r, (e, i, l)|
+            e.concat(r[:errors])
+            i.merge!(r[:import]) { |_k, v1, v2| v1.to_i + v2.to_i }
+            l.concat(r[:leftovers])
+          end
         end
 
-        def fetch_indexed_objects(objects)
-          ids = objects.map { |object| object.respond_to?(:id) ? object.id : object }
-          result = client.search index: index_name,
-                                 type: type_name,
-                                 fields: '_parent',
-                                 body: { filter: { ids: { values: ids } } },
-                                 search_type: 'scan',
-                                 scroll: '1m'
-
-          indexed_objects = {}
+        def payload_errors(errors)
+          errors.each_with_object({}) do |error, result|
+            action = error.keys.first.to_sym
+            item = error.values.first
+            error = item['error']
+            id = item['_id']
 
-          while result = client.scroll(scroll_id: result['_scroll_id'], scroll: '1m') do
-            break if result['hits']['hits'].empty?
-
-            result['hits']['hits'].map do |hit|
-              parent = hit.has_key?('_parent') ? hit['_parent'] : hit['fields']['_parent']
-              indexed_objects[hit['_id']] = { parent: parent }
-            end
+            result[action] ||= {}
+            result[action][error] ||= []
+            result[action][error].push(id)
           end
-
-          indexed_objects
         end
       end
     end

data/lib/chewy/type/mapping.rb

@@ -8,6 +8,8 @@ module Chewy
         class_attribute :_templates
         class_attribute :_agg_defs
         self._agg_defs = {}
+        class_attribute :outdated_sync_field
+        self.outdated_sync_field = :updated_at
       end
 
       module ClassMethods
@@ -15,10 +17,11 @@ module Chewy
         # definition. Use it only if you need to pass options for root
         # object mapping, such as `date_detection` or `dynamic_date_formats`
         #
+        # @example
         #   class UsersIndex < Chewy::Index
         #     define_type User do
         #       # root object defined implicitly and optionless for current type
-        #       field :full_name, type: 'string'
+        #       field :full_name, type: 'keyword'
         #     end
         #   end
         #
@@ -26,32 +29,37 @@ module Chewy
         #     define_type Car do
         #       # explicit root definition with additional options
         #       root dynamic_date_formats: ['yyyy-MM-dd'] do
-        #         field :model_name, type: 'string'
+        #         field :model_name, type: 'keyword'
         #       end
         #     end
         #   end
         #
-        def root options = {}, &block
-          raise "Root is already defined" if root_object
-          build_root(options, &block)
+        def root(**options)
+          self.root_object ||= Chewy::Fields::Root.new(type_name, Chewy.default_root_options.merge(options))
+          root_object.update_options!(options)
+          yield if block_given?
+          root_object
         end
 
         # Defines mapping field for current type
         #
+        # @example
         #   class UsersIndex < Chewy::Index
         #     define_type User do
         #       # passing all the options to field definition:
-        #       field :full_name, type: 'string', analyzer: 'special'
+        #       field :full_name, analyzer: 'special'
         #     end
         #   end
         #
         # The `type` is optional and defaults to `string` if not defined:
         #
+        # @example
         #   field :full_name
         #
         # Also, multiple fields might be defined with one call and
         # with the same options:
         #
+        # @example
         #   field :first_name, :last_name, analyzer: 'special'
         #
         # The only special option in the field definition
@@ -59,31 +67,35 @@ module Chewy
         # method will be called for the indexed object. Also
         # `:value` might be a proc or indexed object method name:
         #
+        # @example
         #   class User < ActiveRecord::Base
         #     def user_full_name
         #       [first_name, last_name].join(' ')
         #     end
         #   end
         #
-        #   field :full_name, type: 'string', value: :user_full_name
+        #   field :full_name, type: 'keyword', value: :user_full_name
         #
         # The proc evaluates inside the indexed object context if
         # its arity is 0 and in present contexts if there is an argument:
         #
-        #   field :full_name, type: 'string', value: -> { [first_name, last_name].join(' ') }
+        # @example
+        #   field :full_name, type: 'keyword', value: -> { [first_name, last_name].join(' ') }
         #
         #   separator = ' '
-        #   field :full_name, type: 'string', value: ->(user) { [user.first_name, user.last_name].join(separator) }
+        #   field :full_name, type: 'keyword', value: ->(user) { [user.first_name, user.last_name].join(separator) }
         #
         # If array was returned as value - it will be put in index as well.
         #
-        #   field :tags, type: 'string', value: -> { tags.map(&:name) }
+        # @example
+        #   field :tags, type: 'keyword', value: -> { tags.map(&:name) }
         #
         # Fields supports nesting in case of `object` field type. If
         # `user.quiz` will return an array of objects, then result index content
         # will be an array of hashes, if `user.quiz` is not a collection association
         # then just values hash will be put in the index.
         #
+        # @example
         #   field :quiz do
         #     field :question, :answer
         #     field :score, type: 'integer'
@@ -91,28 +103,28 @@ module Chewy
         #
         # Nested fields are composed from nested objects:
         #
+        # @example
         #   field :name, value: -> { name_translations } do
         #     field :ru, value: ->(name) { name['ru'] }
         #     field :en, value: ->(name) { name['en'] }
         #   end
         #
-        # Off course it is possible to define object fields contents dynamically
+        # Of course it is possible to define object fields contents dynamically
         # but make sure evaluation proc returns hash:
         #
+        # @example
         #   field :name, type: 'object', value: -> { name_translations }
         #
         # The special case is multi_field. If type options and block are
         # both present field is treated as a multi-field. In that case field
         # composition changes satisfy elasticsearch rules:
         #
-        #   field :full_name, type: 'string', analyzer: 'name', value: ->{ full_name.try(:strip) } do
+        # @example
+        #   field :full_name, type: 'text', analyzer: 'name', value: ->{ full_name.try(:strip) } do
         #     field :sorted, analyzer: 'sorted'
         #   end
         #
-        def field *args, &block
-          options = args.extract_options!
-          build_root
-
+        def field(*args, **options, &block)
           if args.size > 1
             args.map { |name| field(name, options) }
           else
@@ -122,9 +134,9 @@ module Chewy
 
         # Defines an aggregation that can be bound to a query or filter
         #
-        #   Suppose that a user has posts and each post has ratings
-        #   avg_post_rating is the mean of all ratings
-        #
+        # @example
+        #   # Suppose that a user has posts and each post has ratings
+        #   # avg_post_rating is the mean of all ratings
         #   class UsersIndex < Chewy::Index
         #     define_type User do
         #       field :posts do
@@ -136,64 +148,68 @@ module Chewy
         #       end
         #     end
         #   end
-        def agg *args, &block
-          options = args.extract_options!
-          build_root
-          self._agg_defs = _agg_defs.merge(args.first => block)
+        def agg(name, &block)
+          self._agg_defs = _agg_defs.merge(name => block)
         end
         alias_method :aggregation, :agg
 
         # Defines dynamic template in mapping root objects
         #
+        # @example
         #   class CarsIndex < Chewy::Index
         #     define_type Car do
-        #       template 'model.*', type: 'string', analyzer: 'special'
-        #       field 'model', type: 'object' # here we can put { ru: 'Мерседес', en: 'Mercedes' }
+        #       template 'model.*', type: 'text', analyzer: 'special'
+        #       field 'model', type: 'object' # here we can put { de: 'Der Mercedes', en: 'Mercedes' }
         #                                     # and template will be applyed to this field
         #     end
         #   end
         #
         # Name for each template is generated with the following
-        # rule: "template_#{dynamic_templates.size + 1}".
+        # rule: `template_#!{dynamic_templates.size + 1}`.
         #
+        # @example Templates
         #   template 'tit*', mapping_hash
         #   template 'title.*', mapping_hash # dot in template causes "path_match" using
         #   template /tit.+/, mapping_hash # using "match_pattern": "regexp"
         #   template /title\..+/, mapping_hash # "\." - escaped dot causes "path_match" using
-        #   template /tit.+/, 'string', mapping_hash # "match_mapping_type" as the optionsl second argument
+        #   template /tit.+/, type: 'text', mapping_hash # "match_mapping_type" as the optionsl second argument
         #   template template42: {match: 'hello*', mapping: {type: 'object'}} # or even pass a template as is
         #
-        def template *args
-          build_root.dynamic_template *args
+        def template(*args)
+          root.dynamic_template(*args)
         end
         alias_method :dynamic_template, :template
 
         # Returns compiled mappings hash for current type
         #
         def mappings_hash
-          root_object ? root_object.mappings_hash : {}
+          root.mappings_hash[type_name.to_sym].present? ? root.mappings_hash : {}
+        end
+
+        # Check whether the type has outdated_sync_field defined with a simple value.
+        #
+        # @return [true, false]
+        def supports_outdated_sync?
+          updated_at_field = root.child_hash[outdated_sync_field] if outdated_sync_field
+          !!updated_at_field && updated_at_field.value.nil?
         end
 
       private
 
-        def expand_nested field, &block
+        def expand_nested(field)
+          @_current_field ||= root
+
           if @_current_field
             field.parent = @_current_field
             @_current_field.children.push(field)
           end
 
-          if block
-            previous_field, @_current_field = @_current_field, field
-            block.call
-            @_current_field = previous_field
-          end
-        end
+          return unless block_given?
 
-        def build_root options = {}, &block
-          return root_object if root_object
-          self.root_object = Chewy::Fields::Root.new(type_name, options)
-          expand_nested(root_object, &block)
-          @_current_field = root_object
+          previous_field = @_current_field
+          @_current_field = field
+          yield
+          @_current_field = previous_field
         end
       end
     end