chewy 0.9.0 → 0.10.0

data/lib/chewy/type/import/bulk_builder.rb

@@ -0,0 +1,122 @@
+module Chewy
+  class Type
+    module Import
+      # This class purpose is to build ES client-acceptable bulk
+      # request body from the passed objects for index and deletion.
+      # It handles parent-child relationships as well by fetching
+      # existing documents from ES, taking their `_parent` field and
+      # using it in the bulk body.
+      # If fields are passed - it creates partial update entries except for
+      # the cases when the type has parent and parent_id has been changed.
+      class BulkBuilder
+        # @param type [Chewy::Type] desired type
+        # @param index [Array<Object>] objects to index
+        # @param delete [Array<Object>] objects or ids to delete
+        # @param fields [Array<Symbol, String>] and array of fields for documents update
+        def initialize(type, index: [], delete: [], fields: [])
+          @type = type
+          @index = index
+          @delete = delete
+          @fields = fields.map!(&:to_sym)
+        end
+
+        # Returns ES API-ready bulk requiest body.
+        # @see https://github.com/elastic/elasticsearch-ruby/blob/master/elasticsearch-api/lib/elasticsearch/api/actions/bulk.rb
+        # @return [Array<Hash>] bulk body
+        def bulk_body
+          @bulk_body ||= @index.flat_map(&method(:index_entry)).concat(
+            @delete.flat_map(&method(:delete_entry))
+          )
+        end
+
+        # The only purpose of this method is to cache document ids for
+        # all the passed object for index to avoid ids recalculation.
+        #
+        # @return [Hash[String => Object]] an ids-objects index hash
+        def index_objects_by_id
+          @index_objects_by_id ||= index_object_ids.invert.stringify_keys!
+        end
+
+      private
+
+        def crutches
+          @crutches ||= Chewy::Type::Crutch::Crutches.new @type, @index
+        end
+
+        def parents
+          return unless type_root.parent_id
+
+          @parents ||= begin
+            ids = @index.map do |object|
+              object.respond_to?(:id) ? object.id : object
+            end
+            ids.concat(@delete.map do |object|
+              object.respond_to?(:id) ? object.id : object
+            end)
+            @type.filter(ids: {values: ids}).order('_doc').pluck(:_id, :_parent).to_h
+          end
+        end
+
+        def index_entry(object)
+          entry = {}
+          entry[:_id] = index_object_ids[object] if index_object_ids[object]
+
+          if parents
+            entry[:parent] = type_root.compose_parent(object)
+            parent = entry[:_id].present? && parents[entry[:_id].to_s]
+          end
+
+          if parent && entry[:parent].to_s != parent
+            entry[:data] = @type.compose(object, crutches)
+            [{delete: entry.except(:data).merge(parent: parent)}, {index: entry}]
+          elsif @fields.present?
+            return [] unless entry[:_id]
+            entry[:data] = {doc: @type.compose(object, crutches, fields: @fields)}
+            [{update: entry}]
+          else
+            entry[:data] = @type.compose(object, crutches)
+            [{index: entry}]
+          end
+        end
+
+        def delete_entry(object)
+          entry = {}
+          entry[:_id] = entry_id(object)
+          entry[:_id] ||= object.as_json
+
+          return [] if entry[:_id].blank?
+
+          if parents
+            parent = entry[:_id].present? && parents[entry[:_id].to_s]
+            return [] unless parent
+            entry[:parent] = parent
+          end
+
+          [{delete: entry}]
+        end
+
+        def entry_id(object)
+          if type_root.id
+            type_root.compose_id(object)
+          else
+            id = object.id if object.respond_to?(:id)
+            id ||= object[:id] || object['id'] if object.is_a?(Hash)
+            id = id.to_s if defined?(BSON) && id.is_a?(BSON::ObjectId)
+            id
+          end
+        end
+
+        def index_object_ids
+          @index_object_ids ||= @index.each_with_object({}) do |object, result|
+            id = entry_id(object)
+            result[object] = id if id.present?
+          end
+        end
+
+        def type_root
+          @type_root = @type.send(:build_root)
+        end
+      end
+    end
+  end
+end

data/lib/chewy/type/import/bulk_request.rb

@@ -0,0 +1,76 @@
+module Chewy
+  class Type
+    module Import
+      # Adds additional features to elasticsearch-api bulk method:
+      # * supports Chewy index suffix if necessary;
+      # * supports bulk_size, devides the passed body in chunks
+      #   and peforms a separate request for each chunk;
+      # * returns only errored document entries from the response
+      #   if any present.
+      #
+      # @see https://github.com/elastic/elasticsearch-ruby/blob/master/elasticsearch-api/lib/elasticsearch/api/actions/bulk.rb
+      class BulkRequest
+        # @param type [Chewy::Type] a type for the request
+        # @param suffix [String] an index name optional suffix
+        # @param bulk_size [Integer] bulk size in bytes
+        # @param bulk_options [Hash] options passed to the elasticsearch-api bulk method
+        def initialize(type, suffix: nil, bulk_size: nil, **bulk_options)
+          @type = type
+          @suffix = suffix
+          @bulk_size = bulk_size - 1.kilobyte if bulk_size # 1 kilobyte for request header and newlines
+          @bulk_options = bulk_options
+
+          raise ArgumentError, '`bulk_size` can\'t be less than 1 kilobyte' if @bulk_size && @bulk_size <= 0
+        end
+
+        # Performs a bulk request with the passed body, returns empty
+        # array if everything is fine and array filled with errored
+        # document entries if something went wrong.
+        #
+        # @param body [Array<Hash>] a standard bulk request body
+        # @return [Array<Hash>] an array of bulk errors
+        def perform(body)
+          return [] if body.blank?
+
+          request_bodies(body).each_with_object([]) do |request_body, results|
+            response = @type.client.bulk request_base.merge(body: request_body) if request_body.present?
+
+            next unless response.try(:[], 'errors')
+
+            response_items = (response.try(:[], 'items') || [])
+              .select { |item| item.values.first['error'] }
+            results.concat(response_items)
+          end
+        end
+
+      private
+
+        def request_base
+          @request_base ||= {
+            index: @type.index_name(suffix: @suffix),
+            type: @type.type_name
+          }.merge!(@bulk_options)
+        end
+
+        def request_bodies(body)
+          if @bulk_size
+            pieces = body.each_with_object(['']) do |piece, result|
+              operation, meta = piece.to_a.first
+              data = meta.delete(:data)
+              piece = [{operation => meta}, data].compact.map(&:to_json).join("\n")
+
+              if result.last.bytesize + piece.bytesize > @bulk_size
+                result.push(piece)
+              else
+                result[-1] = [result[-1], piece].reject(&:blank?).join("\n")
+              end
+            end
+            pieces.each { |piece| piece << "\n" }
+          else
+            [body]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/chewy/type/import/journal_builder.rb

@@ -0,0 +1,45 @@
+module Chewy
+  class Type
+    module Import
+      class JournalBuilder
+        def initialize(type, index: [], delete: [])
+          @type = type
+          @index = index
+          @delete = delete
+        end
+
+        def bulk_body
+          Chewy::Type::Import::BulkBuilder.new(
+            Chewy::Stash::Journal,
+            index: [
+              entries(:index, @index),
+              entries(:delete, @delete)
+            ].compact
+          ).bulk_body.each do |item|
+            item.values.first.merge!(
+              _index: Chewy::Stash::Journal.index_name,
+              _type: Chewy::Stash::Journal.type_name
+            )
+          end
+        end
+
+      private
+
+        def entries(action, objects)
+          return unless objects.present?
+          {
+            index_name: @type.index.derivable_name,
+            type_name: @type.type_name,
+            action: action,
+            references: identify(objects).map(&:to_json),
+            created_at: Time.now.utc
+          }
+        end
+
+        def identify(objects)
+          @type.adapter.identify(objects)
+        end
+      end
+    end
+  end
+end

data/lib/chewy/type/import/routine.rb

@@ -0,0 +1,138 @@
+module Chewy
+  class Type
+    module Import
+      # This class performs the import routine for the options and objects given.
+      #
+      # 0. Create target and journal indexes if needed.
+      # 1. Iterate over all the passed objects in batches.
+      # 2. For each batch {#process} method is called:
+      #   * creates a bulk request body;
+      #   * appends journal entries for the current batch to the request body;
+      #   * prepends a leftovers bulk to the request body, which is calculated
+      #     basing on the previous iteration errors;
+      #   * performs the bulk request;
+      #   * composes new leftovers bulk for the next iteration basing on the response errors if `update_failover` is true;
+      #   * appends the rest of unfixable errors to the instance level errors array.
+      # 4. Perform the request for the last leftovers bulk if present using {#process_leftovers}.
+      # 3. Return the result errors array.
+      #
+      # At the moment, it tries to restore only from the partial document update errors in cases
+      # when the document doesn't exist only if `update_failover` option is true. In order to
+      # restore, it indexes such an objects completely on the next iteration.
+      #
+      # @see Chewy::Type::Import::ClassMethods#import
+      class Routine
+        BULK_OPTIONS = %i[
+          suffix bulk_size
+          refresh timeout fields pipeline
+          consistency replication
+          wait_for_active_shards routing _source _source_exclude _source_include
+        ].freeze
+
+        DEFAULT_OPTIONS = {
+          refresh: true,
+          update_fields: [],
+          update_failover: true,
+          batch_size: Chewy::Type::Adapter::Base::BATCH_SIZE
+        }.freeze
+
+        attr_reader :options, :parallel_options, :errors, :stats, :leftovers
+
+        # Basically, processes passed options, extracting bulk request specific options.
+        # @param type [Chewy::Type] chewy type
+        # @param options [Hash] import options, see {Chewy::Type::Import::ClassMethods#import}
+        def initialize(type, **options)
+          @type = type
+          @options = options
+          @options.reverse_merge!(@type._default_import_options)
+          @options.reverse_merge!(journal: Chewy.configuration[:journal])
+          @options.reverse_merge!(DEFAULT_OPTIONS)
+          @bulk_options = @options.slice(*BULK_OPTIONS)
+          @parallel_options = @options.delete(:parallel)
+          if @parallel_options && !@parallel_options.is_a?(Hash)
+            @parallel_options = if @parallel_options.is_a?(Integer)
+              {in_processes: @parallel_options}
+            else
+              {}
+            end
+          end
+          @errors = []
+          @stats = {}
+          @leftovers = []
+        end
+
+        # Creates the journal index and the type corresponding index if necessary.
+        # @return [Object] whatever
+        def create_indexes!
+          Chewy::Stash.create if @options[:journal]
+          return if Chewy.configuration[:skip_index_creation_on_import]
+          @type.index.create!(@bulk_options.slice(:suffix)) unless @type.index.exists?
+        end
+
+        # The main process method. Converts passed objects to thr bulk request body,
+        # appends journal entires, performs this request and handles errors performing
+        # failover procedures if applicable.
+        #
+        # @param index [Array<Object>] any acceptable objects for indexing
+        # @param delete [Array<Object>] any acceptable objects for deleting
+        # @return [true, false] the result of the request, true if no errors
+        def process(index: [], delete: [])
+          bulk_builder = BulkBuilder.new(@type, index: index, delete: delete, fields: @options[:update_fields])
+          bulk_body = bulk_builder.bulk_body
+
+          if @options[:journal]
+            journal_builder = JournalBuilder.new(@type, index: index, delete: delete)
+            bulk_body.concat(journal_builder.bulk_body)
+          end
+
+          bulk_body.unshift(*flush_leftovers)
+
+          perform_bulk(bulk_body) do |response|
+            @leftovers = extract_leftovers(response, bulk_builder.index_objects_by_id)
+            @stats[:index] = @stats[:index].to_i + index.count if index.present?
+            @stats[:delete] = @stats[:delete].to_i + delete.count if delete.present?
+          end
+        end
+
+        # Performs a bulk request for the passed body.
+        #
+        # @param body [Array<Hash>] a standard bulk request body
+        # @return [true, false] the result of the request, true if no errors
+        def perform_bulk(body)
+          response = bulk.perform(body)
+          yield response if block_given?
+          Chewy.wait_for_status
+          @errors.concat(response)
+          response.blank?
+        end
+
+      private
+
+        def flush_leftovers
+          leftovers = @leftovers
+          @leftovers = []
+          leftovers
+        end
+
+        def extract_leftovers(errors, index_objects_by_id)
+          return [] unless @options[:update_fields].present? && @options[:update_failover] && errors.present?
+
+          failed_partial_updates = errors.select do |item|
+            item.keys.first == 'update' && item.values.first['error']['type'] == 'document_missing_exception'
+          end
+          failed_ids_hash = failed_partial_updates.index_by { |item| item.values.first['_id'].to_s }
+          failed_ids_for_reimport = failed_ids_hash.keys & index_objects_by_id.keys
+          errors_to_cleanup = failed_ids_hash.values_at(*failed_ids_for_reimport)
+          errors_to_cleanup.each { |error| errors.delete(error) }
+
+          failed_objects = index_objects_by_id.values_at(*failed_ids_for_reimport)
+          BulkBuilder.new(@type, index: failed_objects).bulk_body
+        end
+
+        def bulk
+          @bulk ||= BulkRequest.new(@type, **@bulk_options)
+        end
+      end
+    end
+  end
+end

data/lib/chewy/type/mapping.rb

@@ -8,6 +8,8 @@ module Chewy
         class_attribute :_templates
         class_attribute :_agg_defs
         self._agg_defs = {}
+        class_attribute :outdated_sync_field
+        self.outdated_sync_field = :updated_at
       end
 
       module ClassMethods
@@ -173,6 +175,14 @@ module Chewy
           root_object ? root_object.mappings_hash : {}
         end
 
+        # Check whether the type has outdated_sync_field defined with a simple value.
+        #
+        # @return [true, false]
+        def supports_outdated_sync?
+          updated_at_field = root_object.child_hash[outdated_sync_field] if root_object && outdated_sync_field
+          !!updated_at_field && updated_at_field.value.nil?
+        end
+
       private
 
         def expand_nested(field, &block)
@@ -191,7 +201,7 @@ module Chewy
 
         def build_root(options = {}, &block)
           return root_object if root_object
-          self.root_object = Chewy::Fields::Root.new(type_name, options)
+          self.root_object = Chewy::Fields::Root.new(type_name, Chewy.default_root_options.merge(options))
           expand_nested(root_object, &block)
           @_current_field = root_object
         end

data/lib/chewy/type/observe.rb

@@ -34,7 +34,7 @@ module Chewy
         def extract_callback_options!(args)
           options = args.extract_options!
           result = options.each_key.with_object({}) do |key, hash|
-            hash[key] = options.delete(key) if [:if, :unless].include?(key)
+            hash[key] = options.delete(key) if %i[if unless].include?(key)
           end
           args.push(options) unless options.empty?
           result

data/lib/chewy/type/syncer.rb

@@ -0,0 +1,220 @@
+module Chewy
+  class Type
+    # This class is able to find missing and outdated documents in the ES
+    # comparing ids from the data source and the ES index. Also, if `outdated_sync_field`
+    # existss in the index definition, it performs comparison of this field
+    # values for each source object and corresponding ES document. Usually,
+    # this field is `updated_at` and if its value in the source is not equal
+    # to the value in the index - this means that this document outdated and
+    # should be reindexed.
+    #
+    # To fetch necessary data from the source it uses adapter method
+    # {Chewy::Type::Adapter::Base#import_fields}, in case when the Object
+    # adapter is used it makes sense to read corresponding documentation.
+    #
+    # If `parallel` option is passed to the initializer - it will fetch surce and
+    # index data in parallel and then perform outdated objects calculation in
+    # parallel processes. Also, further import (if required) will be performed
+    # in parallel as well.
+    #
+    # @note
+    #   In rails 4.0 time converted to json with the precision of seconds
+    #   without milliseconds used, so outdated check is not so precise there.
+    #
+    #   ATTENTION: synchronization may be slow in case when synchronized tables
+    #   are missing compound index on primary key and `outdated_sync_field`.
+    #
+    # @see Chewy::Type::Actions::ClassMethods#sync
+    class Syncer
+      DEFAULT_SYNC_BATCH_SIZE = 20_000
+      ISO_DATETIME = /\A(\d{4})-(\d\d)-(\d\d) (\d\d):(\d\d):(\d\d)(\.\d+)?\z/
+      OUTDATED_IDS_WORKER = lambda do |outdated_sync_field_type, source_data_hash, index_data|
+        index_data.each_with_object([]) do |(id, index_sync_value), result|
+          next unless source_data_hash[id]
+
+          outdated = if outdated_sync_field_type == 'date'
+            !Chewy::Type::Syncer.dates_equal(typecast_date(source_data_hash[id]), DateTime.iso8601(index_sync_value))
+          else
+            source_data_hash[id] != index_sync_value
+          end
+
+          result.push(id) if outdated
+        end
+      end
+      SOURCE_OR_INDEX_DATA_WORKER = lambda do |syncer, type|
+        result = case type
+        when :source
+          syncer.send(:fetch_source_data)
+        when :index
+          syncer.send(:fetch_index_data)
+        end
+        {type => result}
+      end
+
+      def self.typecast_date(string)
+        if string.is_a?(String) && (match = ISO_DATETIME.match(string))
+          microsec = (match[7].to_r * 1_000_000).to_i
+          date = "#{match[1]}-#{match[2]}-#{match[3]}T#{match[4]}:#{match[5]}:#{match[6]}.#{format('%06d', microsec)}+00:00"
+          DateTime.iso8601(date)
+        else
+          string
+        end
+      end
+
+      # Compares times with ms precision.
+      def self.dates_equal(one, two)
+        [one.to_i, one.usec / 1000] == [two.to_i, two.usec / 1000]
+      end
+
+      # In ActiveSupport ~> 4.0 json dumpled times without any
+      # milliseconds, so ES stored time with the seconds precision.
+      if ActiveSupport::VERSION::STRING < '4.1.0'
+        def self.dates_equal(one, two)
+          one.to_i == two.to_i
+        end
+      end
+
+      # @param type [Chewy::Type] chewy type
+      # @param parallel [true, Integer, Hash] options for parallel execution or the number of processes
+      def initialize(type, parallel: nil)
+        @type = type
+        @parallel = if !parallel || parallel.is_a?(Hash)
+          parallel
+        elsif parallel.is_a?(Integer)
+          {in_processes: parallel}
+        else
+          {}
+        end
+      end
+
+      # Finds all the missing and outdated ids and performs import for them.
+      #
+      # @return [Integer, nil] the amount of missing and outdated documents reindexed, nil in case of errors
+      def perform
+        ids = missing_ids | outdated_ids
+        return 0 if ids.blank?
+        @type.import(ids, parallel: @parallel) && ids.count
+      end
+
+      # Finds ids of all the objects that are not indexed yet or deleted
+      # from the source already.
+      #
+      # @return [Array<String>] an array of missing ids from both sides
+      def missing_ids
+        return [] if source_data.blank?
+
+        @missing_ids ||= begin
+          source_data_ids = data_ids(source_data)
+          index_data_ids = data_ids(index_data)
+
+          (source_data_ids - index_data_ids).concat(index_data_ids - source_data_ids)
+        end
+      end
+
+      # If type supports outdated sync, it compares for the values of the
+      # type `outdated_sync_field` for each object and document in the source
+      # and index and returns the ids of entities which which are having
+      # different values there.
+      #
+      # @see Chewy::Type::Mapping::ClassMethods#supports_outdated_sync?
+      # @return [Array<String>] an array of outdated ids
+      def outdated_ids
+        return [] if source_data.blank? || index_data.blank? || !@type.supports_outdated_sync?
+        @outdated_ids ||= begin
+          if @parallel
+            parallel_outdated_ids
+          else
+            linear_outdated_ids
+          end
+        end
+      end
+
+    private
+
+      def source_data
+        @source_data ||= source_and_index_data.first
+      end
+
+      def index_data
+        @index_data ||= source_and_index_data.second
+      end
+
+      def source_and_index_data
+        @source_and_index_data ||= begin
+          if @parallel
+            ::ActiveRecord::Base.connection.close if defined?(::ActiveRecord::Base)
+            result = ::Parallel.map(%i[source index], @parallel, &SOURCE_OR_INDEX_DATA_WORKER.curry[self])
+            ::ActiveRecord::Base.connection.reconnect! if defined?(::ActiveRecord::Base)
+            if result.first.keys.first == :source
+              [result.first.values.first, result.second.values.first]
+            else
+              [result.second.values.first, result.first.values.first]
+            end
+          else
+            [fetch_source_data, fetch_index_data]
+          end
+        end
+      end
+
+      def fetch_source_data
+        if @type.supports_outdated_sync?
+          @type.adapter.import_fields(fields: [@type.outdated_sync_field], batch_size: DEFAULT_SYNC_BATCH_SIZE, typecast: false).to_a.flatten(1).each do |data|
+            data[0] = data[0].to_s
+          end
+        else
+          @type.adapter.import_fields(batch_size: DEFAULT_SYNC_BATCH_SIZE, typecast: false).to_a.flatten(1).map(&:to_s)
+        end
+      end
+
+      def fetch_index_data
+        if @type.supports_outdated_sync?
+          @type.pluck(:_id, @type.outdated_sync_field).each do |data|
+            data[0] = data[0].to_s
+          end
+        else
+          @type.pluck(:_id).map(&:to_s)
+        end
+      end
+
+      def data_ids(data)
+        return data unless @type.supports_outdated_sync?
+        data.map(&:first)
+      end
+
+      def linear_outdated_ids
+        OUTDATED_IDS_WORKER.call(outdated_sync_field_type, source_data.to_h, index_data)
+      end
+
+      def parallel_outdated_ids
+        size = processor_count.zero? ? index_data.size : (index_data.size / processor_count.to_f).ceil
+        batches = index_data.each_slice(size)
+
+        ::ActiveRecord::Base.connection.close if defined?(::ActiveRecord::Base)
+        result = ::Parallel.map(batches, @parallel, &OUTDATED_IDS_WORKER.curry[outdated_sync_field_type, source_data.to_h]).flatten(1)
+        ::ActiveRecord::Base.connection.reconnect! if defined?(::ActiveRecord::Base)
+        result
+      end
+
+      def processor_count
+        @processor_count ||= @parallel[:in_processes] || @parallel[:in_threads] || ::Parallel.processor_count
+      end
+
+      def outdated_sync_field_type
+        return @outdated_sync_field_type if instance_variable_defined?(:@outdated_sync_field_type)
+        return unless @type.outdated_sync_field
+
+        mappings = @type.client.indices.get_mapping(
+          index: @type.index_name,
+          type: @type.type_name
+        ).values.first.fetch('mappings', {})
+
+        @outdated_sync_field_type = mappings
+          .fetch(@type.type_name, {})
+          .fetch('properties', {})
+          .fetch(@type.outdated_sync_field.to_s, {})['type']
+      rescue Elasticsearch::Transport::Transport::Errors::NotFound
+        nil
+      end
+    end
+  end
+end