chewy 0.9.0 → 5.2.0

data/lib/chewy/type/observe.rb

@@ -9,14 +9,6 @@ module Chewy
           method = args.first
 
           proc do
-            backreference = if method && method.to_s == 'self'
-              self
-            elsif method
-              send(method)
-            else
-              instance_eval(&block)
-            end
-
             reference = if type_name.is_a?(Proc)
               if type_name.arity.zero?
                 instance_exec(&type_name)
@@ -27,14 +19,26 @@ module Chewy
               type_name
             end
 
-            Chewy.derive_type(reference).update_index(backreference, options)
+            type = Chewy.derive_type(reference)
+
+            next if Chewy.strategy.current.name == :bypass
+
+            backreference = if method && method.to_s == 'self'
+              self
+            elsif method
+              send(method)
+            else
+              instance_eval(&block)
+            end
+
+            type.update_index(backreference, options)
           end
         end
 
         def extract_callback_options!(args)
           options = args.extract_options!
           result = options.each_key.with_object({}) do |key, hash|
-            hash[key] = options.delete(key) if [:if, :unless].include?(key)
+            hash[key] = options.delete(key) if %i[if unless].include?(key)
           end
           args.push(options) unless options.empty?
           result
@@ -59,10 +63,10 @@ module Chewy
           update_proc = Observe.update_proc(type_name, *args, &block)
 
           if Chewy.use_after_commit_callbacks
-            after_commit(callback_options, &update_proc)
+            after_commit(**callback_options, &update_proc)
           else
-            after_save(callback_options, &update_proc)
-            after_destroy(callback_options, &update_proc)
+            after_save(**callback_options, &update_proc)
+            after_destroy(**callback_options, &update_proc)
           end
         end
       end

data/lib/chewy/type/syncer.rb

@@ -0,0 +1,222 @@
+module Chewy
+  class Type
+    # This class is able to find missing and outdated documents in the ES
+    # comparing ids from the data source and the ES index. Also, if `outdated_sync_field`
+    # existss in the index definition, it performs comparison of this field
+    # values for each source object and corresponding ES document. Usually,
+    # this field is `updated_at` and if its value in the source is not equal
+    # to the value in the index - this means that this document outdated and
+    # should be reindexed.
+    #
+    # To fetch necessary data from the source it uses adapter method
+    # {Chewy::Type::Adapter::Base#import_fields}, in case when the Object
+    # adapter is used it makes sense to read corresponding documentation.
+    #
+    # If `parallel` option is passed to the initializer - it will fetch surce and
+    # index data in parallel and then perform outdated objects calculation in
+    # parallel processes. Also, further import (if required) will be performed
+    # in parallel as well.
+    #
+    # @note
+    #   In rails 4.0 time converted to json with the precision of seconds
+    #   without milliseconds used, so outdated check is not so precise there.
+    #
+    #   ATTENTION: synchronization may be slow in case when synchronized tables
+    #   are missing compound index on primary key and `outdated_sync_field`.
+    #
+    # @see Chewy::Type::Actions::ClassMethods#sync
+    class Syncer
+      DEFAULT_SYNC_BATCH_SIZE = 20_000
+      ISO_DATETIME = /\A(\d{4})-(\d\d)-(\d\d) (\d\d):(\d\d):(\d\d)(\.\d+)?\z/
+      OUTDATED_IDS_WORKER = lambda do |outdated_sync_field_type, source_data_hash, type, total, index_data|
+        ::Process.setproctitle("chewy [#{type}]: sync outdated calculation (#{::Parallel.worker_number + 1}/#{total})") if type
+        index_data.each_with_object([]) do |(id, index_sync_value), result|
+          next unless source_data_hash[id]
+
+          outdated = if outdated_sync_field_type == 'date'
+            !Chewy::Type::Syncer.dates_equal(typecast_date(source_data_hash[id]), Time.iso8601(index_sync_value))
+          else
+            source_data_hash[id] != index_sync_value
+          end
+
+          result.push(id) if outdated
+        end
+      end
+      SOURCE_OR_INDEX_DATA_WORKER = lambda do |syncer, type, kind|
+        ::Process.setproctitle("chewy [#{type}]: sync fetching data (#{kind})")
+        result = case kind
+        when :source
+          syncer.send(:fetch_source_data)
+        when :index
+          syncer.send(:fetch_index_data)
+        end
+        {kind => result}
+      end
+
+      def self.typecast_date(string)
+        if string.is_a?(String) && (match = ISO_DATETIME.match(string))
+          microsec = (match[7].to_r * 1_000_000).to_i
+          date = "#{match[1]}-#{match[2]}-#{match[3]}T#{match[4]}:#{match[5]}:#{match[6]}.#{format('%06d', microsec)}+00:00"
+          Time.iso8601(date)
+        else
+          string
+        end
+      end
+
+      # Compares times with ms precision.
+      def self.dates_equal(one, two)
+        [one.to_i, one.usec / 1000] == [two.to_i, two.usec / 1000]
+      end
+
+      # In ActiveSupport ~> 4.0 json dumpled times without any
+      # milliseconds, so ES stored time with the seconds precision.
+      if ActiveSupport::VERSION::STRING < '4.1.0'
+        def self.dates_equal(one, two)
+          one.to_i == two.to_i
+        end
+      end
+
+      # @param type [Chewy::Type] chewy type
+      # @param parallel [true, Integer, Hash] options for parallel execution or the number of processes
+      def initialize(type, parallel: nil)
+        @type = type
+        @parallel = if !parallel || parallel.is_a?(Hash)
+          parallel
+        elsif parallel.is_a?(Integer)
+          {in_processes: parallel}
+        else
+          {}
+        end
+      end
+
+      # Finds all the missing and outdated ids and performs import for them.
+      #
+      # @return [Integer, nil] the amount of missing and outdated documents reindexed, nil in case of errors
+      def perform
+        ids = missing_ids | outdated_ids
+        return 0 if ids.blank?
+        @type.import(ids, parallel: @parallel) && ids.count
+      end
+
+      # Finds ids of all the objects that are not indexed yet or deleted
+      # from the source already.
+      #
+      # @return [Array<String>] an array of missing ids from both sides
+      def missing_ids
+        return [] if source_data.blank?
+
+        @missing_ids ||= begin
+          source_data_ids = data_ids(source_data)
+          index_data_ids = data_ids(index_data)
+
+          (source_data_ids - index_data_ids).concat(index_data_ids - source_data_ids)
+        end
+      end
+
+      # If type supports outdated sync, it compares for the values of the
+      # type `outdated_sync_field` for each object and document in the source
+      # and index and returns the ids of entities which which are having
+      # different values there.
+      #
+      # @see Chewy::Type::Mapping::ClassMethods#supports_outdated_sync?
+      # @return [Array<String>] an array of outdated ids
+      def outdated_ids
+        return [] if source_data.blank? || index_data.blank? || !@type.supports_outdated_sync?
+        @outdated_ids ||= begin
+          if @parallel
+            parallel_outdated_ids
+          else
+            linear_outdated_ids
+          end
+        end
+      end
+
+    private
+
+      def source_data
+        @source_data ||= source_and_index_data.first
+      end
+
+      def index_data
+        @index_data ||= source_and_index_data.second
+      end
+
+      def source_and_index_data
+        @source_and_index_data ||= begin
+          if @parallel
+            ::ActiveRecord::Base.connection.close if defined?(::ActiveRecord::Base)
+            result = ::Parallel.map(%i[source index], @parallel, &SOURCE_OR_INDEX_DATA_WORKER.curry[self, @type])
+            ::ActiveRecord::Base.connection.reconnect! if defined?(::ActiveRecord::Base)
+            if result.first.keys.first == :source
+              [result.first.values.first, result.second.values.first]
+            else
+              [result.second.values.first, result.first.values.first]
+            end
+          else
+            [fetch_source_data, fetch_index_data]
+          end
+        end
+      end
+
+      def fetch_source_data
+        if @type.supports_outdated_sync?
+          @type.adapter.import_fields(fields: [@type.outdated_sync_field], batch_size: DEFAULT_SYNC_BATCH_SIZE, typecast: false).to_a.flatten(1).each do |data|
+            data[0] = data[0].to_s
+          end
+        else
+          @type.adapter.import_fields(batch_size: DEFAULT_SYNC_BATCH_SIZE, typecast: false).to_a.flatten(1).map(&:to_s)
+        end
+      end
+
+      def fetch_index_data
+        if @type.supports_outdated_sync?
+          @type.pluck(:_id, @type.outdated_sync_field).each do |data|
+            data[0] = data[0].to_s
+          end
+        else
+          @type.pluck(:_id).map(&:to_s)
+        end
+      end
+
+      def data_ids(data)
+        return data unless @type.supports_outdated_sync?
+        data.map(&:first)
+      end
+
+      def linear_outdated_ids
+        OUTDATED_IDS_WORKER.call(outdated_sync_field_type, source_data.to_h, nil, nil, index_data)
+      end
+
+      def parallel_outdated_ids
+        size = processor_count.zero? ? index_data.size : (index_data.size / processor_count.to_f).ceil
+        batches = index_data.each_slice(size)
+
+        ::ActiveRecord::Base.connection.close if defined?(::ActiveRecord::Base)
+        result = ::Parallel.map(batches, @parallel, &OUTDATED_IDS_WORKER.curry[outdated_sync_field_type, source_data.to_h, @type, batches.size]).flatten(1)
+        ::ActiveRecord::Base.connection.reconnect! if defined?(::ActiveRecord::Base)
+        result
+      end
+
+      def processor_count
+        @processor_count ||= @parallel[:in_processes] || @parallel[:in_threads] || ::Parallel.processor_count
+      end
+
+      def outdated_sync_field_type
+        return @outdated_sync_field_type if instance_variable_defined?(:@outdated_sync_field_type)
+        return unless @type.outdated_sync_field
+
+        mappings = @type.client.indices.get_mapping(
+          index: @type.index_name,
+          type: @type.type_name
+        ).values.first.fetch('mappings', {})
+
+        @outdated_sync_field_type = mappings
+          .fetch(@type.type_name, {})
+          .fetch('properties', {})
+          .fetch(@type.outdated_sync_field.to_s, {})['type']
+      rescue Elasticsearch::Transport::Transport::Errors::NotFound
+        nil
+      end
+    end
+  end
+end

data/lib/chewy/type/witchcraft.rb

@@ -35,17 +35,20 @@ module Chewy
           !!_witchcraft
         end
 
-        def cauldron
-          @cauldron ||= Cauldron.new(self)
+        def cauldron(**options)
+          (@cauldron ||= {})[options] ||= Cauldron.new(self, **options)
         end
       end
 
       class Cauldron
         attr_reader :locals
 
-        def initialize(type)
+        # @param type [Chewy::Type] type for composition
+        # @param fields [Array<Symbol>] restricts the fields for composition
+        def initialize(type, fields: [])
           @type = type
           @locals = []
+          @fields = fields
         end
 
         def brew(object, crutches = nil)
@@ -55,9 +58,9 @@ module Chewy
       private
 
         def alicorn
-          @alicorn ||= class_eval <<-RUBY
+          @alicorn ||= singleton_class.class_eval <<-RUBY, __FILE__, __LINE__ + 1
             -> (locals, object0, crutches) do
-              #{composed_values(@type.root_object, 0)}
+              #{composed_values(@type.root, 0)}
             end
           RUBY
         end
@@ -91,7 +94,7 @@ module Chewy
         end
 
         def non_proc_values(field, nesting)
-          non_proc_fields = non_proc_fields_for(field)
+          non_proc_fields = non_proc_fields_for(field, nesting)
           object = "object#{nesting}"
 
           if non_proc_fields.present?
@@ -99,14 +102,16 @@ module Chewy
               (if #{object}.is_a?(Hash)
                 {
                   #{non_proc_fields.map do |f|
-                    fetcher = "#{object}.has_key?(:#{f.name}) ? #{object}[:#{f.name}] : #{object}['#{f.name}']"
-                    "#{f.name}: #{composed_value(f, fetcher, nesting)}"
+                    key_name = f.value.is_a?(Symbol) || f.value.is_a?(String) ? f.value : f.name
+                    fetcher = "#{object}.has_key?(:#{key_name}) ? #{object}[:#{key_name}] : #{object}['#{key_name}']"
+                    "'#{f.name}'.freeze => #{composed_value(f, fetcher, nesting)}"
                   end.join(', ')}
                 }
               else
                 {
                   #{non_proc_fields.map do |f|
-                    "#{f.name}: #{composed_value(f, "#{object}.#{f.name}", nesting)}"
+                    method_name = f.value.is_a?(Symbol) || f.value.is_a?(String) ? f.value : f.name
+                    "'#{f.name}'.freeze => #{composed_value(f, "#{object}.#{method_name}", nesting)}"
                   end.join(', ')}
                 }
               end)
@@ -117,13 +122,13 @@ module Chewy
         end
 
         def proc_values(field, nesting)
-          proc_fields = proc_fields_for(field)
+          proc_fields = proc_fields_for(field, nesting)
 
           if proc_fields.present?
             <<-RUBY
               {
                 #{proc_fields.map do |f|
-                  "#{f.name}: (#{composed_value(f, source_for(f.value, nesting), nesting)})"
+                  "'#{f.name}'.freeze => (#{composed_value(f, source_for(f.value, nesting), nesting)})"
                 end.join(', ')}
               }
             RUBY
@@ -132,14 +137,26 @@ module Chewy
           end
         end
 
-        def non_proc_fields_for(parent)
+        def non_proc_fields_for(parent, nesting)
           return [] unless parent
-          (parent.children || []).select { |field| !(field.value && field.value.is_a?(Proc)) }
+          fields = (parent.children || []).reject { |field| field.value.is_a?(Proc) }
+
+          if nesting.zero? && @fields.present?
+            fields.select { |f| @fields.include?(f.name) }
+          else
+            fields
+          end
         end
 
-        def proc_fields_for(parent)
+        def proc_fields_for(parent, nesting)
           return [] unless parent
-          (parent.children || []).select { |field| field.value && field.value.is_a?(Proc) }
+          fields = (parent.children || []).select { |field| field.value.is_a?(Proc) }
+
+          if nesting.zero? && @fields.present?
+            fields.select { |f| @fields.include?(f.name) }
+          else
+            fields
+          end
         end
 
         def source_for(proc, nesting)
@@ -166,7 +183,6 @@ module Chewy
               locals.push(proc.binding.eval(variable.to_s))
               source = replace_local(source, variable, locals.size - 1)
             end
-
           end
 
           Unparser.unparse(source)

data/lib/chewy/type/wrapper.rb

@@ -3,22 +3,48 @@ module Chewy
     module Wrapper
       extend ActiveSupport::Concern
 
-      attr_accessor :attributes, :_data, :_object
+      included do
+        attr_accessor :_data, :_object
+        attr_reader :attributes
+      end
+
+      module ClassMethods
+        def build(hit)
+          attributes = (hit['_source'] || {})
+            .reverse_merge(id: hit['_id'])
+            .merge!(_score: hit['_score'])
+            .merge!(_explanation: hit['_explanation'])
+
+          wrapper = new(attributes)
+          wrapper._data = hit
+          wrapper
+        end
+      end
 
       def initialize(attributes = {})
         @attributes = attributes.stringify_keys
       end
 
       def ==(other)
+        return true if super
+
         if other.is_a?(Chewy::Type)
           self.class == other.class && (respond_to?(:id) ? id == other.id : attributes == other.attributes)
         elsif other.respond_to?(:id)
-          id.to_s == other.id.to_s
+          self.class.adapter.target.is_a?(Class) &&
+            other.is_a?(self.class.adapter.target) &&
+            id.to_s == other.id.to_s
         else
           false
         end
       end
 
+      %w[_id _type _index].each do |name|
+        define_method name do
+          _data[name]
+        end
+      end
+
       def method_missing(method, *args, &block)
         m = method.to_s
         if (name = highlight_name(m))
@@ -44,11 +70,11 @@ module Chewy
       end
 
       def attribute_defined?(attribute)
-        self.class.root_object && self.class.root_object.children.find { |a| a.name.to_s == attribute }.present?
+        self.class.root && self.class.root.children.find { |a| a.name.to_s == attribute }.present?
       end
 
       def highlight(attribute)
-        _data['highlight'][attribute].first
+        _data['highlight'][attribute].first if highlight?(attribute)
       end
 
       def highlight?(attribute)