chewy 0.8.4 → 5.1.0

data/lib/chewy/type/observe.rb

@@ -8,7 +8,21 @@ module Chewy
           options = args.extract_options!
           method = args.first
 
-          Proc.new do
+          proc do
+            reference = if type_name.is_a?(Proc)
+              if type_name.arity.zero?
+                instance_exec(&type_name)
+              else
+                type_name.call(self)
+              end
+            else
+              type_name
+            end
+
+            type = Chewy.derive_type(reference)
+
+            next if Chewy.strategy.current.name == :bypass
+
             backreference = if method && method.to_s == 'self'
               self
             elsif method
@@ -17,25 +31,17 @@ module Chewy
               instance_eval(&block)
             end
 
-            reference = if type_name.is_a?(Proc)
-              type_name.arity == 0 ?
-                instance_exec(&type_name) :
-                type_name.call(self)
-            else
-              type_name
-            end
-
-            Chewy.derive_type(reference).update_index(backreference, options)
+            type.update_index(backreference, options)
           end
         end
 
         def extract_callback_options!(args)
           options = args.extract_options!
-          options.each_key.with_object({}) { |key, hash|
-            hash[key] = options.delete(key) if [:if, :unless].include?(key)
-          }.tap {
-            args.push(options) unless options.empty?
-          }
+          result = options.each_key.with_object({}) do |key, hash|
+            hash[key] = options.delete(key) if %i[if unless].include?(key)
+          end
+          args.push(options) unless options.empty?
+          result
         end
       end
 

data/lib/chewy/type/syncer.rb

@@ -0,0 +1,222 @@
+module Chewy
+  class Type
+    # This class is able to find missing and outdated documents in the ES
+    # comparing ids from the data source and the ES index. Also, if `outdated_sync_field`
+    # existss in the index definition, it performs comparison of this field
+    # values for each source object and corresponding ES document. Usually,
+    # this field is `updated_at` and if its value in the source is not equal
+    # to the value in the index - this means that this document outdated and
+    # should be reindexed.
+    #
+    # To fetch necessary data from the source it uses adapter method
+    # {Chewy::Type::Adapter::Base#import_fields}, in case when the Object
+    # adapter is used it makes sense to read corresponding documentation.
+    #
+    # If `parallel` option is passed to the initializer - it will fetch surce and
+    # index data in parallel and then perform outdated objects calculation in
+    # parallel processes. Also, further import (if required) will be performed
+    # in parallel as well.
+    #
+    # @note
+    #   In rails 4.0 time converted to json with the precision of seconds
+    #   without milliseconds used, so outdated check is not so precise there.
+    #
+    #   ATTENTION: synchronization may be slow in case when synchronized tables
+    #   are missing compound index on primary key and `outdated_sync_field`.
+    #
+    # @see Chewy::Type::Actions::ClassMethods#sync
+    class Syncer
+      DEFAULT_SYNC_BATCH_SIZE = 20_000
+      ISO_DATETIME = /\A(\d{4})-(\d\d)-(\d\d) (\d\d):(\d\d):(\d\d)(\.\d+)?\z/
+      OUTDATED_IDS_WORKER = lambda do |outdated_sync_field_type, source_data_hash, type, total, index_data|
+        ::Process.setproctitle("chewy [#{type}]: sync outdated calculation (#{::Parallel.worker_number + 1}/#{total})") if type
+        index_data.each_with_object([]) do |(id, index_sync_value), result|
+          next unless source_data_hash[id]
+
+          outdated = if outdated_sync_field_type == 'date'
+            !Chewy::Type::Syncer.dates_equal(typecast_date(source_data_hash[id]), Time.iso8601(index_sync_value))
+          else
+            source_data_hash[id] != index_sync_value
+          end
+
+          result.push(id) if outdated
+        end
+      end
+      SOURCE_OR_INDEX_DATA_WORKER = lambda do |syncer, type, kind|
+        ::Process.setproctitle("chewy [#{type}]: sync fetching data (#{kind})")
+        result = case kind
+        when :source
+          syncer.send(:fetch_source_data)
+        when :index
+          syncer.send(:fetch_index_data)
+        end
+        {kind => result}
+      end
+
+      def self.typecast_date(string)
+        if string.is_a?(String) && (match = ISO_DATETIME.match(string))
+          microsec = (match[7].to_r * 1_000_000).to_i
+          date = "#{match[1]}-#{match[2]}-#{match[3]}T#{match[4]}:#{match[5]}:#{match[6]}.#{format('%06d', microsec)}+00:00"
+          Time.iso8601(date)
+        else
+          string
+        end
+      end
+
+      # Compares times with ms precision.
+      def self.dates_equal(one, two)
+        [one.to_i, one.usec / 1000] == [two.to_i, two.usec / 1000]
+      end
+
+      # In ActiveSupport ~> 4.0 json dumpled times without any
+      # milliseconds, so ES stored time with the seconds precision.
+      if ActiveSupport::VERSION::STRING < '4.1.0'
+        def self.dates_equal(one, two)
+          one.to_i == two.to_i
+        end
+      end
+
+      # @param type [Chewy::Type] chewy type
+      # @param parallel [true, Integer, Hash] options for parallel execution or the number of processes
+      def initialize(type, parallel: nil)
+        @type = type
+        @parallel = if !parallel || parallel.is_a?(Hash)
+          parallel
+        elsif parallel.is_a?(Integer)
+          {in_processes: parallel}
+        else
+          {}
+        end
+      end
+
+      # Finds all the missing and outdated ids and performs import for them.
+      #
+      # @return [Integer, nil] the amount of missing and outdated documents reindexed, nil in case of errors
+      def perform
+        ids = missing_ids | outdated_ids
+        return 0 if ids.blank?
+        @type.import(ids, parallel: @parallel) && ids.count
+      end
+
+      # Finds ids of all the objects that are not indexed yet or deleted
+      # from the source already.
+      #
+      # @return [Array<String>] an array of missing ids from both sides
+      def missing_ids
+        return [] if source_data.blank?
+
+        @missing_ids ||= begin
+          source_data_ids = data_ids(source_data)
+          index_data_ids = data_ids(index_data)
+
+          (source_data_ids - index_data_ids).concat(index_data_ids - source_data_ids)
+        end
+      end
+
+      # If type supports outdated sync, it compares for the values of the
+      # type `outdated_sync_field` for each object and document in the source
+      # and index and returns the ids of entities which which are having
+      # different values there.
+      #
+      # @see Chewy::Type::Mapping::ClassMethods#supports_outdated_sync?
+      # @return [Array<String>] an array of outdated ids
+      def outdated_ids
+        return [] if source_data.blank? || index_data.blank? || !@type.supports_outdated_sync?
+        @outdated_ids ||= begin
+          if @parallel
+            parallel_outdated_ids
+          else
+            linear_outdated_ids
+          end
+        end
+      end
+
+    private
+
+      def source_data
+        @source_data ||= source_and_index_data.first
+      end
+
+      def index_data
+        @index_data ||= source_and_index_data.second
+      end
+
+      def source_and_index_data
+        @source_and_index_data ||= begin
+          if @parallel
+            ::ActiveRecord::Base.connection.close if defined?(::ActiveRecord::Base)
+            result = ::Parallel.map(%i[source index], @parallel, &SOURCE_OR_INDEX_DATA_WORKER.curry[self, @type])
+            ::ActiveRecord::Base.connection.reconnect! if defined?(::ActiveRecord::Base)
+            if result.first.keys.first == :source
+              [result.first.values.first, result.second.values.first]
+            else
+              [result.second.values.first, result.first.values.first]
+            end
+          else
+            [fetch_source_data, fetch_index_data]
+          end
+        end
+      end
+
+      def fetch_source_data
+        if @type.supports_outdated_sync?
+          @type.adapter.import_fields(fields: [@type.outdated_sync_field], batch_size: DEFAULT_SYNC_BATCH_SIZE, typecast: false).to_a.flatten(1).each do |data|
+            data[0] = data[0].to_s
+          end
+        else
+          @type.adapter.import_fields(batch_size: DEFAULT_SYNC_BATCH_SIZE, typecast: false).to_a.flatten(1).map(&:to_s)
+        end
+      end
+
+      def fetch_index_data
+        if @type.supports_outdated_sync?
+          @type.pluck(:_id, @type.outdated_sync_field).each do |data|
+            data[0] = data[0].to_s
+          end
+        else
+          @type.pluck(:_id).map(&:to_s)
+        end
+      end
+
+      def data_ids(data)
+        return data unless @type.supports_outdated_sync?
+        data.map(&:first)
+      end
+
+      def linear_outdated_ids
+        OUTDATED_IDS_WORKER.call(outdated_sync_field_type, source_data.to_h, nil, nil, index_data)
+      end
+
+      def parallel_outdated_ids
+        size = processor_count.zero? ? index_data.size : (index_data.size / processor_count.to_f).ceil
+        batches = index_data.each_slice(size)
+
+        ::ActiveRecord::Base.connection.close if defined?(::ActiveRecord::Base)
+        result = ::Parallel.map(batches, @parallel, &OUTDATED_IDS_WORKER.curry[outdated_sync_field_type, source_data.to_h, @type, batches.size]).flatten(1)
+        ::ActiveRecord::Base.connection.reconnect! if defined?(::ActiveRecord::Base)
+        result
+      end
+
+      def processor_count
+        @processor_count ||= @parallel[:in_processes] || @parallel[:in_threads] || ::Parallel.processor_count
+      end
+
+      def outdated_sync_field_type
+        return @outdated_sync_field_type if instance_variable_defined?(:@outdated_sync_field_type)
+        return unless @type.outdated_sync_field
+
+        mappings = @type.client.indices.get_mapping(
+          index: @type.index_name,
+          type: @type.type_name
+        ).values.first.fetch('mappings', {})
+
+        @outdated_sync_field_type = mappings
+          .fetch(@type.type_name, {})
+          .fetch('properties', {})
+          .fetch(@type.outdated_sync_field.to_s, {})['type']
+      rescue Elasticsearch::Transport::Transport::Errors::NotFound
+        nil
+      end
+    end
+  end
+end

data/lib/chewy/type/witchcraft.rb

@@ -1,10 +1,11 @@
-def try_require lib
-  require lib
+begin
+  require 'method_source'
+  require 'parser/current'
+  require 'unparser'
 rescue LoadError
+  nil
 end
 
-%w[method_source parser/current unparser].each { |lib| try_require lib }
-
 module Chewy
   class Type
     module Witchcraft
@@ -22,9 +23,9 @@ module Chewy
 
         def check_requirements!
           messages = []
-          messages << "MethodSource gem is required for the Witchcraft™, please add `gem 'method_source'` to your Gemfile" unless Proc.method_defined?(:source)
-          messages << "Parser gem is required for the Witchcraft™, please add `gem 'parser'` to your Gemfile" unless '::Parser'.safe_constantize
-          messages << "Unparser gem is required for the Witchcraft™, please add `gem 'unparser'` to your Gemfile" unless '::Unparser'.safe_constantize
+          messages << "MethodSource gem is required for the Witchcraft, please add `gem 'method_source'` to your Gemfile" unless Proc.method_defined?(:source)
+          messages << "Parser gem is required for the Witchcraft, please add `gem 'parser'` to your Gemfile" unless '::Parser'.safe_constantize
+          messages << "Unparser gem is required for the Witchcraft, please add `gem 'unparser'` to your Gemfile" unless '::Unparser'.safe_constantize
           messages = messages.join("\n")
 
           raise messages if messages.present?
@@ -34,26 +35,32 @@ module Chewy
           !!_witchcraft
         end
 
-        def cauldron
-          @cauldron ||= Cauldron.new(self)
+        def cauldron(**options)
+          (@cauldron ||= {})[options] ||= Cauldron.new(self, **options)
         end
       end
 
       class Cauldron
-        def initialize(type)
+        attr_reader :locals
+
+        # @param type [Chewy::Type] type for composition
+        # @param fields [Array<Symbol>] restricts the fields for composition
+        def initialize(type, fields: [])
           @type = type
+          @locals = []
+          @fields = fields
         end
 
         def brew(object, crutches = nil)
-          alicorn.call(object, crutches).as_json
+          alicorn.call(locals, object, crutches).as_json
         end
 
       private
 
         def alicorn
-          @alicorn ||= class_eval <<-RUBY
-            -> (object0, crutches) do
-              #{composed_values(@type.root_object, 0)}
+          @alicorn ||= class_eval <<-RUBY, __FILE__, __LINE__ + 1
+            -> (locals, object0, crutches) do
+              #{composed_values(@type.root, 0)}
             end
           RUBY
         end
@@ -87,55 +94,69 @@ module Chewy
         end
 
         def non_proc_values(field, nesting)
-          non_proc_fields = non_proc_fields_for(field)
+          non_proc_fields = non_proc_fields_for(field, nesting)
           object = "object#{nesting}"
 
           if non_proc_fields.present?
             <<-RUBY
               (if #{object}.is_a?(Hash)
                 {
-                  #{non_proc_fields.map do |field|
-                    fetcher = "#{object}.has_key?(:#{field.name}) ? #{object}[:#{field.name}] : #{object}['#{field.name}']"
-                    "#{field.name}: #{composed_value(field, fetcher, nesting)}"
+                  #{non_proc_fields.map do |f|
+                    key_name = f.value.is_a?(Symbol) || f.value.is_a?(String) ? f.value : f.name
+                    fetcher = "#{object}.has_key?(:#{key_name}) ? #{object}[:#{key_name}] : #{object}['#{key_name}']"
+                    "'#{f.name}'.freeze => #{composed_value(f, fetcher, nesting)}"
                   end.join(', ')}
                 }
               else
                 {
-                  #{non_proc_fields.map do |field|
-                    "#{field.name}: #{composed_value(field, "#{object}.#{field.name}", nesting)}"
+                  #{non_proc_fields.map do |f|
+                    method_name = f.value.is_a?(Symbol) || f.value.is_a?(String) ? f.value : f.name
+                    "'#{f.name}'.freeze => #{composed_value(f, "#{object}.#{method_name}", nesting)}"
                   end.join(', ')}
                 }
               end)
             RUBY
           else
-            "{}"
+            '{}'
           end
         end
 
         def proc_values(field, nesting)
-          proc_fields = proc_fields_for(field)
+          proc_fields = proc_fields_for(field, nesting)
 
           if proc_fields.present?
             <<-RUBY
               {
-                #{proc_fields.map do |field|
-                  "#{field.name}: (#{composed_value(field, source_for(field.value, nesting), nesting)})"
+                #{proc_fields.map do |f|
+                  "'#{f.name}'.freeze => (#{composed_value(f, source_for(f.value, nesting), nesting)})"
                 end.join(', ')}
               }
             RUBY
           else
-            "{}"
+            '{}'
           end
         end
 
-        def non_proc_fields_for(parent)
+        def non_proc_fields_for(parent, nesting)
           return [] unless parent
-          (parent.children || []).select { |field| !(field.value && field.value.is_a?(Proc)) }
+          fields = (parent.children || []).reject { |field| field.value.is_a?(Proc) }
+
+          if nesting.zero? && @fields.present?
+            fields.select { |f| @fields.include?(f.name) }
+          else
+            fields
+          end
         end
 
-        def proc_fields_for(parent)
+        def proc_fields_for(parent, nesting)
           return [] unless parent
-          (parent.children || []).select { |field| field.value && field.value.is_a?(Proc) }
+          fields = (parent.children || []).select { |field| field.value.is_a?(Proc) }
+
+          if nesting.zero? && @fields.present?
+            fields.select { |f| @fields.include?(f.name) }
+          else
+            fields
+          end
         end
 
         def source_for(proc, nesting)
@@ -145,26 +166,35 @@ module Chewy
           raise "No lambdas found, try to reformat your code:\n`#{proc.source}`" unless lambdas
 
           source = lambdas.first
-          if proc.arity == 0
+          proc_params = proc.parameters.map(&:second)
+
+          if proc.arity.zero?
             source = replace_self(source, :"object#{nesting}")
             source = replace_send(source, :"object#{nesting}")
           elsif proc.arity < 0
             raise "Splat arguments are unsupported by witchcraft:\n`#{proc.source}"
           else
             (nesting + 1).times do |n|
-              source = replace_lvar(source, proc.parameters[n][1], :"object#{n}") if proc.parameters[n]
+              source = replace_lvar(source, proc_params[n], :"object#{n}") if proc_params[n]
+            end
+            source = replace_lvar(source, proc_params[nesting + 1], :crutches) if proc_params[nesting + 1]
+
+            binding_variable_list(source).each do |variable|
+              locals.push(proc.binding.eval(variable.to_s))
+              source = replace_local(source, variable, locals.size - 1)
             end
-            source = replace_lvar(source, proc.parameters[nesting + 1][1], :crutches) if proc.parameters[nesting + 1]
           end
+
           Unparser.unparse(source)
         end
 
         def exctract_lambdas(node)
+          return unless node.is_a?(Parser::AST::Node)
           if node.type == :block && node.children[0].type == :send && node.children[0].to_a == [nil, :lambda]
-            node.children[2]
+            [node.children[2]]
           else
             node.children.map { |child| exctract_lambdas(child) }.flatten.compact
-          end if node.is_a?(Parser::AST::Node)
+          end
         end
 
         def replace_lvar(node, old_variable, new_variable)
@@ -202,6 +232,31 @@ module Chewy
             node
           end
         end
+
+        def replace_local(node, variable, local_index)
+          if node.is_a?(Parser::AST::Node)
+            if node.type == :send && node.children.to_a == [nil, variable]
+              node.updated(nil, [
+                Parser::AST::Node.new(:lvar, [:locals]),
+                :[],
+                Parser::AST::Node.new(:int, [local_index])
+              ])
+            else
+              node.updated(nil, node.children.map { |child| replace_local(child, variable, local_index) })
+            end
+          else
+            node
+          end
+        end
+
+        def binding_variable_list(node)
+          return unless node.is_a?(Parser::AST::Node)
+          if node.type == :send && node.children[0].nil?
+            node.children[1]
+          else
+            node.children.map { |child| binding_variable_list(child) }.flatten.compact.uniq
+          end
+        end
       end
     end
   end

data/lib/chewy/type/wrapper.rb

@@ -3,50 +3,82 @@ module Chewy
     module Wrapper
       extend ActiveSupport::Concern
 
-      attr_accessor :attributes, :_data, :_object
+      included do
+        attr_accessor :_data, :_object
+        attr_reader :attributes
+      end
+
+      module ClassMethods
+        def build(hit)
+          attributes = (hit['_source'] || {})
+            .reverse_merge(id: hit['_id'])
+            .merge!(_score: hit['_score'])
+            .merge!(_explanation: hit['_explanation'])
+
+          wrapper = new(attributes)
+          wrapper._data = hit
+          wrapper
+        end
+      end
 
       def initialize(attributes = {})
         @attributes = attributes.stringify_keys
       end
 
       def ==(other)
+        return true if super
+
         if other.is_a?(Chewy::Type)
           self.class == other.class && (respond_to?(:id) ? id == other.id : attributes == other.attributes)
         elsif other.respond_to?(:id)
-          id.to_s == other.id.to_s
+          self.class.adapter.target.is_a?(Class) &&
+            other.is_a?(self.class.adapter.target) &&
+            id.to_s == other.id.to_s
         else
           false
         end
       end
 
-      def method_missing(method_name, *args, &block)
-        method_name.to_s.match(/_highlight\z/) do |match|
-          return highlight(match.pre_match) if highlight?(match.pre_match)
+      %w[_id _type _index].each do |name|
+        define_method name do
+          _data[name]
         end
-        return @attributes[method_name.to_s] if @attributes.key?(method_name.to_s)
-        return nil if attribute_defined?(method_name.to_s)
-        super
       end
 
-      def respond_to_missing?(method_name, include_private = false)
-        method_name.to_s.match(/_highlight\z/) { |m| highlight?(m.pre_match) } ||
-          @attributes.key?(method_name.to_s) ||
-          attribute_defined?(method_name.to_s) ||
+      def method_missing(method, *args, &block)
+        m = method.to_s
+        if (name = highlight_name(m))
+          highlight(name)
+        elsif @attributes.key?(m)
+          @attributes[m]
+        elsif attribute_defined?(m)
+          nil
+        else
           super
+        end
+      end
+
+      def respond_to_missing?(method, include_private = false)
+        m = method.to_s
+        highlight_name(m) || @attributes.key?(m) || attribute_defined?(m) || super
       end
 
-      private
+    private
+
+      def highlight_name(method)
+        method.sub(/_highlight\z/, '') if method.end_with?('_highlight')
+      end
 
       def attribute_defined?(attribute)
-        self.class.root_object && self.class.root_object.children.find { |a| a.name.to_s == attribute }.present?
+        self.class.root && self.class.root.children.find { |a| a.name.to_s == attribute }.present?
       end
 
       def highlight(attribute)
-        _data["highlight"][attribute].first
+        _data['highlight'][attribute].first if highlight?(attribute)
       end
 
       def highlight?(attribute)
-        _data.key?("highlight") && _data["highlight"].key?(attribute)
+        _data.key?('highlight') && _data['highlight'].key?(attribute)
       end
     end
   end