chewy 0.10.0 → 6.0.0

data/lib/chewy/type/import/routine.rb

@@ -13,7 +13,7 @@ module Chewy
       #   * performs the bulk request;
       #   * composes new leftovers bulk for the next iteration basing on the response errors if `update_failover` is true;
       #   * appends the rest of unfixable errors to the instance level errors array.
-      # 4. Perform the request for the last leftovers bulk if present using {#process_leftovers}.
+      # 4. Perform the request for the last leftovers bulk if present using {#extract_leftovers}.
       # 3. Return the result errors array.
       #
       # At the moment, it tries to restore only from the partial document update errors in cases
@@ -64,9 +64,9 @@ module Chewy
         # Creates the journal index and the type corresponding index if necessary.
         # @return [Object] whatever
         def create_indexes!
-          Chewy::Stash.create if @options[:journal]
+          Chewy::Stash::Journal.create if @options[:journal]
           return if Chewy.configuration[:skip_index_creation_on_import]
-          @type.index.create!(@bulk_options.slice(:suffix)) unless @type.index.exists?
+          @type.index.create!(**@bulk_options.slice(:suffix)) unless @type.index.exists?
         end
 
         # The main process method. Converts passed objects to thr bulk request body,

data/lib/chewy/type/mapping.rb

@@ -17,10 +17,11 @@ module Chewy
         # definition. Use it only if you need to pass options for root
         # object mapping, such as `date_detection` or `dynamic_date_formats`
         #
+        # @example
         #   class UsersIndex < Chewy::Index
         #     define_type User do
         #       # root object defined implicitly and optionless for current type
-        #       field :full_name, type: 'string'
+        #       field :full_name, type: 'keyword'
         #     end
         #   end
         #
@@ -28,32 +29,37 @@ module Chewy
         #     define_type Car do
         #       # explicit root definition with additional options
         #       root dynamic_date_formats: ['yyyy-MM-dd'] do
-        #         field :model_name, type: 'string'
+        #         field :model_name, type: 'keyword'
         #       end
         #     end
         #   end
         #
-        def root(options = {}, &block)
-          raise 'Root is already defined' if root_object
-          build_root(options, &block)
+        def root(**options)
+          self.root_object ||= Chewy::Fields::Root.new(type_name, **Chewy.default_root_options.merge(options))
+          root_object.update_options!(**options)
+          yield if block_given?
+          root_object
         end
 
         # Defines mapping field for current type
         #
+        # @example
         #   class UsersIndex < Chewy::Index
         #     define_type User do
         #       # passing all the options to field definition:
-        #       field :full_name, type: 'string', analyzer: 'special'
+        #       field :full_name, analyzer: 'special'
         #     end
         #   end
         #
         # The `type` is optional and defaults to `string` if not defined:
         #
+        # @example
         #   field :full_name
         #
         # Also, multiple fields might be defined with one call and
         # with the same options:
         #
+        # @example
         #   field :first_name, :last_name, analyzer: 'special'
         #
         # The only special option in the field definition
@@ -61,31 +67,35 @@ module Chewy
         # method will be called for the indexed object. Also
         # `:value` might be a proc or indexed object method name:
         #
+        # @example
         #   class User < ActiveRecord::Base
         #     def user_full_name
         #       [first_name, last_name].join(' ')
         #     end
         #   end
         #
-        #   field :full_name, type: 'string', value: :user_full_name
+        #   field :full_name, type: 'keyword', value: :user_full_name
         #
         # The proc evaluates inside the indexed object context if
         # its arity is 0 and in present contexts if there is an argument:
         #
-        #   field :full_name, type: 'string', value: -> { [first_name, last_name].join(' ') }
+        # @example
+        #   field :full_name, type: 'keyword', value: -> { [first_name, last_name].join(' ') }
         #
         #   separator = ' '
-        #   field :full_name, type: 'string', value: ->(user) { [user.first_name, user.last_name].join(separator) }
+        #   field :full_name, type: 'keyword', value: ->(user) { [user.first_name, user.last_name].join(separator) }
         #
         # If array was returned as value - it will be put in index as well.
         #
-        #   field :tags, type: 'string', value: -> { tags.map(&:name) }
+        # @example
+        #   field :tags, type: 'keyword', value: -> { tags.map(&:name) }
         #
         # Fields supports nesting in case of `object` field type. If
         # `user.quiz` will return an array of objects, then result index content
         # will be an array of hashes, if `user.quiz` is not a collection association
         # then just values hash will be put in the index.
         #
+        # @example
         #   field :quiz do
         #     field :question, :answer
         #     field :score, type: 'integer'
@@ -93,6 +103,7 @@ module Chewy
         #
         # Nested fields are composed from nested objects:
         #
+        # @example
         #   field :name, value: -> { name_translations } do
         #     field :ru, value: ->(name) { name['ru'] }
         #     field :en, value: ->(name) { name['en'] }
@@ -101,32 +112,31 @@ module Chewy
         # Of course it is possible to define object fields contents dynamically
         # but make sure evaluation proc returns hash:
         #
+        # @example
         #   field :name, type: 'object', value: -> { name_translations }
         #
         # The special case is multi_field. If type options and block are
         # both present field is treated as a multi-field. In that case field
         # composition changes satisfy elasticsearch rules:
         #
-        #   field :full_name, type: 'string', analyzer: 'name', value: ->{ full_name.try(:strip) } do
+        # @example
+        #   field :full_name, type: 'text', analyzer: 'name', value: ->{ full_name.try(:strip) } do
         #     field :sorted, analyzer: 'sorted'
         #   end
         #
-        def field(*args, &block)
-          options = args.extract_options!
-          build_root
-
+        def field(*args, **options, &block)
           if args.size > 1
-            args.map { |name| field(name, options) }
+            args.map { |name| field(name, **options) }
           else
-            expand_nested(Chewy::Fields::Base.new(args.first, options), &block)
+            expand_nested(Chewy::Fields::Base.new(args.first, **options), &block)
           end
         end
 
         # Defines an aggregation that can be bound to a query or filter
         #
-        #   Suppose that a user has posts and each post has ratings
-        #   avg_post_rating is the mean of all ratings
-        #
+        # @example
+        #   # Suppose that a user has posts and each post has ratings
+        #   # avg_post_rating is the mean of all ratings
         #   class UsersIndex < Chewy::Index
         #     define_type User do
         #       field :posts do
@@ -139,72 +149,68 @@ module Chewy
         #     end
         #   end
         def agg(name, &block)
-          build_root
           self._agg_defs = _agg_defs.merge(name => block)
         end
         alias_method :aggregation, :agg
 
         # Defines dynamic template in mapping root objects
         #
+        # @example
         #   class CarsIndex < Chewy::Index
         #     define_type Car do
-        #       template 'model.*', type: 'string', analyzer: 'special'
+        #       template 'model.*', type: 'text', analyzer: 'special'
         #       field 'model', type: 'object' # here we can put { de: 'Der Mercedes', en: 'Mercedes' }
         #                                     # and template will be applyed to this field
         #     end
         #   end
         #
         # Name for each template is generated with the following
-        # rule: "template_#{dynamic_templates.size + 1}".
+        # rule: `template_#!{dynamic_templates.size + 1}`.
         #
+        # @example Templates
         #   template 'tit*', mapping_hash
         #   template 'title.*', mapping_hash # dot in template causes "path_match" using
         #   template /tit.+/, mapping_hash # using "match_pattern": "regexp"
         #   template /title\..+/, mapping_hash # "\." - escaped dot causes "path_match" using
-        #   template /tit.+/, 'string', mapping_hash # "match_mapping_type" as the optionsl second argument
+        #   template /tit.+/, type: 'text', mapping_hash # "match_mapping_type" as the optionsl second argument
         #   template template42: {match: 'hello*', mapping: {type: 'object'}} # or even pass a template as is
         #
         def template(*args)
-          build_root.dynamic_template(*args)
+          root.dynamic_template(*args)
         end
         alias_method :dynamic_template, :template
 
         # Returns compiled mappings hash for current type
         #
         def mappings_hash
-          root_object ? root_object.mappings_hash : {}
+          root.mappings_hash[type_name.to_sym].present? ? root.mappings_hash : {}
         end
 
         # Check whether the type has outdated_sync_field defined with a simple value.
         #
         # @return [true, false]
         def supports_outdated_sync?
-          updated_at_field = root_object.child_hash[outdated_sync_field] if root_object && outdated_sync_field
+          updated_at_field = root.child_hash[outdated_sync_field] if outdated_sync_field
           !!updated_at_field && updated_at_field.value.nil?
         end
 
       private
 
-        def expand_nested(field, &block)
+        def expand_nested(field)
+          @_current_field ||= root
+
           if @_current_field
             field.parent = @_current_field
             @_current_field.children.push(field)
           end
 
-          return unless block
+          return unless block_given?
 
           previous_field = @_current_field
           @_current_field = field
           yield
           @_current_field = previous_field
         end
-
-        def build_root(options = {}, &block)
-          return root_object if root_object
-          self.root_object = Chewy::Fields::Root.new(type_name, Chewy.default_root_options.merge(options))
-          expand_nested(root_object, &block)
-          @_current_field = root_object
-        end
       end
     end
   end

data/lib/chewy/type/observe.rb

@@ -9,14 +9,6 @@ module Chewy
           method = args.first
 
           proc do
-            backreference = if method && method.to_s == 'self'
-              self
-            elsif method
-              send(method)
-            else
-              instance_eval(&block)
-            end
-
             reference = if type_name.is_a?(Proc)
               if type_name.arity.zero?
                 instance_exec(&type_name)
@@ -27,7 +19,19 @@ module Chewy
               type_name
             end
 
-            Chewy.derive_type(reference).update_index(backreference, options)
+            type = Chewy.derive_type(reference)
+
+            next if Chewy.strategy.current.name == :bypass
+
+            backreference = if method && method.to_s == 'self'
+              self
+            elsif method
+              send(method)
+            else
+              instance_eval(&block)
+            end
+
+            type.update_index(backreference, options)
           end
         end
 
@@ -59,10 +63,10 @@ module Chewy
           update_proc = Observe.update_proc(type_name, *args, &block)
 
           if Chewy.use_after_commit_callbacks
-            after_commit(callback_options, &update_proc)
+            after_commit(**callback_options, &update_proc)
           else
-            after_save(callback_options, &update_proc)
-            after_destroy(callback_options, &update_proc)
+            after_save(**callback_options, &update_proc)
+            after_destroy(**callback_options, &update_proc)
           end
         end
       end

data/lib/chewy/type/syncer.rb

@@ -2,7 +2,7 @@ module Chewy
   class Type
     # This class is able to find missing and outdated documents in the ES
     # comparing ids from the data source and the ES index. Also, if `outdated_sync_field`
-    # existss in the index definition, it performs comparison of this field
+    # exists in the index definition, it performs comparison of this field
     # values for each source object and corresponding ES document. Usually,
     # this field is `updated_at` and if its value in the source is not equal
     # to the value in the index - this means that this document outdated and
@@ -28,12 +28,13 @@ module Chewy
     class Syncer
       DEFAULT_SYNC_BATCH_SIZE = 20_000
       ISO_DATETIME = /\A(\d{4})-(\d\d)-(\d\d) (\d\d):(\d\d):(\d\d)(\.\d+)?\z/
-      OUTDATED_IDS_WORKER = lambda do |outdated_sync_field_type, source_data_hash, index_data|
+      OUTDATED_IDS_WORKER = lambda do |outdated_sync_field_type, source_data_hash, type, total, index_data|
+        ::Process.setproctitle("chewy [#{type}]: sync outdated calculation (#{::Parallel.worker_number + 1}/#{total})") if type
         index_data.each_with_object([]) do |(id, index_sync_value), result|
           next unless source_data_hash[id]
 
           outdated = if outdated_sync_field_type == 'date'
-            !Chewy::Type::Syncer.dates_equal(typecast_date(source_data_hash[id]), DateTime.iso8601(index_sync_value))
+            !Chewy::Type::Syncer.dates_equal(typecast_date(source_data_hash[id]), Time.iso8601(index_sync_value))
           else
             source_data_hash[id] != index_sync_value
           end
@@ -41,21 +42,22 @@ module Chewy
           result.push(id) if outdated
         end
       end
-      SOURCE_OR_INDEX_DATA_WORKER = lambda do |syncer, type|
-        result = case type
+      SOURCE_OR_INDEX_DATA_WORKER = lambda do |syncer, type, kind|
+        ::Process.setproctitle("chewy [#{type}]: sync fetching data (#{kind})")
+        result = case kind
         when :source
           syncer.send(:fetch_source_data)
         when :index
           syncer.send(:fetch_index_data)
         end
-        {type => result}
+        {kind => result}
       end
 
       def self.typecast_date(string)
         if string.is_a?(String) && (match = ISO_DATETIME.match(string))
           microsec = (match[7].to_r * 1_000_000).to_i
           date = "#{match[1]}-#{match[2]}-#{match[3]}T#{match[4]}:#{match[5]}:#{match[6]}.#{format('%06d', microsec)}+00:00"
-          DateTime.iso8601(date)
+          Time.iso8601(date)
         else
           string
         end
@@ -143,7 +145,7 @@ module Chewy
         @source_and_index_data ||= begin
           if @parallel
             ::ActiveRecord::Base.connection.close if defined?(::ActiveRecord::Base)
-            result = ::Parallel.map(%i[source index], @parallel, &SOURCE_OR_INDEX_DATA_WORKER.curry[self])
+            result = ::Parallel.map(%i[source index], @parallel, &SOURCE_OR_INDEX_DATA_WORKER.curry[self, @type])
             ::ActiveRecord::Base.connection.reconnect! if defined?(::ActiveRecord::Base)
             if result.first.keys.first == :source
               [result.first.values.first, result.second.values.first]
@@ -182,7 +184,7 @@ module Chewy
       end
 
       def linear_outdated_ids
-        OUTDATED_IDS_WORKER.call(outdated_sync_field_type, source_data.to_h, index_data)
+        OUTDATED_IDS_WORKER.call(outdated_sync_field_type, source_data.to_h, nil, nil, index_data)
       end
 
       def parallel_outdated_ids
@@ -190,7 +192,7 @@ module Chewy
         batches = index_data.each_slice(size)
 
         ::ActiveRecord::Base.connection.close if defined?(::ActiveRecord::Base)
-        result = ::Parallel.map(batches, @parallel, &OUTDATED_IDS_WORKER.curry[outdated_sync_field_type, source_data.to_h]).flatten(1)
+        result = ::Parallel.map(batches, @parallel, &OUTDATED_IDS_WORKER.curry[outdated_sync_field_type, source_data.to_h, @type, batches.size]).flatten(1)
         ::ActiveRecord::Base.connection.reconnect! if defined?(::ActiveRecord::Base)
         result
       end
@@ -203,10 +205,9 @@ module Chewy
         return @outdated_sync_field_type if instance_variable_defined?(:@outdated_sync_field_type)
         return unless @type.outdated_sync_field
 
-        mappings = @type.client.indices.get_mapping(
-          index: @type.index_name,
-          type: @type.type_name
-        ).values.first.fetch('mappings', {})
+        args = {index: @type.index_name, type: @type.type_name}
+        args[:include_type_name] = true if Runtime.version >= '6.7.0'
+        mappings = @type.client.indices.get_mapping(**args).values.first.fetch('mappings', {})
 
         @outdated_sync_field_type = mappings
           .fetch(@type.type_name, {})

data/lib/chewy/type/witchcraft.rb

@@ -58,9 +58,9 @@ module Chewy
       private
 
         def alicorn
-          @alicorn ||= class_eval <<-RUBY
+          @alicorn ||= singleton_class.class_eval <<-RUBY, __FILE__, __LINE__ + 1
             -> (locals, object0, crutches) do
-              #{composed_values(@type.root_object, 0)}
+              #{composed_values(@type.root, 0)}
             end
           RUBY
         end
@@ -79,7 +79,9 @@ module Chewy
           if field.children.present? && !field.multi_field?
             <<-RUBY
               (result#{nesting} = #{fetcher}
-              if result#{nesting}.respond_to?(:to_ary)
+              if result#{nesting}.nil?
+                nil
+              elsif result#{nesting}.respond_to?(:to_ary)
                 result#{nesting}.map do |object#{nesting}|
                   #{composed_values(field, nesting)}
                 end
@@ -102,14 +104,16 @@ module Chewy
               (if #{object}.is_a?(Hash)
                 {
                   #{non_proc_fields.map do |f|
-                    fetcher = "#{object}.has_key?(:#{f.name}) ? #{object}[:#{f.name}] : #{object}['#{f.name}']"
+                    key_name = f.value.is_a?(Symbol) || f.value.is_a?(String) ? f.value : f.name
+                    fetcher = "#{object}.has_key?(:#{key_name}) ? #{object}[:#{key_name}] : #{object}['#{key_name}']"
                     "'#{f.name}'.freeze => #{composed_value(f, fetcher, nesting)}"
                   end.join(', ')}
                 }
               else
                 {
                   #{non_proc_fields.map do |f|
-                    "'#{f.name}'.freeze => #{composed_value(f, "#{object}.#{f.name}", nesting)}"
+                    method_name = f.value.is_a?(Symbol) || f.value.is_a?(String) ? f.value : f.name
+                    "'#{f.name}'.freeze => #{composed_value(f, "#{object}.#{method_name}", nesting)}"
                   end.join(', ')}
                 }
               end)
@@ -137,7 +141,7 @@ module Chewy
 
         def non_proc_fields_for(parent, nesting)
           return [] unless parent
-          fields = (parent.children || []).reject { |field| field.value && field.value.is_a?(Proc) }
+          fields = (parent.children || []).reject { |field| field.value.is_a?(Proc) }
 
           if nesting.zero? && @fields.present?
             fields.select { |f| @fields.include?(f.name) }
@@ -148,7 +152,7 @@ module Chewy
 
         def proc_fields_for(parent, nesting)
           return [] unless parent
-          fields = (parent.children || []).select { |field| field.value && field.value.is_a?(Proc) }
+          fields = (parent.children || []).select { |field| field.value.is_a?(Proc) }
 
           if nesting.zero? && @fields.present?
             fields.select { |f| @fields.include?(f.name) }

data/lib/chewy/type/wrapper.rb

@@ -41,7 +41,7 @@ module Chewy
 
       %w[_id _type _index].each do |name|
         define_method name do
-          data[name]
+          _data[name]
         end
       end
 
@@ -49,6 +49,8 @@ module Chewy
         m = method.to_s
         if (name = highlight_name(m))
           highlight(name)
+        elsif (name = highlight_names(m))
+          highlights(name)
         elsif @attributes.key?(m)
           @attributes[m]
         elsif attribute_defined?(m)
@@ -60,7 +62,7 @@ module Chewy
 
       def respond_to_missing?(method, include_private = false)
         m = method.to_s
-        highlight_name(m) || @attributes.key?(m) || attribute_defined?(m) || super
+        highlight_name(m) || highlight_names(m) || @attributes.key?(m) || attribute_defined?(m) || super
       end
 
     private
@@ -69,12 +71,20 @@ module Chewy
         method.sub(/_highlight\z/, '') if method.end_with?('_highlight')
       end
 
+      def highlight_names(method)
+        method.sub(/_highlights\z/, '') if method.end_with?('_highlights')
+      end
+
       def attribute_defined?(attribute)
-        self.class.root_object && self.class.root_object.children.find { |a| a.name.to_s == attribute }.present?
+        self.class.root && self.class.root.children.find { |a| a.name.to_s == attribute }.present?
       end
 
       def highlight(attribute)
-        _data['highlight'][attribute].first
+        _data['highlight'][attribute].first if highlight?(attribute)
+      end
+
+      def highlights(attribute)
+        _data['highlight'][attribute] if highlight?(attribute)
       end
 
       def highlight?(attribute)