chewy 5.0.0 → 7.1.0

data/lib/chewy/search/response.rb

@@ -23,7 +23,7 @@ module Chewy
       #
       # @return [Integer]
       def total
-        @total ||= hits_root['total'] || 0
+        @total ||= hits_root.fetch('total', {}).fetch('value', 0)
       end
 
       # Response `max_score` field.

data/lib/chewy/search/scrolling.rb

@@ -28,19 +28,22 @@ module Chewy
         return enum_for(:scroll_batches, batch_size: batch_size, scroll: scroll) unless block_given?
 
         result = perform(size: batch_size, scroll: scroll)
-        total = [raw_limit_value, result.fetch('hits', {}).fetch('total', 0)].compact.min
+        total = [raw_limit_value, result.fetch('hits', {}).fetch('total', {}).fetch('value', 0)].compact.min
         last_batch_size = total % batch_size
         fetched = 0
+        scroll_id = nil
 
         loop do
           hits = result.fetch('hits', {}).fetch('hits', [])
           fetched += hits.size
           hits = hits.first(last_batch_size) if last_batch_size != 0 && fetched >= total
           yield(hits) if hits.present?
-          break if fetched >= total
           scroll_id = result['_scroll_id']
+          break if fetched >= total
           result = perform_scroll(scroll: scroll, scroll_id: scroll_id)
         end
+      ensure
+        Chewy.client.clear_scroll(scroll_id: scroll_id) if scroll_id
       end
 
       # @!method scroll_hits(batch_size: 1000, scroll: '1m')
@@ -125,11 +128,9 @@ module Chewy
 
       def perform_scroll(body)
         ActiveSupport::Notifications.instrument 'search_query.chewy',
-          request: body, indexes: _indexes, types: _types,
-          index: _indexes.one? ? _indexes.first : _indexes,
-          type: _types.one? ? _types.first : _types do
-          Chewy.client.scroll(body)
-        end
+          notification_payload(request: body) do
+            Chewy.client.scroll(body)
+          end
       end
     end
   end

data/lib/chewy/stash.rb

@@ -24,7 +24,7 @@ module Chewy
       # @param since_time [Time, DateTime] a timestamp from which we load a journal
       # @param only [Chewy::Index, Array<Chewy::Index>] journal entries related to these indices will be loaded only
       def self.entries(since_time, only: [])
-        self.for(only).filter(range: {created_at: {gt: since_time}})
+        self.for(only).filter(range: {created_at: {gt: since_time}}).filter.minimum_should_match(1)
       end
 
       # Cleans up all the journal entries until the specified time. If nothing is
@@ -46,11 +46,8 @@ module Chewy
         types = something.flat_map { |s| Chewy.derive_types(s) }
         return none if something.present? && types.blank?
         scope = all
-        types.group_by(&:index).each do |index, index_types|
-          scope = scope.or(
-            filter(term: {index_name: index.derivable_name})
-              .filter(terms: {type_name: index_types.map(&:type_name)})
-          )
+        types.map(&:index).uniq.each do |index|
+          scope = scope.or(filter(term: {index_name: index.derivable_name}))
         end
         scope
       end

data/lib/chewy/strategy/active_job.rb

@@ -11,7 +11,7 @@ module Chewy
     #
     class ActiveJob < Atomic
       class Worker < ::ActiveJob::Base
-        queue_as :chewy
+        queue_as { Chewy.settings.dig(:active_job, :queue) || 'chewy' }
 
         def perform(type, ids, options = {})
           options[:refresh] = !Chewy.disable_refresh_async if Chewy.disable_refresh_async

data/lib/chewy/strategy/sidekiq.rb

@@ -33,7 +33,7 @@ module Chewy
     private
 
       def sidekiq_queue
-        Chewy.settings.fetch(:sidekiq, {})[:queue] || 'chewy'
+        Chewy.settings.dig(:sidekiq, :queue) || 'chewy'
       end
     end
   end

data/lib/chewy/type.rb

@@ -14,7 +14,10 @@ require 'chewy/type/witchcraft'
 
 module Chewy
   class Type
-    IMPORT_OPTIONS_KEYS = %i[batch_size bulk_size refresh consistency replication raw_import journal pipeline].freeze
+    IMPORT_OPTIONS_KEYS = %i[
+      batch_size bulk_size consistency direct_import journal
+      pipeline raw_import refresh replication
+    ].freeze
 
     include Search
     include Mapping

data/lib/chewy/type/adapter/active_record.rb

@@ -22,7 +22,7 @@ module Chewy
         end
 
         def import_scope(scope, options)
-          pluck_in_batches(scope, options.slice(:batch_size)).inject(true) do |result, ids|
+          pluck_in_batches(scope, **options.slice(:batch_size)).inject(true) do |result, ids|
             objects = if options[:raw_import]
               raw_default_scope_where_ids_in(ids, options[:raw_import])
             else

data/lib/chewy/type/adapter/mongoid.rb

@@ -25,7 +25,7 @@ module Chewy
         end
 
         def import_scope(scope, options)
-          pluck_in_batches(scope, options.slice(:batch_size)).map do |ids|
+          pluck_in_batches(scope, **options.slice(:batch_size)).map do |ids|
             yield grouped_objects(default_scope_where_ids_in(ids))
           end.all?
         end

data/lib/chewy/type/adapter/orm.rb

@@ -45,6 +45,7 @@ module Chewy
         # Import options:
         #
         #   <tt>:batch_size</tt> - import batch size, 1000 objects by default
+        #   <tt>:direct_import</tt> - import objects without reloading
         #
         # Method handles destroyed objects as well. In case of objects ORM scope
         # or array passed, objects, responding with true to `destroyed?` method will be deleted
@@ -75,10 +76,10 @@ module Chewy
         def import(*args, &block)
           collection, options = import_args(*args)
 
-          if collection.is_a?(relation_class)
-            import_scope(collection, options, &block)
-          else
+          if !collection.is_a?(relation_class) || options[:direct_import]
             import_objects(collection, options, &block)
+          else
+            import_scope(collection, options, &block)
           end
         end
 
@@ -89,7 +90,7 @@ module Chewy
 
           if options[:fields].present? || collection.is_a?(relation_class)
             collection = all_scope_where_ids_in(identify(collection)) unless collection.is_a?(relation_class)
-            pluck_in_batches(collection, options.slice(:fields, :batch_size, :typecast), &block)
+            pluck_in_batches(collection, **options.slice(:fields, :batch_size, :typecast), &block)
           else
             identify(collection).each_slice(options[:batch_size]) do |batch|
               yield batch
@@ -119,6 +120,8 @@ module Chewy
           indexed = collection_ids.each_slice(options[:batch_size]).map do |ids|
             batch = if options[:raw_import]
               raw_default_scope_where_ids_in(ids, options[:raw_import])
+            elsif options[:direct_import]
+              hash.values_at(*ids.map(&:to_s))
             else
               default_scope_where_ids_in(ids)
             end

data/lib/chewy/type/adapter/sequel.rb

@@ -22,7 +22,7 @@ module Chewy
         end
 
         def import_scope(scope, options)
-          pluck_in_batches(scope, options.slice(:batch_size)).inject(true) do |result, ids|
+          pluck_in_batches(scope, **options.slice(:batch_size)).inject(true) do |result, ids|
             result & yield(grouped_objects(default_scope_where_ids_in(ids).all))
           end
         end

data/lib/chewy/type/import.rb

@@ -10,7 +10,7 @@ module Chewy
 
       IMPORT_WORKER = lambda do |type, options, total, ids, index|
         ::Process.setproctitle("chewy [#{type}]: import data (#{index + 1}/#{total})")
-        routine = Routine.new(type, options)
+        routine = Routine.new(type, **options)
         type.adapter.import(*ids, routine.options) do |action_objects|
           routine.process(**action_objects)
         end
@@ -19,7 +19,7 @@ module Chewy
 
       LEFTOVERS_WORKER = lambda do |type, options, total, body, index|
         ::Process.setproctitle("chewy [#{type}]: import leftovers (#{index + 1}/#{total})")
-        routine = Routine.new(type, options)
+        routine = Routine.new(type, **options)
         routine.perform_bulk(body)
         routine.errors
       end
@@ -67,6 +67,7 @@ module Chewy
         # @option options [String] suffix an index name suffix, used for zero-downtime reset mostly, no suffix by default
         # @option options [Integer] bulk_size bulk API chunk size in bytes; if passed, the request is performed several times for each chunk, empty by default
         # @option options [Integer] batch_size passed to the adapter import method, used to split imported objects in chunks, 1000 by default
+        # @option options [Boolean] direct_import skips object reloading in ORM adapter, `false` by default
         # @option options [true, false] journal enables imported objects journaling, false by default
         # @option options [Array<Symbol, String>] update_fields list of fields for the partial import, empty by default
         # @option options [true, false] update_failover enables full objects reimport in cases of partial update errors, `true` by default
@@ -126,8 +127,9 @@ module Chewy
       private
 
         def import_routine(*args)
-          return if args.first.blank? && !args.first.nil?
-          routine = Routine.new(self, args.extract_options!)
+          return if !args.first.nil? && empty_objects_or_scope?(args.first)
+
+          routine = Routine.new(self, **args.extract_options!)
           routine.create_indexes!
 
           if routine.parallel_options
@@ -137,6 +139,14 @@ module Chewy
           end
         end
 
+        def empty_objects_or_scope?(objects_or_scope)
+          if objects_or_scope.respond_to?(:empty?)
+            objects_or_scope.empty?
+          else
+            objects_or_scope.blank?
+          end
+        end
+
         def import_linear(objects, routine)
           ActiveSupport::Notifications.instrument 'import_objects.chewy', type: self do |payload|
             adapter.import(*objects, routine.options) do |action_objects|

data/lib/chewy/type/import/bulk_request.rb

@@ -47,22 +47,23 @@ module Chewy
 
         def request_base
           @request_base ||= {
-            index: @type.index_name(suffix: @suffix),
-            type: @type.type_name
+            index: @type.index_name(suffix: @suffix)
           }.merge!(@bulk_options)
         end
 
         def request_bodies(body)
           if @bulk_size
+            serializer = ::Elasticsearch::API.serializer
             pieces = body.each_with_object(['']) do |piece, result|
               operation, meta = piece.to_a.first
               data = meta.delete(:data)
-              piece = [{operation => meta}, data].compact.map(&:to_json).join("\n")
+              piece = serializer.dump(operation => meta)
+              piece << "\n" << serializer.dump(data) if data.present?
 
               if result.last.bytesize + piece.bytesize > @bulk_size
                 result.push(piece)
               else
-                result[-1] = [result[-1], piece].reject(&:blank?).join("\n")
+                result[-1].blank? ? (result[-1] = piece) : (result[-1] << "\n" << piece)
               end
             end
             pieces.each { |piece| piece << "\n" }

data/lib/chewy/type/import/journal_builder.rb

@@ -17,8 +17,7 @@ module Chewy
             ].compact
           ).bulk_body.each do |item|
             item.values.first.merge!(
-              _index: Chewy::Stash::Journal.index_name,
-              _type: Chewy::Stash::Journal::Journal.type_name
+              _index: Chewy::Stash::Journal.index_name
             )
           end
         end
@@ -31,7 +30,7 @@ module Chewy
             index_name: @type.index.derivable_name,
             type_name: @type.type_name,
             action: action,
-            references: identify(objects).map(&:to_json).map(&Base64.method(:encode64)),
+            references: identify(objects).map { |item| Base64.encode64(::Elasticsearch::API.serializer.dump(item)) },
             created_at: Time.now.utc
           }
         end

data/lib/chewy/type/import/routine.rb

@@ -66,11 +66,11 @@ module Chewy
         def create_indexes!
           Chewy::Stash::Journal.create if @options[:journal]
           return if Chewy.configuration[:skip_index_creation_on_import]
-          @type.index.create!(@bulk_options.slice(:suffix)) unless @type.index.exists?
+          @type.index.create!(**@bulk_options.slice(:suffix)) unless @type.index.exists?
         end
 
-        # The main process method. Converts passed objects to thr bulk request body,
-        # appends journal entires, performs this request and handles errors performing
+        # The main process method. Converts passed objects to the bulk request body,
+        # appends journal entries, performs this request and handles errors performing
         # failover procedures if applicable.
         #
         # @param index [Array<Object>] any acceptable objects for indexing

data/lib/chewy/type/mapping.rb

@@ -35,8 +35,8 @@ module Chewy
         #   end
         #
         def root(**options)
-          self.root_object ||= Chewy::Fields::Root.new(type_name, Chewy.default_root_options.merge(options))
-          root_object.update_options!(options)
+          self.root_object ||= Chewy::Fields::Root.new(type_name, **Chewy.default_root_options.merge(options))
+          root_object.update_options!(**options)
           yield if block_given?
           root_object
         end
@@ -126,9 +126,9 @@ module Chewy
         #
         def field(*args, **options, &block)
           if args.size > 1
-            args.map { |name| field(name, options) }
+            args.map { |name| field(name, **options) }
           else
-            expand_nested(Chewy::Fields::Base.new(args.first, options), &block)
+            expand_nested(Chewy::Fields::Base.new(args.first, **options), &block)
           end
         end
 
@@ -183,7 +183,7 @@ module Chewy
         # Returns compiled mappings hash for current type
         #
         def mappings_hash
-          root.mappings_hash[type_name.to_sym].present? ? root.mappings_hash : {}
+          root.mappings_hash
         end
 
         # Check whether the type has outdated_sync_field defined with a simple value.

data/lib/chewy/type/observe.rb

@@ -63,10 +63,10 @@ module Chewy
           update_proc = Observe.update_proc(type_name, *args, &block)
 
           if Chewy.use_after_commit_callbacks
-            after_commit(callback_options, &update_proc)
+            after_commit(**callback_options, &update_proc)
           else
-            after_save(callback_options, &update_proc)
-            after_destroy(callback_options, &update_proc)
+            after_save(**callback_options, &update_proc)
+            after_destroy(**callback_options, &update_proc)
           end
         end
       end

data/lib/chewy/type/syncer.rb

@@ -2,7 +2,7 @@ module Chewy
   class Type
     # This class is able to find missing and outdated documents in the ES
     # comparing ids from the data source and the ES index. Also, if `outdated_sync_field`
-    # existss in the index definition, it performs comparison of this field
+    # exists in the index definition, it performs comparison of this field
     # values for each source object and corresponding ES document. Usually,
     # this field is `updated_at` and if its value in the source is not equal
     # to the value in the index - this means that this document outdated and
@@ -205,13 +205,9 @@ module Chewy
         return @outdated_sync_field_type if instance_variable_defined?(:@outdated_sync_field_type)
         return unless @type.outdated_sync_field
 
-        mappings = @type.client.indices.get_mapping(
-          index: @type.index_name,
-          type: @type.type_name
-        ).values.first.fetch('mappings', {})
+        mappings = @type.client.indices.get_mapping(index: @type.index_name).values.first.fetch('mappings', {})
 
         @outdated_sync_field_type = mappings
-          .fetch(@type.type_name, {})
           .fetch('properties', {})
           .fetch(@type.outdated_sync_field.to_s, {})['type']
       rescue Elasticsearch::Transport::Transport::Errors::NotFound

data/lib/chewy/type/witchcraft.rb

@@ -58,7 +58,7 @@ module Chewy
       private
 
         def alicorn
-          @alicorn ||= class_eval <<-RUBY, __FILE__, __LINE__ + 1
+          @alicorn ||= singleton_class.class_eval <<-RUBY, __FILE__, __LINE__ + 1
             -> (locals, object0, crutches) do
               #{composed_values(@type.root, 0)}
             end
@@ -79,7 +79,9 @@ module Chewy
           if field.children.present? && !field.multi_field?
             <<-RUBY
               (result#{nesting} = #{fetcher}
-              if result#{nesting}.respond_to?(:to_ary)
+              if result#{nesting}.nil?
+                nil
+              elsif result#{nesting}.respond_to?(:to_ary)
                 result#{nesting}.map do |object#{nesting}|
                   #{composed_values(field, nesting)}
                 end

data/lib/chewy/type/wrapper.rb

@@ -41,7 +41,7 @@ module Chewy
 
       %w[_id _type _index].each do |name|
         define_method name do
-          data[name]
+          _data[name]
         end
       end
 
@@ -49,6 +49,8 @@ module Chewy
         m = method.to_s
         if (name = highlight_name(m))
           highlight(name)
+        elsif (name = highlight_names(m))
+          highlights(name)
         elsif @attributes.key?(m)
           @attributes[m]
         elsif attribute_defined?(m)
@@ -60,7 +62,7 @@ module Chewy
 
       def respond_to_missing?(method, include_private = false)
         m = method.to_s
-        highlight_name(m) || @attributes.key?(m) || attribute_defined?(m) || super
+        highlight_name(m) || highlight_names(m) || @attributes.key?(m) || attribute_defined?(m) || super
       end
 
     private
@@ -69,6 +71,10 @@ module Chewy
         method.sub(/_highlight\z/, '') if method.end_with?('_highlight')
       end
 
+      def highlight_names(method)
+        method.sub(/_highlights\z/, '') if method.end_with?('_highlights')
+      end
+
       def attribute_defined?(attribute)
         self.class.root && self.class.root.children.find { |a| a.name.to_s == attribute }.present?
       end
@@ -77,6 +83,10 @@ module Chewy
         _data['highlight'][attribute].first if highlight?(attribute)
       end
 
+      def highlights(attribute)
+        _data['highlight'][attribute] if highlight?(attribute)
+      end
+
       def highlight?(attribute)
         _data.key?('highlight') && _data['highlight'].key?(attribute)
       end

data/lib/chewy/version.rb

@@ -1,3 +1,3 @@
 module Chewy
-  VERSION = '5.0.0'.freeze
+  VERSION = '7.1.0'.freeze
 end

data/lib/tasks/chewy.rake

@@ -23,22 +23,22 @@ end
 namespace :chewy do
   desc 'Destroys, recreates and imports data for the specified indexes or all of them'
   task reset: :environment do |_task, args|
-    Chewy::RakeHelper.reset(parse_classes(args.extras))
+    Chewy::RakeHelper.reset(**parse_classes(args.extras))
   end
 
   desc 'Resets data for the specified indexes or all of them only if the index specification is changed'
   task upgrade: :environment do |_task, args|
-    Chewy::RakeHelper.upgrade(parse_classes(args.extras))
+    Chewy::RakeHelper.upgrade(**parse_classes(args.extras))
   end
 
   desc 'Updates data for the specified indexes/types or all of them'
   task update: :environment do |_task, args|
-    Chewy::RakeHelper.update(parse_classes(args.extras))
+    Chewy::RakeHelper.update(**parse_classes(args.extras))
   end
 
   desc 'Synchronizes data for the specified indexes/types or all of them'
   task sync: :environment do |_task, args|
-    Chewy::RakeHelper.sync(parse_classes(args.extras))
+    Chewy::RakeHelper.sync(**parse_classes(args.extras))
   end
 
   desc 'Resets all the indexes with the specification changed and synchronizes the rest of them'
@@ -50,22 +50,22 @@ namespace :chewy do
   namespace :parallel do
     desc 'Parallel version of `rake chewy:reset`'
     task reset: :environment do |_task, args|
-      Chewy::RakeHelper.reset(parse_parallel_args(args.extras))
+      Chewy::RakeHelper.reset(**parse_parallel_args(args.extras))
     end
 
     desc 'Parallel version of `rake chewy:upgrade`'
     task upgrade: :environment do |_task, args|
-      Chewy::RakeHelper.upgrade(parse_parallel_args(args.extras))
+      Chewy::RakeHelper.upgrade(**parse_parallel_args(args.extras))
     end
 
     desc 'Parallel version of `rake chewy:update`'
     task update: :environment do |_task, args|
-      Chewy::RakeHelper.update(parse_parallel_args(args.extras))
+      Chewy::RakeHelper.update(**parse_parallel_args(args.extras))
     end
 
     desc 'Parallel version of `rake chewy:sync`'
     task sync: :environment do |_task, args|
-      Chewy::RakeHelper.sync(parse_parallel_args(args.extras))
+      Chewy::RakeHelper.sync(**parse_parallel_args(args.extras))
     end
 
     desc 'Parallel version of `rake chewy:deploy`'
@@ -79,12 +79,12 @@ namespace :chewy do
   namespace :journal do
     desc 'Applies changes that were done after the specified time for the specified indexes/types or all of them'
     task apply: :environment do |_task, args|
-      Chewy::RakeHelper.journal_apply(parse_journal_args(args.extras))
+      Chewy::RakeHelper.journal_apply(**parse_journal_args(args.extras))
     end
 
     desc 'Removes journal records created before the specified timestamp for the specified indexes/types or all of them'
     task clean: :environment do |_task, args|
-      Chewy::RakeHelper.journal_clean(parse_journal_args(args.extras))
+      Chewy::RakeHelper.journal_clean(**parse_journal_args(args.extras))
     end
   end