chewy 0.10.0 → 6.0.0

data/lib/chewy/search/scrolling.rb

@@ -31,16 +31,19 @@ module Chewy
         total = [raw_limit_value, result.fetch('hits', {}).fetch('total', 0)].compact.min
         last_batch_size = total % batch_size
         fetched = 0
+        scroll_id = nil
 
         loop do
           hits = result.fetch('hits', {}).fetch('hits', [])
           fetched += hits.size
           hits = hits.first(last_batch_size) if last_batch_size != 0 && fetched >= total
           yield(hits) if hits.present?
-          break if fetched >= total
           scroll_id = result['_scroll_id']
+          break if fetched >= total
           result = perform_scroll(scroll: scroll, scroll_id: scroll_id)
         end
+      ensure
+        Chewy.client.clear_scroll(scroll_id: scroll_id) if scroll_id
       end
 
       # @!method scroll_hits(batch_size: 1000, scroll: '1m')
@@ -125,11 +128,9 @@ module Chewy
 
       def perform_scroll(body)
         ActiveSupport::Notifications.instrument 'search_query.chewy',
-          request: body, indexes: _indexes, types: _types,
-          index: _indexes.one? ? _indexes.first : _indexes,
-          type: _types.one? ? _types.first : _types do
-          Chewy.client.scroll(body)
-        end
+          notification_payload(request: body) do
+            Chewy.client.scroll(body)
+          end
       end
     end
   end

data/lib/chewy/stash.rb

@@ -1,26 +1,23 @@
 module Chewy
   # This class is the main storage for Chewy service data,
-  # Now index raw specifications are stored in the `chewy_stash`
-  # index. In the future the journal will be moved here as well.
+  # Now index raw specifications are stored in the `chewy_specifications`
+  # index.
+  # Journal entries are stored in `chewy_journal`
   #
   # @see Chewy::Index::Specification
-  class Stash < Chewy::Index
-    index_name 'chewy_stash'
+  module Stash
+    class Specification < Chewy::Index
+      index_name 'chewy_specifications'
 
-    define_type :specification do
-      default_import_options journal: false
+      define_type :specification do
+        default_import_options journal: false
 
-      field :value, index: 'no'
+        field :specification, type: 'binary'
+      end
     end
 
-    define_type :journal do # rubocop:disable Metrics/BlockLength
-      default_import_options journal: false
-
-      field :index_name, type: 'string', index: 'not_analyzed'
-      field :type_name, type: 'string', index: 'not_analyzed'
-      field :action, type: 'string', index: 'not_analyzed'
-      field :references, type: 'string', index: 'no'
-      field :created_at, type: 'date'
+    class Journal < Chewy::Index
+      index_name 'chewy_journal'
 
       # Loads all entries since the specified time.
       #
@@ -52,18 +49,30 @@ module Chewy
         types.group_by(&:index).each do |index, index_types|
           scope = scope.or(
             filter(term: {index_name: index.derivable_name})
-            .filter(terms: {type_name: index_types.map(&:type_name)})
+              .filter(terms: {type_name: index_types.map(&:type_name)})
           )
         end
         scope
       end
 
-      def type
-        @type ||= Chewy.derive_type("#{index_name}##{type_name}")
-      end
+      define_type :journal do
+        default_import_options journal: false
+
+        field :index_name, type: 'keyword'
+        field :type_name, type: 'keyword'
+        field :action, type: 'keyword'
+        field :references, type: 'binary'
+        field :created_at, type: 'date'
 
-      def references
-        @references ||= Array.wrap(@attributes['references']).map { |r| JSON.load(r) } # rubocop:disable Security/JSONLoad
+        def type
+          @type ||= Chewy.derive_type("#{index_name}##{type_name}")
+        end
+
+        def references
+          @references ||= Array.wrap(@attributes['references']).map do |item|
+            JSON.load(Base64.decode64(item)) # rubocop:disable Security/JSONLoad
+          end
+        end
       end
     end
   end

data/lib/chewy/strategy/active_job.rb

@@ -11,7 +11,7 @@ module Chewy
     #
     class ActiveJob < Atomic
       class Worker < ::ActiveJob::Base
-        queue_as :chewy
+        queue_as { Chewy.settings.dig(:active_job, :queue) || 'chewy' }
 
         def perform(type, ids, options = {})
           options[:refresh] = !Chewy.disable_refresh_async if Chewy.disable_refresh_async

data/lib/chewy/strategy/atomic.rb

@@ -18,7 +18,7 @@ module Chewy
 
       def update(type, objects, _options = {})
         @stash[type] ||= []
-        @stash[type] |= type.send(:build_root).id ? Array.wrap(objects) : type.adapter.identify(objects)
+        @stash[type] |= type.root.id ? Array.wrap(objects) : type.adapter.identify(objects)
       end
 
       def leave

data/lib/chewy/strategy/sidekiq.rb

@@ -33,7 +33,7 @@ module Chewy
     private
 
       def sidekiq_queue
-        Chewy.settings.fetch(:sidekiq, {})[:queue] || 'chewy'
+        Chewy.settings.dig(:sidekiq, :queue) || 'chewy'
       end
     end
   end

data/lib/chewy/type.rb

@@ -14,7 +14,10 @@ require 'chewy/type/witchcraft'
 
 module Chewy
   class Type
-    IMPORT_OPTIONS_KEYS = %i[batch_size bulk_size refresh consistency replication raw_import journal].freeze
+    IMPORT_OPTIONS_KEYS = %i[
+      batch_size bulk_size consistency direct_import journal
+      pipeline raw_import refresh replication
+    ].freeze
 
     include Search
     include Mapping
@@ -34,7 +37,7 @@ module Chewy
       # Chewy index current type belongs to. Defined inside `Chewy.create_type`
       #
       def index
-        raise NotImplementedError, 'Looks like this type ws defined outside the index scope and `.index` method is undefined for it'
+        raise NotImplementedError, 'Looks like this type was defined outside the index scope and `.index` method is undefined for it'
       end
 
       # Current type adapter. Defined inside `Chewy.create_type`, derived from

data/lib/chewy/type/adapter/active_record.rb

@@ -22,7 +22,7 @@ module Chewy
         end
 
         def import_scope(scope, options)
-          pluck_in_batches(scope, options.slice(:batch_size)).inject(true) do |result, ids|
+          pluck_in_batches(scope, **options.slice(:batch_size)).inject(true) do |result, ids|
             objects = if options[:raw_import]
               raw_default_scope_where_ids_in(ids, options[:raw_import])
             else

data/lib/chewy/type/adapter/base.rb

@@ -41,19 +41,19 @@ module Chewy
         #
         #   { delete: [object_or_id1, object_or_id2], index: [object3, object4, object5] }
         #
-        # @yield batch [Array<Object>] each batch of objects
+        # @yieldparam _batch [Array<Object>] each batch of objects
         # @return [true, false] returns true if all the block call returns true and false otherwise
-        def import(*_args, &_block)
+        def import(_batch, &_block)
           raise NotImplementedError
         end
 
         # Unlike {#import} fetches only ids (references) to the imported objects,
         # using the same procedures as {#import}.
         #
-        # @param fields [Array<Symbol>] additional fields to fetch
-        # @param batch_size [Integer] batch size, defaults to 1000
-        # @yield batch [Array<Object>] each batch of objects
-        def import_fields(*_args, &_block)
+        # @param _fields [Array<Symbol>] additional fields to fetch
+        # @param _batch_size [Integer] batch size, defaults to 1000
+        # @yieldparam batch [Array<Object>] each batch of objects
+        def import_fields(_fields, _batch_size, &_block)
           raise NotImplementedError
         end
 
@@ -61,9 +61,9 @@ module Chewy
         # an array of references to the passed objects. Returns ids if possible.
         # Otherwise - and array of objects themselves.
         #
-        # @param batch_size [Integer] batch size, defaults to 1000
-        # @yield batch [Array<Object>] each batch of objects
-        def import_references(*_args, &_block)
+        # @param _batch_size [Integer] batch size, defaults to 1000
+        # @yieldparam batch [Array<Object>] each batch of objects
+        def import_references(_batch_size, &_block)
           raise NotImplementedError
         end
 

data/lib/chewy/type/adapter/mongoid.rb

@@ -17,9 +17,7 @@ module Chewy
       private
 
         def cleanup_default_scope!
-          if Chewy.logger && @default_scope.options.values_at(:sort, :limit, :skip).compact.present?
-            Chewy.logger.warn('Default type scope order, limit and offset are ignored and will be nullified')
-          end
+          Chewy.logger.warn('Default type scope order, limit and offset are ignored and will be nullified') if Chewy.logger && @default_scope.options.values_at(:sort, :limit, :skip).compact.present?
 
           @default_scope.options.delete(:limit)
           @default_scope.options.delete(:skip)
@@ -27,7 +25,7 @@ module Chewy
         end
 
         def import_scope(scope, options)
-          pluck_in_batches(scope, options.slice(:batch_size)).map do |ids|
+          pluck_in_batches(scope, **options.slice(:batch_size)).map do |ids|
             yield grouped_objects(default_scope_where_ids_in(ids))
           end.all?
         end

data/lib/chewy/type/adapter/orm.rb

@@ -45,6 +45,7 @@ module Chewy
         # Import options:
         #
         #   <tt>:batch_size</tt> - import batch size, 1000 objects by default
+        #   <tt>:direct_import</tt> - import objects without reloading
         #
         # Method handles destroyed objects as well. In case of objects ORM scope
         # or array passed, objects, responding with true to `destroyed?` method will be deleted
@@ -75,10 +76,10 @@ module Chewy
         def import(*args, &block)
           collection, options = import_args(*args)
 
-          if collection.is_a?(relation_class)
-            import_scope(collection, options, &block)
-          else
+          if !collection.is_a?(relation_class) || options[:direct_import]
             import_objects(collection, options, &block)
+          else
+            import_scope(collection, options, &block)
           end
         end
 
@@ -89,7 +90,7 @@ module Chewy
 
           if options[:fields].present? || collection.is_a?(relation_class)
             collection = all_scope_where_ids_in(identify(collection)) unless collection.is_a?(relation_class)
-            pluck_in_batches(collection, options.slice(:fields, :batch_size, :typecast), &block)
+            pluck_in_batches(collection, **options.slice(:fields, :batch_size, :typecast), &block)
           else
             identify(collection).each_slice(options[:batch_size]) do |batch|
               yield batch
@@ -119,6 +120,8 @@ module Chewy
           indexed = collection_ids.each_slice(options[:batch_size]).map do |ids|
             batch = if options[:raw_import]
               raw_default_scope_where_ids_in(ids, options[:raw_import])
+            elsif options[:direct_import]
+              hash.values_at(*ids.map(&:to_s))
             else
               default_scope_where_ids_in(ids)
             end

data/lib/chewy/type/adapter/sequel.rb

@@ -16,15 +16,13 @@ module Chewy
       private
 
         def cleanup_default_scope!
-          if Chewy.logger && @default_scope != @default_scope.unordered.unlimited
-            Chewy.logger.warn('Default type scope order, limit and offset are ignored and will be nullified')
-          end
+          Chewy.logger.warn('Default type scope order, limit and offset are ignored and will be nullified') if Chewy.logger && @default_scope != @default_scope.unordered.unlimited
 
           @default_scope = @default_scope.unordered.unlimited
         end
 
         def import_scope(scope, options)
-          pluck_in_batches(scope, options.slice(:batch_size)).inject(true) do |result, ids|
+          pluck_in_batches(scope, **options.slice(:batch_size)).inject(true) do |result, ids|
             result & yield(grouped_objects(default_scope_where_ids_in(ids).all))
           end
         end
@@ -34,7 +32,7 @@ module Chewy
         end
 
         def full_column_name(column)
-          "#{target.table_name}__#{column}".to_sym
+          ::Sequel.qualify(target.table_name, column)
         end
 
         def all_scope
@@ -55,7 +53,7 @@ module Chewy
         def pluck_in_batches(scope, fields: [], batch_size: nil, **options)
           return enum_for(:pluck_in_batches, scope, fields: fields, batch_size: batch_size, **options) unless block_given?
 
-          scope = scope.unordered.order(::Sequel.asc(full_column_name(primary_key))).limit(batch_size)
+          scope = scope.unordered.order(full_column_name(primary_key).asc).limit(batch_size)
 
           ids = pluck(scope, fields: fields)
           count = 0
@@ -64,7 +62,7 @@ module Chewy
             yield ids
             break if ids.size < batch_size
             last_id = ids.last.is_a?(Array) ? ids.last.first : ids.last
-            ids = pluck(scope.where { |o| o.__send__(full_column_name(primary_key)) > last_id }, fields: fields)
+            ids = pluck(scope.where { |_o| full_column_name(primary_key) > last_id }, fields: fields)
           end
 
           count

data/lib/chewy/type/crutch.rb

@@ -12,7 +12,7 @@ module Chewy
         def initialize(type, collection)
           @type = type
           @collection = collection
-          @type._crutches.keys.each do |name|
+          @type._crutches.each_key do |name|
             singleton_class.class_eval <<-METHOD, __FILE__, __LINE__ + 1
               def #{name}
                 @#{name} ||= @type._crutches[:#{name}].call @collection

data/lib/chewy/type/import.rb

@@ -8,18 +8,18 @@ module Chewy
     module Import
       extend ActiveSupport::Concern
 
-      IMPORT_WORKER = lambda do |type, options, ids|
-        ::Process.setproctitle("chewy import #{type}[#{::Parallel.worker_number}]")
-        routine = Routine.new(type, options)
+      IMPORT_WORKER = lambda do |type, options, total, ids, index|
+        ::Process.setproctitle("chewy [#{type}]: import data (#{index + 1}/#{total})")
+        routine = Routine.new(type, **options)
         type.adapter.import(*ids, routine.options) do |action_objects|
           routine.process(**action_objects)
         end
         {errors: routine.errors, import: routine.stats, leftovers: routine.leftovers}
       end
 
-      LEFTOVERS_WORKER = lambda do |type, options, body|
-        ::Process.setproctitle("chewy import #{type}[#{::Parallel.worker_number}]")
-        routine = Routine.new(type, options)
+      LEFTOVERS_WORKER = lambda do |type, options, total, body, index|
+        ::Process.setproctitle("chewy [#{type}]: import leftovers (#{index + 1}/#{total})")
+        routine = Routine.new(type, **options)
         routine.perform_bulk(body)
         routine.errors
       end
@@ -67,6 +67,7 @@ module Chewy
         # @option options [String] suffix an index name suffix, used for zero-downtime reset mostly, no suffix by default
         # @option options [Integer] bulk_size bulk API chunk size in bytes; if passed, the request is performed several times for each chunk, empty by default
         # @option options [Integer] batch_size passed to the adapter import method, used to split imported objects in chunks, 1000 by default
+        # @option options [Boolean] direct_import skips object reloading in ORM adapter, `false` by default
         # @option options [true, false] journal enables imported objects journaling, false by default
         # @option options [Array<Symbol, String>] update_fields list of fields for the partial import, empty by default
         # @option options [true, false] update_failover enables full objects reimport in cases of partial update errors, `true` by default
@@ -116,17 +117,18 @@ module Chewy
         def compose(object, crutches = nil, fields: [])
           crutches ||= Chewy::Type::Crutch::Crutches.new self, [object]
 
-          if witchcraft? && build_root.children.present?
+          if witchcraft? && root.children.present?
             cauldron(fields: fields).brew(object, crutches)
           else
-            build_root.compose(object, crutches, fields: fields)
+            root.compose(object, crutches, fields: fields)
           end
         end
 
       private
 
         def import_routine(*args)
-          routine = Routine.new(self, args.extract_options!)
+          return if args.first.blank? && !args.first.nil?
+          routine = Routine.new(self, **args.extract_options!)
           routine.create_indexes!
 
           if routine.parallel_options
@@ -155,13 +157,13 @@ module Chewy
             batches = adapter.import_references(*objects, routine.options.slice(:batch_size)).to_a
 
             ::ActiveRecord::Base.connection.close if defined?(::ActiveRecord::Base)
-            results = ::Parallel.map(batches, routine.parallel_options, &IMPORT_WORKER.curry[self, routine.options])
+            results = ::Parallel.map_with_index(batches, routine.parallel_options, &IMPORT_WORKER.curry[self, routine.options, batches.size])
             ::ActiveRecord::Base.connection.reconnect! if defined?(::ActiveRecord::Base)
             errors, import, leftovers = process_parallel_import_results(results)
 
             if leftovers.present?
               batches = leftovers.each_slice(routine.options[:batch_size])
-              results = ::Parallel.map(batches, routine.parallel_options, &LEFTOVERS_WORKER.curry[self, routine.options])
+              results = ::Parallel.map_with_index(batches, routine.parallel_options, &LEFTOVERS_WORKER.curry[self, routine.options, batches.size])
               errors.concat(results.flatten(1))
             end
 

data/lib/chewy/type/import/bulk_builder.rb

@@ -114,7 +114,7 @@ module Chewy
         end
 
         def type_root
-          @type_root = @type.send(:build_root)
+          @type_root ||= @type.root
         end
       end
     end

data/lib/chewy/type/import/bulk_request.rb

@@ -54,15 +54,17 @@ module Chewy
 
         def request_bodies(body)
           if @bulk_size
+            serializer = ::Elasticsearch::API.serializer
             pieces = body.each_with_object(['']) do |piece, result|
               operation, meta = piece.to_a.first
               data = meta.delete(:data)
-              piece = [{operation => meta}, data].compact.map(&:to_json).join("\n")
+              piece = serializer.dump(operation => meta)
+              piece << "\n" << serializer.dump(data) if data.present?
 
               if result.last.bytesize + piece.bytesize > @bulk_size
                 result.push(piece)
               else
-                result[-1] = [result[-1], piece].reject(&:blank?).join("\n")
+                result[-1].blank? ? (result[-1] = piece) : (result[-1] << "\n" << piece)
               end
             end
             pieces.each { |piece| piece << "\n" }

data/lib/chewy/type/import/journal_builder.rb

@@ -10,7 +10,7 @@ module Chewy
 
         def bulk_body
           Chewy::Type::Import::BulkBuilder.new(
-            Chewy::Stash::Journal,
+            Chewy::Stash::Journal::Journal,
             index: [
               entries(:index, @index),
               entries(:delete, @delete)
@@ -18,7 +18,7 @@ module Chewy
           ).bulk_body.each do |item|
             item.values.first.merge!(
               _index: Chewy::Stash::Journal.index_name,
-              _type: Chewy::Stash::Journal.type_name
+              _type: Chewy::Stash::Journal::Journal.type_name
             )
           end
         end
@@ -31,7 +31,7 @@ module Chewy
             index_name: @type.index.derivable_name,
             type_name: @type.type_name,
             action: action,
-            references: identify(objects).map(&:to_json),
+            references: identify(objects).map { |item| Base64.encode64(::Elasticsearch::API.serializer.dump(item)) },
             created_at: Time.now.utc
           }
         end