chewy 0.9.0 → 5.2.0

data/lib/chewy/search/response.rb

@@ -0,0 +1,119 @@
+module Chewy
+  module Search
+    # This class is a ES response hash wrapper.
+    #
+    # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/_the_search_api.html
+    class Response
+      # @param body [Hash] response body hash
+      # @param loader [Chewy::Search::Loader] loader instance
+      def initialize(body, loader, paginator = nil)
+        @body = body
+        @loader = loader
+        @paginator = paginator
+      end
+
+      # Raw response `hits` collection. Returns empty array is something went wrong.
+      #
+      # @return [Array<Hash>]
+      def hits
+        @hits ||= hits_root['hits'] || []
+      end
+
+      # Response `total` field. Returns `0` if something went wrong.
+      #
+      # @return [Integer]
+      def total
+        @total ||= hits_root['total'] || 0
+      end
+
+      # Response `max_score` field.
+      #
+      # @return [Float]
+      def max_score
+        @max_score ||= hits_root['max_score']
+      end
+
+      # Duration of the request handling in ms according to ES.
+      #
+      # @return [Integer]
+      def took
+        @took ||= @body['took']
+      end
+
+      # Has the request been timed out?
+      #
+      # @return [true, false]
+      def timed_out?
+        @timed_out ||= @body['timed_out']
+      end
+
+      # The `suggest` response part. Returns empty hash if suggests
+      # were not requested.
+      #
+      # @return [Hash]
+      def suggest
+        @suggest ||= @body['suggest'] || {}
+      end
+
+      # The `aggregations` response part. Returns empty hash if aggregations
+      # were not requested.
+      #
+      # @return [Hash]
+      def aggs
+        @aggs ||= @body['aggregations'] || {}
+      end
+      alias_method :aggregations, :aggs
+
+      # {Chewy::Type} wrappers collection instantiated on top of hits.
+      #
+      # @return [Array<Chewy::Type>]
+      def wrappers
+        @wrappers ||= hits.map do |hit|
+          @loader.derive_type(hit['_index'], hit['_type']).build(hit)
+        end
+      end
+
+      # ORM/ODM objects that had been a source for Chewy import
+      # and now loaded from the DB using hits ids. Uses
+      # {Chewy::Search::Request#load} passed options for loading.
+      #
+      # @see Chewy::Search::Request#load
+      # @see Chewy::Search::Loader
+      # @return [Array<Object>]
+      def objects
+        @objects ||= begin
+          objects = @loader.load(hits)
+          if @paginator
+            @paginator.call(objects)
+          else
+            objects
+          end
+        end
+      end
+      alias_method :records, :objects
+      alias_method :documents, :objects
+
+      # This method is used in cases when you need to iterate through
+      # both of the collections simultaneously.
+      #
+      # @example
+      #   scope.each do |wrapper|
+      #     scope.object_hash[wrapper]
+      #   end
+      # @see #wrappers
+      # @see #objects
+      # @return [{Chewy::Type => Object}] a hash with wrappers as keys and ORM/ODM objects as values
+      def object_hash
+        @object_hash ||= wrappers.zip(objects).to_h
+      end
+      alias_method :record_hash, :object_hash
+      alias_method :document_hash, :object_hash
+
+    private
+
+      def hits_root
+        @body.fetch('hits', {})
+      end
+    end
+  end
+end

data/lib/chewy/search/scoping.rb

@@ -0,0 +1,50 @@
+module Chewy
+  module Search
+    # This module along with {Chewy::Search} provides an ability to
+    # use names scopes.
+    #
+    # @example
+    #   class UsersIndex < Chewy::Index
+    #     def self.by_name(name)
+    #       query(match: {name: name})
+    #     end
+    #
+    #     define_type :user do
+    #       def self.by_age(age)
+    #         filter(term: {age: age})
+    #       end
+    #     end
+    #   end
+    #
+    #   UsersIndex.limit(10).by_name('Martin')
+    #   # => <UsersIndex::Query {..., :body=>{:size=>10, :query=>{:match=>{:name=>"Martin"}}}}>
+    #   UsersIndex::User.limit(10).by_name('Martin').by_age(42)
+    #   # => <UsersIndex::User::Query {..., :body=>{:size=>10, :query=>{:bool=>{
+    #   #      :must=>{:match=>{:name=>"Martin"}},
+    #   #      :filter=>{:term=>{:age=>42}}}}}}>
+    module Scoping
+      extend ActiveSupport::Concern
+
+      module ClassMethods
+        # The scopes stack.
+        #
+        # @return [Array<Chewy::Search::Reques>] array of scopes
+        def scopes
+          Thread.current[:chewy_scopes] ||= []
+        end
+      end
+
+      # Wraps any method to make it contents be executed inside the
+      # current request scope.
+      #
+      # @see Chewy::Search::ClassMethods#all
+      # @yield executes the block after the current context is put at the top of the scope stack
+      def scoping
+        self.class.scopes.push(self)
+        yield
+      ensure
+        self.class.scopes.pop
+      end
+    end
+  end
+end

data/lib/chewy/search/scrolling.rb

@@ -0,0 +1,134 @@
+module Chewy
+  module Search
+    # This module contains batch requests DSL via ES scroll API. All the methods
+    # are optimized on memory consumption, they are not caching anythig, so
+    # use them when you need to do some single-run stuff on a huge amount of
+    # documents. Don't forget to tune the `scroll` parameter for long-lasting
+    # actions.
+    # All the scroll methods respect the limit value if provided.
+    #
+    # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-scroll.html
+    module Scrolling
+      # Iterates through the documents of the scope in batches. Limit if overrided
+      # by the `batch_size`. There are 2 possible use-cases: with a block or without.
+      #
+      # @param batch_size [Integer] batch size obviously, replaces `size` query parameter
+      # @param scroll [String] cursor expiration time
+      #
+      # @overload scroll_batches(batch_size: 1000, scroll: '1m')
+      #   @example
+      #     PlaceIndex.scroll_batches { |batch| batch.each { |hit| p hit['_id'] } }
+      #   @yieldparam batch [Array<Hash>] block is executed for each batch of hits
+      #
+      # @overload scroll_batches(batch_size: 1000, scroll: '1m')
+      #   @example
+      #     PlaceIndex.scroll_batches.flat_map { |batch| batch.map { |hit| hit['_id'] } }
+      #   @return [Enumerator] a standard ruby Enumerator
+      def scroll_batches(batch_size: Request::DEFAULT_BATCH_SIZE, scroll: Request::DEFAULT_SCROLL)
+        return enum_for(:scroll_batches, batch_size: batch_size, scroll: scroll) unless block_given?
+
+        result = perform(size: batch_size, scroll: scroll)
+        total = [raw_limit_value, result.fetch('hits', {}).fetch('total', 0)].compact.min
+        last_batch_size = total % batch_size
+        fetched = 0
+
+        loop do
+          hits = result.fetch('hits', {}).fetch('hits', [])
+          fetched += hits.size
+          hits = hits.first(last_batch_size) if last_batch_size != 0 && fetched >= total
+          yield(hits) if hits.present?
+          break if fetched >= total
+          scroll_id = result['_scroll_id']
+          result = perform_scroll(scroll: scroll, scroll_id: scroll_id)
+        end
+      end
+
+      # @!method scroll_hits(batch_size: 1000, scroll: '1m')
+      # Iterates through the documents of the scope in batches. Yields each hit separately.
+      #
+      # @param batch_size [Integer] batch size obviously, replaces `size` query parameter
+      # @param scroll [String] cursor expiration time
+      #
+      # @overload scroll_hits(batch_size: 1000, scroll: '1m')
+      #   @example
+      #     PlaceIndex.scroll_hits { |hit| p hit['_id'] }
+      #   @yieldparam hit [Hash] block is executed for each hit
+      #
+      # @overload scroll_hits(batch_size: 1000, scroll: '1m')
+      #   @example
+      #     PlaceIndex.scroll_hits.map { |hit| hit['_id'] }
+      #   @return [Enumerator] a standard ruby Enumerator
+      def scroll_hits(**options)
+        return enum_for(:scroll_hits, **options) unless block_given?
+
+        scroll_batches(**options).each do |batch|
+          batch.each { |hit| yield hit }
+        end
+      end
+
+      # @!method scroll_wrappers(batch_size: 1000, scroll: '1m')
+      # Iterates through the documents of the scope in batches. Yields
+      # each hit wrapped with {Chewy::Type}.
+      #
+      # @param batch_size [Integer] batch size obviously, replaces `size` query parameter
+      # @param scroll [String] cursor expiration time
+      #
+      # @overload scroll_wrappers(batch_size: 1000, scroll: '1m')
+      #   @example
+      #     PlaceIndex.scroll_wrappers { |object| p object.id }
+      #   @yieldparam object [Chewy::Type] block is executed for each hit object
+      #
+      # @overload scroll_wrappers(batch_size: 1000, scroll: '1m')
+      #   @example
+      #     PlaceIndex.scroll_wrappers.map { |object| object.id }
+      #   @return [Enumerator] a standard ruby Enumerator
+      def scroll_wrappers(**options)
+        return enum_for(:scroll_wrappers, **options) unless block_given?
+
+        scroll_hits(**options).each do |hit|
+          yield loader.derive_type(hit['_index'], hit['_type']).build(hit)
+        end
+      end
+
+      # @!method scroll_objects(batch_size: 1000, scroll: '1m')
+      # Iterates through the documents of the scope in batches. Performs load
+      # operation for each batch and then yields each loaded ORM/ODM object.
+      # Uses {Chewy::Search::Request#load} passed options for loading.
+      #
+      # @note If the record is not found it yields nil instead.
+      # @see Chewy::Search::Request#load
+      # @see Chewy::Search::Loader
+      # @param batch_size [Integer] batch size obviously, replaces `size` query parameter
+      # @param scroll [String] cursor expiration time
+      #
+      # @overload scroll_objects(batch_size: 1000, scroll: '1m')
+      #   @example
+      #     PlaceIndex.scroll_objects { |record| p record.id }
+      #   @yieldparam record [Object] block is executed for each record loaded
+      #
+      # @overload scroll_objects(batch_size: 1000, scroll: '1m')
+      #   @example
+      #     PlaceIndex.scroll_objects.map { |record| record.id }
+      #   @return [Enumerator] a standard ruby Enumerator
+      def scroll_objects(**options)
+        return enum_for(:scroll_objects, **options) unless block_given?
+
+        except(:source, :stored_fields, :script_fields, :docvalue_fields)
+          .source(false).scroll_batches(**options).each do |batch|
+            loader.load(batch).each { |object| yield object }
+          end
+      end
+      alias_method :scroll_records, :scroll_objects
+      alias_method :scroll_documents, :scroll_objects
+
+    private
+
+      def perform_scroll(body)
+        ActiveSupport::Notifications.instrument 'search_query.chewy',
+          notification_payload(request: body) do
+            Chewy.client.scroll(body)
+          end
+      end
+    end
+  end
+end

data/lib/chewy/stash.rb

@@ -0,0 +1,79 @@
+module Chewy
+  # This class is the main storage for Chewy service data,
+  # Now index raw specifications are stored in the `chewy_specifications`
+  # index.
+  # Journal entries are stored in `chewy_journal`
+  #
+  # @see Chewy::Index::Specification
+  module Stash
+    class Specification < Chewy::Index
+      index_name 'chewy_specifications'
+
+      define_type :specification do
+        default_import_options journal: false
+
+        field :specification, type: 'binary'
+      end
+    end
+
+    class Journal < Chewy::Index
+      index_name 'chewy_journal'
+
+      # Loads all entries since the specified time.
+      #
+      # @param since_time [Time, DateTime] a timestamp from which we load a journal
+      # @param only [Chewy::Index, Array<Chewy::Index>] journal entries related to these indices will be loaded only
+      def self.entries(since_time, only: [])
+        self.for(only).filter(range: {created_at: {gt: since_time}})
+      end
+
+      # Cleans up all the journal entries until the specified time. If nothing is
+      # specified - cleans up everything.
+      #
+      # @param since_time [Time, DateTime] the time top boundary
+      # @param only [Chewy::Index, Array<Chewy::Index>] indexes to clean up journal entries for
+      def self.clean(until_time = nil, only: [])
+        scope = self.for(only)
+        scope = scope.filter(range: {created_at: {lte: until_time}}) if until_time
+        scope.delete_all
+      end
+
+      # Selects all the journal entries for the specified indices.
+      #
+      # @param indices [Chewy::Index, Array<Chewy::Index>]
+      def self.for(*something)
+        something = something.flatten.compact
+        types = something.flat_map { |s| Chewy.derive_types(s) }
+        return none if something.present? && types.blank?
+        scope = all
+        types.group_by(&:index).each do |index, index_types|
+          scope = scope.or(
+            filter(term: {index_name: index.derivable_name})
+              .filter(terms: {type_name: index_types.map(&:type_name)})
+          )
+        end
+        scope
+      end
+
+      define_type :journal do
+        default_import_options journal: false
+
+        field :index_name, type: 'keyword'
+        field :type_name, type: 'keyword'
+        field :action, type: 'keyword'
+        field :references, type: 'binary'
+        field :created_at, type: 'date'
+
+        def type
+          @type ||= Chewy.derive_type("#{index_name}##{type_name}")
+        end
+
+        def references
+          @references ||= Array.wrap(@attributes['references']).map do |item|
+            JSON.load(Base64.decode64(item)) # rubocop:disable Security/JSONLoad
+          end
+        end
+      end
+    end
+  end
+end

data/lib/chewy/strategy.rb

@@ -17,6 +17,13 @@ rescue LoadError
   nil
 end
 
+begin
+  require 'shoryuken'
+  require 'chewy/strategy/shoryuken'
+rescue LoadError
+  nil
+end
+
 begin
   require 'active_job'
   require 'chewy/strategy/active_job'
@@ -47,14 +54,14 @@ module Chewy
 
     def push(name)
       result = @stack.push resolve(name).new
-      debug "[#{@stack.size}] <- #{current.name}"
+      debug "[#{@stack.size - 1}] <- #{current.name}" if @stack.size > 2
       result
     end
 
     def pop
       raise "Can't pop root strategy" if @stack.one?
-      debug "[#{@stack.size}] -> #{current.name}"
       result = @stack.pop.tap(&:leave)
+      debug "[#{@stack.size}] -> #{result.name}, now #{current.name}" if @stack.size > 1
       result
     end
 
@@ -70,7 +77,7 @@ module Chewy
     def debug(string)
       return unless Chewy.logger && Chewy.logger.debug?
       line = caller.detect { |l| l !~ %r{lib/chewy/strategy.rb:|lib/chewy.rb:} }
-      Chewy.logger.debug(["DEBUG: Chewy strategies stack: #{string}", line.sub(/:in\s.+$/, '')].join(' @ '))
+      Chewy.logger.debug(["Chewy strategies stack: #{string}", line.sub(/:in\s.+$/, '')].join(' @ '))
     end
 
     def resolve(name)

data/lib/chewy/strategy/active_job.rb

@@ -11,9 +11,10 @@ module Chewy
     #
     class ActiveJob < Atomic
       class Worker < ::ActiveJob::Base
-        queue_as :chewy
+        queue_as { Chewy.settings.dig(:active_job, :queue) || 'chewy' }
 
         def perform(type, ids, options = {})
+          options[:refresh] = !Chewy.disable_refresh_async if Chewy.disable_refresh_async
           type.constantize.import!(ids, options)
         end
       end

data/lib/chewy/strategy/atomic.rb

@@ -16,11 +16,9 @@ module Chewy
         @stash = {}
       end
 
-      def update(type, objects, options = {})
-        ActiveSupport::Deprecation.warn('`urgent: true` option is deprecated and is not effective inside `:atomic` strategy, use `Chewy.strategy(:urgent)` strategy instead') if options.key?(:urgent)
-
+      def update(type, objects, _options = {})
         @stash[type] ||= []
-        @stash[type] |= type.send(:build_root).id ? Array.wrap(objects) : type.adapter.identify(objects)
+        @stash[type] |= type.root.id ? Array.wrap(objects) : type.adapter.identify(objects)
       end
 
       def leave