chewy 0.8.4 → 7.3.4

data/lib/chewy/search/response.rb

@@ -0,0 +1,119 @@
+module Chewy
+  module Search
+    # This class is a ES response hash wrapper.
+    #
+    # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/_the_search_api.html
+    class Response
+      # @param body [Hash] response body hash
+      # @param loader [Chewy::Search::Loader] loader instance
+      def initialize(body, loader, paginator = nil)
+        @body = body
+        @loader = loader
+        @paginator = paginator
+      end
+
+      # Raw response `hits` collection. Returns empty array is something went wrong.
+      #
+      # @return [Array<Hash>]
+      def hits
+        @hits ||= hits_root['hits'] || []
+      end
+
+      # Response `total` field. Returns `0` if something went wrong.
+      #
+      # @return [Integer]
+      def total
+        @total ||= hits_root.fetch('total', {}).fetch('value', 0)
+      end
+
+      # Response `max_score` field.
+      #
+      # @return [Float]
+      def max_score
+        @max_score ||= hits_root['max_score']
+      end
+
+      # Duration of the request handling in ms according to ES.
+      #
+      # @return [Integer]
+      def took
+        @took ||= @body['took']
+      end
+
+      # Has the request been timed out?
+      #
+      # @return [true, false]
+      def timed_out?
+        @timed_out ||= @body['timed_out']
+      end
+
+      # The `suggest` response part. Returns empty hash if suggests
+      # were not requested.
+      #
+      # @return [Hash]
+      def suggest
+        @suggest ||= @body['suggest'] || {}
+      end
+
+      # The `aggregations` response part. Returns empty hash if aggregations
+      # were not requested.
+      #
+      # @return [Hash]
+      def aggs
+        @aggs ||= @body['aggregations'] || {}
+      end
+      alias_method :aggregations, :aggs
+
+      # {Chewy::Index} wrappers collection instantiated on top of hits.
+      #
+      # @return [Array<Chewy::Index>]
+      def wrappers
+        @wrappers ||= hits.map do |hit|
+          @loader.derive_index(hit['_index']).build(hit)
+        end
+      end
+
+      # ORM/ODM objects that had been a source for Chewy import
+      # and now loaded from the DB using hits ids. Uses
+      # {Chewy::Search::Request#load} passed options for loading.
+      #
+      # @see Chewy::Search::Request#load
+      # @see Chewy::Search::Loader
+      # @return [Array<Object>]
+      def objects
+        @objects ||= begin
+          objects = @loader.load(hits)
+          if @paginator
+            @paginator.call(objects)
+          else
+            objects
+          end
+        end
+      end
+      alias_method :records, :objects
+      alias_method :documents, :objects
+
+      # This method is used in cases when you need to iterate through
+      # both of the collections simultaneously.
+      #
+      # @example
+      #   scope.each do |wrapper|
+      #     scope.object_hash[wrapper]
+      #   end
+      # @see #wrappers
+      # @see #objects
+      # @return [{Chewy::Index => Object}] a hash with wrappers as keys and ORM/ODM objects as values
+      def object_hash
+        @object_hash ||= wrappers.zip(objects).to_h
+      end
+      alias_method :record_hash, :object_hash
+      alias_method :document_hash, :object_hash
+
+    private
+
+      def hits_root
+        @body.fetch('hits', {})
+      end
+    end
+  end
+end

data/lib/chewy/search/scoping.rb

@@ -0,0 +1,49 @@
+module Chewy
+  module Search
+    # This module along with {Chewy::Search} provides an ability to
+    # use names scopes.
+    #
+    # @example
+    #   class UsersIndex < Chewy::Index
+    #     def self.by_name(name)
+    #       query(match: {name: name})
+    #     end
+    #
+    #
+    #     def self.by_age(age)
+    #       filter(term: {age: age})
+    #     end
+    #   end
+    #
+    #   UsersIndex.limit(10).by_name('Martin')
+    #   # => <UsersIndex::Query {..., :body=>{:size=>10, :query=>{:match=>{:name=>"Martin"}}}}>
+    #   UsersIndex.limit(10).by_name('Martin').by_age(42)
+    #   # => <UsersIndex::Query {..., :body=>{:size=>10, :query=>{:bool=>{
+    #   #      :must=>{:match=>{:name=>"Martin"}},
+    #   #      :filter=>{:term=>{:age=>42}}}}}}>
+    module Scoping
+      extend ActiveSupport::Concern
+
+      module ClassMethods
+        # The scopes stack.
+        #
+        # @return [Array<Chewy::Search::Request>] array of scopes
+        def scopes
+          Chewy.current[:chewy_scopes] ||= []
+        end
+      end
+
+      # Wraps any method to make it contents be executed inside the
+      # current request scope.
+      #
+      # @see Chewy::Search::ClassMethods#all
+      # @yield executes the block after the current context is put at the top of the scope stack
+      def scoping
+        self.class.scopes.push(self)
+        yield
+      ensure
+        self.class.scopes.pop
+      end
+    end
+  end
+end

data/lib/chewy/search/scrolling.rb

@@ -0,0 +1,137 @@
+module Chewy
+  module Search
+    # This module contains batch requests DSL via ES scroll API. All the methods
+    # are optimized on memory consumption, they are not caching anythig, so
+    # use them when you need to do some single-run stuff on a huge amount of
+    # documents. Don't forget to tune the `scroll` parameter for long-lasting
+    # actions.
+    # All the scroll methods respect the limit value if provided.
+    #
+    # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-scroll.html
+    module Scrolling
+      # Iterates through the documents of the scope in batches. Limit if overrided
+      # by the `batch_size`. There are 2 possible use-cases: with a block or without.
+      #
+      # @param batch_size [Integer] batch size obviously, replaces `size` query parameter
+      # @param scroll [String] cursor expiration time
+      #
+      # @overload scroll_batches(batch_size: 1000, scroll: '1m')
+      #   @example
+      #     PlaceIndex.scroll_batches { |batch| batch.each { |hit| p hit['_id'] } }
+      #   @yieldparam batch [Array<Hash>] block is executed for each batch of hits
+      #
+      # @overload scroll_batches(batch_size: 1000, scroll: '1m')
+      #   @example
+      #     PlaceIndex.scroll_batches.flat_map { |batch| batch.map { |hit| hit['_id'] } }
+      #   @return [Enumerator] a standard ruby Enumerator
+      def scroll_batches(batch_size: Request::DEFAULT_BATCH_SIZE, scroll: Request::DEFAULT_SCROLL)
+        return enum_for(:scroll_batches, batch_size: batch_size, scroll: scroll) unless block_given?
+
+        result = perform(size: batch_size, scroll: scroll)
+        total = [raw_limit_value, result.fetch('hits', {}).fetch('total', {}).fetch('value', 0)].compact.min
+        last_batch_size = total % batch_size
+        fetched = 0
+        scroll_id = nil
+
+        loop do
+          hits = result.fetch('hits', {}).fetch('hits', [])
+          fetched += hits.size
+          hits = hits.first(last_batch_size) if last_batch_size != 0 && fetched >= total
+          yield(hits) if hits.present?
+          scroll_id = result['_scroll_id']
+          break if fetched >= total
+
+          result = perform_scroll(scroll: scroll, scroll_id: scroll_id)
+        end
+      ensure
+        Chewy.client.clear_scroll(body: {scroll_id: scroll_id}) if scroll_id
+      end
+
+      # @!method scroll_hits(batch_size: 1000, scroll: '1m')
+      # Iterates through the documents of the scope in batches. Yields each hit separately.
+      #
+      # @param batch_size [Integer] batch size obviously, replaces `size` query parameter
+      # @param scroll [String] cursor expiration time
+      #
+      # @overload scroll_hits(batch_size: 1000, scroll: '1m')
+      #   @example
+      #     PlaceIndex.scroll_hits { |hit| p hit['_id'] }
+      #   @yieldparam hit [Hash] block is executed for each hit
+      #
+      # @overload scroll_hits(batch_size: 1000, scroll: '1m')
+      #   @example
+      #     PlaceIndex.scroll_hits.map { |hit| hit['_id'] }
+      #   @return [Enumerator] a standard ruby Enumerator
+      def scroll_hits(**options, &block)
+        return enum_for(:scroll_hits, **options) unless block_given?
+
+        scroll_batches(**options).each do |batch|
+          batch.each(&block)
+        end
+      end
+
+      # @!method scroll_wrappers(batch_size: 1000, scroll: '1m')
+      # Iterates through the documents of the scope in batches. Yields
+      # each hit wrapped with {Chewy::Index}.
+      #
+      # @param batch_size [Integer] batch size obviously, replaces `size` query parameter
+      # @param scroll [String] cursor expiration time
+      #
+      # @overload scroll_wrappers(batch_size: 1000, scroll: '1m')
+      #   @example
+      #     PlaceIndex.scroll_wrappers { |object| p object.id }
+      #   @yieldparam object [Chewy::Index] block is executed for each hit object
+      #
+      # @overload scroll_wrappers(batch_size: 1000, scroll: '1m')
+      #   @example
+      #     PlaceIndex.scroll_wrappers.map { |object| object.id }
+      #   @return [Enumerator] a standard ruby Enumerator
+      def scroll_wrappers(**options)
+        return enum_for(:scroll_wrappers, **options) unless block_given?
+
+        scroll_hits(**options).each do |hit|
+          yield loader.derive_index(hit['_index']).build(hit)
+        end
+      end
+
+      # @!method scroll_objects(batch_size: 1000, scroll: '1m')
+      # Iterates through the documents of the scope in batches. Performs load
+      # operation for each batch and then yields each loaded ORM/ODM object.
+      # Uses {Chewy::Search::Request#load} passed options for loading.
+      #
+      # @note If the record is not found it yields nil instead.
+      # @see Chewy::Search::Request#load
+      # @see Chewy::Search::Loader
+      # @param batch_size [Integer] batch size obviously, replaces `size` query parameter
+      # @param scroll [String] cursor expiration time
+      #
+      # @overload scroll_objects(batch_size: 1000, scroll: '1m')
+      #   @example
+      #     PlaceIndex.scroll_objects { |record| p record.id }
+      #   @yieldparam record [Object] block is executed for each record loaded
+      #
+      # @overload scroll_objects(batch_size: 1000, scroll: '1m')
+      #   @example
+      #     PlaceIndex.scroll_objects.map { |record| record.id }
+      #   @return [Enumerator] a standard ruby Enumerator
+      def scroll_objects(**options, &block)
+        return enum_for(:scroll_objects, **options) unless block_given?
+
+        except(:source, :stored_fields, :script_fields, :docvalue_fields)
+          .source(false).scroll_batches(**options).each do |batch|
+            loader.load(batch).each(&block)
+          end
+      end
+      alias_method :scroll_records, :scroll_objects
+      alias_method :scroll_documents, :scroll_objects
+
+    private
+
+      def perform_scroll(body)
+        ActiveSupport::Notifications.instrument 'search_query.chewy', notification_payload(request: body) do
+          Chewy.client.scroll(body)
+        end
+      end
+    end
+  end
+end

data/lib/chewy/search.rb

@@ -1,53 +1,93 @@
-require 'chewy/query'
+require 'chewy/search/scoping'
+require 'chewy/search/scrolling'
+require 'chewy/search/query_proxy'
+require 'chewy/search/parameters'
+require 'chewy/search/response'
+require 'chewy/search/loader'
+require 'chewy/search/request'
+require 'chewy/search/pagination/kaminari'
 
 module Chewy
+  # This module being included to any provides an interface to the
+  # request DSL. By default it is included to {Chewy::Index}.
+  #
+  # The class used as a request DSL provider is
+  # inherited from {Chewy::Search::Request}
+  #
+  # Also, the search class is refined with the pagination module {Chewy::Search::Pagination::Kaminari}.
+  #
+  # @example
+  #   PlacesIndex.query(match: {name: 'Moscow'})
+  # @see Chewy::Index
+  # @see Chewy::Search::Request
+  # @see Chewy::Search::ClassMethods
+  # @see Chewy::Search::Pagination::Kaminari
   module Search
     extend ActiveSupport::Concern
 
-    included do
-      singleton_class.delegate :explain, :query_mode, :filter_mode, :post_filter_mode,
-        :timeout, :limit, :offset, :highlight, :min_score, :rescore, :facets, :script_score,
-        :boost_factor, :random_score, :field_value_factor, :decay, :aggregations,
-        :suggest, :none, :strategy, :query, :filter, :post_filter, :boost_mode,
-        :score_mode, :order, :reorder, :only, :types, :delete_all, :find, :total,
-        :total_count, :total_entries, to: :all
-    end
-
     module ClassMethods
+      # This is the entry point for the request composition, however,
+      # most of the {Chewy::Search::Request} methods are delegated
+      # directly as well.
+      #
+      # This method also provides an ability to use names scopes.
+      #
+      # @example
+      #   PlacesIndex.all.limit(10)
+      #   # is basically the same as:
+      #   PlacesIndex.limit(10)
+      # @see Chewy::Search::Request
+      # @see Chewy::Search::Scoping
+      # @return [Chewy::Search::Request] request instance
       def all
-        query_class.scopes.last || query_class.new(self)
+        search_class.scopes.last || search_class.new(self)
       end
 
-      def search_string query, options = {}
-        options = options.merge(
-          index: all._indexes.map(&:index_name),
-          type: all._types.map(&:type_name),
-          q: query)
+      # A simple way to execute search string query.
+      #
+      # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/search-uri-request.html
+      # @return [Hash] the request result
+      def search_string(query, options = {})
+        options = options.merge(all.render.slice(:index).merge(q: query))
         Chewy.client.search(options)
       end
 
+      # Delegates methods from the request class to the index class
+      #
+      # @example
+      #   PlacesIndex.query(match: {name: 'Moscow'})
+      ruby2_keywords def method_missing(name, *args, &block)
+        if search_class::DELEGATED_METHODS.include?(name)
+          all.send(name, *args, &block)
+        else
+          super
+        end
+      end
+
+      def respond_to_missing?(name, _)
+        search_class::DELEGATED_METHODS.include?(name) || super
+      end
+
     private
 
-      def query_class
-        @query_class ||= begin
-          query_class = Class.new(Chewy::Query)
-          if self < Chewy::Type
-            index_scopes = index.scopes - scopes
+      def search_class
+        @search_class ||= build_search_class(Chewy.search_class)
+      end
 
-            delegate_scoped index, query_class, index_scopes
-            delegate_scoped index, self, index_scopes
-          end
-          delegate_scoped self, query_class, scopes
-          const_set('Query', query_class)
-        end
+      def build_search_class(base)
+        search_class = Class.new(base)
+
+        delegate_scoped self, search_class, scopes
+        const_set('Query', search_class)
       end
 
-      def delegate_scoped source, destination, methods
+      def delegate_scoped(source, destination, methods)
         methods.each do |method|
           destination.class_eval do
             define_method method do |*args, &block|
               scoping { source.public_send(method, *args, &block) }
             end
+            ruby2_keywords method
           end
         end
       end

data/lib/chewy/stash.rb

@@ -0,0 +1,68 @@
+module Chewy
+  # This class is the main storage for Chewy service data,
+  # Now index raw specifications are stored in the `chewy_specifications`
+  # index.
+  # Journal entries are stored in `chewy_journal`
+  #
+  # @see Chewy::Index::Specification
+  module Stash
+    class Specification < Chewy::Index
+      index_name 'chewy_specifications'
+
+      default_import_options journal: false
+
+      field :specification, type: 'binary'
+    end
+
+    class Journal < Chewy::Index
+      index_name 'chewy_journal'
+
+      # Loads all entries since the specified time.
+      #
+      # @param since_time [Time, DateTime] a timestamp from which we load a journal
+      # @param only [Chewy::Index, Array<Chewy::Index>] journal entries related to these indices will be loaded only
+      def self.entries(since_time, only: [])
+        self.for(only).filter(range: {created_at: {gt: since_time}}).filter.minimum_should_match(1)
+      end
+
+      # Cleans up all the journal entries until the specified time. If nothing is
+      # specified - cleans up everything.
+      #
+      # @param until_time [Time, DateTime] Clean everything before that date
+      # @param only [Chewy::Index, Array<Chewy::Index>] indexes to clean up journal entries for
+      def self.clean(until_time = nil, only: [], delete_by_query_options: {})
+        scope = self.for(only)
+        scope = scope.filter(range: {created_at: {lte: until_time}}) if until_time
+        scope.delete_all(**delete_by_query_options)
+      end
+
+      # Selects all the journal entries for the specified indices.
+      #
+      # @param indices [Chewy::Index, Array<Chewy::Index>]
+      def self.for(*something)
+        something = something.flatten.compact
+        indexes = something.flat_map { |s| Chewy.derive_name(s) }
+        return none if something.present? && indexes.blank?
+
+        scope = all
+        indexes.each do |index|
+          scope = scope.or(filter(term: {index_name: index.derivable_name}))
+        end
+        scope
+      end
+
+      default_import_options journal: false
+
+      field :index_name, type: 'keyword'
+      field :action, type: 'keyword'
+      field :references, type: 'binary'
+      field :created_at, type: 'date'
+
+      def references
+        @references ||= Array.wrap(@attributes['references']).map do |item|
+          JSON.load(Base64.decode64(item)) # rubocop:disable Security/JSONLoad
+        end
+      end
+    end
+  end
+end

data/lib/chewy/strategy/active_job.rb

@@ -11,10 +11,11 @@ module Chewy
     #
     class ActiveJob < Atomic
       class Worker < ::ActiveJob::Base
-        queue_as :chewy
+        queue_as { Chewy.settings.dig(:active_job, :queue) || 'chewy' }
 
         def perform(type, ids, options = {})
-          type.constantize.import!(ids, options)
+          options[:refresh] = !Chewy.disable_refresh_async if Chewy.disable_refresh_async
+          type.constantize.import!(ids, **options)
         end
       end
 

data/lib/chewy/strategy/atomic.rb

@@ -16,11 +16,9 @@ module Chewy
         @stash = {}
       end
 
-      def update type, objects, options = {}
-        ActiveSupport::Deprecation.warn("`urgent: true` option is deprecated and is not effective inside `:atomic` strategy, use `Chewy.strategy(:urgent)` strategy instead") if options.key?(:urgent)
-
+      def update(type, objects, _options = {})
         @stash[type] ||= []
-        @stash[type] |= type.send(:build_root).id ? Array.wrap(objects) : type.adapter.identify(objects)
+        @stash[type] |= type.root.id ? Array.wrap(objects) : type.adapter.identify(objects)
       end
 
       def leave

data/lib/chewy/strategy/atomic_no_refresh.rb

@@ -0,0 +1,18 @@
+module Chewy
+  class Strategy
+    # This strategy works like atomic but import objects with `refresh=false` parameter.
+    #
+    #   Chewy.strategy(:atomic_no_refresh) do
+    #     User.all.map(&:save) # Does nothing here
+    #     Post.all.map(&:save) # And here
+    #     # It imports all the changed users and posts right here
+    #     # before block leaving with bulk ES API, kinda optimization
+    #   end
+    #
+    class AtomicNoRefresh < Atomic
+      def leave
+        @stash.all? { |type, ids| type.import!(ids, refresh: false) }
+      end
+    end
+  end
+end

data/lib/chewy/strategy/base.rb

@@ -11,16 +11,26 @@ module Chewy
       def name
         self.class.name.demodulize.underscore.to_sym
       end
+
       # This method called when some model tries to update index
       #
-      def update type, objects, options = {}
-        raise UndefinedUpdateStrategy.new(type)
+      def update(type, _objects, _options = {})
+        raise UndefinedUpdateStrategy, type
       end
 
       # This method called when strategy pops from the
       # strategies stack
       #
-      def leave
+      def leave; end
+
+      # This method called when some model record is created or updated.
+      # Normally it will just evaluate all the Chewy callbacks and pass results
+      # to current strategy's update method.
+      # However it's possible to override it to achieve delayed evaluation of
+      # callbacks, e.g. using sidekiq.
+      #
+      def update_chewy_indices(object)
+        object.run_chewy_callbacks
       end
     end
   end

data/lib/chewy/strategy/bypass.rb

@@ -8,8 +8,7 @@ module Chewy
     #   end
     #
     class Bypass < Base
-      def update type, objects, options = {}
-      end
+      def update(type, objects, _options = {}); end
     end
   end
 end