chewy 0.8.4 → 5.1.0

data/lib/chewy/search/response.rb

@@ -0,0 +1,119 @@
+module Chewy
+  module Search
+    # This class is a ES response hash wrapper.
+    #
+    # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/_the_search_api.html
+    class Response
+      # @param body [Hash] response body hash
+      # @param loader [Chewy::Search::Loader] loader instance
+      def initialize(body, loader, paginator = nil)
+        @body = body
+        @loader = loader
+        @paginator = paginator
+      end
+
+      # Raw response `hits` collection. Returns empty array is something went wrong.
+      #
+      # @return [Array<Hash>]
+      def hits
+        @hits ||= hits_root['hits'] || []
+      end
+
+      # Response `total` field. Returns `0` if something went wrong.
+      #
+      # @return [Integer]
+      def total
+        @total ||= hits_root['total'] || 0
+      end
+
+      # Response `max_score` field.
+      #
+      # @return [Float]
+      def max_score
+        @max_score ||= hits_root['max_score']
+      end
+
+      # Duration of the request handling in ms according to ES.
+      #
+      # @return [Integer]
+      def took
+        @took ||= @body['took']
+      end
+
+      # Has the request been timed out?
+      #
+      # @return [true, false]
+      def timed_out?
+        @timed_out ||= @body['timed_out']
+      end
+
+      # The `suggest` response part. Returns empty hash if suggests
+      # were not requested.
+      #
+      # @return [Hash]
+      def suggest
+        @suggest ||= @body['suggest'] || {}
+      end
+
+      # The `aggregations` response part. Returns empty hash if aggregations
+      # were not requested.
+      #
+      # @return [Hash]
+      def aggs
+        @aggs ||= @body['aggregations'] || {}
+      end
+      alias_method :aggregations, :aggs
+
+      # {Chewy::Type} wrappers collection instantiated on top of hits.
+      #
+      # @return [Array<Chewy::Type>]
+      def wrappers
+        @wrappers ||= hits.map do |hit|
+          @loader.derive_type(hit['_index'], hit['_type']).build(hit)
+        end
+      end
+
+      # ORM/ODM objects that had been a source for Chewy import
+      # and now loaded from the DB using hits ids. Uses
+      # {Chewy::Search::Request#load} passed options for loading.
+      #
+      # @see Chewy::Search::Request#load
+      # @see Chewy::Search::Loader
+      # @return [Array<Object>]
+      def objects
+        @objects ||= begin
+          objects = @loader.load(hits)
+          if @paginator
+            @paginator.call(objects)
+          else
+            objects
+          end
+        end
+      end
+      alias_method :records, :objects
+      alias_method :documents, :objects
+
+      # This method is used in cases when you need to iterate through
+      # both of the collections simultaneously.
+      #
+      # @example
+      #   scope.each do |wrapper|
+      #     scope.object_hash[wrapper]
+      #   end
+      # @see #wrappers
+      # @see #objects
+      # @return [{Chewy::Type => Object}] a hash with wrappers as keys and ORM/ODM objects as values
+      def object_hash
+        @object_hash ||= wrappers.zip(objects).to_h
+      end
+      alias_method :record_hash, :object_hash
+      alias_method :document_hash, :object_hash
+
+    private
+
+      def hits_root
+        @body.fetch('hits', {})
+      end
+    end
+  end
+end

data/lib/chewy/search/scoping.rb

@@ -0,0 +1,50 @@
+module Chewy
+  module Search
+    # This module along with {Chewy::Search} provides an ability to
+    # use names scopes.
+    #
+    # @example
+    #   class UsersIndex < Chewy::Index
+    #     def self.by_name(name)
+    #       query(match: {name: name})
+    #     end
+    #
+    #     define_type :user do
+    #       def self.by_age(age)
+    #         filter(term: {age: age})
+    #       end
+    #     end
+    #   end
+    #
+    #   UsersIndex.limit(10).by_name('Martin')
+    #   # => <UsersIndex::Query {..., :body=>{:size=>10, :query=>{:match=>{:name=>"Martin"}}}}>
+    #   UsersIndex::User.limit(10).by_name('Martin').by_age(42)
+    #   # => <UsersIndex::User::Query {..., :body=>{:size=>10, :query=>{:bool=>{
+    #   #      :must=>{:match=>{:name=>"Martin"}},
+    #   #      :filter=>{:term=>{:age=>42}}}}}}>
+    module Scoping
+      extend ActiveSupport::Concern
+
+      module ClassMethods
+        # The scopes stack.
+        #
+        # @return [Array<Chewy::Search::Reques>] array of scopes
+        def scopes
+          Thread.current[:chewy_scopes] ||= []
+        end
+      end
+
+      # Wraps any method to make it contents be executed inside the
+      # current request scope.
+      #
+      # @see Chewy::Search::ClassMethods#all
+      # @yield executes the block after the current context is put at the top of the scope stack
+      def scoping
+        self.class.scopes.push(self)
+        yield
+      ensure
+        self.class.scopes.pop
+      end
+    end
+  end
+end

data/lib/chewy/search/scrolling.rb

@@ -0,0 +1,134 @@
+module Chewy
+  module Search
+    # This module contains batch requests DSL via ES scroll API. All the methods
+    # are optimized on memory consumption, they are not caching anythig, so
+    # use them when you need to do some single-run stuff on a huge amount of
+    # documents. Don't forget to tune the `scroll` parameter for long-lasting
+    # actions.
+    # All the scroll methods respect the limit value if provided.
+    #
+    # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-scroll.html
+    module Scrolling
+      # Iterates through the documents of the scope in batches. Limit if overrided
+      # by the `batch_size`. There are 2 possible use-cases: with a block or without.
+      #
+      # @param batch_size [Integer] batch size obviously, replaces `size` query parameter
+      # @param scroll [String] cursor expiration time
+      #
+      # @overload scroll_batches(batch_size: 1000, scroll: '1m')
+      #   @example
+      #     PlaceIndex.scroll_batches { |batch| batch.each { |hit| p hit['_id'] } }
+      #   @yieldparam batch [Array<Hash>] block is executed for each batch of hits
+      #
+      # @overload scroll_batches(batch_size: 1000, scroll: '1m')
+      #   @example
+      #     PlaceIndex.scroll_batches.flat_map { |batch| batch.map { |hit| hit['_id'] } }
+      #   @return [Enumerator] a standard ruby Enumerator
+      def scroll_batches(batch_size: Request::DEFAULT_BATCH_SIZE, scroll: Request::DEFAULT_SCROLL)
+        return enum_for(:scroll_batches, batch_size: batch_size, scroll: scroll) unless block_given?
+
+        result = perform(size: batch_size, scroll: scroll)
+        total = [raw_limit_value, result.fetch('hits', {}).fetch('total', 0)].compact.min
+        last_batch_size = total % batch_size
+        fetched = 0
+
+        loop do
+          hits = result.fetch('hits', {}).fetch('hits', [])
+          fetched += hits.size
+          hits = hits.first(last_batch_size) if last_batch_size != 0 && fetched >= total
+          yield(hits) if hits.present?
+          break if fetched >= total
+          scroll_id = result['_scroll_id']
+          result = perform_scroll(scroll: scroll, scroll_id: scroll_id)
+        end
+      end
+
+      # @!method scroll_hits(batch_size: 1000, scroll: '1m')
+      # Iterates through the documents of the scope in batches. Yields each hit separately.
+      #
+      # @param batch_size [Integer] batch size obviously, replaces `size` query parameter
+      # @param scroll [String] cursor expiration time
+      #
+      # @overload scroll_hits(batch_size: 1000, scroll: '1m')
+      #   @example
+      #     PlaceIndex.scroll_hits { |hit| p hit['_id'] }
+      #   @yieldparam hit [Hash] block is executed for each hit
+      #
+      # @overload scroll_hits(batch_size: 1000, scroll: '1m')
+      #   @example
+      #     PlaceIndex.scroll_hits.map { |hit| hit['_id'] }
+      #   @return [Enumerator] a standard ruby Enumerator
+      def scroll_hits(**options)
+        return enum_for(:scroll_hits, **options) unless block_given?
+
+        scroll_batches(**options).each do |batch|
+          batch.each { |hit| yield hit }
+        end
+      end
+
+      # @!method scroll_wrappers(batch_size: 1000, scroll: '1m')
+      # Iterates through the documents of the scope in batches. Yields
+      # each hit wrapped with {Chewy::Type}.
+      #
+      # @param batch_size [Integer] batch size obviously, replaces `size` query parameter
+      # @param scroll [String] cursor expiration time
+      #
+      # @overload scroll_wrappers(batch_size: 1000, scroll: '1m')
+      #   @example
+      #     PlaceIndex.scroll_wrappers { |object| p object.id }
+      #   @yieldparam object [Chewy::Type] block is executed for each hit object
+      #
+      # @overload scroll_wrappers(batch_size: 1000, scroll: '1m')
+      #   @example
+      #     PlaceIndex.scroll_wrappers.map { |object| object.id }
+      #   @return [Enumerator] a standard ruby Enumerator
+      def scroll_wrappers(**options)
+        return enum_for(:scroll_wrappers, **options) unless block_given?
+
+        scroll_hits(**options).each do |hit|
+          yield loader.derive_type(hit['_index'], hit['_type']).build(hit)
+        end
+      end
+
+      # @!method scroll_objects(batch_size: 1000, scroll: '1m')
+      # Iterates through the documents of the scope in batches. Performs load
+      # operation for each batch and then yields each loaded ORM/ODM object.
+      # Uses {Chewy::Search::Request#load} passed options for loading.
+      #
+      # @note If the record is not found it yields nil instead.
+      # @see Chewy::Search::Request#load
+      # @see Chewy::Search::Loader
+      # @param batch_size [Integer] batch size obviously, replaces `size` query parameter
+      # @param scroll [String] cursor expiration time
+      #
+      # @overload scroll_objects(batch_size: 1000, scroll: '1m')
+      #   @example
+      #     PlaceIndex.scroll_objects { |record| p record.id }
+      #   @yieldparam record [Object] block is executed for each record loaded
+      #
+      # @overload scroll_objects(batch_size: 1000, scroll: '1m')
+      #   @example
+      #     PlaceIndex.scroll_objects.map { |record| record.id }
+      #   @return [Enumerator] a standard ruby Enumerator
+      def scroll_objects(**options)
+        return enum_for(:scroll_objects, **options) unless block_given?
+
+        except(:source, :stored_fields, :script_fields, :docvalue_fields)
+          .source(false).scroll_batches(**options).each do |batch|
+            loader.load(batch).each { |object| yield object }
+          end
+      end
+      alias_method :scroll_records, :scroll_objects
+      alias_method :scroll_documents, :scroll_objects
+
+    private
+
+      def perform_scroll(body)
+        ActiveSupport::Notifications.instrument 'search_query.chewy',
+          notification_payload(request: body) do
+            Chewy.client.scroll(body)
+          end
+      end
+    end
+  end
+end

data/lib/chewy/search.rb

@@ -1,48 +1,103 @@
+require 'chewy/search/scoping'
 require 'chewy/query'
+require 'chewy/search/scrolling'
+require 'chewy/search/query_proxy'
+require 'chewy/search/parameters'
+require 'chewy/search/response'
+require 'chewy/search/loader'
+require 'chewy/search/request'
+require 'chewy/search/pagination/kaminari'
+require 'chewy/search/pagination/will_paginate'
 
 module Chewy
+  # This module being included to any provides an interface to the
+  # request DSL. By default it is included to {Chewy::Index} and
+  # {Chewy::Type}.
+  #
+  # The class used as a request DSL provider is
+  # inherited from {Chewy::Search::Request} by default, but if you
+  # need ES < 2.0 DSL support - you can switch it to {Chewy::Query}
+  # using {Chewy::Config#search_class}
+  #
+  # Also, the search class is refined with one of the pagination-
+  # providing modules: {Chewy::Search::Pagination::Kaminari} or
+  # {Chewy::Search::Pagination::WillPaginate}.
+  #
+  # @example
+  #   PlacesIndex.query(match: {name: 'Moscow'})
+  #   PlacesIndex::City.query(match: {name: 'Moscow'})
+  # @see Chewy::Index
+  # @see Chewy::Type
+  # @see Chewy::Search::Request
+  # @see Chewy::Search::ClassMethods
+  # @see Chewy::Search::Pagination::Kaminari
+  # @see Chewy::Search::Pagination::WillPaginate
   module Search
     extend ActiveSupport::Concern
 
-    included do
-      singleton_class.delegate :explain, :query_mode, :filter_mode, :post_filter_mode,
-        :timeout, :limit, :offset, :highlight, :min_score, :rescore, :facets, :script_score,
-        :boost_factor, :random_score, :field_value_factor, :decay, :aggregations,
-        :suggest, :none, :strategy, :query, :filter, :post_filter, :boost_mode,
-        :score_mode, :order, :reorder, :only, :types, :delete_all, :find, :total,
-        :total_count, :total_entries, to: :all
-    end
-
     module ClassMethods
+      # This is the entry point for the request composition, however,
+      # most of the {Chewy::Search::Request} methods are delegated
+      # directly as well.
+      #
+      # This method also provides an ability to use names scopes.
+      #
+      # @example
+      #   PlacesIndex.all.limit(10)
+      #   # is basically the same as:
+      #   PlacesIndex.limit(10)
+      # @see Chewy::Search::Request
+      # @see Chewy::Search::Scoping
+      # @return [Chewy::Search::Request] request instance
       def all
-        query_class.scopes.last || query_class.new(self)
+        search_class.scopes.last || search_class.new(self)
       end
 
-      def search_string query, options = {}
-        options = options.merge(
-          index: all._indexes.map(&:index_name),
-          type: all._types.map(&:type_name),
-          q: query)
+      # A simple way to execute search string query.
+      #
+      # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/search-uri-request.html
+      # @return [Hash] the request result
+      def search_string(query, options = {})
+        options = options.merge(all.render.slice(:index, :type).merge(q: query))
         Chewy.client.search(options)
       end
 
+      # Delegates methods from the request class to the index or type class
+      #
+      # @example
+      #   PlacesIndex.query(match: {name: 'Moscow'})
+      #   PlacesIndex::City.query(match: {name: 'Moscow'})
+      def method_missing(name, *args, &block)
+        if search_class::DELEGATED_METHODS.include?(name)
+          all.send(name, *args, &block)
+        else
+          super
+        end
+      end
+
+      def respond_to_missing?(name, _)
+        search_class::DELEGATED_METHODS.include?(name) || super
+      end
+
     private
 
-      def query_class
-        @query_class ||= begin
-          query_class = Class.new(Chewy::Query)
-          if self < Chewy::Type
-            index_scopes = index.scopes - scopes
+      def search_class
+        @search_class ||= build_search_class(Chewy.search_class)
+      end
 
-            delegate_scoped index, query_class, index_scopes
-            delegate_scoped index, self, index_scopes
-          end
-          delegate_scoped self, query_class, scopes
-          const_set('Query', query_class)
+      def build_search_class(base)
+        search_class = Class.new(base)
+
+        if self < Chewy::Type
+          index_scopes = index.scopes - scopes
+          delegate_scoped index, search_class, index_scopes
         end
+
+        delegate_scoped self, search_class, scopes
+        const_set('Query', search_class)
       end
 
-      def delegate_scoped source, destination, methods
+      def delegate_scoped(source, destination, methods)
         methods.each do |method|
           destination.class_eval do
             define_method method do |*args, &block|

data/lib/chewy/stash.rb

@@ -0,0 +1,79 @@
+module Chewy
+  # This class is the main storage for Chewy service data,
+  # Now index raw specifications are stored in the `chewy_specifications`
+  # index.
+  # Journal entries are stored in `chewy_journal`
+  #
+  # @see Chewy::Index::Specification
+  module Stash
+    class Specification < Chewy::Index
+      index_name 'chewy_specifications'
+
+      define_type :specification do
+        default_import_options journal: false
+
+        field :specification, type: 'binary'
+      end
+    end
+
+    class Journal < Chewy::Index
+      index_name 'chewy_journal'
+
+      # Loads all entries since the specified time.
+      #
+      # @param since_time [Time, DateTime] a timestamp from which we load a journal
+      # @param only [Chewy::Index, Array<Chewy::Index>] journal entries related to these indices will be loaded only
+      def self.entries(since_time, only: [])
+        self.for(only).filter(range: {created_at: {gt: since_time}})
+      end
+
+      # Cleans up all the journal entries until the specified time. If nothing is
+      # specified - cleans up everything.
+      #
+      # @param since_time [Time, DateTime] the time top boundary
+      # @param only [Chewy::Index, Array<Chewy::Index>] indexes to clean up journal entries for
+      def self.clean(until_time = nil, only: [])
+        scope = self.for(only)
+        scope = scope.filter(range: {created_at: {lte: until_time}}) if until_time
+        scope.delete_all
+      end
+
+      # Selects all the journal entries for the specified indices.
+      #
+      # @param indices [Chewy::Index, Array<Chewy::Index>]
+      def self.for(*something)
+        something = something.flatten.compact
+        types = something.flat_map { |s| Chewy.derive_types(s) }
+        return none if something.present? && types.blank?
+        scope = all
+        types.group_by(&:index).each do |index, index_types|
+          scope = scope.or(
+            filter(term: {index_name: index.derivable_name})
+              .filter(terms: {type_name: index_types.map(&:type_name)})
+          )
+        end
+        scope
+      end
+
+      define_type :journal do
+        default_import_options journal: false
+
+        field :index_name, type: 'keyword'
+        field :type_name, type: 'keyword'
+        field :action, type: 'keyword'
+        field :references, type: 'binary'
+        field :created_at, type: 'date'
+
+        def type
+          @type ||= Chewy.derive_type("#{index_name}##{type_name}")
+        end
+
+        def references
+          @references ||= Array.wrap(@attributes['references']).map do |item|
+            JSON.load(Base64.decode64(item)) # rubocop:disable Security/JSONLoad
+          end
+        end
+      end
+    end
+  end
+end

data/lib/chewy/strategy/active_job.rb

@@ -14,6 +14,7 @@ module Chewy
         queue_as :chewy
 
         def perform(type, ids, options = {})
+          options[:refresh] = !Chewy.disable_refresh_async if Chewy.disable_refresh_async
           type.constantize.import!(ids, options)
         end
       end

data/lib/chewy/strategy/atomic.rb

@@ -16,11 +16,9 @@ module Chewy
         @stash = {}
       end
 
-      def update type, objects, options = {}
-        ActiveSupport::Deprecation.warn("`urgent: true` option is deprecated and is not effective inside `:atomic` strategy, use `Chewy.strategy(:urgent)` strategy instead") if options.key?(:urgent)
-
+      def update(type, objects, _options = {})
         @stash[type] ||= []
-        @stash[type] |= type.send(:build_root).id ? Array.wrap(objects) : type.adapter.identify(objects)
+        @stash[type] |= type.root.id ? Array.wrap(objects) : type.adapter.identify(objects)
       end
 
       def leave

data/lib/chewy/strategy/base.rb

@@ -11,17 +11,17 @@ module Chewy
       def name
         self.class.name.demodulize.underscore.to_sym
       end
+
       # This method called when some model tries to update index
       #
-      def update type, objects, options = {}
-        raise UndefinedUpdateStrategy.new(type)
+      def update(type, _objects, _options = {})
+        raise UndefinedUpdateStrategy, type
       end
 
       # This method called when strategy pops from the
       # strategies stack
       #
-      def leave
-      end
+      def leave; end
     end
   end
 end

data/lib/chewy/strategy/bypass.rb

@@ -8,8 +8,7 @@ module Chewy
     #   end
     #
     class Bypass < Base
-      def update type, objects, options = {}
-      end
+      def update(type, objects, _options = {}); end
     end
   end
 end

data/lib/chewy/strategy/resque.rb

@@ -14,6 +14,7 @@ module Chewy
         @queue = :chewy
 
         def self.perform(type, ids, options = {})
+          options[:refresh] = !Chewy.disable_refresh_async if Chewy.disable_refresh_async
           type.constantize.import!(ids, options)
         end
       end

data/lib/chewy/strategy/shoryuken.rb

@@ -0,0 +1,40 @@
+module Chewy
+  class Strategy
+    # The strategy works the same way as atomic, but performs
+    # async index update driven by shoryuken
+    #
+    #   Chewy.strategy(:shoryuken) do
+    #     User.all.map(&:save) # Does nothing here
+    #     Post.all.map(&:save) # And here
+    #     # It imports all the changed users and posts right here
+    #   end
+    #
+    class Shoryuken < Atomic
+      class Worker
+        include ::Shoryuken::Worker
+
+        shoryuken_options auto_delete: true,
+                          body_parser: :json
+
+        def perform(_sqs_msg, body)
+          options = body['options'] || {}
+          options[:refresh] = !Chewy.disable_refresh_async if Chewy.disable_refresh_async
+          body['type'].constantize.import!(body['ids'], options.deep_symbolize_keys!)
+        end
+      end
+
+      def leave
+        @stash.each do |type, ids|
+          next if ids.empty?
+          Shoryuken::Worker.perform_async({type: type.name, ids: ids}, queue: shoryuken_queue)
+        end
+      end
+
+    private
+
+      def shoryuken_queue
+        Chewy.settings.fetch(:shoryuken, {})[:queue] || 'chewy'
+      end
+    end
+  end
+end

data/lib/chewy/strategy/sidekiq.rb

@@ -14,15 +14,27 @@ module Chewy
         include ::Sidekiq::Worker
 
         def perform(type, ids, options = {})
+          options[:refresh] = !Chewy.disable_refresh_async if Chewy.disable_refresh_async
           type.constantize.import!(ids, options)
         end
       end
 
       def leave
         @stash.each do |type, ids|
-          Chewy::Strategy::Sidekiq::Worker.perform_async(type.name, ids) unless ids.empty?
+          next if ids.empty?
+          ::Sidekiq::Client.push(
+            'queue' => sidekiq_queue,
+            'class' => Chewy::Strategy::Sidekiq::Worker,
+            'args'  => [type.name, ids]
+          )
         end
       end
+
+    private
+
+      def sidekiq_queue
+        Chewy.settings.fetch(:sidekiq, {})[:queue] || 'chewy'
+      end
     end
   end
 end

data/lib/chewy/strategy/urgent.rb

@@ -9,7 +9,7 @@ module Chewy
     #   end
     #
     class Urgent < Base
-      def update type, objects, options = {}
+      def update(type, objects, _options = {})
         type.import!(Array.wrap(objects))
       end
     end