chewy 0.10.1 → 7.3.2

data/lib/chewy/search/response.rb

@@ -23,7 +23,7 @@ module Chewy
       #
       # @return [Integer]
       def total
-        @total ||= hits_root['total'] || 0
+        @total ||= hits_root.fetch('total', {}).fetch('value', 0)
       end
 
       # Response `max_score` field.
@@ -64,12 +64,12 @@ module Chewy
       end
       alias_method :aggregations, :aggs
 
-      # {Chewy::Type} wrappers collection instantiated on top of hits.
+      # {Chewy::Index} wrappers collection instantiated on top of hits.
       #
-      # @return [Array<Chewy::Type>]
+      # @return [Array<Chewy::Index>]
       def wrappers
         @wrappers ||= hits.map do |hit|
-          @loader.derive_type(hit['_index'], hit['_type']).build(hit)
+          @loader.derive_index(hit['_index']).build(hit)
         end
       end
 
@@ -102,7 +102,7 @@ module Chewy
       #   end
       # @see #wrappers
       # @see #objects
-      # @return [{Chewy::Type => Object}] a hash with wrappers as keys and ORM/ODM objects as values
+      # @return [{Chewy::Index => Object}] a hash with wrappers as keys and ORM/ODM objects as values
       def object_hash
         @object_hash ||= wrappers.zip(objects).to_h
       end

data/lib/chewy/search/scoping.rb

@@ -9,17 +9,16 @@ module Chewy
     #       query(match: {name: name})
     #     end
     #
-    #     define_type :user do
-    #       def self.by_age(age)
-    #         filter(term: {age: age})
-    #       end
+    #
+    #     def self.by_age(age)
+    #       filter(term: {age: age})
     #     end
     #   end
     #
     #   UsersIndex.limit(10).by_name('Martin')
     #   # => <UsersIndex::Query {..., :body=>{:size=>10, :query=>{:match=>{:name=>"Martin"}}}}>
-    #   UsersIndex::User.limit(10).by_name('Martin').by_age(42)
-    #   # => <UsersIndex::User::Query {..., :body=>{:size=>10, :query=>{:bool=>{
+    #   UsersIndex.limit(10).by_name('Martin').by_age(42)
+    #   # => <UsersIndex::Query {..., :body=>{:size=>10, :query=>{:bool=>{
     #   #      :must=>{:match=>{:name=>"Martin"}},
     #   #      :filter=>{:term=>{:age=>42}}}}}}>
     module Scoping
@@ -28,9 +27,9 @@ module Chewy
       module ClassMethods
         # The scopes stack.
         #
-        # @return [Array<Chewy::Search::Reques>] array of scopes
+        # @return [Array<Chewy::Search::Request>] array of scopes
         def scopes
-          Thread.current[:chewy_scopes] ||= []
+          Chewy.current[:chewy_scopes] ||= []
         end
       end
 

data/lib/chewy/search/scrolling.rb

@@ -28,19 +28,23 @@ module Chewy
         return enum_for(:scroll_batches, batch_size: batch_size, scroll: scroll) unless block_given?
 
         result = perform(size: batch_size, scroll: scroll)
-        total = [raw_limit_value, result.fetch('hits', {}).fetch('total', 0)].compact.min
+        total = [raw_limit_value, result.fetch('hits', {}).fetch('total', {}).fetch('value', 0)].compact.min
         last_batch_size = total % batch_size
         fetched = 0
+        scroll_id = nil
 
         loop do
           hits = result.fetch('hits', {}).fetch('hits', [])
           fetched += hits.size
           hits = hits.first(last_batch_size) if last_batch_size != 0 && fetched >= total
           yield(hits) if hits.present?
-          break if fetched >= total
           scroll_id = result['_scroll_id']
+          break if fetched >= total
+
           result = perform_scroll(scroll: scroll, scroll_id: scroll_id)
         end
+      ensure
+        Chewy.client.clear_scroll(body: {scroll_id: scroll_id}) if scroll_id
       end
 
       # @!method scroll_hits(batch_size: 1000, scroll: '1m')
@@ -58,17 +62,17 @@ module Chewy
       #   @example
       #     PlaceIndex.scroll_hits.map { |hit| hit['_id'] }
       #   @return [Enumerator] a standard ruby Enumerator
-      def scroll_hits(**options)
+      def scroll_hits(**options, &block)
         return enum_for(:scroll_hits, **options) unless block_given?
 
         scroll_batches(**options).each do |batch|
-          batch.each { |hit| yield hit }
+          batch.each(&block)
         end
       end
 
       # @!method scroll_wrappers(batch_size: 1000, scroll: '1m')
       # Iterates through the documents of the scope in batches. Yields
-      # each hit wrapped with {Chewy::Type}.
+      # each hit wrapped with {Chewy::Index}.
       #
       # @param batch_size [Integer] batch size obviously, replaces `size` query parameter
       # @param scroll [String] cursor expiration time
@@ -76,7 +80,7 @@ module Chewy
       # @overload scroll_wrappers(batch_size: 1000, scroll: '1m')
       #   @example
       #     PlaceIndex.scroll_wrappers { |object| p object.id }
-      #   @yieldparam object [Chewy::Type] block is executed for each hit object
+      #   @yieldparam object [Chewy::Index] block is executed for each hit object
       #
       # @overload scroll_wrappers(batch_size: 1000, scroll: '1m')
       #   @example
@@ -86,7 +90,7 @@ module Chewy
         return enum_for(:scroll_wrappers, **options) unless block_given?
 
         scroll_hits(**options).each do |hit|
-          yield loader.derive_type(hit['_index'], hit['_type']).build(hit)
+          yield loader.derive_index(hit['_index']).build(hit)
         end
       end
 
@@ -110,12 +114,12 @@ module Chewy
       #   @example
       #     PlaceIndex.scroll_objects.map { |record| record.id }
       #   @return [Enumerator] a standard ruby Enumerator
-      def scroll_objects(**options)
+      def scroll_objects(**options, &block)
         return enum_for(:scroll_objects, **options) unless block_given?
 
         except(:source, :stored_fields, :script_fields, :docvalue_fields)
           .source(false).scroll_batches(**options).each do |batch|
-            loader.load(batch).each { |object| yield object }
+            loader.load(batch).each(&block)
           end
       end
       alias_method :scroll_records, :scroll_objects
@@ -124,10 +128,7 @@ module Chewy
     private
 
       def perform_scroll(body)
-        ActiveSupport::Notifications.instrument 'search_query.chewy',
-          request: body, indexes: _indexes, types: _types,
-          index: _indexes.one? ? _indexes.first : _indexes,
-          type: _types.one? ? _types.first : _types do
+        ActiveSupport::Notifications.instrument 'search_query.chewy', notification_payload(request: body) do
           Chewy.client.scroll(body)
         end
       end

data/lib/chewy/search.rb

@@ -1,5 +1,4 @@
 require 'chewy/search/scoping'
-require 'chewy/query'
 require 'chewy/search/scrolling'
 require 'chewy/search/query_proxy'
 require 'chewy/search/parameters'
@@ -7,31 +6,22 @@ require 'chewy/search/response'
 require 'chewy/search/loader'
 require 'chewy/search/request'
 require 'chewy/search/pagination/kaminari'
-require 'chewy/search/pagination/will_paginate'
 
 module Chewy
   # This module being included to any provides an interface to the
-  # request DSL. By default it is included to {Chewy::Index} and
-  # {Chewy::Type}.
+  # request DSL. By default it is included to {Chewy::Index}.
   #
   # The class used as a request DSL provider is
-  # inherited from {Chewy::Search::Request} by default, but if you
-  # need ES < 2.0 DSL support - you can switch it to {Chewy::Query}
-  # using {Chewy::Config#search_class}
+  # inherited from {Chewy::Search::Request}
   #
-  # Also, the search class is refined with one of the pagination-
-  # providing modules: {Chewy::Search::Pagination::Kaminari} or
-  # {Chewy::Search::Pagination::WillPaginate}.
+  # Also, the search class is refined with the pagination module {Chewy::Search::Pagination::Kaminari}.
   #
   # @example
   #   PlacesIndex.query(match: {name: 'Moscow'})
-  #   PlacesIndex::City.query(match: {name: 'Moscow'})
   # @see Chewy::Index
-  # @see Chewy::Type
   # @see Chewy::Search::Request
   # @see Chewy::Search::ClassMethods
   # @see Chewy::Search::Pagination::Kaminari
-  # @see Chewy::Search::Pagination::WillPaginate
   module Search
     extend ActiveSupport::Concern
 
@@ -58,20 +48,15 @@ module Chewy
       # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/search-uri-request.html
       # @return [Hash] the request result
       def search_string(query, options = {})
-        options = options.merge(
-          index: all._indexes.map(&:index_name),
-          type: all._types.map(&:type_name),
-          q: query
-        )
+        options = options.merge(all.render.slice(:index).merge(q: query))
         Chewy.client.search(options)
       end
 
-      # Delegates methods from the request class to the index or type class
+      # Delegates methods from the request class to the index class
       #
       # @example
       #   PlacesIndex.query(match: {name: 'Moscow'})
-      #   PlacesIndex::City.query(match: {name: 'Moscow'})
-      def method_missing(name, *args, &block)
+      ruby2_keywords def method_missing(name, *args, &block)
         if search_class::DELEGATED_METHODS.include?(name)
           all.send(name, *args, &block)
         else
@@ -91,12 +76,7 @@ module Chewy
 
       def build_search_class(base)
         search_class = Class.new(base)
-        if self < Chewy::Type
-          index_scopes = index.scopes - scopes
 
-          delegate_scoped index, search_class, index_scopes
-          delegate_scoped index, self, index_scopes
-        end
         delegate_scoped self, search_class, scopes
         const_set('Query', search_class)
       end
@@ -107,6 +87,7 @@ module Chewy
             define_method method do |*args, &block|
               scoping { source.public_send(method, *args, &block) }
             end
+            ruby2_keywords method
           end
         end
       end

data/lib/chewy/stash.rb

@@ -1,44 +1,39 @@
 module Chewy
   # This class is the main storage for Chewy service data,
-  # Now index raw specifications are stored in the `chewy_stash`
-  # index. In the future the journal will be moved here as well.
+  # Now index raw specifications are stored in the `chewy_specifications`
+  # index.
+  # Journal entries are stored in `chewy_journal`
   #
   # @see Chewy::Index::Specification
-  class Stash < Chewy::Index
-    index_name 'chewy_stash'
+  module Stash
+    class Specification < Chewy::Index
+      index_name 'chewy_specifications'
 
-    define_type :specification do
       default_import_options journal: false
 
-      field :value, index: 'no'
+      field :specification, type: 'binary'
     end
 
-    define_type :journal do # rubocop:disable Metrics/BlockLength
-      default_import_options journal: false
-
-      field :index_name, type: 'string', index: 'not_analyzed'
-      field :type_name, type: 'string', index: 'not_analyzed'
-      field :action, type: 'string', index: 'not_analyzed'
-      field :references, type: 'string', index: 'no'
-      field :created_at, type: 'date'
+    class Journal < Chewy::Index
+      index_name 'chewy_journal'
 
       # Loads all entries since the specified time.
       #
       # @param since_time [Time, DateTime] a timestamp from which we load a journal
       # @param only [Chewy::Index, Array<Chewy::Index>] journal entries related to these indices will be loaded only
       def self.entries(since_time, only: [])
-        self.for(only).filter(range: {created_at: {gt: since_time}})
+        self.for(only).filter(range: {created_at: {gt: since_time}}).filter.minimum_should_match(1)
       end
 
       # Cleans up all the journal entries until the specified time. If nothing is
       # specified - cleans up everything.
       #
-      # @param since_time [Time, DateTime] the time top boundary
+      # @param until_time [Time, DateTime] Clean everything before that date
       # @param only [Chewy::Index, Array<Chewy::Index>] indexes to clean up journal entries for
-      def self.clean(until_time = nil, only: [])
+      def self.clean(until_time = nil, only: [], delete_by_query_options: {})
         scope = self.for(only)
         scope = scope.filter(range: {created_at: {lte: until_time}}) if until_time
-        scope.delete_all
+        scope.delete_all(**delete_by_query_options)
       end
 
       # Selects all the journal entries for the specified indices.
@@ -46,24 +41,27 @@ module Chewy
       # @param indices [Chewy::Index, Array<Chewy::Index>]
       def self.for(*something)
         something = something.flatten.compact
-        types = something.flat_map { |s| Chewy.derive_types(s) }
-        return none if something.present? && types.blank?
+        indexes = something.flat_map { |s| Chewy.derive_name(s) }
+        return none if something.present? && indexes.blank?
+
         scope = all
-        types.group_by(&:index).each do |index, index_types|
-          scope = scope.or(
-            filter(term: {index_name: index.derivable_name})
-            .filter(terms: {type_name: index_types.map(&:type_name)})
-          )
+        indexes.each do |index|
+          scope = scope.or(filter(term: {index_name: index.derivable_name}))
         end
         scope
       end
 
-      def type
-        @type ||= Chewy.derive_type("#{index_name}##{type_name}")
-      end
+      default_import_options journal: false
+
+      field :index_name, type: 'keyword'
+      field :action, type: 'keyword'
+      field :references, type: 'binary'
+      field :created_at, type: 'date'
 
       def references
-        @references ||= Array.wrap(@attributes['references']).map { |r| JSON.load(r) } # rubocop:disable Security/JSONLoad
+        @references ||= Array.wrap(@attributes['references']).map do |item|
+          JSON.load(Base64.decode64(item)) # rubocop:disable Security/JSONLoad
+        end
       end
     end
   end

data/lib/chewy/strategy/active_job.rb

@@ -11,11 +11,11 @@ module Chewy
     #
     class ActiveJob < Atomic
       class Worker < ::ActiveJob::Base
-        queue_as :chewy
+        queue_as { Chewy.settings.dig(:active_job, :queue) || 'chewy' }
 
         def perform(type, ids, options = {})
           options[:refresh] = !Chewy.disable_refresh_async if Chewy.disable_refresh_async
-          type.constantize.import!(ids, options)
+          type.constantize.import!(ids, **options)
         end
       end
 

data/lib/chewy/strategy/atomic.rb

@@ -18,7 +18,7 @@ module Chewy
 
       def update(type, objects, _options = {})
         @stash[type] ||= []
-        @stash[type] |= type.send(:build_root).id ? Array.wrap(objects) : type.adapter.identify(objects)
+        @stash[type] |= type.root.id ? Array.wrap(objects) : type.adapter.identify(objects)
       end
 
       def leave

data/lib/chewy/strategy/atomic_no_refresh.rb

@@ -0,0 +1,18 @@
+module Chewy
+  class Strategy
+    # This strategy works like atomic but import objects with `refresh=false` parameter.
+    #
+    #   Chewy.strategy(:atomic_no_refresh) do
+    #     User.all.map(&:save) # Does nothing here
+    #     Post.all.map(&:save) # And here
+    #     # It imports all the changed users and posts right here
+    #     # before block leaving with bulk ES API, kinda optimization
+    #   end
+    #
+    class AtomicNoRefresh < Atomic
+      def leave
+        @stash.all? { |type, ids| type.import!(ids, refresh: false) }
+      end
+    end
+  end
+end

data/lib/chewy/strategy/base.rb

@@ -22,6 +22,16 @@ module Chewy
       # strategies stack
       #
       def leave; end
+
+      # This method called when some model record is created or updated.
+      # Normally it will just evaluate all the Chewy callbacks and pass results
+      # to current strategy's update method.
+      # However it's possible to override it to achieve delayed evaluation of
+      # callbacks, e.g. using sidekiq.
+      #
+      def update_chewy_indices(object)
+        object.run_chewy_callbacks
+      end
     end
   end
 end

data/lib/chewy/strategy/delayed_sidekiq/scheduler.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+require_relative '../../index'
+
+# The class is responsible for accumulating in redis [type, ids]
+# that were requested to be reindexed during `latency` seconds.
+# The reindex job is going to be scheduled after a `latency` seconds.
+# that job is going to read accumulated [type, ids] from the redis
+# and reindex all them at once.
+module Chewy
+  class Strategy
+    class DelayedSidekiq
+      require_relative 'worker'
+
+      class Scheduler
+        DEFAULT_TTL = 60 * 60 * 24 # in seconds
+        DEFAULT_LATENCY = 10
+        DEFAULT_MARGIN = 2
+        DEFAULT_QUEUE = 'chewy'
+        KEY_PREFIX = 'chewy:delayed_sidekiq'
+        FALLBACK_FIELDS = 'all'
+        FIELDS_IDS_SEPARATOR = ';'
+        IDS_SEPARATOR = ','
+
+        def initialize(type, ids, options = {})
+          @type = type
+          @ids = ids
+          @options = options
+        end
+
+        # the diagram:
+        #
+        #  inputs:
+        #  latency == 2
+        #  reindex_time = Time.current
+        #
+        #  Parallel OR Sequential triggers of reindex:          |  What is going on in reindex store (Redis):
+        #  --------------------------------------------------------------------------------------------------
+        #                                                       |
+        #  process 1 (reindex_time):                            |  chewy:delayed_sidekiq:CitiesIndex:1679347866 = [1]
+        #    Schedule.new(CitiesIndex, [1]).postpone            |  chewy:delayed_sidekiq:timechunks = [{ score: 1679347866, "chewy:delayed_sidekiq:CitiesIndex:1679347866"}]
+        #                                                       |  & schedule a DelayedSidekiq::Worker at 1679347869 (at + 3)
+        #                                                       |    it will zpop chewy:delayed_sidekiq:timechunks up to 1679347866 score and reindex all ids with zpoped keys
+        #                                                       |      chewy:delayed_sidekiq:CitiesIndex:1679347866
+        #                                                       |
+        #                                                       |
+        #  process 2 (reindex_time):                            |  chewy:delayed_sidekiq:CitiesIndex:1679347866 = [1, 2]
+        #    Schedule.new(CitiesIndex, [2]).postpone            |  chewy:delayed_sidekiq:timechunks = [{ score: 1679347866, "chewy:delayed_sidekiq:CitiesIndex:1679347866"}]
+        #                                                       |  & do not schedule a new worker
+        #                                                       |
+        #                                                       |
+        #  process 1 (reindex_time + (latency - 1).seconds):    |  chewy:delayed_sidekiq:CitiesIndex:1679347866 = [1, 2, 3]
+        #    Schedule.new(CitiesIndex, [3]).postpone            |  chewy:delayed_sidekiq:timechunks = [{ score: 1679347866, "chewy:delayed_sidekiq:CitiesIndex:1679347866"}]
+        #                                                       |  & do not schedule a new worker
+        #                                                       |
+        #                                                       |
+        #  process 2 (reindex_time + (latency + 1).seconds):    |  chewy:delayed_sidekiq:CitiesIndex:1679347866 = [1, 2, 3]
+        #    Schedule.new(CitiesIndex, [4]).postpone            |  chewy:delayed_sidekiq:CitiesIndex:1679347868 = [4]
+        #                                                       |  chewy:delayed_sidekiq:timechunks = [
+        #                                                       |    { score: 1679347866, "chewy:delayed_sidekiq:CitiesIndex:1679347866"}
+        #                                                       |    { score: 1679347868, "chewy:delayed_sidekiq:CitiesIndex:1679347868"}
+        #                                                       |  ]
+        #                                                       |  & schedule a DelayedSidekiq::Worker at 1679347871 (at + 3)
+        #                                                       |    it will zpop chewy:delayed_sidekiq:timechunks up to 1679347868 score and reindex all ids with zpoped keys
+        #                                                       |      chewy:delayed_sidekiq:CitiesIndex:1679347866 (in case of failed previous reindex),
+        #                                                       |      chewy:delayed_sidekiq:CitiesIndex:1679347868
+        def postpone
+          ::Sidekiq.redis do |redis|
+            # warning: Redis#sadd will always return an Integer in Redis 5.0.0. Use Redis#sadd? instead
+            if redis.respond_to?(:sadd?)
+              redis.sadd?(timechunk_key, serialize_data)
+            else
+              redis.sadd(timechunk_key, serialize_data)
+            end
+
+            redis.expire(timechunk_key, ttl)
+
+            unless redis.zrank(timechunks_key, timechunk_key)
+              redis.zadd(timechunks_key, at, timechunk_key)
+              redis.expire(timechunks_key, ttl)
+
+              ::Sidekiq::Client.push(
+                'queue' => sidekiq_queue,
+                'at' => at + margin,
+                'class' => Chewy::Strategy::DelayedSidekiq::Worker,
+                'args' => [type_name, at]
+              )
+            end
+          end
+        end
+
+      private
+
+        attr_reader :type, :ids, :options
+
+        # this method returns predictable value that jumps by latency value
+        # another words each latency seconds it return the same value
+        def at
+          @at ||= begin
+            schedule_at = latency.seconds.from_now.to_f
+
+            (schedule_at - (schedule_at % latency)).to_i
+          end
+        end
+
+        def fields
+          options[:update_fields].presence || [FALLBACK_FIELDS]
+        end
+
+        def timechunks_key
+          "#{KEY_PREFIX}:#{type_name}:timechunks"
+        end
+
+        def timechunk_key
+          "#{KEY_PREFIX}:#{type_name}:#{at}"
+        end
+
+        def serialize_data
+          [ids.join(IDS_SEPARATOR), fields.join(IDS_SEPARATOR)].join(FIELDS_IDS_SEPARATOR)
+        end
+
+        def type_name
+          type.name
+        end
+
+        def latency
+          strategy_config.latency || DEFAULT_LATENCY
+        end
+
+        def margin
+          strategy_config.margin || DEFAULT_MARGIN
+        end
+
+        def ttl
+          strategy_config.ttl || DEFAULT_TTL
+        end
+
+        def sidekiq_queue
+          Chewy.settings.dig(:sidekiq, :queue) || DEFAULT_QUEUE
+        end
+
+        def strategy_config
+          type.strategy_config.delayed_sidekiq
+        end
+      end
+    end
+  end
+end

data/lib/chewy/strategy/delayed_sidekiq/worker.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Chewy
+  class Strategy
+    class DelayedSidekiq
+      class Worker
+        include ::Sidekiq::Worker
+
+        def perform(type, score, options = {})
+          options[:refresh] = !Chewy.disable_refresh_async if Chewy.disable_refresh_async
+
+          ::Sidekiq.redis do |redis|
+            timechunks_key = "#{Scheduler::KEY_PREFIX}:#{type}:timechunks"
+            timechunk_keys = redis.zrangebyscore(timechunks_key, -1, score)
+            members = timechunk_keys.flat_map { |timechunk_key| redis.smembers(timechunk_key) }.compact
+
+            # extract ids and fields & do the reset of records
+            ids, fields = extract_ids_and_fields(members)
+            options[:update_fields] = fields if fields
+
+            index = type.constantize
+            index.strategy_config.delayed_sidekiq.reindex_wrapper.call do
+              options.any? ? index.import!(ids, **options) : index.import!(ids)
+            end
+
+            redis.del(timechunk_keys)
+            redis.zremrangebyscore(timechunks_key, -1, score)
+          end
+        end
+
+      private
+
+        def extract_ids_and_fields(members)
+          ids = []
+          fields = []
+
+          members.each do |member|
+            member_ids, member_fields = member.split(Scheduler::FIELDS_IDS_SEPARATOR).map do |v|
+              v.split(Scheduler::IDS_SEPARATOR)
+            end
+            ids |= member_ids
+            fields |= member_fields
+          end
+
+          fields = nil if fields.include?(Scheduler::FALLBACK_FIELDS)
+
+          [ids.map(&:to_i), fields]
+        end
+      end
+    end
+  end
+end

data/lib/chewy/strategy/delayed_sidekiq.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Chewy
+  class Strategy
+    class DelayedSidekiq < Sidekiq
+      require_relative 'delayed_sidekiq/scheduler'
+
+      def leave
+        @stash.each do |type, ids|
+          next if ids.empty?
+
+          DelayedSidekiq::Scheduler.new(type, ids).postpone
+        end
+      end
+    end
+  end
+end