chewy 0.9.0 → 5.2.0

data/lib/chewy/query/nodes/query.rb

@@ -9,9 +9,9 @@ module Chewy
 
         def __render__
           if @options.key?(:cache)
-            { fquery: { query: @query, _cache: !!@options[:cache] } }
+            {fquery: {query: @query, _cache: !!@options[:cache]}}
           else
-            { query: @query }
+            {query: @query}
           end
         end
       end

data/lib/chewy/query/nodes/range.rb

@@ -12,8 +12,8 @@ module Chewy
         def initialize(name, *args)
           @name = name.to_s
           @options = args.extract_options!
-          @range = @options.reject { |k, _v| ![:gt, :lt].include?(k) }
-          @bounds = @options.reject { |k, _v| ![:left_closed, :right_closed].include?(k) }
+          @range = @options.select { |k, _v| %i[gt lt].include?(k) }
+          @bounds = @options.select { |k, _v| %i[left_closed right_closed].include?(k) }
           execution = EXECUTION[args.first.to_sym] if args.first
           @options[:execution] = execution if execution
         end
@@ -51,11 +51,11 @@ module Chewy
           body[@bounds[:left_closed] ? :gte : :gt] = @range[:gt] if @range.key?(:gt)
           body[@bounds[:right_closed] ? :lte : :lt] = @range[:lt] if @range.key?(:lt)
 
-          filter = { @name => body }
+          filter = {@name => body}
           filter[:_cache] = !!@options[:cache] if @options.key?(:cache)
           filter.merge!(@options.slice(:execution))
 
-          { range: filter }
+          {range: filter}
         end
       end
     end

data/lib/chewy/query/nodes/regexp.rb

@@ -2,7 +2,7 @@ module Chewy
   class Query
     module Nodes
       class Regexp < Expr
-        FLAGS = %w(all anystring automaton complement empty intersection interval none).freeze
+        FLAGS = %w[all anystring automaton complement empty intersection interval none].freeze
 
         def initialize(name, regexp, *args)
           @name = name.to_s
@@ -14,11 +14,11 @@ module Chewy
 
         def __render__
           body = if @options[:flags]
-            { value: @regexp, flags: @options[:flags].map(&:to_s).map(&:upcase).uniq.join('|') }
+            {value: @regexp, flags: @options[:flags].map(&:to_s).map(&:upcase).uniq.join('|')}
           else
             @regexp
           end
-          filter = { @name => body }
+          filter = {@name => body}
           if @options.key?(:cache)
             filter[:_cache] = !!@options[:cache]
             filter[:_cache_key] = if @options[:cache].is_a?(TrueClass) || @options[:cache].is_a?(FalseClass)
@@ -27,7 +27,7 @@ module Chewy
               @options[:cache]
             end
           end
-          { regexp: filter }
+          {regexp: filter}
         end
       end
     end

data/lib/chewy/query/nodes/script.rb

@@ -5,14 +5,14 @@ module Chewy
         def initialize(script, params = {})
           @script = script
           @params = params
-          @options = params.reject { |k, _v| ![:cache].include?(k) }
+          @options = params.select { |k, _v| [:cache].include?(k) }
         end
 
         def __render__
-          script = { script: @script }
+          script = {script: @script}
           script[:params] = @params if @params.present?
           script[:_cache] = !!@options[:cache] if @options.key?(:cache)
-          { script: script }
+          {script: script}
         end
       end
     end

data/lib/chewy/query/pagination.rb

@@ -8,9 +8,18 @@ module Chewy
       def total
         _response['hits'].try(:[], 'total') || 0
       end
-
       alias_method :total_count, :total
       alias_method :total_entries, :total
+
+    private
+
+      def raw_limit_value
+        criteria.request_options[:size]
+      end
+
+      def raw_offset_value
+        criteria.request_options[:from]
+      end
     end
   end
 end

data/lib/chewy/railtie.rb

@@ -14,6 +14,8 @@ module Chewy
         if Rails.application.config.respond_to?(:assets) && env['PATH_INFO'].start_with?(Rails.application.config.assets.prefix)
           @app.call(env)
         else
+          Chewy.logger.info("Chewy request strategy is `#{Chewy.request_strategy}`") if Chewy.logger && @request_strategy != Chewy.request_strategy
+          @request_strategy = Chewy.request_strategy
           Chewy.strategy(Chewy.request_strategy) { @app.call(env) }
         end
       end
@@ -41,13 +43,12 @@ module Chewy
     end
 
     console do |app|
-      Chewy.logger = ActiveRecord::Base.logger if defined?(ActiveRecord)
-
       if app.sandbox?
         Chewy.strategy(:bypass)
       else
         Chewy.strategy(:urgent)
       end
+      puts "Chewy console strategy is `#{Chewy.strategy.current.name}`"
     end
 
     initializer 'chewy.logger', after: 'active_record.logger' do
@@ -67,7 +68,7 @@ module Chewy
     end
 
     initializer 'chewy.request_strategy' do |app|
-      app.config.middleware.insert_after(Rails::Rack::Logger, RequestStrategy)
+      app.config.middleware.insert_before(ActionDispatch::ShowExceptions, RequestStrategy)
     end
 
     initializer 'chewy.add_indices_path' do |_app|

data/lib/chewy/rake_helper.rb

@@ -1,85 +1,302 @@
 module Chewy
   module RakeHelper
+    IMPORT_CALLBACK = lambda do |output, _name, start, finish, _id, payload|
+      duration = (finish - start).ceil
+      stats = payload.fetch(:import, {}).map { |key, count| "#{key} #{count}" }.join(', ')
+      output.puts "  Imported #{payload[:type]} in #{human_duration(duration)}, stats: #{stats}"
+      if payload[:errors]
+        payload[:errors].each do |action, errors|
+          output.puts "    #{action.to_s.humanize} errors:"
+          errors.each do |error, documents|
+            output.puts "      `#{error}`"
+            output.puts "        on #{documents.count} documents: #{documents}"
+          end
+        end
+      end
+    end
+
+    JOURNAL_CALLBACK = lambda do |output, _, _, _, _, payload|
+      count = payload[:groups].values.map(&:size).sum
+      targets = payload[:groups].keys.sort_by(&:derivable_name)
+      output.puts "  Applying journal to #{targets}, #{count} entries, stage #{payload[:stage]}"
+    end
+
     class << self
-      def subscribed_task_stats
-        import_callback = lambda do |_name, start, finish, _id, payload|
-          duration = (finish - start).round(2)
-          puts "  Imported #{payload[:type]} for #{duration}s, documents total: #{payload[:import].try(:[], :index).to_i}"
-          if payload[:errors]
-            payload[:errors].each do |action, errors|
-              puts "    #{action.to_s.humanize} errors:"
-              errors.each do |error, documents|
-                puts "      `#{error}`"
-                puts "        on #{documents.count} documents: #{documents}"
+      # Performs zero-downtime reindexing of all documents for the specified indexes
+      #
+      # @example
+      #   Chewy::RakeHelper.reset # resets everything
+      #   Chewy::RakeHelper.reset(only: 'cities') # resets only CitiesIndex
+      #   Chewy::RakeHelper.reset(only: ['cities', CountriesIndex]) # resets CitiesIndex and CountriesIndex
+      #   Chewy::RakeHelper.reset(except: CitiesIndex) # resets everything, but CitiesIndex
+      #   Chewy::RakeHelper.reset(only: ['cities', 'countries'], except: CitiesIndex) # resets only CountriesIndex
+      #
+      # @param only [Array<Chewy::Index, String>, Chewy::Index, String] index or indexes to reset; if nothing is passed - uses all the indexes defined in the app
+      # @param except [Array<Chewy::Index, String>, Chewy::Index, String] index or indexes to exclude from processing
+      # @param parallel [true, Integer, Hash] any acceptable parallel options for import
+      # @param output [IO] output io for logging
+      # @return [Array<Chewy::Index>] indexes that were reset
+      def reset(only: nil, except: nil, parallel: nil, output: STDOUT)
+        subscribed_task_stats(output) do
+          indexes_from(only: only, except: except).each do |index|
+            reset_one(index, output, parallel: parallel)
+          end
+        end
+      end
+
+      # Performs zero-downtime reindexing of all documents for the specified
+      # indexes only if a particular index specification was changed.
+      #
+      # @example
+      #   Chewy::RakeHelper.upgrade # resets everything
+      #   Chewy::RakeHelper.upgrade(only: 'cities') # resets only CitiesIndex
+      #   Chewy::RakeHelper.upgrade(only: ['cities', CountriesIndex]) # resets CitiesIndex and CountriesIndex
+      #   Chewy::RakeHelper.upgrade(except: CitiesIndex) # resets everything, but CitiesIndex
+      #   Chewy::RakeHelper.upgrade(only: ['cities', 'countries'], except: CitiesIndex) # resets only CountriesIndex
+      #
+      # @param only [Array<Chewy::Index, String>, Chewy::Index, String] index or indexes to reset; if nothing is passed - uses all the indexes defined in the app
+      # @param except [Array<Chewy::Index, String>, Chewy::Index, String] index or indexes to exclude from processing
+      # @param parallel [true, Integer, Hash] any acceptable parallel options for import
+      # @param output [IO] output io for logging
+      # @return [Array<Chewy::Index>] indexes that were actually reset
+      def upgrade(only: nil, except: nil, parallel: nil, output: STDOUT)
+        subscribed_task_stats(output) do
+          indexes = indexes_from(only: only, except: except)
+
+          changed_indexes = indexes.select do |index|
+            index.specification.changed?
+          end
+
+          if changed_indexes.present?
+            indexes.each do |index|
+              if changed_indexes.include?(index)
+                reset_one(index, output, parallel: parallel)
+              else
+                output.puts "Skipping #{index}, the specification didn't change"
               end
             end
+          else
+            output.puts 'No index specification was changed'
           end
+
+          changed_indexes
         end
-        journal_callback = lambda do |_, _, _, _, payload|
-          puts "Applying journal. Stage #{payload[:stage]}"
+      end
+
+      # Performs full update for each passed type if the corresponding index exists.
+      #
+      # @example
+      #   Chewy::RakeHelper.update # updates everything
+      #   Chewy::RakeHelper.update(only: 'places') # updates only PlacesIndex::City and PlacesIndex::Country
+      #   Chewy::RakeHelper.update(only: 'places#city') # updates PlacesIndex::City only
+      #   Chewy::RakeHelper.update(except: PlacesIndex::Country) # updates everything, but PlacesIndex::Country
+      #   Chewy::RakeHelper.update(only: 'places', except: 'places#country') # updates PlacesIndex::City only
+      #
+      # @param only [Array<Chewy::Index, Chewy::Type, String>, Chewy::Index, Chewy::Type, String] indexes or types to update; if nothing is passed - uses all the types defined in the app
+      # @param except [Array<Chewy::Index, Chewy::Type, String>, Chewy::Index, Chewy::Type, String] indexes or types to exclude from processing
+      # @param parallel [true, Integer, Hash] any acceptable parallel options for import
+      # @param output [IO] output io for logging
+      # @return [Array<Chewy::Type>] types that were actually updated
+      def update(only: nil, except: nil, parallel: nil, output: STDOUT)
+        subscribed_task_stats(output) do
+          types_from(only: only, except: except).group_by(&:index).each_with_object([]) do |(index, types), update_types|
+            if index.exists?
+              output.puts "Updating #{index}"
+              types.each { |type| type.import(parallel: parallel) }
+              update_types.concat(types)
+            else
+              output.puts "Skipping #{index}, it does not exists (use rake chewy:reset[#{index.derivable_name}] to create and update it)"
+            end
+          end
         end
-        ActiveSupport::Notifications.subscribed(journal_callback, 'apply_journal.chewy') do
-          ActiveSupport::Notifications.subscribed(import_callback, 'import_objects.chewy') do
-            yield
+      end
+
+      # Performs synchronization for each passed index if it exists.
+      #
+      # @example
+      #   Chewy::RakeHelper.sync # synchronizes everything
+      #   Chewy::RakeHelper.sync(only: 'places') # synchronizes only PlacesIndex::City and PlacesIndex::Country
+      #   Chewy::RakeHelper.sync(only: 'places#city') # synchronizes PlacesIndex::City only
+      #   Chewy::RakeHelper.sync(except: PlacesIndex::Country) # synchronizes everything, but PlacesIndex::Country
+      #   Chewy::RakeHelper.sync(only: 'places', except: 'places#country') # synchronizes PlacesIndex::City only
+      #
+      # @param only [Array<Chewy::Index, Chewy::Type, String>, Chewy::Index, Chewy::Type, String] indexes or types to synchronize; if nothing is passed - uses all the types defined in the app
+      # @param except [Array<Chewy::Index, Chewy::Type, String>, Chewy::Index, Chewy::Type, String] indexes or types to exclude from processing
+      # @param parallel [true, Integer, Hash] any acceptable parallel options for sync
+      # @param output [IO] output io for logging
+      # @return [Array<Chewy::Type>] types that were actually updated
+      def sync(only: nil, except: nil, parallel: nil, output: STDOUT)
+        subscribed_task_stats(output) do
+          types_from(only: only, except: except).each_with_object([]) do |type, synced_types|
+            output.puts "Synchronizing #{type}"
+            output.puts "  #{type} doesn't support outdated synchronization" unless type.supports_outdated_sync?
+            time = Time.now
+            sync_result = type.sync(parallel: parallel)
+            if !sync_result
+              output.puts "  Something went wrong with the #{type} synchronization"
+            elsif sync_result[:count] > 0
+              output.puts "  Missing documents: #{sync_result[:missing]}" if sync_result[:missing].present?
+              output.puts "  Outdated documents: #{sync_result[:outdated]}" if sync_result[:outdated].present?
+              synced_types.push(type)
+            else
+              output.puts "  Skipping #{type}, up to date"
+            end
+            output.puts "  Took #{human_duration(Time.now - time)}"
           end
         end
       end
 
-      def eager_load_chewy!
-        dirs = Chewy::Railtie.all_engines.map { |engine| engine.paths[Chewy.configuration[:indices_path]] }.compact.map(&:existent).flatten.uniq
+      # Applies changes that were done after the specified time for the
+      # specified indexes/types or all of them.
+      #
+      # @example
+      #   Chewy::RakeHelper.journal_apply(time: 1.minute.ago) # applies entries created for the last minute
+      #   Chewy::RakeHelper.journal_apply(time: 1.minute.ago, only: 'places') # applies only PlacesIndex::City and PlacesIndex::Country entries reated for the last minute
+      #   Chewy::RakeHelper.journal_apply(time: 1.minute.ago, only: 'places#city') # applies PlacesIndex::City entries reated for the last minute only
+      #   Chewy::RakeHelper.journal_apply(time: 1.minute.ago, except: PlacesIndex::Country) # applies everything, but PlacesIndex::Country entries reated for the last minute
+      #   Chewy::RakeHelper.journal_apply(time: 1.minute.ago, only: 'places', except: 'places#country') # applies PlacesIndex::City entries reated for the last minute only
+      #
+      # @param time [Time, DateTime] use only journal entries created after this time
+      # @param only [Array<Chewy::Index, Chewy::Type, String>, Chewy::Index, Chewy::Type, String] indexes or types to synchronize; if nothing is passed - uses all the types defined in the app
+      # @param except [Array<Chewy::Index, Chewy::Type, String>, Chewy::Index, Chewy::Type, String] indexes or types to exclude from processing
+      # @param output [IO] output io for logging
+      # @return [Array<Chewy::Type>] types that were actually updated
+      def journal_apply(time: nil, only: nil, except: nil, output: STDOUT)
+        raise ArgumentError, 'Please specify the time to start with' unless time
+        subscribed_task_stats(output) do
+          output.puts "Applying journal entries created after #{time}"
+          count = Chewy::Journal.new(types_from(only: only, except: except)).apply(time)
+          output.puts 'No journal entries were created after the specified time' if count.zero?
+        end
+      end
 
-        dirs.each do |dir|
-          Dir.glob(File.join(dir, '**/*.rb')).each do |file|
-            require_dependency file
-          end
+      # Removes journal records created before the specified timestamp for
+      # the specified indexes/types or all of them.
+      #
+      # @example
+      #   Chewy::RakeHelper.journal_clean # cleans everything
+      #   Chewy::RakeHelper.journal_clean(time: 1.minute.ago) # leaves only entries created for the last minute
+      #   Chewy::RakeHelper.journal_clean(only: 'places') # cleans only PlacesIndex::City and PlacesIndex::Country entries
+      #   Chewy::RakeHelper.journal_clean(only: 'places#city') # cleans PlacesIndex::City entries only
+      #   Chewy::RakeHelper.journal_clean(except: PlacesIndex::Country) # cleans everything, but PlacesIndex::Country entries
+      #   Chewy::RakeHelper.journal_clean(only: 'places', except: 'places#country') # cleans PlacesIndex::City entries only
+      #
+      # @param time [Time, DateTime] clean all the journal entries created before this time
+      # @param only [Array<Chewy::Index, Chewy::Type, String>, Chewy::Index, Chewy::Type, String] indexes or types to synchronize; if nothing is passed - uses all the types defined in the app
+      # @param except [Array<Chewy::Index, Chewy::Type, String>, Chewy::Index, Chewy::Type, String] indexes or types to exclude from processing
+      # @param output [IO] output io for logging
+      # @return [Array<Chewy::Type>] types that were actually updated
+      def journal_clean(time: nil, only: nil, except: nil, output: STDOUT)
+        subscribed_task_stats(output) do
+          output.puts "Cleaning journal entries created before #{time}" if time
+          response = Chewy::Journal.new(types_from(only: only, except: except)).clean(time)
+          count = response['deleted'] || response['_indices']['_all']['deleted']
+          output.puts "Cleaned up #{count} journal entries"
         end
       end
 
+      # Eager loads and returns all the indexes defined in the application
+      # except Chewy::Stash::Specification and Chewy::Stash::Journal.
+      #
+      # @return [Array<Chewy::Index>] indexes found
       def all_indexes
-        eager_load_chewy!
-        Chewy::Index.descendants
+        Chewy.eager_load!
+        Chewy::Index.descendants - [Chewy::Stash::Journal, Chewy::Stash::Specification]
       end
 
-      def normalize_index(index)
-        return index if index.is_a?(Chewy::Index)
-        "#{index.to_s.gsub(/index\z/i, '').camelize}Index".constantize
+      def normalize_indexes(*identifiers)
+        identifiers.flatten(1).map { |identifier| normalize_index(identifier) }
       end
 
-      def normalize_indexes(*indexes)
-        indexes.flatten.map { |index| normalize_index(index) }
+      def normalize_index(identifier)
+        return identifier if identifier.is_a?(Class) && identifier < Chewy::Index
+        "#{identifier.to_s.gsub(/identifier\z/i, '').camelize}Index".constantize
       end
 
-      # Performs zero downtime reindexing of all documents in the specified index.
-      def reset_index(*indexes)
-        normalize_indexes(indexes).each do |index|
-          puts "Resetting #{index}"
-          time = Time.now
-          index.reset!((time.to_f * 1000).round)
-          if index.journal?
-            Chewy::Journal.create
-            Chewy::Journal::Apply.since(time, only: [index])
+      def subscribed_task_stats(output = STDOUT)
+        start = Time.now
+        ActiveSupport::Notifications.subscribed(JOURNAL_CALLBACK.curry[output], 'apply_journal.chewy') do
+          ActiveSupport::Notifications.subscribed(IMPORT_CALLBACK.curry[output], 'import_objects.chewy') do
+            yield
           end
         end
+        output.puts "Total: #{human_duration(Time.now - start)}"
+      end
+
+      def reset_index(*indexes)
+        ActiveSupport::Deprecation.warn '`Chewy::RakeHelper.reset_index` is deprecated and will be removed soon, use `Chewy::RakeHelper.reset` instead'
+        reset(only: indexes)
       end
 
-      # Performs zero downtime reindexing of all documents across all indices.
       def reset_all(*except)
-        reset_index(all_indexes - normalize_indexes(except))
+        ActiveSupport::Deprecation.warn '`Chewy::RakeHelper.reset_all` is deprecated and will be removed soon, use `Chewy::RakeHelper.reset` instead'
+        reset(except: except)
       end
 
       def update_index(*indexes)
-        normalize_indexes(indexes).each do |index|
-          puts "Updating #{index}"
-          if index.exists?
-            index.import
-          else
-            puts "Index `#{index.index_name}` does not exists. Use rake chewy:reset[#{index.index_name}] to create and update it."
-          end
-        end
+        ActiveSupport::Deprecation.warn '`Chewy::RakeHelper.update_index` is deprecated and will be removed soon, use `Chewy::RakeHelper.update` instead'
+        update(only: indexes)
       end
 
       def update_all(*except)
-        update_index(all_indexes - normalize_indexes(except))
+        ActiveSupport::Deprecation.warn '`Chewy::RakeHelper.update_all` is deprecated and will be removed soon, use `Chewy::RakeHelper.update` instead'
+        update(except: except)
+      end
+
+    private
+
+      def indexes_from(only: nil, except: nil)
+        indexes = if only.present?
+          normalize_indexes(Array.wrap(only))
+        else
+          all_indexes
+        end
+
+        indexes = if except.present?
+          indexes - normalize_indexes(Array.wrap(except))
+        else
+          indexes
+        end
+
+        indexes.sort_by(&:derivable_name)
+      end
+
+      def types_from(only: nil, except: nil)
+        types = if only.present?
+          normalize_types(Array.wrap(only))
+        else
+          all_indexes.flat_map(&:types)
+        end
+
+        types = if except.present?
+          types - normalize_types(Array.wrap(except))
+        else
+          types
+        end
+
+        types.sort_by(&:derivable_name)
+      end
+
+      def normalize_types(*identifiers)
+        identifiers.flatten(1).flat_map { |identifier| normalize_type(identifier) }
+      end
+
+      def normalize_type(identifier)
+        Chewy.derive_types(identifier)
+      end
+
+      def human_duration(seconds)
+        [[60, :s], [60, :m], [24, :h]].map do |amount, unit|
+          if seconds > 0
+            seconds, n = seconds.divmod(amount)
+            "#{n.to_i}#{unit}"
+          end
+        end.compact.reverse.join(' ')
+      end
+
+      def reset_one(index, output, parallel: false)
+        output.puts "Resetting #{index}"
+        index.reset!((Time.now.to_f * 1000).round, parallel: parallel)
       end
     end
   end