chewy 0.8.4 → 5.1.0

data/lib/chewy/index.rb

@@ -2,6 +2,7 @@ require 'chewy/search'
 require 'chewy/index/actions'
 require 'chewy/index/aliases'
 require 'chewy/index/settings'
+require 'chewy/index/specification'
 
 module Chewy
   class Index
@@ -17,159 +18,259 @@ module Chewy
     class_attribute :_settings
     self._settings = Chewy::Index::Settings.new
 
-    # Setups or returns ElasticSearch index name
-    #
-    #   class UsersIndex < Chewy::Index
-    #   end
-    #   UsersIndex.index_name # => 'users'
-    #
-    #   class UsersIndex < Chewy::Index
-    #     index_name 'dudes'
-    #   end
-    #   UsersIndex.index_name # => 'dudes'
-    #
-    def self.index_name(suggest = nil)
-      if suggest
-        @index_name = build_index_name(suggest, prefix: default_prefix)
-      else
-        @index_name ||= begin
-          build_index_name(
-            name.sub(/Index\Z/, '').demodulize.underscore,
-            prefix: default_prefix
-          ) if name
+    class << self
+      # @overload index_name(suggest)
+      #   If suggested name is passed, it is set up as the new base name for
+      #   the index. Used for the index base name redefinition.
+      #
+      #   @example
+      #     class UsersIndex < Chewy::Index
+      #       index_name :legacy_users
+      #     end
+      #     UsersIndex.index_name # => 'legacy_users'
+      #
+      #   @param suggest [String, Symbol] suggested base name
+      #   @return [String] new base name
+      #
+      # @overload index_name(prefix: nil, suffix: nil)
+      #   If suggested name is not passed, returns the base name accompanied
+      #   with the prefix (if any) and suffix (if passed).
+      #
+      #   @example
+      #     class UsersIndex < Chewy::Index
+      #     end
+      #
+      #     Chewy.settings = {prefix: 'test'}
+      #     UsersIndex.index_name # => 'test_users'
+      #     UsersIndex.index_name(prefix: 'foobar') # => 'foobar_users'
+      #     UsersIndex.index_name(suffix: '2017') # => 'test_users_2017'
+      #     UsersIndex.index_name(prefix: '', suffix: '2017') # => 'users_2017'
+      #
+      #   @param prefix [String] index name prefix, uses {.prefix} method by default
+      #   @param suffix [String] index name suffix, used for creating several indexes for the same alias during the zero-downtime reset
+      #   @raise [UndefinedIndex] if the base name is blank
+      #   @return [String] result index name
+      def index_name(suggest = nil, prefix: nil, suffix: nil)
+        if suggest
+          @base_name = suggest.to_s.presence
+        else
+          [
+            prefix || prefix_with_deprecation,
+            base_name,
+            suffix
+          ].reject(&:blank?).join('_')
         end
       end
-      @index_name or raise UndefinedIndex
-    end
 
-    # Prefix to use
-    #
-    def self.default_prefix
-      Chewy.configuration[:prefix]
-    end
+      # Base name for the index. Uses the default value inferred from the
+      # class name unless redefined.
+      #
+      # @example
+      #   class Namespace::UsersIndex < Chewy::Index
+      #   end
+      #   UsersIndex.index_name # => 'users'
+      #
+      #   Class.new(Chewy::Index).base_name # => raises UndefinedIndex
+      #
+      # @raise [UndefinedIndex] when the base name is blank
+      # @return [String] current base name
+      def base_name
+        @base_name ||= name.sub(/Index\z/, '').demodulize.underscore if name
+        raise UndefinedIndex if @base_name.blank?
+        @base_name
+      end
 
-    # Defines type for the index. Arguments depends on adapter used. For
-    # ActiveRecord you can pass model or scope and options
-    #
-    #   class CarsIndex < Chewy::Index
-    #     define_type Car do
-    #       ...
-    #     end # defines VehiclesIndex::Car type
-    #   end
-    #
-    # Type name might be passed in complicated cases:
-    #
-    #   class VehiclesIndex < Chewy::Index
-    #     define_type Vehicle.cars.includes(:manufacturer), name: 'cars' do
-    #        ...
-    #     end # defines VehiclesIndex::Cars type
-    #
-    #     define_type Vehicle.motocycles.includes(:manufacturer), name: 'motocycles' do
-    #        ...
-    #     end # defines VehiclesIndex::Motocycles type
-    #   end
-    #
-    # For plain objects:
-    #
-    #   class PlanesIndex < Chewy::Index
-    #     define_type :plane do
-    #       ...
-    #     end # defines PlanesIndex::Plane type
-    #   end
-    #
-    # The main difference between using plain objects or ActiveRecord models for indexing
-    # is import. If you will call `CarsIndex::Car.import` - it will import all the cars
-    # automatically, while `PlanesIndex::Plane.import(my_planes)` requires import data to be
-    # passed.
-    #
-    def self.define_type(target, options = {}, &block)
-      type_class = Chewy.create_type(self, target, options, &block)
-      self.type_hash = type_hash.merge(type_class.type_name => type_class)
-
-      unless respond_to?(type_class.type_name)
-        class_eval <<-METHOD, __FILE__, __LINE__ + 1
-          def self.#{type_class.type_name}
-            ActiveSupport::Deprecation.warn("`#{self}.#{type_class.type_name}` accessor is deprecated and will be removed soon. Use `#{type_class}` directly instead.")
-            type_hash['#{type_class.type_name}']
-          end
-        METHOD
+      # Similar to the {.base_name} but respects the class namespace, also,
+      # can't be redefined. Used to reference index with the string identifier
+      #
+      # @example
+      #   class Namespace::UsersIndex < Chewy::Index
+      #   end
+      #   UsersIndex.derivable_name # => 'namespace/users'
+      #
+      #   Class.new(Chewy::Index).derivable_name # => nil
+      #
+      # @return [String, nil] derivable name or nil when it is impossible to calculate
+      def derivable_name
+        @derivable_name ||= name.sub(/Index\z/, '').underscore if name
       end
-    end
 
-    # Types method has double usage.
-    # If no arguments are passed - it returns array of defined types:
-    #
-    #   UsersIndex.types # => [UsersIndex::Admin, UsersIndex::Manager, UsersIndex::User]
-    #
-    # If arguments are passed it treats like a part of chainable query DSL and
-    # adds types array for index to select.
-    #
-    #   UsersIndex.filters { name =~ 'ro' }.types(:admin, :manager)
-    #   UsersIndex.types(:admin, :manager).filters { name =~ 'ro' } # the same as the first example
-    #
-    def self.types *args
-      if args.present?
-        all.types *args
-      else
-        type_hash.values
+      # Used as a default value for {.index_name}. Return prefix from the configuration
+      # but can be redefined per-index to be more dynamic.
+      #
+      # @example
+      #   class UsersIndex < Chewy::Index
+      #     def self.prefix
+      #       'foobar'
+      #     end
+      #   end
+      #   UsersIndex.index_name # => 'foobar_users'
+      #
+      # @return [String] prefix
+      def prefix
+        Chewy.configuration[:prefix]
       end
-    end
 
-    # Returns defined types names:
-    #
-    #   UsersIndex.type_names # => ['admin', 'manager', 'user']
-    #
-    def self.type_names
-      type_hash.keys
-    end
+      # Defines type for the index. Arguments depends on adapter used. For
+      # ActiveRecord you can pass model or scope and options
+      #
+      #   class CarsIndex < Chewy::Index
+      #     define_type Car do
+      #       ...
+      #     end # defines VehiclesIndex::Car type
+      #   end
+      #
+      # Type name might be passed in complicated cases:
+      #
+      #   class VehiclesIndex < Chewy::Index
+      #     define_type Vehicle.cars.includes(:manufacturer), name: 'cars' do
+      #        ...
+      #     end # defines VehiclesIndex::Cars type
+      #
+      #     define_type Vehicle.motocycles.includes(:manufacturer), name: 'motocycles' do
+      #        ...
+      #     end # defines VehiclesIndex::Motocycles type
+      #   end
+      #
+      # For plain objects:
+      #
+      #   class PlanesIndex < Chewy::Index
+      #     define_type :plane do
+      #       ...
+      #     end # defines PlanesIndex::Plane type
+      #   end
+      #
+      # The main difference between using plain objects or ActiveRecord models for indexing
+      # is import. If you will call `CarsIndex::Car.import` - it will import all the cars
+      # automatically, while `PlanesIndex::Plane.import(my_planes)` requires import data to be
+      # passed.
+      #
+      def define_type(target, options = {}, &block)
+        type_class = Chewy.create_type(self, target, options, &block)
+        self.type_hash = type_hash.merge(type_class.type_name => type_class)
+      end
 
-    # Used as a part of index definition DSL. Defines settings:
-    #
-    #   class UsersIndex < Chewy::Index
-    #     settings analysis: {
-    #       analyzer: {
-    #         name: {
-    #           tokenizer: 'standard',
-    #           filter: ['lowercase', 'icu_folding', 'names_nysiis']
-    #         }
-    #       }
-    #     }
-    #   end
-    #
-    # See http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/indices-update-settings.html
-    # for more details
-    #
-    # It is possible to store analyzers settings in Chewy repositories
-    # and link them form index class. See `Chewy::Index::Settings` for details.
-    #
-    def self.settings(params)
-      self._settings = Chewy::Index::Settings.new params
-    end
+      # Types method has double usage.
+      # If no arguments are passed - it returns array of defined types:
+      #
+      #   UsersIndex.types # => [UsersIndex::Admin, UsersIndex::Manager, UsersIndex::User]
+      #
+      # If arguments are passed it treats like a part of chainable query DSL and
+      # adds types array for index to select.
+      #
+      #   UsersIndex.filters { name =~ 'ro' }.types(:admin, :manager)
+      #   UsersIndex.types(:admin, :manager).filters { name =~ 'ro' } # the same as the first example
+      #
+      def types(*args)
+        if args.present?
+          all.types(*args)
+        else
+          type_hash.values
+        end
+      end
 
-    # Returns list of public class methods defined in current index
-    #
-    def self.scopes
-      public_methods - Chewy::Index.public_methods - type_names.map(&:to_sym)
-    end
+      # Returns defined types names:
+      #
+      #   UsersIndex.type_names # => ['admin', 'manager', 'user']
+      #
+      def type_names
+        type_hash.keys
+      end
 
-  private
+      # Returns named type:
+      #
+      #    UserIndex.type('admin') # => UsersIndex::Admin
+      #
+      def type(type_name)
+        type_hash.fetch(type_name) { raise UndefinedType, "Unknown type in #{name}: #{type_name}" }
+      end
 
-    def self.build_index_name *args
-      options = args.extract_options!
-      [options[:prefix], args.first || index_name, options[:suffix]].reject(&:blank?).join(?_)
-    end
+      # Used as a part of index definition DSL. Defines settings:
+      #
+      #   class UsersIndex < Chewy::Index
+      #     settings analysis: {
+      #       analyzer: {
+      #         name: {
+      #           tokenizer: 'standard',
+      #           filter: ['lowercase', 'icu_folding', 'names_nysiis']
+      #         }
+      #       }
+      #     }
+      #   end
+      #
+      # See http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/indices-update-settings.html
+      # for more details
+      #
+      # It is possible to store analyzers settings in Chewy repositories
+      # and link them form index class. See `Chewy::Index::Settings` for details.
+      #
+      def settings(params = {}, &block)
+        self._settings = Chewy::Index::Settings.new(params, &block)
+      end
 
-    def self.settings_hash
-      _settings.to_hash
-    end
+      # Returns list of public class methods defined in current index
+      #
+      def scopes
+        public_methods - Chewy::Index.public_methods
+      end
 
-    def self.mappings_hash
-      mappings = types.map(&:mappings_hash).inject(:merge)
-      mappings.present? ? {mappings: mappings} : {}
-    end
+      def settings_hash
+        _settings.to_hash
+      end
+
+      def mappings_hash
+        mappings = types.map(&:mappings_hash).inject(:merge)
+        mappings.present? ? {mappings: mappings} : {}
+      end
+
+      # Returns a hash containing the index settings and mappings
+      # Used for the ES index creation as body.
+      #
+      # @see Chewy::Index::Specification
+      # @return [Hash] specification as a hash
+      def specification_hash
+        [settings_hash, mappings_hash].inject(:merge)
+      end
+
+      def index_params
+        ActiveSupport::Deprecation.warn '`Chewy::Index.index_params` is deprecated and will be removed soon, use `Chewy::Index.specification_hash`'
+        specification_hash
+      end
+
+      # @see Chewy::Index::Specification
+      # @return [Chewy::Index::Specification] a specification object instance for this particular index
+      def specification
+        @specification ||= Specification.new(self)
+      end
 
-    def self.index_params
-      [settings_hash, mappings_hash].inject(:merge)
+      def derivable_index_name
+        ActiveSupport::Deprecation.warn '`Chewy::Index.derivable_index_name` is deprecated and will be removed soon, use `Chewy::Index.derivable_name` instead'
+        derivable_name
+      end
+
+      # Handling old default_prefix if it is not defined.
+      def method_missing(name, *args, &block) # rubocop:disable Style/MethodMissing
+        if name == :default_prefix
+          ActiveSupport::Deprecation.warn '`Chewy::Index.default_prefix` is deprecated and will be removed soon, use `Chewy::Index.prefix` instead'
+          prefix
+        else
+          super
+        end
+      end
+
+      def prefix_with_deprecation
+        if respond_to?(:default_prefix)
+          ActiveSupport::Deprecation.warn '`Chewy::Index.default_prefix` is deprecated and will be removed soon, define `Chewy::Index.prefix` method instead'
+          default_prefix
+        else
+          prefix
+        end
+      end
+
+      def build_index_name(*args)
+        ActiveSupport::Deprecation.warn '`Chewy::Index.build_index_name` is deprecated and will be removed soon, use `Chewy::Index.index_name` instead'
+        index_name(args.extract_options!)
+      end
     end
   end
 end

data/lib/chewy/journal.rb

@@ -0,0 +1,55 @@
+module Chewy
+  # A class to perform journal-related actions for the specified indexes/types.
+  #
+  # @example
+  #   journal = Chewy::Journal.new('places#city', UsersIndex)
+  #   journal.apply(20.minutes.ago)
+  #   journal.clean
+  #
+  class Journal
+    # @param only [Array<String, Chewy::Index, Chewy::Type>] indexes/types or even string references to perform actions on
+    def initialize(*only)
+      @only = only
+    end
+
+    # Applies all changes that were done since the specified time to the
+    # specified indexes/types.
+    #
+    # @param since_time [Time, DateTime] timestamp from which changes will be applied
+    # @param retries [Integer] maximum number of attempts to make journal empty, 10 by default
+    # @return [Integer] the amount of journal entries found
+    def apply(since_time, retries: 10, **import_options)
+      stage = 1
+      since_time -= 1
+      count = 0
+      while stage <= retries
+        entries = Chewy::Stash::Journal.entries(since_time, only: @only).to_a.presence or break
+        count += entries.size
+        groups = reference_groups(entries)
+        ActiveSupport::Notifications.instrument 'apply_journal.chewy', stage: stage, groups: groups
+        groups.each do |type, references|
+          type.import(references, import_options.merge(journal: false))
+        end
+        stage += 1
+        since_time = entries.map(&:created_at).max
+      end
+      count
+    end
+
+    # Cleans journal for the specified indexes/types.
+    #
+    # @param until_time [Time, DateTime] time to clean up until it
+    # @return [Hash] delete_by_query ES API call result
+    def clean(until_time = nil)
+      Chewy::Stash::Journal.clean(until_time, only: @only)
+    end
+
+  private
+
+    def reference_groups(entries)
+      entries.group_by(&:type).map do |type, grouped_entries|
+        [type, grouped_entries.map(&:references).inject(:|)]
+      end.to_h
+    end
+  end
+end

data/lib/chewy/log_subscriber.rb

@@ -16,17 +16,17 @@ module Chewy
       render_action('Delete by Query', event) { |payload| payload[:request] }
     end
 
-    def render_action action, event, &block
+    def render_action(action, event)
       payload = event.payload
-      description = block.call(payload)
+      description = yield(payload)
 
-      if description.present?
-        subject = payload[:type].presence || payload[:index]
-        action = "#{subject} #{action} (#{event.duration.round(1)}ms)"
-        action = color(action, GREEN, true)
+      return if description.blank?
 
-        debug("  #{action} #{description}")
-      end
+      subject = payload[:type].presence || payload[:index]
+      action = "#{subject} #{action} (#{event.duration.round(1)}ms)"
+      action = color(action, GREEN, true)
+
+      debug("  #{action} #{description}")
     end
   end
 end

data/lib/chewy/minitest/helpers.rb

@@ -0,0 +1,77 @@
+require_relative 'search_index_receiver'
+
+module Chewy
+  module Minitest
+    module Helpers
+      extend ActiveSupport::Concern
+
+      # Assert that an index *changes* during a block.
+      # @param index [Chewy::Type] the index / type to watch, eg EntitiesIndex::Entity.
+      # @param strategy [Symbol] the Chewy strategy to use around the block. See Chewy docs.
+      # @param bypass_actual_index [true, false]
+      #   True to preempt the http call to Elastic, false otherwise.
+      #   Should be set to true unless actually testing search functionality.
+      #
+      # @return [SearchIndexReceiver] for optional further assertions on the nature of the index changes.
+      #
+      def assert_indexes(index, strategy: :atomic, bypass_actual_index: true)
+        type = Chewy.derive_type index
+        receiver = SearchIndexReceiver.new
+
+        bulk_method = type.method :bulk
+        # Manually mocking #bulk because we need to properly capture `self`
+        bulk_mock = lambda do |*bulk_args|
+          receiver.catch bulk_args, self
+
+          bulk_method.call(*bulk_args) unless bypass_actual_index
+
+          {}
+        end
+
+        type.define_singleton_method :bulk, bulk_mock
+
+        Chewy.strategy(strategy) do
+          yield
+        end
+
+        type.define_singleton_method :bulk, bulk_method
+
+        assert_includes receiver.updated_indexes, index, "Expected #{index} to be updated but it wasn't"
+
+        receiver
+      end
+
+      # Run indexing for the database changes during the block provided.
+      # By default, indexing is run at the end of the block.
+      # @param strategy [Symbol] the Chewy index update strategy see Chewy docs.
+      def run_indexing(strategy: :atomic)
+        Chewy.strategy strategy do
+          yield
+        end
+      end
+
+      module ClassMethods
+        # Declare that all tests in this file require real indexing, always.
+        # In my completely unscientific experiments, this roughly doubled test runtime.
+        # Use with trepidation.
+        def index_everything!
+          setup do
+            Chewy.strategy :urgent
+          end
+
+          teardown do
+            Chewy.strategy.pop
+          end
+        end
+      end
+
+      included do
+        teardown do
+          # always destroy indexes between tests
+          # Prevent croll pollution of test cases due to indexing
+          Chewy.massacre
+        end
+      end
+    end
+  end
+end

data/lib/chewy/minitest/search_index_receiver.rb

@@ -0,0 +1,80 @@
+# Test helper class to provide minitest hooks for Chewy::Index testing.
+#
+# @note Intended to be used in conjunction with a test helper which mocks over the #bulk
+#   method on a {Chewy::Type} class. (See SearchTestHelper)
+#
+# The class will capture the data from the *param on the Chewy::Type.bulk method and
+# aggregate the data for test analysis.
+class SearchIndexReceiver
+  def initialize
+    @mutations = {}
+  end
+
+  # @param bulk_params [Hash] the bulk_params that should be sent to the Chewy::Type.bulk method.
+  # @param type [Chewy::Type] the type executing this query.
+  def catch(bulk_params, type)
+    Array.wrap(bulk_params).map { |y| y[:body] }.flatten.each do |update|
+      if update[:delete]
+        mutation_for(type).deletes << update[:delete][:_id]
+      elsif update[:index]
+        mutation_for(type).indexes << update[:index]
+      end
+    end
+  end
+
+  # @param index [Chewy::Index] return only index requests to the specified {Chewy::Type} index.
+  # @return [Hash] the index changes captured by the mock.
+  def indexes_for(index = nil)
+    if index
+      mutation_for(index).indexes
+    else
+      Hash[
+        @mutations.map { |a, b| [a, b.indexes] }
+      ]
+    end
+  end
+  alias_method :indexes, :indexes_for
+
+  # @param index [Chewy::Index] return only delete requests to the specified {Chewy::Type} index.
+  # @return [Hash] the index deletes captured by the mock.
+  def deletes_for(index = nil)
+    if index
+      mutation_for(index).deletes
+    else
+      Hash[
+        @mutations.map { |a, b| [a, b.deletes] }
+      ]
+    end
+  end
+  alias_method :deletes, :deletes_for
+
+  # Check to see if a given object has been indexed.
+  # @param obj [#id] obj the object to look for.
+  # @param type [Chewy::Type] what type the object should be indexed as.
+  # @return [true, false] if the object was indexed.
+  def indexed?(obj, type)
+    indexes_for(type).map { |i| i[:_id] }.include? obj.id
+  end
+
+  # Check to see if a given object has been deleted.
+  # @param obj [#id] obj the object to look for.
+  # @param type [Chewy::Type] what type the object should have been deleted from.
+  # @return [true, false] if the object was deleted.
+  def deleted?(obj, type)
+    deletes_for(type).include? obj.id
+  end
+
+  # @return [Array<Chewy::Type>] a list of types indexes changed.
+  def updated_indexes
+    @mutations.keys
+  end
+
+private
+
+  # Get the mutation object for a given type.
+  # @param type [Chewy::Type] the index type to fetch.
+  # @return [#indexes, #deletes] an object with a list of indexes and a list of deletes.
+  def mutation_for(type)
+    @mutations[type] ||= OpenStruct.new(indexes: [], deletes: [])
+  end
+end

data/lib/chewy/minitest.rb

@@ -0,0 +1 @@
+require 'chewy/minitest/helpers'