chewy 0.8.4 → 7.3.4

data/lib/chewy/index.rb

@@ -1,175 +1,251 @@
 require 'chewy/search'
 require 'chewy/index/actions'
+require 'chewy/index/adapter/active_record'
+require 'chewy/index/adapter/object'
 require 'chewy/index/aliases'
+require 'chewy/index/crutch'
+require 'chewy/index/import'
+require 'chewy/index/mapping'
+require 'chewy/index/observe'
 require 'chewy/index/settings'
+require 'chewy/index/specification'
+require 'chewy/index/syncer'
+require 'chewy/index/witchcraft'
+require 'chewy/index/wrapper'
 
 module Chewy
   class Index
+    IMPORT_OPTIONS_KEYS = %i[
+      batch_size bulk_size consistency direct_import journal
+      pipeline raw_import refresh replication
+    ].freeze
+
+    STRATEGY_OPTIONS = {
+      delayed_sidekiq: %i[latency margin ttl reindex_wrapper]
+    }.freeze
+
     include Search
     include Actions
     include Aliases
+    include Import
+    include Mapping
+    include Observe
+    include Crutch
+    include Witchcraft
+    include Wrapper
 
     singleton_class.delegate :client, to: 'Chewy'
 
-    class_attribute :type_hash
-    self.type_hash = {}
+    class_attribute :adapter
+    self.adapter = Chewy::Index::Adapter::Object.new(:default)
+
+    class_attribute :index_scope_defined
 
     class_attribute :_settings
     self._settings = Chewy::Index::Settings.new
 
-    # Setups or returns ElasticSearch index name
-    #
-    #   class UsersIndex < Chewy::Index
-    #   end
-    #   UsersIndex.index_name # => 'users'
-    #
-    #   class UsersIndex < Chewy::Index
-    #     index_name 'dudes'
-    #   end
-    #   UsersIndex.index_name # => 'dudes'
-    #
-    def self.index_name(suggest = nil)
-      if suggest
-        @index_name = build_index_name(suggest, prefix: default_prefix)
-      else
-        @index_name ||= begin
-          build_index_name(
-            name.sub(/Index\Z/, '').demodulize.underscore,
-            prefix: default_prefix
-          ) if name
+    class_attribute :_default_import_options
+    self._default_import_options = {}
+
+    class << self
+      # @overload index_name(suggest)
+      #   If suggested name is passed, it is set up as the new base name for
+      #   the index. Used for the index base name redefinition.
+      #
+      #   @example
+      #     class UsersIndex < Chewy::Index
+      #       index_name :legacy_users
+      #     end
+      #     UsersIndex.index_name # => 'legacy_users'
+      #
+      #   @param suggest [String, Symbol] suggested base name
+      #   @return [String] new base name
+      #
+      # @overload index_name(prefix: nil, suffix: nil)
+      #   If suggested name is not passed, returns the base name accompanied
+      #   with the prefix (if any) and suffix (if passed).
+      #
+      #   @example
+      #     class UsersIndex < Chewy::Index
+      #     end
+      #
+      #     Chewy.settings = {prefix: 'test'}
+      #     UsersIndex.index_name # => 'test_users'
+      #     UsersIndex.index_name(prefix: 'foobar') # => 'foobar_users'
+      #     UsersIndex.index_name(suffix: '2017') # => 'test_users_2017'
+      #     UsersIndex.index_name(prefix: '', suffix: '2017') # => 'users_2017'
+      #
+      #   @param prefix [String] index name prefix, uses {.prefix} method by default
+      #   @param suffix [String] index name suffix, used for creating several indexes for the same
+      #     alias during the zero-downtime reset
+      #   @raise [UndefinedIndex] if the base name is blank
+      #   @return [String] result index name
+      def index_name(suggest = nil, prefix: nil, suffix: nil)
+        if suggest
+          @base_name = suggest.to_s.presence
+        else
+          [
+            prefix || self.prefix,
+            base_name,
+            suffix
+          ].reject(&:blank?).join('_')
         end
       end
-      @index_name or raise UndefinedIndex
-    end
 
-    # Prefix to use
-    #
-    def self.default_prefix
-      Chewy.configuration[:prefix]
-    end
+      # Base name for the index. Uses the default value inferred from the
+      # class name unless redefined.
+      #
+      # @example
+      #   class Namespace::UsersIndex < Chewy::Index
+      #   end
+      #   UsersIndex.index_name # => 'users'
+      #
+      #   Class.new(Chewy::Index).base_name # => raises UndefinedIndex
+      #
+      # @raise [UndefinedIndex] when the base name is blank
+      # @return [String] current base name
+      def base_name
+        @base_name ||= name.sub(/Index\z/, '').demodulize.underscore if name
+        raise UndefinedIndex if @base_name.blank?
+
+        @base_name
+      end
 
-    # Defines type for the index. Arguments depends on adapter used. For
-    # ActiveRecord you can pass model or scope and options
-    #
-    #   class CarsIndex < Chewy::Index
-    #     define_type Car do
-    #       ...
-    #     end # defines VehiclesIndex::Car type
-    #   end
-    #
-    # Type name might be passed in complicated cases:
-    #
-    #   class VehiclesIndex < Chewy::Index
-    #     define_type Vehicle.cars.includes(:manufacturer), name: 'cars' do
-    #        ...
-    #     end # defines VehiclesIndex::Cars type
-    #
-    #     define_type Vehicle.motocycles.includes(:manufacturer), name: 'motocycles' do
-    #        ...
-    #     end # defines VehiclesIndex::Motocycles type
-    #   end
-    #
-    # For plain objects:
-    #
-    #   class PlanesIndex < Chewy::Index
-    #     define_type :plane do
-    #       ...
-    #     end # defines PlanesIndex::Plane type
-    #   end
-    #
-    # The main difference between using plain objects or ActiveRecord models for indexing
-    # is import. If you will call `CarsIndex::Car.import` - it will import all the cars
-    # automatically, while `PlanesIndex::Plane.import(my_planes)` requires import data to be
-    # passed.
-    #
-    def self.define_type(target, options = {}, &block)
-      type_class = Chewy.create_type(self, target, options, &block)
-      self.type_hash = type_hash.merge(type_class.type_name => type_class)
-
-      unless respond_to?(type_class.type_name)
-        class_eval <<-METHOD, __FILE__, __LINE__ + 1
-          def self.#{type_class.type_name}
-            ActiveSupport::Deprecation.warn("`#{self}.#{type_class.type_name}` accessor is deprecated and will be removed soon. Use `#{type_class}` directly instead.")
-            type_hash['#{type_class.type_name}']
-          end
-        METHOD
+      # Similar to the {.base_name} but respects the class namespace, also,
+      # can't be redefined. Used to reference index with the string identifier
+      #
+      # @example
+      #   class Namespace::UsersIndex < Chewy::Index
+      #   end
+      #   UsersIndex.derivable_name # => 'namespace/users'
+      #
+      #   Class.new(Chewy::Index).derivable_name # => nil
+      #
+      # @return [String, nil] derivable name or nil when it is impossible to calculate
+      def derivable_name
+        @derivable_name ||= name.sub(/Index\z/, '').underscore if name
       end
-    end
 
-    # Types method has double usage.
-    # If no arguments are passed - it returns array of defined types:
-    #
-    #   UsersIndex.types # => [UsersIndex::Admin, UsersIndex::Manager, UsersIndex::User]
-    #
-    # If arguments are passed it treats like a part of chainable query DSL and
-    # adds types array for index to select.
-    #
-    #   UsersIndex.filters { name =~ 'ro' }.types(:admin, :manager)
-    #   UsersIndex.types(:admin, :manager).filters { name =~ 'ro' } # the same as the first example
-    #
-    def self.types *args
-      if args.present?
-        all.types *args
-      else
-        type_hash.values
+      # Used as a default value for {.index_name}. Return prefix from the configuration
+      # but can be redefined per-index to be more dynamic.
+      #
+      # @example
+      #   class UsersIndex < Chewy::Index
+      #     def self.prefix
+      #       'foobar'
+      #     end
+      #   end
+      #   UsersIndex.index_name # => 'foobar_users'
+      #
+      # @return [String] prefix
+      def prefix
+        Chewy.configuration[:prefix]
       end
-    end
 
-    # Returns defined types names:
-    #
-    #   UsersIndex.type_names # => ['admin', 'manager', 'user']
-    #
-    def self.type_names
-      type_hash.keys
-    end
+      # Defines scope and options for the index. Arguments depends on adapter used. For
+      # ActiveRecord you can pass model or scope and options
+      #
+      #   class CarsIndex < Chewy::Index
+      #     index_scope Car
+      #     ...
+      #   end
+      #
+      # For plain objects you can completely omit this directive, unless you need to specify some options:
+      #
+      #   class PlanesIndex < Chewy::Index
+      #     ...
+      #   end
+      #
+      # The main difference between using plain objects or ActiveRecord models for indexing
+      # is import. If you will call `CarsIndex.import` - it will import all the cars
+      # automatically, while `PlanesIndex.import(my_planes)` requires import data to be
+      # passed.
+      #
+      def index_scope(target, options = {})
+        raise 'Index scope is already defined' if index_scope_defined?
+
+        self.adapter = Chewy.adapters.find { |klass| klass.accepts?(target) }.new(target, **options)
+        self.index_scope_defined = true
+      end
 
-    # Used as a part of index definition DSL. Defines settings:
-    #
-    #   class UsersIndex < Chewy::Index
-    #     settings analysis: {
-    #       analyzer: {
-    #         name: {
-    #           tokenizer: 'standard',
-    #           filter: ['lowercase', 'icu_folding', 'names_nysiis']
-    #         }
-    #       }
-    #     }
-    #   end
-    #
-    # See http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/indices-update-settings.html
-    # for more details
-    #
-    # It is possible to store analyzers settings in Chewy repositories
-    # and link them form index class. See `Chewy::Index::Settings` for details.
-    #
-    def self.settings(params)
-      self._settings = Chewy::Index::Settings.new params
-    end
+      # Used as a part of index definition DSL. Defines settings:
+      #
+      #   class UsersIndex < Chewy::Index
+      #     settings analysis: {
+      #       analyzer: {
+      #         name: {
+      #           tokenizer: 'standard',
+      #           filter: ['lowercase', 'icu_folding', 'names_nysiis']
+      #         }
+      #       }
+      #     }
+      #   end
+      #
+      # See http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/indices-update-settings.html
+      # for more details
+      #
+      # It is possible to store analyzers settings in Chewy repositories
+      # and link them form index class. See `Chewy::Index::Settings` for details.
+      #
+      def settings(params = {}, &block)
+        self._settings = Chewy::Index::Settings.new(params, &block)
+      end
 
-    # Returns list of public class methods defined in current index
-    #
-    def self.scopes
-      public_methods - Chewy::Index.public_methods - type_names.map(&:to_sym)
-    end
+      # Returns list of public class methods defined in current index
+      #
+      def scopes
+        public_methods - Chewy::Index.public_methods
+      end
 
-  private
+      def settings_hash
+        _settings.to_hash
+      end
 
-    def self.build_index_name *args
-      options = args.extract_options!
-      [options[:prefix], args.first || index_name, options[:suffix]].reject(&:blank?).join(?_)
-    end
+      def mappings_hash
+        mappings = root.mappings_hash
+        mappings.present? ? {mappings: mappings} : {}
+      end
 
-    def self.settings_hash
-      _settings.to_hash
-    end
+      # Returns a hash containing the index settings and mappings
+      # Used for the ES index creation as body.
+      #
+      # @see Chewy::Index::Specification
+      # @return [Hash] specification as a hash
+      def specification_hash
+        [settings_hash, mappings_hash].inject(:merge)
+      end
 
-    def self.mappings_hash
-      mappings = types.map(&:mappings_hash).inject(:merge)
-      mappings.present? ? {mappings: mappings} : {}
-    end
+      # @see Chewy::Index::Specification
+      # @return [Chewy::Index::Specification] a specification object instance for this particular index
+      def specification
+        @specification ||= Specification.new(self)
+      end
+
+      def default_import_options(params)
+        params.assert_valid_keys(IMPORT_OPTIONS_KEYS)
+        self._default_import_options = _default_import_options.merge(params)
+      end
 
-    def self.index_params
-      [settings_hash, mappings_hash].inject(:merge)
+      def strategy_config(params = {})
+        @strategy_config ||= begin
+          config_struct = Struct.new(*STRATEGY_OPTIONS.keys).new
+
+          STRATEGY_OPTIONS.each_with_object(config_struct) do |(strategy, options), res|
+            res[strategy] = case strategy
+            when :delayed_sidekiq
+              Struct.new(*STRATEGY_OPTIONS[strategy]).new.tap do |config|
+                options.each do |option|
+                  config[option] = params.dig(strategy, option) || Chewy.configuration.dig(:strategy_config, strategy, option)
+                end
+
+                config[:reindex_wrapper] ||= ->(&reindex) { reindex.call } # default wrapper
+              end
+            else
+              raise NotImplementedError, "Unsupported strategy: '#{strategy}'"
+            end
+          end
+        end
+      end
     end
   end
 end

data/lib/chewy/journal.rb

@@ -0,0 +1,66 @@
+module Chewy
+  # A class to perform journal-related actions for the specified indexes/types.
+  #
+  # @example
+  #   journal = Chewy::Journal.new('places', UsersIndex)
+  #   journal.apply(20.minutes.ago)
+  #   journal.clean
+  #
+  class Journal
+    # @param only [Array<String, Chewy::Index>] indexes or string references to perform actions on
+    def initialize(*only)
+      @only = only
+    end
+
+    # Applies all changes that were done since the specified time to the
+    # specified indexes.
+    #
+    # @param since_time [Time, DateTime] timestamp from which changes will be applied
+    # @param fetch_limit [Int] amount of entries to be fetched on each cycle
+    # @return [Integer] the amount of journal entries found
+    def apply(since_time, fetch_limit: 10, **import_options)
+      stage = 1
+      since_time -= 1
+      count = 0
+
+      total_count = entries(since_time, fetch_limit).total_count
+
+      while count < total_count
+        entries = entries(since_time, fetch_limit).to_a.presence or break
+        count += entries.size
+        groups = reference_groups(entries)
+        ActiveSupport::Notifications.instrument 'apply_journal.chewy', stage: stage, groups: groups
+        groups.each do |index, references|
+          index.import(references, import_options.merge(journal: false))
+        end
+        stage += 1
+        since_time = entries.map(&:created_at).max
+      end
+      count
+    end
+
+    # Cleans journal for the specified indexes/types.
+    #
+    # @param until_time [Time, DateTime] time to clean up until it
+    # @return [Hash] delete_by_query ES API call result
+    def clean(until_time = nil, delete_by_query_options: {})
+      Chewy::Stash::Journal.clean(
+        until_time,
+        only: @only,
+        delete_by_query_options: delete_by_query_options.merge(refresh: false)
+      )
+    end
+
+  private
+
+    def entries(since_time, fetch_limit)
+      Chewy::Stash::Journal.entries(since_time, only: @only).order(:created_at).limit(fetch_limit)
+    end
+
+    def reference_groups(entries)
+      entries.group_by(&:index_name)
+        .transform_keys { |index_name| Chewy.derive_name(index_name) }
+        .transform_values { |grouped_entries| grouped_entries.map(&:references).inject(:|) }
+    end
+  end
+end

data/lib/chewy/log_subscriber.rb

@@ -16,17 +16,17 @@ module Chewy
       render_action('Delete by Query', event) { |payload| payload[:request] }
     end
 
-    def render_action action, event, &block
+    def render_action(action, event)
       payload = event.payload
-      description = block.call(payload)
+      description = yield(payload)
 
-      if description.present?
-        subject = payload[:type].presence || payload[:index]
-        action = "#{subject} #{action} (#{event.duration.round(1)}ms)"
-        action = color(action, GREEN, true)
+      return if description.blank?
 
-        debug("  #{action} #{description}")
-      end
+      subject = payload[:type].presence || payload[:index]
+      action = "#{subject} #{action} (#{event.duration.round(1)}ms)"
+      action = color(action, GREEN, true)
+
+      debug("  #{action} #{description}")
     end
   end
 end

data/lib/chewy/minitest/helpers.rb

@@ -0,0 +1,150 @@
+require_relative 'search_index_receiver'
+
+module Chewy
+  module Minitest
+    module Helpers
+      extend ActiveSupport::Concern
+
+      # Assert that an index *changes* during a block.
+      # @param index [Chewy::Index] the index to watch, eg EntitiesIndex.
+      # @param strategy [Symbol] the Chewy strategy to use around the block. See Chewy docs.
+      # @param bypass_actual_index [true, false]
+      #   True to preempt the http call to Elastic, false otherwise.
+      #   Should be set to true unless actually testing search functionality.
+      #
+      # @return [SearchIndexReceiver] for optional further assertions on the nature of the index changes.
+      #
+      def assert_indexes(index, strategy: :atomic, bypass_actual_index: true, &block)
+        index_class = Chewy.derive_name index
+        receiver = SearchIndexReceiver.new
+
+        bulk_method = index_class.method :bulk
+        # Manually mocking #bulk because we need to properly capture `self`
+        bulk_mock = lambda do |*bulk_args|
+          receiver.catch bulk_args, self
+
+          bulk_method.call(*bulk_args) unless bypass_actual_index
+
+          {}
+        end
+
+        index_class.define_singleton_method :bulk, bulk_mock
+
+        Chewy.strategy(strategy, &block)
+
+        index_class.define_singleton_method :bulk, bulk_method
+
+        assert_includes receiver.updated_indexes, index, "Expected #{index} to be updated but it wasn't"
+
+        receiver
+      end
+
+      # Run indexing for the database changes during the block provided.
+      # By default, indexing is run at the end of the block.
+      # @param strategy [Symbol] the Chewy index update strategy see Chewy docs.
+      #
+      def run_indexing(strategy: :atomic, &block)
+        Chewy.strategy strategy, &block
+      end
+
+      # Mock Elasticsearch response
+      # Simple usage - just pass index, expected raw response
+      # and block with the query.
+      #
+      # @param index [Chewy::Index] the index to watch, eg EntitiesIndex.
+      # @param raw_response [Hash] hash with response.
+      #
+      def mock_elasticsearch_response(index, raw_response)
+        mocked_request = Chewy::Search::Request.new(index)
+
+        original_new = Chewy::Search::Request.method(:new)
+
+        Chewy::Search::Request.define_singleton_method(:new) { |*_args| mocked_request }
+
+        original_perform = mocked_request.method(:perform)
+        mocked_request.define_singleton_method(:perform) { raw_response }
+
+        yield
+      ensure
+        mocked_request.define_singleton_method(:perform, original_perform)
+        Chewy::Search::Request.define_singleton_method(:new, original_new)
+      end
+
+      # Mock Elasticsearch response with defined sources
+      # Simple usage - just pass index, expected sources
+      # and block with the query.
+      #
+      # @param index [Chewy::Index] the index to watch, eg EntitiesIndex.
+      # @param hits [Hash] hash with sources.
+      #
+      def mock_elasticsearch_response_sources(index, hits, &block)
+        raw_response = {
+          'took' => 4,
+          'timed_out' => false,
+          '_shards' => {
+            'total' => 1,
+            'successful' => 1,
+            'skipped' => 0,
+            'failed' => 0
+          },
+          'hits' => {
+            'total' => {
+              'value' => hits.count,
+              'relation' => 'eq'
+            },
+            'max_score' => 1.0,
+            'hits' => hits.each_with_index.map do |hit, i|
+              {
+                '_index' => index.index_name,
+                '_type' => '_doc',
+                '_id' => hit[:id] || (i + 1).to_s,
+                '_score' => 3.14,
+                '_source' => hit
+              }
+            end
+          }
+        }
+
+        mock_elasticsearch_response(index, raw_response, &block)
+      end
+
+      # Check the assertion that actual Elasticsearch query is rendered
+      # to the expected query
+      #
+      # @param query [::Query] the actual Elasticsearch query.
+      # @param expected_query [Hash] expected query.
+      #
+      # @return [Boolean]
+      #   True - in the case when actual Elasticsearch query is rendered to the expected query.
+      #   False - in the opposite case.
+      #
+      def assert_elasticsearch_query(query, expected_query)
+        actual_query = query.render
+        assert_equal expected_query, actual_query, "got #{actual_query.inspect} instead of expected query."
+      end
+
+      module ClassMethods
+        # Declare that all tests in this file require real indexing, always.
+        # In my completely unscientific experiments, this roughly doubled test runtime.
+        # Use with trepidation.
+        def index_everything!
+          setup do
+            Chewy.strategy :urgent
+          end
+
+          teardown do
+            Chewy.strategy.pop
+          end
+        end
+      end
+
+      included do
+        teardown do
+          # always destroy indexes between tests
+          # Prevent croll pollution of test cases due to indexing
+          Chewy.massacre
+        end
+      end
+    end
+  end
+end