chewy 6.0.0 → 7.6.0

data/chewy.gemspec

@@ -1,8 +1,8 @@
-lib = File.expand_path('../lib', __FILE__)
+lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'chewy/version'
 
-Gem::Specification.new do |spec| # rubocop:disable BlockLength
+Gem::Specification.new do |spec|
   spec.name          = 'chewy'
   spec.version       = Chewy::VERSION
   spec.authors       = ['Toptal, LLC', 'pyromaniac']
@@ -14,25 +14,10 @@ Gem::Specification.new do |spec| # rubocop:disable BlockLength
 
   spec.files         = `git ls-files`.split($RS)
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
-  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
 
-  spec.add_development_dependency 'appraisal'
-  spec.add_development_dependency 'database_cleaner'
-  spec.add_development_dependency 'elasticsearch-extensions'
-  spec.add_development_dependency 'rake'
-  spec.add_development_dependency 'resque_spec'
-  spec.add_development_dependency 'rspec', '>= 3.7.0'
-  spec.add_development_dependency 'rspec-collection_matchers'
-  spec.add_development_dependency 'rspec-its'
-  spec.add_development_dependency 'rubocop', '0.52.1'
-  spec.add_development_dependency 'sqlite3'
-  spec.add_development_dependency 'timecop'
-
-  spec.add_development_dependency 'method_source'
-  spec.add_development_dependency 'unparser'
-
-  spec.add_dependency 'activesupport', '>= 5.2'
-  spec.add_dependency 'elasticsearch', '>= 6.3.0'
+  spec.add_dependency 'activesupport', '>= 5.2' # Remove with major version bump, 8.x
+  spec.add_dependency 'elasticsearch', '>= 7.14.0', '< 8'
   spec.add_dependency 'elasticsearch-dsl'
+  spec.metadata['rubygems_mfa_required'] = 'true'
 end

data/gemfiles/base.gemfile

@@ -0,0 +1,12 @@
+gem 'database_cleaner'
+gem 'elasticsearch-extensions'
+gem 'method_source'
+gem 'rake'
+gem 'redis', require: false
+gem 'rspec', '>= 3.7.0'
+gem 'rspec-collection_matchers'
+gem 'rspec-its'
+gem 'rubocop', '1.63.4'
+gem 'sqlite3', '~> 1.4'
+gem 'timecop'
+gem 'unparser'

data/gemfiles/rails.6.1.activerecord.gemfile

@@ -1,19 +1,14 @@
-# This file was generated by Appraisal
+source 'https://rubygems.org'
 
-source "https://rubygems.org"
-
-gem "activerecord", "~> 6.1.0"
-gem "activesupport", "~> 6.1.0"
-gem "activejob", "~> 6.1.0"
-gem "resque", require: false
-gem "shoryuken", require: false
-gem "aws-sdk-sqs", require: false
-gem "sidekiq", require: false
-gem "kaminari-core", "~> 1.1.0", require: false
-gem "will_paginate", require: false
-gem "parallel", require: false
+gem 'activejob', '~> 6.1.0'
+gem 'activerecord', '~> 6.1.0'
+gem 'activesupport', '~> 6.1.0'
+gem 'kaminari-core', '~> 1.1.0', require: false
+gem 'parallel', require: false
 gem 'rspec_junit_formatter', '~> 0.4.1'
+gem 'sidekiq', require: false
 
-gem 'rexml' if RUBY_VERSION >= '3.0.0'
+gem 'rexml'
 
-gemspec path: "../"
+gemspec path: '../'
+eval_gemfile 'base.gemfile'

data/gemfiles/rails.7.0.activerecord.gemfile

@@ -0,0 +1,14 @@
+source 'https://rubygems.org'
+
+gem 'activejob', '~> 7.0.0'
+gem 'activerecord', '~> 7.0.0'
+gem 'activesupport', '~> 7.0.0'
+gem 'kaminari-core', '~> 1.1.0', require: false
+gem 'parallel', require: false
+gem 'rspec_junit_formatter', '~> 0.4.1'
+gem 'sidekiq', require: false
+
+gem 'rexml'
+
+gemspec path: '../'
+eval_gemfile 'base.gemfile'

data/gemfiles/rails.7.1.activerecord.gemfile

@@ -0,0 +1,14 @@
+source 'https://rubygems.org'
+
+gem 'activejob', '~> 7.1.0'
+gem 'activerecord', '~> 7.1.0'
+gem 'activesupport', '~> 7.1.0'
+gem 'kaminari-core', '~> 1.1.0', require: false
+gem 'parallel', require: false
+gem 'rspec_junit_formatter', '~> 0.4.1'
+gem 'sidekiq', require: false
+
+gem 'rexml'
+
+gemspec path: '../'
+eval_gemfile 'base.gemfile'

data/lib/chewy/config.rb

@@ -3,43 +3,51 @@ module Chewy
     include Singleton
 
     attr_accessor :settings, :logger,
-      # The first strategy in stack. `:base` by default.
-      # If you need to return to the previous chewy behavior -
-      # just set it to `:bypass`
-      #
-      :root_strategy,
-      # Default request strategy middleware, used in e.g
-      # Rails controllers. See Chewy::Railtie::RequestStrategy
-      # for more info.
-      #
-      :request_strategy,
-      # Use after_commit callbacks for RDBMS instead of
-      # after_save and after_destroy. True by default. Useful
-      # in tests with transactional fixtures or transactional
-      # DatabaseCleaner strategy.
-      #
-      :use_after_commit_callbacks,
-      # Where Chewy expects to find index definitions
-      # within a Rails app folder.
-      :indices_path,
-      # Set index refresh_interval setting to -1 before reset and put the original value after.
-      # If setting not present, put back to default 1s
-      # https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-update-settings.html
-      :reset_disable_refresh_interval,
-      # Set number_of_replicas to 0 before reset and put the original value after
-      # https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-update-settings.html
-      :reset_no_replicas,
-      # Refresh or not when import async (sidekiq, resque, activejob)
-      :disable_refresh_async,
-      # Default options for root of Chewy type. Allows to set default options
-      # for type mappings like `_all`.
-      :default_root_options,
-      # Default field type for any field in any Chewy type. Defaults to 'text'.
-      :default_field_type
+                  # The first strategy in stack. `:base` by default.
+                  # If you need to return to the previous chewy behavior -
+                  # just set it to `:bypass`
+                  #
+                  :root_strategy,
+                  # Default request strategy middleware, used in e.g
+                  # Rails controllers. See Chewy::Railtie::RequestStrategy
+                  # for more info.
+                  #
+                  :request_strategy,
+                  # Rails console strategy, `:urgent` by default.
+                  #
+                  :console_strategy,
+                  # Use after_commit callbacks for RDBMS instead of
+                  # after_save and after_destroy. True by default. Useful
+                  # in tests with transactional fixtures or transactional
+                  # DatabaseCleaner strategy.
+                  #
+                  :use_after_commit_callbacks,
+                  # Where Chewy expects to find index definitions
+                  # within a Rails app folder.
+                  :indices_path,
+                  # Set index refresh_interval setting to -1 before reset and put the original value after.
+                  # If setting not present, put back to default 1s
+                  # https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-update-settings.html
+                  :reset_disable_refresh_interval,
+                  # Set number_of_replicas to 0 before reset and put the original value after
+                  # https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-update-settings.html
+                  :reset_no_replicas,
+                  # Refresh or not when import async (sidekiq, lazy_sidekiq, activejob)
+                  :disable_refresh_async,
+                  # Default options for root of Chewy type. Allows to set default options
+                  # for type mappings like `_all`.
+                  :default_root_options,
+                  # Default field type for any field in any Chewy type. Defaults to 'text'.
+                  :default_field_type,
+                  # Callback called on each search request to be done into ES
+                  :before_es_request_filter,
+                  # Behavior when import scope for index includes order, offset or limit.
+                  # Can be :ignore, :warn, :raise. Defaults to :warn
+                  :import_scope_cleanup_behavior
 
     attr_reader :transport_logger, :transport_tracer,
-      # Chewy search request DSL base class, used by every index.
-      :search_class
+                # Chewy search request DSL base class, used by every index.
+                :search_class
 
     def self.delegated
       public_instance_methods - superclass.public_instance_methods - Singleton.public_instance_methods
@@ -49,6 +57,7 @@ module Chewy
       @settings = {}
       @root_strategy = :base
       @request_strategy = :atomic
+      @console_strategy = :urgent
       @use_after_commit_callbacks = true
       @reset_disable_refresh_interval = false
       @reset_no_replicas = false
@@ -56,16 +65,17 @@ module Chewy
       @indices_path = 'app/chewy'
       @default_root_options = {}
       @default_field_type = 'text'.freeze
+      @import_scope_cleanup_behavior = :warn
       @search_class = build_search_class(Chewy::Search::Request)
     end
 
     def transport_logger=(logger)
-      Chewy.client.transport.logger = logger
+      Chewy.client.transport.transport.logger = logger
       @transport_logger = logger
     end
 
     def transport_tracer=(tracer)
-      Chewy.client.transport.tracer = tracer
+      Chewy.client.transport.transport.tracer = tracer
       @transport_tracer = tracer
     end
 
@@ -123,26 +133,24 @@ module Chewy
   private
 
     def yaml_settings
-      @yaml_settings ||= begin
-        if defined?(Rails::VERSION)
-          file = Rails.root.join('config', 'chewy.yml')
+      @yaml_settings ||= build_yaml_settings || {}
+    end
 
-          if File.exist?(file)
-            yaml = ERB.new(File.read(file)).result
-            hash = YAML.load(yaml) # rubocop:disable Security/YAMLLoad
-            hash[Rails.env].try(:deep_symbolize_keys) if hash
-          end
-        end || {}
-      end
+    def build_yaml_settings
+      return unless defined?(Rails::VERSION)
+
+      file = Rails.root.join('config', 'chewy.yml')
+
+      return unless File.exist?(file)
+
+      yaml = ERB.new(File.read(file)).result
+      hash = YAML.unsafe_load(yaml)
+      hash[Rails.env].try(:deep_symbolize_keys) if hash
     end
 
     def build_search_class(base)
       Class.new(base).tap do |search_class|
-        if defined?(::Kaminari)
-          search_class.send :include, Chewy::Search::Pagination::Kaminari
-        elsif defined?(::WillPaginate)
-          search_class.send :include, Chewy::Search::Pagination::WillPaginate
-        end
+        search_class.send :include, Chewy::Search::Pagination::Kaminari if defined?(::Kaminari)
       end
     end
   end

data/lib/chewy/elastic_client.rb

@@ -0,0 +1,31 @@
+module Chewy
+  # Replacement for Chewy.client
+  class ElasticClient
+    def self.build_es_client(configuration = Chewy.configuration)
+      client_configuration = configuration.deep_dup
+      client_configuration.delete(:prefix) # used by Chewy, not relevant to Elasticsearch::Client
+      block = client_configuration[:transport_options].try(:delete, :proc)
+      ::Elasticsearch::Client.new(client_configuration, &block)
+    end
+
+    def initialize(elastic_client = self.class.build_es_client)
+      @elastic_client = elastic_client
+    end
+
+  private
+
+    def method_missing(name, *args, **kwargs, &block)
+      inspect_payload(name, args, kwargs)
+
+      @elastic_client.__send__(name, *args, **kwargs, &block)
+    end
+
+    def respond_to_missing?(name, _include_private = false)
+      @elastic_client.respond_to?(name) || super
+    end
+
+    def inspect_payload(name, args, kwargs)
+      Chewy.config.before_es_request_filter&.call(name, args, kwargs)
+    end
+  end
+end

data/lib/chewy/errors.rb

@@ -5,15 +5,9 @@ module Chewy
   class UndefinedIndex < Error
   end
 
-  class UndefinedType < Error
-  end
-
-  class UnderivableType < Error
-  end
-
   class UndefinedUpdateStrategy < Error
     def initialize(_type)
-      super <<-MESSAGE
+      super(<<-MESSAGE)
   Index update strategy is undefined for current context.
   Please wrap your code with `Chewy.strategy(:strategy_name) block.`
       MESSAGE
@@ -33,13 +27,16 @@ module Chewy
           message << "        on #{documents.count} documents: #{documents}\n"
         end
       end
-      super message
+      super(message)
     end
   end
 
-  class RemovedFeature < Error
+  class InvalidJoinFieldType < Error
+    def initialize(join_field_type, join_field_name, relations)
+      super("`#{join_field_type}` set for the join field `#{join_field_name}` is not on the :relations list (#{relations})")
+    end
   end
 
-  class PluginMissing < Error
+  class ImportScopeCleanupError < Error
   end
 end

data/lib/chewy/fields/base.rb

@@ -1,8 +1,8 @@
 module Chewy
   module Fields
     class Base
-      attr_reader :name, :options, :value, :children
-      attr_accessor :parent
+      attr_reader :name, :join_options, :options, :children
+      attr_accessor :parent # used by Chewy::Index::Mapping to expand nested fields
 
       def initialize(name, value: nil, **options)
         @name = name.to_sym
@@ -10,9 +10,11 @@ module Chewy
         update_options!(**options)
         @value = value
         @children = []
+        @allowed_relations = find_allowed_relations(options[:relations]) # for join fields
       end
 
       def update_options!(**options)
+        @join_options = options.delete(:join) || {}
         @options = options
       end
 
@@ -31,7 +33,7 @@ module Chewy
           else
             {}
           end
-        mapping.reverse_merge!(options)
+        mapping.reverse_merge!(options.except(:ignore_blank))
         mapping.reverse_merge!(type: (children.present? ? 'object' : Chewy.default_field_type))
 
         {name => mapping}
@@ -40,6 +42,8 @@ module Chewy
       def compose(*objects)
         result = evaluate(objects)
 
+        return {} if result.blank? && ignore_blank?
+
         if children.present? && !multi_field?
           result = if result.respond_to?(:to_ary)
             result.to_ary.map { |item| compose_children(item, *objects) }
@@ -51,22 +55,70 @@ module Chewy
         {name => result}
       end
 
+      def value
+        if join_field?
+          join_type = join_options[:type]
+          join_id = join_options[:id]
+          # memoize
+          @value ||= proc do |object|
+            validate_join_type!(value_by_name_proc(join_type).call(object))
+            # If it's a join field and it has join_id, the value is compound and contains
+            # both name (type) and id of the parent object
+            if value_by_name_proc(join_id).call(object).present?
+              {
+                name: value_by_name_proc(join_type).call(object), # parent type
+                parent: value_by_name_proc(join_id).call(object)  # parent id
+              }
+            else
+              value_by_name_proc(join_type).call(object)
+            end
+          end
+        else
+          @value
+        end
+      end
+
     private
 
-      def evaluate(objects)
-        object = objects.first
+      def geo_point?
+        @options[:type].to_s == 'geo_point'
+      end
+
+      def join_field?
+        @options[:type].to_s == 'join'
+      end
 
+      def ignore_blank?
+        @options.fetch(:ignore_blank) { geo_point? }
+      end
+
+      def evaluate(objects)
         if value.is_a?(Proc)
-          if value.arity.zero?
-            object.instance_exec(&value)
-          elsif value.arity < 0
-            value.call(*object)
-          else
-            value.call(*objects.first(value.arity))
-          end
+          value_by_proc(objects, value)
         else
-          message = value.is_a?(Symbol) || value.is_a?(String) ? value.to_sym : name
+          value_by_name(objects, value)
+        end
+      end
 
+      def value_by_proc(objects, value)
+        object = objects.first
+        if value.arity.zero?
+          object.instance_exec(&value)
+        elsif value.arity.negative?
+          value.call(*object)
+        else
+          value.call(*objects.first(value.arity))
+        end
+      end
+
+      def value_by_name(objects, value)
+        object = objects.first
+        message = value.is_a?(Symbol) || value.is_a?(String) ? value.to_sym : name
+        value_by_name_proc(message).call(object)
+      end
+
+      def value_by_name_proc(message)
+        proc do |object|
           if object.is_a?(Hash)
             if object.key?(message)
               object[message]
@@ -79,6 +131,20 @@ module Chewy
         end
       end
 
+      def validate_join_type!(type)
+        return unless type
+        return if @allowed_relations.include?(type.to_sym)
+
+        raise Chewy::InvalidJoinFieldType.new(type, @name, options[:relations])
+      end
+
+      def find_allowed_relations(relations)
+        return [] unless relations
+        return relations unless relations.is_a?(Hash)
+
+        (relations.keys + relations.values).flatten.uniq
+      end
+
       def compose_children(value, *parent_objects)
         return unless value
 

data/lib/chewy/fields/root.rb

@@ -1,10 +1,7 @@
 module Chewy
   module Fields
     class Root < Chewy::Fields::Base
-      attr_reader :dynamic_templates
-      attr_reader :id
-      attr_reader :parent
-      attr_reader :parent_id
+      attr_reader :dynamic_templates, :id
 
       def initialize(name, **options)
         super(name, **options)
@@ -15,9 +12,7 @@ module Chewy
 
       def update_options!(**options)
         @id = options.fetch(:id, options.fetch(:_id, @id))
-        @parent = options.fetch(:parent, options.fetch(:_parent, @parent))
-        @parent_id = options.fetch(:parent_id, @parent_id)
-        @options.merge!(options.except(:id, :_id, :parent, :_parent, :parent_id, :type))
+        @options.merge!(options.except(:id, :_id, :type))
       end
 
       def mappings_hash
@@ -29,8 +24,7 @@ module Chewy
           mappings[name][:dynamic_templates].concat dynamic_templates
         end
 
-        mappings[name][:_parent] = parent.is_a?(Hash) ? parent : {type: parent} if parent
-        mappings
+        mappings[name]
       end
 
       def dynamic_template(*args)
@@ -54,13 +48,9 @@ module Chewy
         end
       end
 
-      def compose_parent(object)
-        return unless parent_id
-        parent_id.arity.zero? ? object.instance_exec(&parent_id) : parent_id.call(object)
-      end
-
       def compose_id(object)
         return unless id
+
         id.arity.zero? ? object.instance_exec(&id) : id.call(object)
       end
 

data/lib/chewy/index/actions.rb

@@ -59,9 +59,7 @@ module Chewy
 
           body = specification_hash
           body[:aliases] = {general_name => {}} if options[:alias] && suffixed_name != general_name
-          args = {index: suffixed_name, body: body}
-          args[:include_type_name] = true if Runtime.version >= '6.7.0'
-          result = client.indices.create(**args)
+          result = client.indices.create(index: suffixed_name, body: body)
 
           Chewy.wait_for_status if result
           result
@@ -131,37 +129,6 @@ module Chewy
           create! suffix
         end
 
-        # Perform import operation for every defined type
-        #
-        #   UsersIndex.import                           # imports default data for every index type
-        #   UsersIndex.import user: User.active         # imports specified objects for user type and default data for other types
-        #   UsersIndex.import refresh: false            # to disable index refreshing after import
-        #   UsersIndex.import suffix: Time.now.to_i     # imports data to index with specified suffix if such is exists
-        #   UsersIndex.import batch_size: 300           # import batch size
-        #
-        # See [import.rb](lib/chewy/type/import.rb) for more details.
-        #
-        %i[import import!].each do |method|
-          class_eval <<-METHOD, __FILE__, __LINE__ + 1
-            def #{method}(*args)
-              return true if args.first.blank? && !args.first.nil?
-
-              options = args.extract_options!
-              if args.one? && type_names.one?
-                objects = {type_names.first.to_sym => args.first}
-              elsif args.one?
-                fail ArgumentError, "Please pass objects for `#{method}` as a hash with type names"
-              else
-                objects = options.reject { |k, v| !type_names.map(&:to_sym).include?(k) }
-              end
-              types.map do |type|
-                args = [objects[type.type_name.to_sym], options.dup].reject(&:blank?)
-                type.#{method} *args
-              end.all?
-            end
-          METHOD
-        end
-
         # Deletes, creates and imports data to the index. Returns the
         # import result. If index name suffix is passed as the first
         # argument - performs zero-downtime index resetting.
@@ -182,14 +149,18 @@ module Chewy
         def reset!(suffix = nil, apply_journal: true, journal: false, **import_options)
           result = if suffix.present?
             start_time = Time.now
-            indexes = self.indexes
+            indexes = self.indexes - [index_name]
             create! suffix, alias: false
 
             general_name = index_name
             suffixed_name = index_name(suffix: suffix)
 
             optimize_index_settings suffixed_name
-            result = import import_options.merge(suffix: suffix, journal: journal, refresh: !Chewy.reset_disable_refresh_interval)
+            result = import(**import_options.merge(
+              suffix: suffix,
+              journal: journal,
+              refresh: !Chewy.reset_disable_refresh_interval
+            ))
             original_index_settings suffixed_name
 
             delete if indexes.blank?
@@ -205,12 +176,13 @@ module Chewy
             result
           else
             purge!
-            import import_options.merge(journal: journal)
+            import(**import_options.merge(journal: journal))
           end
 
           specification.lock!
           result
         end
+        alias_method :reset, :reset!
 
         # A {Chewy::Journal} instance for the particular index
         #
@@ -219,6 +191,50 @@ module Chewy
           @journal ||= Chewy::Journal.new(self)
         end
 
+        def clear_cache(args = {index: index_name})
+          client.indices.clear_cache(args)
+        end
+
+        def reindex(source: index_name, dest: index_name)
+          client.reindex(
+            {
+              body:
+                {
+                  source: {index: source},
+                  dest: {index: dest}
+                }
+            }
+          )
+        end
+
+        # Adds new fields to an existing data stream or index.
+        # Change the search settings of existing fields.
+        #
+        # @example
+        #   Chewy.client.update_mapping('cities', {properties: {new_field: {type: :text}}})
+        #
+        def update_mapping(name = index_name, body = root.mappings_hash)
+          client.indices.put_mapping(
+            index: name,
+            body: body
+          )['acknowledged']
+        end
+
+        # Performs missing and outdated objects synchronization for the current index.
+        #
+        # @example
+        #   UsersIndex.sync
+        #
+        # @see Chewy::Index::Syncer
+        # @param parallel [true, Integer, Hash] options for parallel execution or the number of processes
+        # @return [Hash{Symbol, Object}, nil] a number of missing and outdated documents re-indexed and their ids,
+        #   nil in case of errors
+        def sync(parallel: nil)
+          syncer = Syncer.new(self, parallel: parallel)
+          count = syncer.perform
+          {count: count, missing: syncer.missing_ids, outdated: syncer.outdated_ids} if count
+        end
+
       private
 
         def optimize_index_settings(index_name)
@@ -244,6 +260,7 @@ module Chewy
 
         def index_settings(setting_name)
           return {} unless settings_hash.key?(:settings) && settings_hash[:settings].key?(:index)
+
           settings_hash[:settings][:index].slice(setting_name)
         end
       end