chewy 5.1.0 → 7.2.7

data/chewy.gemspec

@@ -1,12 +1,12 @@
-lib = File.expand_path('../lib', __FILE__)
+lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'chewy/version'
 
-Gem::Specification.new do |spec| # rubocop:disable BlockLength
+Gem::Specification.new do |spec| # rubocop:disable Metrics/BlockLength
   spec.name          = 'chewy'
   spec.version       = Chewy::VERSION
-  spec.authors       = ['pyromaniac']
-  spec.email         = ['kinwizard@gmail.com']
+  spec.authors       = ['Toptal, LLC', 'pyromaniac']
+  spec.email         = ['open-source@toptal.com', 'kinwizard@gmail.com']
   spec.summary       = 'Elasticsearch ODM client wrapper'
   spec.description   = 'Chewy provides functionality for Elasticsearch index handling, documents import mappings and chainable query DSL'
   spec.homepage      = 'https://github.com/toptal/chewy'
@@ -17,22 +17,20 @@ Gem::Specification.new do |spec| # rubocop:disable BlockLength
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
 
-  spec.add_development_dependency 'appraisal'
   spec.add_development_dependency 'database_cleaner'
   spec.add_development_dependency 'elasticsearch-extensions'
   spec.add_development_dependency 'rake'
-  spec.add_development_dependency 'resque_spec'
-  spec.add_development_dependency 'rspec', '~> 3.7.0'
+  spec.add_development_dependency 'rspec', '>= 3.7.0'
   spec.add_development_dependency 'rspec-collection_matchers'
   spec.add_development_dependency 'rspec-its'
-  spec.add_development_dependency 'rubocop', '0.52.1'
+  spec.add_development_dependency 'rubocop', '1.11'
   spec.add_development_dependency 'sqlite3'
   spec.add_development_dependency 'timecop'
 
   spec.add_development_dependency 'method_source'
   spec.add_development_dependency 'unparser'
 
-  spec.add_dependency 'activesupport', '>= 4.0'
-  spec.add_dependency 'elasticsearch', '>= 2.0.0'
+  spec.add_dependency 'activesupport', '>= 5.2'
+  spec.add_dependency 'elasticsearch', '>= 7.12.0', '< 7.14.0'
   spec.add_dependency 'elasticsearch-dsl'
 end

data/gemfiles/rails.5.2.activerecord.gemfile

@@ -1,16 +1,11 @@
-# This file was generated by Appraisal
+source 'https://rubygems.org'
 
-source "https://rubygems.org"
+gem 'activejob', '~> 5.2.0'
+gem 'activerecord', '~> 5.2.0'
+gem 'activesupport', '~> 5.2.0'
+gem 'kaminari-core', '~> 1.1.0', require: false
+gem 'parallel', require: false
+gem 'rspec_junit_formatter', '~> 0.4.1'
+gem 'sidekiq', require: false
 
-gem "activerecord", "~> 5.2.0"
-gem "activesupport", "~> 5.2.0"
-gem "activejob", "~> 5.2.0"
-gem "resque", require: false
-gem "shoryuken", require: false
-gem "aws-sdk-sqs", require: false
-gem "sidekiq", require: false
-gem "kaminari-core", "~> 1.1.0", require: false
-gem "will_paginate", require: false
-gem "parallel", require: false
-
-gemspec path: "../"
+gemspec path: '../'

data/gemfiles/rails.6.0.activerecord.gemfile

@@ -0,0 +1,11 @@
+source 'https://rubygems.org'
+
+gem 'activejob', '~> 6.0.0'
+gem 'activerecord', '~> 6.0.0'
+gem 'activesupport', '~> 6.0.0'
+gem 'kaminari-core', '~> 1.1.0', require: false
+gem 'parallel', require: false
+gem 'rspec_junit_formatter', '~> 0.4.1'
+gem 'sidekiq', require: false
+
+gemspec path: '../'

data/gemfiles/rails.6.1.activerecord.gemfile

@@ -0,0 +1,13 @@
+source 'https://rubygems.org'
+
+gem 'activejob', '~> 6.1.0'
+gem 'activerecord', '~> 6.1.0'
+gem 'activesupport', '~> 6.1.0'
+gem 'kaminari-core', '~> 1.1.0', require: false
+gem 'parallel', require: false
+gem 'rspec_junit_formatter', '~> 0.4.1'
+gem 'sidekiq', require: false
+
+gem 'rexml' if RUBY_VERSION >= '3.0.0'
+
+gemspec path: '../'

data/gemfiles/rails.7.0.activerecord.gemfile

@@ -0,0 +1,13 @@
+source 'https://rubygems.org'
+
+gem 'activejob', '~> 7.0.0'
+gem 'activerecord', '~> 7.0.0'
+gem 'activesupport', '~> 7.0.0'
+gem 'kaminari-core', '~> 1.1.0', require: false
+gem 'parallel', require: false
+gem 'rspec_junit_formatter', '~> 0.4.1'
+gem 'sidekiq', require: false
+
+gem 'rexml' if RUBY_VERSION >= '3.0.0'
+
+gemspec path: '../'

data/lib/chewy/config.rb

@@ -3,55 +3,46 @@ module Chewy
     include Singleton
 
     attr_accessor :settings, :logger,
-      # Default query compilation mode. `:must` by default.
-      # See Chewy::Query#query_mode for details
-      #
-      :query_mode,
-      # Default filters compilation mode. `:and` by default.
-      # See Chewy::Query#filter_mode for details
-      #
-      :filter_mode,
-      # Default post_filters compilation mode. `nil` by default.
-      # See Chewy::Query#post_filter_mode for details
-      #
-      :post_filter_mode,
-      # The first strategy in stack. `:base` by default.
-      # If you need to return to the previous chewy behavior -
-      # just set it to `:bypass`
-      #
-      :root_strategy,
-      # Default request strategy middleware, used in e.g
-      # Rails controllers. See Chewy::Railtie::RequestStrategy
-      # for more info.
-      #
-      :request_strategy,
-      # Use after_commit callbacks for RDBMS instead of
-      # after_save and after_destroy. True by default. Useful
-      # in tests with transactional fixtures or transactional
-      # DatabaseCleaner strategy.
-      #
-      :use_after_commit_callbacks,
-      # Where Chewy expects to find index definitions
-      # within a Rails app folder.
-      :indices_path,
-      # Set index refresh_interval setting to -1 before reset and put the original value after.
-      # If setting not present, put back to default 1s
-      # https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-update-settings.html
-      :reset_disable_refresh_interval,
-      # Set number_of_replicas to 0 before reset and put the original value after
-      # https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-update-settings.html
-      :reset_no_replicas,
-      # Refresh or not when import async (sidekiq, resque, activejob)
-      :disable_refresh_async,
-      # Default options for root of Chewy type. Allows to set default options
-      # for type mappings like `_all`.
-      :default_root_options,
-      # Default field type for any field in any Chewy type. Defaults to 'string'.
-      :default_field_type
+                  # The first strategy in stack. `:base` by default.
+                  # If you need to return to the previous chewy behavior -
+                  # just set it to `:bypass`
+                  #
+                  :root_strategy,
+                  # Default request strategy middleware, used in e.g
+                  # Rails controllers. See Chewy::Railtie::RequestStrategy
+                  # for more info.
+                  #
+                  :request_strategy,
+                  # Rails console strategy, `:urgent` by default.
+                  #
+                  :console_strategy,
+                  # Use after_commit callbacks for RDBMS instead of
+                  # after_save and after_destroy. True by default. Useful
+                  # in tests with transactional fixtures or transactional
+                  # DatabaseCleaner strategy.
+                  #
+                  :use_after_commit_callbacks,
+                  # Where Chewy expects to find index definitions
+                  # within a Rails app folder.
+                  :indices_path,
+                  # Set index refresh_interval setting to -1 before reset and put the original value after.
+                  # If setting not present, put back to default 1s
+                  # https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-update-settings.html
+                  :reset_disable_refresh_interval,
+                  # Set number_of_replicas to 0 before reset and put the original value after
+                  # https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-update-settings.html
+                  :reset_no_replicas,
+                  # Refresh or not when import async (sidekiq, lazy_sidekiq, activejob)
+                  :disable_refresh_async,
+                  # Default options for root of Chewy type. Allows to set default options
+                  # for type mappings like `_all`.
+                  :default_root_options,
+                  # Default field type for any field in any Chewy type. Defaults to 'text'.
+                  :default_field_type
 
     attr_reader :transport_logger, :transport_tracer,
-      # Chewy search request DSL base class, used by every index.
-      :search_class
+                # Chewy search request DSL base class, used by every index.
+                :search_class
 
     def self.delegated
       public_instance_methods - superclass.public_instance_methods - Singleton.public_instance_methods
@@ -59,10 +50,9 @@ module Chewy
 
     def initialize
       @settings = {}
-      @query_mode = :must
-      @filter_mode = :and
       @root_strategy = :base
       @request_strategy = :atomic
+      @console_strategy = :urgent
       @use_after_commit_callbacks = true
       @reset_disable_refresh_interval = false
       @reset_no_replicas = false
@@ -70,7 +60,7 @@ module Chewy
       @indices_path = 'app/chewy'
       @default_root_options = {}
       @default_field_type = 'text'.freeze
-      self.search_class = Chewy::Search::Request
+      @search_class = build_search_class(Chewy::Search::Request)
     end
 
     def transport_logger=(logger)
@@ -83,10 +73,6 @@ module Chewy
       @transport_tracer = tracer
     end
 
-    def search_class=(base)
-      @search_class = build_search_class(base)
-    end
-
     # Chewy core configurations. There is two ways to set it up:
     # use `Chewy.settings=` method or, for Rails application,
     # create `config/chewy.yml` file. Btw, `config/chewy.yml` supports
@@ -147,7 +133,7 @@ module Chewy
 
           if File.exist?(file)
             yaml = ERB.new(File.read(file)).result
-            hash = YAML.load(yaml) # rubocop:disable Security/YAMLLoad
+            hash = YAML.respond_to?(:unsafe_load) ? YAML.unsafe_load(yaml) : YAML.load(yaml) # rubocop:disable Security/YAMLLoad
             hash[Rails.env].try(:deep_symbolize_keys) if hash
           end
         end || {}
@@ -156,11 +142,7 @@ module Chewy
 
     def build_search_class(base)
       Class.new(base).tap do |search_class|
-        if defined?(::Kaminari)
-          search_class.send :include, Chewy::Search::Pagination::Kaminari
-        elsif defined?(::WillPaginate)
-          search_class.send :include, Chewy::Search::Pagination::WillPaginate
-        end
+        search_class.send :include, Chewy::Search::Pagination::Kaminari if defined?(::Kaminari)
       end
     end
   end

data/lib/chewy/errors.rb

@@ -5,12 +5,6 @@ module Chewy
   class UndefinedIndex < Error
   end
 
-  class UndefinedType < Error
-  end
-
-  class UnderivableType < Error
-  end
-
   class UndefinedUpdateStrategy < Error
     def initialize(_type)
       super <<-MESSAGE
@@ -37,9 +31,9 @@ module Chewy
     end
   end
 
-  class RemovedFeature < Error
-  end
-
-  class PluginMissing < Error
+  class InvalidJoinFieldType < Error
+    def initialize(join_field_type, join_field_name, relations)
+      super("`#{join_field_type}` set for the join field `#{join_field_name}` is not on the :relations list (#{relations})")
+    end
   end
 end

data/lib/chewy/fields/base.rb

@@ -1,18 +1,20 @@
 module Chewy
   module Fields
     class Base
-      attr_reader :name, :options, :value, :children
-      attr_accessor :parent
+      attr_reader :name, :join_options, :options, :children
+      attr_accessor :parent # used by Chewy::Index::Mapping to expand nested fields
 
       def initialize(name, value: nil, **options)
         @name = name.to_sym
         @options = {}
-        update_options!(options)
+        update_options!(**options)
         @value = value
         @children = []
+        @allowed_relations = find_allowed_relations(options[:relations]) # for join fields
       end
 
       def update_options!(**options)
+        @join_options = options.delete(:join) || {}
         @options = options
       end
 
@@ -31,21 +33,17 @@ module Chewy
           else
             {}
           end
-        mapping.reverse_merge!(options)
+        mapping.reverse_merge!(options.except(:ignore_blank))
         mapping.reverse_merge!(type: (children.present? ? 'object' : Chewy.default_field_type))
 
-        # This is done to support ES2 journaling and will be removed soon
-        if mapping[:type] == 'keyword' && Chewy::Runtime.version < '5.0'
-          mapping[:type] = 'string'
-          mapping[:index] = 'not_analyzed'
-        end
-
         {name => mapping}
       end
 
       def compose(*objects)
         result = evaluate(objects)
 
+        return {} if result.blank? && ignore_blank?
+
         if children.present? && !multi_field?
           result = if result.respond_to?(:to_ary)
             result.to_ary.map { |item| compose_children(item, *objects) }
@@ -57,22 +55,70 @@ module Chewy
         {name => result}
       end
 
+      def value
+        if join_field?
+          join_type = join_options[:type]
+          join_id = join_options[:id]
+          # memoize
+          @value ||= proc do |object|
+            validate_join_type!(value_by_name_proc(join_type).call(object))
+            # If it's a join field and it has join_id, the value is compound and contains
+            # both name (type) and id of the parent object
+            if value_by_name_proc(join_id).call(object).present?
+              {
+                name: value_by_name_proc(join_type).call(object), # parent type
+                parent: value_by_name_proc(join_id).call(object)  # parent id
+              }
+            else
+              value_by_name_proc(join_type).call(object)
+            end
+          end
+        else
+          @value
+        end
+      end
+
     private
 
-      def evaluate(objects)
-        object = objects.first
+      def geo_point?
+        @options[:type].to_s == 'geo_point'
+      end
 
+      def join_field?
+        @options[:type].to_s == 'join'
+      end
+
+      def ignore_blank?
+        @options.fetch(:ignore_blank) { geo_point? }
+      end
+
+      def evaluate(objects)
         if value.is_a?(Proc)
-          if value.arity.zero?
-            object.instance_exec(&value)
-          elsif value.arity < 0
-            value.call(*object)
-          else
-            value.call(*objects.first(value.arity))
-          end
+          value_by_proc(objects, value)
         else
-          message = value.is_a?(Symbol) || value.is_a?(String) ? value.to_sym : name
+          value_by_name(objects, value)
+        end
+      end
+
+      def value_by_proc(objects, value)
+        object = objects.first
+        if value.arity.zero?
+          object.instance_exec(&value)
+        elsif value.arity.negative?
+          value.call(*object)
+        else
+          value.call(*objects.first(value.arity))
+        end
+      end
 
+      def value_by_name(objects, value)
+        object = objects.first
+        message = value.is_a?(Symbol) || value.is_a?(String) ? value.to_sym : name
+        value_by_name_proc(message).call(object)
+      end
+
+      def value_by_name_proc(message)
+        proc do |object|
           if object.is_a?(Hash)
             if object.key?(message)
               object[message]
@@ -85,6 +131,20 @@ module Chewy
         end
       end
 
+      def validate_join_type!(type)
+        return unless type
+        return if @allowed_relations.include?(type.to_sym)
+
+        raise Chewy::InvalidJoinFieldType.new(type, @name, options[:relations])
+      end
+
+      def find_allowed_relations(relations)
+        return [] unless relations
+        return relations unless relations.is_a?(Hash)
+
+        (relations.keys + relations.values).flatten.uniq
+      end
+
       def compose_children(value, *parent_objects)
         return unless value
 

data/lib/chewy/fields/root.rb

@@ -1,13 +1,10 @@
 module Chewy
   module Fields
     class Root < Chewy::Fields::Base
-      attr_reader :dynamic_templates
-      attr_reader :id
-      attr_reader :parent
-      attr_reader :parent_id
+      attr_reader :dynamic_templates, :id
 
-      def initialize(*)
-        super
+      def initialize(name, **options)
+        super(name, **options)
 
         @value ||= -> { self }
         @dynamic_templates = []
@@ -15,9 +12,7 @@ module Chewy
 
       def update_options!(**options)
         @id = options.fetch(:id, options.fetch(:_id, @id))
-        @parent = options.fetch(:parent, options.fetch(:_parent, @parent))
-        @parent_id = options.fetch(:parent_id, @parent_id)
-        @options.merge!(options.except(:id, :_id, :parent, :_parent, :parent_id, :type))
+        @options.merge!(options.except(:id, :_id, :type))
       end
 
       def mappings_hash
@@ -29,11 +24,10 @@ module Chewy
           mappings[name][:dynamic_templates].concat dynamic_templates
         end
 
-        mappings[name][:_parent] = parent.is_a?(Hash) ? parent : {type: parent} if parent
-        mappings
+        mappings[name]
       end
 
-      def dynamic_template(*args)
+      ruby2_keywords def dynamic_template(*args)
         options = args.extract_options!.deep_symbolize_keys
         if args.first
           template_name = :"template_#{dynamic_templates.count.next}"
@@ -54,13 +48,9 @@ module Chewy
         end
       end
 
-      def compose_parent(object)
-        return unless parent_id
-        parent_id.arity.zero? ? object.instance_exec(&parent_id) : parent_id.call(object)
-      end
-
       def compose_id(object)
         return unless id
+
         id.arity.zero? ? object.instance_exec(&id) : id.call(object)
       end
 

data/lib/chewy/index/actions.rb

@@ -30,8 +30,8 @@ module Chewy
         # Suffixed index names might be used for zero-downtime mapping change, for example.
         # Description: (http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime/).
         #
-        def create(*args)
-          create!(*args)
+        def create(*args, **kwargs)
+          create!(*args, **kwargs)
         rescue Elasticsearch::Transport::Transport::Errors::BadRequest
           false
         end
@@ -74,7 +74,13 @@ module Chewy
         #   UsersIndex.delete '01-2014' # deletes `users_01-2014` index
         #
         def delete(suffix = nil)
-          result = client.indices.delete index: index_name(suffix: suffix)
+          # Verify that the index_name is really the index_name and not an alias.
+          #
+          #   "The index parameter in the delete index API no longer accepts alias names.
+          #   Instead, it accepts only index names (or wildcards which will expand to matching indices)."
+          #   https://www.elastic.co/guide/en/elasticsearch/reference/6.8/breaking-changes-6.0.html#_delete_index_api_resolves_indices_expressions_only_against_indices
+          index_names = client.indices.get_alias(index: index_name(suffix: suffix)).keys
+          result = client.indices.delete index: index_names.join(',')
           Chewy.wait_for_status if result
           result
           # es-ruby >= 1.0.10 handles Elasticsearch::Transport::Transport::Errors::NotFound
@@ -123,35 +129,6 @@ module Chewy
           create! suffix
         end
 
-        # Perform import operation for every defined type
-        #
-        #   UsersIndex.import                           # imports default data for every index type
-        #   UsersIndex.import user: User.active         # imports specified objects for user type and default data for other types
-        #   UsersIndex.import refresh: false            # to disable index refreshing after import
-        #   UsersIndex.import suffix: Time.now.to_i     # imports data to index with specified suffix if such is exists
-        #   UsersIndex.import batch_size: 300           # import batch size
-        #
-        # See [import.rb](lib/chewy/type/import.rb) for more details.
-        #
-        %i[import import!].each do |method|
-          class_eval <<-METHOD, __FILE__, __LINE__ + 1
-            def #{method}(*args)
-              options = args.extract_options!
-              if args.one? && type_names.one?
-                objects = {type_names.first.to_sym => args.first}
-              elsif args.one?
-                fail ArgumentError, "Please pass objects for `#{method}` as a hash with type names"
-              else
-                objects = options.reject { |k, v| !type_names.map(&:to_sym).include?(k) }
-              end
-              types.map do |type|
-                args = [objects[type.type_name.to_sym], options.dup].reject(&:blank?)
-                type.#{method} *args
-              end.all?
-            end
-          METHOD
-        end
-
         # Deletes, creates and imports data to the index. Returns the
         # import result. If index name suffix is passed as the first
         # argument - performs zero-downtime index resetting.
@@ -172,14 +149,18 @@ module Chewy
         def reset!(suffix = nil, apply_journal: true, journal: false, **import_options)
           result = if suffix.present?
             start_time = Time.now
-            indexes = self.indexes
+            indexes = self.indexes - [index_name]
             create! suffix, alias: false
 
             general_name = index_name
             suffixed_name = index_name(suffix: suffix)
 
             optimize_index_settings suffixed_name
-            result = import import_options.merge(suffix: suffix, journal: journal, refresh: !Chewy.reset_disable_refresh_interval)
+            result = import(**import_options.merge(
+              suffix: suffix,
+              journal: journal,
+              refresh: !Chewy.reset_disable_refresh_interval
+            ))
             original_index_settings suffixed_name
 
             delete if indexes.blank?
@@ -195,12 +176,13 @@ module Chewy
             result
           else
             purge!
-            import import_options.merge(journal: journal)
+            import(**import_options.merge(journal: journal))
           end
 
           specification.lock!
           result
         end
+        alias_method :reset, :reset!
 
         # A {Chewy::Journal} instance for the particular index
         #
@@ -209,6 +191,50 @@ module Chewy
           @journal ||= Chewy::Journal.new(self)
         end
 
+        def clear_cache(args = {index: index_name})
+          client.indices.clear_cache(args)
+        end
+
+        def reindex(source: index_name, dest: index_name)
+          client.reindex(
+            {
+              body:
+                {
+                  source: {index: source},
+                  dest: {index: dest}
+                }
+            }
+          )
+        end
+
+        # Adds new fields to an existing data stream or index.
+        # Change the search settings of existing fields.
+        #
+        # @example
+        #   Chewy.client.update_mapping('cities', {properties: {new_field: {type: :text}}})
+        #
+        def update_mapping(name = index_name, body = root.mappings_hash)
+          client.indices.put_mapping(
+            index: name,
+            body: body
+          )['acknowledged']
+        end
+
+        # Performs missing and outdated objects synchronization for the current index.
+        #
+        # @example
+        #   UsersIndex.sync
+        #
+        # @see Chewy::Index::Syncer
+        # @param parallel [true, Integer, Hash] options for parallel execution or the number of processes
+        # @return [Hash{Symbol, Object}, nil] a number of missing and outdated documents re-indexed and their ids,
+        #   nil in case of errors
+        def sync(parallel: nil)
+          syncer = Syncer.new(self, parallel: parallel)
+          count = syncer.perform
+          {count: count, missing: syncer.missing_ids, outdated: syncer.outdated_ids} if count
+        end
+
       private
 
         def optimize_index_settings(index_name)
@@ -234,6 +260,7 @@ module Chewy
 
         def index_settings(setting_name)
           return {} unless settings_hash.key?(:settings) && settings_hash[:settings].key?(:index)
+
           settings_hash[:settings][:index].slice(setting_name)
         end
       end