chewy 0.8.4 → 7.3.4

data/Rakefile

@@ -4,4 +4,14 @@ require 'elasticsearch/extensions/test/cluster/tasks'
 
 RSpec::Core::RakeTask.new(:spec)
 
-task :default => :spec
+task default: :spec
+
+namespace :es do
+  task :start do
+    Rake.application['elasticsearch:start'].invoke
+  end
+
+  task :stop do
+    Rake.application['elasticsearch:stop'].invoke
+  end
+end

data/chewy.gemspec

@@ -1,13 +1,12 @@
-# coding: utf-8
-lib = File.expand_path('../lib', __FILE__)
+lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'chewy/version'
 
-Gem::Specification.new do |spec|
+Gem::Specification.new do |spec| # rubocop:disable Metrics/BlockLength
   spec.name          = 'chewy'
   spec.version       = Chewy::VERSION
-  spec.authors       = ['pyromaniac']
-  spec.email         = ['kinwizard@gmail.com']
+  spec.authors       = ['Toptal, LLC', 'pyromaniac']
+  spec.email         = ['open-source@toptal.com', 'kinwizard@gmail.com']
   spec.summary       = 'Elasticsearch ODM client wrapper'
   spec.description   = 'Chewy provides functionality for Elasticsearch index handling, documents import mappings and chainable query DSL'
   spec.homepage      = 'https://github.com/toptal/chewy'
@@ -18,24 +17,21 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
 
+  spec.add_development_dependency 'database_cleaner'
+  spec.add_development_dependency 'elasticsearch-extensions'
+  spec.add_development_dependency 'mock_redis'
   spec.add_development_dependency 'rake'
-  spec.add_development_dependency 'appraisal'
-  spec.add_development_dependency 'rspec', '~> 3.4.0'
-  spec.add_development_dependency 'rspec-its'
+  spec.add_development_dependency 'rspec', '>= 3.7.0'
   spec.add_development_dependency 'rspec-collection_matchers'
+  spec.add_development_dependency 'rspec-its'
+  spec.add_development_dependency 'rubocop', '1.11'
   spec.add_development_dependency 'sqlite3'
-  spec.add_development_dependency 'database_cleaner'
-  spec.add_development_dependency 'elasticsearch-extensions'
-  spec.add_development_dependency 'resque_spec'
-  spec.add_development_dependency 'rubysl', '~> 2.0' if RUBY_ENGINE == 'rbx'
+  spec.add_development_dependency 'timecop'
 
   spec.add_development_dependency 'method_source'
-  if RUBY_VERSION < '2.1.0'
-    spec.add_development_dependency 'unparser', '0.2.4'
-  else
-    spec.add_development_dependency 'unparser'
-  end
+  spec.add_development_dependency 'unparser'
 
-  spec.add_dependency 'activesupport', '>= 3.2'
-  spec.add_dependency 'elasticsearch', '>= 1.0.0'
+  spec.add_dependency 'activesupport', '>= 5.2'
+  spec.add_dependency 'elasticsearch', '>= 7.12.0', '< 7.14.0'
+  spec.add_dependency 'elasticsearch-dsl'
 end

data/gemfiles/rails.5.2.activerecord.gemfile

@@ -0,0 +1,11 @@
+source 'https://rubygems.org'
+
+gem 'activejob', '~> 5.2.0'
+gem 'activerecord', '~> 5.2.0'
+gem 'activesupport', '~> 5.2.0'
+gem 'kaminari-core', '~> 1.1.0', require: false
+gem 'parallel', require: false
+gem 'rspec_junit_formatter', '~> 0.4.1'
+gem 'sidekiq', require: false
+
+gemspec path: '../'

data/gemfiles/rails.6.0.activerecord.gemfile

@@ -0,0 +1,11 @@
+source 'https://rubygems.org'
+
+gem 'activejob', '~> 6.0.0'
+gem 'activerecord', '~> 6.0.0'
+gem 'activesupport', '~> 6.0.0'
+gem 'kaminari-core', '~> 1.1.0', require: false
+gem 'parallel', require: false
+gem 'rspec_junit_formatter', '~> 0.4.1'
+gem 'sidekiq', require: false
+
+gemspec path: '../'

data/gemfiles/rails.6.1.activerecord.gemfile

@@ -0,0 +1,13 @@
+source 'https://rubygems.org'
+
+gem 'activejob', '~> 6.1.0'
+gem 'activerecord', '~> 6.1.0'
+gem 'activesupport', '~> 6.1.0'
+gem 'kaminari-core', '~> 1.1.0', require: false
+gem 'parallel', require: false
+gem 'rspec_junit_formatter', '~> 0.4.1'
+gem 'sidekiq', require: false
+
+gem 'rexml' if RUBY_VERSION >= '3.0.0'
+
+gemspec path: '../'

data/gemfiles/rails.7.0.activerecord.gemfile

@@ -0,0 +1,13 @@
+source 'https://rubygems.org'
+
+gem 'activejob', '~> 7.0.0'
+gem 'activerecord', '~> 7.0.0'
+gem 'activesupport', '~> 7.0.0'
+gem 'kaminari-core', '~> 1.1.0', require: false
+gem 'parallel', require: false
+gem 'rspec_junit_formatter', '~> 0.4.1'
+gem 'sidekiq', require: false
+
+gem 'rexml' if RUBY_VERSION >= '3.0.0'
+
+gemspec path: '../'

data/lib/chewy/config.rb

@@ -2,61 +2,73 @@ module Chewy
   class Config
     include Singleton
 
-    attr_accessor :settings, :transport_logger, :transport_tracer, :logger,
-
-      # Default query compilation mode. `:must` by default.
-      # See Chewy::Query#query_mode for details
-      #
-      :query_mode,
-
-      # Default filters compilation mode. `:and` by default.
-      # See Chewy::Query#filter_mode for details
-      #
-      :filter_mode,
-
-      # Default post_filters compilation mode. `nil` by default.
-      # See Chewy::Query#post_filter_mode for details
-      #
-      :post_filter_mode,
-
-      # The first strategy in stack. `:base` by default.
-      # If you need to return to the previous chewy behavior -
-      # just set it to `:bypass`
-      #
-      :root_strategy,
-
-      # Default request strategy middleware, used in e.g
-      # Rails controllers. See Chewy::Railtie::RequestStrategy
-      # for more info.
-      #
-      :request_strategy,
-
-      # Use after_commit callbacks for RDBMS instead of
-      # after_save and after_destroy. True by default. Useful
-      # in tests with transactional fixtures or transactional
-      # DatabaseCleaner strategy.
-      #
-      :use_after_commit_callbacks
+    attr_accessor :settings, :logger,
+                  # The first strategy in stack. `:base` by default.
+                  # If you need to return to the previous chewy behavior -
+                  # just set it to `:bypass`
+                  #
+                  :root_strategy,
+                  # Default request strategy middleware, used in e.g
+                  # Rails controllers. See Chewy::Railtie::RequestStrategy
+                  # for more info.
+                  #
+                  :request_strategy,
+                  # Rails console strategy, `:urgent` by default.
+                  #
+                  :console_strategy,
+                  # Use after_commit callbacks for RDBMS instead of
+                  # after_save and after_destroy. True by default. Useful
+                  # in tests with transactional fixtures or transactional
+                  # DatabaseCleaner strategy.
+                  #
+                  :use_after_commit_callbacks,
+                  # Where Chewy expects to find index definitions
+                  # within a Rails app folder.
+                  :indices_path,
+                  # Set index refresh_interval setting to -1 before reset and put the original value after.
+                  # If setting not present, put back to default 1s
+                  # https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-update-settings.html
+                  :reset_disable_refresh_interval,
+                  # Set number_of_replicas to 0 before reset and put the original value after
+                  # https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-update-settings.html
+                  :reset_no_replicas,
+                  # Refresh or not when import async (sidekiq, lazy_sidekiq, activejob)
+                  :disable_refresh_async,
+                  # Default options for root of Chewy type. Allows to set default options
+                  # for type mappings like `_all`.
+                  :default_root_options,
+                  # Default field type for any field in any Chewy type. Defaults to 'text'.
+                  :default_field_type
+
+    attr_reader :transport_logger, :transport_tracer,
+                # Chewy search request DSL base class, used by every index.
+                :search_class
 
     def self.delegated
-      public_instance_methods - self.superclass.public_instance_methods - Singleton.public_instance_methods
+      public_instance_methods - superclass.public_instance_methods - Singleton.public_instance_methods
     end
 
     def initialize
       @settings = {}
-      @query_mode = :must
-      @filter_mode = :and
       @root_strategy = :base
       @request_strategy = :atomic
+      @console_strategy = :urgent
       @use_after_commit_callbacks = true
+      @reset_disable_refresh_interval = false
+      @reset_no_replicas = false
+      @disable_refresh_async = false
+      @indices_path = 'app/chewy'
+      @default_root_options = {}
+      @default_field_type = 'text'.freeze
+      @search_class = build_search_class(Chewy::Search::Request)
     end
 
-    def transport_logger= logger
+    def transport_logger=(logger)
       Chewy.client.transport.logger = logger
       @transport_logger = logger
     end
 
-    def transport_tracer= tracer
+    def transport_tracer=(tracer)
       Chewy.client.transport.tracer = tracer
       @transport_tracer = tracer
     end
@@ -87,7 +99,7 @@ module Chewy
     #
     #      :wait_for_status - if this option set - chewy actions such
     #      as creating or deleting index, importing data will wait for
-    #      the status specified. Extremely useful for tests under havy
+    #      the status specified. Extremely useful for tests under heavy
     #      indexes manipulations.
     #
     #        test:
@@ -106,30 +118,32 @@ module Chewy
     #
     def configuration
       yaml_settings.merge(settings.deep_symbolize_keys).tap do |configuration|
-        configuration.merge!(logger: transport_logger) if transport_logger
+        configuration[:logger] = transport_logger if transport_logger
+        configuration[:indices_path] ||= indices_path if indices_path
         configuration.merge!(tracer: transport_tracer) if transport_tracer
       end
     end
 
-    def configuration= options
-      ActiveSupport::Deprecation.warn("`Chewy.configuration = {foo: 'bar'}` method is deprecated and will be removed soon, use `Chewy.settings = {foo: 'bar'}` method instead")
-      self.settings = options
-    end
-
   private
 
     def yaml_settings
       @yaml_settings ||= begin
-        if defined?(Rails)
-          file = Rails.root.join(*%w(config chewy.yml))
+        if defined?(Rails::VERSION)
+          file = Rails.root.join('config', 'chewy.yml')
 
           if File.exist?(file)
             yaml = ERB.new(File.read(file)).result
-            hash = YAML.load(yaml)
+            hash = YAML.respond_to?(:unsafe_load) ? YAML.unsafe_load(yaml) : YAML.load(yaml) # rubocop:disable Security/YAMLLoad
             hash[Rails.env].try(:deep_symbolize_keys) if hash
           end
         end || {}
       end
     end
+
+    def build_search_class(base)
+      Class.new(base).tap do |search_class|
+        search_class.send :include, Chewy::Search::Pagination::Kaminari if defined?(::Kaminari)
+      end
+    end
   end
 end

data/lib/chewy/errors.rb

@@ -5,17 +5,11 @@ module Chewy
   class UndefinedIndex < Error
   end
 
-  class UndefinedType < Error
-  end
-
-  class UnderivableType < Error
-  end
-
   class UndefinedUpdateStrategy < Error
-    def initialize type
+    def initialize(_type)
       super <<-MESSAGE
-Index update strategy is undefined in current context.
-Please wrap your code with `Chewy.strategy(:strategy_name) block.`
+  Index update strategy is undefined for current context.
+  Please wrap your code with `Chewy.strategy(:strategy_name) block.`
       MESSAGE
     end
   end
@@ -24,11 +18,11 @@ Please wrap your code with `Chewy.strategy(:strategy_name) block.`
   end
 
   class ImportFailed < Error
-    def initialize type, errors
+    def initialize(type, import_errors)
       message = "Import failed for `#{type}` with:\n"
-      errors.each do |action, errors|
+      import_errors.each do |action, action_errors|
         message << "    #{action.to_s.humanize} errors:\n"
-        errors.each do |error, documents|
+        action_errors.each do |error, documents|
           message << "      `#{error}`\n"
           message << "        on #{documents.count} documents: #{documents}\n"
         end
@@ -37,9 +31,9 @@ Please wrap your code with `Chewy.strategy(:strategy_name) block.`
     end
   end
 
-  class RemovedFeature < Error
-  end
-
-  class PluginMissing < Error
+  class InvalidJoinFieldType < Error
+    def initialize(join_field_type, join_field_name, relations)
+      super("`#{join_field_type}` set for the join field `#{join_field_name}` is not on the :relations list (#{relations})")
+    end
   end
 end

data/lib/chewy/fields/base.rb

@@ -1,13 +1,21 @@
 module Chewy
   module Fields
     class Base
-      attr_reader :name, :options, :value, :children
-      attr_accessor :parent
+      attr_reader :name, :join_options, :options, :children
+      attr_accessor :parent # used by Chewy::Index::Mapping to expand nested fields
 
-      def initialize(name, options = {})
-        @name, @options = name.to_sym, options.deep_symbolize_keys
-        @value = @options.delete(:value)
+      def initialize(name, value: nil, **options)
+        @name = name.to_sym
+        @options = {}
+        update_options!(**options)
+        @value = value
         @children = []
+        @allowed_relations = find_allowed_relations(options[:relations]) # for join fields
+      end
+
+      def update_options!(**options)
+        @join_options = options.delete(:join) || {}
+        @options = options
       end
 
       def multi_field?
@@ -15,52 +23,134 @@ module Chewy
       end
 
       def object_field?
-        (children.present? && options[:type].blank?) || ['object', 'nested'].include?(options[:type].to_s)
+        (children.present? && options[:type].blank?) || %w[object nested].include?(options[:type].to_s)
       end
 
       def mappings_hash
-        mapping = children.present? ? {
-          (multi_field? ? :fields : :properties) => children.map(&:mappings_hash).inject(:merge)
-        } : {}
-        mapping.reverse_merge!(options)
-        mapping.reverse_merge!(type: (children.present? ? 'object' : 'string'))
+        mapping =
+          if children.present?
+            {(multi_field? ? :fields : :properties) => children.map(&:mappings_hash).inject(:merge)}
+          else
+            {}
+          end
+        mapping.reverse_merge!(options.except(:ignore_blank))
+        mapping.reverse_merge!(type: (children.present? ? 'object' : Chewy.default_field_type))
+
         {name => mapping}
       end
 
-      def compose(object, *parent_objects)
-        objects = ([object] + parent_objects.flatten).uniq
+      def compose(*objects)
+        result = evaluate(objects)
 
-        result = if value && value.is_a?(Proc)
-          if value.arity == 0
-            object.instance_exec(&value)
-          elsif value.arity < 0
-            value.call(*object)
+        return {} if result.blank? && ignore_blank?
+
+        if children.present? && !multi_field?
+          result = if result.respond_to?(:to_ary)
+            result.to_ary.map { |item| compose_children(item, *objects) }
           else
-            value.call(*objects.first(value.arity))
+            compose_children(result, *objects)
           end
-        elsif object.is_a?(Hash)
-          if object.has_key?(name)
-            object[name]
-          else
-            object[name.to_s]
+        end
+
+        {name => result}
+      end
+
+      def value
+        if join_field?
+          join_type = join_options[:type]
+          join_id = join_options[:id]
+          # memoize
+          @value ||= proc do |object|
+            validate_join_type!(value_by_name_proc(join_type).call(object))
+            # If it's a join field and it has join_id, the value is compound and contains
+            # both name (type) and id of the parent object
+            if value_by_name_proc(join_id).call(object).present?
+              {
+                name: value_by_name_proc(join_type).call(object), # parent type
+                parent: value_by_name_proc(join_id).call(object)  # parent id
+              }
+            else
+              value_by_name_proc(join_type).call(object)
+            end
           end
         else
-          object.send(name)
+          @value
         end
+      end
+
+    private
+
+      def geo_point?
+        @options[:type].to_s == 'geo_point'
+      end
+
+      def join_field?
+        @options[:type].to_s == 'join'
+      end
+
+      def ignore_blank?
+        @options.fetch(:ignore_blank) { geo_point? }
+      end
 
-        result = if result.respond_to?(:to_ary)
-          result.to_ary.map { |item| compose_children(item, *objects) }
+      def evaluate(objects)
+        if value.is_a?(Proc)
+          value_by_proc(objects, value)
         else
-          compose_children(result, *objects)
-        end if children.present? && !multi_field?
+          value_by_name(objects, value)
+        end
+      end
 
-        {name => result.as_json(root: false)}
+      def value_by_proc(objects, value)
+        object = objects.first
+        if value.arity.zero?
+          object.instance_exec(&value)
+        elsif value.arity.negative?
+          value.call(*object)
+        else
+          value.call(*objects.first(value.arity))
+        end
       end
 
-    private
+      def value_by_name(objects, value)
+        object = objects.first
+        message = value.is_a?(Symbol) || value.is_a?(String) ? value.to_sym : name
+        value_by_name_proc(message).call(object)
+      end
+
+      def value_by_name_proc(message)
+        proc do |object|
+          if object.is_a?(Hash)
+            if object.key?(message)
+              object[message]
+            else
+              object[message.to_s]
+            end
+          else
+            object.send(message)
+          end
+        end
+      end
+
+      def validate_join_type!(type)
+        return unless type
+        return if @allowed_relations.include?(type.to_sym)
+
+        raise Chewy::InvalidJoinFieldType.new(type, @name, options[:relations])
+      end
+
+      def find_allowed_relations(relations)
+        return [] unless relations
+        return relations unless relations.is_a?(Hash)
+
+        (relations.keys + relations.values).flatten.uniq
+      end
 
       def compose_children(value, *parent_objects)
-        children.map { |field| field.compose(value, *parent_objects) }.compact.inject(:merge) if value
+        return unless value
+
+        children.each_with_object({}) do |field, result|
+          result.merge!(field.compose(value, *parent_objects) || {})
+        end
       end
     end
   end

data/lib/chewy/fields/root.rb

@@ -1,20 +1,18 @@
 module Chewy
   module Fields
     class Root < Chewy::Fields::Base
-      attr_reader :dynamic_templates
-      attr_reader :id
-      attr_reader :parent
-      attr_reader :parent_id
-
-      def initialize(*args)
-        super(*args)
-
-        @id = @options.delete(:id) || options.delete(:_id)
-        @parent = @options.delete(:parent) || options.delete(:_parent)
-        @parent_id = @options.delete(:parent_id)
-        @value ||= ->(_) { _ }
+      attr_reader :dynamic_templates, :id
+
+      def initialize(name, **options)
+        super(name, **options)
+
+        @value ||= -> { self }
         @dynamic_templates = []
-        @options.delete(:type)
+      end
+
+      def update_options!(**options)
+        @id = options.fetch(:id, options.fetch(:_id, @id))
+        @options.merge!(options.except(:id, :_id, :type))
       end
 
       def mappings_hash
@@ -26,11 +24,10 @@ module Chewy
           mappings[name][:dynamic_templates].concat dynamic_templates
         end
 
-        mappings[name][:_parent] = parent.is_a?(Hash) ? parent : { type: parent } if parent
-        mappings
+        mappings[name]
       end
 
-      def dynamic_template *args
+      ruby2_keywords def dynamic_template(*args)
         options = args.extract_options!.deep_symbolize_keys
         if args.first
           template_name = :"template_#{dynamic_templates.count.next}"
@@ -51,17 +48,45 @@ module Chewy
         end
       end
 
-      def compose_parent(object)
-        if parent_id
-          parent_id.arity == 0 ? object.instance_exec(&parent_id) : parent_id.call(object)
-        end
+      def compose_id(object)
+        return unless id
+
+        id.arity.zero? ? object.instance_exec(&id) : id.call(object)
       end
 
-      def compose_id(object)
-        if id
-          id.arity == 0 ? object.instance_exec(&id) : id.call(object)
+      # Converts passed object to JSON-ready hash. Used for objects import.
+      #
+      # @param object [Object] a base object for composition
+      # @param crutches [Object] any object that will be passed to every field value proc as a last argument
+      # @param fields [Array<Symbol>] a list of fields to compose, every field will be composed if empty
+      # @return [Hash] JSON-ready hash with stringified keys
+      #
+      def compose(object, crutches = nil, fields: [])
+        result = evaluate([object, crutches])
+
+        if children.present?
+          child_fields = if fields.present?
+            child_hash.slice(*fields).values
+          else
+            children
+          end
+
+          child_fields.each_with_object({}) do |field, memo|
+            memo.merge!(field.compose(result, crutches) || {})
+          end.as_json
+        elsif fields.present?
+          result.as_json(only: fields, root: false)
+        else
+          result.as_json(root: false)
         end
       end
+
+      # Children indexed by name as a hash.
+      #
+      # @return [Hash{Symbol => Chewy::Fields::Base}]
+      def child_hash
+        @child_hash ||= children.index_by(&:name)
+      end
     end
   end
 end