chewy 0.10.1 → 7.3.2

data/lib/chewy/{type → index}/import.rb

@@ -1,43 +1,42 @@
-require 'chewy/type/import/journal_builder'
-require 'chewy/type/import/bulk_builder'
-require 'chewy/type/import/bulk_request'
-require 'chewy/type/import/routine'
+require 'chewy/index/import/journal_builder'
+require 'chewy/index/import/bulk_builder'
+require 'chewy/index/import/bulk_request'
+require 'chewy/index/import/routine'
 
 module Chewy
-  class Type
+  class Index
     module Import
       extend ActiveSupport::Concern
 
-      IMPORT_WORKER = lambda do |type, options, total, ids, index|
-        ::Process.setproctitle("chewy [#{type}]: import data (#{index + 1}/#{total})")
-        routine = Routine.new(type, options)
-        type.adapter.import(*ids, routine.options) do |action_objects|
+      IMPORT_WORKER = lambda do |index, options, total, ids, iteration|
+        ::Process.setproctitle("chewy [#{index}]: import data (#{iteration + 1}/#{total})")
+        routine = Routine.new(index, **options)
+        index.adapter.import(*ids, routine.options) do |action_objects|
           routine.process(**action_objects)
         end
         {errors: routine.errors, import: routine.stats, leftovers: routine.leftovers}
       end
 
-      LEFTOVERS_WORKER = lambda do |type, options, total, body, index|
-        ::Process.setproctitle("chewy [#{type}]: import leftovers (#{index + 1}/#{total})")
-        routine = Routine.new(type, options)
+      LEFTOVERS_WORKER = lambda do |index, options, total, body, iteration|
+        ::Process.setproctitle("chewy [#{index}]: import leftovers (#{iteration + 1}/#{total})")
+        routine = Routine.new(index, **options)
         routine.perform_bulk(body)
         routine.errors
       end
 
       module ClassMethods
         # @!method import(*collection, **options)
-        # Basically, one of the main methods for type. Performs any objects import
-        # to the index for a specified type. Does all the objects handling routines.
+        # Basically, one of the main methods for an index. Performs any objects import
+        # to the index. Does all the objects handling routines.
         # Performs document import by utilizing bulk API. Bulk size and objects batch
         # size are controlled by the corresponding options.
         #
         # It accepts ORM/ODM objects, PORO, hashes, ids which are used by adapter to
-        # fetch objects from the source depenting on the used adapter. It destroys
-        # passed objects from the index if they are not in the default type scope
+        # fetch objects from the source depending on the used adapter. It destroys
+        # passed objects from the index if they are not in the default scope
         # or marked for destruction.
         #
-        # It handles parent-child relationships: if the object parent_id has been
-        # changed it destroys the object and recreates it from scratch.
+        # It handles parent-child relationships with a join field reindexing children when the parent is reindexed.
         #
         # Performs journaling if enabled: it stores all the ids of the imported
         # objects to a specialized index. It is possible to replay particular import
@@ -51,15 +50,15 @@ module Chewy
         #
         # Utilizes `ActiveSupport::Notifications`, so it is possible to get imported
         # objects later by listening to the `import_objects.chewy` queue. It is also
-        # possible to get the list of occured errors from the payload if something
+        # possible to get the list of occurred errors from the payload if something
         # went wrong.
         #
         # Import can also be run in parallel using the Parallel gem functionality.
         #
         # @example
-        #   UsersIndex::User.import(parallel: true) # imports everything in parallel with automatic workers number
-        #   UsersIndex::User.import(parallel: 3) # using 3 workers
-        #   UsersIndex::User.import(parallel: {in_threads: 10}) # in 10 threads
+        #   UsersIndex.import(parallel: true) # imports everything in parallel with automatic workers number
+        #   UsersIndex.import(parallel: 3) # using 3 workers
+        #   UsersIndex.import(parallel: {in_threads: 10}) # in 10 threads
         #
         # @see https://github.com/elastic/elasticsearch-ruby/blob/master/elasticsearch-api/lib/elasticsearch/api/actions/bulk.rb
         # @param collection [Array<Object>] and array or anything to import
@@ -67,13 +66,14 @@ module Chewy
         # @option options [String] suffix an index name suffix, used for zero-downtime reset mostly, no suffix by default
         # @option options [Integer] bulk_size bulk API chunk size in bytes; if passed, the request is performed several times for each chunk, empty by default
         # @option options [Integer] batch_size passed to the adapter import method, used to split imported objects in chunks, 1000 by default
+        # @option options [Boolean] direct_import skips object reloading in ORM adapter, `false` by default
         # @option options [true, false] journal enables imported objects journaling, false by default
         # @option options [Array<Symbol, String>] update_fields list of fields for the partial import, empty by default
         # @option options [true, false] update_failover enables full objects reimport in cases of partial update errors, `true` by default
         # @option options [true, Integer, Hash] parallel enables parallel import processing with the Parallel gem, accepts the number of workers or any Parallel gem acceptable options
         # @return [true, false] false in case of errors
-        def import(*args)
-          import_routine(*args).blank?
+        ruby2_keywords def import(*args)
+          intercept_import_using_strategy(*args).blank?
         end
 
         # @!method import!(*collection, **options)
@@ -83,9 +83,11 @@ module Chewy
         # in case of any import errors.
         #
         # @raise [Chewy::ImportFailed] in case of errors
-        def import!(*args)
-          errors = import_routine(*args)
+        ruby2_keywords def import!(*args)
+          errors = intercept_import_using_strategy(*args)
+
           raise Chewy::ImportFailed.new(self, errors) if errors.present?
+
           true
         end
 
@@ -93,7 +95,7 @@ module Chewy
         # `bulk_size` and `suffix`.
         #
         # @see https://github.com/elastic/elasticsearch-ruby/blob/master/elasticsearch-api/lib/elasticsearch/api/actions/bulk.rb
-        # @see Chewy::Type::Import::Bulk
+        # @see Chewy::Index::Import::BulkRequest
         # @param options [Hash{Symbol => Object}] besides specific import options, it accepts all the options suitable for the bulk API call like `refresh` or `timeout`
         # @option options [String] suffix bulk API chunk size in bytes; if passed, the request is performed several times for each chunk, empty by default
         # @option options [Integer] bulk_size bulk API chunk size in bytes; if passed, the request is performed several times for each chunk, empty by default
@@ -110,23 +112,51 @@ module Chewy
         # or normal composing under the hood.
         #
         # @param object [Object] a data source object
-        # @param crutches [Object] optional crutches object; if ommited - a crutch for the single passed object is created as a fallback
+        # @param crutches [Object] optional crutches object; if omitted - a crutch for the single passed object is created as a fallback
         # @param fields [Array<Symbol>] and array of fields to restrict the generated document
         # @return [Hash] a JSON-ready hash
         def compose(object, crutches = nil, fields: [])
-          crutches ||= Chewy::Type::Crutch::Crutches.new self, [object]
+          crutches ||= Chewy::Index::Crutch::Crutches.new self, [object]
 
-          if witchcraft? && build_root.children.present?
+          if witchcraft? && root.children.present?
             cauldron(fields: fields).brew(object, crutches)
           else
-            build_root.compose(object, crutches, fields: fields)
+            root.compose(object, crutches, fields: fields)
           end
         end
 
       private
 
+        def intercept_import_using_strategy(*args)
+          args_clone = args.deep_dup
+          options = args_clone.extract_options!
+          strategy = options.delete(:strategy)
+
+          return import_routine(*args) if strategy.blank?
+
+          ids = args_clone.flatten
+          return {} if ids.blank?
+          return {argument: {"#{strategy} supports ids only!" => ids}} unless ids.all? do |id|
+            id.respond_to?(:to_i)
+          end
+
+          case strategy
+          when :delayed_sidekiq
+            begin
+              Chewy::Strategy::DelayedSidekiq::Scheduler.new(self, ids, options).postpone
+              {} # success. errors handling convention
+            rescue StandardError => e
+              {scheduler: {e.message => ids}}
+            end
+          else
+            {argument: {"unsupported strategy: '#{strategy}'" => ids}}
+          end
+        end
+
         def import_routine(*args)
-          routine = Routine.new(self, args.extract_options!)
+          return if !args.first.nil? && empty_objects_or_scope?(args.first)
+
+          routine = Routine.new(self, **args.extract_options!)
           routine.create_indexes!
 
           if routine.parallel_options
@@ -136,8 +166,16 @@ module Chewy
           end
         end
 
+        def empty_objects_or_scope?(objects_or_scope)
+          if objects_or_scope.respond_to?(:empty?)
+            objects_or_scope.empty?
+          else
+            objects_or_scope.blank?
+          end
+        end
+
         def import_linear(objects, routine)
-          ActiveSupport::Notifications.instrument 'import_objects.chewy', type: self do |payload|
+          ActiveSupport::Notifications.instrument 'import_objects.chewy', index: self do |payload|
             adapter.import(*objects, routine.options) do |action_objects|
               routine.process(**action_objects)
             end
@@ -151,17 +189,25 @@ module Chewy
         def import_parallel(objects, routine)
           raise "The `parallel` gem is required for parallel import, please add `gem 'parallel'` to your Gemfile" unless '::Parallel'.safe_constantize
 
-          ActiveSupport::Notifications.instrument 'import_objects.chewy', type: self do |payload|
+          ActiveSupport::Notifications.instrument 'import_objects.chewy', index: self do |payload|
             batches = adapter.import_references(*objects, routine.options.slice(:batch_size)).to_a
 
             ::ActiveRecord::Base.connection.close if defined?(::ActiveRecord::Base)
-            results = ::Parallel.map_with_index(batches, routine.parallel_options, &IMPORT_WORKER.curry[self, routine.options, batches.size])
+            results = ::Parallel.map_with_index(
+              batches,
+              routine.parallel_options,
+              &IMPORT_WORKER.curry[self, routine.options, batches.size]
+            )
             ::ActiveRecord::Base.connection.reconnect! if defined?(::ActiveRecord::Base)
             errors, import, leftovers = process_parallel_import_results(results)
 
             if leftovers.present?
               batches = leftovers.each_slice(routine.options[:batch_size])
-              results = ::Parallel.map_with_index(batches, routine.parallel_options, &LEFTOVERS_WORKER.curry[self, routine.options, batches.size])
+              results = ::Parallel.map_with_index(
+                batches,
+                routine.parallel_options,
+                &LEFTOVERS_WORKER.curry[self, routine.options, batches.size]
+              )
               errors.concat(results.flatten(1))
             end
 

data/lib/chewy/{type → index}/mapping.rb

@@ -1,5 +1,5 @@
 module Chewy
-  class Type
+  class Index
     module Mapping
       extend ActiveSupport::Concern
 
@@ -13,47 +13,50 @@ module Chewy
       end
 
       module ClassMethods
-        # Defines root object for mapping and is optional for type
+        # Defines root object for mapping and is optional for index
         # definition. Use it only if you need to pass options for root
         # object mapping, such as `date_detection` or `dynamic_date_formats`
         #
+        # @example
         #   class UsersIndex < Chewy::Index
-        #     define_type User do
-        #       # root object defined implicitly and optionless for current type
-        #       field :full_name, type: 'string'
-        #     end
+        #     index_scope User
+        #     # root object defined implicitly and optionless for current type
+        #     field :full_name, type: 'keyword'
         #   end
         #
         #   class CarsIndex < Chewy::Index
-        #     define_type Car do
-        #       # explicit root definition with additional options
-        #       root dynamic_date_formats: ['yyyy-MM-dd'] do
-        #         field :model_name, type: 'string'
-        #       end
+        #     index_scope Car
+        #     # explicit root definition with additional options
+        #     root dynamic_date_formats: ['yyyy-MM-dd'] do
+        #       field :model_name, type: 'keyword'
         #     end
         #   end
         #
-        def root(options = {}, &block)
-          raise 'Root is already defined' if root_object
-          build_root(options, &block)
+        def root(**options)
+          self.root_object ||= Chewy::Fields::Root.new(:root, **Chewy.default_root_options.merge(options))
+          root_object.update_options!(**options)
+          yield if block_given?
+          root_object
         end
 
-        # Defines mapping field for current type
+        # Defines mapping field for index
         #
+        # @example
         #   class UsersIndex < Chewy::Index
-        #     define_type User do
-        #       # passing all the options to field definition:
-        #       field :full_name, type: 'string', analyzer: 'special'
-        #     end
+        #     index_scope User
+        #     # passing all the options to field definition:
+        #     field :full_name, analyzer: 'special'
         #   end
         #
         # The `type` is optional and defaults to `string` if not defined:
         #
+        # @example
         #   field :full_name
         #
         # Also, multiple fields might be defined with one call and
         # with the same options:
         #
+        # @example
         #   field :first_name, :last_name, analyzer: 'special'
         #
         # The only special option in the field definition
@@ -61,31 +64,35 @@ module Chewy
         # method will be called for the indexed object. Also
         # `:value` might be a proc or indexed object method name:
         #
+        # @example
         #   class User < ActiveRecord::Base
         #     def user_full_name
         #       [first_name, last_name].join(' ')
         #     end
         #   end
         #
-        #   field :full_name, type: 'string', value: :user_full_name
+        #   field :full_name, type: 'keyword', value: :user_full_name
         #
         # The proc evaluates inside the indexed object context if
         # its arity is 0 and in present contexts if there is an argument:
         #
-        #   field :full_name, type: 'string', value: -> { [first_name, last_name].join(' ') }
+        # @example
+        #   field :full_name, type: 'keyword', value: -> { [first_name, last_name].join(' ') }
         #
         #   separator = ' '
-        #   field :full_name, type: 'string', value: ->(user) { [user.first_name, user.last_name].join(separator) }
+        #   field :full_name, type: 'keyword', value: ->(user) { [user.first_name, user.last_name].join(separator) }
         #
         # If array was returned as value - it will be put in index as well.
         #
-        #   field :tags, type: 'string', value: -> { tags.map(&:name) }
+        # @example
+        #   field :tags, type: 'keyword', value: -> { tags.map(&:name) }
         #
         # Fields supports nesting in case of `object` field type. If
         # `user.quiz` will return an array of objects, then result index content
         # will be an array of hashes, if `user.quiz` is not a collection association
         # then just values hash will be put in the index.
         #
+        # @example
         #   field :quiz do
         #     field :question, :answer
         #     field :score, type: 'integer'
@@ -93,6 +100,7 @@ module Chewy
         #
         # Nested fields are composed from nested objects:
         #
+        # @example
         #   field :name, value: -> { name_translations } do
         #     field :ru, value: ->(name) { name['ru'] }
         #     field :en, value: ->(name) { name['en'] }
@@ -101,110 +109,103 @@ module Chewy
         # Of course it is possible to define object fields contents dynamically
         # but make sure evaluation proc returns hash:
         #
+        # @example
         #   field :name, type: 'object', value: -> { name_translations }
         #
         # The special case is multi_field. If type options and block are
         # both present field is treated as a multi-field. In that case field
         # composition changes satisfy elasticsearch rules:
         #
-        #   field :full_name, type: 'string', analyzer: 'name', value: ->{ full_name.try(:strip) } do
+        # @example
+        #   field :full_name, type: 'text', analyzer: 'name', value: ->{ full_name.try(:strip) } do
         #     field :sorted, analyzer: 'sorted'
         #   end
         #
-        def field(*args, &block)
-          options = args.extract_options!
-          build_root
-
+        def field(*args, **options, &block)
           if args.size > 1
-            args.map { |name| field(name, options) }
+            args.map { |name| field(name, **options) }
           else
-            expand_nested(Chewy::Fields::Base.new(args.first, options), &block)
+            expand_nested(Chewy::Fields::Base.new(args.first, **options), &block)
           end
         end
 
         # Defines an aggregation that can be bound to a query or filter
         #
-        #   Suppose that a user has posts and each post has ratings
-        #   avg_post_rating is the mean of all ratings
-        #
+        # @example
+        #   # Suppose that a user has posts and each post has ratings
+        #   # avg_post_rating is the mean of all ratings
         #   class UsersIndex < Chewy::Index
-        #     define_type User do
-        #       field :posts do
-        #         field :rating
-        #       end
-        #
-        #       agg :avg_rating do
-        #         { avg: { field: 'posts.rating' } }
-        #       end
+        #     index_scope User
+        #     field :posts do
+        #       field :rating
+        #     end
+        #
+        #     agg :avg_rating do
+        #       { avg: { field: 'posts.rating' } }
         #     end
         #   end
         def agg(name, &block)
-          build_root
           self._agg_defs = _agg_defs.merge(name => block)
         end
         alias_method :aggregation, :agg
 
         # Defines dynamic template in mapping root objects
         #
+        # @example
         #   class CarsIndex < Chewy::Index
-        #     define_type Car do
-        #       template 'model.*', type: 'string', analyzer: 'special'
-        #       field 'model', type: 'object' # here we can put { de: 'Der Mercedes', en: 'Mercedes' }
-        #                                     # and template will be applyed to this field
-        #     end
+        #     index_scope Car
+        #     template 'model.*', type: 'text', analyzer: 'special'
+        #     field 'model', type: 'object' # here we can put { de: 'Der Mercedes', en: 'Mercedes' }
+        #                                   # and template will be applied to this field
         #   end
         #
         # Name for each template is generated with the following
-        # rule: "template_#{dynamic_templates.size + 1}".
+        # rule: `template_#!{dynamic_templates.size + 1}`.
         #
+        # @example Templates
         #   template 'tit*', mapping_hash
         #   template 'title.*', mapping_hash # dot in template causes "path_match" using
         #   template /tit.+/, mapping_hash # using "match_pattern": "regexp"
         #   template /title\..+/, mapping_hash # "\." - escaped dot causes "path_match" using
-        #   template /tit.+/, 'string', mapping_hash # "match_mapping_type" as the optionsl second argument
+        #   template /tit.+/, type: 'text', mapping_hash # "match_mapping_type" as an optional second argument
         #   template template42: {match: 'hello*', mapping: {type: 'object'}} # or even pass a template as is
         #
-        def template(*args)
-          build_root.dynamic_template(*args)
+        def template(*args, **options)
+          root.dynamic_template(*args, **options)
         end
         alias_method :dynamic_template, :template
 
         # Returns compiled mappings hash for current type
         #
         def mappings_hash
-          root_object ? root_object.mappings_hash : {}
+          root.mappings_hash
         end
 
         # Check whether the type has outdated_sync_field defined with a simple value.
         #
         # @return [true, false]
         def supports_outdated_sync?
-          updated_at_field = root_object.child_hash[outdated_sync_field] if root_object && outdated_sync_field
+          updated_at_field = root.child_hash[outdated_sync_field] if outdated_sync_field
           !!updated_at_field && updated_at_field.value.nil?
         end
 
       private
 
-        def expand_nested(field, &block)
+        def expand_nested(field)
+          @_current_field ||= root
+
           if @_current_field
             field.parent = @_current_field
             @_current_field.children.push(field)
           end
 
-          return unless block
+          return unless block_given?
 
           previous_field = @_current_field
           @_current_field = field
           yield
           @_current_field = previous_field
         end
-
-        def build_root(options = {}, &block)
-          return root_object if root_object
-          self.root_object = Chewy::Fields::Root.new(type_name, Chewy.default_root_options.merge(options))
-          expand_nested(root_object, &block)
-          @_current_field = root_object
-        end
       end
     end
   end

data/lib/chewy/index/observe/active_record_methods.rb

@@ -0,0 +1,87 @@
+module Chewy
+  class Index
+    module Observe
+      module Helpers
+        def update_proc(index_name, *args, &block)
+          options = args.extract_options!
+          method = args.first
+
+          proc do
+            reference = if index_name.is_a?(Proc)
+              if index_name.arity.zero?
+                instance_exec(&index_name)
+              else
+                index_name.call(self)
+              end
+            else
+              index_name
+            end
+
+            index = Chewy.derive_name(reference)
+
+            next if Chewy.strategy.current.name == :bypass
+
+            backreference = if method && method.to_s == 'self'
+              self
+            elsif method
+              send(method)
+            else
+              instance_eval(&block)
+            end
+
+            index.update_index(backreference, options)
+          end
+        end
+
+        def extract_callback_options!(args)
+          options = args.extract_options!
+          result = options.each_key.with_object({}) do |key, hash|
+            hash[key] = options.delete(key) if %i[if unless].include?(key)
+          end
+          args.push(options) unless options.empty?
+          result
+        end
+      end
+
+      extend Helpers
+
+      module ActiveRecordMethods
+        extend ActiveSupport::Concern
+
+        def run_chewy_callbacks
+          chewy_callbacks.each { |callback| callback.call(self) }
+        end
+
+        def update_chewy_indices
+          Chewy.strategy.current.update_chewy_indices(self)
+        end
+
+        included do
+          class_attribute :chewy_callbacks, default: []
+        end
+
+        class_methods do
+          def initialize_chewy_callbacks
+            if Chewy.use_after_commit_callbacks
+              after_commit :update_chewy_indices, on: %i[create update]
+              after_commit :run_chewy_callbacks, on: :destroy
+            else
+              after_save :update_chewy_indices
+              after_destroy :run_chewy_callbacks
+            end
+          end
+
+          ruby2_keywords def update_index(type_name, *args, &block)
+            callback_options = Observe.extract_callback_options!(args)
+            update_proc = Observe.update_proc(type_name, *args, &block)
+            callback = Chewy::Index::Observe::Callback.new(update_proc, callback_options)
+
+            initialize_chewy_callbacks if chewy_callbacks.empty?
+
+            self.chewy_callbacks = chewy_callbacks.dup << callback
+          end
+        end
+      end
+    end
+  end
+end

data/lib/chewy/index/observe/callback.rb

@@ -0,0 +1,34 @@
+module Chewy
+  class Index
+    module Observe
+      class Callback
+        def initialize(executable, filters = {})
+          @executable = executable
+          @if_filter = filters[:if]
+          @unless_filter = filters[:unless]
+        end
+
+        def call(context)
+          return if !@if_filter.nil? && !eval_filter(@if_filter, context)
+          return if !@unless_filter.nil? && eval_filter(@unless_filter, context)
+
+          eval_proc(@executable, context)
+        end
+
+      private
+
+        def eval_filter(filter, context)
+          case filter
+          when Symbol then context.send(filter)
+          when Proc then eval_proc(filter, context)
+          else filter
+          end
+        end
+
+        def eval_proc(executable, context)
+          executable.arity.zero? ? context.instance_exec(&executable) : executable.call(context)
+        end
+      end
+    end
+  end
+end

data/lib/chewy/index/observe.rb

@@ -0,0 +1,17 @@
+require 'chewy/index/observe/callback'
+require 'chewy/index/observe/active_record_methods'
+
+module Chewy
+  class Index
+    module Observe
+      extend ActiveSupport::Concern
+
+      module ClassMethods
+        def update_index(objects, options = {})
+          Chewy.strategy.current.update(self, objects, options)
+          true
+        end
+      end
+    end
+  end
+end

data/lib/chewy/index/settings.rb

@@ -4,12 +4,14 @@ module Chewy
     # hash. At first, you need to store some analyzers or other
     # analysis options to the corresponding repository:
     #
+    # @example
     #   Chewy.analyzer :title_analyzer, type: 'custom', filter: %w(lowercase icu_folding title_nysiis)
     #   Chewy.filter :title_nysiis, type: 'phonetic', encoder: 'nysiis', replace: false
     #
     # `title_nysiis` filter here will be expanded automatically when
     # `title_analyzer` analyser will be used in index settings:
     #
+    # @example
     #   class ProductsIndex < Chewy::Index
     #     settings analysis: {
     #       analyzer: [