chewy 0.8.4 → 7.3.4

data/lib/chewy/index/import.rb

@@ -0,0 +1,243 @@
+require 'chewy/index/import/journal_builder'
+require 'chewy/index/import/bulk_builder'
+require 'chewy/index/import/bulk_request'
+require 'chewy/index/import/routine'
+
+module Chewy
+  class Index
+    module Import
+      extend ActiveSupport::Concern
+
+      IMPORT_WORKER = lambda do |index, options, total, ids, iteration|
+        ::Process.setproctitle("chewy [#{index}]: import data (#{iteration + 1}/#{total})")
+        routine = Routine.new(index, **options)
+        index.adapter.import(*ids, routine.options) do |action_objects|
+          routine.process(**action_objects)
+        end
+        {errors: routine.errors, import: routine.stats, leftovers: routine.leftovers}
+      end
+
+      LEFTOVERS_WORKER = lambda do |index, options, total, body, iteration|
+        ::Process.setproctitle("chewy [#{index}]: import leftovers (#{iteration + 1}/#{total})")
+        routine = Routine.new(index, **options)
+        routine.perform_bulk(body)
+        routine.errors
+      end
+
+      module ClassMethods
+        # @!method import(*collection, **options)
+        # Basically, one of the main methods for an index. Performs any objects import
+        # to the index. Does all the objects handling routines.
+        # Performs document import by utilizing bulk API. Bulk size and objects batch
+        # size are controlled by the corresponding options.
+        #
+        # It accepts ORM/ODM objects, PORO, hashes, ids which are used by adapter to
+        # fetch objects from the source depending on the used adapter. It destroys
+        # passed objects from the index if they are not in the default scope
+        # or marked for destruction.
+        #
+        # It handles parent-child relationships with a join field reindexing children when the parent is reindexed.
+        #
+        # Performs journaling if enabled: it stores all the ids of the imported
+        # objects to a specialized index. It is possible to replay particular import
+        # later to restore the data consistency.
+        #
+        # Performs partial index update using `update` bulk action if any `fields` are
+        # specified. Note that if document doesn't exist yet, an error will be raised
+        # by ES, but import catches this an errors and performs full indexing
+        # for the corresponding documents. This feature can be disabled by setting
+        # `update_failover` to `false`.
+        #
+        # Utilizes `ActiveSupport::Notifications`, so it is possible to get imported
+        # objects later by listening to the `import_objects.chewy` queue. It is also
+        # possible to get the list of occurred errors from the payload if something
+        # went wrong.
+        #
+        # Import can also be run in parallel using the Parallel gem functionality.
+        #
+        # @example
+        #   UsersIndex.import(parallel: true) # imports everything in parallel with automatic workers number
+        #   UsersIndex.import(parallel: 3) # using 3 workers
+        #   UsersIndex.import(parallel: {in_threads: 10}) # in 10 threads
+        #
+        # @see https://github.com/elastic/elasticsearch-ruby/blob/master/elasticsearch-api/lib/elasticsearch/api/actions/bulk.rb
+        # @param collection [Array<Object>] and array or anything to import
+        # @param options [Hash{Symbol => Object}] besides specific import options, it accepts all the options suitable for the bulk API call like `refresh` or `timeout`
+        # @option options [String] suffix an index name suffix, used for zero-downtime reset mostly, no suffix by default
+        # @option options [Integer] bulk_size bulk API chunk size in bytes; if passed, the request is performed several times for each chunk, empty by default
+        # @option options [Integer] batch_size passed to the adapter import method, used to split imported objects in chunks, 1000 by default
+        # @option options [Boolean] direct_import skips object reloading in ORM adapter, `false` by default
+        # @option options [true, false] journal enables imported objects journaling, false by default
+        # @option options [Array<Symbol, String>] update_fields list of fields for the partial import, empty by default
+        # @option options [true, false] update_failover enables full objects reimport in cases of partial update errors, `true` by default
+        # @option options [true, Integer, Hash] parallel enables parallel import processing with the Parallel gem, accepts the number of workers or any Parallel gem acceptable options
+        # @return [true, false] false in case of errors
+        ruby2_keywords def import(*args)
+          intercept_import_using_strategy(*args).blank?
+        end
+
+        # @!method import!(*collection, **options)
+        # (see #import)
+        #
+        # The only difference from {#import} is that it raises an exception
+        # in case of any import errors.
+        #
+        # @raise [Chewy::ImportFailed] in case of errors
+        ruby2_keywords def import!(*args)
+          errors = intercept_import_using_strategy(*args)
+
+          raise Chewy::ImportFailed.new(self, errors) if errors.present?
+
+          true
+        end
+
+        # Wraps elasticsearch API bulk method, adds additional features like
+        # `bulk_size` and `suffix`.
+        #
+        # @see https://github.com/elastic/elasticsearch-ruby/blob/master/elasticsearch-api/lib/elasticsearch/api/actions/bulk.rb
+        # @see Chewy::Index::Import::BulkRequest
+        # @param options [Hash{Symbol => Object}] besides specific import options, it accepts all the options suitable for the bulk API call like `refresh` or `timeout`
+        # @option options [String] suffix bulk API chunk size in bytes; if passed, the request is performed several times for each chunk, empty by default
+        # @option options [Integer] bulk_size bulk API chunk size in bytes; if passed, the request is performed several times for each chunk, empty by default
+        # @option options [Array<Hash>] body elasticsearch API bulk method body
+        # @return [Hash] tricky transposed errors hash, empty if everything is fine
+        def bulk(**options)
+          error_items = BulkRequest.new(self, **options).perform(options[:body])
+          Chewy.wait_for_status
+
+          payload_errors(error_items)
+        end
+
+        # Composes a single document from the passed object. Uses either witchcraft
+        # or normal composing under the hood.
+        #
+        # @param object [Object] a data source object
+        # @param crutches [Object] optional crutches object; if omitted - a crutch for the single passed object is created as a fallback
+        # @param fields [Array<Symbol>] and array of fields to restrict the generated document
+        # @return [Hash] a JSON-ready hash
+        def compose(object, crutches = nil, fields: [])
+          crutches ||= Chewy::Index::Crutch::Crutches.new self, [object]
+
+          if witchcraft? && root.children.present?
+            cauldron(fields: fields).brew(object, crutches)
+          else
+            root.compose(object, crutches, fields: fields)
+          end
+        end
+
+      private
+
+        def intercept_import_using_strategy(*args)
+          args_clone = args.deep_dup
+          options = args_clone.extract_options!
+          strategy = options.delete(:strategy)
+
+          return import_routine(*args) if strategy.blank?
+
+          ids = args_clone.flatten
+          return {} if ids.blank?
+          return {argument: {"#{strategy} supports ids only!" => ids}} unless ids.all? do |id|
+            id.respond_to?(:to_i)
+          end
+
+          case strategy
+          when :delayed_sidekiq
+            begin
+              Chewy::Strategy::DelayedSidekiq::Scheduler.new(self, ids, options).postpone
+              {} # success. errors handling convention
+            rescue StandardError => e
+              {scheduler: {e.message => ids}}
+            end
+          else
+            {argument: {"unsupported strategy: '#{strategy}'" => ids}}
+          end
+        end
+
+        def import_routine(*args)
+          return if !args.first.nil? && empty_objects_or_scope?(args.first)
+
+          routine = Routine.new(self, **args.extract_options!)
+          routine.create_indexes!
+
+          if routine.parallel_options
+            import_parallel(args, routine)
+          else
+            import_linear(args, routine)
+          end
+        end
+
+        def empty_objects_or_scope?(objects_or_scope)
+          if objects_or_scope.respond_to?(:empty?)
+            objects_or_scope.empty?
+          else
+            objects_or_scope.blank?
+          end
+        end
+
+        def import_linear(objects, routine)
+          ActiveSupport::Notifications.instrument 'import_objects.chewy', index: self do |payload|
+            adapter.import(*objects, routine.options) do |action_objects|
+              routine.process(**action_objects)
+            end
+            routine.perform_bulk(routine.leftovers)
+            payload[:import] = routine.stats
+            payload[:errors] = payload_errors(routine.errors) if routine.errors.present?
+            payload[:errors]
+          end
+        end
+
+        def import_parallel(objects, routine)
+          raise "The `parallel` gem is required for parallel import, please add `gem 'parallel'` to your Gemfile" unless '::Parallel'.safe_constantize
+
+          ActiveSupport::Notifications.instrument 'import_objects.chewy', index: self do |payload|
+            batches = adapter.import_references(*objects, routine.options.slice(:batch_size)).to_a
+
+            ::ActiveRecord::Base.connection.close if defined?(::ActiveRecord::Base)
+            results = ::Parallel.map_with_index(
+              batches,
+              routine.parallel_options,
+              &IMPORT_WORKER.curry[self, routine.options, batches.size]
+            )
+            ::ActiveRecord::Base.connection.reconnect! if defined?(::ActiveRecord::Base)
+            errors, import, leftovers = process_parallel_import_results(results)
+
+            if leftovers.present?
+              batches = leftovers.each_slice(routine.options[:batch_size])
+              results = ::Parallel.map_with_index(
+                batches,
+                routine.parallel_options,
+                &LEFTOVERS_WORKER.curry[self, routine.options, batches.size]
+              )
+              errors.concat(results.flatten(1))
+            end
+
+            payload[:import] = import
+            payload[:errors] = payload_errors(errors) if errors.present?
+            payload[:errors]
+          end
+        end
+
+        def process_parallel_import_results(results)
+          results.each_with_object([[], {}, []]) do |r, (e, i, l)|
+            e.concat(r[:errors])
+            i.merge!(r[:import]) { |_k, v1, v2| v1.to_i + v2.to_i }
+            l.concat(r[:leftovers])
+          end
+        end
+
+        def payload_errors(errors)
+          errors.each_with_object({}) do |error, result|
+            action = error.keys.first.to_sym
+            item = error.values.first
+            error = item['error']
+            id = item['_id']
+
+            result[action] ||= {}
+            result[action][error] ||= []
+            result[action][error].push(id)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/chewy/{type → index}/mapping.rb

@@ -1,5 +1,5 @@
 module Chewy
-  class Type
+  class Index
     module Mapping
       extend ActiveSupport::Concern
 
@@ -8,50 +8,55 @@ module Chewy
         class_attribute :_templates
         class_attribute :_agg_defs
         self._agg_defs = {}
+        class_attribute :outdated_sync_field
+        self.outdated_sync_field = :updated_at
       end
 
       module ClassMethods
-        # Defines root object for mapping and is optional for type
+        # Defines root object for mapping and is optional for index
         # definition. Use it only if you need to pass options for root
         # object mapping, such as `date_detection` or `dynamic_date_formats`
         #
+        # @example
         #   class UsersIndex < Chewy::Index
-        #     define_type User do
-        #       # root object defined implicitly and optionless for current type
-        #       field :full_name, type: 'string'
-        #     end
+        #     index_scope User
+        #     # root object defined implicitly and optionless for current type
+        #     field :full_name, type: 'keyword'
         #   end
         #
         #   class CarsIndex < Chewy::Index
-        #     define_type Car do
-        #       # explicit root definition with additional options
-        #       root dynamic_date_formats: ['yyyy-MM-dd'] do
-        #         field :model_name, type: 'string'
-        #       end
+        #     index_scope Car
+        #     # explicit root definition with additional options
+        #     root dynamic_date_formats: ['yyyy-MM-dd'] do
+        #       field :model_name, type: 'keyword'
         #     end
         #   end
         #
-        def root options = {}, &block
-          raise "Root is already defined" if root_object
-          build_root(options, &block)
+        def root(**options)
+          self.root_object ||= Chewy::Fields::Root.new(:root, **Chewy.default_root_options.merge(options))
+          root_object.update_options!(**options)
+          yield if block_given?
+          root_object
         end
 
-        # Defines mapping field for current type
+        # Defines mapping field for index
         #
+        # @example
         #   class UsersIndex < Chewy::Index
-        #     define_type User do
-        #       # passing all the options to field definition:
-        #       field :full_name, type: 'string', analyzer: 'special'
-        #     end
+        #     index_scope User
+        #     # passing all the options to field definition:
+        #     field :full_name, analyzer: 'special'
         #   end
         #
         # The `type` is optional and defaults to `string` if not defined:
         #
+        # @example
         #   field :full_name
         #
         # Also, multiple fields might be defined with one call and
         # with the same options:
         #
+        # @example
         #   field :first_name, :last_name, analyzer: 'special'
         #
         # The only special option in the field definition
@@ -59,31 +64,35 @@ module Chewy
         # method will be called for the indexed object. Also
         # `:value` might be a proc or indexed object method name:
         #
+        # @example
         #   class User < ActiveRecord::Base
         #     def user_full_name
         #       [first_name, last_name].join(' ')
         #     end
         #   end
         #
-        #   field :full_name, type: 'string', value: :user_full_name
+        #   field :full_name, type: 'keyword', value: :user_full_name
         #
         # The proc evaluates inside the indexed object context if
         # its arity is 0 and in present contexts if there is an argument:
         #
-        #   field :full_name, type: 'string', value: -> { [first_name, last_name].join(' ') }
+        # @example
+        #   field :full_name, type: 'keyword', value: -> { [first_name, last_name].join(' ') }
         #
         #   separator = ' '
-        #   field :full_name, type: 'string', value: ->(user) { [user.first_name, user.last_name].join(separator) }
+        #   field :full_name, type: 'keyword', value: ->(user) { [user.first_name, user.last_name].join(separator) }
         #
         # If array was returned as value - it will be put in index as well.
         #
-        #   field :tags, type: 'string', value: -> { tags.map(&:name) }
+        # @example
+        #   field :tags, type: 'keyword', value: -> { tags.map(&:name) }
         #
         # Fields supports nesting in case of `object` field type. If
         # `user.quiz` will return an array of objects, then result index content
         # will be an array of hashes, if `user.quiz` is not a collection association
         # then just values hash will be put in the index.
         #
+        # @example
         #   field :quiz do
         #     field :question, :answer
         #     field :score, type: 'integer'
@@ -91,109 +100,111 @@ module Chewy
         #
         # Nested fields are composed from nested objects:
         #
+        # @example
         #   field :name, value: -> { name_translations } do
         #     field :ru, value: ->(name) { name['ru'] }
         #     field :en, value: ->(name) { name['en'] }
         #   end
         #
-        # Off course it is possible to define object fields contents dynamically
+        # Of course it is possible to define object fields contents dynamically
         # but make sure evaluation proc returns hash:
         #
+        # @example
         #   field :name, type: 'object', value: -> { name_translations }
         #
         # The special case is multi_field. If type options and block are
         # both present field is treated as a multi-field. In that case field
         # composition changes satisfy elasticsearch rules:
         #
-        #   field :full_name, type: 'string', analyzer: 'name', value: ->{ full_name.try(:strip) } do
+        # @example
+        #   field :full_name, type: 'text', analyzer: 'name', value: ->{ full_name.try(:strip) } do
         #     field :sorted, analyzer: 'sorted'
         #   end
         #
-        def field *args, &block
-          options = args.extract_options!
-          build_root
-
+        def field(*args, **options, &block)
           if args.size > 1
-            args.map { |name| field(name, options) }
+            args.map { |name| field(name, **options) }
           else
-            expand_nested(Chewy::Fields::Base.new(args.first, options), &block)
+            expand_nested(Chewy::Fields::Base.new(args.first, **options), &block)
           end
         end
 
         # Defines an aggregation that can be bound to a query or filter
         #
-        #   Suppose that a user has posts and each post has ratings
-        #   avg_post_rating is the mean of all ratings
-        #
+        # @example
+        #   # Suppose that a user has posts and each post has ratings
+        #   # avg_post_rating is the mean of all ratings
         #   class UsersIndex < Chewy::Index
-        #     define_type User do
-        #       field :posts do
-        #         field :rating
-        #       end
-        #
-        #       agg :avg_rating do
-        #         { avg: { field: 'posts.rating' } }
-        #       end
+        #     index_scope User
+        #     field :posts do
+        #       field :rating
+        #     end
+        #
+        #     agg :avg_rating do
+        #       { avg: { field: 'posts.rating' } }
         #     end
         #   end
-        def agg *args, &block
-          options = args.extract_options!
-          build_root
-          self._agg_defs = _agg_defs.merge(args.first => block)
+        def agg(name, &block)
+          self._agg_defs = _agg_defs.merge(name => block)
         end
         alias_method :aggregation, :agg
 
         # Defines dynamic template in mapping root objects
         #
+        # @example
         #   class CarsIndex < Chewy::Index
-        #     define_type Car do
-        #       template 'model.*', type: 'string', analyzer: 'special'
-        #       field 'model', type: 'object' # here we can put { ru: 'Мерседес', en: 'Mercedes' }
-        #                                     # and template will be applyed to this field
-        #     end
+        #     index_scope Car
+        #     template 'model.*', type: 'text', analyzer: 'special'
+        #     field 'model', type: 'object' # here we can put { de: 'Der Mercedes', en: 'Mercedes' }
+        #                                   # and template will be applied to this field
         #   end
         #
         # Name for each template is generated with the following
-        # rule: "template_#{dynamic_templates.size + 1}".
+        # rule: `template_#!{dynamic_templates.size + 1}`.
         #
+        # @example Templates
         #   template 'tit*', mapping_hash
         #   template 'title.*', mapping_hash # dot in template causes "path_match" using
         #   template /tit.+/, mapping_hash # using "match_pattern": "regexp"
         #   template /title\..+/, mapping_hash # "\." - escaped dot causes "path_match" using
-        #   template /tit.+/, 'string', mapping_hash # "match_mapping_type" as the optionsl second argument
+        #   template /tit.+/, type: 'text', mapping_hash # "match_mapping_type" as an optional second argument
         #   template template42: {match: 'hello*', mapping: {type: 'object'}} # or even pass a template as is
         #
-        def template *args
-          build_root.dynamic_template *args
+        def template(*args, **options)
+          root.dynamic_template(*args, **options)
         end
         alias_method :dynamic_template, :template
 
         # Returns compiled mappings hash for current type
         #
         def mappings_hash
-          root_object ? root_object.mappings_hash : {}
+          root.mappings_hash
+        end
+
+        # Check whether the type has outdated_sync_field defined with a simple value.
+        #
+        # @return [true, false]
+        def supports_outdated_sync?
+          updated_at_field = root.child_hash[outdated_sync_field] if outdated_sync_field
+          !!updated_at_field && updated_at_field.value.nil?
         end
 
       private
 
-        def expand_nested field, &block
+        def expand_nested(field)
+          @_current_field ||= root
+
           if @_current_field
             field.parent = @_current_field
             @_current_field.children.push(field)
           end
 
-          if block
-            previous_field, @_current_field = @_current_field, field
-            block.call
-            @_current_field = previous_field
-          end
-        end
+          return unless block_given?
 
-        def build_root options = {}, &block
-          return root_object if root_object
-          self.root_object = Chewy::Fields::Root.new(type_name, options)
-          expand_nested(root_object, &block)
-          @_current_field = root_object
+          previous_field = @_current_field
+          @_current_field = field
+          yield
+          @_current_field = previous_field
         end
       end
     end

data/lib/chewy/index/observe/active_record_methods.rb

@@ -0,0 +1,87 @@
+module Chewy
+  class Index
+    module Observe
+      module Helpers
+        def update_proc(index_name, *args, &block)
+          options = args.extract_options!
+          method = args.first
+
+          proc do
+            reference = if index_name.is_a?(Proc)
+              if index_name.arity.zero?
+                instance_exec(&index_name)
+              else
+                index_name.call(self)
+              end
+            else
+              index_name
+            end
+
+            index = Chewy.derive_name(reference)
+
+            next if Chewy.strategy.current.name == :bypass
+
+            backreference = if method && method.to_s == 'self'
+              self
+            elsif method
+              send(method)
+            else
+              instance_eval(&block)
+            end
+
+            index.update_index(backreference, options)
+          end
+        end
+
+        def extract_callback_options!(args)
+          options = args.extract_options!
+          result = options.each_key.with_object({}) do |key, hash|
+            hash[key] = options.delete(key) if %i[if unless].include?(key)
+          end
+          args.push(options) unless options.empty?
+          result
+        end
+      end
+
+      extend Helpers
+
+      module ActiveRecordMethods
+        extend ActiveSupport::Concern
+
+        def run_chewy_callbacks
+          chewy_callbacks.each { |callback| callback.call(self) }
+        end
+
+        def update_chewy_indices
+          Chewy.strategy.current.update_chewy_indices(self)
+        end
+
+        included do
+          class_attribute :chewy_callbacks, default: []
+        end
+
+        class_methods do
+          def initialize_chewy_callbacks
+            if Chewy.use_after_commit_callbacks
+              after_commit :update_chewy_indices, on: %i[create update]
+              after_commit :run_chewy_callbacks, on: :destroy
+            else
+              after_save :update_chewy_indices
+              after_destroy :run_chewy_callbacks
+            end
+          end
+
+          ruby2_keywords def update_index(type_name, *args, &block)
+            callback_options = Observe.extract_callback_options!(args)
+            update_proc = Observe.update_proc(type_name, *args, &block)
+            callback = Chewy::Index::Observe::Callback.new(update_proc, callback_options)
+
+            initialize_chewy_callbacks if chewy_callbacks.empty?
+
+            self.chewy_callbacks = chewy_callbacks.dup << callback
+          end
+        end
+      end
+    end
+  end
+end

data/lib/chewy/index/observe/callback.rb

@@ -0,0 +1,34 @@
+module Chewy
+  class Index
+    module Observe
+      class Callback
+        def initialize(executable, filters = {})
+          @executable = executable
+          @if_filter = filters[:if]
+          @unless_filter = filters[:unless]
+        end
+
+        def call(context)
+          return if !@if_filter.nil? && !eval_filter(@if_filter, context)
+          return if !@unless_filter.nil? && eval_filter(@unless_filter, context)
+
+          eval_proc(@executable, context)
+        end
+
+      private
+
+        def eval_filter(filter, context)
+          case filter
+          when Symbol then context.send(filter)
+          when Proc then eval_proc(filter, context)
+          else filter
+          end
+        end
+
+        def eval_proc(executable, context)
+          executable.arity.zero? ? context.instance_exec(&executable) : executable.call(context)
+        end
+      end
+    end
+  end
+end

data/lib/chewy/index/observe.rb

@@ -0,0 +1,17 @@
+require 'chewy/index/observe/callback'
+require 'chewy/index/observe/active_record_methods'
+
+module Chewy
+  class Index
+    module Observe
+      extend ActiveSupport::Concern
+
+      module ClassMethods
+        def update_index(objects, options = {})
+          Chewy.strategy.current.update(self, objects, options)
+          true
+        end
+      end
+    end
+  end
+end