chewy 0.10.1 → 7.3.2

data/lib/chewy/index/specification.rb

@@ -2,21 +2,21 @@ module Chewy
   class Index
     # Index specification is a combination of index settings and
     # mappings. The idea behind this class is that specification
-    # can be locked in the `Chewy::Stash` between resets, so it is
-    # possible to track changes. In the future it is planned to
-    # be way smarter but right now `rake chewy:deploy` checks
-    # if there were changes and resets the index only if anything
-    # was changed. Otherwise, the index reset is skipped.
+    # can be locked in the `Chewy::Stash::Specification` between
+    # resets, so it is possible to track changes. In the future
+    # it is planned to be way smarter but right now `rake chewy:deploy`
+    # checks if there were changes and resets the index only if
+    # anything was changed. Otherwise, the index reset is skipped.
     #
-    # @see Chewy::Stash
+    # @see Chewy::Stash::Specification
     class Specification
-      # @see Chewy::Index.specification
+      # @see Chewy::Index::Specification
       # @param index [Chewy::Index] Just a chewy index
       def initialize(index)
         @index = index
       end
 
-      # Stores the current index specification to the `Chewy::Stash`
+      # Stores the current index specification to the `Chewy::Stash::Specification`
       # as json.
       #
       # @raise [Chewy::ImportFailed] if something went wrong
@@ -24,7 +24,7 @@ module Chewy
       def lock!
         Chewy::Stash::Specification.import!([
           id: @index.derivable_name,
-          value: current.to_json
+          specification: Base64.encode64(current.to_json)
         ], journal: false)
       end
 
@@ -34,7 +34,10 @@ module Chewy
       # @return [Hash] hash produced with JSON parser
       def locked
         filter = {ids: {values: [@index.derivable_name]}}
-        JSON.parse(Chewy::Stash::Specification.filter(filter).first.try!(:value) || '{}')
+        document = Chewy::Stash::Specification.filter(filter).first
+        return {} unless document
+
+        JSON.load(Base64.decode64(document.specification)) # rubocop:disable Security/JSONLoad
       end
 
       # Simply returns `Chewy::Index.specification_hash`, but

data/lib/chewy/{type → index}/syncer.rb

@@ -1,18 +1,18 @@
 module Chewy
-  class Type
+  class Index
     # This class is able to find missing and outdated documents in the ES
     # comparing ids from the data source and the ES index. Also, if `outdated_sync_field`
-    # existss in the index definition, it performs comparison of this field
+    # exists in the index definition, it performs comparison of this field
     # values for each source object and corresponding ES document. Usually,
     # this field is `updated_at` and if its value in the source is not equal
     # to the value in the index - this means that this document outdated and
     # should be reindexed.
     #
     # To fetch necessary data from the source it uses adapter method
-    # {Chewy::Type::Adapter::Base#import_fields}, in case when the Object
+    # {Chewy::Index::Adapter::Base#import_fields}, in case when the Object
     # adapter is used it makes sense to read corresponding documentation.
     #
-    # If `parallel` option is passed to the initializer - it will fetch surce and
+    # If `parallel` option is passed to the initializer - it will fetch source and
     # index data in parallel and then perform outdated objects calculation in
     # parallel processes. Also, further import (if required) will be performed
     # in parallel as well.
@@ -24,17 +24,17 @@ module Chewy
     #   ATTENTION: synchronization may be slow in case when synchronized tables
     #   are missing compound index on primary key and `outdated_sync_field`.
     #
-    # @see Chewy::Type::Actions::ClassMethods#sync
+    # @see Chewy::Index::Actions::ClassMethods#sync
     class Syncer
       DEFAULT_SYNC_BATCH_SIZE = 20_000
-      ISO_DATETIME = /\A(\d{4})-(\d\d)-(\d\d) (\d\d):(\d\d):(\d\d)(\.\d+)?\z/
-      OUTDATED_IDS_WORKER = lambda do |outdated_sync_field_type, source_data_hash, type, total, index_data|
-        ::Process.setproctitle("chewy [#{type}]: sync outdated calculation (#{::Parallel.worker_number + 1}/#{total})") if type
+      ISO_DATETIME = /\A(\d{4})-(\d\d)-(\d\d) (\d\d):(\d\d):(\d\d)(\.\d+)?\z/.freeze
+      OUTDATED_IDS_WORKER = lambda do |outdated_sync_field_type, source_data_hash, index, total, index_data|
+        ::Process.setproctitle("chewy [#{index}]: sync outdated calculation (#{::Parallel.worker_number + 1}/#{total})") if index
         index_data.each_with_object([]) do |(id, index_sync_value), result|
           next unless source_data_hash[id]
 
           outdated = if outdated_sync_field_type == 'date'
-            !Chewy::Type::Syncer.dates_equal(typecast_date(source_data_hash[id]), DateTime.iso8601(index_sync_value))
+            !Chewy::Index::Syncer.dates_equal(typecast_date(source_data_hash[id]), Time.iso8601(index_sync_value))
           else
             source_data_hash[id] != index_sync_value
           end
@@ -42,8 +42,8 @@ module Chewy
           result.push(id) if outdated
         end
       end
-      SOURCE_OR_INDEX_DATA_WORKER = lambda do |syncer, type, kind|
-        ::Process.setproctitle("chewy [#{type}]: sync fetching data (#{kind})")
+      SOURCE_OR_INDEX_DATA_WORKER = lambda do |syncer, index, kind|
+        ::Process.setproctitle("chewy [#{index}]: sync fetching data (#{kind})")
         result = case kind
         when :source
           syncer.send(:fetch_source_data)
@@ -56,8 +56,11 @@ module Chewy
       def self.typecast_date(string)
         if string.is_a?(String) && (match = ISO_DATETIME.match(string))
           microsec = (match[7].to_r * 1_000_000).to_i
-          date = "#{match[1]}-#{match[2]}-#{match[3]}T#{match[4]}:#{match[5]}:#{match[6]}.#{format('%06d', microsec)}+00:00"
-          DateTime.iso8601(date)
+          day = "#{match[1]}-#{match[2]}-#{match[3]}"
+          time_with_seconds = "#{match[4]}:#{match[5]}:#{match[6]}"
+          microseconds = format('%06d', microsec)
+          date = "#{day}T#{time_with_seconds}.#{microseconds}+00:00"
+          Time.iso8601(date)
         else
           string
         end
@@ -68,18 +71,10 @@ module Chewy
         [one.to_i, one.usec / 1000] == [two.to_i, two.usec / 1000]
       end
 
-      # In ActiveSupport ~> 4.0 json dumpled times without any
-      # milliseconds, so ES stored time with the seconds precision.
-      if ActiveSupport::VERSION::STRING < '4.1.0'
-        def self.dates_equal(one, two)
-          one.to_i == two.to_i
-        end
-      end
-
-      # @param type [Chewy::Type] chewy type
+      # @param index [Chewy::Index] chewy index
       # @param parallel [true, Integer, Hash] options for parallel execution or the number of processes
-      def initialize(type, parallel: nil)
-        @type = type
+      def initialize(index, parallel: nil)
+        @index = index
         @parallel = if !parallel || parallel.is_a?(Hash)
           parallel
         elsif parallel.is_a?(Integer)
@@ -95,7 +90,8 @@ module Chewy
       def perform
         ids = missing_ids | outdated_ids
         return 0 if ids.blank?
-        @type.import(ids, parallel: @parallel) && ids.count
+
+        @index.import(ids, parallel: @parallel) && ids.count
       end
 
       # Finds ids of all the objects that are not indexed yet or deleted
@@ -113,21 +109,19 @@ module Chewy
         end
       end
 
-      # If type supports outdated sync, it compares for the values of the
-      # type `outdated_sync_field` for each object and document in the source
-      # and index and returns the ids of entities which which are having
-      # different values there.
+      # If index supports outdated sync, it compares the values of the
+      # `outdated_sync_field` for each object and document in the source
+      # and index and returns the ids of entities which differ.
       #
-      # @see Chewy::Type::Mapping::ClassMethods#supports_outdated_sync?
+      # @see Chewy::Index::Mapping::ClassMethods#supports_outdated_sync?
       # @return [Array<String>] an array of outdated ids
       def outdated_ids
-        return [] if source_data.blank? || index_data.blank? || !@type.supports_outdated_sync?
-        @outdated_ids ||= begin
-          if @parallel
-            parallel_outdated_ids
-          else
-            linear_outdated_ids
-          end
+        return [] if source_data.blank? || index_data.blank? || !@index.supports_outdated_sync?
+
+        @outdated_ids ||= if @parallel
+          parallel_outdated_ids
+        else
+          linear_outdated_ids
         end
       end
 
@@ -142,44 +136,48 @@ module Chewy
       end
 
       def source_and_index_data
-        @source_and_index_data ||= begin
-          if @parallel
-            ::ActiveRecord::Base.connection.close if defined?(::ActiveRecord::Base)
-            result = ::Parallel.map(%i[source index], @parallel, &SOURCE_OR_INDEX_DATA_WORKER.curry[self, @type])
-            ::ActiveRecord::Base.connection.reconnect! if defined?(::ActiveRecord::Base)
-            if result.first.keys.first == :source
-              [result.first.values.first, result.second.values.first]
-            else
-              [result.second.values.first, result.first.values.first]
-            end
+        @source_and_index_data ||= if @parallel
+          ::ActiveRecord::Base.connection.close if defined?(::ActiveRecord::Base)
+          result = ::Parallel.map(%i[source index], @parallel, &SOURCE_OR_INDEX_DATA_WORKER.curry[self, @index])
+          ::ActiveRecord::Base.connection.reconnect! if defined?(::ActiveRecord::Base)
+          if result.first.keys.first == :source
+            [result.first.values.first, result.second.values.first]
           else
-            [fetch_source_data, fetch_index_data]
+            [result.second.values.first, result.first.values.first]
           end
+        else
+          [fetch_source_data, fetch_index_data]
         end
       end
 
       def fetch_source_data
-        if @type.supports_outdated_sync?
-          @type.adapter.import_fields(fields: [@type.outdated_sync_field], batch_size: DEFAULT_SYNC_BATCH_SIZE, typecast: false).to_a.flatten(1).each do |data|
+        if @index.supports_outdated_sync?
+          import_fields_args = {
+            fields: [@index.outdated_sync_field],
+            batch_size: DEFAULT_SYNC_BATCH_SIZE,
+            typecast: false
+          }
+          @index.adapter.import_fields(import_fields_args).to_a.flatten(1).each do |data|
             data[0] = data[0].to_s
           end
         else
-          @type.adapter.import_fields(batch_size: DEFAULT_SYNC_BATCH_SIZE, typecast: false).to_a.flatten(1).map(&:to_s)
+          @index.adapter.import_fields(batch_size: DEFAULT_SYNC_BATCH_SIZE, typecast: false).to_a.flatten(1).map(&:to_s)
         end
       end
 
       def fetch_index_data
-        if @type.supports_outdated_sync?
-          @type.pluck(:_id, @type.outdated_sync_field).each do |data|
+        if @index.supports_outdated_sync?
+          @index.pluck(:_id, @index.outdated_sync_field).each do |data|
             data[0] = data[0].to_s
           end
         else
-          @type.pluck(:_id).map(&:to_s)
+          @index.pluck(:_id).map(&:to_s)
         end
       end
 
       def data_ids(data)
-        return data unless @type.supports_outdated_sync?
+        return data unless @index.supports_outdated_sync?
+
         data.map(&:first)
       end
 
@@ -192,7 +190,12 @@ module Chewy
         batches = index_data.each_slice(size)
 
         ::ActiveRecord::Base.connection.close if defined?(::ActiveRecord::Base)
-        result = ::Parallel.map(batches, @parallel, &OUTDATED_IDS_WORKER.curry[outdated_sync_field_type, source_data.to_h, @type, batches.size]).flatten(1)
+        curried_outdated_ids_worker = OUTDATED_IDS_WORKER.curry[outdated_sync_field_type, source_data.to_h, @index, batches.size]
+        result = ::Parallel.map(
+          batches,
+          @parallel,
+          &curried_outdated_ids_worker
+        ).flatten(1)
         ::ActiveRecord::Base.connection.reconnect! if defined?(::ActiveRecord::Base)
         result
       end
@@ -203,17 +206,13 @@ module Chewy
 
       def outdated_sync_field_type
         return @outdated_sync_field_type if instance_variable_defined?(:@outdated_sync_field_type)
-        return unless @type.outdated_sync_field
+        return unless @index.outdated_sync_field
 
-        mappings = @type.client.indices.get_mapping(
-          index: @type.index_name,
-          type: @type.type_name
-        ).values.first.fetch('mappings', {})
+        mappings = @index.client.indices.get_mapping(index: @index.index_name).values.first.fetch('mappings', {})
 
         @outdated_sync_field_type = mappings
-          .fetch(@type.type_name, {})
           .fetch('properties', {})
-          .fetch(@type.outdated_sync_field.to_s, {})['type']
+          .fetch(@index.outdated_sync_field.to_s, {})['type']
       rescue Elasticsearch::Transport::Transport::Errors::NotFound
         nil
       end

data/lib/chewy/{type → index}/witchcraft.rb

@@ -7,7 +7,7 @@ rescue LoadError
 end
 
 module Chewy
-  class Type
+  class Index
     module Witchcraft
       extend ActiveSupport::Concern
 
@@ -43,10 +43,10 @@ module Chewy
       class Cauldron
         attr_reader :locals
 
-        # @param type [Chewy::Type] type for composition
+        # @param index [Chewy::Index] index for composition
         # @param fields [Array<Symbol>] restricts the fields for composition
-        def initialize(type, fields: [])
-          @type = type
+        def initialize(index, fields: [])
+          @index = index
           @locals = []
           @fields = fields
         end
@@ -58,9 +58,9 @@ module Chewy
       private
 
         def alicorn
-          @alicorn ||= class_eval <<-RUBY
+          @alicorn ||= singleton_class.class_eval <<-RUBY, __FILE__, __LINE__ + 1
             -> (locals, object0, crutches) do
-              #{composed_values(@type.root_object, 0)}
+              #{composed_values(@index.root, 0)}
             end
           RUBY
         end
@@ -79,7 +79,9 @@ module Chewy
           if field.children.present? && !field.multi_field?
             <<-RUBY
               (result#{nesting} = #{fetcher}
-              if result#{nesting}.respond_to?(:to_ary)
+              if result#{nesting}.nil?
+                nil
+              elsif result#{nesting}.respond_to?(:to_ary)
                 result#{nesting}.map do |object#{nesting}|
                   #{composed_values(field, nesting)}
                 end
@@ -139,6 +141,7 @@ module Chewy
 
         def non_proc_fields_for(parent, nesting)
           return [] unless parent
+
           fields = (parent.children || []).reject { |field| field.value.is_a?(Proc) }
 
           if nesting.zero? && @fields.present?
@@ -150,6 +153,7 @@ module Chewy
 
         def proc_fields_for(parent, nesting)
           return [] unless parent
+
           fields = (parent.children || []).select { |field| field.value.is_a?(Proc) }
 
           if nesting.zero? && @fields.present?
@@ -171,7 +175,7 @@ module Chewy
           if proc.arity.zero?
             source = replace_self(source, :"object#{nesting}")
             source = replace_send(source, :"object#{nesting}")
-          elsif proc.arity < 0
+          elsif proc.arity.negative?
             raise "Splat arguments are unsupported by witchcraft:\n`#{proc.source}"
           else
             (nesting + 1).times do |n|
@@ -190,6 +194,7 @@ module Chewy
 
         def exctract_lambdas(node)
           return unless node.is_a?(Parser::AST::Node)
+
           if node.type == :block && node.children[0].type == :send && node.children[0].to_a == [nil, :lambda]
             [node.children[2]]
           else
@@ -212,7 +217,7 @@ module Chewy
         def replace_send(node, variable)
           if node.is_a?(Parser::AST::Node)
             if node.type == :send && node.children[0].nil?
-              node.updated(nil, [Parser::AST::Node.new(:lvar, [variable]), *node.children[1..-1]])
+              node.updated(nil, [Parser::AST::Node.new(:lvar, [variable]), *node.children[1..]])
             else
               node.updated(nil, node.children.map { |child| replace_send(child, variable) })
             end
@@ -251,6 +256,7 @@ module Chewy
 
         def binding_variable_list(node)
           return unless node.is_a?(Parser::AST::Node)
+
           if node.type == :send && node.children[0].nil?
             node.children[1]
           else

data/lib/chewy/{type → index}/wrapper.rb

@@ -1,5 +1,5 @@
 module Chewy
-  class Type
+  class Index
     module Wrapper
       extend ActiveSupport::Concern
 
@@ -28,7 +28,7 @@ module Chewy
       def ==(other)
         return true if super
 
-        if other.is_a?(Chewy::Type)
+        if other.is_a?(Chewy::Index)
           self.class == other.class && (respond_to?(:id) ? id == other.id : attributes == other.attributes)
         elsif other.respond_to?(:id)
           self.class.adapter.target.is_a?(Class) &&
@@ -41,7 +41,7 @@ module Chewy
 
       %w[_id _type _index].each do |name|
         define_method name do
-          data[name]
+          _data[name]
         end
       end
 
@@ -49,6 +49,8 @@ module Chewy
         m = method.to_s
         if (name = highlight_name(m))
           highlight(name)
+        elsif (name = highlight_names(m))
+          highlights(name)
         elsif @attributes.key?(m)
           @attributes[m]
         elsif attribute_defined?(m)
@@ -60,7 +62,7 @@ module Chewy
 
       def respond_to_missing?(method, include_private = false)
         m = method.to_s
-        highlight_name(m) || @attributes.key?(m) || attribute_defined?(m) || super
+        highlight_name(m) || highlight_names(m) || @attributes.key?(m) || attribute_defined?(m) || super
       end
 
     private
@@ -69,12 +71,20 @@ module Chewy
         method.sub(/_highlight\z/, '') if method.end_with?('_highlight')
       end
 
+      def highlight_names(method)
+        method.sub(/_highlights\z/, '') if method.end_with?('_highlights')
+      end
+
       def attribute_defined?(attribute)
-        self.class.root_object && self.class.root_object.children.find { |a| a.name.to_s == attribute }.present?
+        self.class.root && self.class.root.children.find { |a| a.name.to_s == attribute }.present?
       end
 
       def highlight(attribute)
-        _data['highlight'][attribute].first
+        _data['highlight'][attribute].first if highlight?(attribute)
+      end
+
+      def highlights(attribute)
+        _data['highlight'][attribute] if highlight?(attribute)
       end
 
       def highlight?(attribute)