chewy 0.8.4 → 7.3.4

data/lib/chewy/index/settings.rb

@@ -1,16 +1,17 @@
 module Chewy
   class Index
-
     # Stores ElasticSearch index settings and resolves `analysis`
     # hash. At first, you need to store some analyzers or other
     # analysis options to the corresponding repository:
     #
+    # @example
     #   Chewy.analyzer :title_analyzer, type: 'custom', filter: %w(lowercase icu_folding title_nysiis)
     #   Chewy.filter :title_nysiis, type: 'phonetic', encoder: 'nysiis', replace: false
     #
     # `title_nysiis` filter here will be expanded automatically when
     # `title_analyzer` analyser will be used in index settings:
     #
+    # @example
     #   class ProductsIndex < Chewy::Index
     #     settings analysis: {
     #       analyzer: [
@@ -24,14 +25,17 @@ module Chewy
     # might be used as well.
     #
     class Settings
-      def initialize(params = {})
+      def initialize(params = {}, &block)
         @params = params
+        @proc_params = block
       end
 
       def to_hash
         settings = @params.deep_symbolize_keys
+        settings.merge!((@proc_params.call || {}).deep_symbolize_keys) if @proc_params
 
         settings[:analysis] = resolve_analysis(settings[:analysis]) if settings[:analysis]
+
         if settings[:index] || Chewy.configuration[:index]
           settings[:index] = (Chewy.configuration[:index] || {})
             .deep_merge((settings[:index] || {}).deep_symbolize_keys)
@@ -45,14 +49,15 @@ module Chewy
       def resolve_analysis(analysis)
         analyzer = resolve(analysis[:analyzer], Chewy.analyzers)
 
-        options = [:tokenizer, :filter, :char_filter].each.with_object({}) do |type, result|
+        options = %i[tokenizer filter char_filter].each.with_object({}) do |type, result|
           dependencies = collect_dependencies(type, analyzer)
           resolved = resolve(dependencies.push(analysis[type]), Chewy.send(type.to_s.pluralize))
           result.merge!(type => resolved) if resolved.present?
         end
 
-        options.merge!(analyzer: analyzer) if analyzer.present?
-        options
+        options[:analyzer] = analyzer if analyzer.present?
+        analysis = analysis.except(:analyzer, :tokenizer, :filter, :char_filter)
+        analysis.merge(options)
       end
 
       def collect_dependencies(type, analyzer)

data/lib/chewy/index/specification.rb

@@ -0,0 +1,61 @@
+module Chewy
+  class Index
+    # Index specification is a combination of index settings and
+    # mappings. The idea behind this class is that specification
+    # can be locked in the `Chewy::Stash::Specification` between
+    # resets, so it is possible to track changes. In the future
+    # it is planned to be way smarter but right now `rake chewy:deploy`
+    # checks if there were changes and resets the index only if
+    # anything was changed. Otherwise, the index reset is skipped.
+    #
+    # @see Chewy::Stash::Specification
+    class Specification
+      # @see Chewy::Index::Specification
+      # @param index [Chewy::Index] Just a chewy index
+      def initialize(index)
+        @index = index
+      end
+
+      # Stores the current index specification to the `Chewy::Stash::Specification`
+      # as json.
+      #
+      # @raise [Chewy::ImportFailed] if something went wrong
+      # @return [true] if everything is fine
+      def lock!
+        Chewy::Stash::Specification.import!([
+          id: @index.derivable_name,
+          specification: Base64.encode64(current.to_json)
+        ], journal: false)
+      end
+
+      # Returns the last locked specification as ruby hash. Returns
+      # empty hash if nothing is stored yet.
+      #
+      # @return [Hash] hash produced with JSON parser
+      def locked
+        filter = {ids: {values: [@index.derivable_name]}}
+        document = Chewy::Stash::Specification.filter(filter).first
+        return {} unless document
+
+        JSON.load(Base64.decode64(document.specification)) # rubocop:disable Security/JSONLoad
+      end
+
+      # Simply returns `Chewy::Index.specification_hash`, but
+      # prepared for JSON with `as_json` method. This means all the
+      # keys are strings and there are only values of types handled in JSON.
+      #
+      # @see Chewy::Index.specification_hash
+      # @return [Hash] a JSON-ready hash
+      def current
+        @index.specification_hash.as_json
+      end
+
+      # Compares previously locked and current specifications.
+      #
+      # @return [true, false] the result of comparison
+      def changed?
+        current != locked
+      end
+    end
+  end
+end

data/lib/chewy/index/syncer.rb

@@ -0,0 +1,221 @@
+module Chewy
+  class Index
+    # This class is able to find missing and outdated documents in the ES
+    # comparing ids from the data source and the ES index. Also, if `outdated_sync_field`
+    # exists in the index definition, it performs comparison of this field
+    # values for each source object and corresponding ES document. Usually,
+    # this field is `updated_at` and if its value in the source is not equal
+    # to the value in the index - this means that this document outdated and
+    # should be reindexed.
+    #
+    # To fetch necessary data from the source it uses adapter method
+    # {Chewy::Index::Adapter::Base#import_fields}, in case when the Object
+    # adapter is used it makes sense to read corresponding documentation.
+    #
+    # If `parallel` option is passed to the initializer - it will fetch source and
+    # index data in parallel and then perform outdated objects calculation in
+    # parallel processes. Also, further import (if required) will be performed
+    # in parallel as well.
+    #
+    # @note
+    #   In rails 4.0 time converted to json with the precision of seconds
+    #   without milliseconds used, so outdated check is not so precise there.
+    #
+    #   ATTENTION: synchronization may be slow in case when synchronized tables
+    #   are missing compound index on primary key and `outdated_sync_field`.
+    #
+    # @see Chewy::Index::Actions::ClassMethods#sync
+    class Syncer
+      DEFAULT_SYNC_BATCH_SIZE = 20_000
+      ISO_DATETIME = /\A(\d{4})-(\d\d)-(\d\d) (\d\d):(\d\d):(\d\d)(\.\d+)?\z/.freeze
+      OUTDATED_IDS_WORKER = lambda do |outdated_sync_field_type, source_data_hash, index, total, index_data|
+        ::Process.setproctitle("chewy [#{index}]: sync outdated calculation (#{::Parallel.worker_number + 1}/#{total})") if index
+        index_data.each_with_object([]) do |(id, index_sync_value), result|
+          next unless source_data_hash[id]
+
+          outdated = if outdated_sync_field_type == 'date'
+            !Chewy::Index::Syncer.dates_equal(typecast_date(source_data_hash[id]), Time.iso8601(index_sync_value))
+          else
+            source_data_hash[id] != index_sync_value
+          end
+
+          result.push(id) if outdated
+        end
+      end
+      SOURCE_OR_INDEX_DATA_WORKER = lambda do |syncer, index, kind|
+        ::Process.setproctitle("chewy [#{index}]: sync fetching data (#{kind})")
+        result = case kind
+        when :source
+          syncer.send(:fetch_source_data)
+        when :index
+          syncer.send(:fetch_index_data)
+        end
+        {kind => result}
+      end
+
+      def self.typecast_date(string)
+        if string.is_a?(String) && (match = ISO_DATETIME.match(string))
+          microsec = (match[7].to_r * 1_000_000).to_i
+          day = "#{match[1]}-#{match[2]}-#{match[3]}"
+          time_with_seconds = "#{match[4]}:#{match[5]}:#{match[6]}"
+          microseconds = format('%06d', microsec)
+          date = "#{day}T#{time_with_seconds}.#{microseconds}+00:00"
+          Time.iso8601(date)
+        else
+          string
+        end
+      end
+
+      # Compares times with ms precision.
+      def self.dates_equal(one, two)
+        [one.to_i, one.usec / 1000] == [two.to_i, two.usec / 1000]
+      end
+
+      # @param index [Chewy::Index] chewy index
+      # @param parallel [true, Integer, Hash] options for parallel execution or the number of processes
+      def initialize(index, parallel: nil)
+        @index = index
+        @parallel = if !parallel || parallel.is_a?(Hash)
+          parallel
+        elsif parallel.is_a?(Integer)
+          {in_processes: parallel}
+        else
+          {}
+        end
+      end
+
+      # Finds all the missing and outdated ids and performs import for them.
+      #
+      # @return [Integer, nil] the amount of missing and outdated documents reindexed, nil in case of errors
+      def perform
+        ids = missing_ids | outdated_ids
+        return 0 if ids.blank?
+
+        @index.import(ids, parallel: @parallel) && ids.count
+      end
+
+      # Finds ids of all the objects that are not indexed yet or deleted
+      # from the source already.
+      #
+      # @return [Array<String>] an array of missing ids from both sides
+      def missing_ids
+        return [] if source_data.blank?
+
+        @missing_ids ||= begin
+          source_data_ids = data_ids(source_data)
+          index_data_ids = data_ids(index_data)
+
+          (source_data_ids - index_data_ids).concat(index_data_ids - source_data_ids)
+        end
+      end
+
+      # If index supports outdated sync, it compares the values of the
+      # `outdated_sync_field` for each object and document in the source
+      # and index and returns the ids of entities which differ.
+      #
+      # @see Chewy::Index::Mapping::ClassMethods#supports_outdated_sync?
+      # @return [Array<String>] an array of outdated ids
+      def outdated_ids
+        return [] if source_data.blank? || index_data.blank? || !@index.supports_outdated_sync?
+
+        @outdated_ids ||= if @parallel
+          parallel_outdated_ids
+        else
+          linear_outdated_ids
+        end
+      end
+
+    private
+
+      def source_data
+        @source_data ||= source_and_index_data.first
+      end
+
+      def index_data
+        @index_data ||= source_and_index_data.second
+      end
+
+      def source_and_index_data
+        @source_and_index_data ||= if @parallel
+          ::ActiveRecord::Base.connection.close if defined?(::ActiveRecord::Base)
+          result = ::Parallel.map(%i[source index], @parallel, &SOURCE_OR_INDEX_DATA_WORKER.curry[self, @index])
+          ::ActiveRecord::Base.connection.reconnect! if defined?(::ActiveRecord::Base)
+          if result.first.keys.first == :source
+            [result.first.values.first, result.second.values.first]
+          else
+            [result.second.values.first, result.first.values.first]
+          end
+        else
+          [fetch_source_data, fetch_index_data]
+        end
+      end
+
+      def fetch_source_data
+        if @index.supports_outdated_sync?
+          import_fields_args = {
+            fields: [@index.outdated_sync_field],
+            batch_size: DEFAULT_SYNC_BATCH_SIZE,
+            typecast: false
+          }
+          @index.adapter.import_fields(import_fields_args).to_a.flatten(1).each do |data|
+            data[0] = data[0].to_s
+          end
+        else
+          @index.adapter.import_fields(batch_size: DEFAULT_SYNC_BATCH_SIZE, typecast: false).to_a.flatten(1).map(&:to_s)
+        end
+      end
+
+      def fetch_index_data
+        if @index.supports_outdated_sync?
+          @index.pluck(:_id, @index.outdated_sync_field).each do |data|
+            data[0] = data[0].to_s
+          end
+        else
+          @index.pluck(:_id).map(&:to_s)
+        end
+      end
+
+      def data_ids(data)
+        return data unless @index.supports_outdated_sync?
+
+        data.map(&:first)
+      end
+
+      def linear_outdated_ids
+        OUTDATED_IDS_WORKER.call(outdated_sync_field_type, source_data.to_h, nil, nil, index_data)
+      end
+
+      def parallel_outdated_ids
+        size = processor_count.zero? ? index_data.size : (index_data.size / processor_count.to_f).ceil
+        batches = index_data.each_slice(size)
+
+        ::ActiveRecord::Base.connection.close if defined?(::ActiveRecord::Base)
+        curried_outdated_ids_worker = OUTDATED_IDS_WORKER.curry[outdated_sync_field_type, source_data.to_h, @index, batches.size]
+        result = ::Parallel.map(
+          batches,
+          @parallel,
+          &curried_outdated_ids_worker
+        ).flatten(1)
+        ::ActiveRecord::Base.connection.reconnect! if defined?(::ActiveRecord::Base)
+        result
+      end
+
+      def processor_count
+        @processor_count ||= @parallel[:in_processes] || @parallel[:in_threads] || ::Parallel.processor_count
+      end
+
+      def outdated_sync_field_type
+        return @outdated_sync_field_type if instance_variable_defined?(:@outdated_sync_field_type)
+        return unless @index.outdated_sync_field
+
+        mappings = @index.client.indices.get_mapping(index: @index.index_name).values.first.fetch('mappings', {})
+
+        @outdated_sync_field_type = mappings
+          .fetch('properties', {})
+          .fetch(@index.outdated_sync_field.to_s, {})['type']
+      rescue Elasticsearch::Transport::Transport::Errors::NotFound
+        nil
+      end
+    end
+  end
+end

data/lib/chewy/{type → index}/witchcraft.rb

@@ -1,12 +1,13 @@
-def try_require lib
-  require lib
+begin
+  require 'method_source'
+  require 'parser/current'
+  require 'unparser'
 rescue LoadError
+  nil
 end
 
-%w[method_source parser/current unparser].each { |lib| try_require lib }
-
 module Chewy
-  class Type
+  class Index
     module Witchcraft
       extend ActiveSupport::Concern
 
@@ -22,9 +23,9 @@ module Chewy
 
         def check_requirements!
           messages = []
-          messages << "MethodSource gem is required for the Witchcraft™, please add `gem 'method_source'` to your Gemfile" unless Proc.method_defined?(:source)
-          messages << "Parser gem is required for the Witchcraft™, please add `gem 'parser'` to your Gemfile" unless '::Parser'.safe_constantize
-          messages << "Unparser gem is required for the Witchcraft™, please add `gem 'unparser'` to your Gemfile" unless '::Unparser'.safe_constantize
+          messages << "MethodSource gem is required for the Witchcraft, please add `gem 'method_source'` to your Gemfile" unless Proc.method_defined?(:source)
+          messages << "Parser gem is required for the Witchcraft, please add `gem 'parser'` to your Gemfile" unless '::Parser'.safe_constantize
+          messages << "Unparser gem is required for the Witchcraft, please add `gem 'unparser'` to your Gemfile" unless '::Unparser'.safe_constantize
           messages = messages.join("\n")
 
           raise messages if messages.present?
@@ -34,26 +35,32 @@ module Chewy
           !!_witchcraft
         end
 
-        def cauldron
-          @cauldron ||= Cauldron.new(self)
+        def cauldron(**options)
+          (@cauldron ||= {})[options] ||= Cauldron.new(self, **options)
         end
       end
 
       class Cauldron
-        def initialize(type)
-          @type = type
+        attr_reader :locals
+
+        # @param index [Chewy::Index] index for composition
+        # @param fields [Array<Symbol>] restricts the fields for composition
+        def initialize(index, fields: [])
+          @index = index
+          @locals = []
+          @fields = fields
         end
 
         def brew(object, crutches = nil)
-          alicorn.call(object, crutches).as_json
+          alicorn.call(locals, object, crutches).as_json
         end
 
       private
 
         def alicorn
-          @alicorn ||= class_eval <<-RUBY
-            -> (object0, crutches) do
-              #{composed_values(@type.root_object, 0)}
+          @alicorn ||= singleton_class.class_eval <<-RUBY, __FILE__, __LINE__ + 1
+            -> (locals, object0, crutches) do
+              #{composed_values(@index.root, 0)}
             end
           RUBY
         end
@@ -72,7 +79,9 @@ module Chewy
           if field.children.present? && !field.multi_field?
             <<-RUBY
               (result#{nesting} = #{fetcher}
-              if result#{nesting}.respond_to?(:to_ary)
+              if result#{nesting}.nil?
+                nil
+              elsif result#{nesting}.respond_to?(:to_ary)
                 result#{nesting}.map do |object#{nesting}|
                   #{composed_values(field, nesting)}
                 end
@@ -87,55 +96,71 @@ module Chewy
         end
 
         def non_proc_values(field, nesting)
-          non_proc_fields = non_proc_fields_for(field)
+          non_proc_fields = non_proc_fields_for(field, nesting)
           object = "object#{nesting}"
 
           if non_proc_fields.present?
             <<-RUBY
               (if #{object}.is_a?(Hash)
                 {
-                  #{non_proc_fields.map do |field|
-                    fetcher = "#{object}.has_key?(:#{field.name}) ? #{object}[:#{field.name}] : #{object}['#{field.name}']"
-                    "#{field.name}: #{composed_value(field, fetcher, nesting)}"
+                  #{non_proc_fields.map do |f|
+                    key_name = f.value.is_a?(Symbol) || f.value.is_a?(String) ? f.value : f.name
+                    fetcher = "#{object}.has_key?(:#{key_name}) ? #{object}[:#{key_name}] : #{object}['#{key_name}']"
+                    "'#{f.name}'.freeze => #{composed_value(f, fetcher, nesting)}"
                   end.join(', ')}
                 }
               else
                 {
-                  #{non_proc_fields.map do |field|
-                    "#{field.name}: #{composed_value(field, "#{object}.#{field.name}", nesting)}"
+                  #{non_proc_fields.map do |f|
+                    method_name = f.value.is_a?(Symbol) || f.value.is_a?(String) ? f.value : f.name
+                    "'#{f.name}'.freeze => #{composed_value(f, "#{object}.#{method_name}", nesting)}"
                   end.join(', ')}
                 }
               end)
             RUBY
           else
-            "{}"
+            '{}'
           end
         end
 
         def proc_values(field, nesting)
-          proc_fields = proc_fields_for(field)
+          proc_fields = proc_fields_for(field, nesting)
 
           if proc_fields.present?
             <<-RUBY
               {
-                #{proc_fields.map do |field|
-                  "#{field.name}: (#{composed_value(field, source_for(field.value, nesting), nesting)})"
+                #{proc_fields.map do |f|
+                  "'#{f.name}'.freeze => (#{composed_value(f, source_for(f.value, nesting), nesting)})"
                 end.join(', ')}
               }
             RUBY
           else
-            "{}"
+            '{}'
           end
         end
 
-        def non_proc_fields_for(parent)
+        def non_proc_fields_for(parent, nesting)
           return [] unless parent
-          (parent.children || []).select { |field| !(field.value && field.value.is_a?(Proc)) }
+
+          fields = (parent.children || []).reject { |field| field.value.is_a?(Proc) }
+
+          if nesting.zero? && @fields.present?
+            fields.select { |f| @fields.include?(f.name) }
+          else
+            fields
+          end
         end
 
-        def proc_fields_for(parent)
+        def proc_fields_for(parent, nesting)
           return [] unless parent
-          (parent.children || []).select { |field| field.value && field.value.is_a?(Proc) }
+
+          fields = (parent.children || []).select { |field| field.value.is_a?(Proc) }
+
+          if nesting.zero? && @fields.present?
+            fields.select { |f| @fields.include?(f.name) }
+          else
+            fields
+          end
         end
 
         def source_for(proc, nesting)
@@ -145,26 +170,36 @@ module Chewy
           raise "No lambdas found, try to reformat your code:\n`#{proc.source}`" unless lambdas
 
           source = lambdas.first
-          if proc.arity == 0
+          proc_params = proc.parameters.map(&:second)
+
+          if proc.arity.zero?
             source = replace_self(source, :"object#{nesting}")
             source = replace_send(source, :"object#{nesting}")
-          elsif proc.arity < 0
+          elsif proc.arity.negative?
             raise "Splat arguments are unsupported by witchcraft:\n`#{proc.source}"
           else
             (nesting + 1).times do |n|
-              source = replace_lvar(source, proc.parameters[n][1], :"object#{n}") if proc.parameters[n]
+              source = replace_lvar(source, proc_params[n], :"object#{n}") if proc_params[n]
+            end
+            source = replace_lvar(source, proc_params[nesting + 1], :crutches) if proc_params[nesting + 1]
+
+            binding_variable_list(source).each do |variable|
+              locals.push(proc.binding.eval(variable.to_s))
+              source = replace_local(source, variable, locals.size - 1)
             end
-            source = replace_lvar(source, proc.parameters[nesting + 1][1], :crutches) if proc.parameters[nesting + 1]
           end
+
           Unparser.unparse(source)
         end
 
         def exctract_lambdas(node)
+          return unless node.is_a?(Parser::AST::Node)
+
           if node.type == :block && node.children[0].type == :send && node.children[0].to_a == [nil, :lambda]
-            node.children[2]
+            [node.children[2]]
           else
             node.children.map { |child| exctract_lambdas(child) }.flatten.compact
-          end if node.is_a?(Parser::AST::Node)
+          end
         end
 
         def replace_lvar(node, old_variable, new_variable)
@@ -182,7 +217,7 @@ module Chewy
         def replace_send(node, variable)
           if node.is_a?(Parser::AST::Node)
             if node.type == :send && node.children[0].nil?
-              node.updated(nil, [Parser::AST::Node.new(:lvar, [variable]), *node.children[1..-1]])
+              node.updated(nil, [Parser::AST::Node.new(:lvar, [variable]), *node.children[1..]])
             else
               node.updated(nil, node.children.map { |child| replace_send(child, variable) })
             end
@@ -202,6 +237,32 @@ module Chewy
             node
           end
         end
+
+        def replace_local(node, variable, local_index)
+          if node.is_a?(Parser::AST::Node)
+            if node.type == :send && node.children.to_a == [nil, variable]
+              node.updated(nil, [
+                Parser::AST::Node.new(:lvar, [:locals]),
+                :[],
+                Parser::AST::Node.new(:int, [local_index])
+              ])
+            else
+              node.updated(nil, node.children.map { |child| replace_local(child, variable, local_index) })
+            end
+          else
+            node
+          end
+        end
+
+        def binding_variable_list(node)
+          return unless node.is_a?(Parser::AST::Node)
+
+          if node.type == :send && node.children[0].nil?
+            node.children[1]
+          else
+            node.children.map { |child| binding_variable_list(child) }.flatten.compact.uniq
+          end
+        end
       end
     end
   end

data/lib/chewy/index/wrapper.rb

@@ -0,0 +1,95 @@
+module Chewy
+  class Index
+    module Wrapper
+      extend ActiveSupport::Concern
+
+      included do
+        attr_accessor :_data, :_object
+        attr_reader :attributes
+      end
+
+      module ClassMethods
+        def build(hit)
+          attributes = (hit['_source'] || {})
+            .reverse_merge(id: hit['_id'])
+            .merge!(_score: hit['_score'])
+            .merge!(_explanation: hit['_explanation'])
+
+          wrapper = new(attributes)
+          wrapper._data = hit
+          wrapper
+        end
+      end
+
+      def initialize(attributes = {})
+        @attributes = attributes.stringify_keys
+      end
+
+      def ==(other)
+        return true if super
+
+        if other.is_a?(Chewy::Index)
+          self.class == other.class && (respond_to?(:id) ? id == other.id : attributes == other.attributes)
+        elsif other.respond_to?(:id)
+          self.class.adapter.target.is_a?(Class) &&
+            other.is_a?(self.class.adapter.target) &&
+            id.to_s == other.id.to_s
+        else
+          false
+        end
+      end
+
+      %w[_id _type _index].each do |name|
+        define_method name do
+          _data[name]
+        end
+      end
+
+      def method_missing(method, *args, &block)
+        m = method.to_s
+        if (name = highlight_name(m))
+          highlight(name)
+        elsif (name = highlight_names(m))
+          highlights(name)
+        elsif @attributes.key?(m)
+          @attributes[m]
+        elsif attribute_defined?(m)
+          nil
+        else
+          super
+        end
+      end
+
+      def respond_to_missing?(method, include_private = false)
+        m = method.to_s
+        highlight_name(m) || highlight_names(m) || @attributes.key?(m) || attribute_defined?(m) || super
+      end
+
+    private
+
+      def highlight_name(method)
+        method.sub(/_highlight\z/, '') if method.end_with?('_highlight')
+      end
+
+      def highlight_names(method)
+        method.sub(/_highlights\z/, '') if method.end_with?('_highlights')
+      end
+
+      def attribute_defined?(attribute)
+        self.class.root && self.class.root.children.find { |a| a.name.to_s == attribute }.present?
+      end
+
+      def highlight(attribute)
+        _data['highlight'][attribute].first if highlight?(attribute)
+      end
+
+      def highlights(attribute)
+        _data['highlight'][attribute] if highlight?(attribute)
+      end
+
+      def highlight?(attribute)
+        _data.key?('highlight') && _data['highlight'].key?(attribute)
+      end
+    end
+  end
+end