chewy 0.9.0 → 5.2.0

data/lib/chewy/index/actions.rb

@@ -30,8 +30,8 @@ module Chewy
         # Suffixed index names might be used for zero-downtime mapping change, for example.
         # Description: (http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime/).
         #
-        def create(*args)
-          create!(*args)
+        def create(*args, **kwargs)
+          create!(*args, **kwargs)
         rescue Elasticsearch::Transport::Transport::Errors::BadRequest
           false
         end
@@ -52,18 +52,14 @@ module Chewy
         # Suffixed index names might be used for zero-downtime mapping change, for example.
         # Description: (http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime/).
         #
-        def create!(*args)
-          options = args.extract_options!.reverse_merge!(alias: true)
-          name = build_index_name(suffix: args.first)
+        def create!(suffix = nil, **options)
+          options.reverse_merge!(alias: true)
+          general_name = index_name
+          suffixed_name = index_name(suffix: suffix)
 
-          if Chewy::Runtime.version >= 1.1
-            body = index_params
-            body[:aliases] = { index_name => {} } if options[:alias] && name != index_name
-            result = client.indices.create(index: name, body: body)
-          else
-            result = client.indices.create(index: name, body: index_params)
-            result &&= client.indices.put_alias(index: name, name: index_name) if options[:alias] && name != index_name
-          end
+          body = specification_hash
+          body[:aliases] = {general_name => {}} if options[:alias] && suffixed_name != general_name
+          result = client.indices.create(index: suffixed_name, body: body)
 
           Chewy.wait_for_status if result
           result
@@ -78,7 +74,13 @@ module Chewy
         #   UsersIndex.delete '01-2014' # deletes `users_01-2014` index
         #
         def delete(suffix = nil)
-          result = client.indices.delete index: build_index_name(suffix: suffix)
+          # Verify that the index_name is really the index_name and not an alias.
+          #
+          #   "The index parameter in the delete index API no longer accepts alias names.
+          #   Instead, it accepts only index names (or wildcards which will expand to matching indices)."
+          #   https://www.elastic.co/guide/en/elasticsearch/reference/6.8/breaking-changes-6.0.html#_delete_index_api_resolves_indices_expressions_only_against_indices
+          index_names = client.indices.get_alias(index: index_name(suffix: suffix)).keys
+          result = client.indices.delete index: index_names.join(',')
           Chewy.wait_for_status if result
           result
           # es-ruby >= 1.0.10 handles Elasticsearch::Transport::Transport::Errors::NotFound
@@ -137,10 +139,17 @@ module Chewy
         #
         # See [import.rb](lib/chewy/type/import.rb) for more details.
         #
-        [:import, :import!].each do |method|
+        %i[import import!].each do |method|
           class_eval <<-METHOD, __FILE__, __LINE__ + 1
-            def #{method} options = {}
-              objects = options.reject { |k, v| !type_names.map(&:to_sym).include?(k) }
+            def #{method}(*args)
+              options = args.extract_options!
+              if args.one? && type_names.one?
+                objects = {type_names.first.to_sym => args.first}
+              elsif args.one?
+                fail ArgumentError, "Please pass objects for `#{method}` as a hash with type names"
+              else
+                objects = options.reject { |k, v| !type_names.map(&:to_sym).include?(k) }
+              end
               types.map do |type|
                 args = [objects[type.type_name.to_sym], options.dup].reject(&:blank?)
                 type.#{method} *args
@@ -149,33 +158,89 @@ module Chewy
           METHOD
         end
 
-        # Deletes, creates and imports data to the index.
-        # Returns import result
-        #
-        #   UsersIndex.reset!
+        # Deletes, creates and imports data to the index. Returns the
+        # import result. If index name suffix is passed as the first
+        # argument - performs zero-downtime index resetting.
         #
-        # If index name suffix passed as the first argument - performs
-        # zero-downtime index resetting (described here:
-        # http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime/).
+        # It also applies journal if anything was journaled during the
+        # reset.
         #
-        #   UsersIndex.reset! Time.now.to_i, journal: true
-        #
-        def reset!(suffix = nil, journal: false)
-          if suffix.present? && (indexes = self.indexes).present?
+        # @example
+        #   UsersIndex.reset!
+        #   UsersIndex.reset! Time.now.to_i
+        #
+        # @see http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime
+        # @param suffix [String] a suffix for the newly created index
+        # @param apply_journal [true, false] if true, journal is applied after the import is completed
+        # @param journal [true, false] journaling is switched off for import during reset by default
+        # @param import_options [Hash] options, passed to the import call
+        # @return [true, false] false in case of errors
+        def reset!(suffix = nil, apply_journal: true, journal: false, **import_options)
+          result = if suffix.present?
+            start_time = Time.now
+            indexes = self.indexes
             create! suffix, alias: false
-            result = import suffix: suffix, journal: journal
-            client.indices.update_aliases body: { actions: [
+
+            general_name = index_name
+            suffixed_name = index_name(suffix: suffix)
+
+            optimize_index_settings suffixed_name
+            result = import import_options.merge(suffix: suffix, journal: journal, refresh: !Chewy.reset_disable_refresh_interval)
+            original_index_settings suffixed_name
+
+            delete if indexes.blank?
+            client.indices.update_aliases body: {actions: [
               *indexes.map do |index|
-                { remove: { index: index, alias: index_name } }
+                {remove: {index: index, alias: general_name}}
               end,
-              { add: { index: build_index_name(suffix: suffix), alias: index_name } }
-            ] }
+              {add: {index: suffixed_name, alias: general_name}}
+            ]}
             client.indices.delete index: indexes if indexes.present?
+
+            self.journal.apply(start_time, **import_options) if apply_journal
             result
           else
-            purge! suffix
-            import journal: journal
+            purge!
+            import import_options.merge(journal: journal)
           end
+
+          specification.lock!
+          result
+        end
+
+        # A {Chewy::Journal} instance for the particular index
+        #
+        # @return [Chewy::Journal] journal instance
+        def journal
+          @journal ||= Chewy::Journal.new(self)
+        end
+
+      private
+
+        def optimize_index_settings(index_name)
+          settings = {}
+          settings[:refresh_interval] = -1 if Chewy.reset_disable_refresh_interval
+          settings[:number_of_replicas] = 0 if Chewy.reset_no_replicas
+          update_settings index_name, settings: settings if settings.any?
+        end
+
+        def original_index_settings(index_name)
+          settings = {}
+          if Chewy.reset_disable_refresh_interval
+            settings.merge! index_settings(:refresh_interval)
+            settings[:refresh_interval] = '1s' if settings.empty?
+          end
+          settings.merge! index_settings(:number_of_replicas) if Chewy.reset_no_replicas
+          update_settings index_name, settings: settings if settings.any?
+        end
+
+        def update_settings(index_name, **options)
+          client.indices.put_settings index: index_name, body: {index: options[:settings]}
+        end
+
+        def index_settings(setting_name)
+          return {} unless settings_hash.key?(:settings) && settings_hash[:settings].key?(:index)
+          settings_hash[:settings][:index].slice(setting_name)
         end
       end
     end

data/lib/chewy/index/aliases.rb

@@ -11,7 +11,8 @@ module Chewy
         end
 
         def aliases
-          client.indices.get_alias(index: index_name, name: '*')[index_name].try(:[], 'aliases').try(:keys) || []
+          name = index_name
+          client.indices.get_alias(index: name, name: '*')[name].try(:[], 'aliases').try(:keys) || []
         rescue Elasticsearch::Transport::Transport::Errors::NotFound
           []
         end

data/lib/chewy/index/settings.rb

@@ -4,12 +4,14 @@ module Chewy
     # hash. At first, you need to store some analyzers or other
     # analysis options to the corresponding repository:
     #
+    # @example
     #   Chewy.analyzer :title_analyzer, type: 'custom', filter: %w(lowercase icu_folding title_nysiis)
     #   Chewy.filter :title_nysiis, type: 'phonetic', encoder: 'nysiis', replace: false
     #
     # `title_nysiis` filter here will be expanded automatically when
     # `title_analyzer` analyser will be used in index settings:
     #
+    # @example
     #   class ProductsIndex < Chewy::Index
     #     settings analysis: {
     #       analyzer: [
@@ -23,20 +25,23 @@ module Chewy
     # might be used as well.
     #
     class Settings
-      def initialize(params = {})
+      def initialize(params = {}, &block)
         @params = params
+        @proc_params = block
       end
 
       def to_hash
         settings = @params.deep_symbolize_keys
+        settings.merge!((@proc_params.call || {}).deep_symbolize_keys) if @proc_params
 
         settings[:analysis] = resolve_analysis(settings[:analysis]) if settings[:analysis]
+
         if settings[:index] || Chewy.configuration[:index]
           settings[:index] = (Chewy.configuration[:index] || {})
             .deep_merge((settings[:index] || {}).deep_symbolize_keys)
         end
 
-        settings.present? ? { settings: settings } : {}
+        settings.present? ? {settings: settings} : {}
       end
 
     private
@@ -44,14 +49,15 @@ module Chewy
       def resolve_analysis(analysis)
         analyzer = resolve(analysis[:analyzer], Chewy.analyzers)
 
-        options = [:tokenizer, :filter, :char_filter].each.with_object({}) do |type, result|
+        options = %i[tokenizer filter char_filter].each.with_object({}) do |type, result|
           dependencies = collect_dependencies(type, analyzer)
           resolved = resolve(dependencies.push(analysis[type]), Chewy.send(type.to_s.pluralize))
           result.merge!(type => resolved) if resolved.present?
         end
 
         options[:analyzer] = analyzer if analyzer.present?
-        options
+        analysis = analysis.except(:analyzer, :tokenizer, :filter, :char_filter)
+        analysis.merge(options)
       end
 
       def collect_dependencies(type, analyzer)
@@ -66,7 +72,7 @@ module Chewy
             else
               name_or_hash = name_or_hash.to_sym
               resolved = repository[name_or_hash]
-              resolved ? { name_or_hash => resolved } : {}
+              resolved ? {name_or_hash => resolved} : {}
             end
             result.merge!(options)
           end

data/lib/chewy/index/specification.rb

@@ -0,0 +1,60 @@
+module Chewy
+  class Index
+    # Index specification is a combination of index settings and
+    # mappings. The idea behind this class is that specification
+    # can be locked in the `Chewy::Stash::Specification` between
+    # resets, so it is possible to track changes. In the future
+    # it is planned to be way smarter but right now `rake chewy:deploy`
+    # checks if there were changes and resets the index only if
+    # anything was changed. Otherwise, the index reset is skipped.
+    #
+    # @see Chewy::Stash::Specification
+    class Specification
+      # @see Chewy::Index::Specification
+      # @param index [Chewy::Index] Just a chewy index
+      def initialize(index)
+        @index = index
+      end
+
+      # Stores the current index specification to the `Chewy::Stash::Specification`
+      # as json.
+      #
+      # @raise [Chewy::ImportFailed] if something went wrong
+      # @return [true] if everything is fine
+      def lock!
+        Chewy::Stash::Specification.import!([
+          id: @index.derivable_name,
+          specification: Base64.encode64(current.to_json)
+        ], journal: false)
+      end
+
+      # Returns the last locked specification as ruby hash. Returns
+      # empty hash if nothing is stored yet.
+      #
+      # @return [Hash] hash produced with JSON parser
+      def locked
+        filter = {ids: {values: [@index.derivable_name]}}
+        document = Chewy::Stash::Specification.filter(filter).first
+        return {} unless document
+        JSON.load(Base64.decode64(document.specification)) # rubocop:disable Security/JSONLoad
+      end
+
+      # Simply returns `Chewy::Index.specification_hash`, but
+      # prepared for JSON with `as_json` method. This means all the
+      # keys are strings and there are only values of types handled in JSON.
+      #
+      # @see Chewy::Index.specification_hash
+      # @return [Hash] a JSON-ready hash
+      def current
+        @index.specification_hash.as_json
+      end
+
+      # Compares previously locked and current specifications.
+      #
+      # @return [true, false] the result of comparison
+      def changed?
+        current != locked
+      end
+    end
+  end
+end

data/lib/chewy/journal.rb

@@ -1,107 +1,55 @@
-require 'chewy/journal/entry'
-require 'chewy/journal/query'
-require 'chewy/journal/apply'
-require 'chewy/journal/clean'
-
 module Chewy
+  # A class to perform journal-related actions for the specified indexes/types.
+  #
+  # @example
+  #   journal = Chewy::Journal.new('places#city', UsersIndex)
+  #   journal.apply(20.minutes.ago)
+  #   journal.clean
+  #
   class Journal
-    JOURNAL_MAPPING = {
-      journal: {
-        properties: {
-          index_name: { type: 'string', index: 'not_analyzed' },
-          type_name: { type: 'string', index: 'not_analyzed' },
-          action: { type: 'string', index: 'not_analyzed' },
-          object_ids: { type: 'string', index: 'not_analyzed' },
-          created_at: { type: 'date' }
-        }
-      }
-    }.freeze
-
-    def initialize(index)
-      @records = []
-      @index = index
+    # @param only [Array<String, Chewy::Index, Chewy::Type>] indexes/types or even string references to perform actions on
+    def initialize(*only)
+      @only = only
     end
 
-    def add(action_objects)
-      @records +=
-        action_objects.map do |action, objects|
-          {
-            index_name: @index.derivable_index_name,
-            type_name: @index.type_name,
-            action: action,
-            object_ids: identify(objects),
-            created_at: Time.now.to_i
-          }
+    # Applies all changes that were done since the specified time to the
+    # specified indexes/types.
+    #
+    # @param since_time [Time, DateTime] timestamp from which changes will be applied
+    # @param retries [Integer] maximum number of attempts to make journal empty, 10 by default
+    # @return [Integer] the amount of journal entries found
+    def apply(since_time, retries: 10, **import_options)
+      stage = 1
+      since_time -= 1
+      count = 0
+      while stage <= retries
+        entries = Chewy::Stash::Journal.entries(since_time, only: @only).to_a.presence or break
+        count += entries.size
+        groups = reference_groups(entries)
+        ActiveSupport::Notifications.instrument 'apply_journal.chewy', stage: stage, groups: groups
+        groups.each do |type, references|
+          type.import(references, import_options.merge(journal: false))
         end
-    end
-
-    def bulk_body
-      @records.map do |record|
-        {
-          create: {
-            _index: self.class.index_name,
-            _type: self.class.type_name,
-            data: record
-          }
-        }
+        stage += 1
+        since_time = entries.map(&:created_at).max
       end
+      count
     end
 
-    def any_records?
-      @records.any?
+    # Cleans journal for the specified indexes/types.
+    #
+    # @param until_time [Time, DateTime] time to clean up until it
+    # @return [Hash] delete_by_query ES API call result
+    def clean(until_time = nil)
+      Chewy::Stash::Journal.clean(until_time, only: @only)
     end
 
   private
 
-    def identify(objects)
-      @index.adapter.identify(objects)
-    end
-
-    class << self
-      def exists?
-        Chewy.client.indices.exists? index: index_name
-      end
-
-      def index_name
-        [
-          Chewy.configuration[:prefix],
-          Chewy.configuration[:journal_name] || 'chewy_journal'
-        ].reject(&:blank?).join('_')
-      end
-
-      def type_name
-        JOURNAL_MAPPING.keys.first
-      end
-
-      def create
-        return if exists?
-        Chewy.client.indices.create index: index_name, body: { settings: { index: Chewy.configuration[:index] }, mappings: JOURNAL_MAPPING }
-        Chewy.wait_for_status
-      end
-
-      def delete!
-        delete or raise Elasticsearch::Transport::Transport::Errors::NotFound
-      end
-
-      def delete
-        result = Chewy.client.indices.delete index: index_name
-        Chewy.wait_for_status if result
-        result
-      rescue Elasticsearch::Transport::Transport::Errors::NotFound
-        false
-      end
-
-      def apply_changes_from(*args)
-        Apply.since(*args)
-      end
-
-      def entries_from(*args)
-        Entry.since(*args)
-      end
-
-      def clean_until(*args)
-        Clean.until(*args)
-      end
+    def reference_groups(entries)
+      entries.group_by(&:type).map do |type, grouped_entries|
+        [type, grouped_entries.map(&:references).inject(:|)]
+      end.to_h
     end
   end
 end