chewy 0.9.0 → 0.10.0

data/lib/chewy/index/actions.rb

@@ -52,17 +52,18 @@ module Chewy
         # Suffixed index names might be used for zero-downtime mapping change, for example.
         # Description: (http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime/).
         #
-        def create!(*args)
-          options = args.extract_options!.reverse_merge!(alias: true)
-          name = build_index_name(suffix: args.first)
+        def create!(suffix = nil, **options)
+          options.reverse_merge!(alias: true)
+          general_name = index_name
+          suffixed_name = index_name(suffix: suffix)
 
           if Chewy::Runtime.version >= 1.1
-            body = index_params
-            body[:aliases] = { index_name => {} } if options[:alias] && name != index_name
-            result = client.indices.create(index: name, body: body)
+            body = specification_hash
+            body[:aliases] = {general_name => {}} if options[:alias] && suffixed_name != general_name
+            result = client.indices.create(index: suffixed_name, body: body)
           else
-            result = client.indices.create(index: name, body: index_params)
-            result &&= client.indices.put_alias(index: name, name: index_name) if options[:alias] && name != index_name
+            result = client.indices.create(index: suffixed_name, body: specification_hash)
+            result &&= client.indices.put_alias(index: suffixed_name, name: general_name) if options[:alias] && name != index_name
           end
 
           Chewy.wait_for_status if result
@@ -78,7 +79,7 @@ module Chewy
         #   UsersIndex.delete '01-2014' # deletes `users_01-2014` index
         #
         def delete(suffix = nil)
-          result = client.indices.delete index: build_index_name(suffix: suffix)
+          result = client.indices.delete index: index_name(suffix: suffix)
           Chewy.wait_for_status if result
           result
           # es-ruby >= 1.0.10 handles Elasticsearch::Transport::Transport::Errors::NotFound
@@ -137,7 +138,7 @@ module Chewy
         #
         # See [import.rb](lib/chewy/type/import.rb) for more details.
         #
-        [:import, :import!].each do |method|
+        %i[import import!].each do |method|
           class_eval <<-METHOD, __FILE__, __LINE__ + 1
             def #{method} options = {}
               objects = options.reject { |k, v| !type_names.map(&:to_sym).include?(k) }
@@ -149,33 +150,89 @@ module Chewy
           METHOD
         end
 
-        # Deletes, creates and imports data to the index.
-        # Returns import result
+        # Deletes, creates and imports data to the index. Returns the
+        # import result. If index name suffix is passed as the first
+        # argument - performs zero-downtime index resetting.
         #
-        #   UsersIndex.reset!
-        #
-        # If index name suffix passed as the first argument - performs
-        # zero-downtime index resetting (described here:
-        # http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime/).
-        #
-        #   UsersIndex.reset! Time.now.to_i, journal: true
+        # It also applies journal if anything was journaled during the
+        # reset.
         #
-        def reset!(suffix = nil, journal: false)
-          if suffix.present? && (indexes = self.indexes).present?
+        # @example
+        #   UsersIndex.reset!
+        #   UsersIndex.reset! Time.now.to_i
+        #
+        # @see http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime
+        # @param suffix [String] a suffix for the newly created index
+        # @param apply_journal [true, false] if true, journal is applied after the import is completed
+        # @param journal [true, false] journalig is switched off for import during reset by default
+        # @param import_options [Hash] options, passed to the import call
+        # @return [true, false] false in case of errors
+        def reset!(suffix = nil, apply_journal: true, journal: false, **import_options)
+          result = if suffix.present?
+            start_time = Time.now
+            indexes = self.indexes
             create! suffix, alias: false
-            result = import suffix: suffix, journal: journal
-            client.indices.update_aliases body: { actions: [
+
+            general_name = index_name
+            suffixed_name = index_name(suffix: suffix)
+
+            optimize_index_settings suffixed_name
+            result = import import_options.merge(suffix: suffix, journal: journal, refresh: !Chewy.reset_disable_refresh_interval)
+            original_index_settings suffixed_name
+
+            delete if indexes.blank?
+            client.indices.update_aliases body: {actions: [
               *indexes.map do |index|
-                { remove: { index: index, alias: index_name } }
+                {remove: {index: index, alias: general_name}}
               end,
-              { add: { index: build_index_name(suffix: suffix), alias: index_name } }
-            ] }
+              {add: {index: suffixed_name, alias: general_name}}
+            ]}
             client.indices.delete index: indexes if indexes.present?
+
+            self.journal.apply(start_time, **import_options) if apply_journal
             result
           else
-            purge! suffix
-            import journal: journal
+            purge!
+            import import_options.merge(journal: journal)
           end
+
+          specification.lock!
+          result
+        end
+
+        # A {Chewy::Journal} instance for the particular index
+        #
+        # @return [Chewy::Journal] journal instance
+        def journal
+          @journal ||= Chewy::Journal.new(self)
+        end
+
+      private
+
+        def optimize_index_settings(index_name)
+          settings = {}
+          settings[:refresh_interval] = -1 if Chewy.reset_disable_refresh_interval
+          settings[:number_of_replicas] = 0 if Chewy.reset_no_replicas
+          update_settings index_name, settings: settings if settings.any?
+        end
+
+        def original_index_settings(index_name)
+          settings = {}
+          if Chewy.reset_disable_refresh_interval
+            settings.merge! index_settings(:refresh_interval)
+            settings[:refresh_interval] = '1s' if settings.empty?
+          end
+          settings.merge! index_settings(:number_of_replicas) if Chewy.reset_no_replicas
+          update_settings index_name, settings: settings if settings.any?
+        end
+
+        def update_settings(index_name, **options)
+          client.indices.put_settings index: index_name, body: {index: options[:settings]}
+        end
+
+        def index_settings(setting_name)
+          return {} unless settings_hash.key?(:settings) || settings_hash[:settings].key?(:index)
+          settings_hash[:settings][:index].slice(setting_name)
         end
       end
     end

data/lib/chewy/index/aliases.rb

@@ -11,7 +11,8 @@ module Chewy
         end
 
         def aliases
-          client.indices.get_alias(index: index_name, name: '*')[index_name].try(:[], 'aliases').try(:keys) || []
+          name = index_name
+          client.indices.get_alias(index: name, name: '*')[name].try(:[], 'aliases').try(:keys) || []
         rescue Elasticsearch::Transport::Transport::Errors::NotFound
           []
         end

data/lib/chewy/index/settings.rb

@@ -23,20 +23,23 @@ module Chewy
     # might be used as well.
     #
     class Settings
-      def initialize(params = {})
+      def initialize(params = {}, &block)
         @params = params
+        @proc_params = block
       end
 
       def to_hash
         settings = @params.deep_symbolize_keys
+        settings.merge!((@proc_params.call || {}).deep_symbolize_keys) if @proc_params
 
         settings[:analysis] = resolve_analysis(settings[:analysis]) if settings[:analysis]
+
         if settings[:index] || Chewy.configuration[:index]
           settings[:index] = (Chewy.configuration[:index] || {})
             .deep_merge((settings[:index] || {}).deep_symbolize_keys)
         end
 
-        settings.present? ? { settings: settings } : {}
+        settings.present? ? {settings: settings} : {}
       end
 
     private
@@ -44,14 +47,15 @@ module Chewy
       def resolve_analysis(analysis)
         analyzer = resolve(analysis[:analyzer], Chewy.analyzers)
 
-        options = [:tokenizer, :filter, :char_filter].each.with_object({}) do |type, result|
+        options = %i[tokenizer filter char_filter].each.with_object({}) do |type, result|
           dependencies = collect_dependencies(type, analyzer)
           resolved = resolve(dependencies.push(analysis[type]), Chewy.send(type.to_s.pluralize))
           result.merge!(type => resolved) if resolved.present?
         end
 
         options[:analyzer] = analyzer if analyzer.present?
-        options
+        analysis = analysis.except(:analyzer, :tokenizer, :filter, :char_filter)
+        analysis.merge(options)
       end
 
       def collect_dependencies(type, analyzer)
@@ -66,7 +70,7 @@ module Chewy
             else
               name_or_hash = name_or_hash.to_sym
               resolved = repository[name_or_hash]
-              resolved ? { name_or_hash => resolved } : {}
+              resolved ? {name_or_hash => resolved} : {}
             end
             result.merge!(options)
           end

data/lib/chewy/index/specification.rb

@@ -0,0 +1,58 @@
+module Chewy
+  class Index
+    # Index specification is a combination of index settings and
+    # mappings. The idea behind this class is that specification
+    # can be locked in the `Chewy::Stash` between resets, so it is
+    # possible to track changes. In the future it is planned to
+    # be way smarter but right now `rake chewy:deploy` checks
+    # if there were changes and resets the index only if anything
+    # was changed. Otherwise, the index reset is skipped.
+    #
+    # @see Chewy::Stash
+    class Specification
+      # @see Chewy::Index.specification
+      # @param index [Chewy::Index] Just a chewy index
+      def initialize(index)
+        @index = index
+      end
+
+      # Stores the current index specification to the `Chewy::Stash`
+      # as json.
+      #
+      # @raise [Chewy::ImportFailed] if something went wrong
+      # @return [true] if everything is fine
+      def lock!
+        Chewy::Stash::Specification.import!([
+          id: @index.derivable_name,
+          value: current.to_json
+        ], journal: false)
+      end
+
+      # Returns the last locked specification as ruby hash. Returns
+      # empty hash if nothing is stored yet.
+      #
+      # @return [Hash] hash produced with JSON parser
+      def locked
+        filter = {ids: {values: [@index.derivable_name]}}
+        JSON.parse(Chewy::Stash::Specification.filter(filter).first.try!(:value) || '{}')
+      end
+
+      # Simply returns `Chewy::Index.specification_hash`, but
+      # prepared for JSON with `as_json` method. This means all the
+      # keys are strings and there are only values of types handled in JSON.
+      #
+      # @see Chewy::Index.specification_hash
+      # @return [Hash] a JSON-ready hash
+      def current
+        @index.specification_hash.as_json
+      end
+
+      # Compares previously locked and current specifications.
+      #
+      # @return [true, false] the result of comparison
+      def changed?
+        current != locked
+      end
+    end
+  end
+end

data/lib/chewy/journal.rb

@@ -1,107 +1,55 @@
-require 'chewy/journal/entry'
-require 'chewy/journal/query'
-require 'chewy/journal/apply'
-require 'chewy/journal/clean'
-
 module Chewy
+  # A class to perform journal-related actions for the specified indexes/types.
+  #
+  # @example
+  #   journal = Chewy::Journal.new('places#city', UsersIndex)
+  #   journal.apply(20.minutes.ago)
+  #   journal.clean
+  #
   class Journal
-    JOURNAL_MAPPING = {
-      journal: {
-        properties: {
-          index_name: { type: 'string', index: 'not_analyzed' },
-          type_name: { type: 'string', index: 'not_analyzed' },
-          action: { type: 'string', index: 'not_analyzed' },
-          object_ids: { type: 'string', index: 'not_analyzed' },
-          created_at: { type: 'date' }
-        }
-      }
-    }.freeze
-
-    def initialize(index)
-      @records = []
-      @index = index
+    # @param only [Array<String, Chewy::Index, Chewy::Type>] indexes/types or even string references to perform actions on
+    def initialize(*only)
+      @only = only
     end
 
-    def add(action_objects)
-      @records +=
-        action_objects.map do |action, objects|
-          {
-            index_name: @index.derivable_index_name,
-            type_name: @index.type_name,
-            action: action,
-            object_ids: identify(objects),
-            created_at: Time.now.to_i
-          }
+    # Applies all changes that were done since the specified time to the
+    # specified indexes/types.
+    #
+    # @param since_time [Time, DateTime] timestamp from which changes will be applied
+    # @param retries [Integer] maximum number of attempts to make journal empty, 10 by default
+    # @return [Integer] the amount of journal entries found
+    def apply(since_time, retries: 10, **import_options)
+      stage = 1
+      since_time -= 1
+      count = 0
+      while stage <= retries
+        entries = Chewy::Stash::Journal.entries(since_time, only: @only).to_a.presence or break
+        count += entries.size
+        groups = reference_groups(entries)
+        ActiveSupport::Notifications.instrument 'apply_journal.chewy', stage: stage, groups: groups
+        groups.each do |type, references|
+          type.import(references, import_options.merge(journal: false))
         end
-    end
-
-    def bulk_body
-      @records.map do |record|
-        {
-          create: {
-            _index: self.class.index_name,
-            _type: self.class.type_name,
-            data: record
-          }
-        }
+        stage += 1
+        since_time = entries.map(&:created_at).max
       end
+      count
     end
 
-    def any_records?
-      @records.any?
+    # Cleans journal for the specified indexes/types.
+    #
+    # @param until_time [Time, DateTime] time to clean up until it
+    # @return [Hash] delete_by_query ES API call result
+    def clean(until_time = nil)
+      Chewy::Stash::Journal.clean(until_time, only: @only)
     end
 
   private
 
-    def identify(objects)
-      @index.adapter.identify(objects)
-    end
-
-    class << self
-      def exists?
-        Chewy.client.indices.exists? index: index_name
-      end
-
-      def index_name
-        [
-          Chewy.configuration[:prefix],
-          Chewy.configuration[:journal_name] || 'chewy_journal'
-        ].reject(&:blank?).join('_')
-      end
-
-      def type_name
-        JOURNAL_MAPPING.keys.first
-      end
-
-      def create
-        return if exists?
-        Chewy.client.indices.create index: index_name, body: { settings: { index: Chewy.configuration[:index] }, mappings: JOURNAL_MAPPING }
-        Chewy.wait_for_status
-      end
-
-      def delete!
-        delete or raise Elasticsearch::Transport::Transport::Errors::NotFound
-      end
-
-      def delete
-        result = Chewy.client.indices.delete index: index_name
-        Chewy.wait_for_status if result
-        result
-      rescue Elasticsearch::Transport::Transport::Errors::NotFound
-        false
-      end
-
-      def apply_changes_from(*args)
-        Apply.since(*args)
-      end
-
-      def entries_from(*args)
-        Entry.since(*args)
-      end
-
-      def clean_until(*args)
-        Clean.until(*args)
-      end
+    def reference_groups(entries)
+      entries.group_by(&:type).map do |type, grouped_entries|
+        [type, grouped_entries.map(&:references).inject(:|)]
+      end.to_h
     end
   end
 end

data/lib/chewy/query.rb

@@ -1,6 +1,5 @@
 require 'chewy/query/criteria'
 require 'chewy/query/filters'
-require 'chewy/query/scoping'
 require 'chewy/query/loading'
 require 'chewy/query/pagination'
 
@@ -14,9 +13,18 @@ module Chewy
   #
   class Query
     include Enumerable
-    include Scoping
     include Loading
     include Pagination
+    include Chewy::Search::Scoping
+
+    DELEGATED_METHODS = %i[
+      explain query_mode filter_mode post_filter_mode
+      timeout limit offset highlight min_score rescore facets script_score
+      boost_factor weight random_score field_value_factor decay aggregations
+      suggest none strategy query filter post_filter boost_mode
+      score_mode order reorder only types delete_all find total
+      total_count total_entries unlimited script_fields track_scores preference
+    ].to_set.freeze
 
     delegate :each, :count, :size, to: :_collection
     alias_method :to_ary, :to_a
@@ -373,7 +381,7 @@ module Chewy
     #                  }]
     #                } } }
     def script_score(script, options = {})
-      scoring = { script_score: { script: script }.merge(options) }
+      scoring = {script_score: {script: script}.merge(options)}
       chain { criteria.update_scores scoring }
     end
 
@@ -441,7 +449,7 @@ module Chewy
     #                  }]
     #                } } }
     def random_score(seed = Time.now, options = {})
-      scoring = options.merge(random_score: { seed: seed.to_i })
+      scoring = options.merge(random_score: {seed: seed.to_i})
       chain { criteria.update_scores scoring }
     end
 
@@ -510,8 +518,8 @@ module Chewy
     def decay(function, field, options = {})
       field_options = options.extract!(:origin, :scale, :offset, :decay).delete_if { |_, v| v.nil? }
       scoring = options.merge(function => {
-                                field => field_options
-                              })
+        field => field_options
+      })
       chain { criteria.update_scores scoring }
     end
 
@@ -540,8 +548,8 @@ module Chewy
       @_named_aggs ||= _build_named_aggs
       @_fully_qualified_named_aggs ||= _build_fqn_aggs
       if params
-        params = { params => @_named_aggs[params] } if params.is_a?(Symbol)
-        params = { params => _get_fully_qualified_named_agg(params) } if params.is_a?(String) && params =~ /\A\S+#\S+\.\S+\z/
+        params = {params => @_named_aggs[params]} if params.is_a?(Symbol)
+        params = {params => _get_fully_qualified_named_agg(params)} if params.is_a?(String) && params =~ /\A\S+#\S+\.\S+\z/
         chain { criteria.update_aggregations params }
       else
         _response['aggregations'] || {}
@@ -605,7 +613,7 @@ module Chewy
       end
     end
 
-    # Marks the criteria as having zero records. This scope  always returns empty array
+    # Marks the criteria as having zero documents. This scope  always returns empty array
     # without touching the elasticsearch server.
     # All the chained calls of methods don't affect the result
     #
@@ -898,10 +906,10 @@ module Chewy
     end
 
     # Sets <tt>search_type</tt> for request.
-    # For instance, one can use <tt>search_type=count</tt> to fetch only total count of records or to fetch only aggregations without fetching records.
+    # For instance, one can use <tt>search_type=count</tt> to fetch only total count of documents or to fetch only aggregations without fetching documents.
     #
     #   scope = UsersIndex.search_type(:count)
-    #   scope.count == 0  # no records actually fetched
+    #   scope.count == 0  # no documents actually fetched
     #   scope.total == 10 # but we know a total count of them
     #
     #   scope = UsersIndex.aggs(max_age: { max: { field: 'age' } }).search_type(:count)
@@ -924,7 +932,7 @@ module Chewy
       chain { criteria.merge!(other.criteria) }
     end
 
-    # Deletes all records matching a query.
+    # Deletes all documents matching a query.
     #
     #   UsersIndex.delete_all
     #   UsersIndex.filter{ age <= 42 }.delete_all
@@ -932,20 +940,31 @@ module Chewy
     #   UsersIndex::User.filter{ age <= 42 }.delete_all
     #
     def delete_all
-      if Runtime.version > '2.0'
+      if Runtime.version >= '2.0'
         plugins = Chewy.client.nodes.info(plugins: true)['nodes'].values.map { |item| item['plugins'] }.flatten
         raise PluginMissing, 'install delete-by-query plugin' unless plugins.find { |item| item['name'] == 'delete-by-query' }
       end
+
       request = chain { criteria.update_options simple: true }.send(:_request)
+
       ActiveSupport::Notifications.instrument 'delete_query.chewy',
         request: request, indexes: _indexes, types: _types,
         index: _indexes.one? ? _indexes.first : _indexes,
         type: _types.one? ? _types.first : _types do
-        Chewy.client.delete_by_query(request)
-      end
+          if Runtime.version >= '2.0'
+            path = Elasticsearch::API::Utils.__pathify(
+              Elasticsearch::API::Utils.__listify(request[:index]),
+              Elasticsearch::API::Utils.__listify(request[:type]),
+              '/_query'
+            )
+            Chewy.client.perform_request(Elasticsearch::API::HTTP_DELETE, path, {}, request[:body]).body
+          else
+            Chewy.client.delete_by_query(request)
+          end
+        end
     end
 
-    # Find all records matching a query.
+    # Find all documents matching a query.
     #
     #   UsersIndex.find(42)
     #   UsersIndex.filter{ age <= 42 }.find(42)
@@ -1008,8 +1027,8 @@ module Chewy
 
   protected
 
-    def initialize_clone(other)
-      @criteria = other.criteria.clone
+    def initialize_clone(origin)
+      @criteria = origin.criteria.clone
       reset
     end
 
@@ -1026,7 +1045,7 @@ module Chewy
     def _request
       @_request ||= begin
         request = criteria.request_body
-        request[:index] = _indexes.map(&:index_name)
+        request[:index] = _indexes_hash.keys
         request[:type] = _types.map(&:type_name)
         request
       end
@@ -1048,14 +1067,7 @@ module Chewy
 
     def _results
       @_results ||= (criteria.none? || _response == {} ? [] : _response['hits']['hits']).map do |hit|
-        attributes = (hit['_source'] || {})
-          .reverse_merge(id: hit['_id'])
-          .merge!(_score: hit['_score'])
-          .merge!(_explanation: hit['_explanation'])
-
-        wrapper = _derive_index(hit['_index']).type(hit['_type']).new(attributes)
-        wrapper._data = hit
-        wrapper
+        _derive_type(hit['_index'], hit['_type']).build(hit)
       end
     end
 
@@ -1070,6 +1082,10 @@ module Chewy
       end
     end
 
+    def _derive_type(index, type)
+      (@types_cache ||= {})[[index, type]] ||= _derive_index(index).type(type)
+    end
+
     def _derive_index(index_name)
       (@derive_index ||= {})[index_name] ||= _indexes_hash[index_name] ||
         _indexes_hash[_indexes_hash.keys.sort_by(&:length).reverse.detect { |name| index_name.start_with?(name) }]