elastomer-client 2.3.0 → 3.0.0

data/lib/elastomer/client/delete_by_query.rb

@@ -1,120 +1,13 @@
 module Elastomer
   class Client
-
-    # Delete documents from one or more indices and one or more types based
-    # on a query.
+    # Execute delete_by_query using the native _delete_by_query API if supported
+    # or the application-level implementation.
     #
-    # The return value follows the format returned by the Elasticsearch Delete
-    # by Query plugin: https://github.com/elastic/elasticsearch/blob/master/docs/plugins/delete-by-query.asciidoc#response-body
-    #
-    # Internally, this method uses a combination of scroll and bulk delete
-    # instead of the Delete by Query API, which was removed in Elasticsearch
-    # 2.0.
-    #
-    # query  - The query body as a Hash
-    # params - Parameters Hash
-    #
-    # Examples
-    #
-    #   # request body query
-    #   delete_by_query({:query => {:match_all => {}}}, :type => 'tweet')
-    #
-    #   # same thing but using the URI request method
-    #   delete_by_query(nil, { :q => '*:*', :type => 'tweet' })
-    #
-    # See https://www.elastic.co/guide/en/elasticsearch/plugins/current/delete-by-query-usage.html
-    #
-    # Returns a Hash of statistics about the delete operations, for example:
-    #
-    #   {
-    #     "took" : 639,
-    #     "_indices" : {
-    #       "_all" : {
-    #         "found" : 5901,
-    #         "deleted" : 5901,
-    #         "missing" : 0,
-    #         "failed" : 0
-    #       },
-    #       "twitter" : {
-    #         "found" : 5901,
-    #         "deleted" : 5901,
-    #         "missing" : 0,
-    #         "failed" : 0
-    #       }
-    #     },
-    #     "failures" : [ ]
-    #   }
+    # Warning: These implementations have different parameters and return types.
+    # If you want to use one or the other consistently, use Elastomer::Client#native_delete_by_query
+    # or Elastomer::Client#app_delete_by_query directly.
     def delete_by_query(query, params = {})
-      DeleteByQuery.new(self, query, params).execute
+      send(version_support.delete_by_query_method, query, params)
     end
-
-    class DeleteByQuery
-
-      # Create a new DeleteByQuery command for deleting documents matching a
-      # query
-      #
-      # client - Elastomer::Client used for HTTP requests to the server
-      # query  - The query used to find documents to delete
-      # params - Other URL parameters
-      def initialize(client, query, params = {})
-        @client = client
-        @query = query
-        @params = params
-        @response_stats = { "took" => 0, "_indices" => { "_all" => {} }, "failures" => [] }
-      end
-
-      attr_reader :client, :query, :params, :response_stats
-
-      # Internal: Determine whether or not an HTTP status code is in the range
-      # 200 to 299
-      #
-      # status - HTTP status code
-      #
-      # Returns a boolean
-      def is_ok?(status)
-        status.between?(200, 299)
-      end
-
-      # Internal: Tally the contributions of an item to the found, deleted,
-      # missing, and failed counts for the summary statistics
-      #
-      # item - An element of the items array from a bulk response
-      #
-      # Returns a Hash of counts for each category
-      def categorize(item)
-        {
-          "found" => item["found"] || item["status"] == 409 ? 1 : 0,
-          "deleted" => is_ok?(item["status"]) ? 1 : 0,
-          "missing" => !item["found"]  && !item.key?("error") ? 1 : 0,
-          "failed" => item.key?("error") ? 1 : 0,
-        }
-      end
-
-      # Internal: Combine a response item with the existing statistics
-      #
-      # item - A bulk response item
-      def accumulate(item)
-        item = item["delete"]
-        (@response_stats["_indices"][item["_index"]] ||= {}).merge!(categorize(item)) { |_, n, m| n + m }
-        @response_stats["_indices"]["_all"].merge!(categorize(item)) { |_, n, m| n + m }
-        @response_stats["failures"] << item unless is_ok? item["status"]
-      end
-
-      # Perform the Delete by Query action
-      #
-      # Returns a Hash of statistics about the bulk operation
-      def execute
-        ops = Enumerator.new do |yielder|
-          @client.scan(@query, @params.merge(:_source => false)).each_document do |hit|
-            yielder.yield([:delete, hit.select { |key, _| ["_index", "_type", "_id", "_routing"].include?(key) }])
-          end
-        end
-
-        stats = @client.bulk_stream_items(ops, @params) { |item| accumulate(item) }
-        @response_stats["took"] = stats["took"]
-        @response_stats
-      end
-
-    end  # DeleteByQuery
-  end  # Client
-end  # Elastomer
+  end
+end

data/lib/elastomer/client/docs.rb

@@ -40,7 +40,9 @@ module Elastomer
       # There are several other document attributes that control how
       # Elasticsearch will index the document. They are listed below. Please
       # refer to the Elasticsearch documentation for a full explanation of each
-      # and how it affects the indexing process.
+      # and how it affects the indexing process. These indexing directives vary
+      # by Elasticsearch version. Attempting to use a directive not supported
+      # by the Elasticsearch server will raise an exception.
       #
       #   :_id
       #   :_type
@@ -49,12 +51,18 @@ module Elastomer
       #   :_op_type
       #   :_routing
       #   :_parent
-      #   :_timestamp
-      #   :_ttl
-      #   :_consistency
-      #   :_replication
       #   :_refresh
       #
+      # Elasticsearch 2.X only:
+      #
+      #   :_timestamp (deprecated)
+      #   :_ttl (deprecated)
+      #   :_consistency
+      #
+      # Elasticsearch 5.x only:
+      #
+      #   :_wait_for_active_shards
+      #
       # If any of these attributes are present in the document they will be
       # removed from the document before it is indexed. This means that the
       # document will be modified by this method.
@@ -65,6 +73,9 @@ module Elastomer
       # See https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html
       #
       # Returns the response body as a Hash
+      #
+      # Raises Elastomer::Client::IllegalArgument if an unsupported indexing
+      # directive is used.
       def index( document, params = {} )
         overrides = from_document document
         params = update_params(params, overrides)
@@ -246,19 +257,45 @@ module Elastomer
         response.body
       end
 
-      # Delete documents from one or more indices and one or more types based
+      # Delete documents by query following either the native or
+      # application-level delete by query method.
+      #
+      # NOTE: The parameters and response format varies by version. To have more
+      # control over this, use app_delete_by_query or native_delete_by_query
+      # directly.
+      def delete_by_query(query, params = nil)
+        send(@client.version_support.delete_by_query_method, query, params)
+      end
+
+      # DEPRECATED: Delete documents from one or more indices and one or more types based
       # on a query. This method supports both the "request body" query and the
       # "URI request" query. When using the request body semantics, the query
       # hash must contain the :query key. Otherwise we assume a URI request is
       # being made.
       #
-      # See Client#delete_by_query for more information.
+      # See Client#app_delete_by_query for more information.
       #
       # Returns a Hash of statistics about the delete operations
-      def delete_by_query(query, params = nil)
+      def app_delete_by_query(query, params = nil)
         query, params = extract_params(query) if params.nil?
 
-        @client.delete_by_query(query, update_params(params))
+        @client.app_delete_by_query(query, update_params(params))
+      end
+
+      # Delete documents from one or more indices and one or more types based
+      # on a query using Elasticsearch's _delete_by_query API.
+      #
+      # See Client#native_delete_by_query for more information.
+      #
+      # Returns a Hash of statistics about the delete operations as returned by
+      # _delete_by_query.
+      #
+      # Raises Elastomer::Client::IncompatibleVersionException if this version
+      # of Elasticsearch does not support _delete_by_query.
+      def native_delete_by_query(query, params = {})
+        query, params = extract_params(query) if params.nil?
+
+        @client.native_delete_by_query(query, update_params(params))
       end
 
       # Matches a provided or existing document to the stored percolator
@@ -517,9 +554,6 @@ Percolate
         client.multi_percolate(params, &block)
       end
 
-      SPECIAL_KEYS = %w[index type id version version_type op_type routing parent timestamp ttl consistency replication refresh].freeze
-      SPECIAL_KEYS_HASH = SPECIAL_KEYS.inject({}) { |h, k| h[k.to_sym] = "_#{k}"; h }.freeze
-
       # Internal: Given a `document` generate an options hash that will
       # override parameters based on the content of the document. The document
       # will be returned as the value of the :body key.
@@ -530,14 +564,30 @@ Percolate
       # document - A document Hash or JSON encoded String.
       #
       # Returns an options Hash extracted from the document.
+      #
+      # Raises Elastomer::Client::IllegalArgument if an unsupported indexing
+      # directive is used.
       def from_document( document )
         opts = {:body => document}
 
         if document.is_a? Hash
-          SPECIAL_KEYS_HASH.each do |key, field|
+          client.version_support.indexing_directives.each do |key, field|
             opts[key] = document.delete field if document.key? field
             opts[key] = document.delete field.to_sym if document.key? field.to_sym
           end
+
+          # COMPATIBILITY
+          # Fail fast if a consumer is attempting to use an indexing parameter
+          # for a different version of Elasticsearch. Elasticsearch 5+ is strict
+          # about parameter names so we need to either ignore these (in which
+          # case they would be indexed with the document) or fail-fast.
+          # Elasticsearch 2.X will happily ignore unknown parameters, but we
+          # felt it was best to consistently fail fast.
+          client.version_support.unsupported_indexing_directives.each do |key, field|
+            if document.key?(field) || document.key?(field.to_sym)
+              raise IllegalArgument, "Elasticsearch #{client.version} does not support the '#{key}' indexing parameter"
+            end
+          end
         end
 
         opts

data/lib/elastomer/client/errors.rb

@@ -104,5 +104,13 @@ module Elastomer
       ::Elastomer::Client.const_set(error_name, Class.new(Error))
     end
 
-  end  # Client
-end  # Elastomer
+    # Exception for operations that are unsupported with the version of
+    # Elasticsearch being used.
+    IncompatibleVersionException = Class.new Error
+
+    # Exception for client-detected and server-raised invalid Elasticsearch
+    # request parameter.
+    IllegalArgument = Class.new Error
+
+  end
+end

data/lib/elastomer/client/index.rb

@@ -230,7 +230,8 @@ module Elastomer
       #
       # Returns the response body as a Hash
       def analyze( text, params = {} )
-        response = client.get "{/index}/_analyze", update_params(params, :body => text.to_s, :action => "index.analyze")
+        body = text.is_a?(Hash) ? text : {text: text.to_s}
+        response = client.get "{/index}/_analyze", update_params(params, body: body, action: "index.analyze")
         response.body
       end
 
@@ -261,19 +262,21 @@ module Elastomer
         response.body
       end
 
-      # Optimize one or more indices. Optimizing an index allows for faster
-      # search operations but can be resource intensive.
+      # Force merge one or more indices. Force merging an index allows to
+      # reduce the number of segments but can be resource intensive.
       #
       # params - Parameters Hash
-      #   :index - set to "_all" to optimize all indices
+      #   :index - set to "_all" to force merge all indices
       #
-      # See https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-optimize.html
+      # See https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-forcemerge.html
       #
       # Returns the response body as a Hash
-      def optimize( params = {} )
-        response = client.post "{/index}/_optimize", update_params(params, :action => "index.optimize")
+      def forcemerge( params = {} )
+        response = client.post "{/index}/_forcemerge", update_params(params, :action => "index.forcemerge")
         response.body
       end
+      # DEPRECATED:  ES 5.X has removed the `/_optimize` endpoint.
+      alias_method :optimize, :forcemerge
 
       # Provides insight into ongoing index shard recoveries. Recovery status
       # may be reported for specific indices, or cluster-wide.
@@ -523,14 +526,39 @@ module Elastomer
         Warmer.new(client, name, warmer_name)
       end
 
+      # Delete documents by query following either the native or
+      # application-level delete by query method.
+      #
+      # NOTE: The parameters and response format varies by version. To have more
+      # control over this, use app_delete_by_query or native_delete_by_query
+      # directly.
+      def delete_by_query(query, params = nil)
+        docs.send(client.version_support.delete_by_query_method, query, params)
+      end
+
+      # DEPRECATED: Delete documents from one or more indices and one or more types based
+      # on a query using application-level logic.
+      #
+      # See Client#app_delete_by_query for more information.
+      #
+      # Returns a Hash of statistics about the delete operations simulating the
+      # Elasticsearch 2.x delete by query plugin's output.
+      def app_delete_by_query(query, params = nil)
+        docs.app_delete_by_query(query, params)
+      end
+
       # Delete documents from one or more indices and one or more types based
-      # on a query.
+      # on a query using Elasticsearch's _delete_by_query API.
       #
-      # See Client#delete_by_query for more information.
+      # See Client#native_delete_by_query for more information.
       #
-      # Returns a Hash of statistics about the delete operations
-      def delete_by_query(query, params = nil)
-        docs.delete_by_query(query, params)
+      # Returns a Hash of statistics about the delete operations as returned by
+      # _delete_by_query.
+      #
+      # Raises Elastomer::Client::IncompatibleVersionException if this version
+      # of Elasticsearch does not support _delete_by_query.
+      def native_delete_by_query(query, params = nil)
+        docs.native_delete_by_query(query, params)
       end
 
       # Constructs a Percolator with the given id on this index.

data/lib/elastomer/client/multi_percolate.rb

@@ -80,7 +80,7 @@ module Elastomer
       #
       # Returns this MultiPercolate instance.
       def percolate(doc, header = {})
-        add_to_actions(:percolate => header)
+        add_to_actions(:percolate => @params.merge(header))
         add_to_actions(:doc => doc)
       end
 
@@ -93,7 +93,7 @@ module Elastomer
       #
       # Returns this MultiPercolate instance.
       def count(doc, header = {})
-        add_to_actions(:count => header)
+        add_to_actions(:count => @params.merge(header))
         add_to_actions(:doc => doc)
       end
 

data/lib/elastomer/client/native_delete_by_query.rb

@@ -0,0 +1,60 @@
+module Elastomer
+  class Client
+    # Delete documents based on a query using the Elasticsearch _delete_by_query API.
+    #
+    # query  - The query body as a Hash
+    # params - Parameters Hash
+    #
+    # Examples
+    #
+    #   # request body query
+    #   native_delete_by_query({:query => {:match_all => {}}}, :type => 'tweet')
+    #
+    # See https://www.elastic.co/guide/en/elasticsearch/reference/5.6/docs-delete-by-query.html
+    #
+    # Returns a Hash containing the _delete_by_query response body.
+    def native_delete_by_query(query, parameters = {})
+      NativeDeleteByQuery.new(self, query, parameters).execute
+    end
+
+    class NativeDeleteByQuery
+      attr_reader :client, :query, :parameters
+
+      PARAMETERS = %i[
+        conflicts
+        index
+        q
+        refresh
+        routing
+        scroll_size
+        timeout
+        type
+        wait_for_active_shards
+        wait_for_completion
+      ].to_set.freeze
+
+      def initialize(client, query, parameters)
+        unless client.version_support.native_delete_by_query?
+          raise IncompatibleVersionException, "Elasticsearch '#{client.version}' does not support _delete_by_query"
+        end
+
+        parameters.keys.each do |key|
+          unless PARAMETERS.include?(key) || PARAMETERS.include?(key.to_sym)
+            raise IllegalArgument, "'#{key}' is not a valid _delete_by_query parameter"
+          end
+        end
+
+        @client = client
+        @query = query
+        @parameters = parameters
+
+      end
+
+      def execute
+        # TODO: Require index parameter. type is optional.
+        response = client.post("/{index}{/type}/_delete_by_query", parameters.merge(body: query))
+        response.body
+      end
+    end
+  end
+end

data/lib/elastomer/client/percolator.rb

@@ -12,6 +12,9 @@ module Elastomer
         @client = client
         @index_name = client.assert_param_presence(index_name, "index name")
         @id = client.assert_param_presence(id, "id")
+
+        # COMPATIBILITY
+        @percolator_type = client.version_support.percolator_type
       end
 
       attr_reader :client, :index_name, :id
@@ -25,7 +28,7 @@ module Elastomer
       #
       # Returns the response body as a Hash
       def create(body, params = {})
-        response = client.put("/{index}/.percolator/{id}", defaults.merge(params.merge(:body => body, :action => "percolator.create")))
+        response = client.put("/{index}/#{@percolator_type}/{id}", defaults.merge(params.merge(:body => body, :action => "percolator.create")))
         response.body
       end
 
@@ -38,7 +41,7 @@ module Elastomer
       #
       # Returns the response body as a Hash
       def get(params = {})
-        response = client.get("/{index}/.percolator/{id}", defaults.merge(params.merge(:action => "percolator.get")))
+        response = client.get("/{index}/#{@percolator_type}/{id}", defaults.merge(params.merge(:action => "percolator.get")))
         response.body
       end
 
@@ -51,7 +54,7 @@ module Elastomer
       #
       # Returns the response body as a Hash
       def delete(params = {})
-        response = client.delete("/{index}/.percolator/{id}", defaults.merge(params.merge(:action => "percolator.delete")))
+        response = client.delete("/{index}/#{@percolator_type}/{id}", defaults.merge(params.merge(:action => "percolator.delete")))
         response.body
       end