elastomer-client 2.3.0 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: efb8ef5cb70b5ee2dff9d968876ec9b2412650f2
-  data.tar.gz: ef5a5fab5b54be5581cddef9e0741f0cae962341
+  metadata.gz: 7db106056a2ea4d14012e82621bc4c4073c85c15
+  data.tar.gz: ed047e4e19b8f45e7c553998cb0935ea59c1a94e
 SHA512:
-  metadata.gz: d3d7c0fe1ef38202642a185670511ad10e99bacd8a57c8f02ab03adefa0133fad0485511cb9a7d70251533b58031987454686783930340060dc641914c21a05d
-  data.tar.gz: b2f69a64ad1e5c6de4f2a6ded6c00f4dbf3c639340bab1003cd7ad9fc80cfef8522df9bfe0be5c9f1e27fb00f7295c194f331c83228c2881ea60f36bdd99eb5f
+  metadata.gz: cb92e602580ce508195626093783510df3320ba20416cac7c879fcf80839ce87fbb64dd0af7c8f37b0a2775cf9f0c3d603a692251b8ab9c7c87bcfe867b955a4
+  data.tar.gz: 423da9494641de0a33a90dc26fdae67564469291a1120202015a5f9daecab22da06bbbf9c1e351a41a825b031563e90066ba9585b483d5c257ad3044c64ba449

data/.gitignore

@@ -8,3 +8,5 @@
 Gemfile.lock
 *.gem
 tags
+.byebug_history
+*.swp

data/.travis.yml

@@ -6,10 +6,6 @@ env:
   - ES_VERSION=2.3.5 ES_DOWNLOAD_URL=https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/${ES_VERSION}/elasticsearch-${ES_VERSION}.tar.gz
   - ES_VERSION=5.6.4 ES_DOWNLOAD_URL=https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ES_VERSION}.tar.gz
 
-matrix:
-  allow_failures:
-    - env: ES_VERSION=5.6.4 ES_DOWNLOAD_URL=https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ES_VERSION}.tar.gz
-
 # Configure a specific version of Elasticsearch
 # See: https://docs.travis-ci.com/user/database-setup/#Installing-ElasticSearch-on-trusty-container-based-infrastructure
 install:

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 3.0.0 (2017-12-15)
+- Fixed swapped args in {Client,Index}#multi\_percolate count calls using block API
+- Support for Elasticsearch 5.x
+- Uses Elasticsearch's built-in `_delete_by_query` when supported
+- GET and HEAD requests are retried when possible
+- Add support for `_tasks` API
+- Replace `scan` queries with `scroll` sorted by `doc_id`
+
 ## 2.3.0 (2017-11-29)
 - Remove Elasticsearch 1.x and earlier code paths
 - Fix CI and configure an Elasticsearch 5.6 build

data/docker/docker-compose.cibuild.yml

@@ -0,0 +1,8 @@
+version: "2.1"
+
+networks:
+  default:
+    ipam:
+      config:
+        - subnet: 192.168.0.0/26
+          gateway: 192.168.0.1

data/docker/docker-compose.es24.yml

@@ -0,0 +1,34 @@
+version: "2.1"
+
+services:
+  elasticsearch2.4:
+    image: elasticsearch:2.4.6
+    container_name: es2.4
+    environment:
+      - cluster.name=elastomer2.4
+      - bootstrap.memory_lock=true
+      - discovery.type=single-node
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    mem_limit: 1g
+    cap_add:
+      - IPC_LOCK
+    volumes:
+      - esrepos:/usr/share/elasticsearch/repos
+      - ./elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
+    ports:
+      - 127.0.0.1:19200:9200
+
+volumes:
+  esrepos:
+    driver: local
+    driver_opts:
+      device: tmpfs
+      type: tmpfs
+      o: size=100m,uid=102,gid=102

data/docker/docker-compose.es56.yml

@@ -0,0 +1,37 @@
+version: "2.1"
+
+services:
+  elasticsearch5.6:
+    image: docker.elastic.co/elasticsearch/elasticsearch:5.6.4
+    container_name: es5.6
+    environment:
+      - cluster.name=elastomer5.6
+      - bootstrap.memory_lock=true
+      - discovery.type=single-node
+      - xpack.monitoring.enabled=false
+      - xpack.security.enabled=false
+      - xpack.watcher.enabled=false
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    mem_limit: 1g
+    cap_add:
+      - IPC_LOCK
+    volumes:
+      - esrepos:/usr/share/elasticsearch/repos
+      - ./elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
+    ports:
+      - 127.0.0.1:19200:9200
+
+volumes:
+  esrepos:
+    driver: local
+    driver_opts:
+      device: tmpfs
+      type: tmpfs
+      o: size=100m,uid=102,gid=102

data/docker/elasticsearch.yml

@@ -0,0 +1,15 @@
+cluster.name: "docker-cluster"
+
+network.host: 0.0.0.0
+
+discovery.zen.minimum_master_nodes: 1
+
+path:
+  data: /usr/share/elasticsearch/data
+  logs: /usr/share/elasticsearch/logs
+  repo: /usr/share/elasticsearch/repos
+
+transport.tcp.port: 9300
+http.port: 9200
+http.max_content_length: 50mb
+

data/docs/index.md

@@ -137,8 +137,8 @@ Let's take a look at some simple event log maintenance using elastomer-client.
 # the previous month's event log
 index = client.index "event-log-2014-09"
 
-# optimize the index to have only 1 segment file (expunges deleted documents)
-index.optimize \
+# force merge the index to have only 1 segment file (expunges deleted documents)
+index.force merge \
   :max_num_segments => 1,
   :wait_for_merge   => true
 

data/docs/notifications.md

@@ -63,7 +63,7 @@ The event namespace is `request.client.elastomer`.
 - index.get_mapping
 - index.get_settings
 - index.open
-- index.optimize
+- index.forcemerge
 - index.recovery
 - index.refresh
 - index.segments

data/elastomer-client.gemspec

@@ -28,7 +28,9 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "activesupport",      ">= 3.0"
   spec.add_development_dependency "minitest",           "~> 5.10"
   spec.add_development_dependency "minitest-fail-fast", "~> 0.1.0"
+  spec.add_development_dependency "minitest-focus",     "~> 1.1", ">= 1.1.2"
   spec.add_development_dependency "webmock",            "~> 2.3"
   spec.add_development_dependency "awesome_print",      "~> 1.8"
+  spec.add_development_dependency "pry-byebug",         "~> 3.4"
   spec.add_development_dependency "rake"
 end

data/lib/elastomer/client.rb

@@ -4,16 +4,23 @@ require "multi_json"
 require "semantic"
 
 require "elastomer/version"
+require "elastomer/version_support"
 
 module Elastomer
 
   class Client
     MAX_REQUEST_SIZE = 250 * 2**20  # 250 MB
 
+    RETRYABLE_METHODS = %i[get head].freeze
+
     # Create a new client that can be used to make HTTP requests to the
-    # Elasticsearch server.
+    # Elasticsearch server. If you use `:max_retries`, then any GET or HEAD
+    # request will be retried up to this many times with a 75ms delay between
+    # the retry attempts. Only non-fatal exceptions will retried automatically.
+    #
+    # see lib/elastomer/client/errors.rb#L92-L94
     #
-    # opts - The options Hash
+    # Method params:
     #   :host - the host as a String
     #   :port - the port number of the server
     #   :url  - the URL as a String (overrides :host and :port)
@@ -22,25 +29,43 @@ module Elastomer
     #   :adapter      - the Faraday adapter to use (defaults to :excon)
     #   :opaque_id    - set to `true` to use the 'X-Opaque-Id' request header
     #   :max_request_size - the maximum allowed request size in bytes (defaults to 250 MB)
+    #   :max_retries      - the maximum number of request retires (defaults to 0)
+    #   :retry_delay      - delay in seconds between retries (defaults to 0.075)
     #
-    def initialize( opts = {} )
-      host = opts.fetch :host, "localhost"
-      port = opts.fetch :port, 9200
-      @url = opts.fetch :url,  "http://#{host}:#{port}"
+    def initialize(host: "localhost", port: 9200, url: nil,
+                   read_timeout: 5, open_timeout: 2, max_retries: 0, retry_delay: 0.075,
+                   opaque_id: false, adapter: Faraday.default_adapter, max_request_size: MAX_REQUEST_SIZE)
+
+      @url = url || "http://#{host}:#{port}"
 
       uri = Addressable::URI.parse @url
       @host = uri.host
       @port = uri.port
 
-      @read_timeout     = opts.fetch :read_timeout, 5
-      @open_timeout     = opts.fetch :open_timeout, 2
-      @adapter          = opts.fetch :adapter, Faraday.default_adapter
-      @opaque_id        = opts.fetch :opaque_id, false
-      @max_request_size = opts.fetch :max_request_size, MAX_REQUEST_SIZE
+      @read_timeout     = read_timeout
+      @open_timeout     = open_timeout
+      @max_retries      = max_retries
+      @retry_delay      = retry_delay
+      @adapter          = adapter
+      @opaque_id        = opaque_id
+      @max_request_size = max_request_size
     end
 
     attr_reader :host, :port, :url
-    attr_reader :read_timeout, :open_timeout, :max_request_size
+    attr_reader :read_timeout, :open_timeout
+    attr_reader :max_retries, :retry_delay, :max_request_size
+
+    # Returns a duplicate of this Client connection configured in the exact same
+    # fashion.
+    def dup
+      self.class.new \
+          url:              url,
+          read_timeout:     read_timeout,
+          open_timeout:     open_timeout,
+          adapter:          @adapter,
+          opaque_id:        @opaque_id,
+          max_request_size: max_request_size
+    end
 
     # Returns true if the server is available; returns false otherwise.
     def ping
@@ -90,6 +115,15 @@ module Elastomer
       end
     end
 
+    # Internal: Reset the client by removing the current Faraday::Connection. A
+    # new connection will be established on the next request.
+    #
+    # Returns this Client instance.
+    def reset!
+      @connection = nil
+      self
+    end
+
     # Internal: Sends an HTTP HEAD request to the server.
     #
     # path   - The path as a String
@@ -153,15 +187,18 @@ module Elastomer
     # params - Parameters Hash
     #   :body         - Will be used as the request body
     #   :read_timeout - Optional read timeout (in seconds) for the request
+    #   :max_retires  - Optional retry number for the request
     #
     # Returns a Faraday::Response
     # Raises an Elastomer::Client::Error on 4XX and 5XX responses
     # rubocop:disable Metrics/MethodLength
     def request( method, path, params )
-      read_timeout = params.delete :read_timeout
-      body = extract_body params
-      path = expand_path path, params
+      read_timeout = params.delete(:read_timeout)
+      request_max_retries = params.delete(:max_retries) || max_retries
+      body = extract_body(params)
+      path = expand_path(path, params)
 
+      params[:retries] = retries = 0
       instrument(path, body, params) do
         begin
           response =
@@ -195,14 +232,35 @@ module Elastomer
 
         # wrap Faraday errors with appropriate Elastomer::Client error classes
         rescue Faraday::Error::ClientError => boom
-          error_name = boom.class.name.split("::").last
-          error_class = Elastomer::Client.const_get(error_name) rescue Elastomer::Client::Error
-          raise error_class.new(boom, method.upcase, path)
+          error = wrap_faraday_error(boom, method, path)
+          if error.retry? && RETRYABLE_METHODS.include?(method) && (retries += 1) <= request_max_retries
+            params[:retries] = retries
+            sleep retry_delay
+            retry
+          end
+          raise error
+        rescue OpaqueIdError => boom
+          reset!
+          raise boom
         end
       end
     end
     # rubocop:enable Metrics/MethodLength
 
+    # Internal: Returns a new Elastomer::Client error that wraps the given
+    # Faraday error. A generic Error is returned if we cannot wrap the given
+    # Faraday error.
+    #
+    #   error  - The Faraday error
+    #   method - The request method
+    #   path   - The request path
+    #
+    def wrap_faraday_error(error, method, path)
+      error_name  = error.class.name.split("::").last
+      error_class = Elastomer::Client.const_get(error_name) rescue Elastomer::Client::Error
+      error_class.new(error, method.upcase, path)
+    end
+
     # Internal: Extract the :body from the params Hash and convert it to a
     # JSON String format. If the params Hash does not contain a :body then no
     # action is taken and `nil` is returned.
@@ -263,6 +321,7 @@ module Elastomer
       query_values = params.dup
       query_values.delete :action
       query_values.delete :context
+      query_values.delete :retries
 
       template.keys.map(&:to_sym).each do |key|
         value = query_values.delete key
@@ -305,21 +364,11 @@ module Elastomer
       raise ServerError, response if response.status >= 500
 
       if response.body.is_a?(Hash) && (error = response.body["error"])
-        # ES 2.X style
-        if error.is_a?(Hash)
-          root_cause = Array(error["root_cause"]).first || error
-          case root_cause["type"]
-          when "index_not_found_exception"; raise IndexNotFoundError, response
-          when "query_parsing_exception"; raise QueryParsingError, response
-          end
-
-        # ES 1.X style
-        elsif error.is_a?(String)
-          case error
-          when %r/IndexMissingException/; raise IndexNotFoundError, response
-          when %r/QueryParsingException/; raise QueryParsingError, response
-          when %r/ParseException/; raise QueryParsingError, response
-          end
+        root_cause = Array(error["root_cause"]).first || error
+        case root_cause["type"]
+        when "index_not_found_exception"; raise IndexNotFoundError, response
+        when "illegal_argument_exception"; raise IllegalArgument, response
+        when *version_support.query_parse_exception; raise QueryParsingError, response
         end
 
         raise RequestError, response
@@ -361,6 +410,10 @@ module Elastomer
       end
     end
 
+    def version_support
+      @version_support ||= VersionSupport.new(version)
+    end
+
   end  # Client
 end  # Elastomer
 

data/lib/elastomer/client/app_delete_by_query.rb

@@ -0,0 +1,158 @@
+module Elastomer
+  class Client
+
+    # DEPRECATED: Delete documents from one or more indices and one or more
+    # types based on a query.
+    #
+    # The return value follows the format returned by the Elasticsearch Delete
+    # by Query plugin: https://github.com/elastic/elasticsearch/blob/v2.4.6/docs/plugins/delete-by-query.asciidoc
+    #
+    # Internally, this method uses a combination of scroll and bulk delete
+    # instead of the Delete by Query API, which was removed in Elasticsearch
+    # 2.0.
+    #
+    # For native _delete_by_query functionality in Elasticsearch 5+, see the
+    # native_delete_by_query method.
+    #
+    # query  - The query body as a Hash
+    # params - Parameters Hash
+    #
+    # Examples
+    #
+    #   # request body query
+    #   app_delete_by_query({:query => {:match_all => {}}}, :type => 'tweet')
+    #
+    #   # same thing but using the URI request method
+    #   app_delete_by_query(nil, { :q => '*:*', :type => 'tweet' })
+    #
+    # See https://www.elastic.co/guide/en/elasticsearch/plugins/current/delete-by-query-usage.html
+    #
+    # Returns a Hash of statistics about the delete operations, for example:
+    #
+    #   {
+    #     "took" : 639,
+    #     "_indices" : {
+    #       "_all" : {
+    #         "found" : 5901,
+    #         "deleted" : 5901,
+    #         "missing" : 0,
+    #         "failed" : 0
+    #       },
+    #       "twitter" : {
+    #         "found" : 5901,
+    #         "deleted" : 5901,
+    #         "missing" : 0,
+    #         "failed" : 0
+    #       }
+    #     },
+    #     "failures" : [ ]
+    #   }
+    def app_delete_by_query(query, params = {})
+      AppDeleteByQuery.new(self, query, params).execute
+    end
+
+    class AppDeleteByQuery
+
+      SEARCH_PARAMETERS = %i[
+        index
+        type
+        q
+        df
+        analyzer
+        analyze_wildcard
+        batched_reduce_size
+        default_operator
+        lenient
+        explain
+        _source
+        stored_fields
+        sort
+        track_scores
+        timeout
+        terminate_after
+        from
+        size
+        search_type
+        scroll
+      ].to_set.freeze
+
+
+      # Create a new DeleteByQuery command for deleting documents matching a
+      # query
+      #
+      # client - Elastomer::Client used for HTTP requests to the server
+      # query  - The query used to find documents to delete
+      # params - Other URL parameters
+      def initialize(client, query, params = {})
+        @client = client
+        @query = query
+        @params = params
+        @response_stats = { "took" => 0, "_indices" => { "_all" => {} }, "failures" => [] }
+      end
+
+      attr_reader :client, :query, :params, :response_stats
+
+      # Internal: Determine whether or not an HTTP status code is in the range
+      # 200 to 299
+      #
+      # status - HTTP status code
+      #
+      # Returns a boolean
+      def is_ok?(status)
+        status.between?(200, 299)
+      end
+
+      # Internal: Tally the contributions of an item to the found, deleted,
+      # missing, and failed counts for the summary statistics
+      #
+      # item - An element of the items array from a bulk response
+      #
+      # Returns a Hash of counts for each category
+      def categorize(item)
+        {
+          "found" => item["found"] || item["status"] == 409 ? 1 : 0,
+          "deleted" => is_ok?(item["status"]) ? 1 : 0,
+          "missing" => !item["found"]  && !item.key?("error") ? 1 : 0,
+          "failed" => item.key?("error") ? 1 : 0,
+        }
+      end
+
+      # Internal: Combine a response item with the existing statistics
+      #
+      # item - A bulk response item
+      def accumulate(item)
+        item = item["delete"]
+        (@response_stats["_indices"][item["_index"]] ||= {}).merge!(categorize(item)) { |_, n, m| n + m }
+        @response_stats["_indices"]["_all"].merge!(categorize(item)) { |_, n, m| n + m }
+        @response_stats["failures"] << item unless is_ok? item["status"]
+      end
+
+      # Perform the Delete by Query action
+      #
+      # Returns a Hash of statistics about the bulk operation
+      def execute
+        ops = Enumerator.new do |yielder|
+          @client.scan(@query, search_params).each_document do |hit|
+            yielder.yield([:delete, hit.select { |key, _| ["_index", "_type", "_id", "_routing"].include?(key) }])
+          end
+        end
+
+        stats = @client.bulk_stream_items(ops, bulk_params) { |item| accumulate(item) }
+        @response_stats["took"] = stats["took"]
+        @response_stats
+      end
+
+      # Internal: Remove parameters that are not valid for the _search endpoint
+      def search_params
+        params = @params.merge(:_source => false)
+        params.select {|p, _| SEARCH_PARAMETERS.include? p}
+      end
+
+      # Internal: Remove parameters that are not valid for the _bulk endpoint
+      def bulk_params
+        @params.clone.tap { |p| p.delete(:q) }
+      end
+
+    end  # AppDeleteByQuery
+  end  # Client
+end  # Elastomer