elastomer-client 2.0.1 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ffa99916d4477ed77f1e67b038ddc8de6c70dcf4
-  data.tar.gz: 5700c42af44a02684b86d29d9381cdcacd0aaf50
+  metadata.gz: b2f48910ab846142b9a55a868ce832dbc4a8c1ec
+  data.tar.gz: 3dd8c645093818313fd9f4fb5a974591bcd33516
 SHA512:
-  metadata.gz: 6f73679c1b09cafca5499fb515795025528b194cc0794aac1ecdcac2d3a084fa69c7f08f89837de61ecaea1d75d48f9e668d5df15351b8eaa9c8c7246a412f29
-  data.tar.gz: 169c324aac607fd8e3531a2c07aa8191df935390e603d8f7498fec3b093cc2a7a039df6e5280d9bf92018a2dc0e7723a97f02408bb9c90aad3abc88ee90989f9
+  metadata.gz: 94812c6c1f76f535858d04a1fa91fafa86d65e57990d56adea77117a374b450e1d1d5267afa5d937e12490a318be32d9fee399f6758f9977fa3c8803acc9f9c0
+  data.tar.gz: ec5999dafd6c14225eea303331db69ce85b19819e2fcda67df2f45ef78f95e7b7eaa73849c0a4fcb2e8c7bba819e731ec444f4f7c9e4b3797233bd07c144a79a

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 2.1.0 (2016-01-02)
+- Added enforcement of maximum request size
+- Added some exception wrapping
+
 ## 2.0.1 (2016-09-01)
 - Fix bug in delete by query when routing is required
 

data/lib/elastomer/client.rb

@@ -8,6 +8,7 @@ require "elastomer/version"
 module Elastomer
 
   class Client
+    MAX_REQUEST_SIZE = 250 * 2**20  # 250 MB
 
     # Create a new client that can be used to make HTTP requests to the
     # Elasticsearch server.
@@ -20,6 +21,7 @@ module Elastomer
     #   :open_timeout - the timeout in seconds when opening an HTTP connection
     #   :adapter      - the Faraday adapter to use (defaults to :excon)
     #   :opaque_id    - set to `true` to use the 'X-Opaque-Id' request header
+    #   :max_request_size - the maximum allowed request size in bytes (defaults to 250 MB)
     #
     def initialize( opts = {} )
       host = opts.fetch :host, "localhost"
@@ -30,14 +32,15 @@ module Elastomer
       @host = uri.host
       @port = uri.port
 
-      @read_timeout = opts.fetch :read_timeout, 5
-      @open_timeout = opts.fetch :open_timeout, 2
-      @adapter      = opts.fetch :adapter, Faraday.default_adapter
-      @opaque_id    = opts.fetch :opaque_id, false
+      @read_timeout     = opts.fetch :read_timeout, 5
+      @open_timeout     = opts.fetch :open_timeout, 2
+      @adapter          = opts.fetch :adapter, Faraday.default_adapter
+      @opaque_id        = opts.fetch :opaque_id, false
+      @max_request_size = opts.fetch :max_request_size, MAX_REQUEST_SIZE
     end
 
     attr_reader :host, :port, :url
-    attr_reader :read_timeout, :open_timeout
+    attr_reader :read_timeout, :open_timeout, :max_request_size
 
     # Returns true if the server is available; returns false otherwise.
     def ping
@@ -71,9 +74,10 @@ module Elastomer
     # Returns a Faraday::Connection
     def connection
       @connection ||= Faraday.new(url) do |conn|
-        conn.request  :encode_json
-        conn.response :parse_json
-        conn.request  :opaque_id if @opaque_id
+        conn.request(:encode_json)
+        conn.response(:parse_json)
+        conn.request(:opaque_id) if @opaque_id
+        conn.request(:limit_size, :max_request_size => max_request_size) if max_request_size
 
         if @adapter.is_a?(Array)
           conn.adapter(*@adapter)
@@ -295,7 +299,27 @@ module Elastomer
     # containing and 'error' field.
     def handle_errors( response )
       raise ServerError, response if response.status >= 500
-      raise RequestError, response if response.body.is_a?(Hash) && response.body["error"]
+
+      if response.body.is_a?(Hash) && (error = response.body["error"])
+        # ES 2.X style
+        if error.is_a?(Hash)
+          root_cause = Array(error["root_cause"]).first || error
+          case root_cause["type"]
+          when "index_not_found_exception"; raise IndexNotFoundError, response
+          when "query_parsing_exception"; raise QueryParsingError, response
+          end
+
+        # ES 1.X style
+        elsif error.is_a?(String)
+          case error
+          when %r/IndexMissingException/; raise IndexNotFoundError, response
+          when %r/QueryParsingException/; raise QueryParsingError, response
+          when %r/ParseException/; raise QueryParsingError, response
+          end
+        end
+
+        raise RequestError, response
+      end
 
       response
     end

data/lib/elastomer/client/bulk.rb

@@ -158,7 +158,7 @@ module Elastomer
     # immediately.
     #
     class Bulk
-      DEFAULT_REQUEST_SIZE = 10 * 2**20 # 10 MB
+      DEFAULT_REQUEST_SIZE = 10 * 2**20  # 10 MB
 
       # Create a new bulk client for handling some of the details of
       # accumulating documents to index and then formatting them properly for
@@ -194,7 +194,10 @@ module Elastomer
         if value.nil?
           @request_size = nil
         else
-          @request_size = value.to_i <= 0 ? nil : value.to_i
+          value = value.to_i
+          value = nil if value <= 0
+          value = client.max_request_size if value > client.max_request_size
+          @request_size = value
         end
       end
 
@@ -370,26 +373,55 @@ module Elastomer
       # document - Optional document for the action as a Hash or JSON encoded String
       #
       # Returns the response from the bulk call if one was made or nil.
+      # Raises RequestSizeError if the given action is larger than the
+      #        configured requst size or the client.max_request_size
       def add_to_actions( action, document = nil )
-        action = MultiJson.dump action
-        @actions << action
-        @current_request_size += action.bytesize
-        @current_action_count += 1
-
-        unless document.nil?
-          document = MultiJson.dump document unless String === document
-          @actions << document
-          @current_request_size += document.bytesize
+        action = MultiJson.dump(action)
+        size = action.bytesize
+
+        if document
+          document = MultiJson.dump(document) unless document.is_a?(String)
+          size += document.bytesize
         end
 
-        if (request_size && @current_request_size >= request_size) ||
-           (action_count && @current_action_count >= action_count)
-          call
-        else
-          nil
+        check_action_size!(size)
+
+        response = nil
+        begin
+          response = call if ready_to_send?(size)
+        rescue StandardError => err
+          raise err
+        ensure
+          @actions << action
+          @actions << document unless document.nil?
+          @current_request_size += size
+          @current_action_count += 1
         end
+
+        response
+      end
+
+      # Internal: Determines if adding `size` more bytes and one more action
+      # will bring the current bulk request over the `request_size` limit or the
+      # `action_count` limit. If this method returns true, then it is time to
+      # dispatch the bulk request.
+      #
+      # Returns `true` of `false`
+      def ready_to_send?( size )
+        total_request_size = @current_request_size + size
+        total_action_count = @current_action_count + 1
+
+        (request_size && total_request_size >= request_size) ||
+        (action_count && total_action_count >  action_count)
       end
 
-    end  # Bulk
-  end  # Client
-end  # Elastomer
+      # Internal: Raises a RequestSizeError if the given size is larger than
+      # the configured requst size or the client.max_request_size
+      def check_action_size!( size )
+        return unless (request_size && size > request_size) || (size > client.max_request_size)
+        raise RequestSizeError, "Bulk action of size `#{size}` is too large"
+      end
+
+    end
+  end
+end

data/lib/elastomer/client/errors.rb

@@ -81,6 +81,12 @@ module Elastomer
     SSLError         = Class.new Error
     ServerError      = Class.new Error
     RequestError     = Class.new Error
+    RequestSizeError = Class.new Error
+
+    # Provide some nice errors for common Elasticsearch exceptions. These are
+    # all subclasses of the more general RequestError
+    IndexNotFoundError = Class.new RequestError
+    QueryParsingError  = Class.new RequestError
 
     ServerError.fatal      = false
     TimeoutError.fatal     = false

data/lib/elastomer/middleware/limit_size.rb

@@ -0,0 +1,30 @@
+module Elastomer
+  module Middleware
+
+    # Request middleware that raises an exception if the request body exceeds a
+    # `max_request_size`.
+    class LimitSize < Faraday::Middleware
+
+      def initialize(app = nil, options = {})
+        super(app)
+        @max_request_size = options.fetch(:max_request_size)
+      end
+
+      attr_reader :max_request_size
+
+      def call(env)
+        if body = env[:body]
+          if body.is_a?(String) && body.bytesize > max_request_size
+            raise ::Elastomer::Client::RequestSizeError,
+              "Request of size `#{body.bytesize}` exceeds the maximum requst size: #{max_request_size}"
+          end
+        end
+        @app.call(env)
+      end
+
+    end
+  end
+end
+
+Faraday::Request.register_middleware \
+  :limit_size => ::Elastomer::Middleware::LimitSize

data/lib/elastomer/version.rb

@@ -1,5 +1,5 @@
 module Elastomer
-  VERSION = "2.0.1"
+  VERSION = "2.1.0"
 
   def self.version
     VERSION

data/test/client/bulk_test.rb

@@ -177,14 +177,14 @@ describe Elastomer::Client::Bulk do
         ary << b.index(document)
       }
       ary.compact!
-      assert_equal 3, ary.length
+      assert_equal 4, ary.length
 
       document = {:_id => 10, :_type => "tweet", :author => "pea53", :message => "tweet 10 is a 102 character request"}
       ary << b.index(document)
     end
     ary.compact!
 
-    assert_equal 4, ary.length
+    assert_equal 5, ary.length
     ary.each { |a| a["items"].each { |b| assert_bulk_index(b) } }
 
     @index.refresh
@@ -208,7 +208,7 @@ describe Elastomer::Client::Bulk do
         ary << b.index(document)
       }
       ary.compact!
-      assert_equal 3, ary.length
+      assert_equal 2, ary.length
 
       document = {:_id => 10, :_type => "tweet", :author => "pea53", :message => "this is tweet number 10"}
       ary << b.index(document)
@@ -224,6 +224,30 @@ describe Elastomer::Client::Bulk do
     assert_equal 10, h["hits"]["total"]
   end
 
+  it "rejects documents that excceed the allowed bulk request size" do
+    ary = []
+    ary << @index.bulk(:request_size => 300) do |b|
+      2.times { |num|
+        document = {:_id => num, :_type => "tweet", :author => "pea53", :message => "tweet #{num} is a 100 character request"}
+        ary << b.index(document)
+      }
+      ary.compact!
+      assert_equal 0, ary.length
+
+      document = {:_id => 342, :_type => "tweet", :author => "pea53", :message => "a"*290}
+      assert_raises(Elastomer::Client::RequestSizeError) { b.index(document) }
+    end
+    ary.compact!
+
+    assert_equal 1, ary.length
+    ary.each { |a| a["items"].each { |b| assert_bulk_index(b) } }
+
+    @index.refresh
+    h = @index.docs.search :q => "*:*", :size => 0
+
+    assert_equal 2, h["hits"]["total"]
+  end
+
   it "uses :id from parameters" do
     response = @index.bulk do |b|
       document = { :_type => "tweet", :author => "pea53", :message => "just a test tweet" }

data/test/client/docs_test.rb

@@ -279,6 +279,14 @@ describe Elastomer::Client::Docs do
     end
   end
 
+  it "generates QueryParsingError exceptions on bad input when searching" do
+    query = {:query => {:query_string => {:query => "OR should fail"}}}
+    assert_raises(Elastomer::Client::QueryParsingError) { @docs.search(query, :type => %w[doc1 doc2]) }
+
+    query = {:query => {:foo_is_not_valid => {}}}
+    assert_raises(Elastomer::Client::QueryParsingError) { @docs.search(query, :type => %w[doc1 doc2]) }
+  end
+
   it "counts documents" do
     h = @docs.count :q => "*:*"
     assert_equal 0, h["count"]

data/test/client/index_test.rb

@@ -259,6 +259,13 @@ describe Elastomer::Client::Index do
     assert_equal %w[just few words analyze], tokens
   end
 
+  describe "when an index does not exist" do
+    it "raises an IndexNotFoundError on delete" do
+      index = $client.index("index-that-does-not-exist")
+      assert_raises(Elastomer::Client::IndexNotFoundError) { index.delete }
+    end
+  end
+
   describe "when an index exists" do
     before do
       suggest = {

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: elastomer-client
 version: !ruby/object:Gem::Version
-  version: 2.0.1
+  version: 2.1.0
 platform: ruby
 authors:
 - Tim Pease
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-01 00:00:00.000000000 Z
+date: 2016-09-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -215,6 +215,7 @@ files:
 - lib/elastomer/client/warmer.rb
 - lib/elastomer/core_ext/time.rb
 - lib/elastomer/middleware/encode_json.rb
+- lib/elastomer/middleware/limit_size.rb
 - lib/elastomer/middleware/opaque_id.rb
 - lib/elastomer/middleware/parse_json.rb
 - lib/elastomer/notifications.rb
@@ -267,7 +268,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.5
+rubygems_version: 2.2.3
 signing_key: 
 specification_version: 4
 summary: A library for interacting with Elasticsearch