td-client 0.8.80 → 0.8.81

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6b90ad6943946fa449277adfcb6cf73151d06f5d
-  data.tar.gz: 3955d6677c5ff35ec1da44ed517d8ab69fca7b47
+  metadata.gz: f6639c3d87e9a018b4d8270c64165afc64ddb5c1
+  data.tar.gz: f0d2a64b5cb60ec6bc0b43f53f6ea0f8136e236f
 SHA512:
-  metadata.gz: 21a28c9c7d85fa5b70eca4f9d841cbea5ee19a240140201046a12a3bfd13e52293a6bd6cbccfa4c4b2526b41689f9c027923f0f2b9260853491e839e2a7f603f
-  data.tar.gz: ae79f478833ade0e3ab01c30e8eacdbc1ea3b1cbae613e9a68bead92b746b6b5165c1b9106abee517fb865a949813d9ef48eceff44a85bef7cf40bdb524c0109
+  metadata.gz: 7fe4f5336064479b42b5f867bd54014530aae360b175e3953aa3d82ee06822361d2c41219b9f64179791c08e918102de712ad323de39ef8f5e06ea2990d17e95
+  data.tar.gz: dd8dffeca422ea5d9e57ece36ed2655b4a2b57ef072dcf697dddd6f6c01d8a8a8ba04fd1f946d442982242d1028101830a8c6aefed676350ce168708c8d90324

data/lib/td/client/api.rb

@@ -235,19 +235,17 @@ private
 
   # @param [String] url
   # @param [Hash] params
-  # @param [Hash] opt
   # @yield [response]
-  def get(url, params=nil, opt={}, &block)
+  def get(url, params=nil, &block)
     guard_no_sslv3 do
-      do_get(url, params, opt, &block)
+      do_get(url, params, &block)
     end
   end
 
   # @param [String] url
   # @param [Hash] params
-  # @param [Hash] opt
   # @yield [response]
-  def do_get(url, params=nil, opt={})
+  def do_get(url, params=nil, &block)
     client, header = new_client
     client.send_timeout = @send_timeout
     client.receive_timeout = @read_timeout
@@ -270,49 +268,23 @@ private
     # for both exceptions and 500+ errors retrying is enabled by default.
     # The total number of retries cumulatively should not exceed 10 minutes / 600 seconds
     response = nil
-    etag = nil
-    current_total_chunk_size = 0
-    body = String.new unless block_given?
     begin # this block is to allow retry (redo) in the begin part of the begin-rescue block
       begin
-        if etag
-          header['If-Range'] = etag
-          header['Range'] = "bytes=#{current_total_chunk_size}-"
-        else
-          etag = nil
+        if block
           current_total_chunk_size = 0
-          body.clear if body
-        end
-
-        if block_given?
           response = client.get(target, params, header) {|res, chunk|
-            current_total_chunk_size += chunk.bytesize if res.status == 200
-            yield res, chunk, current_total_chunk_size
+            current_total_chunk_size += chunk.bytesize
+            block.call(res, chunk, current_total_chunk_size)
           }
+
+          # XXX ext/openssl raises EOFError in case where underlying connection causes an error,
+          #     and msgpack-ruby that used in block handles it as an end of stream == no exception.
+          #     Therefor, check content size.
+          validate_content_length!(response, current_total_chunk_size) if @ssl
         else
           response = client.get(target, params, header)
-          if response.status == 200
-            current_total_chunk_size += response.body.bytesize
-            body << response.body
-          end
-        end
 
-        # XXX ext/openssl raises EOFError in case where underlying connection causes an error,
-        #     but httpclient ignores it. Therefor, check content size.
-        #     https://github.com/nahi/httpclient/issues/296
-        if expected_size = response.header['Content-Range'].first
-          expected_size = expected_size[/\d+$/]
-        else
-          expected_size = response.header['Content-Length'].first
-        end
-        if expected_size
-          expected_size = expected_size.to_i
-          if expected_size != current_total_chunk_size
-            if expected_size < current_total_chunk_size
-              etag = false
-            end
-            raise IncompleteError, "#{expected_size} bytes expected, but got #{current_total_chunk_size} bytes"
-          end
+          validate_content_length!(response, response.body.bytesize) if @ssl
         end
 
         status = response.code
@@ -325,9 +297,7 @@ private
           redo # restart from beginning of do-while loop
         end
       rescue Errno::ECONNREFUSED, Errno::ECONNRESET, Timeout::Error, EOFError, OpenSSL::SSL::SSLError, SocketError, IncompleteError => e
-        if opt[:resume]
-          etag = response.header['ETag'].first if etag != false
-        elsif block_given?
+        if block_given?
           raise e
         end
         $stderr.print "#{e.class}: #{e.message}. "
@@ -353,24 +323,29 @@ private
       puts "DEBUG:   body:   " + response.body.to_s
     end
 
-    body = inflate_body(response, body) unless block_given?
+    body = block ? response.body : inflate_body(response)
 
     return [response.code.to_s, body, response]
   end
 
-  def inflate_body(response, body=response.body)
-    return body if (ce = response.header['Content-Encoding']).empty?
+  def validate_content_length!(response, body_size)
+    content_length = response.header['Content-Length'].first
+    raise IncompleteError if @ssl && content_length && content_length.to_i != body_size
+  end
+
+  def inflate_body(response)
+    return response.body if (ce = response.header['Content-Encoding']).empty?
 
     if ce.include?('gzip')
       infl = Zlib::Inflate.new(Zlib::MAX_WBITS + 16)
       begin
-        infl.inflate(body)
+        infl.inflate(response.body)
       ensure
         infl.close
       end
     else
       # NOTE maybe for content-encoding is msgpack.gz ?
-      Zlib::Inflate.inflate(body)
+      Zlib::Inflate.inflate(response.body)
     end
   end
 
@@ -628,7 +603,7 @@ private
         error['stacktrace'] = js['stacktrace']
       end
     rescue JSON::ParserError
-      error['message'] = res.body
+      error['message'] = res.body[0,1000].dump
     end
 
     error

data/lib/td/client/api/job.rb

@@ -115,14 +115,13 @@ module Job
   # @param [String] job_id
   # @return [Array]
   def job_result(job_id)
-    code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>'msgpack'}, {:resume => true})
-    if code != "200"
-      raise_error("Get job result failed", res)
-    end
     result = []
-    MessagePack::Unpacker.new.feed_each(body) {|row|
-      result << row
-    }
+    unpacker = MessagePack::Unpacker.new
+    job_result_download(job_id) do |chunk|
+      unpacker.feed_each(chunk) do |row|
+        result << row
+      end
+    end
     return result
   end
 
@@ -133,24 +132,17 @@ module Job
   # @param [IO] io
   # @param [Proc] block
   # @return [nil, String]
-  def job_result_format(job_id, format, io=nil, &block)
+  def job_result_format(job_id, format, io=nil)
     if io
-      infl = nil
-      code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>format}, {:resume => true}) {|res, chunk, current_total_chunk_size|
-        if res.code != 200
-          raise_error("Get job result failed", res)
-        end
-
-        infl ||= create_inflalte_or_null_inflate(res)
-
-        io.write infl.inflate(chunk)
-        block.call(current_total_chunk_size) if block_given?
-      }
+      job_result_download(job_id, format) do |chunk, total|
+        io.write chunk
+        yield total if block_given?
+      end
       nil
     else
-      code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>format}, {:resume => true})
-      if code != "200"
-        raise_error("Get job result failed", res)
+      body = String.new
+      job_result_download(job_id, format) do |chunk|
+        body << chunk
       end
       body
     end
@@ -163,22 +155,11 @@ module Job
   # @return [nil]
   def job_result_each(job_id, &block)
     upkr = MessagePack::Unpacker.new
-    infl = nil
-
-    get("/v3/job/result/#{e job_id}", {'format'=>'msgpack'}, {:resume => true}) {|res, chunk, current_total_chunk_size|
-      if res.code != 200
-        raise_error("Get job result failed", res)
-      end
-
-      # default to decompressing the response since format is fixed to 'msgpack'
-      infl ||= create_inflate(res)
-
-      inflated_fragment = infl.inflate(chunk)
-      upkr.feed_each(inflated_fragment, &block)
-    }
+    # default to decompressing the response since format is fixed to 'msgpack'
+    job_result_download(job_id) do |chunk|
+      upkr.feed_each(chunk, &block)
+    end
     nil
-  ensure
-    infl.close if infl
   end
 
   # block is optional and must accept 1 argument
@@ -186,50 +167,30 @@ module Job
   # @param [String] job_id
   # @param [Proc] block
   # @return [nil]
-  def job_result_each_with_compr_size(job_id, &block)
+  def job_result_each_with_compr_size(job_id)
     upkr = MessagePack::Unpacker.new
-    infl = nil
-
-    get("/v3/job/result/#{e job_id}", {'format'=>'msgpack'}, {:resume => true}) {|res, chunk, current_total_chunk_size|
-      if res.code != 200
-        raise_error("Get job result failed", res)
-      end
-
-      # default to decompressing the response since format is fixed to 'msgpack'
-      infl ||= create_inflate(res)
-
-      inflated_fragment = infl.inflate(chunk)
-      upkr.feed_each(inflated_fragment) {|unpacked|
-        block.call(unpacked, current_total_chunk_size) if block_given?
+    # default to decompressing the response since format is fixed to 'msgpack'
+    job_result_download(job_id) do |chunk, total|
+      upkr.feed_each(chunk) {|unpacked|
+        yield unpacked, total if block_given?
       }
-    }
+    end
     nil
-  ensure
-    infl.close if infl
   end
 
   # @param [String] job_id
   # @param [String] format
   # @return [String]
-  def job_result_raw(job_id, format, io = nil, &block)
-    body = nil
-
-    get("/v3/job/result/#{e job_id}", {'format'=>format}, {:resume => true}) {|res, chunk, current_total_chunk_size|
-      unless res.ok?
-        raise_error("Get job result failed", res)
-      end
-
+  def job_result_raw(job_id, format, io = nil)
+    body = io ? nil : String.new
+    job_result_download(job_id, format, false) do |chunk, total|
       if io
         io.write(chunk)
-        block.call(current_total_chunk_size) if block_given?
+        yield total if block_given?
       else
-        if body
-          body += chunk
-        else
-          body = chunk
-        end
+        body << chunk
       end
-    }
+    end
     body
   end
 
@@ -287,6 +248,117 @@ module Job
 
   private
 
+  def validate_content_length_with_range(response, current_total_chunk_size)
+    if expected_size = response.header['Content-Range'][0]
+      expected_size = expected_size[/\d+$/].to_i
+    elsif expected_size = response.header['Content-Length'][0]
+      expected_size = expected_size.to_i
+    end
+
+    if expected_size.nil?
+    elsif current_total_chunk_size < expected_size
+      # too small
+      # NOTE:
+      #   ext/openssl raises EOFError in case where underlying connection
+      #   causes an error, but httpclient ignores it.
+      #   https://github.com/nahi/httpclient/blob/v3.2.8/lib/httpclient/session.rb#L1003
+      raise EOFError, 'httpclient IncompleteError'
+    elsif current_total_chunk_size > expected_size
+      # too large
+      raise_error("Get job result failed", response)
+    end
+  end
+
+  def job_result_download(job_id, format='msgpack', autodecode=true)
+    client, header = new_client
+    client.send_timeout = @send_timeout
+    client.receive_timeout = @read_timeout
+    header['Accept-Encoding'] = 'deflate, gzip'
+
+    url = build_endpoint("/v3/job/result/#{e job_id}", @host)
+    params = {'format' => format}
+
+    unless ENV['TD_CLIENT_DEBUG'].nil?
+      puts "DEBUG: REST GET call:"
+      puts "DEBUG:   header: " + header.to_s
+      puts "DEBUG:   url:    " + url.to_s
+      puts "DEBUG:   params: " + params.to_s
+    end
+
+    # up to 7 retries with exponential (base 2) back-off starting at 'retry_delay'
+    retry_delay = @retry_delay
+    cumul_retry_delay = 0
+    current_total_chunk_size = 0
+    infl = nil
+    begin # LOOP of Network/Server errors
+      response = nil
+      client.get(url, params, header) do |res, chunk|
+        unless response
+          case res.status
+          when 200
+            if current_total_chunk_size != 0
+              # try to resume but the server returns 200
+              raise_error("Get job result failed", res)
+            end
+          when 206 # resuming
+          else
+            if res.status/100 == 5 && cumul_retry_delay < @max_cumul_retry_delay
+              $stderr.puts "Error #{res.status}: #{get_error(res)}. Retrying after #{retry_delay} seconds..."
+              sleep retry_delay
+              cumul_retry_delay += retry_delay
+              retry_delay *= 2
+              redo
+            end
+            raise_error("Get job result failed", res)
+          end
+          if infl.nil? && autodecode
+            case res.header['Content-Encoding'][0].to_s.downcase
+            when 'gzip'
+              infl = Zlib::Inflate.new(Zlib::MAX_WBITS + 16)
+            when 'deflate'
+              infl = Zlib::Inflate.new
+            end
+          end
+        end
+        response = res
+        current_total_chunk_size += chunk.bytesize
+        chunk = infl.inflate(chunk) if infl
+        yield chunk, current_total_chunk_size
+      end
+
+      # completed?
+      validate_content_length_with_range(response, current_total_chunk_size)
+    rescue Errno::ECONNREFUSED, Errno::ECONNRESET, Timeout::Error, EOFError, OpenSSL::SSL::SSLError, SocketError => e
+      if response # at least a chunk is downloaded
+        if etag = response.header['ETag'][0]
+          header['If-Range'] = etag
+          header['Range'] = "bytes=#{current_total_chunk_size}-"
+        end
+      end
+
+      $stderr.print "#{e.class}: #{e.message}. "
+      if cumul_retry_delay < @max_cumul_retry_delay
+        $stderr.puts "Retrying after #{retry_delay} seconds..."
+        sleep retry_delay
+        cumul_retry_delay += retry_delay
+        retry_delay *= 2
+        retry
+      end
+      raise
+    end
+
+    unless ENV['TD_CLIENT_DEBUG'].nil?
+      puts "DEBUG: REST GET response:"
+      puts "DEBUG:   header: " + response.header.to_s
+      puts "DEBUG:   status: " + response.code.to_s
+      puts "DEBUG:   body:   " + response.body.to_s
+    end
+
+    nil
+  ensure
+    infl.close if infl
+  end
+
   class NullInflate
     def inflate(chunk)
       chunk

data/lib/td/client/version.rb

@@ -1,5 +1,5 @@
 module TreasureData
   class Client
-    VERSION = '0.8.80'
+    VERSION = '0.8.81'
   end
 end

data/spec/td/client/api_spec.rb

@@ -178,7 +178,7 @@ describe API do
 
       let(:api) { API.new(nil, endpoint: endpoint) }
       let :packed do
-        s = StringIO.new
+        s = StringIO.new(String.new)
         Zlib::GzipWriter.wrap(s) do |f|
           f << ['hello', 'world'].to_json
         end
@@ -195,7 +195,7 @@ describe API do
       end
 
       subject (:get_api_call) {
-        api.job_result_format(12345, 'json', StringIO.new)
+        api.job_result_format(12345, 'json', StringIO.new(String.new))
       }
 
       context 'without ssl' do

data/spec/td/client/job_api_spec.rb

@@ -166,7 +166,7 @@ describe 'Job API' do
 
   describe 'job_result' do
     let :packed do
-      s = StringIO.new
+      s = StringIO.new(String.new)
       pk = MessagePack::Packer.new(s)
       pk.write('hello')
       pk.write('world')
@@ -181,6 +181,74 @@ describe 'Job API' do
       expect(api.job_result(12345)).to eq(['hello', 'world'])
     end
 
+    it '200->200 cannot resume' do
+      sz = packed.bytesize / 3
+      stub_api_request(:get, '/v3/job/result/12345').
+        with(:query => {'format' => 'msgpack'}).
+        to_return(
+          :headers => {
+            'Content-Length' => packed.bytesize,
+            'Etag' => '"abcdefghijklmn"',
+          },
+          :body => packed[0, sz]
+        )
+      stub_api_request(:get, '/v3/job/result/12345').
+        with(
+          :headers => {
+            'If-Range' => '"abcdefghijklmn"',
+            'Range' => "bytes=#{sz}-",
+          },
+          :query => {'format' => 'msgpack'}
+        ).
+        to_return(
+          :status => 200,
+          :headers => {
+            'Content-Length' => packed.bytesize - sz,
+            'Content-Range' => "bytes #{sz}-#{packed.bytesize-1}/#{packed.bytesize}",
+            'Etag' => '"abcdefghijklmn"',
+          },
+          :body => packed
+        )
+      expect(api).to receive(:sleep).once
+      expect($stderr).to receive(:print)
+      expect($stderr).to receive(:puts)
+      expect{api.job_result(12345)}.to raise_error(TreasureData::APIError)
+    end
+
+    it '200->403 cannot resume' do
+      sz = packed.bytesize / 3
+      stub_api_request(:get, '/v3/job/result/12345').
+        with(:query => {'format' => 'msgpack'}).
+        to_return(
+          :headers => {
+            'Content-Length' => packed.bytesize,
+            'Etag' => '"abcdefghijklmn"',
+          },
+          :body => packed[0, sz]
+        )
+      stub_api_request(:get, '/v3/job/result/12345').
+        with(
+          :headers => {
+            'If-Range' => '"abcdefghijklmn"',
+            'Range' => "bytes=#{sz}-",
+          },
+          :query => {'format' => 'msgpack'}
+        ).
+        to_return(
+          :status => 403,
+          :headers => {
+            'Content-Length' => packed.bytesize - sz,
+            'Content-Range' => "bytes #{sz}-#{packed.bytesize-1}/#{packed.bytesize}",
+            'Etag' => '"abcdefghijklmn"',
+          },
+          :body => packed
+        )
+      expect(api).to receive(:sleep).once
+      expect($stderr).to receive(:print)
+      expect($stderr).to receive(:puts)
+      expect{api.job_result(12345)}.to raise_error(TreasureData::APIError)
+    end
+
     it 'can resume' do
       sz = packed.bytesize / 3
       stub_api_request(:get, '/v3/job/result/12345').
@@ -201,6 +269,7 @@ describe 'Job API' do
           :query => {'format' => 'msgpack'}
         ).
         to_return(
+          :status => 206,
           :headers => {
             'Content-Length' => packed.bytesize - sz,
             'Content-Range' => "bytes #{sz}-#{packed.bytesize-1}/#{packed.bytesize}",
@@ -213,11 +282,63 @@ describe 'Job API' do
       expect($stderr).to receive(:puts)
       expect(api.job_result(12345)).to eq ['hello', 'world']
     end
+
+    it '200->500->206 can resume' do
+      sz = packed.bytesize / 3
+      stub_api_request(:get, '/v3/job/result/12345').
+        with(:query => {'format' => 'msgpack'}).
+        to_return(
+          :headers => {
+            'Content-Length' => packed.bytesize,
+            'Etag' => '"abcdefghijklmn"',
+          },
+          :body => packed[0, sz]
+        )
+      stub_api_request(:get, '/v3/job/result/12345').
+        with(
+          :headers => {
+            'If-Range' => '"abcdefghijklmn"',
+            'Range' => "bytes=#{sz}-",
+          },
+          :query => {'format' => 'msgpack'}
+        ).
+        to_return(
+          :status => 500,
+          :headers => {
+            'Content-Length' => packed.bytesize - sz,
+            'Content-Range' => "bytes #{sz}-#{packed.bytesize-1}/#{packed.bytesize}",
+            'Etag' => '"abcdefghijklmn"',
+          },
+          :body => packed
+        )
+      stub_api_request(:get, '/v3/job/result/12345').
+        with(
+          :headers => {
+            'If-Range' => '"abcdefghijklmn"',
+            'Range' => "bytes=#{sz}-",
+          },
+          :query => {'format' => 'msgpack'}
+        ).
+        to_return(
+          :status => 206,
+          :headers => {
+            'Content-Length' => packed.bytesize - sz,
+            'Content-Range' => "bytes #{sz}-#{packed.bytesize-1}/#{packed.bytesize}",
+            'Etag' => '"abcdefghijklmn"',
+          },
+          :body => packed[sz, packed.bytesize - sz]
+        )
+      expect(api).to receive(:sleep).once
+      expect($stderr).to receive(:print)
+      expect($stderr).to receive(:puts)
+      expect(api.job_result(12345)).to eq ['hello', 'world']
+    end
+
   end
 
   describe 'job_result_format' do
     let :packed do
-      s = StringIO.new
+      s = StringIO.new(String.new)
       Zlib::GzipWriter.wrap(s) do |f|
         f << ['hello', 'world'].to_json
       end
@@ -286,6 +407,7 @@ describe 'Job API' do
               :query => {'format' => 'json'}
             ).
           to_return(
+            :status => 206,
             :headers => {
               'Content-Encoding' => 'gzip',
               'Content-Length' => packed.bytesize-sz,
@@ -312,7 +434,7 @@ describe 'Job API' do
 
   describe 'job_result_each' do
     let :packed do
-      s = StringIO.new
+      s = StringIO.new(String.new)
       Zlib::GzipWriter.wrap(s) do |f|
         pk = MessagePack::Packer.new(f)
         pk.write('hello')
@@ -357,6 +479,7 @@ describe 'Job API' do
           :query => {'format' => 'msgpack'}
         ).
         to_return(
+          :status => 206,
           :headers => {
             'Content-Length' => packed.bytesize-sz,
             'Content-Range' => "bytes #{sz}-#{packed.bytesize-1}/#{packed.bytesize}",
@@ -386,7 +509,7 @@ describe 'Job API' do
       #   pk.flush
       # end
       # s.string
-      "\u001F\x8B\b\u0000#\xA1\x93T\u0000\u0003[\x9A\x91\x9A\x93\x93\xBF\xB4<\xBF('\u0005\u0000e 0\xB3\f\u0000\u0000\u0000"
+      "\x1F\x8B\b\x00#\xA1\x93T\x00\x03[\x9A\x91\x9A\x93\x93\xBF\xB4<\xBF('\x05\x00e 0\xB3\f\x00\x00\x00".force_encoding(Encoding::ASCII_8BIT)
     end
 
     it 'yields job result for each row with progress' do
@@ -424,6 +547,7 @@ describe 'Job API' do
           :query => {'format' => 'msgpack'}
         ).
         to_return(
+          :status => 206,
           :headers => {
             'Content-Length' => packed.bytesize - sz,
             'Content-Range' => "bytes #{sz}-#{packed.bytesize-1}/#{packed.bytesize}",
@@ -498,6 +622,7 @@ describe 'Job API' do
           :query => {'format' => 'msgpack.gz'}
         ).
         to_return(
+          :status => 206,
           :headers => {
             'Content-Length' => packed.bytesize - sz,
             'Content-Range' => "bytes #{sz}-#{packed.bytesize-1}/#{packed.bytesize}",
@@ -508,7 +633,7 @@ describe 'Job API' do
       expect(api).to receive(:sleep).once
       expect($stderr).to receive(:print)
       expect($stderr).to receive(:puts)
-      sio = StringIO.new(''.force_encoding(Encoding::ASCII_8BIT))
+      sio = StringIO.new(String.new)
       api.job_result_raw(12345, 'msgpack.gz', sio)
       expect(sio.string).to eq(packed)
     end
@@ -521,4 +646,185 @@ describe 'Job API' do
       expect(api.kill(12345)).to eq('status')
     end
   end
+
+  describe 'job_result_download' do
+    let (:data){ [[1, 'hello', nil], [2, 'world', true], [3, '!', false]] }
+    let :formatted do
+      case format
+      when 'json'
+        data.map{|a| JSON(a) }.join("\n")
+      when 'msgpack'
+        pk = MessagePack::Packer.new
+        data.each{|x| pk.write(x) }
+        pk.to_str
+      else
+        raise
+      end
+    end
+    let :gziped do
+      s = StringIO.new(String.new)
+      Zlib::GzipWriter.wrap(s) do |f|
+        f.write formatted
+      end
+      s.string
+    end
+    let :deflated do
+      Zlib.deflate(formatted)
+    end
+    subject do
+      str = ''
+      api.__send__(:job_result_download, 12345, format){|x| str << x }
+      str
+    end
+    context '200' do
+      before do
+        sz = packed.bytesize / 3
+        stub_api_request(:get, '/v3/job/result/12345').
+          with(:query => {'format' => format}).
+          to_return(
+            :headers => {
+              'Content-Encoding' => content_encoding,
+              'Content-Length' => packed.bytesize,
+              'Etag' => '"abcdefghijklmn"',
+            },
+            :body => packed
+          )
+        expect(api).not_to receive(:sleep)
+        expect($stderr).not_to receive(:print)
+        expect($stderr).not_to receive(:puts)
+      end
+      context 'Content-Encoding: gzip' do
+        let (:content_encoding){ 'gzip' }
+        let (:packed){ gziped }
+        context 'msgpack' do
+          let (:format){ 'msgpack' }
+          it { is_expected.to eq formatted }
+        end
+        context 'json' do
+          let (:format){ 'json' }
+          it { is_expected.to eq formatted }
+        end
+      end
+      context 'Content-Encoding: deflate' do
+        let (:content_encoding){ 'deflate' }
+        let (:packed){ deflated }
+        context 'msgpack' do
+          let (:format){ 'msgpack' }
+          it { is_expected.to eq formatted }
+        end
+        context 'json' do
+          let (:format){ 'json' }
+          it { is_expected.to eq formatted }
+        end
+      end
+    end
+
+    context '200 -> 206' do
+      before do
+        sz = packed.bytesize / 3
+        stub_api_request(:get, '/v3/job/result/12345').
+          with(:query => {'format' => format}).
+          to_return(
+            :headers => {
+              'Content-Encoding' => content_encoding,
+              'Content-Length' => packed.bytesize,
+              'Etag' => '"abcdefghijklmn"',
+            },
+            :body => packed[0, sz]
+        )
+        stub_api_request(:get, '/v3/job/result/12345').
+          with(
+            :headers => {
+              'If-Range' => '"abcdefghijklmn"',
+              'Range' => "bytes=#{sz}-",
+          },
+          :query => {'format' => format}
+        ).
+          to_return(
+            :status => 206,
+            :headers => {
+              'Content-Encoding' => content_encoding,
+              'Content-Length' => packed.bytesize - sz,
+              'Content-Range' => "bytes #{sz}-#{packed.bytesize-1}/#{packed.bytesize}",
+              'Etag' => '"abcdefghijklmn"',
+            },
+            :body => packed[sz, packed.bytesize - sz]
+        )
+        expect(api).to receive(:sleep).once
+        allow($stderr).to receive(:print)
+        allow($stderr).to receive(:puts)
+      end
+      context 'Content-Encoding: gzip' do
+        let (:content_encoding){ 'gzip' }
+        let (:packed){ gziped }
+        context 'msgpack' do
+          let (:format){ 'msgpack' }
+          it { is_expected.to eq formatted }
+        end
+        context 'json' do
+          let (:format){ 'json' }
+          it { is_expected.to eq formatted }
+        end
+      end
+      context 'Content-Encoding: deflate' do
+        let (:content_encoding){ 'deflate' }
+        let (:packed){ deflated }
+        context 'msgpack' do
+          let (:format){ 'msgpack' }
+          it { is_expected.to eq formatted }
+        end
+        context 'json' do
+          let (:format){ 'json' }
+          it { is_expected.to eq formatted }
+        end
+      end
+    end
+
+    context 'without autodecode' do
+      before do
+        sz = packed.bytesize / 3
+        stub_api_request(:get, '/v3/job/result/12345').
+          with(:query => {'format' => format}).
+          to_return(
+            :headers => {
+              'Content-Length' => packed.bytesize,
+              'Etag' => '"abcdefghijklmn"',
+            },
+            :body => packed
+          )
+        expect(api).not_to receive(:sleep)
+        expect($stderr).not_to receive(:print)
+        expect($stderr).not_to receive(:puts)
+      end
+      subject do
+        str = ''
+        api.__send__(:job_result_download, 12345, format, false){|x| str << x }
+        str
+      end
+      context 'Content-Encoding: gzip' do
+        let (:content_encoding){ 'gzip' }
+        let (:packed){ gziped }
+        context 'msgpack' do
+          let (:format){ 'msgpack' }
+          it { is_expected.to eq packed }
+        end
+        context 'json' do
+          let (:format){ 'json' }
+          it { is_expected.to eq packed }
+        end
+      end
+      context 'Content-Encoding: deflate' do
+        let (:content_encoding){ 'deflate' }
+        let (:packed){ deflated }
+        context 'msgpack' do
+          let (:format){ 'msgpack' }
+          it { is_expected.to eq packed }
+        end
+        context 'json' do
+          let (:format){ 'json' }
+          it { is_expected.to eq packed }
+        end
+      end
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: td-client
 version: !ruby/object:Gem::Version
-  version: 0.8.80
+  version: 0.8.81
 platform: ruby
 authors:
 - Treasure Data, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-05-30 00:00:00.000000000 Z
+date: 2016-06-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack