td-client 0.8.79 → 0.8.80

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 361771ba9d0d5fd00765c9ce956c2853ae4b3333
-  data.tar.gz: b6863fc4474b7598417d3e1fc776f585577b0b8b
+  metadata.gz: 6b90ad6943946fa449277adfcb6cf73151d06f5d
+  data.tar.gz: 3955d6677c5ff35ec1da44ed517d8ab69fca7b47
 SHA512:
-  metadata.gz: 9a36906b042f9fa5f41288a34ae679e54a80af6709f03ca41f94935728ca0450bf2f45adb5c63c348546448d5824c13db3351d6e1f3521963cd453d2f400ca8e
-  data.tar.gz: 3e564144569d8b2dd1b3ee1895a3a8117846802565d7581395deb5246ebc2588237c64868308e609e75dc9a8160c9a12f9d20687dfc254eb43a52623c51c87ac
+  metadata.gz: 21a28c9c7d85fa5b70eca4f9d841cbea5ee19a240140201046a12a3bfd13e52293a6bd6cbccfa4c4b2526b41689f9c027923f0f2b9260853491e839e2a7f603f
+  data.tar.gz: ae79f478833ade0e3ab01c30e8eacdbc1ea3b1cbae613e9a68bead92b746b6b5165c1b9106abee517fb865a949813d9ef48eceff44a85bef7cf40bdb524c0109

data/lib/td/client/api.rb

@@ -247,7 +247,7 @@ private
   # @param [Hash] params
   # @param [Hash] opt
   # @yield [response]
-  def do_get(url, params=nil, opt={}, &block)
+  def do_get(url, params=nil, opt={})
     client, header = new_client
     client.send_timeout = @send_timeout
     client.receive_timeout = @read_timeout
@@ -272,27 +272,48 @@ private
     response = nil
     etag = nil
     current_total_chunk_size = 0
+    body = String.new unless block_given?
     begin # this block is to allow retry (redo) in the begin part of the begin-rescue block
       begin
         if etag
           header['If-Range'] = etag
           header['Range'] = "bytes=#{current_total_chunk_size}-"
         else
+          etag = nil
           current_total_chunk_size = 0
+          body.clear if body
         end
-        if block
+
+        if block_given?
           response = client.get(target, params, header) {|res, chunk|
-            current_total_chunk_size += chunk.bytesize
-            block.call(res, chunk, current_total_chunk_size)
+            current_total_chunk_size += chunk.bytesize if res.status == 200
+            yield res, chunk, current_total_chunk_size
           }
         else
           response = client.get(target, params, header)
-          current_total_chunk_size += response.body.bytesize
+          if response.status == 200
+            current_total_chunk_size += response.body.bytesize
+            body << response.body
+          end
         end
+
         # XXX ext/openssl raises EOFError in case where underlying connection causes an error,
         #     but httpclient ignores it. Therefor, check content size.
         #     https://github.com/nahi/httpclient/issues/296
-        validate_content_length!(response, current_total_chunk_size)
+        if expected_size = response.header['Content-Range'].first
+          expected_size = expected_size[/\d+$/]
+        else
+          expected_size = response.header['Content-Length'].first
+        end
+        if expected_size
+          expected_size = expected_size.to_i
+          if expected_size != current_total_chunk_size
+            if expected_size < current_total_chunk_size
+              etag = false
+            end
+            raise IncompleteError, "#{expected_size} bytes expected, but got #{current_total_chunk_size} bytes"
+          end
+        end
 
         status = response.code
         # retry if the HTTP error code is 500 or higher and we did not run out of retrying attempts
@@ -305,7 +326,7 @@ private
         end
       rescue Errno::ECONNREFUSED, Errno::ECONNRESET, Timeout::Error, EOFError, OpenSSL::SSL::SSLError, SocketError, IncompleteError => e
         if opt[:resume]
-          etag = response.header['ETag'].first
+          etag = response.header['ETag'].first if etag != false
         elsif block_given?
           raise e
         end
@@ -332,35 +353,24 @@ private
       puts "DEBUG:   body:   " + response.body.to_s
     end
 
-    body = block ? response.body : inflate_body(response)
+    body = inflate_body(response, body) unless block_given?
 
     return [response.code.to_s, body, response]
   end
 
-  def validate_content_length!(response, body_size)
-    if content_length = response.header['Content-Range'].first
-      content_length = content_length[/\d+$/]
-    else
-      content_length = response.header['Content-Length'].first
-    end
-    if content_length && content_length.to_i != body_size
-      raise IncompleteError, "#{content_length} bytes expected, but got #{body_size} bytes"
-    end
-  end
-
-  def inflate_body(response)
-    return response.body if (ce = response.header['Content-Encoding']).empty?
+  def inflate_body(response, body=response.body)
+    return body if (ce = response.header['Content-Encoding']).empty?
 
     if ce.include?('gzip')
       infl = Zlib::Inflate.new(Zlib::MAX_WBITS + 16)
       begin
-        infl.inflate(response.body)
+        infl.inflate(body)
       ensure
         infl.close
       end
     else
       # NOTE maybe for content-encoding is msgpack.gz ?
-      Zlib::Inflate.inflate(response.body)
+      Zlib::Inflate.inflate(body)
     end
   end
 

data/lib/td/client/api/job.rb

@@ -115,7 +115,7 @@ module Job
   # @param [String] job_id
   # @return [Array]
   def job_result(job_id)
-    code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>'msgpack'})
+    code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>'msgpack'}, {:resume => true})
     if code != "200"
       raise_error("Get job result failed", res)
     end
@@ -136,7 +136,7 @@ module Job
   def job_result_format(job_id, format, io=nil, &block)
     if io
       infl = nil
-      code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>format}) {|res, chunk, current_total_chunk_size|
+      code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>format}, {:resume => true}) {|res, chunk, current_total_chunk_size|
         if res.code != 200
           raise_error("Get job result failed", res)
         end
@@ -148,7 +148,7 @@ module Job
       }
       nil
     else
-      code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>format})
+      code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>format}, {:resume => true})
       if code != "200"
         raise_error("Get job result failed", res)
       end
@@ -165,7 +165,7 @@ module Job
     upkr = MessagePack::Unpacker.new
     infl = nil
 
-    get("/v3/job/result/#{e job_id}", {'format'=>'msgpack'}) {|res, chunk, current_total_chunk_size|
+    get("/v3/job/result/#{e job_id}", {'format'=>'msgpack'}, {:resume => true}) {|res, chunk, current_total_chunk_size|
       if res.code != 200
         raise_error("Get job result failed", res)
       end
@@ -190,7 +190,7 @@ module Job
     upkr = MessagePack::Unpacker.new
     infl = nil
 
-    get("/v3/job/result/#{e job_id}", {'format'=>'msgpack'}) {|res, chunk, current_total_chunk_size|
+    get("/v3/job/result/#{e job_id}", {'format'=>'msgpack'}, {:resume => true}) {|res, chunk, current_total_chunk_size|
       if res.code != 200
         raise_error("Get job result failed", res)
       end

data/lib/td/client/version.rb

@@ -1,5 +1,5 @@
 module TreasureData
   class Client
-    VERSION = '0.8.79'
+    VERSION = '0.8.80'
   end
 end

data/spec/td/client/account_api_spec.rb

@@ -14,7 +14,7 @@ describe 'Account API' do
     it 'returns account properties' do
       stub_api_request(:get, "/v3/account/show").
         to_return(:body => {'account' => {'id' => 1, 'plan' => 0, 'storage_size' => 2, 'guaranteed_cores' => 3, 'maximum_cores' => 4, 'created_at' => '2014-12-14T17:24:00+0900'}}.to_json)
-      api.show_account.should == [1, 0, 2, 3, 4, "2014-12-14T17:24:00+0900"]
+      expect(api.show_account).to eq([1, 0, 2, 3, 4, "2014-12-14T17:24:00+0900"])
     end
   end
 
@@ -25,10 +25,10 @@ describe 'Account API' do
       stub_api_request(:get, "/v3/account/core_utilization", :query => {'from' => from, 'to' => to}).
         to_return(:body => {'from' => from, 'to' => to, 'interval' => 1, 'history' => ['dummy']}.to_json)
       r = api.account_core_utilization(from, to)
-      r[0].should == Time.parse(from)
-      r[1].should == Time.parse(to)
-      r[2].should == 1
-      r[3].should == ['dummy']
+      expect(r[0]).to eq(Time.parse(from))
+      expect(r[1]).to eq(Time.parse(to))
+      expect(r[2]).to eq(1)
+      expect(r[3]).to eq(['dummy'])
     end
   end
 end

data/spec/td/client/api_spec.rb

@@ -40,26 +40,26 @@ describe API do
 
     it 'normalize_database_name should return normalized data' do
       INVALID_NAMES.each_pair {|ng,ok|
-        API.normalize_database_name(ng).should == ok
+        expect(API.normalize_database_name(ng)).to eq(ok)
       }
-      lambda {
+      expect {
         API.normalize_database_name('')
-      }.should raise_error(RuntimeError)
+      }.to raise_error(RuntimeError)
     end
 
     it 'normalize_table_name should return normalized data' do
       INVALID_NAMES.each_pair {|ng,ok|
-        API.normalize_table_name(ng).should == ok
+        expect(API.normalize_table_name(ng)).to eq(ok)
       }
       # empty
-      lambda {
+      expect {
         API.normalize_table_name('')
-      }.should raise_error(RuntimeError)
+      }.to raise_error(RuntimeError)
     end
 
     it 'normalize_database_name should return valid data' do
       VALID_NAMES.each {|ok|
-        API.normalize_database_name(ok).should == ok
+        expect(API.normalize_database_name(ok)).to eq(ok)
       }
     end
   end
@@ -68,19 +68,19 @@ describe API do
     describe "'validate_database_name'" do
       it 'should raise a ParameterValidationError exceptions' do
         INVALID_NAMES.each_pair {|ng,ok|
-          lambda {
+          expect {
             API.validate_database_name(ng)
-          }.should raise_error(ParameterValidationError)
+          }.to raise_error(ParameterValidationError)
         }
         # empty
-        lambda {
+        expect {
           API.validate_database_name('')
-        }.should raise_error(ParameterValidationError)
+        }.to raise_error(ParameterValidationError)
       end
 
       it 'should return valid data' do
         VALID_NAMES.each {|ok|
-          API.validate_database_name(ok).should == ok
+          expect(API.validate_database_name(ok)).to eq(ok)
         }
       end
     end
@@ -88,18 +88,18 @@ describe API do
     describe "'validate_table_name'" do
       it 'should raise a ParameterValidationError exception' do
         INVALID_NAMES.each_pair {|ng,ok|
-          lambda {
+          expect {
             API.validate_table_name(ng)
-          }.should raise_error(ParameterValidationError)
+          }.to raise_error(ParameterValidationError)
         }
-        lambda {
+        expect {
           API.validate_table_name('')
-        }.should raise_error(ParameterValidationError)
+        }.to raise_error(ParameterValidationError)
       end
 
       it 'should return valid data' do
         VALID_NAMES.each {|ok|
-          API.validate_database_name(ok).should == ok
+          expect(API.validate_database_name(ok)).to eq(ok)
         }
       end
     end
@@ -107,19 +107,19 @@ describe API do
     describe "'validate_result_set_name'" do
       it 'should raise a ParameterValidationError exception' do
         INVALID_NAMES.each_pair {|ng,ok|
-          lambda {
+          expect {
             API.validate_result_set_name(ng)
-          }.should raise_error(ParameterValidationError)
+          }.to raise_error(ParameterValidationError)
         }
         # empty
-        lambda {
+        expect {
           API.validate_result_set_name('')
-        }.should raise_error(ParameterValidationError)
+        }.to raise_error(ParameterValidationError)
       end
 
       it 'should return valid data' do
         VALID_NAMES.each {|ok|
-          API.validate_result_set_name(ok).should == ok
+          expect(API.validate_result_set_name(ok)).to eq(ok)
         }
       end
     end
@@ -127,18 +127,18 @@ describe API do
     describe "'validate_column_name'" do
       it 'should raise a ParameterValidationError exception' do
         ['/', '', 'D'].each { |ng|
-          lambda {
+          expect {
             API.validate_column_name(ng)
-          }.should raise_error(ParameterValidationError)
+          }.to raise_error(ParameterValidationError)
         }
       end
 
       it 'should return valid data' do
         VALID_NAMES.each {|ok|
-          API.validate_column_name(ok).should == ok
+          expect(API.validate_column_name(ok)).to eq(ok)
         }
         # columns can be as short as 2 characters
-        API.validate_column_name('ab').should == 'ab'
+        expect(API.validate_column_name('ab')).to eq('ab')
       end
     end
 
@@ -146,30 +146,30 @@ describe API do
     describe "'generic validate_name'" do
       it 'should raise a ParameterValidationError exception' do
         INVALID_NAMES.each_pair {|ng,ok|
-          lambda {
+          expect {
             API.validate_name("generic", 3, 256, ng)
-          }.should raise_error(ParameterValidationError)
+          }.to raise_error(ParameterValidationError)
         }
         # empty
-        lambda {
+        expect {
           API.validate_name("generic", 3, 256, '')
-        }.should raise_error(ParameterValidationError)
+        }.to raise_error(ParameterValidationError)
         # too short - one less than left limit
-        lambda {
+        expect {
           API.validate_name("generic", 3, 256, 'ab')
-        }.should raise_error(ParameterValidationError)
+        }.to raise_error(ParameterValidationError)
       end
 
       it 'should return valid data' do
         VALID_NAMES.each {|ok|
-          API.validate_name("generic", 3, 256, ok).should == ok
+          expect(API.validate_name("generic", 3, 256, ok)).to eq(ok)
         }
         # esplore left boundary
-        API.validate_name("generic", 2, 256, 'ab').should == 'ab'
-        API.validate_name("generic", 1, 256, 'a').should == 'a'
+        expect(API.validate_name("generic", 2, 256, 'ab')).to eq('ab')
+        expect(API.validate_name("generic", 1, 256, 'a')).to eq('a')
         # explore right boundary
-        API.validate_name("generic", 3, 256, 'a' * 256).should == 'a' * 256
-        API.validate_name("generic", 3, 128, 'a' * 128).should == 'a' * 128
+        expect(API.validate_name("generic", 3, 256, 'a' * 256)).to eq('a' * 256)
+        expect(API.validate_name("generic", 3, 128, 'a' * 128)).to eq('a' * 128)
       end
     end
 
@@ -204,7 +204,7 @@ describe API do
         let(:content_length) { {'Content-Length' => packed.size} }
 
         it 'not called #completed_body?' do
-          api.should_not_receive(:completed_body?)
+          expect(api).not_to receive(:completed_body?)
 
           get_api_call
         end
@@ -223,21 +223,13 @@ describe API do
         end
 
         context 'with Content-Length' do
-          context 'macth Content-Length and body.size' do
+          context 'match Content-Length and body.size' do
             let(:content_length) { {'Content-Length' => packed.size} }
 
             it 'api accuess succeded' do
               expect { get_api_call }.not_to raise_error
             end
           end
-
-          context 'not macth Content-Length and body.size' do
-            let(:content_length) { {'Content-Length' => packed.size + 1} }
-
-            it 'api accuess succeded' do
-              expect { get_api_call }.to raise_error(TreasureData::API::IncompleteError)
-            end
-          end
         end
       end
     end

data/spec/td/client/bulk_import_spec.rb

@@ -29,7 +29,7 @@ describe 'BulkImport API' do
       stub_api_request(:post, "/v3/bulk_import/create/#{e(bi_name)}/#{e(db_name)}/#{e(table_name)}").
         to_return(:body => {'bulk_import' => bi_name}.to_json)
 
-      api.create_bulk_import(bi_name, db_name, table_name).should be_nil
+      expect(api.create_bulk_import(bi_name, db_name, table_name)).to be_nil
     end
 
     it 'should return 422 error with invalid name' do
@@ -70,7 +70,7 @@ describe 'BulkImport API' do
     it 'runs' do
       stub_api_request(:post, '/v3/bulk_import/delete/name').
         with(:body => 'foo=bar')
-      api.delete_bulk_import('name', 'foo' => 'bar').should == nil
+      expect(api.delete_bulk_import('name', 'foo' => 'bar')).to eq(nil)
     end
   end
 
@@ -78,7 +78,7 @@ describe 'BulkImport API' do
     it 'runs' do
       stub_api_request(:get, '/v3/bulk_import/show/name').
         to_return(:body => {'status' => 'status', 'other' => 'other'}.to_json)
-      api.show_bulk_import('name')['status'].should == 'status'
+      expect(api.show_bulk_import('name')['status']).to eq('status')
     end
   end
 
@@ -87,7 +87,7 @@ describe 'BulkImport API' do
       stub_api_request(:get, '/v3/bulk_import/list').
         with(:query => 'foo=bar').
         to_return(:body => {'bulk_imports' => %w(1 2 3)}.to_json)
-      api.list_bulk_imports('foo' => 'bar').should == %w(1 2 3)
+      expect(api.list_bulk_imports('foo' => 'bar')).to eq(%w(1 2 3))
     end
   end
 
@@ -96,7 +96,7 @@ describe 'BulkImport API' do
       stub_api_request(:get, '/v3/bulk_import/list_parts/name').
         with(:query => 'foo=bar').
         to_return(:body => {'parts' => %w(1 2 3)}.to_json)
-      api.list_bulk_import_parts('name', 'foo' => 'bar').should == %w(1 2 3)
+      expect(api.list_bulk_import_parts('name', 'foo' => 'bar')).to eq(%w(1 2 3))
     end
   end
 
@@ -109,7 +109,7 @@ describe 'BulkImport API' do
       stub_request(:put, 'http://api.treasure-data.com/v3/bulk_import/upload_part/name/part').
         with(:body => '12345')
       File.open(t.path) do |f|
-        api.bulk_import_upload_part('name', 'part', f, 5).should == nil
+        expect(api.bulk_import_upload_part('name', 'part', f, 5)).to eq(nil)
       end
     end
 
@@ -122,7 +122,7 @@ describe 'BulkImport API' do
         stub_request(:put, 'http://api.treasure-data.com/v3/bulk_import/upload_part/name/' + CGI.escape('日本語(Japanese)'.encode('UTF-8'))).
           with(:body => '12345')
         File.open(t.path) do |f|
-          api.bulk_import_upload_part('name', '日本語(Japanese)'.encode('Windows-31J'), f, 5).should == nil
+          expect(api.bulk_import_upload_part('name', '日本語(Japanese)'.encode('Windows-31J'), f, 5)).to eq(nil)
         end
       end
     end
@@ -131,21 +131,21 @@ describe 'BulkImport API' do
   describe 'bulk_import_delete_part' do
     it 'runs' do
       stub_api_request(:post, '/v3/bulk_import/delete_part/name/part')
-      api.bulk_import_delete_part('name', 'part').should == nil
+      expect(api.bulk_import_delete_part('name', 'part')).to eq(nil)
     end
   end
 
   describe 'freeze_bulk_import' do
     it 'runs' do
       stub_api_request(:post, '/v3/bulk_import/freeze/name')
-      api.freeze_bulk_import('name').should == nil
+      expect(api.freeze_bulk_import('name')).to eq(nil)
     end
   end
 
   describe 'unfreeze_bulk_import' do
     it 'runs' do
       stub_api_request(:post, '/v3/bulk_import/unfreeze/name')
-      api.unfreeze_bulk_import('name').should == nil
+      expect(api.unfreeze_bulk_import('name')).to eq(nil)
     end
   end
 
@@ -153,7 +153,7 @@ describe 'BulkImport API' do
     it 'runs' do
       stub_api_request(:post, '/v3/bulk_import/perform/name').
         to_return(:body => {'job_id' => 12345}.to_json)
-      api.perform_bulk_import('name').should == '12345'
+      expect(api.perform_bulk_import('name')).to eq('12345')
     end
   end
 
@@ -161,7 +161,7 @@ describe 'BulkImport API' do
     it 'runs' do
       stub_api_request(:post, '/v3/bulk_import/commit/name').
         to_return(:body => {'job_id' => 12345}.to_json)
-      api.commit_bulk_import('name').should == nil
+      expect(api.commit_bulk_import('name')).to eq(nil)
     end
   end
 
@@ -169,19 +169,19 @@ describe 'BulkImport API' do
     it 'returns [] on empty' do
       stub_api_request(:get, '/v3/bulk_import/error_records/name').
         to_return(:body => '')
-      api.bulk_import_error_records('name').should == []
+      expect(api.bulk_import_error_records('name')).to eq([])
     end
 
     it 'returns nil on empty if block given' do
       stub_api_request(:get, '/v3/bulk_import/error_records/name').
         to_return(:body => '')
-      api.bulk_import_error_records('name'){}.should == nil
+      expect(api.bulk_import_error_records('name'){}).to eq(nil)
     end
 
     it 'returns unpacked result' do
       stub_api_request(:get, '/v3/bulk_import/error_records/name').
         to_return(:body => packed)
-      api.bulk_import_error_records('name').should == [[1, '2', 3.0], [4, '5', 6.0], [7, '8', 9.0]]
+      expect(api.bulk_import_error_records('name')).to eq([[1, '2', 3.0], [4, '5', 6.0], [7, '8', 9.0]])
     end
 
     it 'yields unpacked result if block given' do
@@ -191,7 +191,7 @@ describe 'BulkImport API' do
       api.bulk_import_error_records('name') do |row|
         result << row
       end
-      result.should == [[1, '2', 3.0], [4, '5', 6.0], [7, '8', 9.0]]
+      expect(result).to eq([[1, '2', 3.0], [4, '5', 6.0], [7, '8', 9.0]])
     end
   end
 end