td-client 0.8.78 → 0.8.79

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b7b08bc82aed5007cc2f9b0500847ca78520cc78
-  data.tar.gz: 1fa61539241cce5a9152f23629f46c5c95a7ec01
+  metadata.gz: 361771ba9d0d5fd00765c9ce956c2853ae4b3333
+  data.tar.gz: b6863fc4474b7598417d3e1fc776f585577b0b8b
 SHA512:
-  metadata.gz: cd680197fab594cf321145804b06b99d9070cecb97115625b7a489d94b4baae685f47744a866172f20baf6d50b7e9c9eca0287b20ab0d4692cfeec38aa5e9448
-  data.tar.gz: 64fc10d47a765efc4bab85fb24fa12b0cbc459336467f21765007057807117c6177f936f642c84da608d15fa09ddb9dacf5e08a3b50fcb96f900860f78c9c02e
+  metadata.gz: 9a36906b042f9fa5f41288a34ae679e54a80af6709f03ca41f94935728ca0450bf2f45adb5c63c348546448d5824c13db3351d6e1f3521963cd453d2f400ca8e
+  data.tar.gz: 3e564144569d8b2dd1b3ee1895a3a8117846802565d7581395deb5246ebc2588237c64868308e609e75dc9a8160c9a12f9d20687dfc254eb43a52623c51c87ac

data/lib/td/client.rb

@@ -356,7 +356,7 @@ class Client
     raise ArgumentError, "'cron' option is required" unless opts[:cron] || opts['cron']
     raise ArgumentError, "'query' option is required" unless opts[:query] || opts['query']
     start = @api.create_schedule(name, opts)
-    return Time.parse(start)
+    return start && Time.parse(start)
   end
 
   # @param [String] name

data/lib/td/client/api.rb

@@ -235,16 +235,19 @@ private
 
   # @param [String] url
   # @param [Hash] params
+  # @param [Hash] opt
   # @yield [response]
-  def get(url, params=nil, &block)
+  def get(url, params=nil, opt={}, &block)
     guard_no_sslv3 do
-      do_get(url, params, &block)
+      do_get(url, params, opt, &block)
     end
   end
+
   # @param [String] url
   # @param [Hash] params
+  # @param [Hash] opt
   # @yield [response]
-  def do_get(url, params=nil, &block)
+  def do_get(url, params=nil, opt={}, &block)
     client, header = new_client
     client.send_timeout = @send_timeout
     client.receive_timeout = @read_timeout
@@ -267,24 +270,29 @@ private
     # for both exceptions and 500+ errors retrying is enabled by default.
     # The total number of retries cumulatively should not exceed 10 minutes / 600 seconds
     response = nil
+    etag = nil
+    current_total_chunk_size = 0
     begin # this block is to allow retry (redo) in the begin part of the begin-rescue block
       begin
-        if block
+        if etag
+          header['If-Range'] = etag
+          header['Range'] = "bytes=#{current_total_chunk_size}-"
+        else
           current_total_chunk_size = 0
+        end
+        if block
           response = client.get(target, params, header) {|res, chunk|
             current_total_chunk_size += chunk.bytesize
             block.call(res, chunk, current_total_chunk_size)
           }
-
-          # XXX ext/openssl raises EOFError in case where underlying connection causes an error,
-          #     and msgpack-ruby that used in block handles it as an end of stream == no exception.
-          #     Therefor, check content size.
-          validate_content_length!(response, current_total_chunk_size) if @ssl
         else
           response = client.get(target, params, header)
-
-          validate_content_length!(response, response.body.bytesize) if @ssl
+          current_total_chunk_size += response.body.bytesize
         end
+        # XXX ext/openssl raises EOFError in case where underlying connection causes an error,
+        #     but httpclient ignores it. Therefor, check content size.
+        #     https://github.com/nahi/httpclient/issues/296
+        validate_content_length!(response, current_total_chunk_size)
 
         status = response.code
         # retry if the HTTP error code is 500 or higher and we did not run out of retrying attempts
@@ -296,7 +304,9 @@ private
           redo # restart from beginning of do-while loop
         end
       rescue Errno::ECONNREFUSED, Errno::ECONNRESET, Timeout::Error, EOFError, OpenSSL::SSL::SSLError, SocketError, IncompleteError => e
-        if block_given?
+        if opt[:resume]
+          etag = response.header['ETag'].first
+        elsif block_given?
           raise e
         end
         $stderr.print "#{e.class}: #{e.message}. "
@@ -328,8 +338,14 @@ private
   end
 
   def validate_content_length!(response, body_size)
-    content_length = response.header['Content-Length'].first
-    raise IncompleteError if @ssl && content_length && content_length.to_i != body_size
+    if content_length = response.header['Content-Range'].first
+      content_length = content_length[/\d+$/]
+    else
+      content_length = response.header['Content-Length'].first
+    end
+    if content_length && content_length.to_i != body_size
+      raise IncompleteError, "#{content_length} bytes expected, but got #{body_size} bytes"
+    end
   end
 
   def inflate_body(response)

data/lib/td/client/api/job.rb

@@ -214,8 +214,8 @@ module Job
   def job_result_raw(job_id, format, io = nil, &block)
     body = nil
 
-    get("/v3/job/result/#{e job_id}", {'format'=>format}) {|res, chunk, current_total_chunk_size|
-      if res.code != 200
+    get("/v3/job/result/#{e job_id}", {'format'=>format}, {:resume => true}) {|res, chunk, current_total_chunk_size|
+      unless res.ok?
         raise_error("Get job result failed", res)
       end
 

data/lib/td/client/api/schedule.rb

@@ -14,7 +14,7 @@ module Schedule
     if code != "200"
       raise_error("Create schedule failed", res)
     end
-    js = checked_json(body, %w[start])
+    js = checked_json(body)
     return js['start']
   end
 

data/lib/td/client/version.rb

@@ -1,5 +1,5 @@
 module TreasureData
   class Client
-    VERSION = '0.8.78'
+    VERSION = '0.8.79'
   end
 end

data/spec/spec_helper.rb

@@ -10,10 +10,10 @@ unless ENV['APPVEYOR']
       require 'simplecov'
       require 'coveralls'
 
-      SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter[
+      SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter.new([
         SimpleCov::Formatter::HTMLFormatter,
         Coveralls::SimpleCov::Formatter
-      ]
+      ])
       SimpleCov.start("test_frameworks")
     end
   rescue NameError

data/spec/td/client/job_api_spec.rb

@@ -311,6 +311,53 @@ describe 'Job API' do
         api.job_result_raw(12345, 'json').should == 'raw binary'
       end
     end
+
+    let :packed do
+      # Hard code fixture data to make the size stable
+      # s = StringIO.new
+      # Zlib::GzipWriter.wrap(s) do |f|
+      #   pk = MessagePack::Packer.new(f)
+      #   pk.write('hello')
+      #   pk.write('world')
+      #   pk.flush
+      # end
+      # s.string
+      "\x1F\x8B\b\x00#\xA1\x93T\x00\x03[\x9A\x91\x9A\x93\x93\xBF\xB4<\xBF('\x05\x00e 0\xB3\f\x00\x00\x00".force_encoding(Encoding::ASCII_8BIT)
+    end
+
+    it 'can resume' do
+      stub_api_request(:get, '/v3/job/result/12345').
+        with(:query => {'format' => 'msgpack.gz'}).
+        to_return(
+          :headers => {
+            'Content-Length' => 32,
+            'Etag' => '"abcdefghijklmn"',
+          },
+          :body => packed[0, 20]
+        )
+      stub_api_request(:get, '/v3/job/result/12345').
+        with(
+          :headers => {
+            'If-Range' => '"abcdefghijklmn"',
+            'Range' => 'bytes=20-',
+          },
+          :query => {'format' => 'msgpack.gz'}
+        ).
+        to_return(
+          :headers => {
+            'Content-Length' => 12,
+            'Content-Range' => 'bytes 20-31/32',
+            'Etag' => '"abcdefghijklmn"',
+          },
+          :body => packed[20, 12]
+        )
+      expect(api).to receive(:sleep).once
+      expect($stderr).to receive(:print)
+      expect($stderr).to receive(:puts)
+      sio = StringIO.new(''.force_encoding(Encoding::ASCII_8BIT))
+      api.job_result_raw(12345, 'msgpack.gz', sio)
+      sio.string.should == packed
+    end
   end
 
   describe 'kill' do

data/spec/td/client/sched_api_spec.rb

@@ -23,6 +23,14 @@ describe 'Schedule API' do
       api.create_schedule(sched_name, opts.merge('type' => 'hive')).should == start.to_s
     end
 
+    it 'should create a dummy schedule' do
+      stub_api_request(:post, "/v3/schedule/create/#{e(sched_name)}").
+        with(:body => opts.merge('type' => 'hive')).
+        to_return(:body => {'name' => sched_name, 'start' => nil}.to_json)
+
+      api.create_schedule(sched_name, opts.merge('type' => 'hive')).should be_nil
+    end
+
     it 'should return 422 error with invalid name' do
       name = '1'
       err_msg = "Validation failed: Name is too short" # " (minimum is 3 characters)"

data/spec/td/client_sched_spec.rb

@@ -15,6 +15,31 @@ describe 'Schedule Command' do
     client
   end
 
+  describe 'create' do
+    let :opts do
+      {:database => db_name, :cron => '', :type => 'hive', :query => 'select 1;'}
+    end
+
+    before do
+      stub_api_request(:post, "/v3/schedule/create/#{e(sched_name)}").
+      with(:body => opts).
+        to_return(:body => {'name' => sched_name, 'start' => start}.to_json)
+    end
+    context 'start is now' do
+      let (:start){ Time.now.round }
+      it 'returns Time object' do
+        client.create_schedule(sched_name, opts).should == start
+      end
+    end
+
+    context 'start is nil' do
+      let (:start){ nil }
+      it do
+        client.create_schedule(sched_name, opts).should == start
+      end
+    end
+  end
+
   describe 'history' do
     let :opts do
       {'database' => db_name}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: td-client
 version: !ruby/object:Gem::Version
-  version: 0.8.78
+  version: 0.8.79
 platform: ruby
 authors:
 - Treasure Data, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-26 00:00:00.000000000 Z
+date: 2016-04-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack