td-client 0.8.67 → 0.8.68

data/lib/td/client/model.rb

@@ -426,13 +426,15 @@ end
 
 
 class ScheduledJob < Job
+  attr_reader :scheduled_at
+
   def initialize(client, scheduled_at, *super_args)
     super(client, *super_args)
-    @scheduled_at = scheduled_at
-  end
-
-  def scheduled_at
-    @scheduled_at ? Time.parse(@scheduled_at) : nil
+    if scheduled_at.to_s.empty?
+      @scheduled_at = nil
+    else
+      @scheduled_at = Time.parse(scheduled_at) rescue nil
+    end
   end
 end
 

data/lib/td/client/version.rb

@@ -1,5 +1,5 @@
 module TreasureData
   class Client
-    VERSION = '0.8.67'
+    VERSION = '0.8.68'
   end
 end

data/lib/td/core_ext/openssl/ssl/sslcontext/set_params.rb

@@ -0,0 +1,18 @@
+require 'openssl'
+module OpenSSL
+  module SSL
+    class SSLContext
+
+      # For disabling SSLv3 connection in favor of POODLE Attack protection
+      #
+      # Allow 'options' customize through Thread local storage since
+      # Net::HTTP does not support 'options' configuration.
+      #
+      alias original_set_params set_params
+      def set_params(params={})
+        original_set_params(params)
+        self.options |= OP_NO_SSLv3 if Thread.current[:SET_SSL_OP_NO_SSLv3]
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -1,9 +1,13 @@
 require 'rubygems'
 
-begin
-  require 'simplecov'
-  SimpleCov.start
-rescue LoadError
+if ENV['SIMPLE_COV']
+  begin
+    require 'simplecov'
+    SimpleCov.start do
+      add_filter '/spec/'
+    end
+  rescue LoadError
+  end
 end
 
 require 'rspec'

data/spec/td/client/access_control_api_spec.rb

@@ -0,0 +1,37 @@
+require 'spec_helper'
+require 'td/client/spec_resources'
+require 'json'
+
+describe 'Access Control API' do
+  include_context 'spec symbols'
+  include_context 'common helper'
+
+  let :api do
+    # no retry for GET
+    API.new(nil, {:max_cumul_retry_delay => -1})
+  end
+
+  describe 'all apis' do
+    it 'is deprecated' do
+      stub_api_request(:post, "/v3/acl/grant").to_return(:status => 500)
+      expect {
+        api.grant_access_control('subject', 'action', 'scope', [])
+      }.to raise_error(TreasureData::APIError)
+
+      stub_api_request(:post, "/v3/acl/revoke").to_return(:status => 500)
+      expect {
+        api.revoke_access_control('subject', 'action', 'scope')
+      }.to raise_error(TreasureData::APIError)
+
+      stub_api_request(:get, "/v3/acl/test", :query => {'user' => 'user', 'action' => 'action', 'scope' => 'scope'}).to_return(:status => 422)
+      expect {
+        api.test_access_control('user', 'action', 'scope')
+      }.to raise_error(TreasureData::APIError)
+
+      stub_api_request(:get, "/v3/acl/list").to_return(:status => 500)
+      expect {
+        api.list_access_controls
+      }.to raise_error(TreasureData::APIError)
+    end
+  end
+end

data/spec/td/client/account_api_spec.rb

@@ -0,0 +1,34 @@
+require 'spec_helper'
+require 'td/client/spec_resources'
+require 'json'
+
+describe 'Account API' do
+  include_context 'spec symbols'
+  include_context 'common helper'
+
+  let :api do
+    API.new(nil)
+  end
+
+  describe 'show_account' do
+    it 'returns account properties' do
+      stub_api_request(:get, "/v3/account/show").
+        to_return(:body => {'account' => {'id' => 1, 'plan' => 0, 'storage_size' => 2, 'guaranteed_cores' => 3, 'maximum_cores' => 4, 'created_at' => '2014-12-14T17:24:00+0900'}}.to_json)
+      api.show_account.should == [1, 0, 2, 3, 4, "2014-12-14T17:24:00+0900"]
+    end
+  end
+
+  describe 'account_core_utilization' do
+    it 'returns core utilization' do
+      from = '2014-12-01T00:00:00+0900'
+      to = '2015-01-01T00:00:00+0900'
+      stub_api_request(:get, "/v3/account/core_utilization", :query => {'from' => from, 'to' => to}).
+        to_return(:body => {'from' => from, 'to' => to, 'interval' => 1, 'history' => ['dummy']}.to_json)
+      r = api.account_core_utilization(from, to)
+      r[0].should == Time.parse(from)
+      r[1].should == Time.parse(to)
+      r[2].should == 1
+      r[3].should == ['dummy']
+    end
+  end
+end

data/spec/td/client/api_ssl_connection_spec.rb

@@ -20,13 +20,16 @@ describe 'API SSL connection' do
     }.to raise_error OpenSSL::SSL::SSLError
   end
 
-  it 'should success to connect TLSv1 only server' do
-    @server = setup_server(:TLSv1)
-    api = API.new(nil, :endpoint => "https://localhost:#{@serverport}", :retry_post_requests => false)
-    api.ssl_ca_file = File.join(DIR, 'ca-all.cert')
-    expect {
-      api.delete_database('no_such_database')
-    }.to raise_error TreasureData::NotFoundError
+  # 1.8.7's net/http does not call SSLContext#set_params. Use 1.8.7 where you don't care security.
+  if RUBY_VERSION >= '1.9.0'
+    it 'should success to connect TLSv1 only server' do
+      @server = setup_server(:TLSv1)
+      api = API.new(nil, :endpoint => "https://localhost:#{@serverport}", :retry_post_requests => false)
+      api.ssl_ca_file = File.join(DIR, 'ca-all.cert')
+      expect {
+        api.delete_database('no_such_database')
+      }.to raise_error TreasureData::NotFoundError
+    end
   end
 
   def setup_server(ssl_version)

data/spec/td/client/bulk_import_spec.rb

@@ -1,5 +1,8 @@
+# encoding: utf-8
+
 require 'spec_helper'
 require 'td/client/spec_resources'
+require 'tempfile'
 
 describe 'BulkImport API' do
   include_context 'spec symbols'
@@ -9,6 +12,18 @@ describe 'BulkImport API' do
     API.new(nil)
   end
 
+  let :packed do
+    s = StringIO.new
+    Zlib::GzipWriter.wrap(s) do |f|
+      pk = MessagePack::Packer.new(f)
+      pk.write([1, '2', 3.0])
+      pk.write([4, '5', 6.0])
+      pk.write([7, '8', 9.0])
+      pk.flush
+    end
+    s.string
+  end
+
   describe 'create_bulk_import' do
     it 'should create a new bulk_import' do
       stub_api_request(:post, "/v3/bulk_import/create/#{e(bi_name)}/#{e(db_name)}/#{e(table_name)}").
@@ -50,4 +65,133 @@ describe 'BulkImport API' do
       }.to raise_error(TreasureData::APIError, /#{err_msg}/)
     end
   end
+
+  describe 'delete_bulk_import' do
+    it 'runs' do
+      stub_api_request(:post, '/v3/bulk_import/delete/name').
+        with(:body => 'foo=bar')
+      api.delete_bulk_import('name', 'foo' => 'bar').should == nil
+    end
+  end
+
+  describe 'show_bulk_import' do
+    it 'runs' do
+      stub_api_request(:get, '/v3/bulk_import/show/name').
+        to_return(:body => {'status' => 'status', 'other' => 'other'}.to_json)
+      api.show_bulk_import('name')['status'].should == 'status'
+    end
+  end
+
+  describe 'list_bulk_imports' do
+    it 'runs' do
+      stub_api_request(:get, '/v3/bulk_import/list').
+        with(:query => 'foo=bar').
+        to_return(:body => {'bulk_imports' => %w(1 2 3)}.to_json)
+      api.list_bulk_imports('foo' => 'bar').should == %w(1 2 3)
+    end
+  end
+
+  describe 'list_bulk_import_parts' do
+    it 'runs' do
+      stub_api_request(:get, '/v3/bulk_import/list_parts/name').
+        with(:query => 'foo=bar').
+        to_return(:body => {'parts' => %w(1 2 3)}.to_json)
+      api.list_bulk_import_parts('name', 'foo' => 'bar').should == %w(1 2 3)
+    end
+  end
+
+  describe 'bulk_import_upload_part' do
+    it 'runs' do
+      t = Tempfile.new('bulk_import_spec')
+      File.open(t.path, 'w') do |f|
+        f << '12345'
+      end
+      stub_request(:put, 'http://api.treasure-data.com/v3/bulk_import/upload_part/name/part').
+        with(:body => '12345')
+      File.open(t.path) do |f|
+        api.bulk_import_upload_part('name', 'part', f, 5).should == nil
+      end
+    end
+
+    if ''.respond_to?(:encode)
+      it 'encodes part_name in UTF-8' do
+        t = Tempfile.new('bulk_import_spec')
+        File.open(t.path, 'w') do |f|
+          f << '12345'
+        end
+        stub_request(:put, 'http://api.treasure-data.com/v3/bulk_import/upload_part/name/' + CGI.escape('日本語(Japanese)'.encode('UTF-8'))).
+          with(:body => '12345')
+        File.open(t.path) do |f|
+          api.bulk_import_upload_part('name', '日本語(Japanese)'.encode('Windows-31J'), f, 5).should == nil
+        end
+      end
+    end
+  end
+
+  describe 'bulk_import_delete_part' do
+    it 'runs' do
+      stub_api_request(:post, '/v3/bulk_import/delete_part/name/part')
+      api.bulk_import_delete_part('name', 'part').should == nil
+    end
+  end
+
+  describe 'freeze_bulk_import' do
+    it 'runs' do
+      stub_api_request(:post, '/v3/bulk_import/freeze/name')
+      api.freeze_bulk_import('name').should == nil
+    end
+  end
+
+  describe 'unfreeze_bulk_import' do
+    it 'runs' do
+      stub_api_request(:post, '/v3/bulk_import/unfreeze/name')
+      api.unfreeze_bulk_import('name').should == nil
+    end
+  end
+
+  describe 'perform_bulk_import' do
+    it 'runs' do
+      stub_api_request(:post, '/v3/bulk_import/perform/name').
+        to_return(:body => {'job_id' => 12345}.to_json)
+      api.perform_bulk_import('name').should == '12345'
+    end
+  end
+
+  describe 'commit_bulk_import' do
+    it 'runs' do
+      stub_api_request(:post, '/v3/bulk_import/commit/name').
+        to_return(:body => {'job_id' => 12345}.to_json)
+      api.commit_bulk_import('name').should == nil
+    end
+  end
+
+  describe 'bulk_import_error_records' do
+    it 'returns [] on empty' do
+      stub_api_request(:get, '/v3/bulk_import/error_records/name').
+        to_return(:body => '')
+      api.bulk_import_error_records('name').should == []
+    end
+
+    it 'returns nil on empty if block given' do
+      stub_api_request(:get, '/v3/bulk_import/error_records/name').
+        to_return(:body => '')
+      api.bulk_import_error_records('name'){}.should == nil
+    end
+
+    it 'returns unpacked result' do
+      stub_api_request(:get, '/v3/bulk_import/error_records/name').
+        to_return(:body => packed)
+      api.bulk_import_error_records('name').should == [[1, '2', 3.0], [4, '5', 6.0], [7, '8', 9.0]]
+    end
+
+    it 'yields unpacked result if block given' do
+      stub_api_request(:get, '/v3/bulk_import/error_records/name').
+        to_return(:body => packed)
+      result = []
+      api.bulk_import_error_records('name') do |row|
+        result << row
+      end
+      result.should == [[1, '2', 3.0], [4, '5', 6.0], [7, '8', 9.0]]
+    end
+  end
 end

data/spec/td/client/import_api_spec.rb

@@ -0,0 +1,73 @@
+require 'spec_helper'
+require 'td/client/spec_resources'
+require 'json'
+require 'tempfile'
+
+describe 'Import API' do
+  include_context 'spec symbols'
+  include_context 'common helper'
+
+  let :api do
+    API.new(nil, :endpoint => 'https://api.treasuredata.com')
+  end
+
+  let :api_old do
+    API.new(nil, :endpoint => 'http://api.treasure-data.com')
+  end
+
+  describe 'import' do
+    it 'runs with unique_id' do
+      t = Tempfile.new('import_api_spec')
+      File.open(t.path, 'w') do |f|
+        f << '12345'
+      end
+      stub_request(:put, "https://api-import.treasuredata.com/v3/table/import_with_id/db/table/unique_id/format").
+        with(:body => '12345').
+        to_return(:body => '{"elapsed_time":"1.23"}')
+      File.open(t.path) do |f|
+        api.import('db', 'table', 'format', f, 5, 'unique_id').should == 1.23
+      end
+    end
+
+    it 'runs without unique_id' do
+      t = Tempfile.new('import_api_spec')
+      File.open(t.path, 'w') do |f|
+        f << '12345'
+      end
+      stub_request(:put, "https://api-import.treasuredata.com/v3/table/import/db/table/format").
+        with(:body => '12345').
+        to_return(:body => '{"elapsed_time":"1.23"}')
+      File.open(t.path) do |f|
+        api.import('db', 'table', 'format', f, 5).should == 1.23
+      end
+    end
+
+    it 'runs for old endpoint' do
+      t = Tempfile.new('import_api_spec')
+      File.open(t.path, 'w') do |f|
+        f << '12345'
+      end
+      stub_request(:put, "http://api-import.treasure-data.com/v3/table/import/db/table/format").
+        with(:body => '12345').
+        to_return(:body => '{"elapsed_time":"1.23"}')
+      File.open(t.path) do |f|
+        api_old.import('db', 'table', 'format', f, 5).should == 1.23
+      end
+    end
+
+    it 'raises APIError' do
+      t = Tempfile.new('import_api_spec')
+      File.open(t.path, 'w') do |f|
+        f << '12345'
+      end
+      stub_request(:put, "https://api-import.treasuredata.com/v3/table/import/db/table/format").
+        with(:body => '12345').
+        to_return(:status => 500)
+      File.open(t.path) do |f|
+        expect {
+          api.import('db', 'table', 'format', f, 5)
+        }.to raise_error(TreasureData::APIError)
+      end
+    end
+  end
+end

data/spec/td/client/job_api_spec.rb

@@ -1,6 +1,8 @@
 require 'spec_helper'
 require 'td/client/spec_resources'
 require 'json'
+require 'webrick'
+require 'logger'
 
 describe 'Job API' do
   include_context 'spec symbols'
@@ -11,7 +13,7 @@ describe 'Job API' do
   end
 
   let :api_with_short_retry do
-    API.new(nil, {:max_cumul_retry_delay => 10})
+    API.new(nil, {:max_cumul_retry_delay => 0})
   end
 
   describe 'list_jobs' do
@@ -159,4 +161,125 @@ describe 'Job API' do
       job_id.should == '1'
     end
   end
+
+  describe 'job_result' do
+    let :packed do
+      s = StringIO.new
+      pk = MessagePack::Packer.new(s)
+      pk.write('hello')
+      pk.write('world')
+      pk.flush
+      s.string
+    end
+
+    it 'returns job result' do
+      stub_api_request(:get, '/v3/job/result/12345').
+        with(:query => {'format' => 'msgpack'}).
+        to_return(:body => packed)
+      api.job_result(12345).should == ['hello', 'world']
+    end
+  end
+
+  describe 'job_result_format' do
+    let :packed do
+      s = StringIO.new
+      Zlib::GzipWriter.wrap(s) do |f|
+        f << ['hello', 'world'].to_json
+      end
+      s.string
+    end
+
+    it 'returns formatted job result' do
+      stub_api_request(:get, '/v3/job/result/12345').
+        with(:query => {'format' => 'json'}).
+        to_return(
+          :headers => {'Content-Encoding' => 'gzip'},
+          :body => packed
+        )
+      api.job_result_format(12345, 'json').should == ['hello', 'world'].to_json
+    end
+
+    it 'writes formatted job result' do
+      stub_api_request(:get, '/v3/job/result/12345').
+        with(:query => {'format' => 'json'}).
+        to_return(
+          :headers => {'Content-Encoding' => 'gzip'},
+          :body => packed
+        )
+      s = StringIO.new
+      api.job_result_format(12345, 'json', s)
+      s.string.should == ['hello', 'world'].to_json
+    end
+  end
+
+  describe 'job_result_each' do
+    let :packed do
+      s = StringIO.new
+      Zlib::GzipWriter.wrap(s) do |f|
+        pk = MessagePack::Packer.new(f)
+        pk.write('hello')
+        pk.write('world')
+        pk.flush
+      end
+      s.string
+    end
+
+    it 'yields job result for each row' do
+      stub_api_request(:get, '/v3/job/result/12345').
+        with(:query => {'format' => 'msgpack'}).
+        to_return(
+          :headers => {'Content-Encoding' => 'gzip'},
+          :body => packed
+        )
+      result = []
+      api.job_result_each(12345) do |row|
+        result << row
+      end
+      result.should == ['hello', 'world']
+    end
+  end
+
+  describe 'job_result_each_with_compr_size' do
+    let :packed do
+      s = StringIO.new
+      Zlib::GzipWriter.wrap(s) do |f|
+        pk = MessagePack::Packer.new(f)
+        pk.write('hello')
+        pk.write('world')
+        pk.flush
+      end
+      s.string
+    end
+
+    it 'yields job result for each row with progress' do
+      stub_api_request(:get, '/v3/job/result/12345').
+        with(:query => {'format' => 'msgpack'}).
+        to_return(
+          :headers => {'Content-Encoding' => 'gzip'},
+          :body => packed
+        )
+      result = []
+      api.job_result_each_with_compr_size(12345) do |row, size|
+        result << [row, size]
+      end
+      result.should == [['hello', 32], ['world', 32]]
+    end
+  end
+
+  describe 'job_result_raw' do
+    it 'returns raw result' do
+      stub_api_request(:get, '/v3/job/result/12345').
+        with(:query => {'format' => 'json'}).
+        to_return(:body => 'raw binary')
+      api.job_result_raw(12345, 'json').should == 'raw binary'
+    end
+  end
+
+  describe 'kill' do
+    it 'kills a job' do
+      stub_api_request(:post, '/v3/job/kill/12345').
+        to_return(:body => {'former_status' => 'status'}.to_json)
+      api.kill(12345).should == 'status'
+    end
+  end
 end