td-client 0.8.62 → 0.8.63

data/lib/td/client/version.rb

@@ -1,5 +1,5 @@
 module TreasureData
   class Client
-    VERSION = '0.8.62'
+    VERSION = '0.8.63'
   end
 end

data/spec/spec_helper.rb

@@ -14,6 +14,7 @@ include WebMock::API
 $LOAD_PATH << File.dirname(__FILE__)+"../lib"
 require 'td-client'
 require 'msgpack'
+require 'json'
 
 include TreasureData
 

data/spec/td/client/api_spec.rb

@@ -126,15 +126,11 @@ describe API do
 
     describe "'validate_column_name'" do
       it 'should raise a ParameterValidationError exception' do
-        INVALID_NAMES.each_pair {|ng,ok|
+        ['/', '', 'D'].each { |ng|
           lambda {
             API.validate_column_name(ng)
           }.should raise_error(ParameterValidationError)
         }
-        # empty
-        lambda {
-          API.validate_column_name('')
-        }.should raise_error(ParameterValidationError)
       end
 
       it 'should return valid data' do

data/spec/td/client/bulk_import_spec.rb

@@ -18,14 +18,14 @@ describe 'BulkImport API' do
     end
 
     it 'should return 422 error with invalid name' do
-      name = '1'
+      name = 'D'
       err_msg = "Validation failed: Name is too short" # " (minimum is 3 characters)"
       stub_api_request(:post, "/v3/bulk_import/create/#{e(name)}/#{e(db_name)}/#{e(table_name)}")
-        .to_return(:status => 404, :body => {'message' => err_msg}.to_json)
+        .to_return(:status => 422, :body => {'message' => err_msg}.to_json)
 
       expect {
         api.create_bulk_import(name, db_name, table_name)
-      }.to raise_error(TreasureData::APIError, /^#{err_msg}/)
+      }.to raise_error(TreasureData::APIError, /#{err_msg}/)
     end
 
     it 'should return 404 error with non exist database name' do
@@ -36,7 +36,7 @@ describe 'BulkImport API' do
 
       expect {
         api.create_bulk_import(bi_name, db, table_name)
-      }.to raise_error(TreasureData::APIError, /^#{err_msg}/)
+      }.to raise_error(TreasureData::APIError, /#{err_msg}/)
     end
 
     it 'should return 404 error with non exist table name' do
@@ -47,7 +47,7 @@ describe 'BulkImport API' do
 
       expect {
         api.create_bulk_import(bi_name, db_name, table)
-      }.to raise_error(TreasureData::APIError, /^#{err_msg}/)
+      }.to raise_error(TreasureData::APIError, /#{err_msg}/)
     end
   end
 end

data/spec/td/client/db_api_spec.rb

@@ -9,9 +9,14 @@ describe 'Database API' do
     API.new(nil)
   end
 
-  describe 'create_database' do
+  let :client do
+    Client.new(apikey)
+  end
+
+  describe "'create_database' API" do
     it 'should create a new database' do
-      stub_api_request(:post, "/v3/database/create/#{e(db_name)}").to_return(:body => {'database' => db_name}.to_json)
+      stub_api_request(:post, "/v3/database/create/#{e(db_name)}")
+        .to_return(:body => {'database' => db_name}.to_json)
 
       api.create_database(db_name).should be_true
     end
@@ -19,20 +24,100 @@ describe 'Database API' do
     it 'should return 400 error with invalid name' do
       invalid_name = 'a'
       err_msg = "Name must be 3 to 256 characters, got #{invalid_name.length} characters. name = '#{invalid_name}'"
-      stub_api_request(:post, "/v3/database/create/#{e(invalid_name)}").to_return(:status => 400, :body => {'message' => err_msg}.to_json)
+      stub_api_request(:post, "/v3/database/create/#{e(invalid_name)}")
+        .to_return(:status => 400, :body => {'message' => err_msg}.to_json)
 
       expect {
         api.create_database(invalid_name)
-      }.to raise_error(TreasureData::APIError, /^#{err_msg}/)
+      }.to raise_error(TreasureData::APIError, /#{err_msg}/)
     end
 
     it 'should return 409 error with duplicated name' do
       err_msg = "Database #{db_name} already exists"
-      stub_api_request(:post, "/v3/database/create/#{e(db_name)}").to_return(:status => 409, :body => {'message' => err_msg}.to_json)
+      stub_api_request(:post, "/v3/database/create/#{e(db_name)}")
+        .to_return(:status => 409, :body => {'message' => err_msg}.to_json)
 
       expect {
         api.create_database(db_name)
-      }.to raise_error(TreasureData::APIError, /^#{err_msg}/)
+      }.to raise_error(TreasureData::AlreadyExistsError, /#{err_msg}/)
+    end
+  end
+
+  describe "'list_databases' API" do
+    it 'should list the databases with count, created_at, updated_at, organization, and permission' do
+      databases = [
+        ["db_1", 111, "2013-01-21 01:51:41 UTC", "2014-01-21 01:51:41 UTC", nil, "administrator"],
+        ["db_2", 222, "2013-02-22 02:52:42 UTC", "2014-02-22 02:52:42 UTC", nil, "full_access"],
+        ["db_3", 333, "2013-03-23 03:53:43 UTC", "2014-03-23 03:53:43 UTC", nil, "import_only"],
+        ["db_4", 444, "2013-04-24 04:54:44 UTC", "2014-04-24 04:54:44 UTC", nil, "query_only"]
+      ]
+      stub_api_request(:get, "/v3/database/list")
+        .to_return(:body => {'databases' => [
+          {'name' => databases[0][0], 'count' => databases[0][1], 'created_at' => databases[0][2], 'updated_at' => databases[0][3], 'organization' => databases[0][4], 'permission' => databases[0][5]},
+          {'name' => databases[1][0], 'count' => databases[1][1], 'created_at' => databases[1][2], 'updated_at' => databases[1][3], 'organization' => databases[1][4], 'permission' => databases[1][5]},
+          {'name' => databases[2][0], 'count' => databases[2][1], 'created_at' => databases[2][2], 'updated_at' => databases[2][3], 'organization' => databases[2][4], 'permission' => databases[2][5]},
+          {'name' => databases[3][0], 'count' => databases[3][1], 'created_at' => databases[3][2], 'updated_at' => databases[3][3], 'organization' => databases[3][4], 'permission' => databases[3][5]}
+        ]}.to_json)
+
+      db_list = api.list_databases
+      databases.each {|db|
+        expect(db_list[db[0]]).to eq(db[1..-1])
+      }
+    end
+  end
+
+  describe "'databases' Client API" do
+    it 'should return an array of Databases objects containing name, count, created_at, updated_at, organization, and permission' do
+      databases = [
+        ["db_1", 111, "2013-01-21 01:51:41 UTC", "2014-01-21 01:51:41 UTC", nil, "administrator"],
+        ["db_2", 222, "2013-02-22 02:52:42 UTC", "2014-02-22 02:52:42 UTC", nil, "full_access"],
+        ["db_3", 333, "2013-03-23 03:53:43 UTC", "2014-03-23 03:53:43 UTC", nil, "import_only"],
+        ["db_4", 444, "2013-04-24 04:54:44 UTC", "2014-04-24 04:54:44 UTC", nil, "query_only"]
+      ]
+      stub_api_request(:get, "/v3/database/list")
+        .to_return(:body => {'databases' => [
+          {'name' => databases[0][0], 'count' => databases[0][1], 'created_at' => databases[0][2], 'updated_at' => databases[0][3], 'organization' => databases[0][4], 'permission' => databases[0][5]},
+          {'name' => databases[1][0], 'count' => databases[1][1], 'created_at' => databases[1][2], 'updated_at' => databases[1][3], 'organization' => databases[1][4], 'permission' => databases[1][5]},
+          {'name' => databases[2][0], 'count' => databases[2][1], 'created_at' => databases[2][2], 'updated_at' => databases[2][3], 'organization' => databases[2][4], 'permission' => databases[2][5]},
+          {'name' => databases[3][0], 'count' => databases[3][1], 'created_at' => databases[3][2], 'updated_at' => databases[3][3], 'organization' => databases[3][4], 'permission' => databases[3][5]}
+        ]}.to_json)
+
+      db_list = client.databases
+      databases.length.times {|i|
+        expect(db_list[i].name).to          eq(databases[i][0])
+        expect(db_list[i].count).to         eq(databases[i][1])
+        expect(db_list[i].created_at).to    eq(Time.parse(databases[i][2]))
+        expect(db_list[i].updated_at).to    eq(Time.parse(databases[i][3]))
+        expect(db_list[i].org_name).to      eq(databases[i][4])
+        expect(db_list[i].permission).to    eq(databases[i][5].to_sym)
+      }
+    end
+  end
+
+  describe "'database' Client API" do
+    it "should return the Databases object corresponding to the name and containing count, created_at, updated_at, organization, and permission" do
+      databases = [
+        ["db_1", 111, "2013-01-21 01:51:41 UTC", "2014-01-21 01:51:41 UTC", nil, "administrator"],
+        ["db_2", 222, "2013-02-22 02:52:42 UTC", "2014-02-22 02:52:42 UTC", nil, "full_access"],
+        ["db_3", 333, "2013-03-23 03:53:43 UTC", "2014-03-23 03:53:43 UTC", nil, "import_only"],
+        ["db_4", 444, "2013-04-24 04:54:44 UTC", "2014-04-24 04:54:44 UTC", nil, "query_only"]
+      ]
+      stub_api_request(:get, "/v3/database/list")
+        .to_return(:body => {'databases' => [
+          {'name' => databases[0][0], 'count' => databases[0][1], 'created_at' => databases[0][2], 'updated_at' => databases[0][3], 'organization' => databases[0][4], 'permission' => databases[0][5]},
+          {'name' => databases[1][0], 'count' => databases[1][1], 'created_at' => databases[1][2], 'updated_at' => databases[1][3], 'organization' => databases[1][4], 'permission' => databases[1][5]},
+          {'name' => databases[2][0], 'count' => databases[2][1], 'created_at' => databases[2][2], 'updated_at' => databases[2][3], 'organization' => databases[2][4], 'permission' => databases[2][5]},
+          {'name' => databases[3][0], 'count' => databases[3][1], 'created_at' => databases[3][2], 'updated_at' => databases[3][3], 'organization' => databases[3][4], 'permission' => databases[3][5]}
+        ]}.to_json)
+
+      i = 1
+      db = client.database(databases[i][0])
+      expect(db.name).to        eq(databases[i][0])
+      expect(db.count).to       eq(databases[i][1])
+      expect(db.created_at).to  eq(Time.parse(databases[i][2]))
+      expect(db.updated_at).to  eq(Time.parse(databases[i][3]))
+      expect(db.org_name).to    eq(databases[i][4])
+      expect(db.permission).to  eq(databases[i][5].to_sym)
     end
   end
 end

data/spec/td/client/export_api_spec.rb

@@ -32,7 +32,7 @@ describe 'Export API' do
 
       expect {
         api.export(db_name, table_name, invalid_type)
-      }.to raise_error(TreasureData::APIError, /^#{err_msg}/)
+      }.to raise_error(TreasureData::APIError, /#{err_msg}/)
     end
 
     # TODO: Add other parameters spec

data/spec/td/client/job_api_spec.rb

@@ -1,5 +1,6 @@
 require 'spec_helper'
 require 'td/client/spec_resources'
+require 'json'
 
 describe 'Job API' do
   include_context 'spec symbols'
@@ -13,9 +14,35 @@ describe 'Job API' do
     it 'should returns 20 jobs by default' do
       stub_api_request(:get, "/v3/job/list", :query => {'from' => '0'}).to_return(:body => {'jobs' => raw_jobs}.to_json)
       jobs = api.list_jobs
+      puts jobs.size
       jobs.size.should == 20
     end
 
+    (0...MAX_JOB).each {|i|
+      it "should get the correct fields for job #{i} of 20" do
+        job = raw_jobs[i]
+        stub_api_request(:get, "/v3/job/list", :query => {'from' => '0'}).to_return(:body => {'jobs' => raw_jobs}.to_json)
+
+        jobs = api.list_jobs
+        jobs[i..i].map {|job_id, type, status, query, start_at, end_at, cpu_time,
+                      result_size, result_url, priority, retry_limit, org, db|
+          job_id.should == job['job_id']
+          type.should == job['type']
+          status.should == job['status']
+          query.should == job['query']
+          start_at.should == job['start_at']
+          end_at.should == job['end_at']
+          cpu_time.should == job['cpu_time']
+          result_size.should == job['result_size']
+          result_url.should == job['result_url']
+          priority.should == job['priority']
+          retry_limit.should == job['retry_limit']
+          org.should == job['organization']
+          db.should == job['database']
+        }
+      end
+    }
+
     it 'should returns 10 jobs with to parameter' do
       stub_api_request(:get, "/v3/job/list", :query => {'from' => '0', 'to' => '10'}).to_return(:body => {'jobs' => raw_jobs[0...10]}.to_json)
       jobs = api.list_jobs(0, 10)
@@ -28,15 +55,19 @@ describe 'Job API' do
       jobs = api.list_jobs(0, nil, 'error')
       jobs.size.should == error_jobs.size
     end
+
+    #it 'should contain the result_size field' do
+
   end
 
   describe 'show_job' do
     (0...MAX_JOB).each { |i|
-      it "should get a job of id #{i}" do
+      it "should get the correct fields for job #{i}" do
         job = raw_jobs[i]
         stub_api_request(:get, "/v3/job/show/#{e(i)}").to_return(:body => job.to_json)
 
-        type, query, status, url, debug, start_at, end_at, result_url, hive_result_schema, priority, retry_limit, org, db = api.show_job(i)
+        type, query, status, url, debug, start_at, end_at, cpu_time,
+          result_size, result_url, hive_result_schema, priority, retry_limit, org, db = api.show_job(i)
         type.should == job['type']
         query.should == job['query']
         status.should == job['status']
@@ -44,6 +75,8 @@ describe 'Job API' do
         debug.should == job['debug']
         start_at.should == job['start_at']
         end_at.should == job['end_at']
+        cpu_time.should == job['cpu_time']
+        result_size.should == job['result_size']
         result_url.should == job['result_url']
         hive_result_schema.should == job['hive_result_schema']
         result_url.should == job['result_url']
@@ -60,7 +93,7 @@ describe 'Job API' do
 
       expect {
         api.show_job(unknown_id)
-      }.to raise_error(TreasureData::NotFoundError, /^Couldn't find Job with account_id = #{account_id}, id = #{unknown_id}/)
+      }.to raise_error(TreasureData::NotFoundError, /Couldn't find Job with account_id = #{account_id}, id = #{unknown_id}/)
     end
 
     it 'should return an error with invalid id' do
@@ -70,13 +103,13 @@ describe 'Job API' do
 
       expect {
         api.show_job(invalid_id)
-      }.to raise_error(TreasureData::APIError, /^'job_id' parameter is required but missing/)
+      }.to raise_error(TreasureData::APIError, /'job_id' parameter is required but missing/)
     end
   end
 
   describe 'job status' do
     (0...MAX_JOB).each { |i|
-      it 'should return the status of a job #{i}' do
+      it "should return the status of a job #{i}" do
         job_id = i.to_s
         raw_job = raw_jobs[i]
         result_job = {

data/spec/td/client/partial_delete_api_spec.rb

@@ -38,7 +38,7 @@ describe 'PartialDelete API' do
 
       expect {
         api.partial_delete(db, table_name, to, from)
-      }.to raise_error(TreasureData::APIError, /^#{err_msg}/)
+      }.to raise_error(TreasureData::APIError, /#{err_msg}/)
     end
 
     it 'should return 404 error with non exist table name' do
@@ -49,7 +49,7 @@ describe 'PartialDelete API' do
 
       expect {
         api.partial_delete(db_name, table, to, from)
-      }.to raise_error(TreasureData::APIError, /^#{err_msg}/)
+      }.to raise_error(TreasureData::APIError, /#{err_msg}/)
     end
 
     # TODO: Add from / to parameters spec

data/spec/td/client/result_api_spec.rb

@@ -26,7 +26,7 @@ describe 'Result API' do
 
       expect {
         api.create_result(name, result_url)
-      }.to raise_error(TreasureData::APIError, /^#{err_msg}/)
+      }.to raise_error(TreasureData::APIError, /#{err_msg}/)
     end
 
     it 'should return 422 error without url' do
@@ -37,7 +37,7 @@ describe 'Result API' do
 
       expect {
         api.create_result(result_name, false)
-      }.to raise_error(TreasureData::APIError, /^#{err_msg}/)
+      }.to raise_error(TreasureData::APIError, /#{err_msg}/)
     end
 
     it 'should return 409 error with duplicated name' do
@@ -48,7 +48,7 @@ describe 'Result API' do
 
       expect {
         api.create_result(result_name, result_url)
-      }.to raise_error(TreasureData::APIError, /^#{err_msg}/)
+      }.to raise_error(TreasureData::APIError, /#{err_msg}/)
     end
   end
 end

data/spec/td/client/sched_api_spec.rb

@@ -16,7 +16,8 @@ describe 'Schedule API' do
 
     it 'should create a new schedule' do
       start = Time.now
-      stub_api_request(:post, "/v3/schedule/create/#{e(sched_name)}").with(opts.merge('type' => 'hive'))
+      stub_api_request(:post, "/v3/schedule/create/#{e(sched_name)}")
+        .with(opts.merge('type' => 'hive'))
         .to_return(:body => {'name' => sched_name, 'start' => start.to_s}.to_json)
 
       api.create_schedule(sched_name, opts).should == start.to_s
@@ -25,12 +26,44 @@ describe 'Schedule API' do
     it 'should return 422 error with invalid name' do
       name = '1'
       err_msg = "Validation failed: Name is too short" # " (minimum is 3 characters)"
-      stub_api_request(:post, "/v3/schedule/create/#{e(name)}").with(opts.merge('type' => 'hive'))
+      stub_api_request(:post, "/v3/schedule/create/#{e(name)}")
+        .with(opts.merge('type' => 'hive'))
         .to_return(:status => 422, :body => {'message' => err_msg}.to_json)
 
       expect {
         api.create_schedule(name, opts)
-      }.to raise_error(TreasureData::APIError, /^#{err_msg}/)
+      }.to raise_error(TreasureData::APIError, /#{err_msg}/)
+    end
+  end
+
+  describe 'update_schedule' do
+    let :pig_query do
+      "OUT = FOREACH (GROUP plt364 ALL) GENERATE COUNT(plt364);\n" * 200
+    end
+    let :opts do
+      {'cron' => cron, 'query' => pig_query, 'database' => db_name}
+    end
+
+    it 'should not return 414 even if the query text is very long' do
+      stub_api_request(:post, "/v3/schedule/update/#{e(sched_name)}")
+        .with(opts.merge('type' => 'pig'))
+        .to_return(:body => {'name' => sched_name, 'query' => pig_query}.to_json)
+
+      err_msg = "Update schedule failed: Request-URI Too Long"
+      expect {
+        api.update_schedule(sched_name, opts)
+      }.not_to raise_error
+    end
+
+    it 'should update the schedule with the new query' do
+      stub_api_request(:post, "/v3/schedule/update/#{e(sched_name)}")
+        .with(opts.merge('type' => 'pig'))
+        .to_return(:body => {'name' => sched_name, 'query' => pig_query}.to_json)
+
+      stub_api_request(:get, "/v3/schedule/list")
+        .to_return(:body => {'schedules' => [{'name' => sched_name, 'query' => pig_query}]}.to_json)
+
+      expect(api.list_schedules.first[2]).to eq(pig_query)
     end
   end
 end

data/spec/td/client/spec_resources.rb

@@ -2,6 +2,10 @@ require 'spec_helper'
 require 'td/client/model'
 
 shared_context 'spec symbols' do
+  let :apikey do
+    '1/0123456789ABCDEFG'
+  end
+
   let :db_name do
     'db_test'
   end
@@ -66,22 +70,24 @@ shared_context 'job resources' do
       job_type = types[i % types.size]
       status = i.odd? ? 'success' : 'error'
       {
-        "job_id"      => i,
-        "url"         => "https://console.treasure-data.com/jobs/#{i.to_s}?target=query",
-        "database"    => dbs[i % dbs.size].to_s,
-        "status"      => status,
-        "type"        => job_type[0].to_sym,
-        "query"       => "select #{i}",
-        "priority"    => i % 3,
-        "result"      => nil,
-        "created_at"  => created_at.to_s,
-        "updated_at"  => (created_at + (i * 10)).to_s,
-        "start_at"    => (created_at + (i * 10 * 60)).to_s,
-        "end_at"      => (created_at + (i * 10 * 3600)).to_s,
-        'retry_limit' => 10,
-        'organization' => nil,
+        "job_id"             => i,
+        "url"                => "https://console.treasure-data.com/jobs/#{i.to_s}?target=query",
+        "database"           => dbs[i % dbs.size].to_s,
+        "status"             => status,
+        "type"               => job_type[0].to_sym,
+        "query"              => "select #{i}",
+        "priority"           => i % 3,
+        "result"             => nil,
+        "created_at"         => created_at.to_s,
+        "updated_at"         => (created_at + (i * 10)).to_s,
+        "start_at"           => (created_at + (i * 10 * 60)).to_s,
+        "end_at"             => (created_at + (i * 10 * 3600)).to_s,
+        "cpu_time"           => i * 100 + i,
+        "result_size"        => i * 1000,
+        'retry_limit'        => 10,
+        'organization'       => nil,
         'hive_result_schema' => nil,
-        'debug' => { 
+        'debug' => {
           'stderr' => "job #{i} #{status}",
           'cmdout' => "job #{i} command",
         }