td-client 0.8.84 → 0.8.85
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/td/client.rb +12 -4
- data/lib/td/client/api.rb +19 -15
- data/lib/td/client/api/export.rb +12 -0
- data/lib/td/client/api/job.rb +4 -2
- data/lib/td/client/model.rb +28 -7
- data/lib/td/client/version.rb +1 -1
- data/spec/td/client/api_spec.rb +4 -4
- data/spec/td/client/export_api_spec.rb +10 -0
- data/spec/td/client/job_api_spec.rb +5 -3
- data/spec/td/client/model_job_spec.rb +1 -1
- data/spec/td/client/spec_resources.rb +1 -0
- data/spec/td/client_spec.rb +1 -0
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 60a54746395659411ee0e31937d4456acf1161db
|
4
|
+
data.tar.gz: 86d0d880f287ef9ee6919fdfac91ad1211d887fa
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 177e498d108552a2b549755455a34e6ac811cf8f06510cf2cb612b53f33ac13876c524fb8390f84459efb52ec00b2316ed8b0fb8a77daf2b11c56674e47bc318
|
7
|
+
data.tar.gz: fd06041a13ded4aeddb1514ce10f0225140d72a70a4dafa2305c77935415b59230d9c31b54378ab5ed04217b1cca4f06a19d1147f54e6c476209ab490adf816a
|
data/lib/td/client.rb
CHANGED
@@ -182,10 +182,10 @@ class Client
|
|
182
182
|
results = @api.list_jobs(from, to, status, conditions)
|
183
183
|
results.map {|job_id, type, status, query, start_at, end_at, cpu_time,
|
184
184
|
result_size, result_url, priority, retry_limit, org, db,
|
185
|
-
duration|
|
185
|
+
duration, num_records|
|
186
186
|
Job.new(self, job_id, type, query, status, nil, nil, start_at, end_at, cpu_time,
|
187
187
|
result_size, nil, result_url, nil, priority, retry_limit, org, db,
|
188
|
-
duration)
|
188
|
+
duration, num_records)
|
189
189
|
}
|
190
190
|
end
|
191
191
|
|
@@ -194,9 +194,9 @@ class Client
|
|
194
194
|
def job(job_id)
|
195
195
|
job_id = job_id.to_s
|
196
196
|
type, query, status, url, debug, start_at, end_at, cpu_time,
|
197
|
-
result_size, result_url, hive_result_schema, priority, retry_limit, org, db = @api.show_job(job_id)
|
197
|
+
result_size, result_url, hive_result_schema, priority, retry_limit, org, db, num_records = @api.show_job(job_id)
|
198
198
|
Job.new(self, job_id, type, query, status, url, debug, start_at, end_at, cpu_time,
|
199
|
-
result_size, nil, result_url, hive_result_schema, priority, retry_limit, org, db)
|
199
|
+
result_size, nil, result_url, hive_result_schema, priority, retry_limit, org, db, num_records)
|
200
200
|
end
|
201
201
|
|
202
202
|
# @param [String] job_id
|
@@ -254,6 +254,14 @@ class Client
|
|
254
254
|
Job.new(self, job_id, :export, nil)
|
255
255
|
end
|
256
256
|
|
257
|
+
# @param [String] target_job_id
|
258
|
+
# @param [Hash] opts
|
259
|
+
# @return [Job]
|
260
|
+
def result_export(target_job_id, opts={})
|
261
|
+
job_id = @api.result_export(target_job_id, opts)
|
262
|
+
Job.new(self, job_id, :result_export, nil)
|
263
|
+
end
|
264
|
+
|
257
265
|
# @param [String] db_name
|
258
266
|
# @param [String] table_name
|
259
267
|
# @param [Fixnum] to
|
data/lib/td/client/api.rb
CHANGED
@@ -149,15 +149,27 @@ class API
|
|
149
149
|
end
|
150
150
|
|
151
151
|
name = name.to_s
|
152
|
-
if
|
153
|
-
|
152
|
+
if max_len
|
153
|
+
if name.length < min_len || name.length > max_len
|
154
|
+
raise ParameterValidationError,
|
155
|
+
"#{target.capitalize} name must be between #{min_len} and #{max_len} characters long. Got #{name.length} " +
|
156
|
+
(name.length == 1 ? "character" : "characters") + "."
|
157
|
+
end
|
158
|
+
else
|
159
|
+
if min_len == 1
|
160
|
+
if name.empty?
|
161
|
+
raise ParameterValidationError,
|
154
162
|
"Empty #{target} name is not allowed"
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
163
|
+
end
|
164
|
+
else
|
165
|
+
if name.length < min_len
|
166
|
+
raise ParameterValidationError,
|
167
|
+
"#{target.capitalize} name must be longer than #{min_len} characters. Got #{name.length} " +
|
159
168
|
(name.length == 1 ? "character" : "characters") + "."
|
169
|
+
end
|
170
|
+
end
|
160
171
|
end
|
172
|
+
|
161
173
|
unless name =~ /^([a-z0-9_]+)$/
|
162
174
|
raise ParameterValidationError,
|
163
175
|
"#{target.capitalize} name must only consist of lower-case alpha-numeric characters and '_'."
|
@@ -184,25 +196,17 @@ class API
|
|
184
196
|
# @param [String] name
|
185
197
|
def self.validate_column_name(name)
|
186
198
|
target = 'column'
|
187
|
-
min_len = 1
|
188
|
-
max_len = 128
|
189
199
|
name = name.to_s
|
190
200
|
if name.empty?
|
191
201
|
raise ParameterValidationError,
|
192
202
|
"Empty #{target} name is not allowed"
|
193
203
|
end
|
194
|
-
if name.length < min_len || name.length > max_len
|
195
|
-
raise ParameterValidationError,
|
196
|
-
"#{target.capitalize} name must be between #{min_len} and #{max_len} characters long. Got #{name.length} " +
|
197
|
-
(name.length == 1 ? "character" : "characters") + "."
|
198
|
-
end
|
199
|
-
|
200
204
|
name
|
201
205
|
end
|
202
206
|
|
203
207
|
# @param [String] name
|
204
208
|
def self.validate_sql_alias_name(name)
|
205
|
-
validate_name("sql_alias", 1,
|
209
|
+
validate_name("sql_alias", 1, nil, name)
|
206
210
|
end
|
207
211
|
|
208
212
|
# @param [String] name
|
data/lib/td/client/api/export.rb
CHANGED
@@ -22,5 +22,17 @@ module Export
|
|
22
22
|
return js['job_id'].to_s
|
23
23
|
end
|
24
24
|
|
25
|
+
# => jobId:String
|
26
|
+
# @param [String] target_job_id
|
27
|
+
# @param [Hash] opts
|
28
|
+
# @return [String] job_id
|
29
|
+
def result_export(target_job_id, opts={})
|
30
|
+
code, body, res = post("/v3/job/result_export/#{target_job_id}", opts)
|
31
|
+
if code != "200"
|
32
|
+
raise_error("Result Export failed", res)
|
33
|
+
end
|
34
|
+
js = checked_json(body, %w[job_id])
|
35
|
+
return js['job_id'].to_s
|
36
|
+
end
|
25
37
|
end
|
26
38
|
end
|
data/lib/td/client/api/job.rb
CHANGED
@@ -36,9 +36,10 @@ module Job
|
|
36
36
|
priority = m['priority']
|
37
37
|
retry_limit = m['retry_limit']
|
38
38
|
duration = m['duration']
|
39
|
+
num_records = m['num_records']
|
39
40
|
result << [job_id, type, status, query, start_at, end_at, cpu_time,
|
40
41
|
result_size, result_url, priority, retry_limit, nil, database,
|
41
|
-
duration]
|
42
|
+
duration, num_records]
|
42
43
|
}
|
43
44
|
return result
|
44
45
|
end
|
@@ -63,6 +64,7 @@ module Job
|
|
63
64
|
end_at = js['end_at']
|
64
65
|
cpu_time = js['cpu_time']
|
65
66
|
result_size = js['result_size'] # compressed result size in msgpack.gz format
|
67
|
+
num_records = js['num_records']
|
66
68
|
result = js['result'] # result target URL
|
67
69
|
hive_result_schema = (js['hive_result_schema'] || '')
|
68
70
|
if hive_result_schema.empty?
|
@@ -97,7 +99,7 @@ module Job
|
|
97
99
|
priority = js['priority']
|
98
100
|
retry_limit = js['retry_limit']
|
99
101
|
return [type, query, status, url, debug, start_at, end_at, cpu_time,
|
100
|
-
result_size, result, hive_result_schema, priority, retry_limit, nil, database]
|
102
|
+
result_size, result, hive_result_schema, priority, retry_limit, nil, database, num_records]
|
101
103
|
end
|
102
104
|
|
103
105
|
# @param [String] job_id
|
data/lib/td/client/model.rb
CHANGED
@@ -402,9 +402,10 @@ class Job < Model
|
|
402
402
|
# @param [String] org_name
|
403
403
|
# @param [String] db_name
|
404
404
|
# @param [Fixnum] duration
|
405
|
+
# @param [Fixnum] num_records
|
405
406
|
def initialize(client, job_id, type, query, status=nil, url=nil, debug=nil, start_at=nil, end_at=nil, cpu_time=nil,
|
406
407
|
result_size=nil, result=nil, result_url=nil, hive_result_schema=nil, priority=nil, retry_limit=nil,
|
407
|
-
org_name=nil, db_name=nil, duration=nil)
|
408
|
+
org_name=nil, db_name=nil, duration=nil, num_records=nil)
|
408
409
|
super(client)
|
409
410
|
@job_id = job_id
|
410
411
|
@type = type
|
@@ -423,6 +424,7 @@ class Job < Model
|
|
423
424
|
@retry_limit = retry_limit
|
424
425
|
@db_name = db_name
|
425
426
|
@duration = duration
|
427
|
+
@num_records = num_records
|
426
428
|
end
|
427
429
|
|
428
430
|
# @!attribute [r] job_id
|
@@ -433,26 +435,35 @@ class Job < Model
|
|
433
435
|
# @!attribute [r] org_name
|
434
436
|
# @!attribute [r] db_name
|
435
437
|
# @!attribute [r] duration
|
438
|
+
# @!attribute [r] num_records
|
436
439
|
attr_reader :job_id, :type, :result_url
|
437
440
|
attr_reader :priority, :retry_limit, :org_name, :db_name
|
438
|
-
attr_reader :duration
|
441
|
+
attr_reader :duration, :num_records
|
439
442
|
|
440
443
|
# @option timeout [Integer,nil] timeout in second
|
441
444
|
# @option wait_interval [Integer,nil] interval in second of polling the job status
|
442
445
|
# @param detail [Boolean] update job detail or not
|
443
446
|
# @param verbose [Boolean] out retry log to stderr or not
|
444
|
-
def wait(
|
445
|
-
|
447
|
+
def wait(*args)
|
448
|
+
opthash = Hash.try_convert(args.last)
|
449
|
+
if opthash
|
450
|
+
args.pop
|
451
|
+
detail = opthash.fetch(:detail, false)
|
452
|
+
verbose = opthash.fetch(:verbose, ENV['TD_CLIENT_DEBUG'])
|
453
|
+
end
|
454
|
+
timeout = args[0]
|
455
|
+
wait_interval = args[1] || 2
|
456
|
+
deadline = monotonic_clock + timeout if timeout
|
446
457
|
timeout_klass = Class.new(Exception)
|
447
458
|
begin
|
448
459
|
if timeout
|
449
|
-
if deadline <=
|
460
|
+
if deadline <= monotonic_clock
|
450
461
|
raise timeout_klass, "timeout (#{timeout}) exceeded wait_interval=#{wait_interval}"
|
451
462
|
end
|
452
463
|
end
|
453
464
|
sleep wait_interval
|
465
|
+
detail ? update_status! : update_progress!
|
454
466
|
yield self if block_given?
|
455
|
-
detail ? update_status! : update_progress!
|
456
467
|
rescue timeout_klass
|
457
468
|
raise Timeout::Error, $!.message
|
458
469
|
rescue Timeout::Error, SystemCallError, EOFError, SocketError, HTTPClient::ConnectTimeoutError
|
@@ -606,7 +617,7 @@ class Job < Model
|
|
606
617
|
def update_status!
|
607
618
|
type, query, status, url, debug, start_at, end_at, cpu_time,
|
608
619
|
result_size, result_url, hive_result_schema, priority, retry_limit,
|
609
|
-
org_name, db_name = @client.api.show_job(@job_id)
|
620
|
+
org_name, db_name , num_records = @client.api.show_job(@job_id)
|
610
621
|
@query = query
|
611
622
|
@status = status
|
612
623
|
@url = url
|
@@ -620,8 +631,18 @@ class Job < Model
|
|
620
631
|
@priority = priority
|
621
632
|
@retry_limit = retry_limit
|
622
633
|
@db_name = db_name
|
634
|
+
@num_records = num_records
|
623
635
|
self
|
624
636
|
end
|
637
|
+
|
638
|
+
private
|
639
|
+
def monotonic_clock
|
640
|
+
if defined?(Process.clock_gettime)
|
641
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
642
|
+
else
|
643
|
+
Time.now.to_i
|
644
|
+
end
|
645
|
+
end
|
625
646
|
end
|
626
647
|
|
627
648
|
|
data/lib/td/client/version.rb
CHANGED
data/spec/td/client/api_spec.rb
CHANGED
@@ -126,7 +126,7 @@ describe API do
|
|
126
126
|
|
127
127
|
describe "'validate_column_name'" do
|
128
128
|
it 'should raise a ParameterValidationError exception' do
|
129
|
-
[''
|
129
|
+
[''].each { |ng|
|
130
130
|
expect {
|
131
131
|
API.validate_column_name(ng)
|
132
132
|
}.to raise_error(ParameterValidationError)
|
@@ -137,7 +137,7 @@ describe API do
|
|
137
137
|
VALID_NAMES.each {|ok|
|
138
138
|
expect(API.validate_column_name(ok)).to eq(ok)
|
139
139
|
}
|
140
|
-
['a', 'a'*
|
140
|
+
['a', 'a'*255].each {|ok|
|
141
141
|
expect(API.validate_column_name(ok)).to eq(ok)
|
142
142
|
}
|
143
143
|
end
|
@@ -145,7 +145,7 @@ describe API do
|
|
145
145
|
|
146
146
|
describe "'validate_sql_alias_name'" do
|
147
147
|
it 'should raise a ParameterValidationError exception' do
|
148
|
-
[''
|
148
|
+
[''].each { |ng|
|
149
149
|
expect{API.validate_sql_alias_name(ng)}.to raise_error(ParameterValidationError)
|
150
150
|
}
|
151
151
|
valid = ("a".."z").to_a.join<<("0".."9").to_a.join<<"_"
|
@@ -159,7 +159,7 @@ describe API do
|
|
159
159
|
VALID_NAMES.each {|ok|
|
160
160
|
expect(API.validate_sql_alias_name(ok)).to eq(ok)
|
161
161
|
}
|
162
|
-
['a', '_a', 'a_', 'a'*
|
162
|
+
['a', '_a', 'a_', 'a'*512].each {|ok|
|
163
163
|
expect(API.validate_sql_alias_name(ok)).to eq(ok)
|
164
164
|
}
|
165
165
|
end
|
@@ -37,5 +37,15 @@ describe 'Export API' do
|
|
37
37
|
|
38
38
|
# TODO: Add other parameters spec
|
39
39
|
end
|
40
|
+
|
41
|
+
describe 'result_export' do
|
42
|
+
it 'should export result successfully' do
|
43
|
+
params = {'result' => 'mysql://user:pass@host.com/database/table'}
|
44
|
+
stub_api_request(:post, "/v3/job/result_export/100").with(:body => params).
|
45
|
+
to_return(:body => {'job_id' => '101'}.to_json)
|
46
|
+
|
47
|
+
expect(api.result_export(100, params)).to eq('101')
|
48
|
+
end
|
49
|
+
end
|
40
50
|
end
|
41
51
|
|
@@ -31,7 +31,7 @@ describe 'Job API' do
|
|
31
31
|
jobs = api.list_jobs
|
32
32
|
jobs[i..i].map {|job_id, type, status, query, start_at, end_at, cpu_time,
|
33
33
|
result_size, result_url, priority, retry_limit, org, db,
|
34
|
-
duration|
|
34
|
+
duration, num_records|
|
35
35
|
expect(job_id).to eq(job['job_id'])
|
36
36
|
expect(type).to eq(job['type'])
|
37
37
|
expect(status).to eq(job['status'])
|
@@ -46,6 +46,7 @@ describe 'Job API' do
|
|
46
46
|
expect(org).to eq(job['organization'])
|
47
47
|
expect(db).to eq(job['database'])
|
48
48
|
expect(duration).to eq(job['duration'])
|
49
|
+
expect(num_records).to eq(job['num_records'])
|
49
50
|
}
|
50
51
|
end
|
51
52
|
}
|
@@ -74,7 +75,7 @@ describe 'Job API' do
|
|
74
75
|
stub_api_request(:get, "/v3/job/show/#{e(i)}").to_return(:body => job.to_json)
|
75
76
|
|
76
77
|
type, query, status, url, debug, start_at, end_at, cpu_time,
|
77
|
-
result_size, result_url, hive_result_schema, priority, retry_limit, org, db = api.show_job(i)
|
78
|
+
result_size, result_url, hive_result_schema, priority, retry_limit, org, db, num_records = api.show_job(i)
|
78
79
|
expect(type).to eq(job['type'])
|
79
80
|
expect(query).to eq(job['query'])
|
80
81
|
expect(status).to eq(job['status'])
|
@@ -90,6 +91,7 @@ describe 'Job API' do
|
|
90
91
|
expect(priority).to eq(job['priority'])
|
91
92
|
expect(org).to eq(job['organization'])
|
92
93
|
expect(db).to eq(job['database'])
|
94
|
+
expect(num_records).to eq(job['num_records'])
|
93
95
|
end
|
94
96
|
}
|
95
97
|
|
@@ -669,7 +671,7 @@ describe 'Job API' do
|
|
669
671
|
s.string
|
670
672
|
end
|
671
673
|
let :deflated do
|
672
|
-
Zlib.deflate(formatted)
|
674
|
+
Zlib::Deflate.deflate(formatted)
|
673
675
|
end
|
674
676
|
subject do
|
675
677
|
str = ''
|
@@ -26,7 +26,7 @@ describe 'Job Model' do
|
|
26
26
|
'job_id', 'type', 'query', 'status', 'url', 'debug',
|
27
27
|
'start_at', 'end_at', 'cpu_time', 'result_size', 'result', 'result_url',
|
28
28
|
'hive_result_schema', 'priority', 'retry_limit', 'org_name', 'db_name',
|
29
|
-
'duration'
|
29
|
+
'duration', 'num_records'
|
30
30
|
].map {|name| job_attributes[name]}
|
31
31
|
end
|
32
32
|
|
data/spec/td/client_spec.rb
CHANGED
@@ -39,6 +39,7 @@ describe 'Command' do
|
|
39
39
|
expect(job.org_name).to eq raw_jobs[i]['organization']
|
40
40
|
expect(job.db_name).to eq raw_jobs[i]['database']
|
41
41
|
expect(job.duration).to eq raw_jobs[i]['duration']
|
42
|
+
expect(job.num_records).to eq raw_jobs[i]['num_records']
|
42
43
|
end
|
43
44
|
end
|
44
45
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: td-client
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.8.
|
4
|
+
version: 0.8.85
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Treasure Data, Inc.
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-11-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: msgpack
|
@@ -214,7 +214,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
214
214
|
version: '0'
|
215
215
|
requirements: []
|
216
216
|
rubyforge_project:
|
217
|
-
rubygems_version: 2.5.
|
217
|
+
rubygems_version: 2.5.2
|
218
218
|
signing_key:
|
219
219
|
specification_version: 4
|
220
220
|
summary: Treasure Data API library for Ruby
|