td-client 0.8.85 → 0.9.0dev1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/td/client.rb +8 -16
- data/lib/td/client/api.rb +46 -62
- data/lib/td/client/api/bulk_import.rb +2 -1
- data/lib/td/client/api/bulk_load.rb +3 -3
- data/lib/td/client/api/export.rb +0 -12
- data/lib/td/client/api/import.rb +2 -3
- data/lib/td/client/api/job.rb +71 -145
- data/lib/td/client/api/schedule.rb +1 -1
- data/lib/td/client/api_error.rb +0 -5
- data/lib/td/client/model.rb +28 -91
- data/lib/td/client/version.rb +1 -1
- data/spec/spec_helper.rb +5 -5
- data/spec/td/client/account_api_spec.rb +5 -5
- data/spec/td/client/api_spec.rb +51 -69
- data/spec/td/client/api_ssl_connection_spec.rb +1 -1
- data/spec/td/client/bulk_import_spec.rb +29 -28
- data/spec/td/client/bulk_load_spec.rb +35 -60
- data/spec/td/client/db_api_spec.rb +1 -1
- data/spec/td/client/export_api_spec.rb +1 -11
- data/spec/td/client/import_api_spec.rb +10 -85
- data/spec/td/client/job_api_spec.rb +61 -567
- data/spec/td/client/model_job_spec.rb +10 -27
- data/spec/td/client/model_schedule_spec.rb +2 -2
- data/spec/td/client/partial_delete_api_spec.rb +1 -1
- data/spec/td/client/result_api_spec.rb +3 -3
- data/spec/td/client/sched_api_spec.rb +4 -12
- data/spec/td/client/server_status_api_spec.rb +2 -2
- data/spec/td/client/spec_resources.rb +0 -1
- data/spec/td/client/table_api_spec.rb +14 -14
- data/spec/td/client/user_api_spec.rb +12 -12
- data/spec/td/client_sched_spec.rb +6 -31
- data/spec/td/client_spec.rb +0 -1
- metadata +97 -42
- data/spec/td/client/api_error_spec.rb +0 -77
- data/spec/td/client/model_schema_spec.rb +0 -134
@@ -14,24 +14,25 @@ describe 'BulkImport API' do
|
|
14
14
|
|
15
15
|
let :packed do
|
16
16
|
s = StringIO.new
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
17
|
+
pk = MessagePack::Packer.new(s)
|
18
|
+
pk.write([1, '2', 3.0])
|
19
|
+
pk.write([4, '5', 6.0])
|
20
|
+
pk.write([7, '8', 9.0])
|
21
|
+
pk.flush
|
22
|
+
s.rewind
|
23
|
+
out = StringIO.new
|
24
|
+
Zlib::GzipWriter.wrap(out) do |f|
|
25
|
+
f.write s.read
|
26
|
+
end
|
27
|
+
out.string
|
25
28
|
end
|
26
29
|
|
27
|
-
let(:endpoint_domain) { TreasureData::API::DEFAULT_IMPORT_ENDPOINT }
|
28
|
-
|
29
30
|
describe 'create_bulk_import' do
|
30
31
|
it 'should create a new bulk_import' do
|
31
32
|
stub_api_request(:post, "/v3/bulk_import/create/#{e(bi_name)}/#{e(db_name)}/#{e(table_name)}").
|
32
33
|
to_return(:body => {'bulk_import' => bi_name}.to_json)
|
33
34
|
|
34
|
-
|
35
|
+
api.create_bulk_import(bi_name, db_name, table_name).should be_nil
|
35
36
|
end
|
36
37
|
|
37
38
|
it 'should return 422 error with invalid name' do
|
@@ -72,7 +73,7 @@ describe 'BulkImport API' do
|
|
72
73
|
it 'runs' do
|
73
74
|
stub_api_request(:post, '/v3/bulk_import/delete/name').
|
74
75
|
with(:body => 'foo=bar')
|
75
|
-
|
76
|
+
api.delete_bulk_import('name', 'foo' => 'bar').should == nil
|
76
77
|
end
|
77
78
|
end
|
78
79
|
|
@@ -80,7 +81,7 @@ describe 'BulkImport API' do
|
|
80
81
|
it 'runs' do
|
81
82
|
stub_api_request(:get, '/v3/bulk_import/show/name').
|
82
83
|
to_return(:body => {'status' => 'status', 'other' => 'other'}.to_json)
|
83
|
-
|
84
|
+
api.show_bulk_import('name')['status'].should == 'status'
|
84
85
|
end
|
85
86
|
end
|
86
87
|
|
@@ -89,7 +90,7 @@ describe 'BulkImport API' do
|
|
89
90
|
stub_api_request(:get, '/v3/bulk_import/list').
|
90
91
|
with(:query => 'foo=bar').
|
91
92
|
to_return(:body => {'bulk_imports' => %w(1 2 3)}.to_json)
|
92
|
-
|
93
|
+
api.list_bulk_imports('foo' => 'bar').should == %w(1 2 3)
|
93
94
|
end
|
94
95
|
end
|
95
96
|
|
@@ -98,7 +99,7 @@ describe 'BulkImport API' do
|
|
98
99
|
stub_api_request(:get, '/v3/bulk_import/list_parts/name').
|
99
100
|
with(:query => 'foo=bar').
|
100
101
|
to_return(:body => {'parts' => %w(1 2 3)}.to_json)
|
101
|
-
|
102
|
+
api.list_bulk_import_parts('name', 'foo' => 'bar').should == %w(1 2 3)
|
102
103
|
end
|
103
104
|
end
|
104
105
|
|
@@ -108,10 +109,10 @@ describe 'BulkImport API' do
|
|
108
109
|
File.open(t.path, 'w') do |f|
|
109
110
|
f << '12345'
|
110
111
|
end
|
111
|
-
stub_request(:put,
|
112
|
+
stub_request(:put, 'http://api.treasure-data.com/v3/bulk_import/upload_part/name/part').
|
112
113
|
with(:body => '12345')
|
113
114
|
File.open(t.path) do |f|
|
114
|
-
|
115
|
+
api.bulk_import_upload_part('name', 'part', f, 5).should == nil
|
115
116
|
end
|
116
117
|
end
|
117
118
|
|
@@ -121,10 +122,10 @@ describe 'BulkImport API' do
|
|
121
122
|
File.open(t.path, 'w') do |f|
|
122
123
|
f << '12345'
|
123
124
|
end
|
124
|
-
stub_request(:put,
|
125
|
+
stub_request(:put, 'http://api.treasure-data.com/v3/bulk_import/upload_part/name/' + CGI.escape('日本語(Japanese)'.encode('UTF-8'))).
|
125
126
|
with(:body => '12345')
|
126
127
|
File.open(t.path) do |f|
|
127
|
-
|
128
|
+
api.bulk_import_upload_part('name', '日本語(Japanese)'.encode('Windows-31J'), f, 5).should == nil
|
128
129
|
end
|
129
130
|
end
|
130
131
|
end
|
@@ -133,21 +134,21 @@ describe 'BulkImport API' do
|
|
133
134
|
describe 'bulk_import_delete_part' do
|
134
135
|
it 'runs' do
|
135
136
|
stub_api_request(:post, '/v3/bulk_import/delete_part/name/part')
|
136
|
-
|
137
|
+
api.bulk_import_delete_part('name', 'part').should == nil
|
137
138
|
end
|
138
139
|
end
|
139
140
|
|
140
141
|
describe 'freeze_bulk_import' do
|
141
142
|
it 'runs' do
|
142
143
|
stub_api_request(:post, '/v3/bulk_import/freeze/name')
|
143
|
-
|
144
|
+
api.freeze_bulk_import('name').should == nil
|
144
145
|
end
|
145
146
|
end
|
146
147
|
|
147
148
|
describe 'unfreeze_bulk_import' do
|
148
149
|
it 'runs' do
|
149
150
|
stub_api_request(:post, '/v3/bulk_import/unfreeze/name')
|
150
|
-
|
151
|
+
api.unfreeze_bulk_import('name').should == nil
|
151
152
|
end
|
152
153
|
end
|
153
154
|
|
@@ -155,7 +156,7 @@ describe 'BulkImport API' do
|
|
155
156
|
it 'runs' do
|
156
157
|
stub_api_request(:post, '/v3/bulk_import/perform/name').
|
157
158
|
to_return(:body => {'job_id' => 12345}.to_json)
|
158
|
-
|
159
|
+
api.perform_bulk_import('name').should == '12345'
|
159
160
|
end
|
160
161
|
end
|
161
162
|
|
@@ -163,7 +164,7 @@ describe 'BulkImport API' do
|
|
163
164
|
it 'runs' do
|
164
165
|
stub_api_request(:post, '/v3/bulk_import/commit/name').
|
165
166
|
to_return(:body => {'job_id' => 12345}.to_json)
|
166
|
-
|
167
|
+
api.commit_bulk_import('name').should == nil
|
167
168
|
end
|
168
169
|
end
|
169
170
|
|
@@ -171,19 +172,19 @@ describe 'BulkImport API' do
|
|
171
172
|
it 'returns [] on empty' do
|
172
173
|
stub_api_request(:get, '/v3/bulk_import/error_records/name').
|
173
174
|
to_return(:body => '')
|
174
|
-
|
175
|
+
api.bulk_import_error_records('name').should == []
|
175
176
|
end
|
176
177
|
|
177
178
|
it 'returns nil on empty if block given' do
|
178
179
|
stub_api_request(:get, '/v3/bulk_import/error_records/name').
|
179
180
|
to_return(:body => '')
|
180
|
-
|
181
|
+
api.bulk_import_error_records('name'){}.should == nil
|
181
182
|
end
|
182
183
|
|
183
184
|
it 'returns unpacked result' do
|
184
185
|
stub_api_request(:get, '/v3/bulk_import/error_records/name').
|
185
186
|
to_return(:body => packed)
|
186
|
-
|
187
|
+
api.bulk_import_error_records('name').should == [[1, '2', 3.0], [4, '5', 6.0], [7, '8', 9.0]]
|
187
188
|
end
|
188
189
|
|
189
190
|
it 'yields unpacked result if block given' do
|
@@ -193,7 +194,7 @@ describe 'BulkImport API' do
|
|
193
194
|
api.bulk_import_error_records('name') do |row|
|
194
195
|
result << row
|
195
196
|
end
|
196
|
-
|
197
|
+
result.should == [[1, '2', 3.0], [4, '5', 6.0], [7, '8', 9.0]]
|
197
198
|
end
|
198
199
|
end
|
199
200
|
end
|
@@ -113,9 +113,9 @@ describe 'BulkImport API' do
|
|
113
113
|
stub_api_request(:post, '/v3/bulk_loads/guess').
|
114
114
|
with(:body => original_config.to_json).
|
115
115
|
to_return(:body => guessed_config.to_json)
|
116
|
-
|
116
|
+
api.bulk_load_guess(
|
117
117
|
original_config
|
118
|
-
)
|
118
|
+
).should == guessed_config
|
119
119
|
end
|
120
120
|
|
121
121
|
it 'raises an error' do
|
@@ -134,23 +134,23 @@ describe 'BulkImport API' do
|
|
134
134
|
with(:body => original_config.to_json).
|
135
135
|
to_return(:status => 500, :body => guessed_config.to_json)
|
136
136
|
begin
|
137
|
-
|
137
|
+
retry_api.bulk_load_guess(
|
138
138
|
original_config
|
139
|
-
)
|
139
|
+
).should != nil
|
140
140
|
rescue TreasureData::APIError => e
|
141
|
-
|
141
|
+
e.message.should =~ /^500: BulkLoad configuration guess failed/
|
142
142
|
end
|
143
143
|
end
|
144
144
|
|
145
145
|
it 'perform retries on connection failure' do
|
146
146
|
api = retry_api
|
147
|
-
|
147
|
+
api.instance_eval { @api }.stub(:post).and_raise(SocketError.new('>>'))
|
148
148
|
begin
|
149
149
|
retry_api.bulk_load_guess(
|
150
150
|
original_config
|
151
151
|
)
|
152
152
|
rescue SocketError => e
|
153
|
-
|
153
|
+
e.message.should == '>> (Retried 1 times in 1 seconds)'
|
154
154
|
end
|
155
155
|
end
|
156
156
|
end
|
@@ -160,9 +160,9 @@ describe 'BulkImport API' do
|
|
160
160
|
stub_api_request(:post, '/v3/bulk_loads/guess').
|
161
161
|
with(:body => original_config.to_json).
|
162
162
|
to_return(:body => guessed_config.to_json)
|
163
|
-
|
163
|
+
api.bulk_load_guess(
|
164
164
|
original_config
|
165
|
-
)
|
165
|
+
).should == guessed_config
|
166
166
|
end
|
167
167
|
|
168
168
|
it 'raises an error' do
|
@@ -181,23 +181,23 @@ describe 'BulkImport API' do
|
|
181
181
|
with(:body => original_config.to_json).
|
182
182
|
to_return(:status => 500, :body => guessed_config.to_json)
|
183
183
|
begin
|
184
|
-
|
184
|
+
retry_api.bulk_load_guess(
|
185
185
|
original_config
|
186
|
-
)
|
186
|
+
).should != nil
|
187
187
|
rescue TreasureData::APIError => e
|
188
|
-
|
188
|
+
e.message.should =~ /^500: BulkLoad configuration guess failed/
|
189
189
|
end
|
190
190
|
end
|
191
191
|
|
192
192
|
it 'perform retries on connection failure' do
|
193
193
|
api = retry_api
|
194
|
-
|
194
|
+
api.instance_eval { @api }.stub(:post).and_raise(SocketError.new('>>'))
|
195
195
|
begin
|
196
196
|
retry_api.bulk_load_guess(
|
197
197
|
original_config
|
198
198
|
)
|
199
199
|
rescue SocketError => e
|
200
|
-
|
200
|
+
e.message.should == '>> (Retried 1 times in 1 seconds)'
|
201
201
|
end
|
202
202
|
end
|
203
203
|
end
|
@@ -207,9 +207,9 @@ describe 'BulkImport API' do
|
|
207
207
|
stub_api_request(:post, '/v3/bulk_loads/preview').
|
208
208
|
with(:body => guessed_config.to_json).
|
209
209
|
to_return(:body => preview_result.to_json)
|
210
|
-
|
210
|
+
api.bulk_load_preview(
|
211
211
|
guessed_config
|
212
|
-
)
|
212
|
+
).should == preview_result
|
213
213
|
end
|
214
214
|
|
215
215
|
it 'raises an error' do
|
@@ -232,11 +232,11 @@ describe 'BulkImport API' do
|
|
232
232
|
stub_api_request(:post, '/v3/job/issue/bulkload/database').
|
233
233
|
with(:body => expected_request.to_json).
|
234
234
|
to_return(:body => {'job_id' => 12345}.to_json)
|
235
|
-
|
235
|
+
api.bulk_load_issue(
|
236
236
|
'database',
|
237
237
|
'table',
|
238
238
|
guessed_config
|
239
|
-
)
|
239
|
+
).should == '12345'
|
240
240
|
end
|
241
241
|
end
|
242
242
|
|
@@ -245,14 +245,14 @@ describe 'BulkImport API' do
|
|
245
245
|
stub_api_request(:get, '/v3/bulk_loads').
|
246
246
|
to_return(:body => [bulk_load_session, bulk_load_session].to_json)
|
247
247
|
result = api.bulk_load_list
|
248
|
-
|
249
|
-
|
248
|
+
result.size.should == 2
|
249
|
+
result.first.should == bulk_load_session
|
250
250
|
end
|
251
251
|
|
252
252
|
it 'returns empty' do
|
253
253
|
stub_api_request(:get, '/v3/bulk_loads').
|
254
254
|
to_return(:body => [].to_json)
|
255
|
-
|
255
|
+
api.bulk_load_list.size.should == 0
|
256
256
|
end
|
257
257
|
end
|
258
258
|
|
@@ -268,7 +268,7 @@ describe 'BulkImport API' do
|
|
268
268
|
stub_api_request(:post, '/v3/bulk_loads').
|
269
269
|
with(:body => expected_request.to_json).
|
270
270
|
to_return(:body => bulk_load_session.to_json)
|
271
|
-
|
271
|
+
api.bulk_load_create(
|
272
272
|
'nahi_test_1',
|
273
273
|
'database',
|
274
274
|
'table',
|
@@ -278,7 +278,7 @@ describe 'BulkImport API' do
|
|
278
278
|
timezone: 'Asia/Tokyo',
|
279
279
|
delay: 3600
|
280
280
|
}
|
281
|
-
)
|
281
|
+
).should == bulk_load_session
|
282
282
|
end
|
283
283
|
|
284
284
|
it 'accepts empty option' do
|
@@ -289,12 +289,12 @@ describe 'BulkImport API' do
|
|
289
289
|
stub_api_request(:post, '/v3/bulk_loads').
|
290
290
|
with(:body => expected_request.to_json).
|
291
291
|
to_return(:body => bulk_load_session.to_json)
|
292
|
-
|
292
|
+
api.bulk_load_create(
|
293
293
|
'nahi_test_1',
|
294
294
|
'database',
|
295
295
|
'table',
|
296
296
|
guessed_config
|
297
|
-
)
|
297
|
+
).should == bulk_load_session
|
298
298
|
end
|
299
299
|
|
300
300
|
it 'accepts time_column option' do
|
@@ -306,7 +306,7 @@ describe 'BulkImport API' do
|
|
306
306
|
stub_api_request(:post, '/v3/bulk_loads').
|
307
307
|
with(:body => expected_request.to_json).
|
308
308
|
to_return(:body => bulk_load_session.to_json)
|
309
|
-
|
309
|
+
api.bulk_load_create(
|
310
310
|
'nahi_test_1',
|
311
311
|
'database',
|
312
312
|
'table',
|
@@ -314,7 +314,7 @@ describe 'BulkImport API' do
|
|
314
314
|
{
|
315
315
|
time_column: 'c0'
|
316
316
|
}
|
317
|
-
)
|
317
|
+
).should == bulk_load_session
|
318
318
|
end
|
319
319
|
end
|
320
320
|
|
@@ -322,7 +322,7 @@ describe 'BulkImport API' do
|
|
322
322
|
it 'returns bulk_load_session' do
|
323
323
|
stub_api_request(:get, '/v3/bulk_loads/nahi_test_1').
|
324
324
|
to_return(:body => bulk_load_session.to_json)
|
325
|
-
|
325
|
+
api.bulk_load_show('nahi_test_1').should == bulk_load_session
|
326
326
|
end
|
327
327
|
end
|
328
328
|
|
@@ -331,35 +331,10 @@ describe 'BulkImport API' do
|
|
331
331
|
stub_api_request(:put, '/v3/bulk_loads/nahi_test_1').
|
332
332
|
with(:body => bulk_load_session.to_json).
|
333
333
|
to_return(:body => bulk_load_session.to_json)
|
334
|
-
|
334
|
+
api.bulk_load_update(
|
335
335
|
'nahi_test_1',
|
336
336
|
bulk_load_session
|
337
|
-
)
|
338
|
-
end
|
339
|
-
|
340
|
-
it 'returns updated bulk_load_session' do
|
341
|
-
updated_bulk_load_session = bulk_load_session.merge({'timezone' => 'America/Los Angeles'})
|
342
|
-
stub_api_request(:put, '/v3/bulk_loads/nahi_test_1').
|
343
|
-
with(:body => updated_bulk_load_session.to_json).
|
344
|
-
to_return(:body => updated_bulk_load_session.to_json)
|
345
|
-
expect(api.bulk_load_update(
|
346
|
-
'nahi_test_1',
|
347
|
-
updated_bulk_load_session
|
348
|
-
)).to eq updated_bulk_load_session
|
349
|
-
end
|
350
|
-
|
351
|
-
it 'can remove the cron schedule ' do
|
352
|
-
updated_bulk_load_session = bulk_load_session.merge({'cron' => ''})
|
353
|
-
# NOTE: currently the API just ignores an empty 'cron' specification update
|
354
|
-
# I am assuming that once fixed, the API will return a nil for cron if unscheduled.
|
355
|
-
expected_bulk_load_session = (bulk_load_session['cron'] = nil)
|
356
|
-
stub_api_request(:put, '/v3/bulk_loads/nahi_test_1').
|
357
|
-
with(:body => updated_bulk_load_session.to_json).
|
358
|
-
to_return(:body => expected_bulk_load_session.to_json)
|
359
|
-
expect(api.bulk_load_update(
|
360
|
-
'nahi_test_1',
|
361
|
-
updated_bulk_load_session
|
362
|
-
)).to eq expected_bulk_load_session
|
337
|
+
).should == bulk_load_session
|
363
338
|
end
|
364
339
|
end
|
365
340
|
|
@@ -367,7 +342,7 @@ describe 'BulkImport API' do
|
|
367
342
|
it 'returns updated bulk_load_session' do
|
368
343
|
stub_api_request(:delete, '/v3/bulk_loads/nahi_test_1').
|
369
344
|
to_return(:body => bulk_load_session.to_json)
|
370
|
-
|
345
|
+
api.bulk_load_delete('nahi_test_1').should == bulk_load_session
|
371
346
|
end
|
372
347
|
end
|
373
348
|
|
@@ -376,8 +351,8 @@ describe 'BulkImport API' do
|
|
376
351
|
stub_api_request(:get, '/v3/bulk_loads/nahi_test_1/jobs').
|
377
352
|
to_return(:body => [bulk_load_job, bulk_load_job].to_json)
|
378
353
|
result = api.bulk_load_history('nahi_test_1')
|
379
|
-
|
380
|
-
|
354
|
+
result.size.should == 2
|
355
|
+
result.first.should == bulk_load_job
|
381
356
|
end
|
382
357
|
end
|
383
358
|
|
@@ -386,7 +361,7 @@ describe 'BulkImport API' do
|
|
386
361
|
stub_api_request(:post, '/v3/bulk_loads/nahi_test_1/jobs').
|
387
362
|
with(:body => '{}').
|
388
363
|
to_return(:body => {'job_id' => 12345}.to_json)
|
389
|
-
|
364
|
+
api.bulk_load_run('nahi_test_1').should == '12345'
|
390
365
|
end
|
391
366
|
|
392
367
|
it 'accepts scheduled_time' do
|
@@ -394,7 +369,7 @@ describe 'BulkImport API' do
|
|
394
369
|
stub_api_request(:post, '/v3/bulk_loads/nahi_test_1/jobs').
|
395
370
|
with(:body => {scheduled_time: now.to_s}.to_json).
|
396
371
|
to_return(:body => {'job_id' => 12345}.to_json)
|
397
|
-
|
372
|
+
api.bulk_load_run('nahi_test_1', now).should == '12345'
|
398
373
|
end
|
399
374
|
end
|
400
375
|
|
@@ -18,7 +18,7 @@ describe 'Database API' do
|
|
18
18
|
stub_api_request(:post, "/v3/database/create/#{e(db_name)}").
|
19
19
|
to_return(:body => {'database' => db_name}.to_json)
|
20
20
|
|
21
|
-
|
21
|
+
api.create_database(db_name).should be true
|
22
22
|
end
|
23
23
|
|
24
24
|
it 'should return 400 error with invalid name' do
|
@@ -20,7 +20,7 @@ describe 'Export API' do
|
|
20
20
|
stub_api_request(:post, "/v3/export/run/#{e(db_name)}/#{e(table_name)}").with(:body => params.merge('storage_type' => storage_type)).
|
21
21
|
to_return(:body => {'database' => db_name, 'job_id' => '1', 'debug' => {}}.to_json)
|
22
22
|
|
23
|
-
|
23
|
+
api.export(db_name, table_name, storage_type, params).should == '1'
|
24
24
|
end
|
25
25
|
|
26
26
|
it 'should return 400 error with invalid storage type' do
|
@@ -37,15 +37,5 @@ describe 'Export API' do
|
|
37
37
|
|
38
38
|
# TODO: Add other parameters spec
|
39
39
|
end
|
40
|
-
|
41
|
-
describe 'result_export' do
|
42
|
-
it 'should export result successfully' do
|
43
|
-
params = {'result' => 'mysql://user:pass@host.com/database/table'}
|
44
|
-
stub_api_request(:post, "/v3/job/result_export/100").with(:body => params).
|
45
|
-
to_return(:body => {'job_id' => '101'}.to_json)
|
46
|
-
|
47
|
-
expect(api.result_export(100, params)).to eq('101')
|
48
|
-
end
|
49
|
-
end
|
50
40
|
end
|
51
41
|
|