td-client 0.8.85 → 0.9.0dev1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/td/client.rb +8 -16
- data/lib/td/client/api.rb +46 -62
- data/lib/td/client/api/bulk_import.rb +2 -1
- data/lib/td/client/api/bulk_load.rb +3 -3
- data/lib/td/client/api/export.rb +0 -12
- data/lib/td/client/api/import.rb +2 -3
- data/lib/td/client/api/job.rb +71 -145
- data/lib/td/client/api/schedule.rb +1 -1
- data/lib/td/client/api_error.rb +0 -5
- data/lib/td/client/model.rb +28 -91
- data/lib/td/client/version.rb +1 -1
- data/spec/spec_helper.rb +5 -5
- data/spec/td/client/account_api_spec.rb +5 -5
- data/spec/td/client/api_spec.rb +51 -69
- data/spec/td/client/api_ssl_connection_spec.rb +1 -1
- data/spec/td/client/bulk_import_spec.rb +29 -28
- data/spec/td/client/bulk_load_spec.rb +35 -60
- data/spec/td/client/db_api_spec.rb +1 -1
- data/spec/td/client/export_api_spec.rb +1 -11
- data/spec/td/client/import_api_spec.rb +10 -85
- data/spec/td/client/job_api_spec.rb +61 -567
- data/spec/td/client/model_job_spec.rb +10 -27
- data/spec/td/client/model_schedule_spec.rb +2 -2
- data/spec/td/client/partial_delete_api_spec.rb +1 -1
- data/spec/td/client/result_api_spec.rb +3 -3
- data/spec/td/client/sched_api_spec.rb +4 -12
- data/spec/td/client/server_status_api_spec.rb +2 -2
- data/spec/td/client/spec_resources.rb +0 -1
- data/spec/td/client/table_api_spec.rb +14 -14
- data/spec/td/client/user_api_spec.rb +12 -12
- data/spec/td/client_sched_spec.rb +6 -31
- data/spec/td/client_spec.rb +0 -1
- metadata +97 -42
- data/spec/td/client/api_error_spec.rb +0 -77
- data/spec/td/client/model_schema_spec.rb +0 -134
@@ -14,24 +14,25 @@ describe 'BulkImport API' do
|
|
14
14
|
|
15
15
|
let :packed do
|
16
16
|
s = StringIO.new
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
17
|
+
pk = MessagePack::Packer.new(s)
|
18
|
+
pk.write([1, '2', 3.0])
|
19
|
+
pk.write([4, '5', 6.0])
|
20
|
+
pk.write([7, '8', 9.0])
|
21
|
+
pk.flush
|
22
|
+
s.rewind
|
23
|
+
out = StringIO.new
|
24
|
+
Zlib::GzipWriter.wrap(out) do |f|
|
25
|
+
f.write s.read
|
26
|
+
end
|
27
|
+
out.string
|
25
28
|
end
|
26
29
|
|
27
|
-
let(:endpoint_domain) { TreasureData::API::DEFAULT_IMPORT_ENDPOINT }
|
28
|
-
|
29
30
|
describe 'create_bulk_import' do
|
30
31
|
it 'should create a new bulk_import' do
|
31
32
|
stub_api_request(:post, "/v3/bulk_import/create/#{e(bi_name)}/#{e(db_name)}/#{e(table_name)}").
|
32
33
|
to_return(:body => {'bulk_import' => bi_name}.to_json)
|
33
34
|
|
34
|
-
|
35
|
+
api.create_bulk_import(bi_name, db_name, table_name).should be_nil
|
35
36
|
end
|
36
37
|
|
37
38
|
it 'should return 422 error with invalid name' do
|
@@ -72,7 +73,7 @@ describe 'BulkImport API' do
|
|
72
73
|
it 'runs' do
|
73
74
|
stub_api_request(:post, '/v3/bulk_import/delete/name').
|
74
75
|
with(:body => 'foo=bar')
|
75
|
-
|
76
|
+
api.delete_bulk_import('name', 'foo' => 'bar').should == nil
|
76
77
|
end
|
77
78
|
end
|
78
79
|
|
@@ -80,7 +81,7 @@ describe 'BulkImport API' do
|
|
80
81
|
it 'runs' do
|
81
82
|
stub_api_request(:get, '/v3/bulk_import/show/name').
|
82
83
|
to_return(:body => {'status' => 'status', 'other' => 'other'}.to_json)
|
83
|
-
|
84
|
+
api.show_bulk_import('name')['status'].should == 'status'
|
84
85
|
end
|
85
86
|
end
|
86
87
|
|
@@ -89,7 +90,7 @@ describe 'BulkImport API' do
|
|
89
90
|
stub_api_request(:get, '/v3/bulk_import/list').
|
90
91
|
with(:query => 'foo=bar').
|
91
92
|
to_return(:body => {'bulk_imports' => %w(1 2 3)}.to_json)
|
92
|
-
|
93
|
+
api.list_bulk_imports('foo' => 'bar').should == %w(1 2 3)
|
93
94
|
end
|
94
95
|
end
|
95
96
|
|
@@ -98,7 +99,7 @@ describe 'BulkImport API' do
|
|
98
99
|
stub_api_request(:get, '/v3/bulk_import/list_parts/name').
|
99
100
|
with(:query => 'foo=bar').
|
100
101
|
to_return(:body => {'parts' => %w(1 2 3)}.to_json)
|
101
|
-
|
102
|
+
api.list_bulk_import_parts('name', 'foo' => 'bar').should == %w(1 2 3)
|
102
103
|
end
|
103
104
|
end
|
104
105
|
|
@@ -108,10 +109,10 @@ describe 'BulkImport API' do
|
|
108
109
|
File.open(t.path, 'w') do |f|
|
109
110
|
f << '12345'
|
110
111
|
end
|
111
|
-
stub_request(:put,
|
112
|
+
stub_request(:put, 'http://api.treasure-data.com/v3/bulk_import/upload_part/name/part').
|
112
113
|
with(:body => '12345')
|
113
114
|
File.open(t.path) do |f|
|
114
|
-
|
115
|
+
api.bulk_import_upload_part('name', 'part', f, 5).should == nil
|
115
116
|
end
|
116
117
|
end
|
117
118
|
|
@@ -121,10 +122,10 @@ describe 'BulkImport API' do
|
|
121
122
|
File.open(t.path, 'w') do |f|
|
122
123
|
f << '12345'
|
123
124
|
end
|
124
|
-
stub_request(:put,
|
125
|
+
stub_request(:put, 'http://api.treasure-data.com/v3/bulk_import/upload_part/name/' + CGI.escape('日本語(Japanese)'.encode('UTF-8'))).
|
125
126
|
with(:body => '12345')
|
126
127
|
File.open(t.path) do |f|
|
127
|
-
|
128
|
+
api.bulk_import_upload_part('name', '日本語(Japanese)'.encode('Windows-31J'), f, 5).should == nil
|
128
129
|
end
|
129
130
|
end
|
130
131
|
end
|
@@ -133,21 +134,21 @@ describe 'BulkImport API' do
|
|
133
134
|
describe 'bulk_import_delete_part' do
|
134
135
|
it 'runs' do
|
135
136
|
stub_api_request(:post, '/v3/bulk_import/delete_part/name/part')
|
136
|
-
|
137
|
+
api.bulk_import_delete_part('name', 'part').should == nil
|
137
138
|
end
|
138
139
|
end
|
139
140
|
|
140
141
|
describe 'freeze_bulk_import' do
|
141
142
|
it 'runs' do
|
142
143
|
stub_api_request(:post, '/v3/bulk_import/freeze/name')
|
143
|
-
|
144
|
+
api.freeze_bulk_import('name').should == nil
|
144
145
|
end
|
145
146
|
end
|
146
147
|
|
147
148
|
describe 'unfreeze_bulk_import' do
|
148
149
|
it 'runs' do
|
149
150
|
stub_api_request(:post, '/v3/bulk_import/unfreeze/name')
|
150
|
-
|
151
|
+
api.unfreeze_bulk_import('name').should == nil
|
151
152
|
end
|
152
153
|
end
|
153
154
|
|
@@ -155,7 +156,7 @@ describe 'BulkImport API' do
|
|
155
156
|
it 'runs' do
|
156
157
|
stub_api_request(:post, '/v3/bulk_import/perform/name').
|
157
158
|
to_return(:body => {'job_id' => 12345}.to_json)
|
158
|
-
|
159
|
+
api.perform_bulk_import('name').should == '12345'
|
159
160
|
end
|
160
161
|
end
|
161
162
|
|
@@ -163,7 +164,7 @@ describe 'BulkImport API' do
|
|
163
164
|
it 'runs' do
|
164
165
|
stub_api_request(:post, '/v3/bulk_import/commit/name').
|
165
166
|
to_return(:body => {'job_id' => 12345}.to_json)
|
166
|
-
|
167
|
+
api.commit_bulk_import('name').should == nil
|
167
168
|
end
|
168
169
|
end
|
169
170
|
|
@@ -171,19 +172,19 @@ describe 'BulkImport API' do
|
|
171
172
|
it 'returns [] on empty' do
|
172
173
|
stub_api_request(:get, '/v3/bulk_import/error_records/name').
|
173
174
|
to_return(:body => '')
|
174
|
-
|
175
|
+
api.bulk_import_error_records('name').should == []
|
175
176
|
end
|
176
177
|
|
177
178
|
it 'returns nil on empty if block given' do
|
178
179
|
stub_api_request(:get, '/v3/bulk_import/error_records/name').
|
179
180
|
to_return(:body => '')
|
180
|
-
|
181
|
+
api.bulk_import_error_records('name'){}.should == nil
|
181
182
|
end
|
182
183
|
|
183
184
|
it 'returns unpacked result' do
|
184
185
|
stub_api_request(:get, '/v3/bulk_import/error_records/name').
|
185
186
|
to_return(:body => packed)
|
186
|
-
|
187
|
+
api.bulk_import_error_records('name').should == [[1, '2', 3.0], [4, '5', 6.0], [7, '8', 9.0]]
|
187
188
|
end
|
188
189
|
|
189
190
|
it 'yields unpacked result if block given' do
|
@@ -193,7 +194,7 @@ describe 'BulkImport API' do
|
|
193
194
|
api.bulk_import_error_records('name') do |row|
|
194
195
|
result << row
|
195
196
|
end
|
196
|
-
|
197
|
+
result.should == [[1, '2', 3.0], [4, '5', 6.0], [7, '8', 9.0]]
|
197
198
|
end
|
198
199
|
end
|
199
200
|
end
|
@@ -113,9 +113,9 @@ describe 'BulkImport API' do
|
|
113
113
|
stub_api_request(:post, '/v3/bulk_loads/guess').
|
114
114
|
with(:body => original_config.to_json).
|
115
115
|
to_return(:body => guessed_config.to_json)
|
116
|
-
|
116
|
+
api.bulk_load_guess(
|
117
117
|
original_config
|
118
|
-
)
|
118
|
+
).should == guessed_config
|
119
119
|
end
|
120
120
|
|
121
121
|
it 'raises an error' do
|
@@ -134,23 +134,23 @@ describe 'BulkImport API' do
|
|
134
134
|
with(:body => original_config.to_json).
|
135
135
|
to_return(:status => 500, :body => guessed_config.to_json)
|
136
136
|
begin
|
137
|
-
|
137
|
+
retry_api.bulk_load_guess(
|
138
138
|
original_config
|
139
|
-
)
|
139
|
+
).should != nil
|
140
140
|
rescue TreasureData::APIError => e
|
141
|
-
|
141
|
+
e.message.should =~ /^500: BulkLoad configuration guess failed/
|
142
142
|
end
|
143
143
|
end
|
144
144
|
|
145
145
|
it 'perform retries on connection failure' do
|
146
146
|
api = retry_api
|
147
|
-
|
147
|
+
api.instance_eval { @api }.stub(:post).and_raise(SocketError.new('>>'))
|
148
148
|
begin
|
149
149
|
retry_api.bulk_load_guess(
|
150
150
|
original_config
|
151
151
|
)
|
152
152
|
rescue SocketError => e
|
153
|
-
|
153
|
+
e.message.should == '>> (Retried 1 times in 1 seconds)'
|
154
154
|
end
|
155
155
|
end
|
156
156
|
end
|
@@ -160,9 +160,9 @@ describe 'BulkImport API' do
|
|
160
160
|
stub_api_request(:post, '/v3/bulk_loads/guess').
|
161
161
|
with(:body => original_config.to_json).
|
162
162
|
to_return(:body => guessed_config.to_json)
|
163
|
-
|
163
|
+
api.bulk_load_guess(
|
164
164
|
original_config
|
165
|
-
)
|
165
|
+
).should == guessed_config
|
166
166
|
end
|
167
167
|
|
168
168
|
it 'raises an error' do
|
@@ -181,23 +181,23 @@ describe 'BulkImport API' do
|
|
181
181
|
with(:body => original_config.to_json).
|
182
182
|
to_return(:status => 500, :body => guessed_config.to_json)
|
183
183
|
begin
|
184
|
-
|
184
|
+
retry_api.bulk_load_guess(
|
185
185
|
original_config
|
186
|
-
)
|
186
|
+
).should != nil
|
187
187
|
rescue TreasureData::APIError => e
|
188
|
-
|
188
|
+
e.message.should =~ /^500: BulkLoad configuration guess failed/
|
189
189
|
end
|
190
190
|
end
|
191
191
|
|
192
192
|
it 'perform retries on connection failure' do
|
193
193
|
api = retry_api
|
194
|
-
|
194
|
+
api.instance_eval { @api }.stub(:post).and_raise(SocketError.new('>>'))
|
195
195
|
begin
|
196
196
|
retry_api.bulk_load_guess(
|
197
197
|
original_config
|
198
198
|
)
|
199
199
|
rescue SocketError => e
|
200
|
-
|
200
|
+
e.message.should == '>> (Retried 1 times in 1 seconds)'
|
201
201
|
end
|
202
202
|
end
|
203
203
|
end
|
@@ -207,9 +207,9 @@ describe 'BulkImport API' do
|
|
207
207
|
stub_api_request(:post, '/v3/bulk_loads/preview').
|
208
208
|
with(:body => guessed_config.to_json).
|
209
209
|
to_return(:body => preview_result.to_json)
|
210
|
-
|
210
|
+
api.bulk_load_preview(
|
211
211
|
guessed_config
|
212
|
-
)
|
212
|
+
).should == preview_result
|
213
213
|
end
|
214
214
|
|
215
215
|
it 'raises an error' do
|
@@ -232,11 +232,11 @@ describe 'BulkImport API' do
|
|
232
232
|
stub_api_request(:post, '/v3/job/issue/bulkload/database').
|
233
233
|
with(:body => expected_request.to_json).
|
234
234
|
to_return(:body => {'job_id' => 12345}.to_json)
|
235
|
-
|
235
|
+
api.bulk_load_issue(
|
236
236
|
'database',
|
237
237
|
'table',
|
238
238
|
guessed_config
|
239
|
-
)
|
239
|
+
).should == '12345'
|
240
240
|
end
|
241
241
|
end
|
242
242
|
|
@@ -245,14 +245,14 @@ describe 'BulkImport API' do
|
|
245
245
|
stub_api_request(:get, '/v3/bulk_loads').
|
246
246
|
to_return(:body => [bulk_load_session, bulk_load_session].to_json)
|
247
247
|
result = api.bulk_load_list
|
248
|
-
|
249
|
-
|
248
|
+
result.size.should == 2
|
249
|
+
result.first.should == bulk_load_session
|
250
250
|
end
|
251
251
|
|
252
252
|
it 'returns empty' do
|
253
253
|
stub_api_request(:get, '/v3/bulk_loads').
|
254
254
|
to_return(:body => [].to_json)
|
255
|
-
|
255
|
+
api.bulk_load_list.size.should == 0
|
256
256
|
end
|
257
257
|
end
|
258
258
|
|
@@ -268,7 +268,7 @@ describe 'BulkImport API' do
|
|
268
268
|
stub_api_request(:post, '/v3/bulk_loads').
|
269
269
|
with(:body => expected_request.to_json).
|
270
270
|
to_return(:body => bulk_load_session.to_json)
|
271
|
-
|
271
|
+
api.bulk_load_create(
|
272
272
|
'nahi_test_1',
|
273
273
|
'database',
|
274
274
|
'table',
|
@@ -278,7 +278,7 @@ describe 'BulkImport API' do
|
|
278
278
|
timezone: 'Asia/Tokyo',
|
279
279
|
delay: 3600
|
280
280
|
}
|
281
|
-
)
|
281
|
+
).should == bulk_load_session
|
282
282
|
end
|
283
283
|
|
284
284
|
it 'accepts empty option' do
|
@@ -289,12 +289,12 @@ describe 'BulkImport API' do
|
|
289
289
|
stub_api_request(:post, '/v3/bulk_loads').
|
290
290
|
with(:body => expected_request.to_json).
|
291
291
|
to_return(:body => bulk_load_session.to_json)
|
292
|
-
|
292
|
+
api.bulk_load_create(
|
293
293
|
'nahi_test_1',
|
294
294
|
'database',
|
295
295
|
'table',
|
296
296
|
guessed_config
|
297
|
-
)
|
297
|
+
).should == bulk_load_session
|
298
298
|
end
|
299
299
|
|
300
300
|
it 'accepts time_column option' do
|
@@ -306,7 +306,7 @@ describe 'BulkImport API' do
|
|
306
306
|
stub_api_request(:post, '/v3/bulk_loads').
|
307
307
|
with(:body => expected_request.to_json).
|
308
308
|
to_return(:body => bulk_load_session.to_json)
|
309
|
-
|
309
|
+
api.bulk_load_create(
|
310
310
|
'nahi_test_1',
|
311
311
|
'database',
|
312
312
|
'table',
|
@@ -314,7 +314,7 @@ describe 'BulkImport API' do
|
|
314
314
|
{
|
315
315
|
time_column: 'c0'
|
316
316
|
}
|
317
|
-
)
|
317
|
+
).should == bulk_load_session
|
318
318
|
end
|
319
319
|
end
|
320
320
|
|
@@ -322,7 +322,7 @@ describe 'BulkImport API' do
|
|
322
322
|
it 'returns bulk_load_session' do
|
323
323
|
stub_api_request(:get, '/v3/bulk_loads/nahi_test_1').
|
324
324
|
to_return(:body => bulk_load_session.to_json)
|
325
|
-
|
325
|
+
api.bulk_load_show('nahi_test_1').should == bulk_load_session
|
326
326
|
end
|
327
327
|
end
|
328
328
|
|
@@ -331,35 +331,10 @@ describe 'BulkImport API' do
|
|
331
331
|
stub_api_request(:put, '/v3/bulk_loads/nahi_test_1').
|
332
332
|
with(:body => bulk_load_session.to_json).
|
333
333
|
to_return(:body => bulk_load_session.to_json)
|
334
|
-
|
334
|
+
api.bulk_load_update(
|
335
335
|
'nahi_test_1',
|
336
336
|
bulk_load_session
|
337
|
-
)
|
338
|
-
end
|
339
|
-
|
340
|
-
it 'returns updated bulk_load_session' do
|
341
|
-
updated_bulk_load_session = bulk_load_session.merge({'timezone' => 'America/Los Angeles'})
|
342
|
-
stub_api_request(:put, '/v3/bulk_loads/nahi_test_1').
|
343
|
-
with(:body => updated_bulk_load_session.to_json).
|
344
|
-
to_return(:body => updated_bulk_load_session.to_json)
|
345
|
-
expect(api.bulk_load_update(
|
346
|
-
'nahi_test_1',
|
347
|
-
updated_bulk_load_session
|
348
|
-
)).to eq updated_bulk_load_session
|
349
|
-
end
|
350
|
-
|
351
|
-
it 'can remove the cron schedule ' do
|
352
|
-
updated_bulk_load_session = bulk_load_session.merge({'cron' => ''})
|
353
|
-
# NOTE: currently the API just ignores an empty 'cron' specification update
|
354
|
-
# I am assuming that once fixed, the API will return a nil for cron if unscheduled.
|
355
|
-
expected_bulk_load_session = (bulk_load_session['cron'] = nil)
|
356
|
-
stub_api_request(:put, '/v3/bulk_loads/nahi_test_1').
|
357
|
-
with(:body => updated_bulk_load_session.to_json).
|
358
|
-
to_return(:body => expected_bulk_load_session.to_json)
|
359
|
-
expect(api.bulk_load_update(
|
360
|
-
'nahi_test_1',
|
361
|
-
updated_bulk_load_session
|
362
|
-
)).to eq expected_bulk_load_session
|
337
|
+
).should == bulk_load_session
|
363
338
|
end
|
364
339
|
end
|
365
340
|
|
@@ -367,7 +342,7 @@ describe 'BulkImport API' do
|
|
367
342
|
it 'returns updated bulk_load_session' do
|
368
343
|
stub_api_request(:delete, '/v3/bulk_loads/nahi_test_1').
|
369
344
|
to_return(:body => bulk_load_session.to_json)
|
370
|
-
|
345
|
+
api.bulk_load_delete('nahi_test_1').should == bulk_load_session
|
371
346
|
end
|
372
347
|
end
|
373
348
|
|
@@ -376,8 +351,8 @@ describe 'BulkImport API' do
|
|
376
351
|
stub_api_request(:get, '/v3/bulk_loads/nahi_test_1/jobs').
|
377
352
|
to_return(:body => [bulk_load_job, bulk_load_job].to_json)
|
378
353
|
result = api.bulk_load_history('nahi_test_1')
|
379
|
-
|
380
|
-
|
354
|
+
result.size.should == 2
|
355
|
+
result.first.should == bulk_load_job
|
381
356
|
end
|
382
357
|
end
|
383
358
|
|
@@ -386,7 +361,7 @@ describe 'BulkImport API' do
|
|
386
361
|
stub_api_request(:post, '/v3/bulk_loads/nahi_test_1/jobs').
|
387
362
|
with(:body => '{}').
|
388
363
|
to_return(:body => {'job_id' => 12345}.to_json)
|
389
|
-
|
364
|
+
api.bulk_load_run('nahi_test_1').should == '12345'
|
390
365
|
end
|
391
366
|
|
392
367
|
it 'accepts scheduled_time' do
|
@@ -394,7 +369,7 @@ describe 'BulkImport API' do
|
|
394
369
|
stub_api_request(:post, '/v3/bulk_loads/nahi_test_1/jobs').
|
395
370
|
with(:body => {scheduled_time: now.to_s}.to_json).
|
396
371
|
to_return(:body => {'job_id' => 12345}.to_json)
|
397
|
-
|
372
|
+
api.bulk_load_run('nahi_test_1', now).should == '12345'
|
398
373
|
end
|
399
374
|
end
|
400
375
|
|
@@ -18,7 +18,7 @@ describe 'Database API' do
|
|
18
18
|
stub_api_request(:post, "/v3/database/create/#{e(db_name)}").
|
19
19
|
to_return(:body => {'database' => db_name}.to_json)
|
20
20
|
|
21
|
-
|
21
|
+
api.create_database(db_name).should be true
|
22
22
|
end
|
23
23
|
|
24
24
|
it 'should return 400 error with invalid name' do
|
@@ -20,7 +20,7 @@ describe 'Export API' do
|
|
20
20
|
stub_api_request(:post, "/v3/export/run/#{e(db_name)}/#{e(table_name)}").with(:body => params.merge('storage_type' => storage_type)).
|
21
21
|
to_return(:body => {'database' => db_name, 'job_id' => '1', 'debug' => {}}.to_json)
|
22
22
|
|
23
|
-
|
23
|
+
api.export(db_name, table_name, storage_type, params).should == '1'
|
24
24
|
end
|
25
25
|
|
26
26
|
it 'should return 400 error with invalid storage type' do
|
@@ -37,15 +37,5 @@ describe 'Export API' do
|
|
37
37
|
|
38
38
|
# TODO: Add other parameters spec
|
39
39
|
end
|
40
|
-
|
41
|
-
describe 'result_export' do
|
42
|
-
it 'should export result successfully' do
|
43
|
-
params = {'result' => 'mysql://user:pass@host.com/database/table'}
|
44
|
-
stub_api_request(:post, "/v3/job/result_export/100").with(:body => params).
|
45
|
-
to_return(:body => {'job_id' => '101'}.to_json)
|
46
|
-
|
47
|
-
expect(api.result_export(100, params)).to eq('101')
|
48
|
-
end
|
49
|
-
end
|
50
40
|
end
|
51
41
|
|