td-client 1.0.0-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/data/ca-bundle.crt +3448 -0
- data/lib/td-client.rb +1 -0
- data/lib/td/client.rb +606 -0
- data/lib/td/client/api.rb +707 -0
- data/lib/td/client/api/access_control.rb +74 -0
- data/lib/td/client/api/account.rb +45 -0
- data/lib/td/client/api/bulk_import.rb +184 -0
- data/lib/td/client/api/bulk_load.rb +172 -0
- data/lib/td/client/api/database.rb +50 -0
- data/lib/td/client/api/export.rb +38 -0
- data/lib/td/client/api/import.rb +38 -0
- data/lib/td/client/api/job.rb +390 -0
- data/lib/td/client/api/partial_delete.rb +27 -0
- data/lib/td/client/api/result.rb +46 -0
- data/lib/td/client/api/schedule.rb +120 -0
- data/lib/td/client/api/server_status.rb +21 -0
- data/lib/td/client/api/table.rb +132 -0
- data/lib/td/client/api/user.rb +134 -0
- data/lib/td/client/api_error.rb +37 -0
- data/lib/td/client/compat_gzip_reader.rb +22 -0
- data/lib/td/client/model.rb +816 -0
- data/lib/td/client/version.rb +5 -0
- data/lib/td/core_ext/openssl/ssl/sslcontext/set_params.rb +18 -0
- data/spec/spec_helper.rb +63 -0
- data/spec/td/client/access_control_api_spec.rb +37 -0
- data/spec/td/client/account_api_spec.rb +34 -0
- data/spec/td/client/api_error_spec.rb +77 -0
- data/spec/td/client/api_spec.rb +269 -0
- data/spec/td/client/api_ssl_connection_spec.rb +109 -0
- data/spec/td/client/bulk_import_spec.rb +199 -0
- data/spec/td/client/bulk_load_spec.rb +401 -0
- data/spec/td/client/db_api_spec.rb +123 -0
- data/spec/td/client/export_api_spec.rb +51 -0
- data/spec/td/client/import_api_spec.rb +148 -0
- data/spec/td/client/job_api_spec.rb +833 -0
- data/spec/td/client/model_job_spec.rb +136 -0
- data/spec/td/client/model_schedule_spec.rb +26 -0
- data/spec/td/client/model_schema_spec.rb +134 -0
- data/spec/td/client/partial_delete_api_spec.rb +58 -0
- data/spec/td/client/result_api_spec.rb +77 -0
- data/spec/td/client/sched_api_spec.rb +109 -0
- data/spec/td/client/server_status_api_spec.rb +25 -0
- data/spec/td/client/spec_resources.rb +99 -0
- data/spec/td/client/table_api_spec.rb +226 -0
- data/spec/td/client/user_api_spec.rb +118 -0
- data/spec/td/client_sched_spec.rb +79 -0
- data/spec/td/client_spec.rb +46 -0
- metadata +271 -0
@@ -0,0 +1,109 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'td/client/api'
|
3
|
+
require 'logger'
|
4
|
+
require 'webrick'
|
5
|
+
require 'webrick/https'
|
6
|
+
|
7
|
+
# Workaround for https://github.com/jruby/jruby-openssl/issues/78
|
8
|
+
# With recent JRuby + jruby-openssl, X509CRL#extentions_to_text causes
|
9
|
+
# StringIndexOOBException when we try to dump SSL Server Certificate.
|
10
|
+
# when one of extensions has "" as value.
|
11
|
+
# This hack is from httpclient https://github.com/nahi/httpclient/blob/master/lib/httpclient/util.rb#L27-L46
|
12
|
+
if defined? JRUBY_VERSION
|
13
|
+
require 'openssl'
|
14
|
+
require 'java'
|
15
|
+
module OpenSSL
|
16
|
+
module X509
|
17
|
+
class Certificate
|
18
|
+
java_import 'java.security.cert.Certificate'
|
19
|
+
java_import 'java.security.cert.CertificateFactory'
|
20
|
+
java_import 'java.io.ByteArrayInputStream'
|
21
|
+
def to_text
|
22
|
+
cf = CertificateFactory.getInstance('X.509')
|
23
|
+
cf.generateCertificate(ByteArrayInputStream.new(self.to_der.to_java_bytes)).toString
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
describe 'API SSL connection' do
|
31
|
+
DIR = File.dirname(File.expand_path(__FILE__))
|
32
|
+
|
33
|
+
after :each do
|
34
|
+
@server.shutdown if @server
|
35
|
+
end
|
36
|
+
|
37
|
+
it 'should fail to connect SSLv3 only server' do
|
38
|
+
@server = setup_server(:SSLv3)
|
39
|
+
api = API.new(nil, :endpoint => "https://localhost:#{@serverport}", :retry_post_requests => false)
|
40
|
+
api.ssl_ca_file = File.join(DIR, 'ca-all.cert')
|
41
|
+
expect {
|
42
|
+
api.delete_database('no_such_database')
|
43
|
+
}.to raise_error OpenSSL::SSL::SSLError
|
44
|
+
end
|
45
|
+
|
46
|
+
# 1.8.7's net/http does not call SSLContext#set_params. Use 1.8.7 where you don't care security.
|
47
|
+
if RUBY_VERSION >= '1.9.0'
|
48
|
+
it 'should success to connect TLSv1 only server' do
|
49
|
+
@server = setup_server(:TLSv1)
|
50
|
+
api = API.new(nil, :endpoint => "https://localhost:#{@serverport}", :retry_post_requests => false)
|
51
|
+
api.ssl_ca_file = File.join(DIR, 'ca-all.cert')
|
52
|
+
expect {
|
53
|
+
api.delete_database('no_such_database')
|
54
|
+
}.to raise_error TreasureData::NotFoundError
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
def setup_server(ssl_version)
|
59
|
+
logger = Logger.new(STDERR)
|
60
|
+
logger.level = Logger::Severity::FATAL # avoid logging SSLError (ERROR level)
|
61
|
+
@server = WEBrick::HTTPServer.new(
|
62
|
+
:BindAddress => "localhost",
|
63
|
+
:Logger => logger,
|
64
|
+
:Port => 0,
|
65
|
+
:AccessLog => [],
|
66
|
+
:DocumentRoot => '.',
|
67
|
+
:SSLEnable => true,
|
68
|
+
:SSLCACertificateFile => File.join(DIR, 'ca.cert'),
|
69
|
+
:SSLCertificate => cert('server.cert'),
|
70
|
+
:SSLPrivateKey => key('server.key')
|
71
|
+
)
|
72
|
+
@serverport = @server.config[:Port]
|
73
|
+
@server.mount(
|
74
|
+
'/hello',
|
75
|
+
WEBrick::HTTPServlet::ProcHandler.new(method(:do_hello).to_proc)
|
76
|
+
)
|
77
|
+
@server.ssl_context.ssl_version = ssl_version
|
78
|
+
@server_thread = start_server_thread(@server)
|
79
|
+
return @server
|
80
|
+
end
|
81
|
+
|
82
|
+
def do_hello(req, res)
|
83
|
+
res['content-type'] = 'text/html'
|
84
|
+
res.body = "hello"
|
85
|
+
end
|
86
|
+
|
87
|
+
def start_server_thread(server)
|
88
|
+
t = Thread.new {
|
89
|
+
Thread.current.abort_on_exception = true
|
90
|
+
server.start
|
91
|
+
}
|
92
|
+
while server.status != :Running
|
93
|
+
sleep 0.1
|
94
|
+
unless t.alive?
|
95
|
+
t.join
|
96
|
+
raise
|
97
|
+
end
|
98
|
+
end
|
99
|
+
t
|
100
|
+
end
|
101
|
+
|
102
|
+
def cert(filename)
|
103
|
+
OpenSSL::X509::Certificate.new(File.read(File.join(DIR, filename)))
|
104
|
+
end
|
105
|
+
|
106
|
+
def key(filename)
|
107
|
+
OpenSSL::PKey::RSA.new(File.read(File.join(DIR, filename)))
|
108
|
+
end
|
109
|
+
end
|
@@ -0,0 +1,199 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
require 'spec_helper'
|
4
|
+
require 'td/client/spec_resources'
|
5
|
+
require 'tempfile'
|
6
|
+
|
7
|
+
describe 'BulkImport API' do
|
8
|
+
include_context 'spec symbols'
|
9
|
+
include_context 'common helper'
|
10
|
+
|
11
|
+
let :api do
|
12
|
+
API.new(nil)
|
13
|
+
end
|
14
|
+
|
15
|
+
let :packed do
|
16
|
+
s = StringIO.new
|
17
|
+
Zlib::GzipWriter.wrap(s) do |f|
|
18
|
+
pk = MessagePack::Packer.new(f)
|
19
|
+
pk.write([1, '2', 3.0])
|
20
|
+
pk.write([4, '5', 6.0])
|
21
|
+
pk.write([7, '8', 9.0])
|
22
|
+
pk.flush
|
23
|
+
end
|
24
|
+
s.string
|
25
|
+
end
|
26
|
+
|
27
|
+
let(:endpoint_domain) { TreasureData::API::DEFAULT_IMPORT_ENDPOINT }
|
28
|
+
|
29
|
+
describe 'create_bulk_import' do
|
30
|
+
it 'should create a new bulk_import' do
|
31
|
+
stub_api_request(:post, "/v3/bulk_import/create/#{e(bi_name)}/#{e(db_name)}/#{e(table_name)}").
|
32
|
+
to_return(:body => {'bulk_import' => bi_name}.to_json)
|
33
|
+
|
34
|
+
expect(api.create_bulk_import(bi_name, db_name, table_name)).to be_nil
|
35
|
+
end
|
36
|
+
|
37
|
+
it 'should return 422 error with invalid name' do
|
38
|
+
name = 'D'
|
39
|
+
err_msg = "Validation failed: Name is too short" # " (minimum is 3 characters)"
|
40
|
+
stub_api_request(:post, "/v3/bulk_import/create/#{e(name)}/#{e(db_name)}/#{e(table_name)}").
|
41
|
+
to_return(:status => 422, :body => {'message' => err_msg}.to_json)
|
42
|
+
|
43
|
+
expect {
|
44
|
+
api.create_bulk_import(name, db_name, table_name)
|
45
|
+
}.to raise_error(TreasureData::APIError, /#{err_msg}/)
|
46
|
+
end
|
47
|
+
|
48
|
+
it 'should return 404 error with non exist database name' do
|
49
|
+
db = 'no_such_db'
|
50
|
+
err_msg = "Couldn't find UserDatabase with name = #{db}"
|
51
|
+
stub_api_request(:post, "/v3/bulk_import/create/#{e(bi_name)}/#{e(db)}/#{e(table_name)}").
|
52
|
+
to_return(:status => 404, :body => {'message' => err_msg}.to_json)
|
53
|
+
|
54
|
+
expect {
|
55
|
+
api.create_bulk_import(bi_name, db, table_name)
|
56
|
+
}.to raise_error(TreasureData::APIError, /#{err_msg}/)
|
57
|
+
end
|
58
|
+
|
59
|
+
it 'should return 404 error with non exist table name' do
|
60
|
+
table = 'no_such_table'
|
61
|
+
err_msg = "Couldn't find UserTableReference with name = #{table}"
|
62
|
+
stub_api_request(:post, "/v3/bulk_import/create/#{e(bi_name)}/#{e(db_name)}/#{e(table)}").
|
63
|
+
to_return(:status => 404, :body => {'message' => err_msg}.to_json)
|
64
|
+
|
65
|
+
expect {
|
66
|
+
api.create_bulk_import(bi_name, db_name, table)
|
67
|
+
}.to raise_error(TreasureData::APIError, /#{err_msg}/)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
describe 'delete_bulk_import' do
|
72
|
+
it 'runs' do
|
73
|
+
stub_api_request(:post, '/v3/bulk_import/delete/name').
|
74
|
+
with(:body => 'foo=bar')
|
75
|
+
expect(api.delete_bulk_import('name', 'foo' => 'bar')).to eq(nil)
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
describe 'show_bulk_import' do
|
80
|
+
it 'runs' do
|
81
|
+
stub_api_request(:get, '/v3/bulk_import/show/name').
|
82
|
+
to_return(:body => {'status' => 'status', 'other' => 'other'}.to_json)
|
83
|
+
expect(api.show_bulk_import('name')['status']).to eq('status')
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
describe 'list_bulk_imports' do
|
88
|
+
it 'runs' do
|
89
|
+
stub_api_request(:get, '/v3/bulk_import/list').
|
90
|
+
with(:query => 'foo=bar').
|
91
|
+
to_return(:body => {'bulk_imports' => %w(1 2 3)}.to_json)
|
92
|
+
expect(api.list_bulk_imports('foo' => 'bar')).to eq(%w(1 2 3))
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
describe 'list_bulk_import_parts' do
|
97
|
+
it 'runs' do
|
98
|
+
stub_api_request(:get, '/v3/bulk_import/list_parts/name').
|
99
|
+
with(:query => 'foo=bar').
|
100
|
+
to_return(:body => {'parts' => %w(1 2 3)}.to_json)
|
101
|
+
expect(api.list_bulk_import_parts('name', 'foo' => 'bar')).to eq(%w(1 2 3))
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
describe 'bulk_import_upload_part' do
|
106
|
+
it 'runs' do
|
107
|
+
t = Tempfile.new('bulk_import_spec')
|
108
|
+
File.open(t.path, 'w') do |f|
|
109
|
+
f << '12345'
|
110
|
+
end
|
111
|
+
stub_request(:put, "https://#{TreasureData::API::DEFAULT_ENDPOINT}/v3/bulk_import/upload_part/name/part").
|
112
|
+
with(:body => '12345')
|
113
|
+
File.open(t.path) do |f|
|
114
|
+
expect(api.bulk_import_upload_part('name', 'part', f, 5)).to eq(nil)
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
118
|
+
if ''.respond_to?(:encode)
|
119
|
+
it 'encodes part_name in UTF-8' do
|
120
|
+
t = Tempfile.new('bulk_import_spec')
|
121
|
+
File.open(t.path, 'w') do |f|
|
122
|
+
f << '12345'
|
123
|
+
end
|
124
|
+
stub_request(:put, "https://#{TreasureData::API::DEFAULT_ENDPOINT}/v3/bulk_import/upload_part/name/" + CGI.escape('日本語(Japanese)'.encode('UTF-8'))).
|
125
|
+
with(:body => '12345')
|
126
|
+
File.open(t.path) do |f|
|
127
|
+
expect(api.bulk_import_upload_part('name', '日本語(Japanese)'.encode('Windows-31J'), f, 5)).to eq(nil)
|
128
|
+
end
|
129
|
+
end
|
130
|
+
end
|
131
|
+
end
|
132
|
+
|
133
|
+
describe 'bulk_import_delete_part' do
|
134
|
+
it 'runs' do
|
135
|
+
stub_api_request(:post, '/v3/bulk_import/delete_part/name/part')
|
136
|
+
expect(api.bulk_import_delete_part('name', 'part')).to eq(nil)
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
140
|
+
describe 'freeze_bulk_import' do
|
141
|
+
it 'runs' do
|
142
|
+
stub_api_request(:post, '/v3/bulk_import/freeze/name')
|
143
|
+
expect(api.freeze_bulk_import('name')).to eq(nil)
|
144
|
+
end
|
145
|
+
end
|
146
|
+
|
147
|
+
describe 'unfreeze_bulk_import' do
|
148
|
+
it 'runs' do
|
149
|
+
stub_api_request(:post, '/v3/bulk_import/unfreeze/name')
|
150
|
+
expect(api.unfreeze_bulk_import('name')).to eq(nil)
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
describe 'perform_bulk_import' do
|
155
|
+
it 'runs' do
|
156
|
+
stub_api_request(:post, '/v3/bulk_import/perform/name').
|
157
|
+
to_return(:body => {'job_id' => 12345}.to_json)
|
158
|
+
expect(api.perform_bulk_import('name')).to eq('12345')
|
159
|
+
end
|
160
|
+
end
|
161
|
+
|
162
|
+
describe 'commit_bulk_import' do
|
163
|
+
it 'runs' do
|
164
|
+
stub_api_request(:post, '/v3/bulk_import/commit/name').
|
165
|
+
to_return(:body => {'job_id' => 12345}.to_json)
|
166
|
+
expect(api.commit_bulk_import('name')).to eq(nil)
|
167
|
+
end
|
168
|
+
end
|
169
|
+
|
170
|
+
describe 'bulk_import_error_records' do
|
171
|
+
it 'returns [] on empty' do
|
172
|
+
stub_api_request(:get, '/v3/bulk_import/error_records/name').
|
173
|
+
to_return(:body => '')
|
174
|
+
expect(api.bulk_import_error_records('name')).to eq([])
|
175
|
+
end
|
176
|
+
|
177
|
+
it 'returns nil on empty if block given' do
|
178
|
+
stub_api_request(:get, '/v3/bulk_import/error_records/name').
|
179
|
+
to_return(:body => '')
|
180
|
+
expect(api.bulk_import_error_records('name'){}).to eq(nil)
|
181
|
+
end
|
182
|
+
|
183
|
+
it 'returns unpacked result' do
|
184
|
+
stub_api_request(:get, '/v3/bulk_import/error_records/name').
|
185
|
+
to_return(:body => packed)
|
186
|
+
expect(api.bulk_import_error_records('name')).to eq([[1, '2', 3.0], [4, '5', 6.0], [7, '8', 9.0]])
|
187
|
+
end
|
188
|
+
|
189
|
+
it 'yields unpacked result if block given' do
|
190
|
+
stub_api_request(:get, '/v3/bulk_import/error_records/name').
|
191
|
+
to_return(:body => packed)
|
192
|
+
result = []
|
193
|
+
api.bulk_import_error_records('name') do |row|
|
194
|
+
result << row
|
195
|
+
end
|
196
|
+
expect(result).to eq([[1, '2', 3.0], [4, '5', 6.0], [7, '8', 9.0]])
|
197
|
+
end
|
198
|
+
end
|
199
|
+
end
|
@@ -0,0 +1,401 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'td/client/spec_resources'
|
3
|
+
require 'tempfile'
|
4
|
+
|
5
|
+
describe 'BulkImport API' do
|
6
|
+
include_context 'spec symbols'
|
7
|
+
include_context 'common helper'
|
8
|
+
|
9
|
+
let :api do
|
10
|
+
Client.new(nil, {:max_cumul_retry_delay => -1})
|
11
|
+
end
|
12
|
+
|
13
|
+
let :retry_api do
|
14
|
+
API.new(nil, {:retry_delay => 1, :max_cumul_retry_delay => 1})
|
15
|
+
end
|
16
|
+
|
17
|
+
let :original_config do
|
18
|
+
{
|
19
|
+
:config => {
|
20
|
+
:type => "s3_file",
|
21
|
+
:access_key_id => "key",
|
22
|
+
:secret_access_key => "secret",
|
23
|
+
:endpoint => "s3.amazonaws.com",
|
24
|
+
:bucket => "td-bulk-loader-test-tokyo",
|
25
|
+
:path_prefix => "in/nahi/sample"
|
26
|
+
}
|
27
|
+
}
|
28
|
+
end
|
29
|
+
|
30
|
+
let :guessed_config do
|
31
|
+
{
|
32
|
+
"config" => {
|
33
|
+
"type" => "s3_file",
|
34
|
+
"access_key_id" => "key",
|
35
|
+
"secret_access_key" => "secret",
|
36
|
+
"endpoint" => "s3.amazonaws.com",
|
37
|
+
"bucket" => "td-bulk-loader-test-tokyo",
|
38
|
+
"path_prefix" => "in/nahi/sample",
|
39
|
+
"parser" => {
|
40
|
+
"charset" => "UTF-8",
|
41
|
+
"newline" => "LF",
|
42
|
+
"type" => "csv",
|
43
|
+
"delimiter" => ",",
|
44
|
+
"header_line" => false,
|
45
|
+
"columns" => [
|
46
|
+
{"name" => "time", "type" => "long"},
|
47
|
+
{"name" => "c1", "type" => "long"},
|
48
|
+
{"name" => "c2", "type" => "string"},
|
49
|
+
{"name" => "c3", "type" => "string"},
|
50
|
+
]
|
51
|
+
},
|
52
|
+
"decoders" => [
|
53
|
+
{"type" => "gzip"}
|
54
|
+
]
|
55
|
+
}
|
56
|
+
}
|
57
|
+
end
|
58
|
+
|
59
|
+
let :preview_result do
|
60
|
+
{
|
61
|
+
"schema" => [
|
62
|
+
{"index" => 0, "name" => "c0", "type" => "string"},
|
63
|
+
{"index" => 1, "name" => "c1", "type" => "long"},
|
64
|
+
{"index" => 2, "name" => "c2", "type" => "string"},
|
65
|
+
{"index" => 3, "name" => "c3", "type" => "string"}
|
66
|
+
],
|
67
|
+
"records" => [
|
68
|
+
["19920116", 32864, "06612", "00195"],
|
69
|
+
["19910729", 14824, "07706", "00058"],
|
70
|
+
["19950708", 27559, "03244", "00034"],
|
71
|
+
["19931010", 11270, "03459", "00159"],
|
72
|
+
["19981117", 20461, "01409", "00128"],
|
73
|
+
["19981117", 20461, "00203", "00128"],
|
74
|
+
["19930108", 44402, "01489", "00001"],
|
75
|
+
["19960104", 16528, "04848", "00184"],
|
76
|
+
["19960104", 16528, "01766", "00184"],
|
77
|
+
["19881022", 26114, "06960", "00175"]
|
78
|
+
]
|
79
|
+
}
|
80
|
+
end
|
81
|
+
|
82
|
+
let :bulk_load_session do
|
83
|
+
guessed_config.dup.merge(
|
84
|
+
{
|
85
|
+
"name" => "nahi_test_1",
|
86
|
+
"cron" => "@daily",
|
87
|
+
"timezone" => "Asia/Tokyo",
|
88
|
+
"delay" => 3600
|
89
|
+
}
|
90
|
+
)
|
91
|
+
end
|
92
|
+
|
93
|
+
let :bulk_load_job do
|
94
|
+
guessed_config.dup.merge(
|
95
|
+
{
|
96
|
+
"job_id" => 123456,
|
97
|
+
"account_id" => 1,
|
98
|
+
"status" => "success",
|
99
|
+
"records" => 10,
|
100
|
+
"schema" => [["c0", "string", ""], ["c1", "long", ""], ["c2", "string", ""], ["c3", "string", ""]],
|
101
|
+
"database" => {"id" => 189263, "name" => "nahidb"},
|
102
|
+
"table" => {"id" => 176281, "name" => "bulkload_import_test"},
|
103
|
+
"created_at" => 1426738133,
|
104
|
+
"updated_at" => 1426738145,
|
105
|
+
"start_at" => 1426738134,
|
106
|
+
"end_at" => 1426738144
|
107
|
+
}
|
108
|
+
)
|
109
|
+
end
|
110
|
+
|
111
|
+
describe 'guess' do
|
112
|
+
it 'returns guessed json' do
|
113
|
+
stub_api_request(:post, '/v3/bulk_loads/guess').
|
114
|
+
with(:body => original_config.to_json).
|
115
|
+
to_return(:body => guessed_config.to_json)
|
116
|
+
expect(api.bulk_load_guess(
|
117
|
+
original_config
|
118
|
+
)).to eq(guessed_config)
|
119
|
+
end
|
120
|
+
|
121
|
+
it 'raises an error' do
|
122
|
+
stub_api_request(:post, '/v3/bulk_loads/guess').
|
123
|
+
with(:body => original_config.to_json).
|
124
|
+
to_return(:status => 500, :body => guessed_config.to_json)
|
125
|
+
expect {
|
126
|
+
api.bulk_load_guess(
|
127
|
+
original_config
|
128
|
+
)
|
129
|
+
}.to raise_error(TreasureData::APIError)
|
130
|
+
end
|
131
|
+
|
132
|
+
it 'perform redo on 500 error' do
|
133
|
+
stub_api_request(:post, '/v3/bulk_loads/guess').
|
134
|
+
with(:body => original_config.to_json).
|
135
|
+
to_return(:status => 500, :body => guessed_config.to_json)
|
136
|
+
begin
|
137
|
+
expect(retry_api.bulk_load_guess(
|
138
|
+
original_config
|
139
|
+
)).to != nil
|
140
|
+
rescue TreasureData::APIError => e
|
141
|
+
expect(e.message).to match(/^500: BulkLoad configuration guess failed/)
|
142
|
+
end
|
143
|
+
end
|
144
|
+
|
145
|
+
it 'perform retries on connection failure' do
|
146
|
+
api = retry_api
|
147
|
+
allow(api.instance_eval { @api }).to receive(:post).and_raise(SocketError.new('>>'))
|
148
|
+
begin
|
149
|
+
retry_api.bulk_load_guess(
|
150
|
+
original_config
|
151
|
+
)
|
152
|
+
rescue SocketError => e
|
153
|
+
expect(e.message).to eq('>> (Retried 1 times in 1 seconds)')
|
154
|
+
end
|
155
|
+
end
|
156
|
+
end
|
157
|
+
|
158
|
+
describe 'guess with old format' do
|
159
|
+
it 'returns guessed json' do
|
160
|
+
stub_api_request(:post, '/v3/bulk_loads/guess').
|
161
|
+
with(:body => original_config.to_json).
|
162
|
+
to_return(:body => guessed_config.to_json)
|
163
|
+
expect(api.bulk_load_guess(
|
164
|
+
original_config
|
165
|
+
)).to eq(guessed_config)
|
166
|
+
end
|
167
|
+
|
168
|
+
it 'raises an error' do
|
169
|
+
stub_api_request(:post, '/v3/bulk_loads/guess').
|
170
|
+
with(:body => original_config.to_json).
|
171
|
+
to_return(:status => 500, :body => guessed_config.to_json)
|
172
|
+
expect {
|
173
|
+
api.bulk_load_guess(
|
174
|
+
original_config
|
175
|
+
)
|
176
|
+
}.to raise_error(TreasureData::APIError)
|
177
|
+
end
|
178
|
+
|
179
|
+
it 'perform redo on 500 error' do
|
180
|
+
stub_api_request(:post, '/v3/bulk_loads/guess').
|
181
|
+
with(:body => original_config.to_json).
|
182
|
+
to_return(:status => 500, :body => guessed_config.to_json)
|
183
|
+
begin
|
184
|
+
expect(retry_api.bulk_load_guess(
|
185
|
+
original_config
|
186
|
+
)).to != nil
|
187
|
+
rescue TreasureData::APIError => e
|
188
|
+
expect(e.message).to match(/^500: BulkLoad configuration guess failed/)
|
189
|
+
end
|
190
|
+
end
|
191
|
+
|
192
|
+
it 'perform retries on connection failure' do
|
193
|
+
api = retry_api
|
194
|
+
allow(api.instance_eval { @api }).to receive(:post).and_raise(SocketError.new('>>'))
|
195
|
+
begin
|
196
|
+
retry_api.bulk_load_guess(
|
197
|
+
original_config
|
198
|
+
)
|
199
|
+
rescue SocketError => e
|
200
|
+
expect(e.message).to eq('>> (Retried 1 times in 1 seconds)')
|
201
|
+
end
|
202
|
+
end
|
203
|
+
end
|
204
|
+
|
205
|
+
describe 'preview' do
|
206
|
+
it 'returns preview json' do
|
207
|
+
stub_api_request(:post, '/v3/bulk_loads/preview').
|
208
|
+
with(:body => guessed_config.to_json).
|
209
|
+
to_return(:body => preview_result.to_json)
|
210
|
+
expect(api.bulk_load_preview(
|
211
|
+
guessed_config
|
212
|
+
)).to eq(preview_result)
|
213
|
+
end
|
214
|
+
|
215
|
+
it 'raises an error' do
|
216
|
+
stub_api_request(:post, '/v3/bulk_loads/preview').
|
217
|
+
with(:body => guessed_config.to_json).
|
218
|
+
to_return(:status => 500, :body => preview_result.to_json)
|
219
|
+
expect {
|
220
|
+
api.bulk_load_preview(
|
221
|
+
guessed_config
|
222
|
+
)
|
223
|
+
}.to raise_error(TreasureData::APIError)
|
224
|
+
end
|
225
|
+
end
|
226
|
+
|
227
|
+
describe 'issue' do
|
228
|
+
it 'returns job id' do
|
229
|
+
expected_request = guessed_config.dup
|
230
|
+
expected_request['database'] = 'database'
|
231
|
+
expected_request['table'] = 'table'
|
232
|
+
stub_api_request(:post, '/v3/job/issue/bulkload/database').
|
233
|
+
with(:body => expected_request.to_json).
|
234
|
+
to_return(:body => {'job_id' => 12345}.to_json)
|
235
|
+
expect(api.bulk_load_issue(
|
236
|
+
'database',
|
237
|
+
'table',
|
238
|
+
guessed_config
|
239
|
+
)).to eq('12345')
|
240
|
+
end
|
241
|
+
end
|
242
|
+
|
243
|
+
describe 'list' do
|
244
|
+
it 'returns BulkLoadSession' do
|
245
|
+
stub_api_request(:get, '/v3/bulk_loads').
|
246
|
+
to_return(:body => [bulk_load_session, bulk_load_session].to_json)
|
247
|
+
result = api.bulk_load_list
|
248
|
+
expect(result.size).to eq(2)
|
249
|
+
expect(result.first).to eq(bulk_load_session)
|
250
|
+
end
|
251
|
+
|
252
|
+
it 'returns empty' do
|
253
|
+
stub_api_request(:get, '/v3/bulk_loads').
|
254
|
+
to_return(:body => [].to_json)
|
255
|
+
expect(api.bulk_load_list.size).to eq(0)
|
256
|
+
end
|
257
|
+
end
|
258
|
+
|
259
|
+
describe 'create' do
|
260
|
+
it 'returns registered bulk_load_session' do
|
261
|
+
expected_request = guessed_config.dup
|
262
|
+
expected_request['name'] = 'nahi_test_1'
|
263
|
+
expected_request['cron'] = '@daily'
|
264
|
+
expected_request['timezone'] = 'Asia/Tokyo'
|
265
|
+
expected_request['delay'] = 3600
|
266
|
+
expected_request['database'] = 'database'
|
267
|
+
expected_request['table'] = 'table'
|
268
|
+
stub_api_request(:post, '/v3/bulk_loads').
|
269
|
+
with(:body => expected_request.to_json).
|
270
|
+
to_return(:body => bulk_load_session.to_json)
|
271
|
+
expect(api.bulk_load_create(
|
272
|
+
'nahi_test_1',
|
273
|
+
'database',
|
274
|
+
'table',
|
275
|
+
guessed_config,
|
276
|
+
{
|
277
|
+
cron: '@daily',
|
278
|
+
timezone: 'Asia/Tokyo',
|
279
|
+
delay: 3600
|
280
|
+
}
|
281
|
+
)).to eq(bulk_load_session)
|
282
|
+
end
|
283
|
+
|
284
|
+
it 'accepts empty option' do
|
285
|
+
expected_request = guessed_config.dup
|
286
|
+
expected_request['name'] = 'nahi_test_1'
|
287
|
+
expected_request['database'] = 'database'
|
288
|
+
expected_request['table'] = 'table'
|
289
|
+
stub_api_request(:post, '/v3/bulk_loads').
|
290
|
+
with(:body => expected_request.to_json).
|
291
|
+
to_return(:body => bulk_load_session.to_json)
|
292
|
+
expect(api.bulk_load_create(
|
293
|
+
'nahi_test_1',
|
294
|
+
'database',
|
295
|
+
'table',
|
296
|
+
guessed_config
|
297
|
+
)).to eq(bulk_load_session)
|
298
|
+
end
|
299
|
+
|
300
|
+
it 'accepts time_column option' do
|
301
|
+
expected_request = guessed_config.dup
|
302
|
+
expected_request['name'] = 'nahi_test_1'
|
303
|
+
expected_request['time_column'] = 'c0'
|
304
|
+
expected_request['database'] = 'database'
|
305
|
+
expected_request['table'] = 'table'
|
306
|
+
stub_api_request(:post, '/v3/bulk_loads').
|
307
|
+
with(:body => expected_request.to_json).
|
308
|
+
to_return(:body => bulk_load_session.to_json)
|
309
|
+
expect(api.bulk_load_create(
|
310
|
+
'nahi_test_1',
|
311
|
+
'database',
|
312
|
+
'table',
|
313
|
+
guessed_config,
|
314
|
+
{
|
315
|
+
time_column: 'c0'
|
316
|
+
}
|
317
|
+
)).to eq(bulk_load_session)
|
318
|
+
end
|
319
|
+
end
|
320
|
+
|
321
|
+
describe 'show' do
|
322
|
+
it 'returns bulk_load_session' do
|
323
|
+
stub_api_request(:get, '/v3/bulk_loads/nahi_test_1').
|
324
|
+
to_return(:body => bulk_load_session.to_json)
|
325
|
+
expect(api.bulk_load_show('nahi_test_1')).to eq(bulk_load_session)
|
326
|
+
end
|
327
|
+
end
|
328
|
+
|
329
|
+
describe 'update' do
|
330
|
+
it 'returns updated bulk_load_session' do
|
331
|
+
stub_api_request(:put, '/v3/bulk_loads/nahi_test_1').
|
332
|
+
with(:body => bulk_load_session.to_json).
|
333
|
+
to_return(:body => bulk_load_session.to_json)
|
334
|
+
expect(api.bulk_load_update(
|
335
|
+
'nahi_test_1',
|
336
|
+
bulk_load_session
|
337
|
+
)).to eq(bulk_load_session)
|
338
|
+
end
|
339
|
+
|
340
|
+
it 'returns updated bulk_load_session' do
|
341
|
+
updated_bulk_load_session = bulk_load_session.merge({'timezone' => 'America/Los Angeles'})
|
342
|
+
stub_api_request(:put, '/v3/bulk_loads/nahi_test_1').
|
343
|
+
with(:body => updated_bulk_load_session.to_json).
|
344
|
+
to_return(:body => updated_bulk_load_session.to_json)
|
345
|
+
expect(api.bulk_load_update(
|
346
|
+
'nahi_test_1',
|
347
|
+
updated_bulk_load_session
|
348
|
+
)).to eq updated_bulk_load_session
|
349
|
+
end
|
350
|
+
|
351
|
+
it 'can remove the cron schedule ' do
|
352
|
+
updated_bulk_load_session = bulk_load_session.merge({'cron' => ''})
|
353
|
+
# NOTE: currently the API just ignores an empty 'cron' specification update
|
354
|
+
# I am assuming that once fixed, the API will return a nil for cron if unscheduled.
|
355
|
+
expected_bulk_load_session = (bulk_load_session['cron'] = nil)
|
356
|
+
stub_api_request(:put, '/v3/bulk_loads/nahi_test_1').
|
357
|
+
with(:body => updated_bulk_load_session.to_json).
|
358
|
+
to_return(:body => expected_bulk_load_session.to_json)
|
359
|
+
expect(api.bulk_load_update(
|
360
|
+
'nahi_test_1',
|
361
|
+
updated_bulk_load_session
|
362
|
+
)).to eq expected_bulk_load_session
|
363
|
+
end
|
364
|
+
end
|
365
|
+
|
366
|
+
describe 'delete' do
|
367
|
+
it 'returns updated bulk_load_session' do
|
368
|
+
stub_api_request(:delete, '/v3/bulk_loads/nahi_test_1').
|
369
|
+
to_return(:body => bulk_load_session.to_json)
|
370
|
+
expect(api.bulk_load_delete('nahi_test_1')).to eq(bulk_load_session)
|
371
|
+
end
|
372
|
+
end
|
373
|
+
|
374
|
+
describe 'history' do
|
375
|
+
it 'returns list of jobs' do
|
376
|
+
stub_api_request(:get, '/v3/bulk_loads/nahi_test_1/jobs').
|
377
|
+
to_return(:body => [bulk_load_job, bulk_load_job].to_json)
|
378
|
+
result = api.bulk_load_history('nahi_test_1')
|
379
|
+
expect(result.size).to eq(2)
|
380
|
+
expect(result.first).to eq(bulk_load_job)
|
381
|
+
end
|
382
|
+
end
|
383
|
+
|
384
|
+
describe 'run' do
|
385
|
+
it 'returns job_id' do
|
386
|
+
stub_api_request(:post, '/v3/bulk_loads/nahi_test_1/jobs').
|
387
|
+
with(:body => '{}').
|
388
|
+
to_return(:body => {'job_id' => 12345}.to_json)
|
389
|
+
expect(api.bulk_load_run('nahi_test_1')).to eq('12345')
|
390
|
+
end
|
391
|
+
|
392
|
+
it 'accepts scheduled_time' do
|
393
|
+
now = Time.now.to_i
|
394
|
+
stub_api_request(:post, '/v3/bulk_loads/nahi_test_1/jobs').
|
395
|
+
with(:body => {scheduled_time: now.to_s}.to_json).
|
396
|
+
to_return(:body => {'job_id' => 12345}.to_json)
|
397
|
+
expect(api.bulk_load_run('nahi_test_1', now)).to eq('12345')
|
398
|
+
end
|
399
|
+
end
|
400
|
+
|
401
|
+
end
|