td 0.7.2 → 0.7.3
Sign up to get free protection for your applications and to get access to all the features.
- data/ChangeLog +6 -0
- data/lib/td/api.rb +307 -262
- data/lib/td/client.rb +337 -0
- data/lib/td/command/account.rb +8 -8
- data/lib/td/command/common.rb +8 -8
- data/lib/td/command/database.rb +7 -7
- data/lib/td/command/import.rb +7 -7
- data/lib/td/command/list.rb +2 -2
- data/lib/td/command/query.rb +8 -26
- data/lib/td/command/server.rb +3 -3
- data/lib/td/command/table.rb +8 -8
- data/lib/td/command/td.rb +6 -8
- data/lib/td/config.rb +11 -2
- data/lib/td/version.rb +1 -1
- metadata +4 -5
- data/lib/td/api_iface.rb +0 -374
- data/lib/td/error.rb +0 -29
data/ChangeLog
CHANGED
data/lib/td/api.rb
CHANGED
@@ -1,356 +1,401 @@
|
|
1
|
-
require 'time'
|
2
|
-
require 'td/api_iface'
|
3
|
-
require 'td/error'
|
4
1
|
|
5
|
-
module
|
2
|
+
module TreasureData
|
6
3
|
|
7
|
-
class API
|
8
|
-
def self.authenticate(user, password)
|
9
|
-
iface = APIInterface.new(nil)
|
10
|
-
apikey = iface.authenticate(user, password)
|
11
|
-
new(apikey)
|
12
|
-
end
|
13
4
|
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
5
|
+
class APIError < StandardError
|
6
|
+
end
|
7
|
+
|
8
|
+
class AuthError < APIError
|
9
|
+
end
|
10
|
+
|
11
|
+
class AlreadyExistsError < APIError
|
12
|
+
end
|
18
13
|
|
14
|
+
class NotFoundError < APIError
|
15
|
+
end
|
16
|
+
|
17
|
+
|
18
|
+
class API
|
19
19
|
def initialize(apikey)
|
20
|
-
|
20
|
+
require 'json'
|
21
|
+
@apikey = apikey
|
21
22
|
end
|
22
23
|
|
23
|
-
|
24
|
+
# TODO error check & raise appropriate errors
|
24
25
|
|
25
|
-
|
26
|
-
|
26
|
+
attr_reader :apikey
|
27
|
+
|
28
|
+
def self.validate_database_name(name)
|
29
|
+
name = name.to_s
|
30
|
+
if name.empty?
|
31
|
+
raise "Empty name is not allowed"
|
32
|
+
end
|
33
|
+
if name.length < 3 || 32 < name.length
|
34
|
+
raise "Name must be 3 to 32 characters, got #{name.length} characters."
|
35
|
+
end
|
36
|
+
unless name =~ /^([a-z0-9_]+)$/
|
37
|
+
raise "Name must consist only of alphabets, numbers, '_'."
|
38
|
+
end
|
27
39
|
end
|
28
40
|
|
29
|
-
def
|
30
|
-
|
41
|
+
def self.validate_table_name(name)
|
42
|
+
validate_database_name(name)
|
31
43
|
end
|
32
44
|
|
33
|
-
|
34
|
-
|
35
|
-
|
45
|
+
####
|
46
|
+
## Database API
|
47
|
+
##
|
48
|
+
|
49
|
+
# => [name:String]
|
50
|
+
def list_databases
|
51
|
+
code, body, res = get("/v3/database/list")
|
52
|
+
if code != "200"
|
53
|
+
raise_error("List databases failed", res)
|
54
|
+
end
|
55
|
+
# TODO format check
|
56
|
+
js = JSON.load(body)
|
57
|
+
names = js["databases"].map {|dbinfo| dbinfo['name'] }
|
58
|
+
return names
|
36
59
|
end
|
37
60
|
|
38
61
|
# => true
|
39
|
-
def delete_database(
|
40
|
-
|
62
|
+
def delete_database(db)
|
63
|
+
code, body, res = post("/v3/database/delete/#{e db}")
|
64
|
+
if code != "200"
|
65
|
+
raise_error("Delete database failed", res)
|
66
|
+
end
|
67
|
+
return true
|
41
68
|
end
|
42
69
|
|
43
|
-
# =>
|
44
|
-
def
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
70
|
+
# => true
|
71
|
+
def create_database(db)
|
72
|
+
code, body, res = post("/v3/database/create/#{e db}")
|
73
|
+
if code != "200"
|
74
|
+
raise_error("Create database failed", res)
|
75
|
+
end
|
76
|
+
return true
|
49
77
|
end
|
50
78
|
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
79
|
+
|
80
|
+
####
|
81
|
+
## Table API
|
82
|
+
##
|
83
|
+
|
84
|
+
# => {name:String => [type:Symbol, count:Integer]}
|
85
|
+
def list_tables(db)
|
86
|
+
code, body, res = get("/v3/table/list/#{e db}")
|
87
|
+
if code != "200"
|
88
|
+
raise_error("List tables failed", res)
|
89
|
+
end
|
90
|
+
# TODO format check
|
91
|
+
js = JSON.load(body)
|
92
|
+
result = {}
|
93
|
+
js["tables"].map {|m|
|
94
|
+
name = m['name']
|
95
|
+
type = (m['type'] || '?').to_sym
|
96
|
+
count = (m['count'] || 0).to_i # TODO?
|
97
|
+
result[name] = [type, count]
|
58
98
|
}
|
59
|
-
|
99
|
+
return result
|
60
100
|
end
|
61
101
|
|
62
102
|
# => true
|
63
|
-
def create_table(
|
64
|
-
|
103
|
+
def create_table(db, table, type)
|
104
|
+
code, body, res = post("/v3/table/create/#{e db}/#{e table}/#{type}")
|
105
|
+
if code != "200"
|
106
|
+
raise_error("Create #{type} table failed", res)
|
107
|
+
end
|
108
|
+
return true
|
65
109
|
end
|
66
110
|
|
67
111
|
# => true
|
68
|
-
def create_log_table(
|
69
|
-
create_table(
|
112
|
+
def create_log_table(db, table)
|
113
|
+
create_table(db, table, :log)
|
70
114
|
end
|
71
115
|
|
72
116
|
# => true
|
73
|
-
def create_item_table(
|
74
|
-
create_table(
|
117
|
+
def create_item_table(db, table)
|
118
|
+
create_table(db, table, :item)
|
75
119
|
end
|
76
120
|
|
77
121
|
# => type:Symbol
|
78
|
-
def delete_table(
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
}
|
122
|
+
def delete_table(db, table)
|
123
|
+
code, body, res = post("/v3/table/delete/#{e db}/#{e table}")
|
124
|
+
if code != "200"
|
125
|
+
raise_error("Drop table failed", res)
|
126
|
+
end
|
127
|
+
# TODO format check
|
128
|
+
js = JSON.load(body)
|
129
|
+
type = (js['type'] || '?').to_sym
|
130
|
+
return type
|
88
131
|
end
|
89
132
|
|
90
|
-
# => Table
|
91
|
-
def table(db_name, table_name)
|
92
|
-
m = @iface.list_tables(db_name)
|
93
|
-
m.each_pair {|name,(type,count)|
|
94
|
-
if name == table_name
|
95
|
-
return Table.new(self, db_name, name, type, count)
|
96
|
-
end
|
97
|
-
}
|
98
|
-
raise NotFoundError, "Table '#{db_name}.#{table_name}' does not exist"
|
99
|
-
end
|
100
133
|
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
Job.new(self, job_id, :hive, q) # TODO url
|
105
|
-
end
|
134
|
+
####
|
135
|
+
## Job API
|
136
|
+
##
|
106
137
|
|
107
|
-
# => [
|
108
|
-
def
|
109
|
-
|
110
|
-
|
111
|
-
|
138
|
+
# => [(jobId:String, type:Symbol, status:String, start_at:String, end_at:String)]
|
139
|
+
def list_jobs(from=0, to=nil)
|
140
|
+
params = {}
|
141
|
+
params['from'] = from.to_s if from
|
142
|
+
params['to'] = to.to_s if to
|
143
|
+
code, body, res = get("/v3/job/list", params)
|
144
|
+
if code != "200"
|
145
|
+
raise_error("List jobs failed", res)
|
146
|
+
end
|
147
|
+
# TODO format check
|
148
|
+
js = JSON.load(body)
|
149
|
+
result = []
|
150
|
+
js['jobs'].each {|m|
|
151
|
+
job_id = m['job_id']
|
152
|
+
type = (m['type'] || '?').to_sym
|
153
|
+
status = m['status']
|
154
|
+
query = m['query']
|
155
|
+
start_at = m['start_at']
|
156
|
+
end_at = m['end_at']
|
157
|
+
result << [job_id, type, status, query, start_at, end_at]
|
112
158
|
}
|
159
|
+
return result
|
113
160
|
end
|
114
161
|
|
115
|
-
# =>
|
116
|
-
def
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
type
|
125
|
-
|
162
|
+
# => (type:Symbol, status:String, result:String, url:String)
|
163
|
+
def show_job(job_id)
|
164
|
+
code, body, res = get("/v3/job/show/#{e job_id}")
|
165
|
+
if code != "200"
|
166
|
+
raise_error("Show job failed", res)
|
167
|
+
end
|
168
|
+
# TODO format check
|
169
|
+
js = JSON.load(body)
|
170
|
+
# TODO debug
|
171
|
+
type = (js['type'] || '?').to_sym # TODO
|
172
|
+
query = js['query']
|
173
|
+
status = js['status']
|
174
|
+
debug = js['debug']
|
175
|
+
url = js['url']
|
176
|
+
start_at = js['start_at']
|
177
|
+
end_at = js['end_at']
|
178
|
+
return [type, query, status, url, debug, start_at, end_at]
|
126
179
|
end
|
127
180
|
|
128
|
-
# => result:[{column:String=>value:Object]
|
129
181
|
def job_result(job_id)
|
130
|
-
|
182
|
+
require 'msgpack'
|
183
|
+
code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>'msgpack'})
|
184
|
+
if code != "200"
|
185
|
+
raise_error("Get job result failed", res)
|
186
|
+
end
|
187
|
+
result = []
|
188
|
+
MessagePack::Unpacker.new.feed_each(body) {|row|
|
189
|
+
result << row
|
190
|
+
}
|
191
|
+
return result
|
131
192
|
end
|
132
193
|
|
133
|
-
# => result:String
|
134
194
|
def job_result_format(job_id, format)
|
135
|
-
|
195
|
+
# TODO chunked encoding
|
196
|
+
code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>format})
|
197
|
+
if code != "200"
|
198
|
+
raise_error("Get job result failed", res)
|
199
|
+
end
|
200
|
+
return body
|
136
201
|
end
|
137
202
|
|
138
|
-
# => nil
|
139
203
|
def job_result_each(job_id, &block)
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
@iface.import(db_name, table_name, format, stream, stream_size)
|
146
|
-
end
|
147
|
-
|
148
|
-
def self.validate_database_name(name)
|
149
|
-
name = name.to_s
|
150
|
-
if name.empty?
|
151
|
-
raise "Empty name is not allowed"
|
152
|
-
end
|
153
|
-
if name.length < 3 || 32 < name.length
|
154
|
-
raise "Name must be 3 to 32 characters, got #{name.length} characters."
|
204
|
+
# TODO chunked encoding
|
205
|
+
require 'msgpack'
|
206
|
+
code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>'msgpack'})
|
207
|
+
if code != "200"
|
208
|
+
raise_error("Get job result failed", res)
|
155
209
|
end
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
def self.validate_table_name(name)
|
162
|
-
validate_database_name(name)
|
163
|
-
end
|
164
|
-
end
|
165
|
-
|
166
|
-
end
|
167
|
-
|
168
|
-
|
169
|
-
module TD
|
170
|
-
|
171
|
-
class APIObject
|
172
|
-
def initialize(api)
|
173
|
-
@api = api
|
174
|
-
end
|
175
|
-
end
|
176
|
-
|
177
|
-
class Database < APIObject
|
178
|
-
def initialize(api, db_name, tables=nil)
|
179
|
-
super(api)
|
180
|
-
@db_name = db_name
|
181
|
-
@tables = tables
|
182
|
-
end
|
183
|
-
|
184
|
-
def name
|
185
|
-
@db_name
|
210
|
+
result = []
|
211
|
+
MessagePack::Unpacker.new.feed_each(body) {|row|
|
212
|
+
yield row
|
213
|
+
}
|
214
|
+
nil
|
186
215
|
end
|
187
216
|
|
188
|
-
def
|
189
|
-
|
190
|
-
|
217
|
+
def job_result_raw(job_id, format)
|
218
|
+
code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>format})
|
219
|
+
if code != "200"
|
220
|
+
raise_error("Get job result failed", res)
|
221
|
+
end
|
222
|
+
return body
|
191
223
|
end
|
192
224
|
|
193
|
-
|
194
|
-
|
225
|
+
# => jobId:String
|
226
|
+
def hive_query(q, db=nil)
|
227
|
+
code, body, res = post("/v3/job/issue/hive/#{e db}", {'query'=>q})
|
228
|
+
if code != "200"
|
229
|
+
raise_error("Query failed", res)
|
230
|
+
end
|
231
|
+
# TODO format check
|
232
|
+
js = JSON.load(body)
|
233
|
+
return js['job_id'].to_s
|
195
234
|
end
|
196
235
|
|
197
|
-
def create_log_table(name)
|
198
|
-
create_table(name, :log)
|
199
|
-
end
|
200
236
|
|
201
|
-
|
202
|
-
|
203
|
-
|
237
|
+
####
|
238
|
+
## Import API
|
239
|
+
##
|
204
240
|
|
205
|
-
|
206
|
-
|
241
|
+
# => time:Float
|
242
|
+
def import(db, table, format, stream, stream_size=stream.lstat.size)
|
243
|
+
code, body, res = put("/v3/table/import/#{e db}/#{e table}/#{format}", stream, stream_size)
|
244
|
+
if code[0] != ?2
|
245
|
+
raise_error("Import failed", res)
|
246
|
+
end
|
247
|
+
# TODO format check
|
248
|
+
js = JSON.load(body)
|
249
|
+
time = js['time'].to_f
|
250
|
+
return time
|
207
251
|
end
|
208
252
|
|
209
|
-
def delete
|
210
|
-
@api.delete_database(@db_name)
|
211
|
-
end
|
212
253
|
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
end
|
254
|
+
####
|
255
|
+
## User API
|
256
|
+
##
|
217
257
|
|
218
|
-
|
219
|
-
def
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
258
|
+
# apikey:String
|
259
|
+
def authenticate(user, password)
|
260
|
+
code, body, res = post("/v3/user/authenticate", {'user'=>user, 'password'=>password})
|
261
|
+
if code != "200"
|
262
|
+
raise_error("Authentication failed", res)
|
263
|
+
end
|
264
|
+
# TODO format check
|
265
|
+
js = JSON.load(body)
|
266
|
+
apikey = js['apikey']
|
267
|
+
return apikey
|
225
268
|
end
|
226
269
|
|
227
|
-
|
270
|
+
####
|
271
|
+
## Server Status API
|
272
|
+
##
|
228
273
|
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
274
|
+
# => status:String
|
275
|
+
def server_status
|
276
|
+
code, body, res = get('/v3/system/server_status')
|
277
|
+
if code != "200"
|
278
|
+
return "Server is down (#{code})"
|
279
|
+
end
|
280
|
+
# TODO format check
|
281
|
+
js = JSON.load(body)
|
282
|
+
status = js['status']
|
283
|
+
return status
|
284
|
+
end
|
285
|
+
|
286
|
+
private
|
287
|
+
host = 'api.treasure-data.com'
|
288
|
+
port = 80
|
289
|
+
if e = ENV['TD_API_SERVER']
|
290
|
+
host, port_ = e.split(':',2)
|
291
|
+
port_ = port_.to_i
|
292
|
+
port = port_ if port_ != 0
|
293
|
+
end
|
294
|
+
|
295
|
+
HOST = host
|
296
|
+
PORT = port
|
297
|
+
USE_SSL = false
|
298
|
+
BASE_URL = ''
|
299
|
+
|
300
|
+
def get(url, params=nil)
|
301
|
+
http, header = new_http
|
302
|
+
|
303
|
+
path = BASE_URL + url
|
304
|
+
if params && !params.empty?
|
305
|
+
path << "?"+params.map {|k,v|
|
306
|
+
"#{k}=#{e v}"
|
307
|
+
}.join('&')
|
308
|
+
end
|
236
309
|
|
237
|
-
|
238
|
-
@table_name
|
239
|
-
end
|
310
|
+
request = Net::HTTP::Get.new(path, header)
|
240
311
|
|
241
|
-
|
242
|
-
|
312
|
+
response = http.request(request)
|
313
|
+
return [response.code, response.body, response]
|
243
314
|
end
|
244
315
|
|
245
|
-
def
|
246
|
-
|
247
|
-
end
|
248
|
-
end
|
316
|
+
def post(url, params=nil)
|
317
|
+
http, header = new_http
|
249
318
|
|
250
|
-
|
251
|
-
def initialize(api, job_id, type, query, status=nil, url=nil, debug=nil, start_at=nil, end_at=nil, result=nil)
|
252
|
-
super(api)
|
253
|
-
@job_id = job_id
|
254
|
-
@type = type
|
255
|
-
@url = url
|
256
|
-
@query = query
|
257
|
-
@status = status
|
258
|
-
@debug = debug
|
259
|
-
@start_at = start_at
|
260
|
-
@end_at = end_at
|
261
|
-
@result = result
|
262
|
-
end
|
319
|
+
path = BASE_URL + url
|
263
320
|
|
264
|
-
|
321
|
+
request = Net::HTTP::Post.new(path, header)
|
322
|
+
request.set_form_data(params) if params
|
265
323
|
|
266
|
-
|
267
|
-
|
324
|
+
response = http.request(request)
|
325
|
+
return [response.code, response.body, response]
|
268
326
|
end
|
269
327
|
|
270
|
-
def
|
271
|
-
|
272
|
-
@query
|
273
|
-
end
|
328
|
+
def put(url, stream, stream_size)
|
329
|
+
http, header = new_http
|
274
330
|
|
275
|
-
|
276
|
-
update_status! unless @status
|
277
|
-
@status
|
278
|
-
end
|
331
|
+
path = BASE_URL + url
|
279
332
|
|
280
|
-
|
281
|
-
|
282
|
-
@url
|
283
|
-
end
|
333
|
+
header['Content-Type'] = 'application/octet-stream'
|
334
|
+
header['Content-Length'] = stream_size.to_s
|
284
335
|
|
285
|
-
|
286
|
-
|
287
|
-
|
288
|
-
|
336
|
+
request = Net::HTTP::Put.new(url, header)
|
337
|
+
if request.respond_to?(:body_stream=)
|
338
|
+
request.body_stream = stream
|
339
|
+
else # Ruby 1.8
|
340
|
+
request.body = stream.read
|
341
|
+
end
|
289
342
|
|
290
|
-
|
291
|
-
|
292
|
-
@start_at && !@start_at.empty? ? Time.parse(@start_at) : nil
|
343
|
+
response = http.request(request)
|
344
|
+
return [response.code, response.body, response]
|
293
345
|
end
|
294
346
|
|
295
|
-
def
|
296
|
-
|
297
|
-
|
298
|
-
end
|
347
|
+
def new_http
|
348
|
+
require 'net/http'
|
349
|
+
require 'time'
|
299
350
|
|
300
|
-
|
301
|
-
|
302
|
-
|
303
|
-
|
351
|
+
http = Net::HTTP.new(HOST, PORT)
|
352
|
+
if USE_SSL
|
353
|
+
http.use_ssl = true
|
354
|
+
http.verify_mode = OpenSSL::SSL::VERIFY_PEER
|
355
|
+
store = OpenSSL::X509::Store.new
|
356
|
+
http.cert_store = store
|
304
357
|
end
|
305
|
-
@result
|
306
|
-
end
|
307
358
|
|
308
|
-
|
309
|
-
return nil unless finished?
|
310
|
-
@api.job_result_format(@job_id, format)
|
311
|
-
end
|
359
|
+
#http.read_timeout = options[:read_timeout]
|
312
360
|
|
313
|
-
|
314
|
-
if @
|
315
|
-
|
316
|
-
else
|
317
|
-
@api.job_result_each(@job_id, &block)
|
361
|
+
header = {}
|
362
|
+
if @apikey
|
363
|
+
header['Authorization'] = "TD1 #{apikey}"
|
318
364
|
end
|
319
|
-
|
320
|
-
end
|
365
|
+
header['Date'] = Time.now.rfc2822
|
321
366
|
|
322
|
-
|
323
|
-
update_status! unless @status
|
324
|
-
if @status == "success" || @status == "error"
|
325
|
-
return true
|
326
|
-
else
|
327
|
-
return false
|
328
|
-
end
|
367
|
+
return http, header
|
329
368
|
end
|
330
369
|
|
331
|
-
def
|
332
|
-
|
333
|
-
|
370
|
+
def raise_error(msg, res)
|
371
|
+
begin
|
372
|
+
js = JSON.load(res.body)
|
373
|
+
msg = js['message']
|
374
|
+
error_code = js['error_code']
|
334
375
|
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
|
376
|
+
if res.code == "404"
|
377
|
+
raise NotFoundError, "#{error_code}: #{msg}"
|
378
|
+
elsif res.code == "409"
|
379
|
+
raise AlreadyExistsError, "#{error_code}: #{msg}"
|
380
|
+
else
|
381
|
+
raise APIError, "#{error_code}: #{msg}"
|
382
|
+
end
|
339
383
|
|
340
|
-
|
341
|
-
|
342
|
-
|
384
|
+
rescue
|
385
|
+
if res.code == "404"
|
386
|
+
raise NotFoundError, "#{msg}: #{res.body}"
|
387
|
+
elsif res.code == "409"
|
388
|
+
raise AlreadyExistsError, "#{msg}: #{res.body}"
|
389
|
+
else
|
390
|
+
raise APIError, "#{msg}: #{res.body}"
|
391
|
+
end
|
392
|
+
end
|
393
|
+
# TODO error
|
343
394
|
end
|
344
395
|
|
345
|
-
def
|
346
|
-
|
347
|
-
|
348
|
-
@status = status
|
349
|
-
@url = url
|
350
|
-
@debug = debug
|
351
|
-
@start_at = start_at
|
352
|
-
@end_at = end_at
|
353
|
-
self
|
396
|
+
def e(s)
|
397
|
+
require 'cgi'
|
398
|
+
CGI.escape(s.to_s)
|
354
399
|
end
|
355
400
|
end
|
356
401
|
|