td 0.7.0 → 0.7.1
Sign up to get free protection for your applications and to get access to all the features.
- data/ChangeLog +10 -0
- data/lib/td/api.rb +59 -17
- data/lib/td/api_iface.rb +46 -4
- data/lib/td/command/database.rb +2 -1
- data/lib/td/command/query.rb +24 -7
- data/lib/td/command/table.rb +2 -0
- data/lib/td/version.rb +1 -1
- metadata +4 -4
data/ChangeLog
CHANGED
@@ -1,9 +1,19 @@
|
|
1
1
|
|
2
|
+
== 2011-08-15 version 0.7.1
|
3
|
+
|
4
|
+
* Validate name of a database/table on create-database/create-log-table
|
5
|
+
subcommands
|
6
|
+
* Uses /v3/job/result?format=msgpack API to get result of a job
|
7
|
+
* -d, --database DB_NAME option is now required on query subcommand
|
8
|
+
* API server can be changed using TD_API_SERVER=HOST:PORT environment variable
|
9
|
+
|
10
|
+
|
2
11
|
== 2011-08-06 version 0.7.0
|
3
12
|
|
4
13
|
* import subcommand accepts UNIX time integer value on --json and --msgpack format
|
5
14
|
* Renamed command name (trd -> td)
|
6
15
|
|
16
|
+
|
7
17
|
== 2011-07-18 version 0.6.3
|
8
18
|
|
9
19
|
* show-jobs: shows elapsed time
|
data/lib/td/api.rb
CHANGED
@@ -99,7 +99,7 @@ class API
|
|
99
99
|
end
|
100
100
|
|
101
101
|
# => Job
|
102
|
-
def query(
|
102
|
+
def query(db_name, q)
|
103
103
|
job_id = @iface.hive_query(q, db_name)
|
104
104
|
Job.new(self, job_id, :hive, q) # TODO url
|
105
105
|
end
|
@@ -108,27 +108,54 @@ class API
|
|
108
108
|
def jobs(from=nil, to=nil)
|
109
109
|
js = @iface.list_jobs(from, to)
|
110
110
|
js.map {|job_id,type,status,query,start_at,end_at|
|
111
|
-
Job.new(self, job_id, type, query, status, nil, nil,
|
111
|
+
Job.new(self, job_id, type, query, status, nil, nil, start_at, end_at)
|
112
112
|
}
|
113
113
|
end
|
114
114
|
|
115
115
|
# => Job
|
116
116
|
def job(job_id)
|
117
117
|
job_id = job_id.to_s
|
118
|
-
type, query, status,
|
119
|
-
Job.new(self, job_id, type, query, status, url,
|
118
|
+
type, query, status, url, debug, start_at, end_at = @iface.show_job(job_id)
|
119
|
+
Job.new(self, job_id, type, query, status, url, debug, start_at, end_at)
|
120
120
|
end
|
121
121
|
|
122
|
-
# => type:Symbol,
|
122
|
+
# => type:Symbol, url:String
|
123
123
|
def job_status(job_id)
|
124
|
-
type, query, status,
|
125
|
-
return query, status,
|
124
|
+
type, query, status, url, debug, start_at, end_at = @iface.show_job(job_id)
|
125
|
+
return query, status, url, debug, start_at, end_at
|
126
|
+
end
|
127
|
+
|
128
|
+
# => result:[{column:String=>value:Object]
|
129
|
+
def job_result(job_id)
|
130
|
+
@iface.job_result(job_id)
|
131
|
+
end
|
132
|
+
|
133
|
+
# => nil
|
134
|
+
def job_result_each(job_id, &block)
|
135
|
+
@iface.job_result_each(job_id, &block)
|
126
136
|
end
|
127
137
|
|
128
138
|
# => time:Flaot
|
129
139
|
def import(db_name, table_name, format, stream, stream_size=stream.lstat.size)
|
130
140
|
@iface.import(db_name, table_name, format, stream, stream_size)
|
131
141
|
end
|
142
|
+
|
143
|
+
def self.validate_database_name(name)
|
144
|
+
name = name.to_s
|
145
|
+
if name.empty?
|
146
|
+
raise "Empty name is not allowed"
|
147
|
+
end
|
148
|
+
if name.length < 3 || 32 < name.length
|
149
|
+
raise "Name must be 3 to 32 characters, got #{name.length} characters."
|
150
|
+
end
|
151
|
+
unless name =~ /^([a-z0-9_]+)$/
|
152
|
+
raise "Name must consist only of alphabets, numbers, '_'."
|
153
|
+
end
|
154
|
+
end
|
155
|
+
|
156
|
+
def self.validate_table_name(name)
|
157
|
+
validate_database_name(name)
|
158
|
+
end
|
132
159
|
end
|
133
160
|
|
134
161
|
end
|
@@ -216,17 +243,17 @@ class Table < APIObject
|
|
216
243
|
end
|
217
244
|
|
218
245
|
class Job < APIObject
|
219
|
-
def initialize(api, job_id, type, query, status=nil, url=nil,
|
246
|
+
def initialize(api, job_id, type, query, status=nil, url=nil, debug=nil, start_at=nil, end_at=nil, result=nil)
|
220
247
|
super(api)
|
221
248
|
@job_id = job_id
|
222
249
|
@type = type
|
223
250
|
@url = url
|
224
251
|
@query = query
|
225
252
|
@status = status
|
226
|
-
@result = result
|
227
253
|
@debug = debug
|
228
254
|
@start_at = start_at
|
229
255
|
@end_at = end_at
|
256
|
+
@result = result
|
230
257
|
end
|
231
258
|
|
232
259
|
attr_reader :job_id, :type
|
@@ -266,12 +293,20 @@ class Job < APIObject
|
|
266
293
|
end
|
267
294
|
|
268
295
|
def result
|
269
|
-
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
296
|
+
unless @result
|
297
|
+
return nil unless finished?
|
298
|
+
@result = @api.job_result(@job_id)
|
299
|
+
end
|
300
|
+
@result
|
301
|
+
end
|
302
|
+
|
303
|
+
def result_each(&block)
|
304
|
+
if @result
|
305
|
+
@result.each(&block)
|
306
|
+
else
|
307
|
+
@api.job_result_each(&block)
|
308
|
+
end
|
309
|
+
nil
|
275
310
|
end
|
276
311
|
|
277
312
|
def finished?
|
@@ -287,11 +322,18 @@ class Job < APIObject
|
|
287
322
|
!finished?
|
288
323
|
end
|
289
324
|
|
325
|
+
def success?
|
326
|
+
@status == "success"
|
327
|
+
end
|
328
|
+
|
329
|
+
def error?
|
330
|
+
@status == "error"
|
331
|
+
end
|
332
|
+
|
290
333
|
def update_status!
|
291
|
-
query, status,
|
334
|
+
query, status, url, debug, start_at, end_at = @api.job_status(@job_id)
|
292
335
|
@query = query
|
293
336
|
@status = status
|
294
|
-
@result = result
|
295
337
|
@url = url
|
296
338
|
@debug = debug
|
297
339
|
@start_at = start_at
|
data/lib/td/api_iface.rb
CHANGED
@@ -141,12 +141,46 @@ class APIInterface
|
|
141
141
|
type = (js['type'] || '?').to_sym # TODO
|
142
142
|
query = js['query']
|
143
143
|
status = js['status']
|
144
|
-
result = js['result']
|
145
144
|
debug = js['debug']
|
146
145
|
url = js['url']
|
147
146
|
start_at = js['start_at']
|
148
147
|
end_at = js['end_at']
|
149
|
-
return [type, query, status,
|
148
|
+
return [type, query, status, url, debug, start_at, end_at]
|
149
|
+
end
|
150
|
+
|
151
|
+
def job_result(job_id)
|
152
|
+
require 'msgpack'
|
153
|
+
code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>'msgpack'})
|
154
|
+
if code != "200"
|
155
|
+
raise_error("Get job result failed", res)
|
156
|
+
end
|
157
|
+
result = []
|
158
|
+
MessagePack::Unpacker.new.feed_each(body) {|row|
|
159
|
+
result << row
|
160
|
+
}
|
161
|
+
return result
|
162
|
+
end
|
163
|
+
|
164
|
+
def job_result_each(job_id, &block)
|
165
|
+
# TODO chunked encoding
|
166
|
+
require 'msgpack'
|
167
|
+
code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>'msgpack'})
|
168
|
+
if code != "200"
|
169
|
+
raise_error("Get job result failed", res)
|
170
|
+
end
|
171
|
+
result = []
|
172
|
+
MessagePack::Unpacker.new.feed_each(body) {|row|
|
173
|
+
yield row
|
174
|
+
}
|
175
|
+
nil
|
176
|
+
end
|
177
|
+
|
178
|
+
def job_result_raw(job_id, format)
|
179
|
+
code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>format})
|
180
|
+
if code != "200"
|
181
|
+
raise_error("Get job result failed", res)
|
182
|
+
end
|
183
|
+
return body
|
150
184
|
end
|
151
185
|
|
152
186
|
# => jobId:String
|
@@ -211,8 +245,16 @@ class APIInterface
|
|
211
245
|
end
|
212
246
|
|
213
247
|
private
|
214
|
-
|
215
|
-
|
248
|
+
host = 'api.treasure-data.com'
|
249
|
+
port = 80
|
250
|
+
if e = ENV['TD_API_SERVER']
|
251
|
+
host, port_ = e.split(':',2)
|
252
|
+
port_ = port_.to_i
|
253
|
+
port = port_ if port_ != 0
|
254
|
+
end
|
255
|
+
|
256
|
+
HOST = host
|
257
|
+
PORT = port
|
216
258
|
USE_SSL = false
|
217
259
|
BASE_URL = ''
|
218
260
|
|
data/lib/td/command/database.rb
CHANGED
@@ -9,6 +9,8 @@ module Command
|
|
9
9
|
conf = cmd_config
|
10
10
|
api = cmd_api(conf)
|
11
11
|
|
12
|
+
API.validate_database_name(db_name)
|
13
|
+
|
12
14
|
begin
|
13
15
|
api.create_database(db_name)
|
14
16
|
rescue AlreadyExistsError
|
@@ -76,7 +78,6 @@ module Command
|
|
76
78
|
alias show_dbs show_databases
|
77
79
|
alias create_db create_database
|
78
80
|
alias drop_db drop_database
|
79
|
-
|
80
81
|
end
|
81
82
|
end
|
82
83
|
|
data/lib/td/command/query.rb
CHANGED
@@ -8,7 +8,7 @@ module Command
|
|
8
8
|
op.banner << "\noptions:\n"
|
9
9
|
|
10
10
|
db_name = nil
|
11
|
-
op.on('-d', '--database DB_NAME', 'use the database') {|s|
|
11
|
+
op.on('-d', '--database DB_NAME', 'use the database (required)') {|s|
|
12
12
|
db_name = s
|
13
13
|
}
|
14
14
|
|
@@ -22,11 +22,14 @@ module Command
|
|
22
22
|
conf = cmd_config
|
23
23
|
api = cmd_api(conf)
|
24
24
|
|
25
|
-
|
26
|
-
|
25
|
+
unless db_name
|
26
|
+
$stderr.puts "-d, --database DB_NAME option is required."
|
27
|
+
exit 1
|
27
28
|
end
|
28
29
|
|
29
|
-
|
30
|
+
find_database(api, db_name)
|
31
|
+
|
32
|
+
job = api.query(db_name, sql)
|
30
33
|
|
31
34
|
$stderr.puts "Job #{job.job_id} is started."
|
32
35
|
$stderr.puts "Use '#{$prog} job #{job.job_id}' to show the status."
|
@@ -36,7 +39,7 @@ module Command
|
|
36
39
|
wait_job(job)
|
37
40
|
puts "Status : #{job.status}"
|
38
41
|
puts "Result :"
|
39
|
-
puts
|
42
|
+
puts render_result(job.result)
|
40
43
|
end
|
41
44
|
end
|
42
45
|
|
@@ -116,12 +119,12 @@ module Command
|
|
116
119
|
if wait && !job.finished?
|
117
120
|
wait_job(job)
|
118
121
|
puts "Result :"
|
119
|
-
puts
|
122
|
+
puts render_result(job.result)
|
120
123
|
|
121
124
|
else
|
122
125
|
if job.finished?
|
123
126
|
puts "Result :"
|
124
|
-
puts
|
127
|
+
puts render_result(job.result)
|
125
128
|
end
|
126
129
|
|
127
130
|
if verbose
|
@@ -162,6 +165,20 @@ module Command
|
|
162
165
|
stderr_lines += stderr.size
|
163
166
|
end
|
164
167
|
end
|
168
|
+
|
169
|
+
def render_result(result)
|
170
|
+
require 'json'
|
171
|
+
rows = result.map {|row|
|
172
|
+
row.map {|v|
|
173
|
+
if v.is_a?(String)
|
174
|
+
v.to_s
|
175
|
+
else
|
176
|
+
v.to_json
|
177
|
+
end
|
178
|
+
}
|
179
|
+
}
|
180
|
+
puts cmd_render_table(rows, :max_width=>10000)
|
181
|
+
end
|
165
182
|
end
|
166
183
|
end
|
167
184
|
|
data/lib/td/command/table.rb
CHANGED
data/lib/td/version.rb
CHANGED
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: td
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 1
|
5
5
|
prerelease: false
|
6
6
|
segments:
|
7
7
|
- 0
|
8
8
|
- 7
|
9
|
-
-
|
10
|
-
version: 0.7.
|
9
|
+
- 1
|
10
|
+
version: 0.7.1
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Sadayuki Furuhashi
|
@@ -15,7 +15,7 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2011-08-
|
18
|
+
date: 2011-08-15 00:00:00 +09:00
|
19
19
|
default_executable: td
|
20
20
|
dependencies:
|
21
21
|
- !ruby/object:Gem::Dependency
|