td 0.11.1 → 0.11.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +8 -8
- data/ChangeLog +29 -0
- data/README.rdoc +64 -0
- data/Rakefile +11 -0
- data/dist/exe.rake +1 -1
- data/dist/pkg.rake +2 -2
- data/lib/td/command/account.rb +18 -5
- data/lib/td/command/acl.rb +1 -1
- data/lib/td/command/apikey.rb +2 -3
- data/lib/td/command/bulk_import.rb +7 -7
- data/lib/td/command/common.rb +91 -6
- data/lib/td/command/db.rb +2 -2
- data/lib/td/command/export.rb +1 -1
- data/lib/td/command/import.rb +48 -18
- data/lib/td/command/job.rb +11 -64
- data/lib/td/command/list.rb +6 -5
- data/lib/td/command/query.rb +1 -4
- data/lib/td/command/result.rb +5 -3
- data/lib/td/command/runner.rb +33 -12
- data/lib/td/command/sample.rb +4 -4
- data/lib/td/command/sched.rb +7 -8
- data/lib/td/command/server.rb +21 -0
- data/lib/td/command/status.rb +3 -3
- data/lib/td/command/table.rb +7 -8
- data/lib/td/command/update.rb +3 -3
- data/lib/td/command/user.rb +7 -7
- data/lib/td/config.rb +45 -20
- data/lib/td/updater.rb +391 -378
- data/lib/td/version.rb +1 -1
- data/td.gemspec +14 -14
- metadata +6 -6
data/lib/td/command/db.rb
CHANGED
@@ -41,7 +41,7 @@ module Command
|
|
41
41
|
|
42
42
|
if dbs.empty?
|
43
43
|
$stderr.puts "There are no databases."
|
44
|
-
$stderr.puts "Use '#{$prog} " + Config.
|
44
|
+
$stderr.puts "Use '#{$prog} " + Config.cl_options_string + "db:create <db_name>' to create a database."
|
45
45
|
end
|
46
46
|
end
|
47
47
|
|
@@ -61,7 +61,7 @@ module Command
|
|
61
61
|
end
|
62
62
|
|
63
63
|
$stderr.puts "Database '#{db_name}' is created."
|
64
|
-
$stderr.puts "Use '#{$prog} " + Config.
|
64
|
+
$stderr.puts "Use '#{$prog} " + Config.cl_options_string + "table:create #{db_name} <table_name>' to create a table."
|
65
65
|
end
|
66
66
|
|
67
67
|
def db_delete(op)
|
data/lib/td/command/export.rb
CHANGED
@@ -65,7 +65,7 @@ module Command
|
|
65
65
|
job = client.export(db_name, table_name, "s3", s3_opts)
|
66
66
|
|
67
67
|
$stderr.puts "Export job #{job.job_id} is queued."
|
68
|
-
$stderr.puts "Use '#{$prog} " + Config.
|
68
|
+
$stderr.puts "Use '#{$prog} " + Config.cl_options_string + "job:show #{job.job_id}' to show the status."
|
69
69
|
end
|
70
70
|
|
71
71
|
private
|
data/lib/td/command/import.rb
CHANGED
@@ -150,7 +150,16 @@ module Command
|
|
150
150
|
|
151
151
|
private
|
152
152
|
def set_sysprops_endpoint(sysprops)
|
153
|
-
|
153
|
+
# optional, if not provided a default is used from the ruby client library
|
154
|
+
begin
|
155
|
+
endpoint = Config.endpoint
|
156
|
+
rescue ConfigNotFoundError => e
|
157
|
+
# rescue the ConfigNotFoundError exception which originates when
|
158
|
+
# the config file is not found or an endpoint is not provided
|
159
|
+
# either through -e / --endpoint or TD_API_SERVER environment
|
160
|
+
# variable.
|
161
|
+
end
|
162
|
+
|
154
163
|
if endpoint
|
155
164
|
require 'uri'
|
156
165
|
|
@@ -160,23 +169,31 @@ module Command
|
|
160
169
|
when 'http', 'https'
|
161
170
|
host = uri.host
|
162
171
|
port = uri.port
|
163
|
-
|
164
|
-
|
165
|
-
|
172
|
+
# NOTE: Config.secure option is ignored in favor
|
173
|
+
# of defining ssl based on the URL scheme
|
174
|
+
ssl = (uri.scheme == 'https')
|
166
175
|
else
|
167
|
-
if
|
168
|
-
|
176
|
+
# uri scheme is not set if the endpoint is
|
177
|
+
# like 'api.treasuredata.com:80'. In that case the
|
178
|
+
# uri object does not contain any useful info.
|
179
|
+
if uri.port # invalid URI
|
169
180
|
raise "Invalid endpoint: #{endpoint}"
|
170
181
|
end
|
171
182
|
|
172
183
|
# generic URI
|
173
184
|
host, port = endpoint.split(':', 2)
|
174
185
|
port = port.to_i
|
175
|
-
#
|
176
|
-
|
177
|
-
|
178
|
-
|
186
|
+
# Config.secure = false is the --insecure option was used
|
187
|
+
if Config.secure
|
188
|
+
port = 443 if port == 0
|
189
|
+
ssl = true
|
190
|
+
else
|
191
|
+
port = 80 if port == 0
|
192
|
+
ssl = false
|
193
|
+
end
|
194
|
+
end
|
179
195
|
|
196
|
+
sysprops << "-Dtd.api.server.scheme=#{ssl ? 'https' : 'http'}://"
|
180
197
|
sysprops << "-Dtd.api.server.host=#{host}"
|
181
198
|
sysprops << "-Dtd.api.server.port=#{port}"
|
182
199
|
end
|
@@ -184,17 +201,30 @@ module Command
|
|
184
201
|
|
185
202
|
private
|
186
203
|
def set_sysprops_http_proxy(sysprops)
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
204
|
+
uri_string = ENV['HTTP_PROXY']
|
205
|
+
return unless uri_string
|
206
|
+
|
207
|
+
require 'uri'
|
208
|
+
uri = URI.parse(uri_string)
|
209
|
+
proxy_host = nil
|
210
|
+
proxy_port = nil
|
211
|
+
|
212
|
+
case uri.scheme
|
213
|
+
when 'http'
|
214
|
+
if uri.host
|
215
|
+
proxy_host = uri.host # host is required
|
216
|
+
proxy_port = uri.port if uri.port # default value of uri.port is 80.
|
191
217
|
end
|
192
|
-
|
218
|
+
when 'https'
|
219
|
+
raise ParameterConfigurationError,
|
220
|
+
"HTTP proxy URL must use 'http' protocol. Example format: 'http://localhost:3128'."
|
221
|
+
else
|
222
|
+
proxy_host, proxy_port = uri_string.split(':', 2)
|
193
223
|
proxy_port = (proxy_port ? proxy_port.to_i : 80)
|
194
|
-
|
195
|
-
sysprops << "-Dhttp.proxyHost=#{proxy_host}"
|
196
|
-
sysprops << "-Dhttp.proxyPort=#{proxy_port}"
|
197
224
|
end
|
225
|
+
|
226
|
+
sysprops << "-Dhttp.proxyHost=#{proxy_host}" if proxy_host
|
227
|
+
sysprops << "-Dhttp.proxyPort=#{proxy_port}" if proxy_port
|
198
228
|
end
|
199
229
|
|
200
230
|
end
|
data/lib/td/command/job.rb
CHANGED
@@ -72,8 +72,8 @@ module Command
|
|
72
72
|
rows = []
|
73
73
|
jobs.each {|job|
|
74
74
|
start = job.start_at
|
75
|
-
elapsed = humanize_elapsed_time(start, job.end_at)
|
76
|
-
cpu_time = humanize_time(job.cpu_time, true)
|
75
|
+
elapsed = Command.humanize_elapsed_time(start, job.end_at)
|
76
|
+
cpu_time = Command.humanize_time(job.cpu_time, true)
|
77
77
|
priority = job_priority_name_of(job.priority)
|
78
78
|
rows << {
|
79
79
|
:JobID => job.job_id,
|
@@ -181,7 +181,7 @@ module Command
|
|
181
181
|
end
|
182
182
|
# if the job is done and is of type hive, show the Map-Reduce cumulated CPU time
|
183
183
|
if job.finished? && [:hive].include?(job.type)
|
184
|
-
puts "CPU time : #{humanize_time(job.cpu_time, true)}"
|
184
|
+
puts "CPU time : #{Command.humanize_time(job.cpu_time, true)}"
|
185
185
|
end
|
186
186
|
|
187
187
|
if wait && !job.finished?
|
@@ -196,7 +196,7 @@ module Command
|
|
196
196
|
end
|
197
197
|
|
198
198
|
else
|
199
|
-
if [:hive, :pig, :impala, :presto].include?(job.type) && !exclude
|
199
|
+
if [:hive, :pig, :impala, :presto].include?(job.type) && !exclude && job.finished?
|
200
200
|
puts "Result :"
|
201
201
|
begin
|
202
202
|
show_result(job, output, limit, format, render_opts)
|
@@ -294,26 +294,6 @@ module Command
|
|
294
294
|
|
295
295
|
def write_result(job, output, limit, format, render_opts={})
|
296
296
|
|
297
|
-
# start progress indicator
|
298
|
-
line_len = 0
|
299
|
-
start_time = last_time = Time.now.to_i
|
300
|
-
base_msg = "WARNING: the query result is being written"
|
301
|
-
if !output.nil?
|
302
|
-
msg = base_msg + "\r"
|
303
|
-
line_len += msg.length
|
304
|
-
print msg
|
305
|
-
end
|
306
|
-
|
307
|
-
# this lambda is passed on as a block when outputing the results to file
|
308
|
-
progress = lambda {
|
309
|
-
if (time = Time.now.to_i) - last_time > 5
|
310
|
-
msg = base_msg + ": #{humanize_time(time - start_time)} elapsed" + " " * 10 + "\r"
|
311
|
-
line_len = msg.length
|
312
|
-
print msg
|
313
|
-
last_time = time
|
314
|
-
end
|
315
|
-
}
|
316
|
-
|
317
297
|
# the next 3 formats allow writing to both a file and stdout
|
318
298
|
|
319
299
|
case format
|
@@ -327,9 +307,6 @@ module Command
|
|
327
307
|
f.write Yajl.dump(row)
|
328
308
|
n_rows += 1
|
329
309
|
break if output.nil? and !limit.nil? and n_rows == limit
|
330
|
-
|
331
|
-
# update progress indicator
|
332
|
-
progress.call if !output.nil?
|
333
310
|
}
|
334
311
|
f.write "]"
|
335
312
|
}
|
@@ -357,9 +334,6 @@ module Command
|
|
357
334
|
n_rows += 1
|
358
335
|
writer.flush if n_rows % 100 == 0 # flush every 100 recods
|
359
336
|
break if output.nil? and !limit.nil? and n_rows == limit
|
360
|
-
|
361
|
-
# update progress indicator
|
362
|
-
progress.call if !output.nil?
|
363
337
|
}
|
364
338
|
}
|
365
339
|
|
@@ -388,9 +362,6 @@ module Command
|
|
388
362
|
n_rows += 1
|
389
363
|
f.flush if n_rows % 100 == 0 # flush every 100 recods
|
390
364
|
break if output.nil? and !limit.nil? and n_rows == limit
|
391
|
-
|
392
|
-
# update progress indicator
|
393
|
-
progress.call if !output.nil?
|
394
365
|
}
|
395
366
|
}
|
396
367
|
|
@@ -402,7 +373,7 @@ module Command
|
|
402
373
|
"Format 'msgpack' does not support writing to stdout"
|
403
374
|
end
|
404
375
|
open_file(output, "wb") {|f|
|
405
|
-
job.result_format('msgpack', f
|
376
|
+
job.result_format('msgpack', f)
|
406
377
|
}
|
407
378
|
|
408
379
|
when 'msgpack.gz'
|
@@ -411,18 +382,12 @@ module Command
|
|
411
382
|
"Format 'msgpack' does not support writing to stdout"
|
412
383
|
end
|
413
384
|
open_file(output, "wb") {|f|
|
414
|
-
job.result_format('msgpack.gz', f
|
385
|
+
job.result_format('msgpack.gz', f)
|
415
386
|
}
|
416
387
|
|
417
388
|
else
|
418
389
|
raise "Unknown format #{format.inspect}"
|
419
390
|
end
|
420
|
-
|
421
|
-
# clear the progress indicator
|
422
|
-
if !output.nil?
|
423
|
-
print "\r"
|
424
|
-
print " " * (line_len + 5) + "\r"
|
425
|
-
end
|
426
391
|
end
|
427
392
|
|
428
393
|
def open_file(output, mode)
|
@@ -443,17 +408,10 @@ module Command
|
|
443
408
|
require 'yajl'
|
444
409
|
|
445
410
|
if format.nil?
|
446
|
-
|
447
|
-
# start progress indicator
|
448
|
-
start_time = last_time = Time.now.to_i
|
449
|
-
base_msg = "WARNING: the query result is being downloaded"
|
450
|
-
msg = base_msg + "\r"
|
451
|
-
line_len = msg.length
|
452
|
-
print msg
|
453
|
-
|
454
411
|
# display result in tabular format
|
455
412
|
rows = []
|
456
413
|
n_rows = 0
|
414
|
+
print "WARNING: the query result is being downloaded...\r"
|
457
415
|
job.result_each {|row|
|
458
416
|
# TODO limit number of rows to show
|
459
417
|
rows << row.map {|v|
|
@@ -461,20 +419,8 @@ module Command
|
|
461
419
|
}
|
462
420
|
n_rows += 1
|
463
421
|
break if !limit.nil? and n_rows == limit
|
464
|
-
|
465
|
-
# progress indication
|
466
|
-
time = Time.now.to_i
|
467
|
-
if time - last_time > 5
|
468
|
-
msg = base_msg + ": #{humanize_time(time - start_time)} elapsed" + " " * 10 + "\r"
|
469
|
-
line_len = msg.length
|
470
|
-
print msg
|
471
|
-
last_time = time
|
472
|
-
end
|
473
422
|
}
|
474
|
-
|
475
|
-
# clear the progress indicator
|
476
|
-
print "\r"
|
477
|
-
print " " * (line_len + 5) + "\r"
|
423
|
+
print " " * 100, "\r" # make sure the previous WARNING is cleared over
|
478
424
|
|
479
425
|
render_opts[:max_width] = 10000
|
480
426
|
if job.hive_result_schema
|
@@ -512,6 +458,7 @@ module Command
|
|
512
458
|
}
|
513
459
|
return nil
|
514
460
|
end
|
515
|
-
|
516
|
-
end
|
461
|
+
|
462
|
+
end # module Command
|
463
|
+
end # module TrasureData
|
517
464
|
|
data/lib/td/command/list.rb
CHANGED
@@ -1,4 +1,3 @@
|
|
1
|
-
|
2
1
|
module TreasureData
|
3
2
|
module Command
|
4
3
|
module List
|
@@ -46,7 +45,7 @@ module List
|
|
46
45
|
|
47
46
|
def cmd_usage(msg=nil)
|
48
47
|
puts self.to_s
|
49
|
-
puts "
|
48
|
+
puts "Error: #{msg}" if msg
|
50
49
|
exit 1
|
51
50
|
end
|
52
51
|
end
|
@@ -309,6 +308,7 @@ module List
|
|
309
308
|
# TODO acl:test
|
310
309
|
|
311
310
|
add_list 'server:status', %w[], 'Show status of the Treasure Data server'
|
311
|
+
add_list 'server:endpoint', %w[api_endpoint], "Set the Treasure Data API server's endpoint (must be a valid URI)", "td server:endpoint 'https://api.treasuredata.com'"
|
312
312
|
|
313
313
|
add_list 'sample:apache', %w[path.json], 'Create a sample log file'
|
314
314
|
|
@@ -396,7 +396,8 @@ module List
|
|
396
396
|
add_guess 'server-status', 'server:status'
|
397
397
|
|
398
398
|
finishup
|
399
|
-
|
400
|
-
end
|
401
|
-
end
|
399
|
+
|
400
|
+
end # module List
|
401
|
+
end # module Command
|
402
|
+
end # module TreasureData
|
402
403
|
|
data/lib/td/command/query.rb
CHANGED
@@ -119,9 +119,6 @@ module Command
|
|
119
119
|
if result_url
|
120
120
|
require 'td/command/result'
|
121
121
|
result_url = build_result_url(result_url, result_user, result_ask_password)
|
122
|
-
if result_url =~ /^td:/
|
123
|
-
validate_td_result_url(result_url)
|
124
|
-
end
|
125
122
|
end
|
126
123
|
|
127
124
|
client = get_client
|
@@ -134,7 +131,7 @@ module Command
|
|
134
131
|
job = client.query(db_name, sql, result_url, priority, retry_limit, opts)
|
135
132
|
|
136
133
|
puts "Job #{job.job_id} is queued."
|
137
|
-
puts "Use '#{$prog} " + Config.
|
134
|
+
puts "Use '#{$prog} " + Config.cl_options_string + "job:show #{job.job_id}' to show the status."
|
138
135
|
#puts "See #{job.url} to see the progress."
|
139
136
|
|
140
137
|
if wait
|
data/lib/td/command/result.rb
CHANGED
@@ -11,7 +11,7 @@ module Command
|
|
11
11
|
|
12
12
|
unless r
|
13
13
|
$stderr.puts "Result URL '#{name}' does not exist."
|
14
|
-
$stderr.puts "Use '#{$prog} " + Config.
|
14
|
+
$stderr.puts "Use '#{$prog} " + Config.cl_options_string + "result:create #{name} <URL>' to create the URL."
|
15
15
|
exit 1
|
16
16
|
end
|
17
17
|
|
@@ -40,7 +40,7 @@ module Command
|
|
40
40
|
|
41
41
|
if rs.empty?
|
42
42
|
$stderr.puts "There are no result URLs."
|
43
|
-
$stderr.puts "Use '#{$prog} " + Config.
|
43
|
+
$stderr.puts "Use '#{$prog} " + Config.cl_options_string + "result:create <name> <url>' to create a result URL."
|
44
44
|
end
|
45
45
|
end
|
46
46
|
|
@@ -123,9 +123,11 @@ module Command
|
|
123
123
|
url
|
124
124
|
end
|
125
125
|
|
126
|
+
# DEPRECATED: relying on API server side validation which will return
|
127
|
+
# immediately after query submission with error code 422.
|
126
128
|
private
|
127
129
|
def validate_td_result_url(url)
|
128
|
-
re = /td:\/\/[^@]*@\/(.*)\/(
|
130
|
+
re = /td:\/\/[^@]*@\/(.*)\/([^?]+)/
|
129
131
|
match = re.match(url)
|
130
132
|
if match.nil?
|
131
133
|
raise ParameterConfigurationError, "Treasure Data result output invalid URL format"
|
data/lib/td/command/runner.rb
CHANGED
@@ -1,4 +1,3 @@
|
|
1
|
-
|
2
1
|
module TreasureData
|
3
2
|
module Command
|
4
3
|
|
@@ -6,11 +5,12 @@ class Runner
|
|
6
5
|
def initialize
|
7
6
|
@config_path = nil
|
8
7
|
@apikey = nil
|
8
|
+
@endpoint = nil
|
9
9
|
@prog_name = nil
|
10
|
-
@
|
10
|
+
@insecure = false
|
11
11
|
end
|
12
12
|
|
13
|
-
attr_accessor :apikey, :config_path, :prog_name, :
|
13
|
+
attr_accessor :apikey, :endpoint, :config_path, :prog_name, :insecure
|
14
14
|
|
15
15
|
def run(argv=ARGV)
|
16
16
|
require 'td/version'
|
@@ -58,7 +58,7 @@ Additional commands:
|
|
58
58
|
Type 'td help COMMAND' for more information on a specific command.
|
59
59
|
EOF
|
60
60
|
if errmsg
|
61
|
-
puts "
|
61
|
+
puts "Error: #{errmsg}"
|
62
62
|
exit 1
|
63
63
|
else
|
64
64
|
exit 0
|
@@ -66,13 +66,16 @@ EOF
|
|
66
66
|
end
|
67
67
|
end
|
68
68
|
|
69
|
+
# there local vars are loaded with the values of the options below
|
70
|
+
# here they are preloaded with the defaults
|
69
71
|
config_path = @config_path
|
70
72
|
apikey = @apikey
|
73
|
+
endpoint = @endpoint
|
71
74
|
insecure = nil
|
72
75
|
$verbose = false
|
73
76
|
#$debug = false
|
74
77
|
|
75
|
-
op.on('-c', '--config PATH', "path to
|
78
|
+
op.on('-c', '--config PATH', "path to the configuration file (default: ~/.td/td.conf)") {|s|
|
76
79
|
config_path = s
|
77
80
|
}
|
78
81
|
|
@@ -80,7 +83,15 @@ EOF
|
|
80
83
|
apikey = s
|
81
84
|
}
|
82
85
|
|
83
|
-
op.on('--
|
86
|
+
op.on('-e', '--endpoint API_SERVER', "specify the URL for API server to use (default: https://api.treasuredata.com)." ,
|
87
|
+
" The URL must contain a scheme (http:// or https:// prefix) to be valid.",
|
88
|
+
" Valid IPv4 addresses are accepted as well in place of the host name.") {|e|
|
89
|
+
require 'td/command/common'
|
90
|
+
Command.validate_api_endpoint(e)
|
91
|
+
endpoint = e
|
92
|
+
}
|
93
|
+
|
94
|
+
op.on('--insecure', "Insecure access: disable SSL (enabled by default)") {|b|
|
84
95
|
insecure = true
|
85
96
|
}
|
86
97
|
|
@@ -106,6 +117,8 @@ EOF
|
|
106
117
|
usage nil if argv.empty?
|
107
118
|
cmd = argv.shift
|
108
119
|
|
120
|
+
# NOTE: these information are loaded from by each command through
|
121
|
+
# 'TreasureData::Command::get_client' from 'lib/td/command/common.rb'
|
109
122
|
require 'td/config'
|
110
123
|
if config_path
|
111
124
|
Config.path = config_path
|
@@ -114,6 +127,10 @@ EOF
|
|
114
127
|
Config.apikey = apikey
|
115
128
|
Config.cl_apikey = true
|
116
129
|
end
|
130
|
+
if endpoint
|
131
|
+
Config.endpoint = endpoint
|
132
|
+
Config.cl_endpoint = true
|
133
|
+
end
|
117
134
|
if insecure
|
118
135
|
Config.secure = false
|
119
136
|
end
|
@@ -145,8 +162,13 @@ EOF
|
|
145
162
|
# List of new exceptions:
|
146
163
|
# => ParameterConfigurationError
|
147
164
|
# => BulkImportExecutionError
|
148
|
-
|
149
|
-
|
165
|
+
# => UpUpdateError
|
166
|
+
require 'td/client/api'
|
167
|
+
# => APIError
|
168
|
+
unless [ParameterConfigurationError, BulkImportExecutionError, UpdateError,
|
169
|
+
APIError]
|
170
|
+
.include?(e.class) && ENV['TD_TOOLBELT_DEBUG'].nil?
|
171
|
+
$stderr.puts "Error #{$!.class}: backtrace:"
|
150
172
|
$!.backtrace.each {|b|
|
151
173
|
$stderr.puts " #{b}"
|
152
174
|
}
|
@@ -167,9 +189,8 @@ EOS
|
|
167
189
|
end
|
168
190
|
return 0
|
169
191
|
end
|
170
|
-
end
|
171
|
-
|
192
|
+
end # class Runner
|
172
193
|
|
173
|
-
end
|
174
|
-
end
|
194
|
+
end # module Command
|
195
|
+
end # module TreasureData
|
175
196
|
|