td 0.11.1 → 0.11.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -41,7 +41,7 @@ module Command
41
41
 
42
42
  if dbs.empty?
43
43
  $stderr.puts "There are no databases."
44
- $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "db:create <db_name>' to create a database."
44
+ $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "db:create <db_name>' to create a database."
45
45
  end
46
46
  end
47
47
 
@@ -61,7 +61,7 @@ module Command
61
61
  end
62
62
 
63
63
  $stderr.puts "Database '#{db_name}' is created."
64
- $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "table:create #{db_name} <table_name>' to create a table."
64
+ $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "table:create #{db_name} <table_name>' to create a table."
65
65
  end
66
66
 
67
67
  def db_delete(op)
@@ -65,7 +65,7 @@ module Command
65
65
  job = client.export(db_name, table_name, "s3", s3_opts)
66
66
 
67
67
  $stderr.puts "Export job #{job.job_id} is queued."
68
- $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "job:show #{job.job_id}' to show the status."
68
+ $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "job:show #{job.job_id}' to show the status."
69
69
  end
70
70
 
71
71
  private
@@ -150,7 +150,16 @@ module Command
150
150
 
151
151
  private
152
152
  def set_sysprops_endpoint(sysprops)
153
- endpoint = ENV['TD_API_SERVER']
153
+ # optional, if not provided a default is used from the ruby client library
154
+ begin
155
+ endpoint = Config.endpoint
156
+ rescue ConfigNotFoundError => e
157
+ # rescue the ConfigNotFoundError exception which originates when
158
+ # the config file is not found or an endpoint is not provided
159
+ # either through -e / --endpoint or TD_API_SERVER environment
160
+ # variable.
161
+ end
162
+
154
163
  if endpoint
155
164
  require 'uri'
156
165
 
@@ -160,23 +169,31 @@ module Command
160
169
  when 'http', 'https'
161
170
  host = uri.host
162
171
  port = uri.port
163
- ssl = uri.scheme == 'https'
164
-
165
- port = 80 if port == 443 and ssl
172
+ # NOTE: Config.secure option is ignored in favor
173
+ # of defining ssl based on the URL scheme
174
+ ssl = (uri.scheme == 'https')
166
175
  else
167
- if uri.port
168
- # invalid URI
176
+ # uri scheme is not set if the endpoint is
177
+ # like 'api.treasuredata.com:80'. In that case the
178
+ # uri object does not contain any useful info.
179
+ if uri.port # invalid URI
169
180
  raise "Invalid endpoint: #{endpoint}"
170
181
  end
171
182
 
172
183
  # generic URI
173
184
  host, port = endpoint.split(':', 2)
174
185
  port = port.to_i
175
- # TODO support ssl
176
- port = 80 if port == 0
177
- ssl = false
178
- end
186
+ # Config.secure = false is the --insecure option was used
187
+ if Config.secure
188
+ port = 443 if port == 0
189
+ ssl = true
190
+ else
191
+ port = 80 if port == 0
192
+ ssl = false
193
+ end
194
+ end
179
195
 
196
+ sysprops << "-Dtd.api.server.scheme=#{ssl ? 'https' : 'http'}://"
180
197
  sysprops << "-Dtd.api.server.host=#{host}"
181
198
  sysprops << "-Dtd.api.server.port=#{port}"
182
199
  end
@@ -184,17 +201,30 @@ module Command
184
201
 
185
202
  private
186
203
  def set_sysprops_http_proxy(sysprops)
187
- http_proxy = ENV['HTTP_PROXY']
188
- if http_proxy
189
- if http_proxy =~ /\Ahttp:\/\/(.*)\z/
190
- http_proxy = $~[1]
204
+ uri_string = ENV['HTTP_PROXY']
205
+ return unless uri_string
206
+
207
+ require 'uri'
208
+ uri = URI.parse(uri_string)
209
+ proxy_host = nil
210
+ proxy_port = nil
211
+
212
+ case uri.scheme
213
+ when 'http'
214
+ if uri.host
215
+ proxy_host = uri.host # host is required
216
+ proxy_port = uri.port if uri.port # default value of uri.port is 80.
191
217
  end
192
- proxy_host, proxy_port = http_proxy.split(':', 2)
218
+ when 'https'
219
+ raise ParameterConfigurationError,
220
+ "HTTP proxy URL must use 'http' protocol. Example format: 'http://localhost:3128'."
221
+ else
222
+ proxy_host, proxy_port = uri_string.split(':', 2)
193
223
  proxy_port = (proxy_port ? proxy_port.to_i : 80)
194
-
195
- sysprops << "-Dhttp.proxyHost=#{proxy_host}"
196
- sysprops << "-Dhttp.proxyPort=#{proxy_port}"
197
224
  end
225
+
226
+ sysprops << "-Dhttp.proxyHost=#{proxy_host}" if proxy_host
227
+ sysprops << "-Dhttp.proxyPort=#{proxy_port}" if proxy_port
198
228
  end
199
229
 
200
230
  end
@@ -72,8 +72,8 @@ module Command
72
72
  rows = []
73
73
  jobs.each {|job|
74
74
  start = job.start_at
75
- elapsed = humanize_elapsed_time(start, job.end_at)
76
- cpu_time = humanize_time(job.cpu_time, true)
75
+ elapsed = Command.humanize_elapsed_time(start, job.end_at)
76
+ cpu_time = Command.humanize_time(job.cpu_time, true)
77
77
  priority = job_priority_name_of(job.priority)
78
78
  rows << {
79
79
  :JobID => job.job_id,
@@ -181,7 +181,7 @@ module Command
181
181
  end
182
182
  # if the job is done and is of type hive, show the Map-Reduce cumulated CPU time
183
183
  if job.finished? && [:hive].include?(job.type)
184
- puts "CPU time : #{humanize_time(job.cpu_time, true)}"
184
+ puts "CPU time : #{Command.humanize_time(job.cpu_time, true)}"
185
185
  end
186
186
 
187
187
  if wait && !job.finished?
@@ -196,7 +196,7 @@ module Command
196
196
  end
197
197
 
198
198
  else
199
- if [:hive, :pig, :impala, :presto].include?(job.type) && !exclude
199
+ if [:hive, :pig, :impala, :presto].include?(job.type) && !exclude && job.finished?
200
200
  puts "Result :"
201
201
  begin
202
202
  show_result(job, output, limit, format, render_opts)
@@ -294,26 +294,6 @@ module Command
294
294
 
295
295
  def write_result(job, output, limit, format, render_opts={})
296
296
 
297
- # start progress indicator
298
- line_len = 0
299
- start_time = last_time = Time.now.to_i
300
- base_msg = "WARNING: the query result is being written"
301
- if !output.nil?
302
- msg = base_msg + "\r"
303
- line_len += msg.length
304
- print msg
305
- end
306
-
307
- # this lambda is passed on as a block when outputing the results to file
308
- progress = lambda {
309
- if (time = Time.now.to_i) - last_time > 5
310
- msg = base_msg + ": #{humanize_time(time - start_time)} elapsed" + " " * 10 + "\r"
311
- line_len = msg.length
312
- print msg
313
- last_time = time
314
- end
315
- }
316
-
317
297
  # the next 3 formats allow writing to both a file and stdout
318
298
 
319
299
  case format
@@ -327,9 +307,6 @@ module Command
327
307
  f.write Yajl.dump(row)
328
308
  n_rows += 1
329
309
  break if output.nil? and !limit.nil? and n_rows == limit
330
-
331
- # update progress indicator
332
- progress.call if !output.nil?
333
310
  }
334
311
  f.write "]"
335
312
  }
@@ -357,9 +334,6 @@ module Command
357
334
  n_rows += 1
358
335
  writer.flush if n_rows % 100 == 0 # flush every 100 recods
359
336
  break if output.nil? and !limit.nil? and n_rows == limit
360
-
361
- # update progress indicator
362
- progress.call if !output.nil?
363
337
  }
364
338
  }
365
339
 
@@ -388,9 +362,6 @@ module Command
388
362
  n_rows += 1
389
363
  f.flush if n_rows % 100 == 0 # flush every 100 recods
390
364
  break if output.nil? and !limit.nil? and n_rows == limit
391
-
392
- # update progress indicator
393
- progress.call if !output.nil?
394
365
  }
395
366
  }
396
367
 
@@ -402,7 +373,7 @@ module Command
402
373
  "Format 'msgpack' does not support writing to stdout"
403
374
  end
404
375
  open_file(output, "wb") {|f|
405
- job.result_format('msgpack', f, &progress)
376
+ job.result_format('msgpack', f)
406
377
  }
407
378
 
408
379
  when 'msgpack.gz'
@@ -411,18 +382,12 @@ module Command
411
382
  "Format 'msgpack' does not support writing to stdout"
412
383
  end
413
384
  open_file(output, "wb") {|f|
414
- job.result_format('msgpack.gz', f, &progress)
385
+ job.result_format('msgpack.gz', f)
415
386
  }
416
387
 
417
388
  else
418
389
  raise "Unknown format #{format.inspect}"
419
390
  end
420
-
421
- # clear the progress indicator
422
- if !output.nil?
423
- print "\r"
424
- print " " * (line_len + 5) + "\r"
425
- end
426
391
  end
427
392
 
428
393
  def open_file(output, mode)
@@ -443,17 +408,10 @@ module Command
443
408
  require 'yajl'
444
409
 
445
410
  if format.nil?
446
-
447
- # start progress indicator
448
- start_time = last_time = Time.now.to_i
449
- base_msg = "WARNING: the query result is being downloaded"
450
- msg = base_msg + "\r"
451
- line_len = msg.length
452
- print msg
453
-
454
411
  # display result in tabular format
455
412
  rows = []
456
413
  n_rows = 0
414
+ print "WARNING: the query result is being downloaded...\r"
457
415
  job.result_each {|row|
458
416
  # TODO limit number of rows to show
459
417
  rows << row.map {|v|
@@ -461,20 +419,8 @@ module Command
461
419
  }
462
420
  n_rows += 1
463
421
  break if !limit.nil? and n_rows == limit
464
-
465
- # progress indication
466
- time = Time.now.to_i
467
- if time - last_time > 5
468
- msg = base_msg + ": #{humanize_time(time - start_time)} elapsed" + " " * 10 + "\r"
469
- line_len = msg.length
470
- print msg
471
- last_time = time
472
- end
473
422
  }
474
-
475
- # clear the progress indicator
476
- print "\r"
477
- print " " * (line_len + 5) + "\r"
423
+ print " " * 100, "\r" # make sure the previous WARNING is cleared over
478
424
 
479
425
  render_opts[:max_width] = 10000
480
426
  if job.hive_result_schema
@@ -512,6 +458,7 @@ module Command
512
458
  }
513
459
  return nil
514
460
  end
515
- end
516
- end
461
+
462
+ end # module Command
463
+ end # module TrasureData
517
464
 
@@ -1,4 +1,3 @@
1
-
2
1
  module TreasureData
3
2
  module Command
4
3
  module List
@@ -46,7 +45,7 @@ module List
46
45
 
47
46
  def cmd_usage(msg=nil)
48
47
  puts self.to_s
49
- puts "error: #{msg}" if msg
48
+ puts "Error: #{msg}" if msg
50
49
  exit 1
51
50
  end
52
51
  end
@@ -309,6 +308,7 @@ module List
309
308
  # TODO acl:test
310
309
 
311
310
  add_list 'server:status', %w[], 'Show status of the Treasure Data server'
311
+ add_list 'server:endpoint', %w[api_endpoint], "Set the Treasure Data API server's endpoint (must be a valid URI)", "td server:endpoint 'https://api.treasuredata.com'"
312
312
 
313
313
  add_list 'sample:apache', %w[path.json], 'Create a sample log file'
314
314
 
@@ -396,7 +396,8 @@ module List
396
396
  add_guess 'server-status', 'server:status'
397
397
 
398
398
  finishup
399
- end
400
- end
401
- end
399
+
400
+ end # module List
401
+ end # module Command
402
+ end # module TreasureData
402
403
 
@@ -119,9 +119,6 @@ module Command
119
119
  if result_url
120
120
  require 'td/command/result'
121
121
  result_url = build_result_url(result_url, result_user, result_ask_password)
122
- if result_url =~ /^td:/
123
- validate_td_result_url(result_url)
124
- end
125
122
  end
126
123
 
127
124
  client = get_client
@@ -134,7 +131,7 @@ module Command
134
131
  job = client.query(db_name, sql, result_url, priority, retry_limit, opts)
135
132
 
136
133
  puts "Job #{job.job_id} is queued."
137
- puts "Use '#{$prog} " + Config.cl_apikey_string + "job:show #{job.job_id}' to show the status."
134
+ puts "Use '#{$prog} " + Config.cl_options_string + "job:show #{job.job_id}' to show the status."
138
135
  #puts "See #{job.url} to see the progress."
139
136
 
140
137
  if wait
@@ -11,7 +11,7 @@ module Command
11
11
 
12
12
  unless r
13
13
  $stderr.puts "Result URL '#{name}' does not exist."
14
- $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "result:create #{name} <URL>' to create the URL."
14
+ $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "result:create #{name} <URL>' to create the URL."
15
15
  exit 1
16
16
  end
17
17
 
@@ -40,7 +40,7 @@ module Command
40
40
 
41
41
  if rs.empty?
42
42
  $stderr.puts "There are no result URLs."
43
- $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "result:create <name> <url>' to create a result URL."
43
+ $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "result:create <name> <url>' to create a result URL."
44
44
  end
45
45
  end
46
46
 
@@ -123,9 +123,11 @@ module Command
123
123
  url
124
124
  end
125
125
 
126
+ # DEPRECATED: relying on API server side validation which will return
127
+ # immediately after query submission with error code 422.
126
128
  private
127
129
  def validate_td_result_url(url)
128
- re = /td:\/\/[^@]*@\/(.*)\/(.*)?/
130
+ re = /td:\/\/[^@]*@\/(.*)\/([^?]+)/
129
131
  match = re.match(url)
130
132
  if match.nil?
131
133
  raise ParameterConfigurationError, "Treasure Data result output invalid URL format"
@@ -1,4 +1,3 @@
1
-
2
1
  module TreasureData
3
2
  module Command
4
3
 
@@ -6,11 +5,12 @@ class Runner
6
5
  def initialize
7
6
  @config_path = nil
8
7
  @apikey = nil
8
+ @endpoint = nil
9
9
  @prog_name = nil
10
- @secure = true
10
+ @insecure = false
11
11
  end
12
12
 
13
- attr_accessor :apikey, :config_path, :prog_name, :secure
13
+ attr_accessor :apikey, :endpoint, :config_path, :prog_name, :insecure
14
14
 
15
15
  def run(argv=ARGV)
16
16
  require 'td/version'
@@ -58,7 +58,7 @@ Additional commands:
58
58
  Type 'td help COMMAND' for more information on a specific command.
59
59
  EOF
60
60
  if errmsg
61
- puts "error: #{errmsg}"
61
+ puts "Error: #{errmsg}"
62
62
  exit 1
63
63
  else
64
64
  exit 0
@@ -66,13 +66,16 @@ EOF
66
66
  end
67
67
  end
68
68
 
69
+ # there local vars are loaded with the values of the options below
70
+ # here they are preloaded with the defaults
69
71
  config_path = @config_path
70
72
  apikey = @apikey
73
+ endpoint = @endpoint
71
74
  insecure = nil
72
75
  $verbose = false
73
76
  #$debug = false
74
77
 
75
- op.on('-c', '--config PATH', "path to config file (~/.td/td.conf)") {|s|
78
+ op.on('-c', '--config PATH', "path to the configuration file (default: ~/.td/td.conf)") {|s|
76
79
  config_path = s
77
80
  }
78
81
 
@@ -80,7 +83,15 @@ EOF
80
83
  apikey = s
81
84
  }
82
85
 
83
- op.on('--insecure', "Insecure access: disable SSL") { |b|
86
+ op.on('-e', '--endpoint API_SERVER', "specify the URL for API server to use (default: https://api.treasuredata.com)." ,
87
+ " The URL must contain a scheme (http:// or https:// prefix) to be valid.",
88
+ " Valid IPv4 addresses are accepted as well in place of the host name.") {|e|
89
+ require 'td/command/common'
90
+ Command.validate_api_endpoint(e)
91
+ endpoint = e
92
+ }
93
+
94
+ op.on('--insecure', "Insecure access: disable SSL (enabled by default)") {|b|
84
95
  insecure = true
85
96
  }
86
97
 
@@ -106,6 +117,8 @@ EOF
106
117
  usage nil if argv.empty?
107
118
  cmd = argv.shift
108
119
 
120
+ # NOTE: these information are loaded from by each command through
121
+ # 'TreasureData::Command::get_client' from 'lib/td/command/common.rb'
109
122
  require 'td/config'
110
123
  if config_path
111
124
  Config.path = config_path
@@ -114,6 +127,10 @@ EOF
114
127
  Config.apikey = apikey
115
128
  Config.cl_apikey = true
116
129
  end
130
+ if endpoint
131
+ Config.endpoint = endpoint
132
+ Config.cl_endpoint = true
133
+ end
117
134
  if insecure
118
135
  Config.secure = false
119
136
  end
@@ -145,8 +162,13 @@ EOF
145
162
  # List of new exceptions:
146
163
  # => ParameterConfigurationError
147
164
  # => BulkImportExecutionError
148
- unless [ParameterConfigurationError, BulkImportExecutionError].include?(e.class) && ENV['TD_TOOLBELT_DEBUG'].nil?
149
- $stderr.puts "error #{$!.class}: backtrace:"
165
+ # => UpUpdateError
166
+ require 'td/client/api'
167
+ # => APIError
168
+ unless [ParameterConfigurationError, BulkImportExecutionError, UpdateError,
169
+ APIError]
170
+ .include?(e.class) && ENV['TD_TOOLBELT_DEBUG'].nil?
171
+ $stderr.puts "Error #{$!.class}: backtrace:"
150
172
  $!.backtrace.each {|b|
151
173
  $stderr.puts " #{b}"
152
174
  }
@@ -167,9 +189,8 @@ EOS
167
189
  end
168
190
  return 0
169
191
  end
170
- end
171
-
192
+ end # class Runner
172
193
 
173
- end
174
- end
194
+ end # module Command
195
+ end # module TreasureData
175
196