td 0.11.2 → 0.11.3

data/lib/td/command/job.rb

@@ -84,13 +84,14 @@ module Command
         :Start => (start ? start.localtime : ''),
         :Elapsed => elapsed.rjust(11),
         :CPUTime => cpu_time.rjust(17),
+        :ResultSize => (job.result_size ? Command.humanize_bytesize(job.result_size, 2) : ""),
         :Priority => priority,
         :Result => job.result_url
       }
     }
 
     puts cmd_render_table(rows,
-      :fields => [:JobID, :Status, :Start, :Elapsed, :CPUTime, :Priority, :Result, :Type, :Database, :Query],
+      :fields => [:JobID, :Status, :Start, :Elapsed, :CPUTime, :ResultSize, :Priority, :Result, :Type, :Database, :Query],
       :max_width => 1000,
       :render_format => op.render_format
     )
@@ -180,8 +181,13 @@ module Command
       puts "Destination : #{job.query}"
     end
     # if the job is done and is of type hive, show the Map-Reduce cumulated CPU time
-    if job.finished? && [:hive].include?(job.type)
-      puts "CPU time    : #{Command.humanize_time(job.cpu_time, true)}"
+    if job.finished?
+      if [:hive].include?(job.type)
+        puts "CPU time    : #{Command.humanize_time(job.cpu_time, true)}"
+      end
+      if [:hive, :pig, :impala, :presto].include?(job.type)
+        puts "Result size : #{Command.humanize_bytesize(job.result_size, 2)}"
+      end
     end
 
     if wait && !job.finished?
@@ -285,7 +291,7 @@ module Command
   def show_result(job, output, limit, format, render_opts={})
     if output
       write_result(job, output, limit, format, render_opts)
-      puts "written to #{output} in #{format} format"
+      puts "\rwritten to #{output} in #{format} format" + " " * 50
     else
       # every format that is allowed on stdout
       render_result(job, limit, format, render_opts)
@@ -302,13 +308,20 @@ module Command
       open_file(output, "w") {|f|
         f.write "["
         n_rows = 0
-        job.result_each {|row|
+        unless output.nil?
+          indicator = Command::SizeBasedDownloadProgressIndicator.new(
+            "NOTE: the job result is being written to #{output} in json format",
+            job.result_size, 0.1, 1)
+        end
+        job.result_each_with_compr_size {|row, compr_size|
+          indicator.update(compr_size) unless output.nil?
           f.write ",\n" if n_rows > 0
           f.write Yajl.dump(row)
           n_rows += 1
           break if output.nil? and !limit.nil? and n_rows == limit
         }
         f.write "]"
+        indicator.finish unless output.nil?
       }
       puts if output.nil?
 
@@ -318,15 +331,21 @@ module Command
 
       open_file(output, "w") {|f|
         writer = CSV.new(f)
-        n_rows = 0
         # output headers
         if render_opts[:header] && job.hive_result_schema
-          writer << job.hive_result_schema.map {|name,type|
+          writer << job.hive_result_schema.map {|name, type|
             name
           }
         end
         # output data
-        job.result_each {|row|
+        n_rows = 0
+        unless output.nil?
+          indicator = Command::SizeBasedDownloadProgressIndicator.new(
+            "NOTE: the job result is being written to #{output} in csv format",
+            job.result_size, 0.1, 1)
+        end
+        job.result_each_with_compr_size {|row, compr_size|
+          indicator.update(compr_size) unless output.nil?
           # TODO limit the # of columns
           writer << row.map {|col|
             dump_column(col)
@@ -335,6 +354,7 @@ module Command
           writer.flush if n_rows % 100 == 0 # flush every 100 recods
           break if output.nil? and !limit.nil? and n_rows == limit
         }
+        indicator.finish unless output.nil?
       }
 
     when 'tsv'
@@ -350,7 +370,13 @@ module Command
         end
         # output data
         n_rows = 0
-        job.result_each {|row|
+        unless output.nil?
+          indicator = Command::SizeBasedDownloadProgressIndicator.new(
+            "NOTE: the job result is being written to #{output} in tsv format",
+            job.result_size, 0.1, 1)
+        end
+        job.result_each_with_compr_size {|row, compr_size|
+          indicator.update(compr_size) unless output.nil?
           n_cols = 0
           row.each {|col|
             f.write "\t" if n_cols > 0
@@ -363,6 +389,7 @@ module Command
           f.flush if n_rows % 100 == 0 # flush every 100 recods
           break if output.nil? and !limit.nil? and n_rows == limit
         }
+        indicator.finish unless output.nil?
       }
 
     # these last 2 formats are only valid if writing the result to file through the -o/--output option.
@@ -373,7 +400,13 @@ module Command
               "Format 'msgpack' does not support writing to stdout"
       end
       open_file(output, "wb") {|f|
-        job.result_format('msgpack', f)
+        indicator = Command::SizeBasedDownloadProgressIndicator.new(
+          "NOTE: the job result is being written to #{output} in msgpack format",
+          job.result_size, 0.1, 1)
+        job.result_format('msgpack', f) {|compr_size|
+          indicator.update(compr_size)
+        }
+        indicator.finish
       }
 
     when 'msgpack.gz'
@@ -382,7 +415,13 @@ module Command
               "Format 'msgpack' does not support writing to stdout"
       end
       open_file(output, "wb") {|f|
-        job.result_format('msgpack.gz', f)
+        indicator = Command::SizeBasedDownloadProgressIndicator.new(
+          "NOTE: the job result is being written to #{output} in msgpack.gz format",
+          job.result_size, 0.1, 1)
+        job.result_format('msgpack.gz', f) {|compr_size|
+          indicator.update(compr_size)
+        }
+        indicator.finish
       }
 
     else
@@ -411,9 +450,11 @@ module Command
       # display result in tabular format
       rows = []
       n_rows = 0
-      print "WARNING: the query result is being downloaded...\r"
-      job.result_each {|row|
-        # TODO limit number of rows to show
+
+      indicator = Command::SizeBasedDownloadProgressIndicator.new(
+        "WARNING: the job result is being downloaded...", job.result_size, 0.1, 1)
+      job.result_each_with_compr_size {|row, compr_size|
+        indicator.update(compr_size)
         rows << row.map {|v|
           dump_column(v)
         }
@@ -427,7 +468,8 @@ module Command
         render_opts[:change_fields] = job.hive_result_schema.map { |name,type| name }
       end
 
-      puts cmd_render_table(rows, render_opts)
+      print "\r" + " " * 50
+      puts "\r" + cmd_render_table(rows, render_opts)
     else
       # display result in any of: json, csv, tsv.
       # msgpack and mspgpack.gz are not supported for stdout output

data/lib/td/command/list.rb

@@ -3,17 +3,19 @@ module Command
 module List
 
   class CommandParser < OptionParser
-    def initialize(name, req_args, opt_args, varlen, argv)
+    def initialize(name, req_args, opt_args, varlen, argv, req_conn)
       super()
+      @name = name
       @req_args = req_args
       @opt_args = opt_args
       @varlen = varlen
       @argv = argv
       @has_options = false
       @message = ''
+      @cmd_requires_connectivity = req_conn
     end
 
-    attr_accessor :message
+    attr_accessor :message, :name, :cmd_requires_connectivity
 
     def on(*argv)
       @has_options = true
@@ -51,15 +53,16 @@ module List
   end
 
   class CommandOption
-    def initialize(name, args, description, examples)
+    def initialize(name, args, description, examples = [], req_conn = true)
       @name = name
       @args = args
       @description = description.to_s
       @examples = examples
+      @req_conn = req_conn
       @override_message = nil
     end
 
-    attr_reader :name, :args, :description, :examples
+    attr_reader :name, :args, :description, :examples, :req_conn
     attr_accessor :override_message
 
     def compile!
@@ -84,7 +87,7 @@ module List
 
     def create_optparse(argv)
       compile!
-      op = CommandParser.new(@name, @req_args, @opt_args, @varlen, argv)
+      op = CommandParser.new(@name, @req_args, @opt_args, @varlen, argv, @req_conn)
 
       message = "usage:\n"
       message << "  $ #{File.basename($0)} #{@usage_args}\n"
@@ -129,8 +132,8 @@ module List
   HELP_EXCLUDE = [/^help/, /^account/, /^update/, /^user/, /^acl/]
   USAGE_EXCLUDE = [/bulk_import:upload_part\z/, /bulk_import:delete_part\z/]
 
-  def self.add_list(name, args, description, *examples)
-    LIST << COMMAND[name] = CommandOption.new(name, args, description, examples)
+  def self.add_list(name, args, description, examples = [], cmd_req_conn = true)
+    LIST << COMMAND[name] = CommandOption.new(name, args, description, examples, cmd_req_conn)
   end
 
   def self.add_alias(new_cmd, old_cmd)
@@ -157,7 +160,7 @@ module List
       require "td/command/#{group}"
       cmd = name.gsub(/[\:\-]/, '_')
       m = Object.new.extend(Command).method(cmd)
-      return Proc.new {|args| m.call(c.create_optparse(args)) }
+      return Proc.new { |args| m.call(c.create_optparse(args)) }, c.req_conn
     end
     nil
   end
@@ -213,85 +216,85 @@ module List
     }
   end
 
-  add_list 'db:list', %w[], 'Show list of tables in a database', 'db:list', 'dbs'
-  add_list 'db:show', %w[db], 'Describe information of a database', 'db example_db'
-  add_list 'db:create', %w[db], 'Create a database', 'db:create example_db'
-  add_list 'db:delete', %w[db], 'Delete a database', 'db:delete example_db'
-
-  add_list 'table:list', %w[db?], 'Show list of tables', 'table:list', 'table:list example_db', 'tables'
-  add_list 'table:show', %w[db table], 'Describe information of a table', 'table example_db table1'
-  add_list 'table:create', %w[db table], 'Create a table', 'table:create example_db table1'
-  add_list 'table:delete', %w[db table], 'Delete a table', 'table:delete example_db table1'
-  add_list 'table:import', %w[db table files_], 'Parse and import files to a table', 'table:import example_db table1 --apache access.log', 'table:import example_db table1 --json -t time - < test.json'
-  add_list 'table:export', %w[db table], 'Dump logs in a table to the specified storage', 'table:export example_db table1 --s3-bucket mybucket -k KEY_ID -s SECRET_KEY'
-  add_list 'table:swap', %w[db table1 table2], 'Swap names of two tables', 'table:swap example_db table1 table2'
-  add_list 'table:tail', %w[db table], 'Get recently imported logs', 'table:tail example_db table1', 'table:tail example_db table1 -t "2011-01-02 03:04:05" -n 30'
-  add_list 'table:partial_delete', %w[db table], 'Delete logs from the table within the specified time range', 'table:partial_delete example_db table1 --from 1341000000 --to 1341003600'
-  add_list 'table:expire', %w[db table expire_days], 'Expire data in table after specified number of days', 'table:expire example_db table1 30'
-
-  add_list 'bulk_import:list', %w[], 'List bulk import sessions', 'bulk_import:list'
-  add_list 'bulk_import:show', %w[name], 'Show list of uploaded parts', 'bulk_import:show'
-  add_list 'bulk_import:create', %w[name db table], 'Create a new bulk import session to the the table', 'bulk_import:create logs_201201 example_db event_logs'
-  add_list 'bulk_import:prepare_parts', %w[files_], 'Convert files into part file format', 'bulk_import:prepare_parts logs/*.csv --format csv --columns time,uid,price,count --time-column "time" -o parts/'
-  add_list 'bulk_import:upload_part', %w[name id path.msgpack.gz], 'Upload or re-upload a file into a bulk import session', 'bulk_import:upload_part logs_201201 01h data-201201-01.msgpack.gz'
-  add_list 'bulk_import:upload_parts', %w[name files_], 'Upload or re-upload files into a bulk import session', 'bulk_import:upload_parts parts/* --parallel 4'
-  add_list 'bulk_import:delete_part', %w[name id], 'Delete a uploaded file from a bulk import session', 'bulk_import:delete_part logs_201201 01h'
-  add_list 'bulk_import:delete_parts', %w[name ids_], 'Delete uploaded files from a bulk import session', 'bulk_import:delete_parts logs_201201 01h 02h 03h'
-  add_list 'bulk_import:perform', %w[name], 'Start to validate and convert uploaded files', 'bulk_import:perform logs_201201'
-  add_list 'bulk_import:error_records', %w[name], 'Show records which did not pass validations', 'bulk_import:error_records logs_201201'
-  add_list 'bulk_import:commit', %w[name], 'Start to commit a performed bulk import session', 'bulk_import:commit logs_201201'
-  add_list 'bulk_import:delete', %w[name], 'Delete a bulk import session', 'bulk_import:delete logs_201201'
-  add_list 'bulk_import:freeze', %w[name], 'Reject succeeding uploadings to a bulk import session', 'bulk_import:freeze logs_201201'
-  add_list 'bulk_import:unfreeze', %w[name], 'Unfreeze a frozen bulk import session', 'bulk_import:unfreeze logs_201201'
-
-  add_list 'import:list', %w[], 'List bulk import sessions', 'import:list'
-  add_list 'import:show', %w[name], 'Show list of uploaded parts', 'import:show'
-  add_list 'import:create', %w[name db table], 'Create a new bulk import session to the the table', 'import:create logs_201201 example_db event_logs'
-  add_list 'import:jar_version', %w[], 'Show import jar version', 'import:jar_version'
-  add_list 'import:jar_update', %w[], 'Update import jar to the latest version', 'import:jar_update'
-  add_list 'import:prepare', %w[files_], 'Convert files into part file format', 'import:prepare logs/*.csv --format csv --columns time,uid,price,count --time-column "time" -o parts/'
-  add_list 'import:upload', %w[name files_], 'Upload or re-upload files into a bulk import session', 'import:upload parts/* --parallel 4'
-  add_list 'import:auto', %w[name files_], 'Upload files and automatically perform and commit the data', 'import:auto parts/* --parallel 4'
-  add_list 'import:perform', %w[name], 'Start to validate and convert uploaded files', 'import:perform logs_201201'
-  add_list 'import:error_records', %w[name], 'Show records which did not pass validations', 'import:error_records logs_201201'
-  add_list 'import:commit', %w[name], 'Start to commit a performed bulk import session', 'import:commit logs_201201'
-  add_list 'import:delete', %w[name], 'Delete a bulk import session', 'import:delete logs_201201'
-  add_list 'import:freeze', %w[name], 'Reject succeeding uploadings to a bulk import session', 'import:freeze logs_201201'
-  add_list 'import:unfreeze', %w[name], 'Unfreeze a frozen bulk import session', 'import:unfreeze logs_201201'
-
-  add_list 'result:list', %w[], 'Show list of result URLs', 'result:list', 'results'
-  add_list 'result:show', %w[name], 'Describe information of a result URL', 'result name'
-  add_list 'result:create', %w[name URL], 'Create a result URL', 'result:create name mysql://my-server/mydb'
-  add_list 'result:delete', %w[name], 'Delete a result URL', 'result:delete name'
-
-  add_list 'status', %w[], 'Show schedules, jobs, tables and results', 'status', 's'
-
-  add_list 'schema:show', %w[db table], 'Show schema of a table', 'schema example_db table1'
-  add_list 'schema:set', %w[db table columns_?], 'Set new schema on a table', 'schema:set example_db table1 user:string size:int'
-  add_list 'schema:add', %w[db table columns_], 'Add new columns to a table', 'schema:add example_db table1 user:string size:int'
-  add_list 'schema:remove', %w[db table columns_], 'Remove columns from a table', 'schema:remove example_db table1 user size'
-
-  add_list 'sched:list', %w[], 'Show list of schedules', 'sched:list', 'scheds'
-  add_list 'sched:create', %w[name cron sql?], 'Create a schedule', 'sched:create sched1 "0 * * * *" -d example_db "select count(*) from table1" -r rset1',
-                                                                    'sched:create sched1 "0 * * * *" -d example_db -q query.txt -r rset2'
-  add_list 'sched:delete', %w[name], 'Delete a schedule', 'sched:delete sched1'
-  add_list 'sched:update', %w[name], 'Modify a schedule', 'sched:update sched1 -s "0 */2 * * *" -d my_db -t "Asia/Tokyo" -D 3600'
-  add_list 'sched:history', %w[name max?], 'Show history of scheduled queries', 'sched sched1 --page 1'
-  add_list 'sched:run', %w[name time], 'Run scheduled queries for the specified time', 'sched:run sched1 "2013-01-01 00:00:00" -n 6'
-
-  add_list 'query', %w[sql?], 'Issue a query', 'query -d example_db -w -r rset1 "select count(*) from table1"',
-                                               'query -d example_db -w -r rset1 -q query.txt'
-
-  add_list 'job:show', %w[job_id], 'Show status and result of a job', 'job:show 1461'
-  add_list 'job:status', %w[job_id], 'Show status progress of a job', 'job:status 1461'
-  add_list 'job:list', %w[max?], 'Show list of jobs', 'jobs', 'jobs --page 1'
-  add_list 'job:kill', %w[job_id], 'Kill or cancel a job', 'job:kill 1461'
+  add_list 'db:list', %w[], 'Show list of tables in a database', ['db:list', 'dbs']
+  add_list 'db:show', %w[db], 'Describe information of a database', ['db example_db']
+  add_list 'db:create', %w[db], 'Create a database', ['db:create example_db']
+  add_list 'db:delete', %w[db], 'Delete a database', ['db:delete example_db']
+
+  add_list 'table:list', %w[db?], 'Show list of tables', ['table:list', 'table:list example_db', 'tables']
+  add_list 'table:show', %w[db table], 'Describe information of a table', ['table example_db table1']
+  add_list 'table:create', %w[db table], 'Create a table', ['table:create example_db table1']
+  add_list 'table:delete', %w[db table], 'Delete a table', ['table:delete example_db table1']
+  add_list 'table:import', %w[db table files_], 'Parse and import files to a table', ['table:import example_db table1 --apache access.log', 'table:import example_db table1 --json -t time - < test.json']
+  add_list 'table:export', %w[db table], 'Dump logs in a table to the specified storage', ['table:export example_db table1 --s3-bucket mybucket -k KEY_ID -s SECRET_KEY']
+  add_list 'table:swap', %w[db table1 table2], 'Swap names of two tables', ['table:swap example_db table1 table2']
+  add_list 'table:tail', %w[db table], 'Get recently imported logs', ['table:tail example_db table1', 'table:tail example_db table1 -t "2011-01-02 03:04:05" -n 30']
+  add_list 'table:partial_delete', %w[db table], 'Delete logs from the table within the specified time range', ['table:partial_delete example_db table1 --from 1341000000 --to 1341003600']
+  add_list 'table:expire', %w[db table expire_days], 'Expire data in table after specified number of days', ['table:expire example_db table1 30']
+
+  add_list 'bulk_import:list', %w[], 'List bulk import sessions', ['bulk_import:list']
+  add_list 'bulk_import:show', %w[name], 'Show list of uploaded parts', ['bulk_import:show']
+  add_list 'bulk_import:create', %w[name db table], 'Create a new bulk import session to the the table', ['bulk_import:create logs_201201 example_db event_logs']
+  add_list 'bulk_import:prepare_parts', %w[files_], 'Convert files into part file format', ['bulk_import:prepare_parts logs/*.csv --format csv --columns time,uid,price,count --time-column "time" -o parts/']
+  add_list 'bulk_import:upload_part', %w[name id path.msgpack.gz], 'Upload or re-upload a file into a bulk import session', ['bulk_import:upload_part logs_201201 01h data-201201-01.msgpack.gz']
+  add_list 'bulk_import:upload_parts', %w[name files_], 'Upload or re-upload files into a bulk import session', ['bulk_import:upload_parts parts/* --parallel 4']
+  add_list 'bulk_import:delete_part', %w[name id], 'Delete a uploaded file from a bulk import session', ['bulk_import:delete_part logs_201201 01h']
+  add_list 'bulk_import:delete_parts', %w[name ids_], 'Delete uploaded files from a bulk import session', ['bulk_import:delete_parts logs_201201 01h 02h 03h']
+  add_list 'bulk_import:perform', %w[name], 'Start to validate and convert uploaded files', ['bulk_import:perform logs_201201']
+  add_list 'bulk_import:error_records', %w[name], 'Show records which did not pass validations', ['bulk_import:error_records logs_201201']
+  add_list 'bulk_import:commit', %w[name], 'Start to commit a performed bulk import session', ['bulk_import:commit logs_201201']
+  add_list 'bulk_import:delete', %w[name], 'Delete a bulk import session', ['bulk_import:delete logs_201201']
+  add_list 'bulk_import:freeze', %w[name], 'Reject succeeding uploadings to a bulk import session', ['bulk_import:freeze logs_201201']
+  add_list 'bulk_import:unfreeze', %w[name], 'Unfreeze a frozen bulk import session', ['bulk_import:unfreeze logs_201201']
+
+  add_list 'import:list', %w[], 'List bulk import sessions', ['import:list']
+  add_list 'import:show', %w[name], 'Show list of uploaded parts', ['import:show']
+  add_list 'import:create', %w[name db table], 'Create a new bulk import session to the the table', ['import:create logs_201201 example_db event_logs']
+  add_list 'import:jar_version', %w[], 'Show import jar version', ['import:jar_version'], false
+  add_list 'import:jar_update', %w[], 'Update import jar to the latest version', ['import:jar_update']
+  add_list 'import:prepare', %w[files_], 'Convert files into part file format', ['import:prepare logs/*.csv --format csv --columns time,uid,price,count --time-column "time" -o parts/'], false
+  add_list 'import:upload', %w[name files_], 'Upload or re-upload files into a bulk import session', ['import:upload parts/* --parallel 4']
+  add_list 'import:auto', %w[name files_], 'Upload files and automatically perform and commit the data', ['import:auto parts/* --parallel 4']
+  add_list 'import:perform', %w[name], 'Start to validate and convert uploaded files', ['import:perform logs_201201']
+  add_list 'import:error_records', %w[name], 'Show records which did not pass validations', ['import:error_records logs_201201']
+  add_list 'import:commit', %w[name], 'Start to commit a performed bulk import session', ['import:commit logs_201201']
+  add_list 'import:delete', %w[name], 'Delete a bulk import session', ['import:delete logs_201201']
+  add_list 'import:freeze', %w[name], 'Reject succeeding uploadings to a bulk import session', ['import:freeze logs_201201']
+  add_list 'import:unfreeze', %w[name], 'Unfreeze a frozen bulk import session', ['import:unfreeze logs_201201']
+
+  add_list 'result:list', %w[], 'Show list of result URLs', ['result:list', 'results']
+  add_list 'result:show', %w[name], 'Describe information of a result URL', ['result name']
+  add_list 'result:create', %w[name URL], 'Create a result URL', ['result:create name mysql://my-server/mydb']
+  add_list 'result:delete', %w[name], 'Delete a result URL', ['result:delete name']
+
+  add_list 'status', %w[], 'Show schedules, jobs, tables and results', ['status', 's']
+
+  add_list 'schema:show', %w[db table], 'Show schema of a table', ['schema example_db table1']
+  add_list 'schema:set', %w[db table columns_?], 'Set new schema on a table', ['schema:set example_db table1 user:string size:int']
+  add_list 'schema:add', %w[db table columns_], 'Add new columns to a table', ['schema:add example_db table1 user:string size:int']
+  add_list 'schema:remove', %w[db table columns_], 'Remove columns from a table', ['schema:remove example_db table1 user size']
+
+  add_list 'sched:list', %w[], 'Show list of schedules', ['sched:list', 'scheds']
+  add_list 'sched:create', %w[name cron sql?], 'Create a schedule', ['sched:create sched1 "0 * * * *" -d example_db "select count(*) from table1" -r rset1',
+                                                                    'sched:create sched1 "0 * * * *" -d example_db -q query.txt -r rset2']
+  add_list 'sched:delete', %w[name], 'Delete a schedule', ['sched:delete sched1']
+  add_list 'sched:update', %w[name], 'Modify a schedule', ['sched:update sched1 -s "0 */2 * * *" -d my_db -t "Asia/Tokyo" -D 3600']
+  add_list 'sched:history', %w[name max?], 'Show history of scheduled queries', ['sched sched1 --page 1']
+  add_list 'sched:run', %w[name time], 'Run scheduled queries for the specified time', ['sched:run sched1 "2013-01-01 00:00:00" -n 6']
+
+  add_list 'query', %w[sql?], 'Issue a query', ['query -d example_db -w -r rset1 "select count(*) from table1"',
+                                               'query -d example_db -w -r rset1 -q query.txt']
+
+  add_list 'job:show', %w[job_id], 'Show status and result of a job', ['job:show 1461']
+  add_list 'job:status', %w[job_id], 'Show status progress of a job', ['job:status 1461']
+  add_list 'job:list', %w[max?], 'Show list of jobs', ['jobs', 'jobs --page 1']
+  add_list 'job:kill', %w[job_id], 'Kill or cancel a job', ['job:kill 1461']
 
   add_list 'account', %w[user_name?], 'Setup a Treasure Data account'
   add_list 'account:usage', %w[user_name?], 'Show resource usage information'
   add_list 'password:change', %w[], 'Change password'
-  add_list 'apikey:show', %w[], 'Show Treasure Data API key'
-  add_list 'apikey:set', %w[apikey], 'Set Treasure Data API key'
+  add_list 'apikey:show', %w[], 'Show Treasure Data API key', [], false
+  add_list 'apikey:set', %w[apikey], 'Set Treasure Data API key', [], false
 
   add_list 'user:list', %w[], 'Show list of users'
   add_list 'user:show', %w[name], 'Show an user'
@@ -308,12 +311,12 @@ module List
   # TODO acl:test
 
   add_list 'server:status', %w[], 'Show status of the Treasure Data server'
-  add_list 'server:endpoint', %w[api_endpoint], "Set the Treasure Data API server's endpoint (must be a valid URI)", "td server:endpoint 'https://api.treasuredata.com'"
+  add_list 'server:endpoint', %w[api_endpoint], "Set the Treasure Data API server's endpoint (must be a valid URI)", ["td server:endpoint 'https://api.treasuredata.com'"]
 
-  add_list 'sample:apache', %w[path.json], 'Create a sample log file'
+  add_list 'sample:apache', %w[path.json], 'Create a sample log file', [], false
 
-  add_list 'help:all', %w[], 'Show usage of all commands'
-  add_list 'help', %w[command], 'Show usage of a command'
+  add_list 'help:all', %w[], 'Show usage of all commands', [], false
+  add_list 'help', %w[command], 'Show usage of a command', [], false
 
   add_list 'update', %w[], 'Update td and related libraries for TreasureData toolbelt'
 

data/lib/td/command/runner.rb

@@ -59,9 +59,9 @@ Type 'td help COMMAND' for more information on a specific command.
 EOF
         if errmsg
           puts "Error: #{errmsg}"
-          exit 1
+          return 1
         else
-          exit 0
+          return 0
         end
       end
     end
@@ -104,17 +104,17 @@ EOF
     #}
 
     op.on('-h', '--help', "show help") {
-      usage nil
+      return usage nil
     }
 
     op.on('--version', "show version") {
       puts op.version
-      exit
+      return 0
     }
 
     begin
       op.order!(argv)
-      usage nil if argv.empty?
+      return usage nil if argv.empty?
       cmd = argv.shift
 
       # NOTE: these information are loaded from by each command through
@@ -135,7 +135,7 @@ EOF
         Config.secure = false
       end
     rescue
-      usage $!.to_s
+      return usage $!.to_s
     end
 
     require 'td/command/list'
@@ -144,37 +144,49 @@ EOF
       Encoding.default_external = 'UTF-8' if Encoding.respond_to?(:default_external)
     end
 
-    method = Command::List.get_method(cmd)
+    method, cmd_req_connectivity = Command::List.get_method(cmd)
     unless method
       $stderr.puts "'#{cmd}' is not a td command. Run '#{$prog}' to show the list."
       Command::List.show_guess(cmd)
-      exit 1
+      return 1
     end
 
     begin
+      # test the connectivity with the API endpoint
+      if cmd_req_connectivity && Config.cl_endpoint
+        Command.test_api_endpoint(Config.endpoint)
+      end
       method.call(argv)
     rescue ConfigError
       $stderr.puts "TreasureData account is not configured yet."
       $stderr.puts "Run '#{$prog} account' first."
     rescue => e
-      # work in progress look ahead development: new exceptions are rendered as simple
-      # error messages unless the TD_TOOLBELT_DEBUG variable is not empty.
-      # List of new exceptions:
-      # => ParameterConfigurationError
-      # => BulkImportExecutionError
-      # => UpUpdateError
+      # known exceptions are rendered as simple error messages unless the
+      # TD_TOOLBELT_DEBUG variable is set or the -v / --verbose option is used.
+      # List of known exceptions:
+      #   => ParameterConfigurationError
+      #   => BulkImportExecutionError
+      #   => UpUpdateError
+      #   => ImportError
       require 'td/client/api'
-      # => APIError
-      unless [ParameterConfigurationError, BulkImportExecutionError, UpdateError,
-              APIError]
-              .include?(e.class) && ENV['TD_TOOLBELT_DEBUG'].nil?
+      #   => APIError
+      #   => ForbiddenError
+      #   => NotFoundError
+      #   => AuthError
+      if ![ParameterConfigurationError, BulkImportExecutionError, UpdateError, ImportError,
+            APIError, ForbiddenError, NotFoundError, AuthError].include?(e.class) ||
+         !ENV['TD_TOOLBELT_DEBUG'].nil? || $verbose
         $stderr.puts "Error #{$!.class}: backtrace:"
-        $!.backtrace.each {|b|
-          $stderr.puts "  #{b}"
+        $!.backtrace.each {|bt|
+          $stderr.puts "  #{bt}"
         }
         puts ""
       end
-      puts "Error: " + $!.to_s
+      print "Error: "
+      if [ForbiddenError, NotFoundError, AuthError].include?(e.class)
+        print "#{e.class} - "
+      end
+      puts $!.to_s
 
       require 'socket'
       if e.is_a?(::SocketError)

data/lib/td/command/sched.rb

@@ -136,6 +136,7 @@ module Command
   def sched_update(op)
     require 'td/command/job'  # job_priority_id_of
 
+    newname = nil
     cron = nil
     sql = nil
     db_name = nil
@@ -146,6 +147,9 @@ module Command
     retry_limit = nil
     type = nil
 
+    op.on('-n', '--newname NAME', 'change the schedule\'s name') {|n|
+      newname = n
+    }
     op.on('-s', '--schedule CRON', 'change the schedule') {|s|
       cron = s
     }
@@ -183,10 +187,10 @@ module Command
       type = s
     }
 
-
-    name = op.cmd_parse
+    curname = op.cmd_parse
 
     params = {}
+    params['name'] = newname if newname
     params['cron'] = cron if cron
     params['query'] = sql if sql
     params['database'] = db_name if db_name
@@ -205,15 +209,19 @@ module Command
     client = get_client
 
     begin
-      client.update_schedule(name, params)
+      client.update_schedule(curname, params)
     rescue NotFoundError
       cmd_debug_error $!
-      $stderr.puts "Schedule '#{name}' does not exist."
+      $stderr.puts "Schedule '#{curname}' does not exist."
       $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "sched:list' to show list of the schedules."
       exit 1
     end
 
-    $stderr.puts "Schedule '#{name}' is updated."
+    if newname && curname != newname
+      puts "Schedule '#{curname}' is updated and its name changed to '#{newname}'."
+    else
+      puts "Schedule '#{curname}' is updated."
+    end
   end
 
   def sched_history(op)

data/lib/td/command/server.rb

@@ -12,13 +12,16 @@ module Command
   def server_endpoint(op)
     endpoint = op.cmd_parse
 
-    Command.validate_api_endpoint(endpoint)
-
     if Config.cl_endpoint and endpoint != Config.endpoint
       raise ParameterConfigurationError,
-            "You specified the API server endpoint in the command options as well (-e / --endpoint option) but it does not match the value provided to the 'server:endpoint' command. Please remove the option or ensure the endpoints URLs match each other."
+            "You specified the API server endpoint in the command options as well (-e / --endpoint " +
+            "option) but it does not match the value provided to the 'server:endpoint' command. " +
+            "Please remove the option or ensure the endpoints URLs match each other."
     end
 
+    Command.validate_api_endpoint(endpoint)
+    Coomand.test_api_endpoint(endpoint)
+
     conf = nil
     begin
       conf = Config.read