td 0.11.9 → 0.11.10

data/lib/td/command/job.rb

@@ -1,5 +1,8 @@
+require 'td/command/options'
+
 module TreasureData
 module Command
+  include Options
 
   # TODO
   JOB_WAIT_MAX_RETRY_COUNT_ON_NETWORK_ERROR = 10
@@ -49,6 +52,7 @@ module Command
     op.on('--slow [SECONDS]', 'show slow queries (default threshold: 3600 seconds)', Integer) {|i|
       slower_than = i || 3600
     }
+
     set_render_format_option(op)
 
     max = op.cmd_parse
@@ -86,69 +90,29 @@ module Command
         :CPUTime => cpu_time.rjust(17),
         :ResultSize => (job.result_size ? Command.humanize_bytesize(job.result_size, 2) : ""),
         :Priority => priority,
-        :Result => job.result_url
+        :Result => job.result_url,
+        :Duration => job.duration ? Time.at(job.duration).utc.strftime("%X") : nil
       }
     }
 
-    puts cmd_render_table(rows,
-      :fields => [:JobID, :Status, :Start, :Elapsed, :CPUTime, :ResultSize, :Priority, :Result, :Type, :Database, :Query],
+    $stdout.puts cmd_render_table(rows,
+      :fields => [:JobID, :Status, :Start, :Elapsed, :CPUTime, :ResultSize, :Priority, :Result, :Type, :Database, :Query, :Duration],
       :max_width => 1000,
       :render_format => op.render_format
     )
   end
 
   def job_show(op)
-    verbose = nil
-    wait = false
-    output = nil
-    format = nil
-    render_opts = {:header => false}
-    limit = nil
-    exclude = false
-
-    op.on('-v', '--verbose', 'show logs', TrueClass) {|b|
-      verbose = b
-    }
-    op.on('-w', '--wait', 'wait for finishing the job', TrueClass) {|b|
-      wait = b
-    }
-    op.on('-G', '--vertical', 'use vertical table to show results', TrueClass) {|b|
-      render_opts[:vertical] = b
-    }
-    op.on('-o', '--output PATH', 'write result to the file') {|s|
-      unless Dir.exist?(File.dirname(s))
-        s = File.expand_path(s)
-      end
-      output = s
-      format = 'tsv' if format.nil?
-    }
-    op.on('-f', '--format FORMAT', 'format of the result to write to the file (tsv, csv, json, msgpack, and msgpack.gz)') {|s|
-      unless ['tsv', 'csv', 'json', 'msgpack', 'msgpack.gz'].include?(s)
-        raise "Unknown format #{s.dump}. Supported formats are: tsv, csv, json, msgpack, and msgpack.gz"
-      end
-      format = s
-    }
-    op.on('-l', '--limit ROWS', 'limit the number of result rows shown when not outputting to file') {|s|
-      unless s.to_i > 0
-        raise "Invalid limit number. Must be a positive integer"
-      end
-      limit = s.to_i
-    }
-    op.on('-c', '--column-header', 'output of the columns\' header when the schema is available',
-                                   '  for the table (only applies to tsv and csv formats)', TrueClass) {|b|
-      render_opts[:header] = b;
-    }
-    op.on('-x', '--exclude', 'do not automatically retrieve the job result', TrueClass) {|b|
-      exclude = b
-    }
-
-    op.on('--null STRING', "null expression in csv or tsv") {|s|
-      render_opts[:null_expr] = s.to_s
-    }
-
+    options = job_show_options(op)
     job_id = op.cmd_parse
 
-    # parameter concurrency validation
+    verbose     = options[:verbose]
+    wait        = options[:wait]
+    output      = options[:output]
+    format      = options[:format]
+    render_opts = options[:render_opts]
+    limit       = options[:limit]
+    exclude     = options[:exclude]
 
     if output.nil? && format
       unless ['tsv', 'csv', 'json'].include?(format)
@@ -176,7 +140,7 @@ module Command
     job_id = op.cmd_parse
     client = get_client
 
-    puts client.job_status(job_id)
+    $stdout.puts client.job_status(job_id)
   end
 
   def job_kill(op)
@@ -203,27 +167,27 @@ private
     client = get_client
     job = client.job(job_id)
 
-    puts "JobID       : #{job.job_id}"
+    $stdout.puts "JobID       : #{job.job_id}"
     #puts "URL         : #{job.url}"
-    puts "Status      : #{job.status}"
-    puts "Type        : #{job.type}"
-    puts "Database    : #{job.db_name}"
+    $stdout.puts "Status      : #{job.status}"
+    $stdout.puts "Type        : #{job.type}"
+    $stdout.puts "Database    : #{job.db_name}"
     # exclude some fields from bulk_import_perform type jobs
     if [:hive, :pig, :impala, :presto].include?(job.type)
-      puts "Priority    : #{job_priority_name_of(job.priority)}"
-      puts "Retry limit : #{job.retry_limit}"
-      puts "Output      : #{job.result_url}"
-      puts "Query       : #{job.query}"
+      $stdout.puts "Priority    : #{job_priority_name_of(job.priority)}"
+      $stdout.puts "Retry limit : #{job.retry_limit}"
+      $stdout.puts "Output      : #{job.result_url}"
+      $stdout.puts "Query       : #{job.query}"
     elsif job.type == :bulk_import_perform
-      puts "Destination : #{job.query}"
+      $stdout.puts "Destination : #{job.query}"
     end
     # if the job is done and is of type hive, show the Map-Reduce cumulated CPU time
     if job.finished?
       if [:hive].include?(job.type)
-        puts "CPU time    : #{Command.humanize_time(job.cpu_time, true)}"
+        $stdout.puts "CPU time    : #{Command.humanize_time(job.cpu_time, true)}"
       end
       if [:hive, :pig, :impala, :presto].include?(job.type)
-        puts "Result size : #{Command.humanize_bytesize(job.result_size, 2)}"
+        $stdout.puts "Result size : #{Command.humanize_bytesize(job.result_size, 2)}"
       end
     end
 
@@ -239,23 +203,23 @@ private
 
       if verbose
         if !job.debug['cmdout'].nil?
-          puts ""
-          puts "Output:"
+          $stdout.puts ""
+          $stdout.puts "Output:"
           job.debug['cmdout'].to_s.split("\n").each {|line|
-            puts "  " + line
+            $stdout.puts "  " + line
           }
         end
         if !job.debug['stderr'].nil?
-          puts ""
-          puts "Details:"
+          $stdout.puts ""
+          $stdout.puts "Details:"
           job.debug['stderr'].to_s.split("\n").each {|line|
-            puts "  " + line
+            $stdout.puts "  " + line
           }
         end
       end
     end
 
-    puts "\rUse '-v' option to show detailed messages." + " " * 20 unless verbose
+    $stdout.puts "\rUse '-v' option to show detailed messages." + " " * 20 unless verbose
   end
 
   def wait_job(job, first_call = false)
@@ -281,7 +245,7 @@ private
       cmdout = job.debug['cmdout'].to_s.split("\n")[cmdout_lines..-1] || []
       stderr = job.debug['stderr'].to_s.split("\n")[stderr_lines..-1] || []
       (cmdout + stderr).each {|line|
-        puts "  "+line
+        $stdout.puts "  "+line
       }
       cmdout_lines += cmdout.size
       stderr_lines += stderr.size
@@ -294,7 +258,7 @@ private
     max_cumul_retry_delay = 200
     cumul_retry_delay = 0
 
-    puts "Result      :"
+    $stdout.puts "Result      :"
     begin
       show_result(job, output, limit, format, render_opts)
     rescue TreasureData::NotFoundError => e
@@ -320,7 +284,7 @@ private
   def show_result(job, output, limit, format, render_opts={})
     if output
       write_result(job, output, limit, format, render_opts)
-      puts "\rwritten to #{output} in #{format} format" + " " * 50
+      $stdout.puts "\rwritten to #{output} in #{format} format" + " " * 50
     else
       # every format that is allowed on stdout
       render_result(job, limit, format, render_opts)
@@ -331,10 +295,19 @@ private
 
     # the next 3 formats allow writing to both a file and stdout
 
+
+    if output
+      if output.is_a?(String)
+        tempfile = "#{output}.tmp"
+      else # File or Tempfile
+        tempfile = "#{output.path}.tmp"
+      end
+    end
+
     case format
     when 'json'
       require 'yajl'
-      open_file(output, "w") {|f|
+      open_file(tempfile || output, "w") {|f|
         f.write "["
         n_rows = 0
         unless output.nil?
@@ -352,13 +325,13 @@ private
         f.write "]"
         indicator.finish unless output.nil?
       }
-      puts if output.nil?
+      $stdout.puts if output.nil?
 
     when 'csv'
       require 'yajl'
       require 'csv'
 
-      open_file(output, "w") {|f|
+      open_file(tempfile || output, "w") {|f|
         writer = CSV.new(f)
         # output headers
         if render_opts[:header] && job.hive_result_schema
@@ -391,7 +364,7 @@ private
     when 'tsv'
       require 'yajl'
 
-      open_file(output, "w") {|f|
+      open_file(tempfile || output, "w") {|f|
         # output headers
         if render_opts[:header] && job.hive_result_schema
           f.write job.hive_result_schema.map {|name, type| name}.join("\t") + "\n"
@@ -423,7 +396,7 @@ private
         raise ParameterConfigurationError,
               "Format 'msgpack' does not support writing to stdout"
       end
-      open_file(output, "wb") {|f|
+      open_file(tempfile || output, "wb") {|f|
         indicator = Command::SizeBasedDownloadProgressIndicator.new(
           "NOTE: the job result is being written to #{output} in msgpack format",
           job.result_size, 0.1, 1)
@@ -438,7 +411,7 @@ private
         raise ParameterConfigurationError,
               "Format 'msgpack' does not support writing to stdout"
       end
-      open_file(output, "wb") {|f|
+      open_file(tempfile || output, "wb") {|f|
         indicator = Command::SizeBasedDownloadProgressIndicator.new(
           "NOTE: the job result is being written to #{output} in msgpack.gz format",
           job.result_size, 0.1, 1)
@@ -451,6 +424,10 @@ private
     else
       raise "Unknown format #{format.inspect}"
     end
+
+    if tempfile && File.exists?(tempfile)
+      FileUtils.mv(tempfile, output.respond_to?(:path) ? output.path : output)
+    end
   end
 
   def open_file(output, mode)
@@ -485,15 +462,15 @@ private
         n_rows += 1
         break if !limit.nil? and n_rows == limit
       }
-      print " " * 100, "\r" # make sure the previous WARNING is cleared over
+      $stdout.print " " * 100, "\r" # make sure the previous WARNING is cleared over
 
       render_opts[:max_width] = 10000
       if job.hive_result_schema
         render_opts[:change_fields] = job.hive_result_schema.map { |name,type| name }
       end
 
-      print "\r" + " " * 50
-      puts "\r" + cmd_render_table(rows, render_opts)
+      $stdout.print "\r" + " " * 50
+      $stdout.puts "\r" + cmd_render_table(rows, render_opts)
     else
       # display result in any of: json, csv, tsv.
       # msgpack and mspgpack.gz are not supported for stdout output

data/lib/td/command/list.rb

@@ -15,7 +15,7 @@ module List
       @cmd_requires_connectivity = req_conn
     end
 
-    attr_accessor :message, :name, :cmd_requires_connectivity
+    attr_accessor :message, :name, :cmd_requires_connectivity, :argv
 
     def on(*argv)
       @has_options = true
@@ -46,8 +46,8 @@ module List
     end
 
     def cmd_usage(msg=nil)
-      puts self.to_s
-      puts "Error: #{msg}" if msg
+      $stdout.puts self.to_s
+      $stdout.puts "Error: #{msg}" if msg
       exit 1
     end
   end
@@ -118,7 +118,15 @@ module List
 
     def usage
       compile!
-      "%-40s   # %s" % [@usage_args, @description]
+      description_lines = @description.split("\n")
+      strout = "%-40s   # %s" % [@usage_args, description_lines[0]]
+      if description_lines.length > 1
+        description_lines[1..-1].each {|lines|
+          strout += "\n"
+          strout += "%-40s     # %s" % ["", lines]
+        }
+      end
+      strout
     end
 
     def group
@@ -181,9 +189,9 @@ module List
       next if HELP_EXCLUDE.any? {|pattern| pattern =~ c.name }
       if before_group != c.group
         before_group = c.group
-        puts ""
+        $stdout.puts ""
       end
-      puts "#{indent}#{c.usage}"
+      $stdout.puts "#{indent}#{c.usage}"
     }
   end
 
@@ -281,6 +289,10 @@ module List
   add_list 'sched:update', %w[name], 'Modify a schedule', ['sched:update sched1 -s "0 */2 * * *" -d my_db -t "Asia/Tokyo" -D 3600']
   add_list 'sched:history', %w[name max?], 'Show history of scheduled queries', ['sched sched1 --page 1']
   add_list 'sched:run', %w[name time], 'Run scheduled queries for the specified time', ['sched:run sched1 "2013-01-01 00:00:00" -n 6']
+  add_list 'sched:result', %w[name], "Show status and result of the last job ran.\n" +
+                                 "--last [N] option enables to show the result before N from the last.\n" +
+                                 "The other options are identical to those of the 'job:show' command.",
+                                 ['sched:result NAME | sched:result NAME --last | sched:result NAME --last 3']
 
   add_list 'query', %w[sql?], 'Issue a query', ['query -d example_db -w -r rset1 "select count(*) from table1"',
                                                'query -d example_db -w -r rset1 -q query.txt']
@@ -356,6 +368,7 @@ module List
   add_alias 'schedule:create', 'sched:create'
   add_alias 'schedule:delete', 'sched:delete'
   add_alias 'schedule:history', 'sched:history'
+  add_alias 'schedule:result', 'sched:result'
   add_alias 'schedule:hist', 'sched:history'
   add_alias 'sched:hist', 'sched:history'
   add_alias 'sched', 'sched:history'

data/lib/td/command/options.rb

@@ -0,0 +1,61 @@
+module TreasureData
+module Command
+module Options
+
+  def job_show_options(op)
+    opts = {}
+    opts[:verbose] = nil
+    opts[:wait] = false
+    opts[:output] = nil
+    opts[:format] = nil
+    opts[:render_opts] = {:header => false}
+    opts[:limit] = nil
+    opts[:exclude] = false
+
+    op.on('-v', '--verbose', 'show logs', TrueClass) {|b|
+      opts[:verbose] = b
+    }
+    op.on('-w', '--wait', 'wait for finishing the job', TrueClass) {|b|
+      opts[:wait] = b
+    }
+    op.on('-G', '--vertical', 'use vertical table to show results', TrueClass) {|b|
+      opts[:render_opts][:vertical] = b
+    }
+    op.on('-o', '--output PATH', 'write result to the file') {|s|
+      unless Dir.exist?(File.dirname(s))
+        s = File.expand_path(s)
+      end
+      opts[:output] = s
+      opts[:format] ||= 'tsv'
+    }
+    op.on('-f', '--format FORMAT', 'format of the result to write to the file (tsv, csv, json, msgpack, and msgpack.gz)') {|s|
+      unless ['tsv', 'csv', 'json', 'msgpack', 'msgpack.gz'].include?(s)
+        raise "Unknown format #{s.dump}. Supported formats are: tsv, csv, json, msgpack, and msgpack.gz"
+      end
+      opts[:format] = s
+    }
+    op.on('-l', '--limit ROWS', 'limit the number of result rows shown when not outputting to file') {|s|
+      unless s.to_i > 0
+        raise "Invalid limit number. Must be a positive integer"
+      end
+      opts[:limit] = s.to_i
+    }
+    op.on('-c', '--column-header', 'output of the columns\' header when the schema is available',
+                                   '  for the table (only applies to tsv and csv formats)', TrueClass) {|b|
+      opts[:render_opts][:header] = b;
+    }
+    op.on('-x', '--exclude', 'do not automatically retrieve the job result', TrueClass) {|b|
+      opts[:exclude] = b
+    }
+
+    op.on('--null STRING', "null expression in csv or tsv") {|s|
+      opts[:render_opts][:null_expr] = s.to_s
+    }
+
+    # CAUTION: this opts is filled by op.cmd_parse
+    opts
+  end
+
+end # module Options
+end # module Command
+end # module TreasureData

data/lib/td/command/password.rb

@@ -10,7 +10,7 @@ module Command
 
     begin
       system "stty -echo"  # TODO termios
-      print "Old password (typing will be hidden): "
+      $stdout.print "Old password (typing will be hidden): "
       old_password = STDIN.gets || ""
       old_password = old_password[0..-2]  # strip \n
     rescue Interrupt
@@ -18,7 +18,7 @@ module Command
       exit 1
     ensure
       system "stty echo"   # TODO termios
-      print "\n"
+      $stdout.print "\n"
     end
 
     if old_password.empty?
@@ -29,7 +29,7 @@ module Command
     3.times do
       begin
         system "stty -echo"  # TODO termios
-        print "New password (typing will be hidden): "
+        $stdout.print "New password (typing will be hidden): "
         password = STDIN.gets || ""
         password = password[0..-2]  # strip \n
       rescue Interrupt
@@ -37,7 +37,7 @@ module Command
         exit 1
       ensure
         system "stty echo"   # TODO termios
-        print "\n"
+        $stdout.print "\n"
       end
 
       if password.empty?
@@ -47,7 +47,7 @@ module Command
 
       begin
         system "stty -echo"  # TODO termios
-        print "Retype new password: "
+        $stdout.print "Retype new password: "
         password2 = STDIN.gets || ""
         password2 = password2[0..-2]  # strip \n
       rescue Interrupt
@@ -55,14 +55,14 @@ module Command
         exit 1
       ensure
         system "stty echo"   # TODO termios
-        print "\n"
+        $stdout.print "\n"
       end
 
       if password == password2
         break
       end
 
-      puts "Doesn't match."
+      $stdout.puts "Doesn't match."
     end
 
     client = get_client(:ssl => true)

data/lib/td/command/query.rb

@@ -16,6 +16,7 @@ module Command
     type = nil
     limit = nil
     exclude = false
+    pool_name = nil
 
     op.on('-d', '--database DB_NAME', 'use the database (required)') {|s|
       db_name = s
@@ -62,13 +63,13 @@ module Command
     op.on('-q', '--query PATH', 'use file instead of inline query') {|s|
       query = File.open(s) { |f| f.read.strip }
     }
-    op.on('-T', '--type TYPE', 'set query type (hive, pig, impala, presto)') {|s|
+    op.on('-T', '--type TYPE', 'set query type (hive, pig, presto)') {|s|
       type = s.to_sym
     }
     op.on('--sampling DENOMINATOR', 'OBSOLETE - enable random sampling to reduce records 1/DENOMINATOR', Integer) {|i|
-      puts "WARNING: the random sampling feature enabled through the '--sampling' option was removed and does no longer"
-      puts "         have any effect. It is left for backwards compatibility with older scripts using 'td'."
-      puts
+      $stdout.puts "WARNING: the random sampling feature enabled through the '--sampling' option was removed and does no longer"
+      $stdout.puts "         have any effect. It is left for backwards compatibility with older scripts using 'td'."
+      $stdout.puts
     }
     op.on('-l', '--limit ROWS', 'limit the number of result rows shown when not outputting to file') {|s|
       unless s.to_i > 0
@@ -77,11 +78,14 @@ module Command
       limit = s.to_i
     }
     op.on('-c', '--column-header', 'output of the columns\' header when the schema is available for the table (only applies to tsv and csv formats)', TrueClass) {|b|
-      render_opts[:header] = b;
+      render_opts[:header] = b
     }
     op.on('-x', '--exclude', 'do not automatically retrieve the job result', TrueClass) {|b|
       exclude = b
     }
+    op.on('-O', '--pool-name NAME', 'specify resource pool by name') {|s|
+      pool_name = s
+    }
 
     sql = op.cmd_parse
 
@@ -130,17 +134,18 @@ module Command
 
     opts = {}
     opts['type'] = type if type
+    opts['pool_name'] = pool_name if pool_name
     job = client.query(db_name, sql, result_url, priority, retry_limit, opts)
 
-    puts "Job #{job.job_id} is queued."
-    puts "Use '#{$prog} " + Config.cl_options_string + "job:show #{job.job_id}' to show the status."
+    $stdout.puts "Job #{job.job_id} is queued."
+    $stdout.puts "Use '#{$prog} " + Config.cl_options_string + "job:show #{job.job_id}' to show the status."
     #puts "See #{job.url} to see the progress."
 
     if wait
       wait_job(job, true)
-      puts "Status     : #{job.status}"
+      $stdout.puts "Status     : #{job.status}"
       if job.success? && !exclude
-        puts "Result     :"
+        $stdout.puts "Result     :"
         begin
           show_result(job, output, limit, format, render_opts)
         rescue TreasureData::NotFoundError => e

data/lib/td/command/result.rb

@@ -15,8 +15,8 @@ module Command
       exit 1
     end
 
-    puts "Name : #{r.name}"
-    puts "URL  : #{r.url}"
+    $stdout.puts "Name : #{r.name}"
+    $stdout.puts "URL  : #{r.url}"
   end
 
   def result_list(op)
@@ -36,7 +36,7 @@ module Command
       map[:Name]
     }
 
-    puts cmd_render_table(rows, :fields => [:Name, :URL], :render_format => op.render_format)
+    $stdout.puts cmd_render_table(rows, :fields => [:Name, :URL], :render_format => op.render_format)
 
     if rs.empty?
       $stderr.puts "There are no result URLs."
@@ -93,7 +93,7 @@ module Command
     if ask_password
       begin
         system "stty -echo"  # TODO termios
-        print "Password (typing will be hidden): "
+        $stdout.print "Password (typing will be hidden): "
         password = STDIN.gets || ""
         password = password[0..-2]  # strip \n
       rescue Interrupt
@@ -101,7 +101,7 @@ module Command
         exit 1
       ensure
         system "stty echo"   # TODO termios
-        print "\n"
+        $stdout.print "\n"
       end
     end
 

data/lib/td/command/runner.rb

@@ -32,9 +32,9 @@ EOF
     (class << self;self;end).module_eval do
       define_method(:usage) do |errmsg|
         require 'td/command/list'
-        puts op.to_s
-        puts ""
-        puts <<EOF
+        $stdout.puts op.to_s
+        $stdout.puts ""
+        $stdout.puts <<EOF
 Basic commands:
 
   db             # create/delete/list databases
@@ -59,7 +59,7 @@ Additional commands:
 Type 'td help COMMAND' for more information on a specific command.
 EOF
         if errmsg
-          puts "Error: #{errmsg}"
+          $stdout.puts "Error: #{errmsg}"
           return 1
         else
           return 0
@@ -116,7 +116,7 @@ EOF
     }
 
     op.on('--version', "show version") {
-      puts op.version
+      $stdout.puts op.version
       return 0
     }
 
@@ -191,13 +191,13 @@ EOF
         $!.backtrace.each {|bt|
           $stderr.puts "  #{bt}"
         }
-        puts ""
+        $stdout.puts ""
       end
-      print "Error: "
+      $stdout.print "Error: "
       if [ForbiddenError, NotFoundError, AuthError].include?(e.class)
-        print "#{e.class} - "
+        $stdout.print "#{e.class} - "
       end
-      puts $!.to_s
+      $stdout.puts $!.to_s
 
       require 'socket'
       if e.is_a?(::SocketError)