td 0.11.9 → 0.11.10

data/ChangeLog

@@ -1,3 +1,16 @@
+== 2015-05-19 version 0.11.10
+
+* Updated 'td-client' gem dependency to 0.8.70. The new gem removed client
+  side validation for connector commands to allow additional plugin configs
+  we're going to add in the future.
+* connector:guess dumps configuration file to stdout that is guessed by the
+  server to check the config file easier.
+* connector:preview shows full column name (not truncated) in the table.
+* add --pool-name option to job registration.
+* add sched:result command to show job results for a saved query.
+* save to temporary file while downloading job result in job:show.
+* show job duration in job list.
+
 == 2015-04-17 version 0.11.9
 
 * add connector commands.

data/lib/td/command/account.rb

@@ -41,10 +41,10 @@ module Command
       end
     end
 
-    puts "Enter your Treasure Data credentials."
+    $stdout.puts "Enter your Treasure Data credentials."
     unless user_name
       begin
-        print "Email: "
+        $stdout.print "Email: "
         line = STDIN.gets || ""
         user_name = line.strip
       rescue Interrupt
@@ -62,14 +62,14 @@ module Command
 
     3.times do
       begin
-        print "Password (typing will be hidden): "
+        $stdout.print "Password (typing will be hidden): "
         password = get_password
       rescue Interrupt
         $stderr.print "\ncanceled."
         exit 1
       ensure
         system "stty echo"   # TODO termios
-        print "\n"
+        $stdout.print "\n"
       end
 
       if password.empty?
@@ -91,7 +91,7 @@ module Command
     end
     return unless client
 
-    puts "Authenticated successfully."
+    $stdout.puts "Authenticated successfully."
 
     conf ||= Config.new
     conf["account.user"] = user_name

data/lib/td/command/acl.rb

@@ -14,7 +14,7 @@ module Command
       rows << {:Subject => ac.subject, :Action => ac.action, :Scope => ac.scope, :"Grant option" => ac.grant_option}
     }
 
-    puts cmd_render_table(rows, :fields => [:Subject, :Action, :Scope, :"Grant option"])
+    $stdout.puts cmd_render_table(rows, :fields => [:Subject, :Action, :Scope, :"Grant option"])
 
     if rows.empty?
       $stderr.puts "There are no access controls."

data/lib/td/command/apikey.rb

@@ -4,7 +4,7 @@ module Command
 
   def apikey_show(op)
     if Config.apikey
-      puts Config.apikey
+      $stdout.puts Config.apikey
       return
     end
 
@@ -20,7 +20,7 @@ module Command
       exit 1
     end
 
-    puts conf['account.apikey']
+    $stdout.puts conf['account.apikey']
   end
 
   def apikey_set(op)
@@ -55,8 +55,8 @@ module Command
     conf["account.apikey"] = apikey
     conf.save
 
-    puts "API key is set."
-    puts "Use '#{$prog} db:create <db_name>' to create a database."
+    $stdout.puts "API key is set."
+    $stdout.puts "Use '#{$prog} db:create <db_name>' to create a database."
   end
 
 end

data/lib/td/command/bulk_import.rb

@@ -16,7 +16,7 @@ module Command
       rows << {:Name=>bi.name, :Table=>"#{bi.database}.#{bi.table}", :Status=>bi.status.to_s.capitalize, :Frozen=>bi.upload_frozen? ? 'Frozen' : '', :JobID=>bi.job_id, :"Valid Parts"=>bi.valid_parts, :"Error Parts"=>bi.error_parts, :"Valid Records"=>bi.valid_records, :"Error Records"=>bi.error_records}
     }
 
-    puts cmd_render_table(rows, :fields => [:Name, :Table, :Status, :Frozen, :JobID, :"Valid Parts", :"Error Parts", :"Valid Records", :"Error Records"], :max_width=>200, :render_format => op.render_format)
+    $stdout.puts cmd_render_table(rows, :fields => [:Name, :Table, :Status, :Frozen, :JobID, :"Valid Parts", :"Error Parts", :"Valid Records", :"Error Records"], :max_width=>200, :render_format => op.render_format)
 
     if rows.empty?
       $stderr.puts "There are no bulk import sessions."
@@ -75,7 +75,7 @@ module Command
 
     list = client.list_bulk_import_parts(name)
     list.each {|name|
-      puts "  #{name}"
+      $stdout.puts "  #{name}"
     }
   end
 
@@ -100,6 +100,7 @@ module Command
     part_prefix = ""
     auto_perform = false
     parallel = 2
+    pool_name = nil
 
     op.on('-P', '--prefix NAME', 'add prefix to parts name') {|s|
       part_prefix = s
@@ -113,6 +114,9 @@ module Command
     op.on('--parallel NUM', 'perform uploading in parallel (default: 2; max 8)', Integer) {|i|
       parallel = i
     }
+    op.on('-O', '--pool-name NAME', 'specify resource pool by name') {|s|
+      pool_name = s
+    }
 
     name, *files = op.cmd_parse
 
@@ -176,8 +180,10 @@ module Command
     $stderr.puts "done."
 
     if auto_perform
+      opts = {}
+      opts['pool_name'] = pool_name if pool_name
       client = get_client
-      job = client.perform_bulk_import(name)
+      job = client.perform_bulk_import(name, opts)
 
       $stderr.puts "Job #{job.job_id} is queued."
       $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "job:show [-w] #{job.job_id}' to show the status."
@@ -185,7 +191,7 @@ module Command
   end
 
   def bulk_import_upload_parts2(op)
-    puts "This command is moved to 'td import:upload' since 0.10.85."
+    $stdout.puts "This command is moved to 'td import:upload' since 0.10.85."
   end
 
   # obsoleted
@@ -223,6 +229,7 @@ module Command
   def bulk_import_perform(op)
     wait = false
     force = false
+    pool_name = nil
 
     op.on('-w', '--wait', 'wait for finishing the job', TrueClass) {|b|
       wait = b
@@ -230,6 +237,9 @@ module Command
     op.on('-f', '--force', 'force start performing', TrueClass) {|b|
       force = b
     }
+    op.on('-O', '--pool-name NAME', 'specify resource pool by name') {|s|
+      pool_name = s
+    }
 
     name = op.cmd_parse
 
@@ -252,7 +262,9 @@ module Command
       end
     end
 
-    job = client.perform_bulk_import(name)
+    opts = {}
+    opts['pool_name'] = pool_name if pool_name
+    job = client.perform_bulk_import(name, opts)
 
     $stderr.puts "Job #{job.job_id} is queued."
     $stderr.puts "Use '#{$prog} " + Config.cl_options_string + "job:show [-w] #{job.job_id}' to show the status."
@@ -338,7 +350,7 @@ module Command
 
     require 'yajl'
     client.bulk_import_error_records(name) {|r|
-      puts Yajl.dump(r)
+      $stdout.puts Yajl.dump(r)
     }
   end
 
@@ -459,7 +471,7 @@ module Command
   end
 
   def bulk_import_prepare_parts2(op)
-    puts "This command is moved to 'td import:prepare' since 0.10.85."
+    $stdout.puts "This command is moved to 'td import:prepare' since 0.10.85."
   end
 
   private

data/lib/td/command/common.rb

@@ -2,6 +2,8 @@ require 'td/version'
 
 module TreasureData
 
+require "td/config"
+
 autoload :API, 'td/client/api'
 autoload :Client, 'td/client'
 autoload :Database, 'td/client'
@@ -249,7 +251,7 @@ EOS
     3.times do
       begin
         system "stty -echo"  # TODO termios
-        print "Password (typing will be hidden): "
+        $stdout.print "Password (typing will be hidden): "
         password = STDIN.gets || ""
         password = password[0..-2]  # strip \n
       rescue Interrupt
@@ -257,7 +259,7 @@ EOS
         exit 1
       ensure
         system "stty echo"   # TODO termios
-        print "\n"
+        $stdout.print "\n"
       end
 
       if password.empty?
@@ -328,7 +330,7 @@ EOS
   class DownloadProgressIndicator
     def initialize(msg)
       @base_msg = msg
-      print @base_msg + " " * 10
+      $stdout.print @base_msg + " " * 10
     end
   end
 
@@ -343,8 +345,8 @@ EOS
     def update
       if (time = Time.now.to_i) - @last_time >= @periodicity
         msg = "\r#{@base_msg}: #{Command.humanize_elapsed_time(@start_time, time)} elapsed"
-        print "\r" + " " * (msg.length + 10)
-        print msg
+        $stdout.print "\r" + " " * (msg.length + 10)
+        $stdout.print msg
         @last_time = time
         true
       else
@@ -353,7 +355,7 @@ EOS
     end
 
     def finish
-      puts "\r#{@base_msg}...done" + " " * 20
+      $stdout.puts "\r#{@base_msg}...done" + " " * 20
     end
   end
 
@@ -378,15 +380,15 @@ EOS
     def update(curr_size)
       if @size.nil? || @size == 0
         msg = "\r#{@base_msg}: #{Command.humanize_bytesize(curr_size)}"
-        print msg
+        $stdout.print msg
         true
       else
         ratio = (curr_size.to_f * 100 / @size).round(1)
         if ratio >= (@last_perc_step + @perc_step) &&
            (!@min_periodicity || (time = Time.now.to_i) - @last_time >= @min_periodicity)
           msg = "\r#{@base_msg}: #{Command.humanize_bytesize(curr_size)} / #{ratio}%"
-          print "\r" + " " * (msg.length + 10)
-          print msg
+          $stdout.print "\r" + " " * (msg.length + 10)
+          $stdout.print msg
           @last_perc_step = ratio
           @last_time = time
           true
@@ -397,7 +399,7 @@ EOS
     end
 
     def finish
-      print "\r#{@base_msg}...done" + " " * 20
+      $stdout.print "\r#{@base_msg}...done" + " " * 20
     end
   end
 

data/lib/td/command/connector.rb

@@ -40,7 +40,7 @@ module Command
       bucket = path_components.shift.sub(/\//, '')
       path_prefix = path_components.join.sub(/\//, '')
 
-      job = API::BulkLoad::BulkLoad.from_hash(
+      job = {
         :config => {
           :type => type,
           :access_key_id => id,
@@ -49,7 +49,7 @@ module Command
           :bucket => bucket,
           :path_prefix => path_prefix,
         }
-      ).validate
+      }
     end
 
     client = get_client
@@ -57,16 +57,20 @@ module Command
 
     create_bulkload_job_file_backup(out)
     if /\.json\z/ =~ out
-      config_str = JSON.pretty_generate(job.to_h)
+      config_str = JSON.pretty_generate(job)
     else
-      config_str = YAML.dump(job.to_h)
+      config_str = YAML.dump(job)
     end
     File.open(out, 'w') do |f|
       f << config_str
     end
 
-    puts "Created #{out} file."
-    puts "Use '#{$prog} " + Config.cl_options_string + "connector:preview #{out}' to see bulk load preview."
+    $stdout.puts "Guessed configuration:"
+    $stdout.puts
+    $stdout.puts config_str
+    $stdout.puts
+    $stdout.puts "Created #{out} file."
+    $stdout.puts "Use '#{$prog} " + Config.cl_options_string + "connector:preview #{out}' to see bulk load preview."
   end
 
   def connector_preview(op)
@@ -76,23 +80,23 @@ module Command
     client = get_client()
     preview = client.bulk_load_preview(job)
 
-    cols = preview.schema.sort_by { |col|
+    cols = preview['schema'].sort_by { |col|
       col['index']
     }
     fields = cols.map { |col| col['name'] + ':' + col['type'] }
     types = cols.map { |col| col['type'] }
-    rows = preview.records.map { |row|
+    rows = preview['records'].map { |row|
       cols = {}
       row.each_with_index do |col, idx|
         cols[fields[idx]] = col.inspect
       end
       cols
-    } 
+    }
 
-    puts cmd_render_table(rows, :fields => fields, :render_format => op.render_format)
+    $stdout.puts cmd_render_table(rows, :fields => fields, :render_format => op.render_format, :resize => false)
 
-    puts "Update #{config_file} and use '#{$prog} " + Config.cl_options_string + "connector:preview #{config_file}' to preview again."
-    puts "Use '#{$prog} " + Config.cl_options_string + "connector:issue #{config_file}' to run Server-side bulk load."
+    $stdout.puts "Update #{config_file} and use '#{$prog} " + Config.cl_options_string + "connector:preview #{config_file}' to preview again."
+    $stdout.puts "Use '#{$prog} " + Config.cl_options_string + "connector:issue #{config_file}' to run Server-side bulk load."
   end
 
   def connector_issue(op)
@@ -116,8 +120,8 @@ module Command
     client = get_client()
     job_id = client.bulk_load_issue(database, table, job)
 
-    puts "Job #{job_id} is queued."
-    puts "Use '#{$prog} " + Config.cl_options_string + "job:show #{job_id}' to show the status."
+    $stdout.puts "Job #{job_id} is queued."
+    $stdout.puts "Use '#{$prog} " + Config.cl_options_string + "job:show #{job_id}' to show the status."
 
     if wait
       wait_connector_job(client, job_id, exclude)
@@ -135,10 +139,10 @@ module Command
     rows = client.bulk_load_list().sort_by { |e|
       e['name']
     }.map { |e|
-      Hash[fields.zip(e.to_h.values_at(*keys))]
+      Hash[fields.zip(e.values_at(*keys))]
     }
 
-    puts cmd_render_table(rows, :fields => fields, :render_format => op.render_format)
+    $stdout.puts cmd_render_table(rows, :fields => fields, :render_format => op.render_format)
   end
 
   def connector_create(op)
@@ -195,8 +199,8 @@ module Command
 
     client = get_client()
     session = client.bulk_load_delete(name)
-    puts 'Deleted session'
-    puts '--'
+    $stdout.puts 'Deleted session'
+    $stdout.puts '--'
     dump_connector_session(session)
   end
 
@@ -219,7 +223,7 @@ module Command
         :Duration => (e.end_at.nil? ? Time.now.to_i : e.end_at) - e.start_at,
       }
     }
-    puts cmd_render_table(rows, :fields => fields, :render_format => op.render_format)
+    $stdout.puts cmd_render_table(rows, :fields => fields, :render_format => op.render_format)
   end
 
   def connector_run(op)
@@ -231,8 +235,8 @@ module Command
 
     client = get_client()
     job_id = client.bulk_load_run(name)
-    puts "Job #{job_id} is queued."
-    puts "Use '#{$prog} " + Config.cl_options_string + "job:show #{job_id}' to show the status."
+    $stdout.puts "Job #{job_id} is queued."
+    $stdout.puts "Use '#{$prog} " + Config.cl_options_string + "job:show #{job_id}' to show the status."
 
     if wait
       wait_connector_job(client, job_id, exclude)
@@ -263,7 +267,7 @@ private
     if file_type(config_str) == :yaml
       config_str = JSON.pretty_generate(YAML.load(config_str))
     end
-    API::BulkLoad::BulkLoad.from_json(config_str)
+    JSON.load(config_str)
   end
 
   def create_bulkload_job_file_backup(out)
@@ -279,20 +283,20 @@ private
   end
 
   def dump_connector_session(session)
-    puts "Name     : #{session.name}"
-    puts "Cron     : #{session.cron}"
-    puts "Timezone : #{session.timezone}"
-    puts "Delay    : #{session.delay}"
-    puts "Database : #{session.database}"
-    puts "Table    : #{session.table}"
-    puts "Config"
-    puts YAML.dump(session.config.to_h)
+    $stdout.puts "Name     : #{session.name}"
+    $stdout.puts "Cron     : #{session.cron}"
+    $stdout.puts "Timezone : #{session.timezone}"
+    $stdout.puts "Delay    : #{session.delay}"
+    $stdout.puts "Database : #{session.database}"
+    $stdout.puts "Table    : #{session.table}"
+    $stdout.puts "Config"
+    $stdout.puts YAML.dump(session.config)
   end
 
   def wait_connector_job(client, job_id, exclude)
     job = client.job(job_id)
     wait_job(job, true)
-    puts "Status     : #{job.status}"
+    $stdout.puts "Status     : #{job.status}"
   end
 
 end

data/lib/td/command/db.rb

@@ -22,7 +22,7 @@ module Command
       [map[:Type].size, map[:Table]]
     }
 
-    puts cmd_render_table(rows, :fields => [:Table, :Type, :Count, :Schema], :render_format => op.render_format)
+    $stdout.puts cmd_render_table(rows, :fields => [:Table, :Type, :Count, :Schema], :render_format => op.render_format)
   end
 
   def db_list(op)
@@ -37,7 +37,7 @@ module Command
     dbs.each {|db|
       rows << {:Name=>db.name, :Count=>db.count}
     }
-    puts cmd_render_table(rows, :fields => [:Name, :Count], :render_format => op.render_format)
+    $stdout.puts cmd_render_table(rows, :fields => [:Name, :Count], :render_format => op.render_format)
 
     if dbs.empty?
       $stderr.puts "There are no databases."

data/lib/td/command/export.rb

@@ -12,6 +12,7 @@ module Command
     aws_secret_access_key = nil
     file_prefix = nil
     file_format = "json.gz" # default
+    pool_name = nil
 
     op.on('-w', '--wait', 'wait until the job is completed', TrueClass) {|b|
       wait = b
@@ -38,6 +39,9 @@ module Command
       raise ArgumentError, "#{s} is not a supported file format" unless SUPPORTED_FORMATS.include?(s)
       file_format = s
     }
+    op.on('-O', '--pool-name NAME', 'specify resource pool by name') {|s|
+      pool_name = s
+    }
 
     db_name, table_name = op.cmd_parse
 
@@ -70,6 +74,7 @@ module Command
     s3_opts['bucket'] = s3_bucket
     s3_opts['access_key_id'] = aws_access_key_id
     s3_opts['secret_access_key'] = aws_secret_access_key
+    s3_opts['pool_name'] = pool_name if pool_name
 
     job = client.export(db_name, table_name, "s3", s3_opts)
 
@@ -78,7 +83,7 @@ module Command
 
     if wait && !job.finished?
       wait_job(job)
-      puts "Status     : #{job.status}"
+      $stdout.puts "Status     : #{job.status}"
     end
   end
 

data/lib/td/command/help.rb

@@ -13,7 +13,7 @@ module Command
 
     elsif c.name != cmd && c.group == cmd
       # group command
-      puts List.cmd_usage(cmd)
+      $stdout.puts List.cmd_usage(cmd)
       exit 1
 
     else
@@ -26,8 +26,8 @@ module Command
     cmd = op.cmd_parse
 
     TreasureData::Command::List.show_help(op.summary_indent)
-    puts ""
-    puts "Type '#{$prog} help COMMAND' for more information on a specific command."
+    $stdout.puts ""
+    $stdout.puts "Type '#{$prog} help COMMAND' for more information on a specific command."
   end
 
 end

data/lib/td/command/import.rb

@@ -27,7 +27,7 @@ module Command
   def import_jar_version(op)
     op.cmd_parse
     version = find_version_file
-    puts "td-import-java #{File.open(version, 'r').read}"
+    $stdout.puts "td-import-java #{File.open(version, 'r').read}"
   end
 
   def import_jar_update(op)
@@ -90,7 +90,7 @@ module Command
       if op.cmd_requires_connectivity
         raise e
       else
-        puts "Warning: JAR update skipped for connectivity issues"
+        $stdout.puts "Warning: JAR update skipped for connectivity issues"
       end
     end
 
@@ -303,7 +303,7 @@ module Command
     installed_path = File.join(File.expand_path('../../..', File.dirname(__FILE__)), 'java')
     config = Command.find_files("logging.properties", [Updater.jarfile_dest_path, installed_path])
     if config.empty?
-      puts "Cannot find 'logging.properties' file in '#{Updater.jarfile_dest_path}' or " +
+      $stdout.puts "Cannot find 'logging.properties' file in '#{Updater.jarfile_dest_path}' or " +
            "'#{installed_path}'." unless ENV['TD_TOOLBELT_DEBUG'].nil?
       []
     else