td 0.10.95 → 0.10.96

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 4be1e9e6df99daff7484f415d97c0e07b1f8323b
+  data.tar.gz: e5a02f18e5ddc92ce3853064982dac0276c78b51
+SHA512:
+  metadata.gz: 84a6b09a04a928bc8cd07230fc21e041d451bb43d42614821623507d3bfcd55ffaa4d93bdc8372178d11937a5830df5cabef97a82bd53cd6992032b036e9331e
+  data.tar.gz: 6aa3f40c432bc25e2a8430595f071e92388f58ffb47f4e472d113100e58dc440ed49ebd1b3d9886ba36950979456791cf5e248d918e6e7fefffe29d3d4c689e9

data/ChangeLog

@@ -1,3 +1,11 @@
+== 2013-12-04 version 0.10.96
+
+* Add --format option to some subcommands to specify render format. json, csv and tsv.
+* job:show and query subcommand now accept --format option without --output
+* Show -k option in command result message if -k option is given.
+* Support import:jar_update subcommand to update td-import seperately
+
+
 == 2013-11-20 version 0.10.95
 
 * Fix import:prepare command. It cannot parse time-format options that includes ',' charactor.

data/lib/td/command/account.rb

@@ -84,7 +84,7 @@ module Command
     conf["account.apikey"] = client.apikey
     conf.save
 
-    $stderr.puts "Use '#{$prog} db:create <db_name>' to create a database."
+    $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "db:create <db_name>' to create a database."
   end
 
   def account_usage(op)

data/lib/td/command/acl.rb

@@ -18,7 +18,7 @@ module Command
 
     if rows.empty?
       $stderr.puts "There are no access controls."
-      $stderr.puts "Use '#{$prog} acl:grant <subject> <action> <scope>' to grant permissions."
+      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "acl:grant <subject> <action> <scope>' to grant permissions."
     end
   end
 

data/lib/td/command/aggr.rb

@@ -18,7 +18,7 @@ module Command
 
     if rows.empty?
       $stderr.puts "There are no aggregation schemas."
-      $stderr.puts "Use '#{$prog} aggr:create <name>' to create a aggregation schema."
+      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "aggr:create <name>' to create a aggregation schema."
     end
   end
 
@@ -130,7 +130,7 @@ module Command
     rescue NotFoundError
       cmd_debug_error $!
       $stderr.puts "Aggregation schema '#{name}' does not exist."
-      $stderr.puts "Use '#{$prog} aggr:create #{name}' to create the aggregation schema."
+      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "aggr:create #{name}' to create the aggregation schema."
       exit 1
     rescue AlreadyExistsError
       $stderr.puts "Aggregation log entry '#{entry_name}' already exists."
@@ -164,7 +164,7 @@ module Command
     rescue NotFoundError
       cmd_debug_error $!
       $stderr.puts "Aggregation schema '#{name}' does not exist."
-      $stderr.puts "Use '#{$prog} aggr:create #{name}' to create the aggregation schema."
+      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "aggr:create #{name}' to create the aggregation schema."
       exit 1
     rescue AlreadyExistsError
       $stderr.puts "Aggregation attribute entry '#{entry_name}' already exists."

data/lib/td/command/bulk_import.rb

@@ -3,6 +3,8 @@ module TreasureData
 module Command
 
   def bulk_import_list(op)
+    set_render_format_option(op)
+
     op.cmd_parse
 
     client = get_client
@@ -14,11 +16,11 @@ module Command
       rows << {:Name=>bi.name, :Table=>"#{bi.database}.#{bi.table}", :Status=>bi.status.to_s.capitalize, :Frozen=>bi.upload_frozen? ? 'Frozen' : '', :JobID=>bi.job_id, :"Valid Parts"=>bi.valid_parts, :"Error Parts"=>bi.error_parts, :"Valid Records"=>bi.valid_records, :"Error Records"=>bi.error_records}
     }
 
-    puts cmd_render_table(rows, :fields => [:Name, :Table, :Status, :Frozen, :JobID, :"Valid Parts", :"Error Parts", :"Valid Records", :"Error Records"], :max_width=>200)
+    puts cmd_render_table(rows, :fields => [:Name, :Table, :Status, :Frozen, :JobID, :"Valid Parts", :"Error Parts", :"Valid Records", :"Error Records"], :max_width=>200, :render_format => op.render_format)
 
     if rows.empty?
       $stderr.puts "There are no bulk import sessions."
-      $stderr.puts "Use '#{$prog} bulk_import:create <name> <db> <table>' to create a session."
+      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "bulk_import:create <name> <db> <table>' to create a session."
     end
   end
 
@@ -60,7 +62,7 @@ module Command
     bi = bis.find {|bi| name == bi.name }
     unless bi
       $stderr.puts "Bulk import session '#{name}' does not exist."
-      $stderr.puts "Use '#{$prog} bulk_import:create <name> <db> <table>' to create a session."
+      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "bulk_import:create <name> <db> <table>' to create a session."
       exit 1
     end
 
@@ -183,7 +185,7 @@ module Command
       job = client.perform_bulk_import(name)
 
       $stderr.puts "Job #{job.job_id} is queued."
-      $stderr.puts "Use '#{$prog} job:show [-w] #{job.job_id}' to show the status."
+      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "job:show [-w] #{job.job_id}' to show the status."
     end
   end
 
@@ -245,7 +247,7 @@ module Command
         if bi.status == 'performing'
           $stderr.puts "Bulk import session '#{name}' is already performing."
           $stderr.puts "Add '-f' option to force start."
-          $stderr.puts "Use '#{$prog} job:kill #{bi.job_id}' to cancel the last trial."
+          $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "job:kill #{bi.job_id}' to cancel the last trial."
           exit 1
         elsif bi.status == 'ready'
           $stderr.puts "Bulk import session '#{name}' is already ready to commit."
@@ -258,7 +260,7 @@ module Command
     job = client.perform_bulk_import(name)
 
     $stderr.puts "Job #{job.job_id} is queued."
-    $stderr.puts "Use '#{$prog} job:show [-w] #{job.job_id}' to show the status."
+    $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "job:show [-w] #{job.job_id}' to show the status."
 
     if wait
       require 'td/command/job'  # wait_job
@@ -285,13 +287,13 @@ module Command
     bi = bis.find {|bi| name == bi.name }
     unless bi
       $stderr.puts "Bulk import session '#{name}' does not exist."
-      $stderr.puts "Use '#{$prog} bulk_import:create <name> <db> <table>' to create a session."
+      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "bulk_import:create <name> <db> <table>' to create a session."
       exit 1
     end
 
     if bi.status == "uploading" || bi.status == "performing"
       $stderr.puts "Bulk import session '#{name}' is not performed."
-      $stderr.puts "Use '#{$prog} bulk_import:perform <name>' to run."
+      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "bulk_import:perform <name>' to run."
       exit 1
     end
 

data/lib/td/command/common.rb

@@ -37,9 +37,49 @@ module Command
     get_client(opts)
   end
 
+  def set_render_format_option(op)
+    def op.render_format
+      @_render_format
+    end
+    op.on('-f', '--format FORMAT', 'format of the result rendering (tsv, csv, json or table. default is table)') {|s|
+      unless ['tsv', 'csv', 'json', 'table'].include?(s)
+        raise "Unknown format #{s.dump}. Supported format: tsv, csv, json, table"
+      end
+      op.instance_variable_set(:@_render_format, s)
+    }
+  end
+
   def cmd_render_table(rows, *opts)
     require 'hirb'
-    Hirb::Helpers::Table.render(rows, *opts)
+
+    options = opts.first
+    format = options.delete(:render_format)
+
+    case format
+    when 'csv', 'tsv'
+      require 'csv'
+      headers = options[:fields]
+      csv_opts = {}
+      csv_opts[:col_sep] = "\t" if format == 'tsv'
+      CSV.generate('', csv_opts) { |csv|
+        csv << headers
+        rows.each { |row|
+          r = []
+          headers.each { |field|
+            r << row[field]
+          }
+          csv << r
+        }
+      }
+    when 'json'
+      require 'yajl'
+
+      Yajl.dump(rows)
+    when 'table'
+      Hirb::Helpers::Table.render(rows, *opts)
+    else
+      Hirb::Helpers::Table.render(rows, *opts)
+    end
   end
 
   def normalized_message

data/lib/td/command/db.rb

@@ -3,6 +3,8 @@ module TreasureData
 module Command
 
   def db_show(op)
+    set_render_format_option(op)
+
     db_name = op.cmd_parse
 
     client = get_client
@@ -20,10 +22,12 @@ module Command
       [map[:Type].size, map[:Table]]
     }
 
-    puts cmd_render_table(rows, :fields => [:Table, :Type, :Count, :Schema])
+    puts cmd_render_table(rows, :fields => [:Table, :Type, :Count, :Schema], :render_format => op.render_format)
   end
 
   def db_list(op)
+    set_render_format_option(op)
+
     op.cmd_parse
 
     client = get_client
@@ -33,11 +37,11 @@ module Command
     dbs.each {|db|
       rows << {:Name=>db.name, :Count=>db.count}
     }
-    puts cmd_render_table(rows, :fields => [:Name, :Count])
+    puts cmd_render_table(rows, :fields => [:Name, :Count], :render_format => op.render_format)
 
     if dbs.empty?
       $stderr.puts "There are no databases."
-      $stderr.puts "Use '#{$prog} db:create <db_name>' to create a database."
+      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "db:create <db_name>' to create a database."
     end
   end
 
@@ -57,7 +61,7 @@ module Command
     end
 
     $stderr.puts "Database '#{db_name}' is created."
-    $stderr.puts "Use '#{$prog} table:create #{db_name} <table_name>' to create a table."
+    $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "table:create #{db_name} <table_name>' to create a table."
   end
 
   def db_delete(op)

data/lib/td/command/export.rb

@@ -65,7 +65,7 @@ module Command
     job = client.export(db_name, table_name, "s3", s3_opts)
 
     $stderr.puts "Export job #{job.job_id} is queued."
-    $stderr.puts "Use '#{$prog} job:show #{job.job_id}' to show the status."
+    $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "job:show #{job.job_id}' to show the status."
   end
 
   private

data/lib/td/command/import.rb

@@ -1,8 +1,10 @@
+require 'td/updater'
 
 module TreasureData
 module Command
 
   BASE_PATH = File.expand_path('../../..', File.dirname(__FILE__))
+  UPDATED_PATH = File.join(Updater.home_directory, ".td", "java")
 
   JAVA_COMMAND = "java"
   JAVA_MAIN_CLASS = "com.treasure_data.td_import.BulkImportCommand"
@@ -28,6 +30,34 @@ module Command
     puts "td-import-java #{File.open(vfile, 'r').read}"
   end
 
+  def import_jar_update(op)
+    last_updated = existence_jar_updated_time
+
+    require 'rexml/document'
+    require 'open-uri'
+    require 'fileutils'
+
+    doc = REXML::Document.new(open('http://maven.treasure-data.com/com/treasure_data/td-import/maven-metadata.xml') { |f| f.read })
+    updated = Time.strptime(REXML::XPath.match(doc, '/metadata/versioning/lastUpdated').first.text, "%Y%m%d%H%M%S")
+    version = REXML::XPath.match(doc, '/metadata/versioning/release').first.text
+
+    # Convert into UTF to compare time correctly
+    updated = (updated + updated.gmt_offset).utc unless updated.gmt?
+    last_updated = last_updated.utc unless last_updated.gmt?
+
+    if updated > last_updated
+      FileUtils.mkdir_p(UPDATED_PATH) unless File.exist?(UPDATED_PATH)
+      File.open(File.join(UPDATED_PATH, 'VERSION'), 'w') { |f| f.print "#{version} via import:jar_update" }
+      File.open(File.join(UPDATED_PATH, 'td-import-java.version'), 'w') { |f| f.print "#{version} #{updated}" }
+      File.open(File.join(UPDATED_PATH, 'td-import.jar'), 'wb') { |f|
+        f.print Updater.fetch("http://maven.treasure-data.com/com/treasure_data/td-import/#{version}/td-import-#{version}.jar")
+      }
+      puts "Installed td-import.jar #{version} into #{UPDATED_PATH}"
+    else
+      puts 'Installed td-import.jar is latest version'
+    end
+  end
+
   def import_prepare(op)
     import_by_java('prepare')
   end
@@ -113,7 +143,7 @@ module Command
 
   private
   def find_td_import_jar
-    libjars = Dir.glob("#{BASE_PATH}/java/**/*.jar")
+    libjars = find_files('*.jar')
     found = libjars.find { |path| File.basename(path) =~ /^td-import/ }
     if found.nil?
       $stderr.puts "td-import.jar is not found."
@@ -201,9 +231,22 @@ module Command
 
   private
   def find_version_file
-    vfile = Dir.glob("#{BASE_PATH}/java/**/VERSION")
+    vfile = find_files('VERSION')
     vfile
   end
 
+  def existence_jar_updated_time
+    require 'time'
+
+    content = File.open(find_files("td-import-java.version").first).read
+    index = content.index(' ')
+    Time.parse(content[index + 1..-1].strip)
+  end
+
+  def find_files(target)
+    files = Dir.glob("#{UPDATED_PATH}/**/#{target}")
+    return files unless files.empty?
+    Dir.glob("#{BASE_PATH}/java/**/#{target}")
+  end
 end
 end

data/lib/td/command/job.rb

@@ -50,6 +50,7 @@ module Command
     op.on('--slow [SECONDS]', 'show slow queries (default threshold: 3600 seconds)', Integer) { |i|
       slower_than = i || 3600
     }
+    set_render_format_option(op)
 
     max = op.cmd_parse
 
@@ -76,14 +77,14 @@ module Command
       rows << {:JobID => job.job_id, :Database => job.db_name, :Status => job.status, :Type => job.type, :Query => job.query.to_s, :Start => (start ? start.localtime : ''), :Elapsed => elapsed, :Priority => priority, :Result => job.result_url}
     }
 
-    puts cmd_render_table(rows, :fields => [:JobID, :Status, :Start, :Elapsed, :Priority, :Result, :Type, :Database, :Query], :max_width => 140)
+    puts cmd_render_table(rows, :fields => [:JobID, :Status, :Start, :Elapsed, :Priority, :Result, :Type, :Database, :Query], :max_width => 140, :render_format => op.render_format)
   end
 
   def job_show(op)
     verbose = nil
     wait = false
     output = nil
-    format = 'tsv'
+    format = nil
     render_opts = {}
     exclude = false
 
@@ -98,6 +99,7 @@ module Command
     }
     op.on('-o', '--output PATH', 'write result to the file') {|s|
       output = s
+      format = 'tsv' if format.nil?
     }
     op.on('-f', '--format FORMAT', 'format of the result to write to the file (tsv, csv, json or msgpack)') {|s|
       unless ['tsv', 'csv', 'json', 'msgpack', 'msgpack.gz'].include?(s)
@@ -111,6 +113,12 @@ module Command
 
     job_id = op.cmd_parse
 
+    if output.nil? && format
+      unless ['tsv', 'csv', 'json'].include?(format)
+        raise "Supported formats are only tsv, csv and json without --output option"
+      end
+    end
+
     client = get_client
 
     job = client.job(job_id)
@@ -222,7 +230,7 @@ module Command
       write_result(job, output, format)
       puts "written to #{output} in #{format} format"
     else
-      render_result(job, render_opts)
+      render_result(job, render_opts, format)
     end
   end
 
@@ -231,7 +239,7 @@ module Command
     when 'json'
       require 'yajl'
       first = true
-      File.open(output, "w") {|f|
+      open_file(output, "w") { |f|
         f.write "["
         job.result_each {|row|
           if first
@@ -243,21 +251,24 @@ module Command
         }
         f.write "]"
       }
+      puts if output.nil?
 
     when 'msgpack'
-      File.open(output, "wb") {|f|
+      open_file(output, "wb") { |f|
         job.result_format('msgpack', f)
       }
 
     when 'msgpack.gz'
-      File.open(output, "wb") {|f|
+      open_file(output, "wb") { |f|
         job.result_format('msgpack.gz', f)
       }
 
     when 'csv'
       require 'yajl'
       require 'csv'
-      CSV.open(output, "w") {|writer|
+
+      open_file(output, "w") { |f|
+        writer = CSV.new(f)
         job.result_each {|row|
           writer << row.map {|col| dump_column(col) }
         }
@@ -265,7 +276,7 @@ module Command
 
     when 'tsv'
       require 'yajl'
-      File.open(output, "w") {|f|
+      open_file(output, "w") { |f|
         job.result_each {|row|
           first = true
           row.each {|col|
@@ -285,22 +296,41 @@ module Command
     end
   end
 
-  def render_result(job, opts)
+  def open_file(output, mode)
+    f = nil
+    if output.nil?
+      yield STDOUT
+    else
+      f = File.open(output, mode)
+      yield f
+    end
+  ensure
+    if f
+      f.close unless f.closed?
+    end
+  end
+
+  def render_result(job, opts, format = nil)
     require 'yajl'
-    rows = []
-    job.result_each {|row|
-      # TODO limit number of rows to show
-      rows << row.map {|v|
-        dump_column(v)
+
+    if format.nil?
+      rows = []
+      job.result_each {|row|
+        # TODO limit number of rows to show
+        rows << row.map {|v|
+          dump_column(v)
+        }
       }
-    }
 
-    opts[:max_width] = 10000
-    if job.hive_result_schema
-      opts[:change_fields] = job.hive_result_schema.map {|name,type| name }
-    end
+      opts[:max_width] = 10000
+      if job.hive_result_schema
+        opts[:change_fields] = job.hive_result_schema.map {|name,type| name }
+      end
 
-    puts cmd_render_table(rows, opts)
+      puts cmd_render_table(rows, opts)
+    else
+      write_result(job, nil, format)
+    end
   end
 
   def dump_column(v)

data/lib/td/command/list.rb

@@ -249,6 +249,7 @@ module List
   add_list 'import:show', %w[name], 'Show list of uploaded parts', 'import:show'
   add_list 'import:create', %w[name db table], 'Create a new bulk import session to the the table', 'import:create logs_201201 example_db event_logs'
   add_list 'import:java_version', %w[], 'Show version', 'import:java_version'
+  add_list 'import:jar_update', %w[], 'Update import jar', 'import:jar_update'
   add_list 'import:prepare', %w[files_], 'Convert files into part file format', 'import:prepare logs/*.csv --format csv --columns time,uid,price,count --time-column "time" -o parts/'
   add_list 'import:upload', %w[name files_], 'Upload or re-upload files into a bulk import session', 'import:upload parts/* --parallel 4'
   add_list 'import:auto', %w[name files_], 'Upload files and automatically perform and commit the data', 'import:auto parts/* --parallel 4'
@@ -282,7 +283,7 @@ module List
   add_list 'query', %w[sql?], 'Issue a query', 'query -d example_db -w -r rset1 "select count(*) from table1"',
                                                'query -d example_db -w -r rset1 -q query.txt'
 
-  add_list 'job:show', %w[job_id], 'Show status and result of a job', 'job 1461'
+  add_list 'job:show', %w[job_id], 'Show status and result of a job', 'job:show 1461'
   add_list 'job:status', %w[job_id], 'Show status progress of a job', 'job:status 1461'
   add_list 'job:list', %w[max?], 'Show list of jobs', 'jobs', 'jobs --page 1'
   add_list 'job:kill', %w[job_id], 'Kill or cancel a job', 'job:kill 1461'

data/lib/td/command/query.rb

@@ -6,7 +6,7 @@ module Command
     db_name = nil
     wait = false
     output = nil
-    format = 'tsv'
+    format = nil
     render_opts = {}
     result_url = nil
     result_user = nil
@@ -29,6 +29,7 @@ module Command
     }
     op.on('-o', '--output PATH', 'write result to the file') {|s|
       output = s
+      format = 'tsv' if format.nil?
     }
     op.on('-f', '--format FORMAT', 'format of the result to write to the file (tsv, csv, json or msgpack)') {|s|
       unless ['tsv', 'csv', 'json', 'msgpack'].include?(s)
@@ -69,6 +70,12 @@ module Command
 
     sql = op.cmd_parse
 
+    if output.nil? && format
+      unless ['tsv', 'csv', 'json'].include?(format)
+        raise "Supported formats are only tsv, csv and json without --output option"
+      end
+    end
+
     unless db_name
       $stderr.puts "-d, --database DB_NAME option is required."
       exit 1
@@ -101,7 +108,7 @@ module Command
     job = client.query(db_name, sql, result_url, priority, retry_limit, opts)
 
     $stderr.puts "Job #{job.job_id} is queued."
-    $stderr.puts "Use '#{$prog} job:show #{job.job_id}' to show the status."
+    $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "job:show #{job.job_id}' to show the status."
     #$stderr.puts "See #{job.url} to see the progress."
 
     if wait && !job.finished?
@@ -121,3 +128,4 @@ module Command
 end
 end
 
+

data/lib/td/command/result.rb

@@ -4,7 +4,6 @@ module Command
 
   def result_show(op)
     name = op.cmd_parse
-
     client = get_client
 
     rs = client.results
@@ -12,7 +11,7 @@ module Command
 
     unless r
       $stderr.puts "Result URL '#{name}' does not exist."
-      $stderr.puts "Use '#{$prog} result:create #{name} <URL>' to create the URL."
+      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "result:create #{name} <URL>' to create the URL."
       exit 1
     end
 
@@ -21,6 +20,8 @@ module Command
   end
 
   def result_list(op)
+    set_render_format_option(op)
+
     op.cmd_parse
 
     client = get_client
@@ -35,11 +36,11 @@ module Command
       map[:Name]
     }
 
-    puts cmd_render_table(rows, :fields => [:Name, :URL])
+    puts cmd_render_table(rows, :fields => [:Name, :URL], :render_format => op.render_format)
 
     if rs.empty?
       $stderr.puts "There are no result URLs."
-      $stderr.puts "Use '#{$prog} result:create <name> <url>' to create a result URL."
+      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "result:create <name> <url>' to create a result URL."
     end
   end
 

data/lib/td/command/runner.rb

@@ -21,7 +21,7 @@ class Runner
     $prog = @prog_name || File.basename($0)
 
     op = OptionParser.new
-    op.version = TreasureData::VERSION
+    op.version = VERSION
     op.banner = <<EOF
 usage: #{$prog} [options] COMMAND [args]
 
@@ -109,13 +109,14 @@ EOF
 
       require 'td/config'
       if config_path
-        TreasureData::Config.path = config_path
+        Config.path = config_path
       end
       if apikey
-        TreasureData::Config.apikey = apikey
+        Config.apikey = apikey
+        Config.cl_apikey = true
       end
       if insecure
-        TreasureData::Config.secure = false
+        Config.secure = false
       end
     rescue
       usage $!.to_s
@@ -127,16 +128,16 @@ EOF
       Encoding.default_external = 'UTF-8' if Encoding.respond_to?(:default_external)
     end
 
-    method = TreasureData::Command::List.get_method(cmd)
+    method = Command::List.get_method(cmd)
     unless method
       $stderr.puts "'#{cmd}' is not a td command. Run '#{$prog}' to show the list."
-      TreasureData::Command::List.show_guess(cmd)
+      Command::List.show_guess(cmd)
       exit 1
     end
 
     begin
       method.call(argv)
-    rescue TreasureData::ConfigError
+    rescue ConfigError
       $stderr.puts "TreasureData account is not configured yet."
       $stderr.puts "Run '#{$prog} account' first."
     rescue => e

data/lib/td/command/sample.rb

@@ -25,7 +25,7 @@ module Command
 
     $stderr.puts "Create #{fname} with #{i} records whose time is"
     $stderr.puts "from #{Time.at(last_time)} to #{Time.at(t)}."
-    $stderr.puts "Use '#{$prog} table:import <db> <table> --json #{fname}' to import this file."
+    $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "table:import <db> <table> --json #{fname}' to import this file."
   end
 
 end

data/lib/td/command/sched.rb

@@ -5,6 +5,8 @@ module Command
   def sched_list(op)
     require 'td/command/job'  # job_priority_name_of
 
+    set_render_format_option(op)
+
     op.cmd_parse
 
     client = get_client
@@ -19,7 +21,7 @@ module Command
       map[:Name]
     }
 
-    puts cmd_render_table(rows, :fields => [:Name, :Cron, :Timezone, :"Next schedule", :Delay, :Priority, :Result, :Database, :Query], :max_width=>500)
+    puts cmd_render_table(rows, :fields => [:Name, :Cron, :Timezone, :"Next schedule", :Delay, :Priority, :Result, :Database, :Query], :max_width=>500, :render_format => op.render_format)
   end
 
   def sched_create(op)
@@ -119,7 +121,7 @@ module Command
     rescue NotFoundError
       cmd_debug_error $!
       $stderr.puts "Schedule '#{name}' does not exist."
-      $stderr.puts "Use '#{$prog} sched:list' to show list of the schedules."
+      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "sched:list' to show list of the schedules."
       exit 1
     end
 
@@ -196,7 +198,7 @@ module Command
     rescue NotFoundError
       cmd_debug_error $!
       $stderr.puts "Schedule '#{name}' does not exist."
-      $stderr.puts "Use '#{$prog} sched:list' to show list of the schedules."
+      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "sched:list' to show list of the schedules."
       exit 1
     end
 
@@ -215,6 +217,7 @@ module Command
     op.on('-s', '--skip N', 'skip N schedules', Integer) {|i|
       skip = i
     }
+    set_render_format_option(op)
 
     name, max = op.cmd_parse
 
@@ -231,7 +234,7 @@ module Command
     rescue NotFoundError
       cmd_debug_error $!
       $stderr.puts "Schedule '#{name}' does not exist."
-      $stderr.puts "Use '#{$prog} sched:list' to show list of the schedules."
+      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "sched:list' to show list of the schedules."
       exit 1
     end
 
@@ -254,7 +257,7 @@ module Command
       rows << {:Time => j.scheduled_at.localtime, :JobID => j.job_id, :Status => j.status, :Priority => job_priority_name_of(j.priority), :Result=>j.result_url}
     }
 
-    puts cmd_render_table(rows, :fields => [:JobID, :Time, :Status, :Priority, :Result])
+    puts cmd_render_table(rows, :fields => [:JobID, :Time, :Status, :Priority, :Result], :render_format => op.render_format)
   end
 
   def sched_run(op)
@@ -263,6 +266,7 @@ module Command
     op.on('-n', '--num N', 'number of jobs to run', Integer) {|i|
       num = i
     }
+    set_render_format_option(op)
 
     name, time = op.cmd_parse
 
@@ -286,7 +290,7 @@ module Command
     rescue NotFoundError
       cmd_debug_error $!
       $stderr.puts "Schedule '#{name}' does not exist."
-      $stderr.puts "Use '#{$prog} sched:list' to show list of the schedules."
+      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "sched:list' to show list of the schedules."
       exit 1
     end
 
@@ -296,7 +300,7 @@ module Command
     }
 
     $stderr.puts "Scheduled #{num} jobs from #{t}."
-    puts cmd_render_table(rows, :fields => [:JobID, :Time], :max_width=>500)
+    puts cmd_render_table(rows, :fields => [:JobID, :Time], :max_width=>500, :render_format => op.render_format)
   end
 
 end

data/lib/td/command/table.rb

@@ -65,7 +65,7 @@ module Command
     rescue NotFoundError
       cmd_debug_error $!
       $stderr.puts "Database '#{db_name}' does not exist."
-      $stderr.puts "Use '#{$prog} db:create #{db_name}' to create the database."
+      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "db:create #{db_name}' to create the database."
       exit 1
     rescue AlreadyExistsError
       cmd_debug_error $!
@@ -111,7 +111,7 @@ module Command
     rescue NotFoundError
       cmd_debug_error $!
       $stderr.puts "Table '#{db_name}.#{table_name}' does not exist."
-      $stderr.puts "Use '#{$prog} table:list #{db_name}' to show list of the tables."
+      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "table:list #{db_name}' to show list of the tables."
       exit 1
     end
 
@@ -121,6 +121,7 @@ module Command
   def table_list(op)
     require 'parallel'
 
+    format = 'table'
     num_threads = 4
     show_size_in_bytes = false
 
@@ -130,6 +131,7 @@ module Command
     op.on('--show-bytes', 'show estimated table size in bytes') {
       show_size_in_bytes = true
     }
+    set_render_format_option(op)
 
     db_name = op.cmd_parse
 
@@ -169,18 +171,18 @@ module Command
       [map[:Database], map[:Type].size, map[:Table]]
     }
 
-    puts cmd_render_table(rows, :fields => [:Database, :Table, :Type, :Count, :Size, 'Last import', 'Last log timestamp', :Schema], :max_width=>500)
+    puts cmd_render_table(rows, :fields => [:Database, :Table, :Type, :Count, :Size, 'Last import', 'Last log timestamp', :Schema], :max_width=>500, :render_format => op.render_format)
 
     if rows.empty?
       if db_name
         $stderr.puts "Database '#{db_name}' has no tables."
-        $stderr.puts "Use '#{$prog} table:create <db> <table>' to create a table."
+        $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "table:create <db> <table>' to create a table."
       elsif databases.empty?
         $stderr.puts "There are no databases."
-        $stderr.puts "Use '#{$prog} db:create <db>' to create a database."
+        $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "db:create <db>' to create a database."
       else
         $stderr.puts "There are no tables."
-        $stderr.puts "Use '#{$prog} table:create <db> <table>' to create a table."
+        $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "table:create <db> <table>' to create a table."
       end
     end
   end
@@ -292,7 +294,7 @@ module Command
     #    from = Time.parse(s).to_i
     #  end
     #}
-    op.on('-w', '--wait', 'wait for finishing the job', TrueClass) {|b|
+    op.on('-w', '--wait', 'wait until the job is completed', TrueClass) {|b|
       wait = b
     }
     op.on('--s3-bucket NAME', 'name of the s3 bucket to output') {|s|
@@ -319,7 +321,7 @@ module Command
     job = table.export('s3', opts)
 
     $stderr.puts "Export job #{job.job_id} is queued."
-    $stderr.puts "Use '#{$prog} job:show #{job.job_id}' to show the status."
+    $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "job:show #{job.job_id}' to show the status."
 
     if wait && !job.finished?
       wait_job(job)
@@ -378,7 +380,7 @@ module Command
     job = client.partial_delete(db_name, table_name, to, from, opts)
 
     $stderr.puts "Partial delete job #{job.job_id} is queued."
-    $stderr.puts "Use '#{$prog} job:show #{job.job_id}' to show the status."
+    $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "job:show #{job.job_id}' to show the status."
 
     if wait && !job.finished?
       wait_job(job)

data/lib/td/command/user.rb

@@ -11,7 +11,7 @@ module Command
     user = users.find {|user| name == user.name }
     unless user
       $stderr.puts "User '#{name}' does not exist."
-      $stderr.puts "Use '#{$prog} user:create <name>' to create an user."
+      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "user:create <name>' to create an user."
       exit 1
     end
 
@@ -20,6 +20,8 @@ module Command
   end
 
   def user_list(op)
+    set_render_format_option(op)
+
     op.cmd_parse
 
     client = get_client
@@ -31,11 +33,11 @@ module Command
       rows << {:Name => user.name, :Email => user.email}
     }
 
-    puts cmd_render_table(rows, :fields => [:Name, :Email])
+    puts cmd_render_table(rows, :fields => [:Name, :Email], :render_format => op.render_format)
 
     if rows.empty?
       $stderr.puts "There are no users."
-      $stderr.puts "Use '#{$prog} user:create <name>' to create an users."
+      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "user:create <name>' to create an users."
     end
   end
 
@@ -118,7 +120,7 @@ module Command
     client.add_user(name, nil, email, password)
 
     $stderr.puts "User '#{name}' is created."
-    $stderr.puts "Use '#{$prog} user:apikeys #{name}' to show the API key."
+    $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "user:apikeys #{name}' to show the API key."    
   end
 
   def user_delete(op)
@@ -144,12 +146,12 @@ module Command
       client.add_apikey(name)
     rescue TreasureData::NotFoundError
       $stderr.puts "User '#{name}' does not exist."
-      $stderr.puts "Use '#{$prog} users' to show users."
+      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "users' to show users."
       exit 1
     end
 
     $stderr.puts "Added an API key to user '#{name}'."
-    $stderr.puts "Use '#{$prog} user:apikeys #{name}' to show the API key"
+    $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "user:apikeys #{name}' to show the API key"
   end
 
   def user_apikey_remove(op)
@@ -161,8 +163,8 @@ module Command
       client.remove_apikey(name, key)
     rescue TreasureData::NotFoundError
       $stderr.puts "User '#{name}' or API key '#{key}' does not exist."
-      $stderr.puts "Use '#{$prog} users' to show users."
-      $stderr.puts "Use '#{$prog} user:apikeys '#{key}' to show API keys"
+      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "users' to show users."
+      $stderr.puts "Use '#{$prog} " + Config.cl_apikey_string + "user:apikeys '#{key}' to show API keys"
       exit 1
     end
 
@@ -170,6 +172,8 @@ module Command
   end
 
   def user_apikey_list(op)
+    set_render_format_option(op)
+
     name = op.cmd_parse
 
     client = get_client
@@ -181,7 +185,7 @@ module Command
       rows << {:Key => key}
     }
 
-    puts cmd_render_table(rows, :fields => [:Key])
+    puts cmd_render_table(rows, :fields => [:Key], :render_format => op.render_format)
   end
 
   def user_password_change(op)

data/lib/td/config.rb

@@ -91,6 +91,7 @@ class Config
   @@path = ENV['TREASURE_DATA_CONFIG_PATH'] || ENV['TD_CONFIG_PATH'] || File.join(ENV['HOME'], '.td', 'td.conf')
   @@apikey = ENV['TREASURE_DATA_API_KEY'] || ENV['TD_API_KEY']
   @@apikey = nil if @@apikey == ""
+  @@cl_apikey = false # flag to indicate whether an apikey has been provided through the command-line
   @@secure = true
 
   def self.path
@@ -109,6 +110,22 @@ class Config
     @@apikey = apikey
   end
 
+  def self.cl_apikey
+    @@cl_apikey
+  end
+
+  def self.cl_apikey=(flag)
+    @@cl_apikey = flag
+  end
+
+  def self.cl_apikey_string
+    if @@cl_apikey
+      "-k #{@@apikey} "
+    else
+      ""
+    end
+  end
+
   def self.secure
     @@secure
   end

data/lib/td/version.rb

@@ -1,5 +1,5 @@
 module TreasureData
 
-VERSION = '0.10.95'
+VERSION = '0.10.96'
 
 end

metadata

@@ -1,34 +1,32 @@
 --- !ruby/object:Gem::Specification
 name: td
 version: !ruby/object:Gem::Version
-  version: 0.10.95
-  prerelease: 
+  version: 0.10.96
 platform: ruby
 authors:
 - Treasure Data, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-11-20 00:00:00.000000000 Z
+date: 2013-12-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.4.4
-    - - ! '!='
+    - - '!='
       - !ruby/object:Gem::Version
         version: 0.5.0
-    - - ! '!='
+    - - '!='
       - !ruby/object:Gem::Version
         version: 0.5.1
-    - - ! '!='
+    - - '!='
       - !ruby/object:Gem::Version
         version: 0.5.2
-    - - ! '!='
+    - - '!='
       - !ruby/object:Gem::Version
         version: 0.5.3
     - - <
@@ -37,21 +35,20 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.4.4
-    - - ! '!='
+    - - '!='
       - !ruby/object:Gem::Version
         version: 0.5.0
-    - - ! '!='
+    - - '!='
       - !ruby/object:Gem::Version
         version: 0.5.1
-    - - ! '!='
+    - - '!='
       - !ruby/object:Gem::Version
         version: 0.5.2
-    - - ! '!='
+    - - '!='
       - !ruby/object:Gem::Version
         version: 0.5.3
     - - <
@@ -60,7 +57,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -68,7 +64,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -76,23 +71,20 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: hirb
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.4.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.4.5
 - !ruby/object:Gem::Dependency
   name: parallel
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -100,7 +92,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -108,7 +99,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: td-client
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -116,7 +106,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -124,7 +113,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: td-logger
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -132,7 +120,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -140,7 +127,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rubyzip
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -148,7 +134,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -156,7 +141,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -164,7 +148,6 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -172,7 +155,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -180,7 +162,6 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -188,7 +169,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -196,7 +176,6 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -279,33 +258,26 @@ files:
 - td.gemspec
 homepage: http://treasure-data.com/
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: -3232454928558496332
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: -3232454928558496332
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.0.3
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: CLI to manage data on Treasure Data, the Hadoop-based cloud data warehousing
 test_files:
 - spec/file_reader/filter_spec.rb