td 0.8.0 → 0.9.0

data/ChangeLog

@@ -1,4 +1,10 @@
 
+== 2011-09-08 version 0.9.0
+
+* Changed subcommands grammer: action-group -> group:action
+* Fixed import subcommand to strip \n
+
+
 == 2011-08-21 version 0.8.0
 
 * Splits API libraries to td-client gem

data/README.rdoc

@@ -1,5 +1,12 @@
 = Treasure Data command line tool
 
+This CUI utility wraps the td-client-ruby (https://github.com/treasure-data/td-client-ruby),
+the REST API for managing databases and jobs on the Treasure Data Cloud.
+
+For more about Treasure Data, see <http://treasure-data.com/>.
+
+For full documentation see <http://docs.treasure-data.com/>.
+
 = Getting Started
 
 Install td command as a gem.
@@ -10,9 +17,19 @@ See help message for details.
 
     > td
 
+You need to authorize the account, before executing any other commands.
+
+    > td account
 
-== Copyright
+= Sample Workflow
+
+    > td account -f                       # authorize an account
+    user: k@treasure-data.com
+    password: **********
+    > td database:create mydb             # create a database
+    > td table:create mydb www_access     # create a table
+
+= Copyright
 
 Copyright:: Copyright (c) 2011 Treasure Data Inc.
 License::   Apache License, Version 2.0
-

data/bin/td

@@ -1,6 +1,7 @@
 #!/usr/bin/env ruby
 # -*- coding: utf-8 -*-
-#require 'rubygems' unless defined?(gem)
+require 'rubygems' unless defined?(gem)
+gem 'td-client'
 here = File.dirname(__FILE__)
 $LOAD_PATH << File.expand_path(File.join(here, '..', 'lib'))
 require 'td/command/td'

data/lib/td/command/common.rb

@@ -9,62 +9,10 @@ autoload :Schema, 'td/client'
 autoload :Job, 'td/client'
 
 module Command
-  private
-  def cmd_opt(name, *args)
-    if args.last.to_s =~ /_$/
-      multi = true
-      args.push args.pop.to_s[0..-2]+'...'
-    elsif args.last.to_s =~ /_\?$/
-      multi = true
-      args.push args.pop.to_s[0..-3]+'...?'
-    end
-
-    req_args, opt_args = args.partition {|a| a.to_s !~ /\?$/ }
-    opt_args = opt_args.map {|a| a.to_s[0..-2].to_sym }
-    args = req_args + opt_args
-
-    args_line = req_args.map {|a| "<#{a}>" }
-    args_line.concat opt_args.map {|a| "[#{a}]" }
-    args_line = args_line.join(' ')
-
-    description = List.get_description(name)
 
-    op = OptionParser.new
-    op.summary_indent = "  "
-    op.banner = <<EOF
-usage: #{$prog} #{name} #{args_line}
-
-description:
-#{description.split("\n").map {|l| "  #{l}" }.join("\n")}
-EOF
-
-    (class<<op;self;end).module_eval do
-      define_method(:cmd_usage) do |msg|
-        puts op.to_s
-        puts ""
-        puts "error: #{msg}" if msg
-        exit 1
-      end
-
-      define_method(:cmd_parse) do
-        begin
-          parse!(ARGV)
-          if ARGV.length < req_args.length || (!multi && ARGV.length > args.length)
-            cmd_usage nil
-          end
-          if ARGV.length <= 1
-            ARGV[0]
-          else
-            ARGV
-          end
-        rescue
-          cmd_usage $!
-        end
-      end
-
-    end
-
-    op
+  private
+  def get_option(name)
+    List.get_option(name)
   end
 
   def get_client
@@ -95,32 +43,41 @@ EOF
     end
   end
 
-  def find_database(client, db_name)
+  def get_database(client, db_name)
     begin
       return client.database(db_name)
     rescue
       cmd_debug_error $!
       $stderr.puts $!
-      $stderr.puts "Use '#{$prog} show-databases' to show the list of databases."
+      $stderr.puts "Use '#{$prog} database:list' to show the list of databases."
       exit 1
     end
     db
   end
 
-  def find_table(client, db_name, table_name, type=nil)
-    db = find_database(client, db_name)
+  def parse_table_ident(table_ident)
+    db_name, table_name = table_ident.split('.', 2)
+    unless table_name
+      $stderr.puts "Invalid table identifier '#{table_ident}'; expected 'DB.TABLE'"
+      exit 1
+    end
+    return db_name, table_name
+  end
+
+  def get_table(client, db_name, table_name)
+    db = get_database(client, db_name)
     begin
       table = db.table(table_name)
     rescue
       $stderr.puts $!
-      $stderr.puts "Use '#{$prog} show-tables #{db_name}' to show the list of tables."
+      $stderr.puts "Use '#{$prog} table:list #{db_name}' to show the list of tables."
       exit 1
     end
-    if type && table.type != type
-      $stderr.puts "Table '#{db_name}.#{table_name} is not a #{type} table but a #{table.type} table"
-    end
+    #if type && table.type != type
+    #  $stderr.puts "Table '#{db_name}.#{table_name} is not a #{type} table but a #{table.type} table"
+    #end
     table
   end
+
 end
 end
-

data/lib/td/command/{database.rb → db.rb}

@@ -2,14 +2,52 @@
 module TreasureData
 module Command
 
-  def create_database
-    op = cmd_opt 'create-database', :db_name
+  def db_show(op)
     db_name = op.cmd_parse
 
     client = get_client
 
+    db = get_database(client, db_name)
+
+    rows = []
+    db.tables.each {|table|
+      pschema = table.schema.fields.map {|f|
+        "#{f.name}:#{f.type}"
+      }.join(', ')
+      rows << {:Table => table.name, :Type => table.type.to_s, :Count => table.count.to_s, :Schema=>pschema.to_s}
+    }
+    rows = rows.sort_by {|map|
+      [map[:Type].size, map[:Table]]
+    }
+
+    puts cmd_render_table(rows, :fields => [:Table, :Type, :Count, :Schema])
+  end
+
+  def db_list(op)
+    op.cmd_parse
+
+    client = get_client
+    dbs = client.databases
+
+    rows = []
+    dbs.each {|db|
+      rows << {:Name => db.name}
+    }
+    puts cmd_render_table(rows, :fields => [:Name])
+
+    if dbs.empty?
+      $stderr.puts "There are no databases."
+      $stderr.puts "Use '#{$prog} create-database <db_name>' to create a database."
+    end
+  end
+
+  def db_create(op)
+    db_name = op.cmd_parse
+
     API.validate_database_name(db_name)
 
+    client = get_client
+
     begin
       client.create_database(db_name)
     rescue AlreadyExistsError
@@ -21,11 +59,7 @@ module Command
     $stderr.puts "Use '#{$prog} create-log-table #{db_name} <table_name>' to create a table."
   end
 
-  def drop_database
-    op = cmd_opt 'drop-database', :db_name
-
-    op.banner << "\noptions:\n"
-
+  def db_delete(op)
     force = false
     op.on('-f', '--force', 'clear tables and delete the database', TrueClass) {|b|
       force = true
@@ -46,35 +80,12 @@ module Command
       db.delete
     rescue NotFoundError
       $stderr.puts "Database '#{db_name}' does not exist."
-      eixt 1
+      exit 1
     end
 
     $stderr.puts "Database '#{db_name}' is deleted."
   end
 
-  def show_databases
-    op = cmd_opt 'show-databases'
-    op.cmd_parse
-
-    client = get_client
-
-    dbs = client.databases
-
-    rows = []
-    dbs.each {|db|
-      rows << {:Name => db.name}
-    }
-    puts cmd_render_table(rows, :fields => [:Name])
-
-    if dbs.empty?
-      $stderr.puts "There are no databases."
-      $stderr.puts "Use '#{$prog} create-database <db_name>' to create a database."
-    end
-  end
-
-  alias show_dbs show_databases
-  alias create_db create_database
-  alias drop_db drop_database
 end
 end
 

data/lib/td/command/help.rb

@@ -0,0 +1,19 @@
+
+module TreasureData
+module Command
+
+  def help(op)
+    cmd = op.cmd_parse
+
+    op = List.get_option(cmd)
+    unless op
+      $stderr.puts "'#{cmd}' is not a td command. Run '#{$prog}' to show the list."
+      List.show_guess(cmd)
+      exit 1
+    end
+
+    puts op.message
+  end
+
+end
+end

data/lib/td/command/import.rb

@@ -16,17 +16,13 @@ module Command
   # TODO import-item
   # TODO tail
 
-  def import
-    op = cmd_opt 'import', :db_name, :table_name, :files_
-
+  def table_import(op)
     op.banner << "\nsupported formats:\n"
     op.banner << "  apache\n"
     op.banner << "  syslog\n"
     op.banner << "  msgpack\n"
     op.banner << "  json\n"
 
-    op.banner << "\noptions:\n"
-
     format = 'apache'
     time_key = nil
 
@@ -54,7 +50,8 @@ module Command
       time_key = s
     }
 
-    db_name, table_name, *paths = op.cmd_parse
+    table_ident, *paths = op.cmd_parse
+    db_name, table_name = parse_table_ident(table_ident)
 
     client = get_client
 
@@ -81,7 +78,7 @@ module Command
       parser = TextParser.new(names, regexp, time_format)
     end
 
-    find_table(client, db_name, table_name, :log)
+    get_table(client, db_name, table_name)
 
     require 'zlib'
 
@@ -176,6 +173,7 @@ module Command
       file.each_line {|line|
         i += 1
         begin
+          line.rstrip!
           m = @regexp.match(line)
           unless m
             raise "invalid log format at #{path}:#{i}"

data/lib/td/command/job.rb

@@ -0,0 +1,235 @@
+
+module TreasureData
+module Command
+
+  def job_list(op)
+    page = 0
+    skip = 0
+
+    op.on('-p', '--page PAGE', 'skip N pages', Integer) {|i|
+      page = i
+    }
+    op.on('-s', '--skip N', 'skip N jobs', Integer) {|i|
+      skip = i
+    }
+
+    max = op.cmd_parse
+
+    max = (max || 20).to_i
+
+    client = get_client
+
+    if page
+      skip += max * page
+    end
+    jobs = client.jobs(skip, skip+max-1)
+
+    rows = []
+    jobs.each {|job|
+      start = job.start_at
+      finish = job.end_at
+      if start
+        if !finish
+          finish = Time.now.utc
+        end
+        e = finish.to_i - start.to_i
+        elapsed = ''
+        if e >= 3600
+          elapsed << "#{e/3600}h "
+          e %= 3600
+          elapsed << "% 2dm " % (e/60)
+          e %= 60
+          elapsed << "% 2dsec" % e
+        elsif e >= 60
+          elapsed << "% 2dm " % (e/60)
+          e %= 60
+          elapsed << "% 2dsec" % e
+        else
+          elapsed << "% 2dsec" % e
+        end
+      else
+        elapsed = ''
+      end
+      elapsed = "% 10s" % elapsed  # right aligned
+
+      rows << {:JobID => job.job_id, :Status => job.status, :Query => job.query.to_s, :Start => start, :Elapsed => elapsed}
+    }
+
+    puts cmd_render_table(rows, :fields => [:JobID, :Status, :Start, :Elapsed, :Query])
+  end
+
+  def job_show(op)
+    verbose = nil
+    wait = false
+    output = nil
+    format = 'tsv'
+
+    op.on('-v', '--verbose', 'show logs', TrueClass) {|b|
+      verbose = b
+    }
+    op.on('-w', '--wait', 'wait for finishing the job', TrueClass) {|b|
+      wait = b
+    }
+    op.on('-o', '--output PATH', 'write result to the file') {|s|
+      output = s
+    }
+    op.on('-f', '--format FORMAT', 'format of the result to write to the file (tsv, csv, json or msgpack)') {|s|
+      unless ['tsv', 'csv', 'json', 'msgpack'].include?(s)
+        raise "Unknown format #{s.dump}. Supported format: tsv, csv, json, msgpack"
+      end
+      format = s
+    }
+
+    job_id = op.cmd_parse
+
+    client = get_client
+
+    job = client.job(job_id)
+
+    puts "JobID      : #{job.job_id}"
+    puts "URL        : #{job.url}"
+    puts "Status     : #{job.status}"
+    puts "Query      : #{job.query}"
+
+    if wait && !job.finished?
+      wait_job(job)
+      if job.success?
+        puts "Result     :"
+        show_result(job, output, format)
+      end
+
+    else
+      if job.success?
+        puts "Result     :"
+        show_result(job, output, format)
+      end
+
+      if verbose
+        puts ""
+        puts "cmdout:"
+        job.debug['cmdout'].to_s.split("\n").each {|line|
+          puts "  "+line
+        }
+        puts ""
+        puts "stderr:"
+        job.debug['stderr'].to_s.split("\n").each {|line|
+          puts "  "+line
+        }
+      end
+    end
+
+    $stderr.puts "Use '-v' option to show detailed messages." unless verbose
+  end
+
+  def job_kill(op)
+    job_id = op.cmd_parse
+
+    client = get_client
+
+    client.kill_job(job_id)
+    # TODO error
+  end
+
+  private
+  def wait_job(job)
+    $stderr.puts "running..."
+
+    cmdout_lines = 0
+    stderr_lines = 0
+
+    until job.finished?
+      sleep 2
+
+      job.update_status!
+
+      cmdout = job.debug['cmdout'].to_s.split("\n")[cmdout_lines..-1] || []
+      stderr = job.debug['stderr'].to_s.split("\n")[stderr_lines..-1] || []
+      (cmdout + stderr).each {|line|
+        puts "  "+line
+      }
+      cmdout_lines += cmdout.size
+      stderr_lines += stderr.size
+    end
+  end
+
+  def show_result(job, output, format)
+    if output
+      write_result(job, output, format)
+      puts "written to #{output} in #{format} format"
+    else
+      render_result(job)
+    end
+  end
+
+  def write_result(job, output, format)
+    case format
+    when 'json'
+      require 'json'
+      first = true
+      File.open(output, "w") {|f|
+        f.write "["
+        job.result_each {|row|
+          if first
+            first = false
+          else
+            f.write ","
+          end
+          f.write row.to_json
+        }
+        f.write "]"
+      }
+
+    when 'msgpack'
+      File.open(output, "w") {|f|
+        f.write job.result_format('msgpack')
+      }
+
+    when 'csv'
+      require 'json'
+      require 'csv'
+      CSV.open(output, "w") {|writer|
+        job.result_each {|row|
+          writer << row.map {|col| col.is_a?(String) ? col.to_s : col.to_json }
+        }
+      }
+
+    when 'tsv'
+      require 'json'
+      File.open(output, "w") {|f|
+        job.result_each {|row|
+          first = true
+          row.each {|col|
+            if first
+              first = false
+            else
+              f.write "\t"
+            end
+            f.write col.is_a?(String) ? col.to_s : col.to_json
+          }
+          f.write "\n"
+        }
+      }
+
+    else
+      raise "Unknown format #{format.inspect}"
+    end
+  end
+
+  def render_result(job)
+    require 'json'
+    rows = []
+    job.result_each {|row|
+      # TODO limit number of rows to show
+      rows << row.map {|v|
+        if v.is_a?(String)
+          v.to_s
+        else
+          v.to_json
+        end
+      }
+    }
+    puts cmd_render_table(rows, :max_width=>10000)
+  end
+end
+end
+