td 0.8.0 → 0.9.0

data/lib/td/command/list.rb

@@ -3,79 +3,128 @@ module TreasureData
 module Command
 module List
 
+  class CommandOption < OptionParser
+    def initialize(name, args, description)
+      super()
+
+      @name = name
+      @description = description.to_s
+
+      if args.last.to_s =~ /_$/
+        @varlen = true
+        args.push args.pop.to_s[0..-2]+'...'
+      elsif args.last.to_s =~ /_\?$/
+        @varlen = true
+        args.push args.pop.to_s[0..-3]+'...?'
+      end
+
+      @req_args, opt_args = args.partition {|a| a.to_s !~ /\?$/ }
+      @opt_args = opt_args.map {|a| a.to_s[0..-2].to_sym }
+      @args = @req_args + @opt_args
+
+      @usage_args = "#{@name}"
+      @req_args.each {|a| @usage_args << " <#{a}>" }
+      @opt_args.each {|a| @usage_args << " [#{a}]" }
+
+      @has_options = false
+
+      self.summary_indent = "  "
+
+      banner  = "usage:\n"
+      banner << "  $ #{File.basename($0)} #{@usage_args}\n"
+      banner << "\n"
+      banner << "description:\n"
+      @description.split("\n").each {|l|
+        banner << "  #{l}\n"
+      }
+      self.banner = banner
+
+      @message = nil
+    end
+
+    attr_accessor :message
+
+    def message
+      @message || banner
+    end
+
+    def banner
+      s = super.dup
+      if @has_options
+        s << "\n"
+        s << "options:\n"
+      end
+      s << "\n"
+      s
+    end
+
+    def usage
+      "%-40s   # %s" % [@usage_args, @description]
+    end
+
+    def cmd_parse(argv=ARGV)
+      parse!(argv)
+      if argv.length < @req_args.length || (!@varlen && argv.length > @args.length)
+        cmd_usage nil
+      end
+      if argv.length <= 1
+        return argv[0]
+      else
+        return argv
+      end
+    rescue
+      cmd_usage $!
+    end
+
+    def cmd_usage(msg=nil)
+      puts self.message
+      if msg
+        puts ""
+        puts "error: #{msg}"
+      end
+      exit 1
+    end
+
+    def group
+      name.split(':', 2).first
+    end
+
+    def on(*args)
+      @has_options = true
+      super
+    end
+
+    attr_reader :name
+    attr_reader :description
+  end
+
   LIST = []
-  ALIASES = {}
+  COMMAND = {}
   GUESS = {}
+  HELP_EXCLUDE = ['help', 'account']
 
-  def self.add_list(file, cmd, description)
-    LIST << [cmd, file, description]
+  def self.add_list(name, args, description)
+    LIST << COMMAND[name] = CommandOption.new(name, args, description)
   end
 
   def self.add_alias(new_cmd, old_cmd)
-    ALIASES[new_cmd] = old_cmd
-  end
-
-  def self.get_description(command)
-    LIST.each {|cmd,file,description|
-      if cmd == command
-        return description
-      end
-    }
-    nil
+    COMMAND[new_cmd] = COMMAND[old_cmd]
   end
 
   def self.add_guess(wrong, correct)
     GUESS[wrong] = correct
   end
 
-  add_list 'list', 'help', 'Show usage of a command'
-  add_list 'account', 'account', 'Setup a Treasure Data account'
-  add_list 'server', 'server-status', 'Show status of the Treasure Data server'
-  add_list 'database', 'show-databases', 'Show list of databases'
-  add_list 'table', 'show-tables', 'Show list of tables'
-  add_list 'database', 'create-database', 'Create a database'
-  add_list 'table', 'create-log-table', 'Create a log table'
-  #add_list 'table', 'create-item-table', 'Create a item table'
-  add_list 'table', 'set-schema', 'Set a schema on a table'
-  add_list 'table', 'describe-table', 'Describe information of a table'
-  add_list 'database', 'drop-database', 'Delete a database'
-  add_list 'table', 'drop-table', 'Delete a table'
-  add_list 'query', 'query', 'Start a query'
-  add_list 'query', 'job', 'Show status and result of a job'
-  add_list 'query', 'show-jobs', 'Show list of jobs'
-  add_list 'import', 'import', 'Import files to a table'
-  add_list 'list', 'version', 'Show version'
-
-  add_alias 'show-dbs', 'show-databases'
-  add_alias 'databases', 'show-databases'
-  add_alias 'dbs', 'show-databases'
-  add_alias 'create-db', 'create-databases'
-  add_alias 'drop-db', 'create-databases'
-  add_alias 'tables', 'show-tables'
-  add_alias 'table', 'describe-table'
-  add_alias 'show-table', 'describe-table'
-  add_alias 'delete-database', 'drop-database'
-  add_alias 'delete-table', 'drop-table'
-  add_alias 'jobs', 'show-jobs'
-  add_alias 'update-schema', 'set-schema'
-
-  add_guess 'create-table', 'create-log-table'
-  add_guess 'drop-log-table', 'drop-table'
-  #add_guess 'drop-item-table', 'drop-table'
-  add_guess 'delete-log-table', 'drop-table'
-  #add_guess 'delete-item-table', 'drop-table'
-  add_guess 'show-job', 'job'
-
-  def self.get_method(command)
-    command = ALIASES[command] || command
-    LIST.each {|cmd,file,description|
-      if cmd == command
-        require 'td/command/common'
-        require "td/command/#{file}"
-        name = command.gsub('-','_')
-        return Object.new.extend(Command).method(name)
-      end
-    }
+  def self.get_method(name)
+    if op = COMMAND[name]
+      name = op.name
+      group, action = op.group
+      require 'td/command/common'
+      require "td/command/#{group}"
+      cmd = name.gsub(':', '_')
+      m = Object.new.extend(Command).method(cmd)
+      return Proc.new { m.call(op) }
+    end
     nil
   end
 
@@ -85,37 +134,117 @@ module List
     end
   end
 
-  def self.help(indent)
-    LIST.map {|cmd,file,description|
-      if cmd != 'help'
-        "#{indent}%-18s %s" % [cmd, description.split("\n").first]
-      end
-    }.join("\n")
+  def self.get_option(name)
+    COMMAND[name]
   end
-end
-
-def help
-  op = cmd_opt 'help', :command
-  cmd = op.cmd_parse
 
-  ARGV.clear
-  ARGV[0] = '--help'
+  def self.show_help(indent='  ')
+    before_group = nil
+    LIST.each {|op|
+      next if HELP_EXCLUDE.include?(op.name)
+      if before_group != op.group
+        before_group = op.group
+        puts ""
+      end
+      puts "#{indent}#{op.usage}"
+    }
+  end
 
-  method = List.get_method(cmd)
-  unless method
-    $stderr.puts "'#{cmd}' is not a td command. Run '#{$prog}' to show the list."
-    List.show_guess(cmd)
-    exit 1
+  def self.get_group(group)
+    LIST.map {|op|
+      op.group == group
+    }
   end
 
-  method.call
-end
+  def self.finishup
+    groups = {}
+    LIST.each {|op|
+      (groups[op.group] ||= []) << op
+    }
+    groups.each_pair {|group,ops|
+      if ops.size > 1 && xop = COMMAND[group].dup
+        msg = %[Additional commands, type "#{File.basename($0)} help COMMAND" for more details:\n\n]
+        ops.each {|op|
+          msg << %[  #{op.usage}\n]
+        }
+        msg << %[\n]
+        xop.message = msg
+        COMMAND[group] = xop
+      end
+    }
+  end
 
-def version
-  require 'td/version'
-  puts "td-#{TreasureData::VERSION}"
+  add_list 'db:list', %w[], 'Show list of tables in a database'
+  add_list 'db:show', %w[db], 'Describe a information of a database'
+  add_list 'db:create', %w[db], 'Create a database'
+  add_list 'db:delete', %w[db], 'Delete a database'
+
+  add_list 'table:list', %w[db?], 'Show list of tables'
+  add_list 'table:show', %w[db table], 'Describe a information of a table'
+  add_list 'table:create', %w[db table], 'Create a table'
+  add_list 'table:delete', %w[db table], 'Delete a table'
+  add_list 'table:import', %w[db table files_], 'Parse and import files to a table'
+
+  add_list 'schema:show', %w[db table], 'Show schema of a table'
+  add_list 'schema:set', %w[db table columns_?], 'Set new schema on a table'
+  add_list 'schema:add', %w[db table columns_], 'Add new columns to a table'
+  add_list 'schema:remove', %w[db table columns_], 'Remove columns from a table'
+
+  add_list 'query', %w[sql], 'Issue a query'
+
+  add_list 'job:show', %w[job_id], 'Show status and result of a job'
+  add_list 'job:list', %w[max?], 'Show list of jobs'
+  #add_list 'job:kill', %w[job_id], 'Kill a job'
+
+  add_list 'account', %w[], 'Setup a Treasure Data account'
+
+  add_list 'server:status', %w[], 'Show status of the Treasure Data server'
+
+  add_list 'help', %w[command], 'Show usage of a command'
+
+  # aliases
+  add_alias 'db', 'db:show'
+  add_alias 'dbs', 'db:list'
+
+  add_alias 'database:show', 'db:show'
+  add_alias 'database:list', 'db:list'
+  add_alias 'database:create', 'db:create'
+  add_alias 'database:delete', 'db:delete'
+  add_alias 'database', 'db:show'
+  add_alias 'databases', 'db:list'
+
+  add_alias 'table', 'table:show'
+  add_alias 'tables', 'table:list'
+
+  add_alias 'schema', 'schema:show'
+
+  add_alias 'job', 'job:show'
+  add_alias 'jobs', 'job:list'
+
+  # backward compatibility
+  add_alias 'show-databases',   'db:list'
+  add_alias 'show-dbs',         'db:list'
+  add_alias 'create-database',  'db:create'
+  add_alias 'create-db',        'db:create'
+  add_alias 'drop-database',    'db:delete'
+  add_alias 'drop-db',          'db:delete'
+  add_alias 'delete-database',  'db:delete'
+  add_alias 'delete-db',        'db:delete'
+  add_alias 'show-tables',      'table:list'
+  add_alias 'show-table',       'table:show'
+  add_alias 'create-log-table', 'table:create'
+  add_alias 'create-table',     'table:create'
+  add_alias 'drop-log-table',   'table:delete'
+  add_alias 'drop-table',       'table:delete'
+  add_alias 'delete-log-table', 'table:delete'
+  add_alias 'delete-table',     'table:delete'
+  add_guess 'show-job',         'job:show'
+  add_guess 'show-jobs',        'job:list'
+  add_guess 'server-status',    'server:status'
+  add_alias 'import',           'table:import'
+
+  finishup
 end
-
 end
 end
 

data/lib/td/command/query.rb

@@ -2,11 +2,7 @@
 module TreasureData
 module Command
 
-  def query
-    op = cmd_opt 'query', :sql
-
-    op.banner << "\noptions:\n"
-
+  def query(op)
     db_name = nil
     wait = false
     output = nil
@@ -37,12 +33,12 @@ module Command
       exit 1
     end
 
-    find_database(client, db_name)
+    get_database(client, db_name)
 
     job = client.query(db_name, sql)
 
     $stderr.puts "Job #{job.job_id} is started."
-    $stderr.puts "Use '#{$prog} job #{job.job_id}' to show the status."
+    $stderr.puts "Use '#{$prog} job:show #{job.job_id}' to show the status."
     $stderr.puts "See #{job.url} to see the progress."
 
     if wait && !job.finished?
@@ -55,233 +51,7 @@ module Command
     end
   end
 
-  def show_jobs
-    op = cmd_opt 'show-jobs', :max?
-
-    op.banner << "\noptions:\n"
-
-    page = 0
-    skip = 0
-
-    op.on('-p', '--page PAGE', 'skip N pages', Integer) {|i|
-      page = i
-    }
-    op.on('-s', '--skip N', 'skip N jobs', Integer) {|i|
-      skip = i
-    }
-
-    max = op.cmd_parse
-
-    max = (max || 20).to_i
-
-    client = get_client
-
-    if page
-      skip += max * page
-    end
-    jobs = client.jobs(skip, skip+max-1)
-
-    rows = []
-    jobs.each {|job|
-      start = job.start_at
-      finish = job.end_at
-      if start
-        if !finish
-          finish = Time.now.utc
-        end
-        e = finish.to_i - start.to_i
-        elapsed = ''
-        if e >= 3600
-          elapsed << "#{e/3600}h "
-          e %= 3600
-          elapsed << "% 2dm " % (e/60)
-          e %= 60
-          elapsed << "% 2dsec" % e
-        elsif e >= 60
-          elapsed << "% 2dm " % (e/60)
-          e %= 60
-          elapsed << "% 2dsec" % e
-        else
-          elapsed << "% 2dsec" % e
-        end
-      else
-        elapsed = ''
-      end
-      elapsed = "% 10s" % elapsed  # right aligned
-
-      rows << {:JobID => job.job_id, :Status => job.status, :Query => job.query.to_s, :Start => start, :Elapsed => elapsed}
-    }
-
-    puts cmd_render_table(rows, :fields => [:JobID, :Status, :Start, :Elapsed, :Query])
-  end
-
-  def job
-    op = cmd_opt 'job', :job_id
-
-    op.banner << "\noptions:\n"
-
-    verbose = nil
-    wait = false
-    output = nil
-    format = 'tsv'
-
-    op.on('-v', '--verbose', 'show logs', TrueClass) {|b|
-      verbose = b
-    }
-    op.on('-w', '--wait', 'wait for finishing the job', TrueClass) {|b|
-      wait = b
-    }
-    op.on('-o', '--output PATH', 'write result to the file') {|s|
-      output = s
-    }
-    op.on('-f', '--format FORMAT', 'format of the result to write to the file (tsv, csv, json or msgpack)') {|s|
-      unless ['tsv', 'csv', 'json', 'msgpack'].include?(s)
-        raise "Unknown format #{s.dump}. Supported format: tsv, csv, json, msgpack"
-      end
-      format = s
-    }
-
-    job_id = op.cmd_parse
-
-    client = get_client
-
-    job = client.job(job_id)
-
-    puts "JobID      : #{job.job_id}"
-    puts "URL        : #{job.url}"
-    puts "Status     : #{job.status}"
-    puts "Query      : #{job.query}"
-
-    if wait && !job.finished?
-      wait_job(job)
-      if job.success?
-        puts "Result     :"
-        show_result(job, output, format)
-      end
-
-    else
-      if job.success?
-        puts "Result     :"
-        show_result(job, output, format)
-      end
-
-      if verbose
-        puts ""
-        puts "cmdout:"
-        job.debug['cmdout'].to_s.split("\n").each {|line|
-          puts "  "+line
-        }
-        puts ""
-        puts "stderr:"
-        job.debug['stderr'].to_s.split("\n").each {|line|
-          puts "  "+line
-        }
-      end
-    end
-
-    $stderr.puts "Use '-v' option to show detailed messages." unless verbose
-  end
-
-  private
-  def wait_job(job)
-    $stderr.puts "running..."
-
-    cmdout_lines = 0
-    stderr_lines = 0
-
-    until job.finished?
-      sleep 2
-
-      job.update_status!
-
-      cmdout = job.debug['cmdout'].to_s.split("\n")[cmdout_lines..-1] || []
-      stderr = job.debug['stderr'].to_s.split("\n")[stderr_lines..-1] || []
-      (cmdout + stderr).each {|line|
-        puts "  "+line
-      }
-      cmdout_lines += cmdout.size
-      stderr_lines += stderr.size
-    end
-  end
-
-  def show_result(job, output, format)
-    if output
-      write_result(job, output, format)
-      puts "written to #{output} in #{format} format"
-    else
-      render_result(job)
-    end
-  end
-
-  def write_result(job, output, format)
-    case format
-    when 'json'
-      require 'json'
-      first = true
-      File.open(output, "w") {|f|
-        f.write "["
-        job.result_each {|row|
-          if first
-            first = false
-          else
-            f.write ","
-          end
-          f.write row.to_json
-        }
-        f.write "]"
-      }
-
-    when 'msgpack'
-      File.open(output, "w") {|f|
-        f.write job.result_format('msgpack')
-      }
-
-    when 'csv'
-      require 'json'
-      require 'csv'
-      CSV.open(output, "w") {|writer|
-        job.result_each {|row|
-          writer << row.map {|col| col.is_a?(String) ? col.to_s : col.to_json }
-        }
-      }
-
-    when 'tsv'
-      require 'json'
-      File.open(output, "w") {|f|
-        job.result_each {|row|
-          first = true
-          row.each {|col|
-            if first
-              first = false
-            else
-              f.write "\t"
-            end
-            f.write col.is_a?(String) ? col.to_s : col.to_json
-          }
-          f.write "\n"
-        }
-      }
-
-    else
-      raise "Unknown format #{format.inspect}"
-    end
-  end
-
-  def render_result(job)
-    require 'json'
-    rows = []
-    job.result_each {|row|
-      # TODO limit number of rows to show
-      rows << row.map {|v|
-        if v.is_a?(String)
-          v.to_s
-        else
-          v.to_json
-        end
-      }
-    }
-    puts cmd_render_table(rows, :max_width=>10000)
-  end
+  require 'td/command/job'  # wait_job
 end
 end