td 0.10.22 → 0.10.23

data/ChangeLog

@@ -1,4 +1,12 @@
 
+== 2012-04-26 version 0.10.23
+
+* Added result:list, result:create and result:delete subcommands
+* query subcommand supports -r <result URL> option
+* sched:create subcommand supports -r <result URL> option
+* Added table:export subcommand
+
+
 == 2012-04-04 version 0.10.22
 
 * Added missing data/sample_apache.json file

data/lib/td/command/common.rb

@@ -15,12 +15,17 @@ module Command
     @render_indent = ''
   end
 
-  def get_client
+  def get_client(opts={})
     apikey = Config.apikey
     unless apikey
       raise ConfigError, "Account is not configured."
     end
-    Client.new(apikey)
+    Client.new(apikey, opts)
+  end
+
+  def get_ssl_client(opts={})
+    opts[:ssl] = true
+    get_client(opts)
   end
 
   def cmd_render_table(rows, *opts)
@@ -96,5 +101,29 @@ module Command
     table
   end
 
+  def ask_password(max=3, &block)
+    3.times do
+      begin
+        system "stty -echo"  # TODO termios
+        print "Password (typing will be hidden): "
+        password = STDIN.gets || ""
+        password = password[0..-2]  # strip \n
+      rescue Interrupt
+        $stderr.print "\ncanceled."
+        exit 1
+      ensure
+        system "stty echo"   # TODO termios
+        print "\n"
+      end
+
+      if password.empty?
+        $stderr.puts "canceled."
+        exit 0
+      end
+
+      yield password
+    end
+  end
+
 end
 end

data/lib/td/command/export.rb

@@ -0,0 +1,84 @@
+
+module TreasureData
+module Command
+
+  def table_export(op)
+    from = nil
+    to = nil
+    s3_bucket = nil
+    aws_access_key_id = nil
+    aws_secret_access_key = nil
+    file_format = "json.gz"
+
+    op.on('-f', '--from TIME', 'export data which is newer than or same with the TIME') {|s|
+      from = export_parse_time(s)
+    }
+    op.on('-t', '--to TIME', 'export data which is older than the TIME') {|s|
+      to = export_parse_time(s)
+    }
+    op.on('-b', '--s3-bucket NAME', 'name of the destination S3 bucket (required)') {|s|
+      s3_bucket = s
+    }
+    op.on('-k', '--aws-key-id KEY_ID', 'AWS access key id to export data (required)') {|s|
+      aws_access_key_id = s
+    }
+    op.on('-s', '--aws-secret-key SECRET_KEY', 'AWS secret access key to export data (required)') {|s|
+      aws_secret_access_key = s
+    }
+
+    db_name, table_name = op.cmd_parse
+
+    unless s3_bucket
+      $stderr.puts "-b, --s3-bucket NAME option is required."
+      exit 1
+    end
+
+    unless aws_access_key_id
+      $stderr.puts "-k, --aws-key-id KEY_ID option is required."
+      exit 1
+    end
+
+    unless aws_secret_access_key
+      $stderr.puts "-s, --aws-secret-key SECRET_KEY option is required."
+      exit 1
+    end
+
+    client = get_client
+
+    get_table(client, db_name, table_name)
+
+    client = get_ssl_client
+
+    s3_opts = {}
+    s3_opts['from'] = from.to_s if from
+    s3_opts['to'] = to.to_s if to
+    s3_opts['file_format'] = file_format
+    s3_opts['bucket'] = s3_bucket
+    s3_opts['access_key_id'] = aws_access_key_id
+    s3_opts['secret_access_key'] = aws_secret_access_key
+
+    job = client.export(db_name, table_name, "s3", s3_opts)
+
+    $stderr.puts "Export job #{job.job_id} is queued."
+    $stderr.puts "Use '#{$prog} job:show #{job.job_id}' to show the status."
+  end
+
+  private
+  def export_parse_time(time)
+    if time.to_i.to_s == time.to_s
+      # UNIX time
+      return time.to_i
+    else
+      require 'time'
+      begin
+        return Time.parse(time).to_i
+      rescue
+        $stderr.puts "invalid time format: #{time}"
+        exit 1
+      end
+    end
+  end
+
+end
+end
+

data/lib/td/command/help.rb

@@ -5,14 +5,21 @@ module Command
   def help(op)
     cmd = op.cmd_parse
 
-    usage = List.cmd_usage(cmd)
-    unless usage
-      $stderr.puts "'#{cmd}' is not a td command. Run '#{$prog}' to show the list."
-      List.show_guess(cmd)
+    c = List.get_option(cmd)
+    if c == nil
+       $stderr.puts "'#{cmd}' is not a td command. Run '#{$prog}' to show the list."
+       List.show_guess(cmd)
+       exit 1
+
+    elsif c.name != cmd && c.group == cmd
+      # group command
+      puts List.cmd_usage(cmd)
       exit 1
-    end
 
-    puts usage
+    else
+      method = List.get_method(cmd)
+      method.call(['--help'])
+    end
   end
 
   def help_all(op)

data/lib/td/command/job.rb

@@ -38,7 +38,7 @@ module Command
       next if error && !job.error?
       start = job.start_at
       elapsed = cmd_format_elapsed(start, job.end_at)
-      rows << {:JobID => job.job_id, :Status => job.status, :Query => job.query.to_s, :Start => (start ? start.localtime : ''), :Elapsed => elapsed, :Result => job.rset_name}
+      rows << {:JobID => job.job_id, :Status => job.status, :Query => job.query.to_s, :Start => (start ? start.localtime : ''), :Elapsed => elapsed, :Result => job.result_url}
     }
 
     puts cmd_render_table(rows, :fields => [:JobID, :Status, :Start, :Elapsed, :Result, :Query])
@@ -76,7 +76,7 @@ module Command
     puts "URL          : #{job.url}"
     puts "Status       : #{job.status}"
     puts "Query        : #{job.query}"
-    puts "Result table : #{job.rset_name}"
+    puts "Result table : #{job.result_url}"
 
     if wait && !job.finished?
       wait_job(job)

data/lib/td/command/list.rb

@@ -143,9 +143,10 @@ module List
 
   def self.cmd_usage(name)
     if c = COMMAND[name]
-      c.create_optparse([]).cmd_usage
+      c.create_optparse([]).to_s
+    else
+      nil
     end
-    nil
   end
 
   def self.get_method(name)
@@ -211,24 +212,22 @@ module List
   end
 
   add_list 'db:list', %w[], 'Show list of tables in a database', 'db:list', 'dbs'
-  add_list 'db:show', %w[db], 'Describe a information of a database', 'db example_db'
+  add_list 'db:show', %w[db], 'Describe information of a database', 'db example_db'
   add_list 'db:create', %w[db], 'Create a database', 'db:create example_db'
   add_list 'db:delete', %w[db], 'Delete a database', 'db:delete example_db'
 
   add_list 'table:list', %w[db?], 'Show list of tables', 'table:list', 'table:list example_db', 'tables'
-  add_list 'table:show', %w[db table], 'Describe a information of a table', 'table example_db table1'
+  add_list 'table:show', %w[db table], 'Describe information of a table', 'table example_db table1'
   add_list 'table:create', %w[db table], 'Create a table', 'table:create example_db table1'
   add_list 'table:delete', %w[db table], 'Delete a table', 'table:delete example_db table1'
   add_list 'table:import', %w[db table files_], 'Parse and import files to a table', 'table:import example_db table1 --apache access.log', 'table:import example_db table1 --json -t time - < test.json'
-  add_list 'table:export', %w[db table], 'Dump logs in a table to the specified storage', 'table:export example_db table1 --s3-bucket mybucket'
+  add_list 'table:export', %w[db table], 'Dump logs in a table to the specified storage', 'table:export example_db table1 --s3-bucket mybucket -k KEY_ID -s SECRET_KEY'
   add_list 'table:tail', %w[db table], 'Get recently imported logs', 'table:tail example_db table1', 'table:tail example_db table1 -t "2011-01-02 03:04:05" -n 30'
 
-  add_list 'result:info', %w[], 'Show information of the MySQL server', 'result:info'
-  add_list 'result:list', %w[], 'Show list of result tables', 'result:list', 'results'
-  add_list 'result:create', %w[name], 'Create a result table', 'result:create rset1'
-  add_list 'result:delete', %w[name], 'Delete a result table', 'result:delete rset1'
-  add_list 'result:connect', %w[sql?], 'Connect to the server using mysql command', 'result:connect'
-  #add_list 'result:get', %w[name], 'Download dump of the result table'
+  add_list 'result:list', %w[], 'Show list of result URLs', 'result:list', 'results'
+  add_list 'result:show', %w[name], 'Describe information of a result URL', 'result mydb'
+  add_list 'result:create', %w[name URL], 'Create a result URL', 'result:create mydb mysql://my-server/mydb'
+  add_list 'result:delete', %w[name], 'Delete a result URL', 'result:delete mydb'
 
   add_list 'status', %w[], 'Show schedules, jobs, tables and results', 'status', 's'
 
@@ -284,7 +283,7 @@ module List
   add_alias 'table', 'table:show'
   add_alias 'tables', 'table:list'
 
-  add_alias 'result', 'help'  # dummy
+  add_alias 'result', 'result:show'
   add_alias 'results', 'result:list'
 
   add_alias 'schema', 'schema:show'

data/lib/td/command/query.rb

@@ -7,7 +7,9 @@ module Command
     wait = false
     output = nil
     format = 'tsv'
-    result = nil
+    result_url = nil
+    result_user = nil
+    result_ask_password = false
 
     op.on('-d', '--database DB_NAME', 'use the database (required)') {|s|
       db_name = s
@@ -15,9 +17,6 @@ module Command
     op.on('-w', '--wait', 'wait for finishing the job', TrueClass) {|b|
       wait = b
     }
-    op.on('-r', '--result RESULT_TABLE', 'write result to the result table (use result:create command)') {|s|
-      result = s
-    }
     op.on('-o', '--output PATH', 'write result to the file') {|s|
       output = s
     }
@@ -27,6 +26,15 @@ module Command
       end
       format = s
     }
+    op.on('-r', '--result RESULT_URL', 'write result to the URL (see also result:create subcommand)') {|s|
+      result_url = s
+    }
+    op.on('-u', '--user NAME', 'set user name for the result URL') {|s|
+      result_user = s
+    }
+    op.on('-p', '--password', 'ask password for the result URL') {|s|
+      result_ask_password = true
+    }
 
     sql = op.cmd_parse
 
@@ -35,12 +43,17 @@ module Command
       exit 1
     end
 
+    if result_url
+      require 'td/command/result'
+      result_url = build_result_url(result_url, result_user, result_ask_password)
+    end
+
     client = get_client
 
     # local existance check
     get_database(client, db_name)
 
-    job = client.query(db_name, sql, result)
+    job = client.query(db_name, sql, result_url)
 
     $stderr.puts "Job #{job.job_id} is queued."
     $stderr.puts "Use '#{$prog} job:show #{job.job_id}' to show the status."

data/lib/td/command/result.rb

@@ -2,19 +2,22 @@
 module TreasureData
 module Command
 
-  def result_info(op)
-    op.cmd_parse
+  def result_show(op)
+    name = op.cmd_parse
 
     client = get_client
 
-    info = client.result_set_info
+    rs = client.results
+    r = rs.find {|r| name == r.name }
+
+    unless r
+      $stderr.puts "Result URL '#{name}' does not exist."
+      $stderr.puts "Use '#{$prog} result:create #{name} <URL>' to create the URL."
+      exit 1
+    end
 
-    puts "Type       : #{info.type}"
-    puts "Host       : #{info.host}"
-    puts "Port       : #{info.port}"
-    puts "User       : #{info.user}"
-    puts "Password   : #{info.password}"
-    puts "Database   : #{info.database}"
+    puts "Name         : #{r.name}"
+    puts "URL          : #{r.url}"
   end
 
   def result_list(op)
@@ -22,39 +25,51 @@ module Command
 
     client = get_client
 
-    rsets = client.result_sets
+    rs = client.results
 
     rows = []
-    rsets.each {|rset|
-      rows << {:Name => rset.name}
+    rs.each {|r|
+      rows << {:Name => r.name, :URL => r.url}
     }
     rows = rows.sort_by {|map|
       map[:Name]
     }
 
-    puts cmd_render_table(rows, :fields => [:Name])
+    puts cmd_render_table(rows, :fields => [:Name, :URL])
 
-    if rsets.empty?
-      $stderr.puts "There are result tables."
-      $stderr.puts "Use '#{$prog} result:create <name>' to create a result table."
+    if rs.empty?
+      $stderr.puts "There are no result URLs."
+      $stderr.puts "Use '#{$prog} result:create <name> <url>' to create a result URL."
     end
   end
 
   def result_create(op)
-    name = op.cmd_parse
+    result_user = nil
+    result_ask_password = false
+
+    op.on('-u', '--user NAME', 'set user name for authentication') {|s|
+      result_user = s
+    }
+    op.on('-p', '--password', 'ask password for authentication') {|s|
+      result_ask_password = true
+    }
+
+    name, url = op.cmd_parse
 
     API.validate_database_name(name)
 
     client = get_client
 
+    url = build_result_url(url, result_user, result_ask_password)
+
     begin
-      client.create_result_set(name)
+      client.create_result(name, url)
     rescue AlreadyExistsError
-      $stderr.puts "Result table '#{name}' already exists."
+      $stderr.puts "Result URL '#{name}' already exists."
       exit 1
     end
 
-    $stderr.puts "Result table '#{name}' is created."
+    $stderr.puts "Result URL '#{name}' is created."
   end
 
   def result_delete(op)
@@ -63,49 +78,49 @@ module Command
     client = get_client
 
     begin
-      client.delete_result_set(name)
+      client.delete_result(name)
     rescue NotFoundError
-      $stderr.puts "Result table '#{name}' does not exist."
+      $stderr.puts "Result URL '#{name}' does not exist."
       exit 1
     end
 
-    $stderr.puts "Result table '#{name}' is deleted."
+    $stderr.puts "Result URL '#{name}' is deleted."
   end
 
-  def result_connect(op)
-    mysql = 'mysql'
-
-    op.on('-e', '--execute MYSQL', 'mysql command') {|s|
-      mysql = s
-    }
-
-    sql = op.cmd_parse
-
-    client = get_client
-
-    info = client.result_set_info
-
-    cmd = [mysql, '-h', info.host, '-P', info.port.to_s, '-u', info.user, "--password=#{info.password}", info.database]
-
-    cmd_start = Time.now
-
-    if sql
-      IO.popen(cmd, "w") {|io|
-        io.write sql
-        io.close
-      }
-    else
-      STDERR.puts "> #{cmd.join(' ')}"
-      system(*cmd)
+  private
+  def build_result_url(url, user, ask_password)
+    if ask_password
+      begin
+        system "stty -echo"  # TODO termios
+        print "Password (typing will be hidden): "
+        password = STDIN.gets || ""
+        password = password[0..-2]  # strip \n
+      rescue Interrupt
+        $stderr.print "\ncanceled."
+        exit 1
+      ensure
+        system "stty echo"   # TODO termios
+        print "\n"
+      end
     end
 
-    cmd_alive = Time.now - cmd_start
-    if $?.to_i != 0 && cmd_alive < 1.0
-      STDERR.puts "Command died within 1 second with exit code #{$?.to_i}."
-      STDERR.puts "Please confirm mysql command is installed."
+    ups = nil
+    if user && password
+      require 'cgi'
+      ups = "#{CGI.escape(user)}:#{CGI.escape(password)}@"
+    elsif user
+      require 'cgi'
+      ups = "#{CGI.escape(user)}@"
+    elsif password
+      require 'cgi'
+      ups = ":#{CGI.escape(password)}@"
+    end
+    if ups
+      url = url.sub(/\A([\w]+:(?:\/\/)?)/, "\\1#{ups}")
     end
-  end
 
+    url
+  end
 end
 end
 

data/lib/td/command/runner.rb

@@ -37,10 +37,10 @@ EOF
 Basic commands:
 
   db             # create/delete/list databases
-  table          # create/delete/list/import/tail tables
+  table          # create/delete/list/import/export/tail tables
   query          # issue a query
   job            # show/kill/list jobs
-  result         # create/delete/list/connect/get MySQL result set
+  result         # create/delete/list result URLs
 
 Additional commands:
 
@@ -83,6 +83,10 @@ EOF
     #	$debug = b
     #}
 
+    op.on('-h', '--help', "show help") {
+      usage nil
+    }
+
     begin
       op.order!(argv)
       usage nil if argv.empty?

data/lib/td/command/sched.rb

@@ -11,7 +11,7 @@ module Command
 
     rows = []
     scheds.each {|sched|
-      rows << {:Name => sched.name, :Cron => sched.cron, :Timezone => sched.timezone, :Delay => sched.delay, :Result => sched.rset_name, :Database => sched.database, :Query => sched.query, :"Next schedule" => sched.next_time ? sched.next_time.localtime : nil }
+      rows << {:Name => sched.name, :Cron => sched.cron, :Timezone => sched.timezone, :Delay => sched.delay, :Result => sched.result_url, :Database => sched.database, :Query => sched.query, :"Next schedule" => sched.next_time ? sched.next_time.localtime : nil }
     }
     rows = rows.sort_by {|map|
       map[:Name]
@@ -22,22 +22,30 @@ module Command
 
   def sched_create(op)
     db_name = nil
-    result = nil
     timezone = nil
     delay = 0
+    result_url = nil
+    result_user = nil
+    result_ask_password = false
 
     op.on('-d', '--database DB_NAME', 'use the database (required)') {|s|
       db_name = s
     }
-    op.on('-r', '--result RESULT_TABLE', 'write result to the result table (use result:create command)') {|s|
-      result = s
-    }
     op.on('-t', '--timezone TZ', 'name of the timezone (like Asia/Tokyo)') {|s|
       timezone = s
     }
     op.on('-D', '--delay SECONDS', 'delay time of the schedule', Integer) {|i|
       delay = i
     }
+    op.on('-r', '--result RESULT_URL', 'write result to the URL (see also result:create subcommand)') {|s|
+      result_url = s
+    }
+    op.on('-u', '--user NAME', 'set user name for the result URL') {|s|
+      result_user = s
+    }
+    op.on('-p', '--password', 'ask password for the result URL') {|s|
+      result_ask_password = true
+    }
 
     name, cron, sql = op.cmd_parse
 
@@ -46,13 +54,18 @@ module Command
       exit 1
     end
 
+    if result_url
+      require 'td/command/result'
+      result_url = build_result_url(result_url, result_user, result_ask_password)
+    end
+
     client = get_client
 
     # local existance check
     get_database(client, db_name)
 
     begin
-      first_time = client.create_schedule(name, :cron=>cron, :query=>sql, :database=>db_name, :result=>result, :timezone=>timezone, :delay=>delay)
+      first_time = client.create_schedule(name, :cron=>cron, :query=>sql, :database=>db_name, :result=>result_url, :timezone=>timezone, :delay=>delay)
     rescue AlreadyExistsError
       cmd_debug_error $!
       $stderr.puts "Schedule '#{name}' already exists."
@@ -172,14 +185,14 @@ module Command
       puts "Timezone     : #{s.timezone}"
       puts "Delay        : #{s.delay} sec"
       puts "Next         : #{s.next_time}"
-      puts "Result       : #{s.rset_name}"
+      puts "Result       : #{s.result_url}"
       puts "Database     : #{s.database}"
       puts "Query        : #{s.query}"
     end
 
     rows = []
     history.each {|j|
-      rows << {:Time => j.scheduled_at.localtime, :JobID => j.job_id, :Status => j.status, :Result=>j.rset_name}
+      rows << {:Time => j.scheduled_at.localtime, :JobID => j.job_id, :Status => j.status, :Result=>j.result_url}
     }
 
     puts cmd_render_table(rows, :fields => [:JobID, :Time, :Status, :Result])

data/lib/td/command/table.rb

@@ -239,6 +239,7 @@ module Command
   end
 
   require 'td/command/import'  # table:import
+  require 'td/command/export'  # table:export
   require 'td/command/job'  # wait_job
 end
 end

data/lib/td/version.rb

@@ -1,5 +1,5 @@
 module TreasureData
 
-VERSION = '0.10.22'
+VERSION = '0.10.23'
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: td
 version: !ruby/object:Gem::Version
-  version: 0.10.22
+  version: 0.10.23
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-04 00:00:00.000000000Z
+date: 2012-04-27 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
-  requirement: &70323068907380 !ruby/object:Gem::Requirement
+  requirement: &70287442084840 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 0.4.4
   type: :runtime
   prerelease: false
-  version_requirements: *70323068907380
+  version_requirements: *70287442084840
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &70323068906820 !ruby/object:Gem::Requirement
+  requirement: &70287442084360 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: 1.4.3
   type: :runtime
   prerelease: false
-  version_requirements: *70323068906820
+  version_requirements: *70287442084360
 - !ruby/object:Gem::Dependency
   name: hirb
-  requirement: &70323068906340 !ruby/object:Gem::Requirement
+  requirement: &70287442083880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,21 +43,21 @@ dependencies:
         version: 0.4.5
   type: :runtime
   prerelease: false
-  version_requirements: *70323068906340
+  version_requirements: *70287442083880
 - !ruby/object:Gem::Dependency
   name: td-client
-  requirement: &70323068905860 !ruby/object:Gem::Requirement
+  requirement: &70287442083400 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.8.14
+        version: 0.8.15
   type: :runtime
   prerelease: false
-  version_requirements: *70323068905860
+  version_requirements: *70287442083400
 - !ruby/object:Gem::Dependency
   name: td-logger
-  requirement: &70323068905360 !ruby/object:Gem::Requirement
+  requirement: &70287442082860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,7 +65,7 @@ dependencies:
         version: 0.3.8
   type: :runtime
   prerelease: false
-  version_requirements: *70323068905360
+  version_requirements: *70287442082860
 description: 
 email: 
 executables:
@@ -83,6 +83,7 @@ files:
 - lib/td/command/apikey.rb
 - lib/td/command/common.rb
 - lib/td/command/db.rb
+- lib/td/command/export.rb
 - lib/td/command/help.rb
 - lib/td/command/import.rb
 - lib/td/command/job.rb