td 0.7.1 → 0.7.2

data/ChangeLog

@@ -1,10 +1,17 @@
 
+== 2011-08-15 version 0.7.2
+
+* Supports TD_API_KEY and TD_CONFIG_PATH environment variable
+* query, show-job: supports --output and --format options
+* show-jobs: supports --page, --skip, --from and --around options
+
+
 == 2011-08-15 version 0.7.1
 
 * Validate name of a database/table on create-database/create-log-table
 subcommands
-* Uses /v3/job/result?format=msgpack API to get result of a job
 * -d, --database DB_NAME option is now required on query subcommand
+* Uses /v3/job/result?format=msgpack API to get result of a job
 * API server can be changed using TD_API_SERVER=HOST:PORT environment variable
 
 

data/lib/td/api.rb

@@ -130,15 +130,20 @@ class API
     @iface.job_result(job_id)
   end
 
+  # => result:String
+  def job_result_format(job_id, format)
+    @iface.job_result_format(job_id, format)
+  end
+
   # => nil
   def job_result_each(job_id, &block)
     @iface.job_result_each(job_id, &block)
   end
 
   # => time:Flaot
-	def import(db_name, table_name, format, stream, stream_size=stream.lstat.size)
+  def import(db_name, table_name, format, stream, stream_size=stream.lstat.size)
     @iface.import(db_name, table_name, format, stream, stream_size)
-	end
+  end
 
   def self.validate_database_name(name)
     name = name.to_s
@@ -300,11 +305,16 @@ class Job < APIObject
     @result
   end
 
+  def result_format(format)
+    return nil unless finished?
+    @api.job_result_format(@job_id, format)
+  end
+
   def result_each(&block)
     if @result
       @result.each(&block)
     else
-      @api.job_result_each(&block)
+      @api.job_result_each(@job_id, &block)
     end
     nil
   end
@@ -323,10 +333,12 @@ class Job < APIObject
   end
 
   def success?
+    update_status! unless @status
     @status == "success"
   end
 
   def error?
+    update_status! unless @status
     @status == "error"
   end
 

data/lib/td/api_iface.rb

@@ -161,6 +161,15 @@ class APIInterface
     return result
   end
 
+  def job_result_format(job_id, format)
+    # TODO chunked encoding
+    code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>format})
+    if code != "200"
+      raise_error("Get job result failed", res)
+    end
+    return body
+  end
+
   def job_result_each(job_id, &block)
     # TODO chunked encoding
     require 'msgpack'

data/lib/td/command/account.rb

@@ -17,7 +17,7 @@ module Command
     require 'td/config'
     conf = nil
     begin
-      conf = Config.read($TRD_CONFIG_PATH)
+      conf = Config.read
     rescue ConfigError
     end
     if conf && conf['account.user']
@@ -74,7 +74,7 @@ module Command
     conf ||= Config.new
     conf["account.user"] = user_name
     conf["account.apikey"] = api.apikey
-    conf.save($TRD_CONFIG_PATH)
+    conf.save
 
     $stderr.puts "Use '#{$prog} create-database <db_name>' to create a database."
   end

data/lib/td/command/common.rb

@@ -56,13 +56,8 @@ EOF
     op
   end
 
-  def cmd_config
-    require 'td/config'
-    Config.read($TRD_CONFIG_PATH)
-  end
-
-  def cmd_api(conf)
-    apikey = conf['account.apikey']
+  def cmd_api
+    apikey = Config.apikey
     unless apikey
       raise ConfigError, "Account is not configured."
     end

data/lib/td/command/database.rb

@@ -6,8 +6,7 @@ module Command
     op = cmd_opt 'create-database', :db_name
     db_name = op.cmd_parse
 
-    conf = cmd_config
-    api = cmd_api(conf)
+    api = cmd_api
 
     API.validate_database_name(db_name)
 
@@ -34,8 +33,7 @@ module Command
 
     db_name = op.cmd_parse
 
-    conf = cmd_config
-    api = cmd_api(conf)
+    api = cmd_api
 
     begin
       db = api.database(db_name)
@@ -58,8 +56,7 @@ module Command
     op = cmd_opt 'show-databases'
     op.cmd_parse
 
-    conf = cmd_config
-    api = cmd_api(conf)
+    api = cmd_api
 
     dbs = api.databases
 

data/lib/td/command/import.rb

@@ -56,8 +56,7 @@ module Command
 
     db_name, table_name, *paths = op.cmd_parse
 
-    conf = cmd_config
-    api = cmd_api(conf)
+    api = cmd_api
 
     case format
     when 'json', 'msgpack'

data/lib/td/command/query.rb

@@ -8,19 +8,29 @@ module Command
     op.banner << "\noptions:\n"
 
     db_name = nil
+    wait = false
+    output = nil
+    format = 'tsv'
+
     op.on('-d', '--database DB_NAME', 'use the database (required)') {|s|
       db_name = s
     }
-
-    wait = false
     op.on('-w', '--wait', 'wait for finishing the job', TrueClass) {|b|
       wait = b
     }
+    op.on('-o', '--output PATH', 'write result to the file') {|s|
+      output = s
+    }
+    op.on('-f', '--format FORMAT', 'format of the result to write to the file (tsv, csv, json or msgpack)') {|s|
+      unless ['tsv', 'csv', 'json', 'msgpack'].include?(s)
+        raise "Unknown format #{s.dump}. Supported format: tsv, csv, json, msgpack"
+      end
+      format = s
+    }
 
     sql = op.cmd_parse
 
-    conf = cmd_config
-    api = cmd_api(conf)
+    api = cmd_api
 
     unless db_name
       $stderr.puts "-d, --database DB_NAME option is required."
@@ -38,22 +48,56 @@ module Command
     if wait && !job.finished?
       wait_job(job)
       puts "Status     : #{job.status}"
-      puts "Result     :"
-      puts render_result(job.result)
+      if job.success?
+        puts "Result     :"
+        show_result(job, output, format)
+      end
     end
   end
 
   def show_jobs
-    op = cmd_opt 'show-jobs', :max?, :from?
-    max, from = op.cmd_parse
+    op = cmd_opt 'show-jobs', :max?
+
+    op.banner << "\noptions:\n"
+
+    page = 0
+    skip = 0
+    from = nil
+    around = nil
+
+    op.on('-p', '--page PAGE', 'skip N pages', Integer) {|i|
+      page = i
+    }
+    op.on('-s', '--skip N', 'skip N jobs', Integer) {|i|
+      skip = i
+    }
+    op.on('-f', '--from JOB_ID', 'show jobs from the id', Integer) {|i|
+      from = i
+    }
+    op.on('-a', '--around JOB_ID', 'show jobs around the id', Integer) {|i|
+      around = i
+    }
+
+    max = op.cmd_parse
 
     max = (max || 20).to_i
-    from = (from || 0).to_i
 
-    conf = cmd_config
-    api = cmd_api(conf)
+    api = cmd_api
 
-    jobs = api.jobs(from, from+max-1)
+    if from || around
+      jobs = api.jobs(0, 1)
+      if last = jobs[0]
+        if from
+          skip += last.job_id.to_i - from - (max-1)
+        else
+          skip += last.job_id.to_i - around - (max-1) + (max-1)/2
+        end
+      end
+    end
+    if page
+      skip += max * page
+    end
+    jobs = api.jobs(skip, skip+max-1)
 
     rows = []
     jobs.each {|job|
@@ -95,19 +139,29 @@ module Command
     op.banner << "\noptions:\n"
 
     verbose = nil
+    wait = false
+    output = nil
+    format = 'tsv'
+
     op.on('-v', '--verbose', 'show logs', TrueClass) {|b|
       verbose = b
     }
-
-    wait = false
     op.on('-w', '--wait', 'wait for finishing the job', TrueClass) {|b|
       wait = b
     }
+    op.on('-o', '--output PATH', 'write result to the file') {|s|
+      output = s
+    }
+    op.on('-f', '--format FORMAT', 'format of the result to write to the file (tsv, csv, json or msgpack)') {|s|
+      unless ['tsv', 'csv', 'json', 'msgpack'].include?(s)
+        raise "Unknown format #{s.dump}. Supported format: tsv, csv, json, msgpack"
+      end
+      format = s
+    }
 
     job_id = op.cmd_parse
 
-    conf = cmd_config
-    api = cmd_api(conf)
+    api = cmd_api
 
     job = api.job(job_id)
 
@@ -118,13 +172,15 @@ module Command
 
     if wait && !job.finished?
       wait_job(job)
-      puts "Result     :"
-      puts render_result(job.result)
+      if job.success?
+        puts "Result     :"
+        show_result(job, output, format)
+      end
 
     else
-      if job.finished?
+      if job.success?
         puts "Result     :"
-        puts render_result(job.result)
+        show_result(job, output, format)
       end
 
       if verbose
@@ -166,10 +222,75 @@ module Command
     end
   end
 
-  def render_result(result)
+  def show_result(job, output, format)
+    if output
+      write_result(job, output, format)
+      puts "written to #{output} in #{format} format"
+    else
+      render_result(job)
+    end
+  end
+
+  def write_result(job, output, format)
+    case format
+    when 'json'
+      require 'json'
+      first = true
+      File.open(output, "w") {|f|
+        f.write "["
+        job.result_each {|row|
+          if first
+            first = false
+          else
+            f.write ","
+          end
+          f.write row.to_json
+        }
+        f.write "]"
+      }
+
+    when 'msgpack'
+      File.open(output, "w") {|f|
+        f.write job.result_format('msgpack')
+      }
+
+    when 'csv'
+      require 'json'
+      require 'csv'
+      CSV.open(output, "w") {|writer|
+        job.result_each {|row|
+          writer << row.map {|col| col.is_a?(String) ? col.to_s : col.to_json }
+        }
+      }
+
+    when 'tsv'
+      require 'json'
+      File.open(output, "w") {|f|
+        job.result_each {|row|
+          first = true
+          row.each {|col|
+            if first
+              first = false
+            else
+              f.write "\t"
+            end
+            f.write col.is_a?(String) ? col.to_s : col.to_json
+          }
+          f.write "\n"
+        }
+      }
+
+    else
+      raise "Unknown format #{format.inspect}"
+    end
+  end
+
+  def render_result(job)
     require 'json'
-    rows = result.map {|row|
-      row.map {|v|
+    rows = []
+    job.result_each {|row|
+      # TODO limit number of rows to show
+      rows << row.map {|v|
         if v.is_a?(String)
           v.to_s
         else

data/lib/td/command/table.rb

@@ -3,8 +3,7 @@ module TD
 module Command
 
   def create_table_type(type, db_name, table_name)
-    conf = cmd_config
-    api = cmd_api(conf)
+    api = cmd_api
 
     API.validate_table_name(table_name)
 
@@ -43,8 +42,7 @@ module Command
     op = cmd_opt 'drop-table', :db_name, :table_name
     db_name, table_name = op.cmd_parse
 
-    conf = cmd_config
-    api = cmd_api(conf)
+    api = cmd_api
 
     begin
       api.delete_table(db_name, table_name)
@@ -62,8 +60,7 @@ module Command
     op = cmd_opt 'show-tables', :db_name?
     db_name = op.cmd_parse
 
-    conf = cmd_config
-    api = cmd_api(conf)
+    api = cmd_api
 
     if db_name
       db = find_database(api, db_name)

data/lib/td/command/td.rb

@@ -13,33 +13,38 @@ EOF
 op.summary_indent = "  "
 
 (class<<self;self;end).module_eval do
-	define_method(:usage) do |errmsg|
-		require 'td/command/list'
-		puts op.to_s
-		puts ""
-		puts "commands:"
-		puts TD::Command::List.help(op.summary_indent)
-		puts ""
-		puts "Type 'td help COMMAND' for more information on a specific command."
-		if errmsg
-			puts "error: #{errmsg}"
-			exit 1
-		else
-			exit 0
-		end
-	end
+  define_method(:usage) do |errmsg|
+    require 'td/command/list'
+    puts op.to_s
+    puts ""
+    puts "commands:"
+    puts TD::Command::List.help(op.summary_indent)
+    puts ""
+    puts "Type 'td help COMMAND' for more information on a specific command."
+    if errmsg
+      puts "error: #{errmsg}"
+      exit 1
+    else
+      exit 0
+    end
+  end
 end
 
-config_path = File.join(ENV['HOME'], '.td', 'td.conf')
+config_path = nil
+apikey = nil
 $verbose = false
 #$debug = false
 
 op.on('-c', '--config PATH', "path to config file (~/.td/td.conf)") {|s|
-	config_path = s
+  config_path = s
+}
+
+op.on('-k', '--apikey KEY', "use this API key instead of reading the config file") {|s|
+  apikey = s
 }
 
 op.on('-v', '--verbose', "verbose mode", TrueClass) {|b|
-	$verbose = b
+  $verbose = b
 }
 
 #op.on('-d', '--debug', "debug mode", TrueClass) {|b|
@@ -47,30 +52,37 @@ op.on('-v', '--verbose', "verbose mode", TrueClass) {|b|
 #}
 
 begin
-	op.order!(ARGV)
-	usage nil if ARGV.empty?
-	cmd = ARGV.shift
-	$TRD_CONFIG_PATH = config_path
+  op.order!(ARGV)
+  usage nil if ARGV.empty?
+  cmd = ARGV.shift
+
+  require 'td/config'
+  if config_path
+    TD::Config.path = config_path
+  end
+  if apikey
+    TD::Config.apikey = apikey
+  end
 rescue
-	usage $!.to_s
+  usage $!.to_s
 end
 
 require 'td/command/list'
 
 method = TD::Command::List.get_method(cmd)
 unless method
-	$stderr.puts "'#{cmd}' is not a td command. Run '#{$prog}' to show the list."
+  $stderr.puts "'#{cmd}' is not a td command. Run '#{$prog}' to show the list."
   TD::Command::List.show_guess(cmd)
-	exit 1
+  exit 1
 end
 
 require 'td/error'
 
 begin
-	method.call
+  method.call
 rescue TD::ConfigError
-	$stderr.puts "TreasureData account is not configured yet."
-	$stderr.puts "Run '#{$prog} account' first."
+  $stderr.puts "TreasureData account is not configured yet."
+  $stderr.puts "Run '#{$prog} account' first."
 rescue
   $stderr.puts "error #{$!.class}: backtrace:"
   $!.backtrace.each {|b|

data/lib/td/config.rb

@@ -4,75 +4,96 @@ module TD
 
 
 class Config
-	def initialize
-		@path = nil
-		@conf = {}   # section.key = val
-	end
-
-	def self.read(path, create=false)
-		new.read(path)
-	end
-
-	def [](cate_key)
-		@conf[cate_key]
-	end
-
-	def []=(cate_key, val)
-		@conf[cate_key] = val
-	end
-
-	def save(path=@path)
-		@path = path
-		write
-	end
-
-	def read(path=@path)
-		@path = path
-		begin
-			data = File.read(@path)
-		rescue
-			e = ConfigNotFoundError.new($!.to_s)
+  def initialize
+    @path = nil
+    @conf = {}   # section.key = val
+  end
+
+  def self.read(path=Config.path, create=false)
+    new.read(path)
+  end
+
+  def [](cate_key)
+    @conf[cate_key]
+  end
+
+  def []=(cate_key, val)
+    @conf[cate_key] = val
+  end
+
+  def read(path=@path)
+    @path = path
+    begin
+      data = File.read(@path)
+    rescue
+      e = ConfigNotFoundError.new($!.to_s)
       e.set_backtrace($!.backtrace)
       raise e
-		end
-
-		section = ""
-
-		data.each_line {|line|
-			line.strip!
-			case line
-			when /^#/
-				next
-			when /\[(.+)\]/
-				section = $~[1]
-			when /^(\w+)\s*=\s*(.+?)\s*$/
-				key = $~[1]
-				val = $~[2]
-				@conf["#{section}.#{key}"] = val
-			else
-				raise ConfigParseError, "invalid config line '#{line}' at #{@path}"
-			end
-		}
-
-		self
-	end
-
-	def write
-		require 'fileutils'
-		FileUtils.mkdir_p File.dirname(@path)
-		File.open(@path, "w") {|f|
-			@conf.keys.map {|cate_key|
-				cate_key.split('.',2)
-			}.zip(@conf.values).group_by {|(section,key),val|
-				section
-			}.each {|section,cate_key_vals|
-				f.puts "[#{section}]"
-				cate_key_vals.each {|(section,key),val|
-					f.puts "  #{key} = #{val}"
-				}
-			}
-		}
-	end
+    end
+
+    section = ""
+
+    data.each_line {|line|
+      line.strip!
+      case line
+      when /^#/
+        next
+      when /\[(.+)\]/
+        section = $~[1]
+      when /^(\w+)\s*=\s*(.+?)\s*$/
+        key = $~[1]
+        val = $~[2]
+        @conf["#{section}.#{key}"] = val
+      else
+        raise ConfigParseError, "invalid config line '#{line}' at #{@path}"
+      end
+    }
+
+    self
+  end
+
+  def save(path=@path||Config.path)
+    @path = path
+    write
+  end
+
+  private
+  def write
+    require 'fileutils'
+    FileUtils.mkdir_p File.dirname(@path)
+    File.open(@path, "w") {|f|
+      @conf.keys.map {|cate_key|
+        cate_key.split('.',2)
+      }.zip(@conf.values).group_by {|(section,key),val|
+        section
+      }.each {|section,cate_key_vals|
+        f.puts "[#{section}]"
+        cate_key_vals.each {|(section,key),val|
+          f.puts "  #{key} = #{val}"
+        }
+      }
+    }
+  end
+
+  @@path = ENV['TD_CONFIG_PATH'] || File.join(ENV['HOME'], '.td', 'td.conf')
+  @@apikey = ENV['TD_API_KEY']
+  @@apikey = nil if @@apikey == ""
+
+  def self.path
+    @@path
+  end
+
+  def self.path=(path)
+    @@path = path
+  end
+
+  def self.apikey
+    @@apikey || Config.read['account.apikey']
+  end
+
+  def self.apikey=(apikey)
+    @@apikey = apikey
+  end
 end
 
 

data/lib/td/version.rb

@@ -1,5 +1,5 @@
 module TD
 
-VERSION = '0.7.1'
+VERSION = '0.7.2'
 
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: td
 version: !ruby/object:Gem::Version 
-  hash: 1
+  hash: 7
   prerelease: false
   segments: 
   - 0
   - 7
-  - 1
-  version: 0.7.1
+  - 2
+  version: 0.7.2
 platform: ruby
 authors: 
 - Sadayuki Furuhashi