td 0.7.0 → 0.7.1

data/ChangeLog

@@ -1,9 +1,19 @@
 
+== 2011-08-15 version 0.7.1
+
+* Validate name of a database/table on create-database/create-log-table
+subcommands
+* Uses /v3/job/result?format=msgpack API to get result of a job
+* -d, --database DB_NAME option is now required on query subcommand
+* API server can be changed using TD_API_SERVER=HOST:PORT environment variable
+
+
 == 2011-08-06 version 0.7.0
 
 * import subcommand accepts UNIX time integer value on --json and --msgpack format
 * Renamed command name (trd -> td)
 
+
 == 2011-07-18 version 0.6.3
 
 * show-jobs: shows elapsed time

data/lib/td/api.rb

@@ -99,7 +99,7 @@ class API
   end
 
   # => Job
-  def query(q, db_name=nil)
+  def query(db_name, q)
     job_id = @iface.hive_query(q, db_name)
     Job.new(self, job_id, :hive, q)  # TODO url
   end
@@ -108,27 +108,54 @@ class API
   def jobs(from=nil, to=nil)
     js = @iface.list_jobs(from, to)
     js.map {|job_id,type,status,query,start_at,end_at|
-      Job.new(self, job_id, type, query, status, nil, nil, nil, start_at, end_at)
+      Job.new(self, job_id, type, query, status, nil, nil, start_at, end_at)
     }
   end
 
   # => Job
   def job(job_id)
     job_id = job_id.to_s
-    type, query, status, result, url, debug, start_at, end_at = @iface.show_job(job_id)
-    Job.new(self, job_id, type, query, status, url, result, debug, start_at, end_at)
+    type, query, status, url, debug, start_at, end_at = @iface.show_job(job_id)
+    Job.new(self, job_id, type, query, status, url, debug, start_at, end_at)
   end
 
-  # => type:Symbol, result:String, url:String
+  # => type:Symbol, url:String
   def job_status(job_id)
-    type, query, status, result, url, debug, start_at, end_at = @iface.show_job(job_id)
-    return query, status, result, url, debug, start_at, end_at
+    type, query, status, url, debug, start_at, end_at = @iface.show_job(job_id)
+    return query, status, url, debug, start_at, end_at
+  end
+
+  # => result:[{column:String=>value:Object]
+  def job_result(job_id)
+    @iface.job_result(job_id)
+  end
+
+  # => nil
+  def job_result_each(job_id, &block)
+    @iface.job_result_each(job_id, &block)
   end
 
   # => time:Flaot
 	def import(db_name, table_name, format, stream, stream_size=stream.lstat.size)
     @iface.import(db_name, table_name, format, stream, stream_size)
 	end
+
+  def self.validate_database_name(name)
+    name = name.to_s
+    if name.empty?
+      raise "Empty name is not allowed"
+    end
+    if name.length < 3 || 32 < name.length
+      raise "Name must be 3 to 32 characters, got #{name.length} characters."
+    end
+    unless name =~ /^([a-z0-9_]+)$/
+      raise "Name must consist only of alphabets, numbers, '_'."
+    end
+  end
+
+  def self.validate_table_name(name)
+    validate_database_name(name)
+  end
 end
 
 end
@@ -216,17 +243,17 @@ class Table < APIObject
 end
 
 class Job < APIObject
-  def initialize(api, job_id, type, query, status=nil, url=nil, result=nil, debug=nil, start_at=nil, end_at=nil)
+  def initialize(api, job_id, type, query, status=nil, url=nil, debug=nil, start_at=nil, end_at=nil, result=nil)
     super(api)
     @job_id = job_id
     @type = type
     @url = url
     @query = query
     @status = status
-    @result = result
     @debug = debug
     @start_at = start_at
     @end_at = end_at
+    @result = result
   end
 
   attr_reader :job_id, :type
@@ -266,12 +293,20 @@ class Job < APIObject
   end
 
   def result
-    return nil unless finished?
-    update_status! unless @result
-    @result.split("\n").map {|line|
-      # TODO format of the result is TSV for now
-      line.split("\t")
-    }
+    unless @result
+      return nil unless finished?
+      @result = @api.job_result(@job_id)
+    end
+    @result
+  end
+
+  def result_each(&block)
+    if @result
+      @result.each(&block)
+    else
+      @api.job_result_each(&block)
+    end
+    nil
   end
 
   def finished?
@@ -287,11 +322,18 @@ class Job < APIObject
     !finished?
   end
 
+  def success?
+    @status == "success"
+  end
+
+  def error?
+    @status == "error"
+  end
+
   def update_status!
-    query, status, result, url, debug, start_at, end_at = @api.job_status(@job_id)
+    query, status, url, debug, start_at, end_at = @api.job_status(@job_id)
     @query = query
     @status = status
-    @result = result
     @url = url
     @debug = debug
     @start_at = start_at

data/lib/td/api_iface.rb

@@ -141,12 +141,46 @@ class APIInterface
     type = (js['type'] || '?').to_sym  # TODO
     query = js['query']
     status = js['status']
-    result = js['result']
     debug = js['debug']
     url = js['url']
     start_at = js['start_at']
     end_at = js['end_at']
-    return [type, query, status, result, url, debug, start_at, end_at]
+    return [type, query, status, url, debug, start_at, end_at]
+  end
+
+  def job_result(job_id)
+    require 'msgpack'
+    code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>'msgpack'})
+    if code != "200"
+      raise_error("Get job result failed", res)
+    end
+    result = []
+    MessagePack::Unpacker.new.feed_each(body) {|row|
+      result << row
+    }
+    return result
+  end
+
+  def job_result_each(job_id, &block)
+    # TODO chunked encoding
+    require 'msgpack'
+    code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>'msgpack'})
+    if code != "200"
+      raise_error("Get job result failed", res)
+    end
+    result = []
+    MessagePack::Unpacker.new.feed_each(body) {|row|
+      yield row
+    }
+    nil
+  end
+
+  def job_result_raw(job_id, format)
+    code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>format})
+    if code != "200"
+      raise_error("Get job result failed", res)
+    end
+    return body
   end
 
   # => jobId:String
@@ -211,8 +245,16 @@ class APIInterface
   end
 
   private
-  HOST = ENV['TD_API_SERVER'] || 'api.treasure-data.com'
-  PORT = 80
+  host = 'api.treasure-data.com'
+  port = 80
+  if e = ENV['TD_API_SERVER']
+    host, port_ = e.split(':',2)
+    port_ = port_.to_i
+    port = port_ if port_ != 0
+  end
+
+  HOST = host
+  PORT = port
   USE_SSL = false
   BASE_URL = ''
 

data/lib/td/command/database.rb

@@ -9,6 +9,8 @@ module Command
     conf = cmd_config
     api = cmd_api(conf)
 
+    API.validate_database_name(db_name)
+
     begin
       api.create_database(db_name)
     rescue AlreadyExistsError
@@ -76,7 +78,6 @@ module Command
   alias show_dbs show_databases
   alias create_db create_database
   alias drop_db drop_database
-
 end
 end
 

data/lib/td/command/query.rb

@@ -8,7 +8,7 @@ module Command
     op.banner << "\noptions:\n"
 
     db_name = nil
-    op.on('-d', '--database DB_NAME', 'use the database') {|s|
+    op.on('-d', '--database DB_NAME', 'use the database (required)') {|s|
       db_name = s
     }
 
@@ -22,11 +22,14 @@ module Command
     conf = cmd_config
     api = cmd_api(conf)
 
-    if db_name
-      find_database(api, db_name)
+    unless db_name
+      $stderr.puts "-d, --database DB_NAME option is required."
+      exit 1
     end
 
-    job = api.query(sql, db_name)
+    find_database(api, db_name)
+
+    job = api.query(db_name, sql)
 
     $stderr.puts "Job #{job.job_id} is started."
     $stderr.puts "Use '#{$prog} job #{job.job_id}' to show the status."
@@ -36,7 +39,7 @@ module Command
       wait_job(job)
       puts "Status     : #{job.status}"
       puts "Result     :"
-      puts cmd_render_table(job.result, :max_width=>10000)
+      puts render_result(job.result)
     end
   end
 
@@ -116,12 +119,12 @@ module Command
     if wait && !job.finished?
       wait_job(job)
       puts "Result     :"
-      puts cmd_render_table(job.result, :max_width=>10000)
+      puts render_result(job.result)
 
     else
       if job.finished?
         puts "Result     :"
-        puts cmd_render_table(job.result, :max_width=>10000)
+        puts render_result(job.result)
       end
 
       if verbose
@@ -162,6 +165,20 @@ module Command
       stderr_lines += stderr.size
     end
   end
+
+  def render_result(result)
+    require 'json'
+    rows = result.map {|row|
+      row.map {|v|
+        if v.is_a?(String)
+          v.to_s
+        else
+          v.to_json
+        end
+      }
+    }
+    puts cmd_render_table(rows, :max_width=>10000)
+  end
 end
 end
 

data/lib/td/command/table.rb

@@ -6,6 +6,8 @@ module Command
     conf = cmd_config
     api = cmd_api(conf)
 
+    API.validate_table_name(table_name)
+
     begin
       api.create_table(db_name, table_name, type)
     rescue NotFoundError

data/lib/td/version.rb

@@ -1,5 +1,5 @@
 module TD
 
-VERSION = '0.7.0'
+VERSION = '0.7.1'
 
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: td
 version: !ruby/object:Gem::Version 
-  hash: 3
+  hash: 1
   prerelease: false
   segments: 
   - 0
   - 7
-  - 0
-  version: 0.7.0
+  - 1
+  version: 0.7.1
 platform: ruby
 authors: 
 - Sadayuki Furuhashi
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-08-06 00:00:00 +09:00
+date: 2011-08-15 00:00:00 +09:00
 default_executable: td
 dependencies: 
 - !ruby/object:Gem::Dependency