td 0.7.5 → 0.8.0

data/ChangeLog

@@ -1,4 +1,9 @@
 
+== 2011-08-21 version 0.8.0
+
+* Splits API libraries to td-client gem
+
+
 == 2011-08-18 version 0.7.5
 
 * set-schema: adds column instead of replacing all columns

data/lib/td/command/common.rb

@@ -1,7 +1,7 @@
 
 module TreasureData
 
-autoload :API, 'td/api'
+autoload :API, 'td/client/api'
 autoload :Client, 'td/client'
 autoload :Database, 'td/client'
 autoload :Table, 'td/client'

data/lib/td/version.rb

@@ -1,5 +1,5 @@
 module TreasureData
 
-VERSION = '0.7.5'
+VERSION = '0.8.0'
 
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: td
 version: !ruby/object:Gem::Version 
-  hash: 9
+  hash: 63
   prerelease: false
   segments: 
   - 0
-  - 7
-  - 5
-  version: 0.7.5
+  - 8
+  - 0
+  version: 0.8.0
 platform: ruby
 authors: 
 - Sadayuki Furuhashi
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-08-18 00:00:00 +09:00
+date: 2011-08-21 00:00:00 +09:00
 default_executable: td
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -66,6 +66,22 @@ dependencies:
         version: 0.4.5
   type: :runtime
   version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: td-client
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 63
+        segments: 
+        - 0
+        - 8
+        - 0
+        version: 0.8.0
+  type: :runtime
+  version_requirements: *id004
 description: 
 email: 
 executables: 
@@ -76,8 +92,6 @@ extra_rdoc_files:
 - ChangeLog
 - README.rdoc
 files: 
-- lib/td/api.rb
-- lib/td/client.rb
 - lib/td/command/account.rb
 - lib/td/command/common.rb
 - lib/td/command/database.rb

data/lib/td/api.rb

@@ -1,455 +0,0 @@
-
-module TreasureData
-
-
-class APIError < StandardError
-end
-
-class AuthError < APIError
-end
-
-class AlreadyExistsError < APIError
-end
-
-class NotFoundError < APIError
-end
-
-
-class API
-  def initialize(apikey)
-    require 'json'
-    @apikey = apikey
-  end
-
-  # TODO error check & raise appropriate errors
-
-  attr_reader :apikey
-
-  def self.validate_database_name(name)
-    name = name.to_s
-    if name.empty?
-      raise "Empty name is not allowed"
-    end
-    if name.length < 3 || 32 < name.length
-      raise "Name must be 3 to 32 characters, got #{name.length} characters."
-    end
-    unless name =~ /^([a-z0-9_]+)$/
-      raise "Name must consist only of alphabets, numbers, '_'."
-    end
-    name
-  end
-
-  def self.validate_table_name(name)
-    validate_database_name(name)
-  end
-
-  def self.validate_column_name(name)
-    name = name.to_s
-    if name.empty?
-      raise "Empty column name is not allowed"
-    end
-    if 32 < name.length
-      raise "Column name must be to 32 characters, got #{name.length} characters."
-    end
-    unless name =~ /^([a-z0-9_]+)$/
-      raise "Column name must consist only of alphabets, numbers, '_'."
-    end
-  end
-
-  def self.normalize_type_name(name)
-    case name
-    when /int/i, /integer/i
-      "int"
-    when /long/i, /bigint/i
-      "long"
-    when /string/i
-      "string"
-    when /float/i
-      "float"
-    when /double/i
-      "double"
-    else
-      raise "Type name must eather of int, long, string float or double"
-    end
-  end
-
-  ####
-  ## Database API
-  ##
-
-  # => [name:String]
-  def list_databases
-    code, body, res = get("/v3/database/list")
-    if code != "200"
-      raise_error("List databases failed", res)
-    end
-    # TODO format check
-    js = JSON.load(body)
-    names = js["databases"].map {|dbinfo| dbinfo['name'] }
-    return names
-  end
-
-  # => true
-  def delete_database(db)
-    code, body, res = post("/v3/database/delete/#{e db}")
-    if code != "200"
-      raise_error("Delete database failed", res)
-    end
-    return true
-  end
-
-  # => true
-  def create_database(db)
-    code, body, res = post("/v3/database/create/#{e db}")
-    if code != "200"
-      raise_error("Create database failed", res)
-    end
-    return true
-  end
-
-
-  ####
-  ## Table API
-  ##
-
-  # => {name:String => [type:Symbol, count:Integer]}
-  def list_tables(db)
-    code, body, res = get("/v3/table/list/#{e db}")
-    if code != "200"
-      raise_error("List tables failed", res)
-    end
-    # TODO format check
-    js = JSON.load(body)
-    result = {}
-    js["tables"].map {|m|
-      name = m['name']
-      type = (m['type'] || '?').to_sym
-      count = (m['count'] || 0).to_i  # TODO?
-      schema = JSON.parse(m['schema'] || '[]')
-      result[name] = [type, schema, count]
-    }
-    return result
-  end
-
-  def create_log_or_item_table(db, table, type)
-    code, body, res = post("/v3/table/create/#{e db}/#{e table}/#{type}")
-    if code != "200"
-      raise_error("Create #{type} table failed", res)
-    end
-    return true
-  end
-  private :create_log_or_item_table
-
-  # => true
-  def create_log_table(db, table)
-    create_table(db, table, :log)
-  end
-
-  # => true
-  def create_item_table(db, table)
-    create_table(db, table, :item)
-  end
-
-  def create_table(db, table, type)
-    schema = schema.to_s
-    code, body, res = post("/v3/table/create/#{e db}/#{e table}/#{type}")
-    if code != "200"
-      raise_error("Create #{type} table failed", res)
-    end
-    return true
-  end
-  private :create_table
-
-  # => true
-  def update_schema(db, table, schema_json)
-    code, body, res = post("/v3/table/update-schema/#{e db}/#{e table}", {'schema'=>schema_json})
-    if code != "200"
-      raise_error("Create schema table failed", res)
-    end
-    return true
-  end
-
-  # => type:Symbol
-  def delete_table(db, table)
-    code, body, res = post("/v3/table/delete/#{e db}/#{e table}")
-    if code != "200"
-      raise_error("Drop table failed", res)
-    end
-    # TODO format check
-    js = JSON.load(body)
-    type = (js['type'] || '?').to_sym
-    return type
-  end
-
-
-  ####
-  ## Job API
-  ##
-
-  # => [(jobId:String, type:Symbol, status:String, start_at:String, end_at:String)]
-  def list_jobs(from=0, to=nil)
-    params = {}
-    params['from'] = from.to_s if from
-    params['to'] = to.to_s if to
-    code, body, res = get("/v3/job/list", params)
-    if code != "200"
-      raise_error("List jobs failed", res)
-    end
-    # TODO format check
-    js = JSON.load(body)
-    result = []
-    js['jobs'].each {|m|
-      job_id = m['job_id']
-      type = (m['type'] || '?').to_sym
-      status = m['status']
-      query = m['query']
-      start_at = m['start_at']
-      end_at = m['end_at']
-      result << [job_id, type, status, query, start_at, end_at]
-    }
-    return result
-  end
-
-  # => (type:Symbol, status:String, result:String, url:String)
-  def show_job(job_id)
-    code, body, res = get("/v3/job/show/#{e job_id}")
-    if code != "200"
-      raise_error("Show job failed", res)
-    end
-    # TODO format check
-    js = JSON.load(body)
-    # TODO debug
-    type = (js['type'] || '?').to_sym  # TODO
-    query = js['query']
-    status = js['status']
-    debug = js['debug']
-    url = js['url']
-    start_at = js['start_at']
-    end_at = js['end_at']
-    return [type, query, status, url, debug, start_at, end_at]
-  end
-
-  def job_result(job_id)
-    require 'msgpack'
-    code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>'msgpack'})
-    if code != "200"
-      raise_error("Get job result failed", res)
-    end
-    result = []
-    MessagePack::Unpacker.new.feed_each(body) {|row|
-      result << row
-    }
-    return result
-  end
-
-  def job_result_format(job_id, format)
-    # TODO chunked encoding
-    code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>format})
-    if code != "200"
-      raise_error("Get job result failed", res)
-    end
-    return body
-  end
-
-  def job_result_each(job_id, &block)
-    # TODO chunked encoding
-    require 'msgpack'
-    code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>'msgpack'})
-    if code != "200"
-      raise_error("Get job result failed", res)
-    end
-    result = []
-    MessagePack::Unpacker.new.feed_each(body) {|row|
-      yield row
-    }
-    nil
-  end
-
-  def job_result_raw(job_id, format)
-    code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>format})
-    if code != "200"
-      raise_error("Get job result failed", res)
-    end
-    return body
-  end
-
-  # => jobId:String
-  def hive_query(q, db=nil)
-    code, body, res = post("/v3/job/issue/hive/#{e db}", {'query'=>q})
-    if code != "200"
-      raise_error("Query failed", res)
-    end
-    # TODO format check
-    js = JSON.load(body)
-    return js['job_id'].to_s
-  end
-
-
-  ####
-  ## Import API
-  ##
-
-  # => time:Float
-  def import(db, table, format, stream, stream_size=stream.lstat.size)
-    code, body, res = put("/v3/table/import/#{e db}/#{e table}/#{format}", stream, stream_size)
-    if code[0] != ?2
-      raise_error("Import failed", res)
-    end
-    # TODO format check
-    js = JSON.load(body)
-    time = js['time'].to_f
-    return time
-  end
-
-
-  ####
-  ## User API
-  ##
-
-  # apikey:String
-  def authenticate(user, password)
-    code, body, res = post("/v3/user/authenticate", {'user'=>user, 'password'=>password})
-    if code != "200"
-      raise_error("Authentication failed", res)
-    end
-    # TODO format check
-    js = JSON.load(body)
-    apikey = js['apikey']
-    return apikey
-  end
-
-  ####
-  ## Server Status API
-  ##
-
-  # => status:String
-  def server_status
-    code, body, res = get('/v3/system/server_status')
-    if code != "200"
-      return "Server is down (#{code})"
-    end
-    # TODO format check
-    js = JSON.load(body)
-    status = js['status']
-    return status
-  end
-
-  private
-  host = 'api.treasure-data.com'
-  port = 80
-  if e = ENV['TD_API_SERVER']
-    host, port_ = e.split(':',2)
-    port_ = port_.to_i
-    port = port_ if port_ != 0
-  end
-
-  HOST = host
-  PORT = port
-  USE_SSL = false
-  BASE_URL = ''
-
-  def get(url, params=nil)
-    http, header = new_http
-
-    path = BASE_URL + url
-    if params && !params.empty?
-      path << "?"+params.map {|k,v|
-        "#{k}=#{e v}"
-      }.join('&')
-    end
-
-    request = Net::HTTP::Get.new(path, header)
-
-    response = http.request(request)
-    return [response.code, response.body, response]
-  end
-
-  def post(url, params=nil)
-    http, header = new_http
-
-    path = BASE_URL + url
-
-    request = Net::HTTP::Post.new(path, header)
-    request.set_form_data(params) if params
-
-    response = http.request(request)
-    return [response.code, response.body, response]
-  end
-
-  def put(url, stream, stream_size)
-    http, header = new_http
-
-    path = BASE_URL + url
-
-    header['Content-Type'] = 'application/octet-stream'
-    header['Content-Length'] = stream_size.to_s
-
-    request = Net::HTTP::Put.new(url, header)
-    if request.respond_to?(:body_stream=)
-      request.body_stream = stream
-    else  # Ruby 1.8
-      request.body = stream.read
-    end
-
-    response = http.request(request)
-    return [response.code, response.body, response]
-  end
-
-  def new_http
-    require 'net/http'
-    require 'time'
-
-    http = Net::HTTP.new(HOST, PORT)
-    if USE_SSL
-      http.use_ssl = true
-      http.verify_mode = OpenSSL::SSL::VERIFY_PEER
-      store = OpenSSL::X509::Store.new
-      http.cert_store = store
-    end
-
-    #http.read_timeout = options[:read_timeout]
-
-    header = {}
-    if @apikey
-      header['Authorization'] = "TD1 #{apikey}"
-    end
-    header['Date'] = Time.now.rfc2822
-
-    return http, header
-  end
-
-  def raise_error(msg, res)
-    begin
-      js = JSON.load(res.body)
-      msg = js['message']
-      error_code = js['error_code']
-
-      if res.code == "404"
-        raise NotFoundError, "#{error_code}: #{msg}"
-      elsif res.code == "409"
-        raise AlreadyExistsError, "#{error_code}: #{msg}"
-      else
-        raise APIError, "#{error_code}: #{msg}"
-      end
-
-    rescue
-      if res.code == "404"
-        raise NotFoundError, "#{msg}: #{res.body}"
-      elsif res.code == "409"
-        raise AlreadyExistsError, "#{msg}: #{res.body}"
-      else
-        raise APIError, "#{msg}: #{res.body}"
-      end
-    end
-    # TODO error
-  end
-
-  def e(s)
-    require 'cgi'
-    CGI.escape(s.to_s)
-  end
-end
-
-
-end
-

data/lib/td/client.rb

@@ -1,380 +0,0 @@
-require 'time'
-require 'td/api'
-
-module TreasureData
-
-class Client
-  def self.authenticate(user, password)
-    api = API.new(nil)
-    apikey = api.authenticate(user, password)
-    new(apikey)
-  end
-
-  def self.server_status
-    api = API.new(nil)
-    api.server_status
-  end
-
-  def initialize(apikey)
-    @api = API.new(apikey)
-  end
-
-  attr_reader :api
-
-  def apikey
-    @api.apikey
-  end
-
-  def server_status
-    @api.server_status
-  end
-
-  # => true
-  def create_database(db_name)
-    @api.create_database(db_name)
-  end
-
-  # => true
-  def delete_database(db_name)
-    @api.delete_database(db_name)
-  end
-
-  # => [Database]
-  def databases
-    names = @api.list_databases
-    names.map {|db_name|
-      Database.new(self, db_name)
-    }
-  end
-
-  # => Database
-  def database(db_name)
-    names = @api.list_databases
-    names.each {|n|
-      if n == db_name
-        return Database.new(self, db_name)
-      end
-    }
-    raise NotFoundError, "Database '#{db_name}' does not exist"
-  end
-
-  # => true
-  def create_log_table(db_name, table_name)
-    @api.create_log_table(db_name, table_name)
-  end
-
-  # => true
-  def create_item_table(db_name, table_name)
-    @api.create_item_table(db_name, table_name)
-  end
-
-  # => true
-  def update_schema(db_name, table_name, schema)
-    @api.update_schema(db_name, table_name, schema.to_json)
-  end
-
-  # => type:Symbol
-  def delete_table(db_name, table_name)
-    @api.delete_table(db_name, table_name)
-  end
-
-  # => [Table]
-  def tables(db_name)
-    m = @api.list_tables(db_name)
-    m.map {|table_name,(type,schema,count)|
-      schema = Schema.new.from_json(schema)
-      Table.new(self, db_name, table_name, type, schema, count)
-    }
-  end
-
-  # => Table
-  def table(db_name, table_name)
-    tables(db_name).each {|t|
-      if t.name == table_name
-        return t
-      end
-    }
-    raise NotFoundError, "Table '#{db_name}.#{table_name}' does not exist"
-  end
-
-  # => Job
-  def query(db_name, q)
-    job_id = @api.hive_query(q, db_name)
-    Job.new(self, job_id, :hive, q)  # TODO url
-  end
-
-  # => [Job=]
-  def jobs(from=nil, to=nil)
-    js = @api.list_jobs(from, to)
-    js.map {|job_id,type,status,query,start_at,end_at|
-      Job.new(self, job_id, type, query, status, nil, nil, start_at, end_at)
-    }
-  end
-
-  # => Job
-  def job(job_id)
-    job_id = job_id.to_s
-    type, query, status, url, debug, start_at, end_at = @api.show_job(job_id)
-    Job.new(self, job_id, type, query, status, url, debug, start_at, end_at)
-  end
-
-  # => type:Symbol, url:String
-  def job_status(job_id)
-    type, query, status, url, debug, start_at, end_at = @api.show_job(job_id)
-    return query, status, url, debug, start_at, end_at
-  end
-
-  # => result:[{column:String=>value:Object]
-  def job_result(job_id)
-    @api.job_result(job_id)
-  end
-
-  # => result:String
-  def job_result_format(job_id, format)
-    @api.job_result_format(job_id, format)
-  end
-
-  # => nil
-  def job_result_each(job_id, &block)
-    @api.job_result_each(job_id, &block)
-  end
-
-  # => time:Flaot
-  def import(db_name, table_name, format, stream, stream_size=stream.lstat.size)
-    @api.import(db_name, table_name, format, stream, stream_size)
-  end
-end
-
-
-class Model
-  def initialize(client)
-    @client = client
-  end
-end
-
-class Database < Model
-  def initialize(client, db_name, tables=nil)
-    super(client)
-    @db_name = db_name
-    @tables = tables
-  end
-
-  def name
-    @db_name
-  end
-
-  def tables
-    update_tables! unless @tables
-    @tables
-  end
-
-  def create_log_table(name)
-    @client.create_log_table(@db_name, name)
-  end
-
-  def create_item_table(name)
-    @client.create_item_table(@db_name, name)
-  end
-
-  def table(table_name)
-    @client.table(@db_name, table_name)
-  end
-
-  def delete
-    @client.delete_database(@db_name)
-  end
-
-  def update_tables!
-    @tables = @client.tables(@db_name)
-  end
-end
-
-class Table < Model
-  def initialize(client, db_name, table_name, type, schema, count)
-    super(client)
-    @db_name = db_name
-    @table_name = table_name
-    @type = type
-    @schema = schema
-    @count = count
-  end
-
-  attr_reader :type, :db_name, :table_name, :schema, :count
-
-  alias database_name db_name
-  alias name table_name
-
-  def database
-    @client.database(@db_name)
-  end
-
-  def identifier
-    "#{@db_name}.#{@table_name}"
-  end
-
-  def delete
-    @client.delete_table(@db_name, @table_name)
-  end
-end
-
-class Schema
-  class Field
-    def initialize(name, type)
-      @name = name
-      @type = type
-    end
-    attr_reader :name
-    attr_reader :type
-  end
-
-  def self.parse(cols)
-    fields = cols.split(',').map {|col|
-      name, type, *_ = col.split(':')
-      Field.new(name, type)
-    }
-    Schema.new(fields)
-  end
-
-  def initialize(fields=[])
-    @fields = fields
-  end
-
-  attr_reader :fields
-
-  def add_field(name, type)
-    @fields << Field.new(name, type)
-  end
-
-  def merge(schema)
-    nf = @fields.dup
-    schema.fields.each {|f|
-      if i = nf.find_index {|sf| sf.name == f.name }
-        nf[i] = f
-      else
-        nf << f
-      end
-    }
-    Schema.new(nf)
-  end
-
-  def to_json(*args)
-    @fields.map {|f| [f.name, f.type] }.to_json(*args)
-  end
-
-  def from_json(obj)
-    @fields = obj.map {|f|
-      Field.new(f[0], f[1])
-    }
-    self
-  end
-end
-
-class Job < Model
-  def initialize(client, job_id, type, query, status=nil, url=nil, debug=nil, start_at=nil, end_at=nil, result=nil)
-    super(client)
-    @job_id = job_id
-    @type = type
-    @url = url
-    @query = query
-    @status = status
-    @debug = debug
-    @start_at = start_at
-    @end_at = end_at
-    @result = result
-  end
-
-  attr_reader :job_id, :type
-
-  def wait(timeout=nil)
-    # TODO
-  end
-
-  def query
-    update_status! unless @query
-    @query
-  end
-
-  def status
-    update_status! unless @status
-    @status
-  end
-
-  def url
-    update_status! unless @url
-    @url
-  end
-
-  def debug
-    update_status! unless @debug
-    @debug
-  end
-
-  def start_at
-    update_status! unless @start_at
-    @start_at && !@start_at.empty? ? Time.parse(@start_at) : nil
-  end
-
-  def end_at
-    update_status! unless @end_at
-    @end_at && !@end_at.empty? ? Time.parse(@end_at) : nil
-  end
-
-  def result
-    unless @result
-      return nil unless finished?
-      @result = @client.job_result(@job_id)
-    end
-    @result
-  end
-
-  def result_format(format)
-    return nil unless finished?
-    @client.job_result_format(@job_id, format)
-  end
-
-  def result_each(&block)
-    if @result
-      @result.each(&block)
-    else
-      @client.job_result_each(@job_id, &block)
-    end
-    nil
-  end
-
-  def finished?
-    update_status! unless @status
-    if @status == "success" || @status == "error"
-      return true
-    else
-      return false
-    end
-  end
-
-  def running?
-    !finished?
-  end
-
-  def success?
-    update_status! unless @status
-    @status == "success"
-  end
-
-  def error?
-    update_status! unless @status
-    @status == "error"
-  end
-
-  def update_status!
-    query, status, url, debug, start_at, end_at = @client.job_status(@job_id)
-    @query = query
-    @status = status
-    @url = url
-    @debug = debug
-    @start_at = start_at
-    @end_at = end_at
-    self
-  end
-end
-
-end
-