td-client 0.8.0

data/ChangeLog

@@ -0,0 +1,5 @@
+
+== 2011-08-21 version 0.8.0
+
+* First release
+

data/README.rdoc

@@ -0,0 +1,12 @@
+= Treasure Data API library for Ruby
+
+= Getting Started
+
+    > gem install td-client
+
+
+== Copyright
+
+Copyright:: Copyright (c) 2011 Treasure Data Inc.
+License::   Apache License, Version 2.0
+

data/lib/td-client.rb

@@ -0,0 +1 @@
+require File.join(File.dirname(__FILE__), 'td', 'client')

data/lib/td/client.rb

@@ -0,0 +1,152 @@
+
+module TreasureData
+
+require 'td/client/api'
+require 'td/client/model'
+
+
+class Client
+  def self.authenticate(user, password)
+    api = API.new(nil)
+    apikey = api.authenticate(user, password)
+    new(apikey)
+  end
+
+  def self.server_status
+    api = API.new(nil)
+    api.server_status
+  end
+
+  def initialize(apikey)
+    @api = API.new(apikey)
+  end
+
+  attr_reader :api
+
+  def apikey
+    @api.apikey
+  end
+
+  def server_status
+    @api.server_status
+  end
+
+  # => true
+  def create_database(db_name)
+    @api.create_database(db_name)
+  end
+
+  # => true
+  def delete_database(db_name)
+    @api.delete_database(db_name)
+  end
+
+  # => [Database]
+  def databases
+    names = @api.list_databases
+    names.map {|db_name|
+      Database.new(self, db_name)
+    }
+  end
+
+  # => Database
+  def database(db_name)
+    names = @api.list_databases
+    names.each {|n|
+      if n == db_name
+        return Database.new(self, db_name)
+      end
+    }
+    raise NotFoundError, "Database '#{db_name}' does not exist"
+  end
+
+  # => true
+  def create_log_table(db_name, table_name)
+    @api.create_log_table(db_name, table_name)
+  end
+
+  # => true
+  def create_item_table(db_name, table_name)
+    @api.create_item_table(db_name, table_name)
+  end
+
+  # => true
+  def update_schema(db_name, table_name, schema)
+    @api.update_schema(db_name, table_name, schema.to_json)
+  end
+
+  # => type:Symbol
+  def delete_table(db_name, table_name)
+    @api.delete_table(db_name, table_name)
+  end
+
+  # => [Table]
+  def tables(db_name)
+    m = @api.list_tables(db_name)
+    m.map {|table_name,(type,schema,count)|
+      schema = Schema.new.from_json(schema)
+      Table.new(self, db_name, table_name, type, schema, count)
+    }
+  end
+
+  # => Table
+  def table(db_name, table_name)
+    tables(db_name).each {|t|
+      if t.name == table_name
+        return t
+      end
+    }
+    raise NotFoundError, "Table '#{db_name}.#{table_name}' does not exist"
+  end
+
+  # => Job
+  def query(db_name, q)
+    job_id = @api.hive_query(q, db_name)
+    Job.new(self, job_id, :hive, q)  # TODO url
+  end
+
+  # => [Job=]
+  def jobs(from=nil, to=nil)
+    js = @api.list_jobs(from, to)
+    js.map {|job_id,type,status,query,start_at,end_at|
+      Job.new(self, job_id, type, query, status, nil, nil, start_at, end_at)
+    }
+  end
+
+  # => Job
+  def job(job_id)
+    job_id = job_id.to_s
+    type, query, status, url, debug, start_at, end_at = @api.show_job(job_id)
+    Job.new(self, job_id, type, query, status, url, debug, start_at, end_at)
+  end
+
+  # => type:Symbol, url:String
+  def job_status(job_id)
+    type, query, status, url, debug, start_at, end_at = @api.show_job(job_id)
+    return query, status, url, debug, start_at, end_at
+  end
+
+  # => result:[{column:String=>value:Object]
+  def job_result(job_id)
+    @api.job_result(job_id)
+  end
+
+  # => result:String
+  def job_result_format(job_id, format)
+    @api.job_result_format(job_id, format)
+  end
+
+  # => nil
+  def job_result_each(job_id, &block)
+    @api.job_result_each(job_id, &block)
+  end
+
+  # => time:Flaot
+  def import(db_name, table_name, format, stream, size)
+    @api.import(db_name, table_name, format, stream, size)
+  end
+end
+
+
+end
+

data/lib/td/client/api.rb

@@ -0,0 +1,460 @@
+
+module TreasureData
+
+
+class APIError < StandardError
+end
+
+class AuthError < APIError
+end
+
+class AlreadyExistsError < APIError
+end
+
+class NotFoundError < APIError
+end
+
+
+class API
+  def initialize(apikey)
+    require 'json'
+    require 'time'
+    @apikey = apikey
+  end
+
+  # TODO error check & raise appropriate errors
+
+  attr_reader :apikey
+
+  def self.validate_database_name(name)
+    name = name.to_s
+    if name.empty?
+      raise "Empty name is not allowed"
+    end
+    if name.length < 3 || 32 < name.length
+      raise "Name must be 3 to 32 characters, got #{name.length} characters."
+    end
+    unless name =~ /^([a-z0-9_]+)$/
+      raise "Name must consist only of alphabets, numbers, '_'."
+    end
+    name
+  end
+
+  def self.validate_table_name(name)
+    validate_database_name(name)
+  end
+
+  def self.validate_column_name(name)
+    name = name.to_s
+    if name.empty?
+      raise "Empty column name is not allowed"
+    end
+    if 32 < name.length
+      raise "Column name must be to 32 characters, got #{name.length} characters."
+    end
+    unless name =~ /^([a-z0-9_]+)$/
+      raise "Column name must consist only of alphabets, numbers, '_'."
+    end
+  end
+
+  def self.normalize_type_name(name)
+    case name
+    when /int/i, /integer/i
+      "int"
+    when /long/i, /bigint/i
+      "long"
+    when /string/i
+      "string"
+    when /float/i
+      "float"
+    when /double/i
+      "double"
+    else
+      raise "Type name must eather of int, long, string float or double"
+    end
+  end
+
+  ####
+  ## Database API
+  ##
+
+  # => [name:String]
+  def list_databases
+    code, body, res = get("/v3/database/list")
+    if code != "200"
+      raise_error("List databases failed", res)
+    end
+    # TODO format check
+    js = JSON.load(body)
+    names = js["databases"].map {|dbinfo| dbinfo['name'] }
+    return names
+  end
+
+  # => true
+  def delete_database(db)
+    code, body, res = post("/v3/database/delete/#{e db}")
+    if code != "200"
+      raise_error("Delete database failed", res)
+    end
+    return true
+  end
+
+  # => true
+  def create_database(db)
+    code, body, res = post("/v3/database/create/#{e db}")
+    if code != "200"
+      raise_error("Create database failed", res)
+    end
+    return true
+  end
+
+
+  ####
+  ## Table API
+  ##
+
+  # => {name:String => [type:Symbol, count:Integer]}
+  def list_tables(db)
+    code, body, res = get("/v3/table/list/#{e db}")
+    if code != "200"
+      raise_error("List tables failed", res)
+    end
+    # TODO format check
+    js = JSON.load(body)
+    result = {}
+    js["tables"].map {|m|
+      name = m['name']
+      type = (m['type'] || '?').to_sym
+      count = (m['count'] || 0).to_i  # TODO?
+      schema = JSON.parse(m['schema'] || '[]')
+      result[name] = [type, schema, count]
+    }
+    return result
+  end
+
+  def create_log_or_item_table(db, table, type)
+    code, body, res = post("/v3/table/create/#{e db}/#{e table}/#{type}")
+    if code != "200"
+      raise_error("Create #{type} table failed", res)
+    end
+    return true
+  end
+  private :create_log_or_item_table
+
+  # => true
+  def create_log_table(db, table)
+    create_table(db, table, :log)
+  end
+
+  # => true
+  def create_item_table(db, table)
+    create_table(db, table, :item)
+  end
+
+  def create_table(db, table, type)
+    schema = schema.to_s
+    code, body, res = post("/v3/table/create/#{e db}/#{e table}/#{type}")
+    if code != "200"
+      raise_error("Create #{type} table failed", res)
+    end
+    return true
+  end
+  private :create_table
+
+  # => true
+  def update_schema(db, table, schema_json)
+    code, body, res = post("/v3/table/update-schema/#{e db}/#{e table}", {'schema'=>schema_json})
+    if code != "200"
+      raise_error("Create schema table failed", res)
+    end
+    return true
+  end
+
+  # => type:Symbol
+  def delete_table(db, table)
+    code, body, res = post("/v3/table/delete/#{e db}/#{e table}")
+    if code != "200"
+      raise_error("Drop table failed", res)
+    end
+    # TODO format check
+    js = JSON.load(body)
+    type = (js['type'] || '?').to_sym
+    return type
+  end
+
+
+  ####
+  ## Job API
+  ##
+
+  # => [(jobId:String, type:Symbol, status:String, start_at:String, end_at:String)]
+  def list_jobs(from=0, to=nil)
+    params = {}
+    params['from'] = from.to_s if from
+    params['to'] = to.to_s if to
+    code, body, res = get("/v3/job/list", params)
+    if code != "200"
+      raise_error("List jobs failed", res)
+    end
+    # TODO format check
+    js = JSON.load(body)
+    result = []
+    js['jobs'].each {|m|
+      job_id = m['job_id']
+      type = (m['type'] || '?').to_sym
+      status = m['status']
+      query = m['query']
+      start_at = m['start_at']
+      end_at = m['end_at']
+      result << [job_id, type, status, query, start_at, end_at]
+    }
+    return result
+  end
+
+  # => (type:Symbol, status:String, result:String, url:String)
+  def show_job(job_id)
+    code, body, res = get("/v3/job/show/#{e job_id}")
+    if code != "200"
+      raise_error("Show job failed", res)
+    end
+    # TODO format check
+    js = JSON.load(body)
+    # TODO debug
+    type = (js['type'] || '?').to_sym  # TODO
+    query = js['query']
+    status = js['status']
+    debug = js['debug']
+    url = js['url']
+    start_at = js['start_at']
+    end_at = js['end_at']
+    return [type, query, status, url, debug, start_at, end_at]
+  end
+
+  def job_result(job_id)
+    require 'msgpack'
+    code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>'msgpack'})
+    if code != "200"
+      raise_error("Get job result failed", res)
+    end
+    result = []
+    MessagePack::Unpacker.new.feed_each(body) {|row|
+      result << row
+    }
+    return result
+  end
+
+  def job_result_format(job_id, format)
+    # TODO chunked encoding
+    code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>format})
+    if code != "200"
+      raise_error("Get job result failed", res)
+    end
+    return body
+  end
+
+  def job_result_each(job_id, &block)
+    # TODO chunked encoding
+    require 'msgpack'
+    code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>'msgpack'})
+    if code != "200"
+      raise_error("Get job result failed", res)
+    end
+    result = []
+    MessagePack::Unpacker.new.feed_each(body) {|row|
+      yield row
+    }
+    nil
+  end
+
+  def job_result_raw(job_id, format)
+    code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>format})
+    if code != "200"
+      raise_error("Get job result failed", res)
+    end
+    return body
+  end
+
+  # => jobId:String
+  def hive_query(q, db=nil)
+    code, body, res = post("/v3/job/issue/hive/#{e db}", {'query'=>q})
+    if code != "200"
+      raise_error("Query failed", res)
+    end
+    # TODO format check
+    js = JSON.load(body)
+    return js['job_id'].to_s
+  end
+
+
+  ####
+  ## Import API
+  ##
+
+  # => time:Float
+  def import(db, table, format, stream, size)
+    code, body, res = put("/v3/table/import/#{e db}/#{e table}/#{format}", stream, size)
+    if code[0] != ?2
+      raise_error("Import failed", res)
+    end
+    # TODO format check
+    js = JSON.load(body)
+    time = js['time'].to_f
+    return time
+  end
+
+
+  ####
+  ## User API
+  ##
+
+  # apikey:String
+  def authenticate(user, password)
+    code, body, res = post("/v3/user/authenticate", {'user'=>user, 'password'=>password})
+    if code != "200"
+      raise_error("Authentication failed", res)
+    end
+    # TODO format check
+    js = JSON.load(body)
+    apikey = js['apikey']
+    return apikey
+  end
+
+  ####
+  ## Server Status API
+  ##
+
+  # => status:String
+  def server_status
+    code, body, res = get('/v3/system/server_status')
+    if code != "200"
+      return "Server is down (#{code})"
+    end
+    # TODO format check
+    js = JSON.load(body)
+    status = js['status']
+    return status
+  end
+
+  private
+  host = 'api.treasure-data.com'
+  port = 80
+  if e = ENV['TD_API_SERVER']
+    host, port_ = e.split(':',2)
+    port_ = port_.to_i
+    port = port_ if port_ != 0
+  end
+
+  HOST = host
+  PORT = port
+  USE_SSL = false
+  BASE_URL = ''
+
+  def get(url, params=nil)
+    http, header = new_http
+
+    path = BASE_URL + url
+    if params && !params.empty?
+      path << "?"+params.map {|k,v|
+        "#{k}=#{e v}"
+      }.join('&')
+    end
+
+    request = Net::HTTP::Get.new(path, header)
+
+    response = http.request(request)
+    return [response.code, response.body, response]
+  end
+
+  def post(url, params=nil)
+    http, header = new_http
+
+    path = BASE_URL + url
+
+    request = Net::HTTP::Post.new(path, header)
+    request.set_form_data(params) if params
+
+    response = http.request(request)
+    return [response.code, response.body, response]
+  end
+
+  def put(url, stream, size)
+    http, header = new_http
+
+    path = BASE_URL + url
+
+    header['Content-Type'] = 'application/octet-stream'
+    header['Content-Length'] = size.to_s
+
+    request = Net::HTTP::Put.new(url, header)
+    if stream.class.name == 'StringIO'
+      request.body = stream.string
+    else
+      if request.respond_to?(:body_stream=)
+        request.body_stream = stream
+      else  # Ruby 1.8
+        request.body = stream.read
+      end
+    end
+
+    response = http.request(request)
+    return [response.code, response.body, response]
+  end
+
+  def new_http
+    require 'net/http'
+    require 'time'
+
+    http = Net::HTTP.new(HOST, PORT)
+    if USE_SSL
+      http.use_ssl = true
+      http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+      store = OpenSSL::X509::Store.new
+      http.cert_store = store
+    end
+
+    #http.read_timeout = options[:read_timeout]
+
+    header = {}
+    if @apikey
+      header['Authorization'] = "TD1 #{apikey}"
+    end
+    header['Date'] = Time.now.rfc2822
+
+    return http, header
+  end
+
+  def raise_error(msg, res)
+    begin
+      js = JSON.load(res.body)
+      msg = js['message']
+      error_code = js['error_code']
+
+      if res.code == "404"
+        raise NotFoundError, "#{error_code}: #{msg}"
+      elsif res.code == "409"
+        raise AlreadyExistsError, "#{error_code}: #{msg}"
+      else
+        raise APIError, "#{error_code}: #{msg}"
+      end
+
+    rescue
+      if res.code == "404"
+        raise NotFoundError, "#{msg}: #{res.body}"
+      elsif res.code == "409"
+        raise AlreadyExistsError, "#{msg}: #{res.body}"
+      else
+        raise APIError, "#{msg}: #{res.body}"
+      end
+    end
+    # TODO error
+  end
+
+  def e(s)
+    require 'cgi'
+    CGI.escape(s.to_s)
+  end
+end
+
+
+end
+

data/lib/td/client/model.rb

@@ -0,0 +1,237 @@
+
+module TreasureData
+
+
+class Model
+  def initialize(client)
+    @client = client
+  end
+end
+
+class Database < Model
+  def initialize(client, db_name, tables=nil)
+    super(client)
+    @db_name = db_name
+    @tables = tables
+  end
+
+  def name
+    @db_name
+  end
+
+  def tables
+    update_tables! unless @tables
+    @tables
+  end
+
+  def create_log_table(name)
+    @client.create_log_table(@db_name, name)
+  end
+
+  def create_item_table(name)
+    @client.create_item_table(@db_name, name)
+  end
+
+  def table(table_name)
+    @client.table(@db_name, table_name)
+  end
+
+  def delete
+    @client.delete_database(@db_name)
+  end
+
+  def update_tables!
+    @tables = @client.tables(@db_name)
+  end
+end
+
+class Table < Model
+  def initialize(client, db_name, table_name, type, schema, count)
+    super(client)
+    @db_name = db_name
+    @table_name = table_name
+    @type = type
+    @schema = schema
+    @count = count
+  end
+
+  attr_reader :type, :db_name, :table_name, :schema, :count
+
+  alias database_name db_name
+  alias name table_name
+
+  def database
+    @client.database(@db_name)
+  end
+
+  def identifier
+    "#{@db_name}.#{@table_name}"
+  end
+
+  def delete
+    @client.delete_table(@db_name, @table_name)
+  end
+end
+
+class Schema
+  class Field
+    def initialize(name, type)
+      @name = name
+      @type = type
+    end
+    attr_reader :name
+    attr_reader :type
+  end
+
+  def self.parse(cols)
+    fields = cols.split(',').map {|col|
+      name, type, *_ = col.split(':')
+      Field.new(name, type)
+    }
+    Schema.new(fields)
+  end
+
+  def initialize(fields=[])
+    @fields = fields
+  end
+
+  attr_reader :fields
+
+  def add_field(name, type)
+    @fields << Field.new(name, type)
+  end
+
+  def merge(schema)
+    nf = @fields.dup
+    schema.fields.each {|f|
+      if i = nf.find_index {|sf| sf.name == f.name }
+        nf[i] = f
+      else
+        nf << f
+      end
+    }
+    Schema.new(nf)
+  end
+
+  def to_json(*args)
+    @fields.map {|f| [f.name, f.type] }.to_json(*args)
+  end
+
+  def from_json(obj)
+    @fields = obj.map {|f|
+      Field.new(f[0], f[1])
+    }
+    self
+  end
+end
+
+class Job < Model
+  def initialize(client, job_id, type, query, status=nil, url=nil, debug=nil, start_at=nil, end_at=nil, result=nil)
+    super(client)
+    @job_id = job_id
+    @type = type
+    @url = url
+    @query = query
+    @status = status
+    @debug = debug
+    @start_at = start_at
+    @end_at = end_at
+    @result = result
+  end
+
+  attr_reader :job_id, :type
+
+  def wait(timeout=nil)
+    # TODO
+  end
+
+  def query
+    update_status! unless @query
+    @query
+  end
+
+  def status
+    update_status! unless @status
+    @status
+  end
+
+  def url
+    update_status! unless @url
+    @url
+  end
+
+  def debug
+    update_status! unless @debug
+    @debug
+  end
+
+  def start_at
+    update_status! unless @start_at
+    @start_at && !@start_at.empty? ? Time.parse(@start_at) : nil
+  end
+
+  def end_at
+    update_status! unless @end_at
+    @end_at && !@end_at.empty? ? Time.parse(@end_at) : nil
+  end
+
+  def result
+    unless @result
+      return nil unless finished?
+      @result = @client.job_result(@job_id)
+    end
+    @result
+  end
+
+  def result_format(format)
+    return nil unless finished?
+    @client.job_result_format(@job_id, format)
+  end
+
+  def result_each(&block)
+    if @result
+      @result.each(&block)
+    else
+      @client.job_result_each(@job_id, &block)
+    end
+    nil
+  end
+
+  def finished?
+    update_status! unless @status
+    if @status == "success" || @status == "error"
+      return true
+    else
+      return false
+    end
+  end
+
+  def running?
+    !finished?
+  end
+
+  def success?
+    update_status! unless @status
+    @status == "success"
+  end
+
+  def error?
+    update_status! unless @status
+    @status == "error"
+  end
+
+  def update_status!
+    query, status, url, debug, start_at, end_at = @client.job_status(@job_id)
+    @query = query
+    @status = status
+    @url = url
+    @debug = debug
+    @start_at = start_at
+    @end_at = end_at
+    self
+  end
+end
+
+
+end
+

data/lib/td/client/version.rb

@@ -0,0 +1,5 @@
+module TreasureData
+
+VERSION = '0.8.0'
+
+end

metadata

@@ -0,0 +1,105 @@
+--- !ruby/object:Gem::Specification 
+name: td-client
+version: !ruby/object:Gem::Version 
+  hash: 63
+  prerelease: false
+  segments: 
+  - 0
+  - 8
+  - 0
+  version: 0.8.0
+platform: ruby
+authors: 
+- Sadayuki Furuhashi
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-08-21 00:00:00 +09:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: msgpack
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 7
+        segments: 
+        - 0
+        - 4
+        - 4
+        version: 0.4.4
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: json
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 1
+        segments: 
+        - 1
+        - 4
+        - 3
+        version: 1.4.3
+  type: :runtime
+  version_requirements: *id002
+description: 
+email: 
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- ChangeLog
+- README.rdoc
+files: 
+- lib/td-client.rb
+- lib/td/client.rb
+- lib/td/client/api.rb
+- lib/td/client/model.rb
+- lib/td/client/version.rb
+- ChangeLog
+- README.rdoc
+has_rdoc: true
+homepage: 
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Treasure Data API library for Ruby
+test_files: []
+