td-client 0.8.67 → 0.8.68

data/lib/td/client/api/access_control.rb

@@ -0,0 +1,62 @@
+class TreasureData::API
+module AccessControl
+
+  ####
+  ## Access Control API
+  ##
+
+  def grant_access_control(subject, action, scope, grant_option)
+    params = {'subject'=>subject, 'action'=>action, 'scope'=>scope, 'grant_option'=>grant_option.to_s}
+    code, body, res = post("/v3/acl/grant", params)
+    if code != "200"
+      raise_error("Granting access control failed", res)
+    end
+    return true
+  end
+
+  def revoke_access_control(subject, action, scope)
+    params = {'subject'=>subject, 'action'=>action, 'scope'=>scope}
+    code, body, res = post("/v3/acl/revoke", params)
+    if code != "200"
+      raise_error("Revoking access control failed", res)
+    end
+    return true
+  end
+
+  # [true, [{subject:String,action:String,scope:String}]]
+  def test_access_control(user, action, scope)
+    params = {'user'=>user, 'action'=>action, 'scope'=>scope}
+    code, body, res = get("/v3/acl/test", params)
+    if code != "200"
+      raise_error("Testing access control failed", res)
+    end
+    js = checked_json(body, %w[permission access_controls])
+    perm = js["permission"]
+    acl = js["access_controls"].map {|roleinfo|
+      subject = roleinfo['subject']
+      action = roleinfo['action']
+      scope = roleinfo['scope']
+      [name, action, scope]
+    }
+    return perm, acl
+  end
+
+  # [{subject:String,action:String,scope:String}]
+  def list_access_controls
+    code, body, res = get("/v3/acl/list")
+    if code != "200"
+      raise_error("Listing access control failed", res)
+    end
+    js = checked_json(body, %w[access_controls])
+    acl = js["access_controls"].map {|roleinfo|
+      subject = roleinfo['subject']
+      action = roleinfo['action']
+      scope = roleinfo['scope']
+      grant_option = roleinfo['grant_option']
+      [subject, action, scope, grant_option]
+    }
+    return acl
+  end
+
+end
+end

data/lib/td/client/api/account.rb

@@ -0,0 +1,41 @@
+class TreasureData::API
+module Account
+
+  ####
+  ## Account API
+  ##
+
+  def show_account
+    code, body, res = get("/v3/account/show")
+    if code != "200"
+      raise_error("Show account failed", res)
+    end
+    js = checked_json(body, %w[account])
+    a = js["account"]
+    account_id = a['id'].to_i
+    plan = a['plan'].to_i
+    storage_size = a['storage_size'].to_i
+    guaranteed_cores = a['guaranteed_cores'].to_i
+    maximum_cores = a['maximum_cores'].to_i
+    created_at = a['created_at']
+    return [account_id, plan, storage_size, guaranteed_cores, maximum_cores, created_at]
+  end
+
+  def account_core_utilization(from, to)
+    params = { }
+    params['from'] = from.to_s if from
+    params['to'] = to.to_s if to
+    code, body, res = get("/v3/account/core_utilization", params)
+    if code != "200"
+      raise_error("Show account failed", res)
+    end
+    js = checked_json(body, %w[from to interval history])
+    from = Time.parse(js['from']).utc
+    to = Time.parse(js['to']).utc
+    interval = js['interval'].to_i
+    history = js['history']
+    return [from, to, interval, history]
+  end
+
+end
+end

data/lib/td/client/api/bulk_import.rb

@@ -0,0 +1,154 @@
+class TreasureData::API
+module BulkImport
+
+  ####
+  ## Bulk import API
+  ##
+
+  # => nil
+  def create_bulk_import(name, db, table, opts={})
+    params = opts.dup
+    code, body, res = post("/v3/bulk_import/create/#{e name}/#{e db}/#{e table}", params)
+    if code != "200"
+      raise_error("Create bulk import failed", res)
+    end
+    return nil
+  end
+
+  # => nil
+  def delete_bulk_import(name, opts={})
+    params = opts.dup
+    code, body, res = post("/v3/bulk_import/delete/#{e name}", params)
+    if code != "200"
+      raise_error("Delete bulk import failed", res)
+    end
+    return nil
+  end
+
+  # => data:Hash
+  def show_bulk_import(name)
+    code, body, res = get("/v3/bulk_import/show/#{name}")
+    if code != "200"
+      raise_error("Show bulk import failed", res)
+    end
+    js = checked_json(body, %w[status])
+    return js
+  end
+
+  # => result:[data:Hash]
+  def list_bulk_imports(opts={})
+    params = opts.dup
+    code, body, res = get("/v3/bulk_import/list", params)
+    if code != "200"
+      raise_error("List bulk imports failed", res)
+    end
+    js = checked_json(body, %w[bulk_imports])
+    return js['bulk_imports']
+  end
+
+  def list_bulk_import_parts(name, opts={})
+    params = opts.dup
+    code, body, res = get("/v3/bulk_import/list_parts/#{e name}", params)
+    if code != "200"
+      raise_error("List bulk import parts failed", res)
+    end
+    js = checked_json(body, %w[parts])
+    return js['parts']
+  end
+
+  # => nil
+  def bulk_import_upload_part(name, part_name, stream, size, opts={})
+    code, body, res = put("/v3/bulk_import/upload_part/#{e name}/#{e part_name}", stream, size)
+    if code[0] != ?2
+      raise_error("Upload a part failed", res)
+    end
+    return nil
+  end
+
+  # => nil
+  def bulk_import_delete_part(name, part_name, opts={})
+    params = opts.dup
+    code, body, res = post("/v3/bulk_import/delete_part/#{e name}/#{e part_name}", params)
+    if code[0] != ?2
+      raise_error("Delete a part failed", res)
+    end
+    return nil
+  end
+
+  # => nil
+  def freeze_bulk_import(name, opts={})
+    params = opts.dup
+    code, body, res = post("/v3/bulk_import/freeze/#{e name}", params)
+    if code != "200"
+      raise_error("Freeze bulk import failed", res)
+    end
+    return nil
+  end
+
+  # => nil
+  def unfreeze_bulk_import(name, opts={})
+    params = opts.dup
+    code, body, res = post("/v3/bulk_import/unfreeze/#{e name}", params)
+    if code != "200"
+      raise_error("Unfreeze bulk import failed", res)
+    end
+    return nil
+  end
+
+  # => jobId:String
+  def perform_bulk_import(name, opts={})
+    params = opts.dup
+    code, body, res = post("/v3/bulk_import/perform/#{e name}", params)
+    if code != "200"
+      raise_error("Perform bulk import failed", res)
+    end
+    js = checked_json(body, %w[job_id])
+    return js['job_id'].to_s
+  end
+
+  # => nil
+  def commit_bulk_import(name, opts={})
+    params = opts.dup
+    code, body, res = post("/v3/bulk_import/commit/#{e name}", params)
+    if code != "200"
+      raise_error("Commit bulk import failed", res)
+    end
+    return nil
+  end
+
+  # => data...
+  def bulk_import_error_records(name, opts={}, &block)
+    params = opts.dup
+    code, body, res = get("/v3/bulk_import/error_records/#{e name}", params)
+    if code != "200"
+      raise_error("Failed to get bulk import error records", res)
+    end
+    if body.nil? || body.empty?
+      if block
+        return nil
+      else
+        return []
+      end
+    end
+    require File.expand_path('../compat_gzip_reader', File.dirname(__FILE__))
+    u = MessagePack::Unpacker.new(Zlib::GzipReader.new(StringIO.new(body)))
+    if block
+      begin
+        u.each(&block)
+      rescue EOFError
+      end
+      nil
+    else
+      result = []
+      begin
+        u.each {|row|
+          result << row
+        }
+      rescue EOFError
+      end
+      return result
+    end
+  end
+
+end
+end

data/lib/td/client/api/database.rb

@@ -0,0 +1,47 @@
+class TreasureData::API
+module Database
+
+  ####
+  ## Database API
+  ##
+
+  # => [name:String]
+  def list_databases
+    code, body, res = get("/v3/database/list")
+    if code != "200"
+      raise_error("List databases failed", res)
+    end
+    js = checked_json(body, %w[databases])
+    result = {}
+    js["databases"].each {|m|
+      name = m['name']
+      count = m['count']
+      created_at = m['created_at']
+      updated_at = m['updated_at']
+      permission = m['permission']
+      result[name] = [count, created_at, updated_at, nil, permission] # set nil to org for API compatibiilty
+    }
+    return result
+  end
+
+  # => true
+  def delete_database(db)
+    code, body, res = post("/v3/database/delete/#{e db}")
+    if code != "200"
+      raise_error("Delete database failed", res)
+    end
+    return true
+  end
+
+  # => true
+  def create_database(db, opts={})
+    params = opts.dup
+    code, body, res = post("/v3/database/create/#{e db}", params)
+    if code != "200"
+      raise_error("Create database failed", res)
+    end
+    return true
+  end
+
+end
+end

data/lib/td/client/api/export.rb

@@ -0,0 +1,21 @@
+class TreasureData::API
+module Export
+
+  ####
+  ## Export API
+  ##
+
+  # => jobId:String
+  def export(db, table, storage_type, opts={})
+    params = opts.dup
+    params['storage_type'] = storage_type
+    code, body, res = post("/v3/export/run/#{e db}/#{e table}", params)
+    if code != "200"
+      raise_error("Export failed", res)
+    end
+    js = checked_json(body, %w[job_id])
+    return js['job_id'].to_s
+  end
+
+end
+end

data/lib/td/client/api/import.rb

@@ -0,0 +1,31 @@
+class TreasureData::API
+module Import
+
+  ####
+  ## Import API
+  ##
+
+  # => time:Float
+  def import(db, table, format, stream, size, unique_id=nil)
+    if unique_id
+      path = "/v3/table/import_with_id/#{e db}/#{e table}/#{unique_id}/#{format}"
+    else
+      path = "/v3/table/import/#{e db}/#{e table}/#{format}"
+    end
+    opts = {}
+    if @host == DEFAULT_ENDPOINT
+      opts[:host] = DEFAULT_IMPORT_ENDPOINT
+    elsif @host == NEW_DEFAULT_ENDPOINT
+      opts[:host] = NEW_DEFAULT_IMPORT_ENDPOINT
+    end
+    code, body, res = put(path, stream, size, opts)
+    if code[0] != ?2
+      raise_error("Import failed", res)
+    end
+    js = checked_json(body, %w[])
+    time = js['elapsed_time'].to_f
+    return time
+  end
+
+end
+end

data/lib/td/client/api/job.rb

@@ -0,0 +1,251 @@
+class TreasureData::API
+module Job
+
+  ####
+  ## Job API
+  ##
+
+  # => [(jobId:String, type:Symbol, status:String, start_at:String, end_at:String, result_url:String)]
+  def list_jobs(from=0, to=nil, status=nil, conditions=nil)
+    params = {}
+    params['from'] = from.to_s if from
+    params['to'] = to.to_s if to
+    params['status'] = status.to_s if status
+    params.merge!(conditions) if conditions
+    code, body, res = get("/v3/job/list", params)
+    if code != "200"
+      raise_error("List jobs failed", res)
+    end
+    js = checked_json(body, %w[jobs])
+    result = []
+    js['jobs'].each {|m|
+      job_id = m['job_id']
+      type = (m['type'] || '?').to_sym
+      database = m['database']
+      status = m['status']
+      query = m['query']
+      start_at = m['start_at']
+      end_at = m['end_at']
+      cpu_time = m['cpu_time']
+      result_size = m['result_size'] # compressed result size in msgpack.gz format
+      result_url = m['result']
+      priority = m['priority']
+      retry_limit = m['retry_limit']
+      result << [job_id, type, status, query, start_at, end_at, cpu_time,
+                 result_size, result_url, priority, retry_limit, nil, database]
+    }
+    return result
+  end
+
+  # => (type:Symbol, status:String, result:String, url:String, result:String)
+  def show_job(job_id)
+    # use v3/job/status instead of v3/job/show to poll finish of a job
+    code, body, res = get("/v3/job/show/#{e job_id}")
+    if code != "200"
+      raise_error("Show job failed", res)
+    end
+    js = checked_json(body, %w[status])
+    # TODO debug
+    type = (js['type'] || '?').to_sym  # TODO
+    database = js['database']
+    query = js['query']
+    status = js['status']
+    debug = js['debug']
+    url = js['url']
+    start_at = js['start_at']
+    end_at = js['end_at']
+    cpu_time = js['cpu_time']
+    result_size = js['result_size'] # compressed result size in msgpack.gz format
+    result = js['result'] # result target URL
+    hive_result_schema = (js['hive_result_schema'] || '')
+    if hive_result_schema.empty?
+      hive_result_schema = nil
+    else
+      begin
+        hive_result_schema = JSON.parse(hive_result_schema)
+      rescue JSON::ParserError => e
+        # this is a workaround for a Known Limitation in the Pig Engine which does not set a default, auto-generated
+        #   column name for anonymous columns (such as the ones that are generated from UDF like COUNT or SUM).
+        # The schema will contain 'nil' for the name of those columns and that breaks the JSON parser since it violates
+        #   the JSON syntax standard.
+        if type == :pig and hive_result_schema !~ /[\{\}]/
+          begin
+            # NOTE: this works because a JSON 2 dimensional array is the same as a Ruby one.
+            #   Any change in the format for the hive_result_schema output may cause a syntax error, in which case
+            #   this lame attempt at fixing the problem will fail and we will be raising the original JSON exception
+            hive_result_schema = eval(hive_result_schema)
+          rescue SyntaxError => ignored_e
+            raise e
+          end
+          hive_result_schema.each_with_index {|col_schema, idx|
+            if col_schema[0].nil?
+              col_schema[0] = "_col#{idx}"
+            end
+          }
+        else
+          raise e
+        end
+      end
+    end
+    priority = js['priority']
+    retry_limit = js['retry_limit']
+    return [type, query, status, url, debug, start_at, end_at, cpu_time,
+            result_size, result, hive_result_schema, priority, retry_limit, nil, database]
+  end
+
+  def job_status(job_id)
+    code, body, res = get("/v3/job/status/#{e job_id}")
+    if code != "200"
+      raise_error("Get job status failed", res)
+    end
+
+    js = checked_json(body, %w[status])
+    return js['status']
+  end
+
+  def job_result(job_id)
+    code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>'msgpack'})
+    if code != "200"
+      raise_error("Get job result failed", res)
+    end
+    result = []
+    MessagePack::Unpacker.new.feed_each(body) {|row|
+      result << row
+    }
+    return result
+  end
+
+  # block is optional and must accept 1 parameter
+  def job_result_format(job_id, format, io=nil, &block)
+    if io
+      code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>format}) {|res|
+        if res.code != "200"
+          raise_error("Get job result failed", res)
+        end
+
+        if ce = res.header['Content-Encoding']
+          res.extend(DeflateReadBodyMixin)
+          res.gzip = true if ce == 'gzip'
+        else
+          res.extend(DirectReadBodyMixin)
+        end
+
+        res.extend(DirectReadBodyMixin)
+        if ce = res.header['Content-Encoding']
+          if ce == 'gzip'
+            infl = Zlib::Inflate.new(Zlib::MAX_WBITS + 16)
+          else
+            infl = Zlib::Inflate.new
+          end
+        end
+
+        total_compr_size = 0
+        res.each_fragment {|fragment|
+          total_compr_size += fragment.size
+          # uncompressed if the 'Content-Enconding' header is set in response
+          fragment = infl.inflate(fragment) if ce
+          io.write(fragment)
+          block.call(total_compr_size) if block_given?
+        }
+      }
+      nil
+    else
+      code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>format})
+      if res.code != "200"
+        raise_error("Get job result failed", res)
+      end
+      body
+    end
+  end
+
+  # block is optional and must accept 1 argument
+  def job_result_each(job_id, &block)
+    get("/v3/job/result/#{e job_id}", {'format'=>'msgpack'}) {|res|
+      if res.code != "200"
+        raise_error("Get job result failed", res)
+      end
+
+      # default to decompressing the response since format is fixed to 'msgpack'
+      res.extend(DeflateReadBodyMixin)
+      res.gzip = (res.header['Content-Encoding'] == 'gzip')
+      upkr = MessagePack::Unpacker.new
+      res.each_fragment {|inflated_fragment|
+        upkr.feed_each(inflated_fragment, &block)
+      }
+    }
+    nil
+  end
+
+  # block is optional and must accept 1 argument
+  def job_result_each_with_compr_size(job_id, &block)
+    get("/v3/job/result/#{e job_id}", {'format'=>'msgpack'}) {|res|
+      if res.code != "200"
+        raise_error("Get job result failed", res)
+      end
+
+      res.extend(DirectReadBodyMixin)
+      if res.header['Content-Encoding'] == 'gzip'
+        infl = Zlib::Inflate.new(Zlib::MAX_WBITS + 16)
+      else
+        infl = Zlib::Inflate.new
+      end
+      upkr = MessagePack::Unpacker.new
+      begin
+        total_compr_size = 0
+        res.each_fragment {|fragment|
+          total_compr_size += fragment.size
+          upkr.feed_each(infl.inflate(fragment)) {|unpacked|
+            block.call(unpacked, total_compr_size) if block_given?
+          }
+        }
+      ensure
+        infl.close
+      end
+    }
+    nil
+  end
+
+  def job_result_raw(job_id, format)
+    code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>format})
+    if code != "200"
+      raise_error("Get job result failed", res)
+    end
+    return body
+  end
+
+  def kill(job_id)
+    code, body, res = post("/v3/job/kill/#{e job_id}")
+    if code != "200"
+      raise_error("Kill job failed", res)
+    end
+    js = checked_json(body, %w[])
+    former_status = js['former_status']
+    return former_status
+  end
+
+  # => jobId:String
+  def hive_query(q, db=nil, result_url=nil, priority=nil, retry_limit=nil, opts={})
+    query(q, :hive, db, result_url, priority, retry_limit, opts)
+  end
+
+  # => jobId:String
+  def pig_query(q, db=nil, result_url=nil, priority=nil, retry_limit=nil, opts={})
+    query(q, :pig, db, result_url, priority, retry_limit, opts)
+  end
+
+  # => jobId:String
+  def query(q, type=:hive, db=nil, result_url=nil, priority=nil, retry_limit=nil, opts={})
+    params = {'query' => q}.merge(opts)
+    params['result'] = result_url if result_url
+    params['priority'] = priority if priority
+    params['retry_limit'] = retry_limit if retry_limit
+    code, body, res = post("/v3/job/issue/#{type}/#{e db}", params)
+    if code != "200"
+      raise_error("Query failed", res)
+    end
+    js = checked_json(body, %w[job_id])
+    return js['job_id'].to_s
+  end
+
+end
+end