td-client 0.8.68 → 0.8.69

data/lib/td/client/api/bulk_load.rb

@@ -0,0 +1,205 @@
+require 'td/client/api/to_hash_struct'
+
+class TreasureData::API
+module BulkLoad
+
+  ####
+  ## BulkLoad (Server-side Bulk loader) API
+  ##
+
+  #  1. POST   /v3/bulk_loads/guess          - stateless non REST API to return guess result as BulkLoadSession [NEW]
+  #  2. POST   /v3/bulk_loads/preview        - stateless non REST API to return preview result as BulkLoadSession [NEW]
+  #
+  #  3. POST   /v3/job/issue/:type/:database - create a job resource to run BulkLoadSession [EXTENDED]
+  #  4. POST   /v3/job/kill/:id              - kill the job [ALREADY EXISTS]
+  #  5. GET    /v3/job/show/:id              - get status of the job [ALREADY EXISTS]
+  #  6. GET    /v3/job/result/:id            - get result of the job [NOT NEEDED IN Q4] ... because backend feature is not yet implemented
+  #
+  #  7. GET    /v3/bulk_loads                - list BulkLoadSession resources [NEW]
+  #  8. POST   /v3/bulk_loads                - create BulkLoadSession [NEW]
+  #  9. GET    /v3/bulk_loads/:name          - get BulkLoadSession [NEW]
+  # 10. PUT    /v3/bulk_loads/:name          - update BulkLoadSession [NEW]
+  # 11. DELETE /v3/bulk_loads/:name          - delete BulkLoadSession [NEW]
+  # 12. GET    /v3/bulk_loads/:name/jobs     - list BulkLoadSession job history [NEW]
+  # 13. POST   /v3/bulk_loads/:name/jobs     - run BulkLoadSession [NEW]
+
+  # The 'BulkLoadSession' resource in td-api is as follows;
+  # {
+  #   "config": {
+  #     "type": "s3",
+  #     "access_key_id": s3 access key id,
+  #     "secret_access_key": s3 secret key,
+  #     "endpoint": s3 endpoint name,
+  #     "bucket": s3 bucket name,
+  #     "path_prefix": "a/prefix/of/files",
+  #     "decoders": []
+  #   },
+  #   "name": account_wide_unique_name,
+  #   "cron": cron_string,
+  #   "timezone": timezone_string,
+  #   "delay": delay_seconds,
+  #   "database": database_name,
+  #   "table": table_name
+  # }
+
+  ## Resource definitions
+
+  class BulkLoad < ToHashStruct.new(:config, :name, :cron, :timezone, :delay, :time_column, :database, :table)
+    class BulkLoadSessionConfig < ToHashStruct.new(:type, :access_key_id, :secret_access_key, :endpoint, :bucket, :path_prefix, :parser, :decoders)
+      def validate_self
+        validate_presence_of :type
+      end
+    end
+
+    model_property :config, BulkLoadSessionConfig
+
+    def validate_self
+      validate_presence_of :config
+    end
+  end
+
+  class BulkLoadPreview < ToHashStruct.new(:schema, :records)
+  end
+
+  class Job < ToHashStruct.new(:job_id, :account_id, :type, :status, :cpu_time, :config, :records, :schema, :database, :table, :priority, :created_at, :updated_at, :start_at, :end_at)
+    model_property :config, BulkLoad::BulkLoadSessionConfig
+  end
+
+  ## API definitions
+
+  LIST = '/v3/bulk_loads'
+  SESSION = LIST +     '/%s'
+  JOB = SESSION +         '/jobs'
+
+  # job: BulkLoad -> BulkLoad
+  def bulk_load_guess(job)
+    # retry_request = true
+    path = LIST + '/guess'
+    res = api { post(path, job.validate.to_json) }
+    unless res.ok?
+      raise_error('BulkLoad configuration guess failed', res)
+    end
+    BulkLoad.from_json(res.body)
+  end
+
+  # job: BulkLoad -> BulkLoadPreview
+  def bulk_load_preview(job)
+    # retry_request = true
+    path = LIST + '/preview'
+    res = api { post(path, job.validate.to_json) }
+    unless res.ok?
+      raise_error('BulkLoad job preview failed', res)
+    end
+    BulkLoadPreview.from_json(res.body)
+  end
+
+  # job: BulkLoad -> String (job_id)
+  def bulk_load_issue(database, table, job)
+    type = 'bulkload'
+    job = job.dup
+    job['database'] = database
+    job['table'] = table
+    path = "/v3/job/issue/#{e type}/#{e database}"
+    res = api { post(path, job.validate.to_json) }
+    unless res.ok?
+      raise_error('BulkLoad job issuing failed', res)
+    end
+    js = checked_json(res.body)
+    js['job_id'].to_s
+  end
+
+  # nil -> [BulkLoad]
+  def bulk_load_list
+    res = api { get(LIST) }
+    unless res.ok?
+      raise_error("BulkLoadSession list retrieve failed", res)
+    end
+    to_ary(res, BulkLoad)
+  end
+
+  # name: String, database: String, table: String, job: BulkLoad -> BulkLoad
+  def bulk_load_create(name, database, table, job, opts = {})
+    job = job.dup
+    job['name'] = name
+    [:cron, :timezone, :delay, :time_column].each do |prop|
+      job[prop.to_s] = opts[prop] if opts.key?(prop)
+    end
+    job['database'] = database
+    job['table'] = table
+    res = api { post(LIST, job.validate.to_json) }
+    unless res.ok?
+      raise_error("BulkLoadSession: #{name} create failed", res)
+    end
+    BulkLoad.from_json(res.body)
+  end
+
+  # name: String -> BulkLoad
+  def bulk_load_show(name)
+    path = session_path(name)
+    res = api { get(path) }
+    unless res.ok?
+      raise_error("BulkLoadSession: #{name} retrieve failed", res)
+    end
+    BulkLoad.from_json(res.body)
+  end
+
+  # name: String, job: BulkLoad -> BulkLoad
+  def bulk_load_update(name, job)
+    path = session_path(name)
+    res = api { put(path, job.validate.to_json) }
+    unless res.ok?
+      raise_error("BulkLoadSession: #{name} update failed", res)
+    end
+    BulkLoad.from_json(res.body)
+  end
+
+  # name: String -> BulkLoad
+  def bulk_load_delete(name)
+    path = session_path(name)
+    res = api { delete(path) }
+    unless res.ok?
+      raise_error("BulkLoadSession: #{name} delete failed", res)
+    end
+    BulkLoad.from_json(res.body)
+  end
+
+  # name: String -> [Job]
+  def bulk_load_history(name)
+    path = job_path(name)
+    res = api { get(path) }
+    unless res.ok?
+      raise_error("history of BulkLoadSession: #{name} retrieve failed", res)
+    end
+    to_ary(res, Job)
+  end
+
+  def bulk_load_run(name, scheduled_time = nil)
+    path = job_path(name)
+    opts = {}
+    opts[:scheduled_time] = scheduled_time unless scheduled_time.nil?
+    res = api { post(path, opts) }
+    unless res.ok?
+      raise_error("BulkLoadSession: #{name} job create failed", res)
+    end
+    js = checked_json(res.body)
+    js['job_id'].to_s
+  end
+
+private
+
+  def session_path(name)
+    SESSION % e(name)
+  end
+
+  def job_path(name)
+    JOB % e(name)
+  end
+
+  def to_ary(res, klass)
+    JSON.parse(res.body).map { |bulk_load|
+      klass.from_hash(bulk_load)
+    }
+  end
+
+end
+end

data/lib/td/client/api/database.rb

@@ -5,7 +5,7 @@ module Database
   ## Database API
   ##
 
-  # => [name:String]
+  # @return [Array<String>] names as array
   def list_databases
     code, body, res = get("/v3/database/list")
     if code != "200"
@@ -24,7 +24,8 @@ module Database
     return result
   end
 
-  # => true
+  # @param [String] db
+  # @return [true]
   def delete_database(db)
     code, body, res = post("/v3/database/delete/#{e db}")
     if code != "200"
@@ -33,7 +34,9 @@ module Database
     return true
   end
 
-  # => true
+  # @param [String] db
+  # @param [Hash] opts
+  # @return [true]
   def create_database(db, opts={})
     params = opts.dup
     code, body, res = post("/v3/database/create/#{e db}", params)

data/lib/td/client/api/export.rb

@@ -6,6 +6,11 @@ module Export
   ##
 
   # => jobId:String
+  # @param [String] db
+  # @param [String] table
+  # @param [String] storage_type
+  # @param [Hash] opts
+  # @return [String] job_id
   def export(db, table, storage_type, opts={})
     params = opts.dup
     params['storage_type'] = storage_type

data/lib/td/client/api/import.rb

@@ -5,7 +5,13 @@ module Import
   ## Import API
   ##
 
-  # => time:Float
+  # @param [String] db
+  # @param [String] table
+  # @param [String] format
+  # @param [String, StringIO] stream
+  # @param [Fixnum] size
+  # @param [String] unique_id
+  # @return [Float] elapsed time
   def import(db, table, format, stream, size, unique_id=nil)
     if unique_id
       path = "/v3/table/import_with_id/#{e db}/#{e table}/#{unique_id}/#{format}"

data/lib/td/client/api/job.rb

@@ -5,7 +5,11 @@ module Job
   ## Job API
   ##
 
-  # => [(jobId:String, type:Symbol, status:String, start_at:String, end_at:String, result_url:String)]
+  # @param [Fixnum] from
+  # @param [Fixnum] to
+  # @param [String] status
+  # @param [Hash] conditions
+  # @return [Array]
   def list_jobs(from=0, to=nil, status=nil, conditions=nil)
     params = {}
     params['from'] = from.to_s if from
@@ -37,7 +41,8 @@ module Job
     return result
   end
 
-  # => (type:Symbol, status:String, result:String, url:String, result:String)
+  # @param [String] job_id
+  # @return [Array]
   def show_job(job_id)
     # use v3/job/status instead of v3/job/show to poll finish of a job
     code, body, res = get("/v3/job/show/#{e job_id}")
@@ -93,6 +98,8 @@ module Job
             result_size, result, hive_result_schema, priority, retry_limit, nil, database]
   end
 
+  # @param [String] job_id
+  # @return [String] HTTP status
   def job_status(job_id)
     code, body, res = get("/v3/job/status/#{e job_id}")
     if code != "200"
@@ -103,6 +110,8 @@ module Job
     return js['status']
   end
 
+  # @param [String] job_id
+  # @return [Array]
   def job_result(job_id)
     code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>'msgpack'})
     if code != "200"
@@ -116,6 +125,12 @@ module Job
   end
 
   # block is optional and must accept 1 parameter
+  #
+  # @param [String] job_id
+  # @param [String] format
+  # @param [IO] io
+  # @param [Proc] block
+  # @return [nil, String]
   def job_result_format(job_id, format, io=nil, &block)
     if io
       code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>format}) {|res|
@@ -159,6 +174,10 @@ module Job
   end
 
   # block is optional and must accept 1 argument
+  # 
+  # @param [String] job_id
+  # @param [Proc] block
+  # @return [nil]
   def job_result_each(job_id, &block)
     get("/v3/job/result/#{e job_id}", {'format'=>'msgpack'}) {|res|
       if res.code != "200"
@@ -177,6 +196,10 @@ module Job
   end
 
   # block is optional and must accept 1 argument
+  # 
+  # @param [String] job_id
+  # @param [Proc] block
+  # @return [nil]
   def job_result_each_with_compr_size(job_id, &block)
     get("/v3/job/result/#{e job_id}", {'format'=>'msgpack'}) {|res|
       if res.code != "200"
@@ -205,6 +228,9 @@ module Job
     nil
   end
 
+  # @param [String] job_id
+  # @param [String] format
+  # @return [String]
   def job_result_raw(job_id, format)
     code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>format})
     if code != "200"
@@ -213,6 +239,8 @@ module Job
     return body
   end
 
+  # @param [String] job_id
+  # @return [String]
   def kill(job_id)
     code, body, res = post("/v3/job/kill/#{e job_id}")
     if code != "200"
@@ -223,17 +251,33 @@ module Job
     return former_status
   end
 
-  # => jobId:String
+  # @param [String] q
+  # @param [String] db
+  # @param [String] result_url
+  # @param [Fixnum] priority
+  # @param [Hash] opts
+  # @return [String] job_id
   def hive_query(q, db=nil, result_url=nil, priority=nil, retry_limit=nil, opts={})
     query(q, :hive, db, result_url, priority, retry_limit, opts)
   end
 
-  # => jobId:String
+  # @param [String] q
+  # @param [String] db
+  # @param [String] result_url
+  # @param [Fixnum] priority
+  # @param [Hash] opts
+  # @return [String] job_id
   def pig_query(q, db=nil, result_url=nil, priority=nil, retry_limit=nil, opts={})
     query(q, :pig, db, result_url, priority, retry_limit, opts)
   end
 
-  # => jobId:String
+  # @param [String] q
+  # @param [Symbol] type
+  # @param [String] db
+  # @param [String] result_url
+  # @param [Fixnum] priority
+  # @param [Hash] opts
+  # @return [String] job_id
   def query(q, type=:hive, db=nil, result_url=nil, priority=nil, retry_limit=nil, opts={})
     params = {'query' => q}.merge(opts)
     params['result'] = result_url if result_url

data/lib/td/client/api/partial_delete.rb

@@ -5,6 +5,12 @@ module PartialDelete
   ## Partial delete API
   ##
 
+  # @param [String] db
+  # @param [String] table
+  # @param [Fixnum] to
+  # @param [Fixnum] from
+  # @param [Hash] opts
+  # @return [String]
   def partial_delete(db, table, to, from, opts={})
     params = opts.dup
     params['to'] = to.to_s

data/lib/td/client/api/result.rb

@@ -5,6 +5,7 @@ module Result
   ## Result API
   ##
 
+  # @return [Array<String>]
   def list_result
     code, body, res = get("/v3/result/list")
     if code != "200"
@@ -18,7 +19,10 @@ module Result
     return result
   end
 
-  # => true
+  # @param [String] name
+  # @param [String] url
+  # @param [Hash] opts
+  # @return [true]
   def create_result(name, url, opts={})
     params = {'url'=>url}.merge(opts)
     code, body, res = post("/v3/result/create/#{e name}", params)
@@ -28,7 +32,8 @@ module Result
     return true
   end
 
-  # => true
+  # @param [String] name
+  # @return [true]
   def delete_result(name)
     code, body, res = post("/v3/result/delete/#{e name}")
     if code != "200"

data/lib/td/client/api/schedule.rb

@@ -5,7 +5,9 @@ module Schedule
   ## Schedule API
   ##
 
-  # => start:String
+  # @param [String] name
+  # @param [Hash] opts
+  # @return [String]
   def create_schedule(name, opts)
     params = opts.update({:type=> opts[:type] || opts['type'] || 'hive'})
     code, body, res = post("/v3/schedule/create/#{e name}", params)
@@ -16,7 +18,8 @@ module Schedule
     return js['start']
   end
 
-  # => cron:String, query:String
+  # @param [String] name
+  # @return [Array]
   def delete_schedule(name)
     code, body, res = post("/v3/schedule/delete/#{e name}")
     if code != "200"
@@ -26,7 +29,7 @@ module Schedule
     return js['cron'], js["query"]
   end
 
-  # => [(name:String, cron:String, query:String, database:String, result_url:String)]
+  # @return [Array]
   def list_schedules
     code, body, res = get("/v3/schedule/list")
     if code != "200"
@@ -50,6 +53,9 @@ module Schedule
     return result
   end
 
+  # @param [String] name
+  # @param [Hash] params
+  # @return [nil]
   def update_schedule(name, params)
     code, body, res = post("/v3/schedule/update/#{e name}", params)
     if code != "200"
@@ -58,6 +64,10 @@ module Schedule
     return nil
   end
 
+  # @param [String] name
+  # @param [Fixnum] from
+  # @param [Fixnum] to
+  # @return [Array]
   def history(name, from=0, to=nil)
     params = {}
     params['from'] = from.to_s if from
@@ -84,6 +94,10 @@ module Schedule
     return result
   end
 
+  # @param [String] name
+  # @param [String] time
+  # @param [Fixnum] num
+  # @return [Array]
   def run_schedule(name, time, num)
     params = {}
     params = {'num' => num} if num