td-client 0.8.62 → 0.8.63

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NDRiYWJjYzc1NDFiOTQzMzBkNjNmMGUwNDcxMDAzOWQ5Njg0NzVmNA==
+    MDBjNmRmZmY5MjZkMGIxY2ZmZjEzMzU3NjhlNDMxYTBhMDc0YzUxMw==
   data.tar.gz: !binary |-
-    Y2JkMzE0NTBhNWQ4YzZiZjY0MDY2ZmY4MmE2YmY4NWYzYmY2N2Q1NA==
+    MjE5ZTgwOTI1OGM4NjEyODdjMDNlYmM5NmQ4NGU5ZTVlYmNkMDMwYg==
 SHA512:
   metadata.gz: !binary |-
-    NTllMjQzNDUyOGVkNWM1MWMxNDY5YWMwOGE4ODNjM2M5N2ZjYjZhYWUzZWQ1
-    YTFlMmMyMzNkNjExZGFjOWUyMzMxODM3ZGQ4YjQ2ZGUzYWZhYzEwY2M3OWE4
-    ODA5ODI2NTQzNGE4ZDM0OTdiMzNjOGNlODFhNzZlNDU3YTY4M2Q=
+    ZDZmNjY2OTkyODExOWUwYmMyNDE0ZGEwMmU5MzU4OTkzMmYzZTMwMzY3MzEw
+    ZjIxZjMyYTkxZDlmYWFjODEwODA3ZmU0MDc0NjE1NzQzMDkxMWViYTQxZWE5
+    YzRlYzRlOGYwYmYzZTlmNDgzYzQ4YmUzYzAxZDVjYjY4N2VjYzk=
   data.tar.gz: !binary |-
-    MjVhNmU2MmI5NDNlZDliNDU1YzQzNjdmYzkwYWQ0YWU2ZWRjMWJjMDE0ZTdl
-    ZDgyMGYxY2Y3OTI4NmM4MDcwODJjMmJkNmE2NThiMGFkYzEyM2VjNTc0NzNh
-    MjQyMWI2MTFiOGIxYTgzNmEzZTEwYjc0NTNmOTVjYjcwZGMwNDk=
+    Zjk1ZDViZGZkOGI2YmE4ZGM2N2ZjMzFkNmNhMDNmMzFkZDY1YzAwYTU1ZmU5
+    MjUwYjRlNjhkYmM5ZmQwZWNmYjM4YmExZDllNzhlZWMwNjZjMjhiZTk5YjVl
+    NzA1NTkyM2FkZjk3ZGQ4ZjRmOWYzNmFmZTMwMTgwM2NlN2YzNGQ=

data/lib/td/client.rb

@@ -1,4 +1,3 @@
-
 module TreasureData
 
 require 'td/client/api'
@@ -55,17 +54,17 @@ class Client
   # => [Database]
   def databases
     m = @api.list_databases
-    m.map {|db_name,(count,created_at,updated_at,org)|
-      Database.new(self, db_name, nil, count, created_at, updated_at, org)
+    m.map {|db_name,(count, created_at, updated_at, org, permission)|
+      Database.new(self, db_name, nil, count, created_at, updated_at, org, permission)
     }
   end
 
   # => Database
   def database(db_name)
     m = @api.list_databases
-    m.each {|name,(count,created_at,updated_at,org)|
+    m.each {|name,(count, created_at, updated_at, org, permission)|
       if name == db_name
-        return Database.new(self, name, nil, count, created_at, updated_at, org)
+        return Database.new(self, name, nil, count, created_at, updated_at, org, permission)
       end
     }
     raise NotFoundError, "Database '#{db_name}' does not exist"
@@ -129,24 +128,28 @@ class Client
   def query(db_name, q, result_url=nil, priority=nil, retry_limit=nil, opts={})
     # for compatibility, assume type is hive unless specifically specified
     type = opts[:type] || opts['type'] || :hive
-    raise ArgumentError, "The specified query type is not supported: #{type}" unless [:hive, :pig, :impala, :presto].include? type
+    raise ArgumentError, "The specified query type is not supported: #{type}" unless [:hive, :pig, :impala, :presto].include?(type)
     job_id = @api.query(q, type, db_name, result_url, priority, retry_limit, opts)
     Job.new(self, job_id, type, q)
   end
 
   # => [Job]
   def jobs(from=nil, to=nil, status=nil, conditions=nil)
-    result = @api.list_jobs(from, to, status, conditions)
-    result.map {|job_id, type, status, query, start_at, end_at, cpu_time, result_url, priority, retry_limit, org, db|
-      Job.new(self, job_id, type, query, status, nil, nil, start_at, end_at, cpu_time, nil, result_url, nil, priority, retry_limit, org, db)
+    results = @api.list_jobs(from, to, status, conditions)
+    results.map {|job_id, type, status, query, start_at, end_at, cpu_time,
+                 result_size, result_url, priority, retry_limit, org, db|
+      Job.new(self, job_id, type, query, status, nil, nil, start_at, end_at, cpu_time,
+              result_size, nil, result_url, nil, priority, retry_limit, org, db)
     }
   end
 
   # => Job
   def job(job_id)
     job_id = job_id.to_s
-    type, query, status, url, debug, start_at, end_at, cpu_time, result_url, hive_result_schema, priority, retry_limit, org, db = @api.show_job(job_id)
-    Job.new(self, job_id, type, query, status, url, debug, start_at, end_at, cpu_time, nil, result_url, hive_result_schema, priority, retry_limit, org, db)
+    type, query, status, url, debug, start_at, end_at, cpu_time,
+      result_size, result_url, hive_result_schema, priority, retry_limit, org, db = @api.show_job(job_id)
+    Job.new(self, job_id, type, query, status, url, debug, start_at, end_at, cpu_time,
+            result_size, nil, result_url, hive_result_schema, priority, retry_limit, org, db)
   end
 
   # => status:String
@@ -160,8 +163,8 @@ class Client
   end
 
   # => result:String
-  def job_result_format(job_id, format, io=nil)
-    @api.job_result_format(job_id, format, io)
+  def job_result_format(job_id, format, io=nil, &block)
+    @api.job_result_format(job_id, format, io, &block)
   end
 
   # => nil
@@ -169,6 +172,11 @@ class Client
     @api.job_result_each(job_id, &block)
   end
 
+  # => nil
+  def job_result_each_with_compr_size(job_id, &block)
+    @api.job_result_each_with_compr_size(job_id, &block)
+  end
+
   # => former_status:String
   def kill(job_id)
     @api.kill(job_id)
@@ -351,7 +359,7 @@ class Client
 
   # => true
   def remove_apikey(user, apikey)
-    @api.remove_apikey(user, apikey)
+    @api.remove_apikey(user, g)
   end
 
   # => true
@@ -388,5 +396,4 @@ class Client
   end
 end
 
-
-end
+end # module TreasureData

data/lib/td/client/api.rb

@@ -2,19 +2,26 @@ require 'td/client/version'
 
 module TreasureData
 
+class ParameterValidationError < StandardError
+end
 
+# Generic API error
 class APIError < StandardError
 end
 
-class ParameterValidationError < StandardError
+# 401 API errors
+class AuthError < APIError
 end
 
-class AuthError < APIError
+# 403 API errors, used for database permissions
+class ForbiddenError < APIError
 end
 
+# 409 API errors
 class AlreadyExistsError < APIError
 end
 
+# 404 API errors
 class NotFoundError < APIError
 end
 
@@ -154,8 +161,8 @@ class API
     if name.length < 3
       name += "_" * (3 - name.length)
     end
-    if 256 < name.length
-      name = name[0,254] + "__"
+    if 255 < name.length
+      name = name[0, 253] + "__"
     end
     name = name.downcase
     name = name.gsub(/[^a-z0-9_]/, '_')
@@ -248,7 +255,8 @@ class API
       count = m['count']
       created_at = m['created_at']
       updated_at = m['updated_at']
-      result[name] = [count, created_at, updated_at, nil] # set nil to org for API compatibiilty
+      permission = m['permission']
+      result[name] = [count, created_at, updated_at, nil, permission] # set nil to org for API compatibiilty
     }
     return result
   end
@@ -419,10 +427,12 @@ class API
       start_at = m['start_at']
       end_at = m['end_at']
       cpu_time = m['cpu_time']
+      result_size = m['result_size'] # compressed result size in msgpack.gz format
       result_url = m['result']
       priority = m['priority']
       retry_limit = m['retry_limit']
-      result << [job_id, type, status, query, start_at, end_at, cpu_time, result_url, priority, retry_limit, nil, database] # same as database
+      result << [job_id, type, status, query, start_at, end_at, cpu_time,
+                 result_size, result_url, priority, retry_limit, nil, database]
     }
     return result
   end
@@ -445,7 +455,8 @@ class API
     start_at = js['start_at']
     end_at = js['end_at']
     cpu_time = js['cpu_time']
-    result = js['result'] # result URL
+    result_size = js['result_size'] # compressed result size in msgpack.gz format
+    result = js['result'] # result target URL
     hive_result_schema = (js['hive_result_schema'] || '')
     if hive_result_schema.empty?
       hive_result_schema = nil
@@ -478,8 +489,8 @@ class API
     end
     priority = js['priority']
     retry_limit = js['retry_limit']
-    return [type, query, status, url, debug, start_at, end_at, cpu_time, result,
-            hive_result_schema, priority, retry_limit, nil, database]
+    return [type, query, status, url, debug, start_at, end_at, cpu_time,
+            result_size, result, hive_result_schema, priority, retry_limit, nil, database]
   end
 
   def job_status(job_id)
@@ -505,14 +516,38 @@ class API
     return result
   end
 
-  def job_result_format(job_id, format, io=nil)
+  # block is optional and must accept 1 parameter
+  def job_result_format(job_id, format, io=nil, &block)
     if io
       code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>format}) {|res|
         if res.code != "200"
           raise_error("Get job result failed", res)
         end
+
+        if ce = res.header['Content-Encoding']
+          require 'zlib'
+          res.extend(DeflateReadBodyMixin)
+          res.gzip = true if ce == 'gzip'
+        else
+          res.extend(DirectReadBodyMixin)
+        end
+
+        res.extend(DirectReadBodyMixin)
+        if ce = res.header['Content-Encoding']
+          if ce == 'gzip'
+            infl = Zlib::Inflate.new(Zlib::MAX_WBITS + 16)
+          else
+            infl = Zlib::Inflate.new
+          end
+        end
+
+        total_compr_size = 0
         res.each_fragment {|fragment|
+          total_compr_size += fragment.size
+          # uncompressed if the 'Content-Enconding' header is set in response
+          fragment = infl.inflate(fragment) if ce
           io.write(fragment)
+          block.call(total_compr_size) if block_given?
         }
       }
       nil
@@ -525,20 +560,57 @@ class API
     end
   end
 
+  # block is optional and must accept 1 argument
   def job_result_each(job_id, &block)
     require 'msgpack'
     get("/v3/job/result/#{e job_id}", {'format'=>'msgpack'}) {|res|
       if res.code != "200"
         raise_error("Get job result failed", res)
       end
-      u = MessagePack::Unpacker.new
-      res.each_fragment {|fragment|
-        u.feed_each(fragment, &block)
+
+      # default to decompressing the response since format is fixed to 'msgpack'
+      res.extend(DeflateReadBodyMixin)
+      res.gzip = (res.header['Content-Encoding'] == 'gzip')
+      upkr = MessagePack::Unpacker.new
+      res.each_fragment {|inflated_fragment|
+        upkr.feed_each(inflated_fragment, &block)
       }
     }
     nil
   end
 
+  # block is optional and must accept 1 argument
+  def job_result_each_with_compr_size(job_id, &block)
+    require 'zlib'
+    require 'msgpack'
+
+    get("/v3/job/result/#{e job_id}", {'format'=>'msgpack'}) {|res|
+      if res.code != "200"
+        raise_error("Get job result failed", res)
+      end
+
+      res.extend(DirectReadBodyMixin)
+      if res.header['Content-Encoding'] == 'gzip'
+        infl = Zlib::Inflate.new(Zlib::MAX_WBITS + 16)
+      else
+        infl = Zlib::Inflate.new
+      end
+      upkr = MessagePack::Unpacker.new
+      begin
+        total_compr_size = 0
+        res.each_fragment {|fragment|
+          total_compr_size += fragment.size
+          upkr.feed_each(infl.inflate(fragment)) {|unpacked|
+            block.call(unpacked, total_compr_size) if block_given?
+          }
+        }
+      ensure
+        infl.close
+      end
+    }
+    nil
+  end
+
   def job_result_raw(job_id, format)
     code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>format})
     if code != "200"
@@ -816,7 +888,7 @@ class API
   end
 
   def update_schedule(name, params)
-    code, body, res = get("/v3/schedule/update/#{e name}", params)
+    code, body, res = post("/v3/schedule/update/#{e name}", params)
     if code != "200"
       raise_error("Update schedule failed", res)
     end
@@ -1116,21 +1188,20 @@ class API
     return status
   end
 
-
   private
   module DeflateReadBodyMixin
     attr_accessor :gzip
 
     def each_fragment(&block)
       if @gzip
-        infl = Zlib::Inflate.new(Zlib::MAX_WBITS+16)
+        infl = Zlib::Inflate.new(Zlib::MAX_WBITS + 16)
       else
         infl = Zlib::Inflate.new
       end
       begin
-        read_body do |fragment|
+        read_body {|fragment|
           block.call infl.inflate(fragment)
-        end
+        }
       ensure
         infl.close
       end
@@ -1165,16 +1236,9 @@ class API
     end
 
     if block
-      response = http.request(request) do |res|
-        if ce = res.header['Content-Encoding']
-          require 'zlib'
-          res.extend(DeflateReadBodyMixin)
-          res.gzip = true if ce == 'gzip'
-        else
-          res.extend(DirectReadBodyMixin)
-        end
+      response = http.request(request) {|res|
         block.call(res)
-      end
+      }
     else
       response = http.request(request)
     end
@@ -1190,7 +1254,7 @@ class API
       if ce = response.header['content-encoding']
         require 'zlib'
         if ce == 'gzip'
-          infl = Zlib::Inflate.new(Zlib::MAX_WBITS+16)
+          infl = Zlib::Inflate.new(Zlib::MAX_WBITS + 16)
           begin
             body = infl.inflate(body)
           ensure
@@ -1330,6 +1394,8 @@ class API
         raise AlreadyExistsError, "#{msg}: #{error_msg}"
       elsif status_code == "401"
         raise AuthError, "#{msg}: #{error_msg}"
+      elsif status_code == "403"
+        raise ForbiddenError, "#{msg}: #{error_msg}"
       else
         raise APIError, "#{status_code}: #{msg}: #{error_msg}"
       end
@@ -1343,6 +1409,8 @@ class API
         raise AlreadyExistsError, "#{msg}: #{res.body}"
       elsif status_code == "401"
         raise AuthError, "#{msg}: #{res.body}"
+      elsif status_code == "403"
+        raise ForbiddenError, "#{msg}: #{res.body}"
       else
         raise APIError, "#{status_code}: #{msg}: #{res.body}"
       end
@@ -1374,6 +1442,4 @@ class API
   end
 end
 
-
-end
-
+end # module TreasureData

data/lib/td/client/model.rb

@@ -28,29 +28,33 @@ class Account < Model
   end
 
   def storage_size_string
-    if @storage_size <= 1024*1024
+    if @storage_size <= 1024 * 1024
       return "0.0 GB"
-    elsif @storage_size <= 60*1024*1024
+    elsif @storage_size <= 60 * 1024 * 1024
       return "0.01 GB"
-    elsif @storage_size <= 60*1024*1024*1024
-      "%.1f GB" % (@storage_size.to_f / (1024*1024*1024))
+    elsif @storage_size <= 60 * 1024 * 1024 * 1024
+      "%.1f GB" % (@storage_size.to_f / (1024 * 1024 * 1024))
     else
-      "%d GB" % (@storage_size.to_f / (1024*1024*1024)).to_i
+      "%d GB" % (@storage_size.to_f / (1024 * 1024 * 1024)).to_i
     end
   end
 end
 
 class Database < Model
-  def initialize(client, db_name, tables=nil, count=nil, created_at=nil, updated_at=nil, org_name=nil)
+  PERMISSIONS = [:administrator, :full_access, :import_only, :query_only]
+  PERMISSION_LIST_TABLES = [:administrator, :full_access]
+
+  def initialize(client, db_name, tables=nil, count=nil, created_at=nil, updated_at=nil, org_name=nil, permission=nil)
     super(client)
     @db_name = db_name
     @tables = tables
     @count = count
     @created_at = created_at
     @updated_at = updated_at
+    @permission = permission.to_sym
   end
 
-  attr_reader :org_name
+  attr_reader :org_name, :permission, :count
 
   def name
     @db_name
@@ -81,8 +85,6 @@ class Database < Model
     @client.query(@db_name, q)
   end
 
-  attr_reader :count
-
   def created_at
     @created_at && !@created_at.empty? ? Time.parse(@created_at) : nil
   end
@@ -93,12 +95,19 @@ class Database < Model
 
   def update_tables!
     @tables = @client.tables(@db_name)
+    # provide Table objects with a reference to the parent Database to avoid
+    # requesting the Database information (such as permission) every time
+    @tables.each {|table|
+      table.database = self
+    }
   end
+
 end
 
 class Table < Model
   def initialize(client, db_name, table_name, type, schema, count, created_at=nil, updated_at=nil, estimated_storage_size=nil, last_import=nil, last_log_timestamp=nil, expire_days=nil, primary_key=nil, primary_key_type=nil)
     super(client)
+    @database = nil
     @db_name = db_name
     @table_name = table_name
     @type = type
@@ -119,6 +128,10 @@ class Table < Model
   alias database_name db_name
   alias name table_name
 
+  def database=(database)
+    @database = database if database.instance_of?(Database)
+  end
+
   def created_at
     @created_at && !@created_at.empty? ? Time.parse(@created_at) : nil
   end
@@ -140,7 +153,13 @@ class Table < Model
   end
 
   def database
-    @client.database(@db_name)
+    update_database! unless @database
+    @database
+  end
+
+  # get the database's permission as if they were the table's
+  def permission
+    database.permission
   end
 
   def identifier
@@ -174,6 +193,10 @@ class Table < Model
       "%d GB" % (@estimated_storage_size.to_f / (1024*1024*1024)).to_i
     end
   end
+
+  def update_database!
+    @database = @client.database(@db_name)
+  end
 end
 
 class Schema
@@ -237,7 +260,9 @@ class Job < Model
   STATUS_KILLED = "killed"
   FINISHED_STATUS = [STATUS_SUCCESS, STATUS_ERROR, STATUS_KILLED]
 
-  def initialize(client, job_id, type, query, status=nil, url=nil, debug=nil, start_at=nil, end_at=nil, cpu_time=nil, result=nil, result_url=nil, hive_result_schema=nil, priority=nil, retry_limit=nil, org_name=nil, db_name=nil)
+  def initialize(client, job_id, type, query, status=nil, url=nil, debug=nil, start_at=nil, end_at=nil, cpu_time=nil,
+                 result_size=nil, result=nil, result_url=nil, hive_result_schema=nil, priority=nil, retry_limit=nil,
+                 org_name=nil, db_name=nil)
     super(client)
     @job_id = job_id
     @type = type
@@ -248,6 +273,7 @@ class Job < Model
     @start_at = start_at
     @end_at = end_at
     @cpu_time = cpu_time
+    @result_size = result_size
     @result = result
     @result_url = result_url
     @hive_result_schema = hive_result_schema
@@ -307,6 +333,11 @@ class Job < Model
     @hive_result_schema
   end
 
+  def result_size
+    update_status! unless @result_size || finished?
+    @result_size
+  end
+
   def result
     unless @result
       return nil unless finished?
@@ -315,9 +346,18 @@ class Job < Model
     @result
   end
 
-  def result_format(format, io=nil)
+  def result_format(format, io=nil, &block)
     return nil unless finished?
-    @client.job_result_format(@job_id, format, io)
+    @client.job_result_format(@job_id, format, io, &block)
+  end
+
+  def result_each_with_compr_size(&block)
+    if @result
+      @result.each(&block)
+    else
+      @client.job_result_each_with_compr_size(@job_id, &block)
+    end
+    nil
   end
 
   def result_each(&block)
@@ -364,7 +404,9 @@ class Job < Model
   end
 
   def update_status!
-    type, query, status, url, debug, start_at, end_at, cpu_time, result_url, hive_result_schema, priority, retry_limit, org_name, db_name = @client.api.show_job(@job_id)
+    type, query, status, url, debug, start_at, end_at, cpu_time,
+      result_size, result_url, hive_result_schema, priority, retry_limit,
+      org_name, db_name = @client.api.show_job(@job_id)
     @query = query
     @status = status
     @url = url
@@ -372,6 +414,7 @@ class Job < Model
     @start_at = start_at
     @end_at = end_at
     @cpu_time = cpu_time
+    @result_size = result_size
     @result_url = result_url
     @hive_result_schema = hive_result_schema
     @priority = priority
@@ -395,7 +438,8 @@ end
 
 
 class Schedule < Model
-  def initialize(client, name, cron, query, database=nil, result_url=nil, timezone=nil, delay=nil, next_time=nil, priority=nil, retry_limit=nil, org_name=nil)
+  def initialize(client, name, cron, query, database=nil, result_url=nil, timezone=nil, delay=nil, next_time=nil,
+                 priority=nil, retry_limit=nil, org_name=nil)
     super(client)
     @name = name
     @cron = cron