td-client 0.8.68 → 0.8.69

data/lib/td/client/api.rb

@@ -3,6 +3,7 @@ require 'td/client/version'
 require 'td/client/api/access_control'
 require 'td/client/api/account'
 require 'td/client/api/bulk_import'
+require 'td/client/api/bulk_load'
 require 'td/client/api/database'
 require 'td/client/api/export'
 require 'td/client/api/import'
@@ -23,6 +24,7 @@ class API
   include API::AccessControl
   include API::Account
   include API::BulkImport
+  include API::BulkLoad
   include API::Database
   include API::Export
   include API::Import
@@ -40,6 +42,8 @@ class API
   NEW_DEFAULT_ENDPOINT = 'api.treasuredata.com'
   NEW_DEFAULT_IMPORT_ENDPOINT = 'api-import.treasuredata.com'
 
+  # @param [String] apikey
+  # @param [Hash] opts
   def initialize(apikey, opts={})
     require 'json'
     require 'time'
@@ -65,6 +69,7 @@ class API
     @read_timeout = opts[:read_timeout] || 600
     @send_timeout = opts[:send_timeout] || 600
     @retry_post_requests = opts[:retry_post_requests] || false
+    @retry_delay = opts[:retry_delay] || 5
     @max_cumul_retry_delay = opts[:max_cumul_retry_delay] || 600
 
     case uri.scheme
@@ -110,12 +115,16 @@ class API
     end
 
     @headers = opts[:headers] || {}
+    @api = api_client("#{@ssl ? 'https' : 'http'}://#{@host}:#{@port}")
   end
 
   # TODO error check & raise appropriate errors
 
+  # @!attribute [r] apikey
   attr_reader :apikey
 
+  # @param [Hash] record
+  # @param [IO] out
   def self.normalized_msgpack(record, out = nil)
     record.keys.each { |k|
       v = record[k]
@@ -126,6 +135,10 @@ class API
     record.to_msgpack(out)
   end
 
+  # @param [String] target
+  # @param [Fixnum] min_len
+  # @param [Fixnum] max_len
+  # @param [String] name
   def self.validate_name(target, min_len, max_len, name)
     if !target.instance_of?(String) || target.empty?
       raise ParameterValidationError,
@@ -150,22 +163,27 @@ class API
     name
   end
 
+  # @param [String] name
   def self.validate_database_name(name)
     validate_name("database", 3, 255, name)
   end
 
+  # @param [String] name
   def self.validate_table_name(name)
     validate_name("table", 3, 255, name)
   end
 
+  # @param [String] name
   def self.validate_result_set_name(name)
     validate_name("result set", 3, 255, name)
   end
 
+  # @param [String] name
   def self.validate_column_name(name)
     validate_name("column", 1, 255, name)
   end
 
+  # @param [String] name
   def self.normalize_database_name(name)
     name = name.to_s
     if name.empty?
@@ -182,11 +200,13 @@ class API
     name
   end
 
+  # @param [String] name
   def self.normalize_table_name(name)
     normalize_database_name(name)
   end
 
   # TODO support array types
+  # @param [String] name
   def self.normalize_type_name(name)
     case name
     when /int/i, /integer/i
@@ -205,12 +225,14 @@ class API
   end
 
   # for fluent-plugin-td / td command to check table existence with import onlt user
+  # @return [String]
   def self.create_empty_gz_data
     io = StringIO.new
     Zlib::GzipWriter.new(io).close
     io.string
   end
 
+  # @param [String] ssl_ca_file
   def ssl_ca_file=(ssl_ca_file)
     @ssl_ca_file = ssl_ca_file
   end
@@ -220,6 +242,7 @@ private
   module DeflateReadBodyMixin
     attr_accessor :gzip
 
+    # @yield [fragment]
     def each_fragment(&block)
       if @gzip
         infl = Zlib::Inflate.new(Zlib::MAX_WBITS + 16)
@@ -238,17 +261,24 @@ private
   end
 
   module DirectReadBodyMixin
+    # @yield [fragment]
     def each_fragment(&block)
       read_body(&block)
     end
   end
 
+  # @param [String] url
+  # @param [Hash] params
+  # @yield [response]
   def get(url, params=nil, &block)
     guard_no_sslv3 do
       do_get(url, params, &block)
     end
   end
 
+  # @param [String] url
+  # @param [Hash] params
+  # @yield [response]
   def do_get(url, params=nil, &block)
     http, header = new_http
 
@@ -270,7 +300,7 @@ private
     end
 
     # up to 7 retries with exponential (base 2) back-off starting at 'retry_delay'
-    retry_delay = 5
+    retry_delay = @retry_delay
     cumul_retry_delay = 0
 
     # for both exceptions and 500+ errors retrying is enabled by default.
@@ -288,7 +318,7 @@ private
 
         status = response.code.to_i
         # retry if the HTTP error code is 500 or higher and we did not run out of retrying attempts
-        if !block_given? && status >= 500 && cumul_retry_delay <= @max_cumul_retry_delay
+        if !block_given? && status >= 500 && cumul_retry_delay < @max_cumul_retry_delay
           $stderr.puts "Error #{status}: #{get_error(response)}. Retrying after #{retry_delay} seconds..."
           sleep retry_delay
           cumul_retry_delay += retry_delay
@@ -300,7 +330,7 @@ private
           raise e
         end
         $stderr.print "#{e.class}: #{e.message}. "
-        if cumul_retry_delay <= @max_cumul_retry_delay
+        if cumul_retry_delay < @max_cumul_retry_delay
           $stderr.puts "Retrying after #{retry_delay} seconds..."
           sleep retry_delay
           cumul_retry_delay += retry_delay
@@ -341,12 +371,16 @@ private
     return [response.code, body, response]
   end
 
+  # @param [String] url
+  # @param [Hash] params
   def post(url, params=nil)
     guard_no_sslv3 do
       do_post(url, params)
     end
   end
 
+  # @param [String] url
+  # @param [Hash] params
   def do_post(url, params=nil)
     http, header = new_http
 
@@ -368,7 +402,7 @@ private
     end
 
     # up to 7 retries with exponential (base 2) back-off starting at 'retry_delay'
-    retry_delay = 5
+    retry_delay = @retry_delay
     cumul_retry_delay = 0
 
     # for both exceptions and 500+ errors retrying can be enabled by initialization
@@ -382,7 +416,7 @@ private
         # if the HTTP error code is 500 or higher and the user requested retrying
         # on post request, attempt a retry
         status = response.code.to_i
-        if @retry_post_requests && status >= 500 && cumul_retry_delay <= @max_cumul_retry_delay
+        if @retry_post_requests && status >= 500 && cumul_retry_delay < @max_cumul_retry_delay
           $stderr.puts "Error #{status}: #{get_error(response)}. Retrying after #{retry_delay} seconds..."
           sleep retry_delay
           cumul_retry_delay += retry_delay
@@ -391,7 +425,7 @@ private
         end
       rescue Errno::ECONNREFUSED, Errno::ECONNRESET, Timeout::Error, EOFError, OpenSSL::SSL::SSLError, SocketError => e
         $stderr.print "#{e.class}: #{e.message}. "
-        if @retry_post_requests && cumul_retry_delay <= @max_cumul_retry_delay
+        if @retry_post_requests && cumul_retry_delay < @max_cumul_retry_delay
           $stderr.puts "Retrying after #{retry_delay} seconds..."
           sleep retry_delay
           cumul_retry_delay += retry_delay
@@ -420,6 +454,10 @@ private
     return [response.code, response.body, response]
   end
 
+  # @param [String] url
+  # @param [String, StringIO] stream
+  # @param [Fixnum] size
+  # @param [Hash] opts
   def put(url, stream, size, opts = {})
     client, header = new_client(opts)
     client.send_timeout = @send_timeout
@@ -457,11 +495,15 @@ private
     end
   end
 
+  # @param [String] url
+  # @param [String] host
+  # @return [String]
   def build_endpoint(url, host)
     schema = @ssl ? 'https' : 'http'
     "#{schema}://#{host}:#{@port}#{@base_path + url}"
   end
 
+  # @yield Disable SSLv3 in given block
   def guard_no_sslv3
     key = :SET_SSL_OP_NO_SSLv3
     backup = Thread.current[key]
@@ -475,6 +517,8 @@ private
     end
   end
 
+  # @param [Hash] opts
+  # @return [http, Hash]
   def new_http(opts = {})
     host = opts[:host] || @host
     http = @http_class.new(host, @port)
@@ -505,6 +549,8 @@ private
     return http, header
   end
 
+  # @param [Hash] opts
+  # @return [HTTPClient, Hash]
   def new_client(opts = {})
     client = HTTPClient.new(@http_proxy, @user_agent)
     client.connect_timeout = @connect_timeout
@@ -527,10 +573,73 @@ private
     return client, header
   end
 
+  # @return [String]
+  def api_client(endpoint)
+    header = {}.merge(@headers)
+    header['Authorization'] = "TD1 #{apikey}" if @apikey
+    header['Content-Type'] = 'application/json; charset=utf-8'
+    client = HTTPClient.new(:proxy => @http_proxy, :agent_name => @user_agent, :base_url => endpoint, :default_header => header)
+    client.connect_timeout = @connect_timeout
+    client.send_timeout = @send_timeout
+    client.receive_timeout = @read_timeout
+    client.transparent_gzip_decompression = true
+    client.debug_dev = STDOUT unless ENV['TD_CLIENT_DEBUG'].nil?
+    client
+  end
+
+  def api(opt = {:retry_request => true}, &block)
+    retry_request = opt[:retry_request]
+    # up to 7 retries with exponential (base 2) back-off starting at 'retry_delay'
+    retry_delay = @retry_delay
+    retry_times = 0
+    cumul_retry_delay = 0
+
+    # for both exceptions and 500+ errors retrying can be enabled by initialization
+    # parameter 'retry_post_requests'. The total number of retries cumulatively
+    # should not exceed 10 minutes / 600 seconds
+    begin # this block is to allow retry (redo) in the begin part of the begin-rescue block
+      begin
+        response = @api.instance_eval &block
+
+        # if the HTTP error code is 500 or higher and the user requested retrying
+        # on post request, attempt a retry
+        status = response.code.to_i
+        if retry_request && status >= 500 && cumul_retry_delay < @max_cumul_retry_delay
+          $stderr.puts "Error #{status}: #{get_error(response)}. Retrying after #{retry_delay} seconds..."
+          sleep retry_delay
+          cumul_retry_delay += retry_delay
+          retry_delay *= 2
+          redo # restart from beginning of do-while loop
+        end
+        return response
+      rescue Errno::ECONNREFUSED, Errno::ECONNRESET, Timeout::Error, EOFError, OpenSSL::SSL::SSLError, SocketError => e
+        $stderr.print "#{e.class}: #{e.message}. "
+        if retry_request
+          if cumul_retry_delay < @max_cumul_retry_delay
+            $stderr.puts "Retrying after #{retry_delay} seconds..."
+            sleep retry_delay
+            cumul_retry_delay += retry_delay
+            retry_delay *= 2
+            retry_times += 1
+            retry
+          else
+            $stderr.puts "Retrying stopped after #{@max_cumul_retry_delay} seconds."
+            e.message << " (Retried #{retry_times} times in #{cumul_retry_delay} seconds)"
+          end
+        else
+          $stderr.puts "No retry should be performed."
+        end
+        raise e
+      end
+    end while false
+  end
+
   def ssl_ca_file
     @ssl_ca_file ||= File.join(File.dirname(__FILE__), '..', '..', '..', 'data', 'ca-bundle.crt')
   end
 
+  # @param [response] res
+  # @return [String]
   def get_error(res)
     begin
       js = JSON.load(res.body)
@@ -549,6 +658,9 @@ private
     error_msg
   end
 
+  # @param [String] msg
+  # @param [response] res
+  # @param [Class] klass
   def raise_error(msg, res, klass=nil)
     status_code = res.code.to_s
     error_msg = get_error(res)
@@ -569,16 +681,22 @@ private
   end
 
   if ''.respond_to?(:encode)
+    # @param [String] s
+    # @return [String]
     def e(s)
       CGI.escape(s.to_s.encode("UTF-8"))
     end
   else
+    # @param [String] s
+    # @return [String]
     def e(s)
       CGI.escape(s.to_s)
     end
   end
 
-  def checked_json(body, required)
+  # @param [String] body
+  # @param [Array] required
+  def checked_json(body, required = [])
     js = nil
     begin
       js = JSON.load(body)

data/lib/td/client/api/access_control.rb

@@ -5,6 +5,11 @@ module AccessControl
   ## Access Control API
   ##
 
+  # @param [String] subject
+  # @param [String] action
+  # @param [String] scope
+  # @param [Array] grant_option
+  # @return [true]
   def grant_access_control(subject, action, scope, grant_option)
     params = {'subject'=>subject, 'action'=>action, 'scope'=>scope, 'grant_option'=>grant_option.to_s}
     code, body, res = post("/v3/acl/grant", params)
@@ -14,6 +19,10 @@ module AccessControl
     return true
   end
 
+  # @param [String] subject
+  # @param [String] action
+  # @param [String] scope
+  # @return [true]
   def revoke_access_control(subject, action, scope)
     params = {'subject'=>subject, 'action'=>action, 'scope'=>scope}
     code, body, res = post("/v3/acl/revoke", params)
@@ -23,7 +32,10 @@ module AccessControl
     return true
   end
 
-  # [true, [{subject:String,action:String,scope:String}]]
+  # @param [String] user
+  # @param [String] action
+  # @param [String] scope
+  # @return [Array]
   def test_access_control(user, action, scope)
     params = {'user'=>user, 'action'=>action, 'scope'=>scope}
     code, body, res = get("/v3/acl/test", params)
@@ -41,7 +53,7 @@ module AccessControl
     return perm, acl
   end
 
-  # [{subject:String,action:String,scope:String}]
+  # @return [Array]
   def list_access_controls
     code, body, res = get("/v3/acl/list")
     if code != "200"

data/lib/td/client/api/account.rb

@@ -5,6 +5,7 @@ module Account
   ## Account API
   ##
 
+  # @return [Array]
   def show_account
     code, body, res = get("/v3/account/show")
     if code != "200"
@@ -21,6 +22,9 @@ module Account
     return [account_id, plan, storage_size, guaranteed_cores, maximum_cores, created_at]
   end
 
+  # @param [Fixnum] from
+  # @param [Fixnum] to
+  # @return [Array]
   def account_core_utilization(from, to)
     params = { }
     params['from'] = from.to_s if from

data/lib/td/client/api/bulk_import.rb

@@ -5,7 +5,11 @@ module BulkImport
   ## Bulk import API
   ##
 
-  # => nil
+  # @param [String] name
+  # @param [String] db
+  # @param [String] table
+  # @param [Hash] opts
+  # @return [nil]
   def create_bulk_import(name, db, table, opts={})
     params = opts.dup
     code, body, res = post("/v3/bulk_import/create/#{e name}/#{e db}/#{e table}", params)
@@ -15,7 +19,9 @@ module BulkImport
     return nil
   end
 
-  # => nil
+  # @param [String] name
+  # @param [Hash] opts
+  # @return [nil]
   def delete_bulk_import(name, opts={})
     params = opts.dup
     code, body, res = post("/v3/bulk_import/delete/#{e name}", params)
@@ -25,7 +31,8 @@ module BulkImport
     return nil
   end
 
-  # => data:Hash
+  # @param [String] name
+  # @return [Hash]
   def show_bulk_import(name)
     code, body, res = get("/v3/bulk_import/show/#{name}")
     if code != "200"
@@ -35,7 +42,8 @@ module BulkImport
     return js
   end
 
-  # => result:[data:Hash]
+  # @param [Hash] opts
+  # @return [Hash]
   def list_bulk_imports(opts={})
     params = opts.dup
     code, body, res = get("/v3/bulk_import/list", params)
@@ -46,6 +54,9 @@ module BulkImport
     return js['bulk_imports']
   end
 
+  # @param [String] name
+  # @param [Hash] opts
+  # @return [nil]
   def list_bulk_import_parts(name, opts={})
     params = opts.dup
     code, body, res = get("/v3/bulk_import/list_parts/#{e name}", params)
@@ -56,7 +67,12 @@ module BulkImport
     return js['parts']
   end
 
-  # => nil
+  # @param [String] name
+  # @param [String] part_name
+  # @param [String, StringIO] stream
+  # @param [Fixnum] size
+  # @param [Hash] opts
+  # @return [nil]
   def bulk_import_upload_part(name, part_name, stream, size, opts={})
     code, body, res = put("/v3/bulk_import/upload_part/#{e name}/#{e part_name}", stream, size)
     if code[0] != ?2
@@ -65,7 +81,10 @@ module BulkImport
     return nil
   end
 
-  # => nil
+  # @param [String] name
+  # @param [String] part_name
+  # @param [Hash] opts
+  # @return [nil]
   def bulk_import_delete_part(name, part_name, opts={})
     params = opts.dup
     code, body, res = post("/v3/bulk_import/delete_part/#{e name}/#{e part_name}", params)
@@ -75,7 +94,9 @@ module BulkImport
     return nil
   end
 
-  # => nil
+  # @param [String] name
+  # @param [Hash] opts
+  # @return [nil]
   def freeze_bulk_import(name, opts={})
     params = opts.dup
     code, body, res = post("/v3/bulk_import/freeze/#{e name}", params)
@@ -85,7 +106,9 @@ module BulkImport
     return nil
   end
 
-  # => nil
+  # @param [String] name
+  # @param [Hash] opts
+  # @return [nil]
   def unfreeze_bulk_import(name, opts={})
     params = opts.dup
     code, body, res = post("/v3/bulk_import/unfreeze/#{e name}", params)
@@ -95,7 +118,9 @@ module BulkImport
     return nil
   end
 
-  # => jobId:String
+  # @param [String] name
+  # @param [Hash] opts
+  # @return [String] job_id
   def perform_bulk_import(name, opts={})
     params = opts.dup
     code, body, res = post("/v3/bulk_import/perform/#{e name}", params)
@@ -106,7 +131,9 @@ module BulkImport
     return js['job_id'].to_s
   end
 
-  # => nil
+  # @param [String] name
+  # @param [Hash] opts
+  # @return [nil]
   def commit_bulk_import(name, opts={})
     params = opts.dup
     code, body, res = post("/v3/bulk_import/commit/#{e name}", params)
@@ -116,7 +143,10 @@ module BulkImport
     return nil
   end
 
-  # => data...
+  # @param [String] name
+  # @param [Hash] opts
+  # @param [Proc] block
+  # @return [Array]
   def bulk_import_error_records(name, opts={}, &block)
     params = opts.dup
     code, body, res = get("/v3/bulk_import/error_records/#{e name}", params)