td-client 0.8.85 → 0.9.0dev1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 60a54746395659411ee0e31937d4456acf1161db
-  data.tar.gz: 86d0d880f287ef9ee6919fdfac91ad1211d887fa
+  metadata.gz: fbabfd3204808160bd8fd4f5a7cd3c5c48eb1b11
+  data.tar.gz: bd585ffc36231bc199d78b36f0e67a8205d6939a
 SHA512:
-  metadata.gz: 177e498d108552a2b549755455a34e6ac811cf8f06510cf2cb612b53f33ac13876c524fb8390f84459efb52ec00b2316ed8b0fb8a77daf2b11c56674e47bc318
-  data.tar.gz: fd06041a13ded4aeddb1514ce10f0225140d72a70a4dafa2305c77935415b59230d9c31b54378ab5ed04217b1cca4f06a19d1147f54e6c476209ab490adf816a
+  metadata.gz: d88e942c5fa4cdb4aa793419d6213ef80b048449e9383eef3cced73e30dc4f8b0a4f8ba39e2ea13ad5ce4fafb462b13f5e48acd213c87e3e9cf52b02a3f9470c
+  data.tar.gz: 356cfac4a8a32e3c574b0f5928ad80bf75e040952fc438c632b7ef3e3e7310950253103870c62277b75632299febb670cf3baaade3aecb67f45123010587d13c

data/lib/td/client.rb

@@ -182,10 +182,10 @@ class Client
     results = @api.list_jobs(from, to, status, conditions)
     results.map {|job_id, type, status, query, start_at, end_at, cpu_time,
                  result_size, result_url, priority, retry_limit, org, db,
-                 duration, num_records|
+                 duration|
       Job.new(self, job_id, type, query, status, nil, nil, start_at, end_at, cpu_time,
               result_size, nil, result_url, nil, priority, retry_limit, org, db,
-              duration, num_records)
+              duration)
     }
   end
 
@@ -194,9 +194,9 @@ class Client
   def job(job_id)
     job_id = job_id.to_s
     type, query, status, url, debug, start_at, end_at, cpu_time,
-      result_size, result_url, hive_result_schema, priority, retry_limit, org, db, num_records = @api.show_job(job_id)
+      result_size, result_url, hive_result_schema, priority, retry_limit, org, db = @api.show_job(job_id)
     Job.new(self, job_id, type, query, status, url, debug, start_at, end_at, cpu_time,
-            result_size, nil, result_url, hive_result_schema, priority, retry_limit, org, db, num_records)
+            result_size, nil, result_url, hive_result_schema, priority, retry_limit, org, db)
   end
 
   # @param [String] job_id
@@ -254,14 +254,6 @@ class Client
     Job.new(self, job_id, :export, nil)
   end
 
-  # @param [String] target_job_id
-  # @param [Hash] opts
-  # @return [Job]
-  def result_export(target_job_id, opts={})
-    job_id = @api.result_export(target_job_id, opts)
-    Job.new(self, job_id, :result_export, nil)
-  end
-
   # @param [String] db_name
   # @param [String] table_name
   # @param [Fixnum] to
@@ -364,7 +356,7 @@ class Client
     raise ArgumentError, "'cron' option is required" unless opts[:cron] || opts['cron']
     raise ArgumentError, "'query' option is required" unless opts[:query] || opts['query']
     start = @api.create_schedule(name, opts)
-    return start && Time.parse(start)
+    return Time.parse(start)
   end
 
   # @param [String] name
@@ -582,9 +574,9 @@ class Client
     @api.bulk_load_show(name)
   end
 
-  # name: String, settings: Hash -> BulkLoad
-  def bulk_load_update(name, settings)
-    @api.bulk_load_update(name, settings)
+  # name: String, job: BulkLoad -> BulkLoad
+  def bulk_load_update(name, job)
+    @api.bulk_load_update(name, job)
   end
 
   # name: String -> BulkLoad

data/lib/td/client/api.rb

@@ -36,19 +36,16 @@ class API
   include API::Table
   include API::User
 
-  DEFAULT_ENDPOINT = 'api.treasuredata.com'
-  DEFAULT_IMPORT_ENDPOINT = 'api-import.treasuredata.com'
+  DEFAULT_ENDPOINT = 'api.treasure-data.com'
+  DEFAULT_IMPORT_ENDPOINT = 'api-import.treasure-data.com'
 
-  # Deprecated. Use DEFAULT_ENDPOINT and DEFAULT_IMPORT_ENDPOINT instead
-  NEW_DEFAULT_ENDPOINT = DEFAULT_ENDPOINT
-  NEW_DEFAULT_IMPORT_ENDPOINT = DEFAULT_IMPORT_ENDPOINT
-  OLD_ENDPOINT = 'api.treasure-data.com'
+  NEW_DEFAULT_ENDPOINT = 'api.treasuredata.com'
+  NEW_DEFAULT_IMPORT_ENDPOINT = 'api-import.treasuredata.com'
 
   class IncompleteError < APIError; end
 
   # @param [String] apikey
   # @param [Hash] opts
-  # for backward compatibility
   def initialize(apikey, opts={})
     require 'json'
     require 'time'
@@ -95,18 +92,12 @@ class API
       # generic URI
       @host, @port = endpoint.split(':', 2)
       @port = @port.to_i
-      if opts[:ssl] === false || @host == TreasureData::API::OLD_ENDPOINT
-        # for backward compatibility, old endpoint specified without ssl option, use http
-        #
-        # opts[:ssl] would be nil if user doesn't specify ssl options,
-        # but connecting to https is the new default behavior (since 0.9)
-        # so check ssl option by `if opts[:ssl] === false` instead of `if opts[:ssl]`
-        # that means if user desire to use http, give `:ssl => false` for initializer such as API.new("APIKEY", :ssl => false)
-        @port = 80 if @port == 0
-        @ssl = false
-      else
+      if opts[:ssl]
         @port = 443 if @port == 0
         @ssl = true
+      else
+        @port = 80 if @port == 0
+        @ssl = false
       end
       @base_path = ''
     end
@@ -149,27 +140,15 @@ class API
     end
 
     name = name.to_s
-    if max_len
-      if name.length < min_len || name.length > max_len
-        raise ParameterValidationError,
-          "#{target.capitalize} name must be between #{min_len} and #{max_len} characters long. Got #{name.length} " +
-          (name.length == 1 ? "character" : "characters") + "."
-      end
-    else
-      if min_len == 1
-        if name.empty?
-          raise ParameterValidationError,
+    if name.empty?
+      raise ParameterValidationError,
             "Empty #{target} name is not allowed"
-        end
-      else
-        if name.length < min_len
-          raise ParameterValidationError,
-            "#{target.capitalize} name must be longer than #{min_len} characters. Got #{name.length} " +
+    end
+    if name.length < min_len || name.length > max_len
+      raise ParameterValidationError,
+            "#{target.capitalize} name must be between #{min_len} and #{max_len} characters long. Got #{name.length} " +
             (name.length == 1 ? "character" : "characters") + "."
-        end
-      end
     end
-
     unless name =~ /^([a-z0-9_]+)$/
       raise ParameterValidationError,
             "#{target.capitalize} name must only consist of lower-case alpha-numeric characters and '_'."
@@ -195,18 +174,7 @@ class API
 
   # @param [String] name
   def self.validate_column_name(name)
-    target = 'column'
-    name = name.to_s
-    if name.empty?
-      raise ParameterValidationError,
-            "Empty #{target} name is not allowed"
-    end
-    name
-  end
-
-  # @param [String] name
-  def self.validate_sql_alias_name(name)
-    validate_name("sql_alias", 1, nil, name)
+    validate_name("column", 1, 255, name)
   end
 
   # @param [String] name
@@ -231,6 +199,25 @@ class API
     normalize_database_name(name)
   end
 
+  # TODO support array types
+  # @param [String] name
+  def self.normalize_type_name(name)
+    case name
+    when /int/i, /integer/i
+      "int"
+    when /long/i, /bigint/i
+      "long"
+    when /string/i
+      "string"
+    when /float/i
+      "float"
+    when /double/i
+      "double"
+    else
+      raise "Type name must either of int, long, string float or double"
+    end
+  end
+
   # for fluent-plugin-td / td command to check table existence with import onlt user
   # @return [String]
   def self.create_empty_gz_data
@@ -254,7 +241,6 @@ private
       do_get(url, params, &block)
     end
   end
-
   # @param [String] url
   # @param [Hash] params
   # @yield [response]
@@ -309,7 +295,7 @@ private
           retry_delay *= 2
           redo # restart from beginning of do-while loop
         end
-      rescue Errno::ECONNREFUSED, Errno::ECONNRESET, Timeout::Error, EOFError, OpenSSL::SSL::SSLError, SocketError, IncompleteError, HTTPClient::TimeoutError => e
+      rescue Errno::ECONNREFUSED, Errno::ECONNRESET, Timeout::Error, EOFError, OpenSSL::SSL::SSLError, SocketError, IncompleteError => e
         if block_given?
           raise e
         end
@@ -605,15 +591,18 @@ private
   end
 
   def parse_error_response(res)
+    error = {}
+
     begin
-      error = JSON.load(res.body)
-      if error
-        error['message']    = error['error'] unless error['message']
+      js = JSON.load(res.body)
+      if js.nil?
+        error['message'] = res.reason
       else
-        error = {'message' => res.reason}
+        error['message']    = js['message'] || js['error']
+        error['stacktrace'] = js['stacktrace']
       end
     rescue JSON::ParserError
-      error = {'message' => res.body}
+      error['message'] = res.body
     end
 
     error
@@ -635,7 +624,6 @@ private
       when "404"
         NotFoundError
       when "409"
-        message = "#{message}: conflicts_with job:#{error["details"]["conflicts_with"]}" if error["details"] && error["details"]["conflicts_with"]
         AlreadyExistsError
       when "401"
         AuthError
@@ -647,15 +635,11 @@ private
       end
     end
 
-    exc = nil
-    if error_class.method_defined?(:conflicts_with) && error["details"] && error["details"]["conflicts_with"]
-      exc = error_class.new(message, error['stacktrace'], error["details"]["conflicts_with"])
-    elsif error_class.method_defined?(:api_backtrace)
-      exc = error_class.new(message, error['stacktrace'])
+    if error_class.method_defined?(:api_backtrace)
+      raise error_class.new(message, error['stacktrace'])
     else
-      exc = error_class.new(message)
+      raise error_class, message
     end
-    raise exc
   end
 
   if ''.respond_to?(:encode)

data/lib/td/client/api/bulk_import.rb

@@ -161,7 +161,8 @@ module BulkImport
       end
     end
     require File.expand_path('../compat_gzip_reader', File.dirname(__FILE__))
-    u = MessagePack::Unpacker.new(Zlib::GzipReader.new(StringIO.new(body)))
+    io = StringIO.new(Zlib::GzipReader.new(StringIO.new(body)).read)
+    u = MessagePack::Unpacker.new(io)
     if block
       begin
         u.each(&block)

data/lib/td/client/api/bulk_load.rb

@@ -116,10 +116,10 @@ module BulkLoad
     JSON.load(res.body)
   end
 
-  # name: String, settings: Hash -> Hash
-  def bulk_load_update(name, settings)
+  # name: String, job: Hash -> Hash
+  def bulk_load_update(name, job)
     path = session_path(name)
-    res = api { put(path, settings.to_json) }
+    res = api { put(path, job.to_json) }
     unless res.ok?
       raise_error("BulkLoadSession: #{name} update failed", res)
     end

data/lib/td/client/api/export.rb

@@ -22,17 +22,5 @@ module Export
     return js['job_id'].to_s
   end
 
-  # => jobId:String
-  # @param [String] target_job_id
-  # @param [Hash] opts
-  # @return [String] job_id
-  def result_export(target_job_id, opts={})
-    code, body, res = post("/v3/job/result_export/#{target_job_id}", opts)
-    if code != "200"
-      raise_error("Result Export failed", res)
-    end
-    js = checked_json(body, %w[job_id])
-    return js['job_id'].to_s
-  end
 end
 end

data/lib/td/client/api/import.rb

@@ -21,9 +21,8 @@ module Import
     opts = {}
     if @host == DEFAULT_ENDPOINT
       opts[:host] = DEFAULT_IMPORT_ENDPOINT
-    elsif @host == TreasureData::API::OLD_ENDPOINT # backward compatibility
-      opts[:host] = 'api-import.treasure-data.com'
-      opts[:ssl] = false
+    elsif @host == NEW_DEFAULT_ENDPOINT
+      opts[:host] = NEW_DEFAULT_IMPORT_ENDPOINT
     end
     code, body, res = put(path, stream, size, opts)
     if code[0] != ?2

data/lib/td/client/api/job.rb

@@ -36,10 +36,9 @@ module Job
       priority = m['priority']
       retry_limit = m['retry_limit']
       duration = m['duration']
-      num_records = m['num_records']
       result << [job_id, type, status, query, start_at, end_at, cpu_time,
                  result_size, result_url, priority, retry_limit, nil, database,
-                 duration, num_records]
+                 duration]
     }
     return result
   end
@@ -64,7 +63,6 @@ module Job
     end_at = js['end_at']
     cpu_time = js['cpu_time']
     result_size = js['result_size'] # compressed result size in msgpack.gz format
-    num_records = js['num_records']
     result = js['result'] # result target URL
     hive_result_schema = (js['hive_result_schema'] || '')
     if hive_result_schema.empty?
@@ -99,7 +97,7 @@ module Job
     priority = js['priority']
     retry_limit = js['retry_limit']
     return [type, query, status, url, debug, start_at, end_at, cpu_time,
-            result_size, result, hive_result_schema, priority, retry_limit, nil, database, num_records]
+            result_size, result, hive_result_schema, priority, retry_limit, nil, database]
   end
 
   # @param [String] job_id
@@ -117,13 +115,14 @@ module Job
   # @param [String] job_id
   # @return [Array]
   def job_result(job_id)
-    result = []
-    unpacker = MessagePack::Unpacker.new
-    job_result_download(job_id) do |chunk|
-      unpacker.feed_each(chunk) do |row|
-        result << row
-      end
+    code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>'msgpack'})
+    if code != "200"
+      raise_error("Get job result failed", res)
     end
+    result = []
+    MessagePack::Unpacker.new.feed_each(body) {|row|
+      result << row
+    }
     return result
   end
 
@@ -134,17 +133,24 @@ module Job
   # @param [IO] io
   # @param [Proc] block
   # @return [nil, String]
-  def job_result_format(job_id, format, io=nil)
+  def job_result_format(job_id, format, io=nil, &block)
     if io
-      job_result_download(job_id, format) do |chunk, total|
-        io.write chunk
-        yield total if block_given?
-      end
+      infl = nil
+      code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>format}) {|res, chunk, current_total_chunk_size|
+        if res.code != 200
+          raise_error("Get job result failed", res)
+        end
+
+        infl ||= create_inflalte_or_null_inflate(res)
+
+        io.write infl.inflate(chunk)
+        block.call(current_total_chunk_size) if block_given?
+      }
       nil
     else
-      body = String.new
-      job_result_download(job_id, format) do |chunk|
-        body << chunk
+      code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>format})
+      if code != "200"
+        raise_error("Get job result failed", res)
       end
       body
     end
@@ -157,11 +163,22 @@ module Job
   # @return [nil]
   def job_result_each(job_id, &block)
     upkr = MessagePack::Unpacker.new
-    # default to decompressing the response since format is fixed to 'msgpack'
-    job_result_download(job_id) do |chunk|
-      upkr.feed_each(chunk, &block)
-    end
+    infl = nil
+
+    get("/v3/job/result/#{e job_id}", {'format'=>'msgpack'}) {|res, chunk, current_total_chunk_size|
+      if res.code != 200
+        raise_error("Get job result failed", res)
+      end
+
+      # default to decompressing the response since format is fixed to 'msgpack'
+      infl ||= create_inflate(res)
+
+      inflated_fragment = infl.inflate(chunk)
+      upkr.feed_each(inflated_fragment, &block)
+    }
     nil
+  ensure
+    infl.close if infl
   end
 
   # block is optional and must accept 1 argument
@@ -169,30 +186,50 @@ module Job
   # @param [String] job_id
   # @param [Proc] block
   # @return [nil]
-  def job_result_each_with_compr_size(job_id)
+  def job_result_each_with_compr_size(job_id, &block)
     upkr = MessagePack::Unpacker.new
-    # default to decompressing the response since format is fixed to 'msgpack'
-    job_result_download(job_id) do |chunk, total|
-      upkr.feed_each(chunk) {|unpacked|
-        yield unpacked, total if block_given?
+    infl = nil
+
+    get("/v3/job/result/#{e job_id}", {'format'=>'msgpack'}) {|res, chunk, current_total_chunk_size|
+      if res.code != 200
+        raise_error("Get job result failed", res)
+      end
+
+      # default to decompressing the response since format is fixed to 'msgpack'
+      infl ||= create_inflate(res)
+
+      inflated_fragment = infl.inflate(chunk)
+      upkr.feed_each(inflated_fragment) {|unpacked|
+        block.call(unpacked, current_total_chunk_size) if block_given?
       }
-    end
+    }
     nil
+  ensure
+    infl.close if infl
   end
 
   # @param [String] job_id
   # @param [String] format
   # @return [String]
-  def job_result_raw(job_id, format, io = nil)
-    body = io ? nil : String.new
-    job_result_download(job_id, format, false) do |chunk, total|
+  def job_result_raw(job_id, format, io = nil, &block)
+    body = nil
+
+    get("/v3/job/result/#{e job_id}", {'format'=>format}) {|res, chunk, current_total_chunk_size|
+      if res.code != 200
+        raise_error("Get job result failed", res)
+      end
+
       if io
         io.write(chunk)
-        yield total if block_given?
+        block.call(current_total_chunk_size) if block_given?
       else
-        body << chunk
+        if body
+          body += chunk
+        else
+          body = chunk
+        end
       end
-    end
+    }
     body
   end
 
@@ -250,117 +287,6 @@ module Job
 
   private
 
-  def validate_content_length_with_range(response, current_total_chunk_size)
-    if expected_size = response.header['Content-Range'][0]
-      expected_size = expected_size[/\d+$/].to_i
-    elsif expected_size = response.header['Content-Length'][0]
-      expected_size = expected_size.to_i
-    end
-
-    if expected_size.nil?
-    elsif current_total_chunk_size < expected_size
-      # too small
-      # NOTE:
-      #   ext/openssl raises EOFError in case where underlying connection
-      #   causes an error, but httpclient ignores it.
-      #   https://github.com/nahi/httpclient/blob/v3.2.8/lib/httpclient/session.rb#L1003
-      raise EOFError, 'httpclient IncompleteError'
-    elsif current_total_chunk_size > expected_size
-      # too large
-      raise_error("Get job result failed", response)
-    end
-  end
-
-  def job_result_download(job_id, format='msgpack', autodecode=true)
-    client, header = new_client
-    client.send_timeout = @send_timeout
-    client.receive_timeout = @read_timeout
-    header['Accept-Encoding'] = 'deflate, gzip'
-
-    url = build_endpoint("/v3/job/result/#{e job_id}", @host)
-    params = {'format' => format}
-
-    unless ENV['TD_CLIENT_DEBUG'].nil?
-      puts "DEBUG: REST GET call:"
-      puts "DEBUG:   header: " + header.to_s
-      puts "DEBUG:   url:    " + url.to_s
-      puts "DEBUG:   params: " + params.to_s
-    end
-
-    # up to 7 retries with exponential (base 2) back-off starting at 'retry_delay'
-    retry_delay = @retry_delay
-    cumul_retry_delay = 0
-    current_total_chunk_size = 0
-    infl = nil
-    begin # LOOP of Network/Server errors
-      response = nil
-      client.get(url, params, header) do |res, chunk|
-        unless response
-          case res.status
-          when 200
-            if current_total_chunk_size != 0
-              # try to resume but the server returns 200
-              raise_error("Get job result failed", res)
-            end
-          when 206 # resuming
-          else
-            if res.status/100 == 5 && cumul_retry_delay < @max_cumul_retry_delay
-              $stderr.puts "Error #{res.status}: #{get_error(res)}. Retrying after #{retry_delay} seconds..."
-              sleep retry_delay
-              cumul_retry_delay += retry_delay
-              retry_delay *= 2
-              redo
-            end
-            raise_error("Get job result failed", res)
-          end
-          if infl.nil? && autodecode
-            case res.header['Content-Encoding'][0].to_s.downcase
-            when 'gzip'
-              infl = Zlib::Inflate.new(Zlib::MAX_WBITS + 16)
-            when 'deflate'
-              infl = Zlib::Inflate.new
-            end
-          end
-        end
-        response = res
-        current_total_chunk_size += chunk.bytesize
-        chunk = infl.inflate(chunk) if infl
-        yield chunk, current_total_chunk_size
-      end
-
-      # completed?
-      validate_content_length_with_range(response, current_total_chunk_size)
-    rescue Errno::ECONNREFUSED, Errno::ECONNRESET, Timeout::Error, EOFError, OpenSSL::SSL::SSLError, SocketError => e
-      if response # at least a chunk is downloaded
-        if etag = response.header['ETag'][0]
-          header['If-Range'] = etag
-          header['Range'] = "bytes=#{current_total_chunk_size}-"
-        end
-      end
-
-      $stderr.print "#{e.class}: #{e.message}. "
-      if cumul_retry_delay < @max_cumul_retry_delay
-        $stderr.puts "Retrying after #{retry_delay} seconds..."
-        sleep retry_delay
-        cumul_retry_delay += retry_delay
-        retry_delay *= 2
-        retry
-      end
-      raise
-    end
-
-    unless ENV['TD_CLIENT_DEBUG'].nil?
-      puts "DEBUG: REST GET response:"
-      puts "DEBUG:   header: " + response.header.to_s
-      puts "DEBUG:   status: " + response.code.to_s
-      puts "DEBUG:   body:   " + response.body.to_s
-    end
-
-    nil
-  ensure
-    infl.close if infl
-  end
-
   class NullInflate
     def inflate(chunk)
       chunk