td-client 0.9.0dev2 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a408ad7fd0f4eec142b6cc52fcdfbf3bc14980fd
-  data.tar.gz: 17891d7c92097c10818735ec7be9db47c2731b88
+  metadata.gz: 140238f578d4cee66cc4ef09c155375719ad1c9e
+  data.tar.gz: cc7df46122c3186fc80c99ca1911aff0bcf8c994
 SHA512:
-  metadata.gz: 58874a549e403387c112f21b14d236218eff0a446057f705cd398eef118603a82bee3a75f85a86ed990c383bdd97b2f7994d44f3df80d271f57806dd6602ef20
-  data.tar.gz: 498c15e6a4d8ce9f7edcd799107aa0c9ec4cd9a4aee435fc0621ab906ef57acf075f5bf01b841f43064883d180571a4edd01effed53ceec84577d71548f660fa
+  metadata.gz: bdd1ba7cfb27afbc22a07845c06cf420afa1d9d40411137573ec2406ceeb923838a65be2e3ce7dbdc5120126a3386aa6234dca79426e9341d57e3a82ebe559d3
+  data.tar.gz: 8045b7325a7492b35db7962ed36d3bb6895d3307990c1e43d7523e0832b9a33b7d2ac4d792a0b8284755342726c231ea994f83d0829a72a19a3b7a891789a738

data/lib/td/client.rb

@@ -182,10 +182,10 @@ class Client
     results = @api.list_jobs(from, to, status, conditions)
     results.map {|job_id, type, status, query, start_at, end_at, cpu_time,
                  result_size, result_url, priority, retry_limit, org, db,
-                 duration|
+                 duration, num_records|
       Job.new(self, job_id, type, query, status, nil, nil, start_at, end_at, cpu_time,
               result_size, nil, result_url, nil, priority, retry_limit, org, db,
-              duration)
+              duration, num_records)
     }
   end
 
@@ -194,9 +194,9 @@ class Client
   def job(job_id)
     job_id = job_id.to_s
     type, query, status, url, debug, start_at, end_at, cpu_time,
-      result_size, result_url, hive_result_schema, priority, retry_limit, org, db = @api.show_job(job_id)
+      result_size, result_url, hive_result_schema, priority, retry_limit, org, db, duration, num_records = @api.show_job(job_id)
     Job.new(self, job_id, type, query, status, url, debug, start_at, end_at, cpu_time,
-            result_size, nil, result_url, hive_result_schema, priority, retry_limit, org, db)
+            result_size, nil, result_url, hive_result_schema, priority, retry_limit, org, db, duration, num_records)
   end
 
   # @param [String] job_id
@@ -254,6 +254,14 @@ class Client
     Job.new(self, job_id, :export, nil)
   end
 
+  # @param [String] target_job_id
+  # @param [Hash] opts
+  # @return [Job]
+  def result_export(target_job_id, opts={})
+    job_id = @api.result_export(target_job_id, opts)
+    Job.new(self, job_id, :result_export, nil)
+  end
+
   # @param [String] db_name
   # @param [String] table_name
   # @param [Fixnum] to
@@ -356,7 +364,7 @@ class Client
     raise ArgumentError, "'cron' option is required" unless opts[:cron] || opts['cron']
     raise ArgumentError, "'query' option is required" unless opts[:query] || opts['query']
     start = @api.create_schedule(name, opts)
-    return Time.parse(start)
+    return start && Time.parse(start)
   end
 
   # @param [String] name
@@ -574,9 +582,9 @@ class Client
     @api.bulk_load_show(name)
   end
 
-  # name: String, job: BulkLoad -> BulkLoad
-  def bulk_load_update(name, job)
-    @api.bulk_load_update(name, job)
+  # name: String, settings: Hash -> BulkLoad
+  def bulk_load_update(name, settings)
+    @api.bulk_load_update(name, settings)
   end
 
   # name: String -> BulkLoad

data/lib/td/client/api.rb

@@ -36,16 +36,19 @@ class API
   include API::Table
   include API::User
 
-  DEFAULT_ENDPOINT = 'api.treasure-data.com'
-  DEFAULT_IMPORT_ENDPOINT = 'api-import.treasure-data.com'
+  DEFAULT_ENDPOINT = 'api.treasuredata.com'
+  DEFAULT_IMPORT_ENDPOINT = 'api-import.treasuredata.com'
 
-  NEW_DEFAULT_ENDPOINT = 'api.treasuredata.com'
-  NEW_DEFAULT_IMPORT_ENDPOINT = 'api-import.treasuredata.com'
+  # Deprecated. Use DEFAULT_ENDPOINT and DEFAULT_IMPORT_ENDPOINT instead
+  NEW_DEFAULT_ENDPOINT = DEFAULT_ENDPOINT
+  NEW_DEFAULT_IMPORT_ENDPOINT = DEFAULT_IMPORT_ENDPOINT
+  OLD_ENDPOINT = 'api.treasure-data.com'
 
   class IncompleteError < APIError; end
 
   # @param [String] apikey
   # @param [Hash] opts
+  # for backward compatibility
   def initialize(apikey, opts={})
     require 'json'
     require 'time'
@@ -92,12 +95,18 @@ class API
       # generic URI
       @host, @port = endpoint.split(':', 2)
       @port = @port.to_i
-      if opts[:ssl]
-        @port = 443 if @port == 0
-        @ssl = true
-      else
+      if opts[:ssl] === false || @host == TreasureData::API::OLD_ENDPOINT
+        # for backward compatibility, old endpoint specified without ssl option, use http
+        #
+        # opts[:ssl] would be nil if user doesn't specify ssl options,
+        # but connecting to https is the new default behavior (since 0.9)
+        # so check ssl option by `if opts[:ssl] === false` instead of `if opts[:ssl]`
+        # that means if user desire to use http, give `:ssl => false` for initializer such as API.new("APIKEY", :ssl => false)
         @port = 80 if @port == 0
         @ssl = false
+      else
+        @port = 443 if @port == 0
+        @ssl = true
       end
       @base_path = ''
     end
@@ -112,12 +121,15 @@ class API
   # @!attribute [r] apikey
   attr_reader :apikey
 
+  MSGPACK_INT64_MAX = 2 ** 64 - 1
+  MSGPACK_INT64_MIN = -1 * (2 ** 63) # it's just same with -2**63, but for readability
+
   # @param [Hash] record
   # @param [IO] out
   def self.normalized_msgpack(record, out = nil)
     record.keys.each { |k|
       v = record[k]
-      if v.kind_of?(Bignum)
+      if v.kind_of?(Integer) && (v > MSGPACK_INT64_MAX || v < MSGPACK_INT64_MIN)
         record[k] = v.to_s
       end
     }
@@ -140,15 +152,27 @@ class API
     end
 
     name = name.to_s
-    if name.empty?
-      raise ParameterValidationError,
+    if max_len
+      if name.length < min_len || name.length > max_len
+        raise ParameterValidationError,
+          "#{target.capitalize} name must be between #{min_len} and #{max_len} characters long. Got #{name.length} " +
+          (name.length == 1 ? "character" : "characters") + "."
+      end
+    else
+      if min_len == 1
+        if name.empty?
+          raise ParameterValidationError,
             "Empty #{target} name is not allowed"
-    end
-    if name.length < min_len || name.length > max_len
-      raise ParameterValidationError,
-            "#{target.capitalize} name must be between #{min_len} and #{max_len} characters long. Got #{name.length} " +
+        end
+      else
+        if name.length < min_len
+          raise ParameterValidationError,
+            "#{target.capitalize} name must be longer than #{min_len} characters. Got #{name.length} " +
             (name.length == 1 ? "character" : "characters") + "."
+        end
+      end
     end
+
     unless name =~ /^([a-z0-9_]+)$/
       raise ParameterValidationError,
             "#{target.capitalize} name must only consist of lower-case alpha-numeric characters and '_'."
@@ -174,7 +198,18 @@ class API
 
   # @param [String] name
   def self.validate_column_name(name)
-    validate_name("column", 1, 255, name)
+    target = 'column'
+    name = name.to_s
+    if name.empty?
+      raise ParameterValidationError,
+            "Empty #{target} name is not allowed"
+    end
+    name
+  end
+
+  # @param [String] name
+  def self.validate_sql_alias_name(name)
+    validate_name("sql_alias", 1, nil, name)
   end
 
   # @param [String] name
@@ -199,25 +234,6 @@ class API
     normalize_database_name(name)
   end
 
-  # TODO support array types
-  # @param [String] name
-  def self.normalize_type_name(name)
-    case name
-    when /int/i, /integer/i
-      "int"
-    when /long/i, /bigint/i
-      "long"
-    when /string/i
-      "string"
-    when /float/i
-      "float"
-    when /double/i
-      "double"
-    else
-      raise "Type name must either of int, long, string float or double"
-    end
-  end
-
   # for fluent-plugin-td / td command to check table existence with import onlt user
   # @return [String]
   def self.create_empty_gz_data
@@ -241,6 +257,7 @@ private
       do_get(url, params, &block)
     end
   end
+
   # @param [String] url
   # @param [Hash] params
   # @yield [response]
@@ -295,7 +312,7 @@ private
           retry_delay *= 2
           redo # restart from beginning of do-while loop
         end
-      rescue Errno::ECONNREFUSED, Errno::ECONNRESET, Timeout::Error, EOFError, OpenSSL::SSL::SSLError, SocketError, IncompleteError => e
+      rescue Errno::ECONNREFUSED, Errno::ECONNRESET, Timeout::Error, EOFError, OpenSSL::SSL::SSLError, SocketError, IncompleteError, HTTPClient::TimeoutError => e
         if block_given?
           raise e
         end
@@ -591,18 +608,15 @@ private
   end
 
   def parse_error_response(res)
-    error = {}
-
     begin
-      js = JSON.load(res.body)
-      if js.nil?
-        error['message'] = res.reason
+      error = JSON.load(res.body)
+      if error
+        error['message']    = error['error'] unless error['message']
       else
-        error['message']    = js['message'] || js['error']
-        error['stacktrace'] = js['stacktrace']
+        error = {'message' => res.reason}
       end
     rescue JSON::ParserError
-      error['message'] = res.body
+      error = {'message' => res.body}
     end
 
     error
@@ -624,6 +638,7 @@ private
       when "404"
         NotFoundError
       when "409"
+        message = "#{message}: conflicts_with job:#{error["details"]["conflicts_with"]}" if error["details"] && error["details"]["conflicts_with"]
         AlreadyExistsError
       when "401"
         AuthError
@@ -635,11 +650,15 @@ private
       end
     end
 
-    if error_class.method_defined?(:api_backtrace)
-      raise error_class.new(message, error['stacktrace'])
+    exc = nil
+    if error_class.method_defined?(:conflicts_with) && error["details"] && error["details"]["conflicts_with"]
+      exc = error_class.new(message, error['stacktrace'], error["details"]["conflicts_with"])
+    elsif error_class.method_defined?(:api_backtrace)
+      exc = error_class.new(message, error['stacktrace'])
     else
-      raise error_class, message
+      exc = error_class.new(message)
     end
+    raise exc
   end
 
   if ''.respond_to?(:encode)

data/lib/td/client/api/bulk_import.rb

@@ -161,8 +161,7 @@ module BulkImport
       end
     end
     require File.expand_path('../compat_gzip_reader', File.dirname(__FILE__))
-    io = StringIO.new(Zlib::GzipReader.new(StringIO.new(body)).read)
-    u = MessagePack::Unpacker.new(io)
+    u = MessagePack::Unpacker.new(Zlib::GzipReader.new(StringIO.new(body)))
     if block
       begin
         u.each(&block)

data/lib/td/client/api/bulk_load.rb

@@ -116,10 +116,10 @@ module BulkLoad
     JSON.load(res.body)
   end
 
-  # name: String, job: Hash -> Hash
-  def bulk_load_update(name, job)
+  # name: String, settings: Hash -> Hash
+  def bulk_load_update(name, settings)
     path = session_path(name)
-    res = api { put(path, job.to_json) }
+    res = api { put(path, settings.to_json) }
     unless res.ok?
       raise_error("BulkLoadSession: #{name} update failed", res)
     end

data/lib/td/client/api/export.rb

@@ -22,5 +22,17 @@ module Export
     return js['job_id'].to_s
   end
 
+  # => jobId:String
+  # @param [String] target_job_id
+  # @param [Hash] opts
+  # @return [String] job_id
+  def result_export(target_job_id, opts={})
+    code, body, res = post("/v3/job/result_export/#{target_job_id}", opts)
+    if code != "200"
+      raise_error("Result Export failed", res)
+    end
+    js = checked_json(body, %w[job_id])
+    return js['job_id'].to_s
+  end
 end
 end

data/lib/td/client/api/import.rb

@@ -21,8 +21,9 @@ module Import
     opts = {}
     if @host == DEFAULT_ENDPOINT
       opts[:host] = DEFAULT_IMPORT_ENDPOINT
-    elsif @host == NEW_DEFAULT_ENDPOINT
-      opts[:host] = NEW_DEFAULT_IMPORT_ENDPOINT
+    elsif @host == TreasureData::API::OLD_ENDPOINT # backward compatibility
+      opts[:host] = 'api-import.treasure-data.com'
+      opts[:ssl] = false
     end
     code, body, res = put(path, stream, size, opts)
     if code[0] != ?2

data/lib/td/client/api/job.rb

@@ -36,9 +36,10 @@ module Job
       priority = m['priority']
       retry_limit = m['retry_limit']
       duration = m['duration']
+      num_records = m['num_records']
       result << [job_id, type, status, query, start_at, end_at, cpu_time,
                  result_size, result_url, priority, retry_limit, nil, database,
-                 duration]
+                 duration, num_records]
     }
     return result
   end
@@ -63,6 +64,8 @@ module Job
     end_at = js['end_at']
     cpu_time = js['cpu_time']
     result_size = js['result_size'] # compressed result size in msgpack.gz format
+    num_records = js['num_records']
+    duration = js['duration']
     result = js['result'] # result target URL
     hive_result_schema = (js['hive_result_schema'] || '')
     if hive_result_schema.empty?
@@ -97,7 +100,7 @@ module Job
     priority = js['priority']
     retry_limit = js['retry_limit']
     return [type, query, status, url, debug, start_at, end_at, cpu_time,
-            result_size, result, hive_result_schema, priority, retry_limit, nil, database]
+            result_size, result, hive_result_schema, priority, retry_limit, nil, database, duration, num_records]
   end
 
   # @param [String] job_id
@@ -115,14 +118,13 @@ module Job
   # @param [String] job_id
   # @return [Array]
   def job_result(job_id)
-    code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>'msgpack'})
-    if code != "200"
-      raise_error("Get job result failed", res)
-    end
     result = []
-    MessagePack::Unpacker.new.feed_each(body) {|row|
-      result << row
-    }
+    unpacker = MessagePack::Unpacker.new
+    job_result_download(job_id) do |chunk|
+      unpacker.feed_each(chunk) do |row|
+        result << row
+      end
+    end
     return result
   end
 
@@ -133,24 +135,17 @@ module Job
   # @param [IO] io
   # @param [Proc] block
   # @return [nil, String]
-  def job_result_format(job_id, format, io=nil, &block)
+  def job_result_format(job_id, format, io=nil)
     if io
-      infl = nil
-      code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>format}) {|res, chunk, current_total_chunk_size|
-        if res.code != 200
-          raise_error("Get job result failed", res)
-        end
-
-        infl ||= create_inflalte_or_null_inflate(res)
-
-        io.write infl.inflate(chunk)
-        block.call(current_total_chunk_size) if block_given?
-      }
+      job_result_download(job_id, format) do |chunk, total|
+        io.write chunk
+        yield total if block_given?
+      end
       nil
     else
-      code, body, res = get("/v3/job/result/#{e job_id}", {'format'=>format})
-      if code != "200"
-        raise_error("Get job result failed", res)
+      body = String.new
+      job_result_download(job_id, format) do |chunk|
+        body << chunk
       end
       body
     end
@@ -163,22 +158,11 @@ module Job
   # @return [nil]
   def job_result_each(job_id, &block)
     upkr = MessagePack::Unpacker.new
-    infl = nil
-
-    get("/v3/job/result/#{e job_id}", {'format'=>'msgpack'}) {|res, chunk, current_total_chunk_size|
-      if res.code != 200
-        raise_error("Get job result failed", res)
-      end
-
-      # default to decompressing the response since format is fixed to 'msgpack'
-      infl ||= create_inflate(res)
-
-      inflated_fragment = infl.inflate(chunk)
-      upkr.feed_each(inflated_fragment, &block)
-    }
+    # default to decompressing the response since format is fixed to 'msgpack'
+    job_result_download(job_id) do |chunk|
+      upkr.feed_each(chunk, &block)
+    end
     nil
-  ensure
-    infl.close if infl
   end
 
   # block is optional and must accept 1 argument
@@ -186,50 +170,30 @@ module Job
   # @param [String] job_id
   # @param [Proc] block
   # @return [nil]
-  def job_result_each_with_compr_size(job_id, &block)
+  def job_result_each_with_compr_size(job_id)
     upkr = MessagePack::Unpacker.new
-    infl = nil
-
-    get("/v3/job/result/#{e job_id}", {'format'=>'msgpack'}) {|res, chunk, current_total_chunk_size|
-      if res.code != 200
-        raise_error("Get job result failed", res)
-      end
-
-      # default to decompressing the response since format is fixed to 'msgpack'
-      infl ||= create_inflate(res)
-
-      inflated_fragment = infl.inflate(chunk)
-      upkr.feed_each(inflated_fragment) {|unpacked|
-        block.call(unpacked, current_total_chunk_size) if block_given?
+    # default to decompressing the response since format is fixed to 'msgpack'
+    job_result_download(job_id) do |chunk, total|
+      upkr.feed_each(chunk) {|unpacked|
+        yield unpacked, total if block_given?
       }
-    }
+    end
     nil
-  ensure
-    infl.close if infl
   end
 
   # @param [String] job_id
   # @param [String] format
   # @return [String]
-  def job_result_raw(job_id, format, io = nil, &block)
-    body = nil
-
-    get("/v3/job/result/#{e job_id}", {'format'=>format}) {|res, chunk, current_total_chunk_size|
-      if res.code != 200
-        raise_error("Get job result failed", res)
-      end
-
+  def job_result_raw(job_id, format, io = nil)
+    body = io ? nil : String.new
+    job_result_download(job_id, format, false) do |chunk, total|
       if io
         io.write(chunk)
-        block.call(current_total_chunk_size) if block_given?
+        yield total if block_given?
       else
-        if body
-          body += chunk
-        else
-          body = chunk
-        end
+        body << chunk
       end
-    }
+    end
     body
   end
 
@@ -287,6 +251,117 @@ module Job
 
   private
 
+  def validate_content_length_with_range(response, current_total_chunk_size)
+    if expected_size = response.header['Content-Range'][0]
+      expected_size = expected_size[/\d+$/].to_i
+    elsif expected_size = response.header['Content-Length'][0]
+      expected_size = expected_size.to_i
+    end
+
+    if expected_size.nil?
+    elsif current_total_chunk_size < expected_size
+      # too small
+      # NOTE:
+      #   ext/openssl raises EOFError in case where underlying connection
+      #   causes an error, but httpclient ignores it.
+      #   https://github.com/nahi/httpclient/blob/v3.2.8/lib/httpclient/session.rb#L1003
+      raise EOFError, 'httpclient IncompleteError'
+    elsif current_total_chunk_size > expected_size
+      # too large
+      raise_error("Get job result failed", response)
+    end
+  end
+
+  def job_result_download(job_id, format='msgpack', autodecode=true)
+    client, header = new_client
+    client.send_timeout = @send_timeout
+    client.receive_timeout = @read_timeout
+    header['Accept-Encoding'] = 'deflate, gzip'
+
+    url = build_endpoint("/v3/job/result/#{e job_id}", @host)
+    params = {'format' => format}
+
+    unless ENV['TD_CLIENT_DEBUG'].nil?
+      puts "DEBUG: REST GET call:"
+      puts "DEBUG:   header: " + header.to_s
+      puts "DEBUG:   url:    " + url.to_s
+      puts "DEBUG:   params: " + params.to_s
+    end
+
+    # up to 7 retries with exponential (base 2) back-off starting at 'retry_delay'
+    retry_delay = @retry_delay
+    cumul_retry_delay = 0
+    current_total_chunk_size = 0
+    infl = nil
+    begin # LOOP of Network/Server errors
+      response = nil
+      client.get(url, params, header) do |res, chunk|
+        unless response
+          case res.status
+          when 200
+            if current_total_chunk_size != 0
+              # try to resume but the server returns 200
+              raise_error("Get job result failed", res)
+            end
+          when 206 # resuming
+          else
+            if res.status/100 == 5 && cumul_retry_delay < @max_cumul_retry_delay
+              $stderr.puts "Error #{res.status}: #{get_error(res)}. Retrying after #{retry_delay} seconds..."
+              sleep retry_delay
+              cumul_retry_delay += retry_delay
+              retry_delay *= 2
+              redo
+            end
+            raise_error("Get job result failed", res)
+          end
+          if infl.nil? && autodecode
+            case res.header['Content-Encoding'][0].to_s.downcase
+            when 'gzip'
+              infl = Zlib::Inflate.new(Zlib::MAX_WBITS + 16)
+            when 'deflate'
+              infl = Zlib::Inflate.new
+            end
+          end
+        end
+        response = res
+        current_total_chunk_size += chunk.bytesize
+        chunk = infl.inflate(chunk) if infl
+        yield chunk, current_total_chunk_size
+      end
+
+      # completed?
+      validate_content_length_with_range(response, current_total_chunk_size)
+    rescue Errno::ECONNREFUSED, Errno::ECONNRESET, Timeout::Error, EOFError, OpenSSL::SSL::SSLError, SocketError => e
+      if response # at least a chunk is downloaded
+        if etag = response.header['ETag'][0]
+          header['If-Range'] = etag
+          header['Range'] = "bytes=#{current_total_chunk_size}-"
+        end
+      end
+
+      $stderr.print "#{e.class}: #{e.message}. "
+      if cumul_retry_delay < @max_cumul_retry_delay
+        $stderr.puts "Retrying after #{retry_delay} seconds..."
+        sleep retry_delay
+        cumul_retry_delay += retry_delay
+        retry_delay *= 2
+        retry
+      end
+      raise
+    end
+
+    unless ENV['TD_CLIENT_DEBUG'].nil?
+      puts "DEBUG: REST GET response:"
+      puts "DEBUG:   header: " + response.header.to_s
+      puts "DEBUG:   status: " + response.code.to_s
+      puts "DEBUG:   body:   " + response.body.to_s
+    end
+
+    nil
+  ensure
+    infl.close if infl
+  end
+
   class NullInflate
     def inflate(chunk)
       chunk