td-client 0.9.0dev2 → 1.0.0

data/lib/td/client/api/schedule.rb

@@ -14,7 +14,7 @@ module Schedule
     if code != "200"
       raise_error("Create schedule failed", res)
     end
-    js = checked_json(body, %w[start])
+    js = checked_json(body)
     return js['start']
   end
 

data/lib/td/client/api_error.rb

@@ -23,6 +23,11 @@ end
 
 # 409 API errors
 class AlreadyExistsError < APIError
+  attr_reader :conflicts_with
+  def initialize(error_message = nil, api_backtrace = nil, conflicts_with=nil)
+    super(error_message, api_backtrace)
+    @conflicts_with = conflicts_with
+  end
 end
 
 # 404 API errors

data/lib/td/client/model.rb

@@ -1,7 +1,7 @@
+require 'timeout'
 
 module TreasureData
 
-
 class Model
   # @param [TreasureData::Client] client
   def initialize(client)
@@ -269,36 +269,62 @@ class Table < Model
   def update_database!
     @database = @client.database(@db_name)
   end
+
+  # @return [String]
+  def inspect
+    %[#<%s:%#0#{1.size*2}x @db_name="%s" @table_name="%s">] %
+    [self.class.name, self.__id__*2, @db_name, @table_name]
+  end
 end
 
 class Schema
   class Field
     # @param [String] name
     # @param [String] type
-    def initialize(name, type)
+    # @param [String] sql_alias
+    def initialize(name, type, sql_alias=nil)
+      if name == 'v' || name == 'time'
+        raise ParameterValidationError, "Column name '#{name}' is reserved."
+      end
+      API.validate_column_name(name)
+      API.validate_sql_alias_name(sql_alias) if sql_alias
       @name = name
       @type = type
+      @sql_alias = sql_alias
     end
 
     # @!attribute [r] name
     # @!attribute [r] type
     attr_reader :name
     attr_reader :type
+    attr_reader :sql_alias
   end
 
-  # @param [String] cols
+  # @param [String] columns
   # @return [Schema]
-  def self.parse(cols)
-    fields = cols.split(',').map {|col|
-      name, type, *_ = col.split(':')
-      Field.new(name, type)
+  def self.parse(columns)
+    schema = Schema.new
+    columns.each {|column|
+      unless /\A(?<name>.*)(?::(?<type>[^:@]+))(?:@(?<sql_alias>[^:@]+))?\z/ =~ column
+        raise ParameterValidationError, "type must be specified"
+      end
+      schema.add_field(name, type, sql_alias)
     }
-    Schema.new(fields)
+    schema
   end
 
   # @param [Array] fields
   def initialize(fields=[])
     @fields = fields
+    @names = {}
+    @fields.each do |f|
+      raise ArgumentError, "Column name '#{f.name}' is duplicated." if @names.key?(f.name)
+      @names[f.name] = true
+      if f.sql_alias
+        raise ArgumentError, "SQL Column alias '#{f.sql_alias}' is duplicated." if @names.key?(f.sql_alias)
+        @names[f.sql_alias] = true
+      end
+    end
   end
 
   # @!attribute [r] fields
@@ -307,8 +333,16 @@ class Schema
   # @param [String] name
   # @param [String] type
   # @return [Array]
-  def add_field(name, type)
-    @fields << Field.new(name, type)
+  def add_field(name, type, sql_alias=nil)
+    if @names.key?(name)
+      raise ParameterValidationError, "Column name '#{name}' is duplicated."
+    end
+    @names[name] = true
+    if sql_alias && @names.key?(sql_alias)
+      raise ParameterValidationError, "SQL Column alias '#{sql_alias}' is duplicated."
+    end
+    @names[sql_alias] = true
+    @fields << Field.new(name, type, sql_alias)
   end
 
   # @param [Schema] schema
@@ -327,14 +361,14 @@ class Schema
 
   # @return [Array<Field>]
   def to_json(*args)
-    @fields.map {|f| [f.name, f.type] }.to_json(*args)
+    @fields.map {|f| f.sql_alias ? [f.name, f.type, f.sql_alias] : [f.name, f.type] }.to_json(*args)
   end
 
   # @param [Object] obj
   # @return [self]
   def from_json(obj)
     @fields = obj.map {|f|
-      Field.new(f[0], f[1])
+      Field.new(*f)
     }
     self
   end
@@ -349,8 +383,6 @@ class Job < Model
   STATUS_KILLED = "killed"
   FINISHED_STATUS = [STATUS_SUCCESS, STATUS_ERROR, STATUS_KILLED]
 
-  class TimeoutError < StandardError; end
-
   # @param [TreasureData::Client] client
   # @param [String] job_id
   # @param [String] type
@@ -370,9 +402,10 @@ class Job < Model
   # @param [String] org_name
   # @param [String] db_name
   # @param [Fixnum] duration
+  # @param [Fixnum] num_records
   def initialize(client, job_id, type, query, status=nil, url=nil, debug=nil, start_at=nil, end_at=nil, cpu_time=nil,
                  result_size=nil, result=nil, result_url=nil, hive_result_schema=nil, priority=nil, retry_limit=nil,
-                 org_name=nil, db_name=nil, duration=nil)
+                 org_name=nil, db_name=nil, duration=nil, num_records=nil)
     super(client)
     @job_id = job_id
     @type = type
@@ -391,6 +424,7 @@ class Job < Model
     @retry_limit = retry_limit
     @db_name = db_name
     @duration = duration
+    @num_records = num_records
   end
 
   # @!attribute [r] job_id
@@ -401,21 +435,40 @@ class Job < Model
   # @!attribute [r] org_name
   # @!attribute [r] db_name
   # @!attribute [r] duration
+  # @!attribute [r] num_records
   attr_reader :job_id, :type, :result_url
   attr_reader :priority, :retry_limit, :org_name, :db_name
-  attr_reader :duration
-
-  def wait(timeout=nil, wait_interval=2)
-    started_at = Time.now
-    until finished?
-      if !timeout || ((Time.now - started_at).abs < timeout && wait_interval <= timeout)
-        sleep wait_interval
-        yield self if block_given?
-      else
-        raise TimeoutError, "timeout"
-      end
-      update_progress!
+  attr_reader :duration, :num_records
+
+  # @option timeout [Integer,nil] timeout in second
+  # @option wait_interval [Integer,nil] interval in second of polling the job status
+  # @param detail [Boolean] update job detail or not
+  # @param verbose [Boolean] out retry log to stderr or not
+  def wait(*args)
+    opthash = Hash.try_convert(args.last)
+    if opthash
+      args.pop
+      detail = opthash.fetch(:detail, false)
+      verbose = opthash.fetch(:verbose, ENV['TD_CLIENT_DEBUG'])
     end
+    timeout = args[0]
+    wait_interval = args[1] || 2
+    deadline = monotonic_clock + timeout if timeout
+    timeout_klass = Class.new(Exception)
+    begin
+      if timeout
+        if deadline <= monotonic_clock
+          raise timeout_klass, "timeout (#{timeout}) exceeded wait_interval=#{wait_interval}"
+        end
+      end
+      sleep wait_interval
+      detail ? update_status! : update_progress!
+      yield self if block_given?
+    rescue timeout_klass
+      raise Timeout::Error, $!.message
+    rescue Timeout::Error, SystemCallError, EOFError, SocketError, HTTPClient::ConnectTimeoutError
+      $stderr.puts "ignore network error (#{$!}); retry..." if verbose
+    end until finished?
   end
 
   def kill!
@@ -564,7 +617,7 @@ class Job < Model
   def update_status!
     type, query, status, url, debug, start_at, end_at, cpu_time,
       result_size, result_url, hive_result_schema, priority, retry_limit,
-      org_name, db_name = @client.api.show_job(@job_id)
+      org_name, db_name , duration, num_records = @client.api.show_job(@job_id)
     @query = query
     @status = status
     @url = url
@@ -578,8 +631,19 @@ class Job < Model
     @priority = priority
     @retry_limit = retry_limit
     @db_name = db_name
+    @duration = duration
+    @num_records = num_records
     self
   end
+
+  private
+  def monotonic_clock
+    if defined?(Process.clock_gettime)
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    else
+      Time.now.to_i
+    end
+  end
 end
 
 

data/lib/td/client/version.rb

@@ -1,5 +1,5 @@
 module TreasureData
   class Client
-    VERSION = '0.9.0dev2'
+    VERSION = '1.0.0'
   end
 end

data/spec/spec_helper.rb

@@ -10,10 +10,10 @@ unless ENV['APPVEYOR']
       require 'simplecov'
       require 'coveralls'
 
-      SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter[
+      SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter.new([
         SimpleCov::Formatter::HTMLFormatter,
         Coveralls::SimpleCov::Formatter
-      ]
+      ])
       SimpleCov.start("test_frameworks")
     end
   rescue NameError
@@ -48,13 +48,13 @@ shared_context 'common helper' do
   end
 
   def stub_api_request(method, path, opts = nil)
-    scheme = 'http'
+    scheme = 'https'
     with_opts = {:headers => headers}
     if opts
-      scheme = 'https' if opts[:ssl]
+      scheme = 'http' if opts[:ssl] == false
       with_opts[:query] = opts[:query] if opts[:query]
     end
-    stub_request(method, "#{scheme}://api.treasure-data.com#{path}").with(with_opts)
+    stub_request(method, "#{scheme}://api.treasuredata.com#{path}").with(with_opts)
   end
 
   def e(s)

data/spec/td/client/account_api_spec.rb

@@ -14,7 +14,7 @@ describe 'Account API' do
     it 'returns account properties' do
       stub_api_request(:get, "/v3/account/show").
         to_return(:body => {'account' => {'id' => 1, 'plan' => 0, 'storage_size' => 2, 'guaranteed_cores' => 3, 'maximum_cores' => 4, 'created_at' => '2014-12-14T17:24:00+0900'}}.to_json)
-      api.show_account.should == [1, 0, 2, 3, 4, "2014-12-14T17:24:00+0900"]
+      expect(api.show_account).to eq([1, 0, 2, 3, 4, "2014-12-14T17:24:00+0900"])
     end
   end
 
@@ -25,10 +25,10 @@ describe 'Account API' do
       stub_api_request(:get, "/v3/account/core_utilization", :query => {'from' => from, 'to' => to}).
         to_return(:body => {'from' => from, 'to' => to, 'interval' => 1, 'history' => ['dummy']}.to_json)
       r = api.account_core_utilization(from, to)
-      r[0].should == Time.parse(from)
-      r[1].should == Time.parse(to)
-      r[2].should == 1
-      r[3].should == ['dummy']
+      expect(r[0]).to eq(Time.parse(from))
+      expect(r[1]).to eq(Time.parse(to))
+      expect(r[2]).to eq(1)
+      expect(r[3]).to eq(['dummy'])
     end
   end
 end

data/spec/td/client/api_error_spec.rb

@@ -0,0 +1,77 @@
+require 'spec_helper'
+
+describe APIError do
+  let (:message){ 'message' }
+  let (:api_backtrace){ double('api_backtrace') }
+  describe 'new' do
+    context '' do
+      it do
+        exc = APIError.new(message, api_backtrace)
+        expect(exc).to be_an(APIError)
+        expect(exc.message).to eq message
+        expect(exc.api_backtrace).to eq api_backtrace
+      end
+    end
+    context 'api_backtrace is ""' do
+      let (:api_backtrace){ '' }
+      it do
+        exc = APIError.new(message, api_backtrace)
+        expect(exc).to be_an(APIError)
+        expect(exc.message).to eq message
+        expect(exc.api_backtrace).to be_nil
+      end
+    end
+    context 'api_backtrace is nil' do
+      let (:api_backtrace){ nil }
+      it do
+        exc = APIError.new(message, api_backtrace)
+        expect(exc).to be_an(APIError)
+        expect(exc.message).to eq message
+        expect(exc.api_backtrace).to be_nil
+      end
+    end
+  end
+end
+
+describe AlreadyExistsError do
+  let (:message){ 'message' }
+  let (:api_backtrace){ double('api_backtrace') }
+  let (:conflicts_with){ '12345' }
+  describe 'new' do
+    context '' do
+      it do
+        exc = AlreadyExistsError.new(message, api_backtrace)
+        expect(exc).to be_an(AlreadyExistsError)
+        expect(exc.message).to eq message
+        expect(exc.api_backtrace).to eq api_backtrace
+      end
+    end
+    context 'api_backtrace is ""' do
+      let (:api_backtrace){ '' }
+      it do
+        exc = AlreadyExistsError.new(message, api_backtrace)
+        expect(exc).to be_an(AlreadyExistsError)
+        expect(exc.message).to eq message
+        expect(exc.api_backtrace).to be_nil
+      end
+    end
+    context 'api_backtrace is nil' do
+      let (:api_backtrace){ nil }
+      it do
+        exc = AlreadyExistsError.new(message, api_backtrace)
+        expect(exc).to be_an(AlreadyExistsError)
+        expect(exc.message).to eq message
+        expect(exc.api_backtrace).to be_nil
+      end
+    end
+    context 'conflict' do
+      it do
+        exc = AlreadyExistsError.new(message, api_backtrace, conflicts_with)
+        expect(exc).to be_an(AlreadyExistsError)
+        expect(exc.message).to eq message
+        expect(exc.api_backtrace).to eq api_backtrace
+        expect(exc.conflicts_with).to eq conflicts_with
+      end
+    end
+  end
+end

data/spec/td/client/api_spec.rb

@@ -28,38 +28,44 @@ describe API do
       h = {'key' => 1111111111111111111111111111111111}
       unpacked = MessagePack.unpack(API.normalized_msgpack(h))
       expect(unpacked['key']).to eq(h['key'].to_s)
+
+      h = {'key' => -1111111111111111111111111111111111}
+      unpacked = MessagePack.unpack(API.normalized_msgpack(h))
+      expect(unpacked['key']).to eq(h['key'].to_s)
     end
 
     it 'normalized_msgpack with out argument should convert Bignum into String' do
-      h = {'key' => 1111111111111111111111111111111111}
+      h = {'key' => 1111111111111111111111111111111111, 'key2' => -1111111111111111111111111111111111, 'key3' => 0}
       out = ''
       API.normalized_msgpack(h, out)
       unpacked = MessagePack.unpack(out)
       expect(unpacked['key']).to eq(h['key'].to_s)
+      expect(unpacked['key2']).to eq(h['key2'].to_s)
+      expect(unpacked['key3']).to eq(h['key3']) # don't touch non-too-big integer
     end
 
     it 'normalize_database_name should return normalized data' do
       INVALID_NAMES.each_pair {|ng,ok|
-        API.normalize_database_name(ng).should == ok
+        expect(API.normalize_database_name(ng)).to eq(ok)
       }
-      lambda {
+      expect {
         API.normalize_database_name('')
-      }.should raise_error(RuntimeError)
+      }.to raise_error(RuntimeError)
     end
 
     it 'normalize_table_name should return normalized data' do
       INVALID_NAMES.each_pair {|ng,ok|
-        API.normalize_table_name(ng).should == ok
+        expect(API.normalize_table_name(ng)).to eq(ok)
       }
       # empty
-      lambda {
+      expect {
         API.normalize_table_name('')
-      }.should raise_error(RuntimeError)
+      }.to raise_error(RuntimeError)
     end
 
     it 'normalize_database_name should return valid data' do
       VALID_NAMES.each {|ok|
-        API.normalize_database_name(ok).should == ok
+        expect(API.normalize_database_name(ok)).to eq(ok)
       }
     end
   end
@@ -68,19 +74,19 @@ describe API do
     describe "'validate_database_name'" do
       it 'should raise a ParameterValidationError exceptions' do
         INVALID_NAMES.each_pair {|ng,ok|
-          lambda {
+          expect {
             API.validate_database_name(ng)
-          }.should raise_error(ParameterValidationError)
+          }.to raise_error(ParameterValidationError)
         }
         # empty
-        lambda {
+        expect {
           API.validate_database_name('')
-        }.should raise_error(ParameterValidationError)
+        }.to raise_error(ParameterValidationError)
       end
 
       it 'should return valid data' do
         VALID_NAMES.each {|ok|
-          API.validate_database_name(ok).should == ok
+          expect(API.validate_database_name(ok)).to eq(ok)
         }
       end
     end
@@ -88,18 +94,18 @@ describe API do
     describe "'validate_table_name'" do
       it 'should raise a ParameterValidationError exception' do
         INVALID_NAMES.each_pair {|ng,ok|
-          lambda {
+          expect {
             API.validate_table_name(ng)
-          }.should raise_error(ParameterValidationError)
+          }.to raise_error(ParameterValidationError)
         }
-        lambda {
+        expect {
           API.validate_table_name('')
-        }.should raise_error(ParameterValidationError)
+        }.to raise_error(ParameterValidationError)
       end
 
       it 'should return valid data' do
         VALID_NAMES.each {|ok|
-          API.validate_database_name(ok).should == ok
+          expect(API.validate_database_name(ok)).to eq(ok)
         }
       end
     end
@@ -107,69 +113,95 @@ describe API do
     describe "'validate_result_set_name'" do
       it 'should raise a ParameterValidationError exception' do
         INVALID_NAMES.each_pair {|ng,ok|
-          lambda {
+          expect {
             API.validate_result_set_name(ng)
-          }.should raise_error(ParameterValidationError)
+          }.to raise_error(ParameterValidationError)
         }
         # empty
-        lambda {
+        expect {
           API.validate_result_set_name('')
-        }.should raise_error(ParameterValidationError)
+        }.to raise_error(ParameterValidationError)
       end
 
       it 'should return valid data' do
         VALID_NAMES.each {|ok|
-          API.validate_result_set_name(ok).should == ok
+          expect(API.validate_result_set_name(ok)).to eq(ok)
         }
       end
     end
 
     describe "'validate_column_name'" do
       it 'should raise a ParameterValidationError exception' do
-        ['/', '', 'D'].each { |ng|
-          lambda {
+        [''].each { |ng|
+          expect {
             API.validate_column_name(ng)
-          }.should raise_error(ParameterValidationError)
+          }.to raise_error(ParameterValidationError)
         }
       end
 
       it 'should return valid data' do
         VALID_NAMES.each {|ok|
-          API.validate_column_name(ok).should == ok
+          expect(API.validate_column_name(ok)).to eq(ok)
+        }
+        ['a', 'a'*255].each {|ok|
+          expect(API.validate_column_name(ok)).to eq(ok)
         }
-        # columns can be as short as 2 characters
-        API.validate_column_name('ab').should == 'ab'
       end
     end
 
+    describe "'validate_sql_alias_name'" do
+      it 'should raise a ParameterValidationError exception' do
+        [''].each { |ng|
+          expect{API.validate_sql_alias_name(ng)}.to raise_error(ParameterValidationError)
+        }
+        valid = ("a".."z").to_a.join<<("0".."9").to_a.join<<"_"
+        ("\x00".."\x7F").each { |ng|
+          next if valid.include?(ng)
+          expect{API.validate_sql_alias_name(ng)}.to raise_error(ParameterValidationError)
+        }
+      end
+
+      it 'should return valid data' do
+        VALID_NAMES.each {|ok|
+          expect(API.validate_sql_alias_name(ok)).to eq(ok)
+        }
+        ['a', '_a', 'a_', 'a'*512].each {|ok|
+          expect(API.validate_sql_alias_name(ok)).to eq(ok)
+        }
+      end
+    end
 
     describe "'generic validate_name'" do
       it 'should raise a ParameterValidationError exception' do
+        # wrong target
+        expect {
+          API.validate_name("", 3, 256, '')
+        }.to raise_error(ParameterValidationError)
         INVALID_NAMES.each_pair {|ng,ok|
-          lambda {
+          expect {
             API.validate_name("generic", 3, 256, ng)
-          }.should raise_error(ParameterValidationError)
+          }.to raise_error(ParameterValidationError)
         }
         # empty
-        lambda {
+        expect {
           API.validate_name("generic", 3, 256, '')
-        }.should raise_error(ParameterValidationError)
+        }.to raise_error(ParameterValidationError)
         # too short - one less than left limit
-        lambda {
+        expect {
           API.validate_name("generic", 3, 256, 'ab')
-        }.should raise_error(ParameterValidationError)
+        }.to raise_error(ParameterValidationError)
       end
 
       it 'should return valid data' do
         VALID_NAMES.each {|ok|
-          API.validate_name("generic", 3, 256, ok).should == ok
+          expect(API.validate_name("generic", 3, 256, ok)).to eq(ok)
         }
         # esplore left boundary
-        API.validate_name("generic", 2, 256, 'ab').should == 'ab'
-        API.validate_name("generic", 1, 256, 'a').should == 'a'
+        expect(API.validate_name("generic", 2, 256, 'ab')).to eq('ab')
+        expect(API.validate_name("generic", 1, 256, 'a')).to eq('a')
         # explore right boundary
-        API.validate_name("generic", 3, 256, 'a' * 256).should == 'a' * 256
-        API.validate_name("generic", 3, 128, 'a' * 128).should == 'a' * 128
+        expect(API.validate_name("generic", 3, 256, 'a' * 256)).to eq('a' * 256)
+        expect(API.validate_name("generic", 3, 128, 'a' * 128)).to eq('a' * 128)
       end
     end
 
@@ -178,7 +210,7 @@ describe API do
 
       let(:api) { API.new(nil, endpoint: endpoint) }
       let :packed do
-        s = StringIO.new
+        s = StringIO.new(String.new)
         Zlib::GzipWriter.wrap(s) do |f|
           f << ['hello', 'world'].to_json
         end
@@ -195,7 +227,7 @@ describe API do
       end
 
       subject (:get_api_call) {
-        api.job_result_format(12345, 'json', StringIO.new)
+        api.job_result_format(12345, 'json', StringIO.new(String.new))
       }
 
       context 'without ssl' do
@@ -204,7 +236,7 @@ describe API do
         let(:content_length) { {'Content-Length' => packed.size} }
 
         it 'not called #completed_body?' do
-          api.should_not_receive(:completed_body?)
+          expect(api).not_to receive(:completed_body?)
 
           get_api_call
         end
@@ -223,21 +255,13 @@ describe API do
         end
 
         context 'with Content-Length' do
-          context 'macth Content-Length and body.size' do
+          context 'match Content-Length and body.size' do
             let(:content_length) { {'Content-Length' => packed.size} }
 
             it 'api accuess succeded' do
               expect { get_api_call }.not_to raise_error
             end
           end
-
-          context 'not macth Content-Length and body.size' do
-            let(:content_length) { {'Content-Length' => packed.size + 1} }
-
-            it 'api accuess succeded' do
-              expect { get_api_call }.to raise_error(TreasureData::API::IncompleteError)
-            end
-          end
         end
       end
     end