td-client 0.8.85 → 0.9.0dev1

data/lib/td/client/api/schedule.rb

@@ -14,7 +14,7 @@ module Schedule
     if code != "200"
       raise_error("Create schedule failed", res)
     end
-    js = checked_json(body)
+    js = checked_json(body, %w[start])
     return js['start']
   end
 

data/lib/td/client/api_error.rb

@@ -23,11 +23,6 @@ end
 
 # 409 API errors
 class AlreadyExistsError < APIError
-  attr_reader :conflicts_with
-  def initialize(error_message = nil, api_backtrace = nil, conflicts_with=nil)
-    super(error_message, api_backtrace)
-    @conflicts_with = conflicts_with
-  end
 end
 
 # 404 API errors

data/lib/td/client/model.rb

@@ -1,7 +1,7 @@
-require 'timeout'
 
 module TreasureData
 
+
 class Model
   # @param [TreasureData::Client] client
   def initialize(client)
@@ -269,62 +269,36 @@ class Table < Model
   def update_database!
     @database = @client.database(@db_name)
   end
-
-  # @return [String]
-  def inspect
-    %[#<%s:%#0#{1.size*2}x @db_name="%s" @table_name="%s">] %
-    [self.class.name, self.__id__*2, @db_name, @table_name]
-  end
 end
 
 class Schema
   class Field
     # @param [String] name
     # @param [String] type
-    # @param [String] sql_alias
-    def initialize(name, type, sql_alias=nil)
-      if name == 'v' || name == 'time'
-        raise ParameterValidationError, "Column name '#{name}' is reserved."
-      end
-      API.validate_column_name(name)
-      API.validate_sql_alias_name(sql_alias) if sql_alias
+    def initialize(name, type)
       @name = name
       @type = type
-      @sql_alias = sql_alias
     end
 
     # @!attribute [r] name
     # @!attribute [r] type
     attr_reader :name
     attr_reader :type
-    attr_reader :sql_alias
   end
 
-  # @param [String] columns
+  # @param [String] cols
   # @return [Schema]
-  def self.parse(columns)
-    schema = Schema.new
-    columns.each {|column|
-      unless /\A(?<name>.*)(?::(?<type>[^:@]+))(?:@(?<sql_alias>[^:@]+))?\z/ =~ column
-        raise ParameterValidationError, "type must be specified"
-      end
-      schema.add_field(name, type, sql_alias)
+  def self.parse(cols)
+    fields = cols.split(',').map {|col|
+      name, type, *_ = col.split(':')
+      Field.new(name, type)
     }
-    schema
+    Schema.new(fields)
   end
 
   # @param [Array] fields
   def initialize(fields=[])
     @fields = fields
-    @names = {}
-    @fields.each do |f|
-      raise ArgumentError, "Column name '#{f.name}' is duplicated." if @names.key?(f.name)
-      @names[f.name] = true
-      if f.sql_alias
-        raise ArgumentError, "SQL Column alias '#{f.sql_alias}' is duplicated." if @names.key?(f.sql_alias)
-        @names[f.sql_alias] = true
-      end
-    end
   end
 
   # @!attribute [r] fields
@@ -333,16 +307,8 @@ class Schema
   # @param [String] name
   # @param [String] type
   # @return [Array]
-  def add_field(name, type, sql_alias=nil)
-    if @names.key?(name)
-      raise ParameterValidationError, "Column name '#{name}' is duplicated."
-    end
-    @names[name] = true
-    if sql_alias && @names.key?(sql_alias)
-      raise ParameterValidationError, "SQL Column alias '#{sql_alias}' is duplicated."
-    end
-    @names[sql_alias] = true
-    @fields << Field.new(name, type, sql_alias)
+  def add_field(name, type)
+    @fields << Field.new(name, type)
   end
 
   # @param [Schema] schema
@@ -361,14 +327,14 @@ class Schema
 
   # @return [Array<Field>]
   def to_json(*args)
-    @fields.map {|f| f.sql_alias ? [f.name, f.type, f.sql_alias] : [f.name, f.type] }.to_json(*args)
+    @fields.map {|f| [f.name, f.type] }.to_json(*args)
   end
 
   # @param [Object] obj
   # @return [self]
   def from_json(obj)
     @fields = obj.map {|f|
-      Field.new(*f)
+      Field.new(f[0], f[1])
     }
     self
   end
@@ -383,6 +349,8 @@ class Job < Model
   STATUS_KILLED = "killed"
   FINISHED_STATUS = [STATUS_SUCCESS, STATUS_ERROR, STATUS_KILLED]
 
+  class TimeoutError < StandardError; end
+
   # @param [TreasureData::Client] client
   # @param [String] job_id
   # @param [String] type
@@ -402,10 +370,9 @@ class Job < Model
   # @param [String] org_name
   # @param [String] db_name
   # @param [Fixnum] duration
-  # @param [Fixnum] num_records
   def initialize(client, job_id, type, query, status=nil, url=nil, debug=nil, start_at=nil, end_at=nil, cpu_time=nil,
                  result_size=nil, result=nil, result_url=nil, hive_result_schema=nil, priority=nil, retry_limit=nil,
-                 org_name=nil, db_name=nil, duration=nil, num_records=nil)
+                 org_name=nil, db_name=nil, duration=nil)
     super(client)
     @job_id = job_id
     @type = type
@@ -424,7 +391,6 @@ class Job < Model
     @retry_limit = retry_limit
     @db_name = db_name
     @duration = duration
-    @num_records = num_records
   end
 
   # @!attribute [r] job_id
@@ -435,40 +401,21 @@ class Job < Model
   # @!attribute [r] org_name
   # @!attribute [r] db_name
   # @!attribute [r] duration
-  # @!attribute [r] num_records
   attr_reader :job_id, :type, :result_url
   attr_reader :priority, :retry_limit, :org_name, :db_name
-  attr_reader :duration, :num_records
-
-  # @option timeout [Integer,nil] timeout in second
-  # @option wait_interval [Integer,nil] interval in second of polling the job status
-  # @param detail [Boolean] update job detail or not
-  # @param verbose [Boolean] out retry log to stderr or not
-  def wait(*args)
-    opthash = Hash.try_convert(args.last)
-    if opthash
-      args.pop
-      detail = opthash.fetch(:detail, false)
-      verbose = opthash.fetch(:verbose, ENV['TD_CLIENT_DEBUG'])
-    end
-    timeout = args[0]
-    wait_interval = args[1] || 2
-    deadline = monotonic_clock + timeout if timeout
-    timeout_klass = Class.new(Exception)
-    begin
-      if timeout
-        if deadline <= monotonic_clock
-          raise timeout_klass, "timeout (#{timeout}) exceeded wait_interval=#{wait_interval}"
-        end
+  attr_reader :duration
+
+  def wait(timeout=nil, wait_interval=2)
+    started_at = Time.now
+    until finished?
+      if !timeout || ((Time.now - started_at).abs < timeout && wait_interval <= timeout)
+        sleep wait_interval
+        yield self if block_given?
+      else
+        raise TimeoutError, "timeout"
       end
-      sleep wait_interval
-      detail ? update_status! : update_progress!
-      yield self if block_given?
-    rescue timeout_klass
-      raise Timeout::Error, $!.message
-    rescue Timeout::Error, SystemCallError, EOFError, SocketError, HTTPClient::ConnectTimeoutError
-      $stderr.puts "ignore network error (#{$!}); retry..." if verbose
-    end until finished?
+      update_progress!
+    end
   end
 
   def kill!
@@ -617,7 +564,7 @@ class Job < Model
   def update_status!
     type, query, status, url, debug, start_at, end_at, cpu_time,
       result_size, result_url, hive_result_schema, priority, retry_limit,
-      org_name, db_name , num_records = @client.api.show_job(@job_id)
+      org_name, db_name = @client.api.show_job(@job_id)
     @query = query
     @status = status
     @url = url
@@ -631,18 +578,8 @@ class Job < Model
     @priority = priority
     @retry_limit = retry_limit
     @db_name = db_name
-    @num_records = num_records
     self
   end
-
-  private
-  def monotonic_clock
-    if defined?(Process.clock_gettime)
-      Process.clock_gettime(Process::CLOCK_MONOTONIC)
-    else
-      Time.now.to_i
-    end
-  end
 end
 
 

data/lib/td/client/version.rb

@@ -1,5 +1,5 @@
 module TreasureData
   class Client
-    VERSION = '0.8.85'
+    VERSION = '0.9.0dev1'
   end
 end

data/spec/spec_helper.rb

@@ -10,10 +10,10 @@ unless ENV['APPVEYOR']
       require 'simplecov'
       require 'coveralls'
 
-      SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter.new([
+      SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter[
         SimpleCov::Formatter::HTMLFormatter,
         Coveralls::SimpleCov::Formatter
-      ])
+      ]
       SimpleCov.start("test_frameworks")
     end
   rescue NameError
@@ -48,13 +48,13 @@ shared_context 'common helper' do
   end
 
   def stub_api_request(method, path, opts = nil)
-    scheme = 'https'
+    scheme = 'http'
     with_opts = {:headers => headers}
     if opts
-      scheme = 'http' if opts[:ssl] == false
+      scheme = 'https' if opts[:ssl]
       with_opts[:query] = opts[:query] if opts[:query]
     end
-    stub_request(method, "#{scheme}://api.treasuredata.com#{path}").with(with_opts)
+    stub_request(method, "#{scheme}://api.treasure-data.com#{path}").with(with_opts)
   end
 
   def e(s)

data/spec/td/client/account_api_spec.rb

@@ -14,7 +14,7 @@ describe 'Account API' do
     it 'returns account properties' do
       stub_api_request(:get, "/v3/account/show").
         to_return(:body => {'account' => {'id' => 1, 'plan' => 0, 'storage_size' => 2, 'guaranteed_cores' => 3, 'maximum_cores' => 4, 'created_at' => '2014-12-14T17:24:00+0900'}}.to_json)
-      expect(api.show_account).to eq([1, 0, 2, 3, 4, "2014-12-14T17:24:00+0900"])
+      api.show_account.should == [1, 0, 2, 3, 4, "2014-12-14T17:24:00+0900"]
     end
   end
 
@@ -25,10 +25,10 @@ describe 'Account API' do
       stub_api_request(:get, "/v3/account/core_utilization", :query => {'from' => from, 'to' => to}).
         to_return(:body => {'from' => from, 'to' => to, 'interval' => 1, 'history' => ['dummy']}.to_json)
       r = api.account_core_utilization(from, to)
-      expect(r[0]).to eq(Time.parse(from))
-      expect(r[1]).to eq(Time.parse(to))
-      expect(r[2]).to eq(1)
-      expect(r[3]).to eq(['dummy'])
+      r[0].should == Time.parse(from)
+      r[1].should == Time.parse(to)
+      r[2].should == 1
+      r[3].should == ['dummy']
     end
   end
 end

data/spec/td/client/api_spec.rb

@@ -40,26 +40,26 @@ describe API do
 
     it 'normalize_database_name should return normalized data' do
       INVALID_NAMES.each_pair {|ng,ok|
-        expect(API.normalize_database_name(ng)).to eq(ok)
+        API.normalize_database_name(ng).should == ok
       }
-      expect {
+      lambda {
         API.normalize_database_name('')
-      }.to raise_error(RuntimeError)
+      }.should raise_error(RuntimeError)
     end
 
     it 'normalize_table_name should return normalized data' do
       INVALID_NAMES.each_pair {|ng,ok|
-        expect(API.normalize_table_name(ng)).to eq(ok)
+        API.normalize_table_name(ng).should == ok
       }
       # empty
-      expect {
+      lambda {
         API.normalize_table_name('')
-      }.to raise_error(RuntimeError)
+      }.should raise_error(RuntimeError)
     end
 
     it 'normalize_database_name should return valid data' do
       VALID_NAMES.each {|ok|
-        expect(API.normalize_database_name(ok)).to eq(ok)
+        API.normalize_database_name(ok).should == ok
       }
     end
   end
@@ -68,19 +68,19 @@ describe API do
     describe "'validate_database_name'" do
       it 'should raise a ParameterValidationError exceptions' do
         INVALID_NAMES.each_pair {|ng,ok|
-          expect {
+          lambda {
             API.validate_database_name(ng)
-          }.to raise_error(ParameterValidationError)
+          }.should raise_error(ParameterValidationError)
         }
         # empty
-        expect {
+        lambda {
           API.validate_database_name('')
-        }.to raise_error(ParameterValidationError)
+        }.should raise_error(ParameterValidationError)
       end
 
       it 'should return valid data' do
         VALID_NAMES.each {|ok|
-          expect(API.validate_database_name(ok)).to eq(ok)
+          API.validate_database_name(ok).should == ok
         }
       end
     end
@@ -88,18 +88,18 @@ describe API do
     describe "'validate_table_name'" do
       it 'should raise a ParameterValidationError exception' do
         INVALID_NAMES.each_pair {|ng,ok|
-          expect {
+          lambda {
             API.validate_table_name(ng)
-          }.to raise_error(ParameterValidationError)
+          }.should raise_error(ParameterValidationError)
         }
-        expect {
+        lambda {
           API.validate_table_name('')
-        }.to raise_error(ParameterValidationError)
+        }.should raise_error(ParameterValidationError)
       end
 
       it 'should return valid data' do
         VALID_NAMES.each {|ok|
-          expect(API.validate_database_name(ok)).to eq(ok)
+          API.validate_database_name(ok).should == ok
         }
       end
     end
@@ -107,95 +107,69 @@ describe API do
     describe "'validate_result_set_name'" do
       it 'should raise a ParameterValidationError exception' do
         INVALID_NAMES.each_pair {|ng,ok|
-          expect {
+          lambda {
             API.validate_result_set_name(ng)
-          }.to raise_error(ParameterValidationError)
+          }.should raise_error(ParameterValidationError)
         }
         # empty
-        expect {
+        lambda {
           API.validate_result_set_name('')
-        }.to raise_error(ParameterValidationError)
+        }.should raise_error(ParameterValidationError)
       end
 
       it 'should return valid data' do
         VALID_NAMES.each {|ok|
-          expect(API.validate_result_set_name(ok)).to eq(ok)
+          API.validate_result_set_name(ok).should == ok
         }
       end
     end
 
     describe "'validate_column_name'" do
       it 'should raise a ParameterValidationError exception' do
-        [''].each { |ng|
-          expect {
+        ['/', '', 'D'].each { |ng|
+          lambda {
             API.validate_column_name(ng)
-          }.to raise_error(ParameterValidationError)
+          }.should raise_error(ParameterValidationError)
         }
       end
 
       it 'should return valid data' do
         VALID_NAMES.each {|ok|
-          expect(API.validate_column_name(ok)).to eq(ok)
-        }
-        ['a', 'a'*255].each {|ok|
-          expect(API.validate_column_name(ok)).to eq(ok)
+          API.validate_column_name(ok).should == ok
         }
+        # columns can be as short as 2 characters
+        API.validate_column_name('ab').should == 'ab'
       end
     end
 
-    describe "'validate_sql_alias_name'" do
-      it 'should raise a ParameterValidationError exception' do
-        [''].each { |ng|
-          expect{API.validate_sql_alias_name(ng)}.to raise_error(ParameterValidationError)
-        }
-        valid = ("a".."z").to_a.join<<("0".."9").to_a.join<<"_"
-        ("\x00".."\x7F").each { |ng|
-          next if valid.include?(ng)
-          expect{API.validate_sql_alias_name(ng)}.to raise_error(ParameterValidationError)
-        }
-      end
-
-      it 'should return valid data' do
-        VALID_NAMES.each {|ok|
-          expect(API.validate_sql_alias_name(ok)).to eq(ok)
-        }
-        ['a', '_a', 'a_', 'a'*512].each {|ok|
-          expect(API.validate_sql_alias_name(ok)).to eq(ok)
-        }
-      end
-    end
 
     describe "'generic validate_name'" do
       it 'should raise a ParameterValidationError exception' do
-        # wrong target
-        expect {
-          API.validate_name("", 3, 256, '')
-        }.to raise_error(ParameterValidationError)
         INVALID_NAMES.each_pair {|ng,ok|
-          expect {
+          lambda {
             API.validate_name("generic", 3, 256, ng)
-          }.to raise_error(ParameterValidationError)
+          }.should raise_error(ParameterValidationError)
         }
         # empty
-        expect {
+        lambda {
           API.validate_name("generic", 3, 256, '')
-        }.to raise_error(ParameterValidationError)
+        }.should raise_error(ParameterValidationError)
         # too short - one less than left limit
-        expect {
+        lambda {
           API.validate_name("generic", 3, 256, 'ab')
-        }.to raise_error(ParameterValidationError)
+        }.should raise_error(ParameterValidationError)
       end
 
       it 'should return valid data' do
         VALID_NAMES.each {|ok|
-          expect(API.validate_name("generic", 3, 256, ok)).to eq(ok)
+          API.validate_name("generic", 3, 256, ok).should == ok
         }
         # esplore left boundary
-        expect(API.validate_name("generic", 2, 256, 'ab')).to eq('ab')
-        expect(API.validate_name("generic", 1, 256, 'a')).to eq('a')
+        API.validate_name("generic", 2, 256, 'ab').should == 'ab'
+        API.validate_name("generic", 1, 256, 'a').should == 'a'
         # explore right boundary
-        expect(API.validate_name("generic", 3, 256, 'a' * 256)).to eq('a' * 256)
-        expect(API.validate_name("generic", 3, 128, 'a' * 128)).to eq('a' * 128)
+        API.validate_name("generic", 3, 256, 'a' * 256).should == 'a' * 256
+        API.validate_name("generic", 3, 128, 'a' * 128).should == 'a' * 128
       end
     end
 
@@ -204,7 +178,7 @@ describe API do
 
       let(:api) { API.new(nil, endpoint: endpoint) }
       let :packed do
-        s = StringIO.new(String.new)
+        s = StringIO.new
         Zlib::GzipWriter.wrap(s) do |f|
           f << ['hello', 'world'].to_json
         end
@@ -221,7 +195,7 @@ describe API do
       end
 
       subject (:get_api_call) {
-        api.job_result_format(12345, 'json', StringIO.new(String.new))
+        api.job_result_format(12345, 'json', StringIO.new)
       }
 
       context 'without ssl' do
@@ -230,7 +204,7 @@ describe API do
         let(:content_length) { {'Content-Length' => packed.size} }
 
         it 'not called #completed_body?' do
-          expect(api).not_to receive(:completed_body?)
+          api.should_not_receive(:completed_body?)
 
           get_api_call
         end
@@ -249,13 +223,21 @@ describe API do
         end
 
         context 'with Content-Length' do
-          context 'match Content-Length and body.size' do
+          context 'macth Content-Length and body.size' do
             let(:content_length) { {'Content-Length' => packed.size} }
 
             it 'api accuess succeded' do
               expect { get_api_call }.not_to raise_error
             end
           end
+
+          context 'not macth Content-Length and body.size' do
+            let(:content_length) { {'Content-Length' => packed.size + 1} }
+
+            it 'api accuess succeded' do
+              expect { get_api_call }.to raise_error(TreasureData::API::IncompleteError)
+            end
+          end
         end
       end
     end