td-client 0.8.72 → 0.8.73

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a0f239495e96d72f93277ab75395f7e5f48596bd
-  data.tar.gz: b707ac4743e65d6bbd950a665f7889b64f1aaebd
+  metadata.gz: 7d0150f3829aa53453b7eea155e5144becda51a1
+  data.tar.gz: 94796cb468c35580f5117d3ea37e8c625ff3b0a3
 SHA512:
-  metadata.gz: 2e219700d74240c7080c8bfc6d717ea9d4efcaa027d9cecfd86c2e21c404d6d9aa4c4c4a150fb7058f8353a93492aa91500b7adc39a6dccc29eb698db1485802
-  data.tar.gz: 8099a83cdd1d941c350d30fd4c1f9d6eae658c799aae719d2e6477f75c22d829b23e4fd677e56d4a59da4bed511ad23342fccb7eaba27cdba92a60ce01b98a33
+  metadata.gz: 0076c76ac4c3c8f022753f4b26558391ff269a4a90afaf769b38395fca9b35989ac94c7951d1d24e9db8e966748b69465cd901cb5617acd162fcddff33ff325d
+  data.tar.gz: e75ea1598b88c4e41f929c0561588363a4e0c6a6a67eb4b62a3757870b5167f57181f56cd7e92798dadd836bf4695add7d954a74a577e6dfad1e3be216bd3989

data/lib/td/client.rb

@@ -93,15 +93,6 @@ class Client
     @api.create_log_table(db_name, table_name)
   end
 
-  # @param [String] db_name
-  # @param [String] table_name
-  # @param [String] primary_key
-  # @param [String] primary_key_type
-  # @return [true]
-  def create_item_table(db_name, table_name, primary_key, primary_key_type)
-    @api.create_item_table(db_name, table_name, primary_key, primary_key_type)
-  end
-
   # Swap table names
   #
   # @param [String] db_name
@@ -139,10 +130,10 @@ class Client
   # @return [Array] Tables
   def tables(db_name)
     m = @api.list_tables(db_name)
-    m.map {|table_name, (type, schema, count, created_at, updated_at, estimated_storage_size, last_import, last_log_timestamp, expire_days, primary_key, primary_key_type)|
+    m.map {|table_name, (type, schema, count, created_at, updated_at, estimated_storage_size, last_import, last_log_timestamp, expire_days)|
       schema = Schema.new.from_json(schema)
       Table.new(self, db_name, table_name, type, schema, count, created_at, updated_at,
-        estimated_storage_size, last_import, last_log_timestamp, expire_days, primary_key, primary_key_type)
+        estimated_storage_size, last_import, last_log_timestamp, expire_days)
     }
   end
 

data/lib/td/client/api.rb

@@ -42,6 +42,8 @@ class API
   NEW_DEFAULT_ENDPOINT = 'api.treasuredata.com'
   NEW_DEFAULT_IMPORT_ENDPOINT = 'api-import.treasuredata.com'
 
+  class IncompleteError < APIError; end
+
   # @param [String] apikey
   # @param [Hash] opts
   def initialize(apikey, opts={})
@@ -260,6 +262,27 @@ private
     end
   end
 
+  module CountReadBodyTotalSize
+    attr_reader :total_fragment_size
+
+    def read_body(&block)
+      return super if @total_fragment_size
+
+      if block_given?
+        @total_fragment_size = 0
+
+        super {|fragment|
+          @total_fragment_size += fragment.size
+          block.call(fragment)
+        }
+      else
+        super().tap {|body|
+          @total_fragment_size = body.size
+        }
+      end
+    end
+  end
+
   module DirectReadBodyMixin
     # @yield [fragment]
     def each_fragment(&block)
@@ -310,12 +333,18 @@ private
       begin
         if block
           response = http.request(request) {|res|
+            res.extend(CountReadBodyTotalSize)
             block.call(res)
           }
         else
           response = http.request(request)
         end
 
+        # XXX ext/openssl raises EOFError in case where underlying connection causes an error,
+        #     and msgpack-ruby that used in block handles it as an end of stream == no exception.
+        #     Therefor, check content size.
+        raise IncompleteError if @ssl && !completed_body?(response)
+
         status = response.code.to_i
         # retry if the HTTP error code is 500 or higher and we did not run out of retrying attempts
         if !block_given? && status >= 500 && cumul_retry_delay < @max_cumul_retry_delay
@@ -325,7 +354,7 @@ private
           retry_delay *= 2
           redo # restart from beginning of do-while loop
         end
-      rescue Errno::ECONNREFUSED, Errno::ECONNRESET, Timeout::Error, EOFError, OpenSSL::SSL::SSLError, SocketError => e
+      rescue Errno::ECONNREFUSED, Errno::ECONNRESET, Timeout::Error, EOFError, OpenSSL::SSL::SSLError, SocketError, IncompleteError => e
         if block_given?
           raise e
         end
@@ -371,6 +400,21 @@ private
     return [response.code, body, response]
   end
 
+  def completed_body?(response)
+    # NOTE If response doesn't have content_length, we assume it succeeds.
+    return true unless (content_length = response.header.content_length)
+
+    if response.body.instance_of? String
+      content_length == response.body.length
+    else
+      if response.respond_to? :total_fragment_size
+        content_length == response.total_fragment_size
+      else
+        true
+      end
+    end
+  end
+
   # @param [String] url
   # @param [Hash] params
   def post(url, params=nil)

data/lib/td/client/api/job.rb

@@ -156,13 +156,11 @@ module Job
           end
         end
 
-        total_compr_size = 0
         res.each_fragment {|fragment|
-          total_compr_size += fragment.size
           # uncompressed if the 'Content-Enconding' header is set in response
           fragment = infl.inflate(fragment) if ce
           io.write(fragment)
-          block.call(total_compr_size) if block_given?
+          block.call(res.total_fragment_size) if block_given?
         }
       }
       nil
@@ -216,11 +214,9 @@ module Job
       end
       upkr = MessagePack::Unpacker.new
       begin
-        total_compr_size = 0
         res.each_fragment {|fragment|
-          total_compr_size += fragment.size
           upkr.feed_each(infl.inflate(fragment)) {|unpacked|
-            block.call(unpacked, total_compr_size) if block_given?
+            block.call(unpacked, res.total_fragment_size) if block_given?
           }
         }
       ensure
@@ -244,11 +240,9 @@ module Job
       if io
         res.extend(DirectReadBodyMixin)
 
-        total_compr_size = 0
         res.each_fragment {|fragment|
-          total_compr_size += fragment.size
           io.write(fragment)
-          block.call(total_compr_size) if block_given?
+          block.call(res.total_fragment_size) if block_given?
         }
       else
         body = res.read_body

data/lib/td/client/api/table.rb

@@ -25,26 +25,11 @@ module Table
       estimated_storage_size = m['estimated_storage_size'].to_i
       schema = JSON.parse(m['schema'] || '[]')
       expire_days = m['expire_days']
-      primary_key = m['primary_key']
-      primary_key_type = m['primary_key_type']
-      result[name] = [type, schema, count, created_at, updated_at, estimated_storage_size, last_import, last_log_timestamp, expire_days, primary_key, primary_key_type]
+      result[name] = [type, schema, count, created_at, updated_at, estimated_storage_size, last_import, last_log_timestamp, expire_days]
     }
     return result
   end
 
-  # @param [String] db
-  # @param [String] table
-  # @param [String] type
-  # @return [true]
-  def create_log_or_item_table(db, table, type)
-    code, body, res = post("/v3/table/create/#{e db}/#{e table}/#{type}")
-    if code != "200"
-      raise_error("Create #{type} table failed", res)
-    end
-    return true
-  end
-  private :create_log_or_item_table
-
   # @param [String] db
   # @param [String] table
   # @return [true]
@@ -52,16 +37,6 @@ module Table
     create_table(db, table, :log)
   end
 
-  # @param [String] db
-  # @param [String] table
-  # @param [String] primary_key
-  # @param [String] primary_key_type
-  # @return [true]
-  def create_item_table(db, table, primary_key, primary_key_type)
-    params = {'primary_key' => primary_key, 'primary_key_type' => primary_key_type}
-    create_table(db, table, :item, params)
-  end
-
   # @param [String] db
   # @param [String] table
   # @param [String] type

data/lib/td/client/model.rb

@@ -101,12 +101,6 @@ class Database < Model
     @client.create_log_table(@db_name, name)
   end
 
-  # @param [String] name
-  # @return [true]
-  def create_item_table(name)
-    @client.create_item_table(@db_name, name)
-  end
-
   # @param [String] table_name
   # @return [Table]
   def table(table_name)
@@ -159,9 +153,7 @@ class Table < Model
   # @param [String] last_import
   # @param [String] last_log_timestamp
   # @param [Fixnum, String] expire_days
-  # @param [String] primary_key
-  # @param [String] primary_key_type
-  def initialize(client, db_name, table_name, type, schema, count, created_at=nil, updated_at=nil, estimated_storage_size=nil, last_import=nil, last_log_timestamp=nil, expire_days=nil, primary_key=nil, primary_key_type=nil)
+  def initialize(client, db_name, table_name, type, schema, count, created_at=nil, updated_at=nil, estimated_storage_size=nil, last_import=nil, last_log_timestamp=nil, expire_days=nil)
     super(client)
     @database = nil
     @db_name = db_name
@@ -175,8 +167,6 @@ class Table < Model
     @last_import = last_import
     @last_log_timestamp = last_log_timestamp
     @expire_days = expire_days
-    @primary_key = primary_key
-    @primary_key_type = primary_key_type
   end
 
   # @!attribute [r] type
@@ -185,9 +175,7 @@ class Table < Model
   # @!attribute [r] schema
   # @!attribute [r] count
   # @!attribute [r] estimated_storage_size
-  # @!attribute [r] primary_key
-  # @!attribute [r] primary_key_type
-  attr_reader :type, :db_name, :table_name, :schema, :count, :estimated_storage_size, :primary_key, :primary_key_type
+  attr_reader :type, :db_name, :table_name, :schema, :count, :estimated_storage_size
 
   alias database_name db_name
   alias name table_name

data/lib/td/client/version.rb

@@ -1,5 +1,5 @@
 module TreasureData
   class Client
-    VERSION = '0.8.72'
+    VERSION = '0.8.73'
   end
 end

data/spec/td/client/api_http_access_spec.rb

@@ -0,0 +1,50 @@
+require 'spec_helper'
+
+describe API do
+  describe '#completed_body?' do
+    let(:api) { TreasureData::API.new('')  }
+    let(:response) { double(:response) }
+
+    subject { api.__send__(:completed_body?, response) }
+
+    context 'response has no content length' do
+      before do
+        response.stub_chain(:header, :content_length).and_return(nil)
+      end
+
+      it { is_expected.to be }
+    end
+
+    context 'response has content length' do
+      let(:content_length) { 10 }
+
+      before do
+        response.stub_chain(:header, :content_length).and_return(content_length)
+      end
+
+      context 'content length equal body size' do
+        before do
+          response.stub(:body).and_return('a' * content_length)
+        end
+
+        it { is_expected.to be }
+      end
+
+      context 'content length lager than body size' do
+        before do
+          response.stub(:body).and_return('a' * (content_length - 1))
+        end
+
+        it { is_expected.not_to be }
+      end
+
+      context 'content length less than body size' do
+        before do
+          response.stub(:body).and_return('a' * (content_length + 1))
+        end
+
+        it { is_expected.not_to be }
+      end
+    end
+  end
+end

data/spec/td/client/api_spec.rb

@@ -172,5 +172,74 @@ describe API do
         API.validate_name("generic", 3, 128, 'a' * 128).should == 'a' * 128
       end
     end
+
+    describe 'checking GET API content length with ssl' do
+      include_context 'common helper'
+
+      let(:api) { API.new(nil, endpoint: endpoint) }
+      let :packed do
+        s = StringIO.new
+        Zlib::GzipWriter.wrap(s) do |f|
+          f << ['hello', 'world'].to_json
+        end
+        s.string
+      end
+
+      before do
+        stub_api_request(:get, '/v3/job/result/12345', ssl: ssl).
+          with(:query => {'format' => 'json'}).
+          to_return(
+            :headers => {'Content-Encoding' => 'gzip'}.merge(content_length),
+            :body => packed
+          )
+      end
+
+      subject (:get_api_call) {
+        api.job_result_format(12345, 'json', StringIO.new)
+      }
+
+      context 'without ssl' do
+        let(:endpoint) { "http://#{API::DEFAULT_ENDPOINT}" }
+        let(:ssl) { false }
+        let(:content_length) { {'Content-Length' => packed.size} }
+
+        it 'not called #completed_body?' do
+          api.should_not_receive(:completed_body?)
+
+          get_api_call
+        end
+      end
+
+      context 'with ssl' do
+        let(:endpoint) { "https://#{API::DEFAULT_ENDPOINT}" }
+        let(:ssl) { true }
+
+        context 'without Content-Length' do
+          let(:content_length) { {} }
+
+          it 'api accuess succeded' do
+            expect { get_api_call }.not_to raise_error
+          end
+        end
+
+        context 'with Content-Length' do
+          context 'macth Content-Length and body.size' do
+            let(:content_length) { {'Content-Length' => packed.size} }
+
+            it 'api accuess succeded' do
+              expect { get_api_call }.not_to raise_error
+            end
+          end
+
+          context 'not macth Content-Length and body.size' do
+            let(:content_length) { {'Content-Length' => packed.size + 1} }
+
+            it 'api accuess succeded' do
+              expect { get_api_call }.to raise_error(TreasureData::API::IncompleteError)
+            end
+          end
+        end
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: td-client
 version: !ruby/object:Gem::Version
-  version: 0.8.72
+  version: 0.8.73
 platform: ruby
 authors:
 - Treasure Data, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-27 00:00:00.000000000 Z
+date: 2015-07-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -232,6 +232,7 @@ files:
 - spec/spec_helper.rb
 - spec/td/client/access_control_api_spec.rb
 - spec/td/client/account_api_spec.rb
+- spec/td/client/api_http_access_spec.rb
 - spec/td/client/api_spec.rb
 - spec/td/client/api_ssl_connection_spec.rb
 - spec/td/client/bulk_import_spec.rb
@@ -276,6 +277,7 @@ summary: Treasure Data API library for Ruby
 test_files:
 - spec/td/client/access_control_api_spec.rb
 - spec/td/client/account_api_spec.rb
+- spec/td/client/api_http_access_spec.rb
 - spec/td/client/api_spec.rb
 - spec/td/client/api_ssl_connection_spec.rb
 - spec/td/client/bulk_import_spec.rb