td-client 0.8.72 → 0.8.73
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/td/client.rb +2 -11
- data/lib/td/client/api.rb +45 -1
- data/lib/td/client/api/job.rb +3 -9
- data/lib/td/client/api/table.rb +1 -26
- data/lib/td/client/model.rb +2 -14
- data/lib/td/client/version.rb +1 -1
- data/spec/td/client/api_http_access_spec.rb +50 -0
- data/spec/td/client/api_spec.rb +69 -0
- metadata +4 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7d0150f3829aa53453b7eea155e5144becda51a1
|
4
|
+
data.tar.gz: 94796cb468c35580f5117d3ea37e8c625ff3b0a3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0076c76ac4c3c8f022753f4b26558391ff269a4a90afaf769b38395fca9b35989ac94c7951d1d24e9db8e966748b69465cd901cb5617acd162fcddff33ff325d
|
7
|
+
data.tar.gz: e75ea1598b88c4e41f929c0561588363a4e0c6a6a67eb4b62a3757870b5167f57181f56cd7e92798dadd836bf4695add7d954a74a577e6dfad1e3be216bd3989
|
data/lib/td/client.rb
CHANGED
@@ -93,15 +93,6 @@ class Client
|
|
93
93
|
@api.create_log_table(db_name, table_name)
|
94
94
|
end
|
95
95
|
|
96
|
-
# @param [String] db_name
|
97
|
-
# @param [String] table_name
|
98
|
-
# @param [String] primary_key
|
99
|
-
# @param [String] primary_key_type
|
100
|
-
# @return [true]
|
101
|
-
def create_item_table(db_name, table_name, primary_key, primary_key_type)
|
102
|
-
@api.create_item_table(db_name, table_name, primary_key, primary_key_type)
|
103
|
-
end
|
104
|
-
|
105
96
|
# Swap table names
|
106
97
|
#
|
107
98
|
# @param [String] db_name
|
@@ -139,10 +130,10 @@ class Client
|
|
139
130
|
# @return [Array] Tables
|
140
131
|
def tables(db_name)
|
141
132
|
m = @api.list_tables(db_name)
|
142
|
-
m.map {|table_name, (type, schema, count, created_at, updated_at, estimated_storage_size, last_import, last_log_timestamp, expire_days
|
133
|
+
m.map {|table_name, (type, schema, count, created_at, updated_at, estimated_storage_size, last_import, last_log_timestamp, expire_days)|
|
143
134
|
schema = Schema.new.from_json(schema)
|
144
135
|
Table.new(self, db_name, table_name, type, schema, count, created_at, updated_at,
|
145
|
-
estimated_storage_size, last_import, last_log_timestamp, expire_days
|
136
|
+
estimated_storage_size, last_import, last_log_timestamp, expire_days)
|
146
137
|
}
|
147
138
|
end
|
148
139
|
|
data/lib/td/client/api.rb
CHANGED
@@ -42,6 +42,8 @@ class API
|
|
42
42
|
NEW_DEFAULT_ENDPOINT = 'api.treasuredata.com'
|
43
43
|
NEW_DEFAULT_IMPORT_ENDPOINT = 'api-import.treasuredata.com'
|
44
44
|
|
45
|
+
class IncompleteError < APIError; end
|
46
|
+
|
45
47
|
# @param [String] apikey
|
46
48
|
# @param [Hash] opts
|
47
49
|
def initialize(apikey, opts={})
|
@@ -260,6 +262,27 @@ private
|
|
260
262
|
end
|
261
263
|
end
|
262
264
|
|
265
|
+
module CountReadBodyTotalSize
|
266
|
+
attr_reader :total_fragment_size
|
267
|
+
|
268
|
+
def read_body(&block)
|
269
|
+
return super if @total_fragment_size
|
270
|
+
|
271
|
+
if block_given?
|
272
|
+
@total_fragment_size = 0
|
273
|
+
|
274
|
+
super {|fragment|
|
275
|
+
@total_fragment_size += fragment.size
|
276
|
+
block.call(fragment)
|
277
|
+
}
|
278
|
+
else
|
279
|
+
super().tap {|body|
|
280
|
+
@total_fragment_size = body.size
|
281
|
+
}
|
282
|
+
end
|
283
|
+
end
|
284
|
+
end
|
285
|
+
|
263
286
|
module DirectReadBodyMixin
|
264
287
|
# @yield [fragment]
|
265
288
|
def each_fragment(&block)
|
@@ -310,12 +333,18 @@ private
|
|
310
333
|
begin
|
311
334
|
if block
|
312
335
|
response = http.request(request) {|res|
|
336
|
+
res.extend(CountReadBodyTotalSize)
|
313
337
|
block.call(res)
|
314
338
|
}
|
315
339
|
else
|
316
340
|
response = http.request(request)
|
317
341
|
end
|
318
342
|
|
343
|
+
# XXX ext/openssl raises EOFError in case where underlying connection causes an error,
|
344
|
+
# and msgpack-ruby that used in block handles it as an end of stream == no exception.
|
345
|
+
# Therefor, check content size.
|
346
|
+
raise IncompleteError if @ssl && !completed_body?(response)
|
347
|
+
|
319
348
|
status = response.code.to_i
|
320
349
|
# retry if the HTTP error code is 500 or higher and we did not run out of retrying attempts
|
321
350
|
if !block_given? && status >= 500 && cumul_retry_delay < @max_cumul_retry_delay
|
@@ -325,7 +354,7 @@ private
|
|
325
354
|
retry_delay *= 2
|
326
355
|
redo # restart from beginning of do-while loop
|
327
356
|
end
|
328
|
-
rescue Errno::ECONNREFUSED, Errno::ECONNRESET, Timeout::Error, EOFError, OpenSSL::SSL::SSLError, SocketError => e
|
357
|
+
rescue Errno::ECONNREFUSED, Errno::ECONNRESET, Timeout::Error, EOFError, OpenSSL::SSL::SSLError, SocketError, IncompleteError => e
|
329
358
|
if block_given?
|
330
359
|
raise e
|
331
360
|
end
|
@@ -371,6 +400,21 @@ private
|
|
371
400
|
return [response.code, body, response]
|
372
401
|
end
|
373
402
|
|
403
|
+
def completed_body?(response)
|
404
|
+
# NOTE If response doesn't have content_length, we assume it succeeds.
|
405
|
+
return true unless (content_length = response.header.content_length)
|
406
|
+
|
407
|
+
if response.body.instance_of? String
|
408
|
+
content_length == response.body.length
|
409
|
+
else
|
410
|
+
if response.respond_to? :total_fragment_size
|
411
|
+
content_length == response.total_fragment_size
|
412
|
+
else
|
413
|
+
true
|
414
|
+
end
|
415
|
+
end
|
416
|
+
end
|
417
|
+
|
374
418
|
# @param [String] url
|
375
419
|
# @param [Hash] params
|
376
420
|
def post(url, params=nil)
|
data/lib/td/client/api/job.rb
CHANGED
@@ -156,13 +156,11 @@ module Job
|
|
156
156
|
end
|
157
157
|
end
|
158
158
|
|
159
|
-
total_compr_size = 0
|
160
159
|
res.each_fragment {|fragment|
|
161
|
-
total_compr_size += fragment.size
|
162
160
|
# uncompressed if the 'Content-Enconding' header is set in response
|
163
161
|
fragment = infl.inflate(fragment) if ce
|
164
162
|
io.write(fragment)
|
165
|
-
block.call(
|
163
|
+
block.call(res.total_fragment_size) if block_given?
|
166
164
|
}
|
167
165
|
}
|
168
166
|
nil
|
@@ -216,11 +214,9 @@ module Job
|
|
216
214
|
end
|
217
215
|
upkr = MessagePack::Unpacker.new
|
218
216
|
begin
|
219
|
-
total_compr_size = 0
|
220
217
|
res.each_fragment {|fragment|
|
221
|
-
total_compr_size += fragment.size
|
222
218
|
upkr.feed_each(infl.inflate(fragment)) {|unpacked|
|
223
|
-
block.call(unpacked,
|
219
|
+
block.call(unpacked, res.total_fragment_size) if block_given?
|
224
220
|
}
|
225
221
|
}
|
226
222
|
ensure
|
@@ -244,11 +240,9 @@ module Job
|
|
244
240
|
if io
|
245
241
|
res.extend(DirectReadBodyMixin)
|
246
242
|
|
247
|
-
total_compr_size = 0
|
248
243
|
res.each_fragment {|fragment|
|
249
|
-
total_compr_size += fragment.size
|
250
244
|
io.write(fragment)
|
251
|
-
block.call(
|
245
|
+
block.call(res.total_fragment_size) if block_given?
|
252
246
|
}
|
253
247
|
else
|
254
248
|
body = res.read_body
|
data/lib/td/client/api/table.rb
CHANGED
@@ -25,26 +25,11 @@ module Table
|
|
25
25
|
estimated_storage_size = m['estimated_storage_size'].to_i
|
26
26
|
schema = JSON.parse(m['schema'] || '[]')
|
27
27
|
expire_days = m['expire_days']
|
28
|
-
|
29
|
-
primary_key_type = m['primary_key_type']
|
30
|
-
result[name] = [type, schema, count, created_at, updated_at, estimated_storage_size, last_import, last_log_timestamp, expire_days, primary_key, primary_key_type]
|
28
|
+
result[name] = [type, schema, count, created_at, updated_at, estimated_storage_size, last_import, last_log_timestamp, expire_days]
|
31
29
|
}
|
32
30
|
return result
|
33
31
|
end
|
34
32
|
|
35
|
-
# @param [String] db
|
36
|
-
# @param [String] table
|
37
|
-
# @param [String] type
|
38
|
-
# @return [true]
|
39
|
-
def create_log_or_item_table(db, table, type)
|
40
|
-
code, body, res = post("/v3/table/create/#{e db}/#{e table}/#{type}")
|
41
|
-
if code != "200"
|
42
|
-
raise_error("Create #{type} table failed", res)
|
43
|
-
end
|
44
|
-
return true
|
45
|
-
end
|
46
|
-
private :create_log_or_item_table
|
47
|
-
|
48
33
|
# @param [String] db
|
49
34
|
# @param [String] table
|
50
35
|
# @return [true]
|
@@ -52,16 +37,6 @@ module Table
|
|
52
37
|
create_table(db, table, :log)
|
53
38
|
end
|
54
39
|
|
55
|
-
# @param [String] db
|
56
|
-
# @param [String] table
|
57
|
-
# @param [String] primary_key
|
58
|
-
# @param [String] primary_key_type
|
59
|
-
# @return [true]
|
60
|
-
def create_item_table(db, table, primary_key, primary_key_type)
|
61
|
-
params = {'primary_key' => primary_key, 'primary_key_type' => primary_key_type}
|
62
|
-
create_table(db, table, :item, params)
|
63
|
-
end
|
64
|
-
|
65
40
|
# @param [String] db
|
66
41
|
# @param [String] table
|
67
42
|
# @param [String] type
|
data/lib/td/client/model.rb
CHANGED
@@ -101,12 +101,6 @@ class Database < Model
|
|
101
101
|
@client.create_log_table(@db_name, name)
|
102
102
|
end
|
103
103
|
|
104
|
-
# @param [String] name
|
105
|
-
# @return [true]
|
106
|
-
def create_item_table(name)
|
107
|
-
@client.create_item_table(@db_name, name)
|
108
|
-
end
|
109
|
-
|
110
104
|
# @param [String] table_name
|
111
105
|
# @return [Table]
|
112
106
|
def table(table_name)
|
@@ -159,9 +153,7 @@ class Table < Model
|
|
159
153
|
# @param [String] last_import
|
160
154
|
# @param [String] last_log_timestamp
|
161
155
|
# @param [Fixnum, String] expire_days
|
162
|
-
|
163
|
-
# @param [String] primary_key_type
|
164
|
-
def initialize(client, db_name, table_name, type, schema, count, created_at=nil, updated_at=nil, estimated_storage_size=nil, last_import=nil, last_log_timestamp=nil, expire_days=nil, primary_key=nil, primary_key_type=nil)
|
156
|
+
def initialize(client, db_name, table_name, type, schema, count, created_at=nil, updated_at=nil, estimated_storage_size=nil, last_import=nil, last_log_timestamp=nil, expire_days=nil)
|
165
157
|
super(client)
|
166
158
|
@database = nil
|
167
159
|
@db_name = db_name
|
@@ -175,8 +167,6 @@ class Table < Model
|
|
175
167
|
@last_import = last_import
|
176
168
|
@last_log_timestamp = last_log_timestamp
|
177
169
|
@expire_days = expire_days
|
178
|
-
@primary_key = primary_key
|
179
|
-
@primary_key_type = primary_key_type
|
180
170
|
end
|
181
171
|
|
182
172
|
# @!attribute [r] type
|
@@ -185,9 +175,7 @@ class Table < Model
|
|
185
175
|
# @!attribute [r] schema
|
186
176
|
# @!attribute [r] count
|
187
177
|
# @!attribute [r] estimated_storage_size
|
188
|
-
|
189
|
-
# @!attribute [r] primary_key_type
|
190
|
-
attr_reader :type, :db_name, :table_name, :schema, :count, :estimated_storage_size, :primary_key, :primary_key_type
|
178
|
+
attr_reader :type, :db_name, :table_name, :schema, :count, :estimated_storage_size
|
191
179
|
|
192
180
|
alias database_name db_name
|
193
181
|
alias name table_name
|
data/lib/td/client/version.rb
CHANGED
@@ -0,0 +1,50 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe API do
|
4
|
+
describe '#completed_body?' do
|
5
|
+
let(:api) { TreasureData::API.new('') }
|
6
|
+
let(:response) { double(:response) }
|
7
|
+
|
8
|
+
subject { api.__send__(:completed_body?, response) }
|
9
|
+
|
10
|
+
context 'response has no content length' do
|
11
|
+
before do
|
12
|
+
response.stub_chain(:header, :content_length).and_return(nil)
|
13
|
+
end
|
14
|
+
|
15
|
+
it { is_expected.to be }
|
16
|
+
end
|
17
|
+
|
18
|
+
context 'response has content length' do
|
19
|
+
let(:content_length) { 10 }
|
20
|
+
|
21
|
+
before do
|
22
|
+
response.stub_chain(:header, :content_length).and_return(content_length)
|
23
|
+
end
|
24
|
+
|
25
|
+
context 'content length equal body size' do
|
26
|
+
before do
|
27
|
+
response.stub(:body).and_return('a' * content_length)
|
28
|
+
end
|
29
|
+
|
30
|
+
it { is_expected.to be }
|
31
|
+
end
|
32
|
+
|
33
|
+
context 'content length lager than body size' do
|
34
|
+
before do
|
35
|
+
response.stub(:body).and_return('a' * (content_length - 1))
|
36
|
+
end
|
37
|
+
|
38
|
+
it { is_expected.not_to be }
|
39
|
+
end
|
40
|
+
|
41
|
+
context 'content length less than body size' do
|
42
|
+
before do
|
43
|
+
response.stub(:body).and_return('a' * (content_length + 1))
|
44
|
+
end
|
45
|
+
|
46
|
+
it { is_expected.not_to be }
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
data/spec/td/client/api_spec.rb
CHANGED
@@ -172,5 +172,74 @@ describe API do
|
|
172
172
|
API.validate_name("generic", 3, 128, 'a' * 128).should == 'a' * 128
|
173
173
|
end
|
174
174
|
end
|
175
|
+
|
176
|
+
describe 'checking GET API content length with ssl' do
|
177
|
+
include_context 'common helper'
|
178
|
+
|
179
|
+
let(:api) { API.new(nil, endpoint: endpoint) }
|
180
|
+
let :packed do
|
181
|
+
s = StringIO.new
|
182
|
+
Zlib::GzipWriter.wrap(s) do |f|
|
183
|
+
f << ['hello', 'world'].to_json
|
184
|
+
end
|
185
|
+
s.string
|
186
|
+
end
|
187
|
+
|
188
|
+
before do
|
189
|
+
stub_api_request(:get, '/v3/job/result/12345', ssl: ssl).
|
190
|
+
with(:query => {'format' => 'json'}).
|
191
|
+
to_return(
|
192
|
+
:headers => {'Content-Encoding' => 'gzip'}.merge(content_length),
|
193
|
+
:body => packed
|
194
|
+
)
|
195
|
+
end
|
196
|
+
|
197
|
+
subject (:get_api_call) {
|
198
|
+
api.job_result_format(12345, 'json', StringIO.new)
|
199
|
+
}
|
200
|
+
|
201
|
+
context 'without ssl' do
|
202
|
+
let(:endpoint) { "http://#{API::DEFAULT_ENDPOINT}" }
|
203
|
+
let(:ssl) { false }
|
204
|
+
let(:content_length) { {'Content-Length' => packed.size} }
|
205
|
+
|
206
|
+
it 'not called #completed_body?' do
|
207
|
+
api.should_not_receive(:completed_body?)
|
208
|
+
|
209
|
+
get_api_call
|
210
|
+
end
|
211
|
+
end
|
212
|
+
|
213
|
+
context 'with ssl' do
|
214
|
+
let(:endpoint) { "https://#{API::DEFAULT_ENDPOINT}" }
|
215
|
+
let(:ssl) { true }
|
216
|
+
|
217
|
+
context 'without Content-Length' do
|
218
|
+
let(:content_length) { {} }
|
219
|
+
|
220
|
+
it 'api accuess succeded' do
|
221
|
+
expect { get_api_call }.not_to raise_error
|
222
|
+
end
|
223
|
+
end
|
224
|
+
|
225
|
+
context 'with Content-Length' do
|
226
|
+
context 'macth Content-Length and body.size' do
|
227
|
+
let(:content_length) { {'Content-Length' => packed.size} }
|
228
|
+
|
229
|
+
it 'api accuess succeded' do
|
230
|
+
expect { get_api_call }.not_to raise_error
|
231
|
+
end
|
232
|
+
end
|
233
|
+
|
234
|
+
context 'not macth Content-Length and body.size' do
|
235
|
+
let(:content_length) { {'Content-Length' => packed.size + 1} }
|
236
|
+
|
237
|
+
it 'api accuess succeded' do
|
238
|
+
expect { get_api_call }.to raise_error(TreasureData::API::IncompleteError)
|
239
|
+
end
|
240
|
+
end
|
241
|
+
end
|
242
|
+
end
|
243
|
+
end
|
175
244
|
end
|
176
245
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: td-client
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.8.
|
4
|
+
version: 0.8.73
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Treasure Data, Inc.
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-07-
|
11
|
+
date: 2015-07-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: msgpack
|
@@ -232,6 +232,7 @@ files:
|
|
232
232
|
- spec/spec_helper.rb
|
233
233
|
- spec/td/client/access_control_api_spec.rb
|
234
234
|
- spec/td/client/account_api_spec.rb
|
235
|
+
- spec/td/client/api_http_access_spec.rb
|
235
236
|
- spec/td/client/api_spec.rb
|
236
237
|
- spec/td/client/api_ssl_connection_spec.rb
|
237
238
|
- spec/td/client/bulk_import_spec.rb
|
@@ -276,6 +277,7 @@ summary: Treasure Data API library for Ruby
|
|
276
277
|
test_files:
|
277
278
|
- spec/td/client/access_control_api_spec.rb
|
278
279
|
- spec/td/client/account_api_spec.rb
|
280
|
+
- spec/td/client/api_http_access_spec.rb
|
279
281
|
- spec/td/client/api_spec.rb
|
280
282
|
- spec/td/client/api_ssl_connection_spec.rb
|
281
283
|
- spec/td/client/bulk_import_spec.rb
|