td-client 0.8.68 → 0.8.69

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/td/client/api.rb CHANGED
@@ -3,6 +3,7 @@ require 'td/client/version'
3
3
  require 'td/client/api/access_control'
4
4
  require 'td/client/api/account'
5
5
  require 'td/client/api/bulk_import'
6
+ require 'td/client/api/bulk_load'
6
7
  require 'td/client/api/database'
7
8
  require 'td/client/api/export'
8
9
  require 'td/client/api/import'
@@ -23,6 +24,7 @@ class API
23
24
  include API::AccessControl
24
25
  include API::Account
25
26
  include API::BulkImport
27
+ include API::BulkLoad
26
28
  include API::Database
27
29
  include API::Export
28
30
  include API::Import
@@ -40,6 +42,8 @@ class API
40
42
  NEW_DEFAULT_ENDPOINT = 'api.treasuredata.com'
41
43
  NEW_DEFAULT_IMPORT_ENDPOINT = 'api-import.treasuredata.com'
42
44
 
45
+ # @param [String] apikey
46
+ # @param [Hash] opts
43
47
  def initialize(apikey, opts={})
44
48
  require 'json'
45
49
  require 'time'
@@ -65,6 +69,7 @@ class API
65
69
  @read_timeout = opts[:read_timeout] || 600
66
70
  @send_timeout = opts[:send_timeout] || 600
67
71
  @retry_post_requests = opts[:retry_post_requests] || false
72
+ @retry_delay = opts[:retry_delay] || 5
68
73
  @max_cumul_retry_delay = opts[:max_cumul_retry_delay] || 600
69
74
 
70
75
  case uri.scheme
@@ -110,12 +115,16 @@ class API
110
115
  end
111
116
 
112
117
  @headers = opts[:headers] || {}
118
+ @api = api_client("#{@ssl ? 'https' : 'http'}://#{@host}:#{@port}")
113
119
  end
114
120
 
115
121
  # TODO error check & raise appropriate errors
116
122
 
123
+ # @!attribute [r] apikey
117
124
  attr_reader :apikey
118
125
 
126
+ # @param [Hash] record
127
+ # @param [IO] out
119
128
  def self.normalized_msgpack(record, out = nil)
120
129
  record.keys.each { |k|
121
130
  v = record[k]
@@ -126,6 +135,10 @@ class API
126
135
  record.to_msgpack(out)
127
136
  end
128
137
 
138
+ # @param [String] target
139
+ # @param [Fixnum] min_len
140
+ # @param [Fixnum] max_len
141
+ # @param [String] name
129
142
  def self.validate_name(target, min_len, max_len, name)
130
143
  if !target.instance_of?(String) || target.empty?
131
144
  raise ParameterValidationError,
@@ -150,22 +163,27 @@ class API
150
163
  name
151
164
  end
152
165
 
166
+ # @param [String] name
153
167
  def self.validate_database_name(name)
154
168
  validate_name("database", 3, 255, name)
155
169
  end
156
170
 
171
+ # @param [String] name
157
172
  def self.validate_table_name(name)
158
173
  validate_name("table", 3, 255, name)
159
174
  end
160
175
 
176
+ # @param [String] name
161
177
  def self.validate_result_set_name(name)
162
178
  validate_name("result set", 3, 255, name)
163
179
  end
164
180
 
181
+ # @param [String] name
165
182
  def self.validate_column_name(name)
166
183
  validate_name("column", 1, 255, name)
167
184
  end
168
185
 
186
+ # @param [String] name
169
187
  def self.normalize_database_name(name)
170
188
  name = name.to_s
171
189
  if name.empty?
@@ -182,11 +200,13 @@ class API
182
200
  name
183
201
  end
184
202
 
203
+ # @param [String] name
185
204
  def self.normalize_table_name(name)
186
205
  normalize_database_name(name)
187
206
  end
188
207
 
189
208
  # TODO support array types
209
+ # @param [String] name
190
210
  def self.normalize_type_name(name)
191
211
  case name
192
212
  when /int/i, /integer/i
@@ -205,12 +225,14 @@ class API
205
225
  end
206
226
 
207
227
  # for fluent-plugin-td / td command to check table existence with import onlt user
228
+ # @return [String]
208
229
  def self.create_empty_gz_data
209
230
  io = StringIO.new
210
231
  Zlib::GzipWriter.new(io).close
211
232
  io.string
212
233
  end
213
234
 
235
+ # @param [String] ssl_ca_file
214
236
  def ssl_ca_file=(ssl_ca_file)
215
237
  @ssl_ca_file = ssl_ca_file
216
238
  end
@@ -220,6 +242,7 @@ private
220
242
  module DeflateReadBodyMixin
221
243
  attr_accessor :gzip
222
244
 
245
+ # @yield [fragment]
223
246
  def each_fragment(&block)
224
247
  if @gzip
225
248
  infl = Zlib::Inflate.new(Zlib::MAX_WBITS + 16)
@@ -238,17 +261,24 @@ private
238
261
  end
239
262
 
240
263
  module DirectReadBodyMixin
264
+ # @yield [fragment]
241
265
  def each_fragment(&block)
242
266
  read_body(&block)
243
267
  end
244
268
  end
245
269
 
270
+ # @param [String] url
271
+ # @param [Hash] params
272
+ # @yield [response]
246
273
  def get(url, params=nil, &block)
247
274
  guard_no_sslv3 do
248
275
  do_get(url, params, &block)
249
276
  end
250
277
  end
251
278
 
279
+ # @param [String] url
280
+ # @param [Hash] params
281
+ # @yield [response]
252
282
  def do_get(url, params=nil, &block)
253
283
  http, header = new_http
254
284
 
@@ -270,7 +300,7 @@ private
270
300
  end
271
301
 
272
302
  # up to 7 retries with exponential (base 2) back-off starting at 'retry_delay'
273
- retry_delay = 5
303
+ retry_delay = @retry_delay
274
304
  cumul_retry_delay = 0
275
305
 
276
306
  # for both exceptions and 500+ errors retrying is enabled by default.
@@ -288,7 +318,7 @@ private
288
318
 
289
319
  status = response.code.to_i
290
320
  # retry if the HTTP error code is 500 or higher and we did not run out of retrying attempts
291
- if !block_given? && status >= 500 && cumul_retry_delay <= @max_cumul_retry_delay
321
+ if !block_given? && status >= 500 && cumul_retry_delay < @max_cumul_retry_delay
292
322
  $stderr.puts "Error #{status}: #{get_error(response)}. Retrying after #{retry_delay} seconds..."
293
323
  sleep retry_delay
294
324
  cumul_retry_delay += retry_delay
@@ -300,7 +330,7 @@ private
300
330
  raise e
301
331
  end
302
332
  $stderr.print "#{e.class}: #{e.message}. "
303
- if cumul_retry_delay <= @max_cumul_retry_delay
333
+ if cumul_retry_delay < @max_cumul_retry_delay
304
334
  $stderr.puts "Retrying after #{retry_delay} seconds..."
305
335
  sleep retry_delay
306
336
  cumul_retry_delay += retry_delay
@@ -341,12 +371,16 @@ private
341
371
  return [response.code, body, response]
342
372
  end
343
373
 
374
+ # @param [String] url
375
+ # @param [Hash] params
344
376
  def post(url, params=nil)
345
377
  guard_no_sslv3 do
346
378
  do_post(url, params)
347
379
  end
348
380
  end
349
381
 
382
+ # @param [String] url
383
+ # @param [Hash] params
350
384
  def do_post(url, params=nil)
351
385
  http, header = new_http
352
386
 
@@ -368,7 +402,7 @@ private
368
402
  end
369
403
 
370
404
  # up to 7 retries with exponential (base 2) back-off starting at 'retry_delay'
371
- retry_delay = 5
405
+ retry_delay = @retry_delay
372
406
  cumul_retry_delay = 0
373
407
 
374
408
  # for both exceptions and 500+ errors retrying can be enabled by initialization
@@ -382,7 +416,7 @@ private
382
416
  # if the HTTP error code is 500 or higher and the user requested retrying
383
417
  # on post request, attempt a retry
384
418
  status = response.code.to_i
385
- if @retry_post_requests && status >= 500 && cumul_retry_delay <= @max_cumul_retry_delay
419
+ if @retry_post_requests && status >= 500 && cumul_retry_delay < @max_cumul_retry_delay
386
420
  $stderr.puts "Error #{status}: #{get_error(response)}. Retrying after #{retry_delay} seconds..."
387
421
  sleep retry_delay
388
422
  cumul_retry_delay += retry_delay
@@ -391,7 +425,7 @@ private
391
425
  end
392
426
  rescue Errno::ECONNREFUSED, Errno::ECONNRESET, Timeout::Error, EOFError, OpenSSL::SSL::SSLError, SocketError => e
393
427
  $stderr.print "#{e.class}: #{e.message}. "
394
- if @retry_post_requests && cumul_retry_delay <= @max_cumul_retry_delay
428
+ if @retry_post_requests && cumul_retry_delay < @max_cumul_retry_delay
395
429
  $stderr.puts "Retrying after #{retry_delay} seconds..."
396
430
  sleep retry_delay
397
431
  cumul_retry_delay += retry_delay
@@ -420,6 +454,10 @@ private
420
454
  return [response.code, response.body, response]
421
455
  end
422
456
 
457
+ # @param [String] url
458
+ # @param [String, StringIO] stream
459
+ # @param [Fixnum] size
460
+ # @param [Hash] opts
423
461
  def put(url, stream, size, opts = {})
424
462
  client, header = new_client(opts)
425
463
  client.send_timeout = @send_timeout
@@ -457,11 +495,15 @@ private
457
495
  end
458
496
  end
459
497
 
498
+ # @param [String] url
499
+ # @param [String] host
500
+ # @return [String]
460
501
  def build_endpoint(url, host)
461
502
  schema = @ssl ? 'https' : 'http'
462
503
  "#{schema}://#{host}:#{@port}#{@base_path + url}"
463
504
  end
464
505
 
506
+ # @yield Disable SSLv3 in given block
465
507
  def guard_no_sslv3
466
508
  key = :SET_SSL_OP_NO_SSLv3
467
509
  backup = Thread.current[key]
@@ -475,6 +517,8 @@ private
475
517
  end
476
518
  end
477
519
 
520
+ # @param [Hash] opts
521
+ # @return [http, Hash]
478
522
  def new_http(opts = {})
479
523
  host = opts[:host] || @host
480
524
  http = @http_class.new(host, @port)
@@ -505,6 +549,8 @@ private
505
549
  return http, header
506
550
  end
507
551
 
552
+ # @param [Hash] opts
553
+ # @return [HTTPClient, Hash]
508
554
  def new_client(opts = {})
509
555
  client = HTTPClient.new(@http_proxy, @user_agent)
510
556
  client.connect_timeout = @connect_timeout
@@ -527,10 +573,73 @@ private
527
573
  return client, header
528
574
  end
529
575
 
576
+ # @return [String]
577
+ def api_client(endpoint)
578
+ header = {}.merge(@headers)
579
+ header['Authorization'] = "TD1 #{apikey}" if @apikey
580
+ header['Content-Type'] = 'application/json; charset=utf-8'
581
+ client = HTTPClient.new(:proxy => @http_proxy, :agent_name => @user_agent, :base_url => endpoint, :default_header => header)
582
+ client.connect_timeout = @connect_timeout
583
+ client.send_timeout = @send_timeout
584
+ client.receive_timeout = @read_timeout
585
+ client.transparent_gzip_decompression = true
586
+ client.debug_dev = STDOUT unless ENV['TD_CLIENT_DEBUG'].nil?
587
+ client
588
+ end
589
+
590
+ def api(opt = {:retry_request => true}, &block)
591
+ retry_request = opt[:retry_request]
592
+ # up to 7 retries with exponential (base 2) back-off starting at 'retry_delay'
593
+ retry_delay = @retry_delay
594
+ retry_times = 0
595
+ cumul_retry_delay = 0
596
+
597
+ # for both exceptions and 500+ errors retrying can be enabled by initialization
598
+ # parameter 'retry_post_requests'. The total number of retries cumulatively
599
+ # should not exceed 10 minutes / 600 seconds
600
+ begin # this block is to allow retry (redo) in the begin part of the begin-rescue block
601
+ begin
602
+ response = @api.instance_eval &block
603
+
604
+ # if the HTTP error code is 500 or higher and the user requested retrying
605
+ # on post request, attempt a retry
606
+ status = response.code.to_i
607
+ if retry_request && status >= 500 && cumul_retry_delay < @max_cumul_retry_delay
608
+ $stderr.puts "Error #{status}: #{get_error(response)}. Retrying after #{retry_delay} seconds..."
609
+ sleep retry_delay
610
+ cumul_retry_delay += retry_delay
611
+ retry_delay *= 2
612
+ redo # restart from beginning of do-while loop
613
+ end
614
+ return response
615
+ rescue Errno::ECONNREFUSED, Errno::ECONNRESET, Timeout::Error, EOFError, OpenSSL::SSL::SSLError, SocketError => e
616
+ $stderr.print "#{e.class}: #{e.message}. "
617
+ if retry_request
618
+ if cumul_retry_delay < @max_cumul_retry_delay
619
+ $stderr.puts "Retrying after #{retry_delay} seconds..."
620
+ sleep retry_delay
621
+ cumul_retry_delay += retry_delay
622
+ retry_delay *= 2
623
+ retry_times += 1
624
+ retry
625
+ else
626
+ $stderr.puts "Retrying stopped after #{@max_cumul_retry_delay} seconds."
627
+ e.message << " (Retried #{retry_times} times in #{cumul_retry_delay} seconds)"
628
+ end
629
+ else
630
+ $stderr.puts "No retry should be performed."
631
+ end
632
+ raise e
633
+ end
634
+ end while false
635
+ end
636
+
530
637
  def ssl_ca_file
531
638
  @ssl_ca_file ||= File.join(File.dirname(__FILE__), '..', '..', '..', 'data', 'ca-bundle.crt')
532
639
  end
533
640
 
641
+ # @param [response] res
642
+ # @return [String]
534
643
  def get_error(res)
535
644
  begin
536
645
  js = JSON.load(res.body)
@@ -549,6 +658,9 @@ private
549
658
  error_msg
550
659
  end
551
660
 
661
+ # @param [String] msg
662
+ # @param [response] res
663
+ # @param [Class] klass
552
664
  def raise_error(msg, res, klass=nil)
553
665
  status_code = res.code.to_s
554
666
  error_msg = get_error(res)
@@ -569,16 +681,22 @@ private
569
681
  end
570
682
 
571
683
  if ''.respond_to?(:encode)
684
+ # @param [String] s
685
+ # @return [String]
572
686
  def e(s)
573
687
  CGI.escape(s.to_s.encode("UTF-8"))
574
688
  end
575
689
  else
690
+ # @param [String] s
691
+ # @return [String]
576
692
  def e(s)
577
693
  CGI.escape(s.to_s)
578
694
  end
579
695
  end
580
696
 
581
- def checked_json(body, required)
697
+ # @param [String] body
698
+ # @param [Array] required
699
+ def checked_json(body, required = [])
582
700
  js = nil
583
701
  begin
584
702
  js = JSON.load(body)
@@ -5,6 +5,11 @@ module AccessControl
5
5
  ## Access Control API
6
6
  ##
7
7
 
8
+ # @param [String] subject
9
+ # @param [String] action
10
+ # @param [String] scope
11
+ # @param [Array] grant_option
12
+ # @return [true]
8
13
  def grant_access_control(subject, action, scope, grant_option)
9
14
  params = {'subject'=>subject, 'action'=>action, 'scope'=>scope, 'grant_option'=>grant_option.to_s}
10
15
  code, body, res = post("/v3/acl/grant", params)
@@ -14,6 +19,10 @@ module AccessControl
14
19
  return true
15
20
  end
16
21
 
22
+ # @param [String] subject
23
+ # @param [String] action
24
+ # @param [String] scope
25
+ # @return [true]
17
26
  def revoke_access_control(subject, action, scope)
18
27
  params = {'subject'=>subject, 'action'=>action, 'scope'=>scope}
19
28
  code, body, res = post("/v3/acl/revoke", params)
@@ -23,7 +32,10 @@ module AccessControl
23
32
  return true
24
33
  end
25
34
 
26
- # [true, [{subject:String,action:String,scope:String}]]
35
+ # @param [String] user
36
+ # @param [String] action
37
+ # @param [String] scope
38
+ # @return [Array]
27
39
  def test_access_control(user, action, scope)
28
40
  params = {'user'=>user, 'action'=>action, 'scope'=>scope}
29
41
  code, body, res = get("/v3/acl/test", params)
@@ -41,7 +53,7 @@ module AccessControl
41
53
  return perm, acl
42
54
  end
43
55
 
44
- # [{subject:String,action:String,scope:String}]
56
+ # @return [Array]
45
57
  def list_access_controls
46
58
  code, body, res = get("/v3/acl/list")
47
59
  if code != "200"
@@ -5,6 +5,7 @@ module Account
5
5
  ## Account API
6
6
  ##
7
7
 
8
+ # @return [Array]
8
9
  def show_account
9
10
  code, body, res = get("/v3/account/show")
10
11
  if code != "200"
@@ -21,6 +22,9 @@ module Account
21
22
  return [account_id, plan, storage_size, guaranteed_cores, maximum_cores, created_at]
22
23
  end
23
24
 
25
+ # @param [Fixnum] from
26
+ # @param [Fixnum] to
27
+ # @return [Array]
24
28
  def account_core_utilization(from, to)
25
29
  params = { }
26
30
  params['from'] = from.to_s if from
@@ -5,7 +5,11 @@ module BulkImport
5
5
  ## Bulk import API
6
6
  ##
7
7
 
8
- # => nil
8
+ # @param [String] name
9
+ # @param [String] db
10
+ # @param [String] table
11
+ # @param [Hash] opts
12
+ # @return [nil]
9
13
  def create_bulk_import(name, db, table, opts={})
10
14
  params = opts.dup
11
15
  code, body, res = post("/v3/bulk_import/create/#{e name}/#{e db}/#{e table}", params)
@@ -15,7 +19,9 @@ module BulkImport
15
19
  return nil
16
20
  end
17
21
 
18
- # => nil
22
+ # @param [String] name
23
+ # @param [Hash] opts
24
+ # @return [nil]
19
25
  def delete_bulk_import(name, opts={})
20
26
  params = opts.dup
21
27
  code, body, res = post("/v3/bulk_import/delete/#{e name}", params)
@@ -25,7 +31,8 @@ module BulkImport
25
31
  return nil
26
32
  end
27
33
 
28
- # => data:Hash
34
+ # @param [String] name
35
+ # @return [Hash]
29
36
  def show_bulk_import(name)
30
37
  code, body, res = get("/v3/bulk_import/show/#{name}")
31
38
  if code != "200"
@@ -35,7 +42,8 @@ module BulkImport
35
42
  return js
36
43
  end
37
44
 
38
- # => result:[data:Hash]
45
+ # @param [Hash] opts
46
+ # @return [Hash]
39
47
  def list_bulk_imports(opts={})
40
48
  params = opts.dup
41
49
  code, body, res = get("/v3/bulk_import/list", params)
@@ -46,6 +54,9 @@ module BulkImport
46
54
  return js['bulk_imports']
47
55
  end
48
56
 
57
+ # @param [String] name
58
+ # @param [Hash] opts
59
+ # @return [nil]
49
60
  def list_bulk_import_parts(name, opts={})
50
61
  params = opts.dup
51
62
  code, body, res = get("/v3/bulk_import/list_parts/#{e name}", params)
@@ -56,7 +67,12 @@ module BulkImport
56
67
  return js['parts']
57
68
  end
58
69
 
59
- # => nil
70
+ # @param [String] name
71
+ # @param [String] part_name
72
+ # @param [String, StringIO] stream
73
+ # @param [Fixnum] size
74
+ # @param [Hash] opts
75
+ # @return [nil]
60
76
  def bulk_import_upload_part(name, part_name, stream, size, opts={})
61
77
  code, body, res = put("/v3/bulk_import/upload_part/#{e name}/#{e part_name}", stream, size)
62
78
  if code[0] != ?2
@@ -65,7 +81,10 @@ module BulkImport
65
81
  return nil
66
82
  end
67
83
 
68
- # => nil
84
+ # @param [String] name
85
+ # @param [String] part_name
86
+ # @param [Hash] opts
87
+ # @return [nil]
69
88
  def bulk_import_delete_part(name, part_name, opts={})
70
89
  params = opts.dup
71
90
  code, body, res = post("/v3/bulk_import/delete_part/#{e name}/#{e part_name}", params)
@@ -75,7 +94,9 @@ module BulkImport
75
94
  return nil
76
95
  end
77
96
 
78
- # => nil
97
+ # @param [String] name
98
+ # @param [Hash] opts
99
+ # @return [nil]
79
100
  def freeze_bulk_import(name, opts={})
80
101
  params = opts.dup
81
102
  code, body, res = post("/v3/bulk_import/freeze/#{e name}", params)
@@ -85,7 +106,9 @@ module BulkImport
85
106
  return nil
86
107
  end
87
108
 
88
- # => nil
109
+ # @param [String] name
110
+ # @param [Hash] opts
111
+ # @return [nil]
89
112
  def unfreeze_bulk_import(name, opts={})
90
113
  params = opts.dup
91
114
  code, body, res = post("/v3/bulk_import/unfreeze/#{e name}", params)
@@ -95,7 +118,9 @@ module BulkImport
95
118
  return nil
96
119
  end
97
120
 
98
- # => jobId:String
121
+ # @param [String] name
122
+ # @param [Hash] opts
123
+ # @return [String] job_id
99
124
  def perform_bulk_import(name, opts={})
100
125
  params = opts.dup
101
126
  code, body, res = post("/v3/bulk_import/perform/#{e name}", params)
@@ -106,7 +131,9 @@ module BulkImport
106
131
  return js['job_id'].to_s
107
132
  end
108
133
 
109
- # => nil
134
+ # @param [String] name
135
+ # @param [Hash] opts
136
+ # @return [nil]
110
137
  def commit_bulk_import(name, opts={})
111
138
  params = opts.dup
112
139
  code, body, res = post("/v3/bulk_import/commit/#{e name}", params)
@@ -116,7 +143,10 @@ module BulkImport
116
143
  return nil
117
144
  end
118
145
 
119
- # => data...
146
+ # @param [String] name
147
+ # @param [Hash] opts
148
+ # @param [Proc] block
149
+ # @return [Array]
120
150
  def bulk_import_error_records(name, opts={}, &block)
121
151
  params = opts.dup
122
152
  code, body, res = get("/v3/bulk_import/error_records/#{e name}", params)