webhdfs 0.9.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 5c3c7efca2c4e4c3a82a12c544dc5d86790e9b19aa050f1269bfb17628be2bb8
4
- data.tar.gz: 8d7dce607082bbe3c2eaaa17eb26d08158eb3844f1d938580727a9dd4be65729
3
+ metadata.gz: 769cd8dca5e3e329a882028bdcd06a3251e997148a7e85fdb68b14b21d37d989
4
+ data.tar.gz: f57beca7f1528b812e4bc99328120c673d93eb7eb48dc97aa5197ec976f7a5b3
5
5
  SHA512:
6
- metadata.gz: b43c91f0fb8b1908813ef31b012083d3613bde72fa7a0228a9d4200f3a140df95c5658a2c2f1d2e9888eb888c8e44ef10018705167bbcbaf9963a17d505eb93c
7
- data.tar.gz: 209843fe4e9afb954d9f0523a9c2117302d7fb1f2d3118589775b5b0ac4f0b8720b27d954924dfbb5b8d8b5dd521790fd3ef8f632556292eb4c465c8e4087edc
6
+ metadata.gz: 8b0ddef524644a473dd37f9bd13d63eb09cd50d6b709444a93d577dfbe8e080f1bde7426c142e20eae992c578674957b53e77d8eacd4aa644d754cd6602d05de
7
+ data.tar.gz: 00a583e338e6f4e3bcd15ebba1437503b4f17f3ee1ef253fec211a05b7959ce7cfa9f97a55cc0f9fc92277e5c7a828a9a63e1d7f56a15160361087f25e4dc3ab
data/README.md CHANGED
@@ -102,6 +102,9 @@ Note that net/https and openssl libraries must be available:
102
102
  Note that [gssapi](https://github.com/zenchild/gssapi) library must be available:
103
103
 
104
104
  client = WebHDFS::Client.new('hostname', 14000)
105
+ # or if you want to use client delegation token with renewing per 8 hours
106
+ client = WebHDFS::Client.new('hostname', 14000, username, nil, nil, nil, {}, 8)
107
+
105
108
  client.kerberos = true
106
109
  client.kerberos_keytab = "/path/to/project.keytab"
107
110
 
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.9.0
1
+ 0.10.0
@@ -28,6 +28,7 @@ module WebHDFS
28
28
  attr_accessor :ssl_version
29
29
  attr_accessor :kerberos, :kerberos_keytab
30
30
  attr_accessor :http_headers
31
+ attr_accessor :kerberos_delegation_token
31
32
 
32
33
  SSL_VERIFY_MODES = [:none, :peer]
33
34
  def ssl_verify_mode=(mode)
@@ -37,7 +38,7 @@ module WebHDFS
37
38
  @ssl_verify_mode = mode
38
39
  end
39
40
 
40
- def initialize(host='localhost', port=50070, username=nil, doas=nil, proxy_address=nil, proxy_port=nil, http_headers={})
41
+ def initialize(host='localhost', port=50070, username=nil, doas=nil, proxy_address=nil, proxy_port=nil, http_headers={}, renew_kerberos_delegation_token_time_hour=nil)
41
42
  @host = host
42
43
  @port = port
43
44
  @username = username
@@ -59,32 +60,63 @@ module WebHDFS
59
60
 
60
61
  @kerberos = false
61
62
  @kerberos_keytab = nil
63
+ @renew_kerberos_delegation_token_time_hour = renew_kerberos_delegation_token_time_hour
64
+ @kerberos_delegation_token = nil
65
+ @kerberos_token_updated_at = Time.now
62
66
  @http_headers = http_headers
63
67
  end
64
68
 
69
+ def should_kerberos_token_updated?
70
+ @kerberos_token_updated_at + (@renew_kerberos_delegation_token_time_hour * 60 * 60) <= Time.now
71
+ end
72
+
73
+
74
+ def get_cached_kerberos_delegation_token(force_renew=nil)
75
+ return @kerberos_delegation_token if @kerberos_delegation_token && !should_kerberos_token_updated? && !force_renew
76
+
77
+ if !@kerberos_delegation_token || force_renew
78
+ @kerberos_delegation_token = get_kerberos_delegation_token(@username)
79
+ @kerberos_token_updated_at = Time.now
80
+ else
81
+ renew_kerberos_delegation_token(@kerberos_delegation_token)
82
+ @kerberos_token_updated_at = Time.now
83
+ end
84
+ @kerberos_delegation_token
85
+ rescue => e
86
+ raise WebHDFS::RequestFailedError, e.message
87
+ end
88
+
65
89
  # curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=CREATE
66
90
  # [&overwrite=<true|false>][&blocksize=<LONG>][&replication=<SHORT>]
67
- # [&permission=<OCTAL>][&buffersize=<INT>]"
91
+ # [&permission=<OCTAL>][&buffersize=<INT>]
92
+ # [&delegation=<DELEGATION_TOKEN>]"
68
93
  def create(path, body, options={})
69
94
  if @httpfs_mode
70
95
  options = options.merge({'data' => 'true'})
71
96
  end
97
+ if @renew_kerberos_delegation_token_time_hour
98
+ options = options.merge('delegation' => get_cached_kerberos_delegation_token)
99
+ end
72
100
  check_options(options, OPT_TABLE['CREATE'])
73
101
  res = operate_requests('PUT', path, 'CREATE', options, body)
74
102
  res.code == '201'
75
103
  end
76
- OPT_TABLE['CREATE'] = ['overwrite', 'blocksize', 'replication', 'permission', 'buffersize', 'data']
104
+ OPT_TABLE['CREATE'] = ['overwrite', 'blocksize', 'replication', 'permission', 'buffersize', 'data', 'delegation']
77
105
 
78
- # curl -i -X POST "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=APPEND[&buffersize=<INT>]"
106
+ # curl -i -X POST "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=APPEND
107
+ # [&buffersize=<INT>][&delegation=<DELEGATION_TOKEN>]"
79
108
  def append(path, body, options={})
80
109
  if @httpfs_mode
81
110
  options = options.merge({'data' => 'true'})
82
111
  end
112
+ if @renew_kerberos_delegation_token_time_hour
113
+ options = options.merge('delegation' => get_cached_kerberos_delegation_token)
114
+ end
83
115
  check_options(options, OPT_TABLE['APPEND'])
84
116
  res = operate_requests('POST', path, 'APPEND', options, body)
85
117
  res.code == '200'
86
118
  end
87
- OPT_TABLE['APPEND'] = ['buffersize', 'data']
119
+ OPT_TABLE['APPEND'] = ['buffersize', 'data', 'delegation']
88
120
 
89
121
  # curl -i -L "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=OPEN
90
122
  # [&offset=<LONG>][&length=<LONG>][&buffersize=<INT>]"
@@ -219,12 +251,25 @@ module WebHDFS
219
251
  OPT_TABLE['SETTIMES'] = ['modificationtime', 'accesstime']
220
252
  alias :settimes :touch
221
253
 
222
- # def delegation_token(user, options={}) # GETDELEGATIONTOKEN
223
- # raise NotImplementedError
224
- # end
225
- # def renew_delegation_token(token, options={}) # RENEWDELEGATIONTOKEN
226
- # raise NotImplementedError
227
- # end
254
+ # curl -i "http://<HOST>:<PORT>/webhdfs/v1/?op=GETDELEGATIONTOKEN&renewer=<USER>"
255
+ def get_kerberos_delegation_token(user, options={})
256
+ options = options.merge({ 'renewer' => user })
257
+ check_options(options, OPT_TABLE['GETDELEGATIONTOKEN'])
258
+ res = operate_requests('GET', '/', 'GETDELEGATIONTOKEN', options)
259
+ check_success_json(res, 'Token')
260
+ JSON.parse(res.body)['Token']['urlString']
261
+ end
262
+ OPT_TABLE['GETDELEGATIONTOKEN'] = ['renewer']
263
+
264
+ # curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/?op=RENEWDELEGATIONTOKEN&token=<DELEGATION_TOKEN>"
265
+ def renew_kerberos_delegation_token(token, options={})
266
+ options = options.merge({ 'token' => token })
267
+ check_options(options, OPT_TABLE['RENEWDELEGATIONTOKEN'])
268
+ res = operate_requests('PUT', '/', 'RENEWDELEGATIONTOKEN', options)
269
+ check_success_json(res, 'long')
270
+ end
271
+ OPT_TABLE['RENEWDELEGATIONTOKEN'] = ['token']
272
+
228
273
  # def cancel_delegation_token(token, options={}) # CANCELDELEGATIONTOKEN
229
274
  # raise NotImplementedError
230
275
  # end
@@ -359,7 +404,8 @@ module WebHDFS
359
404
  end
360
405
  end
361
406
 
362
- if @kerberos and res.code == '307'
407
+ # if delegation token param settled, namenode do not response WWW-Authenticate header
408
+ if @kerberos and res.code == '307' and not params.key?('delegation')
363
409
  itok = (res.header.get_fields('WWW-Authenticate') || ['']).pop.split(/\s+/).last
364
410
  unless itok
365
411
  raise WebHDFS::KerberosError, 'Server does not return WWW-Authenticate header'
@@ -384,6 +430,18 @@ module WebHDFS
384
430
  'Response body is empty...'
385
431
  end
386
432
 
433
+ # when delegation token is expired
434
+ if res.code == '403' and @renew_kerberos_delegation_token_time_hour
435
+ if message.include?('{"RemoteException":{')
436
+ detail = JSON.parse(message) rescue nil
437
+ if detail&.dig('RemoteException', 'exception') == 'InvalidToken' and detail&.dig('RemoteException', 'message')&.include?('HDFS_DELEGATION_TOKEN')
438
+ params = params.merge('token' => get_cached_kerberos_delegation_token(true))
439
+ sleep @retry_interval if @retry_interval > 0
440
+ return request(host, port, method, path, op, params, payload, header, retries+1)
441
+ end
442
+ end
443
+ end
444
+
387
445
  if @retry_known_errors && retries < @retry_times
388
446
  detail = nil
389
447
  if message =~ /^\{"RemoteException":\{/
data/webhdfs.gemspec CHANGED
@@ -13,6 +13,7 @@ Gem::Specification.new do |gem|
13
13
  gem.files = Dir['lib/**/*','test/**/*','*.gemspec','*.md','AUTHORS','COPYING','Gemfile','VERSION']
14
14
  gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
15
15
  gem.require_paths = ['lib']
16
+ gem.licenses = ["Apache-2.0"]
16
17
 
17
18
  gem.add_development_dependency "rake"
18
19
  gem.add_development_dependency "rdoc"
metadata CHANGED
@@ -1,15 +1,15 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: webhdfs
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.9.0
4
+ version: 0.10.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Kazuki Ohta
8
8
  - Satoshi Tagomori
9
- autorequire:
9
+ autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2019-11-11 00:00:00.000000000 Z
12
+ date: 2021-08-01 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: rake
@@ -104,9 +104,10 @@ files:
104
104
  - test/webhdfs/fileutils.rb
105
105
  - webhdfs.gemspec
106
106
  homepage: https://github.com/kzk/webhdfs/
107
- licenses: []
107
+ licenses:
108
+ - Apache-2.0
108
109
  metadata: {}
109
- post_install_message:
110
+ post_install_message:
110
111
  rdoc_options: []
111
112
  require_paths:
112
113
  - lib
@@ -121,8 +122,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
121
122
  - !ruby/object:Gem::Version
122
123
  version: '0'
123
124
  requirements: []
124
- rubygems_version: 3.0.3
125
- signing_key:
125
+ rubygems_version: 3.2.3
126
+ signing_key:
126
127
  specification_version: 4
127
128
  summary: Ruby WebHDFS/HttpFs client
128
129
  test_files: