webhdfs 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +3 -0
- data/VERSION +1 -1
- data/lib/webhdfs/client_v1.rb +70 -12
- data/webhdfs.gemspec +1 -0
- metadata +8 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 769cd8dca5e3e329a882028bdcd06a3251e997148a7e85fdb68b14b21d37d989
|
4
|
+
data.tar.gz: f57beca7f1528b812e4bc99328120c673d93eb7eb48dc97aa5197ec976f7a5b3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8b0ddef524644a473dd37f9bd13d63eb09cd50d6b709444a93d577dfbe8e080f1bde7426c142e20eae992c578674957b53e77d8eacd4aa644d754cd6602d05de
|
7
|
+
data.tar.gz: 00a583e338e6f4e3bcd15ebba1437503b4f17f3ee1ef253fec211a05b7959ce7cfa9f97a55cc0f9fc92277e5c7a828a9a63e1d7f56a15160361087f25e4dc3ab
|
data/README.md
CHANGED
@@ -102,6 +102,9 @@ Note that net/https and openssl libraries must be available:
|
|
102
102
|
Note that [gssapi](https://github.com/zenchild/gssapi) library must be available:
|
103
103
|
|
104
104
|
client = WebHDFS::Client.new('hostname', 14000)
|
105
|
+
# or if you want to use client delegation token with renewing per 8 hours
|
106
|
+
client = WebHDFS::Client.new('hostname', 14000, username, nil, nil, nil, {}, 8)
|
107
|
+
|
105
108
|
client.kerberos = true
|
106
109
|
client.kerberos_keytab = "/path/to/project.keytab"
|
107
110
|
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.10.0
|
data/lib/webhdfs/client_v1.rb
CHANGED
@@ -28,6 +28,7 @@ module WebHDFS
|
|
28
28
|
attr_accessor :ssl_version
|
29
29
|
attr_accessor :kerberos, :kerberos_keytab
|
30
30
|
attr_accessor :http_headers
|
31
|
+
attr_accessor :kerberos_delegation_token
|
31
32
|
|
32
33
|
SSL_VERIFY_MODES = [:none, :peer]
|
33
34
|
def ssl_verify_mode=(mode)
|
@@ -37,7 +38,7 @@ module WebHDFS
|
|
37
38
|
@ssl_verify_mode = mode
|
38
39
|
end
|
39
40
|
|
40
|
-
def initialize(host='localhost', port=50070, username=nil, doas=nil, proxy_address=nil, proxy_port=nil, http_headers={})
|
41
|
+
def initialize(host='localhost', port=50070, username=nil, doas=nil, proxy_address=nil, proxy_port=nil, http_headers={}, renew_kerberos_delegation_token_time_hour=nil)
|
41
42
|
@host = host
|
42
43
|
@port = port
|
43
44
|
@username = username
|
@@ -59,32 +60,63 @@ module WebHDFS
|
|
59
60
|
|
60
61
|
@kerberos = false
|
61
62
|
@kerberos_keytab = nil
|
63
|
+
@renew_kerberos_delegation_token_time_hour = renew_kerberos_delegation_token_time_hour
|
64
|
+
@kerberos_delegation_token = nil
|
65
|
+
@kerberos_token_updated_at = Time.now
|
62
66
|
@http_headers = http_headers
|
63
67
|
end
|
64
68
|
|
69
|
+
def should_kerberos_token_updated?
|
70
|
+
@kerberos_token_updated_at + (@renew_kerberos_delegation_token_time_hour * 60 * 60) <= Time.now
|
71
|
+
end
|
72
|
+
|
73
|
+
|
74
|
+
def get_cached_kerberos_delegation_token(force_renew=nil)
|
75
|
+
return @kerberos_delegation_token if @kerberos_delegation_token && !should_kerberos_token_updated? && !force_renew
|
76
|
+
|
77
|
+
if !@kerberos_delegation_token || force_renew
|
78
|
+
@kerberos_delegation_token = get_kerberos_delegation_token(@username)
|
79
|
+
@kerberos_token_updated_at = Time.now
|
80
|
+
else
|
81
|
+
renew_kerberos_delegation_token(@kerberos_delegation_token)
|
82
|
+
@kerberos_token_updated_at = Time.now
|
83
|
+
end
|
84
|
+
@kerberos_delegation_token
|
85
|
+
rescue => e
|
86
|
+
raise WebHDFS::RequestFailedError, e.message
|
87
|
+
end
|
88
|
+
|
65
89
|
# curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=CREATE
|
66
90
|
# [&overwrite=<true|false>][&blocksize=<LONG>][&replication=<SHORT>]
|
67
|
-
# [&permission=<OCTAL>][&buffersize=<INT>]
|
91
|
+
# [&permission=<OCTAL>][&buffersize=<INT>]
|
92
|
+
# [&delegation=<DELEGATION_TOKEN>]"
|
68
93
|
def create(path, body, options={})
|
69
94
|
if @httpfs_mode
|
70
95
|
options = options.merge({'data' => 'true'})
|
71
96
|
end
|
97
|
+
if @renew_kerberos_delegation_token_time_hour
|
98
|
+
options = options.merge('delegation' => get_cached_kerberos_delegation_token)
|
99
|
+
end
|
72
100
|
check_options(options, OPT_TABLE['CREATE'])
|
73
101
|
res = operate_requests('PUT', path, 'CREATE', options, body)
|
74
102
|
res.code == '201'
|
75
103
|
end
|
76
|
-
OPT_TABLE['CREATE'] = ['overwrite', 'blocksize', 'replication', 'permission', 'buffersize', 'data']
|
104
|
+
OPT_TABLE['CREATE'] = ['overwrite', 'blocksize', 'replication', 'permission', 'buffersize', 'data', 'delegation']
|
77
105
|
|
78
|
-
# curl -i -X POST "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=APPEND
|
106
|
+
# curl -i -X POST "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=APPEND
|
107
|
+
# [&buffersize=<INT>][&delegation=<DELEGATION_TOKEN>]"
|
79
108
|
def append(path, body, options={})
|
80
109
|
if @httpfs_mode
|
81
110
|
options = options.merge({'data' => 'true'})
|
82
111
|
end
|
112
|
+
if @renew_kerberos_delegation_token_time_hour
|
113
|
+
options = options.merge('delegation' => get_cached_kerberos_delegation_token)
|
114
|
+
end
|
83
115
|
check_options(options, OPT_TABLE['APPEND'])
|
84
116
|
res = operate_requests('POST', path, 'APPEND', options, body)
|
85
117
|
res.code == '200'
|
86
118
|
end
|
87
|
-
OPT_TABLE['APPEND'] = ['buffersize', 'data']
|
119
|
+
OPT_TABLE['APPEND'] = ['buffersize', 'data', 'delegation']
|
88
120
|
|
89
121
|
# curl -i -L "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=OPEN
|
90
122
|
# [&offset=<LONG>][&length=<LONG>][&buffersize=<INT>]"
|
@@ -219,12 +251,25 @@ module WebHDFS
|
|
219
251
|
OPT_TABLE['SETTIMES'] = ['modificationtime', 'accesstime']
|
220
252
|
alias :settimes :touch
|
221
253
|
|
222
|
-
#
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
254
|
+
# curl -i "http://<HOST>:<PORT>/webhdfs/v1/?op=GETDELEGATIONTOKEN&renewer=<USER>"
|
255
|
+
def get_kerberos_delegation_token(user, options={})
|
256
|
+
options = options.merge({ 'renewer' => user })
|
257
|
+
check_options(options, OPT_TABLE['GETDELEGATIONTOKEN'])
|
258
|
+
res = operate_requests('GET', '/', 'GETDELEGATIONTOKEN', options)
|
259
|
+
check_success_json(res, 'Token')
|
260
|
+
JSON.parse(res.body)['Token']['urlString']
|
261
|
+
end
|
262
|
+
OPT_TABLE['GETDELEGATIONTOKEN'] = ['renewer']
|
263
|
+
|
264
|
+
# curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/?op=RENEWDELEGATIONTOKEN&token=<DELEGATION_TOKEN>"
|
265
|
+
def renew_kerberos_delegation_token(token, options={})
|
266
|
+
options = options.merge({ 'token' => token })
|
267
|
+
check_options(options, OPT_TABLE['RENEWDELEGATIONTOKEN'])
|
268
|
+
res = operate_requests('PUT', '/', 'RENEWDELEGATIONTOKEN', options)
|
269
|
+
check_success_json(res, 'long')
|
270
|
+
end
|
271
|
+
OPT_TABLE['RENEWDELEGATIONTOKEN'] = ['token']
|
272
|
+
|
228
273
|
# def cancel_delegation_token(token, options={}) # CANCELDELEGATIONTOKEN
|
229
274
|
# raise NotImplementedError
|
230
275
|
# end
|
@@ -359,7 +404,8 @@ module WebHDFS
|
|
359
404
|
end
|
360
405
|
end
|
361
406
|
|
362
|
-
if
|
407
|
+
# if delegation token param settled, namenode do not response WWW-Authenticate header
|
408
|
+
if @kerberos and res.code == '307' and not params.key?('delegation')
|
363
409
|
itok = (res.header.get_fields('WWW-Authenticate') || ['']).pop.split(/\s+/).last
|
364
410
|
unless itok
|
365
411
|
raise WebHDFS::KerberosError, 'Server does not return WWW-Authenticate header'
|
@@ -384,6 +430,18 @@ module WebHDFS
|
|
384
430
|
'Response body is empty...'
|
385
431
|
end
|
386
432
|
|
433
|
+
# when delegation token is expired
|
434
|
+
if res.code == '403' and @renew_kerberos_delegation_token_time_hour
|
435
|
+
if message.include?('{"RemoteException":{')
|
436
|
+
detail = JSON.parse(message) rescue nil
|
437
|
+
if detail&.dig('RemoteException', 'exception') == 'InvalidToken' and detail&.dig('RemoteException', 'message')&.include?('HDFS_DELEGATION_TOKEN')
|
438
|
+
params = params.merge('token' => get_cached_kerberos_delegation_token(true))
|
439
|
+
sleep @retry_interval if @retry_interval > 0
|
440
|
+
return request(host, port, method, path, op, params, payload, header, retries+1)
|
441
|
+
end
|
442
|
+
end
|
443
|
+
end
|
444
|
+
|
387
445
|
if @retry_known_errors && retries < @retry_times
|
388
446
|
detail = nil
|
389
447
|
if message =~ /^\{"RemoteException":\{/
|
data/webhdfs.gemspec
CHANGED
@@ -13,6 +13,7 @@ Gem::Specification.new do |gem|
|
|
13
13
|
gem.files = Dir['lib/**/*','test/**/*','*.gemspec','*.md','AUTHORS','COPYING','Gemfile','VERSION']
|
14
14
|
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
15
15
|
gem.require_paths = ['lib']
|
16
|
+
gem.licenses = ["Apache-2.0"]
|
16
17
|
|
17
18
|
gem.add_development_dependency "rake"
|
18
19
|
gem.add_development_dependency "rdoc"
|
metadata
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: webhdfs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.10.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kazuki Ohta
|
8
8
|
- Satoshi Tagomori
|
9
|
-
autorequire:
|
9
|
+
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2021-08-01 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rake
|
@@ -104,9 +104,10 @@ files:
|
|
104
104
|
- test/webhdfs/fileutils.rb
|
105
105
|
- webhdfs.gemspec
|
106
106
|
homepage: https://github.com/kzk/webhdfs/
|
107
|
-
licenses:
|
107
|
+
licenses:
|
108
|
+
- Apache-2.0
|
108
109
|
metadata: {}
|
109
|
-
post_install_message:
|
110
|
+
post_install_message:
|
110
111
|
rdoc_options: []
|
111
112
|
require_paths:
|
112
113
|
- lib
|
@@ -121,8 +122,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
121
122
|
- !ruby/object:Gem::Version
|
122
123
|
version: '0'
|
123
124
|
requirements: []
|
124
|
-
rubygems_version: 3.
|
125
|
-
signing_key:
|
125
|
+
rubygems_version: 3.2.3
|
126
|
+
signing_key:
|
126
127
|
specification_version: 4
|
127
128
|
summary: Ruby WebHDFS/HttpFs client
|
128
129
|
test_files:
|