webhdfs 0.8.0 → 0.10.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 3f165b8329507d5bd77e98e14b7b6406b5e74e7b
-  data.tar.gz: 14a14eeb4a18cb0670e9966398ae9591e31a54bf
+SHA256:
+  metadata.gz: 7f1fb40375edf3e4364001418a1c4bf364520c586fe4ba9cda1276e76da60137
+  data.tar.gz: eaa642e3f5b85c0d347f7691a88cdd48d441c5bbf8271b599fb246b4ff7e8b63
 SHA512:
-  metadata.gz: bddf6b9166720a926de0f43855d7b24e512701a672e4a9c019a5154e20da22657084688eddaf6ac2f383a76321164fc843dc7af65996782ad946e4c9923144a0
-  data.tar.gz: 6e988dd3694c2dfb015bf0901b890a44e67e0a504ac539a3efcf8979a33ff84769c38c894a3ed7bf287e56596c652eae0cf5b2c6af5f24c2db17e3342320ca18
+  metadata.gz: be479f32138f814ff961ce820d9b0af8c4d7ddf4a5a55c6e2aec6b9e093f654f6931ea53f19554a454372f51d5f6d47fbb80264255f1e5393fdc9102dbcdee65
+  data.tar.gz: b9081147028452f2e6dd46f14e7337a23fcb8067b8c7b8b7e5aefa79eef861b752c2448feeec9d2524c5ded2739fa807986f69f0e0c0ea8a046a3c5eed314127

data/README.md

@@ -102,6 +102,9 @@ Note that net/https and openssl libraries must be available:
 Note that [gssapi](https://github.com/zenchild/gssapi) library must be available:
 
     client = WebHDFS::Client.new('hostname', 14000)
+    # or if you want to use client delegation token with renewing per 8 hours
+    client = WebHDFS::Client.new('hostname', 14000, username, nil, nil, nil, {}, 8)
+
     client.kerberos = true
     client.kerberos_keytab = "/path/to/project.keytab"
 

data/VERSION

@@ -1 +1 @@
-0.8.0
+0.10.2

data/lib/webhdfs/client_v1.rb

@@ -28,6 +28,7 @@ module WebHDFS
     attr_accessor :ssl_version
     attr_accessor :kerberos, :kerberos_keytab
     attr_accessor :http_headers
+    attr_accessor :kerberos_delegation_token
 
     SSL_VERIFY_MODES = [:none, :peer]
     def ssl_verify_mode=(mode)
@@ -37,7 +38,7 @@ module WebHDFS
       @ssl_verify_mode = mode
     end
 
-    def initialize(host='localhost', port=50070, username=nil, doas=nil, proxy_address=nil, proxy_port=nil, http_headers={})
+    def initialize(host='localhost', port=50070, username=nil, doas=nil, proxy_address=nil, proxy_port=nil, http_headers={}, renew_kerberos_delegation_token_time_hour=nil)
       @host = host
       @port = port
       @username = username
@@ -59,32 +60,61 @@ module WebHDFS
 
       @kerberos = false
       @kerberos_keytab = nil
+      @renew_kerberos_delegation_token_time_hour = renew_kerberos_delegation_token_time_hour
+      @kerberos_delegation_token = nil
+      @kerberos_token_updated_at = Time.now
       @http_headers = http_headers
     end
 
+    def should_kerberos_token_updated?
+      @kerberos_token_updated_at + (@renew_kerberos_delegation_token_time_hour * 60 * 60) <= Time.now
+    end
+
+
+    def get_cached_kerberos_delegation_token(force_renew=nil)
+      return @kerberos_delegation_token if @kerberos_delegation_token && !should_kerberos_token_updated? && !force_renew
+
+      if !@kerberos_delegation_token || force_renew
+        @kerberos_delegation_token = get_kerberos_delegation_token(@username)
+        @kerberos_token_updated_at = Time.now
+      else
+        renew_kerberos_delegation_token(@kerberos_delegation_token)
+        @kerberos_token_updated_at = Time.now
+      end
+      @kerberos_delegation_token
+    end
+
     # curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=CREATE
     #                 [&overwrite=<true|false>][&blocksize=<LONG>][&replication=<SHORT>]
-    #                 [&permission=<OCTAL>][&buffersize=<INT>]"
+    #                 [&permission=<OCTAL>][&buffersize=<INT>]
+    #                 [&delegation=<DELEGATION_TOKEN>]"
     def create(path, body, options={})
       if @httpfs_mode
         options = options.merge({'data' => 'true'})
       end
+      if @renew_kerberos_delegation_token_time_hour
+        options = options.merge('delegation' => get_cached_kerberos_delegation_token)
+      end
       check_options(options, OPT_TABLE['CREATE'])
       res = operate_requests('PUT', path, 'CREATE', options, body)
       res.code == '201'
     end
-    OPT_TABLE['CREATE'] = ['overwrite', 'blocksize', 'replication', 'permission', 'buffersize', 'data']
+    OPT_TABLE['CREATE'] = ['overwrite', 'blocksize', 'replication', 'permission', 'buffersize', 'data', 'delegation']
 
-    # curl -i -X POST "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=APPEND[&buffersize=<INT>]"
+    # curl -i -X POST "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=APPEND
+    #                   [&buffersize=<INT>][&delegation=<DELEGATION_TOKEN>]"
     def append(path, body, options={})
       if @httpfs_mode
         options = options.merge({'data' => 'true'})
       end
+      if @renew_kerberos_delegation_token_time_hour
+        options = options.merge('delegation' => get_cached_kerberos_delegation_token)
+      end
       check_options(options, OPT_TABLE['APPEND'])
       res = operate_requests('POST', path, 'APPEND', options, body)
       res.code == '200'
     end
-    OPT_TABLE['APPEND'] = ['buffersize', 'data']
+    OPT_TABLE['APPEND'] = ['buffersize', 'data', 'delegation']
 
     # curl -i -L "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=OPEN
     #                [&offset=<LONG>][&length=<LONG>][&buffersize=<INT>]"
@@ -164,6 +194,13 @@ module WebHDFS
     end
     alias :gethomedirectory :homedir
 
+    # curl -i "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=GETTRASHROOT"
+    def gettrashroot(options={})
+      check_options(options, OPT_TABLE['GETTRASHROOT'])
+      res = operate_requests('GET', '/', 'GETTRASHROOT', options)
+      check_success_json(res, 'Path')
+    end
+
     # curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=SETPERMISSION
     #                 [&permission=<OCTAL>]"
     def chmod(path, mode, options={})
@@ -212,12 +249,25 @@ module WebHDFS
     OPT_TABLE['SETTIMES'] = ['modificationtime', 'accesstime']
     alias :settimes :touch
 
-    # def delegation_token(user, options={}) # GETDELEGATIONTOKEN
-    #   raise NotImplementedError
-    # end
-    # def renew_delegation_token(token, options={}) # RENEWDELEGATIONTOKEN
-    #   raise NotImplementedError
-    # end
+    # curl -i "http://<HOST>:<PORT>/webhdfs/v1/?op=GETDELEGATIONTOKEN&renewer=<USER>"
+    def get_kerberos_delegation_token(user, options={})
+      options = options.merge({ 'renewer' => user })
+      check_options(options, OPT_TABLE['GETDELEGATIONTOKEN'])
+      res = operate_requests('GET', '/', 'GETDELEGATIONTOKEN', options)
+      check_success_json(res, 'Token')
+      JSON.parse(res.body)['Token']['urlString']
+    end
+    OPT_TABLE['GETDELEGATIONTOKEN'] = ['renewer']
+
+    # curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/?op=RENEWDELEGATIONTOKEN&token=<DELEGATION_TOKEN>"
+    def renew_kerberos_delegation_token(token, options={})
+      options = options.merge({ 'token' => token })
+      check_options(options, OPT_TABLE['RENEWDELEGATIONTOKEN'])
+      res = operate_requests('PUT', '/', 'RENEWDELEGATIONTOKEN', options)
+      check_success_json(res, 'long')
+    end
+    OPT_TABLE['RENEWDELEGATIONTOKEN'] = ['token']
+
     # def cancel_delegation_token(token, options={}) # CANCELDELEGATIONTOKEN
     #   raise NotImplementedError
     # end
@@ -352,7 +402,8 @@ module WebHDFS
         end
       end
 
-      if @kerberos and res.code == '307'
+      # if delegation token param settled, namenode do not response WWW-Authenticate header
+      if @kerberos and res.code == '307' and not params.key?('delegation')
         itok = (res.header.get_fields('WWW-Authenticate') || ['']).pop.split(/\s+/).last
         unless itok
           raise WebHDFS::KerberosError, 'Server does not return WWW-Authenticate header'
@@ -377,6 +428,18 @@ module WebHDFS
                     'Response body is empty...'
                   end
 
+        # when delegation token is invalid
+        if res.code == '403' and @renew_kerberos_delegation_token_time_hour && retries < @retry_times
+          if message.include?('{"RemoteException":{')
+            detail = JSON.parse(message) rescue nil
+            if detail&.dig('RemoteException', 'message')&.include?('HDFS_DELEGATION_TOKEN')
+              params = params.merge('token' => get_cached_kerberos_delegation_token(true))
+              sleep @retry_interval if @retry_interval > 0
+              return request(host, port, method, path, op, params, payload, header, retries+1)
+            end
+          end
+        end
+
         if @retry_known_errors && retries < @retry_times
           detail = nil
           if message =~ /^\{"RemoteException":\{/

data/webhdfs.gemspec

@@ -13,6 +13,7 @@ Gem::Specification.new do |gem|
   gem.files       = Dir['lib/**/*','test/**/*','*.gemspec','*.md','AUTHORS','COPYING','Gemfile','VERSION']
   gem.test_files  = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ['lib']
+  gem.licenses    = ["Apache-2.0"]
 
   gem.add_development_dependency "rake"
   gem.add_development_dependency "rdoc"

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: webhdfs
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.10.2
 platform: ruby
 authors:
 - Kazuki Ohta
 - Satoshi Tagomori
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-02-03 00:00:00.000000000 Z
+date: 2021-08-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -104,9 +104,10 @@ files:
 - test/webhdfs/fileutils.rb
 - webhdfs.gemspec
 homepage: https://github.com/kzk/webhdfs/
-licenses: []
+licenses:
+- Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -121,12 +122,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.1
-signing_key: 
+rubygems_version: 3.3.0.dev
+signing_key:
 specification_version: 4
 summary: Ruby WebHDFS/HttpFs client
 test_files:
 - test/test_helper.rb
 - test/webhdfs/fileutils.rb
-has_rdoc: false