webhdfs 0.9.0 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5c3c7efca2c4e4c3a82a12c544dc5d86790e9b19aa050f1269bfb17628be2bb8
-  data.tar.gz: 8d7dce607082bbe3c2eaaa17eb26d08158eb3844f1d938580727a9dd4be65729
+  metadata.gz: 769cd8dca5e3e329a882028bdcd06a3251e997148a7e85fdb68b14b21d37d989
+  data.tar.gz: f57beca7f1528b812e4bc99328120c673d93eb7eb48dc97aa5197ec976f7a5b3
 SHA512:
-  metadata.gz: b43c91f0fb8b1908813ef31b012083d3613bde72fa7a0228a9d4200f3a140df95c5658a2c2f1d2e9888eb888c8e44ef10018705167bbcbaf9963a17d505eb93c
-  data.tar.gz: 209843fe4e9afb954d9f0523a9c2117302d7fb1f2d3118589775b5b0ac4f0b8720b27d954924dfbb5b8d8b5dd521790fd3ef8f632556292eb4c465c8e4087edc
+  metadata.gz: 8b0ddef524644a473dd37f9bd13d63eb09cd50d6b709444a93d577dfbe8e080f1bde7426c142e20eae992c578674957b53e77d8eacd4aa644d754cd6602d05de
+  data.tar.gz: 00a583e338e6f4e3bcd15ebba1437503b4f17f3ee1ef253fec211a05b7959ce7cfa9f97a55cc0f9fc92277e5c7a828a9a63e1d7f56a15160361087f25e4dc3ab

data/README.md

@@ -102,6 +102,9 @@ Note that net/https and openssl libraries must be available:
 Note that [gssapi](https://github.com/zenchild/gssapi) library must be available:
 
     client = WebHDFS::Client.new('hostname', 14000)
+    # or if you want to use client delegation token with renewing per 8 hours
+    client = WebHDFS::Client.new('hostname', 14000, username, nil, nil, nil, {}, 8)
+
     client.kerberos = true
     client.kerberos_keytab = "/path/to/project.keytab"
 

data/VERSION

@@ -1 +1 @@
-0.9.0
+0.10.0

data/lib/webhdfs/client_v1.rb

@@ -28,6 +28,7 @@ module WebHDFS
     attr_accessor :ssl_version
     attr_accessor :kerberos, :kerberos_keytab
     attr_accessor :http_headers
+    attr_accessor :kerberos_delegation_token
 
     SSL_VERIFY_MODES = [:none, :peer]
     def ssl_verify_mode=(mode)
@@ -37,7 +38,7 @@ module WebHDFS
       @ssl_verify_mode = mode
     end
 
-    def initialize(host='localhost', port=50070, username=nil, doas=nil, proxy_address=nil, proxy_port=nil, http_headers={})
+    def initialize(host='localhost', port=50070, username=nil, doas=nil, proxy_address=nil, proxy_port=nil, http_headers={}, renew_kerberos_delegation_token_time_hour=nil)
       @host = host
       @port = port
       @username = username
@@ -59,32 +60,63 @@ module WebHDFS
 
       @kerberos = false
       @kerberos_keytab = nil
+      @renew_kerberos_delegation_token_time_hour = renew_kerberos_delegation_token_time_hour
+      @kerberos_delegation_token = nil
+      @kerberos_token_updated_at = Time.now
       @http_headers = http_headers
     end
 
+    def should_kerberos_token_updated?
+      @kerberos_token_updated_at + (@renew_kerberos_delegation_token_time_hour * 60 * 60) <= Time.now
+    end
+
+
+    def get_cached_kerberos_delegation_token(force_renew=nil)
+      return @kerberos_delegation_token if @kerberos_delegation_token && !should_kerberos_token_updated? && !force_renew
+
+      if !@kerberos_delegation_token || force_renew
+        @kerberos_delegation_token = get_kerberos_delegation_token(@username)
+        @kerberos_token_updated_at = Time.now
+      else
+        renew_kerberos_delegation_token(@kerberos_delegation_token)
+        @kerberos_token_updated_at = Time.now
+      end
+      @kerberos_delegation_token
+    rescue => e
+      raise WebHDFS::RequestFailedError, e.message
+    end
+
     # curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=CREATE
     #                 [&overwrite=<true|false>][&blocksize=<LONG>][&replication=<SHORT>]
-    #                 [&permission=<OCTAL>][&buffersize=<INT>]"
+    #                 [&permission=<OCTAL>][&buffersize=<INT>]
+    #                 [&delegation=<DELEGATION_TOKEN>]"
     def create(path, body, options={})
       if @httpfs_mode
         options = options.merge({'data' => 'true'})
       end
+      if @renew_kerberos_delegation_token_time_hour
+        options = options.merge('delegation' => get_cached_kerberos_delegation_token)
+      end
       check_options(options, OPT_TABLE['CREATE'])
       res = operate_requests('PUT', path, 'CREATE', options, body)
       res.code == '201'
     end
-    OPT_TABLE['CREATE'] = ['overwrite', 'blocksize', 'replication', 'permission', 'buffersize', 'data']
+    OPT_TABLE['CREATE'] = ['overwrite', 'blocksize', 'replication', 'permission', 'buffersize', 'data', 'delegation']
 
-    # curl -i -X POST "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=APPEND[&buffersize=<INT>]"
+    # curl -i -X POST "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=APPEND
+    #                   [&buffersize=<INT>][&delegation=<DELEGATION_TOKEN>]"
     def append(path, body, options={})
       if @httpfs_mode
         options = options.merge({'data' => 'true'})
       end
+      if @renew_kerberos_delegation_token_time_hour
+        options = options.merge('delegation' => get_cached_kerberos_delegation_token)
+      end
       check_options(options, OPT_TABLE['APPEND'])
       res = operate_requests('POST', path, 'APPEND', options, body)
       res.code == '200'
     end
-    OPT_TABLE['APPEND'] = ['buffersize', 'data']
+    OPT_TABLE['APPEND'] = ['buffersize', 'data', 'delegation']
 
     # curl -i -L "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=OPEN
     #                [&offset=<LONG>][&length=<LONG>][&buffersize=<INT>]"
@@ -219,12 +251,25 @@ module WebHDFS
     OPT_TABLE['SETTIMES'] = ['modificationtime', 'accesstime']
     alias :settimes :touch
 
-    # def delegation_token(user, options={}) # GETDELEGATIONTOKEN
-    #   raise NotImplementedError
-    # end
-    # def renew_delegation_token(token, options={}) # RENEWDELEGATIONTOKEN
-    #   raise NotImplementedError
-    # end
+    # curl -i "http://<HOST>:<PORT>/webhdfs/v1/?op=GETDELEGATIONTOKEN&renewer=<USER>"
+    def get_kerberos_delegation_token(user, options={})
+      options = options.merge({ 'renewer' => user })
+      check_options(options, OPT_TABLE['GETDELEGATIONTOKEN'])
+      res = operate_requests('GET', '/', 'GETDELEGATIONTOKEN', options)
+      check_success_json(res, 'Token')
+      JSON.parse(res.body)['Token']['urlString']
+    end
+    OPT_TABLE['GETDELEGATIONTOKEN'] = ['renewer']
+
+    # curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/?op=RENEWDELEGATIONTOKEN&token=<DELEGATION_TOKEN>"
+    def renew_kerberos_delegation_token(token, options={})
+      options = options.merge({ 'token' => token })
+      check_options(options, OPT_TABLE['RENEWDELEGATIONTOKEN'])
+      res = operate_requests('PUT', '/', 'RENEWDELEGATIONTOKEN', options)
+      check_success_json(res, 'long')
+    end
+    OPT_TABLE['RENEWDELEGATIONTOKEN'] = ['token']
+
     # def cancel_delegation_token(token, options={}) # CANCELDELEGATIONTOKEN
     #   raise NotImplementedError
     # end
@@ -359,7 +404,8 @@ module WebHDFS
         end
       end
 
-      if @kerberos and res.code == '307'
+      # if delegation token param settled, namenode do not response WWW-Authenticate header
+      if @kerberos and res.code == '307' and not params.key?('delegation')
         itok = (res.header.get_fields('WWW-Authenticate') || ['']).pop.split(/\s+/).last
         unless itok
           raise WebHDFS::KerberosError, 'Server does not return WWW-Authenticate header'
@@ -384,6 +430,18 @@ module WebHDFS
                     'Response body is empty...'
                   end
 
+        # when delegation token is expired
+        if res.code == '403' and @renew_kerberos_delegation_token_time_hour
+          if message.include?('{"RemoteException":{')
+            detail = JSON.parse(message) rescue nil
+            if detail&.dig('RemoteException', 'exception') == 'InvalidToken' and detail&.dig('RemoteException', 'message')&.include?('HDFS_DELEGATION_TOKEN')
+              params = params.merge('token' => get_cached_kerberos_delegation_token(true))
+              sleep @retry_interval if @retry_interval > 0
+              return request(host, port, method, path, op, params, payload, header, retries+1)
+            end
+          end
+        end
+
         if @retry_known_errors && retries < @retry_times
           detail = nil
           if message =~ /^\{"RemoteException":\{/

data/webhdfs.gemspec

@@ -13,6 +13,7 @@ Gem::Specification.new do |gem|
   gem.files       = Dir['lib/**/*','test/**/*','*.gemspec','*.md','AUTHORS','COPYING','Gemfile','VERSION']
   gem.test_files  = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ['lib']
+  gem.licenses    = ["Apache-2.0"]
 
   gem.add_development_dependency "rake"
   gem.add_development_dependency "rdoc"

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: webhdfs
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 0.10.0
 platform: ruby
 authors:
 - Kazuki Ohta
 - Satoshi Tagomori
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-11-11 00:00:00.000000000 Z
+date: 2021-08-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -104,9 +104,10 @@ files:
 - test/webhdfs/fileutils.rb
 - webhdfs.gemspec
 homepage: https://github.com/kzk/webhdfs/
-licenses: []
+licenses:
+- Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -121,8 +122,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.2.3
+signing_key:
 specification_version: 4
 summary: Ruby WebHDFS/HttpFs client
 test_files: