webhdfs 0.7.4 → 0.10.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 3633c5408544086d96f4274e31f80e193227d3e5
-  data.tar.gz: d882245ff5d655ba2fd87d54b80261859191b111
+SHA256:
+  metadata.gz: 202827bff28692693795a0b47541acfa5198fdc375401cdcdae0509ac1b01440
+  data.tar.gz: 5b23890cae46ba9363aa3facedd60c083e0bfaf74ba59aae233c866a949ba8bf
 SHA512:
-  metadata.gz: 5c3a6ddc8842f41bd40a6ff845059aae459d2e63bc2694b868fba029a8e7aa5ed7c64b097133b69f43986baf43930259303eb0f8d339d74a6cb1d0dd5730c35c
-  data.tar.gz: 644c24ba30a07b4acf078e9ba222dcdfcfcec60e35f072b2b0fe1e61f547e4352a6efef646ff90677b8ea67cf98b4e884b79d15e1a3b285adf0cfc753b40d770
+  metadata.gz: 85dc05b030202720b9baf7d4f60232232ff121fff4e7b49a4e6a9fd563df6ccb0293a52f793781aa2d48b2ecd9516e8851eca1b316d002af2dc5d6734dd8a977
+  data.tar.gz: edef2f1401b86df2b57fc253e9418837a331df79c4a5c388f41e5d88755c1a9944689107faaaee65849a6a7c481d0cd9c9df3b4f14b65598ec54f8daa7ef0f49

data/README.md

@@ -102,6 +102,9 @@ Note that net/https and openssl libraries must be available:
 Note that [gssapi](https://github.com/zenchild/gssapi) library must be available:
 
     client = WebHDFS::Client.new('hostname', 14000)
+    # or if you want to use client delegation token with renewing per 8 hours
+    client = WebHDFS::Client.new('hostname', 14000, username, nil, nil, nil, {}, 8)
+
     client.kerberos = true
     client.kerberos_keytab = "/path/to/project.keytab"
 

data/VERSION

@@ -1 +1 @@
-0.7.4
+0.10.1

data/lib/webhdfs/client_v1.rb

@@ -28,6 +28,7 @@ module WebHDFS
     attr_accessor :ssl_version
     attr_accessor :kerberos, :kerberos_keytab
     attr_accessor :http_headers
+    attr_accessor :kerberos_delegation_token
 
     SSL_VERIFY_MODES = [:none, :peer]
     def ssl_verify_mode=(mode)
@@ -37,7 +38,7 @@ module WebHDFS
       @ssl_verify_mode = mode
     end
 
-    def initialize(host='localhost', port=50070, username=nil, doas=nil, proxy_address=nil, proxy_port=nil, http_headers={})
+    def initialize(host='localhost', port=50070, username=nil, doas=nil, proxy_address=nil, proxy_port=nil, http_headers={}, renew_kerberos_delegation_token_time_hour=nil)
       @host = host
       @port = port
       @username = username
@@ -59,32 +60,61 @@ module WebHDFS
 
       @kerberos = false
       @kerberos_keytab = nil
+      @renew_kerberos_delegation_token_time_hour = renew_kerberos_delegation_token_time_hour
+      @kerberos_delegation_token = nil
+      @kerberos_token_updated_at = Time.now
       @http_headers = http_headers
     end
 
+    def should_kerberos_token_updated?
+      @kerberos_token_updated_at + (@renew_kerberos_delegation_token_time_hour * 60 * 60) <= Time.now
+    end
+
+
+    def get_cached_kerberos_delegation_token(force_renew=nil)
+      return @kerberos_delegation_token if @kerberos_delegation_token && !should_kerberos_token_updated? && !force_renew
+
+      if !@kerberos_delegation_token || force_renew
+        @kerberos_delegation_token = get_kerberos_delegation_token(@username)
+        @kerberos_token_updated_at = Time.now
+      else
+        renew_kerberos_delegation_token(@kerberos_delegation_token)
+        @kerberos_token_updated_at = Time.now
+      end
+      @kerberos_delegation_token
+    end
+
     # curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=CREATE
     #                 [&overwrite=<true|false>][&blocksize=<LONG>][&replication=<SHORT>]
-    #                 [&permission=<OCTAL>][&buffersize=<INT>]"
+    #                 [&permission=<OCTAL>][&buffersize=<INT>]
+    #                 [&delegation=<DELEGATION_TOKEN>]"
     def create(path, body, options={})
       if @httpfs_mode
         options = options.merge({'data' => 'true'})
       end
+      if @renew_kerberos_delegation_token_time_hour
+        options = options.merge('delegation' => get_cached_kerberos_delegation_token)
+      end
       check_options(options, OPT_TABLE['CREATE'])
       res = operate_requests('PUT', path, 'CREATE', options, body)
       res.code == '201'
     end
-    OPT_TABLE['CREATE'] = ['overwrite', 'blocksize', 'replication', 'permission', 'buffersize', 'data']
+    OPT_TABLE['CREATE'] = ['overwrite', 'blocksize', 'replication', 'permission', 'buffersize', 'data', 'delegation']
 
-    # curl -i -X POST "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=APPEND[&buffersize=<INT>]"
+    # curl -i -X POST "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=APPEND
+    #                   [&buffersize=<INT>][&delegation=<DELEGATION_TOKEN>]"
     def append(path, body, options={})
       if @httpfs_mode
         options = options.merge({'data' => 'true'})
       end
+      if @renew_kerberos_delegation_token_time_hour
+        options = options.merge('delegation' => get_cached_kerberos_delegation_token)
+      end
       check_options(options, OPT_TABLE['APPEND'])
       res = operate_requests('POST', path, 'APPEND', options, body)
       res.code == '200'
     end
-    OPT_TABLE['APPEND'] = ['buffersize', 'data']
+    OPT_TABLE['APPEND'] = ['buffersize', 'data', 'delegation']
 
     # curl -i -L "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=OPEN
     #                [&offset=<LONG>][&length=<LONG>][&buffersize=<INT>]"
@@ -164,6 +194,13 @@ module WebHDFS
     end
     alias :gethomedirectory :homedir
 
+    # curl -i "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=GETTRASHROOT"
+    def gettrashroot(options={})
+      check_options(options, OPT_TABLE['GETTRASHROOT'])
+      res = operate_requests('GET', '/', 'GETTRASHROOT', options)
+      check_success_json(res, 'Path')
+    end
+
     # curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=SETPERMISSION
     #                 [&permission=<OCTAL>]"
     def chmod(path, mode, options={})
@@ -212,12 +249,25 @@ module WebHDFS
     OPT_TABLE['SETTIMES'] = ['modificationtime', 'accesstime']
     alias :settimes :touch
 
-    # def delegation_token(user, options={}) # GETDELEGATIONTOKEN
-    #   raise NotImplementedError
-    # end
-    # def renew_delegation_token(token, options={}) # RENEWDELEGATIONTOKEN
-    #   raise NotImplementedError
-    # end
+    # curl -i "http://<HOST>:<PORT>/webhdfs/v1/?op=GETDELEGATIONTOKEN&renewer=<USER>"
+    def get_kerberos_delegation_token(user, options={})
+      options = options.merge({ 'renewer' => user })
+      check_options(options, OPT_TABLE['GETDELEGATIONTOKEN'])
+      res = operate_requests('GET', '/', 'GETDELEGATIONTOKEN', options)
+      check_success_json(res, 'Token')
+      JSON.parse(res.body)['Token']['urlString']
+    end
+    OPT_TABLE['GETDELEGATIONTOKEN'] = ['renewer']
+
+    # curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/?op=RENEWDELEGATIONTOKEN&token=<DELEGATION_TOKEN>"
+    def renew_kerberos_delegation_token(token, options={})
+      options = options.merge({ 'token' => token })
+      check_options(options, OPT_TABLE['RENEWDELEGATIONTOKEN'])
+      res = operate_requests('PUT', '/', 'RENEWDELEGATIONTOKEN', options)
+      check_success_json(res, 'long')
+    end
+    OPT_TABLE['RENEWDELEGATIONTOKEN'] = ['token']
+
     # def cancel_delegation_token(token, options={}) # CANCELDELEGATIONTOKEN
     #   raise NotImplementedError
     # end
@@ -352,7 +402,8 @@ module WebHDFS
         end
       end
 
-      if @kerberos and res.code == '307'
+      # if delegation token param settled, namenode do not response WWW-Authenticate header
+      if @kerberos and res.code == '307' and not params.key?('delegation')
         itok = (res.header.get_fields('WWW-Authenticate') || ['']).pop.split(/\s+/).last
         unless itok
           raise WebHDFS::KerberosError, 'Server does not return WWW-Authenticate header'
@@ -377,6 +428,18 @@ module WebHDFS
                     'Response body is empty...'
                   end
 
+        # when delegation token is expired
+        if res.code == '403' and @renew_kerberos_delegation_token_time_hour
+          if message.include?('{"RemoteException":{')
+            detail = JSON.parse(message) rescue nil
+            if detail&.dig('RemoteException', 'exception') == 'InvalidToken' and detail&.dig('RemoteException', 'message')&.include?('HDFS_DELEGATION_TOKEN')
+              params = params.merge('token' => get_cached_kerberos_delegation_token(true))
+              sleep @retry_interval if @retry_interval > 0
+              return request(host, port, method, path, op, params, payload, header, retries+1)
+            end
+          end
+        end
+
         if @retry_known_errors && retries < @retry_times
           detail = nil
           if message =~ /^\{"RemoteException":\{/

data/webhdfs.gemspec

@@ -7,14 +7,13 @@ Gem::Specification.new do |gem|
   gem.homepage    = "https://github.com/kzk/webhdfs/"
   gem.summary     = gem.description
   gem.version     = File.read("VERSION").strip
-  gem.authors     = ["Kazuki Ohta"]
-  gem.email       = "kazuki.ohta@gmail.com"
+  gem.authors     = ["Kazuki Ohta", "Satoshi Tagomori"]
+  gem.email       = ["kazuki.ohta@gmail.com", "tagomoris@gmail.com"]
   gem.has_rdoc    = false
-  #gem.platform    = Gem::Platform::RUBY
-  gem.files       = `git ls-files`.split("\n")
-  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
-  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.files       = Dir['lib/**/*','test/**/*','*.gemspec','*.md','AUTHORS','COPYING','Gemfile','VERSION']
+  gem.test_files  = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ['lib']
+  gem.licenses    = ["Apache-2.0"]
 
   gem.add_development_dependency "rake"
   gem.add_development_dependency "rdoc"

metadata

@@ -1,14 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: webhdfs
 version: !ruby/object:Gem::Version
-  version: 0.7.4
+  version: 0.10.1
 platform: ruby
 authors:
 - Kazuki Ohta
-autorequire: 
+- Satoshi Tagomori
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-12-22 00:00:00.000000000 Z
+date: 2021-08-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -81,17 +82,17 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 description: Ruby WebHDFS/HttpFs client
-email: kazuki.ohta@gmail.com
+email:
+- kazuki.ohta@gmail.com
+- tagomoris@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".gitignore"
 - AUTHORS
 - COPYING
 - Gemfile
 - README.md
-- Rakefile
 - VERSION
 - lib/webhdfs.rb
 - lib/webhdfs/backport.rb
@@ -103,9 +104,10 @@ files:
 - test/webhdfs/fileutils.rb
 - webhdfs.gemspec
 homepage: https://github.com/kzk/webhdfs/
-licenses: []
+licenses:
+- Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -120,12 +122,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.4.5
-signing_key: 
+rubygems_version: 3.2.3
+signing_key:
 specification_version: 4
 summary: Ruby WebHDFS/HttpFs client
 test_files:
 - test/test_helper.rb
 - test/webhdfs/fileutils.rb
-has_rdoc: false

data/.gitignore

@@ -1,4 +0,0 @@
-/pkg/
-/coverage/
-/vendor/
-Gemfile.lock

data/Rakefile

@@ -1,21 +0,0 @@
-require 'bundler'
-Bundler::GemHelper.install_tasks
-
-require 'rake/testtask'
-
-Rake::TestTask.new(:test) do |test|
-  test.libs << 'lib' << 'test'
-  test.test_files = FileList['test/webhdfs/*.rb']
-  test.verbose = true
-end
-
-task :doc do |t|
-  `bundle exec rdoc --markup=tomdoc --visibility=public --include=lib --exclude=test`
-end
-
-task :coverage do |t|
-  ENV['SIMPLE_COV'] = '1'
-  Rake::Task["test"].invoke
-end
-
-task :default => [:build]