webhdfs 0.7.3 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 683a33677fdfcccdf3bcd483c761e419b569e666
-  data.tar.gz: c759040209eb4ad21af23516737f5844140abed7
+SHA256:
+  metadata.gz: 769cd8dca5e3e329a882028bdcd06a3251e997148a7e85fdb68b14b21d37d989
+  data.tar.gz: f57beca7f1528b812e4bc99328120c673d93eb7eb48dc97aa5197ec976f7a5b3
 SHA512:
-  metadata.gz: 25d7628a72dac4ab37ada639fd63930f094e3b672367f4db3c8f2fb669f97a642a0f1cbfea2de9fcd88c752d502d2287f312632125982d5cf8d7497e19d2d10d
-  data.tar.gz: 4879bd5fa710df17d53ab80c99703c57df6863b15cabafdc704368a47146345abf6ad66ae54fd9dbf6a50d07f3e7a25cbc4ee9ce8a2d1c026a52120645bc0654
+  metadata.gz: 8b0ddef524644a473dd37f9bd13d63eb09cd50d6b709444a93d577dfbe8e080f1bde7426c142e20eae992c578674957b53e77d8eacd4aa644d754cd6602d05de
+  data.tar.gz: 00a583e338e6f4e3bcd15ebba1437503b4f17f3ee1ef253fec211a05b7959ce7cfa9f97a55cc0f9fc92277e5c7a828a9a63e1d7f56a15160361087f25e4dc3ab

data/README.md

@@ -95,15 +95,30 @@ Note that net/https and openssl libraries must be available:
     client.ssl = true
     client.ssl_ca_file = "/path/to/ca_file.pem" # if needed
     client.ssl_varify_mode = :peer # if needed (:none or :peer)
+    client.ssl_version = :TLSv1 # if needed
 
 ### For Kerberos Authentication
 
 Note that [gssapi](https://github.com/zenchild/gssapi) library must be available:
 
     client = WebHDFS::Client.new('hostname', 14000)
+    # or if you want to use client delegation token with renewing per 8 hours
+    client = WebHDFS::Client.new('hostname', 14000, username, nil, nil, nil, {}, 8)
+
     client.kerberos = true
     client.kerberos_keytab = "/path/to/project.keytab"
 
+### For SSL Client Authentication
+
+Note that openssl libraries must be available:
+
+    require 'openssl'
+    
+    client = WebHDFS::Client.new(host, port)
+    client.ssl = true
+    client.ssl_key = OpenSSL::PKey::RSA.new(open('/path/to/key.pem'))
+    client.ssl_cert = OpenSSL::X509::Certificate.new(open('/path/to/cert.pem'))
+
 ## AUTHORS
 
 * Kazuki Ohta <kazuki.ohta@gmail.com>

data/VERSION

@@ -1 +1 @@
-0.7.3
+0.10.0

data/lib/webhdfs/client_v1.rb

@@ -23,8 +23,12 @@ module WebHDFS
     attr_accessor :ssl
     attr_accessor :ssl_ca_file
     attr_reader   :ssl_verify_mode
+    attr_accessor :ssl_cert
+    attr_accessor :ssl_key
+    attr_accessor :ssl_version
     attr_accessor :kerberos, :kerberos_keytab
     attr_accessor :http_headers
+    attr_accessor :kerberos_delegation_token
 
     SSL_VERIFY_MODES = [:none, :peer]
     def ssl_verify_mode=(mode)
@@ -34,7 +38,7 @@ module WebHDFS
       @ssl_verify_mode = mode
     end
 
-    def initialize(host='localhost', port=50070, username=nil, doas=nil, proxy_address=nil, proxy_port=nil, http_headers={})
+    def initialize(host='localhost', port=50070, username=nil, doas=nil, proxy_address=nil, proxy_port=nil, http_headers={}, renew_kerberos_delegation_token_time_hour=nil)
       @host = host
       @port = port
       @username = username
@@ -50,35 +54,69 @@ module WebHDFS
       @ssl = false
       @ssl_ca_file = nil
       @ssl_verify_mode = nil
+      @ssl_cert = nil
+      @ssl_key = nil
+      @ssl_version = nil
 
       @kerberos = false
       @kerberos_keytab = nil
+      @renew_kerberos_delegation_token_time_hour = renew_kerberos_delegation_token_time_hour
+      @kerberos_delegation_token = nil
+      @kerberos_token_updated_at = Time.now
       @http_headers = http_headers
     end
 
+    def should_kerberos_token_updated?
+      @kerberos_token_updated_at + (@renew_kerberos_delegation_token_time_hour * 60 * 60) <= Time.now
+    end
+
+
+    def get_cached_kerberos_delegation_token(force_renew=nil)
+      return @kerberos_delegation_token if @kerberos_delegation_token && !should_kerberos_token_updated? && !force_renew
+
+      if !@kerberos_delegation_token || force_renew
+        @kerberos_delegation_token = get_kerberos_delegation_token(@username)
+        @kerberos_token_updated_at = Time.now
+      else
+        renew_kerberos_delegation_token(@kerberos_delegation_token)
+        @kerberos_token_updated_at = Time.now
+      end
+      @kerberos_delegation_token
+    rescue => e
+      raise WebHDFS::RequestFailedError, e.message
+    end
+
     # curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=CREATE
     #                 [&overwrite=<true|false>][&blocksize=<LONG>][&replication=<SHORT>]
-    #                 [&permission=<OCTAL>][&buffersize=<INT>]"
+    #                 [&permission=<OCTAL>][&buffersize=<INT>]
+    #                 [&delegation=<DELEGATION_TOKEN>]"
     def create(path, body, options={})
       if @httpfs_mode
         options = options.merge({'data' => 'true'})
       end
+      if @renew_kerberos_delegation_token_time_hour
+        options = options.merge('delegation' => get_cached_kerberos_delegation_token)
+      end
       check_options(options, OPT_TABLE['CREATE'])
       res = operate_requests('PUT', path, 'CREATE', options, body)
       res.code == '201'
     end
-    OPT_TABLE['CREATE'] = ['overwrite', 'blocksize', 'replication', 'permission', 'buffersize', 'data']
+    OPT_TABLE['CREATE'] = ['overwrite', 'blocksize', 'replication', 'permission', 'buffersize', 'data', 'delegation']
 
-    # curl -i -X POST "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=APPEND[&buffersize=<INT>]"
+    # curl -i -X POST "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=APPEND
+    #                   [&buffersize=<INT>][&delegation=<DELEGATION_TOKEN>]"
     def append(path, body, options={})
       if @httpfs_mode
         options = options.merge({'data' => 'true'})
       end
+      if @renew_kerberos_delegation_token_time_hour
+        options = options.merge('delegation' => get_cached_kerberos_delegation_token)
+      end
       check_options(options, OPT_TABLE['APPEND'])
       res = operate_requests('POST', path, 'APPEND', options, body)
       res.code == '200'
     end
-    OPT_TABLE['APPEND'] = ['buffersize', 'data']
+    OPT_TABLE['APPEND'] = ['buffersize', 'data', 'delegation']
 
     # curl -i -L "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=OPEN
     #                [&offset=<LONG>][&length=<LONG>][&buffersize=<INT>]"
@@ -158,6 +196,13 @@ module WebHDFS
     end
     alias :gethomedirectory :homedir
 
+    # curl -i "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=GETTRASHROOT"
+    def gettrashroot(options={})
+      check_options(options, OPT_TABLE['GETTRASHROOT'])
+      res = operate_requests('GET', '/', 'GETTRASHROOT', options)
+      check_success_json(res, 'Path')
+    end
+
     # curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=SETPERMISSION
     #                 [&permission=<OCTAL>]"
     def chmod(path, mode, options={})
@@ -206,12 +251,25 @@ module WebHDFS
     OPT_TABLE['SETTIMES'] = ['modificationtime', 'accesstime']
     alias :settimes :touch
 
-    # def delegation_token(user, options={}) # GETDELEGATIONTOKEN
-    #   raise NotImplementedError
-    # end
-    # def renew_delegation_token(token, options={}) # RENEWDELEGATIONTOKEN
-    #   raise NotImplementedError
-    # end
+    # curl -i "http://<HOST>:<PORT>/webhdfs/v1/?op=GETDELEGATIONTOKEN&renewer=<USER>"
+    def get_kerberos_delegation_token(user, options={})
+      options = options.merge({ 'renewer' => user })
+      check_options(options, OPT_TABLE['GETDELEGATIONTOKEN'])
+      res = operate_requests('GET', '/', 'GETDELEGATIONTOKEN', options)
+      check_success_json(res, 'Token')
+      JSON.parse(res.body)['Token']['urlString']
+    end
+    OPT_TABLE['GETDELEGATIONTOKEN'] = ['renewer']
+
+    # curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/?op=RENEWDELEGATIONTOKEN&token=<DELEGATION_TOKEN>"
+    def renew_kerberos_delegation_token(token, options={})
+      options = options.merge({ 'token' => token })
+      check_options(options, OPT_TABLE['RENEWDELEGATIONTOKEN'])
+      res = operate_requests('PUT', '/', 'RENEWDELEGATIONTOKEN', options)
+      check_success_json(res, 'long')
+    end
+    OPT_TABLE['RENEWDELEGATIONTOKEN'] = ['token']
+
     # def cancel_delegation_token(token, options={}) # CANCELDELEGATIONTOKEN
     #   raise NotImplementedError
     # end
@@ -300,6 +358,9 @@ module WebHDFS
                              when :peer then OpenSSL::SSL::VERIFY_PEER
                              end
         end
+        conn.cert = @ssl_cert if @ssl_cert
+        conn.key = @ssl_key if @ssl_key
+        conn.ssl_version = @ssl_version if @ssl_version
       end
 
       gsscli = nil
@@ -343,7 +404,8 @@ module WebHDFS
         end
       end
 
-      if @kerberos and res.code == '307'
+      # if delegation token param settled, namenode do not response WWW-Authenticate header
+      if @kerberos and res.code == '307' and not params.key?('delegation')
         itok = (res.header.get_fields('WWW-Authenticate') || ['']).pop.split(/\s+/).last
         unless itok
           raise WebHDFS::KerberosError, 'Server does not return WWW-Authenticate header'
@@ -368,6 +430,18 @@ module WebHDFS
                     'Response body is empty...'
                   end
 
+        # when delegation token is expired
+        if res.code == '403' and @renew_kerberos_delegation_token_time_hour
+          if message.include?('{"RemoteException":{')
+            detail = JSON.parse(message) rescue nil
+            if detail&.dig('RemoteException', 'exception') == 'InvalidToken' and detail&.dig('RemoteException', 'message')&.include?('HDFS_DELEGATION_TOKEN')
+              params = params.merge('token' => get_cached_kerberos_delegation_token(true))
+              sleep @retry_interval if @retry_interval > 0
+              return request(host, port, method, path, op, params, payload, header, retries+1)
+            end
+          end
+        end
+
         if @retry_known_errors && retries < @retry_times
           detail = nil
           if message =~ /^\{"RemoteException":\{/

data/webhdfs.gemspec

@@ -7,14 +7,13 @@ Gem::Specification.new do |gem|
   gem.homepage    = "https://github.com/kzk/webhdfs/"
   gem.summary     = gem.description
   gem.version     = File.read("VERSION").strip
-  gem.authors     = ["Kazuki Ohta"]
-  gem.email       = "kazuki.ohta@gmail.com"
+  gem.authors     = ["Kazuki Ohta", "Satoshi Tagomori"]
+  gem.email       = ["kazuki.ohta@gmail.com", "tagomoris@gmail.com"]
   gem.has_rdoc    = false
-  #gem.platform    = Gem::Platform::RUBY
-  gem.files       = `git ls-files`.split("\n")
-  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
-  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.files       = Dir['lib/**/*','test/**/*','*.gemspec','*.md','AUTHORS','COPYING','Gemfile','VERSION']
+  gem.test_files  = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ['lib']
+  gem.licenses    = ["Apache-2.0"]
 
   gem.add_development_dependency "rake"
   gem.add_development_dependency "rdoc"

metadata

@@ -1,14 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: webhdfs
 version: !ruby/object:Gem::Version
-  version: 0.7.3
+  version: 0.10.0
 platform: ruby
 authors:
 - Kazuki Ohta
-autorequire: 
+- Satoshi Tagomori
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-11-17 00:00:00.000000000 Z
+date: 2021-08-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -81,17 +82,17 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 description: Ruby WebHDFS/HttpFs client
-email: kazuki.ohta@gmail.com
+email:
+- kazuki.ohta@gmail.com
+- tagomoris@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".gitignore"
 - AUTHORS
 - COPYING
 - Gemfile
 - README.md
-- Rakefile
 - VERSION
 - lib/webhdfs.rb
 - lib/webhdfs/backport.rb
@@ -103,9 +104,10 @@ files:
 - test/webhdfs/fileutils.rb
 - webhdfs.gemspec
 homepage: https://github.com/kzk/webhdfs/
-licenses: []
+licenses:
+- Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -120,12 +122,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.4.5
-signing_key: 
+rubygems_version: 3.2.3
+signing_key:
 specification_version: 4
 summary: Ruby WebHDFS/HttpFs client
 test_files:
 - test/test_helper.rb
 - test/webhdfs/fileutils.rb
-has_rdoc: false

data/.gitignore

@@ -1,4 +0,0 @@
-/pkg/
-/coverage/
-/vendor/
-Gemfile.lock

data/Rakefile

@@ -1,21 +0,0 @@
-require 'bundler'
-Bundler::GemHelper.install_tasks
-
-require 'rake/testtask'
-
-Rake::TestTask.new(:test) do |test|
-  test.libs << 'lib' << 'test'
-  test.test_files = FileList['test/webhdfs/*.rb']
-  test.verbose = true
-end
-
-task :doc do |t|
-  `bundle exec rdoc --markup=tomdoc --visibility=public --include=lib --exclude=test`
-end
-
-task :coverage do |t|
-  ENV['SIMPLE_COV'] = '1'
-  Rake::Task["test"].invoke
-end
-
-task :default => [:build]