webhdfs 0.5.5 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 079a9de419dc30c5280876fdb45dc4e6a3844fc1
-  data.tar.gz: 915c3c102e9ae29a729d6498d2280b2fc94077f3
+  metadata.gz: 722d97c3cd885df9fcdf8d6dc6ffb53b85e0781c
+  data.tar.gz: 7048f1a9c7346a7e82caa942e71a5680cc930dad
 SHA512:
-  metadata.gz: 514504c4024a57b80d81347634c33fa061c82ab28f6f55ff47f9dc5c3dde2efdca053161a912479bb64f2e8e6fef137af1f639848dbe5b1d506b349ac8f6045b
-  data.tar.gz: 6135ce124d4619ed25a5f1aa790e9cbb7f9cc5a56085256d6bd386f9a9eb0c9fc3a9537b7e8ef72f8a7d19a95afc3f42098c2e3ce5874fcd7526dcb469903a25
+  metadata.gz: 404e0069c1e3b01fb74259ba9883ce5a3bb1655775af7481e5270ad8f148699cc689aa3049258161f8e86a6ae4f710509d994ac407323cd8e13fd36f47352efa
+  data.tar.gz: e7765eb4be5c4629c7c1f5b5f9bda3721cfb3b90b99e8d170a72f8107a7c1160f73a463aa770005a8f250d49ca7528be403cc35a87def3791e593302686f2846

data/Gemfile

@@ -1,5 +1,3 @@
-source :rubygems
+source "https://rubygems.org"
 
 gemspec
-
-gem "simplecov", :require => false

data/README.md

@@ -87,6 +87,22 @@ For HttpFs instead of WebHDFS:
     client.proxy_user = 'jack'   # if needed
     client.proxy_pass = 'secret' # if needed
 
+### For SSL
+
+Note that net/https and openssl libraries must be available:
+
+    client = WebHDFS::Client.new('hostname', 4443)
+    client.ssl = true
+    client.ssl_ca_file = "/path/to/ca_file.pem" # if needed
+    client.ssl_varify_mode = :peer # if needed (:none or :peer)
+
+### For Kerberos Authentication
+
+Note that [gssapi](https://github.com/zenchild/gssapi) library must be available:
+
+    client = WebHDFS::Client.new('hostname', 14000)
+    client.kerberos = true
+
 ## AUTHORS
 
 * Kazuki Ohta <kazuki.ohta@gmail.com>

data/VERSION

@@ -1 +1 @@
-0.5.5
+0.6.0

data/lib/webhdfs/client_v1.rb

@@ -19,6 +19,18 @@ module WebHDFS
     attr_accessor :retry_known_errors # default false (not to retry)
     attr_accessor :retry_times        # default 1 (ignored when retry_known_errors is false)
     attr_accessor :retry_interval     # default 1 ([sec], ignored when retry_known_errors is false)
+    attr_accessor :ssl
+    attr_accessor :ssl_ca_file
+    attr_reader   :ssl_verify_mode
+    attr_accessor :kerberos
+
+    SSL_VERIFY_MODES = [:none, :peer]
+    def ssl_verify_mode=(mode)
+      unless SSL_VERIFY_MODES.include? mode
+        raise ArgumentError, "Invalid SSL verify mode #{mode.inspect}"
+      end
+      @ssl_verify_mode = mode
+    end
 
     def initialize(host='localhost', port=50070, username=nil, doas=nil, proxy_address=nil, proxy_port=nil)
       @host = host
@@ -32,6 +44,12 @@ module WebHDFS
       @retry_interval = 1
 
       @httpfs_mode = false
+
+      @ssl = false
+      @ssl_ca_file = nil
+      @ssl_verify_mode = nil
+
+      @kerberos = false
     end
 
     # curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=CREATE
@@ -267,11 +285,39 @@ module WebHDFS
                      else
                        path
                      end
+      if @ssl
+        conn.use_ssl = true
+        conn.ca_file = @ssl_ca_file if @ssl_ca_file
+        if @ssl_verify_mode
+          require 'openssl'
+          conn.verify_mode = case @ssl_verify_mode
+                             when :none then OpenSSL::SSL::VERIFY_NONE
+                             when :peer then OpenSSL::SSL::VERIFY_PEER
+                             end
+        end
+      end
+
+      gsscli = nil
+      if @kerberos
+        require 'base64'
+        require 'gssapi'
+        gsscli = GSSAPI::Simple.new(@host, 'HTTP')
+        token = nil
+        begin
+          token = gsscli.init_context
+        rescue => e
+          raise WebHDFS::KerberosError, e.message
+        end
+        if header
+          header['Authorization'] = "Negotiate #{Base64.strict_encode64(token)}"
+        else
+          header = {'Authorization' => "Negotiate #{Base64.strict_encode64(token)}"}
+        end
+      end
 
       res = nil
-      if !payload.nil? and payload.is_a?(IO)
+      if !payload.nil? and payload.respond_to? :read and payload.respond_to? :size
         req = Net::HTTPGenericRequest.new(method,(payload ? true : false),true,request_path,header)
-        raise WebHDFS::IOError, 'Error reading given IO data source' unless payload.respond_to? :read and payload.respond_to? :size
         raise WebHDFS::ClientError, 'Error accepting given IO resource as data payload, Not valid in methods other than PUT and POST' unless (method == 'PUT' or method == 'POST')
 
         req.body_stream = payload
@@ -281,6 +327,19 @@ module WebHDFS
         res = conn.send_request(method, request_path, payload, header)
       end
 
+      if @kerberos
+        itok = (res.header.get_fields('WWW-Authenticate') || ['']).pop.split(/\s+/).last
+        unless itok
+          raise WebHDFS::KerberosError, 'Server does not return WWW-Authenticate header'
+        end
+
+        begin
+          gsscli.init_context(Base64.strict_decode64(itok))
+        rescue => e
+          raise WebHDFS::KerberosError, e.message
+        end
+      end
+
       case res
       when Net::HTTPSuccess
         res

data/lib/webhdfs/exceptions.rb

@@ -10,3 +10,5 @@ class WebHDFS::ClientError < WebHDFS::Error; end
 class WebHDFS::ServerError < WebHDFS::Error; end
 
 class WebHDFS::RequestFailedError < WebHDFS::Error; end
+
+class WebHDFS::KerberosError < WebHDFS::Error; end

data/lib/webhdfs/fileutils.rb

@@ -8,6 +8,10 @@ module WebHDFS
     @fu_user = nil
     @fu_doas = nil
     @fu_httpfs_mode = false
+    @fu_ssl = false
+    @fu_ssl_ca_file = nil
+    @fu_ssl_verify_mode = nil
+    @fu_kerberos = false
 
     # Public: Set hostname and port number of WebHDFS
     #
@@ -45,6 +49,58 @@ module WebHDFS
     end
     module_function :set_httpfs_mode
 
+    # Public: Set ssl enable/disable
+    #
+    # mode - boolean (default true)
+    #
+    # Examples
+    #
+    #   FileUtils.set_ssl
+    #
+    def set_ssl(mode=true)
+      @fu_ssl = mode
+    end
+    module_function :set_ssl
+
+    # Public: Set ssl ca_file
+    #
+    # ca_file - string
+    #
+    # Examples
+    #
+    #   FileUtils.set_ca_file("/path/to/ca_file.pem")
+    #
+    def set_ssl_ca_file(ca_file)
+      @fu_ssl_ca_file = ca_file
+    end
+    module_function :set_ssl_ca_file
+
+    # Public: Set ssl verify mode
+    #
+    # mode - :none or :peer
+    #
+    # Examples
+    #
+    #   FileUtils.set_ssl_verify_mode(:peer)
+    #
+    def set_ssl_verify_mode(mode)
+      @fu_ssl_verify_mode = mode
+    end
+    module_function :set_ssl_verify_mode
+
+    # Public: Set kerberos authentication enable/disable
+    #
+    # mode - boolean (default true)
+    #
+    # Examples
+    #
+    #   FileUtils.set_kerberos
+    #
+    def set_kerberos(mode=true)
+      @fu_kerberos = mode
+    end
+    module_function :set_kerberos
+
     # Public: Copy local file into HDFS
     #
     # file - local file path
@@ -363,6 +419,10 @@ module WebHDFS
       if @fu_httpfs_mode
         client.httpfs_mode = true
       end
+      client.ssl = true if @fu_ssl
+      client.ssl_ca_file = @fu_ssl_ca_file if @fu_ssl_ca_file
+      client.ssl_verify_mode = @fu_ssl_verify_mode if @fu_ssl_verify_mode
+      client.kerberos = true if @fu_kerberos
       client
     end
     private_module_function :client

data/webhdfs.gemspec

@@ -16,8 +16,8 @@ Gem::Specification.new do |gem|
   gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   gem.require_paths = ['lib']
 
-  gem.add_development_dependency "rake", ">= 0.9.2"
-  gem.add_development_dependency "rdoc", ">= 3.12"
-  gem.add_development_dependency "simplecov", ">= 0.5.4"
-  gem.add_development_dependency "rr", ">= 1.0.0"
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "rdoc"
+  gem.add_development_dependency "simplecov"
+  gem.add_development_dependency "rr"
 end

metadata

@@ -1,78 +1,78 @@
 --- !ruby/object:Gem::Specification
 name: webhdfs
 version: !ruby/object:Gem::Version
-  version: 0.5.5
+  version: 0.6.0
 platform: ruby
 authors:
 - Kazuki Ohta
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-04 00:00:00.000000000 Z
+date: 2014-10-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.9.2
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.9.2
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3.12'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3.12'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.5.4
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.5.4
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rr
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: '0'
 description: Ruby WebHDFS/HttpFs client
 email: kazuki.ohta@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
 - AUTHORS
 - COPYING
 - Gemfile
@@ -97,21 +97,20 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Ruby WebHDFS/HttpFs client
 test_files:
 - test/test_helper.rb
 - test/webhdfs/fileutils.rb
-has_rdoc: false