webhdfs-with-krb5 0.6.1

data/AUTHORS

@@ -0,0 +1,3 @@
+Kazuki Ohta <kazuki.ohta@gmail.com>
+TAGOMORI Satoshi <tagomoris@gmail.com>
+Zixian.shen <zixian.shen@gmail.com>

data/COPYING

@@ -0,0 +1,13 @@
+Copyright (C) 2012 Fluentd Project
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/Gemfile

@@ -0,0 +1,6 @@
+source "http://rubygems.org"
+
+gemspec
+
+gem "simplecov", :require => false
+

data/README.md

@@ -0,0 +1,84 @@
+now revising...
+
+# webhdfs-client - A client library implementation for Hadoop WebHDFS, and HttpFs, for Ruby
+
+The webhdfs-client gem is to access Hadoop WebHDFS (EXPERIMENTAL: and HttpFs). WebHDFS::Client is a client class, and WebHDFS::FileUtils is utility like 'fileutils'.
+
+## Installation
+
+
+## Usage
+
+### WebHDFS::Client
+
+For client object interface:
+
+    require 'webhdfs'
+    client = WebHDFS::Client.new(hostname, port)
+    # or with pseudo username authentication
+    client = WebHDFS::Client.new(hostname, port, username)
+
+To create/append/read files:
+
+    client.create('/path/to/file', data)
+    client.create('/path/to/file', data, :overwrite => false, :blocksize => 268435456, :replication => 5, :permission => 0666)
+    
+    client.append('/path/to/existing/file', data)
+    
+    client.read('/path/to/target') #=> data
+    client.read('/path/to/target' :offset => 2048, :length => 1024) #=> data
+
+To mkdir/rename/delete directories or files:
+
+    client.mkdir('/hdfs/dirname')
+    client.mkdir('/hdfs/dirname', :permission => 0777)
+    
+    client.rename(original_path, dst_path)
+    
+    client.delete(path)
+    client.delete(dir_path, :recursive => true)
+
+To get status or list of files and directories:
+
+    client.stat(file_path) #=> key-value pairs for file status
+    client.list(dir_path)  #=> list of key-value pairs for files in dir_path
+
+And, 'content_summary', 'checksum', 'homedir', 'chmod', 'chown', 'replication' and 'touch' methods available.
+
+### WebHDFS::FileUtils
+
+    require 'webhdfs/fileutils'
+    WebHDFS::FileUtils.set_server(host, port)
+    # or
+    WebHDFS::FileUtils.set_server(host, port, username, doas)
+    
+    WebHDFS::FileUtils.copy_from_local(localpath, hdfspath)
+    WebHDFS::FileUtils.copy_to_local(hdfspath, localpath)
+    
+    WebHDFS::FileUtils.append(path, data)
+
+### For HttpFs
+
+For HttpFs instead of WebHDFS:
+
+    client = WebHDFS::Client('hostname', 14000)
+    client.httpfs_mode = true
+    
+    client.read(path) #=> data
+    
+    # or with webhdfs/filetuils
+    WebHDFS::FileUtils.set_server('hostname', 14000)
+    WebHDFS::FileUtils.set_httpfs_mode
+    WebHDFS::FileUtils.copy_to_local(remote_path, local_path)
+
+## AUTHORS
+
+* Kazuki Ohta <kazuki.ohta@gmail.com>
+* TAGOMORI Satoshi <tagomoris@gmail.com>
+* ZiXian Shen <zixian.shen@gmail.com>
+* Takamatsu Tomoaki <takamatsutomoaki@gmail.com>
+
+## LICENSE
+
+* Copyright: Copyright (c) 2012- Fluentd Project
+* License: Apache License, Version 2.0

data/Rakefile

@@ -0,0 +1,21 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/webhdfs/*.rb']
+  test.verbose = true
+end
+
+task :doc do |t|
+  `bundle exec rdoc --markup=tomdoc --visibility=public --include=lib --exclude=test`
+end
+
+task :coverage do |t|
+  ENV['SIMPLE_COV'] = '1'
+  Rake::Task["test"].invoke
+end
+
+task :default => [:build]

data/VERSION

@@ -0,0 +1 @@
+0.6.1

data/lib/webhdfs/client.rb

@@ -0,0 +1,6 @@
+require_relative 'client_v1'
+
+module WebHDFS
+  class Client < ClientV1
+  end
+end

data/lib/webhdfs/client_v1.rb

@@ -0,0 +1,349 @@
+#require 'net/http'
+require 'httpi'
+require 'krb5_auth'
+require 'uri'
+require 'json'
+
+require_relative 'exceptions'
+
+module WebHDFS
+  class ClientV1
+
+    # This hash table holds command options.
+    OPT_TABLE = {} # internal use only
+
+    attr_accessor :host, :port, :username, :doas
+    attr_accessor :open_timeout # default 30s (in ruby net/http)
+    attr_accessor :read_timeout # default 60s (in ruby net/http)
+    attr_accessor :httpfs_mode
+    attr_accessor :auth_type       # pseudo, kerberos
+    attr_accessor :keytab
+    attr_accessor :pass
+
+
+    def initialize(host='localhost', port=14000, username=nil, doas=nil)
+      @host = host
+      @port = port
+      @username = username
+      @doas = doas
+
+      @httpfs_mode = false
+
+      @auth_type = :pseudo
+      @keytab = nil
+      @pass = nil
+    end
+
+    # curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=CREATESYMLINK&destination=<PATH>
+    #                 [&createParent=<true|false>]"
+    def create_symlink(path, dest, options={})
+      check_options(options, OPT_TABLE['CREATESYMLINK'])
+      unless dest.start_with?('/')
+        dest = '/' + dest
+      end
+      res = operate_requests(:put, path, 'CREATESYMLINK', options.merge({'destination' => dest}))
+      check_success_json(res, 'boolean')
+    end
+    OPT_TABLE['CREATESYMLINK'] = ['createParent']
+
+    # curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=CREATE
+    #                 [&overwrite=<true|false>][&blocksize=<LONG>][&replication=<SHORT>]
+    #                 [&permission=<OCTAL>][&buffersize=<INT>]"
+    def create(path, body, options={})
+      if @httpfs_mode
+        options = options.merge({'data' => 'true'})
+      end
+      check_options(options, OPT_TABLE['CREATE'])
+      res = operate_requests(:put, path, 'CREATE', options, body)
+      res.code == 201
+    end
+    OPT_TABLE['CREATE'] = ['overwrite', 'blocksize', 'replication', 'permission', 'buffersize', 'data']
+
+    # curl -i -X POST "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=APPEND[&buffersize=<INT>]"
+    def append(path, body, options={})
+      if @httpfs_mode
+        options = options.merge({'data' => 'true'})
+      end
+      check_options(options, OPT_TABLE['APPEND'])
+      res = operate_requests(:post, path, 'APPEND', options, body)
+      res.code == 200
+    end
+    OPT_TABLE['APPEND'] = ['buffersize', 'data']
+
+    # curl -i -L "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=OPEN
+    #                [&offset=<LONG>][&length=<LONG>][&buffersize=<INT>]"
+    def read(path, options={})
+      check_options(options, OPT_TABLE['OPEN'])
+      res = operate_requests(:get, path, 'OPEN', options)
+      res.body
+    end
+    OPT_TABLE['OPEN'] = ['offset', 'length', 'buffersize']
+    alias :open :read
+
+    # curl -i -X PUT "http://<HOST>:<PORT>/<PATH>?op=MKDIRS[&permission=<OCTAL>]"
+    def mkdir(path, options={})
+      check_options(options, OPT_TABLE['MKDIRS'])
+      res = operate_requests(:put, path, 'MKDIRS', options)
+      check_success_json(res, 'boolean')
+    end
+    OPT_TABLE['MKDIRS'] = ['permission']
+    alias :mkdirs :mkdir
+
+    # curl -i -X PUT "<HOST>:<PORT>/webhdfs/v1/<PATH>?op=RENAME&destination=<PATH>"
+    def rename(path, dest, options={})
+      check_options(options, OPT_TABLE['RENAME'])
+      unless dest.start_with?('/')
+        dest = '/' + dest
+      end
+      res = operate_requests(:put, path, 'RENAME', options.merge({'destination' => dest}))
+      check_success_json(res, 'boolean')
+    end
+
+    # curl -i -X DELETE "http://<host>:<port>/webhdfs/v1/<path>?op=DELETE
+    #                          [&recursive=<true|false>]"
+    def delete(path, options={})
+      check_options(options, OPT_TABLE['DELETE'])
+      res = operate_requests(:delete, path, 'DELETE', options)
+      check_success_json(res, 'boolean')
+    end
+    OPT_TABLE['DELETE'] = ['recursive']
+
+    # curl -i  "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=GETFILESTATUS"
+    def stat(path, options={})
+      check_options(options, OPT_TABLE['GETFILESTATUS'])
+      res = operate_requests(:get, path, 'GETFILESTATUS', options)
+      check_success_json(res, 'FileStatus')
+    end
+    alias :getfilestatus :stat
+
+    # curl -i  "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=LISTSTATUS"
+    def list(path, options={})
+      check_options(options, OPT_TABLE['LISTSTATUS'])
+      res = operate_requests(:get, path, 'LISTSTATUS', options)
+      check_success_json(res, 'FileStatuses')['FileStatus']
+    end
+    alias :liststatus :list
+
+    # curl -i "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=GETCONTENTSUMMARY"
+    def content_summary(path, options={})
+      check_options(options, OPT_TABLE['GETCONTENTSUMMARY'])
+      res = operate_requests(:get, path, 'GETCONTENTSUMMARY', options)
+      check_success_json(res, 'ContentSummary')
+    end
+    alias :getcontentsummary :content_summary
+
+    # curl -i "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=GETFILECHECKSUM"
+    def checksum(path, options={})
+      check_options(options, OPT_TABLE['GETFILECHECKSUM'])
+      res = operate_requests(:get, path, 'GETFILECHECKSUM', options)
+      check_success_json(res, 'FileChecksum')
+    end
+    alias :getfilechecksum :checksum
+
+    # curl -i "http://<HOST>:<PORT>/webhdfs/v1/?op=GETHOMEDIRECTORY"
+    def homedir(options={})
+      check_options(options, OPT_TABLE['GETHOMEDIRECTORY'])
+      res = operate_requests(:get, '/', 'GETHOMEDIRECTORY', options)
+      check_success_json(res, 'Path')
+    end
+    alias :gethomedirectory :homedir
+
+    # curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=SETPERMISSION
+    #                 [&permission=<OCTAL>]"
+    def chmod(path, mode, options={})
+      check_options(options, OPT_TABLE['SETPERMISSION'])
+      res = operate_requests(:put, path, 'SETPERMISSION', options.merge({'permission' => mode}))
+      res.code == 200
+    end
+    alias :setpermission :chmod
+
+    # curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=SETOWNER
+    #                          [&owner=<USER>][&group=<GROUP>]"
+    def chown(path, options={})
+      check_options(options, OPT_TABLE['SETOWNER'])
+      unless options.has_key?('owner') or options.has_key?('group') or
+          options.has_key?(:owner) or options.has_key?(:group)
+        raise ArgumentError, "'chown' needs at least one of owner or group"
+      end
+      res = operate_requests(:put, path, 'SETOWNER', options)
+      res.code == 200
+    end
+    OPT_TABLE['SETOWNER'] = ['owner', 'group']
+    alias :setowner :chown
+
+    # curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=SETREPLICATION
+    #                           [&replication=<SHORT>]"
+    def replication(path, replnum, options={})
+      check_options(options, OPT_TABLE['SETREPLICATION'])
+      res = operate_requests(:put, path, 'SETREPLICATION', options.merge({'replication' => replnum.to_s}))
+      check_success_json(res, 'boolean')
+    end
+    alias :setreplication :replication
+
+    # curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=SETTIMES
+    #                           [&modificationtime=<TIME>][&accesstime=<TIME>]"
+    # motidicationtime: radix-10 logn integer
+    # accesstime: radix-10 logn integer
+    def touch(path, options={})
+      check_options(options, OPT_TABLE['SETTIMES'])
+      unless options.has_key?('modificationtime') or options.has_key?('accesstime') or
+          options.has_key?(:modificationtime) or options.has_key?(:accesstime)
+        raise ArgumentError, "'chown' needs at least one of modificationtime or accesstime"
+      end
+      res = operate_requests(:put, path, 'SETTIMES', options)
+      res.code == 200
+    end
+    OPT_TABLE['SETTIMES'] = ['modificationtime', 'accesstime']
+    alias :settimes :touch
+
+    # def delegation_token(user, options={}) # GETDELEGATIONTOKEN
+    #   raise NotImplementedError
+    # end
+    # def renew_delegation_token(token, options={}) # RENEWDELEGATIONTOKEN
+    #   raise NotImplementedError
+    # end
+    # def cancel_delegation_token(token, options={}) # CANCELDELEGATIONTOKEN
+    #   raise NotImplementedError
+    # end
+
+    def check_options(options, optdecl=[])
+      ex = options.keys.map(&:to_s) - (optdecl || [])
+      raise ArgumentError, "no such option: #{ex.join(' ')}" unless ex.empty?
+    end
+
+    def check_success_json(res, attr=nil)
+      res.code == 200 and res.headers['Content-Type'].include?('application/json') and (attr.nil? or JSON.parse(res.body)[attr])
+    end
+
+    def api_path(path)
+      if path.start_with?('/')
+        '/webhdfs/v1' + path
+      else
+        '/webhdfs/v1/' + path
+      end
+    end
+
+    def build_path(path, op, params)
+      opts = if @username and @doas
+               {'op' => op, 'user.name' => @username, 'doas' => @doas}
+             elsif @username
+               {'op' => op, 'user.name' => @username}
+             elsif @doas
+               {'op' => op, 'doas' => @doas}
+             else
+               {'op' => op}
+             end
+      query = URI.encode_www_form(params.merge(opts))
+      api_path(path) + '?' + query
+    end
+
+    REDIRECTED_OPERATIONS = ['APPEND' ,'CREATE' , 'OPEN', 'GETFILECHECKSUM']
+    REDIRECTED_CODE = (300..399)
+
+    def operate_requests(method, path, op, params={}, payload=nil)
+      if not @httpfs_mode and REDIRECTED_OPERATIONS.include?(op)
+        res = request(@host, @port, method, path, op, params, nil)
+        unless REDIRECTED_CODE.include?(res.code) and res.headers['Location']
+          msg = "NameNode returns non-redirection (or without location header), code:#{res.code}, body:#{res.body}."
+          raise WebHDFS::RequestFailedError, msg
+        end
+        uri = URI.parse(res.headers['Location'])
+        rpath = if uri.query
+                  uri.path + '?' + uri.query
+                else
+                  uri.path
+                end
+        request(uri.host, uri.port, method, rpath, nil, {}, payload, {'Content-Type' => 'application/octet-stream'})
+      else
+        if @httpfs_mode and not payload.nil?
+          request(@host, @port, method, path, op, params, payload, {'Content-Type' => 'application/octet-stream'})
+        else
+          request(@host, @port, method, path, op, params, payload)
+        end
+      end
+    end
+
+    # IllegalArgumentException      400 Bad Request
+    # UnsupportedOperationException 400 Bad Request
+    # SecurityException             401 Unauthorized
+    # IOException                   403 Forbidden
+    # FileNotFoundException         404 Not Found
+    # RumtimeException              500 Internal Server Error
+    # @param [Object] host
+    # @param [Object] port
+    # @param [Object] method
+    # @param [Object] path
+    # @param [Object] op
+    # @param [Object] params
+    # @param [Object] payload
+    # @param [Object] header
+    def request(host, port, method, path, op=nil, params={}, payload=nil, header=nil)
+      req = HTTPI::Request.new
+      krb5 = nil
+
+      HTTPI.log     = true      # disable logging
+      HTTPI.log_level= :debug
+      HTTPI.adapter = :net_http #  one of [:httpclient, :curb, :net_http]
+
+      if @auth_type == :kerberos
+        if @username
+          krb5 = Krb5Auth::Krb5.new
+          inited = false;
+          begin
+            if @keytab
+              inited = krb5.get_init_creds_keytab(@username, @keytab)
+            elsif @pass
+              inited = krb5.get_init_creds_password(@username, @pass)
+            else
+              raise ArgumentError, "kerberos authentication requires keytab or password"
+            end
+          rescue
+            raise WebHDFS::SecurityError, "kerberos initialization is failed"
+          end
+          if inited
+            krb5.cache
+            HTTPI.adapter = :curb
+            req.auth.gssnegotiate
+          end
+        end
+      end
+      req.open_timeout = @open_timeout if @open_timeout
+      req.read_timeout = @read_timeout if @read_timeout
+      request_path = if op
+                       build_path(path, op, params)
+                     else
+                       path
+                     end
+      req.url = URI::HTTP.build({:host => host, :port => port}) + request_path
+      req.headers = header.nil? ? {} : header        #  MUST BE ASSIGN {} if nil BY zixian.shen
+      req.body =  payload.nil? ? {} : payload        #  MUST BE ASSIGN {} if nil BY zixian.shen
+
+      res = HTTPI.request( method, req )
+      if HTTPI::Response::SuccessfulResponseCodes.include?(res.code)
+        res
+      elsif REDIRECTED_CODE.include?(res.code)
+        res
+      else
+        message = if res.body and not res.body.empty?
+                    res.body.gsub(/\n/, '')
+                  else
+                    'Response body is empty...'
+                  end
+        case res.code
+        when 400
+          raise WebHDFS::ClientError, message
+        when 401
+          raise WebHDFS::SecurityError, message
+        when 403
+          raise WebHDFS::IOError, message
+        when 404
+          raise WebHDFS::FileNotFoundError, message
+        when 500
+          raise WebHDFS::ServerError, message
+        else
+          raise WebHDFS::RequestFailedError, "response code:#{res.code}, message:#{message}"
+        end
+      end
+    end
+  end
+end

data/lib/webhdfs/exceptions.rb

@@ -0,0 +1,12 @@
+module WebHDFS; end
+class WebHDFS::Error < StandardError; end
+
+class WebHDFS::FileNotFoundError < WebHDFS::Error; end
+
+class WebHDFS::IOError < WebHDFS::Error; end
+class WebHDFS::SecurityError < WebHDFS::Error; end
+
+class WebHDFS::ClientError < WebHDFS::Error; end
+class WebHDFS::ServerError < WebHDFS::Error; end
+
+class WebHDFS::RequestFailedError < WebHDFS::Error; end

data/lib/webhdfs/fileutils.rb

@@ -0,0 +1,350 @@
+require_relative 'client'
+
+module WebHDFS
+  module FileUtils
+    # Those values hold NameNode location
+    @fu_host = 'localhost'
+    @fu_port = 50070
+    @fu_user = nil
+    @fu_doas = nil
+    @fu_httpfs_mode = false
+    @fu_auth_type = :pseudo
+    @fu_keytab = nil
+    @fu_pass = nil
+
+    # Public: Set hostname and port number of WebHDFS
+    #
+    # host - hostname
+    # port - port
+    # user - username
+    # doas - proxy user name
+    #
+    # Examples
+    #
+    #   FileUtils.set_server 'localhost', 50070
+    #
+    def set_server(host, port, user=nil, doas=nil)
+      @fu_host = host
+      @fu_port = port
+      @fu_user = user
+      @fu_doas = doas
+    end
+    module_function :set_server
+
+    # Public: Set httpfs mode enable/disable
+    #
+    # mode - boolean (default true)
+    #
+    # Examples
+    #
+    #   FileUtils.set_httpfs_mode
+    #
+    def set_httpfs_mode(mode=true)
+      @fu_httpfs_mode = mode
+    end
+    module_function :set_httpfs_mode
+
+    # Public: Copy local file into HDFS
+    #
+    # file - local file path
+    # path - HDFS file path
+    # options - :overwrite, :blocksize, :replication, :mode, :buffersize, :verbose
+    #
+    # Examples
+    #
+    #   FileUtils.copy_from_local 'local_file', 'remote_file'
+    #
+    def
+    copy_from_local(file, path, options={})
+      opts = options.dup
+      fu_log "copy_from_local local=#{file} hdfs=#{path}" if opts.delete(:verbose)
+      if mode = opts.delete(:mode)
+        mode = ('%03o' % mode) if mode.is_a? Integer
+      else
+        mode = '644'
+      end
+      opts[:permission] = mode
+      opts[:overwrite] ||= true
+
+      client.create(path, File.new(file, 'rb').read(File.size(file)), opts)
+    end
+    module_function :copy_from_local
+
+    # Public: Copy remote HDFS file into local
+    #
+    # path - HDFS file path
+    # file - local file path
+    # options - :offset, :length, :buffersize, :verbose
+    #
+    # Examples
+    #
+    #   FileUtils.copy_to_local 'remote_file', 'local_file'
+    #
+    def copy_to_local(path, file, options={})
+      opts = options.dup
+      fu_log "copy_to_local hdfs=#{path} local=#{file}" if opts.delete(:verbose)
+      File.open(file, "wb") do |f|
+        f.write client.read(path, opts)
+      end
+    end
+    module_function :copy_to_local
+
+    # Public: Append to HDFS file
+    #
+    # path - HDFS file path
+    # body - contents
+    # options - :buffersize, :verbose
+    #
+    # Examples
+    #
+    #   FileUtils.append 'remote_path', 'contents'
+    #
+    def append(path, body, options={})
+      opts = options.dup
+      fu_log "append #{body.bytesize} bytes to #{path}" if opts.delete(:verbose)
+      client.append(path, body, opts)
+    end
+    module_function :append
+
+    # Public: Create one or more directories.
+    #
+    # list - directory name, or list of them
+    # options - :mode, :verbose
+    #
+    # Examples
+    #
+    #   FileUtils.mkdir 'test'
+    #   FileUtils.mkdir %w( tmp data )
+    #   FileUtils.mkdir 'tmp', :mode => 0700
+    #
+    def mkdir(list, options={})
+      opts = options.dup
+      list = [list].flatten
+      fu_log "mkdir #{options[:mode] ? ('-m %03o ' % options[:mode]) : ''}#{list.join ' '}" if opts.delete(:verbose)
+      if mode = opts[:mode]
+        mode = ('0%03o' % mode) if mode.is_a? Integer
+      else
+        mode = '0755'
+      end
+      c = client
+      list.each { |dir|
+        c.mkdir(dir, {:permission => mode})
+      }
+    end
+    module_function :mkdir
+
+    # Public: Create one or more directories recursively.
+    #
+    # list - directory name, or list of them
+    # options - :mode, :verbose
+    #
+    # Examples
+    #
+    #   FileUtils.mkdir_p 'dir/subdir'
+    #   FileUtils.mkdir_p %w( tmp data )
+    #   FileUtils.mkdir_p 'dir/subdir', :mode => 0700
+    #
+    alias mkdir_p mkdir
+    module_function :mkdir_p
+
+    # Public: Remove one or more directories or files.
+    #
+    # list - directory name, or list of them
+    # options - :recursive, :verbose
+    #
+    # Examples
+    #
+    #   FileUtils.rm 'dir'
+    #   FileUtils.rm %w( tmp data )
+    #   FileUtils.rm 'dir', :recursive => true
+    #
+    def rm(list, options={})
+      opts = options.dup
+      list = [list].flatten
+      fu_log "rm #{list.join ' '}" if opts.delete(:verbose)
+      c = client
+      list.each { |dir|
+        c.delete(dir, {:recursive => opts[:recursive] || false})
+      }
+    end
+    module_function :rm
+
+    # Public: Remove one or more directories/files recursively.
+    #
+    # list - directory name, or list of them
+    # options - :verbose
+    #
+    # Examples
+    #
+    #   FileUtils.rmr 'dir'
+    #   FileUtils.rmr %w( tmp data )
+    #   FileUtils.rmr 'dir'
+    #
+    def rmr(list, options={})
+      self.rm(list, options.merge({:recursive => true}))
+    end
+    module_function :rmr
+
+    # Public: Rename a file or directory.
+    #
+    # src - from
+    # dst - to
+    # options - :verbose
+    #
+    # Examples
+    #
+    #   FileUtils.rename 'from', 'to'
+    #
+    def rename(src, dst, options={})
+      opts = options.dup
+      fu_log "rename #{src} #{dst}" if opts.delete(:verbose)
+      client.rename(src, dst, opts)
+    end
+    module_function :rename
+
+    # Public: Change permission of one or more directories/files.
+    #
+    # mode - permission
+    # list - file/directory name or list of them.
+    # options - :verbose
+    #
+    # Examples
+    #
+    #   FileUtils.chmod 0755, 'dir'
+    #   FileUtils.chmod 0644, 'file'
+    #
+    def chmod(mode, list, options={})
+      opts = options.dup
+      list = [list].flatten
+      fu_log sprintf('chmod %o %s', mode, list.join(' ')) if opts.delete(:verbose)
+      mode = ('%03o' % mode) if mode.is_a? Integer
+      c = client
+      list.each { |entry|
+        c.chmod(entry, mode, opts)
+      }
+    end
+    module_function :chmod
+
+    # Public: Change an ownership of one or more directories/files.
+    #
+    # user - username
+    # group - groupname
+    # list - file/directory name or list of them
+    # options - :verbose
+    #
+    # Examples
+    #
+    #   FileUtils.chmod 0755, 'dir'
+    #   FileUtils.chmod 0644, 'file'
+    #
+    def chown(user, group, list, options={})
+      opts = options.dup
+      list = [list].flatten
+      fu_log sprintf('chown %s%s',
+                     [user,group].compact.join(':') + ' ',
+                     list.join(' ')) if opts.delete(:verbose)
+      c = client
+      list.each { |entry|
+        c.chown(entry, {:owner => user, :group => group})
+      }
+    end
+    module_function :chown
+
+    # Public: Set a replication factor of files
+    #
+    # list - file/directory name or list of them
+    # num - replication factor
+    # options - :verbose
+    #
+    # Examples
+    #
+    #   FileUtils.set_repl_factor 'file', 3
+    #
+    def set_repl_factor(list, num, options={})
+      opts = options.dup
+      list = [list].flatten
+      fu_log sprintf('set_repl_factor %s %d',
+                     list.join(' '), num) if opts.delete(:verbose)
+      c = client
+      list.each { |entry|
+        c.replication(entry, num, opts)
+      }
+    end
+    module_function :set_repl_factor
+
+    # Public: Set an access time of files
+    #
+    # list - file/directory name or list of them
+    # time - new access time
+    # options - :verbose
+    #
+    # Examples
+    #
+    #   FileUtils.set_atime 'file', Time.now
+    #
+    def set_atime(list, time, options={})
+      opts = options.dup
+      list = [list].flatten
+      time = time.to_i
+      fu_log sprintf('set_atime %s %d', list.join(' '), time) if opts.delete(:verbose)
+      c = client
+      list.each { |entry|
+        c.touch(entry, {:accesstime => time})
+      }
+    end
+    module_function :set_atime
+
+    # Public: Set a modification time of files
+    #
+    # list - file/directory name or list of them
+    # time - new modification time
+    # options - :verbose
+    #
+    # Examples
+    #
+    #   FileUtils.set_mtime 'file', Time.now
+    #
+    def set_mtime(list, time, options={})
+      opts = options.dup
+      list = [list].flatten
+      time = time.to_i
+      fu_log sprintf('set_mtime %s %d', list.join(' '), time) if opts.delete(:verbose)
+      c = client
+      list.each { |entry|
+        c.touch(entry, {:modificationtime => time})
+      }
+    end
+    module_function :set_mtime
+
+    # Internal: make functin private
+    def self.private_module_function(name)
+      module_function name
+      private_class_method name
+    end
+
+    @fileutils_output = $stderr
+    @fileutils_label  = '[webhdfs]: '
+    # Internal: Logging
+    def fu_log(msg)
+      @fileutils_output ||= $stderr
+      @fileutils_label  ||= ''
+      @fileutils_output.puts @fileutils_label + msg
+    end
+    private_module_function :fu_log
+
+    # Internal
+    def client
+      client = WebHDFS::Client.new(@fu_host, @fu_port, @fu_user, @fu_doas)
+      if @fu_httpfs_mode
+        client.httpfs_mode = true
+      end
+      if @fu_auth_type == :kerberos
+        client.auth_type = @fu_auth_type
+        client.keytab = @fu_keytab
+        client.pass = @fu_pass
+      end
+      client
+    end
+    private_module_function :client
+  end
+end

data/lib/webhdfs.rb

@@ -0,0 +1 @@
+require File.join(File.dirname(__FILE__), 'webhdfs', 'client.rb')

data/test/test_helper.rb

@@ -0,0 +1,19 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'rr'
+require 'test/unit'
+class Test::Unit::TestCase
+  include RR::Adapters::TestUnit
+end
+
+if ENV['SIMPLE_COV']
+  require 'simplecov'
+  SimpleCov.start do 
+    add_filter 'test/'
+    add_filter 'pkg/'
+    add_filter 'vendor/'
+  end
+end
+
+require 'test/unit'

data/test/test_httpi.rb

@@ -0,0 +1,21 @@
+require 'httpi'
+require 'krb5_auth'
+
+krb5 = Krb5Auth::Krb5.new
+#krb5.get_init_creds_password('zen', 'abc123')
+krb5.get_init_creds_keytab("hdfs/api.0.efoxconn.com", "/etc/hadoop/conf/hdfs.keytab")
+krb5.cache
+
+
+HTTPI.adapter = :curb #  one of [:httpclient, :curb, :net_http]
+
+request = HTTPI::Request.new
+request.url = "http://api.0.efoxconn.com:14000/webhdfs/v1?op=gethomedirectory"
+request.auth.gssnegotiate 
+response = HTTPI.request :get,request
+puts response.code
+puts response.headers
+puts response.body
+
+
+krb5.destroy

data/test/test_krb5-auth.rb

@@ -0,0 +1,44 @@
+ require 'rubygems'
+require 'krb5_auth'
+include Krb5Auth
+
+krb5 = Krb5.new
+
+# get the default realm
+default_realm = krb5.get_default_realm
+puts "Default realm is: " + default_realm
+
+# try to cache non-existant data (this should fail and throw an exception)
+begin
+  krb5.cache
+rescue Krb5Auth::Krb5::Exception
+  puts "Failed caching credentials before obtaining them.  Continuing..."
+end
+
+# Get initial credentials for the default principal and default keytab
+krb5.get_init_creds_password("zen", "abc123")
+
+# cache those credentials in the default cache location
+krb5.cache
+
+puts "Principal: " + krb5.get_default_principal
+
+# List all of the credentials in the cache, and expiration times, etc.
+krb5.list_cache.each do |cred|
+  starttime = DateTime.strptime(cred.starttime.to_s, "%s")
+  endtime = DateTime.strptime(cred.endtime.to_s, "%s")
+  puts "Client: " + cred.client + " Server: " + cred.server + " starttime: " + starttime.strftime("%D %T") + " endtime: " + endtime.strftime("%D %T")
+end
+
+# destroy those same credentials from the default cache location
+krb5.destroy
+
+# close the object (releases all memory)
+krb5.close
+
+# now try to use the object again; this should fail and throw an exception
+begin
+  krb5.cache
+rescue Krb5Auth::Krb5::Exception
+  puts "Tried to reuse closed object; continuing..."
+end

data/test/test_webhdfs.rb

@@ -0,0 +1,54 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require "webhdfs"
+
+#puts (300..399).include?(307)
+
+#require "webhdfs/fileutils"
+
+client = WebHDFS::Client.new
+client.host="api.0.efoxconn.com"
+client.port=14000
+client.auth_type=:kerberos
+client.username="zen"
+client.pass_keytab="abc123"
+client.username="hdfs/api.0.efoxconn.com"
+client.pass_keytab="/etc/hadoop/conf/hdfs.keytab"
+
+client.delete('/user/fU3931aa69444d4558aba6b26981994e6e',  {"recursive"=>"true"})
+#client.mkdir('/user/zen')
+#client.content_summary("/")
+myhome = client.homedir
+p myhome
+info={"user"=>"u740a1d36-2e6d-4aa9-b2e7-028750a8c918", "pass"=>"pfa6f202f-b8da-421e-956b-a5383899192b"}
+client.username=info["user"]
+client.pass_keytab=info["pass"]
+myhome = client.homedir
+p myhome
+client.setowner(myhome, {:owner=>info["user"],:group=>info["user"]} )
+#client.create_symlink("/user/gangang/lznew", "/user/zen/newfolder")
+# puts client.mkdir(myhome+"/newfolder/nf1/nf2")
+#puts client.delete(myhome+"/newfolder")
+#client.getfilechecksum("/user/update-java.deb")
+
+#File.open("/tmp/update-java.deb", "wb") do |f|
+#  f.write client.read("/user/update-java.deb")
+#end
+
+#for x in 20..30
+#
+#client.delete('/user/U-abcdef-'+ x.to_s)
+## client.mkdir('/user/U-abcdef-'+ x.to_s)
+#end
+
+#client.setowner('/user/zen', {"owner"=> "zen", "group"=> "zen"} )
+file = "test_helper.rb"
+#for x in 0..10
+#  client.append("/user/VERSION"+x.to_s,
+#                File.new(file, 'rb').read(File.size(file)).to_s )
+#end
+
+#### PUT Curl::Err::SendFailedRewind
+#for x in 20..30
+#  client.create("/user/VERSION"+x.to_s,
+#                File.new(file, 'rb').read(File.size(file)) )
+#end

data/test/webhdfs/fileutils.rb

@@ -0,0 +1,65 @@
+require 'test_helper'
+
+class FileUtilsTest < Test::Unit::TestCase
+  def setup
+    require 'webhdfs'
+    WebHDFS::FileUtils.fu_host = "api.0.efoxconn.com"
+    WebHDFS::FileUtils.fu_port = "14000"
+    WebHDFS::FileUtils.fu_user = "zen"
+    WebHDFS::FileUtils.fu_pass_keytab = "abc123"
+  end
+
+  def test_copy_from_local
+    WebHDFS::FileUtils.copy_from_local('VERSION', 'VERSION', :verbose => true)
+    WebHDFS::FileUtils.copy_to_local('VERSION', 'VERSION2', :verbose => true)
+    WebHDFS::FileUtils.append('VERSION', 'foo-bar-buzz', :verbose => true)
+    WebHDFS::FileUtils.rm('VERSION', :verbose => true)
+  end
+
+  def test_rm
+    WebHDFS::FileUtils.mkdir('foo', :mode => 0777, :verbose => true)
+    WebHDFS::FileUtils.rm('foo', :verbose => true)
+  end
+
+  def test_rmr
+    WebHDFS::FileUtils.mkdir_p('foo/bar/buzz', :mode => 0777, :verbose => true)
+    WebHDFS::FileUtils.rmr('foo', :verbose => true)
+  end
+
+  def test_rename
+    #WebHDFS::FileUtils.mkdir_p('foo', :mode => 0777, :verbose => true)
+    #WebHDFS::FileUtils.rename('foo', 'foo2', :verbose => true)
+    #WebHDFS::FileUtils.rmr('foo2', :verbose => true)
+  end
+
+  def test_chmod
+    WebHDFS::FileUtils.mkdir('foo', :mode => 0777, :verbose => true)
+    WebHDFS::FileUtils.chmod(0755, 'foo', :verbose => true)
+    WebHDFS::FileUtils.chmod(0777, 'foo', :verbose => true)
+    WebHDFS::FileUtils.rm('foo', :verbose => true)
+  end
+
+  def test_chown
+    #WebHDFS::FileUtils.mkdir('foo', :mode => 0777, :verbose => true)
+    #WebHDFS::FileUtils.chown('webuser', 'supergroup', 'foo', :verbose => true)
+    #WebHDFS::FileUtils.rm('foo', :verbose => true)
+  end
+
+  def test_set_repl_factor
+    WebHDFS::FileUtils.mkdir('foo', :mode => 0777, :verbose => true)
+    WebHDFS::FileUtils.set_repl_factor('foo', 5)
+    WebHDFS::FileUtils.rm('foo', :verbose => true)
+  end
+
+  def test_set_atime
+    #WebHDFS::FileUtils.mkdir('foo', :mode => 0777, :verbose => true)
+    #WebHDFS::FileUtils.set_atime('foo', Time.now)
+    #WebHDFS::FileUtils.rm('foo', :verbose => true)
+  end
+
+  def test_set_mtime
+    #WebHDFS::FileUtils.mkdir('foo', :mode => 0777, :verbose => true)
+    #WebHDFS::FileUtils.set_mtime('foo', Time.now)
+    #WebHDFS::FileUtils.rm('foo', :verbose => true)
+  end
+end

data/webhdfs-with-krb5.gemspec

@@ -0,0 +1,26 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = "webhdfs-with-krb5"
+  gem.description = "Ruby WebHDFS/HttpFs client with krb5 authentication"
+  gem.homepage    = ""
+  gem.summary     = gem.description
+  gem.version     = File.read("VERSION").strip
+  gem.authors     = ["Tomoaki Takamatsu"]
+  gem.email       = "takamatsutomoaki@gmail.com"
+  gem.has_rdoc    = false
+  gem.platform    = Gem::Platform::RUBY
+  gem.files       = `git ls-files`.split("\n")
+  #gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  #gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_development_dependency "rake", ">= 0.9.2"
+  gem.add_development_dependency "rdoc", ">= 3.12"
+  gem.add_development_dependency "simplecov", ">= 0.5.4"
+  gem.add_development_dependency "rr", ">= 1.0.0"
+  gem.add_development_dependency "timfel-krb5-auth"
+  gem.add_development_dependency "httpi"
+  gem.add_development_dependency "curb"
+end

metadata

@@ -0,0 +1,177 @@
+--- !ruby/object:Gem::Specification 
+name: webhdfs-with-krb5
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 6
+  - 1
+  version: 0.6.1
+platform: ruby
+authors: 
+- Tomoaki Takamatsu
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2013-05-20 00:00:00 +09:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rake
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        - 9
+        - 2
+        version: 0.9.2
+  type: :development
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: rdoc
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 3
+        - 12
+        version: "3.12"
+  type: :development
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: simplecov
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        - 5
+        - 4
+        version: 0.5.4
+  type: :development
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: rr
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 1
+        - 0
+        - 0
+        version: 1.0.0
+  type: :development
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: timfel-krb5-auth
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: httpi
+  prerelease: false
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id006
+- !ruby/object:Gem::Dependency 
+  name: curb
+  prerelease: false
+  requirement: &id007 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id007
+description: Ruby WebHDFS/HttpFs client with krb5 authentication
+email: takamatsutomoaki@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- AUTHORS
+- COPYING
+- Gemfile
+- README.md
+- Rakefile
+- VERSION
+- lib/webhdfs.rb
+- lib/webhdfs/client.rb
+- lib/webhdfs/client_v1.rb
+- lib/webhdfs/exceptions.rb
+- lib/webhdfs/fileutils.rb
+- test/test_helper.rb
+- test/test_httpi.rb
+- test/test_krb5-auth.rb
+- test/test_webhdfs.rb
+- test/webhdfs/fileutils.rb
+- webhdfs-with-krb5.gemspec
+has_rdoc: true
+homepage: ""
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Ruby WebHDFS/HttpFs client with krb5 authentication
+test_files: []
+