webhdfs-rlz 0.8.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 473d3b0eaffcc87f27f27d0f8dccf2bfe5fe2189
+  data.tar.gz: e0ade8a94b22d7eee85fa72077bac08b23858e8d
+SHA512:
+  metadata.gz: b88255e72919b0141ad697487ad4cc3f9960c73f0cbdefd2e6738e1639d30e660580b9f6e7c232aaf2c352ada461b538131c2ea0da2229b21d0c3959b198e914
+  data.tar.gz: 354d2a40fecbc9e1fa63746f0d05b5b2627f7eeec9d8c4287682371713c260cfb04cf782648d6929f7936e665ad2987bbf05eb082fdf7ba83dd6ef6a66744c6a

data/AUTHORS

@@ -0,0 +1,2 @@
+Kazuki Ohta <kazuki.ohta@gmail.com>
+TAGOMORI Satoshi <tagomoris@gmail.com>

data/COPYING

@@ -0,0 +1,13 @@
+Copyright (C) 2012 Fluentd Project
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/README.md

@@ -0,0 +1,127 @@
+# webhdfs - A client library implementation for Hadoop WebHDFS, and HttpFs, for Ruby
+
+The webhdfs gem is to access Hadoop WebHDFS (EXPERIMENTAL: and HttpFs). WebHDFS::Client is a client class, and WebHDFS::FileUtils is utility like 'fileutils'.
+
+## Installation
+
+    gem install webhdfs
+
+## Usage
+
+### WebHDFS::Client
+
+For client object interface:
+
+    require 'webhdfs'
+    client = WebHDFS::Client.new(hostname, port)
+    # or with pseudo username authentication
+    client = WebHDFS::Client.new(hostname, port, username)
+
+To create/append/read files:
+
+    client.create('/path/to/file', data)
+    client.create('/path/to/file', data, :overwrite => false, :blocksize => 268435456, :replication => 5, :permission => '0666')
+
+    #This does not require whole data in memory, and it can be read chunk by chunk, ex: File data
+    client.create('/path/to/file', file_IO_handle, :overwrite => false, :permission => 0666)
+
+    client.append('/path/to/existing/file', data)
+    
+    client.read('/path/to/target') #=> data
+    client.read('/path/to/target' :offset => 2048, :length => 1024) #=> data
+
+To mkdir/rename/delete directories or files:
+
+    client.mkdir('/hdfs/dirname')
+    client.mkdir('/hdfs/dirname', :permission => '0777')
+    
+    client.rename(original_path, dst_path)
+    
+    client.delete(path)
+    client.delete(dir_path, :recursive => true)
+
+To get status or list of files and directories:
+
+    client.stat(file_path) #=> key-value pairs for file status
+    client.list(dir_path)  #=> list of key-value pairs for files in dir_path
+
+And, 'content_summary', 'checksum', 'homedir', 'chmod', 'chown', 'replication' and 'touch' methods available.
+
+For known errors, automated retries are available. Set `retry_known_errors` option as true.
+
+    #### To retry for LeaseExpiredException automatically
+    client.retry_known_errors = true
+    
+    # client.retry_interval = 1 # [sec], default: 1
+    # client.retry_times = 1 # [times], default: 1
+
+### WebHDFS::FileUtils
+
+    require 'webhdfs/fileutils'
+    WebHDFS::FileUtils.set_server(host, port)
+    # or
+    WebHDFS::FileUtils.set_server(host, port, username, doas)
+    
+    WebHDFS::FileUtils.copy_from_local(localpath, hdfspath)
+    WebHDFS::FileUtils.copy_to_local(hdfspath, localpath)
+    
+    WebHDFS::FileUtils.append(path, data)
+
+### For HttpFs
+
+For HttpFs instead of WebHDFS:
+
+    client = WebHDFS::Client.new('hostname', 14000)
+    client.httpfs_mode = true
+    
+    client.read(path) #=> data
+    
+    # or with webhdfs/filetuils
+    WebHDFS::FileUtils.set_server('hostname', 14000)
+    WebHDFS::FileUtils.set_httpfs_mode
+    WebHDFS::FileUtils.copy_to_local(remote_path, local_path)
+
+### For HTTP Proxy servers
+
+    client = WebHDFS::Client.new('hostname', 14000, 'proxy.server.local', 8080)
+    client.proxy_user = 'jack'   # if needed
+    client.proxy_pass = 'secret' # if needed
+
+### For SSL
+
+Note that net/https and openssl libraries must be available:
+
+    client = WebHDFS::Client.new('hostname', 4443)
+    client.ssl = true
+    client.ssl_ca_file = "/path/to/ca_file.pem" # if needed
+    client.ssl_varify_mode = :peer # if needed (:none or :peer)
+    client.ssl_version = :TLSv1 # if needed
+
+### For Kerberos Authentication
+
+Note that [gssapi](https://github.com/zenchild/gssapi) library must be available:
+
+    client = WebHDFS::Client.new('hostname', 14000)
+    client.kerberos = true
+    client.kerberos_keytab = "/path/to/project.keytab"
+
+### For SSL Client Authentication
+
+Note that openssl libraries must be available:
+
+    require 'openssl'
+    
+    client = WebHDFS::Client.new(host, port)
+    client.ssl = true
+    client.ssl_key = OpenSSL::PKey::RSA.new(open('/path/to/key.pem'))
+    client.ssl_cert = OpenSSL::X509::Certificate.new(open('/path/to/cert.pem'))
+
+## AUTHORS
+
+* Kazuki Ohta <kazuki.ohta@gmail.com>
+* TAGOMORI Satoshi <tagomoris@gmail.com>
+
+## LICENSE
+
+* Copyright: Copyright (c) 2012- Fluentd Project
+* License: Apache License, Version 2.0

data/VERSION

@@ -0,0 +1 @@
+0.8.0

data/lib/webhdfs.rb

@@ -0,0 +1,2 @@
+require File.join(File.dirname(__FILE__), 'webhdfs', 'backport.rb')
+require File.join(File.dirname(__FILE__), 'webhdfs', 'client.rb')

data/lib/webhdfs/backport.rb

@@ -0,0 +1,27 @@
+if RUBY_VERSION =~ /^1\.8\./
+  require 'cgi'
+
+  def require_relative(relative_feature)
+    file = caller.first.split(/:\d/, 2).first
+    raise LoadError, "require_relative is called in #{Regexp.last_match(1)}"\
+                     if /\A\((.*)\)/ =~ file
+    require File.expand_path(relative_feature, File.dirname(file))
+  end
+
+  module URI
+    def self.encode_www_form(enum)
+      enum.map do |key, value|
+        if value.nil?
+          CGI.escape(key)
+        elsif value.respond_to?(:to_ary)
+          value.to_ary.map do |w|
+            str = CGI.escape(key)
+            str << '=' << CGI.escape(w) unless w.nil?
+          end.join('&')
+        else
+          CGI.escape(key.to_s) << '=' << CGI.escape(value.to_s)
+        end
+      end.join('&')
+    end
+  end
+end

data/lib/webhdfs/client.rb

@@ -0,0 +1,6 @@
+require_relative 'client_v2'
+
+module WebHDFS
+  class Client < ClientV2
+  end
+end

data/lib/webhdfs/client_v1.rb

@@ -0,0 +1,411 @@
+require 'net/http'
+require 'uri'
+require 'json'
+require 'addressable/uri'
+
+require_relative 'utilities'
+require_relative 'exceptions'
+
+module WebHDFS
+  class ClientV1
+    # This hash table holds command options.
+    OPT_TABLE = {}.freeze # internal use only
+    KNOWN_ERRORS = ['LeaseExpiredException'].freeze
+
+    attr_accessor :host, :port, :username, :doas, :proxy_address, :proxy_port
+    attr_accessor :proxy_user, :proxy_pass
+    attr_accessor :open_timeout # default 30s (in ruby net/http)
+    attr_accessor :read_timeout # default 60s (in ruby net/http)
+    attr_accessor :httpfs_mode
+    attr_accessor :retry_known_errors # default false (not to retry)
+    attr_accessor :retry_times        # default 1 (ignored when retry_known_errors is false)
+    attr_accessor :retry_interval     # default 1 ([sec], ignored when retry_known_errors is false)
+    attr_accessor :ssl
+    attr_accessor :ssl_ca_file
+    attr_reader   :ssl_verify_mode
+    attr_accessor :ssl_cert
+    attr_accessor :ssl_key
+    attr_accessor :ssl_version
+    attr_accessor :kerberos, :kerberos_keytab
+    attr_accessor :http_headers
+
+    SSL_VERIFY_MODES = [:none, :peer].freeze
+    def ssl_verify_mode=(mode)
+      unless SSL_VERIFY_MODES.include? mode
+        raise ArgumentError, "Invalid SSL verify mode #{mode.inspect}"
+      end
+      @ssl_verify_mode = mode
+    end
+
+    def initialize(host = 'localhost', port = 50_070, username = nil,
+                   doas = nil, proxy_address = nil, proxy_port = nil,
+                   http_headers = {})
+      @host = host
+      @port = port
+      @username = username
+      @doas = doas
+      @proxy_address = proxy_address
+      @proxy_port = proxy_port
+      @retry_known_errors = false
+      @retry_times = @retry_interval = 1
+
+      @httpfs_mode = false
+
+      @ssl = false
+      @ssl_ca_file = nil
+      @ssl_verify_mode = nil
+      @ssl_cert = @ssl_key = nil
+      @ssl_version = nil
+
+      @kerberos = false
+      @kerberos_keytab = nil
+      @http_headers = http_headers
+    end
+
+    # curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=CREATE
+    #                 [&overwrite=<true|false>][&blocksize=<LONG>]
+    #                 [&replication=<SHORT>]
+    #                 [&permission=<OCTAL>][&buffersize=<INT>]"
+    def create(path, body, options = {})
+      options = options.merge('data' => 'true') if @httpfs_mode
+      check_options(options, OPT_TABLE['CREATE'])
+      res = operate_requests('PUT', path, 'CREATE', options, body)
+      res.code == '201'
+    end
+    OPT_TABLE['CREATE'] = %w(overwrite blocksize replication permission
+                             buffersize data)
+
+    # curl -i -X POST "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=
+    #                      APPEND[&buffersize=<INT>]"
+    def append(path, body, options = {})
+      options = options.merge('data' => 'true') if @httpfs_mode
+      check_options(options, OPT_TABLE['APPEND'])
+      res = operate_requests('POST', path, 'APPEND', options, body)
+      res.code == '200'
+    end
+    OPT_TABLE['APPEND'] = %w(buffersize data)
+
+    # curl -i -L "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=OPEN
+    #                [&offset=<LONG>][&length=<LONG>][&buffersize=<INT>]"
+    def read(path, options = {})
+      check_options(options, OPT_TABLE['OPEN'])
+      res = operate_requests('GET', path, 'OPEN', options)
+      res.body
+    end
+    OPT_TABLE['OPEN'] = %w(offset length buffersize)
+    alias open read
+
+    # curl -i -X PUT "http://<HOST>:<PORT>/<PATH>?op=
+    #                     MKDIRS[&permission=<OCTAL>]"
+    def mkdir(path, options = {})
+      check_options(options, OPT_TABLE['MKDIRS'])
+      res = operate_requests('PUT', path, 'MKDIRS', options)
+      check_success_json(res, 'boolean')
+    end
+    OPT_TABLE['MKDIRS'] = ['permission']
+    alias mkdirs mkdir
+
+    # curl -i -X PUT "<HOST>:<PORT>/webhdfs/v1/<PATH>?op=
+    #                       RENAME&destination=<PATH>"
+    def rename(path, dest, options = {})
+      check_options(options, OPT_TABLE['RENAME'])
+      dest = '/' + dest unless dest.start_with?('/')
+      res = operate_requests('PUT', path, 'RENAME',
+                             options.merge('destination' => dest))
+      check_success_json(res, 'boolean')
+    end
+
+    # curl -i -X DELETE "http://<host>:<port>/webhdfs/v1/<path>?op=DELETE
+    #                          [&recursive=<true|false>]"
+    def delete(path, options = {})
+      check_options(options, OPT_TABLE['DELETE'])
+      res = operate_requests('DELETE', path, 'DELETE', options)
+      check_success_json(res, 'boolean')
+    end
+    OPT_TABLE['DELETE'] = ['recursive']
+
+    # curl -i  "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=GETFILESTATUS"
+    def stat(path, options = {})
+      check_options(options, OPT_TABLE['GETFILESTATUS'])
+      res = operate_requests('GET', path, 'GETFILESTATUS', options)
+      check_success_json(res, 'FileStatus')
+    end
+    alias getfilestatus stat
+
+    # curl -i  "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=LISTSTATUS"
+    def list(path, options = {})
+      check_options(options, OPT_TABLE['LISTSTATUS'])
+      res = operate_requests('GET', path, 'LISTSTATUS', options)
+      check_success_json(res, 'FileStatuses')['FileStatus']
+    end
+    alias liststatus list
+
+    # curl -i "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=GETCONTENTSUMMARY"
+    def content_summary(path, options = {})
+      check_options(options, OPT_TABLE['GETCONTENTSUMMARY'])
+      res = operate_requests('GET', path, 'GETCONTENTSUMMARY', options)
+      check_success_json(res, 'ContentSummary')
+    end
+    alias getcontentsummary content_summary
+
+    # curl -i "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=GETFILECHECKSUM"
+    def checksum(path, options = {})
+      check_options(options, OPT_TABLE['GETFILECHECKSUM'])
+      res = operate_requests('GET', path, 'GETFILECHECKSUM', options)
+      check_success_json(res, 'FileChecksum')
+    end
+    alias getfilechecksum checksum
+
+    # curl -i "http://<HOST>:<PORT>/webhdfs/v1/?op=GETHOMEDIRECTORY"
+    def homedir(options = {})
+      check_options(options, OPT_TABLE['GETHOMEDIRECTORY'])
+      res = operate_requests('GET', '/', 'GETHOMEDIRECTORY', options)
+      check_success_json(res, 'Path')
+    end
+    alias gethomedirectory homedir
+
+    # curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=SETPERMISSION
+    #                 [&permission=<OCTAL>]"
+    def chmod(path, mode, options = {})
+      check_options(options, OPT_TABLE['SETPERMISSION'])
+      res = operate_requests('PUT', path, 'SETPERMISSION',
+                             options.merge('permission' => mode))
+      res.code == '200'
+    end
+    alias setpermission chmod
+
+    # curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=SETOWNER
+    #                          [&owner=<USER>][&group=<GROUP>]"
+    def chown(path, options = {})
+      check_options(options, OPT_TABLE['SETOWNER'])
+      unless options.key?('owner') || options.key?('group') ||
+             options.key?(:owner) || options.key?(:group)
+        raise ArgumentError, "'chown' needs at least one of owner or group"
+      end
+      res = operate_requests('PUT', path, 'SETOWNER', options)
+      res.code == '200'
+    end
+    OPT_TABLE['SETOWNER'] = %w(owner group)
+    alias setowner chown
+
+    # curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=SETREPLICATION
+    #                           [&replication=<SHORT>]"
+    def replication(path, replnum, options = {})
+      check_options(options, OPT_TABLE['SETREPLICATION'])
+      res = operate_requests('PUT', path, 'SETREPLICATION',
+                             options.merge('replication' => replnum.to_s))
+      check_success_json(res, 'boolean')
+    end
+    alias setreplication replication
+
+    # curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=SETTIMES
+    #                           [&modificationtime=<TIME>][&accesstime=<TIME>]"
+    # motidicationtime: radix-10 logn integer
+    # accesstime: radix-10 logn integer
+    def touch(path, options = {})
+      check_options(options, OPT_TABLE['SETTIMES'])
+      unless options.key?('modificationtime') || options.key?('accesstime') ||
+             options.key?(:modificationtime) || options.key?(:accesstime)
+        raise ArgumentError, "'chown' needs at least one of " \
+                               'modificationtime or accesstime'
+      end
+      res = operate_requests('PUT', path, 'SETTIMES', options)
+      res.code == '200'
+    end
+    OPT_TABLE['SETTIMES'] = %w(modificationtime accesstime)
+    alias settimes touch
+
+    # def delegation_token(user, options={}) # GETDELEGATIONTOKEN
+    #   raise NotImplementedError
+    # end
+    # def renew_delegation_token(token, options={}) # RENEWDELEGATIONTOKEN
+    #   raise NotImplementedError
+    # end
+    # def cancel_delegation_token(token, options={}) # CANCELDELEGATIONTOKEN
+    #   raise NotImplementedError
+    # end
+
+    def build_path(path, op, params)
+      opts = if @username && @doas
+               { 'op' => op, 'user.name' => @username, 'doas' => @doas }
+             elsif @username
+               { 'op' => op, 'user.name' => @username }
+             elsif @doas
+               { 'op' => op, 'doas' => @doas }
+             else
+               { 'op' => op }
+             end
+      api_path(path) + '?' + URI.encode_www_form(params.merge(opts))
+    end
+
+    REDIRECTED_OPERATIONS = %w(APPEND CREATE OPEN GETFILECHECKSUM).freeze
+    def operate_requests(method, path, op, params = {}, payload = nil)
+      if !@httpfs_mode && REDIRECTED_OPERATIONS.include?(op)
+        res = request(@host, @port, method, path, op, params, nil)
+        unless res.is_a?(Net::HTTPRedirection) && res['location']
+          msg = 'NameNode returns non-redirection (or without location' \
+                " header), code:#{res.code}, body:#{res.body}."
+          raise WebHDFS::RequestFailedError, msg
+        end
+        uri = URI.parse(res['location'])
+        rpath = if uri.query
+                  uri.path + '?' + uri.query
+                else
+                  uri.path
+                end
+        request(uri.host, uri.port, method, rpath, nil, {},
+                payload, 'Content-Type' => 'application/octet-stream')
+      elsif @httpfs_mode && !payload.nil?
+        request(@host, @port, method, path, op, params,
+                payload, 'Content-Type' => 'application/octet-stream')
+      else
+        request(@host, @port, method, path, op, params, payload)
+      end
+    end
+
+    # IllegalArgumentException      400 Bad Request
+    # UnsupportedOperationException 400 Bad Request
+    # SecurityException             401 Unauthorized
+    # IOException                   403 Forbidden
+    # FileNotFoundException         404 Not Found
+    # RumtimeException              500 Internal Server Error
+    def request(host, port, method, path, op = nil, params = {},
+                payload = nil, header = nil, retries = 0)
+      conn = Net::HTTP.new(host, port, @proxy_address, @proxy_port)
+      conn.proxy_user = @proxy_user if @proxy_user
+      conn.proxy_pass = @proxy_pass if @proxy_pass
+      conn.open_timeout = @open_timeout if @open_timeout
+      conn.read_timeout = @read_timeout if @read_timeout
+
+      path = Addressable::URI.escape(path) # make safe for transmission via HTTP
+      request_path = if op
+                       build_path(path, op, params)
+                     else
+                       path
+                     end
+      if @ssl
+        conn.use_ssl = true
+        conn.ca_file = @ssl_ca_file if @ssl_ca_file
+        if @ssl_verify_mode
+          require 'openssl'
+          conn.verify_mode = case @ssl_verify_mode
+                             when :none then OpenSSL::SSL::VERIFY_NONE
+                             when :peer then OpenSSL::SSL::VERIFY_PEER
+                             end
+        end
+        conn.cert = @ssl_cert if @ssl_cert
+        conn.key = @ssl_key if @ssl_key
+        conn.ssl_version = @ssl_version if @ssl_version
+      end
+
+      gsscli = nil
+      if @kerberos
+        require 'base64'
+        require 'gssapi'
+        gsscli = GSSAPI::Simple.new(@host, 'HTTP', @kerberos_keytab)
+        token = nil
+        begin
+          token = gsscli.init_context
+        rescue => e
+          raise WebHDFS::KerberosError, e.message
+        end
+        if header
+          header['Authorization'] = "Negotiate #{Base64.strict_encode64(token)}"
+        else
+          header = { 'Authorization' =>
+                     "Negotiate #{Base64.strict_encode64(token)}" }
+        end
+      else
+        header = {} if header.nil?
+        header = @http_headers.merge(header)
+      end
+
+      res = nil
+      if !payload.nil? && payload.respond_to?(:read) &&
+         payload.respond_to?(:size)
+        req = Net::HTTPGenericRequest.new(method, (payload ? true : false),
+                                          true, request_path, header)
+        raise WebHDFS::ClientError, 'Error accepting given IO resource as' \
+          ' data payload, Not valid in methods' \
+          ' other than PUT and POST' unless method == 'PUT' || method == 'POST'
+
+        req.body_stream = payload
+        req.content_length = payload.size
+        begin
+          res = conn.request(req)
+        rescue => e
+          raise WebHDFS::ServerError, 'Failed to connect to host' \
+                                      " #{host}:#{port}, #{e.message}"
+        end
+      else
+        begin
+          res = conn.send_request(method, request_path, payload, header)
+        rescue => e
+          raise WebHDFS::ServerError, 'Failed to connect to host' \
+                                      " #{host}:#{port}, #{e.message}"
+        end
+      end
+
+      if @kerberos && res.code == '307'
+        itok = (res.header.get_fields('WWW-Authenticate') ||
+                ['']).pop.split(/\s+/).last
+        unless itok
+          raise WebHDFS::KerberosError, 'Server does not return ' \
+                                        'WWW-Authenticate header'
+        end
+
+        begin
+          gsscli.init_context(Base64.strict_decode64(itok))
+        rescue => e
+          raise WebHDFS::KerberosError, e.message
+        end
+      end
+
+      case res
+      when Net::HTTPSuccess
+        res
+      when Net::HTTPRedirection
+        res
+      else
+        message = if res.body && !res.body.empty?
+                    res.body.delete("\n")
+                  else
+                    'Response body is empty...'
+                  end
+
+        if @retry_known_errors && retries < @retry_times
+          detail = nil
+          if message =~ /^\{"RemoteException":\{/
+            begin
+              detail = JSON.parse(message)
+            rescue
+              # ignore broken json response body
+            end
+          end
+          if detail && detail['RemoteException'] &&
+             KNOWN_ERRORS.include?(detail['RemoteException']['exception'])
+            sleep @retry_interval if @retry_interval > 0
+            return request(host, port, method, path, op, params, payload,
+                           header, retries + 1)
+          end
+        end
+
+        case res.code
+        when '400'
+          raise WebHDFS::ClientError, message
+        when '401'
+          raise WebHDFS::SecurityError, message
+        when '403'
+          raise WebHDFS::IOError, message
+        when '404'
+          raise WebHDFS::FileNotFoundError, message
+        when '500'
+          raise WebHDFS::ServerError, message
+        else
+          raise WebHDFS::RequestFailedError, "response code:#{res.code}, " \
+                                             "message:#{message}"
+        end
+      end
+    end
+  end
+end