webhdfs-rlz 0.8.0

data/lib/webhdfs/client_v2.rb

@@ -0,0 +1,249 @@
+require 'net/http'
+require 'uri'
+require 'json'
+require 'addressable/uri'
+
+require_relative 'utilities'
+require_relative 'exceptions'
+require_relative 'proxy'
+require_relative 'ssl'
+require_relative 'kerberos'
+
+module WebHDFS
+  class ClientV2
+    attr_accessor :host, :port, :username, :doas, :proxy
+    attr_accessor :open_timeout # default 30s (in ruby net/http)
+    attr_accessor :read_timeout # default 60s (in ruby net/http)
+    attr_accessor :httpfs_mode
+    attr_accessor :retry_known_errors # default false (not to retry)
+    attr_accessor :retry_times        # default 1 (ignored when retry_known_errors is false)
+    attr_accessor :retry_interval     # default 1 ([sec], ignored when retry_known_errors is false)
+    attr_accessor :ssl
+    attr_accessor :kerberos
+    attr_accessor :http_headers
+
+    # This hash table holds command options.
+    OPT_TABLE = {}.freeze # internal use only
+    OPT_TABLE['CREATE'] = %w(overwrite blocksize replication permission
+                             buffersize data)
+    OPT_TABLE['APPEND'] = %w(buffersize data)
+    OPT_TABLE['OPEN'] = %w(offset length buffersize)
+    OPT_TABLE['MKDIRS'] = ['permission']
+    OPT_TABLE['DELETE'] = ['recursive']
+    OPT_TABLE['SETOWNER'] = %w(owner group)
+    OPT_TABLE['SETTIMES'] = %w(modificationtime accesstime)
+
+    REDIRECTED_OPERATIONS = %w(APPEND CREATE OPEN GETFILECHECKSUM).freeze
+
+    def initialize(host = 'localhost', port = 50_070, username = nil,
+                   doas = nil, proxy_address = nil, proxy_port = nil,
+                   http_headers = {})
+      @host = host
+      @port = port
+      @username = username
+      @doas = doas
+
+      @proxy = WebHDFS::Proxy.new(proxy_address, proxy_port) if proxy_address && proxy_port
+
+      @retry_known_errors = false
+      @retry_times = @retry_interval = 1
+
+      @httpfs_mode = false
+
+      @ssl = nil
+
+      @kerberos = nil
+      @http_headers = http_headers
+    end
+
+    def request_options
+      {
+        username: @username,
+        doas: @doas,
+        proxy: @proxy,
+        ssl: @ssl,
+        kerberos: @kerberos,
+        open_timeout: @open_timeout, read_timeout: @read_timeout,
+        retry_known_errors: @retry_known_errors,
+        retry_times: @retry_times, retry_interval: @retry_interval
+      }
+    end
+
+    # curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=CREATE
+    #                 [&overwrite=<true|false>][&blocksize=<LONG>]
+    #                 [&replication=<SHORT>]
+    #                 [&permission=<OCTAL>][&buffersize=<INT>]"
+    def create(path, body, options = {})
+      options = options.merge('data' => 'true') if @httpfs_mode
+      WebHDFS.check_options(options, OPT_TABLE['CREATE'])
+      res = operate_requests('PUT', path, 'CREATE', options, body)
+      res.code == '201'
+    end
+
+    # curl -i -X POST "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=
+    #                      APPEND[&buffersize=<INT>]"
+    def append(path, body, options = {})
+      options = options.merge('data' => 'true') if @httpfs_mode
+      WebHDFS.check_options(options, OPT_TABLE['APPEND'])
+      res = operate_requests('POST', path, 'APPEND', options, body)
+      res.code == '200'
+    end
+
+    # curl -i -L "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=OPEN
+    #                [&offset=<LONG>][&length=<LONG>][&buffersize=<INT>]"
+    def read(path, options = {})
+      WebHDFS.check_options(options, OPT_TABLE['OPEN'])
+      res = operate_requests('GET', path, 'OPEN', options)
+      res.body
+    end
+
+    alias open read
+
+    # curl -i -X PUT "http://<HOST>:<PORT>/<PATH>?op=
+    #                     MKDIRS[&permission=<OCTAL>]"
+    def mkdir(path, options = {})
+      WebHDFS.check_options(options, OPT_TABLE['MKDIRS'])
+      res = operate_requests('PUT', path, 'MKDIRS', options)
+      WebHDFS.check_success_json(res, 'boolean')
+    end
+
+    alias mkdirs mkdir
+
+    # curl -i -X PUT "<HOST>:<PORT>/webhdfs/v1/<PATH>?op=
+    #                       RENAME&destination=<PATH>"
+    def rename(path, dest, options = {})
+      WebHDFS.check_options(options, OPT_TABLE['RENAME'])
+      dest = '/' + dest unless dest.start_with?('/')
+      res = operate_requests('PUT', path, 'RENAME',
+                             options.merge('destination' => dest))
+      WebHDFS.check_success_json(res, 'boolean')
+    end
+
+    # curl -i -X DELETE "http://<host>:<port>/webhdfs/v1/<path>?op=DELETE
+    #                          [&recursive=<true|false>]"
+    def delete(path, options = {})
+      WebHDFS.check_options(options, OPT_TABLE['DELETE'])
+      res = operate_requests('DELETE', path, 'DELETE', options)
+      WebHDFS.check_success_json(res, 'boolean')
+    end
+
+    # curl -i  "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=GETFILESTATUS"
+    def stat(path, options = {})
+      WebHDFS.check_options(options, OPT_TABLE['GETFILESTATUS'])
+      res = operate_requests('GET', path, 'GETFILESTATUS', options)
+      WebHDFS.check_success_json(res, 'FileStatus')
+    end
+    alias getfilestatus stat
+
+    # curl -i  "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=LISTSTATUS"
+    def list(path, options = {})
+      WebHDFS.check_options(options, OPT_TABLE['LISTSTATUS'])
+      res = operate_requests('GET', path, 'LISTSTATUS', options)
+      WebHDFS.check_success_json(res, 'FileStatuses')['FileStatus']
+    end
+    alias liststatus list
+
+    # curl -i "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=GETCONTENTSUMMARY"
+    def content_summary(path, options = {})
+      WebHDFS.check_options(options, OPT_TABLE['GETCONTENTSUMMARY'])
+      res = operate_requests('GET', path, 'GETCONTENTSUMMARY', options)
+      WebHDFS.check_success_json(res, 'ContentSummary')
+    end
+    alias getcontentsummary content_summary
+
+    # curl -i "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=GETFILECHECKSUM"
+    def checksum(path, options = {})
+      WebHDFS.check_options(options, OPT_TABLE['GETFILECHECKSUM'])
+      res = operate_requests('GET', path, 'GETFILECHECKSUM', options)
+      WebHDFS.check_success_json(res, 'FileChecksum')
+    end
+    alias getfilechecksum checksum
+
+    # curl -i "http://<HOST>:<PORT>/webhdfs/v1/?op=GETHOMEDIRECTORY"
+    def homedir(options = {})
+      WebHDFS.check_options(options, OPT_TABLE['GETHOMEDIRECTORY'])
+      res = operate_requests('GET', '/', 'GETHOMEDIRECTORY', options)
+      WebHDFS.check_success_json(res, 'Path')
+    end
+    alias gethomedirectory homedir
+
+    # curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=SETPERMISSION
+    #                 [&permission=<OCTAL>]"
+    def chmod(path, mode, options = {})
+      WebHDFS.check_options(options, OPT_TABLE['SETPERMISSION'])
+      res = operate_requests('PUT', path, 'SETPERMISSION',
+                             options.merge('permission' => mode))
+      res.code == '200'
+    end
+    alias setpermission chmod
+
+    # curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=SETOWNER
+    #                          [&owner=<USER>][&group=<GROUP>]"
+    def chown(path, options = {})
+      WebHDFS.check_options(options, OPT_TABLE['SETOWNER'])
+      unless options.key?('owner') || options.key?('group') ||
+             options.key?(:owner) || options.key?(:group)
+        raise ArgumentError, "'chown' needs at least one of owner or group"
+      end
+      res = operate_requests('PUT', path, 'SETOWNER', options)
+      res.code == '200'
+    end
+
+    alias setowner chown
+
+    # curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=SETREPLICATION
+    #                           [&replication=<SHORT>]"
+    def replication(path, replnum, options = {})
+      WebHDFS.check_options(options, OPT_TABLE['SETREPLICATION'])
+      res = operate_requests('PUT', path, 'SETREPLICATION',
+                             options.merge('replication' => replnum.to_s))
+      WebHDFS.check_success_json(res, 'boolean')
+    end
+    alias setreplication replication
+
+    # curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=SETTIMES
+    #                           [&modificationtime=<TIME>][&accesstime=<TIME>]"
+    # motidicationtime: radix-10 logn integer
+    # accesstime: radix-10 logn integer
+    def touch(path, options = {})
+      WebHDFS.check_options(options, OPT_TABLE['SETTIMES'])
+      unless options.key?('modificationtime') || options.key?('accesstime') ||
+             options.key?(:modificationtime) || options.key?(:accesstime)
+        raise ArgumentError, "'chown' needs at least one of " \
+                               'modificationtime or accesstime'
+      end
+      res = operate_requests('PUT', path, 'SETTIMES', options)
+      res.code == '200'
+    end
+
+    alias settimes touch
+
+    def operate_requests(method, path, op, params = {}, payload = nil)
+      request = WebHDFS::Request.new(@host, @port, request_options)
+      if !@httpfs_mode && REDIRECTED_OPERATIONS.include?(op)
+        response = request.execute(path, method, nil, nil, op, params, nil)
+        unless response.is_a?(Net::HTTPRedirection) && response['location']
+          msg = 'NameNode returns non-redirection (or without location' \
+                " header), code:#{res.code}, body:#{res.body}."
+          raise WebHDFS::RequestFailedError, msg
+        end
+        uri = URI.parse(response['location'])
+        rpath = if uri.query
+                  uri.path + '?' + uri.query
+                else
+                  uri.path
+                end
+        request = WebHDFS::Request.new(uri.host, uri.port, request_options)
+        request.execute(rpath, method,
+                        { 'Content-Type' => 'application/octet-stream' },
+                        payload, nil, {})
+      elsif @httpfs_mode && !payload.nil?
+        request.execute(path, method,
+                        { 'Content-Type' => 'application/octet-stream' },
+                        payload, op, params)
+      else
+        request.execute(path, method, nil, payload, op, params)
+      end
+    end
+  end
+end

data/lib/webhdfs/exceptions.rb

@@ -0,0 +1,15 @@
+module WebHDFS
+  class Error < StandardError; end
+
+  class FileNotFoundError < WebHDFS::Error; end
+
+  class IOError < WebHDFS::Error; end
+  class SecurityError < WebHDFS::Error; end
+
+  class ClientError < WebHDFS::Error; end
+  class ServerError < WebHDFS::Error; end
+
+  class RequestFailedError < WebHDFS::Error; end
+
+  class KerberosError < WebHDFS::Error; end
+end

data/lib/webhdfs/fileutils.rb

@@ -0,0 +1,439 @@
+require_relative 'client'
+
+module WebHDFS
+  module FileUtils
+    # Those values hold NameNode location
+    @fu_host = 'localhost'
+    @fu_port = 50_070
+    @fu_user = nil
+    @fu_doas = nil
+    @fu_httpfs_mode = false
+    @fu_ssl = false
+    @fu_ssl_ca_file = nil
+    @fu_ssl_verify_mode = nil
+    @fu_kerberos = false
+
+    # Public: Set hostname and port number of WebHDFS
+    #
+    # host - hostname
+    # port - port
+    # user - username
+    # doas - proxy user name
+    # proxy_address - address of the net http proxy to use
+    # proxy_port - port of the net http proxy to use
+    #
+    # Examples
+    #
+    #   FileUtils.set_server 'localhost', 50070
+    #
+    def set_server(host, port, user = nil, doas = nil, proxy_address = nil,
+                   proxy_port = nil)
+      @fu_host = host
+      @fu_port = port
+      @fu_user = user
+      @fu_doas = doas
+      @fu_paddr = proxy_address
+      @fu_pport = proxy_port
+    end
+    module_function :set_server
+
+    # Public: Set httpfs mode enable/disable
+    #
+    # mode - boolean (default true)
+    #
+    # Examples
+    #
+    #   FileUtils.set_httpfs_mode
+    #
+    def set_httpfs_mode(mode = true)
+      @fu_httpfs_mode = mode
+    end
+    module_function :set_httpfs_mode
+
+    # Public: Set ssl enable/disable
+    #
+    # mode - boolean (default true)
+    #
+    # Examples
+    #
+    #   FileUtils.set_ssl
+    #
+    def set_ssl(mode = true)
+      @fu_ssl = mode
+    end
+    module_function :set_ssl
+
+    # Public: Set ssl ca_file
+    #
+    # ca_file - string
+    #
+    # Examples
+    #
+    #   FileUtils.set_ca_file("/path/to/ca_file.pem")
+    #
+    def set_ssl_ca_file(ca_file)
+      @fu_ssl_ca_file = ca_file
+    end
+    module_function :set_ssl_ca_file
+
+    # Public: Set ssl verify mode
+    #
+    # mode - :none or :peer
+    #
+    # Examples
+    #
+    #   FileUtils.set_ssl_verify_mode(:peer)
+    #
+    def set_ssl_verify_mode(mode)
+      @fu_ssl_verify_mode = mode
+    end
+    module_function :set_ssl_verify_mode
+
+    # Public: Set kerberos authentication enable/disable
+    #
+    # mode - boolean (default true)
+    #
+    # Examples
+    #
+    #   FileUtils.set_kerberos
+    #
+    def set_kerberos(mode = true)
+      @fu_kerberos = mode
+    end
+    module_function :set_kerberos
+
+    # Public: Copy local file into HDFS
+    #
+    # file - local file path
+    # path - HDFS file path
+    # options :overwrite, :blocksize, :replication, :mode, :buffersize, :verbose
+    #
+    # Examples
+    #
+    #   FileUtils.copy_from_local 'local_file', 'remote_file'
+    #
+    def copy_from_local(file, path, options = {})
+      opts = options.dup
+      fu_log "copy_from_local local=#{file} " \
+             "hdfs=#{path}" if opts.delete(:verbose)
+      mode = opts.delete(:mode)
+      if mode
+        mode = format('%03o', mode) if mode.is_a? Integer
+      else
+        mode = '644'
+      end
+      opts[:permission] = mode
+      opts[:overwrite] ||= true
+
+      client.create(path, File.new(file, 'rb').read(File.size(file)), opts)
+    end
+    module_function :copy_from_local
+
+    # Public: Copy local file into HDFS with IOStream
+    #
+    # file - local file IO handle
+    # path - HDFS file path
+    # options :overwrite, :blocksize, :replication, :mode, :buffersize, :verbose
+    #
+    # Examples
+    #
+    # FileUtils.copy_from_local_via_stream 'local_file_IO_handle', 'remote_file'
+    #
+    def copy_from_local_via_stream(file, path, options = {})
+      opts = options.dup
+      fu_log "copy_from_local_via_stream local=#{file} " \
+             "hdfs=#{path}" if opts.delete(:verbose)
+      mode = opts.delete(:mode)
+      if mode
+        mode = format('%03o', mode) if mode.is_a? Integer
+      else
+        mode = '644'
+      end
+      opts[:permission] = mode
+      opts[:overwrite] ||= true
+
+      client.create(path, File.new(file, 'rb'), opts)
+    end
+    module_function :copy_from_local_via_stream
+
+    # Public: Copy remote HDFS file into local
+    #
+    # path - HDFS file path
+    # file - local file path
+    # options - :offset, :length, :buffersize, :verbose
+    #
+    # Examples
+    #
+    #   FileUtils.copy_to_local 'remote_file', 'local_file'
+    #
+    def copy_to_local(path, file, options = {})
+      opts = options.dup
+      fu_log "copy_to_local hdfs=#{path} local=#{file}" if opts.delete(:verbose)
+      File.open(file, 'wb') do |f|
+        f.write client.read(path, opts)
+      end
+    end
+    module_function :copy_to_local
+
+    # Public: Append to HDFS file
+    #
+    # path - HDFS file path
+    # body - contents
+    # options - :buffersize, :verbose
+    #
+    # Examples
+    #
+    #   FileUtils.append 'remote_path', 'contents'
+    #
+    def append(path, body, options = {})
+      opts = options.dup
+      fu_log "append #{body.bytesize} bytes to #{path}" if opts.delete(:verbose)
+      client.append(path, body, opts)
+    end
+    module_function :append
+
+    # Public: Create one or more directories.
+    #
+    # list - directory name, or list of them
+    # options - :mode, :verbose
+    #
+    # Examples
+    #
+    #   FileUtils.mkdir 'test'
+    #   FileUtils.mkdir %w( tmp data )
+    #   FileUtils.mkdir 'tmp', :mode => 0700
+    #
+    def mkdir(list, options = {})
+      opts = options.dup
+      list = [list].flatten
+      fu_log "mkdir #{format('-m %03o ', options[:mode]) if options[:mode]}" \
+             "#{list.join ' '}" if opts.delete(:verbose)
+      mode = opts[:mode]
+      if mode
+        mode = format('0%03o', mode) if mode.is_a? Integer
+      else
+        mode = '0755'
+      end
+      c = client
+      list.each do |dir|
+        c.mkdir(dir, permission: mode)
+      end
+    end
+    module_function :mkdir
+
+    # Public: Create one or more directories recursively.
+    #
+    # list - directory name, or list of them
+    # options - :mode, :verbose
+    #
+    # Examples
+    #
+    #   FileUtils.mkdir_p 'dir/subdir'
+    #   FileUtils.mkdir_p %w( tmp data )
+    #   FileUtils.mkdir_p 'dir/subdir', :mode => 0700
+    #
+    alias mkdir_p mkdir
+    module_function :mkdir_p
+
+    # Public: Remove one or more directories or files.
+    #
+    # list - directory name, or list of them
+    # options - :recursive, :verbose
+    #
+    # Examples
+    #
+    #   FileUtils.rm 'dir'
+    #   FileUtils.rm %w( tmp data )
+    #   FileUtils.rm 'dir', :recursive => true
+    #
+    def rm(list, options = {})
+      opts = options.dup
+      list = [list].flatten
+      fu_log "rm #{list.join ' '}" if opts.delete(:verbose)
+      c = client
+      list.each do |dir|
+        c.delete(dir, recursive: opts[:recursive] || false)
+      end
+    end
+    module_function :rm
+
+    # Public: Remove one or more directories/files recursively.
+    #
+    # list - directory name, or list of them
+    # options - :verbose
+    #
+    # Examples
+    #
+    #   FileUtils.rmr 'dir'
+    #   FileUtils.rmr %w( tmp data )
+    #   FileUtils.rmr 'dir'
+    #
+    def rmr(list, options = {})
+      rm(list, options.merge(recursive: true))
+    end
+    module_function :rmr
+
+    # Public: Rename a file or directory.
+    #
+    # src - from
+    # dst - to
+    # options - :verbose
+    #
+    # Examples
+    #
+    #   FileUtils.rename 'from', 'to'
+    #
+    def rename(src, dst, options = {})
+      opts = options.dup
+      fu_log "rename #{src} #{dst}" if opts.delete(:verbose)
+      client.rename(src, dst, opts)
+    end
+    module_function :rename
+
+    # Public: Change permission of one or more directories/files.
+    #
+    # mode - permission
+    # list - file/directory name or list of them.
+    # options - :verbose
+    #
+    # Examples
+    #
+    #   FileUtils.chmod 0755, 'dir'
+    #   FileUtils.chmod 0644, 'file'
+    #
+    def chmod(mode, list, options = {})
+      opts = options.dup
+      list = [list].flatten
+      fu_log format('chmod %o %s', mode,
+                    list.join(' ')) if opts.delete(:verbose)
+      mode = format('%03o', mode) if mode.is_a? Integer
+      c = client
+      list.each do |entry|
+        c.chmod(entry, mode, opts)
+      end
+    end
+    module_function :chmod
+
+    # Public: Change an ownership of one or more directories/files.
+    #
+    # user - username
+    # group - groupname
+    # list - file/directory name or list of them
+    # options - :verbose
+    #
+    # Examples
+    #
+    #   FileUtils.chmod 0755, 'dir'
+    #   FileUtils.chmod 0644, 'file'
+    #
+    def chown(user, group, list, options = {})
+      opts = options.dup
+      list = [list].flatten
+      fu_log format('chown %s%s',
+                    [user, group].compact.join(':') + ' ',
+                    list.join(' ')) if opts.delete(:verbose)
+      c = client
+      list.each do |entry|
+        c.chown(entry, owner: user, group: group)
+      end
+    end
+    module_function :chown
+
+    # Public: Set a replication factor of files
+    #
+    # list - file/directory name or list of them
+    # num - replication factor
+    # options - :verbose
+    #
+    # Examples
+    #
+    #   FileUtils.set_repl_factor 'file', 3
+    #
+    def set_repl_factor(list, num, options = {})
+      opts = options.dup
+      list = [list].flatten
+      fu_log format('set_repl_factor %s %d',
+                    list.join(' '), num) if opts.delete(:verbose)
+      c = client
+      list.each do |entry|
+        c.replication(entry, num, opts)
+      end
+    end
+    module_function :set_repl_factor
+
+    # Public: Set an access time of files
+    #
+    # list - file/directory name or list of them
+    # time - new access time
+    # options - :verbose
+    #
+    # Examples
+    #
+    #   FileUtils.set_atime 'file', Time.now
+    #
+    def set_atime(list, time, options = {})
+      opts = options.dup
+      list = [list].flatten
+      time = time.to_i
+      fu_log format('set_atime %s %d',
+                    list.join(' '), time) if opts.delete(:verbose)
+      c = client
+      list.each do |entry|
+        c.touch(entry, accesstime: time)
+      end
+    end
+    module_function :set_atime
+
+    # Public: Set a modification time of files
+    #
+    # list - file/directory name or list of them
+    # time - new modification time
+    # options - :verbose
+    #
+    # Examples
+    #
+    #   FileUtils.set_mtime 'file', Time.now
+    #
+    def set_mtime(list, time, options = {})
+      opts = options.dup
+      list = [list].flatten
+      time = time.to_i
+      fu_log format('set_mtime %s %d',
+                    list.join(' '), time) if opts.delete(:verbose)
+      c = client
+      list.each do |entry|
+        c.touch(entry, modificationtime: time)
+      end
+    end
+    module_function :set_mtime
+
+    # Internal: make functin private
+    def self.private_module_function(name)
+      module_function name
+      private_class_method name
+    end
+
+    @fileutils_output = $stderr
+    @fileutils_label  = '[webhdfs]: '
+    # Internal: Logging
+    def fu_log(msg)
+      @fileutils_output ||= $stderr
+      @fileutils_label  ||= ''
+      @fileutils_output.puts @fileutils_label + msg
+    end
+    private_module_function :fu_log
+
+    # Internal
+    def client
+      client = WebHDFS::Client.new(@fu_host, @fu_port, @fu_user, @fu_doas,
+                                   @fu_paddr, @fu_pport)
+      client.httpfs_mode = true if @fu_httpfs_mode
+      client.ssl = true if @fu_ssl
+      client.ssl_ca_file = @fu_ssl_ca_file if @fu_ssl_ca_file
+      client.ssl_verify_mode = @fu_ssl_verify_mode if @fu_ssl_verify_mode
+      client.kerberos = true if @fu_kerberos
+      client
+    end
+    private_module_function :client
+  end
+end