webhdfs-rlz 0.8.0

data/lib/webhdfs/kerberos.rb

@@ -0,0 +1,52 @@
+require 'base64'
+require 'gssapi'
+
+require_relative 'exceptions'
+
+module WebHDFS
+  # Kerberos class for http requests
+  class Kerberos
+    attr_read :host, :keytab, :token
+
+    # Constructor
+    def initialize(host, keytab)
+      @host = host
+      @keytab = keytab
+
+      @gsscli = GSSAPI::Simple.new(@host, 'HTTP', @kerberos_keytab)
+      @token = nil
+      begin
+        @token = @gsscli.init_context
+      rescue => token_error
+        raise WebHDFS::KerberosError, token_error.message
+      end
+    end
+
+    # Set the token to header authorization
+    def autorization(header)
+      encoded_token = Base64.strict_encode64(@token)
+      if header
+        header['Authorization'] = "Negotiate #{encoded_token}"
+      else
+        header = { 'Authorization' => "Negotiate #{encoded_token}" }
+      end
+      header
+    end
+
+    def check_response(response)
+      if @kerberos && response.code == '307'
+        itok = (response.header.get_fields('WWW-Authenticate') ||
+                ['']).pop.split(/\s+/).last
+        unless itok
+          raise WebHDFS::KerberosError, 'Server does not return ' \
+                                        'WWW-Authenticate header'
+        end
+        begin
+          @gsscli.init_context(Base64.strict_decode64(itok))
+        rescue => e
+          raise WebHDFS::KerberosError, e.message
+        end
+      end
+    end
+  end
+end

data/lib/webhdfs/proxy.rb

@@ -0,0 +1,28 @@
+module WebHDFS
+  # Proxy class for http requests
+  class Proxy
+    attr_reader :address, :port, :user, :password
+
+    # Constructor
+    def initialize(address, port)
+      @address = address
+      @port = port
+      @user = @password = nil
+    end
+
+    # Set authentication credentials
+    def credentials(user, password)
+      @user = user
+      @password = password
+    end
+
+    # Proxy has authentication?
+    def authentication?
+      if @user && @password
+        true
+      else
+        false
+      end
+    end
+  end
+end

data/lib/webhdfs/prueba.rb

@@ -0,0 +1,41 @@
+require_relative 'utilities'
+
+module WebHDFS
+  class Prueba
+    attr_reader :campo
+
+    def initialize(campo)
+      self.campo = campo
+    end
+
+    def api(path)
+      puts WebHDFS.api_path(path)
+    end
+
+    def campo=(valor)
+      @campo = valor
+      puts 'fijado'
+    end
+
+    def aqui(valor)
+      puts "Aqui: #{valor}"
+      valor = mas(valor)
+      puts "Aqui: #{valor}"
+    end
+
+    def mas(valor)
+      valor += 1
+      puts "Mas: #{valor}"
+
+      valor
+    end
+  end
+end
+
+p = WebHDFS::Prueba.new(20)
+
+p.api("efren")
+
+a = WebHDFS::Prueba.new(20)
+
+a.api("efren") if a

data/lib/webhdfs/request.rb

@@ -0,0 +1,175 @@
+require 'net/http'
+require 'uri'
+require 'json'
+require 'addressable/uri'
+require 'openssl'
+
+require_relative 'utilities'
+require_relative 'exceptions'
+
+module WebHDFS
+  # Class to make http requests
+  class Request
+    attr_reader :host, :port, :username, :doas
+    attr_reader :proxy, :ssl, :kerberos
+    attr_reader :open_timeout, :read_timeout
+
+    KNOWN_ERRORS = ['LeaseExpiredException'].freeze
+
+    # Constructor
+    def initialize(host, port, options = {})
+      @host = host
+      @port = port
+      @username = options[:username]
+      @doas = options[:doas]
+      @proxy = options[:proxy]
+      @ssl = options[:ssl]
+      @kerberos = options[:kerberos]
+      @open_timeout = options[:open_timeout]
+      @read_timeout = options[:read_timeout]
+      @retry_known_errors = options[:retry_known_errors]
+      @retry_times = options[:retry_times]
+      @retry_interval = options[:retry_interval]
+    end
+
+    def connection
+      conn = if @proxy
+               Net::HTTP.new(host, port, @proxy.address, @proxy.port)
+             else
+               Net::HTTP.new(host, port)
+             end
+
+      if @proxy.authentication?
+        conn.proxy_user = @proxy.user
+        conn.proxy_pass = @proxy.password
+      end
+
+      conn.open_timeout = @open_timeout if @open_timeout
+      conn.read_timeout = @read_timeout if @read_timeout
+
+      if @ssl
+        @ssl.apply_to(conn)
+      else
+        conn
+      end
+    end
+
+    def build_path(path, op, params)
+      path = Addressable::URI.escape(path)
+      if op
+        opts = if @username && @doas
+                 { 'op' => op, 'user.name' => @username, 'doas' => @doas }
+               elsif @username
+                 { 'op' => op, 'user.name' => @username }
+               elsif @doas
+                 { 'op' => op, 'doas' => @doas }
+               else
+                 { 'op' => op }
+               end
+        WebHDFS.api_path(path) + '?' + URI.encode_www_form(params.merge(opts))
+      else
+        path
+      end
+    end
+
+    def generic_request(connection, request_path, method, header = nil,
+                        payload = nil)
+      res = nil
+      req = Net::HTTPGenericRequest.new(method, (payload ? true : false),
+                                        true, request_path, header)
+      raise WebHDFS::ClientError, 'Error accepting given IO resource as' \
+       ' data payload, Not valid in methods' \
+       ' other than PUT and POST' unless method == 'PUT' || method == 'POST'
+
+      req.body_stream = payload
+      req.content_length = payload.size
+      begin
+        res = connection.request(req)
+      rescue => e
+        raise WebHDFS::ServerError, 'Failed to connect to host' \
+                                    " #{@host}:#{@port}, #{e.message}"
+      end
+      res
+    end
+
+    def make_request(connection, request_path, method, header = nil,
+                     payload = nil)
+      res = nil
+      if !payload.nil? && payload.respond_to?(:read) &&
+         payload.respond_to?(:size)
+        res = generic_request(connection, request_path, method, header, payload)
+      else
+        begin
+          res = connection.send_request(method, request_path, payload, header)
+        rescue => e
+          raise WebHDFS::ServerError, 'Failed to connect to host' \
+                                      " #{@host}:#{@port}, #{e.message}"
+        end
+      end
+      res
+    end
+
+    def raise_response_error(code, message)
+      case code
+      when '400'
+        raise WebHDFS::ClientError, message
+      when '401'
+        raise WebHDFS::SecurityError, message
+      when '403'
+        raise WebHDFS::IOError, message
+      when '404'
+        raise WebHDFS::FileNotFoundError, message
+      when '500'
+        raise WebHDFS::ServerError, message
+      else
+        raise WebHDFS::RequestFailedError, "response code:#{code}, " \
+                                           "message:#{message}"
+      end
+    end
+
+    # Execute request
+    def execute(path, method, header = nil, payload = nil, op = nil,
+                params = {}, retries = 0)
+      conn = connection
+
+      header = @kerberos.autorization(header) if @kerberos
+
+      request_path = build_path(path, op, params)
+
+      response = make_request(conn, payload)
+
+      @kerberos.check_response(response) if @kerberos
+
+      case response
+      when Net::HTTPSuccess
+        response
+      when Net::HTTPRedirection
+        response
+      else
+        message = if response.body && !response.body.empty?
+                    response.body.delete("\n")
+                  else
+                    'Response body is empty...'
+                  end
+        if @retry_known_errors && retries < @retry_times
+          detail = nil
+          if message =~ /^\{"RemoteException":\{/
+            begin
+              detail = JSON.parse(message)
+
+              if detail['RemoteException'] &&
+                 KNOWN_ERRORS.include?(detail['RemoteException']['exception'])
+                sleep @retry_interval if @retry_interval > 0
+                return execute(path, method, header, payload, op, params,
+                               retries + 1)
+              end
+            rescue
+              # ignore broken json response body
+            end
+          end
+        end
+        raise_response_error(response.code, message)
+      end
+    end
+  end
+end

data/lib/webhdfs/ssl.rb

@@ -0,0 +1,44 @@
+require 'openssl'
+
+module WebHDFS
+  # SSL class for http requests
+  class SSL
+    attr_reader :ca_file, :verify_mode
+    attr_reader :cert, :key, :version
+    SSL_VERIFY_MODES = [:none, :peer].freeze
+
+    # Constructor
+    def initialize(options = {})
+      @ca_file = options[:ca_file]
+      self.verify_mode = options[:verify_mode]
+      @cert = options[:cert]
+      @key = options[:key]
+      @version = options[:version]
+    end
+
+    # Verify valid ssl mode
+    def verify_mode=(mode)
+      unless SSL_VERIFY_MODES.include? mode
+        raise ArgumentError, "Invalid SSL verify mode #{mode.inspect}"
+      end
+      @verify_mode = mode
+    end
+
+    # Apply ssl to a http connection
+    def apply_to(connection)
+      connection.use_ssl = true
+      connection.ca_file = @ca_file if @ca_file
+      if @verify_mode
+        connection.verify_mode = case @verify_mode
+                                 when :none then OpenSSL::SSL::VERIFY_NONE
+                                 when :peer then OpenSSL::SSL::VERIFY_PEER
+                                 end
+      end
+      connection.cert = @cert if @cert
+      connection.key = @key if @key
+      connection.ssl_version = @version if @version
+
+      connection
+    end
+  end
+end

data/lib/webhdfs/utilities.rb

@@ -0,0 +1,25 @@
+require 'json'
+
+# Principal module
+module WebHDFS
+  # Path to access API
+  def self.api_path(path)
+    if path.start_with?('/')
+      '/webhdfs/v1' + path
+    else
+      '/webhdfs/v1/' + path
+    end
+  end
+
+  # Check if json request is success
+  def self.check_success_json(res, attr = nil)
+    res.code == '200' && res.content_type == 'application/json' &&
+      (attr.nil? || JSON.parse(res.body)[attr])
+  end
+
+  # Check if options are valid
+  def self.check_options(options, optdecl = [])
+    ex = options.keys.map(&:to_s) - (optdecl || [])
+    raise ArgumentError, "no such option: #{ex.join(' ')}" unless ex.empty?
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,2 @@
+require 'codeclimate-test-reporter'
+CodeClimate::TestReporter.start

data/test/test_helper.rb

@@ -0,0 +1,20 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'rubygems'
+require 'rr'
+require 'test/unit'
+class Test::Unit::TestCase
+  include RR::Adapters::TestUnit
+end
+
+if ENV['SIMPLE_COV']
+  require 'simplecov'
+  SimpleCov.start do
+    add_filter 'test/'
+    add_filter 'pkg/'
+    add_filter 'vendor/'
+  end
+end
+
+require 'test/unit'

data/test/webhdfs/fileutils.rb

@@ -0,0 +1,69 @@
+require 'test_helper'
+
+class FileUtilsTest < Test::Unit::TestCase
+  def setup
+    require 'lib/webhdfs'
+    require 'lib/webhdfs/fileutils'
+  end
+
+  def test_copy_from_local
+    WebHDFS::FileUtils.copy_from_local('VERSION', 'VERSION', verbose: true)
+    WebHDFS::FileUtils.copy_to_local('VERSION', 'VERSION2', verbose: true)
+    WebHDFS::FileUtils.append('VERSION', 'foo-bar-buzz', verbose: true)
+    WebHDFS::FileUtils.rm('VERSION', verbose: true)
+  end
+
+  def test_copy_from_local_via_stream
+    WebHDFS::FileUtils.copy_from_local_via_stream('VERSION',
+                                                  '/user/jay/VERSION',
+                                                  verbose: true)
+    WebHDFS::FileUtils.rm('VERSION', verbose: true)
+  end
+
+  def test_rm
+    WebHDFS::FileUtils.mkdir('foo', mode: 0777, verbose: true)
+    WebHDFS::FileUtils.rm('foo', verbose: true)
+  end
+
+  def test_rmr
+    WebHDFS::FileUtils.mkdir_p('foo/bar/buzz', mode: 0777, verbose: true)
+    WebHDFS::FileUtils.rmr('foo', verbose: true)
+  end
+
+  def test_rename
+    # WebHDFS::FileUtils.mkdir_p('foo', :mode => 0777, :verbose => true)
+    # WebHDFS::FileUtils.rename('foo', 'foo2', :verbose => true)
+    # WebHDFS::FileUtils.rmr('foo2', :verbose => true)
+  end
+
+  def test_chmod
+    WebHDFS::FileUtils.mkdir('foo', mode: 0777, verbose: true)
+    WebHDFS::FileUtils.chmod(0755, 'foo', verbose: true)
+    WebHDFS::FileUtils.chmod(0777, 'foo', verbose: true)
+    WebHDFS::FileUtils.rm('foo', verbose: true)
+  end
+
+  def test_chown
+    # WebHDFS::FileUtils.mkdir('foo', :mode => 0777, :verbose => true)
+    # WebHDFS::FileUtils.chown('webuser', 'supergroup', 'foo', :verbose => true)
+    # WebHDFS::FileUtils.rm('foo', :verbose => true)
+  end
+
+  def test_set_repl_factor
+    WebHDFS::FileUtils.mkdir('foo', mode: 0777, verbose: true)
+    WebHDFS::FileUtils.set_repl_factor('foo', 2)
+    WebHDFS::FileUtils.rm('foo', verbose: true)
+  end
+
+  def test_set_atime
+    # WebHDFS::FileUtils.mkdir('foo', :mode => 0777, :verbose => true)
+    # WebHDFS::FileUtils.set_atime('foo', Time.now)
+    # WebHDFS::FileUtils.rm('foo', :verbose => true)
+  end
+
+  def test_set_mtime
+    # WebHDFS::FileUtils.mkdir('foo', :mode => 0777, :verbose => true)
+    # WebHDFS::FileUtils.set_mtime('foo', Time.now)
+    # WebHDFS::FileUtils.rm('foo', :verbose => true)
+  end
+end