webhdfs-rlz 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,52 @@
1
+ require 'base64'
2
+ require 'gssapi'
3
+
4
+ require_relative 'exceptions'
5
+
6
+ module WebHDFS
7
+ # Kerberos class for http requests
8
+ class Kerberos
9
+ attr_read :host, :keytab, :token
10
+
11
+ # Constructor
12
+ def initialize(host, keytab)
13
+ @host = host
14
+ @keytab = keytab
15
+
16
+ @gsscli = GSSAPI::Simple.new(@host, 'HTTP', @kerberos_keytab)
17
+ @token = nil
18
+ begin
19
+ @token = @gsscli.init_context
20
+ rescue => token_error
21
+ raise WebHDFS::KerberosError, token_error.message
22
+ end
23
+ end
24
+
25
+ # Set the token to header authorization
26
+ def autorization(header)
27
+ encoded_token = Base64.strict_encode64(@token)
28
+ if header
29
+ header['Authorization'] = "Negotiate #{encoded_token}"
30
+ else
31
+ header = { 'Authorization' => "Negotiate #{encoded_token}" }
32
+ end
33
+ header
34
+ end
35
+
36
+ def check_response(response)
37
+ if @kerberos && response.code == '307'
38
+ itok = (response.header.get_fields('WWW-Authenticate') ||
39
+ ['']).pop.split(/\s+/).last
40
+ unless itok
41
+ raise WebHDFS::KerberosError, 'Server does not return ' \
42
+ 'WWW-Authenticate header'
43
+ end
44
+ begin
45
+ @gsscli.init_context(Base64.strict_decode64(itok))
46
+ rescue => e
47
+ raise WebHDFS::KerberosError, e.message
48
+ end
49
+ end
50
+ end
51
+ end
52
+ end
@@ -0,0 +1,28 @@
1
+ module WebHDFS
2
+ # Proxy class for http requests
3
+ class Proxy
4
+ attr_reader :address, :port, :user, :password
5
+
6
+ # Constructor
7
+ def initialize(address, port)
8
+ @address = address
9
+ @port = port
10
+ @user = @password = nil
11
+ end
12
+
13
+ # Set authentication credentials
14
+ def credentials(user, password)
15
+ @user = user
16
+ @password = password
17
+ end
18
+
19
+ # Proxy has authentication?
20
+ def authentication?
21
+ if @user && @password
22
+ true
23
+ else
24
+ false
25
+ end
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,41 @@
1
+ require_relative 'utilities'
2
+
3
+ module WebHDFS
4
+ class Prueba
5
+ attr_reader :campo
6
+
7
+ def initialize(campo)
8
+ self.campo = campo
9
+ end
10
+
11
+ def api(path)
12
+ puts WebHDFS.api_path(path)
13
+ end
14
+
15
+ def campo=(valor)
16
+ @campo = valor
17
+ puts 'fijado'
18
+ end
19
+
20
+ def aqui(valor)
21
+ puts "Aqui: #{valor}"
22
+ valor = mas(valor)
23
+ puts "Aqui: #{valor}"
24
+ end
25
+
26
+ def mas(valor)
27
+ valor += 1
28
+ puts "Mas: #{valor}"
29
+
30
+ valor
31
+ end
32
+ end
33
+ end
34
+
35
+ p = WebHDFS::Prueba.new(20)
36
+
37
+ p.api("efren")
38
+
39
+ a = WebHDFS::Prueba.new(20)
40
+
41
+ a.api("efren") if a
@@ -0,0 +1,175 @@
1
+ require 'net/http'
2
+ require 'uri'
3
+ require 'json'
4
+ require 'addressable/uri'
5
+ require 'openssl'
6
+
7
+ require_relative 'utilities'
8
+ require_relative 'exceptions'
9
+
10
+ module WebHDFS
11
+ # Class to make http requests
12
+ class Request
13
+ attr_reader :host, :port, :username, :doas
14
+ attr_reader :proxy, :ssl, :kerberos
15
+ attr_reader :open_timeout, :read_timeout
16
+
17
+ KNOWN_ERRORS = ['LeaseExpiredException'].freeze
18
+
19
+ # Constructor
20
+ def initialize(host, port, options = {})
21
+ @host = host
22
+ @port = port
23
+ @username = options[:username]
24
+ @doas = options[:doas]
25
+ @proxy = options[:proxy]
26
+ @ssl = options[:ssl]
27
+ @kerberos = options[:kerberos]
28
+ @open_timeout = options[:open_timeout]
29
+ @read_timeout = options[:read_timeout]
30
+ @retry_known_errors = options[:retry_known_errors]
31
+ @retry_times = options[:retry_times]
32
+ @retry_interval = options[:retry_interval]
33
+ end
34
+
35
+ def connection
36
+ conn = if @proxy
37
+ Net::HTTP.new(host, port, @proxy.address, @proxy.port)
38
+ else
39
+ Net::HTTP.new(host, port)
40
+ end
41
+
42
+ if @proxy.authentication?
43
+ conn.proxy_user = @proxy.user
44
+ conn.proxy_pass = @proxy.password
45
+ end
46
+
47
+ conn.open_timeout = @open_timeout if @open_timeout
48
+ conn.read_timeout = @read_timeout if @read_timeout
49
+
50
+ if @ssl
51
+ @ssl.apply_to(conn)
52
+ else
53
+ conn
54
+ end
55
+ end
56
+
57
+ def build_path(path, op, params)
58
+ path = Addressable::URI.escape(path)
59
+ if op
60
+ opts = if @username && @doas
61
+ { 'op' => op, 'user.name' => @username, 'doas' => @doas }
62
+ elsif @username
63
+ { 'op' => op, 'user.name' => @username }
64
+ elsif @doas
65
+ { 'op' => op, 'doas' => @doas }
66
+ else
67
+ { 'op' => op }
68
+ end
69
+ WebHDFS.api_path(path) + '?' + URI.encode_www_form(params.merge(opts))
70
+ else
71
+ path
72
+ end
73
+ end
74
+
75
+ def generic_request(connection, request_path, method, header = nil,
76
+ payload = nil)
77
+ res = nil
78
+ req = Net::HTTPGenericRequest.new(method, (payload ? true : false),
79
+ true, request_path, header)
80
+ raise WebHDFS::ClientError, 'Error accepting given IO resource as' \
81
+ ' data payload, Not valid in methods' \
82
+ ' other than PUT and POST' unless method == 'PUT' || method == 'POST'
83
+
84
+ req.body_stream = payload
85
+ req.content_length = payload.size
86
+ begin
87
+ res = connection.request(req)
88
+ rescue => e
89
+ raise WebHDFS::ServerError, 'Failed to connect to host' \
90
+ " #{@host}:#{@port}, #{e.message}"
91
+ end
92
+ res
93
+ end
94
+
95
+ def make_request(connection, request_path, method, header = nil,
96
+ payload = nil)
97
+ res = nil
98
+ if !payload.nil? && payload.respond_to?(:read) &&
99
+ payload.respond_to?(:size)
100
+ res = generic_request(connection, request_path, method, header, payload)
101
+ else
102
+ begin
103
+ res = connection.send_request(method, request_path, payload, header)
104
+ rescue => e
105
+ raise WebHDFS::ServerError, 'Failed to connect to host' \
106
+ " #{@host}:#{@port}, #{e.message}"
107
+ end
108
+ end
109
+ res
110
+ end
111
+
112
+ def raise_response_error(code, message)
113
+ case code
114
+ when '400'
115
+ raise WebHDFS::ClientError, message
116
+ when '401'
117
+ raise WebHDFS::SecurityError, message
118
+ when '403'
119
+ raise WebHDFS::IOError, message
120
+ when '404'
121
+ raise WebHDFS::FileNotFoundError, message
122
+ when '500'
123
+ raise WebHDFS::ServerError, message
124
+ else
125
+ raise WebHDFS::RequestFailedError, "response code:#{code}, " \
126
+ "message:#{message}"
127
+ end
128
+ end
129
+
130
+ # Execute request
131
+ def execute(path, method, header = nil, payload = nil, op = nil,
132
+ params = {}, retries = 0)
133
+ conn = connection
134
+
135
+ header = @kerberos.autorization(header) if @kerberos
136
+
137
+ request_path = build_path(path, op, params)
138
+
139
+ response = make_request(conn, payload)
140
+
141
+ @kerberos.check_response(response) if @kerberos
142
+
143
+ case response
144
+ when Net::HTTPSuccess
145
+ response
146
+ when Net::HTTPRedirection
147
+ response
148
+ else
149
+ message = if response.body && !response.body.empty?
150
+ response.body.delete("\n")
151
+ else
152
+ 'Response body is empty...'
153
+ end
154
+ if @retry_known_errors && retries < @retry_times
155
+ detail = nil
156
+ if message =~ /^\{"RemoteException":\{/
157
+ begin
158
+ detail = JSON.parse(message)
159
+
160
+ if detail['RemoteException'] &&
161
+ KNOWN_ERRORS.include?(detail['RemoteException']['exception'])
162
+ sleep @retry_interval if @retry_interval > 0
163
+ return execute(path, method, header, payload, op, params,
164
+ retries + 1)
165
+ end
166
+ rescue
167
+ # ignore broken json response body
168
+ end
169
+ end
170
+ end
171
+ raise_response_error(response.code, message)
172
+ end
173
+ end
174
+ end
175
+ end
@@ -0,0 +1,44 @@
1
+ require 'openssl'
2
+
3
+ module WebHDFS
4
+ # SSL class for http requests
5
+ class SSL
6
+ attr_reader :ca_file, :verify_mode
7
+ attr_reader :cert, :key, :version
8
+ SSL_VERIFY_MODES = [:none, :peer].freeze
9
+
10
+ # Constructor
11
+ def initialize(options = {})
12
+ @ca_file = options[:ca_file]
13
+ self.verify_mode = options[:verify_mode]
14
+ @cert = options[:cert]
15
+ @key = options[:key]
16
+ @version = options[:version]
17
+ end
18
+
19
+ # Verify valid ssl mode
20
+ def verify_mode=(mode)
21
+ unless SSL_VERIFY_MODES.include? mode
22
+ raise ArgumentError, "Invalid SSL verify mode #{mode.inspect}"
23
+ end
24
+ @verify_mode = mode
25
+ end
26
+
27
+ # Apply ssl to a http connection
28
+ def apply_to(connection)
29
+ connection.use_ssl = true
30
+ connection.ca_file = @ca_file if @ca_file
31
+ if @verify_mode
32
+ connection.verify_mode = case @verify_mode
33
+ when :none then OpenSSL::SSL::VERIFY_NONE
34
+ when :peer then OpenSSL::SSL::VERIFY_PEER
35
+ end
36
+ end
37
+ connection.cert = @cert if @cert
38
+ connection.key = @key if @key
39
+ connection.ssl_version = @version if @version
40
+
41
+ connection
42
+ end
43
+ end
44
+ end
@@ -0,0 +1,25 @@
1
+ require 'json'
2
+
3
+ # Principal module
4
+ module WebHDFS
5
+ # Path to access API
6
+ def self.api_path(path)
7
+ if path.start_with?('/')
8
+ '/webhdfs/v1' + path
9
+ else
10
+ '/webhdfs/v1/' + path
11
+ end
12
+ end
13
+
14
+ # Check if json request is success
15
+ def self.check_success_json(res, attr = nil)
16
+ res.code == '200' && res.content_type == 'application/json' &&
17
+ (attr.nil? || JSON.parse(res.body)[attr])
18
+ end
19
+
20
+ # Check if options are valid
21
+ def self.check_options(options, optdecl = [])
22
+ ex = options.keys.map(&:to_s) - (optdecl || [])
23
+ raise ArgumentError, "no such option: #{ex.join(' ')}" unless ex.empty?
24
+ end
25
+ end
@@ -0,0 +1,2 @@
1
+ require 'codeclimate-test-reporter'
2
+ CodeClimate::TestReporter.start
@@ -0,0 +1,20 @@
1
+ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
2
+ $LOAD_PATH.unshift(File.dirname(__FILE__))
3
+
4
+ require 'rubygems'
5
+ require 'rr'
6
+ require 'test/unit'
7
+ class Test::Unit::TestCase
8
+ include RR::Adapters::TestUnit
9
+ end
10
+
11
+ if ENV['SIMPLE_COV']
12
+ require 'simplecov'
13
+ SimpleCov.start do
14
+ add_filter 'test/'
15
+ add_filter 'pkg/'
16
+ add_filter 'vendor/'
17
+ end
18
+ end
19
+
20
+ require 'test/unit'
@@ -0,0 +1,69 @@
1
+ require 'test_helper'
2
+
3
+ class FileUtilsTest < Test::Unit::TestCase
4
+ def setup
5
+ require 'lib/webhdfs'
6
+ require 'lib/webhdfs/fileutils'
7
+ end
8
+
9
+ def test_copy_from_local
10
+ WebHDFS::FileUtils.copy_from_local('VERSION', 'VERSION', verbose: true)
11
+ WebHDFS::FileUtils.copy_to_local('VERSION', 'VERSION2', verbose: true)
12
+ WebHDFS::FileUtils.append('VERSION', 'foo-bar-buzz', verbose: true)
13
+ WebHDFS::FileUtils.rm('VERSION', verbose: true)
14
+ end
15
+
16
+ def test_copy_from_local_via_stream
17
+ WebHDFS::FileUtils.copy_from_local_via_stream('VERSION',
18
+ '/user/jay/VERSION',
19
+ verbose: true)
20
+ WebHDFS::FileUtils.rm('VERSION', verbose: true)
21
+ end
22
+
23
+ def test_rm
24
+ WebHDFS::FileUtils.mkdir('foo', mode: 0777, verbose: true)
25
+ WebHDFS::FileUtils.rm('foo', verbose: true)
26
+ end
27
+
28
+ def test_rmr
29
+ WebHDFS::FileUtils.mkdir_p('foo/bar/buzz', mode: 0777, verbose: true)
30
+ WebHDFS::FileUtils.rmr('foo', verbose: true)
31
+ end
32
+
33
+ def test_rename
34
+ # WebHDFS::FileUtils.mkdir_p('foo', :mode => 0777, :verbose => true)
35
+ # WebHDFS::FileUtils.rename('foo', 'foo2', :verbose => true)
36
+ # WebHDFS::FileUtils.rmr('foo2', :verbose => true)
37
+ end
38
+
39
+ def test_chmod
40
+ WebHDFS::FileUtils.mkdir('foo', mode: 0777, verbose: true)
41
+ WebHDFS::FileUtils.chmod(0755, 'foo', verbose: true)
42
+ WebHDFS::FileUtils.chmod(0777, 'foo', verbose: true)
43
+ WebHDFS::FileUtils.rm('foo', verbose: true)
44
+ end
45
+
46
+ def test_chown
47
+ # WebHDFS::FileUtils.mkdir('foo', :mode => 0777, :verbose => true)
48
+ # WebHDFS::FileUtils.chown('webuser', 'supergroup', 'foo', :verbose => true)
49
+ # WebHDFS::FileUtils.rm('foo', :verbose => true)
50
+ end
51
+
52
+ def test_set_repl_factor
53
+ WebHDFS::FileUtils.mkdir('foo', mode: 0777, verbose: true)
54
+ WebHDFS::FileUtils.set_repl_factor('foo', 2)
55
+ WebHDFS::FileUtils.rm('foo', verbose: true)
56
+ end
57
+
58
+ def test_set_atime
59
+ # WebHDFS::FileUtils.mkdir('foo', :mode => 0777, :verbose => true)
60
+ # WebHDFS::FileUtils.set_atime('foo', Time.now)
61
+ # WebHDFS::FileUtils.rm('foo', :verbose => true)
62
+ end
63
+
64
+ def test_set_mtime
65
+ # WebHDFS::FileUtils.mkdir('foo', :mode => 0777, :verbose => true)
66
+ # WebHDFS::FileUtils.set_mtime('foo', Time.now)
67
+ # WebHDFS::FileUtils.rm('foo', :verbose => true)
68
+ end
69
+ end