webhdfs-rlz 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/AUTHORS +2 -0
- data/COPYING +13 -0
- data/Gemfile +3 -0
- data/README.md +127 -0
- data/VERSION +1 -0
- data/lib/webhdfs.rb +2 -0
- data/lib/webhdfs/backport.rb +27 -0
- data/lib/webhdfs/client.rb +6 -0
- data/lib/webhdfs/client_v1.rb +411 -0
- data/lib/webhdfs/client_v2.rb +249 -0
- data/lib/webhdfs/exceptions.rb +15 -0
- data/lib/webhdfs/fileutils.rb +439 -0
- data/lib/webhdfs/kerberos.rb +52 -0
- data/lib/webhdfs/proxy.rb +28 -0
- data/lib/webhdfs/prueba.rb +41 -0
- data/lib/webhdfs/request.rb +175 -0
- data/lib/webhdfs/ssl.rb +44 -0
- data/lib/webhdfs/utilities.rb +25 -0
- data/spec/spec_helper.rb +2 -0
- data/test/test_helper.rb +20 -0
- data/test/webhdfs/fileutils.rb +69 -0
- data/webhdfs.gemspec +23 -0
- metadata +139 -0
@@ -0,0 +1,52 @@
|
|
1
|
+
require 'base64'
|
2
|
+
require 'gssapi'
|
3
|
+
|
4
|
+
require_relative 'exceptions'
|
5
|
+
|
6
|
+
module WebHDFS
|
7
|
+
# Kerberos class for http requests
|
8
|
+
class Kerberos
|
9
|
+
attr_read :host, :keytab, :token
|
10
|
+
|
11
|
+
# Constructor
|
12
|
+
def initialize(host, keytab)
|
13
|
+
@host = host
|
14
|
+
@keytab = keytab
|
15
|
+
|
16
|
+
@gsscli = GSSAPI::Simple.new(@host, 'HTTP', @kerberos_keytab)
|
17
|
+
@token = nil
|
18
|
+
begin
|
19
|
+
@token = @gsscli.init_context
|
20
|
+
rescue => token_error
|
21
|
+
raise WebHDFS::KerberosError, token_error.message
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
# Set the token to header authorization
|
26
|
+
def autorization(header)
|
27
|
+
encoded_token = Base64.strict_encode64(@token)
|
28
|
+
if header
|
29
|
+
header['Authorization'] = "Negotiate #{encoded_token}"
|
30
|
+
else
|
31
|
+
header = { 'Authorization' => "Negotiate #{encoded_token}" }
|
32
|
+
end
|
33
|
+
header
|
34
|
+
end
|
35
|
+
|
36
|
+
def check_response(response)
|
37
|
+
if @kerberos && response.code == '307'
|
38
|
+
itok = (response.header.get_fields('WWW-Authenticate') ||
|
39
|
+
['']).pop.split(/\s+/).last
|
40
|
+
unless itok
|
41
|
+
raise WebHDFS::KerberosError, 'Server does not return ' \
|
42
|
+
'WWW-Authenticate header'
|
43
|
+
end
|
44
|
+
begin
|
45
|
+
@gsscli.init_context(Base64.strict_decode64(itok))
|
46
|
+
rescue => e
|
47
|
+
raise WebHDFS::KerberosError, e.message
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module WebHDFS
|
2
|
+
# Proxy class for http requests
|
3
|
+
class Proxy
|
4
|
+
attr_reader :address, :port, :user, :password
|
5
|
+
|
6
|
+
# Constructor
|
7
|
+
def initialize(address, port)
|
8
|
+
@address = address
|
9
|
+
@port = port
|
10
|
+
@user = @password = nil
|
11
|
+
end
|
12
|
+
|
13
|
+
# Set authentication credentials
|
14
|
+
def credentials(user, password)
|
15
|
+
@user = user
|
16
|
+
@password = password
|
17
|
+
end
|
18
|
+
|
19
|
+
# Proxy has authentication?
|
20
|
+
def authentication?
|
21
|
+
if @user && @password
|
22
|
+
true
|
23
|
+
else
|
24
|
+
false
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
require_relative 'utilities'
|
2
|
+
|
3
|
+
module WebHDFS
|
4
|
+
class Prueba
|
5
|
+
attr_reader :campo
|
6
|
+
|
7
|
+
def initialize(campo)
|
8
|
+
self.campo = campo
|
9
|
+
end
|
10
|
+
|
11
|
+
def api(path)
|
12
|
+
puts WebHDFS.api_path(path)
|
13
|
+
end
|
14
|
+
|
15
|
+
def campo=(valor)
|
16
|
+
@campo = valor
|
17
|
+
puts 'fijado'
|
18
|
+
end
|
19
|
+
|
20
|
+
def aqui(valor)
|
21
|
+
puts "Aqui: #{valor}"
|
22
|
+
valor = mas(valor)
|
23
|
+
puts "Aqui: #{valor}"
|
24
|
+
end
|
25
|
+
|
26
|
+
def mas(valor)
|
27
|
+
valor += 1
|
28
|
+
puts "Mas: #{valor}"
|
29
|
+
|
30
|
+
valor
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
p = WebHDFS::Prueba.new(20)
|
36
|
+
|
37
|
+
p.api("efren")
|
38
|
+
|
39
|
+
a = WebHDFS::Prueba.new(20)
|
40
|
+
|
41
|
+
a.api("efren") if a
|
@@ -0,0 +1,175 @@
|
|
1
|
+
require 'net/http'
|
2
|
+
require 'uri'
|
3
|
+
require 'json'
|
4
|
+
require 'addressable/uri'
|
5
|
+
require 'openssl'
|
6
|
+
|
7
|
+
require_relative 'utilities'
|
8
|
+
require_relative 'exceptions'
|
9
|
+
|
10
|
+
module WebHDFS
|
11
|
+
# Class to make http requests
|
12
|
+
class Request
|
13
|
+
attr_reader :host, :port, :username, :doas
|
14
|
+
attr_reader :proxy, :ssl, :kerberos
|
15
|
+
attr_reader :open_timeout, :read_timeout
|
16
|
+
|
17
|
+
KNOWN_ERRORS = ['LeaseExpiredException'].freeze
|
18
|
+
|
19
|
+
# Constructor
|
20
|
+
def initialize(host, port, options = {})
|
21
|
+
@host = host
|
22
|
+
@port = port
|
23
|
+
@username = options[:username]
|
24
|
+
@doas = options[:doas]
|
25
|
+
@proxy = options[:proxy]
|
26
|
+
@ssl = options[:ssl]
|
27
|
+
@kerberos = options[:kerberos]
|
28
|
+
@open_timeout = options[:open_timeout]
|
29
|
+
@read_timeout = options[:read_timeout]
|
30
|
+
@retry_known_errors = options[:retry_known_errors]
|
31
|
+
@retry_times = options[:retry_times]
|
32
|
+
@retry_interval = options[:retry_interval]
|
33
|
+
end
|
34
|
+
|
35
|
+
def connection
|
36
|
+
conn = if @proxy
|
37
|
+
Net::HTTP.new(host, port, @proxy.address, @proxy.port)
|
38
|
+
else
|
39
|
+
Net::HTTP.new(host, port)
|
40
|
+
end
|
41
|
+
|
42
|
+
if @proxy.authentication?
|
43
|
+
conn.proxy_user = @proxy.user
|
44
|
+
conn.proxy_pass = @proxy.password
|
45
|
+
end
|
46
|
+
|
47
|
+
conn.open_timeout = @open_timeout if @open_timeout
|
48
|
+
conn.read_timeout = @read_timeout if @read_timeout
|
49
|
+
|
50
|
+
if @ssl
|
51
|
+
@ssl.apply_to(conn)
|
52
|
+
else
|
53
|
+
conn
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def build_path(path, op, params)
|
58
|
+
path = Addressable::URI.escape(path)
|
59
|
+
if op
|
60
|
+
opts = if @username && @doas
|
61
|
+
{ 'op' => op, 'user.name' => @username, 'doas' => @doas }
|
62
|
+
elsif @username
|
63
|
+
{ 'op' => op, 'user.name' => @username }
|
64
|
+
elsif @doas
|
65
|
+
{ 'op' => op, 'doas' => @doas }
|
66
|
+
else
|
67
|
+
{ 'op' => op }
|
68
|
+
end
|
69
|
+
WebHDFS.api_path(path) + '?' + URI.encode_www_form(params.merge(opts))
|
70
|
+
else
|
71
|
+
path
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
def generic_request(connection, request_path, method, header = nil,
|
76
|
+
payload = nil)
|
77
|
+
res = nil
|
78
|
+
req = Net::HTTPGenericRequest.new(method, (payload ? true : false),
|
79
|
+
true, request_path, header)
|
80
|
+
raise WebHDFS::ClientError, 'Error accepting given IO resource as' \
|
81
|
+
' data payload, Not valid in methods' \
|
82
|
+
' other than PUT and POST' unless method == 'PUT' || method == 'POST'
|
83
|
+
|
84
|
+
req.body_stream = payload
|
85
|
+
req.content_length = payload.size
|
86
|
+
begin
|
87
|
+
res = connection.request(req)
|
88
|
+
rescue => e
|
89
|
+
raise WebHDFS::ServerError, 'Failed to connect to host' \
|
90
|
+
" #{@host}:#{@port}, #{e.message}"
|
91
|
+
end
|
92
|
+
res
|
93
|
+
end
|
94
|
+
|
95
|
+
def make_request(connection, request_path, method, header = nil,
|
96
|
+
payload = nil)
|
97
|
+
res = nil
|
98
|
+
if !payload.nil? && payload.respond_to?(:read) &&
|
99
|
+
payload.respond_to?(:size)
|
100
|
+
res = generic_request(connection, request_path, method, header, payload)
|
101
|
+
else
|
102
|
+
begin
|
103
|
+
res = connection.send_request(method, request_path, payload, header)
|
104
|
+
rescue => e
|
105
|
+
raise WebHDFS::ServerError, 'Failed to connect to host' \
|
106
|
+
" #{@host}:#{@port}, #{e.message}"
|
107
|
+
end
|
108
|
+
end
|
109
|
+
res
|
110
|
+
end
|
111
|
+
|
112
|
+
def raise_response_error(code, message)
|
113
|
+
case code
|
114
|
+
when '400'
|
115
|
+
raise WebHDFS::ClientError, message
|
116
|
+
when '401'
|
117
|
+
raise WebHDFS::SecurityError, message
|
118
|
+
when '403'
|
119
|
+
raise WebHDFS::IOError, message
|
120
|
+
when '404'
|
121
|
+
raise WebHDFS::FileNotFoundError, message
|
122
|
+
when '500'
|
123
|
+
raise WebHDFS::ServerError, message
|
124
|
+
else
|
125
|
+
raise WebHDFS::RequestFailedError, "response code:#{code}, " \
|
126
|
+
"message:#{message}"
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
# Execute request
|
131
|
+
def execute(path, method, header = nil, payload = nil, op = nil,
|
132
|
+
params = {}, retries = 0)
|
133
|
+
conn = connection
|
134
|
+
|
135
|
+
header = @kerberos.autorization(header) if @kerberos
|
136
|
+
|
137
|
+
request_path = build_path(path, op, params)
|
138
|
+
|
139
|
+
response = make_request(conn, payload)
|
140
|
+
|
141
|
+
@kerberos.check_response(response) if @kerberos
|
142
|
+
|
143
|
+
case response
|
144
|
+
when Net::HTTPSuccess
|
145
|
+
response
|
146
|
+
when Net::HTTPRedirection
|
147
|
+
response
|
148
|
+
else
|
149
|
+
message = if response.body && !response.body.empty?
|
150
|
+
response.body.delete("\n")
|
151
|
+
else
|
152
|
+
'Response body is empty...'
|
153
|
+
end
|
154
|
+
if @retry_known_errors && retries < @retry_times
|
155
|
+
detail = nil
|
156
|
+
if message =~ /^\{"RemoteException":\{/
|
157
|
+
begin
|
158
|
+
detail = JSON.parse(message)
|
159
|
+
|
160
|
+
if detail['RemoteException'] &&
|
161
|
+
KNOWN_ERRORS.include?(detail['RemoteException']['exception'])
|
162
|
+
sleep @retry_interval if @retry_interval > 0
|
163
|
+
return execute(path, method, header, payload, op, params,
|
164
|
+
retries + 1)
|
165
|
+
end
|
166
|
+
rescue
|
167
|
+
# ignore broken json response body
|
168
|
+
end
|
169
|
+
end
|
170
|
+
end
|
171
|
+
raise_response_error(response.code, message)
|
172
|
+
end
|
173
|
+
end
|
174
|
+
end
|
175
|
+
end
|
data/lib/webhdfs/ssl.rb
ADDED
@@ -0,0 +1,44 @@
|
|
1
|
+
require 'openssl'
|
2
|
+
|
3
|
+
module WebHDFS
|
4
|
+
# SSL class for http requests
|
5
|
+
class SSL
|
6
|
+
attr_reader :ca_file, :verify_mode
|
7
|
+
attr_reader :cert, :key, :version
|
8
|
+
SSL_VERIFY_MODES = [:none, :peer].freeze
|
9
|
+
|
10
|
+
# Constructor
|
11
|
+
def initialize(options = {})
|
12
|
+
@ca_file = options[:ca_file]
|
13
|
+
self.verify_mode = options[:verify_mode]
|
14
|
+
@cert = options[:cert]
|
15
|
+
@key = options[:key]
|
16
|
+
@version = options[:version]
|
17
|
+
end
|
18
|
+
|
19
|
+
# Verify valid ssl mode
|
20
|
+
def verify_mode=(mode)
|
21
|
+
unless SSL_VERIFY_MODES.include? mode
|
22
|
+
raise ArgumentError, "Invalid SSL verify mode #{mode.inspect}"
|
23
|
+
end
|
24
|
+
@verify_mode = mode
|
25
|
+
end
|
26
|
+
|
27
|
+
# Apply ssl to a http connection
|
28
|
+
def apply_to(connection)
|
29
|
+
connection.use_ssl = true
|
30
|
+
connection.ca_file = @ca_file if @ca_file
|
31
|
+
if @verify_mode
|
32
|
+
connection.verify_mode = case @verify_mode
|
33
|
+
when :none then OpenSSL::SSL::VERIFY_NONE
|
34
|
+
when :peer then OpenSSL::SSL::VERIFY_PEER
|
35
|
+
end
|
36
|
+
end
|
37
|
+
connection.cert = @cert if @cert
|
38
|
+
connection.key = @key if @key
|
39
|
+
connection.ssl_version = @version if @version
|
40
|
+
|
41
|
+
connection
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require 'json'
|
2
|
+
|
3
|
+
# Principal module
|
4
|
+
module WebHDFS
|
5
|
+
# Path to access API
|
6
|
+
def self.api_path(path)
|
7
|
+
if path.start_with?('/')
|
8
|
+
'/webhdfs/v1' + path
|
9
|
+
else
|
10
|
+
'/webhdfs/v1/' + path
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
# Check if json request is success
|
15
|
+
def self.check_success_json(res, attr = nil)
|
16
|
+
res.code == '200' && res.content_type == 'application/json' &&
|
17
|
+
(attr.nil? || JSON.parse(res.body)[attr])
|
18
|
+
end
|
19
|
+
|
20
|
+
# Check if options are valid
|
21
|
+
def self.check_options(options, optdecl = [])
|
22
|
+
ex = options.keys.map(&:to_s) - (optdecl || [])
|
23
|
+
raise ArgumentError, "no such option: #{ex.join(' ')}" unless ex.empty?
|
24
|
+
end
|
25
|
+
end
|
data/spec/spec_helper.rb
ADDED
data/test/test_helper.rb
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
2
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
3
|
+
|
4
|
+
require 'rubygems'
|
5
|
+
require 'rr'
|
6
|
+
require 'test/unit'
|
7
|
+
class Test::Unit::TestCase
|
8
|
+
include RR::Adapters::TestUnit
|
9
|
+
end
|
10
|
+
|
11
|
+
if ENV['SIMPLE_COV']
|
12
|
+
require 'simplecov'
|
13
|
+
SimpleCov.start do
|
14
|
+
add_filter 'test/'
|
15
|
+
add_filter 'pkg/'
|
16
|
+
add_filter 'vendor/'
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
require 'test/unit'
|
@@ -0,0 +1,69 @@
|
|
1
|
+
require 'test_helper'
|
2
|
+
|
3
|
+
class FileUtilsTest < Test::Unit::TestCase
|
4
|
+
def setup
|
5
|
+
require 'lib/webhdfs'
|
6
|
+
require 'lib/webhdfs/fileutils'
|
7
|
+
end
|
8
|
+
|
9
|
+
def test_copy_from_local
|
10
|
+
WebHDFS::FileUtils.copy_from_local('VERSION', 'VERSION', verbose: true)
|
11
|
+
WebHDFS::FileUtils.copy_to_local('VERSION', 'VERSION2', verbose: true)
|
12
|
+
WebHDFS::FileUtils.append('VERSION', 'foo-bar-buzz', verbose: true)
|
13
|
+
WebHDFS::FileUtils.rm('VERSION', verbose: true)
|
14
|
+
end
|
15
|
+
|
16
|
+
def test_copy_from_local_via_stream
|
17
|
+
WebHDFS::FileUtils.copy_from_local_via_stream('VERSION',
|
18
|
+
'/user/jay/VERSION',
|
19
|
+
verbose: true)
|
20
|
+
WebHDFS::FileUtils.rm('VERSION', verbose: true)
|
21
|
+
end
|
22
|
+
|
23
|
+
def test_rm
|
24
|
+
WebHDFS::FileUtils.mkdir('foo', mode: 0777, verbose: true)
|
25
|
+
WebHDFS::FileUtils.rm('foo', verbose: true)
|
26
|
+
end
|
27
|
+
|
28
|
+
def test_rmr
|
29
|
+
WebHDFS::FileUtils.mkdir_p('foo/bar/buzz', mode: 0777, verbose: true)
|
30
|
+
WebHDFS::FileUtils.rmr('foo', verbose: true)
|
31
|
+
end
|
32
|
+
|
33
|
+
def test_rename
|
34
|
+
# WebHDFS::FileUtils.mkdir_p('foo', :mode => 0777, :verbose => true)
|
35
|
+
# WebHDFS::FileUtils.rename('foo', 'foo2', :verbose => true)
|
36
|
+
# WebHDFS::FileUtils.rmr('foo2', :verbose => true)
|
37
|
+
end
|
38
|
+
|
39
|
+
def test_chmod
|
40
|
+
WebHDFS::FileUtils.mkdir('foo', mode: 0777, verbose: true)
|
41
|
+
WebHDFS::FileUtils.chmod(0755, 'foo', verbose: true)
|
42
|
+
WebHDFS::FileUtils.chmod(0777, 'foo', verbose: true)
|
43
|
+
WebHDFS::FileUtils.rm('foo', verbose: true)
|
44
|
+
end
|
45
|
+
|
46
|
+
def test_chown
|
47
|
+
# WebHDFS::FileUtils.mkdir('foo', :mode => 0777, :verbose => true)
|
48
|
+
# WebHDFS::FileUtils.chown('webuser', 'supergroup', 'foo', :verbose => true)
|
49
|
+
# WebHDFS::FileUtils.rm('foo', :verbose => true)
|
50
|
+
end
|
51
|
+
|
52
|
+
def test_set_repl_factor
|
53
|
+
WebHDFS::FileUtils.mkdir('foo', mode: 0777, verbose: true)
|
54
|
+
WebHDFS::FileUtils.set_repl_factor('foo', 2)
|
55
|
+
WebHDFS::FileUtils.rm('foo', verbose: true)
|
56
|
+
end
|
57
|
+
|
58
|
+
def test_set_atime
|
59
|
+
# WebHDFS::FileUtils.mkdir('foo', :mode => 0777, :verbose => true)
|
60
|
+
# WebHDFS::FileUtils.set_atime('foo', Time.now)
|
61
|
+
# WebHDFS::FileUtils.rm('foo', :verbose => true)
|
62
|
+
end
|
63
|
+
|
64
|
+
def test_set_mtime
|
65
|
+
# WebHDFS::FileUtils.mkdir('foo', :mode => 0777, :verbose => true)
|
66
|
+
# WebHDFS::FileUtils.set_mtime('foo', Time.now)
|
67
|
+
# WebHDFS::FileUtils.rm('foo', :verbose => true)
|
68
|
+
end
|
69
|
+
end
|