webhdfs-rlz 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/AUTHORS +2 -0
- data/COPYING +13 -0
- data/Gemfile +3 -0
- data/README.md +127 -0
- data/VERSION +1 -0
- data/lib/webhdfs.rb +2 -0
- data/lib/webhdfs/backport.rb +27 -0
- data/lib/webhdfs/client.rb +6 -0
- data/lib/webhdfs/client_v1.rb +411 -0
- data/lib/webhdfs/client_v2.rb +249 -0
- data/lib/webhdfs/exceptions.rb +15 -0
- data/lib/webhdfs/fileutils.rb +439 -0
- data/lib/webhdfs/kerberos.rb +52 -0
- data/lib/webhdfs/proxy.rb +28 -0
- data/lib/webhdfs/prueba.rb +41 -0
- data/lib/webhdfs/request.rb +175 -0
- data/lib/webhdfs/ssl.rb +44 -0
- data/lib/webhdfs/utilities.rb +25 -0
- data/spec/spec_helper.rb +2 -0
- data/test/test_helper.rb +20 -0
- data/test/webhdfs/fileutils.rb +69 -0
- data/webhdfs.gemspec +23 -0
- metadata +139 -0
@@ -0,0 +1,249 @@
|
|
1
|
+
require 'net/http'
|
2
|
+
require 'uri'
|
3
|
+
require 'json'
|
4
|
+
require 'addressable/uri'
|
5
|
+
|
6
|
+
require_relative 'utilities'
|
7
|
+
require_relative 'exceptions'
|
8
|
+
require_relative 'proxy'
|
9
|
+
require_relative 'ssl'
|
10
|
+
require_relative 'kerberos'
|
11
|
+
|
12
|
+
module WebHDFS
|
13
|
+
class ClientV2
|
14
|
+
attr_accessor :host, :port, :username, :doas, :proxy
|
15
|
+
attr_accessor :open_timeout # default 30s (in ruby net/http)
|
16
|
+
attr_accessor :read_timeout # default 60s (in ruby net/http)
|
17
|
+
attr_accessor :httpfs_mode
|
18
|
+
attr_accessor :retry_known_errors # default false (not to retry)
|
19
|
+
attr_accessor :retry_times # default 1 (ignored when retry_known_errors is false)
|
20
|
+
attr_accessor :retry_interval # default 1 ([sec], ignored when retry_known_errors is false)
|
21
|
+
attr_accessor :ssl
|
22
|
+
attr_accessor :kerberos
|
23
|
+
attr_accessor :http_headers
|
24
|
+
|
25
|
+
# This hash table holds command options.
|
26
|
+
OPT_TABLE = {}.freeze # internal use only
|
27
|
+
OPT_TABLE['CREATE'] = %w(overwrite blocksize replication permission
|
28
|
+
buffersize data)
|
29
|
+
OPT_TABLE['APPEND'] = %w(buffersize data)
|
30
|
+
OPT_TABLE['OPEN'] = %w(offset length buffersize)
|
31
|
+
OPT_TABLE['MKDIRS'] = ['permission']
|
32
|
+
OPT_TABLE['DELETE'] = ['recursive']
|
33
|
+
OPT_TABLE['SETOWNER'] = %w(owner group)
|
34
|
+
OPT_TABLE['SETTIMES'] = %w(modificationtime accesstime)
|
35
|
+
|
36
|
+
REDIRECTED_OPERATIONS = %w(APPEND CREATE OPEN GETFILECHECKSUM).freeze
|
37
|
+
|
38
|
+
def initialize(host = 'localhost', port = 50_070, username = nil,
|
39
|
+
doas = nil, proxy_address = nil, proxy_port = nil,
|
40
|
+
http_headers = {})
|
41
|
+
@host = host
|
42
|
+
@port = port
|
43
|
+
@username = username
|
44
|
+
@doas = doas
|
45
|
+
|
46
|
+
@proxy = WebHDFS::Proxy.new(proxy_address, proxy_port) if proxy_address && proxy_port
|
47
|
+
|
48
|
+
@retry_known_errors = false
|
49
|
+
@retry_times = @retry_interval = 1
|
50
|
+
|
51
|
+
@httpfs_mode = false
|
52
|
+
|
53
|
+
@ssl = nil
|
54
|
+
|
55
|
+
@kerberos = nil
|
56
|
+
@http_headers = http_headers
|
57
|
+
end
|
58
|
+
|
59
|
+
def request_options
|
60
|
+
{
|
61
|
+
username: @username,
|
62
|
+
doas: @doas,
|
63
|
+
proxy: @proxy,
|
64
|
+
ssl: @ssl,
|
65
|
+
kerberos: @kerberos,
|
66
|
+
open_timeout: @open_timeout, read_timeout: @read_timeout,
|
67
|
+
retry_known_errors: @retry_known_errors,
|
68
|
+
retry_times: @retry_times, retry_interval: @retry_interval
|
69
|
+
}
|
70
|
+
end
|
71
|
+
|
72
|
+
# curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=CREATE
|
73
|
+
# [&overwrite=<true|false>][&blocksize=<LONG>]
|
74
|
+
# [&replication=<SHORT>]
|
75
|
+
# [&permission=<OCTAL>][&buffersize=<INT>]"
|
76
|
+
def create(path, body, options = {})
|
77
|
+
options = options.merge('data' => 'true') if @httpfs_mode
|
78
|
+
WebHDFS.check_options(options, OPT_TABLE['CREATE'])
|
79
|
+
res = operate_requests('PUT', path, 'CREATE', options, body)
|
80
|
+
res.code == '201'
|
81
|
+
end
|
82
|
+
|
83
|
+
# curl -i -X POST "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=
|
84
|
+
# APPEND[&buffersize=<INT>]"
|
85
|
+
def append(path, body, options = {})
|
86
|
+
options = options.merge('data' => 'true') if @httpfs_mode
|
87
|
+
WebHDFS.check_options(options, OPT_TABLE['APPEND'])
|
88
|
+
res = operate_requests('POST', path, 'APPEND', options, body)
|
89
|
+
res.code == '200'
|
90
|
+
end
|
91
|
+
|
92
|
+
# curl -i -L "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=OPEN
|
93
|
+
# [&offset=<LONG>][&length=<LONG>][&buffersize=<INT>]"
|
94
|
+
def read(path, options = {})
|
95
|
+
WebHDFS.check_options(options, OPT_TABLE['OPEN'])
|
96
|
+
res = operate_requests('GET', path, 'OPEN', options)
|
97
|
+
res.body
|
98
|
+
end
|
99
|
+
|
100
|
+
alias open read
|
101
|
+
|
102
|
+
# curl -i -X PUT "http://<HOST>:<PORT>/<PATH>?op=
|
103
|
+
# MKDIRS[&permission=<OCTAL>]"
|
104
|
+
def mkdir(path, options = {})
|
105
|
+
WebHDFS.check_options(options, OPT_TABLE['MKDIRS'])
|
106
|
+
res = operate_requests('PUT', path, 'MKDIRS', options)
|
107
|
+
WebHDFS.check_success_json(res, 'boolean')
|
108
|
+
end
|
109
|
+
|
110
|
+
alias mkdirs mkdir
|
111
|
+
|
112
|
+
# curl -i -X PUT "<HOST>:<PORT>/webhdfs/v1/<PATH>?op=
|
113
|
+
# RENAME&destination=<PATH>"
|
114
|
+
def rename(path, dest, options = {})
|
115
|
+
WebHDFS.check_options(options, OPT_TABLE['RENAME'])
|
116
|
+
dest = '/' + dest unless dest.start_with?('/')
|
117
|
+
res = operate_requests('PUT', path, 'RENAME',
|
118
|
+
options.merge('destination' => dest))
|
119
|
+
WebHDFS.check_success_json(res, 'boolean')
|
120
|
+
end
|
121
|
+
|
122
|
+
# curl -i -X DELETE "http://<host>:<port>/webhdfs/v1/<path>?op=DELETE
|
123
|
+
# [&recursive=<true|false>]"
|
124
|
+
def delete(path, options = {})
|
125
|
+
WebHDFS.check_options(options, OPT_TABLE['DELETE'])
|
126
|
+
res = operate_requests('DELETE', path, 'DELETE', options)
|
127
|
+
WebHDFS.check_success_json(res, 'boolean')
|
128
|
+
end
|
129
|
+
|
130
|
+
# curl -i "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=GETFILESTATUS"
|
131
|
+
def stat(path, options = {})
|
132
|
+
WebHDFS.check_options(options, OPT_TABLE['GETFILESTATUS'])
|
133
|
+
res = operate_requests('GET', path, 'GETFILESTATUS', options)
|
134
|
+
WebHDFS.check_success_json(res, 'FileStatus')
|
135
|
+
end
|
136
|
+
alias getfilestatus stat
|
137
|
+
|
138
|
+
# curl -i "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=LISTSTATUS"
|
139
|
+
def list(path, options = {})
|
140
|
+
WebHDFS.check_options(options, OPT_TABLE['LISTSTATUS'])
|
141
|
+
res = operate_requests('GET', path, 'LISTSTATUS', options)
|
142
|
+
WebHDFS.check_success_json(res, 'FileStatuses')['FileStatus']
|
143
|
+
end
|
144
|
+
alias liststatus list
|
145
|
+
|
146
|
+
# curl -i "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=GETCONTENTSUMMARY"
|
147
|
+
def content_summary(path, options = {})
|
148
|
+
WebHDFS.check_options(options, OPT_TABLE['GETCONTENTSUMMARY'])
|
149
|
+
res = operate_requests('GET', path, 'GETCONTENTSUMMARY', options)
|
150
|
+
WebHDFS.check_success_json(res, 'ContentSummary')
|
151
|
+
end
|
152
|
+
alias getcontentsummary content_summary
|
153
|
+
|
154
|
+
# curl -i "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=GETFILECHECKSUM"
|
155
|
+
def checksum(path, options = {})
|
156
|
+
WebHDFS.check_options(options, OPT_TABLE['GETFILECHECKSUM'])
|
157
|
+
res = operate_requests('GET', path, 'GETFILECHECKSUM', options)
|
158
|
+
WebHDFS.check_success_json(res, 'FileChecksum')
|
159
|
+
end
|
160
|
+
alias getfilechecksum checksum
|
161
|
+
|
162
|
+
# curl -i "http://<HOST>:<PORT>/webhdfs/v1/?op=GETHOMEDIRECTORY"
|
163
|
+
def homedir(options = {})
|
164
|
+
WebHDFS.check_options(options, OPT_TABLE['GETHOMEDIRECTORY'])
|
165
|
+
res = operate_requests('GET', '/', 'GETHOMEDIRECTORY', options)
|
166
|
+
WebHDFS.check_success_json(res, 'Path')
|
167
|
+
end
|
168
|
+
alias gethomedirectory homedir
|
169
|
+
|
170
|
+
# curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=SETPERMISSION
|
171
|
+
# [&permission=<OCTAL>]"
|
172
|
+
def chmod(path, mode, options = {})
|
173
|
+
WebHDFS.check_options(options, OPT_TABLE['SETPERMISSION'])
|
174
|
+
res = operate_requests('PUT', path, 'SETPERMISSION',
|
175
|
+
options.merge('permission' => mode))
|
176
|
+
res.code == '200'
|
177
|
+
end
|
178
|
+
alias setpermission chmod
|
179
|
+
|
180
|
+
# curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=SETOWNER
|
181
|
+
# [&owner=<USER>][&group=<GROUP>]"
|
182
|
+
def chown(path, options = {})
|
183
|
+
WebHDFS.check_options(options, OPT_TABLE['SETOWNER'])
|
184
|
+
unless options.key?('owner') || options.key?('group') ||
|
185
|
+
options.key?(:owner) || options.key?(:group)
|
186
|
+
raise ArgumentError, "'chown' needs at least one of owner or group"
|
187
|
+
end
|
188
|
+
res = operate_requests('PUT', path, 'SETOWNER', options)
|
189
|
+
res.code == '200'
|
190
|
+
end
|
191
|
+
|
192
|
+
alias setowner chown
|
193
|
+
|
194
|
+
# curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=SETREPLICATION
|
195
|
+
# [&replication=<SHORT>]"
|
196
|
+
def replication(path, replnum, options = {})
|
197
|
+
WebHDFS.check_options(options, OPT_TABLE['SETREPLICATION'])
|
198
|
+
res = operate_requests('PUT', path, 'SETREPLICATION',
|
199
|
+
options.merge('replication' => replnum.to_s))
|
200
|
+
WebHDFS.check_success_json(res, 'boolean')
|
201
|
+
end
|
202
|
+
alias setreplication replication
|
203
|
+
|
204
|
+
# curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=SETTIMES
|
205
|
+
# [&modificationtime=<TIME>][&accesstime=<TIME>]"
|
206
|
+
# motidicationtime: radix-10 logn integer
|
207
|
+
# accesstime: radix-10 logn integer
|
208
|
+
def touch(path, options = {})
|
209
|
+
WebHDFS.check_options(options, OPT_TABLE['SETTIMES'])
|
210
|
+
unless options.key?('modificationtime') || options.key?('accesstime') ||
|
211
|
+
options.key?(:modificationtime) || options.key?(:accesstime)
|
212
|
+
raise ArgumentError, "'chown' needs at least one of " \
|
213
|
+
'modificationtime or accesstime'
|
214
|
+
end
|
215
|
+
res = operate_requests('PUT', path, 'SETTIMES', options)
|
216
|
+
res.code == '200'
|
217
|
+
end
|
218
|
+
|
219
|
+
alias settimes touch
|
220
|
+
|
221
|
+
def operate_requests(method, path, op, params = {}, payload = nil)
|
222
|
+
request = WebHDFS::Request.new(@host, @port, request_options)
|
223
|
+
if !@httpfs_mode && REDIRECTED_OPERATIONS.include?(op)
|
224
|
+
response = request.execute(path, method, nil, nil, op, params, nil)
|
225
|
+
unless response.is_a?(Net::HTTPRedirection) && response['location']
|
226
|
+
msg = 'NameNode returns non-redirection (or without location' \
|
227
|
+
" header), code:#{res.code}, body:#{res.body}."
|
228
|
+
raise WebHDFS::RequestFailedError, msg
|
229
|
+
end
|
230
|
+
uri = URI.parse(response['location'])
|
231
|
+
rpath = if uri.query
|
232
|
+
uri.path + '?' + uri.query
|
233
|
+
else
|
234
|
+
uri.path
|
235
|
+
end
|
236
|
+
request = WebHDFS::Request.new(uri.host, uri.port, request_options)
|
237
|
+
request.execute(rpath, method,
|
238
|
+
{ 'Content-Type' => 'application/octet-stream' },
|
239
|
+
payload, nil, {})
|
240
|
+
elsif @httpfs_mode && !payload.nil?
|
241
|
+
request.execute(path, method,
|
242
|
+
{ 'Content-Type' => 'application/octet-stream' },
|
243
|
+
payload, op, params)
|
244
|
+
else
|
245
|
+
request.execute(path, method, nil, payload, op, params)
|
246
|
+
end
|
247
|
+
end
|
248
|
+
end
|
249
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
module WebHDFS
|
2
|
+
class Error < StandardError; end
|
3
|
+
|
4
|
+
class FileNotFoundError < WebHDFS::Error; end
|
5
|
+
|
6
|
+
class IOError < WebHDFS::Error; end
|
7
|
+
class SecurityError < WebHDFS::Error; end
|
8
|
+
|
9
|
+
class ClientError < WebHDFS::Error; end
|
10
|
+
class ServerError < WebHDFS::Error; end
|
11
|
+
|
12
|
+
class RequestFailedError < WebHDFS::Error; end
|
13
|
+
|
14
|
+
class KerberosError < WebHDFS::Error; end
|
15
|
+
end
|
@@ -0,0 +1,439 @@
|
|
1
|
+
require_relative 'client'
|
2
|
+
|
3
|
+
module WebHDFS
|
4
|
+
module FileUtils
|
5
|
+
# Those values hold NameNode location
|
6
|
+
@fu_host = 'localhost'
|
7
|
+
@fu_port = 50_070
|
8
|
+
@fu_user = nil
|
9
|
+
@fu_doas = nil
|
10
|
+
@fu_httpfs_mode = false
|
11
|
+
@fu_ssl = false
|
12
|
+
@fu_ssl_ca_file = nil
|
13
|
+
@fu_ssl_verify_mode = nil
|
14
|
+
@fu_kerberos = false
|
15
|
+
|
16
|
+
# Public: Set hostname and port number of WebHDFS
|
17
|
+
#
|
18
|
+
# host - hostname
|
19
|
+
# port - port
|
20
|
+
# user - username
|
21
|
+
# doas - proxy user name
|
22
|
+
# proxy_address - address of the net http proxy to use
|
23
|
+
# proxy_port - port of the net http proxy to use
|
24
|
+
#
|
25
|
+
# Examples
|
26
|
+
#
|
27
|
+
# FileUtils.set_server 'localhost', 50070
|
28
|
+
#
|
29
|
+
def set_server(host, port, user = nil, doas = nil, proxy_address = nil,
|
30
|
+
proxy_port = nil)
|
31
|
+
@fu_host = host
|
32
|
+
@fu_port = port
|
33
|
+
@fu_user = user
|
34
|
+
@fu_doas = doas
|
35
|
+
@fu_paddr = proxy_address
|
36
|
+
@fu_pport = proxy_port
|
37
|
+
end
|
38
|
+
module_function :set_server
|
39
|
+
|
40
|
+
# Public: Set httpfs mode enable/disable
|
41
|
+
#
|
42
|
+
# mode - boolean (default true)
|
43
|
+
#
|
44
|
+
# Examples
|
45
|
+
#
|
46
|
+
# FileUtils.set_httpfs_mode
|
47
|
+
#
|
48
|
+
def set_httpfs_mode(mode = true)
|
49
|
+
@fu_httpfs_mode = mode
|
50
|
+
end
|
51
|
+
module_function :set_httpfs_mode
|
52
|
+
|
53
|
+
# Public: Set ssl enable/disable
|
54
|
+
#
|
55
|
+
# mode - boolean (default true)
|
56
|
+
#
|
57
|
+
# Examples
|
58
|
+
#
|
59
|
+
# FileUtils.set_ssl
|
60
|
+
#
|
61
|
+
def set_ssl(mode = true)
|
62
|
+
@fu_ssl = mode
|
63
|
+
end
|
64
|
+
module_function :set_ssl
|
65
|
+
|
66
|
+
# Public: Set ssl ca_file
|
67
|
+
#
|
68
|
+
# ca_file - string
|
69
|
+
#
|
70
|
+
# Examples
|
71
|
+
#
|
72
|
+
# FileUtils.set_ca_file("/path/to/ca_file.pem")
|
73
|
+
#
|
74
|
+
def set_ssl_ca_file(ca_file)
|
75
|
+
@fu_ssl_ca_file = ca_file
|
76
|
+
end
|
77
|
+
module_function :set_ssl_ca_file
|
78
|
+
|
79
|
+
# Public: Set ssl verify mode
|
80
|
+
#
|
81
|
+
# mode - :none or :peer
|
82
|
+
#
|
83
|
+
# Examples
|
84
|
+
#
|
85
|
+
# FileUtils.set_ssl_verify_mode(:peer)
|
86
|
+
#
|
87
|
+
def set_ssl_verify_mode(mode)
|
88
|
+
@fu_ssl_verify_mode = mode
|
89
|
+
end
|
90
|
+
module_function :set_ssl_verify_mode
|
91
|
+
|
92
|
+
# Public: Set kerberos authentication enable/disable
|
93
|
+
#
|
94
|
+
# mode - boolean (default true)
|
95
|
+
#
|
96
|
+
# Examples
|
97
|
+
#
|
98
|
+
# FileUtils.set_kerberos
|
99
|
+
#
|
100
|
+
def set_kerberos(mode = true)
|
101
|
+
@fu_kerberos = mode
|
102
|
+
end
|
103
|
+
module_function :set_kerberos
|
104
|
+
|
105
|
+
# Public: Copy local file into HDFS
|
106
|
+
#
|
107
|
+
# file - local file path
|
108
|
+
# path - HDFS file path
|
109
|
+
# options :overwrite, :blocksize, :replication, :mode, :buffersize, :verbose
|
110
|
+
#
|
111
|
+
# Examples
|
112
|
+
#
|
113
|
+
# FileUtils.copy_from_local 'local_file', 'remote_file'
|
114
|
+
#
|
115
|
+
def copy_from_local(file, path, options = {})
|
116
|
+
opts = options.dup
|
117
|
+
fu_log "copy_from_local local=#{file} " \
|
118
|
+
"hdfs=#{path}" if opts.delete(:verbose)
|
119
|
+
mode = opts.delete(:mode)
|
120
|
+
if mode
|
121
|
+
mode = format('%03o', mode) if mode.is_a? Integer
|
122
|
+
else
|
123
|
+
mode = '644'
|
124
|
+
end
|
125
|
+
opts[:permission] = mode
|
126
|
+
opts[:overwrite] ||= true
|
127
|
+
|
128
|
+
client.create(path, File.new(file, 'rb').read(File.size(file)), opts)
|
129
|
+
end
|
130
|
+
module_function :copy_from_local
|
131
|
+
|
132
|
+
# Public: Copy local file into HDFS with IOStream
|
133
|
+
#
|
134
|
+
# file - local file IO handle
|
135
|
+
# path - HDFS file path
|
136
|
+
# options :overwrite, :blocksize, :replication, :mode, :buffersize, :verbose
|
137
|
+
#
|
138
|
+
# Examples
|
139
|
+
#
|
140
|
+
# FileUtils.copy_from_local_via_stream 'local_file_IO_handle', 'remote_file'
|
141
|
+
#
|
142
|
+
def copy_from_local_via_stream(file, path, options = {})
|
143
|
+
opts = options.dup
|
144
|
+
fu_log "copy_from_local_via_stream local=#{file} " \
|
145
|
+
"hdfs=#{path}" if opts.delete(:verbose)
|
146
|
+
mode = opts.delete(:mode)
|
147
|
+
if mode
|
148
|
+
mode = format('%03o', mode) if mode.is_a? Integer
|
149
|
+
else
|
150
|
+
mode = '644'
|
151
|
+
end
|
152
|
+
opts[:permission] = mode
|
153
|
+
opts[:overwrite] ||= true
|
154
|
+
|
155
|
+
client.create(path, File.new(file, 'rb'), opts)
|
156
|
+
end
|
157
|
+
module_function :copy_from_local_via_stream
|
158
|
+
|
159
|
+
# Public: Copy remote HDFS file into local
|
160
|
+
#
|
161
|
+
# path - HDFS file path
|
162
|
+
# file - local file path
|
163
|
+
# options - :offset, :length, :buffersize, :verbose
|
164
|
+
#
|
165
|
+
# Examples
|
166
|
+
#
|
167
|
+
# FileUtils.copy_to_local 'remote_file', 'local_file'
|
168
|
+
#
|
169
|
+
def copy_to_local(path, file, options = {})
|
170
|
+
opts = options.dup
|
171
|
+
fu_log "copy_to_local hdfs=#{path} local=#{file}" if opts.delete(:verbose)
|
172
|
+
File.open(file, 'wb') do |f|
|
173
|
+
f.write client.read(path, opts)
|
174
|
+
end
|
175
|
+
end
|
176
|
+
module_function :copy_to_local
|
177
|
+
|
178
|
+
# Public: Append to HDFS file
|
179
|
+
#
|
180
|
+
# path - HDFS file path
|
181
|
+
# body - contents
|
182
|
+
# options - :buffersize, :verbose
|
183
|
+
#
|
184
|
+
# Examples
|
185
|
+
#
|
186
|
+
# FileUtils.append 'remote_path', 'contents'
|
187
|
+
#
|
188
|
+
def append(path, body, options = {})
|
189
|
+
opts = options.dup
|
190
|
+
fu_log "append #{body.bytesize} bytes to #{path}" if opts.delete(:verbose)
|
191
|
+
client.append(path, body, opts)
|
192
|
+
end
|
193
|
+
module_function :append
|
194
|
+
|
195
|
+
# Public: Create one or more directories.
|
196
|
+
#
|
197
|
+
# list - directory name, or list of them
|
198
|
+
# options - :mode, :verbose
|
199
|
+
#
|
200
|
+
# Examples
|
201
|
+
#
|
202
|
+
# FileUtils.mkdir 'test'
|
203
|
+
# FileUtils.mkdir %w( tmp data )
|
204
|
+
# FileUtils.mkdir 'tmp', :mode => 0700
|
205
|
+
#
|
206
|
+
def mkdir(list, options = {})
|
207
|
+
opts = options.dup
|
208
|
+
list = [list].flatten
|
209
|
+
fu_log "mkdir #{format('-m %03o ', options[:mode]) if options[:mode]}" \
|
210
|
+
"#{list.join ' '}" if opts.delete(:verbose)
|
211
|
+
mode = opts[:mode]
|
212
|
+
if mode
|
213
|
+
mode = format('0%03o', mode) if mode.is_a? Integer
|
214
|
+
else
|
215
|
+
mode = '0755'
|
216
|
+
end
|
217
|
+
c = client
|
218
|
+
list.each do |dir|
|
219
|
+
c.mkdir(dir, permission: mode)
|
220
|
+
end
|
221
|
+
end
|
222
|
+
module_function :mkdir
|
223
|
+
|
224
|
+
# Public: Create one or more directories recursively.
|
225
|
+
#
|
226
|
+
# list - directory name, or list of them
|
227
|
+
# options - :mode, :verbose
|
228
|
+
#
|
229
|
+
# Examples
|
230
|
+
#
|
231
|
+
# FileUtils.mkdir_p 'dir/subdir'
|
232
|
+
# FileUtils.mkdir_p %w( tmp data )
|
233
|
+
# FileUtils.mkdir_p 'dir/subdir', :mode => 0700
|
234
|
+
#
|
235
|
+
alias mkdir_p mkdir
|
236
|
+
module_function :mkdir_p
|
237
|
+
|
238
|
+
# Public: Remove one or more directories or files.
|
239
|
+
#
|
240
|
+
# list - directory name, or list of them
|
241
|
+
# options - :recursive, :verbose
|
242
|
+
#
|
243
|
+
# Examples
|
244
|
+
#
|
245
|
+
# FileUtils.rm 'dir'
|
246
|
+
# FileUtils.rm %w( tmp data )
|
247
|
+
# FileUtils.rm 'dir', :recursive => true
|
248
|
+
#
|
249
|
+
def rm(list, options = {})
|
250
|
+
opts = options.dup
|
251
|
+
list = [list].flatten
|
252
|
+
fu_log "rm #{list.join ' '}" if opts.delete(:verbose)
|
253
|
+
c = client
|
254
|
+
list.each do |dir|
|
255
|
+
c.delete(dir, recursive: opts[:recursive] || false)
|
256
|
+
end
|
257
|
+
end
|
258
|
+
module_function :rm
|
259
|
+
|
260
|
+
# Public: Remove one or more directories/files recursively.
|
261
|
+
#
|
262
|
+
# list - directory name, or list of them
|
263
|
+
# options - :verbose
|
264
|
+
#
|
265
|
+
# Examples
|
266
|
+
#
|
267
|
+
# FileUtils.rmr 'dir'
|
268
|
+
# FileUtils.rmr %w( tmp data )
|
269
|
+
# FileUtils.rmr 'dir'
|
270
|
+
#
|
271
|
+
def rmr(list, options = {})
|
272
|
+
rm(list, options.merge(recursive: true))
|
273
|
+
end
|
274
|
+
module_function :rmr
|
275
|
+
|
276
|
+
# Public: Rename a file or directory.
|
277
|
+
#
|
278
|
+
# src - from
|
279
|
+
# dst - to
|
280
|
+
# options - :verbose
|
281
|
+
#
|
282
|
+
# Examples
|
283
|
+
#
|
284
|
+
# FileUtils.rename 'from', 'to'
|
285
|
+
#
|
286
|
+
def rename(src, dst, options = {})
|
287
|
+
opts = options.dup
|
288
|
+
fu_log "rename #{src} #{dst}" if opts.delete(:verbose)
|
289
|
+
client.rename(src, dst, opts)
|
290
|
+
end
|
291
|
+
module_function :rename
|
292
|
+
|
293
|
+
# Public: Change permission of one or more directories/files.
|
294
|
+
#
|
295
|
+
# mode - permission
|
296
|
+
# list - file/directory name or list of them.
|
297
|
+
# options - :verbose
|
298
|
+
#
|
299
|
+
# Examples
|
300
|
+
#
|
301
|
+
# FileUtils.chmod 0755, 'dir'
|
302
|
+
# FileUtils.chmod 0644, 'file'
|
303
|
+
#
|
304
|
+
def chmod(mode, list, options = {})
|
305
|
+
opts = options.dup
|
306
|
+
list = [list].flatten
|
307
|
+
fu_log format('chmod %o %s', mode,
|
308
|
+
list.join(' ')) if opts.delete(:verbose)
|
309
|
+
mode = format('%03o', mode) if mode.is_a? Integer
|
310
|
+
c = client
|
311
|
+
list.each do |entry|
|
312
|
+
c.chmod(entry, mode, opts)
|
313
|
+
end
|
314
|
+
end
|
315
|
+
module_function :chmod
|
316
|
+
|
317
|
+
# Public: Change an ownership of one or more directories/files.
|
318
|
+
#
|
319
|
+
# user - username
|
320
|
+
# group - groupname
|
321
|
+
# list - file/directory name or list of them
|
322
|
+
# options - :verbose
|
323
|
+
#
|
324
|
+
# Examples
|
325
|
+
#
|
326
|
+
# FileUtils.chmod 0755, 'dir'
|
327
|
+
# FileUtils.chmod 0644, 'file'
|
328
|
+
#
|
329
|
+
def chown(user, group, list, options = {})
|
330
|
+
opts = options.dup
|
331
|
+
list = [list].flatten
|
332
|
+
fu_log format('chown %s%s',
|
333
|
+
[user, group].compact.join(':') + ' ',
|
334
|
+
list.join(' ')) if opts.delete(:verbose)
|
335
|
+
c = client
|
336
|
+
list.each do |entry|
|
337
|
+
c.chown(entry, owner: user, group: group)
|
338
|
+
end
|
339
|
+
end
|
340
|
+
module_function :chown
|
341
|
+
|
342
|
+
# Public: Set a replication factor of files
|
343
|
+
#
|
344
|
+
# list - file/directory name or list of them
|
345
|
+
# num - replication factor
|
346
|
+
# options - :verbose
|
347
|
+
#
|
348
|
+
# Examples
|
349
|
+
#
|
350
|
+
# FileUtils.set_repl_factor 'file', 3
|
351
|
+
#
|
352
|
+
def set_repl_factor(list, num, options = {})
|
353
|
+
opts = options.dup
|
354
|
+
list = [list].flatten
|
355
|
+
fu_log format('set_repl_factor %s %d',
|
356
|
+
list.join(' '), num) if opts.delete(:verbose)
|
357
|
+
c = client
|
358
|
+
list.each do |entry|
|
359
|
+
c.replication(entry, num, opts)
|
360
|
+
end
|
361
|
+
end
|
362
|
+
module_function :set_repl_factor
|
363
|
+
|
364
|
+
# Public: Set an access time of files
|
365
|
+
#
|
366
|
+
# list - file/directory name or list of them
|
367
|
+
# time - new access time
|
368
|
+
# options - :verbose
|
369
|
+
#
|
370
|
+
# Examples
|
371
|
+
#
|
372
|
+
# FileUtils.set_atime 'file', Time.now
|
373
|
+
#
|
374
|
+
def set_atime(list, time, options = {})
|
375
|
+
opts = options.dup
|
376
|
+
list = [list].flatten
|
377
|
+
time = time.to_i
|
378
|
+
fu_log format('set_atime %s %d',
|
379
|
+
list.join(' '), time) if opts.delete(:verbose)
|
380
|
+
c = client
|
381
|
+
list.each do |entry|
|
382
|
+
c.touch(entry, accesstime: time)
|
383
|
+
end
|
384
|
+
end
|
385
|
+
module_function :set_atime
|
386
|
+
|
387
|
+
# Public: Set a modification time of files
|
388
|
+
#
|
389
|
+
# list - file/directory name or list of them
|
390
|
+
# time - new modification time
|
391
|
+
# options - :verbose
|
392
|
+
#
|
393
|
+
# Examples
|
394
|
+
#
|
395
|
+
# FileUtils.set_mtime 'file', Time.now
|
396
|
+
#
|
397
|
+
def set_mtime(list, time, options = {})
|
398
|
+
opts = options.dup
|
399
|
+
list = [list].flatten
|
400
|
+
time = time.to_i
|
401
|
+
fu_log format('set_mtime %s %d',
|
402
|
+
list.join(' '), time) if opts.delete(:verbose)
|
403
|
+
c = client
|
404
|
+
list.each do |entry|
|
405
|
+
c.touch(entry, modificationtime: time)
|
406
|
+
end
|
407
|
+
end
|
408
|
+
module_function :set_mtime
|
409
|
+
|
410
|
+
# Internal: make functin private
|
411
|
+
def self.private_module_function(name)
|
412
|
+
module_function name
|
413
|
+
private_class_method name
|
414
|
+
end
|
415
|
+
|
416
|
+
@fileutils_output = $stderr
|
417
|
+
@fileutils_label = '[webhdfs]: '
|
418
|
+
# Internal: Logging
|
419
|
+
def fu_log(msg)
|
420
|
+
@fileutils_output ||= $stderr
|
421
|
+
@fileutils_label ||= ''
|
422
|
+
@fileutils_output.puts @fileutils_label + msg
|
423
|
+
end
|
424
|
+
private_module_function :fu_log
|
425
|
+
|
426
|
+
# Internal
|
427
|
+
def client
|
428
|
+
client = WebHDFS::Client.new(@fu_host, @fu_port, @fu_user, @fu_doas,
|
429
|
+
@fu_paddr, @fu_pport)
|
430
|
+
client.httpfs_mode = true if @fu_httpfs_mode
|
431
|
+
client.ssl = true if @fu_ssl
|
432
|
+
client.ssl_ca_file = @fu_ssl_ca_file if @fu_ssl_ca_file
|
433
|
+
client.ssl_verify_mode = @fu_ssl_verify_mode if @fu_ssl_verify_mode
|
434
|
+
client.kerberos = true if @fu_kerberos
|
435
|
+
client
|
436
|
+
end
|
437
|
+
private_module_function :client
|
438
|
+
end
|
439
|
+
end
|