webhdfs-rlz 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,249 @@
1
+ require 'net/http'
2
+ require 'uri'
3
+ require 'json'
4
+ require 'addressable/uri'
5
+
6
+ require_relative 'utilities'
7
+ require_relative 'exceptions'
8
+ require_relative 'proxy'
9
+ require_relative 'ssl'
10
+ require_relative 'kerberos'
11
+
12
+ module WebHDFS
13
+ class ClientV2
14
+ attr_accessor :host, :port, :username, :doas, :proxy
15
+ attr_accessor :open_timeout # default 30s (in ruby net/http)
16
+ attr_accessor :read_timeout # default 60s (in ruby net/http)
17
+ attr_accessor :httpfs_mode
18
+ attr_accessor :retry_known_errors # default false (not to retry)
19
+ attr_accessor :retry_times # default 1 (ignored when retry_known_errors is false)
20
+ attr_accessor :retry_interval # default 1 ([sec], ignored when retry_known_errors is false)
21
+ attr_accessor :ssl
22
+ attr_accessor :kerberos
23
+ attr_accessor :http_headers
24
+
25
+ # This hash table holds command options.
26
+ OPT_TABLE = {}.freeze # internal use only
27
+ OPT_TABLE['CREATE'] = %w(overwrite blocksize replication permission
28
+ buffersize data)
29
+ OPT_TABLE['APPEND'] = %w(buffersize data)
30
+ OPT_TABLE['OPEN'] = %w(offset length buffersize)
31
+ OPT_TABLE['MKDIRS'] = ['permission']
32
+ OPT_TABLE['DELETE'] = ['recursive']
33
+ OPT_TABLE['SETOWNER'] = %w(owner group)
34
+ OPT_TABLE['SETTIMES'] = %w(modificationtime accesstime)
35
+
36
+ REDIRECTED_OPERATIONS = %w(APPEND CREATE OPEN GETFILECHECKSUM).freeze
37
+
38
+ def initialize(host = 'localhost', port = 50_070, username = nil,
39
+ doas = nil, proxy_address = nil, proxy_port = nil,
40
+ http_headers = {})
41
+ @host = host
42
+ @port = port
43
+ @username = username
44
+ @doas = doas
45
+
46
+ @proxy = WebHDFS::Proxy.new(proxy_address, proxy_port) if proxy_address && proxy_port
47
+
48
+ @retry_known_errors = false
49
+ @retry_times = @retry_interval = 1
50
+
51
+ @httpfs_mode = false
52
+
53
+ @ssl = nil
54
+
55
+ @kerberos = nil
56
+ @http_headers = http_headers
57
+ end
58
+
59
+ def request_options
60
+ {
61
+ username: @username,
62
+ doas: @doas,
63
+ proxy: @proxy,
64
+ ssl: @ssl,
65
+ kerberos: @kerberos,
66
+ open_timeout: @open_timeout, read_timeout: @read_timeout,
67
+ retry_known_errors: @retry_known_errors,
68
+ retry_times: @retry_times, retry_interval: @retry_interval
69
+ }
70
+ end
71
+
72
+ # curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=CREATE
73
+ # [&overwrite=<true|false>][&blocksize=<LONG>]
74
+ # [&replication=<SHORT>]
75
+ # [&permission=<OCTAL>][&buffersize=<INT>]"
76
+ def create(path, body, options = {})
77
+ options = options.merge('data' => 'true') if @httpfs_mode
78
+ WebHDFS.check_options(options, OPT_TABLE['CREATE'])
79
+ res = operate_requests('PUT', path, 'CREATE', options, body)
80
+ res.code == '201'
81
+ end
82
+
83
+ # curl -i -X POST "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=
84
+ # APPEND[&buffersize=<INT>]"
85
+ def append(path, body, options = {})
86
+ options = options.merge('data' => 'true') if @httpfs_mode
87
+ WebHDFS.check_options(options, OPT_TABLE['APPEND'])
88
+ res = operate_requests('POST', path, 'APPEND', options, body)
89
+ res.code == '200'
90
+ end
91
+
92
+ # curl -i -L "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=OPEN
93
+ # [&offset=<LONG>][&length=<LONG>][&buffersize=<INT>]"
94
+ def read(path, options = {})
95
+ WebHDFS.check_options(options, OPT_TABLE['OPEN'])
96
+ res = operate_requests('GET', path, 'OPEN', options)
97
+ res.body
98
+ end
99
+
100
+ alias open read
101
+
102
+ # curl -i -X PUT "http://<HOST>:<PORT>/<PATH>?op=
103
+ # MKDIRS[&permission=<OCTAL>]"
104
+ def mkdir(path, options = {})
105
+ WebHDFS.check_options(options, OPT_TABLE['MKDIRS'])
106
+ res = operate_requests('PUT', path, 'MKDIRS', options)
107
+ WebHDFS.check_success_json(res, 'boolean')
108
+ end
109
+
110
+ alias mkdirs mkdir
111
+
112
+ # curl -i -X PUT "<HOST>:<PORT>/webhdfs/v1/<PATH>?op=
113
+ # RENAME&destination=<PATH>"
114
+ def rename(path, dest, options = {})
115
+ WebHDFS.check_options(options, OPT_TABLE['RENAME'])
116
+ dest = '/' + dest unless dest.start_with?('/')
117
+ res = operate_requests('PUT', path, 'RENAME',
118
+ options.merge('destination' => dest))
119
+ WebHDFS.check_success_json(res, 'boolean')
120
+ end
121
+
122
+ # curl -i -X DELETE "http://<host>:<port>/webhdfs/v1/<path>?op=DELETE
123
+ # [&recursive=<true|false>]"
124
+ def delete(path, options = {})
125
+ WebHDFS.check_options(options, OPT_TABLE['DELETE'])
126
+ res = operate_requests('DELETE', path, 'DELETE', options)
127
+ WebHDFS.check_success_json(res, 'boolean')
128
+ end
129
+
130
+ # curl -i "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=GETFILESTATUS"
131
+ def stat(path, options = {})
132
+ WebHDFS.check_options(options, OPT_TABLE['GETFILESTATUS'])
133
+ res = operate_requests('GET', path, 'GETFILESTATUS', options)
134
+ WebHDFS.check_success_json(res, 'FileStatus')
135
+ end
136
+ alias getfilestatus stat
137
+
138
+ # curl -i "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=LISTSTATUS"
139
+ def list(path, options = {})
140
+ WebHDFS.check_options(options, OPT_TABLE['LISTSTATUS'])
141
+ res = operate_requests('GET', path, 'LISTSTATUS', options)
142
+ WebHDFS.check_success_json(res, 'FileStatuses')['FileStatus']
143
+ end
144
+ alias liststatus list
145
+
146
+ # curl -i "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=GETCONTENTSUMMARY"
147
+ def content_summary(path, options = {})
148
+ WebHDFS.check_options(options, OPT_TABLE['GETCONTENTSUMMARY'])
149
+ res = operate_requests('GET', path, 'GETCONTENTSUMMARY', options)
150
+ WebHDFS.check_success_json(res, 'ContentSummary')
151
+ end
152
+ alias getcontentsummary content_summary
153
+
154
+ # curl -i "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=GETFILECHECKSUM"
155
+ def checksum(path, options = {})
156
+ WebHDFS.check_options(options, OPT_TABLE['GETFILECHECKSUM'])
157
+ res = operate_requests('GET', path, 'GETFILECHECKSUM', options)
158
+ WebHDFS.check_success_json(res, 'FileChecksum')
159
+ end
160
+ alias getfilechecksum checksum
161
+
162
+ # curl -i "http://<HOST>:<PORT>/webhdfs/v1/?op=GETHOMEDIRECTORY"
163
+ def homedir(options = {})
164
+ WebHDFS.check_options(options, OPT_TABLE['GETHOMEDIRECTORY'])
165
+ res = operate_requests('GET', '/', 'GETHOMEDIRECTORY', options)
166
+ WebHDFS.check_success_json(res, 'Path')
167
+ end
168
+ alias gethomedirectory homedir
169
+
170
+ # curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=SETPERMISSION
171
+ # [&permission=<OCTAL>]"
172
+ def chmod(path, mode, options = {})
173
+ WebHDFS.check_options(options, OPT_TABLE['SETPERMISSION'])
174
+ res = operate_requests('PUT', path, 'SETPERMISSION',
175
+ options.merge('permission' => mode))
176
+ res.code == '200'
177
+ end
178
+ alias setpermission chmod
179
+
180
+ # curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=SETOWNER
181
+ # [&owner=<USER>][&group=<GROUP>]"
182
+ def chown(path, options = {})
183
+ WebHDFS.check_options(options, OPT_TABLE['SETOWNER'])
184
+ unless options.key?('owner') || options.key?('group') ||
185
+ options.key?(:owner) || options.key?(:group)
186
+ raise ArgumentError, "'chown' needs at least one of owner or group"
187
+ end
188
+ res = operate_requests('PUT', path, 'SETOWNER', options)
189
+ res.code == '200'
190
+ end
191
+
192
+ alias setowner chown
193
+
194
+ # curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=SETREPLICATION
195
+ # [&replication=<SHORT>]"
196
+ def replication(path, replnum, options = {})
197
+ WebHDFS.check_options(options, OPT_TABLE['SETREPLICATION'])
198
+ res = operate_requests('PUT', path, 'SETREPLICATION',
199
+ options.merge('replication' => replnum.to_s))
200
+ WebHDFS.check_success_json(res, 'boolean')
201
+ end
202
+ alias setreplication replication
203
+
204
+ # curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=SETTIMES
205
+ # [&modificationtime=<TIME>][&accesstime=<TIME>]"
206
+ # motidicationtime: radix-10 logn integer
207
+ # accesstime: radix-10 logn integer
208
+ def touch(path, options = {})
209
+ WebHDFS.check_options(options, OPT_TABLE['SETTIMES'])
210
+ unless options.key?('modificationtime') || options.key?('accesstime') ||
211
+ options.key?(:modificationtime) || options.key?(:accesstime)
212
+ raise ArgumentError, "'chown' needs at least one of " \
213
+ 'modificationtime or accesstime'
214
+ end
215
+ res = operate_requests('PUT', path, 'SETTIMES', options)
216
+ res.code == '200'
217
+ end
218
+
219
+ alias settimes touch
220
+
221
+ def operate_requests(method, path, op, params = {}, payload = nil)
222
+ request = WebHDFS::Request.new(@host, @port, request_options)
223
+ if !@httpfs_mode && REDIRECTED_OPERATIONS.include?(op)
224
+ response = request.execute(path, method, nil, nil, op, params, nil)
225
+ unless response.is_a?(Net::HTTPRedirection) && response['location']
226
+ msg = 'NameNode returns non-redirection (or without location' \
227
+ " header), code:#{res.code}, body:#{res.body}."
228
+ raise WebHDFS::RequestFailedError, msg
229
+ end
230
+ uri = URI.parse(response['location'])
231
+ rpath = if uri.query
232
+ uri.path + '?' + uri.query
233
+ else
234
+ uri.path
235
+ end
236
+ request = WebHDFS::Request.new(uri.host, uri.port, request_options)
237
+ request.execute(rpath, method,
238
+ { 'Content-Type' => 'application/octet-stream' },
239
+ payload, nil, {})
240
+ elsif @httpfs_mode && !payload.nil?
241
+ request.execute(path, method,
242
+ { 'Content-Type' => 'application/octet-stream' },
243
+ payload, op, params)
244
+ else
245
+ request.execute(path, method, nil, payload, op, params)
246
+ end
247
+ end
248
+ end
249
+ end
@@ -0,0 +1,15 @@
1
+ module WebHDFS
2
+ class Error < StandardError; end
3
+
4
+ class FileNotFoundError < WebHDFS::Error; end
5
+
6
+ class IOError < WebHDFS::Error; end
7
+ class SecurityError < WebHDFS::Error; end
8
+
9
+ class ClientError < WebHDFS::Error; end
10
+ class ServerError < WebHDFS::Error; end
11
+
12
+ class RequestFailedError < WebHDFS::Error; end
13
+
14
+ class KerberosError < WebHDFS::Error; end
15
+ end
@@ -0,0 +1,439 @@
1
+ require_relative 'client'
2
+
3
+ module WebHDFS
4
+ module FileUtils
5
+ # Those values hold NameNode location
6
+ @fu_host = 'localhost'
7
+ @fu_port = 50_070
8
+ @fu_user = nil
9
+ @fu_doas = nil
10
+ @fu_httpfs_mode = false
11
+ @fu_ssl = false
12
+ @fu_ssl_ca_file = nil
13
+ @fu_ssl_verify_mode = nil
14
+ @fu_kerberos = false
15
+
16
+ # Public: Set hostname and port number of WebHDFS
17
+ #
18
+ # host - hostname
19
+ # port - port
20
+ # user - username
21
+ # doas - proxy user name
22
+ # proxy_address - address of the net http proxy to use
23
+ # proxy_port - port of the net http proxy to use
24
+ #
25
+ # Examples
26
+ #
27
+ # FileUtils.set_server 'localhost', 50070
28
+ #
29
+ def set_server(host, port, user = nil, doas = nil, proxy_address = nil,
30
+ proxy_port = nil)
31
+ @fu_host = host
32
+ @fu_port = port
33
+ @fu_user = user
34
+ @fu_doas = doas
35
+ @fu_paddr = proxy_address
36
+ @fu_pport = proxy_port
37
+ end
38
+ module_function :set_server
39
+
40
+ # Public: Set httpfs mode enable/disable
41
+ #
42
+ # mode - boolean (default true)
43
+ #
44
+ # Examples
45
+ #
46
+ # FileUtils.set_httpfs_mode
47
+ #
48
+ def set_httpfs_mode(mode = true)
49
+ @fu_httpfs_mode = mode
50
+ end
51
+ module_function :set_httpfs_mode
52
+
53
+ # Public: Set ssl enable/disable
54
+ #
55
+ # mode - boolean (default true)
56
+ #
57
+ # Examples
58
+ #
59
+ # FileUtils.set_ssl
60
+ #
61
+ def set_ssl(mode = true)
62
+ @fu_ssl = mode
63
+ end
64
+ module_function :set_ssl
65
+
66
+ # Public: Set ssl ca_file
67
+ #
68
+ # ca_file - string
69
+ #
70
+ # Examples
71
+ #
72
+ # FileUtils.set_ca_file("/path/to/ca_file.pem")
73
+ #
74
+ def set_ssl_ca_file(ca_file)
75
+ @fu_ssl_ca_file = ca_file
76
+ end
77
+ module_function :set_ssl_ca_file
78
+
79
+ # Public: Set ssl verify mode
80
+ #
81
+ # mode - :none or :peer
82
+ #
83
+ # Examples
84
+ #
85
+ # FileUtils.set_ssl_verify_mode(:peer)
86
+ #
87
+ def set_ssl_verify_mode(mode)
88
+ @fu_ssl_verify_mode = mode
89
+ end
90
+ module_function :set_ssl_verify_mode
91
+
92
+ # Public: Set kerberos authentication enable/disable
93
+ #
94
+ # mode - boolean (default true)
95
+ #
96
+ # Examples
97
+ #
98
+ # FileUtils.set_kerberos
99
+ #
100
+ def set_kerberos(mode = true)
101
+ @fu_kerberos = mode
102
+ end
103
+ module_function :set_kerberos
104
+
105
+ # Public: Copy local file into HDFS
106
+ #
107
+ # file - local file path
108
+ # path - HDFS file path
109
+ # options :overwrite, :blocksize, :replication, :mode, :buffersize, :verbose
110
+ #
111
+ # Examples
112
+ #
113
+ # FileUtils.copy_from_local 'local_file', 'remote_file'
114
+ #
115
+ def copy_from_local(file, path, options = {})
116
+ opts = options.dup
117
+ fu_log "copy_from_local local=#{file} " \
118
+ "hdfs=#{path}" if opts.delete(:verbose)
119
+ mode = opts.delete(:mode)
120
+ if mode
121
+ mode = format('%03o', mode) if mode.is_a? Integer
122
+ else
123
+ mode = '644'
124
+ end
125
+ opts[:permission] = mode
126
+ opts[:overwrite] ||= true
127
+
128
+ client.create(path, File.new(file, 'rb').read(File.size(file)), opts)
129
+ end
130
+ module_function :copy_from_local
131
+
132
+ # Public: Copy local file into HDFS with IOStream
133
+ #
134
+ # file - local file IO handle
135
+ # path - HDFS file path
136
+ # options :overwrite, :blocksize, :replication, :mode, :buffersize, :verbose
137
+ #
138
+ # Examples
139
+ #
140
+ # FileUtils.copy_from_local_via_stream 'local_file_IO_handle', 'remote_file'
141
+ #
142
+ def copy_from_local_via_stream(file, path, options = {})
143
+ opts = options.dup
144
+ fu_log "copy_from_local_via_stream local=#{file} " \
145
+ "hdfs=#{path}" if opts.delete(:verbose)
146
+ mode = opts.delete(:mode)
147
+ if mode
148
+ mode = format('%03o', mode) if mode.is_a? Integer
149
+ else
150
+ mode = '644'
151
+ end
152
+ opts[:permission] = mode
153
+ opts[:overwrite] ||= true
154
+
155
+ client.create(path, File.new(file, 'rb'), opts)
156
+ end
157
+ module_function :copy_from_local_via_stream
158
+
159
+ # Public: Copy remote HDFS file into local
160
+ #
161
+ # path - HDFS file path
162
+ # file - local file path
163
+ # options - :offset, :length, :buffersize, :verbose
164
+ #
165
+ # Examples
166
+ #
167
+ # FileUtils.copy_to_local 'remote_file', 'local_file'
168
+ #
169
+ def copy_to_local(path, file, options = {})
170
+ opts = options.dup
171
+ fu_log "copy_to_local hdfs=#{path} local=#{file}" if opts.delete(:verbose)
172
+ File.open(file, 'wb') do |f|
173
+ f.write client.read(path, opts)
174
+ end
175
+ end
176
+ module_function :copy_to_local
177
+
178
+ # Public: Append to HDFS file
179
+ #
180
+ # path - HDFS file path
181
+ # body - contents
182
+ # options - :buffersize, :verbose
183
+ #
184
+ # Examples
185
+ #
186
+ # FileUtils.append 'remote_path', 'contents'
187
+ #
188
+ def append(path, body, options = {})
189
+ opts = options.dup
190
+ fu_log "append #{body.bytesize} bytes to #{path}" if opts.delete(:verbose)
191
+ client.append(path, body, opts)
192
+ end
193
+ module_function :append
194
+
195
+ # Public: Create one or more directories.
196
+ #
197
+ # list - directory name, or list of them
198
+ # options - :mode, :verbose
199
+ #
200
+ # Examples
201
+ #
202
+ # FileUtils.mkdir 'test'
203
+ # FileUtils.mkdir %w( tmp data )
204
+ # FileUtils.mkdir 'tmp', :mode => 0700
205
+ #
206
+ def mkdir(list, options = {})
207
+ opts = options.dup
208
+ list = [list].flatten
209
+ fu_log "mkdir #{format('-m %03o ', options[:mode]) if options[:mode]}" \
210
+ "#{list.join ' '}" if opts.delete(:verbose)
211
+ mode = opts[:mode]
212
+ if mode
213
+ mode = format('0%03o', mode) if mode.is_a? Integer
214
+ else
215
+ mode = '0755'
216
+ end
217
+ c = client
218
+ list.each do |dir|
219
+ c.mkdir(dir, permission: mode)
220
+ end
221
+ end
222
+ module_function :mkdir
223
+
224
+ # Public: Create one or more directories recursively.
225
+ #
226
+ # list - directory name, or list of them
227
+ # options - :mode, :verbose
228
+ #
229
+ # Examples
230
+ #
231
+ # FileUtils.mkdir_p 'dir/subdir'
232
+ # FileUtils.mkdir_p %w( tmp data )
233
+ # FileUtils.mkdir_p 'dir/subdir', :mode => 0700
234
+ #
235
+ alias mkdir_p mkdir
236
+ module_function :mkdir_p
237
+
238
+ # Public: Remove one or more directories or files.
239
+ #
240
+ # list - directory name, or list of them
241
+ # options - :recursive, :verbose
242
+ #
243
+ # Examples
244
+ #
245
+ # FileUtils.rm 'dir'
246
+ # FileUtils.rm %w( tmp data )
247
+ # FileUtils.rm 'dir', :recursive => true
248
+ #
249
+ def rm(list, options = {})
250
+ opts = options.dup
251
+ list = [list].flatten
252
+ fu_log "rm #{list.join ' '}" if opts.delete(:verbose)
253
+ c = client
254
+ list.each do |dir|
255
+ c.delete(dir, recursive: opts[:recursive] || false)
256
+ end
257
+ end
258
+ module_function :rm
259
+
260
+ # Public: Remove one or more directories/files recursively.
261
+ #
262
+ # list - directory name, or list of them
263
+ # options - :verbose
264
+ #
265
+ # Examples
266
+ #
267
+ # FileUtils.rmr 'dir'
268
+ # FileUtils.rmr %w( tmp data )
269
+ # FileUtils.rmr 'dir'
270
+ #
271
+ def rmr(list, options = {})
272
+ rm(list, options.merge(recursive: true))
273
+ end
274
+ module_function :rmr
275
+
276
+ # Public: Rename a file or directory.
277
+ #
278
+ # src - from
279
+ # dst - to
280
+ # options - :verbose
281
+ #
282
+ # Examples
283
+ #
284
+ # FileUtils.rename 'from', 'to'
285
+ #
286
+ def rename(src, dst, options = {})
287
+ opts = options.dup
288
+ fu_log "rename #{src} #{dst}" if opts.delete(:verbose)
289
+ client.rename(src, dst, opts)
290
+ end
291
+ module_function :rename
292
+
293
+ # Public: Change permission of one or more directories/files.
294
+ #
295
+ # mode - permission
296
+ # list - file/directory name or list of them.
297
+ # options - :verbose
298
+ #
299
+ # Examples
300
+ #
301
+ # FileUtils.chmod 0755, 'dir'
302
+ # FileUtils.chmod 0644, 'file'
303
+ #
304
+ def chmod(mode, list, options = {})
305
+ opts = options.dup
306
+ list = [list].flatten
307
+ fu_log format('chmod %o %s', mode,
308
+ list.join(' ')) if opts.delete(:verbose)
309
+ mode = format('%03o', mode) if mode.is_a? Integer
310
+ c = client
311
+ list.each do |entry|
312
+ c.chmod(entry, mode, opts)
313
+ end
314
+ end
315
+ module_function :chmod
316
+
317
+ # Public: Change an ownership of one or more directories/files.
318
+ #
319
+ # user - username
320
+ # group - groupname
321
+ # list - file/directory name or list of them
322
+ # options - :verbose
323
+ #
324
+ # Examples
325
+ #
326
+ # FileUtils.chmod 0755, 'dir'
327
+ # FileUtils.chmod 0644, 'file'
328
+ #
329
+ def chown(user, group, list, options = {})
330
+ opts = options.dup
331
+ list = [list].flatten
332
+ fu_log format('chown %s%s',
333
+ [user, group].compact.join(':') + ' ',
334
+ list.join(' ')) if opts.delete(:verbose)
335
+ c = client
336
+ list.each do |entry|
337
+ c.chown(entry, owner: user, group: group)
338
+ end
339
+ end
340
+ module_function :chown
341
+
342
+ # Public: Set a replication factor of files
343
+ #
344
+ # list - file/directory name or list of them
345
+ # num - replication factor
346
+ # options - :verbose
347
+ #
348
+ # Examples
349
+ #
350
+ # FileUtils.set_repl_factor 'file', 3
351
+ #
352
+ def set_repl_factor(list, num, options = {})
353
+ opts = options.dup
354
+ list = [list].flatten
355
+ fu_log format('set_repl_factor %s %d',
356
+ list.join(' '), num) if opts.delete(:verbose)
357
+ c = client
358
+ list.each do |entry|
359
+ c.replication(entry, num, opts)
360
+ end
361
+ end
362
+ module_function :set_repl_factor
363
+
364
+ # Public: Set an access time of files
365
+ #
366
+ # list - file/directory name or list of them
367
+ # time - new access time
368
+ # options - :verbose
369
+ #
370
+ # Examples
371
+ #
372
+ # FileUtils.set_atime 'file', Time.now
373
+ #
374
+ def set_atime(list, time, options = {})
375
+ opts = options.dup
376
+ list = [list].flatten
377
+ time = time.to_i
378
+ fu_log format('set_atime %s %d',
379
+ list.join(' '), time) if opts.delete(:verbose)
380
+ c = client
381
+ list.each do |entry|
382
+ c.touch(entry, accesstime: time)
383
+ end
384
+ end
385
+ module_function :set_atime
386
+
387
+ # Public: Set a modification time of files
388
+ #
389
+ # list - file/directory name or list of them
390
+ # time - new modification time
391
+ # options - :verbose
392
+ #
393
+ # Examples
394
+ #
395
+ # FileUtils.set_mtime 'file', Time.now
396
+ #
397
+ def set_mtime(list, time, options = {})
398
+ opts = options.dup
399
+ list = [list].flatten
400
+ time = time.to_i
401
+ fu_log format('set_mtime %s %d',
402
+ list.join(' '), time) if opts.delete(:verbose)
403
+ c = client
404
+ list.each do |entry|
405
+ c.touch(entry, modificationtime: time)
406
+ end
407
+ end
408
+ module_function :set_mtime
409
+
410
+ # Internal: make functin private
411
+ def self.private_module_function(name)
412
+ module_function name
413
+ private_class_method name
414
+ end
415
+
416
+ @fileutils_output = $stderr
417
+ @fileutils_label = '[webhdfs]: '
418
+ # Internal: Logging
419
+ def fu_log(msg)
420
+ @fileutils_output ||= $stderr
421
+ @fileutils_label ||= ''
422
+ @fileutils_output.puts @fileutils_label + msg
423
+ end
424
+ private_module_function :fu_log
425
+
426
+ # Internal
427
+ def client
428
+ client = WebHDFS::Client.new(@fu_host, @fu_port, @fu_user, @fu_doas,
429
+ @fu_paddr, @fu_pport)
430
+ client.httpfs_mode = true if @fu_httpfs_mode
431
+ client.ssl = true if @fu_ssl
432
+ client.ssl_ca_file = @fu_ssl_ca_file if @fu_ssl_ca_file
433
+ client.ssl_verify_mode = @fu_ssl_verify_mode if @fu_ssl_verify_mode
434
+ client.kerberos = true if @fu_kerberos
435
+ client
436
+ end
437
+ private_module_function :client
438
+ end
439
+ end