webhdfs 0.5.1 → 0.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/README.md +8 -0
- data/VERSION +1 -1
- data/lib/webhdfs.rb +1 -0
- data/lib/webhdfs/backport.rb +30 -0
- data/lib/webhdfs/client_v1.rb +24 -1
- data/test/test_helper.rb +1 -0
- data/test/webhdfs/fileutils.rb +3 -2
- metadata +16 -25
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: cdf1d7bc03934462d12ea16db3212bdc55ab3d1a
|
4
|
+
data.tar.gz: 11c21a5e4a5894e66179406f207f3e53f6eb87bd
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: ebe43a9a4ede658b61aab20e8dfdf8caf53b5a0a820ed2bd28717872934d36ed21ca6bf8c767c88adedf9005341408f5f3f800d670c165724be07c7577968732
|
7
|
+
data.tar.gz: 3990c04379334c95b3320b7582cf9fe283ae5ed9996c4ba2c6e55acf4a6ab996531781c60fa2e761dfa0bfcf2afe718b8f7739d97323df61113f26f889a3d611
|
data/README.md
CHANGED
@@ -44,6 +44,14 @@ To get status or list of files and directories:
|
|
44
44
|
|
45
45
|
And, 'content_summary', 'checksum', 'homedir', 'chmod', 'chown', 'replication' and 'touch' methods available.
|
46
46
|
|
47
|
+
For known errors, automated retries are available. Set `retry_known_errors` option as true.
|
48
|
+
|
49
|
+
#### To retry for LeaseExpiredException automatically
|
50
|
+
client.retry_known_errors = true
|
51
|
+
|
52
|
+
# client.retry_interval = 1 # [sec], default: 1
|
53
|
+
# clinet.retry_times = 1 # [times], default: 1
|
54
|
+
|
47
55
|
### WebHDFS::FileUtils
|
48
56
|
|
49
57
|
require 'webhdfs/fileutils'
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.5.
|
1
|
+
0.5.2
|
data/lib/webhdfs.rb
CHANGED
@@ -0,0 +1,30 @@
|
|
1
|
+
if RUBY_VERSION =~ /^1\.8\./
|
2
|
+
require 'cgi'
|
3
|
+
def require_relative(relative_feature)
|
4
|
+
file = caller.first.split(/:\d/,2).first
|
5
|
+
raise LoadError, "require_relative is called in #{$1}" if /\A\((.*)\)/ =~ file
|
6
|
+
require File.expand_path(relative_feature, File.dirname(file))
|
7
|
+
end
|
8
|
+
|
9
|
+
module URI
|
10
|
+
def self.encode_www_form(enum)
|
11
|
+
enum.map do |k,v|
|
12
|
+
if v.nil?
|
13
|
+
CGI::escape(k)
|
14
|
+
elsif v.respond_to?(:to_ary)
|
15
|
+
v.to_ary.map do |w|
|
16
|
+
str = CGI::escape(k)
|
17
|
+
unless w.nil?
|
18
|
+
str << '='
|
19
|
+
str << CGI::escape(w)
|
20
|
+
end
|
21
|
+
end.join('&')
|
22
|
+
else
|
23
|
+
str = CGI::escape(k.to_s)
|
24
|
+
str << '='
|
25
|
+
str << CGI::escape(v.to_s)
|
26
|
+
end
|
27
|
+
end.join('&')
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
data/lib/webhdfs/client_v1.rb
CHANGED
@@ -9,17 +9,24 @@ module WebHDFS
|
|
9
9
|
|
10
10
|
# This hash table holds command options.
|
11
11
|
OPT_TABLE = {} # internal use only
|
12
|
+
KNOWN_ERRORS = ['LeaseExpiredException'].freeze
|
12
13
|
|
13
14
|
attr_accessor :host, :port, :username, :doas
|
14
15
|
attr_accessor :open_timeout # default 30s (in ruby net/http)
|
15
16
|
attr_accessor :read_timeout # default 60s (in ruby net/http)
|
16
17
|
attr_accessor :httpfs_mode
|
18
|
+
attr_accessor :retry_known_errors # default false (not to retry)
|
19
|
+
attr_accessor :retry_times # default 1 (ignored when retry_known_errors is false)
|
20
|
+
attr_accessor :retry_interval # default 1 ([sec], ignored when retry_known_errors is false)
|
17
21
|
|
18
22
|
def initialize(host='localhost', port=50070, username=nil, doas=nil)
|
19
23
|
@host = host
|
20
24
|
@port = port
|
21
25
|
@username = username
|
22
26
|
@doas = doas
|
27
|
+
@retry_known_errors = false
|
28
|
+
@retry_times = 1
|
29
|
+
@retry_interval = 1
|
23
30
|
|
24
31
|
@httpfs_mode = false
|
25
32
|
end
|
@@ -245,7 +252,7 @@ module WebHDFS
|
|
245
252
|
# IOException 403 Forbidden
|
246
253
|
# FileNotFoundException 404 Not Found
|
247
254
|
# RumtimeException 500 Internal Server Error
|
248
|
-
def request(host, port, method, path, op=nil, params={}, payload=nil, header=nil)
|
255
|
+
def request(host, port, method, path, op=nil, params={}, payload=nil, header=nil, retries=0)
|
249
256
|
conn = Net::HTTP.new(host, port)
|
250
257
|
conn.open_timeout = @open_timeout if @open_timeout
|
251
258
|
conn.read_timeout = @read_timeout if @read_timeout
|
@@ -269,6 +276,22 @@ module WebHDFS
|
|
269
276
|
else
|
270
277
|
'Response body is empty...'
|
271
278
|
end
|
279
|
+
|
280
|
+
if @retry_known_errors && retries < @retry_times
|
281
|
+
detail = nil
|
282
|
+
if message =~ /^\{\\"RemoteException\\":\\\{/
|
283
|
+
begin
|
284
|
+
detail = JSON.parse(message)
|
285
|
+
rescue
|
286
|
+
# ignore broken json response body
|
287
|
+
end
|
288
|
+
end
|
289
|
+
if detail && detail['RemoteException'] && KNOWN_ERRORS.include?(detail['RemoteException']['exception'])
|
290
|
+
sleep @retry_interval if @retry_interval > 0
|
291
|
+
return request(host, port, method, path, op, params, payload, header, retries+1)
|
292
|
+
end
|
293
|
+
end
|
294
|
+
|
272
295
|
case res.code
|
273
296
|
when '400'
|
274
297
|
raise WebHDFS::ClientError, message
|
data/test/test_helper.rb
CHANGED
data/test/webhdfs/fileutils.rb
CHANGED
@@ -2,7 +2,8 @@ require 'test_helper'
|
|
2
2
|
|
3
3
|
class FileUtilsTest < Test::Unit::TestCase
|
4
4
|
def setup
|
5
|
-
require 'webhdfs'
|
5
|
+
require 'lib/webhdfs'
|
6
|
+
require 'lib/webhdfs/fileutils'
|
6
7
|
end
|
7
8
|
|
8
9
|
def test_copy_from_local
|
@@ -43,7 +44,7 @@ class FileUtilsTest < Test::Unit::TestCase
|
|
43
44
|
|
44
45
|
def test_set_repl_factor
|
45
46
|
WebHDFS::FileUtils.mkdir('foo', :mode => 0777, :verbose => true)
|
46
|
-
WebHDFS::FileUtils.set_repl_factor('foo',
|
47
|
+
WebHDFS::FileUtils.set_repl_factor('foo', 2)
|
47
48
|
WebHDFS::FileUtils.rm('foo', :verbose => true)
|
48
49
|
end
|
49
50
|
|
metadata
CHANGED
@@ -1,78 +1,69 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: webhdfs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
5
|
-
prerelease:
|
4
|
+
version: 0.5.2
|
6
5
|
platform: ruby
|
7
6
|
authors:
|
8
7
|
- Kazuki Ohta
|
9
8
|
autorequire:
|
10
9
|
bindir: bin
|
11
10
|
cert_chain: []
|
12
|
-
date:
|
11
|
+
date: 2013-05-08 00:00:00.000000000 Z
|
13
12
|
dependencies:
|
14
13
|
- !ruby/object:Gem::Dependency
|
15
14
|
name: rake
|
16
15
|
requirement: !ruby/object:Gem::Requirement
|
17
|
-
none: false
|
18
16
|
requirements:
|
19
|
-
- -
|
17
|
+
- - '>='
|
20
18
|
- !ruby/object:Gem::Version
|
21
19
|
version: 0.9.2
|
22
20
|
type: :development
|
23
21
|
prerelease: false
|
24
22
|
version_requirements: !ruby/object:Gem::Requirement
|
25
|
-
none: false
|
26
23
|
requirements:
|
27
|
-
- -
|
24
|
+
- - '>='
|
28
25
|
- !ruby/object:Gem::Version
|
29
26
|
version: 0.9.2
|
30
27
|
- !ruby/object:Gem::Dependency
|
31
28
|
name: rdoc
|
32
29
|
requirement: !ruby/object:Gem::Requirement
|
33
|
-
none: false
|
34
30
|
requirements:
|
35
|
-
- -
|
31
|
+
- - '>='
|
36
32
|
- !ruby/object:Gem::Version
|
37
33
|
version: '3.12'
|
38
34
|
type: :development
|
39
35
|
prerelease: false
|
40
36
|
version_requirements: !ruby/object:Gem::Requirement
|
41
|
-
none: false
|
42
37
|
requirements:
|
43
|
-
- -
|
38
|
+
- - '>='
|
44
39
|
- !ruby/object:Gem::Version
|
45
40
|
version: '3.12'
|
46
41
|
- !ruby/object:Gem::Dependency
|
47
42
|
name: simplecov
|
48
43
|
requirement: !ruby/object:Gem::Requirement
|
49
|
-
none: false
|
50
44
|
requirements:
|
51
|
-
- -
|
45
|
+
- - '>='
|
52
46
|
- !ruby/object:Gem::Version
|
53
47
|
version: 0.5.4
|
54
48
|
type: :development
|
55
49
|
prerelease: false
|
56
50
|
version_requirements: !ruby/object:Gem::Requirement
|
57
|
-
none: false
|
58
51
|
requirements:
|
59
|
-
- -
|
52
|
+
- - '>='
|
60
53
|
- !ruby/object:Gem::Version
|
61
54
|
version: 0.5.4
|
62
55
|
- !ruby/object:Gem::Dependency
|
63
56
|
name: rr
|
64
57
|
requirement: !ruby/object:Gem::Requirement
|
65
|
-
none: false
|
66
58
|
requirements:
|
67
|
-
- -
|
59
|
+
- - '>='
|
68
60
|
- !ruby/object:Gem::Version
|
69
61
|
version: 1.0.0
|
70
62
|
type: :development
|
71
63
|
prerelease: false
|
72
64
|
version_requirements: !ruby/object:Gem::Requirement
|
73
|
-
none: false
|
74
65
|
requirements:
|
75
|
-
- -
|
66
|
+
- - '>='
|
76
67
|
- !ruby/object:Gem::Version
|
77
68
|
version: 1.0.0
|
78
69
|
description: Ruby WebHDFS/HttpFs client
|
@@ -89,6 +80,7 @@ files:
|
|
89
80
|
- Rakefile
|
90
81
|
- VERSION
|
91
82
|
- lib/webhdfs.rb
|
83
|
+
- lib/webhdfs/backport.rb
|
92
84
|
- lib/webhdfs/client.rb
|
93
85
|
- lib/webhdfs/client_v1.rb
|
94
86
|
- lib/webhdfs/exceptions.rb
|
@@ -98,27 +90,26 @@ files:
|
|
98
90
|
- webhdfs.gemspec
|
99
91
|
homepage: ''
|
100
92
|
licenses: []
|
93
|
+
metadata: {}
|
101
94
|
post_install_message:
|
102
95
|
rdoc_options: []
|
103
96
|
require_paths:
|
104
97
|
- lib
|
105
98
|
required_ruby_version: !ruby/object:Gem::Requirement
|
106
|
-
none: false
|
107
99
|
requirements:
|
108
|
-
- -
|
100
|
+
- - '>='
|
109
101
|
- !ruby/object:Gem::Version
|
110
102
|
version: '0'
|
111
103
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
112
|
-
none: false
|
113
104
|
requirements:
|
114
|
-
- -
|
105
|
+
- - '>='
|
115
106
|
- !ruby/object:Gem::Version
|
116
107
|
version: '0'
|
117
108
|
requirements: []
|
118
109
|
rubyforge_project:
|
119
|
-
rubygems_version:
|
110
|
+
rubygems_version: 2.0.0
|
120
111
|
signing_key:
|
121
|
-
specification_version:
|
112
|
+
specification_version: 4
|
122
113
|
summary: Ruby WebHDFS/HttpFs client
|
123
114
|
test_files:
|
124
115
|
- test/test_helper.rb
|