webhdfs 0.5.1 → 0.5.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: cdf1d7bc03934462d12ea16db3212bdc55ab3d1a
4
+ data.tar.gz: 11c21a5e4a5894e66179406f207f3e53f6eb87bd
5
+ SHA512:
6
+ metadata.gz: ebe43a9a4ede658b61aab20e8dfdf8caf53b5a0a820ed2bd28717872934d36ed21ca6bf8c767c88adedf9005341408f5f3f800d670c165724be07c7577968732
7
+ data.tar.gz: 3990c04379334c95b3320b7582cf9fe283ae5ed9996c4ba2c6e55acf4a6ab996531781c60fa2e761dfa0bfcf2afe718b8f7739d97323df61113f26f889a3d611
data/README.md CHANGED
@@ -44,6 +44,14 @@ To get status or list of files and directories:
44
44
 
45
45
  And, 'content_summary', 'checksum', 'homedir', 'chmod', 'chown', 'replication' and 'touch' methods available.
46
46
 
47
+ For known errors, automated retries are available. Set `retry_known_errors` option as true.
48
+
49
+ #### To retry for LeaseExpiredException automatically
50
+ client.retry_known_errors = true
51
+
52
+ # client.retry_interval = 1 # [sec], default: 1
53
+ # clinet.retry_times = 1 # [times], default: 1
54
+
47
55
  ### WebHDFS::FileUtils
48
56
 
49
57
  require 'webhdfs/fileutils'
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.5.1
1
+ 0.5.2
data/lib/webhdfs.rb CHANGED
@@ -1 +1,2 @@
1
+ require File.join(File.dirname(__FILE__), 'webhdfs', 'backport.rb')
1
2
  require File.join(File.dirname(__FILE__), 'webhdfs', 'client.rb')
@@ -0,0 +1,30 @@
1
+ if RUBY_VERSION =~ /^1\.8\./
2
+ require 'cgi'
3
+ def require_relative(relative_feature)
4
+ file = caller.first.split(/:\d/,2).first
5
+ raise LoadError, "require_relative is called in #{$1}" if /\A\((.*)\)/ =~ file
6
+ require File.expand_path(relative_feature, File.dirname(file))
7
+ end
8
+
9
+ module URI
10
+ def self.encode_www_form(enum)
11
+ enum.map do |k,v|
12
+ if v.nil?
13
+ CGI::escape(k)
14
+ elsif v.respond_to?(:to_ary)
15
+ v.to_ary.map do |w|
16
+ str = CGI::escape(k)
17
+ unless w.nil?
18
+ str << '='
19
+ str << CGI::escape(w)
20
+ end
21
+ end.join('&')
22
+ else
23
+ str = CGI::escape(k.to_s)
24
+ str << '='
25
+ str << CGI::escape(v.to_s)
26
+ end
27
+ end.join('&')
28
+ end
29
+ end
30
+ end
@@ -9,17 +9,24 @@ module WebHDFS
9
9
 
10
10
  # This hash table holds command options.
11
11
  OPT_TABLE = {} # internal use only
12
+ KNOWN_ERRORS = ['LeaseExpiredException'].freeze
12
13
 
13
14
  attr_accessor :host, :port, :username, :doas
14
15
  attr_accessor :open_timeout # default 30s (in ruby net/http)
15
16
  attr_accessor :read_timeout # default 60s (in ruby net/http)
16
17
  attr_accessor :httpfs_mode
18
+ attr_accessor :retry_known_errors # default false (not to retry)
19
+ attr_accessor :retry_times # default 1 (ignored when retry_known_errors is false)
20
+ attr_accessor :retry_interval # default 1 ([sec], ignored when retry_known_errors is false)
17
21
 
18
22
  def initialize(host='localhost', port=50070, username=nil, doas=nil)
19
23
  @host = host
20
24
  @port = port
21
25
  @username = username
22
26
  @doas = doas
27
+ @retry_known_errors = false
28
+ @retry_times = 1
29
+ @retry_interval = 1
23
30
 
24
31
  @httpfs_mode = false
25
32
  end
@@ -245,7 +252,7 @@ module WebHDFS
245
252
  # IOException 403 Forbidden
246
253
  # FileNotFoundException 404 Not Found
247
254
  # RumtimeException 500 Internal Server Error
248
- def request(host, port, method, path, op=nil, params={}, payload=nil, header=nil)
255
+ def request(host, port, method, path, op=nil, params={}, payload=nil, header=nil, retries=0)
249
256
  conn = Net::HTTP.new(host, port)
250
257
  conn.open_timeout = @open_timeout if @open_timeout
251
258
  conn.read_timeout = @read_timeout if @read_timeout
@@ -269,6 +276,22 @@ module WebHDFS
269
276
  else
270
277
  'Response body is empty...'
271
278
  end
279
+
280
+ if @retry_known_errors && retries < @retry_times
281
+ detail = nil
282
+ if message =~ /^\{\\"RemoteException\\":\\\{/
283
+ begin
284
+ detail = JSON.parse(message)
285
+ rescue
286
+ # ignore broken json response body
287
+ end
288
+ end
289
+ if detail && detail['RemoteException'] && KNOWN_ERRORS.include?(detail['RemoteException']['exception'])
290
+ sleep @retry_interval if @retry_interval > 0
291
+ return request(host, port, method, path, op, params, payload, header, retries+1)
292
+ end
293
+ end
294
+
272
295
  case res.code
273
296
  when '400'
274
297
  raise WebHDFS::ClientError, message
data/test/test_helper.rb CHANGED
@@ -1,6 +1,7 @@
1
1
  $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
2
2
  $LOAD_PATH.unshift(File.dirname(__FILE__))
3
3
 
4
+ require 'rubygems'
4
5
  require 'rr'
5
6
  require 'test/unit'
6
7
  class Test::Unit::TestCase
@@ -2,7 +2,8 @@ require 'test_helper'
2
2
 
3
3
  class FileUtilsTest < Test::Unit::TestCase
4
4
  def setup
5
- require 'webhdfs'
5
+ require 'lib/webhdfs'
6
+ require 'lib/webhdfs/fileutils'
6
7
  end
7
8
 
8
9
  def test_copy_from_local
@@ -43,7 +44,7 @@ class FileUtilsTest < Test::Unit::TestCase
43
44
 
44
45
  def test_set_repl_factor
45
46
  WebHDFS::FileUtils.mkdir('foo', :mode => 0777, :verbose => true)
46
- WebHDFS::FileUtils.set_repl_factor('foo', 5)
47
+ WebHDFS::FileUtils.set_repl_factor('foo', 2)
47
48
  WebHDFS::FileUtils.rm('foo', :verbose => true)
48
49
  end
49
50
 
metadata CHANGED
@@ -1,78 +1,69 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: webhdfs
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.1
5
- prerelease:
4
+ version: 0.5.2
6
5
  platform: ruby
7
6
  authors:
8
7
  - Kazuki Ohta
9
8
  autorequire:
10
9
  bindir: bin
11
10
  cert_chain: []
12
- date: 2012-08-15 00:00:00.000000000 Z
11
+ date: 2013-05-08 00:00:00.000000000 Z
13
12
  dependencies:
14
13
  - !ruby/object:Gem::Dependency
15
14
  name: rake
16
15
  requirement: !ruby/object:Gem::Requirement
17
- none: false
18
16
  requirements:
19
- - - ! '>='
17
+ - - '>='
20
18
  - !ruby/object:Gem::Version
21
19
  version: 0.9.2
22
20
  type: :development
23
21
  prerelease: false
24
22
  version_requirements: !ruby/object:Gem::Requirement
25
- none: false
26
23
  requirements:
27
- - - ! '>='
24
+ - - '>='
28
25
  - !ruby/object:Gem::Version
29
26
  version: 0.9.2
30
27
  - !ruby/object:Gem::Dependency
31
28
  name: rdoc
32
29
  requirement: !ruby/object:Gem::Requirement
33
- none: false
34
30
  requirements:
35
- - - ! '>='
31
+ - - '>='
36
32
  - !ruby/object:Gem::Version
37
33
  version: '3.12'
38
34
  type: :development
39
35
  prerelease: false
40
36
  version_requirements: !ruby/object:Gem::Requirement
41
- none: false
42
37
  requirements:
43
- - - ! '>='
38
+ - - '>='
44
39
  - !ruby/object:Gem::Version
45
40
  version: '3.12'
46
41
  - !ruby/object:Gem::Dependency
47
42
  name: simplecov
48
43
  requirement: !ruby/object:Gem::Requirement
49
- none: false
50
44
  requirements:
51
- - - ! '>='
45
+ - - '>='
52
46
  - !ruby/object:Gem::Version
53
47
  version: 0.5.4
54
48
  type: :development
55
49
  prerelease: false
56
50
  version_requirements: !ruby/object:Gem::Requirement
57
- none: false
58
51
  requirements:
59
- - - ! '>='
52
+ - - '>='
60
53
  - !ruby/object:Gem::Version
61
54
  version: 0.5.4
62
55
  - !ruby/object:Gem::Dependency
63
56
  name: rr
64
57
  requirement: !ruby/object:Gem::Requirement
65
- none: false
66
58
  requirements:
67
- - - ! '>='
59
+ - - '>='
68
60
  - !ruby/object:Gem::Version
69
61
  version: 1.0.0
70
62
  type: :development
71
63
  prerelease: false
72
64
  version_requirements: !ruby/object:Gem::Requirement
73
- none: false
74
65
  requirements:
75
- - - ! '>='
66
+ - - '>='
76
67
  - !ruby/object:Gem::Version
77
68
  version: 1.0.0
78
69
  description: Ruby WebHDFS/HttpFs client
@@ -89,6 +80,7 @@ files:
89
80
  - Rakefile
90
81
  - VERSION
91
82
  - lib/webhdfs.rb
83
+ - lib/webhdfs/backport.rb
92
84
  - lib/webhdfs/client.rb
93
85
  - lib/webhdfs/client_v1.rb
94
86
  - lib/webhdfs/exceptions.rb
@@ -98,27 +90,26 @@ files:
98
90
  - webhdfs.gemspec
99
91
  homepage: ''
100
92
  licenses: []
93
+ metadata: {}
101
94
  post_install_message:
102
95
  rdoc_options: []
103
96
  require_paths:
104
97
  - lib
105
98
  required_ruby_version: !ruby/object:Gem::Requirement
106
- none: false
107
99
  requirements:
108
- - - ! '>='
100
+ - - '>='
109
101
  - !ruby/object:Gem::Version
110
102
  version: '0'
111
103
  required_rubygems_version: !ruby/object:Gem::Requirement
112
- none: false
113
104
  requirements:
114
- - - ! '>='
105
+ - - '>='
115
106
  - !ruby/object:Gem::Version
116
107
  version: '0'
117
108
  requirements: []
118
109
  rubyforge_project:
119
- rubygems_version: 1.8.21
110
+ rubygems_version: 2.0.0
120
111
  signing_key:
121
- specification_version: 3
112
+ specification_version: 4
122
113
  summary: Ruby WebHDFS/HttpFs client
123
114
  test_files:
124
115
  - test/test_helper.rb