webhdfs 0.5.1 → 0.5.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: cdf1d7bc03934462d12ea16db3212bdc55ab3d1a
+  data.tar.gz: 11c21a5e4a5894e66179406f207f3e53f6eb87bd
+SHA512:
+  metadata.gz: ebe43a9a4ede658b61aab20e8dfdf8caf53b5a0a820ed2bd28717872934d36ed21ca6bf8c767c88adedf9005341408f5f3f800d670c165724be07c7577968732
+  data.tar.gz: 3990c04379334c95b3320b7582cf9fe283ae5ed9996c4ba2c6e55acf4a6ab996531781c60fa2e761dfa0bfcf2afe718b8f7739d97323df61113f26f889a3d611

data/README.md

@@ -44,6 +44,14 @@ To get status or list of files and directories:
 
 And, 'content_summary', 'checksum', 'homedir', 'chmod', 'chown', 'replication' and 'touch' methods available.
 
+For known errors, automated retries are available. Set `retry_known_errors` option as true.
+
+    #### To retry for LeaseExpiredException automatically
+    client.retry_known_errors = true
+    
+    # client.retry_interval = 1 # [sec], default: 1
+    # clinet.retry_times = 1 # [times], default: 1
+
 ### WebHDFS::FileUtils
 
     require 'webhdfs/fileutils'

data/VERSION

@@ -1 +1 @@
-0.5.1
+0.5.2

data/lib/webhdfs.rb

@@ -1 +1,2 @@
+require File.join(File.dirname(__FILE__), 'webhdfs', 'backport.rb')
 require File.join(File.dirname(__FILE__), 'webhdfs', 'client.rb')

data/lib/webhdfs/backport.rb

@@ -0,0 +1,30 @@
+if RUBY_VERSION =~ /^1\.8\./
+  require 'cgi'
+  def require_relative(relative_feature)
+    file = caller.first.split(/:\d/,2).first
+    raise LoadError, "require_relative is called in #{$1}" if /\A\((.*)\)/ =~ file
+    require File.expand_path(relative_feature, File.dirname(file))
+  end
+
+  module URI
+    def self.encode_www_form(enum)
+      enum.map do |k,v|
+        if v.nil?
+          CGI::escape(k)
+        elsif v.respond_to?(:to_ary)
+          v.to_ary.map do |w|
+            str = CGI::escape(k)
+            unless w.nil?
+              str << '='
+              str << CGI::escape(w)
+            end
+          end.join('&')
+        else
+          str = CGI::escape(k.to_s)
+          str << '='
+          str << CGI::escape(v.to_s)
+        end
+      end.join('&')
+    end
+  end
+end

data/lib/webhdfs/client_v1.rb

@@ -9,17 +9,24 @@ module WebHDFS
 
     # This hash table holds command options.
     OPT_TABLE = {} # internal use only
+    KNOWN_ERRORS = ['LeaseExpiredException'].freeze
 
     attr_accessor :host, :port, :username, :doas
     attr_accessor :open_timeout # default 30s (in ruby net/http)
     attr_accessor :read_timeout # default 60s (in ruby net/http)
     attr_accessor :httpfs_mode
+    attr_accessor :retry_known_errors # default false (not to retry)
+    attr_accessor :retry_times        # default 1 (ignored when retry_known_errors is false)
+    attr_accessor :retry_interval     # default 1 ([sec], ignored when retry_known_errors is false)
 
     def initialize(host='localhost', port=50070, username=nil, doas=nil)
       @host = host
       @port = port
       @username = username
       @doas = doas
+      @retry_known_errors = false
+      @retry_times = 1
+      @retry_interval = 1
 
       @httpfs_mode = false
     end
@@ -245,7 +252,7 @@ module WebHDFS
     # IOException                   403 Forbidden
     # FileNotFoundException         404 Not Found
     # RumtimeException              500 Internal Server Error
-    def request(host, port, method, path, op=nil, params={}, payload=nil, header=nil)
+    def request(host, port, method, path, op=nil, params={}, payload=nil, header=nil, retries=0)
       conn = Net::HTTP.new(host, port)
       conn.open_timeout = @open_timeout if @open_timeout
       conn.read_timeout = @read_timeout if @read_timeout
@@ -269,6 +276,22 @@ module WebHDFS
                   else
                     'Response body is empty...'
                   end
+
+        if @retry_known_errors && retries < @retry_times
+          detail = nil
+          if message =~ /^\{\\"RemoteException\\":\\\{/
+            begin
+              detail = JSON.parse(message)
+            rescue
+              # ignore broken json response body
+            end
+          end
+          if detail && detail['RemoteException'] && KNOWN_ERRORS.include?(detail['RemoteException']['exception'])
+            sleep @retry_interval if @retry_interval > 0
+            return request(host, port, method, path, op, params, payload, header, retries+1)
+          end
+        end
+
         case res.code
         when '400'
           raise WebHDFS::ClientError, message

data/test/test_helper.rb

@@ -1,6 +1,7 @@
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 
+require 'rubygems'
 require 'rr'
 require 'test/unit'
 class Test::Unit::TestCase

data/test/webhdfs/fileutils.rb

@@ -2,7 +2,8 @@ require 'test_helper'
 
 class FileUtilsTest < Test::Unit::TestCase
   def setup
-    require 'webhdfs'
+    require 'lib/webhdfs'
+    require 'lib/webhdfs/fileutils'
   end
 
   def test_copy_from_local
@@ -43,7 +44,7 @@ class FileUtilsTest < Test::Unit::TestCase
 
   def test_set_repl_factor
     WebHDFS::FileUtils.mkdir('foo', :mode => 0777, :verbose => true)
-    WebHDFS::FileUtils.set_repl_factor('foo', 5)
+    WebHDFS::FileUtils.set_repl_factor('foo', 2)
     WebHDFS::FileUtils.rm('foo', :verbose => true)
   end
 

metadata

@@ -1,78 +1,69 @@
 --- !ruby/object:Gem::Specification
 name: webhdfs
 version: !ruby/object:Gem::Version
-  version: 0.5.1
-  prerelease: 
+  version: 0.5.2
 platform: ruby
 authors:
 - Kazuki Ohta
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-08-15 00:00:00.000000000 Z
+date: 2013-05-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.9.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.9.2
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '3.12'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '3.12'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.5.4
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.5.4
 - !ruby/object:Gem::Dependency
   name: rr
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.0.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.0.0
 description: Ruby WebHDFS/HttpFs client
@@ -89,6 +80,7 @@ files:
 - Rakefile
 - VERSION
 - lib/webhdfs.rb
+- lib/webhdfs/backport.rb
 - lib/webhdfs/client.rb
 - lib/webhdfs/client_v1.rb
 - lib/webhdfs/exceptions.rb
@@ -98,27 +90,26 @@ files:
 - webhdfs.gemspec
 homepage: ''
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.21
+rubygems_version: 2.0.0
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Ruby WebHDFS/HttpFs client
 test_files:
 - test/test_helper.rb