webhdfs 0.5.4 → 0.5.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2dbe7f5cbc7dd8becf1af99bc7b4291170e09763
-  data.tar.gz: fec022e46547493c2ecb825e14e62c32ad62a7ab
+  metadata.gz: 079a9de419dc30c5280876fdb45dc4e6a3844fc1
+  data.tar.gz: 915c3c102e9ae29a729d6498d2280b2fc94077f3
 SHA512:
-  metadata.gz: c098695dd8248293b86e85e742e9abfd41da0b1a433609cf0f18bec6522a78135c324095c8a2280ee3ae6bb2524e09a8fc30c121eafd0fd68345c64c0a7caef9
-  data.tar.gz: 0b7a52e3ce9d03ff6d26533604309831d4228606bc3e5f1421f5150323a521a820861bc586906b9606bd5fd55c3d50952b0a70911a235565df101bba65efea1e
+  metadata.gz: 514504c4024a57b80d81347634c33fa061c82ab28f6f55ff47f9dc5c3dde2efdca053161a912479bb64f2e8e6fef137af1f639848dbe5b1d506b349ac8f6045b
+  data.tar.gz: 6135ce124d4619ed25a5f1aa790e9cbb7f9cc5a56085256d6bd386f9a9eb0c9fc3a9537b7e8ef72f8a7d19a95afc3f42098c2e3ce5874fcd7526dcb469903a25

data/README.md

@@ -21,7 +21,10 @@ To create/append/read files:
 
     client.create('/path/to/file', data)
     client.create('/path/to/file', data, :overwrite => false, :blocksize => 268435456, :replication => 5, :permission => 0666)
-    
+
+    #This does not require whole data in memory, and it can be read chunk by chunk, ex: File data
+    client.create('/path/to/file', file_IO_handle, :overwrite => false, :permission => 0666)
+
     client.append('/path/to/existing/file', data)
     
     client.read('/path/to/target') #=> data

data/VERSION

@@ -1 +1 @@
-0.5.4
+0.5.5

data/lib/webhdfs/client_v1.rb

@@ -268,7 +268,18 @@ module WebHDFS
                        path
                      end
 
-      res = conn.send_request(method, request_path, payload, header)
+      res = nil
+      if !payload.nil? and payload.is_a?(IO)
+        req = Net::HTTPGenericRequest.new(method,(payload ? true : false),true,request_path,header)
+        raise WebHDFS::IOError, 'Error reading given IO data source' unless payload.respond_to? :read and payload.respond_to? :size
+        raise WebHDFS::ClientError, 'Error accepting given IO resource as data payload, Not valid in methods other than PUT and POST' unless (method == 'PUT' or method == 'POST')
+
+        req.body_stream = payload
+        req.content_length = payload.size
+        res = conn.request(req)
+      else
+        res = conn.send_request(method, request_path, payload, header)
+      end
 
       case res
       when Net::HTTPSuccess

data/lib/webhdfs/fileutils.rb

@@ -70,6 +70,31 @@ module WebHDFS
     end
     module_function :copy_from_local
 
+    # Public: Copy local file into HDFS with IOStream
+    #
+    # file - local file IO handle
+    # path - HDFS file path
+    # options - :overwrite, :blocksize, :replication, :mode, :buffersize, :verbose
+    #
+    # Examples
+    #
+    #   FileUtils.copy_from_local_via_stream 'local_file_IO_handle', 'remote_file'
+    #
+    def copy_from_local_via_stream(file, path, options={})
+      opts = options.dup
+      fu_log "copy_from_local_via_stream local=#{file} hdfs=#{path}" if opts.delete(:verbose)
+      if mode = opts.delete(:mode)
+        mode = ('%03o' % mode) if mode.is_a? Integer
+      else
+        mode = '644'
+      end
+      opts[:permission] = mode
+      opts[:overwrite] ||= true
+
+      client.create(path, File.new(file, 'rb'), opts)
+    end
+    module_function :copy_from_local_via_stream
+
     # Public: Copy remote HDFS file into local
     #
     # path - HDFS file path

data/test/webhdfs/fileutils.rb

@@ -13,6 +13,11 @@ class FileUtilsTest < Test::Unit::TestCase
     WebHDFS::FileUtils.rm('VERSION', :verbose => true)
   end
 
+  def test_copy_from_local_via_stream
+    WebHDFS::FileUtils.copy_from_local_via_stream('VERSION', '/user/jay/VERSION', :verbose => true)
+    WebHDFS::FileUtils.rm('VERSION', :verbose => true)
+  end
+
   def test_rm
     WebHDFS::FileUtils.mkdir('foo', :mode => 0777, :verbose => true)
     WebHDFS::FileUtils.rm('foo', :verbose => true)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: webhdfs
 version: !ruby/object:Gem::Version
-  version: 0.5.4
+  version: 0.5.5
 platform: ruby
 authors:
 - Kazuki Ohta
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-12 00:00:00.000000000 Z
+date: 2013-10-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -107,7 +107,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.2
+rubygems_version: 2.0.3
 signing_key: 
 specification_version: 4
 summary: Ruby WebHDFS/HttpFs client