webhdfs 0.1 → 0.2

data/Rakefile

@@ -9,6 +9,10 @@ Rake::TestTask.new(:test) do |test|
   test.verbose = true
 end
 
+task :doc do |t|
+  `bundle exec rdoc --markup=tomdoc --visibility=public --include=lib --exclude=test`
+end
+
 task :coverage do |t|
   ENV['SIMPLE_COV'] = '1'
   Rake::Task["test"].invoke

data/VERSION

@@ -1 +1 @@
-0.1
+0.2

data/lib/webhdfs/fileutils.rb

@@ -5,13 +5,120 @@ module WebHDFS
     # This hash table holds command options.
     OPT_TABLE = {} # internal use only
 
+    # Those values hold NameNode location
     @fu_host = 'localhost'
     @fu_port = 50070
-    def set_host(host, port)
+
+    # Public: Set hostname and port number of WebHDFS
+    #
+    # host - hostname
+    # port - port
+    #
+    # Examples
+    #
+    #   FileUtils.set_server 'localhost', 50070
+    #
+    def set_server(host, port)
       @fu_host = host
       @fu_port = port
     end
+    module_function :set_server
+
+    # Public: Copy local file into HDFS
+    #
+    # file - local file path
+    # path - HDFS file path
+    # options - :overwrite, :blocksize, :replication, :mode, :buffersize, :verbose
+    #
+    # Examples
+    #
+    #   FileUtils.copy_from_local 'local_file', 'remote_file'
+    #
+    def copy_from_local(file, path, options={})
+      fu_check_options options, OPT_TABLE['copy_from_local']
+      fu_log "copy_from_local local=#{file} hdfs=#{path}" if options[:verbose]
+      if mode = options[:mode]
+        mode = ('0%03o' % mode) if mode.is_a? Integer
+      else
+        mode = '0644'
+      end
+      options[:permission] = mode
+      options[:overwrite] ||= true
+      begin
+        fu_put(path, 'CREATE', options)
+      rescue RestClient::TemporaryRedirect => e
+        # must be redirected
+        raise e unless [301, 302, 307].include? e.response.code
+        # must have location
+        location = e.response.headers[:location]
+        raise e if location.nil? or location.empty?
+        # put contents
+        RestClient.put location, File.new(file, 'rb')
+      end
+    end
+    OPT_TABLE['copy_from_local'] = [:overwrite, :blocksize, :replication, :mode, :buffersize, :verbose]
+    module_function :copy_from_local
+
+    # Public: Copy remote HDFS file into local
+    #
+    # path - HDFS file path
+    # file - local file path
+    # options - :offset, :length, :buffersize, :verbose
+    #
+    # Examples
+    #
+    #   FileUtils.copy_from_local 'remote_file', 'local_file'
+    #
+    def copy_to_local(path, file, options={})
+      fu_check_options options, OPT_TABLE['copy_to_local']
+      fu_log "copy_to_local hdfs=#{path} local=#{file}" if options[:verbose]
+      File.open(file, "wb") do |f|
+        ret = fu_get(path, 'OPEN', options)
+        f.write ret
+      end
+    end
+    OPT_TABLE['copy_to_local'] = [:offset, :length, :buffersize, :verbose]
+    module_function :copy_to_local
 
+    # Public: Append to HDFS file
+    #
+    # path - HDFS file path
+    # body - contents
+    # options - :buffersize, :verbose
+    #
+    # Examples
+    #
+    #   FileUtils.copy_from_local 'local_file', 'remote_file'
+    #
+    def append(path, body, options={})
+      fu_check_options options, OPT_TABLE['append']
+      fu_log "append #{body.bytesize} bytes to #{path}" if options[:verbose]
+      begin
+        fu_post(path, 'APPEND', options)
+      rescue RestClient::TemporaryRedirect => e
+        # must be redirected
+        raise e unless [301, 302, 307].include? e.response.code
+        # must have location
+        location = e.response.headers[:location]
+        raise e if location.nil? or location.empty?
+        # put contents
+        RestClient.post location, body
+      end
+    end
+    OPT_TABLE['append'] = [:buffersize, :verbose]
+    module_function :append
+
+    # Public: Create one or more directories.
+    #
+    # list - directory name, or list of them
+    # options - :mode, :verbose
+    #
+    # Examples
+    #
+    #   FileUtils.mkdir 'test'
+    #   FileUtils.mkdir %w( tmp data )
+    #   FileUtils.mkdir 'tmp', :mode => 0700
+    #
     def mkdir(list, options={})
       fu_check_options options, OPT_TABLE['mkdir']
       list = fu_list(list)
@@ -28,9 +135,31 @@ module WebHDFS
     OPT_TABLE['mkdir'] = [:mode, :verbose]
     module_function :mkdir
 
+    # Public: Create one or more directories recursively.
+    #
+    # list - directory name, or list of them
+    # options - :mode, :verbose
+    #
+    # Examples
+    #
+    #   FileUtils.mkdir_p 'dir/subdir'
+    #   FileUtils.mkdir_p %w( tmp data )
+    #   FileUtils.mkdir_p 'dir/subdir', :mode => 0700
+    #
     alias mkdir_p mkdir
     module_function :mkdir_p
 
+    # Public: Remove one or more directories or files.
+    #
+    # list - directory name, or list of them
+    # options - :recursive, :verbose
+    #
+    # Examples
+    #
+    #   FileUtils.rm 'dir'
+    #   FileUtils.rm %w( tmp data )
+    #   FileUtils.rm 'dir', :recursive => true
+    #
     def rm(list, options={})
       fu_check_options options, OPT_TABLE['rm']
       list = fu_list(list)
@@ -42,13 +171,34 @@ module WebHDFS
     OPT_TABLE['rm'] = [:verbose, :recursive]
     module_function :rm
 
+    # Public: Remove one or more directories/files recursively.
+    #
+    # list - directory name, or list of them
+    # options - :verbose
+    #
+    # Examples
+    #
+    #   FileUtils.rmr 'dir'
+    #   FileUtils.rmr %w( tmp data )
+    #   FileUtils.rmr 'dir'
+    #
     def rmr(list, options={})
       fu_check_options options, OPT_TABLE['rmr']
       self.rm(list, options.merge({:recursive => true}))
     end
-    OPT_TABLE['rmr'] = [:verbose, :recursive]
+    OPT_TABLE['rmr'] = [:verbose]
     module_function :rmr
 
+    # Public: Rename a file or directory.
+    #
+    # src - from
+    # dst - to
+    # options - :verbose
+    #
+    # Examples
+    #
+    #   FileUtils.rename 'from', 'to'
+    #
     def rename(src, dst, options={})
       fu_check_options options, OPT_TABLE['rename']
       fu_log "rename #{src} #{dst}" if options[:verbose]
@@ -57,6 +207,17 @@ module WebHDFS
     OPT_TABLE['rename'] = [:verbose]
     module_function :rename
 
+    # Public: Change permission of one or more directories/files.
+    #
+    # mode - permission
+    # list - file/directory name or list of them.
+    # options - :verbose
+    #
+    # Examples
+    #
+    #   FileUtils.chmod 0755, 'dir'
+    #   FileUtils.chmod 0644, 'file'
+    #
     def chmod(mode, list, options={})
       fu_check_options options, OPT_TABLE['chmod']
       list = fu_list(list)
@@ -69,6 +230,18 @@ module WebHDFS
     OPT_TABLE['chmod'] = [:verbose]
     module_function :chmod
 
+    # Public: Change an ownership of one or more directories/files.
+    #
+    # user - username
+    # group - groupname
+    # list - file/directory name or list of them
+    # options - :verbose
+    #
+    # Examples
+    #
+    #   FileUtils.chmod 0755, 'dir'
+    #   FileUtils.chmod 0644, 'file'
+    #
     def chown(user, group, list, options={})
       fu_check_options options, OPT_TABLE['chown']
       list = fu_list(list)
@@ -82,6 +255,16 @@ module WebHDFS
     OPT_TABLE['chown'] = [:verbose]
     module_function :chown
 
+    # Public: Set a replication factor of files
+    #
+    # list - file/directory name or list of them
+    # num - replication factor
+    # options - :verbose
+    #
+    # Examples
+    #
+    #   FileUtils.set_repl_factor 'file', 3
+    #
     def set_repl_factor(list, num, options={})
       fu_check_options options, OPT_TABLE['set_repl_factor']
       list = fu_list(list)
@@ -94,6 +277,16 @@ module WebHDFS
     OPT_TABLE['set_repl_factor'] = [:verbose]
     module_function :set_repl_factor
 
+    # Public: Set an access time of files
+    #
+    # list - file/directory name or list of them
+    # time - new access time
+    # options - :verbose
+    #
+    # Examples
+    #
+    #   FileUtils.set_atime 'file', Time.now
+    #
     def set_atime(list, time, options={})
       fu_check_options options, OPT_TABLE['set_atime']
       list = fu_list(list)
@@ -101,11 +294,21 @@ module WebHDFS
       fu_log sprintf('set_atime %s %d', list.join(' '), time) if options[:verbose]
       list.each { |dir|
         fu_put(dir, 'SETTIMES', {:accesstime => time})
-      }      
+      }
     end
     OPT_TABLE['set_atime'] = [:verbose]
     module_function :set_atime
 
+    # Public: Set a modification time of files
+    #
+    # list - file/directory name or list of them
+    # time - new modification time
+    # options - :verbose
+    #
+    # Examples
+    #
+    #   FileUtils.set_mtime 'file', Time.now
+    #
     def set_mtime(list, time, options={})
       fu_check_options options, OPT_TABLE['set_mtime']
       list = fu_list(list)
@@ -113,34 +316,52 @@ module WebHDFS
       fu_log sprintf('set_mtime %s %d', list.join(' '), time) if options[:verbose]
       list.each { |dir|
         fu_put(dir, 'SETTIMES', {:modificationtime => time})
-      }      
+      }
     end
     OPT_TABLE['set_mtime'] = [:verbose]
     module_function :set_mtime
 
-    ##
+    # Internal: make functin private
     def self.private_module_function(name)
       module_function name
       private_class_method name
     end
 
+    # Internal: make list
     def fu_list(arg)
       [arg].flatten
     end
     private_module_function :fu_list
 
+    # Internal: HTTP GET
+    def fu_get(path, op, params={}, payload='')
+      url = "http://#{@fu_host}:#{@fu_port}/webhdfs/v1/#{path}"
+      RestClient.get url, :params => params.merge({:op => op})
+    end
+    private_module_function :fu_get
+
+    # Internal: HTTP PUT
     def fu_put(path, op, params={}, payload='')
       url = "http://#{@fu_host}:#{@fu_port}/webhdfs/v1/#{path}"
       RestClient.put url, payload, :params => params.merge({:op => op})
     end
     private_module_function :fu_put
 
+    # Internal: HTTP POST
+    def fu_post(path, op, params={}, payload='')
+      url = "http://#{@fu_host}:#{@fu_port}/webhdfs/v1/#{path}"
+      RestClient.post url, payload, :params => params.merge({:op => op})
+    end
+    private_module_function :fu_post
+
+    # Internal: HTTP DELETE
     def fu_delete(path, op, params={})
       url = "http://#{@fu_host}:#{@fu_port}/webhdfs/v1/#{path}"
       RestClient.delete url, :params => params.merge({:op => op})
     end
     private_module_function :fu_delete
 
+    # Internal: Check options Hash
     def fu_check_options(options, optdecl)
       h = options.dup
       optdecl.each do |opt|
@@ -152,6 +373,7 @@ module WebHDFS
 
     @fileutils_output = $stderr
     @fileutils_label  = '[webhdfs]: '
+    # Internal: Logging
     def fu_log(msg)
       @fileutils_output ||= $stderr
       @fileutils_label  ||= ''

data/test/webhdfs/fileutils.rb

@@ -5,6 +5,13 @@ class FileUtilsTest < Test::Unit::TestCase
     require 'webhdfs'
   end
 
+  def test_copy_from_local
+    WebHDFS::FileUtils.copy_from_local('VERSION', 'VERSION', :verbose => true)
+    WebHDFS::FileUtils.copy_to_local('VERSION', 'VERSION2', :verbose => true)
+    WebHDFS::FileUtils.append('VERSION', 'foo-bar-buzz', :verbose => true)
+    WebHDFS::FileUtils.rm('VERSION', :verbose => true)
+  end
+
   def test_rm
     WebHDFS::FileUtils.mkdir('foo', :mode => 0777, :verbose => true)
     WebHDFS::FileUtils.rm('foo', :verbose => true)

data/webhdfs.gemspec

@@ -18,6 +18,7 @@ Gem::Specification.new do |gem|
 
   gem.add_dependency "rest-client", "~> 1.6.7"
   gem.add_development_dependency "rake", ">= 0.9.2"
+  gem.add_development_dependency "rdoc", ">= 3.12"
   gem.add_development_dependency "simplecov", ">= 0.5.4"
   gem.add_development_dependency "rr", ">= 1.0.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: webhdfs
 version: !ruby/object:Gem::Version
-  version: '0.1'
+  version: '0.2'
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-03-11 00:00:00.000000000Z
+date: 2012-03-14 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
-  requirement: &2154610720 !ruby/object:Gem::Requirement
+  requirement: &2157003340 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 1.6.7
   type: :runtime
   prerelease: false
-  version_requirements: *2154610720
+  version_requirements: *2157003340
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &2154610240 !ruby/object:Gem::Requirement
+  requirement: &2157002860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,21 @@ dependencies:
         version: 0.9.2
   type: :development
   prerelease: false
-  version_requirements: *2154610240
+  version_requirements: *2157002860
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: &2157002400 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '3.12'
+  type: :development
+  prerelease: false
+  version_requirements: *2157002400
 - !ruby/object:Gem::Dependency
   name: simplecov
-  requirement: &2154609780 !ruby/object:Gem::Requirement
+  requirement: &2157001940 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +54,10 @@ dependencies:
         version: 0.5.4
   type: :development
   prerelease: false
-  version_requirements: *2154609780
+  version_requirements: *2157001940
 - !ruby/object:Gem::Dependency
   name: rr
-  requirement: &2154609320 !ruby/object:Gem::Requirement
+  requirement: &2157001480 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,7 +65,7 @@ dependencies:
         version: 1.0.0
   type: :development
   prerelease: false
-  version_requirements: *2154609320
+  version_requirements: *2157001480
 description: Ruby WebHDFS client
 email: kazuki.ohta@gmail.com
 executables: []