webhdfs 0.1

data/.gitignore

@@ -0,0 +1,4 @@
+/pkg/
+/coverage/
+/vendor/
+Gemfile.lock

data/AUTHORS

@@ -0,0 +1 @@
+Kazuki Ohta <kazuki.ohta@gmail.com>

data/Gemfile

@@ -0,0 +1,5 @@
+source :rubygems
+
+gemspec
+
+gem "simplecov", :require => false

data/Rakefile

@@ -0,0 +1,17 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/webhdfs/*.rb']
+  test.verbose = true
+end
+
+task :coverage do |t|
+  ENV['SIMPLE_COV'] = '1'
+  Rake::Task["test"].invoke
+end
+
+task :default => [:build]

data/VERSION

@@ -0,0 +1 @@
+0.1

data/lib/webhdfs.rb

@@ -0,0 +1 @@
+require File.join(File.dirname(__FILE__), 'webhdfs', 'fileutils.rb')

data/lib/webhdfs/fileutils.rb

@@ -0,0 +1,162 @@
+module WebHDFS
+  module FileUtils
+    require 'rest_client'
+
+    # This hash table holds command options.
+    OPT_TABLE = {} # internal use only
+
+    @fu_host = 'localhost'
+    @fu_port = 50070
+    def set_host(host, port)
+      @fu_host = host
+      @fu_port = port
+    end
+
+    def mkdir(list, options={})
+      fu_check_options options, OPT_TABLE['mkdir']
+      list = fu_list(list)
+      fu_log "mkdir #{options[:mode] ? ('-m %03o ' % options[:mode]) : ''}#{list.join ' '}" if options[:verbose]
+      if mode = options[:mode]
+        mode = ('0%03o' % mode) if mode.is_a? Integer
+      else
+        mode = '0755'
+      end
+      list.each { |dir|
+        fu_put(dir, 'MKDIRS', {:permission => mode})
+      }
+    end
+    OPT_TABLE['mkdir'] = [:mode, :verbose]
+    module_function :mkdir
+
+    alias mkdir_p mkdir
+    module_function :mkdir_p
+
+    def rm(list, options={})
+      fu_check_options options, OPT_TABLE['rm']
+      list = fu_list(list)
+      fu_log "rm #{list.join ' '}" if options[:verbose]
+      list.each { |dir|
+        fu_delete(dir, 'DELETE', {:recursive => options[:recursive] || false})
+      }
+    end
+    OPT_TABLE['rm'] = [:verbose, :recursive]
+    module_function :rm
+
+    def rmr(list, options={})
+      fu_check_options options, OPT_TABLE['rmr']
+      self.rm(list, options.merge({:recursive => true}))
+    end
+    OPT_TABLE['rmr'] = [:verbose, :recursive]
+    module_function :rmr
+
+    def rename(src, dst, options={})
+      fu_check_options options, OPT_TABLE['rename']
+      fu_log "rename #{src} #{dst}" if options[:verbose]
+      fu_put(src, 'RENAME', {:destination => dst})
+    end
+    OPT_TABLE['rename'] = [:verbose]
+    module_function :rename
+
+    def chmod(mode, list, options={})
+      fu_check_options options, OPT_TABLE['chmod']
+      list = fu_list(list)
+      fu_log sprintf('chmod %o %s', mode, list.join(' ')) if options[:verbose]
+      mode = ('0%03o' % mode) if mode.is_a? Integer
+      list.each { |dir|
+        fu_put(dir, 'SETPERMISSION', {:permission => mode})
+      }
+    end
+    OPT_TABLE['chmod'] = [:verbose]
+    module_function :chmod
+
+    def chown(user, group, list, options={})
+      fu_check_options options, OPT_TABLE['chown']
+      list = fu_list(list)
+      fu_log sprintf('chown %s%s',
+                     [user,group].compact.join(':') + ' ',
+                     list.join(' ')) if options[:verbose]
+      list.each { |dir|
+        fu_put(dir, 'SETOWNER', {:owner => user, :group => group})
+      }
+    end
+    OPT_TABLE['chown'] = [:verbose]
+    module_function :chown
+
+    def set_repl_factor(list, num, options={})
+      fu_check_options options, OPT_TABLE['set_repl_factor']
+      list = fu_list(list)
+      fu_log sprintf('set_repl_factor %s %d',
+                     list.join(' '), num) if options[:verbose]
+      list.each { |dir|
+        fu_put(dir, 'SETREPLICATION', {:replication => num})
+      }
+    end
+    OPT_TABLE['set_repl_factor'] = [:verbose]
+    module_function :set_repl_factor
+
+    def set_atime(list, time, options={})
+      fu_check_options options, OPT_TABLE['set_atime']
+      list = fu_list(list)
+      time = time.to_i
+      fu_log sprintf('set_atime %s %d', list.join(' '), time) if options[:verbose]
+      list.each { |dir|
+        fu_put(dir, 'SETTIMES', {:accesstime => time})
+      }      
+    end
+    OPT_TABLE['set_atime'] = [:verbose]
+    module_function :set_atime
+
+    def set_mtime(list, time, options={})
+      fu_check_options options, OPT_TABLE['set_mtime']
+      list = fu_list(list)
+      time = time.to_i
+      fu_log sprintf('set_mtime %s %d', list.join(' '), time) if options[:verbose]
+      list.each { |dir|
+        fu_put(dir, 'SETTIMES', {:modificationtime => time})
+      }      
+    end
+    OPT_TABLE['set_mtime'] = [:verbose]
+    module_function :set_mtime
+
+    ##
+    def self.private_module_function(name)
+      module_function name
+      private_class_method name
+    end
+
+    def fu_list(arg)
+      [arg].flatten
+    end
+    private_module_function :fu_list
+
+    def fu_put(path, op, params={}, payload='')
+      url = "http://#{@fu_host}:#{@fu_port}/webhdfs/v1/#{path}"
+      RestClient.put url, payload, :params => params.merge({:op => op})
+    end
+    private_module_function :fu_put
+
+    def fu_delete(path, op, params={})
+      url = "http://#{@fu_host}:#{@fu_port}/webhdfs/v1/#{path}"
+      RestClient.delete url, :params => params.merge({:op => op})
+    end
+    private_module_function :fu_delete
+
+    def fu_check_options(options, optdecl)
+      h = options.dup
+      optdecl.each do |opt|
+        h.delete opt
+      end
+      raise ArgumentError, "no such option: #{h.keys.join(' ')}" unless h.empty?
+    end
+    private_module_function :fu_check_options
+
+    @fileutils_output = $stderr
+    @fileutils_label  = '[webhdfs]: '
+    def fu_log(msg)
+      @fileutils_output ||= $stderr
+      @fileutils_label  ||= ''
+      @fileutils_output.puts @fileutils_label + msg
+    end
+    private_module_function :fu_log
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,19 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'rr'
+require 'test/unit'
+class Test::Unit::TestCase
+  include RR::Adapters::TestUnit
+end
+
+if ENV['SIMPLE_COV']
+  require 'simplecov'
+  SimpleCov.start do 
+    add_filter 'test/'
+    add_filter 'pkg/'
+    add_filter 'vendor/'
+  end
+end
+
+require 'test/unit'

data/test/webhdfs/fileutils.rb

@@ -0,0 +1,54 @@
+require 'test_helper'
+
+class FileUtilsTest < Test::Unit::TestCase
+  def setup
+    require 'webhdfs'
+  end
+
+  def test_rm
+    WebHDFS::FileUtils.mkdir('foo', :mode => 0777, :verbose => true)
+    WebHDFS::FileUtils.rm('foo', :verbose => true)
+  end
+
+  def test_rmr
+    WebHDFS::FileUtils.mkdir_p('foo/bar/buzz', :mode => 0777, :verbose => true)
+    WebHDFS::FileUtils.rmr('foo', :verbose => true)
+  end
+
+  def test_rename
+    #WebHDFS::FileUtils.mkdir_p('foo', :mode => 0777, :verbose => true)
+    #WebHDFS::FileUtils.rename('foo', 'foo2', :verbose => true)
+    #WebHDFS::FileUtils.rmr('foo2', :verbose => true)
+  end
+
+  def test_chmod
+    WebHDFS::FileUtils.mkdir('foo', :mode => 0777, :verbose => true)
+    WebHDFS::FileUtils.chmod(0755, 'foo', :verbose => true)
+    WebHDFS::FileUtils.chmod(0777, 'foo', :verbose => true)
+    WebHDFS::FileUtils.rm('foo', :verbose => true)
+  end
+
+  def test_chown
+    #WebHDFS::FileUtils.mkdir('foo', :mode => 0777, :verbose => true)
+    #WebHDFS::FileUtils.chown('webuser', 'supergroup', 'foo', :verbose => true)
+    #WebHDFS::FileUtils.rm('foo', :verbose => true)
+  end
+
+  def test_set_repl_factor
+    WebHDFS::FileUtils.mkdir('foo', :mode => 0777, :verbose => true)
+    WebHDFS::FileUtils.set_repl_factor('foo', 5)
+    WebHDFS::FileUtils.rm('foo', :verbose => true)
+  end
+
+  def test_set_atime
+    #WebHDFS::FileUtils.mkdir('foo', :mode => 0777, :verbose => true)
+    #WebHDFS::FileUtils.set_atime('foo', Time.now)
+    #WebHDFS::FileUtils.rm('foo', :verbose => true)
+  end
+
+  def test_set_mtime
+    #WebHDFS::FileUtils.mkdir('foo', :mode => 0777, :verbose => true)
+    #WebHDFS::FileUtils.set_mtime('foo', Time.now)
+    #WebHDFS::FileUtils.rm('foo', :verbose => true)
+  end
+end

data/webhdfs.gemspec

@@ -0,0 +1,23 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = "webhdfs"
+  gem.description = "Ruby WebHDFS client"
+  gem.homepage    = ""
+  gem.summary     = gem.description
+  gem.version     = File.read("VERSION").strip
+  gem.authors     = ["Kazuki Ohta"]
+  gem.email       = "kazuki.ohta@gmail.com"
+  gem.has_rdoc    = false
+  #gem.platform    = Gem::Platform::RUBY
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency "rest-client", "~> 1.6.7"
+  gem.add_development_dependency "rake", ">= 0.9.2"
+  gem.add_development_dependency "simplecov", ">= 0.5.4"
+  gem.add_development_dependency "rr", ">= 1.0.0"
+end

metadata

@@ -0,0 +1,101 @@
+--- !ruby/object:Gem::Specification
+name: webhdfs
+version: !ruby/object:Gem::Version
+  version: '0.1'
+  prerelease: 
+platform: ruby
+authors:
+- Kazuki Ohta
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-03-11 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: &2154610720 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.6.7
+  type: :runtime
+  prerelease: false
+  version_requirements: *2154610720
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: &2154610240 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: *2154610240
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: &2154609780 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.5.4
+  type: :development
+  prerelease: false
+  version_requirements: *2154609780
+- !ruby/object:Gem::Dependency
+  name: rr
+  requirement: &2154609320 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :development
+  prerelease: false
+  version_requirements: *2154609320
+description: Ruby WebHDFS client
+email: kazuki.ohta@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- AUTHORS
+- Gemfile
+- Rakefile
+- VERSION
+- lib/webhdfs.rb
+- lib/webhdfs/fileutils.rb
+- test/test_helper.rb
+- test/webhdfs/fileutils.rb
+- webhdfs.gemspec
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.6
+signing_key: 
+specification_version: 3
+summary: Ruby WebHDFS client
+test_files:
+- test/test_helper.rb
+- test/webhdfs/fileutils.rb
+has_rdoc: false