hdfs-viking 0.0.1

data/lib/viking/fileutilz.rb

@@ -0,0 +1,101 @@
+module Viking
+  module FileUtils
+
+    def self.cd(dirname, &block)
+      if block
+        Viking::Dir.chdir(dirname, &block)
+      else
+        Viking::Dir.chdir(dirname)
+      end
+    end
+
+    def self.chdir(dirname, &block)
+      cd(dirname, &block)
+    end
+
+    def self.chmod(mode, list)
+      list = (list.is_a? Array) ? list : [list]
+      list.each do |path|
+        Viking::File.chmod(mode, path)
+      end
+    end
+
+    def self.chown(user, group, list)
+      list = (list.is_a? Array) ? list : [list]
+      list.each do |dir|
+        Viking::File.chown(user, group, dir)
+      end
+    end
+
+    def self.getwd
+      Viking::Dir.getwd
+    end
+
+    def self.makedirs(list)
+      list = (list.is_a? Array) ? list : [list]
+      list.each do |dir|
+        Viking::Dir.mkdir(dir)
+      end
+    end
+
+    def self.mkdir(list)
+      makedirs(list)
+    end
+
+    def self.move(src, dst)
+      Viking::File.move(src, dst)
+    end
+
+    def self.mv(src, dst)
+      move(src, dst)
+    end
+
+    def self.pwd
+      getwd
+    end
+
+    def self.remove(list)
+      list = (list.is_a? Array) ? list : [list]
+      list.each do |dir|
+        remove_file(dir)
+      end
+    end
+
+    def self.remove_dir(path)
+      Viking::Dir.delete(path)
+    end
+
+    def self.remove_entry(path)
+      rm_r(path)
+    end
+
+    def self.remove_file(path)
+      Viking::File.delete(path)
+    end
+
+    def self.rm(list)
+      remove(list)
+    end
+
+    def self.rm_r(list)
+      list = (list.is_a? Array) ? list : [list]
+      list.each do |path|
+        Viking.client.delete(Path.new(path), true)
+      end
+    end
+
+    def self.rmdir(list)
+      list = (list.is_a? Array) ? list : [list]
+      list.each do |dir|
+        remove_dir(dir)
+      end
+    end
+
+    def self.touch(list)
+      list = (list.is_a? Array) ? list : [list]
+      list.each do |path|
+        Viking.client.create_new_file(Path.new(path))
+      end
+    end
+  end
+end

data/lib/viking/version.rb

@@ -0,0 +1,3 @@
+module Viking
+  VERSION = "0.0.1"
+end

data/lib/viking.rb

@@ -0,0 +1,32 @@
+require "lock_jar"
+
+LockJar.load
+
+java_import java.net.URI
+java_import org.apache.hadoop.fs.FileSystem
+java_import org.apache.hadoop.hdfs.DistributedFileSystem
+java_import org.apache.hadoop.conf.Configuration
+java_import org.apache.hadoop.fs.Path
+java_import org.apache.hadoop.fs.permission.FsPermission
+
+module Viking
+
+  def self.configure(config)
+    hostname = config[:host]
+    port     = config[:port]
+
+    path = URI.new("hdfs://#{hostname}:#{port}")
+
+    @client = DistributedFileSystem.new
+    @client.initialize__method(path, Configuration.new)
+  end
+
+  def self.client
+    @client ||= FileSystem.get_local(Configuration.new)
+  end
+end
+
+require "viking/version"
+require 'viking/file'
+require 'viking/dir'
+require 'viking/fileutilz'

data/test/test_dir_class_methods.rb

@@ -0,0 +1,114 @@
+require 'minitest/autorun'
+require 'minitest/pride'
+require File.expand_path('../../lib/viking.rb', __FILE__)
+
+class TestDirClassMethods < MiniTest::Unit::TestCase
+
+  TMP_DIR          = "/tmp/viking"
+  TEST_FILE_1      = "test_file_1.rb"
+  TEST_FILE_2      = "test_file_2.rb"
+  TEST_FILE_PATH_1 = "#{TMP_DIR}/#{TEST_FILE_1}"
+  TEST_FILE_PATH_2 = "#{TMP_DIR}/#{TEST_FILE_2}"
+
+  def setup
+    FileUtils.mkpath(TMP_DIR) unless File.exists?(TMP_DIR)
+    FileUtils.touch(TEST_FILE_PATH_1)
+    FileUtils.touch(TEST_FILE_PATH_2)
+    File.write(TEST_FILE_PATH_2, "test")
+  end
+
+  def teardown
+    FileUtils.remove_dir(TMP_DIR) if File.exists?(TMP_DIR)
+  end
+
+  def test_chdir
+    assert 0 == Viking::Dir.chdir(TMP_DIR)
+    assert Viking::Dir.pwd == TMP_DIR
+
+    Viking::Dir.chdir("/tmp") do |path|
+      assert "/tmp" == path
+      assert "/tmp" == Viking::Dir.pwd
+    end
+
+    assert Viking::Dir.pwd == TMP_DIR
+
+    assert 0 == Viking::Dir.chdir(Dir.pwd)
+  end
+
+  def test_delete
+    FileUtils.mkpath("#{TMP_DIR}/empty")
+
+    assert_raises(Java::JavaIo::IOException) { Viking::Dir.delete(TMP_DIR) }
+    assert_raises(Errno::ENOENT) { Viking::Dir.delete("#{TMP_DIR}/does-not-exists") }
+
+    assert Dir.exist?("#{TMP_DIR}/empty")
+    assert 0 == Viking::Dir.delete("#{TMP_DIR}/empty")
+    assert !Dir.exist?("#{TMP_DIR}/empty")
+  end
+
+  def test_entries
+    assert_raises(Errno::ENOENT) { Viking::Dir.delete("#{TMP_DIR}/does-not-exists") }
+    assert_raises(Errno::ENOENT) { Viking::Dir.delete(TEST_FILE_PATH_1) }
+
+    assert [TEST_FILE_1, TEST_FILE_2] == Viking::Dir.entries(TMP_DIR)
+  end
+
+  def test_exist?
+    assert Viking::Dir.exist?(TMP_DIR)
+    assert !Viking::Dir.exist?(TEST_FILE_PATH_1)
+    assert !Viking::Dir.exist?("#{TMP_DIR}/does-not-exists")
+  end
+
+  def test_exists?
+    assert Viking::Dir.exists?(TMP_DIR)
+    assert !Viking::Dir.exists?(TEST_FILE_PATH_1)
+    assert !Viking::Dir.exists?("#{TMP_DIR}/does-not-exists")
+  end
+
+  def test_foreach
+    file_names = []
+    Viking::Dir.foreach(TMP_DIR) do |file_name|
+      file_names << file_name
+    end
+    assert [TEST_FILE_1, TEST_FILE_2] == file_names
+  end
+
+  def test_getwd
+    assert Dir.pwd == Viking::Dir.getwd
+  end
+
+  def test_home
+    assert Dir.home == Viking::Dir.home
+  end
+
+  def test_mkdir
+    assert !Dir.exist?("#{TMP_DIR}/new/folder/struct")
+    Viking::Dir.mkdir("#{TMP_DIR}/new/folder/struct")
+    assert Dir.exist?("#{TMP_DIR}/new/folder/struct")
+  end
+
+  def test_open
+    assert Viking::Dir.open(TMP_DIR).instance_of? Viking::Dir
+    assert TMP_DIR == Viking::Dir.open(TMP_DIR).path
+
+    Viking::Dir.open(TMP_DIR) do |dir|
+      assert dir.instance_of? Viking::Dir
+      assert TMP_DIR == dir.path
+    end
+  end
+
+  def test_pwd
+    assert Dir.pwd == Viking::Dir.pwd
+  end
+
+  def test_rmdir
+    FileUtils.mkpath("#{TMP_DIR}/empty")
+
+    assert_raises(Java::JavaIo::IOException) { Viking::Dir.delete(TMP_DIR) }
+    assert_raises(Errno::ENOENT) { Viking::Dir.delete("#{TMP_DIR}/does-not-exists") }
+
+    assert Dir.exist?("#{TMP_DIR}/empty")
+    assert 0 == Viking::Dir.delete("#{TMP_DIR}/empty")
+    assert !Dir.exist?("#{TMP_DIR}/empty")
+  end
+end

data/test/test_dir_instance_methods.rb

@@ -0,0 +1,72 @@
+require 'minitest/autorun'
+require 'minitest/pride'
+require File.expand_path('../../lib/viking.rb', __FILE__)
+
+class TestDirInstanceMethods < MiniTest::Unit::TestCase
+
+  TMP_DIR          = "/tmp/viking"
+  TEST_FILE_1      = "test_file_1.rb"
+  TEST_FILE_2      = "test_file_2.rb"
+  TEST_FILE_PATH_1 = "#{TMP_DIR}/#{TEST_FILE_1}"
+  TEST_FILE_PATH_2 = "#{TMP_DIR}/#{TEST_FILE_2}"
+
+  def setup
+    FileUtils.mkpath(TMP_DIR) unless File.exists?(TMP_DIR)
+    FileUtils.touch(TEST_FILE_PATH_1)
+    FileUtils.touch(TEST_FILE_PATH_2)
+    File.write(TEST_FILE_PATH_2, "test")
+  end
+
+  def teardown
+    FileUtils.remove_dir(TMP_DIR) if File.exists?(TMP_DIR)
+  end
+
+  def test_close
+    d = Viking::Dir.new(TMP_DIR)
+    d.read
+    d.close
+    assert_raises(IOError) { d.read }
+    assert_raises(IOError) { d.close }
+  end
+
+  def test_each
+    d = Viking::Dir.new(TMP_DIR)
+
+    entries = []
+    d.each do |entry|
+      entries << entry
+    end
+
+    assert [TEST_FILE_1, TEST_FILE_2] == entries
+  end
+
+  def test_path
+    d = Viking::Dir.new(TMP_DIR)
+    assert TMP_DIR == d.path
+  end
+
+  def test_read
+    d = Viking::Dir.new(TMP_DIR)
+
+    assert TEST_FILE_1 == d.read
+    assert TEST_FILE_2 == d.read
+    assert d.read.nil?
+    assert d.read.nil?
+  end
+
+  def test_rewind
+    d = Viking::Dir.new(TMP_DIR)
+
+    assert TEST_FILE_1 == d.read
+    assert TEST_FILE_2 == d.read
+    assert d.read.nil?
+    assert d.read.nil?
+
+    d.rewind
+
+    assert TEST_FILE_1 == d.read
+    assert TEST_FILE_2 == d.read
+    assert d.read.nil?
+    assert d.read.nil?
+  end
+end

data/test/test_file_class_methods.rb

@@ -0,0 +1,115 @@
+require 'minitest/autorun'
+require 'minitest/pride'
+require File.expand_path('../../lib/viking.rb', __FILE__)
+
+class TestFileClassMethods < MiniTest::Unit::TestCase
+
+  TMP_DIR          = "/tmp/viking"
+  TEST_FILE_1      = "test_file_1.rb"
+  TEST_FILE_2      = "test_file_2.rb"
+  TEST_FILE_PATH_1 = "#{TMP_DIR}/#{TEST_FILE_1}"
+  TEST_FILE_PATH_2 = "#{TMP_DIR}/#{TEST_FILE_2}"
+
+  def setup
+    FileUtils.mkpath(TMP_DIR) unless File.exists?(TMP_DIR)
+    FileUtils.touch(TEST_FILE_PATH_1)
+    FileUtils.touch(TEST_FILE_PATH_2)
+    File.write(TEST_FILE_PATH_2, "test")
+  end
+
+  def teardown
+    FileUtils.remove_dir(TMP_DIR) if File.exists?(TMP_DIR)
+  end
+
+  def test_absolute_path
+    same_level    = "foo.rb"
+    up_level      = "../../foo.rb"
+    up_down_level = "../bar/biz/foo.rb"
+
+    assert File.absolute_path(same_level)    == Viking::File.absolute_path(same_level)
+    assert File.absolute_path(up_level)      == Viking::File.absolute_path(up_level)
+    assert File.absolute_path(up_down_level) == Viking::File.absolute_path(up_down_level)
+  end
+
+  def test_basename
+    assert TEST_FILE_1                == Viking::File.basename(TEST_FILE_PATH_1)
+    assert TEST_FILE_2.sub(".rb", "") == Viking::File.basename(TEST_FILE_PATH_2, ".rb")
+  end
+
+  def test_chmod
+    assert 1     == Viking::File.chmod(777, TEST_FILE_PATH_1)
+    assert 33545 == File.stat(TEST_FILE_PATH_1).mode
+  end
+
+  def test_chown
+    assert 1 == Viking::File.chown(ENV["USER"], nil, TEST_FILE_PATH_1)
+  end
+
+  def test_delete
+    assert 2 == Viking::File.delete(TEST_FILE_PATH_1, TEST_FILE_PATH_2)
+    assert_raises(IOError) { Viking::File.delete(TMP_DIR) }
+  end
+
+  def test_directory?
+    assert Viking::File.directory?(TMP_DIR)
+    assert !Viking::File.directory?(TEST_FILE_PATH_1)
+  end
+
+  def test_exist?
+    assert Viking::File.exists?(TEST_FILE_PATH_1)
+    assert Viking::File.exists?(TMP_DIR)
+    assert !Viking::File.exists?("#{TEST_FILE_PATH_1}foobar")
+  end
+
+  def test_exists?
+    assert Viking::File.exists?(TEST_FILE_PATH_1)
+    assert Viking::File.exists?(TMP_DIR)
+    assert !Viking::File.exists?("#{TEST_FILE_PATH_1}foobar")
+  end
+
+  def test_file?
+    assert !Viking::File.file?(TMP_DIR)
+    assert Viking::File.file?(TEST_FILE_PATH_1)
+  end
+
+  def test_ftype
+    assert 'directory' == Viking::File.ftype(TMP_DIR)
+    assert 'file'      == Viking::File.ftype(TEST_FILE_PATH_1)
+  end
+
+  def test_move
+    Viking::File.move(TEST_FILE_PATH_1, "#{TEST_FILE_PATH_1}.new")
+
+    assert File.exists?("#{TEST_FILE_PATH_1}.new")
+    assert !File.exists?(TEST_FILE_PATH_1)
+  end
+
+  def test_open
+    f = Viking::File.open(TEST_FILE_PATH_1) do |file|
+      file
+    end
+
+    assert f.instance_of? Viking::File
+  end
+
+  def test_rename
+    Viking::File.rename(TEST_FILE_PATH_1, "#{TEST_FILE_PATH_1}.new")
+
+    assert File.exists?("#{TEST_FILE_PATH_1}.new")
+    assert !File.exists?(TEST_FILE_PATH_1)
+  end
+
+  def test_size
+    assert 4 == Viking::File.size(TEST_FILE_PATH_2)
+    assert_raises(Java::JavaIo::FileNotFoundException) { Viking::File.size("#{TMP_DIR}/does_not_exists") }
+  end
+
+  def test_size?
+    assert 4 == Viking::File.size?(TEST_FILE_PATH_2)
+    assert Viking::File.size?("#{TMP_DIR}/does_not_exists").nil?
+  end
+
+  def test_split
+    assert [TMP_DIR, TEST_FILE_1] == Viking::File.split(TEST_FILE_PATH_1)
+  end
+end

data/test/test_file_instance_methods.rb

@@ -0,0 +1,142 @@
+require 'minitest/autorun'
+require 'minitest/pride'
+require File.expand_path('../../lib/viking.rb', __FILE__)
+
+class TestFileInstanceMethods < MiniTest::Unit::TestCase
+
+  TMP_DIR          = "/tmp/viking"
+  TEST_FILE_1      = "test_file_1.rb"
+  TEST_FILE_2      = "test_file_2.rb"
+  TEST_FILE_PATH_1 = "#{TMP_DIR}/#{TEST_FILE_1}"
+  TEST_FILE_PATH_2 = "#{TMP_DIR}/#{TEST_FILE_2}"
+
+  CONTENT = "line1\nline2\nline3\n"
+
+  def setup
+    FileUtils.mkpath(TMP_DIR) unless File.exists?(TMP_DIR)
+    FileUtils.touch(TEST_FILE_PATH_2)
+    File.write(TEST_FILE_PATH_2, CONTENT)
+  end
+
+  def teardown
+    FileUtils.remove_dir(TMP_DIR) if File.exists?(TMP_DIR)
+  end
+
+  def file1
+    Viking::File.new(TEST_FILE_PATH_1)
+  end
+
+  def file2
+    Viking::File.new(TEST_FILE_PATH_2)
+  end
+
+  def test_path
+    assert TEST_FILE_PATH_1 == file1.path
+  end
+
+  def test_size
+    assert CONTENT.bytes.to_a.size == file2.size
+  end
+
+  def test_close
+    f = file2
+    assert !f.closed?
+
+    f.close
+    assert f.closed?
+  end
+
+  def test_each_line
+    lines = []
+    file2.each_line do |line|
+      lines << line
+    end
+    assert CONTENT == lines.join("")
+  end
+
+  def test_each_char
+    chars = []
+    file2.each_char do |char|
+      chars << char
+    end
+    assert chars == CONTENT.bytes.to_a
+  end
+
+  def test_eof
+    f = file2
+    assert !f.eof
+
+    f.getc
+    assert !f.eof
+
+    f.read
+    assert f.eof
+
+    f.close
+  end
+
+  def test_eof?
+    f = file2
+    assert !f.eof?
+
+    (CONTENT.bytes.to_a.size - 1).times { f.getc }
+    assert !f.eof
+
+    f.getc
+    assert f.eof
+
+    f.close
+  end
+
+  def test_getc
+    f = file2
+    assert CONTENT.bytes.first == f.getc
+    f.close
+  end
+
+  def test_gets
+    f = file2
+    assert "#{CONTENT.split("\n").first}\n" == file2.gets("\n", nil)
+    f.close
+
+    f = file2
+    assert CONTENT[0,10] == f.gets(nil, 10)
+    f.close
+
+    f = file2
+    assert "#{CONTENT.split("\n").first}\n" == f.gets("\n", 10)
+    f.close
+  end
+
+  def test_putc
+    f = file1
+    f.putc("a")
+    f.close
+    assert File.read(TEST_FILE_PATH_1) == "a"
+
+    File.delete(TEST_FILE_PATH_1)
+
+    f = file1
+    f.putc("a".bytes.first)
+    f.close
+    assert File.read(TEST_FILE_PATH_1) == "a"
+  end
+
+  def test_puts
+    f = file1
+    f.puts("foo", "bar\n", "biz")
+    f.close
+    assert File.read(TEST_FILE_PATH_1) == "foo\nbar\nbiz\n"
+  end
+
+  def test_read
+    assert CONTENT == file2.read
+  end
+
+  def test_write
+    f = file1
+    f.write("foobar")
+    f.close
+    assert File.read(TEST_FILE_PATH_1) == "foobar"
+  end
+end