hdfs-viking 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c623269fd30fca9866de8499134312561cf37814
+  data.tar.gz: d52b739f42b36d00ea7d4fc8234c00e4f0b45a7a
+SHA512:
+  metadata.gz: 0d1404561bba13db1e7228d53aa7f4e471205151010ccd929aaad3b0f2da1afce10f4cc1c69130eeb36a1fba1f2494e216cac0b29f933f14d01a9e28af69be88
+  data.tar.gz: 4ff4fd3b3d32945a45d004968359defb23fef630b8dd95bfc795f32a2335724bee7fd5fc37b853cba97f31799b063f977c870be8f974298115980816dfb20367

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+Jarfile.lock

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in viking.gemspec
+gemspec

data/Jarfile

@@ -0,0 +1,3 @@
+repository 'https://repository.cloudera.com/artifactory/cloudera-repos'
+jar 'org.apache.hadoop:hadoop-common:2.0.0-cdh4.4.0'
+jar 'org.apache.hadoop:hadoop-hdfs:2.0.0-cdh4.4.0'

data/Jarfile.lock

@@ -0,0 +1,124 @@
+---
+version: 0.7.5
+groups:
+  default:
+    dependencies:
+    - asm:asm:jar:3.1
+    - com.google.code.findbugs:jsr305:jar:1.3.9
+    - com.google.guava:guava:jar:11.0.2
+    - com.google.protobuf:protobuf-java:jar:2.4.0a
+    - com.jcraft:jsch:jar:0.1.42
+    - com.sun.jersey:jersey-core:jar:1.8
+    - com.sun.jersey:jersey-json:jar:1.8
+    - com.sun.jersey:jersey-server:jar:1.8
+    - com.sun.xml.bind:jaxb-impl:jar:2.2.3-1
+    - com.thoughtworks.paranamer:paranamer:jar:2.3
+    - commons-beanutils:commons-beanutils-core:jar:1.8.0
+    - commons-beanutils:commons-beanutils:jar:1.7.0
+    - commons-cli:commons-cli:jar:1.2
+    - commons-codec:commons-codec:jar:1.4
+    - commons-collections:commons-collections:jar:3.2.1
+    - commons-configuration:commons-configuration:jar:1.6
+    - commons-daemon:commons-daemon:jar:1.0.3
+    - commons-digester:commons-digester:jar:1.8
+    - commons-el:commons-el:jar:1.0
+    - commons-httpclient:commons-httpclient:jar:3.1
+    - commons-io:commons-io:jar:2.1
+    - commons-lang:commons-lang:jar:2.5
+    - commons-logging:commons-logging:jar:1.1.1
+    - commons-net:commons-net:jar:3.1
+    - javax.activation:activation:jar:1.1
+    - javax.servlet.jsp:jsp-api:jar:2.1
+    - javax.servlet:servlet-api:jar:2.5
+    - javax.xml.bind:jaxb-api:jar:2.2.2
+    - jline:jline:jar:0.9.94
+    - junit:junit:jar:4.8.2
+    - log4j:log4j:jar:1.2.17
+    - net.java.dev.jets3t:jets3t:jar:0.6.1
+    - net.sf.kosmosfs:kfs:jar:0.3
+    - org.apache.avro:avro:jar:1.7.4
+    - org.apache.commons:commons-compress:jar:1.4.1
+    - org.apache.commons:commons-math:jar:2.1
+    - org.apache.hadoop:hadoop-annotations:jar:2.0.0-cdh4.4.0
+    - org.apache.hadoop:hadoop-auth:jar:2.0.0-cdh4.4.0
+    - org.apache.hadoop:hadoop-common:jar:2.0.0-cdh4.4.0
+    - org.apache.hadoop:hadoop-hdfs:jar:2.0.0-cdh4.4.0
+    - org.apache.zookeeper:zookeeper:jar:3.4.5-cdh4.4.0
+    - org.codehaus.jackson:jackson-core-asl:jar:1.8.8
+    - org.codehaus.jackson:jackson-jaxrs:jar:1.7.1
+    - org.codehaus.jackson:jackson-mapper-asl:jar:1.8.8
+    - org.codehaus.jackson:jackson-xc:jar:1.7.1
+    - org.codehaus.jettison:jettison:jar:1.1
+    - org.mockito:mockito-all:jar:1.8.5
+    - org.mortbay.jetty:jetty-util:jar:6.1.26.cloudera.2
+    - org.mortbay.jetty:jetty:jar:6.1.26.cloudera.2
+    - org.slf4j:slf4j-api:jar:1.6.1
+    - org.slf4j:slf4j-log4j12:jar:1.6.1
+    - org.tukaani:xz:jar:1.0
+    - org.xerial.snappy:snappy-java:jar:1.0.4.1
+    - stax:stax-api:jar:1.0.1
+    - tomcat:jasper-compiler:jar:5.5.23
+    - tomcat:jasper-runtime:jar:5.5.23
+    - xmlenc:xmlenc:jar:0.52
+    artifacts:
+    - jar:org.apache.hadoop:hadoop-common:jar:2.0.0-cdh4.4.0:
+        transitive:
+          com.jcraft:jsch:jar:0.1.42: {}
+          org.mortbay.jetty:jetty-util:jar:6.1.26.cloudera.2: {}
+          com.sun.jersey:jersey-core:jar:1.8: {}
+          com.sun.jersey:jersey-server:jar:1.8:
+            asm:asm:jar:3.1: {}
+          com.google.guava:guava:jar:11.0.2:
+            com.google.code.findbugs:jsr305:jar:1.3.9: {}
+          org.apache.avro:avro:jar:1.7.4:
+            org.apache.commons:commons-compress:jar:1.4.1:
+              org.tukaani:xz:jar:1.0: {}
+            org.xerial.snappy:snappy-java:jar:1.0.4.1: {}
+            com.thoughtworks.paranamer:paranamer:jar:2.3: {}
+          org.apache.commons:commons-math:jar:2.1: {}
+          javax.servlet.jsp:jsp-api:jar:2.1: {}
+          org.codehaus.jackson:jackson-core-asl:jar:1.8.8: {}
+          commons-cli:commons-cli:jar:1.2: {}
+          net.sf.kosmosfs:kfs:jar:0.3: {}
+          org.apache.hadoop:hadoop-auth:jar:2.0.0-cdh4.4.0: {}
+          xmlenc:xmlenc:jar:0.52: {}
+          org.slf4j:slf4j-log4j12:jar:1.6.1: {}
+          commons-el:commons-el:jar:1.0: {}
+          junit:junit:jar:4.8.2: {}
+          org.apache.hadoop:hadoop-annotations:jar:2.0.0-cdh4.4.0: {}
+          org.slf4j:slf4j-api:jar:1.6.1: {}
+          org.mortbay.jetty:jetty:jar:6.1.26.cloudera.2: {}
+          org.mockito:mockito-all:jar:1.8.5: {}
+          javax.servlet:servlet-api:jar:2.5: {}
+          com.sun.jersey:jersey-json:jar:1.8:
+            com.sun.xml.bind:jaxb-impl:jar:2.2.3-1:
+              javax.xml.bind:jaxb-api:jar:2.2.2:
+                javax.activation:activation:jar:1.1: {}
+            org.codehaus.jackson:jackson-jaxrs:jar:1.7.1: {}
+            org.codehaus.jackson:jackson-xc:jar:1.7.1: {}
+            org.codehaus.jettison:jettison:jar:1.1:
+              stax:stax-api:jar:1.0.1: {}
+          com.google.protobuf:protobuf-java:jar:2.4.0a: {}
+          commons-lang:commons-lang:jar:2.5: {}
+          tomcat:jasper-compiler:jar:5.5.23: {}
+          org.codehaus.jackson:jackson-mapper-asl:jar:1.8.8: {}
+          tomcat:jasper-runtime:jar:5.5.23: {}
+          net.java.dev.jets3t:jets3t:jar:0.6.1: {}
+          org.apache.zookeeper:zookeeper:jar:3.4.5-cdh4.4.0:
+            jline:jline:jar:0.9.94: {}
+          commons-io:commons-io:jar:2.1: {}
+          commons-httpclient:commons-httpclient:jar:3.1: {}
+          commons-logging:commons-logging:jar:1.1.1: {}
+          commons-net:commons-net:jar:3.1: {}
+          commons-codec:commons-codec:jar:1.4: {}
+          commons-configuration:commons-configuration:jar:1.6:
+            commons-collections:commons-collections:jar:3.2.1: {}
+            commons-beanutils:commons-beanutils-core:jar:1.8.0: {}
+            commons-digester:commons-digester:jar:1.8:
+              commons-beanutils:commons-beanutils:jar:1.7.0: {}
+          log4j:log4j:jar:1.2.17: {}
+    - jar:org.apache.hadoop:hadoop-hdfs:jar:2.0.0-cdh4.4.0:
+        transitive:
+          commons-daemon:commons-daemon:jar:1.0.3: {}
+remote_repositories:
+- https://repository.cloudera.com/artifactory/cloudera-repos

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Erik Selin
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,49 @@
+# Viking
+
+A familiar jRuby hdfs wrapper.
+
+## Goal
+
+The goal is to provide ways that are similar to the common ruby file system
+api's for interacting with hdfs. All hdfs functionallity is powered by the java
+hdfs classes.
+
+## Status
+
+### Available
+
+ - `File`
+ - `Dir`
+ - `FileUtils`
+
+### Not available yet but on the todo
+
+ - `File#fnmatch`
+ - `File#fnmatch?`
+ - `Dir#glob`
+ - `FileUtils#copy`
+
+## Example usage
+
+    # Set up hdfs config
+    Viking.configure({
+      host: '127.0.0.1',
+      port: 54310
+    })
+
+    # If "/some/data" exists and it is a file then print its content. If it is
+    # a directory then rename it. If it does not exist then we create it.
+    path = "/some/data"
+    if Viking::File.exists? path
+      if Viking::File.file? path
+        Viking::File.open(path) do |file|
+          puts "Reading data from #{f.path}:"
+          puts f.read
+        end
+      else
+        Viking::File.rename(path, "/some/dir")
+      end
+    else
+      Viking::Dir.mkdir(path)
+    end
+

data/Rakefile

@@ -0,0 +1,19 @@
+#!/usr/bin/env rake
+
+require "bundler/gem_tasks"
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << 'lib/viking'
+  t.test_files = FileList['test/test_*.rb']
+  t.verbose = true
+end
+
+task :prepare do
+  require 'lock_jar'
+  LockJar.lock
+  lockfile = File.expand_path( "../Jarfile.lock", __FILE__ )
+  LockJar.install( :lockfile => lockfile )
+end
+
+task :default => :prepare

data/lib/viking/dir.rb

@@ -0,0 +1,137 @@
+module Viking
+  class Dir
+
+    def self.format(dirname)
+      dirname.to_s.sub(/^[^:]*:\/*\//, '/')
+    end
+
+    def self.chdir(dirname=home, &block)
+      if block
+        main_dirname = pwd
+        Viking.client.set_working_directory(Path.new(dirname))
+
+        result = yield dirname
+
+        Viking.client.set_working_directory(Path.new(main_dirname))
+        result
+      else
+        Viking.client.set_working_directory(Path.new(dirname))
+        0
+      end
+    end
+
+    def self.delete(dirname)
+      if exist?(dirname)
+        Viking.client.delete(Path.new(dirname), false)
+        0
+      else
+        raise Errno::ENOENT.new("No such file or directory - No such directory: #{dirname}")
+      end
+    end
+
+    def self.entries(dirname)
+      if exist?(dirname)
+        iterator = Viking.client.list_located_status(Path.new(dirname))
+        entries  = []
+
+        while(iterator.has_next) do
+          entry = format(iterator.next.path)
+          entries << Viking::File.basename(entry)
+        end
+
+        entries
+      else
+        raise Errno::ENOENT.new("No such file or directory - No such directory: #{dirname}")
+      end
+    end
+
+    def self.exist?(dirname)
+      Viking::File.exist?(dirname) && Viking::File.directory?(dirname)
+    end
+
+    def self.exists?(dirname)
+      exist?(dirname)
+    end
+
+    def self.foreach(dirname)
+      entries(dirname).each { |entry| yield entry }
+    end
+
+    def self.getwd
+      format(Viking.client.get_working_directory)
+    end
+
+    def self.home
+      format(Viking.client.get_home_directory)
+    end
+
+    def self.mkdir(dirname)
+      Viking.client.mkdirs(Path.new(dirname))
+      0
+    end
+
+    def self.open(dirname, &block)
+      dir = Viking::Dir.new(dirname)
+      if block
+        value = yield dir
+        dir.close
+        value
+      else
+        dir
+      end
+    end
+
+    def self.pwd
+      getwd
+    end
+
+    def self.rmdir(dirname)
+      delete(dirname)
+    end
+
+    def initialize(path)
+      @path   = path
+      @closed = false
+    end
+
+    def close
+      check_closed
+      @closed = true
+    end
+
+    def each
+      check_closed
+      entries.each do |file_name|
+        yield file_name
+      end
+    end
+
+    def path
+      @path
+    end
+
+    def read
+      check_closed
+      rewind if @entries.nil?
+      entry = @entries[@pos]
+      @pos += 1 unless entry.nil?
+      entry
+    end
+
+    def rewind
+      check_closed
+      @pos     = 0
+      @entries = entries
+    end
+
+    private
+
+    def check_closed
+      raise IOError.new("closed directory") if @closed
+    end
+
+    def entries
+      Viking::Dir.entries(path)
+    end
+  end
+end

data/lib/viking/file.rb

@@ -0,0 +1,224 @@
+module Viking
+  class File
+
+    def self.absolute_path(file_name, dir_string=Viking.client.get_working_directory)
+      dir_string = dir_string.to_s.sub(/[^\/]*:\/*\//, '/')
+      IO::File.absolute_path(file_name, dir_string)
+    end
+
+    def self.basename(file_name, suffix = nil)
+      base = file_name.split("/").last
+      suffix.nil? ? base : base.sub(suffix, '')
+    end
+
+    def self.chmod(mode_int, *file_names)
+      permission = FsPermission.new(mode_int)
+      file_names.each do |file_name|
+        Viking.client.set_permission(Path.new(file_name), permission)
+      end
+      file_names.size
+    end
+
+    def self.chown(owner_name, group_name, *file_names)
+      file_names.each do |file_name|
+        Viking.client.set_owner(Path.new(file_name), owner_name, group_name)
+      end
+      file_names.size
+    end
+
+    def self.delete(*file_names)
+      file_names.each do |file_name|
+        path = Path.new(file_name)
+        if file?(file_name)
+          Viking.client.delete(Path.new(file_name), false)
+        else
+          raise IOError, "File::delete can only delete files. Attempted to delete directory #{file_name}."
+        end
+      end
+      file_names.size
+    end
+
+    def self.directory?(file_name)
+      Viking.client.is_directory(Path.new(file_name))
+    end
+
+    def self.exist?(file_name)
+      exists?(file_name)
+    end
+
+    def self.exists?(file_name)
+      Viking.client.exists(Path.new(file_name))
+    end
+
+    def self.file?(file_name)
+      Viking.client.is_file(Path.new(file_name))
+    end
+
+    def self.ftype(file_name)
+      directory?(file_name) ? 'directory' : 'file'
+    end
+
+    def self.move(old_name, new_name)
+      rename(old_name, new_name)
+    end
+
+    def self.open(file_name, &block)
+      file = Viking::File.new(file_name)
+      if block
+        value = yield file
+        file.close unless file.closed?
+        value
+      else
+        file
+      end
+    end
+
+    def self.rename(old_name, new_name)
+      Viking.client.rename(Path.new(old_name), Path.new(new_name))
+    end
+
+    def self.size(file_name)
+      Viking.client.get_file_status(Path.new(file_name)).get_len
+    end
+
+    def self.size?(file_name)
+      exists?(file_name) ? size(file_name) : nil
+    end
+
+    def self.split(file_name)
+      name = basename(file_name)
+      path = file_name.sub(/\/#{name}$/,'')
+      [path, name]
+    end
+
+    def initialize(file_name)
+      @path     = file_name
+      @reading  = false
+      @writting = false
+      @closed   = false
+    end
+
+    def path
+      @path
+    end
+
+    def size
+      Viking.client.get_file_status(Path.new(path)).get_len
+    end
+
+    def close
+      unless closed?
+        reader.close  if @reading
+        writter.close if @writting
+
+        @reading  = false
+        @writting = false
+
+        @closed = true
+      end
+    end
+
+    def closed?
+      @closed
+    end
+
+    def each_line(sep="\n", limit=nil)
+      while((line = gets(sep, limit)) != nil) do
+        yield line
+      end
+      close
+    end
+
+    def each_char
+      while(char = getc) do
+        yield char
+      end
+      close
+    end
+
+    def eof
+      eof?
+    end
+
+    def eof?
+      current_byte.nil?
+    end
+
+    def getc
+      byte = current_byte
+      next_byte
+      byte
+    end
+
+    def gets(sep=nil, limit=nil)
+      sep_bytes = sep.bytes.to_a if sep
+      bytes     = []
+      while(b = getc) do
+        bytes << b
+        break if sep   && bytes.last(sep_bytes.size) == sep_bytes
+        break if limit && bytes.size == limit
+      end
+      bytes.empty? ? nil : bytes.pack("C*")
+    end
+
+    def putc(obj)
+      if obj.is_a? Numeric
+        writter.java_send :write, [Java::int], obj
+      else
+        byte = obj.to_s.bytes.first
+        writter.write(byte)
+      end
+    end
+
+    def puts(*objs)
+      separator = "\n"
+      string    = objs.reduce("") do |acc, obj|
+        if obj.end_with?(separator)
+          acc << obj
+        else
+          acc << obj
+          acc << separator
+        end
+      end
+      write(string)
+    end
+
+    def read(limit=nil)
+      gets(nil, limit)
+    end
+
+    def write(string)
+      writter.write_bytes(string)
+    end
+
+    private
+
+    def parse_byte(b)
+      b == -1 ? nil : b
+    end
+
+    def next_byte
+      b  = reader.read
+      @b = parse_byte(b)
+    end
+
+    def current_byte
+      next_byte unless @reading
+      @b
+    end
+
+    def reader
+      @reader ||= begin
+        @reading = true
+        Viking.client.open(Path.new(path))
+      end
+    end
+
+    def writter
+      @writter ||= begin
+        @writting = true
+        Viking.client.create(Path.new(path))
+      end
+    end
+  end
+end