ruby-hdfs 0.1.0

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -0,0 +1,22 @@
+# MacOS X
+.DS_Store
+
+# Emacs
+*~
+\#*
+.\#*
+
+# Vim
+*.swp
+
+# General
+coverage
+rdoc
+pkg
+
+# Project-specific
+*.bak
+/ext/hdfs/Makefile
+/ext/hdfs/*.log
+*.o
+*.so

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2010 Alexander Staubo
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,59 @@
+== Hadoop DFS (HDFS) bindings for Ruby
+
+This library provides native C bindings to Hadoop's libhdfs, for interacting with Hadoop DFS.
+
+=== Requirements
+
+You will need:
+
+* Java JDK and JRE (yes, both). The build file will attempt to find it for you.
+* Hadoop's libhdfs. On Ubuntu/Debian you will need libhdfs0 and libhdfs0-dev.
+* Hadoop Core and DFS libraries.
+
+=== Installation
+
+Install from gems. Note that you will need to provide JAVA_HOME so the compiler can find the
+required libraries.
+
+The installation will attempt to discover the location of the libaries, but if it fails, 
+you can try setting the environment variable JAVA_LIB to the library path of the JDK/JRE.
+
+Installing with a specific Java JDK:
+
+  sudo env JAVA_HOME=/usr/lib/jvm/java-6-openjdk gem install ruby-hdfs
+  
+=== Using
+  
+The library also depends on an installation of Hadoop DFS. The Cloudera distribution of 
+Hadoop is pretty good:
+
+  http://www.cloudera.com/distribution
+  
+Sample classpath setup (yes, welcome to JAR hell):
+
+  export CLASSPATH=$CLASSPATH:/usr/lib/hadoop/hadoop-0.18.3-6cloudera0.3.0-core.jar
+  for jarfile in /usr/lib/hadoop/lib/*.jar; do
+    export CLASSPATH=$CLASSPATH:$jarfile
+  done
+
+Wait, there's more. You will also need libjvm.so in your library path, which comes with 
+the JRE. This might work:
+
+  export LD_LIBRARY_PATH=/usr/lib/jvm/java-6-openjdk/jre/lib/i386/server
+
+=== Known issues
+
+libhdfs will sometimes throw exceptions, which will be output instead of caught by Ruby.
+This is annoying but harmless.
+
+=== Building from source
+
+To build from source:
+
+  rake compile
+ 
+On completion, the compiled extension will be available in ext/hdfs.
+
+== Copyright
+
+Copyright (c) 2010 Alexander Staubo. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,58 @@
+require 'rubygems'
+require 'rake'
+require 'rake/extensiontask'  # From rake-compiler gem
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "ruby-hdfs"
+    gem.summary = %Q{Native C bindings to Hadoop's libhdfs, for interacting with Hadoop HDFS.}
+    gem.description = %Q{Native C bindings to Hadoop's libhdfs, for interacting with Hadoop HDFS.}
+    gem.email = "alex@bengler.no"
+    gem.homepage = "http://github.com/alexstaubo/ruby-hdfs"
+    gem.authors = ["Alexander Staubo"]
+    gem.extensions = ["ext/hdfs/extconf.rb"]
+    gem.require_paths = ["lib"]
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end
+
+Rake::ExtensionTask.new('hdfs') do |ext|
+end
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |test|
+    test.libs << 'test'
+    test.pattern = 'test/**/test_*.rb'
+    test.verbose = true
+  end
+rescue LoadError
+  task :rcov do
+    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+  end
+end
+
+task :test => :check_dependencies
+
+task :default => :test
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "ruby-hdfs #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/ext/hdfs/extconf.rb

@@ -0,0 +1,49 @@
+require 'mkmf'
+
+java_home = ENV["JAVA_HOME"]
+unless java_home
+  %w(
+    /usr/lib/jvm/java-6-openjdk
+  ).each do |path|
+    if File.directory?(path)
+      java_home = path
+      $stderr << "Warning: Automatically guessed #{path} as Java home, might not be correct.\n"
+    end
+  end
+  abort("JAVA_HOME needs to be defined.") unless java_home
+end
+puts("Java home: #{java_home}")
+
+java_lib_path = ENV["JAVA_LIB"]
+unless java_lib_path
+  libjvm = "libjvm.so"
+  [
+    "#{java_home}/lib",
+    "#{java_home}/lib/*/client",
+    "#{java_home}/lib/*/server",
+    "#{java_home}/jre/lib",
+    "#{java_home}/jre/lib/*/client",
+    "#{java_home}/jre/lib/*/server"
+  ].each do |glob|
+    Dir.glob(glob).each do |path|
+      if File.exist?(File.join(path, libjvm))
+        java_lib_path ||= path 
+        break
+      end
+    end
+  end
+  abort("Could not determine Java library path (need #{libjvm})") unless java_lib_path
+end
+puts("Java library path: #{java_lib_path}")
+
+java_include_paths = Dir.glob("#{java_home}/include/**/.").map { |s| s.gsub(/\/\.$/, '') }
+puts("Java include paths: #{java_include_paths.join(', ')}")
+java_include_paths.each do |path|
+  $INCFLAGS << " -I#{path}"
+end
+
+dir_config("hdfs")
+find_library("jvm", nil, java_lib_path)
+find_library("hdfs", nil, java_lib_path)
+have_library("c", "main")
+create_makefile("hdfs")

data/ext/hdfs/hdfs.c

@@ -0,0 +1,247 @@
+#include "ruby.h"
+#include "hdfs.h"
+
+#include <assert.h>
+#include <string.h>
+#include <ctype.h>
+
+// Override flag in hdfs.h
+#define HDFS_O_RDONLY 0
+#define HDFS_O_WRONLY 2
+
+static VALUE m_hadoop;
+static VALUE m_dfs;
+static VALUE c_file_system;
+static VALUE c_file;
+static VALUE e_dfs_exception;
+static VALUE e_file_error;
+static VALUE e_could_not_open;
+
+static const HDFS_DEFAULT_BLOCK_SIZE = 134217728;
+static const char* HDFS_DEFAULT_HOST = "localhost";
+static const int HDFS_DEFAULT_PORT = 9000;
+
+/*
+ * Data structs
+ */
+
+typedef struct FSData {
+  hdfsFS fs;
+} FSData;
+
+typedef struct FileData {
+  hdfsFS fs;
+  hdfsFile file;
+} FileData;
+
+void free_fs_data(FSData* data) {
+  if (data && data->fs != NULL) {
+    hdfsDisconnect(data->fs);
+    data->fs = NULL;
+  }
+}
+
+void free_file_data(FileData* data) {
+  if (data && data->file != NULL) {
+    hdfsCloseFile(data->fs, data->file);
+    data->file = NULL;
+  }
+}
+
+/*
+ * File system interface
+ */
+
+VALUE HDFS_File_System_alloc(VALUE klass) {
+  FSData* data = ALLOC_N(FSData, 1);
+  data->fs = NULL;
+  VALUE instance = Data_Wrap_Struct(klass, NULL, free_fs_data, data);
+  return instance;
+}
+
+/**
+ * call-seq:
+ *    hdfs.new -> hdfs
+ *
+ * Creates a new HDFS client connection.
+ */
+VALUE HDFS_File_System_initialize(VALUE self, VALUE host, VALUE port) {
+  FSData* data = NULL;
+  Data_Get_Struct(self, FSData, data);
+  data->fs = hdfsConnect(
+    RTEST(host) ? RSTRING_PTR(host) : HDFS_DEFAULT_HOST,
+    RTEST(port) ? NUM2INT(port) : HDFS_DEFAULT_PORT);  
+  return self;
+}
+
+/**
+ * call-seq:
+ *    hdfs.disconnect -> nil
+ *
+ * Disconnects the client connection.
+ */
+VALUE HDFS_File_System_disconnect(VALUE self) {
+  FSData* data = NULL;
+  Data_Get_Struct(self, FSData, data);
+  if (data->fs != NULL) {
+    hdfsDisconnect(data->fs);
+    data->fs = NULL;
+  }
+  return Qnil;
+}
+
+VALUE HDFS_File_System_delete(VALUE self, VALUE path) {
+  FSData* data = NULL;
+  Data_Get_Struct(self, FSData, data);
+  int value = hdfsDelete(data->fs, RSTRING_PTR(path));
+  return value == 0 ? Qtrue : Qfalse;
+}
+
+VALUE HDFS_File_System_exist(VALUE self, VALUE path) {
+  FSData* data = NULL;
+  Data_Get_Struct(self, FSData, data);
+  int value = hdfsExists(data->fs, RSTRING_PTR(path));
+  return value == 0 ? Qtrue : Qfalse;
+}
+
+/**
+ * call-seq:
+ *    hdfs.open -> file
+ *
+ * Opens a file.
+ */
+VALUE HDFS_File_System_open(VALUE self, VALUE path, VALUE mode, VALUE options) {
+  FSData* data = NULL;
+  Data_Get_Struct(self, FSData, data);
+
+  int flags = 0;
+  if (strcmp("r", STR2CSTR(mode)) == 0) {
+    flags = HDFS_O_RDONLY;
+  } else if (strcmp("w", STR2CSTR(mode)) == 0) {
+    flags = HDFS_O_WRONLY;
+  } else {
+    rb_raise(rb_eArgError, "Mode must be 'r' or 'w'");
+    return;
+  }
+  VALUE r_buffer_size = rb_hash_aref(options, rb_eval_string(":buffer_size"));
+  VALUE r_replication = rb_hash_aref(options, rb_eval_string(":replication"));
+  VALUE r_block_size = rb_hash_aref(options, rb_eval_string(":block_size"));
+  hdfsFile file = hdfsOpenFile(data->fs, RSTRING_PTR(path), flags, 
+    RTEST(r_buffer_size) ? NUM2INT(r_buffer_size) : 0, 
+    RTEST(r_replication) ? NUM2INT(r_replication) : 0, 
+    RTEST(r_block_size) ? NUM2INT(r_block_size) : HDFS_DEFAULT_BLOCK_SIZE);
+  if (file == NULL) {
+    rb_raise(e_could_not_open, "Could not open file %s", RSTRING_PTR(path));
+    return;
+  }
+
+  FileData* file_data = ALLOC_N(FileData, 1);
+  file_data->fs = data->fs;
+  file_data->file = file;
+  VALUE file_instance = Data_Wrap_Struct(c_file, NULL, free_file_data, file_data);
+  return file_instance;
+}
+
+/*
+ * File interface
+ */
+
+VALUE HDFS_File_read(VALUE self, VALUE length) {
+  FileData* data = NULL;
+  Data_Get_Struct(self, FileData, data);
+  char* buffer = ALLOC_N(char, length);
+  MEMZERO(buffer, char, length);
+  tSize bytes_read = hdfsRead(data->fs, data->file, buffer, NUM2INT(length));
+  if (bytes_read == -1) {
+    rb_raise(e_file_error, "Failed to read data");
+  }
+  return rb_tainted_str_new2(buffer);
+}
+
+VALUE HDFS_File_write(VALUE self, VALUE bytes) {
+  FileData* data = NULL;
+  Data_Get_Struct(self, FileData, data);
+  tSize bytes_written = hdfsWrite(data->fs, data->file, RSTRING_PTR(bytes), RSTRING_LEN(bytes));
+  if (bytes_written == -1) {
+    rb_raise(e_file_error, "Failed to write data");
+  }
+  return INT2NUM(bytes_written);
+}
+
+VALUE HDFS_File_tell(VALUE self) {
+  FileData* data = NULL;
+  Data_Get_Struct(self, FileData, data);
+  tSize offset = hdfsTell(data->fs, data->file);
+  if (offset == -1) {
+    rb_raise(e_file_error, "Failed to read position");
+  }
+  return INT2NUM(offset);
+}
+
+VALUE HDFS_File_seek(VALUE self, VALUE offset) {
+  FileData* data = NULL;
+  Data_Get_Struct(self, FileData, data);
+  int result = hdfsSeek(data->fs, data->file, NUM2INT(offset));
+  return result == 0 ? Qtrue : Qfalse;
+}
+
+VALUE HDFS_File_flush(VALUE self) {
+  FileData* data = NULL;
+  Data_Get_Struct(self, FileData, data);
+  int result = hdfsFlush(data->fs, data->file);
+  if (result != 0) {
+    rb_raise(e_file_error, "Flush failed");
+  }
+  return Qnil;
+}
+
+VALUE HDFS_File_available(VALUE self) {
+  FileData* data = NULL;
+  Data_Get_Struct(self, FileData, data);
+  int result = hdfsAvailable(data->fs, data->file);
+  if (result == -1) {
+    rb_raise(e_file_error, "Failed to get available data");
+  }
+  return INT2NUM(result);
+}
+
+VALUE HDFS_File_close(VALUE self) {
+  FileData* data = NULL;
+  Data_Get_Struct(self, FileData, data);
+  if (data->file != NULL) {
+    hdfsCloseFile(data->fs, data->file);
+    data->file = NULL;
+  }
+  return Qnil;
+}
+
+/*
+ * Extension initialization
+ */
+ 
+void Init_hdfs() {
+  m_hadoop = rb_define_module("Hadoop");
+  m_dfs = rb_define_module_under(m_hadoop, "DFS");
+
+  c_file_system = rb_define_class_under(m_dfs, "FileSystem", rb_cObject);
+  rb_define_alloc_func(c_file_system, HDFS_File_System_alloc);
+  rb_define_method(c_file_system, "initialize", HDFS_File_System_initialize, 2);
+  rb_define_method(c_file_system, "disconnect", HDFS_File_System_disconnect, 0);
+  rb_define_method(c_file_system, "open", HDFS_File_System_open, 3);
+  rb_define_method(c_file_system, "delete", HDFS_File_System_delete, 1);
+  rb_define_method(c_file_system, "exist?", HDFS_File_System_exist, 1);
+
+  c_file = rb_define_class_under(m_dfs, "File", rb_cObject);
+  rb_define_method(c_file, "read", HDFS_File_read, 1);
+  rb_define_method(c_file, "write", HDFS_File_write, 1);
+  rb_define_method(c_file, "<<", HDFS_File_write, 1);
+  rb_define_method(c_file, "seek", HDFS_File_seek, 1);
+  rb_define_method(c_file, "tell", HDFS_File_tell, 0);
+  rb_define_method(c_file, "flush", HDFS_File_flush, 0);
+  rb_define_method(c_file, "available", HDFS_File_available, 0);
+  rb_define_method(c_file, "close", HDFS_File_close, 0);
+  
+  e_dfs_exception = rb_define_class_under(m_dfs, "DFSException", rb_eStandardError);
+  e_file_error = rb_define_class_under(m_dfs, "FileError", e_dfs_exception);  
+  e_could_not_open = rb_define_class_under(m_dfs, "CouldNotOpenFileError", e_file_error);  
+}

data/ext/hdfs/hdfs.h

@@ -0,0 +1,414 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFS_HDFS_H
+#define LIBHDFS_HDFS_H
+
+#include <sys/types.h>
+#include <sys/stat.h>
+
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdint.h>
+#include <string.h>
+#include <stdlib.h>
+#include <time.h>
+#include <errno.h>
+
+#include <jni.h>
+
+#ifndef O_RDONLY
+#define O_RDONLY 1
+#endif
+
+#ifndef O_WRONLY 
+#define O_WRONLY 2
+#endif
+
+#ifndef EINTERNAL
+#define EINTERNAL 255 
+#endif
+
+
+/** All APIs set errno to meaningful values */
+#ifdef __cplusplus
+extern  "C" {
+#endif
+
+    /**
+     * Some utility decls used in libhdfs.
+     */
+
+    typedef int32_t   tSize; /// size of data for read/write io ops 
+    typedef time_t    tTime; /// time type
+    typedef int64_t   tOffset;/// offset within the file
+    typedef uint16_t  tPort; /// port
+    typedef enum tObjectKind {
+        kObjectKindFile = 'F',
+        kObjectKindDirectory = 'D',
+    } tObjectKind;
+
+
+    /**
+     * The C reflection of org.apache.org.hadoop.FileSystem .
+     */
+    typedef void* hdfsFS;
+
+    
+    /**
+     * The C equivalent of org.apache.org.hadoop.FSData(Input|Output)Stream .
+     */
+    enum hdfsStreamType
+    {
+        UNINITIALIZED = 0,
+        INPUT = 1,
+        OUTPUT = 2,
+    };
+
+    
+    /**
+     * The 'file-handle' to a file in hdfs.
+     */
+    struct hdfsFile_internal {
+        void* file;
+        enum hdfsStreamType type;
+    };
+    typedef struct hdfsFile_internal* hdfsFile;
+      
+
+    /** 
+     * hdfsConnect - Connect to a hdfs file system.
+     * Connect to the hdfs.
+     * @param host A string containing either a host name, or an ip address
+     * of the namenode of a hdfs cluster. 'host' should be passed as NULL if
+     * you want to connect to local filesystem. 'host' should be passed as
+     * 'default' (and port as 0) to used the 'configured' filesystem
+     * (hadoop-site/hadoop-default.xml).
+     * @param port The port on which the server is listening.
+     * @return Returns a handle to the filesystem or NULL on error.
+     */
+    hdfsFS hdfsConnect(const char* host, tPort port);
+
+
+    /** 
+     * hdfsDisconnect - Disconnect from the hdfs file system.
+     * Disconnect from hdfs.
+     * @param fs The configured filesystem handle.
+     * @return Returns 0 on success, -1 on error.  
+     */
+    int hdfsDisconnect(hdfsFS fs);
+        
+
+    /** 
+     * hdfsOpenFile - Open a hdfs file in given mode.
+     * @param fs The configured filesystem handle.
+     * @param path The full path to the file.
+     * @param flags Either O_RDONLY or O_WRONLY, for read-only or write-only.
+     * @param bufferSize Size of buffer for read/write - pass 0 if you want
+     * to use the default configured values.
+     * @param replication Block replication - pass 0 if you want to use
+     * the default configured values.
+     * @param blocksize Size of block - pass 0 if you want to use the
+     * default configured values.
+     * @return Returns the handle to the open file or NULL on error.
+     */
+    hdfsFile hdfsOpenFile(hdfsFS fs, const char* path, int flags,
+                          int bufferSize, short replication, tSize blocksize);
+
+
+    /** 
+     * hdfsCloseFile - Close an open file. 
+     * @param fs The configured filesystem handle.
+     * @param file The file handle.
+     * @return Returns 0 on success, -1 on error.  
+     */
+    int hdfsCloseFile(hdfsFS fs, hdfsFile file);
+
+
+    /** 
+     * hdfsExists - Checks if a given path exsits on the filesystem 
+     * @param fs The configured filesystem handle.
+     * @param path The path to look for
+     * @return Returns 0 on success, -1 on error.  
+     */
+    int hdfsExists(hdfsFS fs, const char *path);
+
+
+    /** 
+     * hdfsSeek - Seek to given offset in file. 
+     * This works only for files opened in read-only mode. 
+     * @param fs The configured filesystem handle.
+     * @param file The file handle.
+     * @param desiredPos Offset into the file to seek into.
+     * @return Returns 0 on success, -1 on error.  
+     */
+    int hdfsSeek(hdfsFS fs, hdfsFile file, tOffset desiredPos); 
+
+
+    /** 
+     * hdfsTell - Get the current offset in the file, in bytes.
+     * @param fs The configured filesystem handle.
+     * @param file The file handle.
+     * @return Current offset, -1 on error.
+     */
+    tOffset hdfsTell(hdfsFS fs, hdfsFile file);
+
+
+    /** 
+     * hdfsRead - Read data from an open file.
+     * @param fs The configured filesystem handle.
+     * @param file The file handle.
+     * @param buffer The buffer to copy read bytes into.
+     * @param length The length of the buffer.
+     * @return Returns the number of bytes actually read, possibly less
+     * than than length;-1 on error.
+     */
+    tSize hdfsRead(hdfsFS fs, hdfsFile file, void* buffer, tSize length);
+
+
+    /** 
+     * hdfsPread - Positional read of data from an open file.
+     * @param fs The configured filesystem handle.
+     * @param file The file handle.
+     * @param position Position from which to read
+     * @param buffer The buffer to copy read bytes into.
+     * @param length The length of the buffer.
+     * @return Returns the number of bytes actually read, possibly less than
+     * than length;-1 on error.
+     */
+    tSize hdfsPread(hdfsFS fs, hdfsFile file, tOffset position,
+                    void* buffer, tSize length);
+
+
+    /** 
+     * hdfsWrite - Write data into an open file.
+     * @param fs The configured filesystem handle.
+     * @param file The file handle.
+     * @param buffer The data.
+     * @param length The no. of bytes to write. 
+     * @return Returns the number of bytes written, -1 on error.
+     */
+    tSize hdfsWrite(hdfsFS fs, hdfsFile file, const void* buffer,
+                    tSize length);
+
+
+    /** 
+     * hdfsWrite - Flush the data. 
+     * @param fs The configured filesystem handle.
+     * @param file The file handle.
+     * @return Returns 0 on success, -1 on error. 
+     */
+    int hdfsFlush(hdfsFS fs, hdfsFile file);
+
+
+    /**
+     * hdfsAvailable - Number of bytes that can be read from this
+     * input stream without blocking.
+     * @param fs The configured filesystem handle.
+     * @param file The file handle.
+     * @return Returns available bytes; -1 on error. 
+     */
+    int hdfsAvailable(hdfsFS fs, hdfsFile file);
+
+
+    /**
+     * hdfsCopy - Copy file from one filesystem to another.
+     * @param srcFS The handle to source filesystem.
+     * @param src The path of source file. 
+     * @param dstFS The handle to destination filesystem.
+     * @param dst The path of destination file. 
+     * @return Returns 0 on success, -1 on error. 
+     */
+    int hdfsCopy(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst);
+
+
+    /**
+     * hdfsMove - Move file from one filesystem to another.
+     * @param srcFS The handle to source filesystem.
+     * @param src The path of source file. 
+     * @param dstFS The handle to destination filesystem.
+     * @param dst The path of destination file. 
+     * @return Returns 0 on success, -1 on error. 
+     */
+    int hdfsMove(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst);
+
+
+    /**
+     * hdfsDelete - Delete file. 
+     * @param fs The configured filesystem handle.
+     * @param path The path of the file. 
+     * @return Returns 0 on success, -1 on error. 
+     */
+    int hdfsDelete(hdfsFS fs, const char* path);
+
+
+    /**
+     * hdfsRename - Rename file. 
+     * @param fs The configured filesystem handle.
+     * @param oldPath The path of the source file. 
+     * @param newPath The path of the destination file. 
+     * @return Returns 0 on success, -1 on error. 
+     */
+    int hdfsRename(hdfsFS fs, const char* oldPath, const char* newPath);
+
+
+    /** 
+     * hdfsGetWorkingDirectory - Get the current working directory for
+     * the given filesystem.
+     * @param fs The configured filesystem handle.
+     * @param buffer The user-buffer to copy path of cwd into. 
+     * @param bufferSize The length of user-buffer.
+     * @return Returns buffer, NULL on error.
+     */
+    char* hdfsGetWorkingDirectory(hdfsFS fs, char *buffer, size_t bufferSize);
+
+
+    /** 
+     * hdfsSetWorkingDirectory - Set the working directory. All relative
+     * paths will be resolved relative to it.
+     * @param fs The configured filesystem handle.
+     * @param path The path of the new 'cwd'. 
+     * @return Returns 0 on success, -1 on error. 
+     */
+    int hdfsSetWorkingDirectory(hdfsFS fs, const char* path);
+
+
+    /** 
+     * hdfsCreateDirectory - Make the given file and all non-existent
+     * parents into directories.
+     * @param fs The configured filesystem handle.
+     * @param path The path of the directory. 
+     * @return Returns 0 on success, -1 on error. 
+     */
+    int hdfsCreateDirectory(hdfsFS fs, const char* path);
+
+
+    /** 
+     * hdfsSetReplication - Set the replication of the specified
+     * file to the supplied value
+     * @param fs The configured filesystem handle.
+     * @param path The path of the file. 
+     * @return Returns 0 on success, -1 on error. 
+     */
+    int hdfsSetReplication(hdfsFS fs, const char* path, int16_t replication);
+
+
+    /** 
+     * hdfsFileInfo - Information about a file/directory.
+     */
+    typedef struct  {
+        tObjectKind mKind;   /* file or directory */
+        char *mName;         /* the name of the file */
+        tTime mLastMod;      /* the last modification time for the file*/
+        tOffset mSize;       /* the size of the file in bytes */
+        short mReplication;    /* the count of replicas */
+        tOffset mBlockSize;  /* the block size for the file */
+    } hdfsFileInfo;
+
+
+    /** 
+     * hdfsListDirectory - Get list of files/directories for a given
+     * directory-path. hdfsFreeFileInfo should be called to deallocate memory. 
+     * @param fs The configured filesystem handle.
+     * @param path The path of the directory. 
+     * @param numEntries Set to the number of files/directories in path.
+     * @return Returns a dynamically-allocated array of hdfsFileInfo
+     * objects; NULL on error.
+     */
+    hdfsFileInfo *hdfsListDirectory(hdfsFS fs, const char* path,
+                                    int *numEntries);
+
+
+    /** 
+     * hdfsGetPathInfo - Get information about a path as a (dynamically
+     * allocated) single hdfsFileInfo struct. hdfsFreeFileInfo should be
+     * called when the pointer is no longer needed.
+     * @param fs The configured filesystem handle.
+     * @param path The path of the file. 
+     * @return Returns a dynamically-allocated hdfsFileInfo object;
+     * NULL on error.
+     */
+    hdfsFileInfo *hdfsGetPathInfo(hdfsFS fs, const char* path);
+
+
+    /** 
+     * hdfsFreeFileInfo - Free up the hdfsFileInfo array (including fields) 
+     * @param hdfsFileInfo The array of dynamically-allocated hdfsFileInfo
+     * objects.
+     * @param numEntries The size of the array.
+     */
+    void hdfsFreeFileInfo(hdfsFileInfo *hdfsFileInfo, int numEntries);
+
+
+    /** 
+     * hdfsGetHosts - Get hostnames where a particular block (determined by
+     * pos & blocksize) of a file is stored. The last element in the array
+     * is NULL. Due to replication, a single block could be present on
+     * multiple hosts.
+     * @param fs The configured filesystem handle.
+     * @param path The path of the file. 
+     * @param start The start of the block.
+     * @param length The length of the block.
+     * @return Returns a dynamically-allocated 2-d array of blocks-hosts;
+     * NULL on error.
+     */
+    char*** hdfsGetHosts(hdfsFS fs, const char* path, 
+            tOffset start, tOffset length);
+
+
+    /** 
+     * hdfsFreeHosts - Free up the structure returned by hdfsGetHosts
+     * @param hdfsFileInfo The array of dynamically-allocated hdfsFileInfo
+     * objects.
+     * @param numEntries The size of the array.
+     */
+    void hdfsFreeHosts(char ***blockHosts);
+
+
+    /** 
+     * hdfsGetDefaultBlockSize - Get the optimum blocksize.
+     * @param fs The configured filesystem handle.
+     * @return Returns the blocksize; -1 on error. 
+     */
+    tOffset hdfsGetDefaultBlockSize(hdfsFS fs);
+
+
+    /** 
+     * hdfsGetCapacity - Return the raw capacity of the filesystem.  
+     * @param fs The configured filesystem handle.
+     * @return Returns the raw-capacity; -1 on error. 
+     */
+    tOffset hdfsGetCapacity(hdfsFS fs);
+
+
+    /** 
+     * hdfsGetUsed - Return the total raw size of all files in the filesystem.
+     * @param fs The configured filesystem handle.
+     * @return Returns the total-size; -1 on error. 
+     */
+    tOffset hdfsGetUsed(hdfsFS fs);
+    
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*LIBHDFS_HDFS_H*/
+
+/**
+ * vim: ts=4: sw=4: et
+ */

data/test/helper.rb

@@ -0,0 +1,10 @@
+require 'rubygems'
+require 'test/unit'
+require 'shoulda'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'ruby-hdfs'
+
+class Test::Unit::TestCase
+end

data/test/test_ruby-hdfs.rb

@@ -0,0 +1,7 @@
+require 'helper'
+
+class TestRubyHdfs < Test::Unit::TestCase
+  should "probably rename this file and start testing for real" do
+    flunk "hey buddy, you should probably rename this file and start testing for real"
+  end
+end

metadata

@@ -0,0 +1,74 @@
+--- !ruby/object:Gem::Specification 
+name: ruby-hdfs
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- Alexander Staubo
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-03-04 00:00:00 +01:00
+default_executable: 
+dependencies: []
+
+description: Native C bindings to Hadoop's libhdfs, for interacting with Hadoop HDFS.
+email: alex@bengler.no
+executables: []
+
+extensions: 
+- ext/hdfs/extconf.rb
+extra_rdoc_files: 
+- LICENSE
+- README.rdoc
+files: 
+- .document
+- .gitignore
+- LICENSE
+- README.rdoc
+- Rakefile
+- VERSION
+- ext/hdfs/extconf.rb
+- ext/hdfs/hdfs.c
+- ext/hdfs/hdfs.h
+- test/helper.rb
+- test/test_ruby-hdfs.rb
+has_rdoc: true
+homepage: http://github.com/alexstaubo/ruby-hdfs
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.6
+signing_key: 
+specification_version: 3
+summary: Native C bindings to Hadoop's libhdfs, for interacting with Hadoop HDFS.
+test_files: 
+- test/helper.rb
+- test/test_ruby-hdfs.rb