ruby-hdfs-cdh4 0.0.4 → 0.0.5

data/ext/hdfs/hdfs.c

@@ -19,13 +19,13 @@ static VALUE e_file_error;
 static VALUE e_could_not_open;
 static VALUE e_does_not_exist;
 
-static const int32_t HDFS_DEFAULT_BLOCK_SIZE     = 134217728;
 static const int16_t HDFS_DEFAULT_REPLICATION    = 3;
 static const short HDFS_DEFAULT_MODE             = 0644;
 static const char* HDFS_DEFAULT_HOST             = "0.0.0.0";
 static const int HDFS_DEFAULT_RECURSIVE_DELETE   = 0;
 static const int HDFS_DEFAULT_PATH_STRING_LENGTH = 1024;
 static const int HDFS_DEFAULT_PORT               = 8020;
+static const char* HDFS_DEFAULT_USER             = NULL;
 
 /*
  * Data structs
@@ -109,6 +109,12 @@ int octal_decimal(int n) {
   return decimal;
 }
 
+void ensure_file_open(FileData* data) {
+  if (data->file == NULL) {
+    rb_raise(e_file_error, "File is closed");
+  }
+}
+
 /*
  * Copies an hdfsFileInfo struct into a Hadoop::DFS::FileInfo derivative
  * object.
@@ -157,14 +163,17 @@ VALUE HDFS_File_System_alloc(VALUE klass) {
  * call-seq:
  *    hdfs.new(options={}) -> hdfs
  *
- * Creates a new HDFS client connection, returning a new
+ * Creates a new HDFS client connection, configured by options, returning a new
  * Hadoop::DFS::FileSystem object if successful.  If this fails, raises a
  * ConnectError.
  *
- * Options:
- *  :local
- *  :host
- *  :port
+ * options can have the following keys:
+ *
+ *  * *local*: whether to use the local filesystem instead of HDFS
+ *    (default: false)
+ *  * *host*: hostname or IP address of a Hadoop NameNode (default: '0.0.0.0')
+ *  * *port*: port through which to connect to Hadoop NameNode (default: 8020)
+ *  * *user*: user to connect to filesystem as (default: current user)
  */
 VALUE HDFS_File_System_initialize(int argc, VALUE* argv, VALUE self) {
   VALUE options;
@@ -177,26 +186,23 @@ VALUE HDFS_File_System_initialize(int argc, VALUE* argv, VALUE self) {
     options = rb_hash_new();
   }
 
+  VALUE r_user = rb_hash_aref(options, rb_eval_string(":user"));
+  char* hdfs_user = RTEST(r_user) ? RSTRING_PTR(r_user) : 
+      (char*) HDFS_DEFAULT_USER;
+
   VALUE r_local = rb_hash_aref(options, rb_eval_string(":local"));
   if (r_local == Qtrue) {
-    data->fs = hdfsConnect(NULL, 0);
+    data->fs = hdfsConnectAsUser(NULL, 0, hdfs_user);
   } else {
     VALUE r_host = rb_hash_aref(options, rb_eval_string(":host"));
     VALUE r_port = rb_hash_aref(options, rb_eval_string(":port"));
 
     // Sets default values for host and port if not supplied by user.
-    char* hdfs_host = (char*) HDFS_DEFAULT_HOST;
-    int hdfs_port   = HDFS_DEFAULT_PORT;
-
-    if (RTEST(r_host)) {
-      hdfs_host = RSTRING_PTR(r_host);
-    }
-
-    if (RTEST(r_port)) {
-      hdfs_port = NUM2INT(r_port);
-    }
-
-    data->fs = hdfsConnect(hdfs_host, hdfs_port);     
+    char* hdfs_host = RTEST(r_host) ? RSTRING_PTR(r_host) : 
+        (char*) HDFS_DEFAULT_HOST;
+    int hdfs_port   = RTEST(r_port) ? NUM2INT(r_port) :
+        HDFS_DEFAULT_PORT;
+    data->fs = hdfsConnectAsUser(hdfs_host, hdfs_port, hdfs_user);     
   }
  
   if (data->fs == NULL) {
@@ -424,7 +430,7 @@ VALUE HDFS_File_System_cwd(VALUE self) {
 VALUE HDFS_File_System_chgrp(VALUE self, VALUE path, VALUE group) {
   FSData* data = NULL;
   Data_Get_Struct(self, FSData, data);
-  if (hdfsChgrp(data->fs, RSTRING_PTR(path), NULL, RSTRING_PTR(group)) < 0) {
+  if (hdfsChown(data->fs, RSTRING_PTR(path), NULL, RSTRING_PTR(group)) < 0) {
     rb_raise(e_dfs_exception, "Failed to chgrp path: %s to group: %s",
         RSTRING_PTR(path), RSTRING_PTR(group));
     return Qnil;
@@ -538,7 +544,8 @@ VALUE HDFS_File_System_move(int argc, VALUE* argv, VALUE self) {
   }
   if (hdfsMove(data->fs, RSTRING_PTR(from_path), destFS,
       RSTRING_PTR(to_path)) < 0) {
-    rb_raise(e_dfs_exception, "Error while retrieving capacity");
+    rb_raise(e_dfs_exception, "Error while moving path: %s to path: %s",
+        RSTRING_PTR(from_path), RSTRING_PTR(to_path));
     return Qnil;
   }
   return Qtrue;
@@ -599,6 +606,39 @@ VALUE HDFS_File_System_default_block_size_at_path(VALUE self, VALUE path) {
   return LONG2NUM(block_size);
 }
 
+/**
+ * call-seq:
+ *    hdfs.get_hosts(path, start, length) -> retval
+ *
+ * Returns the hostnames of the DataNodes which serve the portion of the file
+ * between the provided start and length bytes.  Raises a DFSException if this
+ * fails.
+ */
+VALUE HDFS_File_System_get_hosts(VALUE self, VALUE path, VALUE start,
+    VALUE length) {
+  FSData* data = NULL;
+  Data_Get_Struct(self, FSData, data);
+  char*** hosts = hdfsGetHosts(data->fs, RSTRING_PTR(path), NUM2LONG(start),
+      NUM2LONG(length));
+  if (hosts == NULL) {
+    rb_raise(e_dfs_exception,
+        "Error while retrieving hosts at path: %s, start: %lu, length: %lu",
+        RSTRING_PTR(path), NUM2LONG(start), NUM2LONG(length));
+    return Qnil;
+  }
+  // Builds a Ruby Array object out of the hosts reported by HDFS.
+  VALUE hosts_array = rb_ary_new();
+  size_t i, j;
+  for (i = 0; hosts[i]; i++) {
+    VALUE cur_block_hosts = rb_ary_new();
+    for (j = 0; hosts[i][j]; j++) {
+      rb_ary_push(cur_block_hosts, rb_str_new2(hosts[i][j]));
+    }
+    rb_ary_push(hosts_array, cur_block_hosts);
+  }
+  return hosts_array;
+}
+
 /**
  * call-seq:
  *    hdfs.used -> retval
@@ -655,8 +695,18 @@ VALUE HDFS_File_System_utime(int argc, VALUE* argv, VALUE self) {
  * call-seq:
  *    hdfs.open(path, mode='r', options={}) -> file
  *
- * Opens a file.  If the file cannot be opened, raises a CouldNotOpenError;
- * otherwise, returns a Hadoop::DFS::File object corresponding to the file.
+ * Opens a file using the supplied mode and options.  If the file cannot be
+ * opened, raises a CouldNotOpenError; otherwise, returns a Hadoop::DFS::File
+ * object corresponding to the file.
+ *
+ * options can have the following keys:
+ *
+ * * *buffer_size*: size in bytes of buffer to use for file accesses
+ *   (default: default buffer size as configured by HDFS)
+ * * *replication*: the number of nodes this file should be replicated against
+ *   (default: default replication as configured by HDFS)
+ * * *block_size*: the HDFS block size in bytes to use for this file
+ *   (default: default block size as configured by HDFS)
  */
 VALUE HDFS_File_System_open(int argc, VALUE* argv, VALUE self) {
   VALUE path, mode, options;
@@ -684,7 +734,7 @@ VALUE HDFS_File_System_open(int argc, VALUE* argv, VALUE self) {
   hdfsFile file = hdfsOpenFile(data->fs, RSTRING_PTR(path), flags, 
     RTEST(r_buffer_size) ? NUM2INT(r_buffer_size) : 0, 
     RTEST(r_replication) ? NUM2INT(r_replication) : 0, 
-    RTEST(r_block_size) ? NUM2INT(r_block_size) : HDFS_DEFAULT_BLOCK_SIZE);
+    RTEST(r_block_size) ? NUM2INT(r_block_size) : 0);
   if (file == NULL) {
     rb_raise(e_could_not_open, "Could not open file %s", RSTRING_PTR(path));
     return Qnil;
@@ -711,6 +761,7 @@ VALUE HDFS_File_System_open(int argc, VALUE* argv, VALUE self) {
 VALUE HDFS_File_read(VALUE self, VALUE length) {
   FileData* data = NULL;
   Data_Get_Struct(self, FileData, data);
+  ensure_file_open(data);
   char* buffer = ALLOC_N(char, length);
   MEMZERO(buffer, char, length);
   tSize bytes_read = hdfsRead(data->fs, data->file, buffer, NUM2INT(length));
@@ -730,6 +781,7 @@ VALUE HDFS_File_read(VALUE self, VALUE length) {
 VALUE HDFS_File_write(VALUE self, VALUE bytes) {
   FileData* data = NULL;
   Data_Get_Struct(self, FileData, data);
+  ensure_file_open(data);
   tSize bytes_written = hdfsWrite(data->fs, data->file, RSTRING_PTR(bytes), RSTRING_LEN(bytes));
   if (bytes_written == -1) {
     rb_raise(e_file_error, "Failed to write data");
@@ -747,6 +799,7 @@ VALUE HDFS_File_write(VALUE self, VALUE bytes) {
 VALUE HDFS_File_tell(VALUE self) {
   FileData* data = NULL;
   Data_Get_Struct(self, FileData, data);
+  ensure_file_open(data);
   tSize offset = hdfsTell(data->fs, data->file);
   if (offset == -1) {
     rb_raise(e_file_error, "Failed to read position");
@@ -764,6 +817,7 @@ VALUE HDFS_File_tell(VALUE self) {
 VALUE HDFS_File_seek(VALUE self, VALUE offset) {
   FileData* data = NULL;
   Data_Get_Struct(self, FileData, data);
+  ensure_file_open(data);
   int result = hdfsSeek(data->fs, data->file, NUM2INT(offset));
   if (result != 0) {
     rb_raise(e_file_error, "Failed to seek to position %d", NUM2INT(offset));
@@ -782,6 +836,7 @@ VALUE HDFS_File_seek(VALUE self, VALUE offset) {
 VALUE HDFS_File_flush(VALUE self) {
   FileData* data = NULL;
   Data_Get_Struct(self, FileData, data);
+  ensure_file_open(data);
   int result = hdfsFlush(data->fs, data->file);
   if (result != 0) {
     rb_raise(e_file_error, "Flush failed");
@@ -799,6 +854,7 @@ VALUE HDFS_File_flush(VALUE self) {
 VALUE HDFS_File_available(VALUE self) {
   FileData* data = NULL;
   Data_Get_Struct(self, FileData, data);
+  ensure_file_open(data);
   int result = hdfsAvailable(data->fs, data->file);
   if (result == -1) {
     rb_raise(e_file_error, "Failed to get available data");
@@ -816,12 +872,47 @@ VALUE HDFS_File_close(VALUE self) {
   FileData* data = NULL;
   Data_Get_Struct(self, FileData, data);
   if (data->file != NULL) {
-    hdfsCloseFile(data->fs, data->file);
+    if (hdfsCloseFile(data->fs, data->file) < 0) {
+      rb_raise(e_file_error, "Could not close file");
+      return Qnil;
+    }
     data->file = NULL;
   }
   return Qtrue;
 }
 
+/**
+ * call-seq:
+ *    file.read_open -> open_for_read
+ *
+ * Returns True if this file is open for reading; otherwise returns False.
+ */
+VALUE HDFS_File_read_open(VALUE self) {
+  FileData* data = NULL;
+  Data_Get_Struct(self, FileData, data);
+  if (data->file) {
+    return hdfsFileIsOpenForRead(data->file) ? Qtrue : Qfalse;
+  } else {
+    return Qfalse;
+  }
+}
+
+/**
+ * call-seq:
+ *    file.write_open -> open_for_write
+ *
+ * Returns True if this file is open for writing; otherwise returns False.
+ */
+VALUE HDFS_File_write_open(VALUE self) {
+  FileData* data = NULL;
+  Data_Get_Struct(self, FileData, data);
+  if (data->file) {
+    return hdfsFileIsOpenForWrite(data->file) ? Qtrue : Qfalse;
+  } else {
+    return Qfalse;
+  }
+}
+
 /**
  * HDFS File Info interface
  */
@@ -1004,16 +1095,21 @@ void Init_hdfs() {
 
   c_file_system = rb_define_class_under(m_dfs, "FileSystem", rb_cObject);
   rb_define_alloc_func(c_file_system, HDFS_File_System_alloc);
-  rb_define_method(c_file_system, "initialize", HDFS_File_System_initialize, -1);
-  rb_define_method(c_file_system, "disconnect", HDFS_File_System_disconnect, 0);
+  rb_define_method(c_file_system, "initialize", HDFS_File_System_initialize,
+      -1);
+  rb_define_method(c_file_system, "disconnect", HDFS_File_System_disconnect,
+      0);
   rb_define_method(c_file_system, "open", HDFS_File_System_open, -1);
   rb_define_method(c_file_system, "delete", HDFS_File_System_delete, -1);
   rb_define_method(c_file_system, "rename", HDFS_File_System_rename, 2);
   rb_define_method(c_file_system, "exist?", HDFS_File_System_exist, 1);
-  rb_define_method(c_file_system, "create_directory", HDFS_File_System_create_directory, 1);
-  rb_define_method(c_file_system, "list_directory", HDFS_File_System_list_directory, 1);
+  rb_define_method(c_file_system, "create_directory",
+      HDFS_File_System_create_directory, 1);
+  rb_define_method(c_file_system, "list_directory",
+      HDFS_File_System_list_directory, 1);
   rb_define_method(c_file_system, "stat", HDFS_File_System_stat, 1);
-  rb_define_method(c_file_system, "set_replication", HDFS_File_System_set_replication, -1);
+  rb_define_method(c_file_system, "set_replication",
+      HDFS_File_System_set_replication, -1);
   rb_define_method(c_file_system, "cd", HDFS_File_System_cd, 1);
   rb_define_method(c_file_system, "cwd", HDFS_File_System_cwd, 0);
   rb_define_method(c_file_system, "chgrp", HDFS_File_System_chgrp, 2);
@@ -1025,6 +1121,7 @@ void Init_hdfs() {
       HDFS_File_System_default_block_size, 0);
   rb_define_method(c_file_system, "default_block_size_at_path",
       HDFS_File_System_default_block_size_at_path, 1);
+  rb_define_method(c_file_system, "get_hosts", HDFS_File_System_get_hosts, 3);
   rb_define_method(c_file_system, "move", HDFS_File_System_move, -1);
   rb_define_method(c_file_system, "used", HDFS_File_System_used, 0);
   rb_define_method(c_file_system, "utime", HDFS_File_System_utime, -1);
@@ -1038,14 +1135,18 @@ void Init_hdfs() {
   rb_define_method(c_file, "flush", HDFS_File_flush, 0);
   rb_define_method(c_file, "available", HDFS_File_available, 0);
   rb_define_method(c_file, "close", HDFS_File_close, 0);
+  rb_define_method(c_file, "read_open?", HDFS_File_read_open, 0);
+  rb_define_method(c_file, "write_open?", HDFS_File_write_open, 0);
 
   c_file_info = rb_define_class_under(m_dfs, "FileInfo", rb_cObject);
   rb_define_method(c_file_info, "block_size", HDFS_File_Info_block_size, 0);
   rb_define_method(c_file_info, "group", HDFS_File_Info_group, 0);
-  rb_define_method(c_file_info, "is_directory?", HDFS_File_Info_is_directory, 0);
+  rb_define_method(c_file_info, "is_directory?", HDFS_File_Info_is_directory,
+      0);
   rb_define_method(c_file_info, "is_file?", HDFS_File_Info_is_file, 0);
   rb_define_method(c_file_info, "last_access", HDFS_File_Info_last_access, 0);
-  rb_define_method(c_file_info, "last_modified", HDFS_File_Info_last_modified, 0);
+  rb_define_method(c_file_info, "last_modified", HDFS_File_Info_last_modified,
+      0);
   rb_define_method(c_file_info, "mode", HDFS_File_Info_mode, 0);
   rb_define_method(c_file_info, "name", HDFS_File_Info_name, 0);
   rb_define_method(c_file_info, "owner", HDFS_File_Info_owner, 0);
@@ -1054,14 +1155,21 @@ void Init_hdfs() {
   rb_define_method(c_file_info, "to_s", HDFS_File_Info_to_s, 0);
 
   c_file_info_file = rb_define_class_under(c_file_info, "File", c_file_info);
-  rb_define_method(c_file_info_file, "is_file?", HDFS_File_Info_File_is_file, 0);
-
-  c_file_info_directory = rb_define_class_under(c_file_info, "Directory", c_file_info);
-  rb_define_method(c_file_info_directory, "is_directory?", HDFS_File_Info_Directory_is_directory, 0);
-
-  e_dfs_exception = rb_define_class_under(m_dfs, "DFSException", rb_eStandardError);
-  e_connect_error = rb_define_class_under(m_dfs, "ConnectError", e_dfs_exception);
+  rb_define_method(c_file_info_file, "is_file?", HDFS_File_Info_File_is_file,
+      0);
+
+  c_file_info_directory = rb_define_class_under(c_file_info, "Directory",
+      c_file_info);
+  rb_define_method(c_file_info_directory, "is_directory?",
+      HDFS_File_Info_Directory_is_directory, 0);
+
+  e_dfs_exception = rb_define_class_under(m_dfs, "DFSException",
+      rb_eStandardError);
+  e_connect_error = rb_define_class_under(m_dfs, "ConnectError",
+      e_dfs_exception);
   e_file_error = rb_define_class_under(m_dfs, "FileError", e_dfs_exception);
-  e_could_not_open = rb_define_class_under(m_dfs, "CouldNotOpenFileError", e_file_error);
-  e_does_not_exist = rb_define_class_under(m_dfs, "DoesNotExistError", e_file_error); 
+  e_could_not_open = rb_define_class_under(m_dfs, "CouldNotOpenFileError",
+      e_file_error);
+  e_does_not_exist = rb_define_class_under(m_dfs, "DoesNotExistError",
+      e_file_error); 
 }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ruby-hdfs-cdh4
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
   prerelease: 
 platform: ruby
 authors:
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-05-22 00:00:00.000000000 Z
+date: 2013-05-25 00:00:00.000000000 Z
 dependencies: []
 description: ruby hadoop libhdfs client with support for cdh4
 email:
@@ -48,7 +48,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.8.10
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: ruby hadoop libhdfs client with support for cdh4