ruby-hdfs-cdh4 0.0.6 → 0.0.7
Sign up to get free protection for your applications and to get access to all the features.
- data/ext/hdfs/hdfs.c +36 -25
- metadata +2 -2
data/ext/hdfs/hdfs.c
CHANGED
@@ -19,12 +19,13 @@ static VALUE e_file_error;
|
|
19
19
|
static VALUE e_could_not_open;
|
20
20
|
static VALUE e_does_not_exist;
|
21
21
|
|
22
|
-
static const
|
23
|
-
static const short HDFS_DEFAULT_MODE = 0644;
|
22
|
+
static const int32_t HDFS_DEFAULT_BUFFER_SIZE = 131072;
|
24
23
|
static const char* HDFS_DEFAULT_HOST = "0.0.0.0";
|
25
|
-
static const
|
24
|
+
static const short HDFS_DEFAULT_MODE = 0644;
|
26
25
|
static const int HDFS_DEFAULT_PATH_STRING_LENGTH = 1024;
|
27
26
|
static const int HDFS_DEFAULT_PORT = 8020;
|
27
|
+
static const int HDFS_DEFAULT_RECURSIVE_DELETE = 0;
|
28
|
+
static const int16_t HDFS_DEFAULT_REPLICATION = 3;
|
28
29
|
static const char* HDFS_DEFAULT_USER = NULL;
|
29
30
|
|
30
31
|
/*
|
@@ -170,7 +171,7 @@ VALUE HDFS_File_System_alloc(VALUE klass) {
|
|
170
171
|
* options can have the following keys:
|
171
172
|
*
|
172
173
|
* * *local*: whether to use the local filesystem instead of HDFS
|
173
|
-
*
|
174
|
+
* (default: false)
|
174
175
|
* * *host*: hostname or IP address of a Hadoop NameNode (default: '0.0.0.0')
|
175
176
|
* * *port*: port through which to connect to Hadoop NameNode (default: 8020)
|
176
177
|
* * *user*: user to connect to filesystem as (default: current user)
|
@@ -179,6 +180,10 @@ VALUE HDFS_File_System_initialize(int argc, VALUE* argv, VALUE self) {
|
|
179
180
|
VALUE options;
|
180
181
|
rb_scan_args(argc, argv, "01", &options);
|
181
182
|
|
183
|
+
if (TYPE(options) != T_HASH) {
|
184
|
+
rb_raise(e_dfs_exception, "options must be of type Hash");
|
185
|
+
}
|
186
|
+
|
182
187
|
FSData* data = NULL;
|
183
188
|
Data_Get_Struct(self, FSData, data);
|
184
189
|
|
@@ -417,7 +422,7 @@ VALUE HDFS_File_System_cwd(VALUE self) {
|
|
417
422
|
rb_raise(e_dfs_exception, "Failed to get current working directory");
|
418
423
|
return Qnil;
|
419
424
|
}
|
420
|
-
return
|
425
|
+
return rb_tainted_str_new2(cur_dir);
|
421
426
|
}
|
422
427
|
|
423
428
|
/**
|
@@ -732,9 +737,9 @@ VALUE HDFS_File_System_open(int argc, VALUE* argv, VALUE self) {
|
|
732
737
|
FSData* data = NULL;
|
733
738
|
Data_Get_Struct(self, FSData, data);
|
734
739
|
hdfsFile file = hdfsOpenFile(data->fs, RSTRING_PTR(path), flags,
|
735
|
-
|
736
|
-
|
737
|
-
|
740
|
+
RTEST(r_buffer_size) ? NUM2INT(r_buffer_size) : 0,
|
741
|
+
RTEST(r_replication) ? NUM2INT(r_replication) : 0,
|
742
|
+
RTEST(r_block_size) ? NUM2INT(r_block_size) : 0);
|
738
743
|
if (file == NULL) {
|
739
744
|
rb_raise(e_could_not_open, "Could not open file %s", RSTRING_PTR(path));
|
740
745
|
return Qnil;
|
@@ -759,16 +764,22 @@ VALUE HDFS_File_System_open(int argc, VALUE* argv, VALUE self) {
|
|
759
764
|
* FileError.
|
760
765
|
*/
|
761
766
|
VALUE HDFS_File_read(VALUE self, VALUE length) {
|
767
|
+
// Checks whether we're reading more data than HDFS client can support.
|
768
|
+
if (NUM2UINT(length) > HDFS_DEFAULT_BUFFER_SIZE) {
|
769
|
+
rb_raise(e_file_error, "Can only read a max of %u bytes from HDFS",
|
770
|
+
HDFS_DEFAULT_BUFFER_SIZE);
|
771
|
+
return Qnil;
|
772
|
+
}
|
762
773
|
FileData* data = NULL;
|
763
774
|
Data_Get_Struct(self, FileData, data);
|
764
775
|
ensure_file_open(data);
|
765
776
|
char* buffer = ALLOC_N(char, length);
|
766
777
|
MEMZERO(buffer, char, length);
|
767
|
-
tSize bytes_read = hdfsRead(data->fs, data->file, buffer,
|
778
|
+
tSize bytes_read = hdfsRead(data->fs, data->file, buffer, NUM2UINT(length));
|
768
779
|
if (bytes_read == -1) {
|
769
780
|
rb_raise(e_file_error, "Failed to read data");
|
770
781
|
}
|
771
|
-
return
|
782
|
+
return rb_tainted_str_new(buffer, bytes_read);
|
772
783
|
}
|
773
784
|
|
774
785
|
/**
|
@@ -776,17 +787,19 @@ VALUE HDFS_File_read(VALUE self, VALUE length) {
|
|
776
787
|
* file.write(bytes) -> num_bytes_written
|
777
788
|
*
|
778
789
|
* Writes the string specified by bytes to the current file object, returning
|
779
|
-
* the number of bytes
|
790
|
+
* the number of bytes written as an Integer. If this fails, raises a
|
791
|
+
* FileError.
|
780
792
|
*/
|
781
793
|
VALUE HDFS_File_write(VALUE self, VALUE bytes) {
|
782
794
|
FileData* data = NULL;
|
783
795
|
Data_Get_Struct(self, FileData, data);
|
784
796
|
ensure_file_open(data);
|
785
|
-
tSize bytes_written = hdfsWrite(data->fs, data->file, RSTRING_PTR(bytes),
|
797
|
+
tSize bytes_written = hdfsWrite(data->fs, data->file, RSTRING_PTR(bytes),
|
798
|
+
RSTRING_LEN(bytes));
|
786
799
|
if (bytes_written == -1) {
|
787
800
|
rb_raise(e_file_error, "Failed to write data");
|
788
801
|
}
|
789
|
-
return
|
802
|
+
return UINT2NUM(bytes_written);
|
790
803
|
}
|
791
804
|
|
792
805
|
/**
|
@@ -800,11 +813,11 @@ VALUE HDFS_File_tell(VALUE self) {
|
|
800
813
|
FileData* data = NULL;
|
801
814
|
Data_Get_Struct(self, FileData, data);
|
802
815
|
ensure_file_open(data);
|
803
|
-
|
816
|
+
tOffset offset = hdfsTell(data->fs, data->file);
|
804
817
|
if (offset == -1) {
|
805
818
|
rb_raise(e_file_error, "Failed to read position");
|
806
819
|
}
|
807
|
-
return
|
820
|
+
return ULONG2NUM(offset);
|
808
821
|
}
|
809
822
|
|
810
823
|
/**
|
@@ -818,9 +831,8 @@ VALUE HDFS_File_seek(VALUE self, VALUE offset) {
|
|
818
831
|
FileData* data = NULL;
|
819
832
|
Data_Get_Struct(self, FileData, data);
|
820
833
|
ensure_file_open(data);
|
821
|
-
|
822
|
-
|
823
|
-
rb_raise(e_file_error, "Failed to seek to position %d", NUM2INT(offset));
|
834
|
+
if (hdfsSeek(data->fs, data->file, NUM2ULONG(offset)) < 0) {
|
835
|
+
rb_raise(e_file_error, "Failed to seek to position %u", NUM2ULONG(offset));
|
824
836
|
}
|
825
837
|
return Qtrue;
|
826
838
|
}
|
@@ -837,8 +849,7 @@ VALUE HDFS_File_flush(VALUE self) {
|
|
837
849
|
FileData* data = NULL;
|
838
850
|
Data_Get_Struct(self, FileData, data);
|
839
851
|
ensure_file_open(data);
|
840
|
-
|
841
|
-
if (result != 0) {
|
852
|
+
if (hdfsFlush(data->fs, data->file) < 0) {
|
842
853
|
rb_raise(e_file_error, "Flush failed");
|
843
854
|
}
|
844
855
|
return Qtrue;
|
@@ -855,11 +866,11 @@ VALUE HDFS_File_available(VALUE self) {
|
|
855
866
|
FileData* data = NULL;
|
856
867
|
Data_Get_Struct(self, FileData, data);
|
857
868
|
ensure_file_open(data);
|
858
|
-
int
|
859
|
-
if (
|
869
|
+
int bytes_available = hdfsAvailable(data->fs, data->file);
|
870
|
+
if (bytes_available < 0) {
|
860
871
|
rb_raise(e_file_error, "Failed to get available data");
|
861
872
|
}
|
862
|
-
return INT2NUM(
|
873
|
+
return INT2NUM(bytes_available);
|
863
874
|
}
|
864
875
|
|
865
876
|
/**
|
@@ -926,7 +937,7 @@ VALUE HDFS_File_write_open(VALUE self) {
|
|
926
937
|
VALUE HDFS_File_Info_block_size(VALUE self) {
|
927
938
|
FileInfo* file_info = NULL;
|
928
939
|
Data_Get_Struct(self, FileInfo, file_info);
|
929
|
-
return
|
940
|
+
return LONG2NUM(file_info->mBlockSize);
|
930
941
|
}
|
931
942
|
|
932
943
|
/**
|
@@ -938,7 +949,7 @@ VALUE HDFS_File_Info_block_size(VALUE self) {
|
|
938
949
|
VALUE HDFS_File_Info_group(VALUE self) {
|
939
950
|
FileInfo* file_info = NULL;
|
940
951
|
Data_Get_Struct(self, FileInfo, file_info);
|
941
|
-
return
|
952
|
+
return rb_str_new2(file_info->mGroup);
|
942
953
|
}
|
943
954
|
|
944
955
|
/**
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ruby-hdfs-cdh4
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.7
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -11,7 +11,7 @@ authors:
|
|
11
11
|
autorequire:
|
12
12
|
bindir: bin
|
13
13
|
cert_chain: []
|
14
|
-
date: 2013-
|
14
|
+
date: 2013-06-14 00:00:00.000000000 Z
|
15
15
|
dependencies: []
|
16
16
|
description: ruby hadoop libhdfs client with support for cdh4
|
17
17
|
email:
|