toastyapps-excelsior 0.0.4 → 0.0.5

data/Rakefile

@@ -8,7 +8,7 @@ PKG_FILES = %w(Rakefile) +
  
 gem_spec = Gem::Specification.new do |gem_spec|
   gem_spec.name = 'excelsior'
-  gem_spec.version = '0.0.4'
+  gem_spec.version = '0.0.5'
   gem_spec.summary = 'A Ruby gem that uses C bindings to read CSV files superfast. I\'m totally serial!'
   gem_spec.description = 'A Ruby gem that uses C bindings to read CSV files superfast. I\'m totally serial!'
   gem_spec.email = 'matt@toastyapps.com'

data/ext/excelsior_reader/excelsior_reader.c

@@ -1,26 +1,18 @@
 
 #line 1 "excelsior_reader.rl"
-#include "ruby.h"
-
-VALUE e_parse(VALUE self, VALUE data) {
- int cs, act, curline = 1, len = 0;
- char *ts = 0, *te = 0, *buf = NULL, *eof = NULL;
- char *p, *pe;
- VALUE rows;
- VALUE arr;
- rows = rb_ary_new();
- arr = rb_ary_new();
- len = RSTRING(data)->len;
- p = RSTRING(data)->ptr;
- pe = p + len;
- eof = pe;
- int has_found = 0;
- 
-#line 31 "excelsior_reader.rl"
+#include <ruby.h>
+
+static ID s_read;
+VALUE arr;
+int has_found = 0;
+#define BUFSIZE 16384
+
 
+#line 23 "excelsior_reader.rl"
 
  
-#line 24 "excelsior_reader.c"
+
+#line 16 "excelsior_reader.c"
 static const char _excelsior_scan_actions[] = {
 	0, 1, 2, 1, 7, 1, 8, 1, 
 	9, 1, 10, 1, 11, 2, 0, 1, 
@@ -79,9 +71,28 @@ static const int excelsior_scan_error = 0;
 static const int excelsior_scan_en_main = 2;
 
 
-#line 34 "excelsior_reader.rl"
- 
-#line 85 "excelsior_reader.c"
+#line 26 "excelsior_reader.rl"
+
+
+VALUE e_rows(int argc, VALUE *argv, VALUE self) {
+  
+  int cs, act, have = 0, nread = 0, curline = 1, text = 0;
+  char *ts = 0, *te = 0, *buf = NULL, *eof = NULL;
+  int buffer_size = BUFSIZE;
+  
+  has_found = 0;
+  VALUE io;
+  int is_io = 0;
+  int done = 0;
+  
+  arr = rb_ary_new();
+  rb_scan_args(argc, argv, "1", &io);
+  
+  is_io = rb_respond_to(io, s_read);
+  buf = ALLOC_N(char, buffer_size);
+  
+  
+#line 96 "excelsior_reader.c"
 	{
 	cs = excelsior_scan_start;
 	ts = 0;
@@ -89,9 +100,41 @@ static const int excelsior_scan_en_main = 2;
 	act = 0;
 	}
 
-#line 35 "excelsior_reader.rl"
- 
-#line 95 "excelsior_reader.c"
+#line 46 "excelsior_reader.rl"
+  
+  while(!done) {
+  
+    int len, space = buffer_size - have;
+    VALUE str;
+    char *p, *pe;
+    p = buf + have;
+  
+    if(is_io) {
+      str = rb_funcall(io, s_read, 1, INT2FIX(space));
+      len = RSTRING_LEN(str);
+      memcpy(p, StringValuePtr(str), len);
+    } else { 
+      // Going to assume it's a string and already in memory
+      //str = io;
+      len = buffer_size = RSTRING_LEN(io);
+      memcpy(p, StringValuePtr(io), len);
+      space = buffer_size - have;
+      pe = p + buffer_size;
+      eof = pe;
+      done = 1;
+    }
+  
+    if(len < space) {
+      done = 1;
+      //p[len++] = 0; can't seem to get it to work with this
+      pe = p + len;
+      eof = pe;
+    } else {
+      pe = p + len; 
+    }
+  
+    
+#line 138 "excelsior_reader.c"
 	{
 	int _klen;
 	unsigned int _trans;
@@ -112,7 +155,7 @@ _resume:
 #line 1 "excelsior_reader.rl"
 	{ts = p;}
 	break;
-#line 116 "excelsior_reader.c"
+#line 159 "excelsior_reader.c"
 		}
 	}
 
@@ -182,31 +225,31 @@ _eof_trans:
 	{te = p+1;}
 	break;
 	case 4:
-#line 26 "excelsior_reader.rl"
+#line 18 "excelsior_reader.rl"
 	{act = 2;}
 	break;
 	case 5:
-#line 27 "excelsior_reader.rl"
+#line 19 "excelsior_reader.rl"
 	{act = 3;}
 	break;
 	case 6:
-#line 28 "excelsior_reader.rl"
+#line 20 "excelsior_reader.rl"
 	{act = 4;}
 	break;
 	case 7:
-#line 25 "excelsior_reader.rl"
-	{te = p+1;{ rb_ary_push(rows, arr); arr = rb_ary_new(); has_found = 0; }}
+#line 17 "excelsior_reader.rl"
+	{te = p+1;{ rb_yield(arr); arr = rb_ary_new(); has_found = 0; }}
 	break;
 	case 8:
-#line 29 "excelsior_reader.rl"
+#line 21 "excelsior_reader.rl"
 	{te = p+1;{ if(has_found == 0) rb_ary_push(arr, Qnil); has_found = 0;}}
 	break;
 	case 9:
-#line 25 "excelsior_reader.rl"
-	{te = p;p--;{ rb_ary_push(rows, arr); arr = rb_ary_new(); has_found = 0; }}
+#line 17 "excelsior_reader.rl"
+	{te = p;p--;{ rb_yield(arr); arr = rb_ary_new(); has_found = 0; }}
 	break;
 	case 10:
-#line 28 "excelsior_reader.rl"
+#line 20 "excelsior_reader.rl"
 	{te = p;p--;{ rb_ary_push(arr, rb_str_new(ts + 1, te - ts - 2)); has_found = 1;}}
 	break;
 	case 11:
@@ -227,7 +270,7 @@ _eof_trans:
 	}
 	}
 	break;
-#line 231 "excelsior_reader.c"
+#line 274 "excelsior_reader.c"
 		}
 	}
 
@@ -244,7 +287,7 @@ _again:
 #line 1 "excelsior_reader.rl"
 	{act = 0;}
 	break;
-#line 248 "excelsior_reader.c"
+#line 291 "excelsior_reader.c"
 		}
 	}
 
@@ -264,18 +307,30 @@ _again:
 	_out: {}
 	}
 
-#line 36 "excelsior_reader.rl"
- if(RARRAY_LEN(arr) > 0) {
-   rb_ary_push(rows, arr);
- }
- return rows;
+#line 79 "excelsior_reader.rl"
+    
+    if(ts != 0) { // we are not at the end
+      have = pe - ts; //so copy stuff back in
+      memmove(buf, ts, have);
+      te = buf + (te - ts);
+      ts = buf;
+    }   
+    
+  }
+  
+  if(RARRAY_LEN(arr) > 0) { // have a last array to yield
+    rb_yield(arr);
+  }
+  
+  return Qnil;
 }
 
 VALUE mExcelsior;
 VALUE cReader;
 
 void Init_excelsior_reader() {
- mExcelsior = rb_define_module("Excelsior");
- cReader = rb_define_class_under(mExcelsior, "Reader", rb_cObject);
- rb_define_singleton_method(cReader, "parse", e_parse, 1);
+  s_read = rb_intern("read");
+  mExcelsior = rb_define_module("Excelsior");
+  cReader = rb_define_class_under(mExcelsior, "Reader", rb_cObject);
+  rb_define_singleton_method(cReader, "rows", e_rows, -1);
 }

data/ext/excelsior_reader/excelsior_reader.rl

@@ -1,19 +1,11 @@
-#include "ruby.h"
+#include <ruby.h>
 
-VALUE e_parse(VALUE self, VALUE data) {
- int cs, act, curline = 1, len = 0;
- char *ts = 0, *te = 0, *buf = NULL, *eof = NULL;
- char *p, *pe;
- VALUE rows;
- VALUE arr;
- rows = rb_ary_new();
- arr = rb_ary_new();
- len = RSTRING(data)->len;
- p = RSTRING(data)->ptr;
- pe = p + len;
- eof = pe;
- int has_found = 0;
- %%{
+static ID s_read;
+VALUE arr;
+int has_found = 0;
+#define BUFSIZE 16384
+
+%%{
    machine excelsior_scan;
    delimeter = ",";
    newline = "\r"? "\n" | "\r" | "\n";
@@ -22,28 +14,91 @@ VALUE e_parse(VALUE self, VALUE data) {
    string = '"' (schar | '""')* '"' ;
    value = letter+;
    main := |*
-     newline { rb_ary_push(rows, arr); arr = rb_ary_new(); has_found = 0; };
+     newline { rb_yield(arr); arr = rb_ary_new(); has_found = 0; };
      space;
      value { rb_ary_push(arr, rb_str_new(ts, te - ts)); has_found = 1;};
      string { rb_ary_push(arr, rb_str_new(ts + 1, te - ts - 2)); has_found = 1;};
      delimeter { if(has_found == 0) rb_ary_push(arr, Qnil); has_found = 0;};
    *|;
- }%%
+}%%
+ 
+%% write data nofinal; 
+
 
- %% write data nofinal;
- %% write init;
- %% write exec;
- if(RARRAY_LEN(arr) > 0) {
-   rb_ary_push(rows, arr);
- }
- return rows;
+VALUE e_rows(int argc, VALUE *argv, VALUE self) {
+  
+  int cs, act, have = 0, nread = 0, curline = 1, text = 0;
+  char *ts = 0, *te = 0, *buf = NULL, *eof = NULL;
+  int buffer_size = BUFSIZE;
+  
+  has_found = 0;
+  VALUE io;
+  int is_io = 0;
+  int done = 0;
+  
+  arr = rb_ary_new();
+  rb_scan_args(argc, argv, "1", &io);
+  
+  is_io = rb_respond_to(io, s_read);
+  buf = ALLOC_N(char, buffer_size);
+  
+  %% write init;
+  
+  while(!done) {
+  
+    int len, space = buffer_size - have;
+    VALUE str;
+    char *p, *pe;
+    p = buf + have;
+  
+    if(is_io) {
+      str = rb_funcall(io, s_read, 1, INT2FIX(space));
+      len = RSTRING_LEN(str);
+      memcpy(p, StringValuePtr(str), len);
+    } else { 
+      // Going to assume it's a string and already in memory
+      //str = io;
+      len = buffer_size = RSTRING_LEN(io);
+      memcpy(p, StringValuePtr(io), len);
+      space = buffer_size - have;
+      pe = p + buffer_size;
+      eof = pe;
+      done = 1;
+    }
+  
+    if(len < space) {
+      done = 1;
+      //p[len++] = 0; can't seem to get it to work with this
+      pe = p + len;
+      eof = pe;
+    } else {
+      pe = p + len; 
+    }
+  
+    %% write exec;
+    
+    if(ts != 0) { // we are not at the end
+      have = pe - ts; //so copy stuff back in
+      memmove(buf, ts, have);
+      te = buf + (te - ts);
+      ts = buf;
+    }   
+    
+  }
+  
+  if(RARRAY_LEN(arr) > 0) { // have a last array to yield
+    rb_yield(arr);
+  }
+  
+  return Qnil;
 }
 
 VALUE mExcelsior;
 VALUE cReader;
 
 void Init_excelsior_reader() {
- mExcelsior = rb_define_module("Excelsior");
- cReader = rb_define_class_under(mExcelsior, "Reader", rb_cObject);
- rb_define_singleton_method(cReader, "parse", e_parse, 1);
+  s_read = rb_intern("read");
+  mExcelsior = rb_define_module("Excelsior");
+  cReader = rb_define_class_under(mExcelsior, "Reader", rb_cObject);
+  rb_define_singleton_method(cReader, "rows", e_rows, -1);
 }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: toastyapps-excelsior
 version: !ruby/object:Gem::Version 
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors: 
 - Matthew Mongeau
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-08-17 21:00:00 -07:00
+date: 2009-08-20 21:00:00 -07:00
 default_executable: 
 dependencies: []