fastcsv 0.0.2 → 0.0.3

data/ext/fastcsv/fastcsv.rl

@@ -5,7 +5,7 @@
 // http://w3c.github.io/csvw/syntax/#ebnf
 
 // CSV implementation.
-// https://github.com/ruby/ruby/blob/master/lib/csv.rb
+// https://github.com/ruby/ruby/blob/trunk/lib/csv.rb
 
 // Ruby C extensions help.
 // https://github.com/ruby/ruby/blob/trunk/README.EXT
@@ -19,15 +19,24 @@ if (enc2 != NULL) { \
   field = rb_str_encode(field, rb_enc_from_encoding(enc), 0, Qnil); \
 }
 
-static VALUE mModule, rb_eParseError;
-static ID s_read, s_to_str, s_internal_encoding, s_external_encoding, s_string, s_encoding;
+#define FREE \
+if (buf != NULL) { \
+  free(buf); \
+} \
+if (row_sep != NULL) { \
+  free(row_sep); \
+}
 
-%%{
-  machine fastcsv;
+static VALUE cClass, cParser, eError;
+static ID s_read, s_row;
 
-  action new_line {
-    curline++;
-  }
+// @see https://github.com/nofxx/georuby_c/blob/b3b91fd90980d7c295ac8f6012d89878ea7cd569/ext/types.h#L22
+typedef struct {
+  char *start;
+} Data;
+
+%%{
+  machine raw_parse;
 
   action open_quote {
     unclosed_line = curline;
@@ -74,7 +83,7 @@ static ID s_read, s_to_str, s_internal_encoding, s_external_encoding, s_string,
         reader++;
       }
 
-      field = rb_enc_str_new(copy, writer - copy, enc);
+      field = rb_enc_str_new(copy, writer - copy, encoding);
       ENCODE;
 
       if (copy != NULL) {
@@ -88,7 +97,35 @@ static ID s_read, s_to_str, s_internal_encoding, s_external_encoding, s_string,
     field = Qnil;
   }
 
+  action mark_row {
+    d->start = p;
+
+    if (len_row_sep) {
+      if (p - mark_row_sep != len_row_sep || row_sep[0] != *mark_row_sep || len_row_sep == 2 && row_sep[1] != *(mark_row_sep + 1)) {
+        FREE;
+
+        rb_raise(eError, "Unquoted fields do not allow \\r or \\n (line %d).", curline - 1);
+      }
+    }
+    else {
+      len_row_sep = p - mark_row_sep;
+      row_sep = ALLOC_N(char, p - mark_row_sep);
+      memcpy(row_sep, mark_row_sep, p - mark_row_sep);
+    }
+  }
+
   action new_row {
+    mark_row_sep = p;
+
+    curline++;
+
+    if (d->start == 0 || p == d->start) {
+      rb_ivar_set(self, s_row, rb_str_new2(""));
+    }
+    else if (p > d->start) {
+      rb_ivar_set(self, s_row, rb_str_new(d->start, p - d->start));
+    }
+
     if (!NIL_P(field) || RARRAY_LEN(row)) { // same as new_field
       rb_ary_push(row, field);
       field = Qnil;
@@ -99,18 +136,26 @@ static ID s_read, s_to_str, s_internal_encoding, s_external_encoding, s_string,
   }
 
   action last_row {
+    if (d->start == 0 || p == d->start) {
+      rb_ivar_set(self, s_row, rb_str_new2(""));
+    }
+    else if (p > d->start) {
+      rb_ivar_set(self, s_row, rb_str_new(d->start, p - d->start));
+    }
+
     if (!NIL_P(field) || RARRAY_LEN(row)) {
       rb_ary_push(row, field);
     }
+
     if (RARRAY_LEN(row)) {
       rb_yield(row);
     }
   }
 
-  EOF = 0 >last_row;
+  EOF = 0;
   quote_char = '"';
   col_sep = ',' >new_field;
-  row_sep = ('\r' '\n'? | '\n') @new_line;
+  row_sep = ('\r' '\n'? | '\n');
   unquoted = (any* -- quote_char -- col_sep -- row_sep - EOF) %read_unquoted;
   quoted = quote_char >open_quote (any - quote_char - EOF | quote_char quote_char | row_sep)* %read_quoted quote_char >close_quote;
   field = unquoted | quoted;
@@ -118,9 +163,9 @@ static ID s_read, s_to_str, s_internal_encoding, s_external_encoding, s_string,
   # @see Ragel Guide: 6.3 Scanners
   # An unquoted field can be zero-length.
   main := |*
-    field col_sep EOF?;
-    field row_sep >new_row EOF?;
-    field EOF;
+    field col_sep;
+    field row_sep >new_row %mark_row;
+    field EOF >last_row;
   *|;
 }%%
 
@@ -130,9 +175,7 @@ static ID s_read, s_to_str, s_internal_encoding, s_external_encoding, s_string,
 #define BUFSIZE 16384
 
 // @see http://rxr.whitequark.org/mri/source/io.c#4845
-static void
-rb_io_ext_int_to_encs(rb_encoding *ext, rb_encoding *intern, rb_encoding **enc, rb_encoding **enc2, int fmode)
-{
+static void rb_io_ext_int_to_encs(rb_encoding *ext, rb_encoding *intern, rb_encoding **enc, rb_encoding **enc2, int fmode) {
   int default_ext = 0;
 
   if (ext == NULL) {
@@ -157,15 +200,17 @@ rb_io_ext_int_to_encs(rb_encoding *ext, rb_encoding *intern, rb_encoding **enc,
   }
 }
 
-VALUE fastcsv(int argc, VALUE *argv, VALUE self) {
+static VALUE raw_parse(int argc, VALUE *argv, VALUE self) {
   int cs, act, have = 0, curline = 1, io = 0;
-  char *ts = 0, *te = 0, *buf = 0, *eof = 0;
+  char *ts = 0, *te = 0, *buf = 0, *eof = 0, *mark_row_sep = 0, *row_sep = NULL;
 
-  VALUE port, opts;
+  VALUE port, opts, r_encoding;
   VALUE row = rb_ary_new(), field = Qnil, bufsize = Qnil;
-  int done = 0, unclosed_line = 0, buffer_size = 0, taint = 0;
+  int done = 0, unclosed_line = 0, len_row_sep = 0, buffer_size = 0, taint = 0;
   rb_encoding *enc = NULL, *enc2 = NULL, *encoding = NULL;
-  VALUE r_encoding;
+
+  Data *d;
+  Data_Get_Struct(self, Data, d);
 
   VALUE option;
   char quote_char = '"';
@@ -174,8 +219,8 @@ VALUE fastcsv(int argc, VALUE *argv, VALUE self) {
   taint = OBJ_TAINTED(port);
   io = rb_respond_to(port, s_read);
   if (!io) {
-    if (rb_respond_to(port, s_to_str)) {
-      port = rb_funcall(port, s_to_str, 0);
+    if (rb_respond_to(port, rb_intern("to_str"))) {
+      port = rb_funcall(port, rb_intern("to_str"), 0);
       StringValue(port);
     }
     else {
@@ -199,7 +244,7 @@ VALUE fastcsv(int argc, VALUE *argv, VALUE self) {
   // @see http://ruby-doc.org/core-2.1.1/IO.html#method-c-new-label-Open+Mode
   option = rb_hash_aref(opts, ID2SYM(rb_intern("encoding")));
   if (TYPE(option) == T_STRING) {
-    // parse_mode_enc is not in header file.
+    // `parse_mode_enc` is not in header file.
     const char *estr = StringValueCStr(option), *ptr;
     char encname[ENCODING_MAXNAMELEN+1];
     int idx, idx2;
@@ -210,17 +255,17 @@ VALUE fastcsv(int argc, VALUE *argv, VALUE self) {
     ptr = strrchr(estr, ':');
     if (ptr) {
       long len = (ptr++) - estr;
-      if (len == 0 || len > ENCODING_MAXNAMELEN) {
+      if (len == 0 || len > ENCODING_MAXNAMELEN) { // ":enc"
         idx = -1;
       }
-      else {
+      else { // "enc2:enc" or "enc:-"
         memcpy(encname, estr, len);
         encname[len] = '\0';
         estr = encname;
         idx = rb_enc_find_index(encname);
       }
     }
-    else {
+    else { // "enc"
       idx = rb_enc_find_index(estr);
     }
 
@@ -228,7 +273,7 @@ VALUE fastcsv(int argc, VALUE *argv, VALUE self) {
       ext_enc = rb_enc_from_index(idx);
     }
     else {
-      if (idx != -2) {
+      if (idx != -2) { // ":enc"
         // `unsupported_encoding` is not in header file.
         rb_warn("Unsupported encoding %s ignored", estr);
       }
@@ -237,11 +282,11 @@ VALUE fastcsv(int argc, VALUE *argv, VALUE self) {
 
     int_enc = NULL;
     if (ptr) {
-      if (*ptr == '-' && *(ptr+1) == '\0') {
+      if (*ptr == '-' && *(ptr+1) == '\0') { // "enc:-"
         /* Special case - "-" => no transcoding */
         int_enc = (rb_encoding *)Qnil;
       }
-      else {
+      else { // "enc2:enc"
         idx2 = rb_enc_find_index(ptr);
         if (idx2 < 0) {
           // `unsupported_encoding` is not in header file.
@@ -262,29 +307,33 @@ VALUE fastcsv(int argc, VALUE *argv, VALUE self) {
     rb_raise(rb_eArgError, ":encoding has to be a String");
   }
 
-  // @see https://github.com/ruby/ruby/blob/70510d026f8d86693dccaba07417488eed09b41d/lib/csv.rb#L1567
-  // @see https://github.com/ruby/ruby/blob/70510d026f8d86693dccaba07417488eed09b41d/lib/csv.rb#L2300
-  if (rb_respond_to(port, s_internal_encoding)) {
-    r_encoding = rb_funcall(port, s_internal_encoding, 0);
+  // @see CSV#raw_encoding
+  // @see https://github.com/ruby/ruby/blob/ab337e61ecb5f42384ba7d710c36faf96a454e5c/lib/csv.rb#L2290
+  if (rb_respond_to(port, rb_intern("internal_encoding"))) {
+    r_encoding = rb_funcall(port, rb_intern("internal_encoding"), 0);
     if (NIL_P(r_encoding)) {
-      r_encoding = rb_funcall(port, s_external_encoding, 0);
+      r_encoding = rb_funcall(port, rb_intern("external_encoding"), 0);
     }
   }
-  else if (rb_respond_to(port, s_string)) {
-    r_encoding = rb_funcall(rb_funcall(port, s_string, 0), s_encoding, 0);
+  else if (rb_respond_to(port, rb_intern("string"))) {
+    r_encoding = rb_funcall(rb_funcall(port, rb_intern("string"), 0), rb_intern("encoding"), 0);
   }
-  else if (rb_respond_to(port, s_encoding)) {
-    r_encoding = rb_funcall(port, s_encoding, 0);
+  else if (rb_respond_to(port, rb_intern("encoding"))) {
+    r_encoding = rb_funcall(port, rb_intern("encoding"), 0);
   }
   else {
     r_encoding = rb_enc_from_encoding(rb_ascii8bit_encoding());
   }
+
+  // @see CSV#initialize
+  // @see https://github.com/ruby/ruby/blob/ab337e61ecb5f42384ba7d710c36faf96a454e5c/lib/csv.rb#L1510
   if (NIL_P(r_encoding)) {
     r_encoding = rb_enc_from_encoding(rb_default_internal_encoding());
   }
   if (NIL_P(r_encoding)) {
     r_encoding = rb_enc_from_encoding(rb_default_external_encoding());
   }
+
   if (enc2 != NULL) {
     encoding = enc2;
   }
@@ -295,11 +344,19 @@ VALUE fastcsv(int argc, VALUE *argv, VALUE self) {
     encoding = rb_enc_get(r_encoding);
   }
 
+  // In case #raw_parse is called multiple times on the same parser. Note that
+  // using IO methods on a re-used parser can cause segmentation faults.
+  rb_ivar_set(self, s_row, Qnil);
+
   buffer_size = BUFSIZE;
   if (rb_ivar_defined(self, rb_intern("@buffer_size")) == Qtrue) {
     bufsize = rb_ivar_get(self, rb_intern("@buffer_size"));
     if (!NIL_P(bufsize)) {
       buffer_size = NUM2INT(bufsize);
+      // buffer_size = 0 can cause segmentation faults.
+      if (buffer_size == 0) {
+        buffer_size = BUFSIZE;
+      }
     }
   }
 
@@ -312,26 +369,34 @@ VALUE fastcsv(int argc, VALUE *argv, VALUE self) {
   while (!done) {
     VALUE str;
     char *p, *pe;
-    int len, space = buffer_size - have, tokstart_diff, tokend_diff;
+    int len, space = buffer_size - have, tokstart_diff, tokend_diff, start_diff, mark_row_sep_diff;
 
     if (io) {
       if (space == 0) {
-         tokstart_diff = ts - buf;
-         tokend_diff = te - buf;
+        // Not moving d->start will cause intermittent segmentation faults.
+        tokstart_diff = ts - buf;
+        tokend_diff = te - buf;
+        start_diff = d->start - buf;
+        mark_row_sep_diff = mark_row_sep - buf;
 
-         buffer_size += BUFSIZE;
-         REALLOC_N(buf, char, buffer_size);
+        buffer_size += BUFSIZE;
+        REALLOC_N(buf, char, buffer_size);
 
-         space = buffer_size - have;
+        space = buffer_size - have;
 
-         ts = buf + tokstart_diff;
-         te = buf + tokend_diff;
+        ts = buf + tokstart_diff;
+        te = buf + tokend_diff;
+        d->start = buf + start_diff;
+        mark_row_sep = buf + mark_row_sep_diff;
       }
       p = buf + have;
 
+      // Reads "`length` bytes without any conversion (binary mode)."
+      // "The resulted string is always ASCII-8BIT encoding."
+      // @see http://www.ruby-doc.org/core-2.1.4/IO.html#method-i-read
       str = rb_funcall(port, s_read, 1, INT2FIX(space));
       if (NIL_P(str)) {
-        // StringIO#read returns nil for empty string.
+        // "`nil` means it met EOF at beginning," e.g. for `StringIO.new("")`.
         len = 0;
       }
       else {
@@ -339,6 +404,7 @@ VALUE fastcsv(int argc, VALUE *argv, VALUE self) {
         memcpy(p, StringValuePtr(str), len);
       }
 
+      // "The 1 to `length`-1 bytes string means it met EOF after reading the result."
       if (len < space) {
         // EOF actions don't work in scanners, so we add a sentinel value.
         // @see http://www.complang.org/pipermail/ragel-users/2007-May/001516.html
@@ -354,22 +420,21 @@ VALUE fastcsv(int argc, VALUE *argv, VALUE self) {
       done = 1;
     }
 
+    if (d->start == 0) {
+      d->start = p;
+    }
+
     pe = p + len;
     %% write exec;
 
-    if (done && cs < fastcsv_first_final) {
-      if (buf != NULL) {
-        free(buf);
-      }
+    if (done && cs < raw_parse_first_final) {
+      FREE;
+
       if (unclosed_line) {
-        rb_raise(rb_eParseError, "Unclosed quoted field on line %d.", unclosed_line);
+        rb_raise(eError, "Unclosed quoted field on line %d.", unclosed_line);
       }
-      // Ruby raises different errors for illegal quoting, depending on whether
-      // a quoted string is followed by a string ("Unclosed quoted field on line
-      // %d.") or by a string ending in a quote ("Missing or stray quote in line
-      // %d"). These precisions are kind of bogus, but we can try using $!.
       else {
-        rb_raise(rb_eParseError, "Illegal quoting in line %d.", curline);
+        rb_raise(eError, "Illegal quoting in line %d.", curline);
       }
     }
 
@@ -384,23 +449,35 @@ VALUE fastcsv(int argc, VALUE *argv, VALUE self) {
     }
   }
 
-  if (buf != NULL) {
-    free(buf);
-  }
+  FREE;
 
   return Qnil;
 }
 
+// @see https://github.com/ruby/ruby/blob/trunk/README.EXT#L616
+static VALUE allocate(VALUE class) {
+  // @see https://github.com/nofxx/georuby_c/blob/b3b91fd90980d7c295ac8f6012d89878ea7cd569/ext/line.c#L66
+  Data *d = ALLOC(Data);
+  d->start = 0;
+  // @see https://github.com/nofxx/georuby_c/blob/b3b91fd90980d7c295ac8f6012d89878ea7cd569/ext/point.h#L26
+  // rb_gc_mark(d->start) or rb_gc_mark(d) cause warning "passing argument 1 of ‘rb_gc_mark’ makes integer from pointer without a cast"
+  // free(d->start) causes error "pointer being freed was not allocated"
+  return Data_Wrap_Struct(class, NULL, free, d);
+}
+
+// @see http://tenderlovemaking.com/2009/12/18/writing-ruby-c-extensions-part-1.html
+// @see http://tenderlovemaking.com/2010/12/11/writing-ruby-c-extensions-part-2.html
 void Init_fastcsv() {
   s_read = rb_intern("read");
-  s_to_str = rb_intern("to_str");
-  s_internal_encoding = rb_intern("internal_encoding");
-  s_external_encoding = rb_intern("external_encoding");
-  s_string = rb_intern("string");
-  s_encoding = rb_intern("encoding");
-
-  mModule = rb_define_module("FastCSV");
-  rb_define_attr(rb_singleton_class(mModule), "buffer_size", 1, 1);
-  rb_define_singleton_method(mModule, "raw_parse", fastcsv, -1);
-  rb_eParseError = rb_define_class_under(mModule, "ParseError", rb_eStandardError);
+  s_row = rb_intern("@row");
+
+  cClass = rb_define_class("FastCSV", rb_const_get(rb_cObject, rb_intern("CSV"))); // class FastCSV < CSV
+  cParser = rb_define_class_under(cClass, "Parser", rb_cObject);                   //   class Parser
+  rb_define_alloc_func(cParser, allocate);                                         //
+  rb_define_method(cParser, "raw_parse", raw_parse, -1);                           //     def raw_parse(port, opts = nil); end
+  rb_define_attr(cParser, "row", 1, 0);                                            //     attr_reader :row
+  rb_define_attr(cParser, "buffer_size", 1, 1);                                    //     attr_accessor :buffer_size
+                                                                                   //   end
+  eError = rb_define_class_under(cClass, "MalformedCSVError", rb_eRuntimeError);   //   class MalformedCSVError < RuntimeError
+                                                                                   // end
 }

data/fastcsv.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name        = "fastcsv"
-  s.version     = '0.0.2'
+  s.version     = '0.0.3'
   s.platform    = Gem::Platform::RUBY
   s.authors     = ["Open North"]
   s.email       = ["info@opennorth.ca"]

data/lib/fastcsv.rb

@@ -1 +1,131 @@
+require 'csv'
+
 require 'fastcsv/fastcsv'
+
+# @see https://github.com/ruby/ruby/blob/ab337e61ecb5f42384ba7d710c36faf96a454e5c/lib/csv.rb
+class FastCSV < CSV
+  def self.raw_parse(*args, &block)
+    Parser.new.raw_parse(*args, &block)
+  end
+
+  def shift
+    # COPY
+    # handle headers not based on document content
+    if header_row? and @return_headers and
+       [Array, String].include? @use_headers.class
+      if @unconverted_fields
+        return add_unconverted_fields(parse_headers, Array.new)
+      else
+        return parse_headers
+      end
+    end
+    # PASTE
+
+    # The CSV library wraps File objects, whereas `FastCSV.raw_parse` accepts
+    # IO-like objects that implement `#read(length)`.
+    begin
+      unless csv = fiber.resume # was unless parse = @io.gets(@row_sep)
+        return nil
+      end
+    rescue FiberError
+      return nil
+    end
+
+    row = parser.row
+
+    # COPY
+    if csv.empty?
+      #
+      # I believe a blank line should be an <tt>Array.new</tt>, not Ruby 1.8
+      # CSV's <tt>[nil]</tt>
+      #
+      if row.empty? # was if parse.empty?
+        @lineno += 1
+        if @skip_blanks
+          return shift # was next
+        elsif @unconverted_fields
+          return add_unconverted_fields(Array.new, Array.new)
+        elsif @use_headers
+          return self.class::Row.new(Array.new, Array.new)
+        else
+          return Array.new
+        end
+      end
+    end
+    # PASTE
+
+    return shift if @skip_lines and @skip_lines.match row # was next if @skip_lines and @skip_lines.match parse
+
+    # COPY
+    @lineno += 1
+
+    # save fields unconverted fields, if needed...
+    unconverted = csv.dup if @unconverted_fields
+
+    # convert fields, if needed...
+    csv = convert_fields(csv) unless @use_headers or @converters.empty?
+    # parse out header rows and handle CSV::Row conversions...
+    csv = parse_headers(csv)  if     @use_headers
+
+    # inject unconverted fields and accessor, if requested...
+    if @unconverted_fields and not csv.respond_to? :unconverted_fields
+      add_unconverted_fields(csv, unconverted)
+    end
+    # PASTE
+
+    csv # was break csv
+  end
+
+  # CSV's delegated and overwritten IO methods move the pointer within the file,
+  # but FastCSV doesn't notice, so we need to recreate the fiber. The old fiber
+  # is garbage collected.
+
+  def pos=(*args)
+    super
+    @parser = nil
+    @fiber = nil
+  end
+  def reopen(*args)
+    super
+    @parser = nil
+    @fiber = nil
+  end
+  def seek(*args)
+    super
+    @parser = nil
+    @fiber = nil
+  end
+  def rewind
+    super
+    @parser = nil
+    @fiber = nil
+  end
+
+private
+
+  def parser
+    @parser ||= Parser.new
+  end
+
+  def fiber
+    # @see http://www.ruby-doc.org/core-2.1.4/Fiber.html
+    @fiber ||= Fiber.new do
+      if @io.respond_to?(:internal_encoding)
+        enc2 = @io.external_encoding
+        enc = @io.internal_encoding || '-'
+        if enc2
+          encoding = "#{enc2}:#{enc}"
+        else
+          encoding = enc
+        end
+      end
+      parser.raw_parse(@io, encoding: encoding) do |row|
+        Fiber.yield(row)
+      end
+    end
+  end
+end
+
+def FastCSV(*args, &block)
+  FastCSV.instance(*args, &block)
+end