rcsv 0.0.6 → 0.0.8

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    rcsv (0.0.6)
+    rcsv (0.0.8)
 
 GEM
   remote: https://rubygems.org/

data/README.md

@@ -4,7 +4,7 @@
 
 Rcsv is a fast CSV parsing library for MRI Ruby. Tested on REE 1.8.7 and Ruby 1.9.3.
 
-Contrary to many other gems that implement their own parsers, Rcsv uses libcsv 3.0.2 (http://sourceforge.net/projects/libcsv/). As long as libcsv's API is stable, getting Rcsv to use newer libcsv version is as simple as updating two files (csv.h and libcsv.c).
+Contrary to many other gems that implement their own parsers, Rcsv uses libcsv 3.0.3 (http://sourceforge.net/projects/libcsv/). As long as libcsv's API is stable, getting Rcsv to use newer libcsv version is as simple as updating two files (csv.h and libcsv.c).
 
 ## Benchmarks
                    user     system      total        real
@@ -48,6 +48,7 @@ Quickstart:
 
 
 Rcsv class exposes a class method *parse* that accepts a CSV string as its first parameter and options hash as its second parameter.
+If block is passed, Rcsv sequentially yields every parsed line to it and the #parse method itself returns nil.
 
 
 Options supported:
@@ -60,7 +61,19 @@ A single-character string that is used as a separator. Default is ",".
 
 A boolean flag. When enabled, allows to parse oddly quoted CSV data without exceptions being raised. Disabled by default.
 
-Anything that does not conform to http://www.ietf.org/rfc/rfc4180.txt should better be parsed with this option enabled.
+Anything that does not conform to http://www.creativyst.com/Doc/Articles/CSV/CSV01.htm should better be parsed with this option enabled.
+
+### :parse_empty_fields_as
+
+A Ruby symbol that specifies how empty CSV fields should be processed. Accepted values:
+
+* :nil_or_string (default) - If empty field is quoted, it is parsed as empty Ruby string. If empty field is not quoted, it is parsed as Nil.
+
+* :nil - Always parse as Nil.
+
+* :string - Always parse as empty string.
+
+This option doesn't affect defaults processing: all empty fields are replaced with default values if the latter are provided (via per-column :default).
 
 ### :offset_rows
 
@@ -73,7 +86,7 @@ If CSV has a header, :columns keys can be strings that are equal to column names
 
 :columns values are in turn hashes that provide parsing options:
 
-* :alias - Object of any type (though usually a Symbol) that is used to as a key that represents column name when :row_as_hash is set.
+* :alias - Object of any type (though usually a Symbol) that is used as a key that represents column name when :row_as_hash is set.
 * :type - A Ruby Symbol that specifies Ruby data type that CSV cell value should be converted into. Supported types: :int, :float, :string, :bool. :string is the default.
 * :default - Object of any type (though usually of the same type that is specified by :type option). If CSV doesn't have any value for a cell, this default value is used.
 * :match - A string. If set, makes Rcsv skip all the rows where any column doesn't match its :match value. Useful for filtering data.
@@ -96,6 +109,10 @@ When :row_as_hash is disabled, return value is represented as array of arrays.
 ### :only_listed_columns
 A boolean flag. If enabled, only parses columns that are listed in :columns. Disabled by default.
 
+### :buffer_size
+An integer. Default is 1MiB (1024 * 1024).
+Specifies a number of bytes that are read at once, thus allowing to read drectly from IO-like objects (files, sockets etc).
+
 
 ## Examples
 
@@ -132,12 +149,29 @@ The result would look like this:
       [ nil, 0, "Vacuum" ]
     ]
 
+And here is an example of passing a block:
+
+    Rcsv.parse(some_csv) { |row|
+      puts row.inspect
+    }
+
+That would display contents of each row without needing to put the whole parsed result array to memory:
+
+    ["a", "b", "c", "d", "e", "f"]
+    ["1", "2", "3", "4", "5", "6"]
+
+
+This way it is possible to read from a File directly, with a 20MiB buffer and parse lines one by one:
+
+    Rcsv.parse(File.open('/some/file.csv'), :buffer_size => 20 * 1024 * 1024) { |row|
+      puts row.inspect
+    }
+
 
 ## To do
 
 * More specs for boolean values
 * Specs for Ruby parse
-* Add custom Ruby callbacks (if block is passed)
 * Add CSV write support
 
 

data/RELNOTES

@@ -0,0 +1,5 @@
+Version 0.0.8
+  * libcsv upgraded to 3.0.3
+  * :parse_empty_fields_as option added
+  * README.md updated with a proper reference to CSV parsing guidelines
+  * RELNOTES added

data/ext/rcsv/csv.h

@@ -8,8 +8,8 @@ extern "C" {
 #endif
 
 #define CSV_MAJOR 3
-#define CSV_MINOR 1
-#define CSV_RELEASE 0
+#define CSV_MINOR 0
+#define CSV_RELEASE 3
 
 /* Error Codes */
 #define CSV_SUCCESS 0
@@ -25,7 +25,9 @@ extern "C" {
 #define CSV_STRICT_FINI 4 /* causes csv_fini to return CSV_EPARSE if last
                              field is quoted and doesn't containg ending 
                              quote */
-#define CSV_APPEND_NULL 8 /* Ensure that all fields are null-ternimated */
+#define CSV_APPEND_NULL 8 /* Ensure that all fields are null-terminated */
+#define CSV_EMPTY_IS_NULL 16 /* Pass null pointer to cb1 function when
+                                empty, unquoted fields are encountered */
 
 
 /* Character values */

data/ext/rcsv/libcsv.c

@@ -25,7 +25,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 #include "csv.h"
 
-#define VERSION "3.0.2"
+#define VERSION "3.0.3"
 
 #define ROW_NOT_BEGUN           0
 #define FIELD_NOT_BEGUN         1
@@ -50,7 +50,9 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
      entry_pos -= spaces; \
    if (p->options & CSV_APPEND_NULL) \
      ((p)->entry_buf[entry_pos]) = '\0'; \
-   if (cb1) \
+   if (cb1 && (p->options & CSV_EMPTY_IS_NULL) && !quoted && entry_pos == 0) \
+     cb1(NULL, entry_pos, data); \
+   else if (cb1) \
      cb1(p->entry_buf, entry_pos, data); \
    pstate = FIELD_NOT_BEGUN; \
    entry_pos = quoted = spaces = 0; \

data/ext/rcsv/rcsv.c

@@ -12,6 +12,7 @@ static VALUE rcsv_parse_error; /* class Rcsv::ParseError << StandardError; end *
 struct rcsv_metadata {
   /* Derived from user-specified options */
   bool row_as_hash;           /* Used to return array of hashes rather than array of arrays */
+  bool empty_field_is_nil;   /* Do we convert empty fields to nils? */
   size_t offset_rows;         /* Number of rows to skip before parsing */
 
   char * row_conversions;     /* A pointer to string/array of row conversions char specifiers */
@@ -30,6 +31,7 @@ struct rcsv_metadata {
   size_t current_col;         /* Current column's index */
   size_t current_row;         /* Current row's index */
 
+  VALUE last_entry;           /* A pointer to the last entry that's going to be appended to result */
   VALUE * result;             /* A pointer to the parsed data */
 };
 
@@ -41,7 +43,6 @@ void end_of_field_callback(void * field, size_t field_size, void * data) {
   struct rcsv_metadata * meta = (struct rcsv_metadata *) data;
   char row_conversion = 0;
   VALUE parsed_field;
-  VALUE last_entry = rb_ary_entry(*(meta->result), -1); /* result.last */
 
   /* No need to parse anything until the end of the line if skip_current_row is set */
   if (meta->skip_current_row) {
@@ -56,6 +57,7 @@ void end_of_field_callback(void * field, size_t field_size, void * data) {
 
   /* Filter by string row values listed in meta->only_rows */
   if ((meta->only_rows != NULL) &&
+      (field_str != NULL) && /* TODO: What if we want to filter out NULLs? */
       (meta->current_col < meta->num_only_rows) &&
       (meta->only_rows[meta->current_col] != NULL) &&
       (strcmp(meta->only_rows[meta->current_col], field_str))) {
@@ -74,8 +76,12 @@ void end_of_field_callback(void * field, size_t field_size, void * data) {
       /* Assigning appropriate default value if applicable. */
       if (meta->current_col < meta->num_row_defaults) {
         parsed_field = meta->row_defaults[meta->current_col];
-      } else { /* By default, default is nil */
-        parsed_field = Qnil;
+      } else { /* It depends on empty_field_is_nil if we convert empty strings to nils */
+        if (meta->empty_field_is_nil || field_str == NULL) {
+          parsed_field = Qnil;
+        } else {
+          parsed_field = rb_str_new2("");
+        }
       }
     } else {
       if (meta->current_col < meta->num_row_conversions) {
@@ -136,10 +142,10 @@ void end_of_field_callback(void * field, size_t field_size, void * data) {
           (int)meta->num_columns
         );
       } else {
-        rb_hash_aset(last_entry, meta->column_names[meta->current_col], parsed_field);
+        rb_hash_aset(meta->last_entry, meta->column_names[meta->current_col], parsed_field); /* last_entry[column_names[current_col]] = field */
       }
     } else { /* Parse into Array */
-      rb_ary_push(last_entry, parsed_field); /* result << field */
+      rb_ary_push(meta->last_entry, parsed_field); /* last_entry << field */
     }
   }
 
@@ -154,16 +160,22 @@ void end_of_line_callback(int last_char, void * data) {
 
   /* If filters didn't match, current row parsing is reverted */
   if (meta->skip_current_row) {
-    rb_ary_pop(*(meta->result)); /* result.pop */
+    /* Do we wanna GC? */
     meta->skip_current_row = false;
+  } else {
+    if (rb_block_given_p()) { /* STREAMING */
+      rb_yield(meta->last_entry);
+    } else {
+      rb_ary_push(*(meta->result), meta->last_entry);
+    }
   }
 
-  /* Add a new empty array/hash for the next line unless EOF reached */
+  /* Re-initialize last_entry unless EOF reached */
   if (last_char != -1) {
     if (meta->row_as_hash) {
-      rb_ary_push(*(meta->result), rb_hash_new()); /* result << {} */
+      meta->last_entry = rb_hash_new(); /* {} */
     } else {
-      rb_ary_push(*(meta->result), rb_ary_new()); /* result << [] */
+      meta->last_entry = rb_ary_new(); /* [] */
     }
   }
 
@@ -175,12 +187,19 @@ void end_of_line_callback(int last_char, void * data) {
   return;
 }
 
+void custom_end_of_line_callback(int last_char, void * data) {
+  struct rcsv_metadata * meta = (struct rcsv_metadata *) data;
+
+  if (!meta->skip_current_row) {
+  }
+}
+
 /* C API */
 
 /* The main method that handles parsing */
 static VALUE rb_rcsv_raw_parse(int argc, VALUE * argv, VALUE self) {
   struct rcsv_metadata meta;
-  VALUE str, options, option;
+  VALUE csvio, csvstr, buffer_size, options, option;
 
   struct csv_parser cp;
   unsigned char csv_options = CSV_STRICT_FINI | CSV_APPEND_NULL;
@@ -191,6 +210,7 @@ static VALUE rb_rcsv_raw_parse(int argc, VALUE * argv, VALUE self) {
 
   /* Setting up some sane defaults */
   meta.row_as_hash = false;
+  meta.empty_field_is_nil = false;
   meta.skip_current_row = false;
   meta.num_columns = 0;
   meta.current_col = 0;
@@ -205,22 +225,34 @@ static VALUE rb_rcsv_raw_parse(int argc, VALUE * argv, VALUE self) {
   meta.column_names = NULL;
   meta.result = (VALUE[]){rb_ary_new()}; /* [] */
 
-  /* str is required, options is optional (pun intended) */
-  rb_scan_args(argc, argv, "11", &str, &options);
-  csv_string = StringValuePtr(str);
-  csv_string_len = strlen(csv_string);
+  /* csvio is required, options is optional (pun intended) */
+  rb_scan_args(argc, argv, "11", &csvio, &options);
 
   /* options ||= nil */
   if (NIL_P(options)) {
     options = rb_hash_new();
   }
 
+  buffer_size = rb_hash_aref(options, ID2SYM(rb_intern("buffer_size")));
+
   /* By default, parsing is strict */
   option = rb_hash_aref(options, ID2SYM(rb_intern("nostrict")));
   if (!option || (option == Qnil)) {
     csv_options |= CSV_STRICT;
   }
 
+  /* By default, empty strings are treated as Nils and quoted empty strings are treated as empty Ruby strings */
+  option = rb_hash_aref(options, ID2SYM(rb_intern("parse_empty_fields_as")));
+  if ((option == Qnil) || (option == ID2SYM(rb_intern("nil_or_string")))) {
+    csv_options |= CSV_EMPTY_IS_NULL;
+  } else if (option == ID2SYM(rb_intern("nil"))) {
+    meta.empty_field_is_nil = true;
+  } else if (option == ID2SYM(rb_intern("string"))) {
+    meta.empty_field_is_nil = false;
+  } else {
+    rb_raise(rcsv_parse_error, "The only valid options for :parse_empty_fields_as are :nil, :string and :nil_or_string, but %s was supplied.", RSTRING_PTR(rb_inspect(option)));
+  }
+
   /* Try to initialize libcsv */
   if (csv_init(&cp, csv_options) == -1) {
     rb_raise(rcsv_parse_error, "Couldn't initialize libcsv");
@@ -283,12 +315,14 @@ static VALUE rb_rcsv_raw_parse(int argc, VALUE * argv, VALUE self) {
     meta.row_conversions = StringValuePtr(option);
   }
 
-  /* Column names should be declared explicitly when parsing fields as Hashes */
+ /* Column names should be declared explicitly when parsing fields as Hashes */
   if (meta.row_as_hash) { /* Only matters for hash results */
     option = rb_hash_aref(options, ID2SYM(rb_intern("column_names"))); 
     if (option == Qnil) {
       rb_raise(rcsv_parse_error, ":row_as_hash requires :column_names to be set.");
     } else {
+      meta.last_entry = rb_hash_new();
+
       meta.num_columns = (size_t)RARRAY_LEN(option);
       meta.column_names = (VALUE*)malloc(meta.num_columns * sizeof(VALUE*));
 
@@ -296,34 +330,37 @@ static VALUE rb_rcsv_raw_parse(int argc, VALUE * argv, VALUE self) {
         meta.column_names[i] = rb_ary_entry(option, i);
       }
     }
-  }
-
-  /* Initializing result with empty Array */
-  if (meta.row_as_hash) {
-    rb_ary_push(*(meta.result), rb_hash_new()); /* [{}] */
   } else {
-    rb_ary_push(*(meta.result), rb_ary_new()); /* [[]] */
+    meta.last_entry = rb_ary_new();
   }
 
-  /* Actual parsing and error handling */
-  if (csv_string_len != csv_parse(&cp, csv_string, strlen(csv_string),
-                                  &end_of_field_callback, &end_of_line_callback, &meta)) {
-    error = csv_error(&cp);
-    switch(error) {
-      case CSV_EPARSE:
-        rb_raise(rcsv_parse_error, "Error when parsing malformed data");
+  while(true) {
+    csvstr = rb_funcall(csvio, rb_intern("read"), 1, buffer_size);
+    if ((csvstr == Qnil) || (RSTRING_LEN(csvstr) == 0)) { break; }
+
+    csv_string = StringValuePtr(csvstr);
+    csv_string_len = strlen(csv_string);
+
+    /* Actual parsing and error handling */
+    if (csv_string_len != csv_parse(&cp, csv_string, csv_string_len,
+                                    &end_of_field_callback, &end_of_line_callback, &meta)) {
+      error = csv_error(&cp);
+      switch(error) {
+        case CSV_EPARSE:
+          rb_raise(rcsv_parse_error, "Error when parsing malformed data");
+          break;
+        case CSV_ENOMEM:
+          rb_raise(rcsv_parse_error, "No memory");
+          break;
+        case CSV_ETOOBIG:
+          rb_raise(rcsv_parse_error, "Field data is too large");
+          break;
+        case CSV_EINVALID:
+          rb_raise(rcsv_parse_error, "%s", (const char *)csv_strerror(error));
         break;
-      case CSV_ENOMEM:
-        rb_raise(rcsv_parse_error, "No memory");
-        break;
-      case CSV_ETOOBIG:
-        rb_raise(rcsv_parse_error, "Field data is too large");
-        break;
-      case CSV_EINVALID:
-        rb_raise(rcsv_parse_error, "%s", (const char *)csv_strerror(error));
-      break;
-      default:
-        rb_raise(rcsv_parse_error, "Failed due to unknown reason");
+        default:
+          rb_raise(rcsv_parse_error, "Failed due to unknown reason");
+      }
     }
   }
 
@@ -344,12 +381,16 @@ static VALUE rb_rcsv_raw_parse(int argc, VALUE * argv, VALUE self) {
   }
 
   /* Remove the last row if it's empty. That happens if CSV file ends with a newline. */
-  if (RARRAY_LEN(rb_ary_entry(*(meta.result), -1)) == 0) {
+  if (RARRAY_LEN(*(meta.result)) && /* meta.result.size != 0 */
+      RARRAY_LEN(rb_ary_entry(*(meta.result), -1)) == 0) {
     rb_ary_pop(*(meta.result));
   }
 
-  /* An array of arrays of strings is returned. */
-  return *(meta.result);
+  if (rb_block_given_p()) {
+    return Qnil; /* STREAMING */
+  } else {
+    return *(meta.result); /* Return accumulated result */
+  }
 }
 
 

data/lib/rcsv/version.rb

@@ -1,3 +1,3 @@
 class Rcsv
-  VERSION = "0.0.6"
+  VERSION = "0.0.8"
 end

data/lib/rcsv.rb

@@ -1,8 +1,10 @@
 require "rcsv/rcsv"
 require "rcsv/version"
 
+require "stringio"
+
 class Rcsv
-  def self.parse(csv_data, options = {})
+  def self.parse(csv_data, options = {}, &block)
     #options = {
       #:column_separator => "\t",
       #:only_listed_columns => true,
@@ -25,16 +27,27 @@ class Rcsv
     raw_options[:col_sep] = options[:column_separator] && options[:column_separator][0] || ','
     raw_options[:offset_rows] = options[:offset_rows] || 0
     raw_options[:nostrict] = options[:nostrict]
+    raw_options[:parse_empty_fields_as] = options[:parse_empty_fields_as]
+    raw_options[:buffer_size] = options[:buffer_size] || 1024 * 1024 # 1 MiB
+
+    if csv_data.is_a?(String)
+      csv_data = StringIO.new(csv_data)
+    elsif !(csv_data.respond_to?(:lines) && csv_data.respond_to?(:read))
+      inspected_csv_data = csv_data.inspect
+      raise ParseError.new("Supplied CSV object #{inspected_csv_data[0..127]}#{inspected_csv_data.size > 128 ? '...' : ''} is neither String nor looks like IO object.")
+    end
+
+    initial_position = csv_data.pos
 
     case options[:header]
     when :use
-      header = self.raw_parse(csv_data.lines.first, raw_options).first
+      header = self.raw_parse(StringIO.new(csv_data.lines.first), raw_options).first
       raw_options[:offset_rows] += 1
     when :skip
       header = (0..(csv_data.lines.first.split(raw_options[:col_sep]).count)).to_a
       raw_options[:offset_rows] += 1
     when :none
-     header = (0..(csv_data.lines.first.split(raw_options[:col_sep]).count)).to_a
+      header = (0..(csv_data.lines.first.split(raw_options[:col_sep]).count)).to_a
     end
 
     raw_options[:row_as_hash] = options[:row_as_hash] # Setting after header parsing
@@ -86,6 +99,7 @@ class Rcsv
       raw_options[:row_conversions] = row_conversions
     end
 
-    return self.raw_parse(csv_data, raw_options)
+    csv_data.pos = initial_position
+    return self.raw_parse(csv_data, raw_options, &block)
   end
 end

data/test/test_rcsv_raw_parse.rb

@@ -3,7 +3,7 @@ require 'rcsv'
 
 class RcsvTest < Test::Unit::TestCase
   def setup
-    @csv_data = File.read('test/test_rcsv.csv')
+    @csv_data = File.open('test/test_rcsv.csv')
   end
 
   def test_rcsv
@@ -20,7 +20,7 @@ class RcsvTest < Test::Unit::TestCase
   end
 
   def test_rcsv_col_sep
-    tsv_data = @csv_data.tr(",", "\t")
+    tsv_data = StringIO.new(@csv_data.read.tr(",", "\t"))
     raw_parsed_tsv_data = Rcsv.raw_parse(tsv_data, :col_sep => "\t")
 
     assert_equal(raw_parsed_tsv_data[0][2], 'EDADEDADEDADEDADEDADEDAD')
@@ -33,8 +33,27 @@ class RcsvTest < Test::Unit::TestCase
     assert_equal(raw_parsed_tsv_data[888][13], "Dallas\t TX")
   end
 
+  def test_buffer_size
+    raw_parsed_csv_data = Rcsv.raw_parse(@csv_data, :buffer_size => 10)
+
+    assert_equal(raw_parsed_csv_data[0][2], 'EDADEDADEDADEDADEDADEDAD')
+    assert_equal(raw_parsed_csv_data[0][13], '$$$908080')
+    assert_equal(raw_parsed_csv_data[0][14], '"')
+    assert_equal(raw_parsed_csv_data[0][15], 'true/false')
+    assert_equal(raw_parsed_csv_data[0][16], nil)
+    assert_equal(raw_parsed_csv_data[9][2], nil)
+    assert_equal(raw_parsed_csv_data[3][6], '""C81E-=; **ECCB; .. 89')
+    assert_equal(raw_parsed_csv_data[888][13], 'Dallas, TX')
+  end
+
+  def test_single_item_csv
+    raw_parsed_csv_data = Rcsv.raw_parse(StringIO.new("Foo"))
+
+    assert_equal(raw_parsed_csv_data, [["Foo"]])
+  end
+
   def test_broken_data
-    broken_data = @csv_data.sub(/"/, '')
+    broken_data = StringIO.new(@csv_data.read.sub(/"/, ''))
 
     assert_raise(Rcsv::ParseError) do
       Rcsv.raw_parse(broken_data)
@@ -42,7 +61,7 @@ class RcsvTest < Test::Unit::TestCase
   end
 
   def test_broken_data_without_strict
-    broken_data = @csv_data.sub(/"/, '')
+    broken_data = StringIO.new(@csv_data.read.sub(/"/, ''))
 
     raw_parsed_csv_data = Rcsv.raw_parse(broken_data, :nostrict => true)
     assert_equal(["DSAdsfksjh", "iii ooo iii", "EDADEDADEDADEDADEDADEDAD", "111 333 555", "NMLKTF", "---==---", "//", "###", "0000000000", "Asdad bvd qwert", ";'''sd", "@@@", "OCTZ", "$$$908080", "\",true/false\nC85A5B9F,85259637,,96,6838,1983-06-14,\"\"\"C4CA-=; **1679; .. 79", "210,11", "908e", "1281-03-09", "7257.4654049904275", "20efe749-50fe-4b6a-a603-7f9cd1dc6c6d", "3", "New York, NY", "u", "2.228169203286535", "t"], raw_parsed_csv_data.first)
@@ -83,7 +102,7 @@ class RcsvTest < Test::Unit::TestCase
   end
 
   def test_row_conversions
-    raw_parsed_csv_data = Rcsv.raw_parse(@csv_data.each_line.to_a[1..-1].join, # skipping string headers
+    raw_parsed_csv_data = Rcsv.raw_parse(StringIO.new(@csv_data.each_line.to_a[1..-1].join), # skipping string headers
                                          :row_conversions => 'sisiisssssfsissf')
 
     assert_equal(raw_parsed_csv_data[0][2], nil)
@@ -94,7 +113,7 @@ class RcsvTest < Test::Unit::TestCase
   end
 
   def test_row_conversions_with_column_exclusions
-    raw_parsed_csv_data = Rcsv.raw_parse(@csv_data.each_line.to_a[1..-1].join, # skipping string headers
+    raw_parsed_csv_data = Rcsv.raw_parse(StringIO.new(@csv_data.each_line.to_a[1..-1].join), # skipping string headers
                                          :row_conversions => 's f issss fsis fb')
 
     assert_equal(raw_parsed_csv_data[0][1], nil)
@@ -153,4 +172,96 @@ class RcsvTest < Test::Unit::TestCase
       'booleator' => 't'
     }, raw_parsed_csv_data[1])
   end
+
+  def test_array_block_streaming
+    raw_parsed_csv_data = []
+
+    result = Rcsv.raw_parse(@csv_data) { |row|
+      raw_parsed_csv_data << row
+    }
+
+    assert_equal(nil, result)
+    assert_equal(raw_parsed_csv_data[0][2], 'EDADEDADEDADEDADEDADEDAD')
+    assert_equal(raw_parsed_csv_data[0][13], '$$$908080')
+    assert_equal(raw_parsed_csv_data[0][14], '"')
+    assert_equal(raw_parsed_csv_data[0][15], 'true/false')
+    assert_equal(raw_parsed_csv_data[0][16], nil)
+    assert_equal(raw_parsed_csv_data[9][2], nil)
+    assert_equal(raw_parsed_csv_data[3][6], '""C81E-=; **ECCB; .. 89')
+    assert_equal(raw_parsed_csv_data[888][13], 'Dallas, TX')
+  end
+
+  def test_hash_block_streaming
+    raw_parsed_csv_data = []
+    result = Rcsv.raw_parse(@csv_data, :row_as_hash => true, :column_names => [
+      'DSAdsfksjh',
+      'iii ooo iii',
+      'EDADEDADEDADEDADEDADEDAD',
+      '111 333 555',
+      'NMLKTF',
+      '---==---',
+      '//',
+      '###',
+      '0000000000',
+      'Asdad bvd qwert',
+      ";'''sd",
+      '@@@',
+      'OCTZ',
+      '$$$908080',
+      '"',
+      'noname',
+      'booleator'
+    ]) { |row|
+      raw_parsed_csv_data << row
+    }
+
+    assert_equal(nil, result)
+    assert_equal({
+      'DSAdsfksjh' => 'C85A5B9F',
+      'iii ooo iii' => '85259637',
+      'EDADEDADEDADEDADEDADEDAD' => nil,
+      '111 333 555' => '96',
+      'NMLKTF' => '6838',
+      '---==---' => '1983-06-14',
+      '//' => '""C4CA-=; **1679; .. 79',
+      '###' => '210,11',
+      '0000000000' => '908e',
+      'Asdad bvd qwert' => '1281-03-09',
+      ";'''sd" => '7257.4654049904275',
+      '@@@' => '20efe749-50fe-4b6a-a603-7f9cd1dc6c6d',
+      'OCTZ' => '3',
+      '$$$908080' => "New York, NY",
+      '"' => 'u',
+      'noname' => '2.228169203286535',
+      'booleator' => 't'
+    }, raw_parsed_csv_data[1])
+  end
+
+  def test_nils_and_empty_strings_default
+    raw_parsed_csv_data = Rcsv.raw_parse(StringIO.new(",\"\",,   ,,\n,,  \"\", \"\" ,,"))
+
+    assert_equal([nil, '', nil, nil, nil, nil], raw_parsed_csv_data[0])
+    assert_equal([nil, nil, '', '', nil, nil], raw_parsed_csv_data[1])
+  end
+
+  def test_nils_and_empty_strings_nil
+    raw_parsed_csv_data = Rcsv.raw_parse(StringIO.new(",\"\",,   ,,\n,,  \"\", \"\" ,,"), :parse_empty_fields_as => :nil)
+
+    assert_equal([nil, nil, nil, nil, nil, nil], raw_parsed_csv_data[0])
+    assert_equal([nil, nil, nil, nil, nil, nil], raw_parsed_csv_data[1])
+  end
+
+  def test_nils_and_empty_strings_string
+    raw_parsed_csv_data = Rcsv.raw_parse(StringIO.new(",\"\",,   ,,\n,,  \"\", \"\" ,,"), :parse_empty_fields_as => :string)
+
+    assert_equal(['', '', '', '', '', ''], raw_parsed_csv_data[0])
+    assert_equal(['', '', '', '', '', ''], raw_parsed_csv_data[1])
+  end
+
+  def test_nils_and_empty_strings_nil_or_string
+    raw_parsed_csv_data = Rcsv.raw_parse(StringIO.new(",\"\",,   ,,\n,,  \"\", \"\" ,,"), :parse_empty_fields_as => :nil_or_string)
+
+    assert_equal([nil, '', nil, nil, nil, nil], raw_parsed_csv_data[0])
+    assert_equal([nil, nil, '', '', nil, nil], raw_parsed_csv_data[1])
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rcsv
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.8
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-11-16 00:00:00.000000000 Z
+date: 2013-01-11 00:00:00.000000000 Z
 dependencies: []
 description: A libcsv-based CSV parser for Ruby
 email:
@@ -26,6 +26,7 @@ files:
 - Gemfile.lock
 - LICENSE
 - README.md
+- RELNOTES
 - Rakefile
 - bench.rb
 - ext/rcsv/csv.h