rcsv 0.0.5

data/ext/rcsv/rcsv.c

@@ -0,0 +1,365 @@
+#include <stdbool.h>
+#include <ruby.h>
+
+#include "csv.h"
+
+static VALUE rcsv_parse_error; // class Rcsv::ParseError << StandardError; end
+
+// It is useful to know exact row/column positions and field contents where parse-time exception was raised
+#define RAISE_WITH_LOCATION(row, column, contents, fmt, ...) \
+  rb_raise(rcsv_parse_error, "[%d:%d '%s'] " fmt, (int)(row), (int)(column), (char *)(contents), ##__VA_ARGS__);
+
+struct rcsv_metadata {
+  // Derived from user-specified options
+  bool row_as_hash;           // Used to return array of hashes rather than array of arrays
+  size_t offset_rows;         // Number of rows to skip before parsing
+
+  char * row_conversions;     // A pointer to string/array of row conversions char specifiers
+  char ** only_rows;          // A pointer to array of strings for only_rows filter
+  VALUE * row_defaults;       // A pointer to array of row defaults
+  VALUE * column_names;       // A pointer to array of column names to be used with hashes
+
+  // Pointer options lengths
+  size_t num_row_conversions; // Number of converter types in row_conversions array
+  size_t num_only_rows;       // Number of items in only_rows filter
+  size_t num_row_defaults;    // Number of default values in row_defaults array
+  size_t num_columns;         // Number of columns detected from column_names.size
+
+  // Internal state
+  bool skip_current_row;      // Used by only_rows filter to skip parsing of the row remainder
+  size_t current_col;         // Current column's index
+  size_t current_row;         // Current row's index
+
+  VALUE * result;             // A pointer to the parsed data
+};
+
+//// Internal callbacks ////
+
+/* This procedure is called for every parsed field */
+void end_of_field_callback(void * field, size_t field_size, void * data) {
+  const char * field_str = (char *)field;
+  struct rcsv_metadata * meta = (struct rcsv_metadata *) data;
+  char row_conversion = 0;
+  VALUE parsed_field;
+  VALUE last_entry = rb_ary_entry(*(meta->result), -1); // result.last
+
+  // No need to parse anything until the end of the line if skip_current_row is set
+  if (meta->skip_current_row) {
+    return;
+  }
+
+  // Skip the row if its position is less than specifed offset
+  if (meta->current_row < meta->offset_rows) {
+    meta->skip_current_row = true;
+    return;
+  }
+
+  // Filter by string row values listed in meta->only_rows.
+  if ((meta->only_rows != NULL) &&
+      (meta->current_col < meta->num_only_rows) &&
+      (meta->only_rows[meta->current_col] != NULL) &&
+      (strcmp(meta->only_rows[meta->current_col], field_str))) {
+    meta->skip_current_row = true;
+    return;
+  }
+
+  // Get row conversion char specifier
+  if (meta->current_col < meta->num_row_conversions) {
+    row_conversion = (char)meta->row_conversions[meta->current_col];
+  }
+
+  // Convert the field from string into Ruby type specified by row_conversion
+  if (row_conversion != ' ') { // spacebar skips the column
+    if (field_size == 0) {
+      // Assigning appropriate default value if applicable.
+      if (meta->current_col < meta->num_row_defaults) {
+        parsed_field = meta->row_defaults[meta->current_col];
+      } else { // By default, default is nil
+        parsed_field = Qnil;
+      }
+    } else {
+      if (meta->current_col < meta->num_row_conversions) {
+        switch (row_conversion){
+          case 's': // String
+            parsed_field = rb_str_new(field_str, field_size); 
+            break;
+          case 'i': // Integer
+            parsed_field = INT2NUM(atol(field_str));
+            break;
+          case 'f': // Float
+            parsed_field = rb_float_new(atof(field_str));
+            break;
+          case 'b': // TrueClass/FalseClass
+            switch (field_str[0]) {
+              case 't':
+              case 'T':
+              case '1':
+                parsed_field = Qtrue;
+                break;
+              case 'f':
+              case 'F':
+              case '0':
+                parsed_field = Qfalse;
+                break;
+              default:
+                RAISE_WITH_LOCATION(
+                  meta->current_row,
+                  meta->current_col,
+                  field_str,
+                  "Bad Boolean value. Valid values are strings where the first character is T/t/1 for true or F/f/0 for false."
+                );
+            }
+            break;
+          default:
+            RAISE_WITH_LOCATION(
+              meta->current_row,
+              meta->current_col,
+              field_str,
+              "Unknown deserializer '%c'.",
+              row_conversion
+            );
+        }
+      } else { // No conversion happens
+        parsed_field = rb_str_new(field_str, field_size); // field
+      }
+    }
+
+    // Assign the value to appropriate hash key if parsing into Hash
+    if (meta->row_as_hash) {
+      if (meta->current_col >= meta->num_columns) {
+        RAISE_WITH_LOCATION(
+          meta->current_row,
+          meta->current_col,
+          field_str,
+          "There are at least %d columns in a row, which is beyond the number of provided column names (%d).",
+          (int)meta->current_col + 1,
+          (int)meta->num_columns
+        );
+      } else {
+        rb_hash_aset(last_entry, meta->column_names[meta->current_col], parsed_field);
+      }
+    } else { // Parse into Array
+      rb_ary_push(last_entry, parsed_field); // result << field
+    }
+  }
+
+  // Increment column counter
+  meta->current_col++;
+  return;
+}
+
+/* This procedure is called for every line ending */
+void end_of_line_callback(int last_char, void * data) {
+  struct rcsv_metadata * meta = (struct rcsv_metadata *) data;
+
+  // If filters didn't match, current row parsing is reverted.
+  if (meta->skip_current_row) {
+    rb_ary_pop(*(meta->result)); // result.pop
+    meta->skip_current_row = false;
+  }
+
+  // Add a new empty array/hash for the next line unless EOF reached.
+  if (last_char != -1) {
+    if (meta->row_as_hash) {
+      rb_ary_push(*(meta->result), rb_hash_new()); // result << {}
+    } else {
+      rb_ary_push(*(meta->result), rb_ary_new()); // result << []
+    }
+  }
+
+  // Resetting column counter.
+  meta->current_col = 0;
+
+  // Incrementing row counter.
+  meta->current_row++;
+  return;
+}
+
+//// C API ////
+
+/* The main method that handles parsing */
+static VALUE rb_rcsv_raw_parse(int argc, VALUE * argv, VALUE self) {
+  struct rcsv_metadata meta;
+  VALUE str, options, option;
+
+  struct csv_parser cp;
+  unsigned char csv_options = CSV_STRICT_FINI | CSV_APPEND_NULL;
+  char * csv_string;
+  size_t csv_string_len;
+  int error;
+  size_t i = 0;
+
+  // Setting up some sane defaults
+  meta.row_as_hash = false;
+  meta.skip_current_row = false;
+  meta.num_columns = 0;
+  meta.current_col = 0;
+  meta.current_row = 0;
+  meta.offset_rows = 0;
+  meta.num_only_rows = 0;
+  meta.num_row_defaults = 0;
+  meta.num_row_conversions = 0;
+  meta.only_rows = NULL;
+  meta.row_defaults = NULL;
+  meta.row_conversions = NULL;
+  meta.column_names = NULL;
+  meta.result = (VALUE[]){rb_ary_new()}; // []
+
+  // str is required, options is optional (pun intended)
+  rb_scan_args(argc, argv, "11", &str, &options);
+  csv_string = StringValuePtr(str);
+  csv_string_len = strlen(csv_string);
+
+  // options ||= nil
+  if (NIL_P(options)) {
+    options = rb_hash_new();
+  }
+
+  // By default, parsing is strict
+  option = rb_hash_aref(options, ID2SYM(rb_intern("nostrict")));
+  if (!option || (option == Qnil)) {
+    csv_options |= CSV_STRICT;
+  }
+
+  // Try to initialize libcsv
+  if (csv_init(&cp, csv_options) == -1) {
+    rb_raise(rcsv_parse_error, "Couldn't initialize libcsv");
+  }
+
+  // By default, parse as Array of Arrays
+  option = rb_hash_aref(options, ID2SYM(rb_intern("row_as_hash")));
+  if (option && (option != Qnil)) {
+    meta.row_as_hash = true;
+  }
+
+  // :col_sep sets the column separator, default is comma (,)
+  option = rb_hash_aref(options, ID2SYM(rb_intern("col_sep")));
+  if (option != Qnil) {
+    csv_set_delim(&cp, (unsigned char)*StringValuePtr(option));
+  }
+
+  // Specify how many rows to skip from the beginning of CSV
+  option = rb_hash_aref(options, ID2SYM(rb_intern("offset_rows")));
+  if (option != Qnil) {
+    meta.offset_rows = (size_t)NUM2INT(option);
+  }
+
+  // :only_rows is a string mask where row is only parsed
+  // if its fields match those in the passed array.
+  // [nil, nil, "ABC"] skips all rows where 3rd column isn't equal to "ABC"
+  option = rb_hash_aref(options, ID2SYM(rb_intern("only_rows")));
+  if (option != Qnil) {
+    meta.num_only_rows = (size_t)RARRAY_LEN(option);
+    meta.only_rows = (char **)malloc(meta.num_only_rows * sizeof(char *));
+
+    for (i = 0; i < meta.num_only_rows; i++) {
+      VALUE only_row = rb_ary_entry(option, i);
+      if (only_row == Qnil) {
+        meta.only_rows[i] = NULL;
+      } else {
+        meta.only_rows[i] = StringValueCStr(only_row);
+      }
+    }
+  }
+
+  // :row_defaults is an array of default values that are assigned to fields containing empty strings
+  // according to matching field positions
+  option = rb_hash_aref(options, ID2SYM(rb_intern("row_defaults")));
+  if (option != Qnil) {
+    meta.num_row_defaults = RARRAY_LEN(option);
+    meta.row_defaults = (VALUE*)malloc(meta.num_row_defaults * sizeof(VALUE*));
+
+    for (i = 0; i < meta.num_row_defaults; i++) {
+      VALUE row_default = rb_ary_entry(option, i);
+      meta.row_defaults[i] = row_default;
+    }
+  }
+
+  // :row_conversions specifies Ruby types that CSV field values should be converted into.
+  // Each char of row_conversions string represents Ruby type for CSV field with matching position.
+  option = rb_hash_aref(options, ID2SYM(rb_intern("row_conversions"))); 
+  if (option != Qnil) {
+    meta.num_row_conversions = RSTRING_LEN(option);
+    meta.row_conversions = StringValuePtr(option);
+  }
+
+  // Column names should be declared explicitly when parsing fields as Hashes
+  if (meta.row_as_hash) { // Only matters for hash results
+    option = rb_hash_aref(options, ID2SYM(rb_intern("column_names"))); 
+    if (option == Qnil) {
+      rb_raise(rcsv_parse_error, ":row_as_hash requires :column_names to be set.");
+    } else {
+      meta.num_columns = (size_t)RARRAY_LEN(option);
+      meta.column_names = (VALUE*)malloc(meta.num_columns * sizeof(VALUE*));
+
+      for (i = 0; i < meta.num_columns; i++) {
+        meta.column_names[i] = rb_ary_entry(option, i);
+      }
+    }
+  }
+
+  // Initializing result with empty Array
+  if (meta.row_as_hash) {
+    rb_ary_push(*(meta.result), rb_hash_new()); // [{}]
+  } else {
+    rb_ary_push(*(meta.result), rb_ary_new()); // [[]]
+  }
+
+  // Actual parsing and error handling
+  if (csv_string_len != csv_parse(&cp, csv_string, strlen(csv_string),
+                                  &end_of_field_callback, &end_of_line_callback, &meta)) {
+    error = csv_error(&cp);
+    switch(error) {
+      case CSV_EPARSE:
+        rb_raise(rcsv_parse_error, "Error when parsing malformed data");
+        break;
+      case CSV_ENOMEM:
+        rb_raise(rcsv_parse_error, "No memory");
+        break;
+      case CSV_ETOOBIG:
+        rb_raise(rcsv_parse_error, "Field data is too large");
+        break;
+      case CSV_EINVALID:
+        rb_raise(rcsv_parse_error, "%s", (const char *)csv_strerror(error));
+      break;
+      default:
+        rb_raise(rcsv_parse_error, "Failed due to unknown reason");
+    }
+  }
+
+  // Flushing libcsv's buffer and freeing up allocated memory
+  csv_fini(&cp, &end_of_field_callback, &end_of_line_callback, &meta);
+  csv_free(&cp);
+
+  if (meta.only_rows != NULL) {
+    free(meta.only_rows);
+  }
+
+  if (meta.row_defaults != NULL) {
+    free(meta.row_defaults);
+  }
+
+  if (meta.column_names != NULL) {
+    free(meta.column_names);
+  }
+
+  // Remove the last row if it's empty. That happens if CSV file ends with a newline.
+  if (RARRAY_LEN(rb_ary_entry(*(meta.result), -1)) == 0) {
+    rb_ary_pop(*(meta.result));
+  }
+
+  // An array of arrays of strings is returned.
+  return *(meta.result);
+}
+
+
+/* Define Ruby API */
+void Init_rcsv(void) {
+  VALUE klass = rb_define_class("Rcsv", rb_cObject); // class Rcsv; end
+
+  // Error is initialized through static variable in order to access it from rb_rcsv_raw_parse
+  rcsv_parse_error = rb_define_class_under(klass, "ParseError", rb_eStandardError);
+
+  // def Rcsv.raw_parse; ...; end
+  rb_define_singleton_method(klass, "raw_parse", rb_rcsv_raw_parse, -1);
+}

data/ext/rcsv/test.rb

@@ -0,0 +1,5 @@
+require './rcsv'
+
+r = Rcsv.parse(['1,2,3,4,5,abc,"def"', '5,,7,8,9,"yo, kmon", foo'].join("\n"), :col_sep => ',')
+
+puts r.inspect

data/lib/lib_csv.rb

@@ -0,0 +1,88 @@
+require 'ffi'
+
+class LibCsv
+  extend FFI::Library
+  ffi_lib 'libcsv'
+
+  class CsvParser < FFI::Struct
+    layout :pstate, :int,
+           :qouted, :int,
+           :spaces, :size_t,
+           :entry_buf, :string,
+           :entry_pos, :size_t,
+           :entry_size, :size_t,
+           :status, :int,
+           :options, :uchar,
+           :quote_char, :uchar,
+           :delim_char, :uchar,
+           :is_space, :pointer,
+           :is_term, :pointer,
+           :blk_size, :size_t,
+           :malloc_func, :pointer,
+           :realloc_func, :pointer,
+           :free_func, :pointer
+  end
+
+  callback :end_of_field_callback, [:pointer, :size_t, :pointer], :void
+  callback :end_of_record_callback, [:int, :pointer], :void
+
+  attach_function :csv_init, [:pointer, :uchar], :int
+  attach_function :csv_parse, [:pointer, :pointer, :size_t, :end_of_field_callback, :end_of_record_callback, :pointer], :size_t
+  attach_function :csv_fini, [:pointer, :end_of_field_callback, :end_of_record_callback, :pointer], :int
+  attach_function :csv_free, [:pointer], :void
+
+  attach_function :csv_set_delim, [:pointer, :uchar], :void
+  attach_function :csv_get_delim, [:pointer], :uchar
+
+  attach_function :csv_error, [:pointer], :int
+  attach_function :csv_strerror, [:int], :string
+
+  def self.parse(string, options = {})
+    pointer = FFI::MemoryPointer.new :char, CsvParser.size, false
+    parser = CsvParser.new pointer
+    result = csv_init(parser, 0)
+
+    if options[:col_sep]
+      csv_set_delim(parser, options[:col_sep].ord)
+    end
+
+    fail "Couldn't initialize libcsv" if result == -1
+
+    result = [[]]
+
+    end_of_field_callback = Proc.new { |p_field, field_size, p_data|
+      str = p_field.read_pointer.null? ? nil : p_field.read_string(field_size)
+      result.last << str
+    }
+
+    end_of_record_callback = Proc.new { |last_char, p_data|
+      result << [] unless last_char == -1
+    }
+
+    original_length = string.bytesize
+    length = nil
+
+    length = csv_parse(parser, string, original_length, end_of_field_callback, end_of_record_callback, nil)
+
+    unless length == original_length
+      case error = csv_error(parser)
+        when CSV_EPARSE
+          fail "Error when parsing malformed data"
+        when CSV_ENOMEM
+          fail "No memory"
+        when CSV_ETOOBIG
+          fail "Too large field data"
+        when CSV_EINVALID
+          fail csv_strerror(error)
+        else
+          fail "Failed due to unknown reason"
+      end
+    end
+
+    csv_fini(parser, end_of_field_callback, end_of_record_callback, nil)
+    csv_free(parser)
+    result.pop if result.last == []
+    
+    return result
+  end
+end