smarter_csv 1.14.2 → 1.14.3

data/ext/smarter_csv/smarter_csv.c.works8

@@ -0,0 +1,193 @@
+#include "ruby.h"
+#include "ruby/encoding.h"
+#include <stdio.h>
+#include <stdbool.h>
+#include <string.h>
+
+#ifndef bool
+  #define bool int
+  #define false ((bool)0)
+  #define true  ((bool)1)
+#endif
+
+VALUE SmarterCSV = Qnil;
+VALUE eMalformedCSVError = Qnil;
+VALUE Parser = Qnil;
+VALUE Qempty_string = Qnil; // shared frozen empty string
+
+static VALUE rb_parse_csv_line(VALUE self, VALUE line, VALUE col_sep, VALUE quote_char, VALUE max_size, VALUE has_quotes_val) {
+  if (RB_TYPE_P(line, T_NIL) == 1) {
+    return rb_ary_new();
+  }
+
+  if (RB_TYPE_P(line, T_STRING) != 1) {
+    rb_raise(rb_eTypeError, "ERROR in SmarterCSV.parse_line: line has to be a string or nil");
+  }
+
+  rb_encoding *encoding = rb_enc_get(line); /* get the encoding from the input line */
+  char *startP = RSTRING_PTR(line); /* may not be null terminated */
+  long line_len = RSTRING_LEN(line);
+  char *endP = startP + line_len; /* points behind the string */
+  char *p = startP;
+
+  char *col_sepP = RSTRING_PTR(col_sep);
+  long col_sep_len = RSTRING_LEN(col_sep);
+
+  char *quoteP = RSTRING_PTR(quote_char);
+  char quote_char_val = quoteP[0];
+  size_t quote_len = strlen(quoteP);
+
+  bool col_sep_found = true;
+
+  VALUE elements = rb_ary_new();
+  VALUE field;
+  long i;
+
+  /* Variables for escaped quote handling */
+  long backslash_count = 0;
+  bool in_quotes = false;
+
+  /* Optimization 1: maintain count instead of calling RARRAY_LEN repeatedly */
+  long element_count = 0;
+
+  /* Optimization 2: cache max_size value if not nil */
+  int max_fields = -1;
+  if (max_size != Qnil) {
+    max_fields = NUM2INT(max_size);
+    if (max_fields < 0) {
+      return rb_ary_new();  // Return empty array early
+    }
+  }
+
+  bool has_quotes = RTEST(has_quotes_val);
+
+  /* Fast-path for clean CSV lines: no quotes, 1-character separator */
+  if (__builtin_expect(!has_quotes && col_sep_len == 1, 1)) {
+    char sep = *col_sepP;
+    char *sep_pos = NULL;
+
+    while ((sep_pos = memchr(p, sep, endP - p))) {
+      if ((max_fields >= 0) && (element_count >= max_fields)) {
+        break;
+      }
+
+      /* check if only spaces */
+      bool only_spaces = true;
+      for (char *s = startP; s < sep_pos; s++) {
+        if (*s != ' ') {
+          only_spaces = false;
+          break;
+        }
+      }
+
+      field = only_spaces ? Qempty_string : rb_enc_str_new(startP, sep_pos - startP, encoding);
+      rb_ary_push(elements, field);
+      element_count++;
+
+      p = sep_pos + 1;
+      startP = p;
+    }
+
+    /* check if the last part of the line needs to be processed */
+    if ((max_fields < 0) || (element_count < max_fields)) {
+      bool only_spaces = true;
+      for (char *s = startP; s < endP; s++) {
+        if (*s != ' ') {
+          only_spaces = false;
+          break;
+        }
+      }
+      field = only_spaces ? Qempty_string : rb_enc_str_new(startP, endP - startP, encoding);
+      rb_ary_push(elements, field);
+    }
+
+    return elements;
+  }
+
+  /* default path for quoted/multi-separator lines */
+  while (p < endP) {
+    /* does the remaining string start with col_sep ? */
+    col_sep_found = true;
+    for(i=0; (i < col_sep_len) && (p+i < endP); i++) {
+      col_sep_found = col_sep_found && (*(p+i) == *(col_sepP+i));
+    }
+    /* if col_sep was found and we're not inside quotes */
+    if (col_sep_found && !in_quotes) {
+      if ((max_fields >= 0) && (element_count >= max_fields)) {
+        break;
+      } else {
+        long field_len = p - startP;
+        char *raw_field = startP;
+
+        /* quote cleanup inline */
+        if (field_len >= 2 && raw_field[0] == quote_char_val && raw_field[field_len - 1] == quote_char_val) {
+          raw_field++;
+          field_len -= 2;
+        }
+
+        field = rb_enc_str_new(raw_field, field_len, encoding);
+        if (strstr(RSTRING_PTR(field), quoteP)) {
+          VALUE doubled = rb_str_new_cstr(quoteP);
+          rb_str_replace(field, rb_funcall(field, rb_intern("gsub"), 2, rb_str_cat(doubled, quoteP, quote_len), rb_str_new(quoteP, quote_len)));
+        }
+
+        rb_ary_push(elements, field);
+        element_count++;
+
+        p += col_sep_len;
+        startP = p;
+        backslash_count = 0; // Reset backslash count at the start of a new field
+      }
+    } else {
+      if (*p == '\\') {
+        backslash_count++;
+      } else {
+        if (*p == quote_char_val) {
+          if (backslash_count % 2 == 0) {
+            /* Even number of backslashes means quote is not escaped */
+            in_quotes = !in_quotes;
+          }
+          /* Else, quote is escaped; do nothing */
+        }
+        backslash_count = 0; // Reset after any character other than backslash
+      }
+      p++;
+    }
+  } /* while */
+
+  /* Check for unclosed quotes at the end of the line */
+  if (in_quotes) {
+    rb_raise(eMalformedCSVError, "Unclosed quoted field detected in line: %s", StringValueCStr(line));
+  }
+
+  /* check if the last part of the line needs to be processed */
+  if ((max_fields < 0) || (element_count < max_fields)) {
+    long field_len = endP - startP;
+    char *raw_field = startP;
+
+    if (field_len >= 2 && raw_field[0] == quote_char_val && raw_field[field_len - 1] == quote_char_val) {
+      raw_field++;
+      field_len -= 2;
+    }
+
+    field = rb_enc_str_new(raw_field, field_len, encoding);
+    if (strstr(RSTRING_PTR(field), quoteP)) {
+      VALUE doubled = rb_str_new_cstr(quoteP);
+      rb_str_replace(field, rb_funcall(field, rb_intern("gsub"), 2, rb_str_cat(doubled, quoteP, quote_len), rb_str_new(quoteP, quote_len)));
+    }
+
+    rb_ary_push(elements, field);
+  }
+
+  return elements;
+}
+
+void Init_smarter_csv(void) {
+  // these modules and the error class are already defined in Ruby code, make them accessible:
+  SmarterCSV = rb_const_get(rb_cObject, rb_intern("SmarterCSV"));
+  Parser = rb_const_get(SmarterCSV, rb_intern("Parser"));
+  eMalformedCSVError = rb_const_get(SmarterCSV, rb_intern("MalformedCSV"));
+  Qempty_string = rb_str_new_literal("");
+  rb_gc_register_address(&Qempty_string);
+  rb_define_module_function(Parser, "parse_csv_line_c", rb_parse_csv_line, 5);
+}

data/ext/smarter_csv/smarter_csv.c.works9

@@ -0,0 +1,196 @@
+#include "ruby.h"
+#include "ruby/encoding.h"
+#include <stdio.h>
+#include <stdbool.h>
+#include <string.h>
+
+#ifndef bool
+  #define bool int
+  #define false ((bool)0)
+  #define true  ((bool)1)
+#endif
+
+VALUE SmarterCSV = Qnil;
+VALUE eMalformedCSVError = Qnil;
+VALUE Parser = Qnil;
+VALUE Qempty_string = Qnil; // shared frozen empty string
+
+typedef struct {
+  const char *start;
+  long length;
+  bool needs_unescape;
+} ParsedField;
+
+static bool detect_and_strip_quotes(const char **start, long *len, char quote_char, bool *found_double) {
+  if (*len < 2 || (*start)[0] != quote_char || (*start)[*len - 1] != quote_char) return false;
+
+  *start = *start + 1;
+  *len = *len - 2;
+
+  for (long i = 0; i < *len - 1; i++) {
+    if ((*start)[i] == quote_char && (*start)[i + 1] == quote_char) {
+      *found_double = true;
+      return true;
+    }
+  }
+  *found_double = false;
+  return true;
+}
+
+static VALUE unescape_quotes(const char *start, long len, char quote_char, rb_encoding *enc) {
+  char *buf = ALLOC_N(char, len);
+  long j = 0;
+
+  for (long i = 0; i < len; i++) {
+    if (start[i] == quote_char && i + 1 < len && start[i + 1] == quote_char) {
+      buf[j++] = quote_char;
+      i++;
+    } else {
+      buf[j++] = start[i];
+    }
+  }
+
+  VALUE str = rb_enc_str_new(buf, j, enc);
+  xfree(buf);
+  return str;
+}
+
+static VALUE rb_parse_csv_line(VALUE self, VALUE line, VALUE col_sep, VALUE quote_char, VALUE max_size, VALUE has_quotes_val) {
+  if (RB_TYPE_P(line, T_NIL)) return rb_ary_new();
+  if (!RB_TYPE_P(line, T_STRING)) rb_raise(rb_eTypeError, "ERROR in SmarterCSV.parse_line: line has to be a string or nil");
+
+  rb_encoding *encoding = rb_enc_get(line);
+  char *startP = RSTRING_PTR(line);
+  long line_len = RSTRING_LEN(line);
+  char *endP = startP + line_len;
+  char *p = startP;
+
+  char *col_sepP = RSTRING_PTR(col_sep);
+  long col_sep_len = RSTRING_LEN(col_sep);
+
+  char *quoteP = RSTRING_PTR(quote_char);
+  char quote_char_val = quoteP[0];
+  size_t quote_len = strlen(quoteP);
+
+  VALUE elements = rb_ary_new();
+
+  long backslash_count = 0;
+  bool in_quotes = false;
+  long element_count = 0;
+
+  int max_fields = -1;
+  if (max_size != Qnil) {
+    max_fields = NUM2INT(max_size);
+    if (max_fields < 0) return rb_ary_new();
+  }
+
+  bool has_quotes = RTEST(has_quotes_val);
+
+  if (__builtin_expect(!has_quotes && col_sep_len == 1, 1)) {
+    char sep = *col_sepP;
+    char *sep_pos = NULL;
+
+    while ((sep_pos = memchr(p, sep, endP - p))) {
+      if ((max_fields >= 0) && (element_count >= max_fields)) break;
+
+      bool only_spaces = true;
+      for (char *s = startP; s < sep_pos; s++) {
+        if (*s != ' ') { only_spaces = false; break; }
+      }
+
+      VALUE field = only_spaces ? Qempty_string : rb_enc_str_new(startP, sep_pos - startP, encoding);
+      rb_ary_push(elements, field);
+      element_count++;
+
+      p = sep_pos + 1;
+      startP = p;
+    }
+
+    if ((max_fields < 0) || (element_count < max_fields)) {
+      bool only_spaces = true;
+      for (char *s = startP; s < endP; s++) {
+        if (*s != ' ') { only_spaces = false; break; }
+      }
+      VALUE field = only_spaces ? Qempty_string : rb_enc_str_new(startP, endP - startP, encoding);
+      rb_ary_push(elements, field);
+    }
+
+    return elements;
+  }
+
+  ParsedField *fields = ALLOC_N(ParsedField, 128);
+  long field_count = 0;
+  char *field_start = startP;
+
+  while (p < endP) {
+    bool col_sep_found = (p + col_sep_len <= endP) && (memcmp(p, col_sepP, col_sep_len) == 0);
+
+    if (col_sep_found && !in_quotes) {
+      if ((max_fields >= 0) && (field_count >= max_fields)) break;
+
+      long field_len = p - field_start;
+      const char *actual_start = field_start;
+      long actual_len = field_len;
+      bool needs_unescape = false;
+      bool quoted = detect_and_strip_quotes(&actual_start, &actual_len, quote_char_val, &needs_unescape);
+      if (!quoted) {
+        for (long i = 0; i < actual_len - 1; i++) {
+          if (actual_start[i] == quote_char_val && actual_start[i + 1] == quote_char_val) {
+            needs_unescape = true;
+            break;
+          }
+        }
+      }
+
+      fields[field_count++] = (ParsedField){ actual_start, actual_len, needs_unescape };
+      p += col_sep_len;
+      field_start = p;
+      backslash_count = 0;
+    } else {
+      if (*p == '\\') {
+        backslash_count++;
+      } else {
+        if (*p == quote_char_val && (backslash_count % 2 == 0)) {
+          in_quotes = !in_quotes;
+        }
+        backslash_count = 0;
+      }
+      p++;
+    }
+  }
+
+  if (in_quotes) {
+    xfree(fields);
+    rb_raise(eMalformedCSVError, "Unclosed quoted field detected in line: %s", StringValueCStr(line));
+  }
+
+  if ((max_fields < 0) || (field_count < max_fields)) {
+    long field_len = endP - field_start;
+    const char *actual_start = field_start;
+    long actual_len = field_len;
+    bool needs_unescape = false;
+    detect_and_strip_quotes(&actual_start, &actual_len, quote_char_val, &needs_unescape);
+
+    fields[field_count++] = (ParsedField){ actual_start, actual_len, needs_unescape };
+  }
+
+  for (long i = 0; i < field_count; i++) {
+    ParsedField f = fields[i];
+    VALUE field = f.length == 0 ? Qempty_string :
+                  (f.needs_unescape ? unescape_quotes(f.start, f.length, quote_char_val, encoding)
+                                    : rb_enc_str_new(f.start, f.length, encoding));
+    rb_ary_push(elements, field);
+  }
+
+  xfree(fields);
+  return elements;
+}
+
+void Init_smarter_csv(void) {
+  SmarterCSV = rb_const_get(rb_cObject, rb_intern("SmarterCSV"));
+  Parser = rb_const_get(SmarterCSV, rb_intern("Parser"));
+  eMalformedCSVError = rb_const_get(SmarterCSV, rb_intern("MalformedCSV"));
+  Qempty_string = rb_str_new_literal("");
+  rb_gc_register_address(&Qempty_string);
+  rb_define_module_function(Parser, "parse_csv_line_c", rb_parse_csv_line, 5);
+}

data/lib/smarter_csv/parser.rb

@@ -2,6 +2,8 @@
 
 module SmarterCSV
   module Parser
+    EMPTY_STRING = ''.freeze
+
     protected
 
     ###
@@ -11,17 +13,16 @@ module SmarterCSV
     ###
     def parse(line, options, header_size = nil)
       # puts "SmarterCSV.parse OPTIONS: #{options[:acceleration]}" if options[:verbose]
+      has_quotes = line.include?(options[:quote_char])
 
       if options[:acceleration] && has_acceleration
         # :nocov:
-        has_quotes = line =~ /#{options[:quote_char]}/
-        elements = parse_csv_line_c(line, options[:col_sep], options[:quote_char], header_size)
-        elements.map!{|x| cleanup_quotes(x, options[:quote_char])} if has_quotes
+        elements = parse_csv_line_c(line, options[:col_sep], options[:quote_char], header_size, has_quotes, options[:strip_whitespace])
         [elements, elements.size]
         # :nocov:
       else
         # puts "WARNING: SmarterCSV is using un-accelerated parsing of lines. Check options[:acceleration]"
-        parse_csv_line_ruby(line, options, header_size)
+        parse_csv_line_ruby(line, options, header_size, has_quotes)
       end
     end
 
@@ -46,7 +47,7 @@ module SmarterCSV
     #
     # Our convention is that empty fields are returned as empty strings, not as nil.
 
-    def parse_csv_line_ruby(line, options, header_size = nil)
+    def parse_csv_line_ruby(line, options, header_size = nil, has_quotes = false)
       return [[], 0] if line.nil?
 
       line_size = line.size
@@ -98,11 +99,13 @@ module SmarterCSV
         elements << cleanup_quotes(line[start..-1], quote)
       end
 
+      elements.map!(&:strip) if options[:strip_whitespace]
       [elements, elements.size]
     end
 
     def cleanup_quotes(field, quote)
-      return field if field.nil?
+      return nil if field.nil?
+      return EMPTY_STRING if field.empty?
 
       # Remove surrounding quotes if present
       if field.start_with?(quote) && field.end_with?(quote)
@@ -110,9 +113,13 @@ module SmarterCSV
       end
 
       # Replace double quotes with a single quote
-      field.gsub!((quote * 2).to_s, quote)
+      field.gsub!(doubled_quote(quote), quote)
 
       field
     end
+
+    def doubled_quote(quote)
+      @doubled_quote ||= (quote * 2).to_s.freeze
+    end
   end
 end

data/lib/smarter_csv/reader.rb

@@ -128,6 +128,7 @@ module SmarterCSV
           line.chomp!(options[:row_sep])
 
           # --- SPLIT LINE & DATA TRANSFORMATIONS ------------------------------------------------------------
+          # we are now stripping whitespace inside the parse() methods
           dataA, data_size = parse(line, options) # we parse the extra columns
 
           if options[:strict]
@@ -141,8 +142,6 @@ module SmarterCSV
             end
           end
 
-          dataA.map!{|x| x.strip} if options[:strip_whitespace]
-
           # if all values are blank, then ignore this line
           next if options[:remove_empty_hashes] && (dataA.empty? || blank?(dataA))
 

data/lib/smarter_csv/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SmarterCSV
-  VERSION = "1.14.2"
+  VERSION = "1.14.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: smarter_csv
 version: !ruby/object:Gem::Version
-  version: 1.14.2
+  version: 1.14.3
 platform: ruby
 authors:
 - Tilo Sloboda
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-04-10 00:00:00.000000000 Z
+date: 2025-05-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: awesome_print
@@ -127,8 +127,20 @@ files:
 - docs/options.md
 - docs/row_col_sep.md
 - docs/value_converters.md
+- ext/smarter_csv/Makefile
 - ext/smarter_csv/extconf.rb
 - ext/smarter_csv/smarter_csv.c
+- ext/smarter_csv/smarter_csv.c.works
+- ext/smarter_csv/smarter_csv.c.works10
+- ext/smarter_csv/smarter_csv.c.works11
+- ext/smarter_csv/smarter_csv.c.works14
+- ext/smarter_csv/smarter_csv.c.works15
+- ext/smarter_csv/smarter_csv.c.works2
+- ext/smarter_csv/smarter_csv.c.works4
+- ext/smarter_csv/smarter_csv.c.works5
+- ext/smarter_csv/smarter_csv.c.works7
+- ext/smarter_csv/smarter_csv.c.works8
+- ext/smarter_csv/smarter_csv.c.works9
 - lib/smarter_csv.rb
 - lib/smarter_csv/auto_detection.rb
 - lib/smarter_csv/errors.rb