smarter_csv 1.14.2 → 1.14.3

data/ext/smarter_csv/smarter_csv.c.works

@@ -0,0 +1,185 @@
+#include "ruby.h"
+#include "ruby/encoding.h"
+#include <stdio.h>
+#include <stdbool.h>
+
+#ifndef bool
+  #define bool int
+  #define false ((bool)0)
+  #define true  ((bool)1)
+#endif
+
+VALUE SmarterCSV = Qnil;
+VALUE eMalformedCSVError = Qnil;
+VALUE Parser = Qnil;
+
+static VALUE Qempty_string;  // shared frozen empty string
+
+static VALUE rb_parse_csv_line(VALUE self, VALUE line, VALUE col_sep, VALUE quote_char, VALUE max_size) {
+  if (RB_TYPE_P(line, T_NIL) == 1) {
+    return rb_ary_new();
+  }
+
+  if (RB_TYPE_P(line, T_STRING) != 1) {
+    rb_raise(rb_eTypeError, "ERROR in SmarterCSV.parse_line: line has to be a string or nil");
+  }
+
+  rb_encoding *encoding = rb_enc_get(line); /* get the encoding from the input line */
+  char *startP = RSTRING_PTR(line); /* may not be null terminated */
+  long line_len = RSTRING_LEN(line);
+  char *endP = startP + line_len; /* points behind the string */
+  char *p = startP;
+
+  char *col_sepP = RSTRING_PTR(col_sep);
+  long col_sep_len = RSTRING_LEN(col_sep);
+
+  char *quoteP = RSTRING_PTR(quote_char);
+  char quote = *quoteP;
+  long quote_count = 0;
+
+  bool col_sep_found = true;
+
+  VALUE elements = rb_ary_new();
+  VALUE field;
+  long i;
+
+  /* Variables for escaped quote handling */
+  long backslash_count = 0;
+  bool in_quotes = false;
+
+  /* Optimization 1: maintain count instead of calling RARRAY_LEN repeatedly */
+  long element_count = 0;
+
+  /* Optimization 2: cache max_size value if not nil */
+  int max_fields = -1;
+  if (max_size != Qnil) {
+    max_fields = NUM2INT(max_size);
+    if (max_fields < 0) {
+      return rb_ary_new();  // Return empty array early
+    }
+  }
+
+  while (p < endP) {
+    /* does the remaining string start with col_sep ? */
+    col_sep_found = true;
+    for (i = 0; (i < col_sep_len) && (p + i < endP); i++) {
+      col_sep_found = col_sep_found && (*(p + i) == *(col_sepP + i));
+    }
+
+    /* if col_sep was found and we're not inside quotes */
+    if (col_sep_found && !in_quotes) {
+      if ((max_fields >= 0) && (element_count >= max_fields)) {
+        break;
+      } else {
+        bool only_spaces = true;
+        for (char *s = startP; s < p; s++) {
+          if (*s != ' ') {
+            only_spaces = false;
+            break;
+          }
+        }
+
+        if (only_spaces) {
+          field = Qempty_string;
+        } else {
+          long field_len = p - startP;
+
+          // fast-path quote cleanup: if field starts and ends with quote, and no doubled quotes inside
+          if (field_len >= 2 && startP[0] == quote && startP[field_len - 1] == quote) {
+            char *inner_start = startP + 1;
+            long inner_len = field_len - 2;
+            bool has_double_quote = false;
+            for (i = 0; i < inner_len - 1; i++) {
+              if (inner_start[i] == quote && inner_start[i + 1] == quote) {
+                has_double_quote = true;
+                break;
+              }
+            }
+            if (!has_double_quote) {
+              field = rb_enc_str_new(inner_start, inner_len, encoding);
+            } else {
+              field = rb_enc_str_new(startP, field_len, encoding);
+            }
+          } else {
+            field = rb_enc_str_new(startP, field_len, encoding);
+          }
+        }
+
+        rb_ary_push(elements, field);
+        element_count++;
+
+        p += col_sep_len;
+        startP = p;
+        backslash_count = 0; // Reset backslash count at the start of a new field
+      }
+    } else {
+      if (*p == '\\') {
+        backslash_count++;
+      } else {
+        if (*p == quote) {
+          if (backslash_count % 2 == 0) {
+            in_quotes = !in_quotes;
+          }
+        }
+        backslash_count = 0;
+      }
+      p++;
+    }
+  }
+
+  if (in_quotes) {
+    rb_raise(eMalformedCSVError, "Unclosed quoted field detected in line: %s", StringValueCStr(line));
+  }
+
+  if ((max_fields < 0) || (element_count < max_fields)) {
+    bool only_spaces = true;
+    for (char *s = startP; s < endP; s++) {
+      if (*s != ' ') {
+        only_spaces = false;
+        break;
+      }
+    }
+
+    if (only_spaces) {
+      field = Qempty_string;
+    } else {
+      long field_len = endP - startP;
+
+      // fast-path quote cleanup on final field
+      if (field_len >= 2 && startP[0] == quote && startP[field_len - 1] == quote) {
+        char *inner_start = startP + 1;
+        long inner_len = field_len - 2;
+        bool has_double_quote = false;
+        for (i = 0; i < inner_len - 1; i++) {
+          if (inner_start[i] == quote && inner_start[i + 1] == quote) {
+            has_double_quote = true;
+            break;
+          }
+        }
+        if (!has_double_quote) {
+          field = rb_enc_str_new(inner_start, inner_len, encoding);
+        } else {
+          field = rb_enc_str_new(startP, field_len, encoding);
+        }
+      } else {
+        field = rb_enc_str_new(startP, field_len, encoding);
+      }
+    }
+
+    rb_ary_push(elements, field);
+  }
+
+  return elements;
+}
+
+void Init_smarter_csv(void) {
+  SmarterCSV = rb_const_get(rb_cObject, rb_intern("SmarterCSV"));
+  Parser = rb_const_get(SmarterCSV, rb_intern("Parser"));
+  eMalformedCSVError = rb_const_get(SmarterCSV, rb_intern("MalformedCSV"));
+
+  Qempty_string = rb_str_new_literal("");
+  rb_obj_freeze(Qempty_string);
+  rb_global_variable(&Qempty_string);
+
+  rb_define_module_function(Parser, "parse_csv_line_c", rb_parse_csv_line, 4);
+}

data/ext/smarter_csv/smarter_csv.c.works10

@@ -0,0 +1,199 @@
+#include "ruby.h"
+#include "ruby/encoding.h"
+#include <stdio.h>
+#include <stdbool.h>
+#include <string.h>
+
+#ifndef bool
+  #define bool int
+  #define false ((bool)0)
+  #define true  ((bool)1)
+#endif
+
+VALUE SmarterCSV = Qnil;
+VALUE eMalformedCSVError = Qnil;
+VALUE Parser = Qnil;
+VALUE Qempty_string = Qnil; // shared frozen empty string
+
+static VALUE unescape_quotes(char *str, long len, char quote_char, rb_encoding *encoding) {
+  char *buf = ALLOC_N(char, len);
+  long j = 0;
+  for (long i = 0; i < len; i++) {
+    if (str[i] == quote_char && i + 1 < len && str[i + 1] == quote_char) {
+      buf[j++] = quote_char;
+      i++; // skip second quote
+    } else {
+      buf[j++] = str[i];
+    }
+  }
+  VALUE out = rb_enc_str_new(buf, j, encoding);
+  xfree(buf);
+  return out;
+}
+
+static VALUE rb_parse_csv_line(VALUE self, VALUE line, VALUE col_sep, VALUE quote_char, VALUE max_size, VALUE has_quotes_val) {
+  if (RB_TYPE_P(line, T_NIL) == 1) {
+    return rb_ary_new();
+  }
+
+  if (RB_TYPE_P(line, T_STRING) != 1) {
+    rb_raise(rb_eTypeError, "ERROR in SmarterCSV.parse_line: line has to be a string or nil");
+  }
+
+  rb_encoding *encoding = rb_enc_get(line);
+  char *startP = RSTRING_PTR(line);
+  long line_len = RSTRING_LEN(line);
+  char *endP = startP + line_len;
+  char *p = startP;
+
+  char *col_sepP = RSTRING_PTR(col_sep);
+  long col_sep_len = RSTRING_LEN(col_sep);
+
+  char *quoteP = RSTRING_PTR(quote_char);
+  char quote_char_val = quoteP[0];
+  size_t quote_len = strlen(quoteP);
+
+  VALUE elements = rb_ary_new();
+  VALUE field;
+
+  long element_count = 0;
+  int max_fields = -1;
+  if (max_size != Qnil) {
+    max_fields = NUM2INT(max_size);
+    if (max_fields < 0) {
+      return rb_ary_new();
+    }
+  }
+
+  bool has_quotes = RTEST(has_quotes_val);
+
+  // === FAST PATH: No quotes and single-character separator ===
+  if (__builtin_expect(!has_quotes && col_sep_len == 1, 1)) {
+    char sep = *col_sepP;
+    char *sep_pos = NULL;
+
+    while ((sep_pos = memchr(p, sep, endP - p))) {
+      if ((max_fields >= 0) && (element_count >= max_fields)) {
+        break;
+      }
+
+      long field_len = sep_pos - startP;
+      char *raw_field = startP;
+
+      if (memchr(raw_field, quote_char_val, field_len)) {
+        field = unescape_quotes(raw_field, field_len, quote_char_val, encoding);
+      } else {
+        field = rb_enc_str_new(raw_field, field_len, encoding);
+      }
+      rb_ary_push(elements, field);
+      element_count++;
+
+      p = sep_pos + 1;
+      startP = p;
+    }
+
+    if ((max_fields < 0) || (element_count < max_fields)) {
+      long field_len = endP - startP;
+      char *raw_field = startP;
+
+      if (memchr(raw_field, quote_char_val, field_len)) {
+        field = unescape_quotes(raw_field, field_len, quote_char_val, encoding);
+      } else {
+        field = rb_enc_str_new(raw_field, field_len, encoding);
+      }
+      rb_ary_push(elements, field);
+    }
+
+    return elements;
+  }
+
+  // === SLOW PATH: Quoted fields or multi-char separator ===
+  long i;
+  long backslash_count = 0;
+  bool in_quotes = false;
+  bool col_sep_found = true;
+
+  while (p < endP) {
+    col_sep_found = true;
+    for (i = 0; (i < col_sep_len) && (p + i < endP); i++) {
+      if (*(p + i) != *(col_sepP + i)) {
+        col_sep_found = false;
+        break;
+      }
+    }
+
+    if (col_sep_found && !in_quotes) {
+      if ((max_fields >= 0) && (element_count >= max_fields)) {
+        break;
+      }
+
+      long field_len = p - startP;
+      char *raw_field = startP;
+
+      bool quoted = (field_len >= 2 && raw_field[0] == quote_char_val && raw_field[field_len - 1] == quote_char_val);
+      if (quoted) {
+        raw_field++;
+        field_len -= 2;
+      }
+
+      if (quoted || memchr(raw_field, quote_char_val, field_len)) {
+        field = unescape_quotes(raw_field, field_len, quote_char_val, encoding);
+      } else {
+        field = rb_enc_str_new(raw_field, field_len, encoding);
+      }
+
+      rb_ary_push(elements, field);
+      element_count++;
+
+      p += col_sep_len;
+      startP = p;
+      backslash_count = 0;
+    } else {
+      if (*p == '\\') {
+        backslash_count++;
+      } else {
+        if (*p == quote_char_val) {
+          if (backslash_count % 2 == 0) {
+            in_quotes = !in_quotes;
+          }
+        }
+        backslash_count = 0;
+      }
+      p++;
+    }
+  }
+
+  if (in_quotes) {
+    rb_raise(eMalformedCSVError, "Unclosed quoted field detected in line: %s", StringValueCStr(line));
+  }
+
+  if ((max_fields < 0) || (element_count < max_fields)) {
+    long field_len = endP - startP;
+    char *raw_field = startP;
+
+    bool quoted = (field_len >= 2 && raw_field[0] == quote_char_val && raw_field[field_len - 1] == quote_char_val);
+    if (quoted) {
+      raw_field++;
+      field_len -= 2;
+    }
+
+    if (quoted || memchr(raw_field, quote_char_val, field_len)) {
+      field = unescape_quotes(raw_field, field_len, quote_char_val, encoding);
+    } else {
+      field = rb_enc_str_new(raw_field, field_len, encoding);
+    }
+
+    rb_ary_push(elements, field);
+  }
+
+  return elements;
+}
+
+void Init_smarter_csv(void) {
+  SmarterCSV = rb_const_get(rb_cObject, rb_intern("SmarterCSV"));
+  Parser = rb_const_get(SmarterCSV, rb_intern("Parser"));
+  eMalformedCSVError = rb_const_get(SmarterCSV, rb_intern("MalformedCSV"));
+  Qempty_string = rb_str_new_literal("");
+  rb_gc_register_address(&Qempty_string);
+  rb_define_module_function(Parser, "parse_csv_line_c", rb_parse_csv_line, 5);
+}

data/ext/smarter_csv/smarter_csv.c.works11

@@ -0,0 +1,189 @@
+#include "ruby.h"
+#include "ruby/encoding.h"
+#include <stdio.h>
+#include <stdbool.h>
+#include <string.h>
+
+#ifndef bool
+  #define bool int
+  #define false ((bool)0)
+  #define true  ((bool)1)
+#endif
+
+VALUE SmarterCSV = Qnil;
+VALUE eMalformedCSVError = Qnil;
+VALUE Parser = Qnil;
+VALUE Qempty_string = Qnil; // shared frozen empty string
+
+static VALUE unescape_quotes(char *str, long len, char quote_char, rb_encoding *encoding) {
+  char *buf = ALLOC_N(char, len);
+  long j = 0;
+  for (long i = 0; i < len; i++) {
+    if (str[i] == quote_char && i + 1 < len && str[i + 1] == quote_char) {
+      buf[j++] = quote_char;
+      i++; // skip second quote
+    } else {
+      buf[j++] = str[i];
+    }
+  }
+  VALUE out = rb_enc_str_new(buf, j, encoding);
+  xfree(buf);
+  return out;
+}
+
+static VALUE rb_parse_csv_line(VALUE self, VALUE line, VALUE col_sep, VALUE quote_char, VALUE max_size, VALUE has_quotes_val) {
+  if (RB_TYPE_P(line, T_NIL) == 1) {
+    return rb_ary_new();
+  }
+
+  if (RB_TYPE_P(line, T_STRING) != 1) {
+    rb_raise(rb_eTypeError, "ERROR in SmarterCSV.parse_line: line has to be a string or nil");
+  }
+
+  rb_encoding *encoding = rb_enc_get(line);
+  char *startP = RSTRING_PTR(line);
+  long line_len = RSTRING_LEN(line);
+  char *endP = startP + line_len;
+  char *p = startP;
+
+  char *col_sepP = RSTRING_PTR(col_sep);
+  long col_sep_len = RSTRING_LEN(col_sep);
+
+  char *quoteP = RSTRING_PTR(quote_char);
+  char quote_char_val = quoteP[0];
+  size_t quote_len = strlen(quoteP);
+
+  VALUE elements = rb_ary_new();
+  VALUE field;
+
+  long element_count = 0;
+  int max_fields = -1;
+  if (max_size != Qnil) {
+    max_fields = NUM2INT(max_size);
+    if (max_fields < 0) {
+      return rb_ary_new();
+    }
+  }
+
+  bool has_quotes = RTEST(has_quotes_val);
+
+  // === FAST PATH: No quotes and single-character separator ===
+  if (!has_quotes && col_sep_len == 1) {
+    char sep = *col_sepP;
+    char *sep_pos = NULL;
+
+    while ((sep_pos = memchr(p, sep, endP - p))) {
+      if ((max_fields >= 0) && (element_count >= max_fields)) {
+        break;
+      }
+
+      long field_len = sep_pos - startP;
+      char *raw_field = startP;
+      field = rb_enc_str_new(raw_field, field_len, encoding);
+      rb_ary_push(elements, field);
+      element_count++;
+
+      p = sep_pos + 1;
+      startP = p;
+    }
+
+    if ((max_fields < 0) || (element_count < max_fields)) {
+      long field_len = endP - startP;
+      char *raw_field = startP;
+      field = rb_enc_str_new(raw_field, field_len, encoding);
+      rb_ary_push(elements, field);
+    }
+
+    return elements;
+  }
+
+  // === SLOW PATH: Quoted fields or multi-char separator ===
+  long i;
+  long backslash_count = 0;
+  bool in_quotes = false;
+  bool col_sep_found = true;
+
+  while (p < endP) {
+    col_sep_found = true;
+    for (i = 0; (i < col_sep_len) && (p + i < endP); i++) {
+      if (*(p + i) != *(col_sepP + i)) {
+        col_sep_found = false;
+        break;
+      }
+    }
+
+    if (col_sep_found && !in_quotes) {
+      if ((max_fields >= 0) && (element_count >= max_fields)) {
+        break;
+      }
+
+      long field_len = p - startP;
+      char *raw_field = startP;
+
+      bool quoted = (field_len >= 2 && raw_field[0] == quote_char_val && raw_field[field_len - 1] == quote_char_val);
+      if (quoted) {
+        raw_field++;
+        field_len -= 2;
+      }
+
+      if (quoted || memchr(raw_field, quote_char_val, field_len)) {
+        field = unescape_quotes(raw_field, field_len, quote_char_val, encoding);
+      } else {
+        field = rb_enc_str_new(raw_field, field_len, encoding);
+      }
+
+      rb_ary_push(elements, field);
+      element_count++;
+
+      p += col_sep_len;
+      startP = p;
+      backslash_count = 0;
+    } else {
+      if (*p == '\\') {
+        backslash_count++;
+      } else {
+        if (*p == quote_char_val) {
+          if (backslash_count % 2 == 0) {
+            in_quotes = !in_quotes;
+          }
+        }
+        backslash_count = 0;
+      }
+      p++;
+    }
+  }
+
+  if (in_quotes) {
+    rb_raise(eMalformedCSVError, "Unclosed quoted field detected in line: %s", StringValueCStr(line));
+  }
+
+  if ((max_fields < 0) || (element_count < max_fields)) {
+    long field_len = endP - startP;
+    char *raw_field = startP;
+
+    bool quoted = (field_len >= 2 && raw_field[0] == quote_char_val && raw_field[field_len - 1] == quote_char_val);
+    if (quoted) {
+      raw_field++;
+      field_len -= 2;
+    }
+
+    if (quoted || memchr(raw_field, quote_char_val, field_len)) {
+      field = unescape_quotes(raw_field, field_len, quote_char_val, encoding);
+    } else {
+      field = rb_enc_str_new(raw_field, field_len, encoding);
+    }
+
+    rb_ary_push(elements, field);
+  }
+
+  return elements;
+}
+
+void Init_smarter_csv(void) {
+  SmarterCSV = rb_const_get(rb_cObject, rb_intern("SmarterCSV"));
+  Parser = rb_const_get(SmarterCSV, rb_intern("Parser"));
+  eMalformedCSVError = rb_const_get(SmarterCSV, rb_intern("MalformedCSV"));
+  Qempty_string = rb_str_new_literal("");
+  rb_gc_register_address(&Qempty_string);
+  rb_define_module_function(Parser, "parse_csv_line_c", rb_parse_csv_line, 5);
+}