smarter_csv 1.14.2 → 1.14.3

data/ext/smarter_csv/smarter_csv.c.works4

@@ -0,0 +1,190 @@
+#include "ruby.h"
+#include "ruby/encoding.h"
+#include <stdio.h>
+#include <stdbool.h>
+
+#ifndef bool
+  #define bool int
+  #define false ((bool)0)
+  #define true  ((bool)1)
+#endif
+
+VALUE SmarterCSV = Qnil;
+VALUE eMalformedCSVError = Qnil;
+VALUE Parser = Qnil;
+
+static VALUE Qempty_string;  // shared frozen empty string
+
+static VALUE rb_parse_csv_line(VALUE self, VALUE line, VALUE col_sep, VALUE quote_char, VALUE max_size, VALUE has_quotes_flag) {
+  if (RB_TYPE_P(line, T_NIL) == 1) {
+    return rb_ary_new();
+  }
+
+  if (RB_TYPE_P(line, T_STRING) != 1) {
+    rb_raise(rb_eTypeError, "ERROR in SmarterCSV.parse_line: line has to be a string or nil");
+  }
+
+  rb_encoding *encoding = rb_enc_get(line); /* get the encoding from the input line */
+  char *startP = RSTRING_PTR(line); /* may not be null terminated */
+  long line_len = RSTRING_LEN(line);
+  char *endP = startP + line_len; /* points behind the string */
+  char *p = startP;
+
+  char *col_sepP = RSTRING_PTR(col_sep);
+  long col_sep_len = RSTRING_LEN(col_sep);
+
+  char *quoteP = RSTRING_PTR(quote_char);
+  char quote = *quoteP;
+  bool has_quotes = RTEST(has_quotes_flag);
+
+  bool col_sep_found = true;
+
+  VALUE elements = rb_ary_new();
+  VALUE field;
+  long i;
+
+  /* Variables for escaped quote handling */
+  long backslash_count = 0;
+  bool in_quotes = false;
+
+  /* Optimization 1: maintain count instead of calling RARRAY_LEN repeatedly */
+  long element_count = 0;
+
+  /* Optimization 2: cache max_size value if not nil */
+  int max_fields = -1;
+  if (max_size != Qnil) {
+    max_fields = NUM2INT(max_size);
+    if (max_fields < 0) {
+      return rb_ary_new();  // Return empty array early
+    }
+  }
+
+  while (p < endP) {
+    /* does the remaining string start with col_sep ? */
+    col_sep_found = true;
+    for (i = 0; (i < col_sep_len) && (p + i < endP); i++) {
+      col_sep_found = col_sep_found && (*(p + i) == *(col_sepP + i));
+    }
+
+    /* if col_sep was found and we're not inside quotes */
+    if (col_sep_found && !in_quotes) {
+      if ((max_fields >= 0) && (element_count >= max_fields)) {
+        break;
+      } else {
+        bool only_spaces = true;
+        for (char *s = startP; s < p; s++) {
+          if (*s != ' ') {
+            only_spaces = false;
+            break;
+          }
+        }
+
+        if (only_spaces) {
+          field = Qempty_string;
+        } else {
+          char *src = startP;
+          long src_len = p - startP;
+          bool is_quoted = has_quotes && src_len >= 2 && src[0] == quote && src[src_len - 1] == quote;
+
+          if (is_quoted) {
+            src += 1;
+            src_len -= 2;
+          }
+
+          char *clean_buf = ALLOC_N(char, src_len);
+          long clean_len = 0;
+
+          for (i = 0; i < src_len; i++) {
+            if (src[i] == quote && (i + 1 < src_len) && src[i + 1] == quote) {
+              clean_buf[clean_len++] = quote;
+              i++;
+            } else {
+              clean_buf[clean_len++] = src[i];
+            }
+          }
+
+          field = rb_enc_str_new(clean_buf, clean_len, encoding);
+          xfree(clean_buf);
+        }
+
+        rb_ary_push(elements, field);
+        element_count++;
+
+        p += col_sep_len;
+        startP = p;
+        backslash_count = 0; // Reset backslash count at the start of a new field
+      }
+    } else {
+      if (*p == '\\') {
+        backslash_count++;
+      } else {
+        if (*p == quote) {
+          if (backslash_count % 2 == 0) {
+            in_quotes = !in_quotes;
+          }
+        }
+        backslash_count = 0;
+      }
+      p++;
+    }
+  }
+
+  if (in_quotes) {
+    rb_raise(eMalformedCSVError, "Unclosed quoted field detected in line: %s", StringValueCStr(line));
+  }
+
+  if ((max_fields < 0) || (element_count < max_fields)) {
+    bool only_spaces = true;
+    for (char *s = startP; s < endP; s++) {
+      if (*s != ' ') {
+        only_spaces = false;
+        break;
+      }
+    }
+
+    if (only_spaces) {
+      field = Qempty_string;
+    } else {
+      char *src = startP;
+      long src_len = endP - startP;
+      bool is_quoted = has_quotes && src_len >= 2 && src[0] == quote && src[src_len - 1] == quote;
+
+      if (is_quoted) {
+        src += 1;
+        src_len -= 2;
+      }
+
+      char *clean_buf = ALLOC_N(char, src_len);
+      long clean_len = 0;
+
+      for (i = 0; i < src_len; i++) {
+        if (src[i] == quote && (i + 1 < src_len) && src[i + 1] == quote) {
+          clean_buf[clean_len++] = quote;
+          i++;
+        } else {
+          clean_buf[clean_len++] = src[i];
+        }
+      }
+
+      field = rb_enc_str_new(clean_buf, clean_len, encoding);
+      xfree(clean_buf);
+    }
+
+    rb_ary_push(elements, field);
+  }
+
+  return elements;
+}
+
+void Init_smarter_csv(void) {
+  SmarterCSV = rb_const_get(rb_cObject, rb_intern("SmarterCSV"));
+  Parser = rb_const_get(SmarterCSV, rb_intern("Parser"));
+  eMalformedCSVError = rb_const_get(SmarterCSV, rb_intern("MalformedCSV"));
+
+  Qempty_string = rb_str_new_literal("");
+  rb_obj_freeze(Qempty_string);
+  rb_global_variable(&Qempty_string);
+
+  rb_define_module_function(Parser, "parse_csv_line_c", rb_parse_csv_line, 5);
+}
+

data/ext/smarter_csv/smarter_csv.c.works5

@@ -0,0 +1,203 @@
+#include "ruby.h"
+#include "ruby/encoding.h"
+#include <stdio.h>
+#include <stdbool.h>
+#include <string.h>
+
+#ifndef bool
+  #define bool int
+  #define false ((bool)0)
+  #define true  ((bool)1)
+#endif
+
+#define STACK_BUF_SIZE 4096
+
+VALUE SmarterCSV = Qnil;
+VALUE eMalformedCSVError = Qnil;
+VALUE Parser = Qnil;
+
+static VALUE Qempty_string;  // shared frozen empty string
+
+static VALUE rb_parse_csv_line(VALUE self, VALUE line, VALUE col_sep, VALUE quote_char, VALUE max_size, VALUE has_quotes_flag) {
+  if (RB_TYPE_P(line, T_NIL) == 1) {
+    return rb_ary_new();
+  }
+
+  if (RB_TYPE_P(line, T_STRING) != 1) {
+    rb_raise(rb_eTypeError, "ERROR in SmarterCSV.parse_line: line has to be a string or nil");
+  }
+
+  rb_encoding *encoding = rb_enc_get(line);
+  char *startP = RSTRING_PTR(line);
+  long line_len = RSTRING_LEN(line);
+  char *endP = startP + line_len;
+  char *p = startP;
+
+  char *col_sepP = RSTRING_PTR(col_sep);
+  long col_sep_len = RSTRING_LEN(col_sep);
+
+  char *quoteP = RSTRING_PTR(quote_char);
+  char quote = *quoteP;
+  bool has_quotes = RTEST(has_quotes_flag);
+
+  bool col_sep_found = true;
+
+  VALUE elements = rb_ary_new();
+  VALUE field;
+  long i;
+
+  long backslash_count = 0;
+  bool in_quotes = false;
+  long element_count = 0;
+
+  int max_fields = -1;
+  if (max_size != Qnil) {
+    max_fields = NUM2INT(max_size);
+    if (max_fields < 0) {
+      return rb_ary_new();  // early return
+    }
+  }
+
+  char stack_buf[STACK_BUF_SIZE];
+
+  while (p < endP) {
+    col_sep_found = true;
+    for (i = 0; (i < col_sep_len) && (p + i < endP); i++) {
+      col_sep_found = col_sep_found && (*(p + i) == *(col_sepP + i));
+    }
+
+    if (__builtin_expect(col_sep_found && !in_quotes, 1)) {
+      if ((max_fields >= 0) && (element_count >= max_fields)) {
+        break;
+      }
+
+      bool only_spaces = true;
+      for (char *s = startP; s < p; s++) {
+        if (*s != ' ') {
+          only_spaces = false;
+          break;
+        }
+      }
+
+      if (only_spaces) {
+        field = Qempty_string;
+      } else {
+        char *src = startP;
+        long src_len = p - startP;
+        bool is_quoted = false;
+
+        if (__builtin_expect(has_quotes, 0)) {
+          is_quoted = src_len >= 2 && src[0] == quote && src[src_len - 1] == quote;
+          if (is_quoted) {
+            src += 1;
+            src_len -= 2;
+          }
+        }
+
+        if (!has_quotes || memchr(src, quote, src_len) == NULL) {
+          field = rb_enc_str_new(src, src_len, encoding);
+        } else {
+          char *clean_buf = src_len < STACK_BUF_SIZE ? stack_buf : ALLOC_N(char, src_len);
+          long clean_len = 0;
+
+          for (i = 0; i < src_len; i++) {
+            if (src[i] == quote && (i + 1 < src_len) && src[i + 1] == quote) {
+              clean_buf[clean_len++] = quote;
+              i++;
+            } else {
+              clean_buf[clean_len++] = src[i];
+            }
+          }
+
+          field = rb_enc_str_new(clean_buf, clean_len, encoding);
+          if (clean_buf != stack_buf) xfree(clean_buf);
+        }
+      }
+
+      rb_ary_push(elements, field);
+      element_count++;
+
+      p += col_sep_len;
+      startP = p;
+      backslash_count = 0;
+    } else {
+      if (*p == '\\') {
+        backslash_count++;
+      } else {
+        if (*p == quote) {
+          if (backslash_count % 2 == 0) {
+            in_quotes = !in_quotes;
+          }
+        }
+        backslash_count = 0;
+      }
+      p++;
+    }
+  }
+
+  if (in_quotes) {
+    rb_raise(eMalformedCSVError, "Unclosed quoted field detected in line: %s", StringValueCStr(line));
+  }
+
+  if ((max_fields < 0) || (element_count < max_fields)) {
+    bool only_spaces = true;
+    for (char *s = startP; s < endP; s++) {
+      if (*s != ' ') {
+        only_spaces = false;
+        break;
+      }
+    }
+
+    if (only_spaces) {
+      field = Qempty_string;
+    } else {
+      char *src = startP;
+      long src_len = endP - startP;
+      bool is_quoted = false;
+
+      if (__builtin_expect(has_quotes, 0)) {
+        is_quoted = src_len >= 2 && src[0] == quote && src[src_len - 1] == quote;
+        if (is_quoted) {
+          src += 1;
+          src_len -= 2;
+        }
+      }
+
+      if (!has_quotes || memchr(src, quote, src_len) == NULL) {
+        field = rb_enc_str_new(src, src_len, encoding);
+      } else {
+        char *clean_buf = src_len < STACK_BUF_SIZE ? stack_buf : ALLOC_N(char, src_len);
+        long clean_len = 0;
+
+        for (i = 0; i < src_len; i++) {
+          if (src[i] == quote && (i + 1 < src_len) && src[i + 1] == quote) {
+            clean_buf[clean_len++] = quote;
+            i++;
+          } else {
+            clean_buf[clean_len++] = src[i];
+          }
+        }
+
+        field = rb_enc_str_new(clean_buf, clean_len, encoding);
+        if (clean_buf != stack_buf) xfree(clean_buf);
+      }
+    }
+
+    rb_ary_push(elements, field);
+  }
+
+  return elements;
+}
+
+void Init_smarter_csv(void) {
+  SmarterCSV = rb_const_get(rb_cObject, rb_intern("SmarterCSV"));
+  Parser = rb_const_get(SmarterCSV, rb_intern("Parser"));
+  eMalformedCSVError = rb_const_get(SmarterCSV, rb_intern("MalformedCSV"));
+
+  Qempty_string = rb_str_new_literal("");
+  rb_obj_freeze(Qempty_string);
+  rb_global_variable(&Qempty_string);
+
+  rb_define_module_function(Parser, "parse_csv_line_c", rb_parse_csv_line, 5);
+}
+

data/ext/smarter_csv/smarter_csv.c.works7

@@ -0,0 +1,217 @@
+#include "ruby.h"
+#include "ruby/encoding.h"
+#include <stdio.h>
+#include <stdbool.h>
+#include <string.h>
+
+#ifndef bool
+  #define bool int
+  #define false ((bool)0)
+  #define true  ((bool)1)
+#endif
+
+VALUE SmarterCSV = Qnil;
+VALUE eMalformedCSVError = Qnil;
+VALUE Parser = Qnil;
+VALUE Qempty_string = Qnil; // shared frozen empty string
+
+static VALUE rb_parse_csv_line(VALUE self, VALUE line, VALUE col_sep, VALUE quote_char, VALUE max_size, VALUE has_quotes_val) {
+  if (RB_TYPE_P(line, T_NIL) == 1) {
+    return rb_ary_new();
+  }
+
+  if (RB_TYPE_P(line, T_STRING) != 1) {
+    rb_raise(rb_eTypeError, "ERROR in SmarterCSV.parse_line: line has to be a string or nil");
+  }
+
+  rb_encoding *encoding = rb_enc_get(line); /* get the encoding from the input line */
+  char *startP = RSTRING_PTR(line); /* may not be null terminated */
+  long line_len = RSTRING_LEN(line);
+  char *endP = startP + line_len; /* points behind the string */
+  char *p = startP;
+
+  char *col_sepP = RSTRING_PTR(col_sep);
+  long col_sep_len = RSTRING_LEN(col_sep);
+
+  char *quoteP = RSTRING_PTR(quote_char);
+  char quote_char_val = quoteP[0];
+
+  bool col_sep_found = true;
+
+  VALUE elements = rb_ary_new();
+  VALUE field;
+  long i;
+
+  /* Variables for escaped quote handling */
+  long backslash_count = 0;
+  bool in_quotes = false;
+
+  /* Optimization 1: maintain count instead of calling RARRAY_LEN repeatedly */
+  long element_count = 0;
+
+  /* Optimization 2: cache max_size value if not nil */
+  int max_fields = -1;
+  if (max_size != Qnil) {
+    max_fields = NUM2INT(max_size);
+    if (max_fields < 0) {
+      return rb_ary_new();  // Return empty array early
+    }
+  }
+
+  bool has_quotes = RTEST(has_quotes_val);
+
+  /* Fast-path for clean CSV lines: no quotes, 1-character separator */
+  if (__builtin_expect(!has_quotes && col_sep_len == 1, 1)) {
+    char sep = *col_sepP;
+    char *sep_pos = NULL;
+
+    while ((sep_pos = memchr(p, sep, endP - p))) {
+      if ((max_fields >= 0) && (element_count >= max_fields)) {
+        break;
+      }
+
+      /* check if only spaces */
+      bool only_spaces = true;
+      for (char *s = startP; s < sep_pos; s++) {
+        if (*s != ' ') {
+          only_spaces = false;
+          break;
+        }
+      }
+
+      field = only_spaces ? Qempty_string : rb_enc_str_new(startP, sep_pos - startP, encoding);
+      rb_ary_push(elements, field);
+      element_count++;
+
+      p = sep_pos + 1;
+      startP = p;
+    }
+
+    /* check if the last part of the line needs to be processed */
+    if ((max_fields < 0) || (element_count < max_fields)) {
+      bool only_spaces = true;
+      for (char *s = startP; s < endP; s++) {
+        if (*s != ' ') {
+          only_spaces = false;
+          break;
+        }
+      }
+      field = only_spaces ? Qempty_string : rb_enc_str_new(startP, endP - startP, encoding);
+      rb_ary_push(elements, field);
+    }
+
+    return elements;
+  }
+
+  /* default path for quoted/multi-separator lines */
+  while (p < endP) {
+    /* does the remaining string start with col_sep ? */
+    col_sep_found = true;
+    for(i=0; (i < col_sep_len) && (p+i < endP); i++) {
+      col_sep_found = col_sep_found && (*(p+i) == *(col_sepP+i));
+    }
+    /* if col_sep was found and we're not inside quotes */
+    if (col_sep_found && !in_quotes) {
+      if ((max_fields >= 0) && (element_count >= max_fields)) {
+        break;
+      } else {
+        long field_len = p - startP;
+        char *raw_field = startP;
+
+        /* Remove surrounding quotes if present */
+        bool is_wrapped = (field_len >= 2 && raw_field[0] == quote_char_val && raw_field[field_len - 1] == quote_char_val);
+        if (is_wrapped) {
+          raw_field++;
+          field_len -= 2;
+        }
+
+        /* Always unescape doubled quotes */
+        char *src = raw_field;
+        char *limit = raw_field + field_len;
+        char *buffer = ALLOC_N(char, field_len);
+        char *dst = buffer;
+
+        while (src < limit) {
+          if (*src == quote_char_val && (src + 1 < limit) && *(src + 1) == quote_char_val) {
+            *dst++ = quote_char_val;
+            src += 2;
+          } else {
+            *dst++ = *src++;
+          }
+        }
+
+        field = rb_enc_str_new(buffer, dst - buffer, encoding);
+        xfree(buffer);
+
+        rb_ary_push(elements, field);
+        element_count++;
+
+        p += col_sep_len;
+        startP = p;
+        backslash_count = 0; // Reset backslash count at the start of a new field
+      }
+    } else {
+      if (*p == '\\') {
+        backslash_count++;
+      } else {
+        if (*p == quote_char_val) {
+          if (backslash_count % 2 == 0) {
+            /* Even number of backslashes means quote is not escaped */
+            in_quotes = !in_quotes;
+          }
+        }
+        backslash_count = 0; // Reset after any character other than backslash
+      }
+      p++;
+    }
+  } /* while */
+
+  /* Check for unclosed quotes at the end of the line */
+  if (in_quotes) {
+    rb_raise(eMalformedCSVError, "Unclosed quoted field detected in line: %s", StringValueCStr(line));
+  }
+
+  /* check if the last part of the line needs to be processed */
+  if ((max_fields < 0) || (element_count < max_fields)) {
+    long field_len = endP - startP;
+    char *raw_field = startP;
+
+    bool is_wrapped = (field_len >= 2 && raw_field[0] == quote_char_val && raw_field[field_len - 1] == quote_char_val);
+    if (is_wrapped) {
+      raw_field++;
+      field_len -= 2;
+    }
+
+    /* Always unescape doubled quotes */
+    char *src = raw_field;
+    char *limit = raw_field + field_len;
+    char *buffer = ALLOC_N(char, field_len);
+    char *dst = buffer;
+
+    while (src < limit) {
+      if (*src == quote_char_val && (src + 1 < limit) && *(src + 1) == quote_char_val) {
+        *dst++ = quote_char_val;
+        src += 2;
+      } else {
+        *dst++ = *src++;
+      }
+    }
+
+    field = rb_enc_str_new(buffer, dst - buffer, encoding);
+    xfree(buffer);
+
+    rb_ary_push(elements, field);
+  }
+
+  return elements;
+}
+
+void Init_smarter_csv(void) {
+  // these modules and the error class are already defined in Ruby code, make them accessible:
+  SmarterCSV = rb_const_get(rb_cObject, rb_intern("SmarterCSV"));
+  Parser = rb_const_get(SmarterCSV, rb_intern("Parser"));
+  eMalformedCSVError = rb_const_get(SmarterCSV, rb_intern("MalformedCSV"));
+  Qempty_string = rb_str_new_literal("");
+  rb_gc_register_address(&Qempty_string);
+  rb_define_module_function(Parser, "parse_csv_line_c", rb_parse_csv_line, 5);
+}