smarter_csv 1.16.5 → 1.16.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9408d23e53325b5366f4033166b5d4f4b45bcfee81a4dfa94b3d0c1959317216
-  data.tar.gz: e05f83f08e01fbb859da5cb898125b72fc911a4ba199992ee8075e8bfe2b6a99
+  metadata.gz: 6e900772903374e904003b0e2219542eed5889619682133731ac15c1392dfcd2
+  data.tar.gz: '03039f42cd4bd03bc8353047a888ef8ff510216de887fa62b0c2bcd8614c2a52'
 SHA512:
-  metadata.gz: a5064055b6b7cb4a9365c139952cdeaedaad57c5636025401d2e87bd07184f406cb055a4d51229ac60b51ea3dbaace380116dc8a9862eabf00e4b4de7b345e33
-  data.tar.gz: 57ceb31ab4b15c36880c63c6f35c2919e8ab80ddd2599e552390a8d84ef3592e52097710f8be978b28c77cc6ae85c10583653e6050d3f4f3b6eb6ec6cb1e6aa9
+  metadata.gz: af60ffbef48ddaeb0a508624572be722d5ec8c748cbf3519fd277161287ffe406e8fa0209f1b9d21eedad36aa9cf7587dc8dbf6939cf346bce54dfa188c19756
+  data.tar.gz: 798f10a02f1add2f42e4a14d5e6abb60c13cdbfe1ade2e3b44f95068503c3827faaf5aa29fcf38eb6031f9ce3f0e68db3cb26b5df8a4afeb50a1a020917b6338

data/CHANGELOG.md

@@ -1,6 +1,16 @@
 
 # SmarterCSV 1.x Change Log
 
+## 1.16.6 (2026-05-21)
+
+RSpec tests: **1,467 → 1,591** (+124 tests)
+
+### Bug Fix
+
+  - fixed [Issue #334](https://github.com/tilo/smarter_csv/issues/334) with escaped double quote followed by comma. Thanks to [conorg](https://github.com/conorg)
+  - fixed bug when using `headers: { except: }`
+  - added more tests
+
 ## 1.16.5 (2026-05-18)
 
 ### Bug Fix

data/CONTRIBUTORS.md

@@ -1,4 +1,4 @@
-# A Big Thank You to all 63 Contributors!!
+# A Big Thank You to all 64 Contributors!!
 
 
 A Big Thank you to everyone who filed issues, sent comments, and who contributed with pull requests:
@@ -66,3 +66,4 @@ A Big Thank you to everyone who filed issues, sent comments, and who contributed
  * [Dom Lebron](https://github.com/biglebronski)
  * [Paho Lurie-Gregg](https://github.com/paholg)
  * [Jonas Staškevičius](https://github.com/pirminis)
+ * [conorg](https://github.com/conorg)

data/README.md

@@ -249,7 +249,7 @@ For reporting issues, please:
   * open a pull-request adding a test that demonstrates the issue
   * mention your version of SmarterCSV, Ruby, Rails
 
-# [A Special Thanks to all 63 Contributors!](CONTRIBUTORS.md) 🎉🎉🎉
+# [A Special Thanks to all 64 Contributors!](CONTRIBUTORS.md) 🎉🎉🎉
 
 
 ## Contributing

data/Rakefile

@@ -21,8 +21,13 @@ RuboCop::RakeTask.new
 
 require "rake/extensiontask"
 
-if RUBY_ENGINE == 'jruby'
-
+if RUBY_ENGINE != 'ruby'
+  # Non-MRI (JRuby, TruffleRuby, ...): there is no C extension to build. Define the
+  # native-build tasks as no-ops so the same default task list works on every engine
+  # and any caller (CI, rake-compiler, downstream) succeeds without trying to build or
+  # copy a .so. Runtime uses the pure-Ruby parser.
+  task :compile # no-op
+  task :clobber # no-op
   task default: %i[spec]
 
 else

data/ext/smarter_csv/extconf.rb

@@ -3,6 +3,15 @@
 require 'mkmf'
 require "rbconfig"
 
+# On non-MRI Rubies (JRuby, TruffleRuby, ...) there is no C extension to build, and trying to build
+# it breaks `gem install` for anything that depends on smarter_csv. Write a no-op Makefile so install
+# succeeds, then stop. At runtime SmarterCSV falls back to its pure-Ruby parser (it checks whether the
+# C functions actually loaded via respond_to?(:parse_csv_line_c)).
+if RUBY_ENGINE != 'ruby'
+  File.write('Makefile', dummy_makefile($srcdir).join)
+  exit 0
+end
+
 if RbConfig::MAKEFILE_CONFIG["CFLAGS"].include?("-g -O3")
   fixed_CFLAGS = RbConfig::MAKEFILE_CONFIG["CFLAGS"].sub("-g -O3", "-O3 $(cflags)")
   puts("Fix CFLAGS: #{RbConfig::MAKEFILE_CONFIG["CFLAGS"]} -> #{fixed_CFLAGS}")

data/ext/smarter_csv/smarter_csv.c

@@ -304,25 +304,40 @@ static VALUE rb_parse_csv_line(VALUE self, VALUE line, VALUE col_sep, VALUE quot
           if (!allow_escaped_quotes || backslash_count % 2 == 0) {
             if (__builtin_expect(quote_boundary_standard, 1)) {
               if (in_quotes) {
-                // closing quote: only valid if followed by col_sep, row_sep, or end of line
-                bool valid_close = (p + 1 >= endP);
-                if (!valid_close) {
-                  valid_close = true;
-                  for (long j = 0; j < col_sep_len; j++) {
-                    if (*(p + 1 + j) != *(col_sepP + j)) { valid_close = false; break; }
+                if (p + 2 < endP && *(p + 1) == quote_char_val) {
+                  /* RFC doubled quote inside a quoted field ("" → ").
+                   * Give this precedence over the closing-quote check, but only
+                   * when another byte follows the doubled pair.
+                   *
+                   * Compatibility note: we intentionally do NOT force terminal
+                   * "" to be consumed here. SmarterCSV has a long-standing lenient
+                   * behavior for malformed tails like ...\"" in :double_quotes mode:
+                   * the final quote may still close the field instead of turning the
+                   * row into an unclosed-quote error. Issue #334 needs doubled-quote
+                   * precedence for ..."",... (more content follows), but we keep the
+                   * historical leniency for terminal ..."". */
+                  p++;
+                } else {
+                  // closing quote: only valid if followed by col_sep, row_sep, or end of line
+                  bool valid_close = (p + 1 >= endP);
+                  if (!valid_close) {
+                    valid_close = true;
+                    for (long j = 0; j < col_sep_len; j++) {
+                      if (*(p + 1 + j) != *(col_sepP + j)) { valid_close = false; break; }
+                    }
                   }
-                }
-                if (!valid_close && row_sep_len > 0) {
-                  valid_close = true;
-                  for (long j = 0; j < row_sep_len; j++) {
-                    if (*(p + 1 + j) != *(row_sepP + j)) { valid_close = false; break; }
+                  if (!valid_close && row_sep_len > 0) {
+                    valid_close = true;
+                    for (long j = 0; j < row_sep_len; j++) {
+                      if (*(p + 1 + j) != *(row_sepP + j)) { valid_close = false; break; }
+                    }
                   }
+                  if (valid_close) {
+                    in_quotes = false;
+                    field_started = true;
+                  }
+                  // else: quote inside quoted field → literal
                 }
-                if (valid_close) {
-                  in_quotes = false;
-                  field_started = true;
-                }
-                // else: quote inside quoted field → literal (handles "" doubling)
               } else if (!field_started) {
                 in_quotes = true;     // opening quote at field boundary
                 field_started = true;
@@ -763,6 +778,11 @@ __attribute__((hot)) static VALUE rb_parse_line_to_hash(VALUE self, VALUE line,
    * the frame stays well below 4 KB and ___chkstk_darwin never fires on ARM64 macOS.
    */
   bool *keep_bitmap = NULL;
+  /* In THIS (non-ctx) function the bitmap is alloca'd to headers_len on every call (see the alloca
+   * sites below), so keep_bitmap[] is exactly headers_len long and headers_len is the correct bound
+   * at all access sites. Do NOT mirror rb_parse_line_to_hash_ctx's keep_bitmap_len here: that variant
+   * caches its bitmap across rows (where @headers can grow), so it must use the captured length; this
+   * one rebuilds per call and does not. */
   bool keep_extra_columns = true; /* extra cols (> headers_len): keep by default */
   bool has_only = false;          /* true when only_headers: filtering is active */
   long early_exit_after = -1;     /* column index after which we stop; -1 = no early exit */
@@ -1081,25 +1101,40 @@ __attribute__((hot)) static VALUE rb_parse_line_to_hash(VALUE self, VALUE line,
             if (!allow_escaped_quotes || backslash_count % 2 == 0) {
               if (__builtin_expect(quote_boundary_standard, 1)) {
                 if (in_quotes) {
-                  // closing quote: only valid if followed by col_sep, row_sep, or end of line
-                  bool valid_close = (p + 1 >= endP);
-                  if (!valid_close) {
-                    valid_close = true;
-                    for (long j = 0; j < col_sep_len; j++) {
-                      if (*(p + 1 + j) != *(col_sepP + j)) { valid_close = false; break; }
+                  if (p + 2 < endP && *(p + 1) == quote_char_val) {
+                    /* RFC doubled quote inside a quoted field ("" → ").
+                     * Give this precedence over the closing-quote check, but only
+                     * when another byte follows the doubled pair.
+                     *
+                     * Compatibility note: we intentionally do NOT force terminal
+                     * "" to be consumed here. SmarterCSV has a long-standing lenient
+                     * behavior for malformed tails like ...\"" in :double_quotes mode:
+                     * the final quote may still close the field instead of turning the
+                     * row into an unclosed-quote error. Issue #334 needs doubled-quote
+                     * precedence for ..."",... (more content follows), but we keep the
+                     * historical leniency for terminal ..."". */
+                    p++;
+                  } else {
+                    // closing quote: only valid if followed by col_sep, row_sep, or end of line
+                    bool valid_close = (p + 1 >= endP);
+                    if (!valid_close) {
+                      valid_close = true;
+                      for (long j = 0; j < col_sep_len; j++) {
+                        if (*(p + 1 + j) != *(col_sepP + j)) { valid_close = false; break; }
+                      }
                     }
-                  }
-                  if (!valid_close && row_sep_len2 > 0) {
-                    valid_close = true;
-                    for (long j = 0; j < row_sep_len2; j++) {
-                      if (*(p + 1 + j) != *(row_sepP2 + j)) { valid_close = false; break; }
+                    if (!valid_close && row_sep_len2 > 0) {
+                      valid_close = true;
+                      for (long j = 0; j < row_sep_len2; j++) {
+                        if (*(p + 1 + j) != *(row_sepP2 + j)) { valid_close = false; break; }
+                      }
                     }
+                    if (valid_close) {
+                      in_quotes = false;
+                      field_started = true;
+                    }
+                    // else: quote inside quoted field → literal
                   }
-                  if (valid_close) {
-                    in_quotes = false;
-                    field_started = true;
-                  }
-                  // else: quote inside quoted field → literal (handles "" doubling)
                 } else if (!field_started) {
                   in_quotes = true;     // opening quote at field boundary
                   field_started = true;
@@ -1429,6 +1464,14 @@ __attribute__((hot)) static VALUE rb_parse_line_to_hash_ctx(VALUE self, VALUE li
   int  numeric_mode        = ctx->numeric_mode;
   VALUE numeric_keys       = ctx->numeric_keys;
   bool *keep_bitmap         = ctx->keep_bitmap;
+  /* keep_bitmap is cached in the context (xmalloc'd once at construction, sized to the header count
+   * THEN). @headers can grow in place as undeclared extra columns appear, so the live headers_len
+   * (re-read each call below) may exceed the bitmap's length. Every keep_bitmap[] access in this
+   * function MUST be bounded by keep_bitmap_len, never headers_len — indices past the bitmap are
+   * extra columns and follow keep_extra_columns. Bounding by the grown headers_len was an
+   * out-of-bounds heap read (the bug). The sibling rb_parse_line_to_hash safely uses headers_len
+   * because it re-allocs its bitmap to headers_len on every call. */
+  long  keep_bitmap_len     = ctx->keep_bitmap_len;
   bool  keep_extra_columns  = ctx->keep_extra_columns;
   long  early_exit_after    = ctx->early_exit_after;
 
@@ -1530,7 +1573,7 @@ __attribute__((hot)) static VALUE rb_parse_line_to_hash_ctx(VALUE self, VALUE li
           while (trim_end >= trim_start && (*trim_end == ' ' || *trim_end == '\t')) trim_end--;
         }
         long trimmed_len = (trim_end >= trim_start) ? (trim_end - trim_start + 1) : 0;
-        if (!keep_bitmap || (element_count < headers_len ? keep_bitmap[element_count] : keep_extra_columns)) {
+        if (!keep_bitmap || (element_count < keep_bitmap_len ? keep_bitmap[element_count] : keep_extra_columns)) {
           if (insert_field_into_hash(&xform, trim_start, trimmed_len, element_count, false, quote_char_val, encoding))
             all_blank = false;
         }
@@ -1551,7 +1594,7 @@ __attribute__((hot)) static VALUE rb_parse_line_to_hash_ctx(VALUE self, VALUE li
           while (trim_end >= trim_start && (*trim_end == ' ' || *trim_end == '\t')) trim_end--;
         }
         long trimmed_len = (trim_end >= trim_start) ? (trim_end - trim_start + 1) : 0;
-        if (!keep_bitmap || (element_count < headers_len ? keep_bitmap[element_count] : keep_extra_columns)) {
+        if (!keep_bitmap || (element_count < keep_bitmap_len ? keep_bitmap[element_count] : keep_extra_columns)) {
           if (insert_field_into_hash(&xform, trim_start, trimmed_len, element_count, false, quote_char_val, encoding))
             all_blank = false;
         }
@@ -1614,7 +1657,7 @@ __attribute__((hot)) static VALUE rb_parse_line_to_hash_ctx(VALUE self, VALUE li
 
         bool has_embedded_quotes = quoted || (trimmed_len > 0 && memchr(trim_start, quote_char_val, trimmed_len));
 
-        if (!keep_bitmap || (element_count < headers_len ? keep_bitmap[element_count] : keep_extra_columns)) {
+        if (!keep_bitmap || (element_count < keep_bitmap_len ? keep_bitmap[element_count] : keep_extra_columns)) {
           if (insert_field_into_hash(&xform, trim_start, trimmed_len, element_count, has_embedded_quotes, quote_char_val, encoding))
             all_blank = false;
         }
@@ -1648,25 +1691,40 @@ __attribute__((hot)) static VALUE rb_parse_line_to_hash_ctx(VALUE self, VALUE li
             if (!allow_escaped_quotes || backslash_count % 2 == 0) {
               if (__builtin_expect(quote_boundary_standard, 1)) {
                 if (in_quotes) {
-                  /* closing quote: only valid if followed by col_sep, row_sep, or end */
-                  bool valid_close = (p + 1 >= endP);
-                  if (!valid_close) {
-                    valid_close = true;
-                    for (long j = 0; j < col_sep_len; j++) {
-                      if (*(p + 1 + j) != *(col_sepP + j)) { valid_close = false; break; }
+                  if (p + 2 < endP && *(p + 1) == quote_char_val) {
+                    /* RFC doubled quote inside a quoted field ("" → ").
+                     * Give this precedence over the closing-quote check, but only
+                     * when another byte follows the doubled pair.
+                     *
+                     * Compatibility note: we intentionally do NOT force terminal
+                     * "" to be consumed here. SmarterCSV has a long-standing lenient
+                     * behavior for malformed tails like ...\"" in :double_quotes mode:
+                     * the final quote may still close the field instead of turning the
+                     * row into an unclosed-quote error. Issue #334 needs doubled-quote
+                     * precedence for ..."",... (more content follows), but we keep the
+                     * historical leniency for terminal ..."". */
+                    p++;
+                  } else {
+                    /* closing quote: only valid if followed by col_sep, row_sep, or end */
+                    bool valid_close = (p + 1 >= endP);
+                    if (!valid_close) {
+                      valid_close = true;
+                      for (long j = 0; j < col_sep_len; j++) {
+                        if (*(p + 1 + j) != *(col_sepP + j)) { valid_close = false; break; }
+                      }
                     }
-                  }
-                  if (!valid_close && row_sep_len2 > 0) {
-                    valid_close = true;
-                    for (long j = 0; j < row_sep_len2; j++) {
-                      if (*(p + 1 + j) != *(row_sepP2 + j)) { valid_close = false; break; }
+                    if (!valid_close && row_sep_len2 > 0) {
+                      valid_close = true;
+                      for (long j = 0; j < row_sep_len2; j++) {
+                        if (*(p + 1 + j) != *(row_sepP2 + j)) { valid_close = false; break; }
+                      }
                     }
+                    if (valid_close) {
+                      in_quotes     = false;
+                      field_started = true;
+                    }
+                    /* else: quote inside quoted field → literal */
                   }
-                  if (valid_close) {
-                    in_quotes     = false;
-                    field_started = true;
-                  }
-                  /* else: quote inside quoted field → literal (handles "" doubling) */
                 } else if (!field_started) {
                   in_quotes     = true;   /* opening quote at field boundary */
                   field_started = true;
@@ -1725,7 +1783,7 @@ __attribute__((hot)) static VALUE rb_parse_line_to_hash_ctx(VALUE self, VALUE li
 
       bool has_embedded_quotes = quoted || (trimmed_len > 0 && memchr(trim_start, quote_char_val, trimmed_len));
 
-      if (!keep_bitmap || (element_count < headers_len ? keep_bitmap[element_count] : keep_extra_columns)) {
+      if (!keep_bitmap || (element_count < keep_bitmap_len ? keep_bitmap[element_count] : keep_extra_columns)) {
         if (insert_field_into_hash(&xform, trim_start, trimmed_len, element_count, has_embedded_quotes, quote_char_val, encoding))
           all_blank = false;
       }
@@ -1753,7 +1811,7 @@ __attribute__((hot)) static VALUE rb_parse_line_to_hash_ctx(VALUE self, VALUE li
   if (!remove_empty_values) {
     ensure_hash_allocated(&xform);
     for (long i = element_count; i < headers_len; i++) {
-      if (!keep_bitmap || keep_bitmap[i]) {
+      if (!keep_bitmap || (i < keep_bitmap_len ? keep_bitmap[i] : keep_extra_columns)) {
         rb_hash_aset(xform.hash, rb_ary_entry(headers, i), Qnil);
       }
     }

data/lib/smarter_csv/parser.rb

@@ -405,15 +405,28 @@ module SmarterCSV
                 if !allow_escaped_quotes || backslash_count % 2 == 0
                   if quote_boundary_standard
                     if in_quotes
-                      # closing quote: only valid if followed by col_sep, row_sep, or end of line
                       next_i = i + 1
-                      if next_i >= bytesize ||
-                         line.getbyte(next_i) == col_sep_byte ||
-                         (row_sep_bytesize > 0 && line.byteslice(next_i, row_sep_bytesize) == row_sep)
+                      if next_i + 1 < bytesize && line.getbyte(next_i) == quote_byte
+                        # RFC doubled quote inside a quoted field ("" → ").
+                        # Give this precedence over the closing-quote check, but only
+                        # when another byte follows the doubled pair.
+                        #
+                        # Compatibility note: we intentionally do NOT force terminal
+                        # "" to be consumed here. SmarterCSV has a long-standing lenient
+                        # behavior for malformed tails like ...\"" in :double_quotes mode:
+                        # the final quote may still close the field instead of turning the
+                        # row into an unclosed-quote error. Issue #334 needs doubled-quote
+                        # precedence for ..."",... (more content follows), but we keep the
+                        # historical leniency for terminal ..."".
+                        i = next_i
+                      # closing quote: only valid if followed by col_sep, row_sep, or end of line
+                      elsif next_i >= bytesize ||
+                            line.getbyte(next_i) == col_sep_byte ||
+                            (row_sep_bytesize > 0 && line.byteslice(next_i, row_sep_bytesize) == row_sep)
                         in_quotes = false
                         field_started = true
                       end
-                      # else: quote inside quoted field → literal (handles "" doubling)
+                      # else: quote inside quoted field → literal
                     elsif !field_started # at field boundary: open quoted field
                       in_quotes = true
                       field_started = true
@@ -510,15 +523,28 @@ module SmarterCSV
                 if !allow_escaped_quotes || backslash_count % 2 == 0
                   if quote_boundary_standard
                     if in_quotes
-                      # closing quote: only valid if followed by col_sep, row_sep, or end of line
                       next_i = i + 1
-                      if next_i >= line_size ||
-                         line[next_i...next_i + col_sep_size] == col_sep ||
-                         (row_sep_size > 0 && line[next_i...next_i + row_sep_size] == row_sep)
+                      if next_i + 1 < line_size && line[next_i] == quote
+                        # RFC doubled quote inside a quoted field ("" → ").
+                        # Give this precedence over the closing-quote check, but only
+                        # when another character follows the doubled pair.
+                        #
+                        # Compatibility note: we intentionally do NOT force terminal
+                        # "" to be consumed here. SmarterCSV has a long-standing lenient
+                        # behavior for malformed tails like ...\"" in :double_quotes mode:
+                        # the final quote may still close the field instead of turning the
+                        # row into an unclosed-quote error. Issue #334 needs doubled-quote
+                        # precedence for ..."",... (more content follows), but we keep the
+                        # historical leniency for terminal ..."".
+                        i = next_i
+                      # closing quote: only valid if followed by col_sep, row_sep, or end of line
+                      elsif next_i >= line_size ||
+                            line[next_i...next_i + col_sep_size] == col_sep ||
+                            (row_sep_size > 0 && line[next_i...next_i + row_sep_size] == row_sep)
                         in_quotes = false
                         field_started = true
                       end
-                      # else: quote inside quoted field → literal (handles "" doubling)
+                      # else: quote inside quoted field → literal
                     elsif !field_started # at field boundary: open quoted field
                       in_quotes = true
                       field_started = true

data/lib/smarter_csv/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SmarterCSV
-  VERSION = "1.16.5"
+  VERSION = "1.16.6"
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: smarter_csv
 version: !ruby/object:Gem::Version
-  version: 1.16.5
+  version: 1.16.6
 platform: ruby
 authors:
 - Tilo Sloboda
 bindir: bin
 cert_chain: []
-date: 2026-05-17 00:00:00.000000000 Z
+date: 2026-05-21 00:00:00.000000000 Z
 dependencies: []
 description: |
   SmarterCSV is a high-performance CSV reader and writer for Ruby focused on