smarter_csv 1.18.0 → 1.19.0

data/lib/smarter_csv/parser.rb

@@ -169,6 +169,24 @@ module SmarterCSV
     def parse_line_to_hash_ruby(line, headers, options, has_quotes = false)
       return [nil, 0] if line.nil?
 
+      # A line with invalid bytes for its encoding (typically Latin-1 data mislabeled as
+      # UTF-8) would make the encoding-aware operations below (split, strip!, ...) raise
+      # ArgumentError. Like the C path, we parse leniently and preserve the field's raw
+      # bytes: process the line as BINARY, then re-tag each field with the original
+      # encoding. Only relabels — never transcodes. Every byte sequence is valid BINARY,
+      # so the recursive call cannot take this branch again.
+      unless line.valid_encoding?
+        original_encoding = line.encoding
+        binary_options = options.dup
+        %i[col_sep quote_char row_sep].each do |opt|
+          binary_options[opt] = options[opt].dup.force_encoding(Encoding::BINARY) if options[opt].is_a?(String)
+        end
+        hash, data_size = parse_line_to_hash_ruby(line.dup.force_encoding(Encoding::BINARY), headers, binary_options, has_quotes)
+        # skip empty strings: relabeling "" is a no-op, and the shared EMPTY_STRING is frozen
+        hash&.transform_values! { |v| v.is_a?(String) && !v.empty? ? v.force_encoding(original_encoding) : v }
+        return [hash, data_size]
+      end
+
       # Chomp trailing row separator
       line = line.chomp(options[:row_sep]) if options[:row_sep]
 
@@ -176,6 +194,14 @@ module SmarterCSV
       strip   = options[:strip_whitespace]
       prefix  = options[:missing_header_prefix]
 
+      # headers: { only: } SHORT-CUT (mirrors the C path's early exit): stop parsing right
+      # after the last wanted column and ignore everything behind it — extra columns are
+      # not discovered (no :column_N growth) and even an unclosed quote in an unwanted
+      # trailing column is ignored. _early_exit_after is the 0-based index of the last
+      # wanted column (set by the reader for only: without missing_headers: :raise).
+      early_exit = options[:_early_exit_after]
+      max_fields = early_exit && early_exit >= 0 ? early_exit + 1 : nil
+
       # Optimization #11: for unquoted lines, build the hash in one pass directly
       # from String#split — no intermediate array returned from parse_csv_line_ruby
       # and no second iteration to convert array → hash. Saves one Array allocation
@@ -187,7 +213,14 @@ module SmarterCSV
       # (default), v.empty? after strip catches both empty and whitespace-only
       # fields without a regex. Most impactful on sparse files (many empty fields).
       unless has_quotes || col_sep == ' '
-        fields = line.split(col_sep, -1)
+        if max_fields
+          # limited split: at most max_fields + 1 elements, the last being the unparsed
+          # remainder of the line — drop it, it is behind the last wanted column
+          fields = line.split(col_sep, max_fields + 1)
+          fields.pop if fields.size == max_fields + 1
+        else
+          fields = line.split(col_sep, -1)
+        end
         n = fields.size
 
         if options[:remove_empty_hashes]
@@ -205,7 +238,10 @@ module SmarterCSV
         fields.each_with_index do |v, i| # C-level iteration, faster than Ruby while counter loop
           next if remove_empty && v.empty?
 
-          hash[i < headers.size ? headers[i] : :"#{prefix}#{i + 1}"] = v
+          # Empty values become the ONE shared frozen empty string (same design as the
+          # C path): the fresh "" from split dies in the next minor GC instead of being
+          # retained per empty field in the results.
+          hash[i < headers.size ? headers[i] : :"#{prefix}#{i + 1}"] = v.empty? ? EMPTY_STRING : v
         end
 
         unless remove_empty
@@ -216,7 +252,9 @@ module SmarterCSV
       end
 
       # Quoted/complex path: parse into elements array, then build hash.
-      elements, data_size = parse_csv_line_ruby(line, options, nil, has_quotes)
+      # max_fields makes parse_csv_line_ruby stop scanning after the last wanted column
+      # (same short-cut as the C path's early exit).
+      elements, data_size = parse_csv_line_ruby(line, options, max_fields, has_quotes)
       return [nil, -1] if data_size == -1 # unclosed quote at EOL → caller stitches next line
 
       # Optimization #6: elements are always String or nil from parse_csv_line_ruby,
@@ -236,7 +274,10 @@ module SmarterCSV
       hash = {}
       i = 0
       while i < n
-        hash[i < headers.size ? headers[i] : :"#{prefix}#{i + 1}"] = elements[i]
+        v = elements[i]
+        # Empty values become the ONE shared frozen empty string (same design as the C path)
+        v = EMPTY_STRING if v.is_a?(String) && v.empty?
+        hash[i < headers.size ? headers[i] : :"#{prefix}#{i + 1}"] = v
         i += 1
       end
 
@@ -320,12 +361,15 @@ module SmarterCSV
       row_sep = options[:row_sep]
       row_sep_size = row_sep.is_a?(String) ? row_sep.size : 0
 
-      # Optimization #1: for the common single-char separator, use direct
-      # character comparison instead of allocating a substring via line[i...i+n].
-      if col_sep_size == 1
-        # Optimization #13: byte-level indexing for single-char separator.
-        # col_sep and quote_char are both validated to be single-byte at option
-        # parsing time. UTF-8 multi-byte continuation bytes (0x80–0xBF) never
+      # Optimization #1: for the common single-BYTE separator, use direct
+      # byte comparison instead of allocating a substring via line[i...i+n].
+      # The gate must be on bytesize, not size: a one-character multi-byte separator
+      # (e.g. 'é') would be scanned by its first byte only, which also occurs as the
+      # lead byte of other characters — the character-level path below handles it.
+      if col_sep.bytesize == 1
+        # Optimization #13: byte-level indexing for single-byte separator.
+        # quote_char is validated to be single-byte at option parsing time.
+        # UTF-8 multi-byte continuation bytes (0x80–0xBF) never
         # alias ASCII delimiter bytes (0x00–0x7F), so byte scanning is safe for
         # UTF-8 strings with ASCII delimiters — no String allocation per character.
         col_sep_byte     = col_sep.getbyte(0)
@@ -356,8 +400,12 @@ module SmarterCSV
           # unquoted (field_started && !in_quotes), remaining quotes are literal and
           # cannot affect parser state — jump directly to the next col_sep.
           # Mirrors Opt #10 for the unquoted side of the same trade-off.
+          # byteindex requires the byte offset to be on a character boundary — after
+          # stepping over the first byte of a multi-byte character, i is mid-character
+          # (a UTF-8 continuation byte, 0b10xxxxxx), so fall back to the byte loop there.
+          # (The Opt #10 quote jump can't be mid-character: i is always at quote_byte + 1.)
           elsif quote_boundary_standard && field_started && !in_quotes
-            next_sep = if BYTEINDEX_AVAILABLE
+            next_sep = if BYTEINDEX_AVAILABLE && (line.getbyte(i) & 0xC0) != 0x80
                          line.byteindex(col_sep, i)
                        else
                          j = i

data/lib/smarter_csv/reader.rb

@@ -75,12 +75,16 @@ module SmarterCSV
     def each
       return enum_for(:each) unless block_given?
 
-      # Force row-by-row mode regardless of chunk_size setting
+      # Force row-by-row mode regardless of chunk_size setting.
+      # The explicit begin/ensure keeps the restore off the enum_for path above,
+      # where original_chunk_size was never captured (it would restore nil).
       original_chunk_size = @options[:chunk_size]
       @options[:chunk_size] = nil
-      process { |row_array, _| yield row_array.first }
-    ensure
-      @options[:chunk_size] = original_chunk_size
+      begin
+        process { |row_array, _| yield row_array.first }
+      ensure
+        @options[:chunk_size] = original_chunk_size
+      end
     end
 
     # Yields each chunk as Array<Hash> plus its 0-based chunk index.
@@ -236,6 +240,15 @@ module SmarterCSV
 
         @quote_escaping_auto = options[:quote_escaping] == :auto
         @use_acceleration    = options[:acceleration] && has_acceleration
+        # The C ParseContext stores separators and the extra-column prefix in fixed-size
+        # buffers (col_sep 7 bytes, row_sep 15, missing_header_prefix 63) and would
+        # silently truncate anything longer — fall back to the pure-Ruby parser for such
+        # exotic options; it handles any length.
+        if @use_acceleration
+          @use_acceleration = false if options[:col_sep].is_a?(String) && options[:col_sep].bytesize > 7
+          @use_acceleration = false if options[:row_sep].is_a?(String) && options[:row_sep].bytesize > 15
+          @use_acceleration = false if options[:missing_header_prefix].is_a?(String) && options[:missing_header_prefix].bytesize > 63
+        end
 
         # The single options hash used on the hot path — for :auto we always try backslash
         # first (C downgrades to RFC internally via Opt #5 when no backslash is found).
@@ -254,8 +267,10 @@ module SmarterCSV
         # Key-cleanup flags — computed once, checked per row via cheap ivar reads.
         # hash.delete(nil) / hash.delete('') only occur when key_mapping maps a header to nil/"".
         # hash.delete(:"") also catches empty headers produced by ,, in the CSV.
-        @delete_nil_keys   = !!options[:key_mapping]
-        @delete_empty_keys = !!options[:key_mapping] || @headers.include?(:"")
+        # A nil header (key_mapping to nil, or nil in user_provided_headers) drops the column
+        @delete_nil_keys   = !!options[:key_mapping] || @headers.include?(nil)
+        # Empty header keys are :"" with symbol keys, '' with strings_as_keys / keep_original_headers
+        @delete_empty_keys = !!options[:key_mapping] || @headers.include?(:"") || @headers.include?('')
 
         # Cache field_size_limit as an ivar (nil when unset → one nil-check per row, no method calls).
         @field_size_limit = options[:field_size_limit]
@@ -406,24 +421,18 @@ module SmarterCSV
                 hash.delete(nil)
                 hash.delete('')
               end
-              hash.delete(:"") if @delete_empty_keys
-
-              if (matcher = options[:nil_values_matching])
-                if options[:remove_empty_values]
-                  hash.delete_if do |_k, v|
-                    str_val = v.is_a?(String) ? v : (v.is_a?(Numeric) ? v.to_s : nil)
-                    str_val && matcher.match?(str_val)
-                  end
-                else
-                  hash.each_key do |k|
-                    v = hash[k]
-                    str_val = v.is_a?(String) ? v : (v.is_a?(Numeric) ? v.to_s : nil)
-                    hash[k] = nil if str_val && matcher.match?(str_val)
-                  end
-                end
+              if @delete_empty_keys
+                hash.delete(:"")
+                hash.delete('')
               end
 
-              if options[:value_converters]
+              if options[:nil_values_matching]
+                # The C parser deferred numeric conversion and zero-removal (see
+                # defer_value_transforms_to_ruby in the extension), so run the full Ruby
+                # pipeline: nil-matching on the raw strings first, then zero-removal,
+                # numeric conversion, and value_converters — the pure-Ruby-path order.
+                hash = hash_transformations(hash, options)
+              elsif options[:value_converters]
                 options[:value_converters].each do |key, converter|
                   hash[key] = converter.respond_to?(:convert) ? converter.convert(hash[key]) : converter.call(hash[key]) if hash.key?(key)
                 end
@@ -632,7 +641,22 @@ module SmarterCSV
       return false unless line.include?(options[:quote_char])
 
       if options[:quote_boundary] == :standard
-        detect_multiline_strict(line, options)
+        case options[:quote_escaping]
+        when :backslash
+          detect_multiline_strict(line, options, true)
+        when :auto
+          if line.include?('\\')
+            # :auto parses with backslash semantics first and retries with RFC semantics
+            # (see @hot_path_options / @quote_escaping_double) — the row is only still
+            # open if BOTH interpretations leave the quote open, mirroring the dual
+            # counting in the non-strict branch below.
+            detect_multiline_strict(line, options, true) && detect_multiline_strict(line, options, false)
+          else
+            detect_multiline_strict(line, options, false)
+          end
+        else
+          detect_multiline_strict(line, options, false)
+        end
       elsif options[:quote_escaping] == :auto
         escaped_count, rfc_count = count_quote_chars_auto(line, options[:quote_char], options[:col_sep])
         # If backslash-aware count is even → line is self-contained either way
@@ -657,7 +681,7 @@ module SmarterCSV
     #   - inside an unquoted field: jump directly to next col_sep via C-level byteindex
     # This makes detect_multiline_strict competitive with parse_csv_line_ruby on the same
     # content, enabling it to serve as a cheap gate in the stitch loop (Opt #18).
-    def detect_multiline_strict(line, options)
+    def detect_multiline_strict(line, options, allow_escaped_quotes = options[:quote_escaping] == :backslash)
       col_sep = options[:col_sep]
       quote   = options[:quote_char]
       strip   = options[:strip_whitespace]
@@ -667,9 +691,26 @@ module SmarterCSV
       row_sep_size  = row_sep.is_a?(String) ? row_sep.size : 0
       in_quotes     = false
       field_started = false
-
-      if col_sep_size == 1
-        # Fast path: byte-level scanning with byteindex skip-ahead (Opt #17)
+      # The gate must agree with the parser, or the stitch loop keeps accumulating a row
+      # the parser would have closed and fabricates "Unclosed quoted field" at EOF. Two
+      # parser behaviors must therefore be modeled here exactly:
+      #   - a doubled quote inside a quoted field ("" → ") takes precedence over the
+      #     closing-quote check when another byte follows the pair (parser.rb, issue #334);
+      #   - with allow_escaped_quotes, a quote preceded by an odd number of backslashes is
+      #     escaped → literal, never a closing quote (detect_multiline passes the flag per
+      #     quote_escaping mode; :auto runs both interpretations).
+
+      # Walk the same string the parser parses: parse_line_to_hash_ruby chomps the trailing
+      # row separator (String#chomp — for "\n" that also removes a trailing "\r\n" or "\r")
+      # before parsing. Without this, a terminal doubled quote or a CRLF line ending flips
+      # the pair-precedence / close-quote decisions at end-of-line.
+      line = line.chomp(row_sep) if row_sep.is_a?(String)
+
+      if col_sep.bytesize == 1
+        # Fast path: byte-level scanning with byteindex skip-ahead (Opt #17).
+        # Gated on bytesize, not size: a one-character multi-byte separator (e.g. 'é')
+        # must take the character-level path below — byte scanning would match its
+        # first byte inside other characters sharing that lead byte.
         col_sep_byte     = col_sep.getbyte(0)
         quote_byte       = quote.getbyte(0)
         row_sep_bytesize = row_sep.is_a?(String) ? row_sep.bytesize : 0
@@ -697,7 +738,10 @@ module SmarterCSV
             # Opt #12 mirror: unquoted field in progress — jump to next col_sep using C-level
             # byteindex (MRI Ruby ≥ 3.2). Fallback for older Ruby / JRuby: manual getbyte loop —
             # kept inline for the same reason as the Opt #10 mirror above.
-            next_sep = if byteindex_available
+            # byteindex requires a character-boundary offset — after stepping over the first
+            # byte of a multi-byte character, i is mid-character (a UTF-8 continuation byte),
+            # so use the byte loop there.
+            next_sep = if byteindex_available && (line.getbyte(i) & 0xC0) != 0x80
                          line.byteindex(col_sep, i)
                        else
                          j = i
@@ -716,15 +760,28 @@ module SmarterCSV
             field_started = false
           elsif b == quote_byte
             if in_quotes
-              # closing quote: only valid if followed by col_sep, row_sep, or end of line
-              next_i = i + 1
-              if next_i >= bytesize ||
-                 line.getbyte(next_i) == col_sep_byte ||
-                 (row_sep_bytesize > 0 && line.byteslice(next_i, row_sep_bytesize) == row_sep)
-                in_quotes     = false
-                field_started = true
+              escaped = false
+              if allow_escaped_quotes
+                k = i - 1
+                k -= 1 while k >= 0 && line.getbyte(k) == 0x5C # '\\'
+                escaped = (i - 1 - k).odd?
+              end
+              unless escaped # escaped quote → literal, stays inside the quoted field
+                next_i = i + 1
+                if next_i + 1 < bytesize && line.getbyte(next_i) == quote_byte
+                  # doubled quote ("" → ") with another byte following: consume the pair,
+                  # stay inside the quoted field (precedence over the closing-quote check;
+                  # terminal "" keeps the parser's lenient close — see parse_csv_line_ruby)
+                  i = next_i
+                # closing quote: only valid if followed by col_sep, row_sep, or end of line
+                elsif next_i >= bytesize ||
+                      line.getbyte(next_i) == col_sep_byte ||
+                      (row_sep_bytesize > 0 && line.byteslice(next_i, row_sep_bytesize) == row_sep)
+                  in_quotes     = false
+                  field_started = true
+                end
+                # else: quote inside quoted field → literal
               end
-              # else: quote inside quoted field → literal (handles "" doubling)
             elsif !field_started # at field boundary: open quoted field
               in_quotes     = true
               field_started = true
@@ -754,15 +811,27 @@ module SmarterCSV
 
           if line[i] == quote
             if in_quotes
-              # closing quote: only valid if followed by col_sep, row_sep, or end of line
-              next_i = i + 1
-              if next_i >= line_size ||
-                 line[next_i...next_i + col_sep_size] == col_sep ||
-                 (row_sep_size > 0 && line[next_i...next_i + row_sep_size] == row_sep)
-                in_quotes     = false
-                field_started = true
+              escaped = false
+              if allow_escaped_quotes
+                k = i - 1
+                k -= 1 while k >= 0 && line[k] == '\\'
+                escaped = (i - 1 - k).odd?
+              end
+              unless escaped # escaped quote → literal, stays inside the quoted field
+                next_i = i + 1
+                if next_i + 1 < line_size && line[next_i] == quote
+                  # doubled quote ("" → ") with another character following: consume the
+                  # pair, stay inside the quoted field (see byte path above)
+                  i = next_i
+                # closing quote: only valid if followed by col_sep, row_sep, or end of line
+                elsif next_i >= line_size ||
+                      line[next_i...next_i + col_sep_size] == col_sep ||
+                      (row_sep_size > 0 && line[next_i...next_i + row_sep_size] == row_sep)
+                  in_quotes     = false
+                  field_started = true
+                end
+                # else: quote inside quoted field → literal
               end
-              # else: quote inside quoted field → literal (handles "" doubling)
             elsif !field_started # at field boundary: open quoted field
               in_quotes     = true
               field_started = true
@@ -793,7 +862,9 @@ module SmarterCSV
     def blank?(value)
       case value
       when String
-        value.empty? || BLANK_RE.match?(value)
+        # A string with invalid bytes for its encoding would make the regex raise —
+        # and it necessarily contains non-blank bytes, so it is not blank.
+        value.empty? || (value.valid_encoding? && BLANK_RE.match?(value))
       when NilClass
         true
       when Array
@@ -845,25 +916,18 @@ module SmarterCSV
           hash.delete(nil)
           hash.delete('')
         end
-        hash.delete(:"") if @delete_empty_keys
-
-        # Only these Ruby-only post-filters remain (user-provided Ruby objects):
-        if (matcher = options[:nil_values_matching])
-          if options[:remove_empty_values]
-            hash.delete_if do |_k, v|
-              str_val = v.is_a?(String) ? v : (v.is_a?(Numeric) ? v.to_s : nil)
-              str_val && matcher.match?(str_val)
-            end
-          else
-            hash.each_key do |k|
-              v = hash[k]
-              str_val = v.is_a?(String) ? v : (v.is_a?(Numeric) ? v.to_s : nil)
-              hash[k] = nil if str_val && matcher.match?(str_val)
-            end
-          end
+        if @delete_empty_keys
+          hash.delete(:"")
+          hash.delete('')
         end
 
-        if options[:value_converters]
+        if options[:nil_values_matching]
+          # The C parser deferred numeric conversion and zero-removal (see
+          # defer_value_transforms_to_ruby in the extension), so run the full Ruby
+          # pipeline: nil-matching on the raw strings first, then zero-removal,
+          # numeric conversion, and value_converters — the pure-Ruby-path order.
+          hash = hash_transformations(hash, options)
+        elsif options[:value_converters]
           options[:value_converters].each do |key, converter|
             hash[key] = converter.respond_to?(:convert) ? converter.convert(hash[key]) : converter.call(hash[key]) if hash.key?(key)
           end

data/lib/smarter_csv/reader_options.rb

@@ -129,20 +129,53 @@ module SmarterCSV
           warn "DEPRECATION WARNING: 'except_headers:' is deprecated. Use 'headers: { except: [...] }' instead." unless @options[:verbose] == :quiet
         end
 
-        # Normalize only_headers/except_headers to arrays of symbols (internal names, read by C extension)
+        # Normalize only_headers/except_headers to arrays of the row-key type (internal names,
+        # read by the C extension too): with strings_as_keys / keep_original_headers the row
+        # keys are Strings, otherwise Symbols — the selectors must match to select anything.
+        string_keys = @options[:strings_as_keys] || @options[:keep_original_headers]
         if @options[:only_headers]
           values = Array(@options[:only_headers])
           bad = values.reject { |v| v.is_a?(Symbol) || v.is_a?(String) }
           raise SmarterCSV::ValidationError, "headers: { only: } elements must be String or Symbol, got: #{bad.map(&:class).uniq.inspect}" if bad.any?
 
-          @options[:only_headers] = values.map(&:to_sym)
+          @options[:only_headers] = string_keys ? values.map(&:to_s) : values.map(&:to_sym)
         end
         if @options[:except_headers]
           values = Array(@options[:except_headers])
           bad = values.reject { |v| v.is_a?(Symbol) || v.is_a?(String) }
           raise SmarterCSV::ValidationError, "headers: { except: } elements must be String or Symbol, got: #{bad.map(&:class).uniq.inspect}" if bad.any?
 
-          @options[:except_headers] = values.map(&:to_sym)
+          @options[:except_headers] = string_keys ? values.map(&:to_s) : values.map(&:to_sym)
+        end
+
+        # The Hash form of convert_values_to_numeric accepts EXACTLY ONE of only:/except:,
+        # with field name(s) (String/Symbol or an Array of them) as the value. Anything else
+        # is an invalid declaration → ValidationError, instead of silently picking a behavior
+        # (the C and Ruby paths used to disagree on these shapes). Values are normalized to
+        # the row-key type, like headers: { only: } above.
+        if (cvn = @options[:convert_values_to_numeric]).is_a?(Hash)
+          unless (cvn.keys - %i[only except]).empty?
+            raise SmarterCSV::ValidationError, "convert_values_to_numeric: only the keys only:/except: are accepted, got: #{cvn.keys.inspect}"
+          end
+          if cvn.key?(:only) && cvn.key?(:except)
+            raise SmarterCSV::ValidationError, "convert_values_to_numeric: cannot use only: and except: at the same time"
+          end
+          unless cvn.key?(:only) || cvn.key?(:except)
+            raise SmarterCSV::ValidationError, "convert_values_to_numeric: the Hash form requires only: or except: with field name(s)"
+          end
+
+          list_key = cvn.key?(:only) ? :only : :except
+          values = cvn[list_key].is_a?(Array) ? cvn[list_key] : [cvn[list_key]]
+          if values.empty?
+            raise SmarterCSV::ValidationError, "convert_values_to_numeric: #{list_key}: must not be empty — expects field name(s)"
+          end
+
+          bad = values.reject { |v| v.is_a?(Symbol) || v.is_a?(String) }
+          unless bad.empty?
+            raise SmarterCSV::ValidationError, "convert_values_to_numeric: #{list_key}: expects field name(s) (String or Symbol), got: #{bad.map(&:class).uniq.inspect}"
+          end
+
+          @options[:convert_values_to_numeric] = { list_key => string_keys ? values.map(&:to_s) : values.map(&:to_sym) }
         end
 
         # Deprecation: remove_values_matching → nil_values_matching
@@ -195,7 +228,10 @@ module SmarterCSV
         errors = []
         errors << "invalid row_sep" if keys.include?(:row_sep) && !option_valid?(options[:row_sep])
         errors << "invalid col_sep" if keys.include?(:col_sep) && !option_valid?(options[:col_sep])
-        errors << "invalid quote_char" if keys.include?(:quote_char) && !option_valid?(options[:quote_char])
+        # quote_char has no auto-detection — :auto is only valid for row_sep and col_sep
+        if keys.include?(:quote_char) && !(options[:quote_char].is_a?(String) && !options[:quote_char].empty?)
+          errors << "invalid quote_char"
+        end
         if keys.include?(:quote_char) && options[:quote_char].is_a?(String) && options[:quote_char].bytesize > 1
           errors << "invalid quote_char: must be a single byte (got #{options[:quote_char].inspect})"
         end
@@ -261,9 +297,11 @@ module SmarterCSV
           warn "WARNING: buffer_size (#{options[:buffer_size]}) < auto_row_sep_chars (#{arc}); bumping buffer_size to #{bumped}" unless quiet
           options[:buffer_size] = bumped
         end
+        # field_size_limit is overrun protection (a hard upper bound against runaway fields),
+        # not per-field validation — small values make no sense and are rejected.
         fsl = options[:field_size_limit]
-        unless fsl.nil? || (fsl.is_a?(Integer) && fsl > 0)
-          errors << "invalid field_size_limit: must be nil or a positive Integer (got #{fsl.inspect})"
+        unless fsl.nil? || (fsl.is_a?(Integer) && fsl >= 4096)
+          errors << "invalid field_size_limit: must be nil or an Integer >= 4096 (got #{fsl.inspect})"
         end
         obr = options[:on_bad_row]
         unless %i[raise skip collect].include?(obr) || obr.respond_to?(:call)

data/lib/smarter_csv/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SmarterCSV
-  VERSION = "1.18.0"
+  VERSION = "1.19.0"
 end

data/lib/smarter_csv/writer.rb

@@ -204,12 +204,12 @@ module SmarterCSV
       str = field.to_s
       return str if @disable_auto_quoting && !force_quotes
 
-      # double-quote fields if we force that, or if the field contains the comma, new-line, or quote character
+      # quote fields if we force that, or if the field contains the col_sep, row_sep, or quote_char
       contains_special_char = str.match(@quote_regex)
       if force_quotes || contains_special_char
-        str = str.gsub(@quote_char, @escaped_quote_char) if contains_special_char # escape double-quote
+        str = str.gsub(@quote_char, @escaped_quote_char) if contains_special_char # escape the quote_char
 
-        "\"#{str}\""
+        "#{@quote_char}#{str}#{@quote_char}"
       else
         str
       end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: smarter_csv
 version: !ruby/object:Gem::Version
-  version: 1.18.0
+  version: 1.19.0
 platform: ruby
 authors:
 - Tilo Sloboda
 bindir: bin
 cert_chain: []
-date: 2026-06-19 00:00:00.000000000 Z
+date: 2026-08-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bigdecimal
@@ -83,6 +83,7 @@ files:
 - docs/upgrade_wizard.html
 - docs/value_converters.md
 - docs/warnings.md
+- ext/smarter_csv/cpu_flags.rb
 - ext/smarter_csv/extconf.rb
 - ext/smarter_csv/smarter_csv.c
 - ext/smarter_csv/vendor/LICENSE-fast_float-MIT
@@ -140,7 +141,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.9
+rubygems_version: 4.0.17
 specification_version: 4
 summary: Fastest end-to-end CSV ingestion for Ruby with smart defaults and Rails-ready
   hash output