smarter_csv 1.16.4 → 1.17.0

data/images/SmarterCSV_1.17.0_vs_previous_Rb-speedup.svg

@@ -0,0 +1,179 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="820" height="668"
+     font-family="ui-monospace, 'Cascadia Code', 'Courier New', monospace" font-size="12">
+<rect width="820" height="668" fill="#ffffff"/>
+<text x="410" y="18" text-anchor="middle" font-size="13" font-weight="bold" fill="#212121">SmarterCSV improvements 1.15.2, 1.16.4, 1.17.0 vs 1.14.4 — Ruby (not accelerated)</text>
+<text x="410" y="32" text-anchor="middle" font-size="10" fill="#9e9e9e">Speedup ratio = baseline version time ÷ newer version time  (higher = newer version is faster)</text>
+<text x="410" y="48" text-anchor="middle" font-size="11" fill="#616161">Ruby 3.4.7  [log scale, best of 40]</text>
+<line x1="220" y1="68" x2="220" y2="580" stroke="#e0e0e0" stroke-width="1"/>
+<text x="220" y="64" text-anchor="middle" font-size="11" fill="#757575">1×</text>
+<line x1="323" y1="68" x2="323" y2="580" stroke="#e0e0e0" stroke-width="1"/>
+<text x="323" y="64" text-anchor="middle" font-size="11" fill="#757575">2×</text>
+<line x1="459" y1="68" x2="459" y2="580" stroke="#e0e0e0" stroke-width="1"/>
+<text x="459" y="64" text-anchor="middle" font-size="11" fill="#757575">5×</text>
+<line x1="561" y1="68" x2="561" y2="580" stroke="#e0e0e0" stroke-width="1"/>
+<text x="561" y="64" text-anchor="middle" font-size="11" fill="#757575">10×</text>
+<line x1="664" y1="68" x2="664" y2="580" stroke="#e0e0e0" stroke-width="1"/>
+<text x="664" y="64" text-anchor="middle" font-size="11" fill="#757575">20×</text>
+<line x1="800" y1="68" x2="800" y2="580" stroke="#e0e0e0" stroke-width="1"/>
+<text x="800" y="64" text-anchor="middle" font-size="11" fill="#757575">50×</text>
+<line x1="220" y1="68" x2="220" y2="580" stroke="#9e9e9e" stroke-width="1.5"/>
+<line x1="220" y1="68" x2="800" y2="68" stroke="#bdbdbd" stroke-width="1"/>
+<rect x="0" y="86" width="820" height="26" fill="#f5f5f5"/>
+<text x="212" y="103" text-anchor="end" font-size="11" fill="#424242">PEOPLE_IMPORT_C</text>
+<rect x="556" y="94" width="10" height="10" fill="#1565C0"/>
+<text x="518" y="103" font-size="10" fill="#1565C0">10.0×</text>
+<rect x="652" y="94" width="10" height="10" fill="#BF360C"/>
+<rect x="658" y="94" width="10" height="10" fill="#2E7D32"/>
+<text x="671" y="103" font-size="10" fill="#BF360C">19×</text>
+<text x="698" y="103" font-size="10" fill="#2E7D32">20×</text>
+<rect x="0" y="112" width="820" height="26" fill="#ffffff"/>
+<text x="212" y="129" text-anchor="end" font-size="11" fill="#424242">PEOPLE_IMPORT_NC</text>
+<rect x="506" y="120" width="10" height="10" fill="#1565C0"/>
+<text x="475" y="129" font-size="10" fill="#1565C0">7.1×</text>
+<rect x="570" y="120" width="10" height="10" fill="#BF360C"/>
+<rect x="586" y="120" width="10" height="10" fill="#2E7D32"/>
+<text x="599" y="129" font-size="10" fill="#BF360C">11×</text>
+<text x="626" y="129" font-size="10" fill="#2E7D32">12×</text>
+<rect x="0" y="138" width="820" height="26" fill="#f5f5f5"/>
+<text x="212" y="155" text-anchor="end" font-size="11" fill="#424242">whitespace_heavy_60k</text>
+<rect x="538" y="146" width="10" height="10" fill="#1565C0"/>
+<rect x="543" y="146" width="10" height="10" fill="#BF360C"/>
+<rect x="572" y="146" width="10" height="10" fill="#2E7D32"/>
+<text x="507" y="155" font-size="10" fill="#1565C0">8.8×</text>
+<text x="585" y="155" font-size="10" fill="#BF360C">9.1×</text>
+<text x="619" y="155" font-size="10" fill="#2E7D32">11×</text>
+<rect x="0" y="164" width="820" height="26" fill="#ffffff"/>
+<text x="212" y="181" text-anchor="end" font-size="11" fill="#424242">PEOPLE_IMPORT_NB</text>
+<rect x="525" y="172" width="10" height="10" fill="#1565C0"/>
+<rect x="544" y="172" width="10" height="10" fill="#BF360C"/>
+<rect x="563" y="172" width="10" height="10" fill="#2E7D32"/>
+<text x="494" y="181" font-size="10" fill="#1565C0">8.1×</text>
+<text x="576" y="181" font-size="10" fill="#BF360C">9.2×</text>
+<text x="610" y="181" font-size="10" fill="#2E7D32">10×</text>
+<rect x="0" y="190" width="820" height="26" fill="#f5f5f5"/>
+<text x="212" y="207" text-anchor="end" font-size="11" fill="#424242">PEOPLE_IMPORT_B</text>
+<rect x="525" y="198" width="10" height="10" fill="#1565C0"/>
+<rect x="538" y="198" width="10" height="10" fill="#BF360C"/>
+<rect x="550" y="198" width="10" height="10" fill="#2E7D32"/>
+<text x="494" y="207" font-size="10" fill="#1565C0">8.1×</text>
+<text x="563" y="207" font-size="10" fill="#BF360C">8.8×</text>
+<text x="597" y="207" font-size="10" fill="#2E7D32">9.6×</text>
+<rect x="0" y="216" width="820" height="26" fill="#ffffff"/>
+<text x="212" y="233" text-anchor="end" font-size="11" fill="#424242">tab_separated_60k</text>
+<rect x="504" y="224" width="10" height="10" fill="#1565C0"/>
+<rect x="509" y="224" width="10" height="10" fill="#BF360C"/>
+<rect x="539" y="224" width="10" height="10" fill="#2E7D32"/>
+<text x="473" y="233" font-size="10" fill="#1565C0">7.0×</text>
+<text x="552" y="233" font-size="10" fill="#BF360C">7.3×</text>
+<text x="586" y="233" font-size="10" fill="#2E7D32">8.9×</text>
+<rect x="0" y="242" width="820" height="26" fill="#f5f5f5"/>
+<text x="212" y="259" text-anchor="end" font-size="11" fill="#424242">multi_char_separator_60k</text>
+<rect x="466" y="250" width="10" height="10" fill="#1565C0"/>
+<text x="435" y="259" font-size="10" fill="#1565C0">5.4×</text>
+<rect x="503" y="250" width="10" height="10" fill="#BF360C"/>
+<rect x="529" y="250" width="10" height="10" fill="#2E7D32"/>
+<text x="542" y="259" font-size="10" fill="#BF360C">7.0×</text>
+<text x="576" y="259" font-size="10" fill="#2E7D32">8.3×</text>
+<rect x="0" y="268" width="820" height="26" fill="#ffffff"/>
+<text x="212" y="285" text-anchor="end" font-size="11" fill="#424242">wide_500_cols_20k</text>
+<rect x="483" y="276" width="10" height="10" fill="#1565C0"/>
+<rect x="483" y="276" width="10" height="10" fill="#BF360C"/>
+<text x="452" y="285" font-size="10" fill="#1565C0">6.1×</text>
+<text x="496" y="285" font-size="10" fill="#BF360C">6.1×</text>
+<rect x="516" y="276" width="10" height="10" fill="#2E7D32"/>
+<text x="529" y="285" font-size="10" fill="#2E7D32">7.6×</text>
+<rect x="0" y="294" width="820" height="26" fill="#f5f5f5"/>
+<text x="212" y="311" text-anchor="end" font-size="11" fill="#424242">utf8_multibyte_60k</text>
+<rect x="475" y="302" width="10" height="10" fill="#1565C0"/>
+<rect x="483" y="302" width="10" height="10" fill="#BF360C"/>
+<rect x="502" y="302" width="10" height="10" fill="#2E7D32"/>
+<text x="444" y="311" font-size="10" fill="#1565C0">5.8×</text>
+<text x="515" y="311" font-size="10" fill="#BF360C">6.1×</text>
+<text x="549" y="311" font-size="10" fill="#2E7D32">6.9×</text>
+<rect x="0" y="320" width="820" height="26" fill="#ffffff"/>
+<text x="212" y="337" text-anchor="end" font-size="11" fill="#424242">many_empty_fields_60k</text>
+<rect x="393" y="328" width="10" height="10" fill="#1565C0"/>
+<text x="362" y="337" font-size="10" fill="#1565C0">3.3×</text>
+<rect x="454" y="328" width="10" height="10" fill="#BF360C"/>
+<rect x="474" y="328" width="10" height="10" fill="#2E7D32"/>
+<text x="487" y="337" font-size="10" fill="#BF360C">5.0×</text>
+<text x="521" y="337" font-size="10" fill="#2E7D32">5.7×</text>
+<rect x="0" y="346" width="820" height="26" fill="#f5f5f5"/>
+<text x="212" y="363" text-anchor="end" font-size="11" fill="#424242">sample_100k</text>
+<rect x="434" y="354" width="10" height="10" fill="#1565C0"/>
+<rect x="439" y="354" width="10" height="10" fill="#BF360C"/>
+<rect x="456" y="354" width="10" height="10" fill="#2E7D32"/>
+<text x="403" y="363" font-size="10" fill="#1565C0">4.4×</text>
+<text x="469" y="363" font-size="10" fill="#BF360C">4.5×</text>
+<text x="503" y="363" font-size="10" fill="#2E7D32">5.1×</text>
+<rect x="0" y="372" width="820" height="26" fill="#ffffff"/>
+<text x="212" y="389" text-anchor="end" font-size="11" fill="#424242">sensor_data_50krows_50cols</text>
+<rect x="454" y="380" width="10" height="10" fill="#1565C0"/>
+<rect x="455" y="380" width="10" height="10" fill="#2E7D32"/>
+<rect x="456" y="380" width="10" height="10" fill="#BF360C"/>
+<text x="423" y="389" font-size="10" fill="#1565C0">5.0×</text>
+<text x="469" y="389" font-size="10" fill="#2E7D32">5.1×</text>
+<text x="503" y="389" font-size="10" fill="#BF360C">5.1×</text>
+<rect x="0" y="398" width="820" height="26" fill="#f5f5f5"/>
+<text x="212" y="415" text-anchor="end" font-size="11" fill="#424242">long_fields_40k</text>
+<rect x="301" y="406" width="10" height="10" fill="#1565C0"/>
+<text x="270" y="415" font-size="10" fill="#1565C0">1.8×</text>
+<rect x="416" y="406" width="10" height="10" fill="#BF360C"/>
+<rect x="416" y="406" width="10" height="10" fill="#2E7D32"/>
+<text x="429" y="415" font-size="10" fill="#BF360C">3.9×</text>
+<text x="463" y="415" font-size="10" fill="#2E7D32">3.9×</text>
+<rect x="0" y="424" width="820" height="26" fill="#ffffff"/>
+<text x="212" y="441" text-anchor="end" font-size="11" fill="#424242">heavy_quoting_60k</text>
+<rect x="268" y="432" width="10" height="10" fill="#1565C0"/>
+<text x="237" y="441" font-size="10" fill="#1565C0">1.4×</text>
+<rect x="371" y="432" width="10" height="10" fill="#BF360C"/>
+<rect x="381" y="432" width="10" height="10" fill="#2E7D32"/>
+<text x="394" y="441" font-size="10" fill="#BF360C">2.9×</text>
+<text x="428" y="441" font-size="10" fill="#2E7D32">3.1×</text>
+<rect x="0" y="450" width="820" height="26" fill="#f5f5f5"/>
+<text x="212" y="467" text-anchor="end" font-size="11" fill="#424242">embedded_separators_60k</text>
+<rect x="277" y="458" width="10" height="10" fill="#1565C0"/>
+<text x="246" y="467" font-size="10" fill="#1565C0">1.5×</text>
+<rect x="364" y="458" width="10" height="10" fill="#BF360C"/>
+<rect x="374" y="458" width="10" height="10" fill="#2E7D32"/>
+<text x="387" y="467" font-size="10" fill="#BF360C">2.7×</text>
+<text x="421" y="467" font-size="10" fill="#2E7D32">2.9×</text>
+<rect x="0" y="476" width="820" height="26" fill="#ffffff"/>
+<text x="212" y="493" text-anchor="end" font-size="11" fill="#424242">worldcities</text>
+<rect x="270" y="484" width="10" height="10" fill="#1565C0"/>
+<text x="239" y="493" font-size="10" fill="#1565C0">1.4×</text>
+<rect x="366" y="484" width="10" height="10" fill="#BF360C"/>
+<rect x="372" y="484" width="10" height="10" fill="#2E7D32"/>
+<text x="385" y="493" font-size="10" fill="#BF360C">2.8×</text>
+<text x="419" y="493" font-size="10" fill="#2E7D32">2.9×</text>
+<rect x="0" y="502" width="820" height="26" fill="#f5f5f5"/>
+<text x="212" y="519" text-anchor="end" font-size="11" fill="#424242">uscities</text>
+<rect x="269" y="510" width="10" height="10" fill="#1565C0"/>
+<text x="238" y="519" font-size="10" fill="#1565C0">1.4×</text>
+<rect x="359" y="510" width="10" height="10" fill="#BF360C"/>
+<rect x="365" y="510" width="10" height="10" fill="#2E7D32"/>
+<text x="378" y="519" font-size="10" fill="#BF360C">2.6×</text>
+<text x="412" y="519" font-size="10" fill="#2E7D32">2.8×</text>
+<rect x="0" y="528" width="820" height="26" fill="#ffffff"/>
+<text x="212" y="545" text-anchor="end" font-size="11" fill="#424242">uszips</text>
+<rect x="273" y="536" width="10" height="10" fill="#1565C0"/>
+<text x="242" y="545" font-size="10" fill="#1565C0">1.5×</text>
+<rect x="356" y="536" width="10" height="10" fill="#BF360C"/>
+<rect x="361" y="536" width="10" height="10" fill="#2E7D32"/>
+<text x="374" y="545" font-size="10" fill="#BF360C">2.6×</text>
+<text x="408" y="545" font-size="10" fill="#2E7D32">2.7×</text>
+<rect x="0" y="554" width="820" height="26" fill="#f5f5f5"/>
+<text x="212" y="571" text-anchor="end" font-size="11" fill="#424242">embedded_newlines_60k</text>
+<rect x="315" y="562" width="10" height="10" fill="#1565C0"/>
+<rect x="341" y="562" width="10" height="10" fill="#BF360C"/>
+<rect x="346" y="562" width="10" height="10" fill="#2E7D32"/>
+<text x="284" y="571" font-size="10" fill="#1565C0">2.0×</text>
+<text x="359" y="571" font-size="10" fill="#BF360C">2.3×</text>
+<text x="393" y="571" font-size="10" fill="#2E7D32">2.4×</text>
+<rect x="223" y="589" width="10" height="10" fill="#1565C0"/>
+<text x="240" y="598" font-size="11" fill="#1565C0">Ruby path (v1.15.2)</text>
+<rect x="223" y="609" width="10" height="10" fill="#BF360C"/>
+<text x="240" y="618" font-size="11" fill="#BF360C">Ruby path (v1.16.4)</text>
+<rect x="223" y="629" width="10" height="10" fill="#2E7D32"/>
+<text x="240" y="638" font-size="11" fill="#2E7D32">Ruby path (v1.17.0)</text>
+</svg>

data/lib/smarter_csv/auto_detection.rb

@@ -8,8 +8,6 @@ module SmarterCSV
     # Otherwise guesses column separator from contents.
     # Raises exception if none is found.
     def guess_column_separator(filehandle, options)
-      skip_lines(filehandle, options)
-
       delimiters = [',', "\t", ';', ':', '|']
 
       line = nil
@@ -29,11 +27,13 @@ module SmarterCSV
           candidates[d] += non_quoted_text.scan(d).count
         end
       end
-      rewind(filehandle)
+      # No lines were read at all — empty file or stream.
+      # Return a safe default and let process_headers raise EmptyFileError.
+      return ',' if line.nil?
 
       if candidates.values.max == 0
         # if the header only contains word characters and whitespace, assume comma separator
-        return ',' if line && line.chomp(options[:row_sep]) =~ /^[\w\s]+$/
+        return ',' if line.chomp(options[:row_sep]) =~ /^[\w\s]+$/
 
         raise SmarterCSV::NoColSepDetected
       end
@@ -41,38 +41,223 @@ module SmarterCSV
       candidates.key(candidates.values.max)
     end
 
-    # limitation: this currently reads the whole file in before making a decision
+    # Lower bound on auto_row_sep_chars. Below this the initial scan would be
+    # too small to reliably catch a row separator on the very first read on
+    # most real-world CSV files. Most well-formed CSVs reveal a clear majority
+    # within ~200 bytes; 512 gives comfortable headroom while keeping cheap
+    # files cheap.
+    MIN_AUTO_ROW_SEP_CHARS = 512
+
+    # Default auto_row_sep_chars. Sized to cover wide-header CSVs (e.g. 100+
+    # columns) in a single read so escalation rarely fires. 4096 also matches
+    # a typical filesystem block, so the OS-level read cost is the same as a
+    # smaller request.
+    DEFAULT_AUTO_ROW_SEP_CHARS = 4096
+
+    # Upper bound on auto_row_sep_chars. Serves three roles:
+    #   1. The hard cap on the user-facing `auto_row_sep_chars` option.
+    #   2. The cap on the doubling escalation inside guess_line_ending —
+    #      a single chunk read never exceeds this.
+    #   3. The hard cap on total bytes scanned during auto-detection
+    #      (`break if buf.bytesize >= MAX_AUTO_ROW_SEP_CHARS`).
+    # All three roles use the same value because beyond this point further
+    # scanning would not improve detection accuracy and only delays parsing.
+    MAX_AUTO_ROW_SEP_CHARS = 65_536
+
+    # Guess the row separator ("\n", "\r\n", or "\r") by counting occurrences
+    # outside of quoted regions, scanning each chunk only once and accumulating
+    # counts across iterations.
+    #
+    # Reads one chunk of options[:auto_row_sep_chars] bytes at a time and
+    # grows up to MAX_AUTO_ROW_SEP_CHARS bytes total while no candidate has
+    # a clear majority (count > sum of the others).
+    #
+    # State carried across iterations:
+    #   * crlf, lf, cr — running counts; never reset
+    #   * in_quote    — true if a previous chunk ended inside a quoted region
+    #   * pending_cr  — true if a previous chunk's last byte was a lone "\r"
+    #                   (deferred so it can pair with a leading "\n" of the
+    #                   next chunk without an extra read).
+    #
+    # Falls back to "\n" (and emits a warning unless verbose: :quiet) when:
+    #   * no known separator is found within MAX_AUTO_ROW_SEP_CHARS bytes — e.g. a file
+    #     that uses an exotic separator like "\u2028"; or
+    #   * a tie between candidates persists across MAX_AUTO_ROW_SEP_CHARS bytes.
+    #
+    # The fallback preserves 14 years of permissive behavior; the warning lets
+    # infrastructure code (logs, captured stderr) surface the ambiguity.
     def guess_line_ending(filehandle, options)
-      counts = {"\n" => 0, "\r" => 0, "\r\n" => 0}
-      quoted_char = false
-
-      # count how many of the pre-defined line-endings we find
-      # ignoring those contained within quote characters
-      last_char = nil
-      lines = 0
-      filehandle.each_char do |c|
-        quoted_char = !quoted_char if c == options[:quote_char]
-        next if quoted_char
-
-        if last_char == "\r"
-          if c == "\n"
-            counts["\r\n"] += 1
+      q = Regexp.escape(options[:quote_char])
+      # Combined regex: matches complete "..." pairs AND unclosed "...\z (open
+      # quote followed by content to end of string). One gsub pass strips both
+      # cases; quote count parity tells us whether an unclosed open existed.
+      # /n flag: byte-level matching, encoding-agnostic.
+      quoted_re = /#{q}[^#{q}]*(?:#{q}|\z)/n
+      quote_str = options[:quote_char].b
+      # Adaptive doubling: the first read is auto_row_sep_chars bytes (default 4096).
+      # Iter 2 reuses the same size so files with a clear separator slightly past
+      # the initial chunk resolve cheaply; iter 3+ doubles each iteration up to
+      # MAX_AUTO_ROW_SEP_CHARS.
+      #
+      # Read pattern with default auto_row_sep_chars = 4096:
+      #   iter | chunk | cumulative
+      #     1  |  4096 |   4096
+      #     2  |  4096 |   8192
+      #     3  |  8192 |  16384
+      #     4  | 16384 |  32768
+      #     5  | 32768 |  65536  (loop ends at MAX_AUTO_ROW_SEP_CHARS)
+      #
+      # MIN_AUTO_ROW_SEP_CHARS is the defensive floor — catches direct callers
+      # that bypass option validation (e.g. tests calling via send). Through the
+      # public process_options pipeline, validation already enforces this floor,
+      # so this .max is inert in normal use.
+      chunk_size = [options[:auto_row_sep_chars].to_i, MIN_AUTO_ROW_SEP_CHARS].max
+      chunk_size = [chunk_size, MAX_AUTO_ROW_SEP_CHARS].min # Defensive cap: do not read beyond MAX
+      bytes_read = false
+      total_bytes = 0
+      crlf = lf = cr = 0
+      in_quote = false       # carries across chunks; an open quote with no close
+      pending_cr = false     # carries across chunks; "\r" deferred for "\r\n" pairing
+      iter = 0
+
+      loop do
+        part = filehandle.read(chunk_size)
+        break if part.nil? || part.empty?
+
+        bytes_read = true
+        total_bytes += part.bytesize
+
+        # Resolve a "\r" left pending from the previous chunk's last byte.
+        # If the new chunk starts with "\n", the pair is "\r\n"; otherwise
+        # the deferred "\r" was a lone "\r" and the new first byte is
+        # processed below. (pending_cr and in_quote can never both be true
+        # at the start of an iteration — see the open-quote handling below.)
+        if pending_cr
+          pending_cr = false
+          if part.getbyte(0) == 0x0A # \n
+            crlf += 1
+            part = part.byteslice(1, part.bytesize - 1)
           else
-            counts["\r"] += 1 # \r are counted after they appeared
+            cr += 1
+            # part stays as-is; the new first byte is processed below.
           end
-        elsif c == "\n"
-          counts["\n"] += 1
         end
-        last_char = c
-        lines += 1
-        break if options[:auto_row_sep_chars] && options[:auto_row_sep_chars] > 0 && lines >= options[:auto_row_sep_chars]
+
+        # Fast path: chunk has no quote char AND we're not carrying an open
+        # quote from a previous chunk. Skip the gsub + index + .b machinery
+        # and count separators directly — most CSV chunks contain no quote
+        # chars. (`include?` is one C-level byte scan, vs gsub + index = two
+        # passes plus a string copy.)
+        if !in_quote && !part.include?(quote_str)
+          unquoted = part
+          if unquoted.end_with?("\r")
+            pending_cr = true
+            unquoted = unquoted.byteslice(0, unquoted.bytesize - 1)
+          end
+          delta_crlf = unquoted.scan("\r\n").size
+          delta_lf   = unquoted.count("\n") - delta_crlf
+          delta_cr   = unquoted.count("\r") - delta_crlf
+          crlf += delta_crlf
+          lf   += delta_lf
+          cr   += delta_cr
+        else
+          # Slow path: chunk contains quote chars or we're carrying in_quote
+          # state from a previous chunk. Convert to binary so index/byteslice
+          # are byte-level (safe even with multibyte UTF-8 content before the
+          # quote position).
+          part = part.b
+
+          if in_quote
+            close_idx = part.index(quote_str)
+            if close_idx
+              in_quote = false
+              part = part.byteslice(close_idx + 1, part.bytesize - close_idx - 1)
+            else
+              # Whole chunk is still inside the quote.
+              part = nil
+            end
+          end
+
+          if part && !part.empty?
+            # Single regex pass: gsub with the combined regex strips every
+            # complete "..." pair AND, if there's an unclosed open quote at
+            # the end, strips "...\z too. After this, no quote chars remain
+            # in `unquoted`.
+            unquoted = part.gsub(quoted_re, '')
+
+            # Parity check on the original chunk's quote count: an odd count
+            # means an unclosed open quote existed (and the gsub stripped its
+            # content along with the open). Set in_quote so the next chunk
+            # will look for the close. (count is a fast C-level byte scan.)
+            in_quote = true if part.count(quote_str).odd?
+
+            if unquoted.end_with?("\r".b)
+              if in_quote
+                # The byte right after this trailing "\r" was the open quote
+                # char (NOT "\n"), so the "\r" is a lone cr — count it now.
+                # Deferring would mispair against the next chunk's first
+                # byte, which is inside the (now-open) quoted region.
+                cr += 1
+              else
+                # No open quote — safe to defer trailing "\r" so it can pair
+                # with the next chunk's leading "\n" if any.
+                pending_cr = true
+              end
+              unquoted = unquoted.byteslice(0, unquoted.bytesize - 1)
+            end
+
+            # Count separators in the new bytes and add to running totals.
+            delta_crlf = unquoted.scan("\r\n".b).size
+            delta_lf   = unquoted.count("\n") - delta_crlf
+            delta_cr   = unquoted.count("\r") - delta_crlf
+            crlf += delta_crlf
+            lf   += delta_lf
+            cr   += delta_cr
+          end
+        end
+
+        # Clear majority: winner strictly greater than the sum of the others.
+        return "\r\n" if crlf > lf + cr
+        return "\n"   if lf   > crlf + cr
+        return "\r"   if cr   > crlf + lf
+
+        break if total_bytes >= MAX_AUTO_ROW_SEP_CHARS
+
+        # Iter 2 keeps the iter-1 chunk size; iter 3+ doubles each iteration,
+        # capped at MAX_AUTO_ROW_SEP_CHARS.
+        iter += 1
+        chunk_size = [chunk_size * 2, MAX_AUTO_ROW_SEP_CHARS].min if iter >= 2
       end
-      rewind(filehandle)
 
-      counts["\r"] += 1 if last_char == "\r"
-      # find the most frequent key/value pair:
-      most_frequent_key, _count = counts.max_by{|_, v| v}
-      most_frequent_key
+      # Empty stream — return harmless fallback; downstream raises EmptyFileError.
+      return "\n" unless bytes_read
+
+      # If we exited with a deferred "\r" (EOF or cap reached and no following
+      # byte to pair it with), count it as a lone "\r" now and re-check majority.
+      # Without this, a file ending in a lone "\r" with no other separators would
+      # fall through to the no-clear-row-sep warning instead of returning "\r".
+      if pending_cr
+        cr += 1
+        return "\r\n" if crlf > lf + cr
+        return "\n"   if lf   > crlf + cr
+        return "\r"   if cr   > crlf + lf
+      end
+
+      unless options[:verbose] == :quiet
+        if crlf == 0 && lf == 0 && cr == 0
+          record_warning(type: :row_sep, code: :no_row_sep_found, severity: :error) do
+            "no row separator found in first #{total_bytes} bytes; " \
+            "defaulting to \"\\n\". Pass row_sep: explicitly if this is wrong."
+          end
+        else
+          record_warning(type: :row_sep, code: :no_clear_row_sep, severity: :error) do
+            "no clear row separator in first #{total_bytes} bytes " \
+            "(saw #{lf}×\"\\n\", #{crlf}×\"\\r\\n\", #{cr}×\"\\r\"); defaulting to \"\\n\". " \
+            "Pass row_sep: explicitly if this is wrong."
+          end
+        end
+      end
+      "\n"
     end
   end
 end

data/lib/smarter_csv/file_io.rb

@@ -20,10 +20,10 @@ module SmarterCSV
       end
     end
 
-    def rewind(filehandle)
+    def rewind_buffer(filehandle)
       @file_line_count = 0
       @csv_line_count = 0
-      filehandle.rewind
+      filehandle.rewind_buffer # this is PeekableIO.rewind_buffer, not io.rewind !
     end
 
     private

data/lib/smarter_csv/hash_transformations.rb

@@ -3,9 +3,17 @@
 module SmarterCSV
   module HashTransformations
     # Frozen regex constants for performance (avoid recompilation on every value)
-    FLOAT_REGEX = /\A[+-]?\d+\.\d+\z/.freeze
-    INTEGER_REGEX = /\A[+-]?\d+\z/.freeze
-    ZERO_REGEX = /\A0+(?:\.0+)?\z/.freeze
+    NUMERIC_REGEX = /\A[+-]?\d+(?:\.\d+)?\z/.freeze
+    # FLOAT_REGEX = /\A[+-]?\d+\.\d+\z/.freeze
+    # INTEGER_REGEX = /\A[+-]?\d+\z/.freeze
+    ZERO_REGEX = /\A[+-]?0+(?:\.0+)?\z/.freeze # could be +0.0
+
+    # First-byte values that can begin a numeric literal — used to skip the numeric
+    # regexes for values that obviously aren't numbers (e.g. city names).
+    ZERO_BYTE  = '0'.ord # 48
+    NINE_BYTE  = '9'.ord # 57
+    PLUS_BYTE  = '+'.ord # 43
+    MINUS_BYTE = '-'.ord # 45
 
     def hash_transformations(hash, options)
       # Modify hash in-place for performance (avoids allocating a second hash per row)
@@ -24,7 +32,11 @@ module SmarterCSV
       # Early return if no transformations needed
       return hash unless remove_empty_values || remove_zero_values || nil_values_matching || convert_to_numeric || value_converters
 
-      keys_to_delete = []
+      # {only:}/{except:} limits on numeric conversion apply only when the option is a Hash;
+      # in the common case (true/false) skip the per-key check entirely.
+      numeric_has_limits = convert_to_numeric.is_a?(Hash)
+      rails = has_rails
+      keys_to_delete = nil # lazily allocated only if something is actually removed
 
       hash.each do |k, v|
         # Nil-ify values matching the pattern (keeps the key; remove_empty_values handles deletion)
@@ -39,23 +51,27 @@ module SmarterCSV
 
         # Check if this key/value should be removed
         # Note: numeric values (Integer/Float) are never blank, so skip the blank check for them
-        if remove_empty_values && !v.is_a?(Numeric) && (has_rails ? v.blank? : blank?(v))
-          keys_to_delete << k
+        if remove_empty_values && !v.is_a?(Numeric) && (rails ? v.blank? : blank?(v))
+          (keys_to_delete ||= []) << k
           next
         end
 
         # Handle both string zeros ("0", "0.0") and numeric zeros (already converted by C)
         if remove_zero_values && ((v.is_a?(String) && ZERO_REGEX.match?(v)) || (v.is_a?(Numeric) && v == 0))
-          keys_to_delete << k
+          (keys_to_delete ||= []) << k
           next
         end
 
         # Convert to numeric if requested
-        if convert_to_numeric && v.is_a?(String) && !limit_execution_for_only_or_except(options, :convert_values_to_numeric, k)
-          if FLOAT_REGEX.match?(v)
-            hash[k] = v.to_f
-          elsif INTEGER_REGEX.match?(v)
-            hash[k] = v.to_i
+        if convert_to_numeric && v.is_a?(String) &&
+           (!numeric_has_limits || !limit_execution_for_only_or_except(options, :convert_values_to_numeric, k))
+          # Fast-reject: the string is already stripped and NUMERIC_REGEX is \A-anchored on a digit or sign,
+          # so a value whose first byte isn't a digit, '+', or '-' cannot be numeric — skip the regex entirely.
+          first_byte = v.getbyte(0)
+          if first_byte && ((first_byte >= ZERO_BYTE && first_byte <= NINE_BYTE) || first_byte == MINUS_BYTE || first_byte == PLUS_BYTE)
+            if NUMERIC_REGEX.match?(v)
+              hash[k] = v.include?('.') ? v.to_f : v.to_i
+            end
           end
         end
 
@@ -67,7 +83,7 @@ module SmarterCSV
       end
 
       # Delete marked keys
-      keys_to_delete.each { |k| hash.delete(k) }
+      keys_to_delete&.each { |key| hash.delete(key) }
 
       hash
     end