smarter_csv 1.16.3 → 1.16.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b40fb76fef88599d7449691806af6f76a131ffcd41e2ee145d5c87f3554a2006
-  data.tar.gz: edb27057973c0a88524579f450dc8ed3ffadbd983de7155a84ff159495c31233
+  metadata.gz: 9f0dc97fe8b296d479efa58b5e404636fe66dbe768e032de987e4c2736b619a4
+  data.tar.gz: 6ffaa0b2f74fb6a48c22a28c21a254a0e9b962bcfb9d1b979e72e54ae446a5c1
 SHA512:
-  metadata.gz: 5b9e2a17ae14a5d7b3dfddd854148f38c8892d67544b51fecba888aa11c540096057460880564b89251194fea513c3e2807c500d93f896826db885daf64271fe
-  data.tar.gz: 90b5aa10c6bc36cbc97662de879deaf61c2417a61e4cb5171924aac991a37c762588ff6413a924bfd083ca10c1fa1739049561f3c6e9de6ff196bdba85c7878f
+  metadata.gz: 8e16f3d049432df188373da120fd4d5f04fd4a49d6a3bb3e91abf6f5722fa6fc90ee302e6db5348349f65290226e0da046cb10d71b2d267fc1be4d63af18107c
+  data.tar.gz: 473ce5f7d1b2bceb7a82898b90c9a19e4076675eea1d748af97d5c7c04abaf15e1cf95a8b774bab40ba5f36cad6ad517e874eab3e63854302ea9d6d4465fefc8

data/.rubocop.yml

@@ -121,6 +121,9 @@ Style/PercentLiteralDelimiters:
 Style/RegexpLiteral:
   Enabled: false
 
+Style/RescueModifier:
+  Enabled: false
+
 Style/SafeNavigation:
   Enabled: false
 
@@ -153,6 +156,9 @@ Style/SymbolArray:
 Style/SymbolProc: # old Ruby versions can't do this
   Enabled: false
 
+Style/TernaryParentheses:
+  Enabled: false
+
 Style/TrailingCommaInArrayLiteral:
   Enabled: false
   EnforcedStyleForMultiline: consistent_comma

data/CHANGELOG.md

@@ -1,6 +1,18 @@
 
 # SmarterCSV 1.x Change Log
 
+## 1.16.4 (2026-04-21) — Bug Fixes
+
+RSpec tests: **1,434 → 1,467** (+33 tests)
+
+### Bug Fixes
+
+* Fixed bug in `SmarterCSV.errors` that could lose collected records when processing raises mid-stream,
+  e.g. when `bad_row_limit:` was exceeded (`TooManyBadRows`), or when a user's block raised through `.process` / `.each` / `.each_chunk`.
+
+* Fixed `enforce_utf8_encoding` incorrectly replacing all non-ASCII bytes when the input string was tagged as `ASCII-8BIT` (binary).
+  The encoding is now relabeled to UTF-8 before transcoding, so only genuinely invalid byte sequences are replaced.
+
 ## 1.16.3 (2026-04-14) — New Feature
 
 RSpec tests: **1,425 → 1,434** (+9 tests)

data/lib/smarter_csv/reader.rb

@@ -147,9 +147,9 @@ module SmarterCSV
           options[:_keep_bitmap]       = keep_flags.map { |f| f ? 1 : 0 }.pack('C*').freeze
           options[:_keep_extra_cols]   = @only_headers_set ? false : true
           options[:_early_exit_after]  = (@only_headers_set && !options[:strict]) ? (keep_flags.rindex(true) || -1) : -1
-          options[:_keep_cols]         = nil   # nil signals C: "filter active, check _keep_bitmap"
+          options[:_keep_cols]         = nil # nil signals C: "filter active, check _keep_bitmap"
         else
-          options[:_keep_cols] = false  # sentinel: no filtering active — C skips all bitmap paths
+          options[:_keep_cols] = false # sentinel: no filtering active — C skips all bitmap paths
           # Do NOT insert _keep_bitmap/_keep_extra_cols/_early_exit_after when unused.
           # Keeping the options hash as small as possible avoids hash table resize and
           # keeps all 10 per-row rb_hash_aref lookups hitting the same cache lines.
@@ -214,18 +214,18 @@ module SmarterCSV
         # on_start / on_chunk / on_complete are optional callables (nil by default).
         # Hooks only fire from `process` (library-controlled iteration). Enumerator
         # modes (each / each_chunk) do not fire hooks — the caller owns the lifecycle.
-        _on_start    = options[:on_start]
-        _on_chunk    = options[:on_chunk]
-        _on_complete = options[:on_complete]
-        _start_time  = Process.clock_gettime(Process::CLOCK_MONOTONIC) if _on_start || _on_complete
+        on_start    = options[:on_start]
+        on_chunk    = options[:on_chunk]
+        on_complete = options[:on_complete]
+        start_time  = Process.clock_gettime(Process::CLOCK_MONOTONIC) if on_start || on_complete
 
-        if _on_start
-          _input_meta = if @input.is_a?(String)
+        if on_start
+          input_meta = if @input.is_a?(String)
                           { input: @input, file_size: (File.size(@input) rescue nil) }
                         else
                           { input: @input.class.name, file_size: nil }
-                        end
-          _on_start.call(_input_meta.merge(col_sep: options[:col_sep], row_sep: options[:row_sep]))
+                       end
+          on_start.call(input_meta.merge(col_sep: options[:col_sep], row_sep: options[:row_sep]))
         end
 
         # now on to processing all the rest of the lines in the CSV file:
@@ -385,7 +385,7 @@ module SmarterCSV
             chunk << hash # append temp result to chunk
 
             if chunk.size >= chunk_size || fh.eof? # if chunk if full, or EOF reached
-              _on_chunk&.call({ chunk_number: @chunk_count + 1, rows_in_chunk: chunk.size, total_rows_so_far: @csv_line_count })
+              on_chunk&.call({ chunk_number: @chunk_count + 1, rows_in_chunk: chunk.size, total_rows_so_far: @csv_line_count })
               # do something with the chunk
               if block_given?
                 yield chunk, @chunk_count # do something with the hashes in the chunk in the block
@@ -414,7 +414,7 @@ module SmarterCSV
 
         # handling of last chunk:
         if !chunk.nil? && chunk.size > 0
-          _on_chunk&.call({ chunk_number: @chunk_count + 1, rows_in_chunk: chunk.size, total_rows_so_far: @csv_line_count })
+          on_chunk&.call({ chunk_number: @chunk_count + 1, rows_in_chunk: chunk.size, total_rows_so_far: @csv_line_count })
           # do something with the chunk
           if block_given?
             yield chunk, @chunk_count # do something with the hashes in the chunk in the block
@@ -425,13 +425,13 @@ module SmarterCSV
           # chunk = [] # initialize for next chunk of data
         end
 
-        if _on_complete
-          _on_complete.call({
-            total_rows:   @csv_line_count,
-            total_chunks: @chunk_count,
-            duration:     Process.clock_gettime(Process::CLOCK_MONOTONIC) - _start_time,
-            bad_rows:     @errors[:bad_row_count] || 0,
-          })
+        if on_complete
+          on_complete.call({
+                             total_rows: @csv_line_count,
+                             total_chunks: @chunk_count,
+                             duration: Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time,
+                             bad_rows: @errors[:bad_row_count] || 0,
+                           })
         end
       ensure
         fh.close if fh.respond_to?(:close)
@@ -662,12 +662,10 @@ module SmarterCSV
             # else: mid-field quote → literal, no state change
           elsif !in_quotes
             # Non-quote character: track whether field has started
-            if strip
-              # rubocop:disable Style/MultipleComparison -- two direct == comparisons are faster than Array#include? in this hot loop
+            if strip # -- two direct == comparisons are faster than Array#include? in this hot loop
               field_started = true unless line[i] == ' ' || line[i] == "\t"
-              # rubocop:enable Style/MultipleComparison
-            else
-              field_started = true
+                          else
+                            field_started = true
             end
           end
           i += 1
@@ -780,9 +778,13 @@ module SmarterCSV
     end
 
     def enforce_utf8_encoding(line, options)
-      # return line unless options[:force_utf8] || options[:file_encoding] !~ /utf-8/i
-
-      line.force_encoding('utf-8').encode('utf-8', invalid: :replace, undef: :replace, replace: options[:invalid_byte_sequence])
+      replace = options[:invalid_byte_sequence]
+      # ASCII_8BIT (Encoding::BINARY is an alias) has no codepoint mapping above 0x7F,
+      # so encode('utf-8', ASCII_8BIT) would replace every non-ASCII byte. Relabel as
+      # UTF-8 first so encode() treats the bytes as already-UTF-8 and only replaces
+      # sequences that are actually invalid.
+      line = line.force_encoding('utf-8') if line.encoding == Encoding::ASCII_8BIT
+      line.encode('utf-8', line.encoding, invalid: :replace, undef: :replace, replace: replace)
     end
 
     def handle_bad_row(error, line, start_csv_line, start_file_line, options)

data/lib/smarter_csv/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SmarterCSV
-  VERSION = "1.16.3"
+  VERSION = "1.16.4"
 end

data/lib/smarter_csv.rb

@@ -78,9 +78,12 @@ module SmarterCSV
   def self.process(input, given_options = {}, &block)
     Thread.current[:current_thread_recent_errors] = {}
     reader = Reader.new(input, given_options)
-    result = reader.process(&block)
-    Thread.current[:current_thread_recent_errors] = reader.errors
-    result
+    reader.process(&block)
+  ensure
+    # Preserve partial error state when processing raises mid-stream
+    # (e.g. TooManyBadRows, or a user block raising). `reader` is nil if
+    # Reader.new itself raised before the local was assigned.
+    Thread.current[:current_thread_recent_errors] = reader.errors if reader
   end
 
   # Convenience method for parsing a CSV string directly.
@@ -109,9 +112,9 @@ module SmarterCSV
   def self.each(input, options = {}, &block)
     Thread.current[:current_thread_recent_errors] = {}
     reader = Reader.new(input, options)
-    result = reader.each(&block)
-    Thread.current[:current_thread_recent_errors] = reader.errors
-    result
+    reader.each(&block)
+  ensure
+    Thread.current[:current_thread_recent_errors] = reader.errors if reader
   end
 
   # Yields each chunk as Array<Hash> plus its 0-based chunk index.
@@ -126,9 +129,9 @@ module SmarterCSV
   def self.each_chunk(input, options = {}, &block)
     Thread.current[:current_thread_recent_errors] = {}
     reader = Reader.new(input, options)
-    result = reader.each_chunk(&block)
-    Thread.current[:current_thread_recent_errors] = reader.errors
-    result
+    reader.each_chunk(&block)
+  ensure
+    Thread.current[:current_thread_recent_errors] = reader.errors if reader
   end
 
   # Returns the errors from the most recent call to .process, .parse, .each, or .each_chunk
@@ -198,7 +201,7 @@ module SmarterCSV
       begin
         yield writer
       ensure
-        writer&.finalize  # must finalize before reading io.string
+        writer&.finalize # must finalize before reading io.string
       end
       io.string
     else

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: smarter_csv
 version: !ruby/object:Gem::Version
-  version: 1.16.3
+  version: 1.16.4
 platform: ruby
 authors:
 - Tilo Sloboda
 bindir: bin
 cert_chain: []
-date: 2026-04-14 00:00:00.000000000 Z
+date: 2026-04-21 00:00:00.000000000 Z
 dependencies: []
 description: |
   SmarterCSV is a high-performance CSV reader and writer for Ruby focused on