smarter_csv 1.16.6 → 1.17.0.pre5

data/lib/smarter_csv/reader.rb

@@ -7,7 +7,8 @@ module SmarterCSV
     include Enumerable
 
     # Default chunk size used by each_chunk when chunk_size is not explicitly set.
-    # A warning is emitted to STDERR so users know to configure it explicitly.
+    # A warning is recorded (and emitted via Rails.logger or Kernel#warn) so users
+    # know to configure it explicitly.
     DEFAULT_CHUNK_SIZE = 100
 
     include ::SmarterCSV::Reader::Options
@@ -21,7 +22,7 @@ module SmarterCSV
 
     attr_reader :input, :options
     attr_reader :csv_line_count, :chunk_count, :file_line_count
-    attr_reader :enforce_utf8, :has_rails, :has_acceleration
+    attr_reader :enforce_utf8, :has_rails, :has_rails_logger, :has_acceleration
     attr_reader :errors, :warnings, :headers, :raw_header, :result
 
     def self.default_options
@@ -30,7 +31,9 @@ module SmarterCSV
 
     # rubocop:disable Naming/MethodName
     def headerA
-      warn "Deprecarion Warning: 'headerA' will be removed in future versions. Use 'headders'"
+      record_warning(type: :deprecation, code: :header_a_method) do
+        "Deprecarion Warning: 'headerA' will be removed in future versions. Use 'headders'"
+      end
       @headerA
     end
     # rubocop:enable Naming/MethodName
@@ -39,6 +42,7 @@ module SmarterCSV
     def initialize(input, given_options = {})
       @input = input
       @has_rails = !!defined?(Rails)
+      @has_rails_logger = defined?(::Rails) && ::Rails.respond_to?(:logger) && !::Rails.logger.nil?
       @csv_line_count = 0
       @chunk_count = 0
       @errors = {}
@@ -47,7 +51,8 @@ module SmarterCSV
       @headers = nil
       @raw_header = nil # header as it appears in the file
       @result = []
-      @warnings = {}
+      @warnings = []
+      @warnings_by_key = {}
       @enforce_utf8 = false # only set to true if needed (after options parsing)
       @options = process_options(given_options)
       # true if it is compiled with accelleration
@@ -87,7 +92,11 @@ module SmarterCSV
 
       chunk_size = @options[:chunk_size]
       if chunk_size.nil?
-        warn "SmarterCSV: chunk_size not set, defaulting to #{DEFAULT_CHUNK_SIZE}. Set chunk_size explicitly to suppress this warning." unless @options[:verbose] == :quiet
+        unless @options[:verbose] == :quiet
+          record_warning(type: :config, code: :chunk_size_default) do
+            "chunk_size not set, defaulting to #{DEFAULT_CHUNK_SIZE}. Set chunk_size explicitly to suppress this warning."
+          end
+        end
         chunk_size = DEFAULT_CHUNK_SIZE
       end
       unless chunk_size.is_a?(Integer) && chunk_size >= 1
@@ -106,23 +115,72 @@ module SmarterCSV
       end
     end
 
-    def process(&block) # rubocop:disable Lint/UnusedMethodArgument
+    def process(&block)
       @enforce_utf8 = options[:force_utf8] || options[:file_encoding] !~ /utf-8/i
       @verbose = options[:verbose]
 
       begin
-        fh = input.respond_to?(:readline) ? input : File.open(input, "r:#{options[:file_encoding]}")
+        fh = input.is_a?(String) ? File.open(input, "r:#{options[:file_encoding]}") : input
+
+        # Rewindable inputs (File, Tempfile, StringIO, Zlib::GzipReader, ...) use
+        # native rewind for auto-detection — no wrapper overhead in the hot loop.
+        # Non-rewindable streams (pipes, STDIN, custom non-seekable IOs) go through
+        # PeekableIO which buffers the first chunk so detection can replay without
+        # seeking the underlying source.
+        has_rewind = seekable?(fh)
+
+        unless has_rewind
+          # buffer_size can be passed directly; otherwise it scales from auto_row_sep_chars.
+          # 2× auto_row_sep_chars ensures the first peek covers the full row_sep scan with
+          # room for col_sep detection. Falls back to DEFAULT_PEEK_SIZE when auto_row_sep_chars
+          # is 0 (scan whole file).
+          buf_size = if options[:buffer_size]
+                       options[:buffer_size].to_i
+                     else
+                       auto_row_sep_chars = options[:auto_row_sep_chars].to_i
+                       auto_row_sep_chars > 0 ? 2 * auto_row_sep_chars : SmarterCSV::PeekableIO::DEFAULT_PEEK_SIZE
+                     end
+          fh = SmarterCSV::PeekableIO.new(fh, options, buffer_size: buf_size)
+        end
 
         if (options[:force_utf8] || options[:file_encoding] =~ /utf-8/i) && (fh.respond_to?(:external_encoding) && fh.external_encoding != Encoding.find('UTF-8') || fh.respond_to?(:encoding) && fh.encoding != Encoding.find('UTF-8'))
-          warn 'WARNING: you are trying to process UTF-8 input, but did not open the input with "b:utf-8" option. See README file "NOTES about File Encodings".' unless options[:verbose] == :quiet
+          unless options[:verbose] == :quiet
+            record_warning(type: :encoding, code: :utf8_missing_binary_mode) do
+              'WARNING: you are trying to process UTF-8 input, but did not open the input with "b:utf-8" option. See README file "NOTES about File Encodings".'
+            end
+          end
         end
 
-        # auto-detect the row separator
-        options[:row_sep] = guess_line_ending(fh, options) if options[:row_sep]&.to_sym == :auto
-        # attempt to auto-detect column separator
-        options[:col_sep] = guess_column_separator(fh, options) if options[:col_sep]&.to_sym == :auto
+        # Auto-detection. Two orchestrations, same detection functions:
+        #   has_rewind=true  → native fh.rewind between passes; BOM is stripped by
+        #                      next_line_with_counts on the first real line.
+        #   has_rewind=false → PeekableIO buffers the first chunk; peek strips BOM,
+        #                      rewind_buffer replays, freeze_buffer! locks the buffer.
+        if options[:row_sep]&.to_sym == :auto || options[:col_sep]&.to_sym == :auto
+          if has_rewind
+            options[:row_sep] = guess_line_ending(fh, options) if options[:row_sep]&.to_sym == :auto
+            fh.rewind
+            @file_line_count = 0
+            @csv_line_count = 0
+            # skip_lines feeds clean data lines to guess_column_separator. When col_sep is
+            # explicit, it's wasted work — the bytes are consumed and rewound. Guard it.
+            skip_lines(fh, options) if options[:skip_lines] && options[:col_sep]&.to_sym == :auto
+            options[:col_sep] = guess_column_separator(fh, options) if options[:col_sep]&.to_sym == :auto
+            fh.rewind
+            @file_line_count = 0
+            @csv_line_count = 0
+          else
+            fh.peek
+            options[:row_sep] = guess_line_ending(fh, options) if options[:row_sep]&.to_sym == :auto
+            rewind_buffer(fh)
+            skip_lines(fh, options) if options[:skip_lines] && options[:col_sep]&.to_sym == :auto
+            options[:col_sep] = guess_column_separator(fh, options) if options[:col_sep]&.to_sym == :auto
+            fh.freeze_buffer!
+            rewind_buffer(fh)
+          end
+        end
 
-        skip_lines(fh, options)
+        skip_lines(fh, options) if options[:skip_lines] # skip comments
 
         # NOTE: we are no longer using header_size
         @headers, _header_size = process_headers(fh, options)
@@ -221,9 +279,9 @@ module SmarterCSV
 
         if on_start
           input_meta = if @input.is_a?(String)
-                          { input: @input, file_size: (File.size(@input) rescue nil) }
-                        else
-                          { input: @input.class.name, file_size: nil }
+                         { input: @input, file_size: (File.size(@input) rescue nil) }
+                       else
+                         { input: @input.class.name, file_size: nil }
                        end
           on_start.call(input_meta.merge(col_sep: options[:col_sep], row_sep: options[:row_sep]))
         end
@@ -515,6 +573,50 @@ module SmarterCSV
 
     private
 
+    # Records a warning into the histogram and emits it to the warning sink.
+    # `@warnings` is an Array of unique (type, code) records with a `count` field.
+    # `@warnings_by_key` is a dedup map keyed by `[type, code]` — key shape must
+    # stay fixed to keep both structures bounded by distinct codes, not by calls.
+    # The block form avoids string allocation on dedup-hit: on a repeat call we
+    # increment count and return without yielding the block.
+    # `dedup: false` bypasses the dedup map (still appends a new record per call);
+    # reserve for per-occurrence warnings where each call carries distinct info.
+    # `severity:` controls the Rails.logger level (`:debug`/`:info`/`:warn`/`:error`/`:fatal`);
+    # the non-Rails fallback is always `Kernel#warn` regardless of severity.
+    # `type` is purely a semantic grouping for callers iterating reader.warnings.
+    def record_warning(type:, code:, severity: :warn, dedup: true)
+      key = [type, code]
+      if dedup && (existing = @warnings_by_key[key])
+        existing[:count] += 1
+        return
+      end
+
+      message = yield
+      record = { type: type, code: code, severity: severity, message: message, count: 1 }
+      @warnings << record
+      @warnings_by_key[key] = record if dedup
+
+      if @has_rails_logger
+        ::Rails.logger.public_send(severity, "SmarterCSV: #{message}")
+      else
+        warn "SmarterCSV: #{message}"
+      end
+    end
+
+    # True when the IO is genuinely seekable — i.e. rewind will succeed at the kernel
+    # level, not just the Ruby method table. IO.pipe readers respond to :rewind but
+    # raise at the kernel layer when called; probing #pos surfaces that at decision time.
+    # Rescue SystemCallError (parent of all Errno::*) because the exact subclass varies
+    # by Ruby implementation: MRI raises Errno::ESPIPE, jruby raises Errno::EPIPE.
+    def seekable?(io)
+      return false unless io.respond_to?(:rewind)
+
+      io.pos if io.respond_to?(:pos)
+      true
+    rescue SystemCallError, IOError, NotImplementedError
+      false
+    end
+
     # Determine if a line has unbalanced quotes requiring multiline stitching.
     # For :auto mode, uses dual counting to avoid false multiline detection.
     # For :standard quote_boundary mode, uses a full state machine so that
@@ -663,9 +765,9 @@ module SmarterCSV
           elsif !in_quotes
             # Non-quote character: track whether field has started
             if strip # -- two direct == comparisons are faster than Array#include? in this hot loop
-              field_started = true unless line[i] == ' ' || line[i] == "\t"
-                          else
-                            field_started = true
+              field_started = true unless [' ', "\t"].include?(line[i])
+            else
+              field_started = true
             end
           end
           i += 1

data/lib/smarter_csv/reader_options.rb

@@ -5,7 +5,7 @@ module SmarterCSV
     module Options
       DEFAULT_OPTIONS = {
         acceleration: true, # if user wants to use accelleration or not
-        auto_row_sep_chars: 500,
+        auto_row_sep_chars: 8_192,
         bad_row_limit: nil,
         chunk_size: nil,
         col_sep: :auto, # was: ',',
@@ -187,6 +187,19 @@ module SmarterCSV
         unless %i[legacy standard].include?(options[:quote_boundary])
           errors << "invalid quote_boundary: must be :legacy or :standard"
         end
+        arc = options[:auto_row_sep_chars]
+        unless arc.is_a?(Integer) && arc >= DEFAULT_OPTIONS[:auto_row_sep_chars]
+          warn "WARNING: invalid auto_row_sep_chars value #{arc.inspect} — must be an Integer >= #{DEFAULT_OPTIONS[:auto_row_sep_chars]}; using default (#{DEFAULT_OPTIONS[:auto_row_sep_chars]})" unless options[:verbose] == :quiet
+          options[:auto_row_sep_chars] = DEFAULT_OPTIONS[:auto_row_sep_chars]
+        end
+        # buffer_size is an internal option used by tests to exercise small-buffer boundary conditions.
+        # It is purposely not part of the public API and purposely not listed in DEFAULT_OPTIONS.
+        if options.key?(:buffer_size)
+          bs = options[:buffer_size]
+          unless bs.is_a?(Integer) && bs > 0
+            errors << "invalid buffer_size: must be a positive Integer (got #{bs.inspect})"
+          end
+        end
         fsl = options[:field_size_limit]
         unless fsl.nil? || (fsl.is_a?(Integer) && fsl > 0)
           errors << "invalid field_size_limit: must be nil or a positive Integer (got #{fsl.inspect})"

data/lib/smarter_csv/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SmarterCSV
-  VERSION = "1.16.6"
+  VERSION = "1.17.0.pre5"
 end

data/lib/smarter_csv.rb

@@ -5,6 +5,7 @@ require "smarter_csv/version"
 require "smarter_csv/errors"
 
 require "smarter_csv/file_io"
+require "smarter_csv/peekable_io"
 require "smarter_csv/reader_options"
 require "smarter_csv/writer_options"
 require "smarter_csv/auto_detection"
@@ -66,24 +67,30 @@ module SmarterCSV
   #   reader = SmarterCSV::Reader.new(input, options)
   #   reader.process # with or without block
   #
-  # After calling any of the class-level methods, errors from the last run are available via:
+  # After calling any of the class-level methods, errors and warnings from the last run
+  # are available via:
   #
-  #   SmarterCSV.errors  # => { bad_row_count: 2, bad_rows: [...] }
+  #   SmarterCSV.errors    # => { bad_row_count: 2, bad_rows: [...] }
+  #   SmarterCSV.warnings  # => [ { type:, code:, message:, count: }, ... ]
   #
-  # This exposes the same reader.errors hash without requiring access to the Reader instance.
-  # Errors are cleared at the start of each call and stored per-thread, so this is safe in
-  # multi-threaded environments (Puma, Sidekiq). Note: only the most recent call's errors
-  # are retained per thread.
+  # These expose the same reader.errors / reader.warnings without requiring access to the
+  # Reader instance. Both are cleared at the start of each call and stored per-thread, so
+  # this is safe in multi-threaded environments (Puma, Sidekiq). Only the most recent
+  # call's errors and warnings are retained per thread.
   #
   def self.process(input, given_options = {}, &block)
     Thread.current[:current_thread_recent_errors] = {}
+    Thread.current[:current_thread_recent_warnings] = []
     reader = Reader.new(input, given_options)
     reader.process(&block)
   ensure
     # Preserve partial error state when processing raises mid-stream
     # (e.g. TooManyBadRows, or a user block raising). `reader` is nil if
     # Reader.new itself raised before the local was assigned.
-    Thread.current[:current_thread_recent_errors] = reader.errors if reader
+    if reader
+      Thread.current[:current_thread_recent_errors] = reader.errors
+      Thread.current[:current_thread_recent_warnings] = reader.warnings
+    end
   end
 
   # Convenience method for parsing a CSV string directly.
@@ -111,10 +118,14 @@ module SmarterCSV
   #   SmarterCSV.each("data.csv").lazy.map { |h| h[:name] }.first(10)
   def self.each(input, options = {}, &block)
     Thread.current[:current_thread_recent_errors] = {}
+    Thread.current[:current_thread_recent_warnings] = []
     reader = Reader.new(input, options)
     reader.each(&block)
   ensure
-    Thread.current[:current_thread_recent_errors] = reader.errors if reader
+    if reader
+      Thread.current[:current_thread_recent_errors] = reader.errors
+      Thread.current[:current_thread_recent_warnings] = reader.warnings
+    end
   end
 
   # Yields each chunk as Array<Hash> plus its 0-based chunk index.
@@ -128,10 +139,14 @@ module SmarterCSV
   #   SmarterCSV.each_chunk("data.csv", chunk_size: 100).with_index { |chunk, i| ... }
   def self.each_chunk(input, options = {}, &block)
     Thread.current[:current_thread_recent_errors] = {}
+    Thread.current[:current_thread_recent_warnings] = []
     reader = Reader.new(input, options)
     reader.each_chunk(&block)
   ensure
-    Thread.current[:current_thread_recent_errors] = reader.errors if reader
+    if reader
+      Thread.current[:current_thread_recent_errors] = reader.errors
+      Thread.current[:current_thread_recent_warnings] = reader.warnings
+    end
   end
 
   # Returns the errors from the most recent call to .process, .parse, .each, or .each_chunk
@@ -149,6 +164,21 @@ module SmarterCSV
     Thread.current[:current_thread_recent_errors] || {}
   end
 
+  # Returns the warnings from the most recent call to .process, .parse, .each, or .each_chunk
+  # on the current thread. Cleared at the start of each new call.
+  #
+  # Each warning is a Hash: { type:, code:, message:, count: }.
+  # Repeated warnings of the same (type, code) are deduped — `count` tracks
+  # the number of occurrences.
+  #
+  # Example:
+  #   SmarterCSV.process('data.csv')
+  #   SmarterCSV.warnings.each { |w| logger.warn("[#{w[:type]}/#{w[:code]}] #{w[:message]} (×#{w[:count]})") }
+  #
+  def self.warnings
+    Thread.current[:current_thread_recent_warnings] || []
+  end
+
   # Convenience method for generating CSV files, IO objects, or in-memory strings.
   #
   # When called WITHOUT a first argument, generates CSV in memory and returns it as a String.
@@ -184,7 +214,6 @@ module SmarterCSV
   #     end
   #   end
   #
-  # rubocop:disable Lint/UnusedMethodArgument
   def self.generate(file_path_or_io = nil, options = {}, &block)
     raise ArgumentError, "SmarterCSV.generate requires a block" unless block_given?
 
@@ -213,5 +242,4 @@ module SmarterCSV
       end
     end
   end
-  # rubocop:enable Lint/UnusedMethodArgument
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: smarter_csv
 version: !ruby/object:Gem::Version
-  version: 1.16.6
+  version: 1.17.0.pre5
 platform: ruby
 authors:
 - Tilo Sloboda
 bindir: bin
 cert_chain: []
-date: 2026-05-21 00:00:00.000000000 Z
+date: 2026-04-28 00:00:00.000000000 Z
 dependencies: []
 description: |
   SmarterCSV is a high-performance CSV reader and writer for Ruby focused on
@@ -62,6 +62,8 @@ files:
 - docs/row_col_sep.md
 - docs/ruby_csv_pitfalls.md
 - docs/value_converters.md
+- docs/warnings.md
+- ext/smarter_csv/Makefile
 - ext/smarter_csv/extconf.rb
 - ext/smarter_csv/smarter_csv.c
 - images/SmarterCSV_1.16.0_vs_RubyCSV_3.3.5_speedup.png
@@ -79,6 +81,7 @@ files:
 - lib/smarter_csv/header_validations.rb
 - lib/smarter_csv/headers.rb
 - lib/smarter_csv/parser.rb
+- lib/smarter_csv/peekable_io.rb
 - lib/smarter_csv/reader.rb
 - lib/smarter_csv/reader_options.rb
 - lib/smarter_csv/version.rb
@@ -109,7 +112,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.11
+rubygems_version: 4.0.6
 specification_version: 4
 summary: Fastest end-to-end CSV ingestion for Ruby with smart defaults and Rails-ready
   hash output