smarter_csv 1.15.2 → 1.16.1

data/lib/smarter_csv/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SmarterCSV
-  VERSION = "1.15.2"
+  VERSION = "1.16.1"
 end

data/lib/smarter_csv/writer.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require 'tempfile'
+require 'stringio'
+require 'set'
 
 module SmarterCSV
   #
@@ -33,6 +35,13 @@ module SmarterCSV
   #   force_quotes: defaults to false
   #   map_headers: defaults to {}, can be a hash of key -> value mappings
   #   value_converters: optional hash of key -> lambda to control serialization
+  #   encoding: optional encoding string for the output file, e.g. 'UTF-8', 'ISO-8859-1'
+  #             supports Ruby's 'external:internal' transcoding notation, e.g. 'ISO-8859-1:UTF-8'
+  #             defaults to nil (system default). Only applies when writing to a file path.
+  #   write_nil_value: string written in place of nil field values (default: '')
+  #   write_empty_value: string written in place of empty-string field values (default: '')
+  #   write_bom: when true, prepends a UTF-8 BOM (\xEF\xBB\xBF) to the output (default: false)
+  #              Useful for Excel compatibility with non-ASCII content.
 
   # IMPORTANT NOTES:
   #  * Data hashes could contain strings or symbols as keys.
@@ -42,18 +51,23 @@ module SmarterCSV
   attr_reader :options, :row_sep, :col_sep, :quote_char, :force_quotes, :discover_headers, :headers, :map_headers, :output_file
 
   class Writer
-    def initialize(file_path, options = {})
+    def initialize(file_path_or_io, options = {})
       @options = options
 
       @row_sep = options[:row_sep] || $/
       @col_sep = options[:col_sep] || ','
       @quote_char = options[:quote_char] || '"'
+      @escaped_quote_char = @quote_char * 2
       @force_quotes = options[:force_quotes] == true
       @quote_headers = options[:quote_headers] == true
       @disable_auto_quoting = options[:disable_auto_quoting] == true
       @value_converters = options[:value_converters] || {}
+      @encoding = options[:encoding]
+      @write_nil_value = options.fetch(:write_nil_value, '')
+      @write_empty_value = options.fetch(:write_empty_value, '')
+      @write_bom = options[:write_bom] == true
       @map_all_keys = @value_converters.has_key?(:_all)
-      @mapped_keys = @value_converters.keys - [:_all]
+      @mapped_keys = Set.new(@value_converters.keys - [:_all])
       @header_converter = options[:header_converter]
 
       @discover_headers = true
@@ -68,9 +82,38 @@ module SmarterCSV
       @headers = options[:map_headers].keys if options.has_key?(:map_headers) && !options.has_key?(:headers)
       @map_headers = options[:map_headers] || {}
 
-      @output_file = File.open(file_path, 'w+')
-      @temp_file = Tempfile.new('tempfile', '/tmp')
+      # Accept an IO-like object (StringIO, IO, etc.) or any path-like object (String, Pathname, etc.)
+      if file_path_or_io.respond_to?(:write)
+        # External IO handed in — we should not close it ourselves.
+        @output_file = file_path_or_io
+        @file_opened_by_us = false
+      else
+        path =
+          if file_path_or_io.respond_to?(:to_path)
+            file_path_or_io.to_path
+          elsif file_path_or_io.is_a?(String)
+            file_path_or_io
+          else
+            raise ArgumentError,
+                  "SmarterCSV::Writer expects an IO-like object (responding to #write) " \
+                  "or a path-like object (responding to #to_path or being a String), " \
+                  "but got #{file_path_or_io.class}"
+          end
+        mode = @encoding ? "w+:#{@encoding}" : 'w+'
+        @output_file = File.open(path, mode)
+        @file_opened_by_us = true
+      end
       @quote_regex = Regexp.union(@col_sep, @row_sep, @quote_char)
+
+      if !@discover_headers && !@headers.empty?
+        # Headers are fully known at construction time — write the header line immediately
+        # and stream data rows directly to @output_file, bypassing the temp file entirely.
+        @temp_file = nil
+        @output_file.write("\xEF\xBB\xBF") if @write_bom
+        write_header_line
+      else
+        @temp_file = Tempfile.new('smarter_csv')
+      end
     end
 
     def <<(data)
@@ -82,30 +125,36 @@ module SmarterCSV
       when NilClass
         # ignore
       else
-        # :nocov:
         raise InvalidInputData, "Invalid data type: #{data.class}. Must be a Hash or an Array."
-        # :nocov:
       end
     end
 
     def finalize
-      mapped_headers = @headers.map { |header| @map_headers[header] || header }
+      if @temp_file
+        # Header-discovery mode: headers were accumulated while writing rows;
+        # now prepend the header line and copy the buffered rows to the output.
+        @output_file.write("\xEF\xBB\xBF") if @write_bom
+        write_header_line
+        @temp_file.rewind
+        @output_file.write(@temp_file.read)
+        @temp_file.close!
+      end
+      # In direct-write mode (@temp_file == nil) the header line and all data rows
+      # were already written to @output_file — nothing left to do but flush and close.
+      @output_file.flush
+      @output_file.close if @file_opened_by_us # only close files we opened; caller owns external IO objects
+    end
 
-      mapped_headers = @headers.map { |header| @header_converter.call(header) } if @header_converter
+    private
 
+    def write_header_line
+      mapped_headers = @headers.map { |header| @map_headers[header] || header }
+      mapped_headers = @headers.map { |header| @header_converter.call(header) } if @header_converter
       force_quotes = @quote_headers || @force_quotes
       mapped_headers = mapped_headers.map { |x| escape_csv_field(x, force_quotes) }
-
-      @temp_file.rewind
       @output_file.write(mapped_headers.join(@col_sep) + @row_sep) unless mapped_headers.empty?
-      @output_file.write(@temp_file.read)
-      @output_file.flush
-      @output_file.close
-      @temp_file.delete
     end
 
-    private
-
     def process_hash(hash)
       if @discover_headers
         hash_keys = hash.keys
@@ -124,10 +173,13 @@ module SmarterCSV
         # then apply general mapping rules
         value = map_all_values(header, value) if @map_all_keys
 
+        value = @write_nil_value if value.nil?
+        value = @write_empty_value if !value.nil? && value.respond_to?(:empty?) && value.empty?
+
         escape_csv_field(value, @force_quotes) # for backwards compatibility
       end
 
-      @temp_file.write(ordered_row.join(@col_sep) + @row_sep) unless ordered_row.empty?
+      (@temp_file || @output_file).write(ordered_row.join(@col_sep) << @row_sep) unless ordered_row.empty?
     end
 
     def map_value(key, value)
@@ -143,9 +195,9 @@ module SmarterCSV
       return str if @disable_auto_quoting && !force_quotes
 
       # double-quote fields if we force that, or if the field contains the comma, new-line, or quote character
-      contains_special_char = str.to_s.match(@quote_regex)
+      contains_special_char = str.match(@quote_regex)
       if force_quotes || contains_special_char
-        str = str.gsub(@quote_char, @quote_char * 2) if contains_special_char # escape double-quote
+        str = str.gsub(@quote_char, @escaped_quote_char) if contains_special_char # escape double-quote
 
         "\"#{str}\""
       else

data/lib/smarter_csv.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'stringio'
 require "smarter_csv/version"
 require "smarter_csv/errors"
 
@@ -64,29 +65,149 @@ module SmarterCSV
   #   reader = SmarterCSV::Reader.new(input, options)
   #   reader.process # with or without block
   #
+  # After calling any of the class-level methods, errors from the last run are available via:
+  #
+  #   SmarterCSV.errors  # => { bad_row_count: 2, bad_rows: [...] }
+  #
+  # This exposes the same reader.errors hash without requiring access to the Reader instance.
+  # Errors are cleared at the start of each call and stored per-thread, so this is safe in
+  # multi-threaded environments (Puma, Sidekiq). Note: only the most recent call's errors
+  # are retained per thread.
+  #
   def self.process(input, given_options = {}, &block)
+    Thread.current[:current_thread_recent_errors] = {}
     reader = Reader.new(input, given_options)
-    reader.process(&block)
+    result = reader.process(&block)
+    Thread.current[:current_thread_recent_errors] = reader.errors
+    result
+  end
+
+  # Convenience method for parsing a CSV string directly.
+  # Equivalent to SmarterCSV.process(StringIO.new(csv_string), options).
+  # Errors from the run are available via SmarterCSV.errors after the call.
+  #
+  # Example:
+  #   data = SmarterCSV.parse("name,age\nAlice,30\nBob,25")
+  #   # => [{name: "Alice", age: 30}, {name: "Bob", age: 25}]
+  #
+  #   SmarterCSV.parse("name,age\nAlice,30") { |chunk| chunk.each { |h| puts h } }
+  #
+  def self.parse(csv_string, options = {}, &block)
+    process(StringIO.new(csv_string), options, &block)
+  end
+
+  # Yields each successfully parsed row as a Hash (row-by-row, Enumerable-compatible).
+  # Returns an Enumerator when called without a block.
+  # When called with a block, errors from the run are available via SmarterCSV.errors after the call.
+  # When called without a block (Enumerator form), use SmarterCSV::Reader directly for error access.
+  #
+  # Examples:
+  #   SmarterCSV.each("data.csv") { |hash| MyModel.upsert(hash) }
+  #   SmarterCSV.each("data.csv").select { |h| h[:country] == "US" }
+  #   SmarterCSV.each("data.csv").lazy.map { |h| h[:name] }.first(10)
+  def self.each(input, options = {}, &block)
+    Thread.current[:current_thread_recent_errors] = {}
+    reader = Reader.new(input, options)
+    result = reader.each(&block)
+    Thread.current[:current_thread_recent_errors] = reader.errors
+    result
+  end
+
+  # Yields each chunk as Array<Hash> plus its 0-based chunk index.
+  # Requires chunk_size to be set in options (must be >= 1).
+  # Returns an Enumerator when called without a block.
+  # When called with a block, errors from the run are available via SmarterCSV.errors after the call.
+  # When called without a block (Enumerator form), use SmarterCSV::Reader directly for error access.
+  #
+  # Examples:
+  #   SmarterCSV.each_chunk("data.csv", chunk_size: 500) { |chunk, i| Sidekiq.push_bulk(chunk) }
+  #   SmarterCSV.each_chunk("data.csv", chunk_size: 100).with_index { |chunk, i| ... }
+  def self.each_chunk(input, options = {}, &block)
+    Thread.current[:current_thread_recent_errors] = {}
+    reader = Reader.new(input, options)
+    result = reader.each_chunk(&block)
+    Thread.current[:current_thread_recent_errors] = reader.errors
+    result
+  end
+
+  # Returns the errors from the most recent call to .process, .parse, .each, or .each_chunk
+  # on the current thread. Cleared at the start of each new call.
+  #
+  # Keys (when on_bad_row: :skip or :collect is used):
+  #   :bad_row_count  — total number of bad rows encountered
+  #   :bad_rows       — array of error records (only with on_bad_row: :collect)
+  #
+  # Example:
+  #   SmarterCSV.process('data.csv', on_bad_row: :skip)
+  #   puts SmarterCSV.errors[:bad_row_count]
+  #
+  def self.errors
+    Thread.current[:current_thread_recent_errors] || {}
   end
 
-  # Convenience method for generating CSV files:
+  # Convenience method for generating CSV files, IO objects, or in-memory strings.
+  #
+  # When called WITHOUT a first argument, generates CSV in memory and returns it as a String.
+  # When called WITH a file path (String/Pathname) or any IO-compatible object (StringIO,
+  # open File handle, etc.), writes to that destination and returns nil.
+  # The caller retains ownership of any IO object passed in — SmarterCSV will not close it.
+  #
+  # Examples:
+  #
+  #   # Return CSV as a String (no file argument)
+  #   csv_string = SmarterCSV.generate(options) do |csv|
+  #     records.each { |r| csv << r }
+  #   end
+  #
+  #   # Write to a file by path
+  #   SmarterCSV.generate('output.csv', options) do |csv|
+  #     MyModel.find_in_batches(batch_size: 100) do |batch|
+  #       batch.each { |record| csv << record.attributes }
+  #     end
+  #   end
+  #
+  #   # Write to a StringIO (e.g. for Rails streaming responses)
+  #   io = StringIO.new
+  #   SmarterCSV.generate(io) do |csv|
+  #     records.each { |r| csv << r }
+  #   end
+  #   send_data io.string, type: 'text/csv'
   #
-  # SmarterCSV.generate(filename, options) do |csv_writer|
-  #   MyModel.find_in_batches(batch_size: 100) do |batch|
-  #    batch.pluck(:name, :description, :instructor).each do |record|
-  #       csv_writer << record
+  #   # Write to an already-open file handle
+  #   File.open('output.csv', 'w') do |f|
+  #     SmarterCSV.generate(f) do |csv|
+  #       records.each { |r| csv << r }
   #     end
   #   end
-  # end
   #
   # rubocop:disable Lint/UnusedMethodArgument
-  def self.generate(filename, options = {}, &block)
-    raise unless block_given?
+  def self.generate(file_path_or_io = nil, options = {}, &block)
+    raise ArgumentError, "SmarterCSV.generate requires a block" unless block_given?
 
-    writer = Writer.new(filename, options)
-    yield writer
-  ensure
-    writer.finalize
+    # When called as generate(options_hash) { }, the hash lands in file_path_or_io
+    if file_path_or_io.is_a?(Hash)
+      options = file_path_or_io
+      file_path_or_io = nil
+    end
+
+    if file_path_or_io.nil?
+      # No destination given — write to an in-memory StringIO and return the result as a String.
+      io = StringIO.new
+      writer = Writer.new(io, options)
+      begin
+        yield writer
+      ensure
+        writer&.finalize  # must finalize before reading io.string
+      end
+      io.string
+    else
+      writer = Writer.new(file_path_or_io, options)
+      begin
+        yield writer
+      ensure
+        writer&.finalize
+      end
+    end
   end
   # rubocop:enable Lint/UnusedMethodArgument
 end

data/smarter_csv.gemspec

@@ -10,16 +10,33 @@ Gem::Specification.new do |spec|
   spec.version       = SmarterCSV::VERSION
   spec.date          = Time.now.utc.strftime('%Y-%m-%d')
 
-  spec.summary       = "Convenient CSV Reading and Writing"
-  spec.description   = "Ruby Gem for convenient reading and writing of CSV files. It has intelligent defaults, and auto-discovery of column and row separators. It imports CSV Files as Array(s) of Hashes, suitable for direct processing with ActiveRecord, kicking-off batch jobs with Sidekiq, parallel processing, or oploading data to S3. Similarly, writing CSV files takes Hashes, or Arrays of Hashes to create a CSV file."
+  spec.summary = "Fastest end-to-end CSV ingestion for Ruby with smart defaults and Rails-ready hash output"
+  spec.description = <<~DESC
+    SmarterCSV is a high-performance CSV reader and writer for Ruby focused on
+    fastest end-to-end ingestion — not just parsing. It returns ready-to-use
+    hashes with configurable header and value transformations, intelligent
+    defaults, and automatic delimiter discovery.
+
+    Built for real-world data pipelines, SmarterCSV supports chunked processing
+    for large files, streaming via Enumerable APIs, and C acceleration
+    to optimize the full ingestion path (parsing + hash construction +
+    conversions).
+
+    Designed to handle messy user-uploaded CSV while remaining easy to integrate
+    with Rails, ActiveRecord imports, Sidekiq jobs, parallel processing, and
+    S3-based workflows.
+  DESC
+
   spec.homepage      = "https://github.com/tilo/smarter_csv"
   spec.license       = 'MIT'
 
   spec.metadata["homepage_uri"] = spec.homepage
   spec.metadata["source_code_uri"] = spec.homepage
   spec.metadata["changelog_uri"] = "https://github.com/tilo/smarter_csv/blob/main/CHANGELOG.md"
+  spec.metadata["documentation_uri"] = "https://github.com/tilo/smarter_csv/tree/main/docs"
+  spec.metadata["bug_tracker_uri"] = "https://github.com/tilo/smarter_csv/issues"
 
-  spec.required_ruby_version = ">= 2.5.0"
+  spec.required_ruby_version = ">= 2.6.0"
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
@@ -29,16 +46,9 @@ Gem::Specification.new do |spec|
         f.match(%r{\A(?:(?:bin|test|spec|features)/|\.(?:git|travis|circleci)|appveyor)}) || f.match(/\.h\z/)
     end
   end
-  spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
-
   spec.executables   = spec.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   spec.require_paths = %w[lib ext]
   spec.extensions = ["ext/smarter_csv/extconf.rb"]
   spec.files += Dir.glob("ext/smarter_csv/**/*")
 
-  spec.add_development_dependency "awesome_print"
-  spec.add_development_dependency "pry"
-  spec.add_development_dependency "rspec"
-  spec.add_development_dependency "rubocop"
-  spec.add_development_dependency "simplecov"
 end

metadata

@@ -1,89 +1,28 @@
 --- !ruby/object:Gem::Specification
 name: smarter_csv
 version: !ruby/object:Gem::Version
-  version: 1.15.2
+  version: 1.16.1
 platform: ruby
 authors:
 - Tilo Sloboda
 bindir: bin
 cert_chain: []
-date: 2026-02-20 00:00:00.000000000 Z
-dependencies:
-- !ruby/object:Gem::Dependency
-  name: awesome_print
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: pry
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rspec
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rubocop
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: simplecov
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-description: Ruby Gem for convenient reading and writing of CSV files. It has intelligent
-  defaults, and auto-discovery of column and row separators. It imports CSV Files
-  as Array(s) of Hashes, suitable for direct processing with ActiveRecord, kicking-off
-  batch jobs with Sidekiq, parallel processing, or oploading data to S3. Similarly,
-  writing CSV files takes Hashes, or Arrays of Hashes to create a CSV file.
+date: 2026-03-16 00:00:00.000000000 Z
+dependencies: []
+description: |
+  SmarterCSV is a high-performance CSV reader and writer for Ruby focused on
+  fastest end-to-end ingestion — not just parsing. It returns ready-to-use
+  hashes with configurable header and value transformations, intelligent
+  defaults, and automatic delimiter discovery.
+
+  Built for real-world data pipelines, SmarterCSV supports chunked processing
+  for large files, streaming via Enumerable APIs, and C acceleration
+  to optimize the full ingestion path (parsing + hash construction +
+  conversions).
+
+  Designed to handle messy user-uploaded CSV while remaining easy to integrate
+  with Rails, ActiveRecord imports, Sidekiq jobs, parallel processing, and
+  S3-based workflows.
 email:
 - tilo.sloboda@gmail.com
 executables: []
@@ -102,16 +41,26 @@ files:
 - Rakefile
 - TO_DO_v2.md
 - docs/_introduction.md
+- docs/bad_row_quarantine.md
 - docs/basic_read_api.md
 - docs/basic_write_api.md
 - docs/batch_processing.md
+- docs/column_selection.md
 - docs/data_transformations.md
 - docs/examples.md
 - docs/header_transformations.md
 - docs/header_validations.md
+- docs/history.md
+- docs/instrumentation.md
+- docs/migrating_from_csv.md
 - docs/options.md
 - docs/parsing_strategy.md
+- docs/real_world_csv.md
+- docs/releases/1.16.0/benchmarks.md
+- docs/releases/1.16.0/changes.md
+- docs/releases/1.16.0/performance_notes.md
 - docs/row_col_sep.md
+- docs/ruby_csv_pitfalls.md
 - docs/value_converters.md
 - ext/smarter_csv/Makefile
 - ext/smarter_csv/extconf.rb
@@ -121,6 +70,12 @@ files:
 - ext/smarter_csv/smarter_csv.bundle.dSYM/Contents/Resources/Relocations/aarch64/smarter_csv.bundle.yml
 - ext/smarter_csv/smarter_csv.c
 - ext/smarter_csv/smarter_csv.o
+- images/SmarterCSV_1.16.0_vs_RubyCSV_3.3.5_speedup.png
+- images/SmarterCSV_1.16.0_vs_RubyCSV_3.3.5_speedup.svg
+- images/SmarterCSV_1.16.0_vs_previous_C-speedup.png
+- images/SmarterCSV_1.16.0_vs_previous_C-speedup.svg
+- images/SmarterCSV_1.16.0_vs_previous_Rb-speedup.png
+- images/SmarterCSV_1.16.0_vs_previous_Rb-speedup.svg
 - lib/smarter_csv.rb
 - lib/smarter_csv/auto_detection.rb
 - lib/smarter_csv/errors.rb
@@ -142,6 +97,8 @@ metadata:
   homepage_uri: https://github.com/tilo/smarter_csv
   source_code_uri: https://github.com/tilo/smarter_csv
   changelog_uri: https://github.com/tilo/smarter_csv/blob/main/CHANGELOG.md
+  documentation_uri: https://github.com/tilo/smarter_csv/tree/main/docs
+  bug_tracker_uri: https://github.com/tilo/smarter_csv/issues
 rdoc_options: []
 require_paths:
 - lib
@@ -150,7 +107,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.5.0
+      version: 2.6.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -159,5 +116,6 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 rubygems_version: 4.0.6
 specification_version: 4
-summary: Convenient CSV Reading and Writing
+summary: Fastest end-to-end CSV ingestion for Ruby with smart defaults and Rails-ready
+  hash output
 test_files: []