smarter_csv 1.17.4 → 1.18.0

data/lib/smarter_csv/hash_transformations.rb

@@ -3,10 +3,11 @@
 module SmarterCSV
   module HashTransformations
     # Frozen regex constants for performance (avoid recompilation on every value)
-    NUMERIC_REGEX = /\A[+-]?\d+(?:\.\d+)?\z/.freeze
+    NUMERIC_REGEX = /\A[+-]?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?\z/.freeze
     # FLOAT_REGEX = /\A[+-]?\d+\.\d+\z/.freeze
     # INTEGER_REGEX = /\A[+-]?\d+\z/.freeze
     ZERO_REGEX = /\A[+-]?0+(?:\.0+)?\z/.freeze # could be +0.0
+    EXPONENT_CHARS = %w[e E].freeze # mantissa scan stops here in significant_digits
 
     # First-byte values that can begin a numeric literal — used to skip the numeric
     # regexes for values that obviously aren't numbers (e.g. city names).
@@ -70,7 +71,13 @@ module SmarterCSV
           first_byte = v.getbyte(0)
           if first_byte && ((first_byte >= ZERO_BYTE && first_byte <= NINE_BYTE) || first_byte == MINUS_BYTE || first_byte == PLUS_BYTE)
             if NUMERIC_REGEX.match?(v)
-              hash[k] = v.include?('.') ? v.to_f : v.to_i
+              # A value with a '.' or an exponent is a decimal → honor decimal_precision;
+              # otherwise it's an integer.
+              hash[k] = if v.include?('.') || v.include?('e') || v.include?('E')
+                          convert_decimal(v, options[:decimal_precision])
+                        else
+                          v.to_i
+                        end
             end
           end
         end
@@ -121,6 +128,48 @@ module SmarterCSV
 
     protected
 
+    # Convert a decimal string (has a '.' or an exponent) to a numeric, honoring
+    # decimal_precision: :float -> Float, :bigdecimal -> BigDecimal, :auto -> Float unless
+    # the value carries more than 16 significant digits (then BigDecimal, no precision loss).
+    def convert_decimal(str, decimal_precision)
+      case decimal_precision
+      when :float
+        str.to_f
+      when :bigdecimal
+        BigDecimal(str)
+      else # :auto
+        # A float token always has a '.' or 'e', so a token of <= 17 bytes holds at most
+        # 16 digits and therefore <= 16 significant digits — skip the per-char scan and go
+        # straight to Float (the common case: coordinates, sensor readings, prices). Only
+        # longer tokens can reach the BigDecimal threshold, so pay for the scan only then.
+        if str.bytesize > 17 && significant_digits(str) > 16
+          BigDecimal(str)
+        else
+          str.to_f
+        end
+      end
+    end
+
+    # Count significant mantissa digits (leading zeros excluded, trailing and fraction
+    # digits included, exponent excluded). Matches the C path's fj_sig_digits / Oj's dec_cnt
+    # so :auto picks Float vs BigDecimal identically on both paths.
+    def significant_digits(str)
+      cnt = 0
+      started = false
+      str.each_char do |c|
+        break if EXPONENT_CHARS.include?(c)
+        next unless c >= '0' && c <= '9'
+
+        if started
+          cnt += 1
+        elsif c != '0'
+          started = true
+          cnt = 1
+        end
+      end
+      cnt
+    end
+
     # acts as a road-block to limit processing when iterating over all k/v pairs of a CSV-hash:
     def limit_execution_for_only_or_except(options, option_name, key)
       if options[option_name].is_a?(Hash)

data/lib/smarter_csv/reader_options.rb

@@ -17,6 +17,7 @@ module SmarterCSV
         collect_raw_lines: true,
         comment_regexp: nil, # was: /\A#/,
         convert_values_to_numeric: true,
+        decimal_precision: :auto, # :auto (Float, but BigDecimal above 16 significant digits), :float, or :bigdecimal
         downcase_header: true,
         duplicate_header_suffix: '', # was: nil,
         field_size_limit: nil, # Integer (bytes) or nil for no limit. Raises FieldSizeLimitExceeded if any
@@ -58,6 +59,12 @@ module SmarterCSV
         with_line_numbers: false,
       }.freeze
 
+      # Options whose canonical value is one of a fixed set of symbols. A string form
+      # (e.g. "backslash" from options round-tripped through JSON or YAML) is coerced to
+      # the matching symbol. Non-string values (a callable for on_bad_row, true/false for
+      # legacy verbose) pass through untouched.
+      SYMBOL_VALUE_OPTIONS = %i[quote_escaping quote_boundary missing_headers on_bad_row verbose decimal_precision].freeze
+
       # NOTE: this is not called when "parse" methods are tested by themselves
       def process_options(given_options = {})
         # Debug output before merge — check raw verbose value (true or :debug)
@@ -77,6 +84,10 @@ module SmarterCSV
 
         @options = DEFAULT_OPTIONS.dup.merge!(given_options)
 
+        # Symbol/string interchangeability: accept either form for every option whose
+        # value is a symbol or a string. Done once here, before any value is read below.
+        normalize_option_value_types!(@options)
+
         # Normalize verbose to a symbol — done once here, stored back into @options.
         # All subsequent checks are free symbol comparisons; no re-evaluation needed.
         #   :quiet  — suppress all warnings and notices (good for production)
@@ -194,6 +205,9 @@ module SmarterCSV
         unless %i[legacy standard].include?(options[:quote_boundary])
           errors << "invalid quote_boundary: must be :legacy or :standard"
         end
+        unless %i[auto float bigdecimal].include?(options[:decimal_precision])
+          errors << "invalid decimal_precision: must be :auto, :float, or :bigdecimal"
+        end
         arc = options[:auto_row_sep_chars]
         min_arc = SmarterCSV::AutoDetection::MIN_AUTO_ROW_SEP_CHARS
         max_arc = SmarterCSV::AutoDetection::MAX_AUTO_ROW_SEP_CHARS
@@ -268,6 +282,16 @@ module SmarterCSV
         raise SmarterCSV::ValidationError, errors.inspect if errors.any?
       end
 
+      # Accept either a symbol or a string for every option whose value is one or the
+      # other, so callers limited to strings (JSON/YAML) behave the same as those passing
+      # symbols, and vice versa. Validation of the resulting value happens later.
+      def normalize_option_value_types!(options)
+        SYMBOL_VALUE_OPTIONS.each do |key|
+          v = options[key]
+          options[key] = v.to_sym if v.is_a?(String)
+        end
+      end
+
       def option_valid?(str)
         return true if str.is_a?(Symbol) && str == :auto
         return true if str.is_a?(String) && !str.empty?

data/lib/smarter_csv/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SmarterCSV
-  VERSION = "1.17.4"
+  VERSION = "1.18.0"
 end

data/lib/smarter_csv.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'stringio'
+require 'bigdecimal' # for decimal_precision: :auto / :bigdecimal
 require "smarter_csv/version"
 require "smarter_csv/errors"
 

data/smarter_csv.gemspec

@@ -40,6 +40,9 @@ Gem::Specification.new do |spec|
 
   spec.required_ruby_version = ">= 2.6.0"
 
+  # bigdecimal is no longer a default gem on Ruby 3.4+; needed for decimal_precision: :auto / :bigdecimal
+  spec.add_dependency "bigdecimal"
+
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
   spec.files = Dir.chdir(__dir__) do

metadata

@@ -1,14 +1,28 @@
 --- !ruby/object:Gem::Specification
 name: smarter_csv
 version: !ruby/object:Gem::Version
-  version: 1.17.4
+  version: 1.18.0
 platform: ruby
 authors:
 - Tilo Sloboda
 bindir: bin
 cert_chain: []
-date: 2026-06-03 00:00:00.000000000 Z
-dependencies: []
+date: 2026-06-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bigdecimal
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: |
   SmarterCSV is a high-performance CSV reader and writer for Ruby focused on
   fastest end-to-end ingestion — not just parsing. It returns ready-to-use
@@ -71,6 +85,10 @@ files:
 - docs/warnings.md
 - ext/smarter_csv/extconf.rb
 - ext/smarter_csv/smarter_csv.c
+- ext/smarter_csv/vendor/LICENSE-fast_float-MIT
+- ext/smarter_csv/vendor/eisel_lemire.h
+- ext/smarter_csv/vendor/eisel_lemire.md
+- ext/smarter_csv/vendor/eisel_lemire_powers.h
 - images/SmarterCSV_1.16.0_vs_RubyCSV_3.3.5_speedup.png
 - images/SmarterCSV_1.16.0_vs_RubyCSV_3.3.5_speedup.svg
 - images/SmarterCSV_1.16.0_vs_previous_C-speedup.png
@@ -122,7 +140,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.11
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Fastest end-to-end CSV ingestion for Ruby with smart defaults and Rails-ready
   hash output