smarter_csv 1.14.4 → 1.15.0

data/lib/smarter_csv/hash_transformations.rb

@@ -2,76 +2,98 @@
 
 module SmarterCSV
   module HashTransformations
+    # Frozen regex constants for performance (avoid recompilation on every value)
+    FLOAT_REGEX = /\A[+-]?\d+\.\d+\z/.freeze
+    INTEGER_REGEX = /\A[+-]?\d+\z/.freeze
+    ZERO_REGEX = /\A0+(?:\.0+)?\z/.freeze
+
     def hash_transformations(hash, options)
-      # there may be unmapped keys, or keys purposedly mapped to nil or an empty key..
-      # make sure we delete any key/value pairs from the hash, which the user wanted to delete:
+      # Modify hash in-place for performance (avoids allocating a second hash per row)
+
+      # Remove nil/empty keys
+      hash.delete(nil)
+      hash.delete('')
+      hash.delete(:"")
+
       remove_empty_values = options[:remove_empty_values] == true
       remove_zero_values = options[:remove_zero_values]
       remove_values_matching = options[:remove_values_matching]
       convert_to_numeric = options[:convert_values_to_numeric]
       value_converters = options[:value_converters]
 
-      hash.each_with_object({}) do |(k, v), new_hash|
-        next if k.nil? || k == '' || k == :""
-        next if remove_empty_values && (has_rails ? v.blank? : blank?(v))
-        next if remove_zero_values && v.is_a?(String) && v =~ /^(0+|0+\.0+)$/ # values are Strings
-        next if remove_values_matching && v =~ remove_values_matching
-
-        # deal with the :only / :except options to :convert_values_to_numeric
-        if convert_to_numeric && !limit_execution_for_only_or_except(options, :convert_values_to_numeric, k)
-          if v =~ /^[+-]?\d+\.\d+$/
-            v = v.to_f
-          elsif v =~ /^[+-]?\d+$/
-            v = v.to_i
-          end
-        end
+      # Early return if no transformations needed
+      return hash unless remove_empty_values || remove_zero_values || remove_values_matching || convert_to_numeric || value_converters
 
-        converter = value_converters[k] if value_converters
-        v = converter.convert(v) if converter
+      keys_to_delete = []
 
-        new_hash[k] = v
-      end
-    end
+      hash.each do |k, v|
+        # Check if this key/value should be removed
+        if remove_empty_values && (has_rails ? v.blank? : blank?(v))
+          keys_to_delete << k
+          next
+        end
 
-    # def hash_transformations(hash, options)
-    #   # there may be unmapped keys, or keys purposedly mapped to nil or an empty key..
-    #   # make sure we delete any key/value pairs from the hash, which the user wanted to delete:
-    #   hash.delete(nil)
-    #   hash.delete('')
-    #   hash.delete(:"")
-
-    #   if options[:remove_empty_values] == true
-    #     hash.delete_if{|_k, v| has_rails ? v.blank? : blank?(v)}
-    #   end
+        if remove_zero_values && v.is_a?(String) && ZERO_REGEX.match?(v)
+          keys_to_delete << k
+          next
+        end
+
+        if remove_values_matching && v.is_a?(String) && remove_values_matching.match?(v)
+          keys_to_delete << k
+          next
+        end
 
-    #   hash.delete_if{|_k, v| !v.nil? && v =~ /^(0+|0+\.0+)$/} if options[:remove_zero_values] # values are Strings
-    #   hash.delete_if{|_k, v| v =~ options[:remove_values_matching]} if options[:remove_values_matching]
+        # Convert to numeric if requested
+        if convert_to_numeric && v.is_a?(String) && !limit_execution_for_only_or_except(options, :convert_values_to_numeric, k)
+          if FLOAT_REGEX.match?(v)
+            hash[k] = v.to_f
+          elsif INTEGER_REGEX.match?(v)
+            hash[k] = v.to_i
+          end
+        end
 
-    #   if options[:convert_values_to_numeric]
-    #     hash.each do |k, v|
-    #       # deal with the :only / :except options to :convert_values_to_numeric
-    #       next if limit_execution_for_only_or_except(options, :convert_values_to_numeric, k)
+        # Apply value converters
+        if value_converters
+          converter = value_converters[k]
+          hash[k] = converter.convert(hash[k]) if converter
+        end
+      end
 
-    #       # convert if it's a numeric value:
-    #       case v
-    #       when /^[+-]?\d+\.\d+$/
-    #         hash[k] = v.to_f
-    #       when /^[+-]?\d+$/
-    #         hash[k] = v.to_i
-    #       end
-    #     end
-    #   end
+      # Delete marked keys
+      keys_to_delete.each { |k| hash.delete(k) }
 
-    #   if options[:value_converters]
-    #     hash.each do |k, v|
-    #       converter = options[:value_converters][k]
-    #       next unless converter
+      hash
+    end
 
-    #       hash[k] = converter.convert(v)
+    # ORIGINAL each_with_object implementation (replaced with in-place modification above)
+    # def hash_transformations(hash, options)
+    #   remove_empty_values = options[:remove_empty_values] == true
+    #   remove_zero_values = options[:remove_zero_values]
+    #   remove_values_matching = options[:remove_values_matching]
+    #   convert_to_numeric = options[:convert_values_to_numeric]
+    #   value_converters = options[:value_converters]
+    #
+    #   hash.each_with_object({}) do |(k, v), new_hash|
+    #     next if k.nil? || k == '' || k == :""
+    #     next if remove_empty_values && (has_rails ? v.blank? : blank?(v))
+    #     next if remove_zero_values && v.is_a?(String) && ZERO_REGEX.match?(v)
+    #     next if remove_values_matching && remove_values_matching.match?(v)
+    #
+    #     if convert_to_numeric && !limit_execution_for_only_or_except(options, :convert_values_to_numeric, k)
+    #       if v.is_a?(String)
+    #         if FLOAT_REGEX.match?(v)
+    #           v = v.to_f
+    #         elsif INTEGER_REGEX.match?(v)
+    #           v = v.to_i
+    #         end
+    #       end
     #     end
+    #
+    #     converter = value_converters[k] if value_converters
+    #     v = converter.convert(v) if converter
+    #
+    #     new_hash[k] = v
     #   end
-
-    #   hash
     # end
 
     protected

data/lib/smarter_csv/header_validations.rb

@@ -14,7 +14,7 @@ module SmarterCSV
       duplicates = header_counts.select { |_, count| count > 1 }
 
       unless duplicates.empty?
-        raise(SmarterCSV::DuplicateHeaders, "Duplicate Headers in CSV: #{duplicates.inspect}")
+        raise SmarterCSV::DuplicateHeaders.new("Duplicate Headers in CSV: #{duplicates.inspect}", duplicates.keys)
       end
     end
 
@@ -26,7 +26,7 @@ module SmarterCSV
         missing_keys = options[:required_keys].select { |k| !headers_set.include?(k) }
 
         unless missing_keys.empty?
-          raise SmarterCSV::MissingKeys, "ERROR: missing attributes: #{missing_keys.join(',')}. Check `reader.headers` for original headers."
+          raise SmarterCSV::MissingKeys.new("ERROR: missing attributes: #{missing_keys.join(',')}. Check `reader.headers` for original headers.", missing_keys)
         end
       end
     end

data/lib/smarter_csv/headers.rb

@@ -12,7 +12,7 @@ module SmarterCSV
       if options[:headers_in_file] # extract the header line
         # process the header line in the CSV file..
         # the first line of a CSV file contains the header .. it might be commented out, so we need to read it anyhow
-        header_line = @raw_header = readline_with_counts(filehandle, options)
+        header_line = @raw_header = next_line_with_counts(filehandle, options)
         header_line = preprocess_header_line(header_line, options)
 
         file_header_array, file_header_size = parse(header_line, options)

data/lib/smarter_csv/parser.rb

@@ -2,7 +2,7 @@
 
 module SmarterCSV
   module Parser
-    EMPTY_STRING = ''.freeze
+    EMPTY_STRING = '' # already frozen
 
     protected
 
@@ -26,6 +26,64 @@ module SmarterCSV
       end
     end
 
+    # Parse a CSV line directly into a hash, with support for extra columns.
+    # Returns [hash_or_nil, data_size] where hash is nil if all values are blank.
+    def parse_line_to_hash(line, headers, options)
+      has_quotes = line.include?(options[:quote_char])
+
+      if options[:acceleration] && has_acceleration
+        # :nocov:
+        parse_line_to_hash_c(
+          line,
+          headers,
+          options[:col_sep],
+          options[:quote_char],
+          options[:missing_header_prefix],
+          has_quotes,
+          options[:strip_whitespace],
+          options[:remove_empty_hashes],
+          options[:remove_empty_values]
+        )
+        # :nocov:
+      else
+        parse_line_to_hash_ruby(line, headers, options, has_quotes)
+      end
+    end
+
+    # Ruby implementation of parse_line_to_hash
+    def parse_line_to_hash_ruby(line, headers, options, has_quotes = false)
+      return [nil, 0] if line.nil?
+
+      # Parse the line into values
+      elements, data_size = parse_csv_line_ruby(line, options, nil, has_quotes)
+
+      # Check if all values are blank
+      if options[:remove_empty_hashes] && (elements.empty? || elements.all? { |v| v.nil? || v.to_s.strip.empty? })
+        return [nil, data_size]
+      end
+
+      # Build the hash - only include keys for values that exist
+      hash = {}
+      elements.each_with_index do |value, i|
+        key = if i < headers.size
+                headers[i]
+              else
+                "#{options[:missing_header_prefix]}#{i + 1}".to_sym
+              end
+        hash[key] = value
+      end
+
+      # Add nil for missing columns only when remove_empty_values is false
+      # (when true, nils would be removed anyway by hash_transformations)
+      unless options[:remove_empty_values]
+        (elements.size...headers.size).each do |i|
+          hash[headers[i]] = nil
+        end
+      end
+
+      [hash, data_size]
+    end
+
     # ------------------------------------------------------------------
     # Ruby equivalent of the C-extension for parse_line
     #
@@ -47,7 +105,7 @@ module SmarterCSV
     #
     # Our convention is that empty fields are returned as empty strings, not as nil.
 
-    def parse_csv_line_ruby(line, options, header_size = nil, has_quotes = false)
+    def parse_csv_line_ruby(line, options, header_size = nil, _has_quotes = false)
       return [[], 0] if line.nil?
 
       line_size = line.size

data/lib/smarter_csv/reader.rb

@@ -81,9 +81,7 @@ module SmarterCSV
         end
 
         # now on to processing all the rest of the lines in the CSV file:
-        # fh.each_line |line|
-        until fh.eof? # we can't use fh.readlines() here, because this would read the whole file into memory at once, and eof => true
-          line = readline_with_counts(fh, options)
+        while (line = next_line_with_counts(fh, options))
 
           # replace invalid byte sequence in UTF-8 with question mark to avoid errors
           line = enforce_utf8_encoding(line, options) if @enforce_utf8
@@ -98,25 +96,22 @@ module SmarterCSV
           multiline = count_quote_chars(line, options[:quote_char]).odd?
 
           while multiline
-            begin
-              next_line = fh.readline(options[:row_sep])
-              next_line = enforce_utf8_encoding(next_line, options) if @enforce_utf8
-              line += next_line
-              @file_line_count += 1
-
-              multiline = count_quote_chars(line, options[:quote_char]).odd?
-            rescue EOFError
+            next_line = fh.gets(options[:row_sep])
+            if next_line.nil?
               # End of file reached. Check if quotes are balanced.
               total_quotes = count_quote_chars(line, options[:quote_char])
               if total_quotes.odd?
                 raise MalformedCSV, "Unclosed quoted field detected in multiline data"
               else
                 # Quotes are balanced; proceed without raising an error.
-                # :nocov:
                 break
-                # :nocov:
               end
             end
+            next_line = enforce_utf8_encoding(next_line, options) if @enforce_utf8
+            line += next_line
+            @file_line_count += 1
+
+            multiline = count_quote_chars(line, options[:quote_char]).odd?
           end
 
           # :nocov:
@@ -129,24 +124,25 @@ module SmarterCSV
 
           # --- SPLIT LINE & DATA TRANSFORMATIONS ------------------------------------------------------------
           # we are now stripping whitespace inside the parse() methods
-          dataA, data_size = parse(line, options) # we parse the extra columns
+          # we create additional columns on-the-fly when we find more data fields than headers
+          hash, data_size = parse_line_to_hash(line, @headers, options)
 
-          if options[:strict]
-            raise SmarterCSV::HeaderSizeMismatch, "extra columns detected on line #{@file_line_count}"
-          else
-            # we create additional columns on-the-fly
-            current_size = @headers.size
-            while current_size < data_size
-              @headers << "#{options[:missing_header_prefix]}#{current_size + 1}".to_sym
-              current_size += 1
+          # Handle extra columns (more data fields than headers)
+          if data_size > @headers.size
+            if options[:strict]
+              raise SmarterCSV::HeaderSizeMismatch, "extra columns detected on line #{@file_line_count}"
+            end
+
+            # Update headers array for subsequent rows
+            while @headers.size < data_size
+              @headers << "#{options[:missing_header_prefix]}#{@headers.size + 1}".to_sym
             end
           end
 
-          # if all values are blank, then ignore this line
-          next if options[:remove_empty_hashes] && (dataA.empty? || blank?(dataA))
+          # if all values were blank (hash is nil) we ignore this CSV line
+          next if hash.nil?
 
           # --- HASH TRANSFORMATIONS ------------------------------------------------------------
-          hash = @headers.zip(dataA).to_h
 
           hash = hash_transformations(hash, options)
 
@@ -170,7 +166,7 @@ module SmarterCSV
             if chunk.size >= chunk_size || fh.eof? # if chunk if full, or EOF reached
               # do something with the chunk
               if block_given?
-                yield chunk # do something with the hashes in the chunk in the block
+                yield chunk, @chunk_count # do something with the hashes in the chunk in the block
               else
                 @result << chunk.dup # Append chunk to result (use .dup to keep a copy after we do chunk.clear)
               end
@@ -183,7 +179,8 @@ module SmarterCSV
 
           else # no chunk handling
             if block_given?
-              yield [hash] # do something with the hash in the block (better to use chunking here)
+              yield [hash], @chunk_count # do something with the hash in the block (better to use chunking here)
+              @chunk_count += 1
             else
               @result << hash
             end
@@ -197,7 +194,7 @@ module SmarterCSV
         if !chunk.nil? && chunk.size > 0
           # do something with the chunk
           if block_given?
-            yield chunk # do something with the hashes in the chunk in the block
+            yield chunk, @chunk_count # do something with the hashes in the chunk in the block
           else
             @result << chunk.dup # Append chunk to result (use .dup to keep a copy after we do chunk.clear)
           end
@@ -218,6 +215,12 @@ module SmarterCSV
     def count_quote_chars(line, quote_char)
       return 0 if line.nil? || quote_char.nil? || quote_char.empty?
 
+      # Use C extension for performance if available (avoids creating a String object per character)
+      if @has_acceleration && SmarterCSV::Parser.respond_to?(:count_quote_chars_c)
+        return SmarterCSV::Parser.count_quote_chars_c(line, quote_char)
+      end
+
+      # Fallback to Ruby implementation
       count = 0
       escaped = false
 

data/lib/smarter_csv/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SmarterCSV
-  VERSION = "1.14.4"
+  VERSION = "1.15.0"
 end

data/smarter_csv.gemspec

@@ -36,7 +36,6 @@ Gem::Specification.new do |spec|
   spec.files += Dir.glob("ext/smarter_csv/**/*")
 
   spec.add_development_dependency "awesome_print"
-  spec.add_development_dependency "codecov"
   spec.add_development_dependency "pry"
   spec.add_development_dependency "rspec"
   spec.add_development_dependency "rubocop"

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: smarter_csv
 version: !ruby/object:Gem::Version
-  version: 1.14.4
+  version: 1.15.0
 platform: ruby
 authors:
 - Tilo Sloboda
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-05-29 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: awesome_print
@@ -24,20 +23,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: codecov
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -127,8 +112,14 @@ files:
 - docs/options.md
 - docs/row_col_sep.md
 - docs/value_converters.md
+- ext/smarter_csv/Makefile
 - ext/smarter_csv/extconf.rb
+- ext/smarter_csv/smarter_csv.bundle
+- ext/smarter_csv/smarter_csv.bundle.dSYM/Contents/Info.plist
+- ext/smarter_csv/smarter_csv.bundle.dSYM/Contents/Resources/DWARF/smarter_csv.bundle
+- ext/smarter_csv/smarter_csv.bundle.dSYM/Contents/Resources/Relocations/aarch64/smarter_csv.bundle.yml
 - ext/smarter_csv/smarter_csv.c
+- ext/smarter_csv/smarter_csv.o
 - lib/smarter_csv.rb
 - lib/smarter_csv/auto_detection.rb
 - lib/smarter_csv/errors.rb
@@ -150,7 +141,6 @@ metadata:
   homepage_uri: https://github.com/tilo/smarter_csv
   source_code_uri: https://github.com/tilo/smarter_csv
   changelog_uri: https://github.com/tilo/smarter_csv/blob/main/CHANGELOG.md
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -166,8 +156,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.4
-signing_key:
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Convenient CSV Reading and Writing
 test_files: []