smarter_csv 1.8.0 → 1.8.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 55400b3977ce35c58d60c4101362b68d99f2dbf7cb6a63956ae3b6ab79fcf1ac
-  data.tar.gz: 41f46d3e4de69a7924ecd2214ba4e37766106469d1b8b257fd752a96204a47fd
+  metadata.gz: 4a6ec6f3a579d9c1e6bfc2c3c9006f64d8c7b705eeca6ec048ea56c688f8ea1c
+  data.tar.gz: ba9a4a289adcc2fc398ae608f9570c28baac57b877852f3ea37c78fa57f2d7e3
 SHA512:
-  metadata.gz: 24ecc14cf9c65efe5c11e4bd20753420aa8ccd7385171cd21eac2e1be92c4896087cdc2a18799fa111c0f36154ad4481daed7f08b752f4fae2b5f27241b8cf6c
-  data.tar.gz: c1d70e18a7ae8057e58cbf73b62f4896dd7030bc5fd2e927669e5ea829f9a3c11daeb9c8b83296dbb46e6f0d23034245b7207882a77b54cc1ca128a581175359
+  metadata.gz: d8f516501a5539e30789e2d18c4d051f50372786d8df1272192c2bc7997470cf5d5e1ae94b776d0d580cb62ed8ffb0f6591ccc2be5d60eae6e421f22f0c92f94
+  data.tar.gz: 2993c59278adb531cf2299c0aa3869c637868f2f4e6421f845e4ade35011f94ee6e226f43e07b206c67c354c2eb38c0a41981ffbdbff00950690b94b06b2aacd

data/CHANGELOG.md

@@ -1,8 +1,21 @@
 
 # SmarterCSV 1.x Change Log
 
-## 1.8.0 (2023-03-18)
+## 1.8.2 (2023-03-21)
+  * bugfix: do not raise `NoColSepDetected` for CSV files with only one column in most cases (issue #222)
+            If the first lines contain non-ASCII characters, and no col_sep is detected, it will still raise `NoColSepDetected`
+
+## 1.8.1 (2023-03-19)
+  * added validation against invalid values for :col_sep, :row_sep, :quote_char (issue #216)
+  * deprecating `required_headers` and replace with `required_keys` (issue #140)
+  * fixed issue with require statement
+
+## 1.8.0 (2023-03-18) BREAKING
   * NEW DEFAULTS: `col_sep: :auto`, `row_sep: :auto`. Fully automatic detection by default.
+    
+    MAKE SURE to rescue `NoColSepDetected` if your CSV files can have unexpected formats, 
+              e.g. from users uploading them to a service, and handle those cases.
+
   * ignore Byte Order Marker (BOM) in first line in file (issues #27, #219)
 
 ## 1.7.4 (2023-01-13)

data/README.md

@@ -73,6 +73,15 @@ $ hexdump -C spec/fixtures/bom_test_feff.csv
 00000040  73 2c 35 36 37 38 0d 0a                           |s,5678..|
 ```
 
+### Examples
+
+Here are some examples to demonstrate the versatility of SmarterCSV.
+
+**It is generally recommended to rescue `SmarterCSVException` or it's sub-classes.**
+
+By default SmarterCSV determines the `row_sep` and `col_sep` values automatically. In cases where the automatic detection fails, an exception will be raised, e.g. `NoColSepDetected`. Rescuing from these exceptions will make sure that you don't miss processing CSV files, in case users upload CSV files with unexpected formats.
+
+In rare cases you may have to manually set these values, after going through the troubleshooting procedure described above.
 
 #### Example 1a: How SmarterCSV processes CSV-files as array of hashes:
 Please note how each hash contains only the keys for columns with non-null values.
@@ -267,7 +276,8 @@ And header and data validations will also be supported in 2.x
      ---------------------------------------------------------------------------------------------------------------------------------
      | :key_mapping                |   nil    | a hash which maps headers from the CSV file to keys in the result hash               |
      | :silence_missing_key        |   false  | ignore missing keys in `key_mapping` if true                                         |
-     | :required_headers           |   nil    | An array. Each of the given headers must be present after header manipulation,       |
+     | :required_keys              |   nil    | An array. Specify the required names AFTER header transformation.                  |
+     | :required_headers           |   nil    | (DEPRECATED / renamed) Use `required_keys` instead                          |
      |                             |          | or an exception is raised   No validation if nil is given.                           |
      | :remove_unmapped_keys       |   false  | when using :key_mapping option, should non-mapped keys / columns be removed?         |
      | :downcase_header            |   true   | downcase all column headers                                                          |

data/TO_DO_v2.md

@@ -0,0 +1,14 @@
+# SmarterCSV v2.0 TO DO List
+
+* add enumerable to speed up parallel processing [issue #66](https://github.com/tilo/smarter_csv/issues/66), [issue #32](https://github.com/tilo/smarter_csv/issues/32)
+* use Procs for validations and transformatoins [issue #118](https://github.com/tilo/smarter_csv/issues/118)
+* make @errors and @warnings work [issue #118](https://github.com/tilo/smarter_csv/issues/118)
+* skip file opening, allow reading from CSV string, e.g. reading from S3 file [issue #120](https://github.com/tilo/smarter_csv/issues/120).
+  Or stream large file from S3 (linked in the issue)
+* Collect all Errors, before surfacing them. Avoid throwing an exception on the first error [issue #133](https://github.com/tilo/smarter_csv/issues/133)
+* Don't call rewind on filehandle
+* [2.0 BUG] :convert_values_to_numeric_unless_leading_zeros drops leading zeros [issue #151](https://github.com/tilo/smarter_csv/issues/151)
+* [2.0 BUG]  convert_to_float saves Proc as @@convert_to_integer [issue #157](https://github.com/tilo/smarter_csv/issues/157)
+* Provide an example for custom Procs for hash_transformations in the docs [issue #174](https://github.com/tilo/smarter_csv/issues/174)
+* Replace remove_empty_values: false [issue #213](https://github.com/tilo/smarter_csv/issues/213)
+

data/ext/smarter_csv/smarter_csv.c

@@ -27,7 +27,6 @@ static VALUE rb_parse_csv_line(VALUE self, VALUE line, VALUE col_sep, VALUE quot
     long col_sep_len = RSTRING_LEN(col_sep);
 
     char *quoteP = RSTRING_PTR(quote_char);
-    long quote_len = RSTRING_LEN(quote_char);
     long quote_count = 0;
 
     bool col_sep_found = true;

data/lib/smarter_csv/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SmarterCSV
-  VERSION = "1.8.0"
+  VERSION = "1.8.2"
 end

data/lib/smarter_csv.rb

@@ -3,24 +3,25 @@
 require_relative "extensions/hash"
 require_relative "smarter_csv/version"
 
-# require_relative "smarter_csv/smarter_csv" unless ENV['CI'] # does not compile/link in CI?
-require 'smarter_csv.bundle' unless ENV['CI'] # does not compile/link in CI?
+require_relative "smarter_csv/smarter_csv" unless ENV['CI'] # does not compile/link in CI?
+# require 'smarter_csv.bundle' unless ENV['CI'] # local testing
 
 module SmarterCSV
   class SmarterCSVException < StandardError; end
   class HeaderSizeMismatch < SmarterCSVException; end
   class IncorrectOption < SmarterCSVException; end
+  class ValidationError < SmarterCSVException; end
   class DuplicateHeaders < SmarterCSVException; end
   class MissingHeaders < SmarterCSVException; end
   class NoColSepDetected < SmarterCSVException; end
-  class KeyMappingError < SmarterCSVException; end
-  class MalformedCSVError < SmarterCSVException; end
+  class KeyMappingError < SmarterCSVException; end # CURRENTLY UNUSED -> version 1.9.0
 
   # first parameter: filename or input object which responds to readline method
   def SmarterCSV.process(input, options = {}, &block)
     options = default_options.merge(options)
     options[:invalid_byte_sequence] = '' if options[:invalid_byte_sequence].nil?
     puts "SmarterCSV OPTIONS: #{options.inspect}" if options[:verbose]
+    validate_options!(options)
 
     headerA = []
     result = []
@@ -214,7 +215,7 @@ module SmarterCSV
         headers_in_file: true,
         invalid_byte_sequence: '',
         keep_original_headers: false,
-        key_mapping_hash: nil,
+        key_mapping: nil,
         quote_char: '"',
         remove_empty_hashes: true,
         remove_empty_values: true,
@@ -222,6 +223,7 @@ module SmarterCSV
         remove_values_matching: nil,
         remove_zero_values: false,
         required_headers: nil,
+        required_keys: nil,
         row_sep: :auto, # was: $/,
         silence_missing_keys: false,
         skip_lines: nil,
@@ -391,15 +393,28 @@ module SmarterCSV
     def guess_column_separator(filehandle, options)
       skip_lines(filehandle, options)
 
-      possible_delimiters = [',', "\t", ';', ':', '|']
+      delimiters = [',', "\t", ';', ':', '|']
 
-      candidates = if options.fetch(:headers_in_file)
-                     candidated_column_separators_from_headers(filehandle, options, possible_delimiters)
-                   else
-                     candidated_column_separators_from_contents(filehandle, options, possible_delimiters)
-                   end
+      line = nil
+      has_header = options[:headers_in_file]
+      candidates = Hash.new(0)
+      count = has_header ? 1 : 5
+      count.times do
+        line = readline_with_counts(filehandle, options)
+        delimiters.each do |d|
+          candidates[d] += line.scan(d).count
+        end
+      rescue EOFError # short files
+        break
+      end
+      rewind(filehandle)
 
-      raise SmarterCSV::NoColSepDetected if candidates.values.max == 0
+      if candidates.values.max == 0
+        # if the header only contains
+        return ',' if line =~ /^\w+$/
+
+        raise SmarterCSV::NoColSepDetected
+      end
 
       candidates.key(candidates.values.max)
     end
@@ -486,13 +501,13 @@ module SmarterCSV
 
       unless options[:user_provided_headers] # wouldn't make sense to re-map user provided headers
         key_mappingH = options[:key_mapping]
+
         # do some key mapping on the keys in the file header
         #   if you want to completely delete a key, then map it to nil or to ''
         if !key_mappingH.nil? && key_mappingH.class == Hash && key_mappingH.keys.size > 0
           unless options[:silence_missing_keys]
             # if silence_missing_keys are not set, raise error if missing header
             missing_keys = key_mappingH.keys - headerA
-
             puts "WARNING: missing header(s): #{missing_keys.join(",")}" unless missing_keys.empty?
           end
 
@@ -510,12 +525,21 @@ module SmarterCSV
         raise SmarterCSV::DuplicateHeaders, "ERROR: duplicate headers: #{duplicate_headers.join(',')}"
       end
 
-      if options[:required_headers] && options[:required_headers].is_a?(Array)
-        missing_headers = []
-        options[:required_headers].each do |k|
-          missing_headers << k unless headerA.include?(k)
+      # deprecate required_headers
+      if !options[:required_headers].nil?
+        puts "DEPRECATION WARNING: please use 'required_keys' instead of 'required headers'"
+        if options[:required_keys].nil?
+          options[:required_keys] = options[:required_headers]
+          options[:required_headers] = nil
         end
-        raise SmarterCSV::MissingHeaders, "ERROR: missing headers: #{missing_headers.join(',')}" unless missing_headers.empty?
+      end
+
+      if options[:required_keys] && options[:required_keys].is_a?(Array)
+        missing_keys = []
+        options[:required_keys].each do |k|
+          missing_keys << k unless headerA.include?(k)
+        end
+        raise SmarterCSV::MissingHeaders, "ERROR: missing attributes: #{missing_keys.join(',')}" unless missing_keys.empty?
       end
 
       @headers = headerA
@@ -546,7 +570,7 @@ module SmarterCSV
 
     def remove_bom(str)
       str_as_hex = str.bytes.map{|x| x.to_s(16)}
-      # if string does not start with one of the bytes above, there is no BOM
+      # if string does not start with one of the bytes, there is no BOM
       return str unless %w[ef fe ff 0].include?(str_as_hex[0])
 
       return str.byteslice(4..-1) if [UTF_32_BOM, UTF_32LE_BOM].include?(str_as_hex[0..3])
@@ -557,34 +581,19 @@ module SmarterCSV
       str
     end
 
-    def candidated_column_separators_from_headers(filehandle, options, delimiters)
-      candidates = Hash.new(0)
-      line = readline_with_counts(filehandle, options.slice(:row_sep))
-
-      delimiters.each do |d|
-        candidates[d] += line.scan(d).count
-      end
-
-      rewind(filehandle)
-
-      candidates
+    def validate_options!(options)
+      keys = options.keys
+      errors = []
+      errors << "invalid row_sep" if keys.include?(:row_sep) && !option_valid?(options[:row_sep])
+      errors << "invalid col_sep" if keys.include?(:col_sep) && !option_valid?(options[:col_sep])
+      errors << "invalid quote_char" if keys.include?(:quote_char) && !option_valid?(options[:quote_char])
+      raise SmarterCSV::ValidationError, errors.inspect if errors.any?
     end
 
-    def candidated_column_separators_from_contents(filehandle, options, delimiters)
-      candidates = Hash.new(0)
-
-      5.times do
-        line = readline_with_counts(filehandle, options.slice(:row_sep))
-        delimiters.each do |d|
-          candidates[d] += line.scan(d).count
-        end
-      rescue EOFError # short files
-        break
-      end
-
-      rewind(filehandle)
-
-      candidates
+    def option_valid?(str)
+      return true if str.is_a?(Symbol) && str == :auto
+      return true if str.is_a?(String) && !str.empty?
+      false
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: smarter_csv
 version: !ruby/object:Gem::Version
-  version: 1.8.0
+  version: 1.8.2
 platform: ruby
 authors:
 - Tilo Sloboda
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-03-19 00:00:00.000000000 Z
+date: 2023-03-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: awesome_print
@@ -112,6 +112,7 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- TO_DO_v2.md
 - ext/smarter_csv/extconf.rb
 - ext/smarter_csv/smarter_csv.c
 - lib/extensions/hash.rb