smarter_csv 1.7.1 → 1.7.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ab42c787dba96369ba0f499294d5421cc2fc9a514b1cff039e5017d19fd2ff4c
-  data.tar.gz: f3605e56395d498169c449657945c00c677a75d17e2a7cbcca1a4f4e65aa45f2
+  metadata.gz: d4046758f38c21262fdec6bc7e13e3a7811c7aee3944d92e0cc36a2a1cfb032a
+  data.tar.gz: 9d111e2f36171ca488034f3af73fc71c7c9f6fde73986d277aeaf1560a066fa2
 SHA512:
-  metadata.gz: a264574b464219fd53a862be345c008fc0a93c076345d62be4c75935a148f9a469ccbea14fb2f821cedaada1f2b4ec875468ef50b3ae173bcbb76a5079d43406
-  data.tar.gz: 978236fa33bf4656eba89d929545c81ec2dc28b179890dd4254fb04cfa9b8646f4040aa2a62d44361dea221714f1cba0d7e4c858d338d30080c4a0e43c003552
+  metadata.gz: c46c5c45dd3fafe66735b2b17b0679c5aaff27b3670140d97bc19e1c825ad91310fa2cf55a12a5c7b0c31ef82fe9cc12a2c4bda0a78b218d80ad5816c01c0d9f
+  data.tar.gz: ba03acd95955f8afeb8e96f16c7cfa2e1605dbaf6fddb7008930294aab83196aed21f57605efb3553799381c1c4811528eee2db221efa50dc82f58bcf9135842

data/CHANGELOG.md

@@ -1,9 +1,16 @@
 
 # SmarterCSV 1.x Change Log
 
+## 1.7.3 (2022-12-05)
+  * new option :silence_missing_keys; if set to true, it ignores missing keys in `key_mapping`
+
+## 1.7.2 (2022-08-29)
+  * new option :with_line_numbers; if set to true, it adds :csv_line_number to each data hash (issue #130)
+  
 ## 1.7.1 (2022-07-31)
-  * bugfix (thanks to Viacheslav Markin, Nicolas Rodriguez)
-## 1.7.0 (2022-06-26)
+  * bugfix for issue #195 #197 #200 which only appeared when called from Rails (thanks to Viacheslav Markin, Nicolas Rodriguez)
+
+## 1.7.0 (2022-06-26) (replaced by 1.7.1)
   * added native code to accellerate line parsing by >10x over 1.6.0
   * added option `acceleration`, defaulting to `true`, to enable native code.
     Disable this option to use the ruby code for line parsing.

data/CONTRIBUTORS.md

@@ -48,3 +48,4 @@ A Big Thank you to everyone who filed issues, sent comments, and who contributed
  * [Viacheslav Markin](https://github.com/KXEinc)
  * [Nicolas Rodriguez](https://github.com/n-rodriguez)
  * [Hirotaka Mizutani ](https://github.com/hirotaka)
+ * [Rahul Chaudhary](https://github.com/rahulch95)

data/README.md

@@ -1,9 +1,12 @@
- [![codecov](https://codecov.io/gh/tilo/smarter_csv/branch/main/graph/badge.svg?token=1L7OD80182)](https://codecov.io/gh/tilo/smarter_csv)
+
+# SmarterCSV
+
+ [![codecov](https://codecov.io/gh/tilo/smarter_csv/branch/main/graph/badge.svg?token=1L7OD80182)](https://codecov.io/gh/tilo/smarter_csv) [![Gem Version](https://badge.fury.io/rb/smarter_csv.svg)](http://badge.fury.io/rb/smarter_csv)
  
-#### Service Announcement
+#### Work towards Future Version 2.0
 
-* Work towards SmarterCSV 2.0 is still on it's way, with much improved features, and more streamlined options.
-  Please check the [2.0-develop branch](https://github.com/tilo/smarter_csv/blob/master/README.md), open any issues and pull requests with mention of v2.0.
+* Work towards SmarterCSV 2.0 is still ongoing, with improved features, and more streamlined options, but consider it as experimental at this time.
+  Please check the [2.0-develop branch](https://github.com/tilo/smarter_csv/tree/2.0-develop), open any issues and pull requests with mention of tag v2.0.
 
 * New versions of SmarterCSV 1.x will soon print a deprecation warning if you set :verbose to true
   See below for list of deprecated options.
@@ -15,11 +18,7 @@
 
 ---------------
 
-# SmarterCSV
-
-[![Build Status](https://secure.travis-ci.org/tilo/smarter_csv.svg?branch=master)](http://travis-ci.com/tilo/smarter_csv) [![Gem Version](https://badge.fury.io/rb/smarter_csv.svg)](http://badge.fury.io/rb/smarter_csv)
-
-#### SmarterCSV 1.x
+#### SmarterCSV 1.x [Current Version]
 
 `smarter_csv` is a Ruby Gem for smarter importing of CSV Files as Array(s) of Hashes, suitable for direct processing with ActiveRecord, parallel processing, or kicking-off batch jobs with Sidekiq.
 
@@ -132,7 +131,21 @@ and how the `process` method returns the number of chunks when called with a blo
 
      => returns number of chunks / rows we processed
 ```
-#### Example 4: Populate a MongoDB Database in Chunks of 100 records with SmarterCSV:
+
+#### Example 4: Reading a CSV-like File, and Processing it with Sidekiq:
+```ruby
+    filename = '/tmp/strange_db_dump'   # a file with CRTL-A as col_separator, and with CTRL-B\n as record_separator (hello iTunes!)
+    options = {
+      :col_sep => "\cA", :row_sep => "\cB\n", :comment_regexp => /^#/,
+      :chunk_size => 100 , :key_mapping => {:export_date => nil, :name => :genre}
+    }
+    n = SmarterCSV.process(filename, options) do |chunk|
+        SidekiqWorkerClass.process_async(chunk ) # pass an array of hashes to Sidekiq workers for parallel processing
+    end
+    => returns number of chunks
+```
+
+#### Example 5: Populate a MongoDB Database in Chunks of 100 records with SmarterCSV:
 ```ruby
     # using chunks:
     filename = '/tmp/some.csv'
@@ -146,18 +159,6 @@ and how the `process` method returns the number of chunks when called with a blo
      => returns number of chunks we processed
 ```
 
-#### Example 5: Reading a CSV-like File, and Processing it with Resque:
-```ruby
-    filename = '/tmp/strange_db_dump'   # a file with CRTL-A as col_separator, and with CTRL-B\n as record_separator (hello iTunes!)
-    options = {
-      :col_sep => "\cA", :row_sep => "\cB\n", :comment_regexp => /^#/,
-      :chunk_size => 100 , :key_mapping => {:export_date => nil, :name => :genre}
-    }
-    n = SmarterCSV.process(filename, options) do |chunk|
-        Resque.enque( ResqueWorkerClass, chunk ) # pass chunks of CSV-data to Resque workers for parallel processing
-    end
-    => returns number of chunks
-```
 #### Example 6: Using Value Converters
 
 NOTE: If you use `key_mappings` and `value_converters`, make sure that the value converters has references the keys based on the final mapped name, not the original name in the CSV file.
@@ -239,6 +240,7 @@ The options and the block are optional.
      |                             |          | You can not combine the :user_provided_headers and :key_mapping options              |
      | :remove_empty_hashes        |   true   | remove / ignore any hashes which don't have any key/value pairs or all empty values  |
      | :verbose                    |   false  | print out line number while processing (to track down problems in input files)       |
+     | :with_line_numbers          |   false  | add :csv_line_number to each data hash                                               |
      ---------------------------------------------------------------------------------------------------------------------------------
 
 #### Deprecated 1.x Options: to be replaced in 2.0
@@ -251,7 +253,8 @@ And header and data validations will also be supported in 2.x
      | Option                      | Default  |  Explanation                                                                         |
      ---------------------------------------------------------------------------------------------------------------------------------
      | :key_mapping                |   nil    | a hash which maps headers from the CSV file to keys in the result hash               |
-     | :required_headers           |   nil    | An array. Eacn of the given headers must be present after header manipulation,       |
+     | :silence_missing_key        |   false  | ignore missing keys in `key_mapping` if true                                         |
+     | :required_headers           |   nil    | An array. Each of the given headers must be present after header manipulation,       |
      |                             |          | or an exception is raised   No validation if nil is given.                           |
      | :remove_unmapped_keys       |   false  | when using :key_mapping option, should non-mapped keys / columns be removed?         |
      | :downcase_header            |   true   | downcase all column headers                                                          |

data/lib/smarter_csv/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SmarterCSV
-  VERSION = "1.7.1"
+  VERSION = "1.7.3"
 end

data/lib/smarter_csv.rb

@@ -2,7 +2,9 @@
 
 require_relative "extensions/hash"
 require_relative "smarter_csv/version"
+
 require_relative "smarter_csv/smarter_csv" unless ENV['CI'] # does not compile/link in CI?
+# require 'smarter_csv.bundle' unless ENV['CI'] # does not compile/link in CI?
 
 module SmarterCSV
   class SmarterCSVException < StandardError; end
@@ -129,6 +131,8 @@ module SmarterCSV
 
         next if options[:remove_empty_hashes] && hash.empty?
 
+        hash[:csv_line_number] = @csv_line_count if options[:with_line_numbers]
+
         if use_chunks
           chunk << hash # append temp result to chunk
 
@@ -223,6 +227,7 @@ module SmarterCSV
         remove_zero_values: false,
         required_headers: nil,
         row_sep: $/,
+        silence_missing_keys: false,
         skip_lines: nil,
         strings_as_keys: false,
         strip_chars_from_headers: nil,
@@ -230,6 +235,7 @@ module SmarterCSV
         user_provided_headers: nil,
         value_converters: nil,
         verbose: false,
+        with_line_numbers: false,
       }
     end
 
@@ -247,12 +253,12 @@ module SmarterCSV
       # puts "SmarterCSV.parse OPTIONS: #{options[:acceleration]}" if options[:verbose]
 
       if options[:acceleration] && has_acceleration?
-        # puts "NOTICE: Accelerated SmarterCSV / #{options[:acceleration]}" if options[:verbose]
+        # :nocov:
         has_quotes = line =~ /#{options[:quote_char]}/
         elements = parse_csv_line_c(line, options[:col_sep], options[:quote_char], header_size)
         elements.map!{|x| cleanup_quotes(x, options[:quote_char])} if has_quotes
         return [elements, elements.size]
-
+        # :nocov:
       else
         # puts "WARNING: SmarterCSV is using un-accelerated parsing of lines. Check options[:acceleration]"
         return parse_csv_line_ruby(line, options, header_size)
@@ -474,9 +480,11 @@ module SmarterCSV
         # do some key mapping on the keys in the file header
         #   if you want to completely delete a key, then map it to nil or to ''
         if !key_mappingH.nil? && key_mappingH.class == Hash && key_mappingH.keys.size > 0
-          # we can't map keys that are not there
-          missing_keys = key_mappingH.keys - headerA
-          puts "WARNING: missing header(s): #{missing_keys.join(",")}" unless missing_keys.empty?
+          unless options[:silence_missing_keys]
+            # if silence_missing_keys are not set, raise error if missing header
+            missing_keys = key_mappingH.keys - headerA
+            puts "WARNING: missing header(s): #{missing_keys.join(",")}" unless missing_keys.empty?
+          end
 
           headerA.map!{|x| key_mappingH.has_key?(x) ? (key_mappingH[x].nil? ? nil : key_mappingH[x]) : (options[:remove_unmapped_keys] ? nil : x)}
         end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: smarter_csv
 version: !ruby/object:Gem::Version
-  version: 1.7.1
+  version: 1.7.3
 platform: ruby
 authors:
 - Tilo Sloboda
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-07-31 00:00:00.000000000 Z
+date: 2022-12-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: awesome_print