smarter_csv 1.6.0 → 1.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fd2cf82aafc3b45257fbdfc594ed8e1d3bf2226e59cbee144b3003d8f79ec6cf
-  data.tar.gz: 95df862865e3123cf86194d47107f140f69f2fc91c20aba01d4004e8bffa5d74
+  metadata.gz: 34a12dae406ef192b3fbac9dd8a4236e18a7a936d4289cc296e49bf3b88fd386
+  data.tar.gz: f317413b7467386b1337938b2288763d1a6da279c6823ad3f4653ff82ea90d39
 SHA512:
-  metadata.gz: df32ae9a380fa4fff0932d56e8a0cacadb8d4ebf7d8124e607f2ba389c3b60f875c300a2137fe04aac2b7eda77850b343af5e58c53e310f90460f96223f3228c
-  data.tar.gz: 107e1dbacdc6293a0c044a91cf237f50fbeab59eb5b032167f55d0fe6c2cf07b079c6cfe296368c03d2c84e64ce3c0e6ad744043397cdc217cd3ab51beb3ab09
+  metadata.gz: ebbd40e8c6ea684200c8efedc12174da1a0a99ab9fae8bcb00f3bfdb8dcac479285644de09003b04b073b46f8ea64cbb29686628e9b7986d3baa07b041ee7dbd
+  data.tar.gz: 9c9ba18bd64474811bbb3be2b350ab3b25a33dbd3e2cc802d697d04dcefeff2cc24150e87be6f6789eedc045a717ced2590efe6e6b6056a5c0b18095edbd0b38

data/.rubocop.yml

@@ -0,0 +1,133 @@
+AllCops:
+  TargetRubyVersion: 2.5 # purposely an old Ruby version
+
+Layout/SpaceBeforeBlockBraces:
+  Enabled: false
+
+Layout/SpaceInsideBlockBraces:
+  Enabled: false
+
+Layout/SpaceInsideHashLiteralBraces:
+  Enabled: false
+
+Layout/SpaceAroundOperators:
+  Enabled: false
+
+Metrics/AbcSize:
+  Enabled: false
+
+Metrics/BlockLength:
+  Enabled: false
+
+Metrics/BlockNesting:
+  Enabled: false
+
+Metrics/CyclomaticComplexity: # BS rule
+  Enabled: false
+
+Metrics/MethodLength:
+  Enabled: false
+
+Metrics/ModuleLength:
+  Enabled: false
+
+Metrics/PerceivedComplexity: # BS rule
+  Enabled: false
+
+Naming/PredicateName:
+  Enabled: false
+
+Naming/VariableName:
+  Enabled: false
+
+Naming/VariableNumber:
+  Enabled: false
+
+Style/ClassEqualityComparison:
+  Enabled: false
+
+Style/ConditionalAssignment:
+  Enabled: false
+
+Style/CommentedKeyword:
+  Enabled: false
+
+Style/Documentation:
+  Enabled: false
+
+Style/DoubleNegation: # that's how to make true boolean results - this rule is nonsense
+  Enabled: false
+
+Style/EmptyElse:
+  Enabled: false
+
+Style/Encoding:
+  Enabled: false
+
+Style/EvalWithLocation:
+  Enabled: false
+
+Style/FormatString:
+  Enabled: false
+
+Style/FormatStringToken:
+  Enabled: false
+
+Style/GuardClause:
+  Enabled: false
+
+Style/IfInsideElse:
+  Enabled: false
+
+Style/IfUnlessModifier:
+  Enabled: false
+
+Style/NestedTernaryOperator:
+  Enabled: false
+
+Style/PreferredHashMethods:
+  Enabled: false
+
+Style/NumericPredicate:
+  Enabled: false
+
+Style/PercentLiteralDelimiters:
+  Enabled: false
+
+Style/RegexpLiteral:
+  Enabled: false
+
+Style/SafeNavigation:
+  Enabled: false
+
+Style/SlicingWithRange:
+  Enabled: false
+
+Style/SpecialGlobalVars: # DANGER: unsafe rule!!
+  Enabled: false
+
+Style/StringLiterals:
+  Enabled: false
+  EnforcedStyle: double_quotes
+
+Style/StringLiteralsInInterpolation:
+  Enabled: false
+  EnforcedStyle: double_quotes
+
+Style/SymbolProc: # old Ruby versions can't do this
+  Enabled: false
+
+Style/TrailingCommaInHashLiteral:
+  Enabled: false
+
+Style/TrailingUnderscoreVariable:
+  Enabled: false
+
+# Style/UnlessModifier:
+#   Enabled: false
+
+Style/ZeroLengthPredicate:
+  Enabled: false
+
+Layout/LineLength:
+  Max: 240

data/CHANGELOG.md

@@ -1,6 +1,34 @@
 
 # SmarterCSV 1.x Change Log
 
+## 1.7.2 (2022-08-29)
+  * new option :with_line_numbers; if set to true, it adds :csv_line_number to each data hash (issue #130)
+  
+## 1.7.1 (2022-07-31)
+  * bugfix for issue #195 #197 #200 which only appeared when called from Rails (thanks to Viacheslav Markin, Nicolas Rodriguez)
+
+## 1.7.0 (2022-06-26) (replaced by 1.7.1)
+  * added native code to accellerate line parsing by >10x over 1.6.0
+  * added option `acceleration`, defaulting to `true`, to enable native code.
+    Disable this option to use the ruby code for line parsing.
+  * increased test coverage to 100%
+  * rubocop changes
+
+## 1.7.0.pre5 (2022-06-20)
+  * fixed compiling
+  * rubocop changes
+  * published pre-release 
+
+## 1.7.0.pre1 (2022-05-23)
+  * added native code to accellerate line parsing by >10x over 1.6.0
+  * added option `acceleration`, defaulting to `true`, to enable native code.
+    Disable this option to use the ruby code for line parsing.
+  * increased test coverage to 100%
+
+## 1.6.1 (2022-05-06)
+  * unused keys in `key_mapping` now generate a warning, no longer raise an exception
+    This is preferable when `key_mapping` is done defensively for variabilities in the CSV files.
+
 ## 1.6.0 (2022-05-03)
   * completely rewrote line parser
   * added methods `SmarterCSV.raw_headers` and `SmarterCSV.headers` to allow easy examination of how the headers are processed.

data/CONTRIBUTORS.md

@@ -45,3 +45,6 @@ A Big Thank you to everyone who filed issues, sent comments, and who contributed
  * [Sp6](https://github.com/sp6)
  * [Joel Fouse](https://github.com/jfouse)
  * [John Dell](https://github.com/spovich)
+ * [Viacheslav Markin](https://github.com/KXEinc)
+ * [Nicolas Rodriguez](https://github.com/n-rodriguez)
+ * [Hirotaka Mizutani ](https://github.com/hirotaka)

data/Gemfile

@@ -1,11 +1,14 @@
+# frozen_string_literal: true
+
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in smarter_csv.gemspec
 gemspec
 
-gem "rake", "< 11"
+gem "rake" # , "< 11"
+gem "rake-compiler"
+
 gem 'pry'
 
-group :test do
-  gem "rspec", "~> 2.99"
-end
+gem "rspec"
+gem "rubocop"

data/README.md

@@ -1,8 +1,12 @@
 
-#### Service Announcement
+# SmarterCSV
+
+ [![codecov](https://codecov.io/gh/tilo/smarter_csv/branch/main/graph/badge.svg?token=1L7OD80182)](https://codecov.io/gh/tilo/smarter_csv) [![Gem Version](https://badge.fury.io/rb/smarter_csv.svg)](http://badge.fury.io/rb/smarter_csv)
+ 
+#### Work towards Future Version 2.0
 
-* Work towards SmarterCSV 2.0 is still on it's way, with much improved features, and more streamlined options.
-  Please check the [2.0-develop branch](https://github.com/tilo/smarter_csv/blob/master/README.md), open any issues and pull requests with mention of v2.0.
+* Work towards SmarterCSV 2.0 is still ongoing, with improved features, and more streamlined options, but consider it as experimental at this time.
+  Please check the [2.0-develop branch](https://github.com/tilo/smarter_csv/tree/2.0-develop), open any issues and pull requests with mention of tag v2.0.
 
 * New versions of SmarterCSV 1.x will soon print a deprecation warning if you set :verbose to true
   See below for list of deprecated options.
@@ -14,16 +18,11 @@
 
 ---------------
 
-# SmarterCSV
-
-[![Build Status](https://secure.travis-ci.org/tilo/smarter_csv.svg?branch=master)](http://travis-ci.com/tilo/smarter_csv) [![Gem Version](https://badge.fury.io/rb/smarter_csv.svg)](http://badge.fury.io/rb/smarter_csv)
+#### SmarterCSV 1.x [Current Version]
 
-#### SmarterCSV 1.x
+`smarter_csv` is a Ruby Gem for smarter importing of CSV Files as Array(s) of Hashes, suitable for direct processing with ActiveRecord, parallel processing, or kicking-off batch jobs with Sidekiq.
 
-`smarter_csv` is now 10 years old, and still kicking! 🎉🎉🎉
-
-`smarter_csv` is a Ruby Gem for smarter importing of CSV Files as Array(s) of Hashes, suitable for direct processing with Mongoid or ActiveRecord,
-and parallel processing with Resque or Sidekiq.
+To create high-quality output, some options are enabled as a default. Please make sure to check the output and tweak the options accordingly.
 
 One `smarter_csv` user wrote:
 
@@ -132,7 +131,21 @@ and how the `process` method returns the number of chunks when called with a blo
 
      => returns number of chunks / rows we processed
 ```
-#### Example 4: Populate a MongoDB Database in Chunks of 100 records with SmarterCSV:
+
+#### Example 4: Reading a CSV-like File, and Processing it with Sidekiq:
+```ruby
+    filename = '/tmp/strange_db_dump'   # a file with CRTL-A as col_separator, and with CTRL-B\n as record_separator (hello iTunes!)
+    options = {
+      :col_sep => "\cA", :row_sep => "\cB\n", :comment_regexp => /^#/,
+      :chunk_size => 100 , :key_mapping => {:export_date => nil, :name => :genre}
+    }
+    n = SmarterCSV.process(filename, options) do |chunk|
+        SidekiqWorkerClass.process_async(chunk ) # pass an array of hashes to Sidekiq workers for parallel processing
+    end
+    => returns number of chunks
+```
+
+#### Example 5: Populate a MongoDB Database in Chunks of 100 records with SmarterCSV:
 ```ruby
     # using chunks:
     filename = '/tmp/some.csv'
@@ -146,18 +159,6 @@ and how the `process` method returns the number of chunks when called with a blo
      => returns number of chunks we processed
 ```
 
-#### Example 5: Reading a CSV-like File, and Processing it with Resque:
-```ruby
-    filename = '/tmp/strange_db_dump'   # a file with CRTL-A as col_separator, and with CTRL-B\n as record_separator (hello iTunes!)
-    options = {
-      :col_sep => "\cA", :row_sep => "\cB\n", :comment_regexp => /^#/,
-      :chunk_size => 100 , :key_mapping => {:export_date => nil, :name => :genre}
-    }
-    n = SmarterCSV.process(filename, options) do |chunk|
-        Resque.enque( ResqueWorkerClass, chunk ) # pass chunks of CSV-data to Resque workers for parallel processing
-    end
-    => returns number of chunks
-```
 #### Example 6: Using Value Converters
 
 NOTE: If you use `key_mappings` and `value_converters`, make sure that the value converters has references the keys based on the final mapped name, not the original name in the CSV file.
@@ -239,6 +240,7 @@ The options and the block are optional.
      |                             |          | You can not combine the :user_provided_headers and :key_mapping options              |
      | :remove_empty_hashes        |   true   | remove / ignore any hashes which don't have any key/value pairs or all empty values  |
      | :verbose                    |   false  | print out line number while processing (to track down problems in input files)       |
+     | :with_line_numbers          |   false  | add :csv_line_number to heach data hash                                              |
      ---------------------------------------------------------------------------------------------------------------------------------
 
 #### Deprecated 1.x Options: to be replaced in 2.0
@@ -346,8 +348,10 @@ Or install it yourself as:
 
 Please [open an Issue on GitHub](https://github.com/tilo/smarter_csv/issues) if you have feedback, new feature requests, or want to report a bug. Thank you!
 
-  * please include a small sample CSV file
-  * please mention your version of SmarterCSV, Ruby, Rails
+For reporting issues, please:
+  * include a small sample CSV file
+  * open a pull-request adding a test that demonstrates the issue
+  * mention your version of SmarterCSV, Ruby, Rails
 
 ## [A Special Thanks to all Contributors!](CONTRIBUTORS.md) 🎉🎉🎉
 

data/Rakefile

@@ -1,19 +1,21 @@
-#!/usr/bin/env rake
+# frozen_string_literal: true
+
 require "bundler/gem_tasks"
-require 'rubygems'
-require 'rake'
 require 'rspec/core/rake_task'
 
-task :default => :spec
+RSpec::Core::RakeTask.new(:spec)
 
-desc "Run RSpec"
-RSpec::Core::RakeTask.new do |t|
-  # t.verbose = false
-end
+require "rubocop/rake_task"
+
+RuboCop::RakeTask.new
 
-desc 'Run spec with coverage'
-task :coverage do
-  ENV['COVERAGE'] = 'true'
-  Rake::Task['spec'].execute
-  `open coverage/index.html`
+require "rake/extensiontask"
+
+task build: :compile
+
+Rake::ExtensionTask.new("smarter_csv") do |ext|
+  ext.ext_dir = "ext/smarter_csv"
 end
+
+# task default: %i[clobber compile spec rubocop]
+task default: %i[clobber compile spec]

data/ext/smarter_csv/extconf.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require 'mkmf'
+
+require "rbconfig"
+if RbConfig::MAKEFILE_CONFIG["CFLAGS"].include?("-g -O3")
+  fixed_CFLAGS = RbConfig::MAKEFILE_CONFIG["CFLAGS"].sub("-g -O3", "$(cflags)")
+  puts("Fix CFLAGS: #{RbConfig::MAKEFILE_CONFIG["CFLAGS"]} -> #{fixed_CFLAGS}")
+  RbConfig::MAKEFILE_CONFIG["CFLAGS"] = fixed_CFLAGS
+end
+
+CONFIG["optflags"] = "-O3"
+
+create_makefile('smarter_csv/smarter_csv')

data/ext/smarter_csv/smarter_csv.c

@@ -0,0 +1,86 @@
+#include "ruby.h"
+#include "ruby/encoding.h"
+#include <stdio.h>
+#include <stdbool.h>
+
+#ifndef bool
+  #define bool int
+  #define false ((bool)0)
+  #define true  ((bool)1)
+#endif
+
+/*
+   max_size: pass nil if no limit is specified
+ */
+static VALUE rb_parse_csv_line(VALUE self, VALUE line, VALUE col_sep, VALUE quote_char, VALUE max_size) {
+  if (RB_TYPE_P(line, T_NIL) == 1) {
+    return rb_ary_new();
+
+  } else if (RB_TYPE_P(line, T_STRING) == 1) {
+    rb_encoding *encoding = rb_enc_get(line); /* get the encoding from the input line */
+    char *startP = RSTRING_PTR(line); /* may not be null terminated */
+    long line_len = RSTRING_LEN(line);
+    char *endP = startP + line_len ; /* points behind the string */
+    char *p = startP;
+
+    char *col_sepP = RSTRING_PTR(col_sep);
+    long col_sep_len = RSTRING_LEN(col_sep);
+
+    char *quoteP = RSTRING_PTR(quote_char);
+    long quote_len = RSTRING_LEN(quote_char);
+    long quote_count = 0;
+
+    bool col_sep_found = true;
+
+    VALUE elements = rb_ary_new();
+    VALUE field;
+    long i;
+
+    while (p < endP) {
+      /* does the remaining string start with col_sep ? */
+      col_sep_found = true;
+      for(i=0; (i < col_sep_len) && (p+i < endP) ; i++) {
+        col_sep_found = col_sep_found && (*(p+i) == *(col_sepP+i));
+      }
+      /* if col_sep was found and we have even quotes */
+      if (col_sep_found && (quote_count % 2 == 0)) {
+        /* if max_size != nil && lements.size >= header_size */
+        if ((max_size != Qnil) && RARRAY_LEN(elements) >= NUM2INT(max_size)) {
+          break;
+        } else {
+          /* push that field with original encoding onto the results */
+          field = rb_enc_str_new(startP, p - startP, encoding);
+          rb_ary_push(elements, field);
+
+          p += col_sep_len;
+          startP = p;
+        }
+      } else {
+        if (*p == *quoteP) {
+          quote_count += 1;
+        }
+        p++;
+      }
+    } /* while */
+
+    /* check if the last part of the line needs to be processed */
+    if ((max_size == Qnil) || RARRAY_LEN(elements) < NUM2INT(max_size)) {
+      /* copy the remaining line as a field with original encoding onto the results */
+      field = rb_enc_str_new(startP, endP - startP, encoding);
+      rb_ary_push(elements, field);
+    }
+
+    return elements;
+  }
+
+  rb_raise(rb_eTypeError, "ERROR in SmarterCSV.parse_line: line has to be a string or nil");
+}
+
+
+VALUE SmarterCSV = Qnil;
+
+void Init_smarter_csv(void) {
+  VALUE SmarterCSV = rb_define_module("SmarterCSV");
+
+  rb_define_module_function(SmarterCSV, "parse_csv_line_c", rb_parse_csv_line, 4);
+}

data/lib/extensions/hash.rb

@@ -1,7 +1,9 @@
+# frozen_string_literal: true
+
 # the following extension for class Hash is needed (from Facets of Ruby library):
 
 class Hash
-  def self.zip(keys,values) # from Facets of Ruby library
-    (keys.zip(values)).to_h
+  def self.zip(keys, values) # from Facets of Ruby library
+    keys.zip(values).to_h
   end
 end

data/lib/smarter_csv/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SmarterCSV
-  VERSION = "1.6.0"
+  VERSION = "1.7.2"
 end