smarter_csv 1.6.1 → 1.7.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9be5f053e15e157d7d28555b4de894d2761d5918203da45f5fc4e6c5adcc2a3f
-  data.tar.gz: a47394f3d1f985960a64abf1a43ce6ebf9b8217af2c01a0c5f053af8c77c09ae
+  metadata.gz: ab42c787dba96369ba0f499294d5421cc2fc9a514b1cff039e5017d19fd2ff4c
+  data.tar.gz: f3605e56395d498169c449657945c00c677a75d17e2a7cbcca1a4f4e65aa45f2
 SHA512:
-  metadata.gz: f27113af8a5771d89ac5c8783f1f69645c24bb576dafd97de17b0d8db8fff74dc396b42450802f9332c9f8b32a02ee18dabbc5dbc2daa91c75f957a678e99099
-  data.tar.gz: 5b2c2f3cbfc17b43c030c4c4c261962818bfbb2ce1a0ecd88f394682b75b75a64a2d8b6afcc0e4b99b97d39ef4b645de143cbb5f595e7aa9d661e66b1a53e98f
+  metadata.gz: a264574b464219fd53a862be345c008fc0a93c076345d62be4c75935a148f9a469ccbea14fb2f821cedaada1f2b4ec875468ef50b3ae173bcbb76a5079d43406
+  data.tar.gz: 978236fa33bf4656eba89d929545c81ec2dc28b179890dd4254fb04cfa9b8646f4040aa2a62d44361dea221714f1cba0d7e4c858d338d30080c4a0e43c003552

data/.rubocop.yml

@@ -0,0 +1,133 @@
+AllCops:
+  TargetRubyVersion: 2.5 # purposely an old Ruby version
+
+Layout/SpaceBeforeBlockBraces:
+  Enabled: false
+
+Layout/SpaceInsideBlockBraces:
+  Enabled: false
+
+Layout/SpaceInsideHashLiteralBraces:
+  Enabled: false
+
+Layout/SpaceAroundOperators:
+  Enabled: false
+
+Metrics/AbcSize:
+  Enabled: false
+
+Metrics/BlockLength:
+  Enabled: false
+
+Metrics/BlockNesting:
+  Enabled: false
+
+Metrics/CyclomaticComplexity: # BS rule
+  Enabled: false
+
+Metrics/MethodLength:
+  Enabled: false
+
+Metrics/ModuleLength:
+  Enabled: false
+
+Metrics/PerceivedComplexity: # BS rule
+  Enabled: false
+
+Naming/PredicateName:
+  Enabled: false
+
+Naming/VariableName:
+  Enabled: false
+
+Naming/VariableNumber:
+  Enabled: false
+
+Style/ClassEqualityComparison:
+  Enabled: false
+
+Style/ConditionalAssignment:
+  Enabled: false
+
+Style/CommentedKeyword:
+  Enabled: false
+
+Style/Documentation:
+  Enabled: false
+
+Style/DoubleNegation: # that's how to make true boolean results - this rule is nonsense
+  Enabled: false
+
+Style/EmptyElse:
+  Enabled: false
+
+Style/Encoding:
+  Enabled: false
+
+Style/EvalWithLocation:
+  Enabled: false
+
+Style/FormatString:
+  Enabled: false
+
+Style/FormatStringToken:
+  Enabled: false
+
+Style/GuardClause:
+  Enabled: false
+
+Style/IfInsideElse:
+  Enabled: false
+
+Style/IfUnlessModifier:
+  Enabled: false
+
+Style/NestedTernaryOperator:
+  Enabled: false
+
+Style/PreferredHashMethods:
+  Enabled: false
+
+Style/NumericPredicate:
+  Enabled: false
+
+Style/PercentLiteralDelimiters:
+  Enabled: false
+
+Style/RegexpLiteral:
+  Enabled: false
+
+Style/SafeNavigation:
+  Enabled: false
+
+Style/SlicingWithRange:
+  Enabled: false
+
+Style/SpecialGlobalVars: # DANGER: unsafe rule!!
+  Enabled: false
+
+Style/StringLiterals:
+  Enabled: false
+  EnforcedStyle: double_quotes
+
+Style/StringLiteralsInInterpolation:
+  Enabled: false
+  EnforcedStyle: double_quotes
+
+Style/SymbolProc: # old Ruby versions can't do this
+  Enabled: false
+
+Style/TrailingCommaInHashLiteral:
+  Enabled: false
+
+Style/TrailingUnderscoreVariable:
+  Enabled: false
+
+# Style/UnlessModifier:
+#   Enabled: false
+
+Style/ZeroLengthPredicate:
+  Enabled: false
+
+Layout/LineLength:
+  Max: 240

data/CHANGELOG.md

@@ -1,8 +1,29 @@
 
 # SmarterCSV 1.x Change Log
 
+## 1.7.1 (2022-07-31)
+  * bugfix (thanks to Viacheslav Markin, Nicolas Rodriguez)
+## 1.7.0 (2022-06-26)
+  * added native code to accellerate line parsing by >10x over 1.6.0
+  * added option `acceleration`, defaulting to `true`, to enable native code.
+    Disable this option to use the ruby code for line parsing.
+  * increased test coverage to 100%
+  * rubocop changes
+
+## 1.7.0.pre5 (2022-06-20)
+  * fixed compiling
+  * rubocop changes
+  * published pre-release 
+
+## 1.7.0.pre1 (2022-05-23)
+  * added native code to accellerate line parsing by >10x over 1.6.0
+  * added option `acceleration`, defaulting to `true`, to enable native code.
+    Disable this option to use the ruby code for line parsing.
+  * increased test coverage to 100%
+
 ## 1.6.1 (2022-05-06)
-  * unused keys in `key_mapping` generate a warning, no longer raise an exception
+  * unused keys in `key_mapping` now generate a warning, no longer raise an exception
+    This is preferable when `key_mapping` is done defensively for variabilities in the CSV files.
 
 ## 1.6.0 (2022-05-03)
   * completely rewrote line parser

data/CONTRIBUTORS.md

@@ -45,3 +45,6 @@ A Big Thank you to everyone who filed issues, sent comments, and who contributed
  * [Sp6](https://github.com/sp6)
  * [Joel Fouse](https://github.com/jfouse)
  * [John Dell](https://github.com/spovich)
+ * [Viacheslav Markin](https://github.com/KXEinc)
+ * [Nicolas Rodriguez](https://github.com/n-rodriguez)
+ * [Hirotaka Mizutani ](https://github.com/hirotaka)

data/Gemfile

@@ -1,11 +1,14 @@
+# frozen_string_literal: true
+
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in smarter_csv.gemspec
 gemspec
 
-gem "rake", "< 11"
+gem "rake" # , "< 11"
+gem "rake-compiler"
+
 gem 'pry'
 
-group :test do
-  gem "rspec", "~> 2.99"
-end
+gem "rspec"
+gem "rubocop"

data/README.md

@@ -1,4 +1,5 @@
-
+ [![codecov](https://codecov.io/gh/tilo/smarter_csv/branch/main/graph/badge.svg?token=1L7OD80182)](https://codecov.io/gh/tilo/smarter_csv)
+ 
 #### Service Announcement
 
 * Work towards SmarterCSV 2.0 is still on it's way, with much improved features, and more streamlined options.
@@ -20,10 +21,9 @@
 
 #### SmarterCSV 1.x
 
-`smarter_csv` is now 10 years old, and still kicking! 🎉🎉🎉
+`smarter_csv` is a Ruby Gem for smarter importing of CSV Files as Array(s) of Hashes, suitable for direct processing with ActiveRecord, parallel processing, or kicking-off batch jobs with Sidekiq.
 
-`smarter_csv` is a Ruby Gem for smarter importing of CSV Files as Array(s) of Hashes, suitable for direct processing with Mongoid or ActiveRecord,
-and parallel processing with Resque or Sidekiq.
+To create high-quality output, some options are enabled as a default. Please make sure to check the output and tweak the options accordingly.
 
 One `smarter_csv` user wrote:
 
@@ -346,8 +346,10 @@ Or install it yourself as:
 
 Please [open an Issue on GitHub](https://github.com/tilo/smarter_csv/issues) if you have feedback, new feature requests, or want to report a bug. Thank you!
 
-  * please include a small sample CSV file
-  * please mention your version of SmarterCSV, Ruby, Rails
+For reporting issues, please:
+  * include a small sample CSV file
+  * open a pull-request adding a test that demonstrates the issue
+  * mention your version of SmarterCSV, Ruby, Rails
 
 ## [A Special Thanks to all Contributors!](CONTRIBUTORS.md) 🎉🎉🎉
 

data/Rakefile

@@ -1,19 +1,21 @@
-#!/usr/bin/env rake
+# frozen_string_literal: true
+
 require "bundler/gem_tasks"
-require 'rubygems'
-require 'rake'
 require 'rspec/core/rake_task'
 
-task :default => :spec
+RSpec::Core::RakeTask.new(:spec)
 
-desc "Run RSpec"
-RSpec::Core::RakeTask.new do |t|
-  # t.verbose = false
-end
+require "rubocop/rake_task"
+
+RuboCop::RakeTask.new
 
-desc 'Run spec with coverage'
-task :coverage do
-  ENV['COVERAGE'] = 'true'
-  Rake::Task['spec'].execute
-  `open coverage/index.html`
+require "rake/extensiontask"
+
+task build: :compile
+
+Rake::ExtensionTask.new("smarter_csv") do |ext|
+  ext.ext_dir = "ext/smarter_csv"
 end
+
+# task default: %i[clobber compile spec rubocop]
+task default: %i[clobber compile spec]

data/ext/smarter_csv/extconf.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require 'mkmf'
+
+require "rbconfig"
+if RbConfig::MAKEFILE_CONFIG["CFLAGS"].include?("-g -O3")
+  fixed_CFLAGS = RbConfig::MAKEFILE_CONFIG["CFLAGS"].sub("-g -O3", "$(cflags)")
+  puts("Fix CFLAGS: #{RbConfig::MAKEFILE_CONFIG["CFLAGS"]} -> #{fixed_CFLAGS}")
+  RbConfig::MAKEFILE_CONFIG["CFLAGS"] = fixed_CFLAGS
+end
+
+CONFIG["optflags"] = "-O3"
+
+create_makefile('smarter_csv/smarter_csv')

data/ext/smarter_csv/smarter_csv.c

@@ -0,0 +1,86 @@
+#include "ruby.h"
+#include "ruby/encoding.h"
+#include <stdio.h>
+#include <stdbool.h>
+
+#ifndef bool
+  #define bool int
+  #define false ((bool)0)
+  #define true  ((bool)1)
+#endif
+
+/*
+   max_size: pass nil if no limit is specified
+ */
+static VALUE rb_parse_csv_line(VALUE self, VALUE line, VALUE col_sep, VALUE quote_char, VALUE max_size) {
+  if (RB_TYPE_P(line, T_NIL) == 1) {
+    return rb_ary_new();
+
+  } else if (RB_TYPE_P(line, T_STRING) == 1) {
+    rb_encoding *encoding = rb_enc_get(line); /* get the encoding from the input line */
+    char *startP = RSTRING_PTR(line); /* may not be null terminated */
+    long line_len = RSTRING_LEN(line);
+    char *endP = startP + line_len ; /* points behind the string */
+    char *p = startP;
+
+    char *col_sepP = RSTRING_PTR(col_sep);
+    long col_sep_len = RSTRING_LEN(col_sep);
+
+    char *quoteP = RSTRING_PTR(quote_char);
+    long quote_len = RSTRING_LEN(quote_char);
+    long quote_count = 0;
+
+    bool col_sep_found = true;
+
+    VALUE elements = rb_ary_new();
+    VALUE field;
+    long i;
+
+    while (p < endP) {
+      /* does the remaining string start with col_sep ? */
+      col_sep_found = true;
+      for(i=0; (i < col_sep_len) && (p+i < endP) ; i++) {
+        col_sep_found = col_sep_found && (*(p+i) == *(col_sepP+i));
+      }
+      /* if col_sep was found and we have even quotes */
+      if (col_sep_found && (quote_count % 2 == 0)) {
+        /* if max_size != nil && lements.size >= header_size */
+        if ((max_size != Qnil) && RARRAY_LEN(elements) >= NUM2INT(max_size)) {
+          break;
+        } else {
+          /* push that field with original encoding onto the results */
+          field = rb_enc_str_new(startP, p - startP, encoding);
+          rb_ary_push(elements, field);
+
+          p += col_sep_len;
+          startP = p;
+        }
+      } else {
+        if (*p == *quoteP) {
+          quote_count += 1;
+        }
+        p++;
+      }
+    } /* while */
+
+    /* check if the last part of the line needs to be processed */
+    if ((max_size == Qnil) || RARRAY_LEN(elements) < NUM2INT(max_size)) {
+      /* copy the remaining line as a field with original encoding onto the results */
+      field = rb_enc_str_new(startP, endP - startP, encoding);
+      rb_ary_push(elements, field);
+    }
+
+    return elements;
+  }
+
+  rb_raise(rb_eTypeError, "ERROR in SmarterCSV.parse_line: line has to be a string or nil");
+}
+
+
+VALUE SmarterCSV = Qnil;
+
+void Init_smarter_csv(void) {
+  VALUE SmarterCSV = rb_define_module("SmarterCSV");
+
+  rb_define_module_function(SmarterCSV, "parse_csv_line_c", rb_parse_csv_line, 4);
+}

data/lib/extensions/hash.rb

@@ -1,7 +1,9 @@
+# frozen_string_literal: true
+
 # the following extension for class Hash is needed (from Facets of Ruby library):
 
 class Hash
-  def self.zip(keys,values) # from Facets of Ruby library
-    (keys.zip(values)).to_h
+  def self.zip(keys, values) # from Facets of Ruby library
+    keys.zip(values).to_h
   end
 end

data/lib/smarter_csv/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SmarterCSV
-  VERSION = "1.6.1"
+  VERSION = "1.7.1"
 end