ndr_import 8.0.0 → 8.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f74d60e93cf1f1d72baeebcbe931fc4225c3e74c11aaa13c324578b261561008
-  data.tar.gz: 70657ad1f113089bd2f719de5b59317310212c730ba34e762d99f9395da8fff2
+  metadata.gz: 214f73452462c2ea7182deaa8c90e4818d8ab6caf3fd604af4a64833e2518b41
+  data.tar.gz: 5d15e20e7b0c13fa3d3cd9169ad70c9028cc64d688906dd965a09ce791e97022
 SHA512:
-  metadata.gz: 91bced100d286a44df6fa193cad58a2ad85f6733d9b8cbbf1970e438ed8a3a7d9785a6d0e8b16396713511d045edd0a6aba00090a903c217e4c71b8e2fde7174
-  data.tar.gz: 4e2e968084f78f25fffebbc793806402fefd515605afc0ac9c81a5d535f0cb7f27a50af3c4dcb733ff5a9a8bc17311084c65593a2fb971ac35c5a4937bbfe777
+  metadata.gz: 7d4080dc6d86e553f8f6a07d2fe4876c6d014ecc8b5b3e729ac089147223ecf2d18f9f9bddfbc64c692e73deb08e27a12e09793fbd155c2a859b29c7efff7c3b
+  data.tar.gz: 46b267832414e13f1580da5fc386bebe34c5debb79d03e288c3f793b6c28158e362eb3971d2f275f68c398a4f8e3293321a585df2be2595369197bb70bf32eef

data/CHANGELOG.md

@@ -1,6 +1,11 @@
 ## [Unreleased]
 *no unreleased changes*
 
+## 8.1.0 / 2019-01-08
+### Added
+* Support `liberal_parsing` when using the delimited helper
+* Support for Ruby 2.6. Rails 5.0 / Ruby 2.4 is now the minimum.
+
 ## 8.0.0 / 2018-11-26
 ### Changed
 * Strip non tabular captured values by default (#28)
@@ -8,6 +13,7 @@
 ### Added
 * Add validations to field mappings (#27)
 * control `ndr_table` serialise order with `encode_with` method (#30)
+* Added a 7-Zip file handler
 
 ## 7.0.0 / 2018-11-09
 ### Changed

data/code_safety.yml

@@ -15,11 +15,11 @@ file safety:
   ".travis.yml":
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 71979e0a602ca5a0ce415c194f10add9959f0116
+    safe_revision: 607c0668f1fffd70d181bc1a31c4f56eed5f6189
   CHANGELOG.md:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 952e45599240ab96b22d7ce223a7551853e50ae5
+    safe_revision: 5ca2f7e9cb0e79134e5fa54ca2316c584696b34a
   CODE_OF_CONDUCT.md:
     comments: 
     reviewed_by: timgentry
@@ -44,10 +44,6 @@ file safety:
     comments: 
     reviewed_by: josh.pencheon
     safe_revision: 5a7f26cecaabab20f4e666776f9166dcc3fa6bfe
-  gemfiles/Gemfile.rails42:
-    comments: 
-    reviewed_by: timgentry
-    safe_revision: 0c5967732100f43e7b60a97a7f0aae60d3641791
   gemfiles/Gemfile.rails50:
     comments: 
     reviewed_by: josh.pencheon
@@ -79,7 +75,7 @@ file safety:
   lib/ndr_import/file/delimited.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 71979e0a602ca5a0ce415c194f10add9959f0116
+    safe_revision: 607c0668f1fffd70d181bc1a31c4f56eed5f6189
   lib/ndr_import/file/docx.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -127,7 +123,7 @@ file safety:
   lib/ndr_import/helpers/file/delimited.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: be12e57519d3737e8d3901d7b01485c6995708dd
+    safe_revision: 607c0668f1fffd70d181bc1a31c4f56eed5f6189
   lib/ndr_import/helpers/file/excel.rb:
     comments: 
     reviewed_by: joshpencheon
@@ -199,7 +195,7 @@ file safety:
   lib/ndr_import/version.rb:
     comments: another check?
     reviewed_by: josh.pencheon
-    safe_revision: 952e45599240ab96b22d7ce223a7551853e50ae5
+    safe_revision: 5ca2f7e9cb0e79134e5fa54ca2316c584696b34a
   lib/ndr_import/xml/table.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -207,7 +203,7 @@ file safety:
   ndr_import.gemspec:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: eca44583e9989159b45e90021dd1c65228447180
+    safe_revision: 607c0668f1fffd70d181bc1a31c4f56eed5f6189
   test/file/base_test.rb:
     comments: 
     reviewed_by: timgentry
@@ -215,7 +211,7 @@ file safety:
   test/file/delimited_test.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: ab9f926a53e84f8ffa826abdd967abee455c89df
+    safe_revision: 607c0668f1fffd70d181bc1a31c4f56eed5f6189
   test/file/docx_test.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -258,8 +254,8 @@ file safety:
     safe_revision: dfc958d44b6c58355445fa395db08a62213ee709
   test/helpers/file/delimited_test.rb:
     comments: 
-    reviewed_by: joshpencheon
-    safe_revision: 377c060bd1e012d6e162a4ab8923a3609aacdf57
+    reviewed_by: josh.pencheon
+    safe_revision: 607c0668f1fffd70d181bc1a31c4f56eed5f6189
   test/helpers/file/excel_test.rb:
     comments: 
     reviewed_by: joshpencheon

data/lib/ndr_import/file/delimited.rb

@@ -1,5 +1,6 @@
 require 'ndr_support/safe_file'
 require 'ndr_import/csv_library'
+require 'ndr_import/helpers/file/delimited'
 require_relative 'registry'
 
 module NdrImport
@@ -8,6 +9,8 @@ module NdrImport
   module File
     # This class is a delimited file handler that returns a single table.
     class Delimited < Base
+      include Helpers::File::Delimited
+
       DELIMITED_COL_SEP = {
         'csv' => nil
       }
@@ -23,64 +26,16 @@ module NdrImport
       def rows
         return enum_for(:rows) unless block_given?
 
-        safe_path = SafeFile.safepath_to_string(@filename)
-
-        # By now, we know `encodings` should let us read the whole
-        # file succesfully; if there are problems, we should crash.
-        CSVLibrary.foreach(safe_path, encodings(safe_path)) do |line|
-          yield line.map(&:to_s)
-        end
-      end
+        col_sep = @options['col_sep']
+        liberal = @options['liberal_parsing'].presence
 
-      # Cache the determined encodings, so rewinding the enumerator doesn't
-      # have to redo this, but equally it is still done lazily:
-      def encodings(safe_path)
-        @encodings ||= determine_encodings!(safe_path)
+        delimited_rows(@filename, col_sep, liberal) { |row| yield row }
       end
 
-      # Derive the source encoding by trying all supported encodings.
-      # Returns first set of working options, or raises if none could be found.
-      def determine_encodings!(safe_path)
-        # delimiter encoding => # FasterCSV encoding string
-        supported_encodings = {
-          'UTF-8'        => 'r:bom|utf-8',
-          'Windows-1252' => 'r:windows-1252:utf-8'
-        }
-
-        successful_options = nil
-        supported_encodings.each do |delimiter_encoding, access_mode|
-          begin
-            col_sep = @options['col_sep']
-            options = {
-              col_sep:         (col_sep || ',').force_encoding(delimiter_encoding),
-              mode:            access_mode,
-              liberal_parsing: @options['liberal_parsing'].presence
-            }
-
-            row_num = 0
-            # Iterate through the file; if we reach the end, this encoding worked:
-            CSVLibrary.foreach(safe_path, options) { |_line| row_num += 1 }
-          rescue ArgumentError => e
-            next if e.message =~ /invalid byte sequence/ # This encoding didn't work
-            raise(e)
-          rescue CSVLibrary::MalformedCSVError => e
-            description = (col_sep ? col_sep.inspect + ' delimited' : 'CSV')
-
-            raise(CSVLibrary::MalformedCSVError, "Invalid #{description} format " \
-              "on row #{row_num + 1} of #{::File.basename(safe_path)}. Original: #{e.message}")
-          end
-
-          # We got this far => encoding choice worked:
-          successful_options = options
-          break
-        end
-
-        # We tried them all, and none worked:
-        unless successful_options
-          raise "None of the encodings #{supported_encodings.values.inspect} were successful!"
-        end
-
-        successful_options
+      # Cache working encodings, so that resetting the enumerator
+      # doesn't mean the need to recalculate this:
+      def determine_encodings!(*)
+        @encoding ||= super
       end
     end
 

data/lib/ndr_import/helpers/file/delimited.rb

@@ -7,32 +7,32 @@ module NdrImport
       # This mixin adds delimited file functionality to unified importers.
       module Delimited
         # Read a plain text CSV file, return an array of the content
-        def read_csv_file(path)
+        def read_csv_file(path, liberal = false)
           # Read the page below when encountering "CSV::IllegalFormatError" error caused by CSV
           # file generated at MAC OS
           # http://stackoverflow.com/questions/1549139/ruby-cannot-parse-excel-file-exported-as-csv-in-os-x
 
-          read_delimited_file(path)
+          read_delimited_file(path, nil, liberal)
         end
 
         # Slurp the entire file into an array of lines.
-        def read_delimited_file(path, col_sep = nil)
-          delimited_rows(path, col_sep).to_a
+        def read_delimited_file(path, col_sep = nil, liberal = false)
+          delimited_rows(path, col_sep, liberal).to_a
         end
 
         # Iterate through the file table by table, yielding each one in turn.
-        def delimited_tables(path, col_sep = nil)
-          return enum_for(:delimited_tables, path, col_sep) unless block_given?
+        def delimited_tables(path, col_sep = nil, liberal = false)
+          return enum_for(:delimited_tables, path, col_sep, liberal) unless block_given?
 
-          yield nil, delimited_rows(path, col_sep)
+          yield nil, delimited_rows(path, col_sep, liberal)
         end
 
         # Iterate through the file line by line, yielding each one in turn.
-        def delimited_rows(path, col_sep = nil)
-          return enum_for(:delimited_rows, path, col_sep) unless block_given?
+        def delimited_rows(path, col_sep = nil, liberal = false)
+          return enum_for(:delimited_rows, path, col_sep, liberal) unless block_given?
 
           safe_path = SafeFile.safepath_to_string(path)
-          encodings = determine_encodings!(safe_path, col_sep)
+          encodings = determine_encodings!(safe_path, col_sep, liberal)
 
           # By now, we know `encodings` should let us read the whole
           # file succesfully; if there are problems, we should crash.
@@ -45,45 +45,59 @@ module NdrImport
 
         # Derive the source encoding by trying all supported encodings.
         # Returns first set of working options, or raises if none could be found.
-        def determine_encodings!(safe_path, col_sep = nil)
+        def determine_encodings!(safe_path, col_sep, liberal)
           # delimiter encoding => # FasterCSV encoding string
           supported_encodings = {
             'UTF-8'        => 'r:bom|utf-8',
             'Windows-1252' => 'r:windows-1252:utf-8'
           }
 
-          successful_options = nil
+          successful_options = try_each_encoding(safe_path, col_sep, liberal, supported_encodings)
+
+          # We tried them all, and none worked:
+          unless successful_options
+            raise "None of the encodings #{supported_encodings.values.inspect} were successful!"
+          end
+
+          successful_options
+        end
+
+        def try_each_encoding(safe_path, col_sep, liberal, supported_encodings)
           supported_encodings.each do |delimiter_encoding, access_mode|
             begin
               options = {
-                :col_sep => (col_sep || ',').force_encoding(delimiter_encoding),
-                :mode    => access_mode
+                col_sep: (col_sep || ',').force_encoding(delimiter_encoding),
+                liberal_parsing: liberal,
+                mode: access_mode
               }
 
               row_num = 0
               # Iterate through the file; if we reach the end, this encoding worked:
               CSVLibrary.foreach(safe_path, options) { |_line| row_num += 1 }
+              return options
             rescue ArgumentError => e
               next if e.message =~ /invalid byte sequence/ # This encoding didn't work
               raise(e)
+            rescue RegexpError => e
+              next if e.message =~ /invalid multibyte character/ # This encoding didn't work
+              raise(e)
             rescue CSVLibrary::MalformedCSVError => e
-              description = (col_sep ? col_sep.inspect + ' delimited' : 'CSV')
-
-              raise(CSVLibrary::MalformedCSVError, "Invalid #{description} format " \
-                "on row #{row_num + 1} of #{::File.basename(safe_path)}. Original: #{e.message}")
+              next if e.message =~ /Invalid byte sequence/ # This encoding didn't work
+              raise malformed_csv_error(e, col_sep, row_num + 1, safe_path)
             end
-
-            # We got this far => encoding choice worked:
-            successful_options = options
-            break
           end
+        end
 
-          # We tried them all, and none worked:
-          unless successful_options
-            fail "None of the encodings #{supported_encodings.values.inspect} were successful!"
-          end
+        def malformed_csv_error(exception, col_sep, line, safe_path)
+          type    = (col_sep ? col_sep.inspect + ' delimited' : 'CSV')
+          message = "Invalid #{type} format on row #{line} of #{::File.basename(safe_path)}"
 
-          successful_options
+          if exception.respond_to?(:line_number)
+            base_message = exception.message.chomp(" in line #{exception.line_number}.")
+            exception.class.new("#{message}. Original: #{base_message}", exception.line_number)
+          else
+            exception.class.new("#{message}. Original: #{exception.message}")
+          end
         end
       end
     end

data/lib/ndr_import/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 # This stores the current version of the NdrImport gem
 module NdrImport
-  VERSION = '8.0.0'.freeze
+  VERSION = '8.1.0'.freeze
 end

data/ndr_import.gemspec

@@ -21,7 +21,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_dependency 'activemodel'
-  spec.add_dependency 'activesupport', '>= 3.2.18', '< 5.3'
+  spec.add_dependency 'activesupport', '~> 5.0'
   spec.add_dependency 'ndr_support', '>= 5.3.2', '< 6'
 
   spec.add_dependency 'rubyzip', '~> 1.2', '>= 1.2.2'

data/test/file/delimited_test.rb

@@ -185,9 +185,9 @@ module NdrImport
           end
         end
 
-        msg = 'Invalid CSV format on row 2 of broken.csv. ' \
-              'Original: Missing or stray quote in line 2'
-        assert_equal msg, exception.message
+        assert_match(/Invalid CSV format on row 2 of broken\.csv\./, exception.message)
+        assert_match(/(Missing or stray quote|col_sep_split)/, exception.message)
+        assert_match(/in line 2/, exception.message)
       end
 
       test 'should report addition details upon failure to read csv line-by-line' do
@@ -207,16 +207,16 @@ module NdrImport
 
         assert rows_yielded.empty?, 'no rows should have been yielded'
 
-        msg = 'Invalid CSV format on row 2 of broken.csv. ' \
-              'Original: Missing or stray quote in line 2'
-        assert_equal msg, exception.message
+        assert_match(/Invalid CSV format on row 2 of broken\.csv\./, exception.message)
+        assert_match(/(Missing or stray quote|col_sep_split)/, exception.message)
+        assert_match(/in line 2/, exception.message)
       end
 
       test 'should only determine encodings once' do
         file_path = @permanent_test_files.join('normal.csv')
         handler = NdrImport::File::Delimited.new(file_path, 'csv', 'col_sep' => nil)
 
-        handler.expects(determine_encodings!: { mode: 'r:bom|utf-8', col_sep: ',' }).once
+        handler.expects(try_each_encoding: { mode: 'r:bom|utf-8', col_sep: ',' }).once
 
         2.times do
           handler.tables.each do |tablename, sheet|

data/test/helpers/file/delimited_test.rb

@@ -59,8 +59,22 @@ class DelimitedTest < ActiveSupport::TestCase
       @importer.read_delimited_file(@permanent_test_files.join('broken.csv'), nil)
     end
 
-    msg = 'Invalid CSV format on row 2 of broken.csv. Original: Missing or stray quote in line 2'
-    assert_equal msg, exception.message
+    assert_match(/Invalid CSV format on row 2 of broken\.csv\./, exception.message)
+    assert_match(/(Missing or stray quote|col_sep_split)/, exception.message)
+    assert_match(/in line 2/, exception.message)
+  end
+
+  test 'should be able to use liberal parsing to overcome minor CSV errors' do
+    file_path = @permanent_test_files.join('malformed.csv')
+    assert_raises(CSVLibrary::MalformedCSVError) do
+      @importer.read_delimited_file(file_path, nil)
+    end
+
+    rows = @importer.read_delimited_file(file_path, nil, true)
+
+    expected_row = ['2'] * 25
+    expected_row << '2"malformed"'
+    assert_equal expected_row, rows[2].sort
   end
 
   test 'should report addition details upon failure to read csv line-by-line' do
@@ -73,8 +87,9 @@ class DelimitedTest < ActiveSupport::TestCase
 
     assert rows_yielded.empty?, 'no rows should have been yielded'
 
-    msg = 'Invalid CSV format on row 2 of broken.csv. Original: Missing or stray quote in line 2'
-    assert_equal msg, exception.message
+    assert_match(/Invalid CSV format on row 2 of broken\.csv\./, exception.message)
+    assert_match(/(Missing or stray quote|col_sep_split)/, exception.message)
+    assert_match(/in line 2/, exception.message)
   end
 
   test 'delimited_tables should read table correctly' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ndr_import
 version: !ruby/object:Gem::Version
-  version: 8.0.0
+  version: 8.1.0
 platform: ruby
 authors:
 - NCRS Development Team
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-26 00:00:00.000000000 Z
+date: 2019-01-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -28,22 +28,16 @@ dependencies:
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 3.2.18
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.3'
+        version: '5.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 3.2.18
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.3'
+        version: '5.0'
 - !ruby/object:Gem::Dependency
   name: ndr_support
   requirement: !ruby/object:Gem::Requirement
@@ -365,7 +359,6 @@ files:
 - README.md
 - Rakefile
 - code_safety.yml
-- gemfiles/Gemfile.rails42
 - gemfiles/Gemfile.rails50
 - gemfiles/Gemfile.rails51
 - gemfiles/Gemfile.rails52
@@ -497,8 +490,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.1
 signing_key: 
 specification_version: 4
 summary: NDR Import

data/gemfiles/Gemfile.rails42

@@ -1,5 +0,0 @@
-source 'https://rubygems.org'
-
-gemspec path: '..'
-
-gem 'activesupport', '~> 4.2.0'