ndr_import 9.0.1 → 10.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6b428af5bcdd6995256d335b2b8ba8511b09c72e62f30db25ad67998c8cd2410
-  data.tar.gz: e89bc44c132429fcc87ab63b19267720b76d3c50a96f3f253f39466e6324dee2
+  metadata.gz: 16cdebb2e3a6809255d5d66971a2db6d5c954731f6def3c73fe0f89d8ea0b7e9
+  data.tar.gz: 575ba2c40ae01f99ebd48f75ff42c2cdf40871cbed87bfb2399de45d9f402e73
 SHA512:
-  metadata.gz: 4c369ed69230675259d9813869a48b4b6730c6348556fe944bb1e603b9bb6b0ebdb0fbf0db0ea4e9c5e156371ebc8442ebf1fc7369d91ca7f8d21fb109eb96e8
-  data.tar.gz: 14c42758b8b0f373c4bd1c599122fe8c095fdc566a339d9da54dda1ae1a17104a62094b553609bd04120c493fa79c885a95ad9db74f108692025d19b691fafab
+  metadata.gz: 37e9bcfe3b8a5cab98bd68fc846eafd386595e0b6b79b9b4bf10ba8bc44a6d24ff998f5ab249964009bb0eebebfe0700bd6a39ddd0dfefc1260b529ff5543e63
+  data.tar.gz: c144108ad4d2f63c918e43ef540f30d49be26897d6c4d82bccda6128aaff5b408c3d93425612bedaef73f865292a3d2a408f0c792e35a9f548e50363804c8d10

data/.github/CODEOWNERS

@@ -0,0 +1,2 @@
+# Admins should have oversight of the version:
+lib/ndr_import/version.rb      @publichealthengland/ndr-admins

data/.github/workflows/lint.yml

@@ -0,0 +1,23 @@
+name: Lint
+
+on: [pull_request]
+
+jobs:
+  rubocop:
+    name: RuboCop
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0 # fetch everything
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: 3.0
+    - name: Install dependencies
+      run: bundle install
+    - name: Run RuboCop against BASE..HEAD changes
+      run: bundle exec rake rubocop:diff origin/${GITHUB_BASE_REF#*/}
+

data/.github/workflows/test.yml

@@ -0,0 +1,72 @@
+name: Test
+
+on: [push]
+
+jobs:
+  test:
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby-version:
+          - 2.6
+          - 2.7
+          - 3.0
+        gemfile:
+          - gemfiles/Gemfile.rails52
+          - gemfiles/Gemfile.rails60
+
+    name: Ruby ${{ matrix.ruby-version }} / Bundle ${{ matrix.gemfile }}
+
+    runs-on: ubuntu-latest
+
+    env:
+      BUNDLE_GEMFILE: ${{ matrix.gemfile }}
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby-version }}
+    - name: Install dependencies
+      run: bundle install
+    - name: Run tests
+      run: bundle exec rake
+
+  # A utility job upon which Branch Protection can depend,
+  # thus remaining agnostic of the matrix.
+  test_matrix:
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    name: Matrix
+    needs: test
+    steps:
+    - name: Check build matrix status
+      if: ${{ needs.test.result != 'success' }}
+      run: exit 1
+
+  notify:
+    # Run only on master, but regardless of whether tests past:
+    if: ${{ always() && github.ref == 'refs/heads/master' }}
+
+    needs: test_matrix
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: 8398a7/action-slack@v3
+      with:
+        status: custom
+        fields: workflow,commit,author
+        custom_payload: |
+          {
+            channel: 'C7FQWGDHP',
+            username: 'CI – ' + '${{ github.repository }}'.split('/')[1],
+            icon_emoji: ':hammer_and_wrench:',
+            attachments: [{
+              color: '${{ needs.test_matrix.result }}' === 'success' ? 'good' : '${{ needs.test_matrix.result }}' === 'failure' ? 'danger' : 'warning',
+              text: `${process.env.AS_WORKFLOW} against \`${{ github.ref }}\` (${process.env.AS_COMMIT}) for ${{ github.actor }} resulted in *${{ needs.test_matrix.result }}*.`
+            }]
+          }
+      env:
+        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

data/CHANGELOG.md

@@ -1,5 +1,35 @@
 ## [Unreleased]
-*no unreleased changes*
+* no relevant changes
+
+## 10.1 / 2021-03-08
+* Allow optional `last_data_column` in NdrImport::Table mappings (#61)
+
+## 10.0 / 2021-02-22
+### Changed
+* By default, escape any control characters found in XML (#60)
+
+## 9.1.0 / 2021-02-01
+### Added
+* `CSVLibrary` is now deprecated.
+* Handle xlsm files
+
+### Fixed
+* Fix `CSVLibrary.foreach` on Ruby 3.0
+* Updated jekyll bundle
+
+## 9.0.3 / 2021-01-04
+### Fixed
+* Address issue importing multiple files against the same table (#54)
+
+### Changed
+* ensure keyword args are properly splatted for ruby 2.7
+
+### Added
+* Ruby 2.7 to travis matrix
+
+## 9.0.2 / 2020-08-14
+### Changed
+* Configure Nokogiri with HUGE for large xml files
 
 ## 9.0.1 / 2020-03-26
 ### Fixed

data/README.md

@@ -1,5 +1,4 @@
-# NdrImport [![Build Status](https://travis-ci.org/PublicHealthEngland/ndr_import.svg?branch=master)](https://travis-ci.org/PublicHealthEngland/ndr_import) [![Gem Version](https://badge.fury.io/rb/ndr_import.svg)](https://badge.fury.io/rb/ndr_import) [![Documentation](https://img.shields.io/badge/ndr_import-docs-blue.svg)](https://www.rubydoc.info/gems/ndr_import)
-
+# NdrImport [![Build Status](https://github.com/publichealthengland/ndr_import/workflows/Test/badge.svg)](https://github.com/publichealthengland/ndr_import/actions?query=workflow%3Atest) [![Gem Version](https://badge.fury.io/rb/ndr_import.svg)](https://rubygems.org/gems/ndr_import) [![Documentation](https://img.shields.io/badge/ndr_import-docs-blue.svg)](https://www.rubydoc.info/gems/ndr_import)
 This is the Public Health England (PHE) National Disease Registers (NDR) Import ETL ruby gem, providing:
 
 1. file import handlers for *extracting* data from delimited files (csv, pipe, tab, thorn), .xls(x) spreadsheets, .doc(x) word documents, PDF, PDF AcroForms, XML, 7-Zip and Zip files.

data/code_safety.yml

@@ -1,5 +1,17 @@
 ---
 file safety:
+  ".github/CODEOWNERS":
+    comments: 
+    reviewed_by: ollietulloch
+    safe_revision: b64ff21375dcde2b8fefe622ee9861f0fea21487
+  ".github/workflows/lint.yml":
+    comments: 
+    reviewed_by: ollietulloch
+    safe_revision: b64ff21375dcde2b8fefe622ee9861f0fea21487
+  ".github/workflows/test.yml":
+    comments: 
+    reviewed_by: ollietulloch
+    safe_revision: b64ff21375dcde2b8fefe622ee9861f0fea21487
   ".gitignore":
     comments: whole file re-reviewed
     reviewed_by: josh.pencheon
@@ -12,14 +24,10 @@ file safety:
     comments: 
     reviewed_by: josh.pencheon
     safe_revision: b09e268ff9c8349b914aa1b7ba888e1d39f97e4a
-  ".travis.yml":
-    comments: 
-    reviewed_by: josh.pencheon
-    safe_revision: d3d9a987befeecb122a448d8d06e66d74da13fb5
   CHANGELOG.md:
     comments: 
-    reviewed_by: josh.pencheon
-    safe_revision: 0a6e63906f9f697c2220a33fa5502cfd7df6ae0e
+    reviewed_by: ollietulloch
+    safe_revision: 2d093cc57a699b527a7d0159e77b91f4409a6e0b
   CODE_OF_CONDUCT.md:
     comments: 
     reviewed_by: timgentry
@@ -38,8 +46,8 @@ file safety:
     safe_revision: 5d185a0aeba6a9cd2ff5e59efadcaeec9be45d8b
   README.md:
     comments: 
-    reviewed_by: josh.pencheon
-    safe_revision: 1bc459db8970dde36e9b240b6dd08cca629664e3
+    reviewed_by: ollietulloch
+    safe_revision: b64ff21375dcde2b8fefe622ee9861f0fea21487
   Rakefile:
     comments: 
     reviewed_by: josh.pencheon
@@ -58,8 +66,8 @@ file safety:
     safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
   docs/Gemfile.lock:
     comments: 
-    reviewed_by: josh.pencheon
-    safe_revision: af56f473382850e132f8b8953594c47a457fe579
+    reviewed_by: ollietulloch
+    safe_revision: ea0149c7739676463a252ffd9fbe4af238762b2b
   docs/_config.yml:
     comments: 
     reviewed_by: josh.pencheon
@@ -146,8 +154,8 @@ file safety:
     safe_revision: 24d6449fd0612552f132dfbf4cada2ae28d0469e
   lib/ndr_import/csv_library.rb:
     comments: 
-    reviewed_by: josh.pencheon
-    safe_revision: be12e57519d3737e8d3901d7b01485c6995708dd
+    reviewed_by: ollietulloch
+    safe_revision: 6b8668967dbd42d7893a0fa5f0aa1ec1c11227e1
   lib/ndr_import/file/acro_form.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -170,8 +178,8 @@ file safety:
     safe_revision: 897f8b648d633368cf2001d17ab89c06a12d445b
   lib/ndr_import/file/excel.rb:
     comments: 
-    reviewed_by: josh.pencheon
-    safe_revision: c3183e522bce50008df576ceb47fe4761ab8f966
+    reviewed_by: ollietulloch
+    safe_revision: 37482c79448bea80033f6f69d97584df330c9861
   lib/ndr_import/file/office_file_helper.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -214,8 +222,8 @@ file safety:
     safe_revision: dfc958d44b6c58355445fa395db08a62213ee709
   lib/ndr_import/helpers/file/delimited.rb:
     comments: 
-    reviewed_by: josh.pencheon
-    safe_revision: 607c0668f1fffd70d181bc1a31c4f56eed5f6189
+    reviewed_by: ollietulloch
+    safe_revision: 4a5cc1d362c632fc1f9242c69982fbce33557e17
   lib/ndr_import/helpers/file/excel.rb:
     comments: 
     reviewed_by: joshpencheon
@@ -230,8 +238,8 @@ file safety:
     safe_revision: 45da71ebd3acbc0fe53755bcd75483ba17cb6924
   lib/ndr_import/helpers/file/xml.rb:
     comments: 
-    reviewed_by: josh.pencheon
-    safe_revision: d2245268ec6a0e4f60c521d171a820f299632c4f
+    reviewed_by: joshpencheon
+    safe_revision: 3947f13e0cbd17f449eba292ad343eeb82116fe9
   lib/ndr_import/helpers/file/xml_streaming.rb:
     comments: uses SafePath and Shellwords when accessing filesystem, or making system
       calls
@@ -271,8 +279,8 @@ file safety:
     safe_revision: bb44ade56a2151706eede2c31142440ccf49e6f6
   lib/ndr_import/non_tabular/table.rb:
     comments: 
-    reviewed_by: josh.pencheon
-    safe_revision: 71979e0a602ca5a0ce415c194f10add9959f0116
+    reviewed_by: ollietulloch
+    safe_revision: f9df064adcfd38f09d83ad8c5496c84188faed98
   lib/ndr_import/non_tabular_file_helper.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -287,20 +295,20 @@ file safety:
     safe_revision: 3c7f827d17aacbf7b811eea67e27553f3b039070
   lib/ndr_import/table.rb:
     comments: uses File.basename
-    reviewed_by: josh.pencheon
-    safe_revision: a69d4a57ddcf13cdc13c27bd2eb91a395fa7ea36
+    reviewed_by: ollietulloch
+    safe_revision: 3cf7473181f7f835b3dfe7822f6833d751805eaf
   lib/ndr_import/universal_importer_helper.rb:
     comments: 
-    reviewed_by: josh.pencheon
-    safe_revision: 55e502bb4445cb8b985e530e8eb26d92b574ded9
+    reviewed_by: ollietulloch
+    safe_revision: 85869d99ae93252b7f3ef2d0a4db817c88d35c9e
   lib/ndr_import/unmapped_data_error.rb:
     comments: 
     reviewed_by: josh.pencheon
     safe_revision: 5cd2cd0b3a1e254d30d4acc28c6731825a1f84f5
   lib/ndr_import/version.rb:
     comments: another check?
-    reviewed_by: josh.pencheon
-    safe_revision: 0a6e63906f9f697c2220a33fa5502cfd7df6ae0e
+    reviewed_by: ollietulloch
+    safe_revision: 2d093cc57a699b527a7d0159e77b91f4409a6e0b
   lib/ndr_import/xml/table.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -309,6 +317,10 @@ file safety:
     comments: 
     reviewed_by: josh.pencheon
     safe_revision: 95e6ee9997d06471fe6f2f169c3c701471086371
+  test/csv_library_test.rb:
+    comments: 
+    reviewed_by: ollietulloch
+    safe_revision: 6b8668967dbd42d7893a0fa5f0aa1ec1c11227e1
   test/file/acro_form_test.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -327,16 +339,16 @@ file safety:
     safe_revision: a69d4a57ddcf13cdc13c27bd2eb91a395fa7ea36
   test/file/excel_test.rb:
     comments: 
-    reviewed_by: josh.pencheon
-    safe_revision: a69d4a57ddcf13cdc13c27bd2eb91a395fa7ea36
+    reviewed_by: ollietulloch
+    safe_revision: 85a080deaa93e4220ad1bf566f29cbdac9b31c0f
   test/file/pdf_test.rb:
     comments: 
     reviewed_by: josh.pencheon
     safe_revision: cb24ed3ea8116730d07f74546cd6fed0738b171d
   test/file/registry_test.rb:
     comments: 
-    reviewed_by: josh.pencheon
-    safe_revision: 5cd2cd0b3a1e254d30d4acc28c6731825a1f84f5
+    reviewed_by: ollietulloch
+    safe_revision: 85a080deaa93e4220ad1bf566f29cbdac9b31c0f
   test/file/seven_zip_test.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -383,8 +395,8 @@ file safety:
     safe_revision: ae75fb49baf028ac8ce08e4bedcd3625ff3ff0cd
   test/helpers/file/xml_test.rb:
     comments: 
-    reviewed_by: timgentry
-    safe_revision: 137170d443ea6bcc0afb18f62202c285ae6501eb
+    reviewed_by: joshpencheon
+    safe_revision: 3947f13e0cbd17f449eba292ad343eeb82116fe9
   test/helpers/file/zip_test.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -495,8 +507,8 @@ file safety:
     safe_revision: 71979e0a602ca5a0ce415c194f10add9959f0116
   test/resources/malformed.xml:
     comments: 
-    reviewed_by: timgentry
-    safe_revision: 137170d443ea6bcc0afb18f62202c285ae6501eb
+    reviewed_by: joshpencheon
+    safe_revision: 3947f13e0cbd17f449eba292ad343eeb82116fe9
   test/resources/malformed_pipe.csv:
     comments: 
     reviewed_by: josh.pencheon
@@ -553,6 +565,10 @@ file safety:
     comments: 
     reviewed_by: timgentry
     safe_revision: 8c30f89f0562ab120769c166d4e93ff839c055f7
+  test/resources/sample_xlsm.xlsm:
+    comments: 
+    reviewed_by: ollietulloch
+    safe_revision: 85a080deaa93e4220ad1bf566f29cbdac9b31c0f
   test/resources/sample_xlsx.xlsx:
     comments: 
     reviewed_by: timgentry
@@ -569,6 +585,10 @@ file safety:
     comments: 
     reviewed_by: timgentry
     safe_revision: 31fb1935f4578729d8786eea41cf0ce0a19be1cd
+  test/resources/two_files_single_table_mapping.zip:
+    comments: 
+    reviewed_by: ollietulloch
+    safe_revision: 830de0f8cb139c5f61525652b424423935cfc7ac
   test/resources/txt_file_xls_extension.xls:
     comments: 
     reviewed_by: timgentry
@@ -601,6 +621,10 @@ file safety:
     comments: 
     reviewed_by: timgentry
     safe_revision: f755c6960182f7dd460c18866cccfdf09178e860
+  test/resources/with-control-chars.xml:
+    comments: 
+    reviewed_by: joshpencheon
+    safe_revision: 3947f13e0cbd17f449eba292ad343eeb82116fe9
   test/resources/xlsx_file_xls_extension.xls:
     comments: 
     reviewed_by: timgentry
@@ -611,16 +635,16 @@ file safety:
     safe_revision: 3c7f827d17aacbf7b811eea67e27553f3b039070
   test/table_test.rb:
     comments: 
-    reviewed_by: josh.pencheon
-    safe_revision: a69d4a57ddcf13cdc13c27bd2eb91a395fa7ea36
+    reviewed_by: ollietulloch
+    safe_revision: 3cf7473181f7f835b3dfe7822f6833d751805eaf
   test/test_helper.rb:
     comments: 
     reviewed_by: josh.pencheon
     safe_revision: 93ccee82fc2165d1ca2d9b03d146ae03e769ea96
   test/universal_importer_helper_test.rb:
     comments: 
-    reviewed_by: josh.pencheon
-    safe_revision: c3183e522bce50008df576ceb47fe4761ab8f966
+    reviewed_by: ollietulloch
+    safe_revision: 85869d99ae93252b7f3ef2d0a4db817c88d35c9e
   test/xml/table_test.rb:
     comments: 
     reviewed_by: josh.pencheon

data/lib/ndr_import/csv_library.rb

@@ -1,38 +1,60 @@
 # This file allows us to choose the CSV library we want to use.
 
 require 'csv'
+require 'active_support/deprecation'
+
 # Using relevant core CSV library.
-CSVLibrary = CSV
+class CSVLibrary < CSV; end
 
 class << CSVLibrary
   # Is the library we're using FasterCSV?
   def fastercsv?
+    deprecate('if you desparately want fastercsv, please use it explicitly')
     not self.const_defined?(:Reader)
   end
 
   # Ensure that we can pass "mode" straight through the underlying IO object
+  #
+  # Note: this could likely be refactored now, as upstream support for something
+  #       very similar was added:
+  #
+  #       https://github.com/ruby/csv/commit/b4edaf2cf1aa36f5c6264c07514b66739b87ceee
+  #
   def foreach(path, **options, &block)
-    return to_enum(__method__, path, options) unless block
-    open(path, options.delete(:mode) || 'r', options) do |csv|
+    deprecate('CSV#foreach exists, with an optional `mode` argument')
+    return to_enum(__method__, path, **options) unless block
+    open(path, options.delete(:mode) || 'r', **options) do |csv|
       csv.each(&block)
     end
   end
 
   def write_csv_to_string(data)
+    deprecate('write_csv_to_string -> generate')
     self.generate do |csv|
       data.each { |line| csv << line }
     end
   end
 
   def write_csv_to_file(data, filepath, mode = 'w')
+    deprecate('write_csv_to_file -> open')
     self.open(filepath, mode) do |csv|
       data.each { |line| csv << line }
     end
   end
 
   def read_csv_from_file(filepath)
+    deprecate('read_csv_from_file -> read')
     self.read(filepath)
   end
+
+  private
+
+  def deprecate(additional_message = nil)
+    ActiveSupport::Deprecation.warn(<<~MESSAGE)
+      CSVLibrary is deprecated, and will be removed in a future version of ndr_import.
+      Please use standard functionality provided by Ruby's CSV library (#{additional_message}).
+    MESSAGE
+  end
 end
 
 # Forward port CSV::Cell, as it is sometimes

data/lib/ndr_import/file/excel.rb

@@ -90,14 +90,14 @@ module NdrImport
         case SafeFile.extname(path).downcase
         when '.xls'
           Roo::Excel.new(SafeFile.safepath_to_string(path))
-        when '.xlsx'
+        when '.xlsm', '.xlsx'
           if @options['file_password']
             Roo::Excelx.new(StringIO.new(decrypted_file_string(path, @options['file_password'])))
           else
             Roo::Excelx.new(SafeFile.safepath_to_string(path))
           end
         else
-          fail "Received file path with unexpected extension #{SafeFile.extname(path)}"
+          raise "Received file path with unexpected extension #{SafeFile.extname(path)}"
         end
       rescue Ole::Storage::FormatError => e
         # TODO: Do we need to remove the new_file after using it?
@@ -105,16 +105,14 @@ module NdrImport
         # try to load the .xls file as an .xlsx file, useful for sources like USOM
         # roo check file extensions in file_type_check (GenericSpreadsheet),
         # so we create a duplicate file in xlsx extension
-        if /(.*)\.xls$/.match(path)
-          new_file_name = SafeFile.basename(path).gsub(/(.*)\.xls$/, '\1_amend.xlsx')
-          new_file_path = SafeFile.dirname(path).join(new_file_name)
-          copy_file(path, new_file_path)
+        raise e.message unless /(.*)\.xls$/.match(path)
 
-          load_workbook(new_file_path)
-        else
-          raise e.message
-        end
-      rescue => e
+        new_file_name = SafeFile.basename(path).gsub(/(.*)\.xls$/, '\1_amend.xlsx')
+        new_file_path = SafeFile.dirname(path).join(new_file_name)
+        copy_file(path, new_file_path)
+
+        load_workbook(new_file_path)
+      rescue RuntimeError, ::Zip::Error => e
         raise ["Unable to read the file '#{path}'", e.message].join('; ')
       end
 
@@ -133,6 +131,6 @@ module NdrImport
       end
     end
 
-    Registry.register(Excel, 'xls', 'xlsx')
+    Registry.register(Excel, 'xls', 'xlsm', 'xlsx')
   end
 end

data/lib/ndr_import/helpers/file/delimited.rb

@@ -32,11 +32,11 @@ module NdrImport
           return enum_for(:delimited_rows, path, col_sep, liberal) unless block_given?
 
           safe_path = SafeFile.safepath_to_string(path)
-          encodings = determine_encodings!(safe_path, col_sep, liberal)
+          options = determine_encodings!(safe_path, col_sep, liberal)
 
-          # By now, we know `encodings` should let us read the whole
+          # By now, we know `options` should let us read the whole
           # file succesfully; if there are problems, we should crash.
-          CSVLibrary.foreach(safe_path, encodings) do |line|
+          CSV.foreach(safe_path, options.delete(:mode), **options) do |line|
             yield line.map(&:to_s)
           end
         end
@@ -46,7 +46,7 @@ module NdrImport
         # Derive the source encoding by trying all supported encodings.
         # Returns first set of working options, or raises if none could be found.
         def determine_encodings!(safe_path, col_sep, liberal)
-          # delimiter encoding => # FasterCSV encoding string
+          # delimiter encoding => # CSV encoding string
           supported_encodings = {
             'UTF-8'        => 'r:bom|utf-8',
             'Windows-1252' => 'r:windows-1252:utf-8'
@@ -67,14 +67,13 @@ module NdrImport
             begin
               options = {
                 col_sep: (col_sep || ',').force_encoding(delimiter_encoding),
-                liberal_parsing: liberal,
-                mode: access_mode
+                liberal_parsing: liberal
               }
 
               row_num = 0
               # Iterate through the file; if we reach the end, this encoding worked:
-              CSVLibrary.foreach(safe_path, options) { |_line| row_num += 1 }
-              return options
+              CSV.foreach(safe_path, access_mode, **options) { |_line| row_num += 1 }
+              return options.merge(mode: access_mode)
             rescue ArgumentError => e
               next if e.message =~ /invalid byte sequence/ # This encoding didn't work
               raise(e)

data/lib/ndr_import/helpers/file/xml.rb

@@ -10,15 +10,20 @@ module NdrImport
 
         private
 
-        def read_xml_file(path)
-          file_data = SafeFile.new(path).read
+        # By default, escapes any control characters found in the XML
+        # - their use is forbidden in XML 1.0, and highly discouraged
+        # in XML 1.1; any found are most likely to be erroneous.
+        def read_xml_file(path, preserve_control_chars: false)
+          file_data = ensure_utf8!(SafeFile.read(path))
+          escape_xml_control_chars!(file_data) unless preserve_control_chars
 
           require 'nokogiri'
 
-          Nokogiri::XML(ensure_utf8! file_data).tap do |doc|
-            doc.encoding = 'UTF-8'
-            emulate_strict_mode_fatal_check!(doc)
-          end
+          doc = Nokogiri::XML(file_data, &:huge)
+          doc.encoding = 'UTF-8'
+          emulate_strict_mode_fatal_check!(doc)
+
+          doc
         end
 
         # Nokogiri can use give a `STRICT` parse option to libxml, but our friendly
@@ -37,11 +42,19 @@ module NdrImport
           end
 
           return unless fatal_errors.any?
+
           raise Nokogiri::XML::SyntaxError, <<~MSG
             The file had #{fatal_errors.length} fatal error(s)!"
             #{fatal_errors.join("\n")}
           MSG
         end
+
+        # In place, escape out any control chars that would cause
+        # libxml to crash. Very few are allowable in XML 1.0, and
+        # remain heavily discouraged in XML 1.1.
+        def escape_xml_control_chars!(data)
+          escape_control_chars!(data)
+        end
       end
     end
   end

data/lib/ndr_import/non_tabular/table.rb

@@ -16,7 +16,7 @@ module NdrImport
 
       include UTF8Encoding
 
-      TABULAR_ONLY_OPTIONS = %w[delimiter liberal_parsing tablename_pattern
+      TABULAR_ONLY_OPTIONS = %w[delimiter last_data_column liberal_parsing tablename_pattern
                                 header_lines footer_lines xml_record_xpath].freeze
 
       NON_TABULAR_OPTIONS = %w[capture_end_line capture_start_line start_line_pattern

data/lib/ndr_import/table.rb

@@ -10,8 +10,9 @@ module NdrImport
     include NdrImport::Mapper
 
     def self.all_valid_options
-      %w[canonical_name delimiter liberal_parsing filename_pattern file_password tablename_pattern
-         header_lines footer_lines format klass columns xml_record_xpath row_identifier]
+      %w[canonical_name delimiter liberal_parsing filename_pattern file_password last_data_column
+         tablename_pattern header_lines footer_lines format klass columns xml_record_xpath
+         row_identifier]
     end
 
     def all_valid_options
@@ -50,8 +51,9 @@ module NdrImport
       @header_best_guess = nil
       @notifier.try(:started)
 
+      last_col = last_column_to_transform
       skip_footer_lines(lines, footer_lines).each do |line|
-        process_line(line, &block)
+        line.is_a?(Array) ? process_line(line[0..last_col], &block) : process_line(line, &block)
       end
 
       @notifier.try(:finished)
@@ -226,5 +228,26 @@ module NdrImport
     def column_names(column_mappings)
       column_mappings.map { |c| (c['column'] || c['standard_mapping']).downcase }
     end
+
+    # If specified in the mapping, stop transforming data at a given index (column)
+    def last_column_to_transform
+      return -1 if last_data_column.nil?
+      return last_data_column - 1 if last_data_column.is_a?(Integer)
+
+      error =  "Unknown 'last_data_column' format: #{last_data_column} " \
+               "(#{last_data_column.class})"
+      raise error unless last_data_column.is_a?(String) && last_data_column =~ /\A[A-Z]+\z/i
+
+      # If it's an excel column label (eg 'K', 'AF', 'DDE'), convert it to an index
+      index_from_column_label
+    end
+
+    def index_from_column_label
+      alphabet_index_hash = ('A'..'Z').map.with_index.to_h
+      index = last_data_column.upcase.chars.inject(0) do |char_index, char|
+        (char_index * 26) + (alphabet_index_hash[char] + 1)
+      end
+      index - 1
+    end
   end # class Table
 end

data/lib/ndr_import/universal_importer_helper.rb

@@ -7,11 +7,30 @@ module NdrImport
   # complexity of enumerating over files and tables (which should be universally useful).
   # It is assumed that the host module/class defines `unzip_path`.
   module UniversalImporterHelper
+    # Helper class to allow multiple source enumerators to contribute to one overall table.
+    class TableEnumProxy
+      include Enumerable
+
+      def initialize
+        @table_enums = []
+      end
+
+      def add_table_enum(table_enum)
+        @table_enums << table_enum
+      end
+
+      def each(&block)
+        return enum_for(:each) unless block
+
+        @table_enums.each { |table_enum| table_enum.each(&block) }
+      end
+    end
+
     def table_enumerators(filename)
-      table_enumerators = {}
+      table_enumerators = Hash.new { |hash, key| hash[key] = TableEnumProxy.new }
 
       extract(filename).each do |table, rows|
-        table_enumerators[table.canonical_name] = table.transform(rows)
+        table_enumerators[table.canonical_name].add_table_enum table.transform(rows)
       end
 
       table_enumerators
@@ -29,9 +48,7 @@ module NdrImport
     def extract(source_file, &block)
       return enum_for(:extract, source_file) unless block
 
-      files = NdrImport::File::Registry.files(source_file,
-                                              'unzip_path' => unzip_path)
-      files.each do |filename|
+      NdrImport::File::Registry.files(source_file, 'unzip_path' => unzip_path).each do |filename|
         # now at the individual file level, can we find the table mapping?
         table_mapping = get_table_mapping(filename, nil)
 

data/lib/ndr_import/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 # This stores the current version of the NdrImport gem
 module NdrImport
-  VERSION = '9.0.1'.freeze
+  VERSION = '10.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ndr_import
 version: !ruby/object:Gem::Version
-  version: 9.0.1
+  version: '10.1'
 platform: ruby
 authors:
 - NCRS Development Team
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-03-26 00:00:00.000000000 Z
+date: 2021-03-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -365,6 +365,9 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/CODEOWNERS"
+- ".github/workflows/lint.yml"
+- ".github/workflows/test.yml"
 - ".gitignore"
 - ".hound.yml"
 - ".rubocop.yml"