ndr_import 9.0.2 → 10.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7dc0801c147101346c5a2e0af96ef9a658bc068088bb54f4f65970bb012e6cf5
-  data.tar.gz: 22823d5d0415a95eebac8151475d6eba7fdbd1744003ee0d88b1dedf86b837b2
+  metadata.gz: 2463ac35899a6db81e345b75b0ea10186530f559460dfca1211ba7694f52b760
+  data.tar.gz: 2f2cf39e959beeb3cfe6bcad033eb2c0695486dce3b640fe3de8b0c2b9b88a2f
 SHA512:
-  metadata.gz: 3c29ef1ab701b94301aca343d8bf8497fda2633d120b3945dc140cb30d1b40b30bfcfb2334fe2bdd6f02d6128f1743ac42601f8129687502503e69c14d77df98
-  data.tar.gz: 49e739dab53bf1276655e01a91dd9253221e041c0f3241b85f362a2d429597fbb4589d14b89b1d83fed7995b043288e72f3bf1bd27d2678e6f7d2c4b1a5c3d8f
+  metadata.gz: c6eca601043ff01ebe910375a36131bdf55ebd3f18664e3db3c07180f007b0f06dc50cc3016634a12764befc1e28d621a09a7b382392f67608aa4d21c83c7f2d
+  data.tar.gz: dc568494bfc4b39b7ee47c7738511274cb85ad9532e380f3c27870682a1a21d2acda4d7d8d855597abdd0643b8afe0ed6e161b74f201fa53640b5c9641f895c5

data/.github/CODEOWNERS

@@ -0,0 +1,2 @@
+# Admins should have oversight of the version:
+lib/ndr_import/version.rb      @publichealthengland/ndr-admins

data/.github/workflows/lint.yml

@@ -0,0 +1,23 @@
+name: Lint
+
+on: [pull_request]
+
+jobs:
+  rubocop:
+    name: RuboCop
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0 # fetch everything
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: 3.0
+    - name: Install dependencies
+      run: bundle install
+    - name: Run RuboCop against BASE..HEAD changes
+      run: bundle exec rake rubocop:diff origin/${GITHUB_BASE_REF#*/}
+

data/.github/workflows/test.yml

@@ -0,0 +1,72 @@
+name: Test
+
+on: [push]
+
+jobs:
+  test:
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby-version:
+          - 2.6
+          - 2.7
+          - 3.0
+        gemfile:
+          - gemfiles/Gemfile.rails52
+          - gemfiles/Gemfile.rails60
+
+    name: Ruby ${{ matrix.ruby-version }} / Bundle ${{ matrix.gemfile }}
+
+    runs-on: ubuntu-latest
+
+    env:
+      BUNDLE_GEMFILE: ${{ matrix.gemfile }}
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby-version }}
+    - name: Install dependencies
+      run: bundle install
+    - name: Run tests
+      run: bundle exec rake
+
+  # A utility job upon which Branch Protection can depend,
+  # thus remaining agnostic of the matrix.
+  test_matrix:
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    name: Matrix
+    needs: test
+    steps:
+    - name: Check build matrix status
+      if: ${{ needs.test.result != 'success' }}
+      run: exit 1
+
+  notify:
+    # Run only on master, but regardless of whether tests past:
+    if: ${{ always() && github.ref == 'refs/heads/master' }}
+
+    needs: test_matrix
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: 8398a7/action-slack@v3
+      with:
+        status: custom
+        fields: workflow,commit,author
+        custom_payload: |
+          {
+            channel: 'C7FQWGDHP',
+            username: 'CI – ' + '${{ github.repository }}'.split('/')[1],
+            icon_emoji: ':hammer_and_wrench:',
+            attachments: [{
+              color: '${{ needs.test_matrix.result }}' === 'success' ? 'good' : '${{ needs.test_matrix.result }}' === 'failure' ? 'danger' : 'warning',
+              text: `${process.env.AS_WORKFLOW} against \`${{ github.ref }}\` (${process.env.AS_COMMIT}) for ${{ github.actor }} resulted in *${{ needs.test_matrix.result }}*.`
+            }]
+          }
+      env:
+        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

data/CHANGELOG.md

@@ -1,6 +1,37 @@
 ## [Unreleased]
 *no unreleased changes*
 
+## 10.1.1 / 2021-03-15
+### Fixed
+* XML: ensure invalid control character *references* are also escaped (#64)
+
+## 10.1.0 / 2021-03-08
+### Added
+* Allow optional `last_data_column` in NdrImport::Table mappings (#61)
+
+## 10.0.0 / 2021-02-22
+### Changed
+* By default, escape any control characters found in XML (#60)
+
+## 9.1.0 / 2021-02-01
+### Added
+* `CSVLibrary` is now deprecated.
+* Handle xlsm files
+
+### Fixed
+* Fix `CSVLibrary.foreach` on Ruby 3.0
+* Updated jekyll bundle
+
+## 9.0.3 / 2021-01-04
+### Fixed
+* Address issue importing multiple files against the same table (#54)
+
+### Changed
+* ensure keyword args are properly splatted for ruby 2.7
+
+### Added
+* Ruby 2.7 to travis matrix
+
 ## 9.0.2 / 2020-08-14
 ### Changed
 * Configure Nokogiri with HUGE for large xml files

data/README.md

@@ -1,5 +1,4 @@
-# NdrImport [![Build Status](https://travis-ci.org/PublicHealthEngland/ndr_import.svg?branch=master)](https://travis-ci.org/PublicHealthEngland/ndr_import) [![Gem Version](https://badge.fury.io/rb/ndr_import.svg)](https://badge.fury.io/rb/ndr_import) [![Documentation](https://img.shields.io/badge/ndr_import-docs-blue.svg)](https://www.rubydoc.info/gems/ndr_import)
-
+# NdrImport [![Build Status](https://github.com/publichealthengland/ndr_import/workflows/Test/badge.svg)](https://github.com/publichealthengland/ndr_import/actions?query=workflow%3Atest) [![Gem Version](https://badge.fury.io/rb/ndr_import.svg)](https://rubygems.org/gems/ndr_import) [![Documentation](https://img.shields.io/badge/ndr_import-docs-blue.svg)](https://www.rubydoc.info/gems/ndr_import)
 This is the Public Health England (PHE) National Disease Registers (NDR) Import ETL ruby gem, providing:
 
 1. file import handlers for *extracting* data from delimited files (csv, pipe, tab, thorn), .xls(x) spreadsheets, .doc(x) word documents, PDF, PDF AcroForms, XML, 7-Zip and Zip files.

data/code_safety.yml

@@ -1,5 +1,17 @@
 ---
 file safety:
+  ".github/CODEOWNERS":
+    comments: 
+    reviewed_by: ollietulloch
+    safe_revision: b64ff21375dcde2b8fefe622ee9861f0fea21487
+  ".github/workflows/lint.yml":
+    comments: 
+    reviewed_by: ollietulloch
+    safe_revision: b64ff21375dcde2b8fefe622ee9861f0fea21487
+  ".github/workflows/test.yml":
+    comments: 
+    reviewed_by: ollietulloch
+    safe_revision: b64ff21375dcde2b8fefe622ee9861f0fea21487
   ".gitignore":
     comments: whole file re-reviewed
     reviewed_by: josh.pencheon
@@ -12,14 +24,10 @@ file safety:
     comments: 
     reviewed_by: josh.pencheon
     safe_revision: b09e268ff9c8349b914aa1b7ba888e1d39f97e4a
-  ".travis.yml":
-    comments: 
-    reviewed_by: josh.pencheon
-    safe_revision: d3d9a987befeecb122a448d8d06e66d74da13fb5
   CHANGELOG.md:
     comments: 
-    reviewed_by: ollietulloch
-    safe_revision: aa006cd76123db2101d145a07d201dc6a709ed6e
+    reviewed_by: josh.pencheon
+    safe_revision: 47fa3633ec2e48f1ee9fb12aad03e817e73c54bf
   CODE_OF_CONDUCT.md:
     comments: 
     reviewed_by: timgentry
@@ -38,8 +46,8 @@ file safety:
     safe_revision: 5d185a0aeba6a9cd2ff5e59efadcaeec9be45d8b
   README.md:
     comments: 
-    reviewed_by: josh.pencheon
-    safe_revision: 1bc459db8970dde36e9b240b6dd08cca629664e3
+    reviewed_by: ollietulloch
+    safe_revision: b64ff21375dcde2b8fefe622ee9861f0fea21487
   Rakefile:
     comments: 
     reviewed_by: josh.pencheon
@@ -59,7 +67,7 @@ file safety:
   docs/Gemfile.lock:
     comments: 
     reviewed_by: ollietulloch
-    safe_revision: 6f274715bb341c3070190f04f67af9500b510580
+    safe_revision: ea0149c7739676463a252ffd9fbe4af238762b2b
   docs/_config.yml:
     comments: 
     reviewed_by: josh.pencheon
@@ -146,8 +154,8 @@ file safety:
     safe_revision: 24d6449fd0612552f132dfbf4cada2ae28d0469e
   lib/ndr_import/csv_library.rb:
     comments: 
-    reviewed_by: josh.pencheon
-    safe_revision: be12e57519d3737e8d3901d7b01485c6995708dd
+    reviewed_by: ollietulloch
+    safe_revision: 6b8668967dbd42d7893a0fa5f0aa1ec1c11227e1
   lib/ndr_import/file/acro_form.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -170,8 +178,8 @@ file safety:
     safe_revision: 897f8b648d633368cf2001d17ab89c06a12d445b
   lib/ndr_import/file/excel.rb:
     comments: 
-    reviewed_by: josh.pencheon
-    safe_revision: c3183e522bce50008df576ceb47fe4761ab8f966
+    reviewed_by: ollietulloch
+    safe_revision: 37482c79448bea80033f6f69d97584df330c9861
   lib/ndr_import/file/office_file_helper.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -214,8 +222,8 @@ file safety:
     safe_revision: dfc958d44b6c58355445fa395db08a62213ee709
   lib/ndr_import/helpers/file/delimited.rb:
     comments: 
-    reviewed_by: josh.pencheon
-    safe_revision: 607c0668f1fffd70d181bc1a31c4f56eed5f6189
+    reviewed_by: ollietulloch
+    safe_revision: 4a5cc1d362c632fc1f9242c69982fbce33557e17
   lib/ndr_import/helpers/file/excel.rb:
     comments: 
     reviewed_by: joshpencheon
@@ -230,8 +238,8 @@ file safety:
     safe_revision: 45da71ebd3acbc0fe53755bcd75483ba17cb6924
   lib/ndr_import/helpers/file/xml.rb:
     comments: 
-    reviewed_by: ollietulloch
-    safe_revision: 4d337bd233f7e60cf9d363c92400f21269a28da7
+    reviewed_by: josh.pencheon
+    safe_revision: 9a6cc769abce5f9bfa5b4f8bd5cda52dfe18b12b
   lib/ndr_import/helpers/file/xml_streaming.rb:
     comments: uses SafePath and Shellwords when accessing filesystem, or making system
       calls
@@ -272,7 +280,7 @@ file safety:
   lib/ndr_import/non_tabular/table.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 71979e0a602ca5a0ce415c194f10add9959f0116
+    safe_revision: f9df064adcfd38f09d83ad8c5496c84188faed98
   lib/ndr_import/non_tabular_file_helper.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -288,19 +296,23 @@ file safety:
   lib/ndr_import/table.rb:
     comments: uses File.basename
     reviewed_by: josh.pencheon
-    safe_revision: a69d4a57ddcf13cdc13c27bd2eb91a395fa7ea36
+    safe_revision: 3cf7473181f7f835b3dfe7822f6833d751805eaf
   lib/ndr_import/universal_importer_helper.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 55e502bb4445cb8b985e530e8eb26d92b574ded9
+    safe_revision: 85869d99ae93252b7f3ef2d0a4db817c88d35c9e
   lib/ndr_import/unmapped_data_error.rb:
     comments: 
     reviewed_by: josh.pencheon
     safe_revision: 5cd2cd0b3a1e254d30d4acc28c6731825a1f84f5
   lib/ndr_import/version.rb:
     comments: another check?
-    reviewed_by: ollietulloch
-    safe_revision: aa006cd76123db2101d145a07d201dc6a709ed6e
+    reviewed_by: josh.pencheon
+    safe_revision: 47fa3633ec2e48f1ee9fb12aad03e817e73c54bf
+  lib/ndr_import/xml/control_char_escaper.rb:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 9a6cc769abce5f9bfa5b4f8bd5cda52dfe18b12b
   lib/ndr_import/xml/table.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -309,6 +321,10 @@ file safety:
     comments: 
     reviewed_by: josh.pencheon
     safe_revision: 95e6ee9997d06471fe6f2f169c3c701471086371
+  test/csv_library_test.rb:
+    comments: 
+    reviewed_by: ollietulloch
+    safe_revision: 6b8668967dbd42d7893a0fa5f0aa1ec1c11227e1
   test/file/acro_form_test.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -327,16 +343,16 @@ file safety:
     safe_revision: a69d4a57ddcf13cdc13c27bd2eb91a395fa7ea36
   test/file/excel_test.rb:
     comments: 
-    reviewed_by: josh.pencheon
-    safe_revision: a69d4a57ddcf13cdc13c27bd2eb91a395fa7ea36
+    reviewed_by: ollietulloch
+    safe_revision: 85a080deaa93e4220ad1bf566f29cbdac9b31c0f
   test/file/pdf_test.rb:
     comments: 
     reviewed_by: josh.pencheon
     safe_revision: cb24ed3ea8116730d07f74546cd6fed0738b171d
   test/file/registry_test.rb:
     comments: 
-    reviewed_by: josh.pencheon
-    safe_revision: 5cd2cd0b3a1e254d30d4acc28c6731825a1f84f5
+    reviewed_by: ollietulloch
+    safe_revision: 85a080deaa93e4220ad1bf566f29cbdac9b31c0f
   test/file/seven_zip_test.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -383,8 +399,8 @@ file safety:
     safe_revision: ae75fb49baf028ac8ce08e4bedcd3625ff3ff0cd
   test/helpers/file/xml_test.rb:
     comments: 
-    reviewed_by: timgentry
-    safe_revision: 137170d443ea6bcc0afb18f62202c285ae6501eb
+    reviewed_by: josh.pencheon
+    safe_revision: 9a6cc769abce5f9bfa5b4f8bd5cda52dfe18b12b
   test/helpers/file/zip_test.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -495,8 +511,8 @@ file safety:
     safe_revision: 71979e0a602ca5a0ce415c194f10add9959f0116
   test/resources/malformed.xml:
     comments: 
-    reviewed_by: timgentry
-    safe_revision: 137170d443ea6bcc0afb18f62202c285ae6501eb
+    reviewed_by: joshpencheon
+    safe_revision: 3947f13e0cbd17f449eba292ad343eeb82116fe9
   test/resources/malformed_pipe.csv:
     comments: 
     reviewed_by: josh.pencheon
@@ -553,6 +569,10 @@ file safety:
     comments: 
     reviewed_by: timgentry
     safe_revision: 8c30f89f0562ab120769c166d4e93ff839c055f7
+  test/resources/sample_xlsm.xlsm:
+    comments: 
+    reviewed_by: ollietulloch
+    safe_revision: 85a080deaa93e4220ad1bf566f29cbdac9b31c0f
   test/resources/sample_xlsx.xlsx:
     comments: 
     reviewed_by: timgentry
@@ -569,6 +589,10 @@ file safety:
     comments: 
     reviewed_by: timgentry
     safe_revision: 31fb1935f4578729d8786eea41cf0ce0a19be1cd
+  test/resources/two_files_single_table_mapping.zip:
+    comments: 
+    reviewed_by: ollietulloch
+    safe_revision: 830de0f8cb139c5f61525652b424423935cfc7ac
   test/resources/txt_file_xls_extension.xls:
     comments: 
     reviewed_by: timgentry
@@ -601,6 +625,22 @@ file safety:
     comments: 
     reviewed_by: timgentry
     safe_revision: f755c6960182f7dd460c18866cccfdf09178e860
+  test/resources/with-control-char-references-in-cdata.xml:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 9a6cc769abce5f9bfa5b4f8bd5cda52dfe18b12b
+  test/resources/with-control-char-references.xml:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 9a6cc769abce5f9bfa5b4f8bd5cda52dfe18b12b
+  test/resources/with-control-chars.xml:
+    comments: 
+    reviewed_by: joshpencheon
+    safe_revision: 3947f13e0cbd17f449eba292ad343eeb82116fe9
+  test/resources/with-non-control-char-references.xml:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 9a6cc769abce5f9bfa5b4f8bd5cda52dfe18b12b
   test/resources/xlsx_file_xls_extension.xls:
     comments: 
     reviewed_by: timgentry
@@ -612,7 +652,7 @@ file safety:
   test/table_test.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: a69d4a57ddcf13cdc13c27bd2eb91a395fa7ea36
+    safe_revision: 3cf7473181f7f835b3dfe7822f6833d751805eaf
   test/test_helper.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -620,7 +660,11 @@ file safety:
   test/universal_importer_helper_test.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: c3183e522bce50008df576ceb47fe4761ab8f966
+    safe_revision: 85869d99ae93252b7f3ef2d0a4db817c88d35c9e
+  test/xml/control_char_escaper_test.rb:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 9a6cc769abce5f9bfa5b4f8bd5cda52dfe18b12b
   test/xml/table_test.rb:
     comments: 
     reviewed_by: josh.pencheon

data/lib/ndr_import/csv_library.rb

@@ -1,38 +1,60 @@
 # This file allows us to choose the CSV library we want to use.
 
 require 'csv'
+require 'active_support/deprecation'
+
 # Using relevant core CSV library.
-CSVLibrary = CSV
+class CSVLibrary < CSV; end
 
 class << CSVLibrary
   # Is the library we're using FasterCSV?
   def fastercsv?
+    deprecate('if you desparately want fastercsv, please use it explicitly')
     not self.const_defined?(:Reader)
   end
 
   # Ensure that we can pass "mode" straight through the underlying IO object
+  #
+  # Note: this could likely be refactored now, as upstream support for something
+  #       very similar was added:
+  #
+  #       https://github.com/ruby/csv/commit/b4edaf2cf1aa36f5c6264c07514b66739b87ceee
+  #
   def foreach(path, **options, &block)
-    return to_enum(__method__, path, options) unless block
-    open(path, options.delete(:mode) || 'r', options) do |csv|
+    deprecate('CSV#foreach exists, with an optional `mode` argument')
+    return to_enum(__method__, path, **options) unless block
+    open(path, options.delete(:mode) || 'r', **options) do |csv|
       csv.each(&block)
     end
   end
 
   def write_csv_to_string(data)
+    deprecate('write_csv_to_string -> generate')
     self.generate do |csv|
       data.each { |line| csv << line }
     end
   end
 
   def write_csv_to_file(data, filepath, mode = 'w')
+    deprecate('write_csv_to_file -> open')
     self.open(filepath, mode) do |csv|
       data.each { |line| csv << line }
     end
   end
 
   def read_csv_from_file(filepath)
+    deprecate('read_csv_from_file -> read')
     self.read(filepath)
   end
+
+  private
+
+  def deprecate(additional_message = nil)
+    ActiveSupport::Deprecation.warn(<<~MESSAGE)
+      CSVLibrary is deprecated, and will be removed in a future version of ndr_import.
+      Please use standard functionality provided by Ruby's CSV library (#{additional_message}).
+    MESSAGE
+  end
 end
 
 # Forward port CSV::Cell, as it is sometimes

data/lib/ndr_import/file/excel.rb

@@ -90,14 +90,14 @@ module NdrImport
         case SafeFile.extname(path).downcase
         when '.xls'
           Roo::Excel.new(SafeFile.safepath_to_string(path))
-        when '.xlsx'
+        when '.xlsm', '.xlsx'
           if @options['file_password']
             Roo::Excelx.new(StringIO.new(decrypted_file_string(path, @options['file_password'])))
           else
             Roo::Excelx.new(SafeFile.safepath_to_string(path))
           end
         else
-          fail "Received file path with unexpected extension #{SafeFile.extname(path)}"
+          raise "Received file path with unexpected extension #{SafeFile.extname(path)}"
         end
       rescue Ole::Storage::FormatError => e
         # TODO: Do we need to remove the new_file after using it?
@@ -105,16 +105,14 @@ module NdrImport
         # try to load the .xls file as an .xlsx file, useful for sources like USOM
         # roo check file extensions in file_type_check (GenericSpreadsheet),
         # so we create a duplicate file in xlsx extension
-        if /(.*)\.xls$/.match(path)
-          new_file_name = SafeFile.basename(path).gsub(/(.*)\.xls$/, '\1_amend.xlsx')
-          new_file_path = SafeFile.dirname(path).join(new_file_name)
-          copy_file(path, new_file_path)
+        raise e.message unless /(.*)\.xls$/.match(path)
 
-          load_workbook(new_file_path)
-        else
-          raise e.message
-        end
-      rescue => e
+        new_file_name = SafeFile.basename(path).gsub(/(.*)\.xls$/, '\1_amend.xlsx')
+        new_file_path = SafeFile.dirname(path).join(new_file_name)
+        copy_file(path, new_file_path)
+
+        load_workbook(new_file_path)
+      rescue RuntimeError, ::Zip::Error => e
         raise ["Unable to read the file '#{path}'", e.message].join('; ')
       end
 
@@ -133,6 +131,6 @@ module NdrImport
       end
     end
 
-    Registry.register(Excel, 'xls', 'xlsx')
+    Registry.register(Excel, 'xls', 'xlsm', 'xlsx')
   end
 end

data/lib/ndr_import/helpers/file/delimited.rb

@@ -32,11 +32,11 @@ module NdrImport
           return enum_for(:delimited_rows, path, col_sep, liberal) unless block_given?
 
           safe_path = SafeFile.safepath_to_string(path)
-          encodings = determine_encodings!(safe_path, col_sep, liberal)
+          options = determine_encodings!(safe_path, col_sep, liberal)
 
-          # By now, we know `encodings` should let us read the whole
+          # By now, we know `options` should let us read the whole
           # file succesfully; if there are problems, we should crash.
-          CSVLibrary.foreach(safe_path, encodings) do |line|
+          CSV.foreach(safe_path, options.delete(:mode), **options) do |line|
             yield line.map(&:to_s)
           end
         end
@@ -46,7 +46,7 @@ module NdrImport
         # Derive the source encoding by trying all supported encodings.
         # Returns first set of working options, or raises if none could be found.
         def determine_encodings!(safe_path, col_sep, liberal)
-          # delimiter encoding => # FasterCSV encoding string
+          # delimiter encoding => # CSV encoding string
           supported_encodings = {
             'UTF-8'        => 'r:bom|utf-8',
             'Windows-1252' => 'r:windows-1252:utf-8'
@@ -67,14 +67,13 @@ module NdrImport
             begin
               options = {
                 col_sep: (col_sep || ',').force_encoding(delimiter_encoding),
-                liberal_parsing: liberal,
-                mode: access_mode
+                liberal_parsing: liberal
               }
 
               row_num = 0
               # Iterate through the file; if we reach the end, this encoding worked:
-              CSVLibrary.foreach(safe_path, options) { |_line| row_num += 1 }
-              return options
+              CSV.foreach(safe_path, access_mode, **options) { |_line| row_num += 1 }
+              return options.merge(mode: access_mode)
             rescue ArgumentError => e
               next if e.message =~ /invalid byte sequence/ # This encoding didn't work
               raise(e)

data/lib/ndr_import/helpers/file/xml.rb

@@ -1,3 +1,4 @@
+require 'ndr_import/xml/control_char_escaper'
 require 'ndr_support/safe_file'
 require 'ndr_support/utf8_encoding'
 
@@ -10,16 +11,21 @@ module NdrImport
 
         private
 
-        def read_xml_file(path)
-          file_data = SafeFile.new(path).read
+        # By default, escapes any control characters found in the XML
+        # - their use is forbidden in XML 1.0, and highly discouraged
+        # in XML 1.1; any found are most likely to be erroneous.
+        def read_xml_file(path, preserve_control_chars: false)
+          file_data = ensure_utf8!(SafeFile.read(path))
 
           require 'nokogiri'
 
-          doc = Nokogiri::XML((ensure_utf8! file_data)) do |config|
-            config.huge
+          doc = nil
+
+          escaping_control_chars_if_necessary(preserve_control_chars, file_data) do
+            doc = Nokogiri::XML(file_data, &:huge)
+            doc.encoding = 'UTF-8'
+            emulate_strict_mode_fatal_check!(doc)
           end
-          doc.encoding = 'UTF-8'
-          emulate_strict_mode_fatal_check!(doc)
 
           doc
         end
@@ -40,11 +46,27 @@ module NdrImport
           end
 
           return unless fatal_errors.any?
+
           raise Nokogiri::XML::SyntaxError, <<~MSG
             The file had #{fatal_errors.length} fatal error(s)!"
             #{fatal_errors.join("\n")}
           MSG
         end
+
+        def escaping_control_chars_if_necessary(preserve_control_chars, file_data)
+          return yield if preserve_control_chars
+
+          tried_escaping = false
+          begin
+            yield
+          rescue Nokogiri::XML::SyntaxError => e
+            raise e if tried_escaping
+
+            NdrImport::Xml::ControlCharEscaper.new(file_data).escape!
+            tried_escaping = true
+            retry
+          end
+        end
       end
     end
   end

data/lib/ndr_import/non_tabular/table.rb

@@ -16,7 +16,7 @@ module NdrImport
 
       include UTF8Encoding
 
-      TABULAR_ONLY_OPTIONS = %w[delimiter liberal_parsing tablename_pattern
+      TABULAR_ONLY_OPTIONS = %w[delimiter last_data_column liberal_parsing tablename_pattern
                                 header_lines footer_lines xml_record_xpath].freeze
 
       NON_TABULAR_OPTIONS = %w[capture_end_line capture_start_line start_line_pattern

data/lib/ndr_import/table.rb

@@ -10,8 +10,9 @@ module NdrImport
     include NdrImport::Mapper
 
     def self.all_valid_options
-      %w[canonical_name delimiter liberal_parsing filename_pattern file_password tablename_pattern
-         header_lines footer_lines format klass columns xml_record_xpath row_identifier]
+      %w[canonical_name delimiter liberal_parsing filename_pattern file_password last_data_column
+         tablename_pattern header_lines footer_lines format klass columns xml_record_xpath
+         row_identifier]
     end
 
     def all_valid_options
@@ -50,8 +51,9 @@ module NdrImport
       @header_best_guess = nil
       @notifier.try(:started)
 
+      last_col = last_column_to_transform
       skip_footer_lines(lines, footer_lines).each do |line|
-        process_line(line, &block)
+        line.is_a?(Array) ? process_line(line[0..last_col], &block) : process_line(line, &block)
       end
 
       @notifier.try(:finished)
@@ -226,5 +228,26 @@ module NdrImport
     def column_names(column_mappings)
       column_mappings.map { |c| (c['column'] || c['standard_mapping']).downcase }
     end
+
+    # If specified in the mapping, stop transforming data at a given index (column)
+    def last_column_to_transform
+      return -1 if last_data_column.nil?
+      return last_data_column - 1 if last_data_column.is_a?(Integer)
+
+      error =  "Unknown 'last_data_column' format: #{last_data_column} " \
+               "(#{last_data_column.class})"
+      raise error unless last_data_column.is_a?(String) && last_data_column =~ /\A[A-Z]+\z/i
+
+      # If it's an excel column label (eg 'K', 'AF', 'DDE'), convert it to an index
+      index_from_column_label
+    end
+
+    def index_from_column_label
+      alphabet_index_hash = ('A'..'Z').map.with_index.to_h
+      index = last_data_column.upcase.chars.inject(0) do |char_index, char|
+        (char_index * 26) + (alphabet_index_hash[char] + 1)
+      end
+      index - 1
+    end
   end # class Table
 end

data/lib/ndr_import/universal_importer_helper.rb

@@ -7,11 +7,30 @@ module NdrImport
   # complexity of enumerating over files and tables (which should be universally useful).
   # It is assumed that the host module/class defines `unzip_path`.
   module UniversalImporterHelper
+    # Helper class to allow multiple source enumerators to contribute to one overall table.
+    class TableEnumProxy
+      include Enumerable
+
+      def initialize
+        @table_enums = []
+      end
+
+      def add_table_enum(table_enum)
+        @table_enums << table_enum
+      end
+
+      def each(&block)
+        return enum_for(:each) unless block
+
+        @table_enums.each { |table_enum| table_enum.each(&block) }
+      end
+    end
+
     def table_enumerators(filename)
-      table_enumerators = {}
+      table_enumerators = Hash.new { |hash, key| hash[key] = TableEnumProxy.new }
 
       extract(filename).each do |table, rows|
-        table_enumerators[table.canonical_name] = table.transform(rows)
+        table_enumerators[table.canonical_name].add_table_enum table.transform(rows)
       end
 
       table_enumerators
@@ -29,9 +48,7 @@ module NdrImport
     def extract(source_file, &block)
       return enum_for(:extract, source_file) unless block
 
-      files = NdrImport::File::Registry.files(source_file,
-                                              'unzip_path' => unzip_path)
-      files.each do |filename|
+      NdrImport::File::Registry.files(source_file, 'unzip_path' => unzip_path).each do |filename|
         # now at the individual file level, can we find the table mapping?
         table_mapping = get_table_mapping(filename, nil)
 

data/lib/ndr_import/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 # This stores the current version of the NdrImport gem
 module NdrImport
-  VERSION = '9.0.2'.freeze
+  VERSION = '10.1.1'
 end

data/lib/ndr_import/xml/control_char_escaper.rb

@@ -0,0 +1,51 @@
+require 'ndr_support/utf8_encoding'
+
+module NdrImport
+  module Xml
+    # A class to remove control characters, and XML entities representing them
+    class ControlCharEscaper
+      include UTF8Encoding
+
+      # Matches XML character reference entities
+      CHARACTER_REFERENCES = /&#(?:(?<decimal>\d+)|x(?<hex>\h+));/.freeze
+
+      attr_reader :data
+
+      def initialize(data)
+        @data = data
+      end
+
+      def escape!
+        unescape_control_char_references!(data)
+        escape_control_chars!(data)
+      end
+
+      private
+
+      def unescape_control_char_references!(data)
+        data.gsub!(CHARACTER_REFERENCES) do |reference|
+          char = try_to_extract_char_from(Regexp.last_match)
+
+          if char&.match?(CONTROL_CHARACTERS)
+            escape_control_chars!(char)
+          else
+            reference
+          end
+        end
+      end
+
+      def try_to_extract_char_from(match)
+        if match.nil?
+          nil
+        elsif match[:decimal]
+          match[:decimal].to_i(10).chr
+        elsif match[:hex]
+          match[:hex].to_i(16).chr
+        end
+      rescue RangeError
+        # Return everything if the match was against junk:
+        match.to_s
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ndr_import
 version: !ruby/object:Gem::Version
-  version: 9.0.2
+  version: 10.1.1
 platform: ruby
 authors:
 - NCRS Development Team
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-08-14 00:00:00.000000000 Z
+date: 2021-03-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -365,6 +365,9 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/CODEOWNERS"
+- ".github/workflows/lint.yml"
+- ".github/workflows/test.yml"
 - ".gitignore"
 - ".hound.yml"
 - ".rubocop.yml"
@@ -424,6 +427,7 @@ files:
 - lib/ndr_import/universal_importer_helper.rb
 - lib/ndr_import/unmapped_data_error.rb
 - lib/ndr_import/version.rb
+- lib/ndr_import/xml/control_char_escaper.rb
 - lib/ndr_import/xml/table.rb
 - ndr_import.gemspec
 homepage: https://github.com/PublicHealthEngland/ndr_import