ndr_import 9.1.0 → 10.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aaf3c826acb51f4d579fb956e9606b5b86e120f3bd561db762d72081f27a1098
-  data.tar.gz: 2b11ddf7dc9b748b4a1ac9cc91b47c47b99677e4bde990c44d1ec20e76efbfc0
+  metadata.gz: '0080a6cf08b7832f1af6261ff2c774a0fd8629b02c363869ac852389f239731c'
+  data.tar.gz: f1f986e3e1d59d65cb260cd555e5c17716228cc3ba61ff58fc2df40fc9abb19e
 SHA512:
-  metadata.gz: 3faaf744255693f04425b6e5ed1ec7198e912d2854b3f482a39d9bc53fe3859ed3bb6ef7855cdfd03959042a530f3a2f528239548676c7159c9029d69b6c160d
-  data.tar.gz: 1aa9d1895d7f874499823c4b87f1874e025b376f239030f3baf40bab5f5ed3ed09f80721205e271446893a9070652df3f26fceb48b59a9eb86ac121fdbfd92c6
+  metadata.gz: bf67d9b21781db778b92e245af52182dab5e1cc41d1637a87f82df953aaeae017fc14254b3e3caa3dcb9e6459489e3661f5610277f04af79e6aaedd00109b299
+  data.tar.gz: '08dc2fe4d3dc4b0dcd34cc361285f04eeacf11afb244fc117d1b6abc125218305f092676c89b94547aa9f6711bcc9682671a74e0c5ed39714e8651b98ad9dfc6'

data/.github/workflows/test.yml

@@ -1,6 +1,13 @@
 name: Test
 
-on: [push]
+on:
+  # Run on all master branch commits
+  push:
+    branches:
+      - master
+
+  # Run against all PRs (from the main repo, or forks)
+  pull_request:
 
 jobs:
   test:

data/CHANGELOG.md

@@ -1,6 +1,23 @@
 ## [Unreleased]
 *no unreleased changes*
 
+## 10.1.2 / 2021-09-28
+### Fixed
+* Bump to `nokogiri` to address CVE-2021-41098
+* Bump `seven_zip_ruby` requirement for Ruby 2.7 support
+
+## 10.1.1 / 2021-03-15
+### Fixed
+* XML: ensure invalid control character *references* are also escaped (#64)
+
+## 10.1.0 / 2021-03-08
+### Added
+* Allow optional `last_data_column` in NdrImport::Table mappings (#61)
+
+## 10.0.0 / 2021-02-22
+### Changed
+* By default, escape any control characters found in XML (#60)
+
 ## 9.1.0 / 2021-02-01
 ### Added
 * `CSVLibrary` is now deprecated.

data/code_safety.yml

@@ -10,8 +10,8 @@ file safety:
     safe_revision: b64ff21375dcde2b8fefe622ee9861f0fea21487
   ".github/workflows/test.yml":
     comments: 
-    reviewed_by: ollietulloch
-    safe_revision: b64ff21375dcde2b8fefe622ee9861f0fea21487
+    reviewed_by: ollietulllch
+    safe_revision: c3dd24e8abefe61f04fa9d3bb71ec9d0ac109bbe
   ".gitignore":
     comments: whole file re-reviewed
     reviewed_by: josh.pencheon
@@ -27,7 +27,7 @@ file safety:
   CHANGELOG.md:
     comments: 
     reviewed_by: ollietulloch
-    safe_revision: d88ded7c260da37200610e4f0b204a4ea2e481f9
+    safe_revision: e938689d115b75074313541ec9d6b2bc60475add
   CODE_OF_CONDUCT.md:
     comments: 
     reviewed_by: timgentry
@@ -67,7 +67,7 @@ file safety:
   docs/Gemfile.lock:
     comments: 
     reviewed_by: ollietulloch
-    safe_revision: ea0149c7739676463a252ffd9fbe4af238762b2b
+    safe_revision: 75877ddc14e99a10e5e751b7034964a9c7a9d1ef
   docs/_config.yml:
     comments: 
     reviewed_by: josh.pencheon
@@ -238,8 +238,8 @@ file safety:
     safe_revision: 45da71ebd3acbc0fe53755bcd75483ba17cb6924
   lib/ndr_import/helpers/file/xml.rb:
     comments: 
-    reviewed_by: ollietulloch
-    safe_revision: 4d337bd233f7e60cf9d363c92400f21269a28da7
+    reviewed_by: josh.pencheon
+    safe_revision: 9a6cc769abce5f9bfa5b4f8bd5cda52dfe18b12b
   lib/ndr_import/helpers/file/xml_streaming.rb:
     comments: uses SafePath and Shellwords when accessing filesystem, or making system
       calls
@@ -279,8 +279,8 @@ file safety:
     safe_revision: bb44ade56a2151706eede2c31142440ccf49e6f6
   lib/ndr_import/non_tabular/table.rb:
     comments: 
-    reviewed_by: josh.pencheon
-    safe_revision: 71979e0a602ca5a0ce415c194f10add9959f0116
+    reviewed_by: ollietulloch
+    safe_revision: 66cff59af2f078152f7459c436d51b57cb93f28e
   lib/ndr_import/non_tabular_file_helper.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -295,12 +295,12 @@ file safety:
     safe_revision: 3c7f827d17aacbf7b811eea67e27553f3b039070
   lib/ndr_import/table.rb:
     comments: uses File.basename
-    reviewed_by: josh.pencheon
-    safe_revision: a69d4a57ddcf13cdc13c27bd2eb91a395fa7ea36
+    reviewed_by: ollietulloch
+    safe_revision: 66cff59af2f078152f7459c436d51b57cb93f28e
   lib/ndr_import/universal_importer_helper.rb:
     comments: 
-    reviewed_by: ollietulloch
-    safe_revision: ee2e74e4ceda4ff48cbda6872a6bdf0874212c21
+    reviewed_by: josh.pencheon
+    safe_revision: 85869d99ae93252b7f3ef2d0a4db817c88d35c9e
   lib/ndr_import/unmapped_data_error.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -308,15 +308,19 @@ file safety:
   lib/ndr_import/version.rb:
     comments: another check?
     reviewed_by: ollietulloch
-    safe_revision: d88ded7c260da37200610e4f0b204a4ea2e481f9
+    safe_revision: da0aed5e8a8659c5f70254f6d76264f8e780d835
+  lib/ndr_import/xml/control_char_escaper.rb:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 9a6cc769abce5f9bfa5b4f8bd5cda52dfe18b12b
   lib/ndr_import/xml/table.rb:
     comments: 
     reviewed_by: josh.pencheon
     safe_revision: 4ab72f84201c2d5f0147b7dfd041f488f6ff0422
   ndr_import.gemspec:
     comments: 
-    reviewed_by: josh.pencheon
-    safe_revision: 95e6ee9997d06471fe6f2f169c3c701471086371
+    reviewed_by: ollietulloch
+    safe_revision: 3b437f0ca271fa962121edecd4559017c2446a3a
   test/csv_library_test.rb:
     comments: 
     reviewed_by: ollietulloch
@@ -395,8 +399,8 @@ file safety:
     safe_revision: ae75fb49baf028ac8ce08e4bedcd3625ff3ff0cd
   test/helpers/file/xml_test.rb:
     comments: 
-    reviewed_by: timgentry
-    safe_revision: 137170d443ea6bcc0afb18f62202c285ae6501eb
+    reviewed_by: josh.pencheon
+    safe_revision: 9a6cc769abce5f9bfa5b4f8bd5cda52dfe18b12b
   test/helpers/file/zip_test.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -507,8 +511,8 @@ file safety:
     safe_revision: 71979e0a602ca5a0ce415c194f10add9959f0116
   test/resources/malformed.xml:
     comments: 
-    reviewed_by: timgentry
-    safe_revision: 137170d443ea6bcc0afb18f62202c285ae6501eb
+    reviewed_by: joshpencheon
+    safe_revision: 3947f13e0cbd17f449eba292ad343eeb82116fe9
   test/resources/malformed_pipe.csv:
     comments: 
     reviewed_by: josh.pencheon
@@ -621,6 +625,22 @@ file safety:
     comments: 
     reviewed_by: timgentry
     safe_revision: f755c6960182f7dd460c18866cccfdf09178e860
+  test/resources/with-control-char-references-in-cdata.xml:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 9a6cc769abce5f9bfa5b4f8bd5cda52dfe18b12b
+  test/resources/with-control-char-references.xml:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 9a6cc769abce5f9bfa5b4f8bd5cda52dfe18b12b
+  test/resources/with-control-chars.xml:
+    comments: 
+    reviewed_by: joshpencheon
+    safe_revision: 3947f13e0cbd17f449eba292ad343eeb82116fe9
+  test/resources/with-non-control-char-references.xml:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 9a6cc769abce5f9bfa5b4f8bd5cda52dfe18b12b
   test/resources/xlsx_file_xls_extension.xls:
     comments: 
     reviewed_by: timgentry
@@ -632,16 +652,20 @@ file safety:
   test/table_test.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: a69d4a57ddcf13cdc13c27bd2eb91a395fa7ea36
+    safe_revision: 3cf7473181f7f835b3dfe7822f6833d751805eaf
   test/test_helper.rb:
     comments: 
     reviewed_by: josh.pencheon
     safe_revision: 93ccee82fc2165d1ca2d9b03d146ae03e769ea96
   test/universal_importer_helper_test.rb:
     comments: 
-    reviewed_by: ollietulloch
-    safe_revision: 830de0f8cb139c5f61525652b424423935cfc7ac
-  test/xml/table_test.rb:
+    reviewed_by: josh.pencheon
+    safe_revision: 85869d99ae93252b7f3ef2d0a4db817c88d35c9e
+  test/xml/control_char_escaper_test.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 4ab72f84201c2d5f0147b7dfd041f488f6ff0422
+    safe_revision: 9a6cc769abce5f9bfa5b4f8bd5cda52dfe18b12b
+  test/xml/table_test.rb:
+    comments: 
+    reviewed_by: ollietulloch
+    safe_revision: 66cff59af2f078152f7459c436d51b57cb93f28e

data/lib/ndr_import/helpers/file/xml.rb

@@ -1,3 +1,4 @@
+require 'ndr_import/xml/control_char_escaper'
 require 'ndr_support/safe_file'
 require 'ndr_support/utf8_encoding'
 
@@ -10,16 +11,21 @@ module NdrImport
 
         private
 
-        def read_xml_file(path)
-          file_data = SafeFile.new(path).read
+        # By default, escapes any control characters found in the XML
+        # - their use is forbidden in XML 1.0, and highly discouraged
+        # in XML 1.1; any found are most likely to be erroneous.
+        def read_xml_file(path, preserve_control_chars: false)
+          file_data = ensure_utf8!(SafeFile.read(path))
 
           require 'nokogiri'
 
-          doc = Nokogiri::XML((ensure_utf8! file_data)) do |config|
-            config.huge
+          doc = nil
+
+          escaping_control_chars_if_necessary(preserve_control_chars, file_data) do
+            doc = Nokogiri::XML(file_data, &:huge)
+            doc.encoding = 'UTF-8'
+            emulate_strict_mode_fatal_check!(doc)
           end
-          doc.encoding = 'UTF-8'
-          emulate_strict_mode_fatal_check!(doc)
 
           doc
         end
@@ -40,11 +46,27 @@ module NdrImport
           end
 
           return unless fatal_errors.any?
+
           raise Nokogiri::XML::SyntaxError, <<~MSG
             The file had #{fatal_errors.length} fatal error(s)!"
             #{fatal_errors.join("\n")}
           MSG
         end
+
+        def escaping_control_chars_if_necessary(preserve_control_chars, file_data)
+          return yield if preserve_control_chars
+
+          tried_escaping = false
+          begin
+            yield
+          rescue Nokogiri::XML::SyntaxError => e
+            raise e if tried_escaping
+
+            NdrImport::Xml::ControlCharEscaper.new(file_data).escape!
+            tried_escaping = true
+            retry
+          end
+        end
       end
     end
   end

data/lib/ndr_import/non_tabular/table.rb

@@ -16,8 +16,8 @@ module NdrImport
 
       include UTF8Encoding
 
-      TABULAR_ONLY_OPTIONS = %w[delimiter liberal_parsing tablename_pattern
-                                header_lines footer_lines xml_record_xpath].freeze
+      TABULAR_ONLY_OPTIONS = %w[delimiter last_data_column liberal_parsing tablename_pattern
+                                header_lines footer_lines xml_record_xpath slurp].freeze
 
       NON_TABULAR_OPTIONS = %w[capture_end_line capture_start_line start_line_pattern
                                end_line_pattern remove_lines start_in_a_record

data/lib/ndr_import/table.rb

@@ -10,8 +10,9 @@ module NdrImport
     include NdrImport::Mapper
 
     def self.all_valid_options
-      %w[canonical_name delimiter liberal_parsing filename_pattern file_password tablename_pattern
-         header_lines footer_lines format klass columns xml_record_xpath row_identifier]
+      %w[canonical_name delimiter liberal_parsing filename_pattern file_password last_data_column
+         tablename_pattern header_lines footer_lines format klass columns xml_record_xpath slurp
+         row_identifier]
     end
 
     def all_valid_options
@@ -50,8 +51,9 @@ module NdrImport
       @header_best_guess = nil
       @notifier.try(:started)
 
+      last_col = last_column_to_transform
       skip_footer_lines(lines, footer_lines).each do |line|
-        process_line(line, &block)
+        line.is_a?(Array) ? process_line(line[0..last_col], &block) : process_line(line, &block)
       end
 
       @notifier.try(:finished)
@@ -226,5 +228,26 @@ module NdrImport
     def column_names(column_mappings)
       column_mappings.map { |c| (c['column'] || c['standard_mapping']).downcase }
     end
+
+    # If specified in the mapping, stop transforming data at a given index (column)
+    def last_column_to_transform
+      return -1 if last_data_column.nil?
+      return last_data_column - 1 if last_data_column.is_a?(Integer)
+
+      error =  "Unknown 'last_data_column' format: #{last_data_column} " \
+               "(#{last_data_column.class})"
+      raise error unless last_data_column.is_a?(String) && last_data_column =~ /\A[A-Z]+\z/i
+
+      # If it's an excel column label (eg 'K', 'AF', 'DDE'), convert it to an index
+      index_from_column_label
+    end
+
+    def index_from_column_label
+      alphabet_index_hash = ('A'..'Z').map.with_index.to_h
+      index = last_data_column.upcase.chars.inject(0) do |char_index, char|
+        (char_index * 26) + (alphabet_index_hash[char] + 1)
+      end
+      index - 1
+    end
   end # class Table
 end

data/lib/ndr_import/universal_importer_helper.rb

@@ -48,9 +48,7 @@ module NdrImport
     def extract(source_file, &block)
       return enum_for(:extract, source_file) unless block
 
-      files = NdrImport::File::Registry.files(source_file,
-                                              'unzip_path' => unzip_path)
-      files.each do |filename|
+      NdrImport::File::Registry.files(source_file, 'unzip_path' => unzip_path).each do |filename|
         # now at the individual file level, can we find the table mapping?
         table_mapping = get_table_mapping(filename, nil)
 

data/lib/ndr_import/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 # This stores the current version of the NdrImport gem
 module NdrImport
-  VERSION = '9.1.0'.freeze
+  VERSION = '10.1.2'
 end

data/lib/ndr_import/xml/control_char_escaper.rb

@@ -0,0 +1,51 @@
+require 'ndr_support/utf8_encoding'
+
+module NdrImport
+  module Xml
+    # A class to remove control characters, and XML entities representing them
+    class ControlCharEscaper
+      include UTF8Encoding
+
+      # Matches XML character reference entities
+      CHARACTER_REFERENCES = /&#(?:(?<decimal>\d+)|x(?<hex>\h+));/.freeze
+
+      attr_reader :data
+
+      def initialize(data)
+        @data = data
+      end
+
+      def escape!
+        unescape_control_char_references!(data)
+        escape_control_chars!(data)
+      end
+
+      private
+
+      def unescape_control_char_references!(data)
+        data.gsub!(CHARACTER_REFERENCES) do |reference|
+          char = try_to_extract_char_from(Regexp.last_match)
+
+          if char&.match?(CONTROL_CHARACTERS)
+            escape_control_chars!(char)
+          else
+            reference
+          end
+        end
+      end
+
+      def try_to_extract_char_from(match)
+        if match.nil?
+          nil
+        elsif match[:decimal]
+          match[:decimal].to_i(10).chr
+        elsif match[:hex]
+          match[:hex].to_i(16).chr
+        end
+      rescue RangeError
+        # Return everything if the match was against junk:
+        match.to_s
+      end
+    end
+  end
+end

data/ndr_import.gemspec

@@ -35,7 +35,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'ooxml_decrypt'
   spec.add_dependency 'pdf-reader', '~> 2.1'
   spec.add_dependency 'roo-xls'
-  spec.add_dependency 'seven_zip_ruby', '~> 1.2'
+  spec.add_dependency 'seven_zip_ruby', '~> 1.3'
   spec.add_dependency 'spreadsheet', '1.2.6'
 
   spec.required_ruby_version = '>= 2.5'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ndr_import
 version: !ruby/object:Gem::Version
-  version: 9.1.0
+  version: 10.1.2
 platform: ruby
 authors:
 - NCRS Development Team
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-02-01 00:00:00.000000000 Z
+date: 2021-09-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -188,14 +188,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '1.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '1.3'
 - !ruby/object:Gem::Dependency
   name: spreadsheet
   requirement: !ruby/object:Gem::Requirement
@@ -427,6 +427,7 @@ files:
 - lib/ndr_import/universal_importer_helper.rb
 - lib/ndr_import/unmapped_data_error.rb
 - lib/ndr_import/version.rb
+- lib/ndr_import/xml/control_char_escaper.rb
 - lib/ndr_import/xml/table.rb
 - ndr_import.gemspec
 homepage: https://github.com/PublicHealthEngland/ndr_import