ndr_import 10.1 → 10.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 16cdebb2e3a6809255d5d66971a2db6d5c954731f6def3c73fe0f89d8ea0b7e9
-  data.tar.gz: 575ba2c40ae01f99ebd48f75ff42c2cdf40871cbed87bfb2399de45d9f402e73
+  metadata.gz: 2463ac35899a6db81e345b75b0ea10186530f559460dfca1211ba7694f52b760
+  data.tar.gz: 2f2cf39e959beeb3cfe6bcad033eb2c0695486dce3b640fe3de8b0c2b9b88a2f
 SHA512:
-  metadata.gz: 37e9bcfe3b8a5cab98bd68fc846eafd386595e0b6b79b9b4bf10ba8bc44a6d24ff998f5ab249964009bb0eebebfe0700bd6a39ddd0dfefc1260b529ff5543e63
-  data.tar.gz: c144108ad4d2f63c918e43ef540f30d49be26897d6c4d82bccda6128aaff5b408c3d93425612bedaef73f865292a3d2a408f0c792e35a9f548e50363804c8d10
+  metadata.gz: c6eca601043ff01ebe910375a36131bdf55ebd3f18664e3db3c07180f007b0f06dc50cc3016634a12764befc1e28d621a09a7b382392f67608aa4d21c83c7f2d
+  data.tar.gz: dc568494bfc4b39b7ee47c7738511274cb85ad9532e380f3c27870682a1a21d2acda4d7d8d855597abdd0643b8afe0ed6e161b74f201fa53640b5c9641f895c5

data/CHANGELOG.md

@@ -1,10 +1,15 @@
 ## [Unreleased]
-* no relevant changes
+*no unreleased changes*
 
-## 10.1 / 2021-03-08
+## 10.1.1 / 2021-03-15
+### Fixed
+* XML: ensure invalid control character *references* are also escaped (#64)
+
+## 10.1.0 / 2021-03-08
+### Added
 * Allow optional `last_data_column` in NdrImport::Table mappings (#61)
 
-## 10.0 / 2021-02-22
+## 10.0.0 / 2021-02-22
 ### Changed
 * By default, escape any control characters found in XML (#60)
 

data/code_safety.yml

@@ -26,8 +26,8 @@ file safety:
     safe_revision: b09e268ff9c8349b914aa1b7ba888e1d39f97e4a
   CHANGELOG.md:
     comments: 
-    reviewed_by: ollietulloch
-    safe_revision: 2d093cc57a699b527a7d0159e77b91f4409a6e0b
+    reviewed_by: josh.pencheon
+    safe_revision: 47fa3633ec2e48f1ee9fb12aad03e817e73c54bf
   CODE_OF_CONDUCT.md:
     comments: 
     reviewed_by: timgentry
@@ -238,8 +238,8 @@ file safety:
     safe_revision: 45da71ebd3acbc0fe53755bcd75483ba17cb6924
   lib/ndr_import/helpers/file/xml.rb:
     comments: 
-    reviewed_by: joshpencheon
-    safe_revision: 3947f13e0cbd17f449eba292ad343eeb82116fe9
+    reviewed_by: josh.pencheon
+    safe_revision: 9a6cc769abce5f9bfa5b4f8bd5cda52dfe18b12b
   lib/ndr_import/helpers/file/xml_streaming.rb:
     comments: uses SafePath and Shellwords when accessing filesystem, or making system
       calls
@@ -279,7 +279,7 @@ file safety:
     safe_revision: bb44ade56a2151706eede2c31142440ccf49e6f6
   lib/ndr_import/non_tabular/table.rb:
     comments: 
-    reviewed_by: ollietulloch
+    reviewed_by: josh.pencheon
     safe_revision: f9df064adcfd38f09d83ad8c5496c84188faed98
   lib/ndr_import/non_tabular_file_helper.rb:
     comments: 
@@ -295,11 +295,11 @@ file safety:
     safe_revision: 3c7f827d17aacbf7b811eea67e27553f3b039070
   lib/ndr_import/table.rb:
     comments: uses File.basename
-    reviewed_by: ollietulloch
+    reviewed_by: josh.pencheon
     safe_revision: 3cf7473181f7f835b3dfe7822f6833d751805eaf
   lib/ndr_import/universal_importer_helper.rb:
     comments: 
-    reviewed_by: ollietulloch
+    reviewed_by: josh.pencheon
     safe_revision: 85869d99ae93252b7f3ef2d0a4db817c88d35c9e
   lib/ndr_import/unmapped_data_error.rb:
     comments: 
@@ -307,8 +307,12 @@ file safety:
     safe_revision: 5cd2cd0b3a1e254d30d4acc28c6731825a1f84f5
   lib/ndr_import/version.rb:
     comments: another check?
-    reviewed_by: ollietulloch
-    safe_revision: 2d093cc57a699b527a7d0159e77b91f4409a6e0b
+    reviewed_by: josh.pencheon
+    safe_revision: 47fa3633ec2e48f1ee9fb12aad03e817e73c54bf
+  lib/ndr_import/xml/control_char_escaper.rb:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 9a6cc769abce5f9bfa5b4f8bd5cda52dfe18b12b
   lib/ndr_import/xml/table.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -395,8 +399,8 @@ file safety:
     safe_revision: ae75fb49baf028ac8ce08e4bedcd3625ff3ff0cd
   test/helpers/file/xml_test.rb:
     comments: 
-    reviewed_by: joshpencheon
-    safe_revision: 3947f13e0cbd17f449eba292ad343eeb82116fe9
+    reviewed_by: josh.pencheon
+    safe_revision: 9a6cc769abce5f9bfa5b4f8bd5cda52dfe18b12b
   test/helpers/file/zip_test.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -621,10 +625,22 @@ file safety:
     comments: 
     reviewed_by: timgentry
     safe_revision: f755c6960182f7dd460c18866cccfdf09178e860
+  test/resources/with-control-char-references-in-cdata.xml:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 9a6cc769abce5f9bfa5b4f8bd5cda52dfe18b12b
+  test/resources/with-control-char-references.xml:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 9a6cc769abce5f9bfa5b4f8bd5cda52dfe18b12b
   test/resources/with-control-chars.xml:
     comments: 
     reviewed_by: joshpencheon
     safe_revision: 3947f13e0cbd17f449eba292ad343eeb82116fe9
+  test/resources/with-non-control-char-references.xml:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 9a6cc769abce5f9bfa5b4f8bd5cda52dfe18b12b
   test/resources/xlsx_file_xls_extension.xls:
     comments: 
     reviewed_by: timgentry
@@ -635,7 +651,7 @@ file safety:
     safe_revision: 3c7f827d17aacbf7b811eea67e27553f3b039070
   test/table_test.rb:
     comments: 
-    reviewed_by: ollietulloch
+    reviewed_by: josh.pencheon
     safe_revision: 3cf7473181f7f835b3dfe7822f6833d751805eaf
   test/test_helper.rb:
     comments: 
@@ -643,8 +659,12 @@ file safety:
     safe_revision: 93ccee82fc2165d1ca2d9b03d146ae03e769ea96
   test/universal_importer_helper_test.rb:
     comments: 
-    reviewed_by: ollietulloch
+    reviewed_by: josh.pencheon
     safe_revision: 85869d99ae93252b7f3ef2d0a4db817c88d35c9e
+  test/xml/control_char_escaper_test.rb:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 9a6cc769abce5f9bfa5b4f8bd5cda52dfe18b12b
   test/xml/table_test.rb:
     comments: 
     reviewed_by: josh.pencheon

data/lib/ndr_import/helpers/file/xml.rb

@@ -1,3 +1,4 @@
+require 'ndr_import/xml/control_char_escaper'
 require 'ndr_support/safe_file'
 require 'ndr_support/utf8_encoding'
 
@@ -15,13 +16,16 @@ module NdrImport
         # in XML 1.1; any found are most likely to be erroneous.
         def read_xml_file(path, preserve_control_chars: false)
           file_data = ensure_utf8!(SafeFile.read(path))
-          escape_xml_control_chars!(file_data) unless preserve_control_chars
 
           require 'nokogiri'
 
-          doc = Nokogiri::XML(file_data, &:huge)
-          doc.encoding = 'UTF-8'
-          emulate_strict_mode_fatal_check!(doc)
+          doc = nil
+
+          escaping_control_chars_if_necessary(preserve_control_chars, file_data) do
+            doc = Nokogiri::XML(file_data, &:huge)
+            doc.encoding = 'UTF-8'
+            emulate_strict_mode_fatal_check!(doc)
+          end
 
           doc
         end
@@ -49,11 +53,19 @@ module NdrImport
           MSG
         end
 
-        # In place, escape out any control chars that would cause
-        # libxml to crash. Very few are allowable in XML 1.0, and
-        # remain heavily discouraged in XML 1.1.
-        def escape_xml_control_chars!(data)
-          escape_control_chars!(data)
+        def escaping_control_chars_if_necessary(preserve_control_chars, file_data)
+          return yield if preserve_control_chars
+
+          tried_escaping = false
+          begin
+            yield
+          rescue Nokogiri::XML::SyntaxError => e
+            raise e if tried_escaping
+
+            NdrImport::Xml::ControlCharEscaper.new(file_data).escape!
+            tried_escaping = true
+            retry
+          end
         end
       end
     end

data/lib/ndr_import/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 # This stores the current version of the NdrImport gem
 module NdrImport
-  VERSION = '10.1'
+  VERSION = '10.1.1'
 end

data/lib/ndr_import/xml/control_char_escaper.rb

@@ -0,0 +1,51 @@
+require 'ndr_support/utf8_encoding'
+
+module NdrImport
+  module Xml
+    # A class to remove control characters, and XML entities representing them
+    class ControlCharEscaper
+      include UTF8Encoding
+
+      # Matches XML character reference entities
+      CHARACTER_REFERENCES = /&#(?:(?<decimal>\d+)|x(?<hex>\h+));/.freeze
+
+      attr_reader :data
+
+      def initialize(data)
+        @data = data
+      end
+
+      def escape!
+        unescape_control_char_references!(data)
+        escape_control_chars!(data)
+      end
+
+      private
+
+      def unescape_control_char_references!(data)
+        data.gsub!(CHARACTER_REFERENCES) do |reference|
+          char = try_to_extract_char_from(Regexp.last_match)
+
+          if char&.match?(CONTROL_CHARACTERS)
+            escape_control_chars!(char)
+          else
+            reference
+          end
+        end
+      end
+
+      def try_to_extract_char_from(match)
+        if match.nil?
+          nil
+        elsif match[:decimal]
+          match[:decimal].to_i(10).chr
+        elsif match[:hex]
+          match[:hex].to_i(16).chr
+        end
+      rescue RangeError
+        # Return everything if the match was against junk:
+        match.to_s
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ndr_import
 version: !ruby/object:Gem::Version
-  version: '10.1'
+  version: 10.1.1
 platform: ruby
 authors:
 - NCRS Development Team
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-03-08 00:00:00.000000000 Z
+date: 2021-03-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -427,6 +427,7 @@ files:
 - lib/ndr_import/universal_importer_helper.rb
 - lib/ndr_import/unmapped_data_error.rb
 - lib/ndr_import/version.rb
+- lib/ndr_import/xml/control_char_escaper.rb
 - lib/ndr_import/xml/table.rb
 - ndr_import.gemspec
 homepage: https://github.com/PublicHealthEngland/ndr_import