ndr_import 9.1.0 → 10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aaf3c826acb51f4d579fb956e9606b5b86e120f3bd561db762d72081f27a1098
-  data.tar.gz: 2b11ddf7dc9b748b4a1ac9cc91b47c47b99677e4bde990c44d1ec20e76efbfc0
+  metadata.gz: b98a9642fed703edb02ce4bc18d5c15869f1dd10d0e072866a84a0b6b9529141
+  data.tar.gz: 8c4aa215b0e87ca31676a96c703789bfb93d22bf3fa32b44ee7169a4ccfa4607
 SHA512:
-  metadata.gz: 3faaf744255693f04425b6e5ed1ec7198e912d2854b3f482a39d9bc53fe3859ed3bb6ef7855cdfd03959042a530f3a2f528239548676c7159c9029d69b6c160d
-  data.tar.gz: 1aa9d1895d7f874499823c4b87f1874e025b376f239030f3baf40bab5f5ed3ed09f80721205e271446893a9070652df3f26fceb48b59a9eb86ac121fdbfd92c6
+  metadata.gz: da1b4ae10264ac9a5ff7d09832c979f81608c6e428052bbe1dc403f5d5cc0d9c1f44348b59d02be340cc48a277cf2bfe84f5fd80560f7ebc3b8379b529f65a4f
+  data.tar.gz: b969c50b4aec9687571c53f1b49b6798ff65ef2c1d4edeb36ab535e8af59a2387fe8d4a5941ef9c1257686e59c990b30de69a1423779dd8bf77ccfc94bd04786

data/CHANGELOG.md

@@ -1,6 +1,10 @@
 ## [Unreleased]
 *no unreleased changes*
 
+## 10.0 / 2021-02-22
+### Changed
+* By default, escape any control characters found in XML (#60)
+
 ## 9.1.0 / 2021-02-01
 ### Added
 * `CSVLibrary` is now deprecated.

data/code_safety.yml

@@ -26,8 +26,8 @@ file safety:
     safe_revision: b09e268ff9c8349b914aa1b7ba888e1d39f97e4a
   CHANGELOG.md:
     comments: 
-    reviewed_by: ollietulloch
-    safe_revision: d88ded7c260da37200610e4f0b204a4ea2e481f9
+    reviewed_by: joshpencheon
+    safe_revision: 8ba7aae5e4839bed03ddc6837dd657ef7720e8ce
   CODE_OF_CONDUCT.md:
     comments: 
     reviewed_by: timgentry
@@ -238,8 +238,8 @@ file safety:
     safe_revision: 45da71ebd3acbc0fe53755bcd75483ba17cb6924
   lib/ndr_import/helpers/file/xml.rb:
     comments: 
-    reviewed_by: ollietulloch
-    safe_revision: 4d337bd233f7e60cf9d363c92400f21269a28da7
+    reviewed_by: joshpencheon
+    safe_revision: 3947f13e0cbd17f449eba292ad343eeb82116fe9
   lib/ndr_import/helpers/file/xml_streaming.rb:
     comments: uses SafePath and Shellwords when accessing filesystem, or making system
       calls
@@ -307,8 +307,8 @@ file safety:
     safe_revision: 5cd2cd0b3a1e254d30d4acc28c6731825a1f84f5
   lib/ndr_import/version.rb:
     comments: another check?
-    reviewed_by: ollietulloch
-    safe_revision: d88ded7c260da37200610e4f0b204a4ea2e481f9
+    reviewed_by: joshpencheon
+    safe_revision: 8ba7aae5e4839bed03ddc6837dd657ef7720e8ce
   lib/ndr_import/xml/table.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -395,8 +395,8 @@ file safety:
     safe_revision: ae75fb49baf028ac8ce08e4bedcd3625ff3ff0cd
   test/helpers/file/xml_test.rb:
     comments: 
-    reviewed_by: timgentry
-    safe_revision: 137170d443ea6bcc0afb18f62202c285ae6501eb
+    reviewed_by: joshpencheon
+    safe_revision: 3947f13e0cbd17f449eba292ad343eeb82116fe9
   test/helpers/file/zip_test.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -507,8 +507,8 @@ file safety:
     safe_revision: 71979e0a602ca5a0ce415c194f10add9959f0116
   test/resources/malformed.xml:
     comments: 
-    reviewed_by: timgentry
-    safe_revision: 137170d443ea6bcc0afb18f62202c285ae6501eb
+    reviewed_by: joshpencheon
+    safe_revision: 3947f13e0cbd17f449eba292ad343eeb82116fe9
   test/resources/malformed_pipe.csv:
     comments: 
     reviewed_by: josh.pencheon
@@ -621,6 +621,10 @@ file safety:
     comments: 
     reviewed_by: timgentry
     safe_revision: f755c6960182f7dd460c18866cccfdf09178e860
+  test/resources/with-control-chars.xml:
+    comments: 
+    reviewed_by: joshpencheon
+    safe_revision: 3947f13e0cbd17f449eba292ad343eeb82116fe9
   test/resources/xlsx_file_xls_extension.xls:
     comments: 
     reviewed_by: timgentry

data/lib/ndr_import/helpers/file/xml.rb

@@ -10,14 +10,16 @@ module NdrImport
 
         private
 
-        def read_xml_file(path)
-          file_data = SafeFile.new(path).read
+        # By default, escapes any control characters found in the XML
+        # - their use is forbidden in XML 1.0, and highly discouraged
+        # in XML 1.1; any found are most likely to be erroneous.
+        def read_xml_file(path, preserve_control_chars: false)
+          file_data = ensure_utf8!(SafeFile.read(path))
+          escape_xml_control_chars!(file_data) unless preserve_control_chars
 
           require 'nokogiri'
 
-          doc = Nokogiri::XML((ensure_utf8! file_data)) do |config|
-            config.huge
-          end
+          doc = Nokogiri::XML(file_data, &:huge)
           doc.encoding = 'UTF-8'
           emulate_strict_mode_fatal_check!(doc)
 
@@ -40,11 +42,19 @@ module NdrImport
           end
 
           return unless fatal_errors.any?
+
           raise Nokogiri::XML::SyntaxError, <<~MSG
             The file had #{fatal_errors.length} fatal error(s)!"
             #{fatal_errors.join("\n")}
           MSG
         end
+
+        # In place, escape out any control chars that would cause
+        # libxml to crash. Very few are allowable in XML 1.0, and
+        # remain heavily discouraged in XML 1.1.
+        def escape_xml_control_chars!(data)
+          escape_control_chars!(data)
+        end
       end
     end
   end

data/lib/ndr_import/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 # This stores the current version of the NdrImport gem
 module NdrImport
-  VERSION = '9.1.0'.freeze
+  VERSION = '10.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ndr_import
 version: !ruby/object:Gem::Version
-  version: 9.1.0
+  version: '10.0'
 platform: ruby
 authors:
 - NCRS Development Team
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-02-01 00:00:00.000000000 Z
+date: 2021-02-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel