ndr_import 8.5.1 → 9.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d8e9942cb20c52476a27af5b45a5bdb532171e7eb4cbf4b1d6cf40164623702a
-  data.tar.gz: 4e67817d307c143c1dbd3f3dbf17798e8864f8f246149b91d73099763d28488f
+  metadata.gz: 7dc0801c147101346c5a2e0af96ef9a658bc068088bb54f4f65970bb012e6cf5
+  data.tar.gz: 22823d5d0415a95eebac8151475d6eba7fdbd1744003ee0d88b1dedf86b837b2
 SHA512:
-  metadata.gz: 9b36561a8c48c9626dff51913c96426cfa10904f31ba8867056f77e5752583b83080b55b23b5f0b30e74c095e91e08d6563ca1f7bf7f7ae265d03501cf384cbc
-  data.tar.gz: a23881b758d08e34d0b82faa177a193402e50c0ea3373c575189f175af1bcd1e030ceeeacd3eb1a3e21b96b82c3aa8b5a1ee41d5f69aae53977609420b5cf139
+  metadata.gz: 3c29ef1ab701b94301aca343d8bf8497fda2633d120b3945dc140cb30d1b40b30bfcfb2334fe2bdd6f02d6128f1743ac42601f8129687502503e69c14d77df98
+  data.tar.gz: 49e739dab53bf1276655e01a91dd9253221e041c0f3241b85f362a2d429597fbb4589d14b89b1d83fed7995b043288e72f3bf1bd27d2678e6f7d2c4b1a5c3d8f

data/.gitignore

@@ -5,7 +5,7 @@
 /gemfiles/Gemfile.*.lock
 /_yardoc/
 /coverage/
-/doc/
+/docs/_site/
 /pkg/
 /spec/reports/
 /tmp/

data/CHANGELOG.md

@@ -1,6 +1,30 @@
 ## [Unreleased]
 *no unreleased changes*
 
+## 9.0.2 / 2020-08-14
+### Changed
+* Configure Nokogiri with HUGE for large xml files
+
+## 9.0.1 / 2020-03-26
+### Fixed
+* bumps to `nokogiri` / `spreadsheet` / `rubyzip` dependencies
+
+## 9.0.0 / 2019-07-31
+### Changed
+* `File::Xml` will now stream XML files by default. Use `slurp: true` for the old behaviour. (#43)
+
+### Added
+* Add `XmlStreaming` helper, for more performant handling of large XML documents with Nokogiri. (#43)
+
+## 8.6.0 / 2019-06-07
+### Added
+* Allow conditional preservation of blank lines when joining lines in non-tabular data (#41)
+
+## 8.5.2 / 2019-05-17
+### Fixed
+* Fixed issue with `file_password` option key as a String or Symbol
+* Tempfiles now take their encoding from the incoming string/stream
+
 ## 8.5.1 / 2019-05-15
 ### Added
 * Add data loader tools (#39)

data/README.md

@@ -1,4 +1,4 @@
-# NdrImport [![Build Status](https://travis-ci.org/PublicHealthEngland/ndr_import.svg?branch=master)](https://travis-ci.org/PublicHealthEngland/ndr_import) [![Gem Version](https://badge.fury.io/rb/ndr_import.svg)](https://badge.fury.io/rb/ndr_import)
+# NdrImport [![Build Status](https://travis-ci.org/PublicHealthEngland/ndr_import.svg?branch=master)](https://travis-ci.org/PublicHealthEngland/ndr_import) [![Gem Version](https://badge.fury.io/rb/ndr_import.svg)](https://badge.fury.io/rb/ndr_import) [![Documentation](https://img.shields.io/badge/ndr_import-docs-blue.svg)](https://www.rubydoc.info/gems/ndr_import)
 
 This is the Public Health England (PHE) National Disease Registers (NDR) Import ETL ruby gem, providing:
 

data/code_safety.yml

@@ -3,7 +3,7 @@ file safety:
   ".gitignore":
     comments: whole file re-reviewed
     reviewed_by: josh.pencheon
-    safe_revision: 3ef51291c413fd5772d61a8394359146a02ae628
+    safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
   ".hound.yml":
     comments: 
     reviewed_by: timgentry
@@ -18,8 +18,8 @@ file safety:
     safe_revision: d3d9a987befeecb122a448d8d06e66d74da13fb5
   CHANGELOG.md:
     comments: 
-    reviewed_by: josh.pencheon
-    safe_revision: 730152642d4d08454d8aa2f76abe2eda086c077c
+    reviewed_by: ollietulloch
+    safe_revision: aa006cd76123db2101d145a07d201dc6a709ed6e
   CODE_OF_CONDUCT.md:
     comments: 
     reviewed_by: timgentry
@@ -39,7 +39,7 @@ file safety:
   README.md:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: e1d967c10059e8c635452838c3f3dd2b969d9ae4
+    safe_revision: 1bc459db8970dde36e9b240b6dd08cca629664e3
   Rakefile:
     comments: 
     reviewed_by: josh.pencheon
@@ -52,6 +52,70 @@ file safety:
     comments: 
     reviewed_by: josh.pencheon
     safe_revision: e1d967c10059e8c635452838c3f3dd2b969d9ae4
+  docs/Gemfile:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
+  docs/Gemfile.lock:
+    comments: 
+    reviewed_by: ollietulloch
+    safe_revision: 6f274715bb341c3070190f04f67af9500b510580
+  docs/_config.yml:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
+  docs/_includes/footer.html:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
+  docs/_includes/header.html:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
+  docs/capturing-data.md:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
+  docs/date-formats.md:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: fa21d6d967bf132800b456b585795beec80b08a3
+  docs/getting-started.md:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: fa21d6d967bf132800b456b585795beec80b08a3
+  docs/identifying-and-splitting-records.md:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
+  docs/inbuilt-cleaning-methods.md:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 694b57ce14e0709fc4d31a1357f8416e98f5de91
+  docs/index.md:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
+  docs/local-code-transformation-in-yaml-mappings.md:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
+  docs/non-tabular-mappings.md:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
+  docs/priority-field-mapping.md:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
+  docs/standard-yaml-mappings.md:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
+  docs/yaml-mapping-user-guide.md:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
   exe/pdf_acro_form_to_yaml:
     comments: 
     reviewed_by: josh.pencheon
@@ -94,8 +158,8 @@ file safety:
     safe_revision: 5cd2cd0b3a1e254d30d4acc28c6731825a1f84f5
   lib/ndr_import/file/base.rb:
     comments: 
-    reviewed_by: timgentry
-    safe_revision: a8451809db86e0e023b0d93826114a33ff896c4f
+    reviewed_by: josh.pencheon
+    safe_revision: c3183e522bce50008df576ceb47fe4761ab8f966
   lib/ndr_import/file/delimited.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -103,11 +167,11 @@ file safety:
   lib/ndr_import/file/docx.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: a69d4a57ddcf13cdc13c27bd2eb91a395fa7ea36
+    safe_revision: 897f8b648d633368cf2001d17ab89c06a12d445b
   lib/ndr_import/file/excel.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: a69d4a57ddcf13cdc13c27bd2eb91a395fa7ea36
+    safe_revision: c3183e522bce50008df576ceb47fe4761ab8f966
   lib/ndr_import/file/office_file_helper.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -139,7 +203,7 @@ file safety:
   lib/ndr_import/file/xml.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 4ab72f84201c2d5f0147b7dfd041f488f6ff0422
+    safe_revision: ae75fb49baf028ac8ce08e4bedcd3625ff3ff0cd
   lib/ndr_import/file/zip.rb:
     comments: 
     reviewed_by: timgentry
@@ -166,8 +230,13 @@ file safety:
     safe_revision: 45da71ebd3acbc0fe53755bcd75483ba17cb6924
   lib/ndr_import/helpers/file/xml.rb:
     comments: 
+    reviewed_by: ollietulloch
+    safe_revision: 4d337bd233f7e60cf9d363c92400f21269a28da7
+  lib/ndr_import/helpers/file/xml_streaming.rb:
+    comments: uses SafePath and Shellwords when accessing filesystem, or making system
+      calls
     reviewed_by: josh.pencheon
-    safe_revision: d2245268ec6a0e4f60c521d171a820f299632c4f
+    safe_revision: 55e502bb4445cb8b985e530e8eb26d92b574ded9
   lib/ndr_import/helpers/file/zip.rb:
     comments: 
     reviewed_by: timgentry
@@ -175,7 +244,7 @@ file safety:
   lib/ndr_import/mapper.rb:
     comments: Writes to a Tempfile, but cleans up. Ruby will respect TMP_DIR
     reviewed_by: josh.pencheon
-    safe_revision: eca44583e9989159b45e90021dd1c65228447180
+    safe_revision: 897f8b648d633368cf2001d17ab89c06a12d445b
   lib/ndr_import/mapping_error.rb:
     comments: 
     reviewed_by: timgentry
@@ -187,7 +256,7 @@ file safety:
   lib/ndr_import/non_tabular/column_mapping.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: f216687d0bba7f2940f74a3353a32be3c900c194
+    safe_revision: bb44ade56a2151706eede2c31142440ccf49e6f6
   lib/ndr_import/non_tabular/line.rb:
     comments: 
     reviewed_by: timgentry
@@ -198,8 +267,8 @@ file safety:
     safe_revision: cf382902508a21a95b80ac4582fbbd117164e80e
   lib/ndr_import/non_tabular/record.rb:
     comments: 
-    reviewed_by: timgentry
-    safe_revision: 63b1e8c275bd63377309d9b124176f39f75576a9
+    reviewed_by: josh.pencheon
+    safe_revision: bb44ade56a2151706eede2c31142440ccf49e6f6
   lib/ndr_import/non_tabular/table.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -223,15 +292,15 @@ file safety:
   lib/ndr_import/universal_importer_helper.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: a69d4a57ddcf13cdc13c27bd2eb91a395fa7ea36
+    safe_revision: 55e502bb4445cb8b985e530e8eb26d92b574ded9
   lib/ndr_import/unmapped_data_error.rb:
     comments: 
     reviewed_by: josh.pencheon
     safe_revision: 5cd2cd0b3a1e254d30d4acc28c6731825a1f84f5
   lib/ndr_import/version.rb:
     comments: another check?
-    reviewed_by: josh.pencheon
-    safe_revision: 730152642d4d08454d8aa2f76abe2eda086c077c
+    reviewed_by: ollietulloch
+    safe_revision: aa006cd76123db2101d145a07d201dc6a709ed6e
   lib/ndr_import/xml/table.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -239,7 +308,7 @@ file safety:
   ndr_import.gemspec:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: d3d9a987befeecb122a448d8d06e66d74da13fb5
+    safe_revision: 95e6ee9997d06471fe6f2f169c3c701471086371
   test/file/acro_form_test.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -283,7 +352,7 @@ file safety:
   test/file/xml_test.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 4ab72f84201c2d5f0147b7dfd041f488f6ff0422
+    safe_revision: ae75fb49baf028ac8ce08e4bedcd3625ff3ff0cd
   test/file/zip_test.rb:
     comments: 
     reviewed_by: timgentry
@@ -308,6 +377,10 @@ file safety:
     comments: 
     reviewed_by: timgentry
     safe_revision: 9abdd6ced1d0c90ce8dd88abee4eb6472c7ff0d6
+  test/helpers/file/xml_streaming_test.rb:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: ae75fb49baf028ac8ce08e4bedcd3625ff3ff0cd
   test/helpers/file/xml_test.rb:
     comments: 
     reviewed_by: timgentry
@@ -330,8 +403,8 @@ file safety:
     safe_revision: a69d4a57ddcf13cdc13c27bd2eb91a395fa7ea36
   test/non_tabular_file_helper_test.rb:
     comments: 
-    reviewed_by: timgentry
-    safe_revision: cf382902508a21a95b80ac4582fbbd117164e80e
+    reviewed_by: josh.pencheon
+    safe_revision: bb44ade56a2151706eede2c31142440ccf49e6f6
   test/pdf_form/table_test.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -356,6 +429,10 @@ file safety:
     comments: 
     reviewed_by: timgentry
     safe_revision: dab4b8a3e4b29d85eccd971e79936982d888cffd
+  test/resources/claims_utf16be_but_isnt.xml:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: ae75fb49baf028ac8ce08e4bedcd3625ff3ff0cd
   test/resources/filesystem_paths.yml:
     comments: 
     reviewed_by: timgentry
@@ -543,7 +620,7 @@ file safety:
   test/universal_importer_helper_test.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 3e3a852b58e8b169535e29029e535a10f6b9cd42
+    safe_revision: c3183e522bce50008df576ceb47fe4761ab8f966
   test/xml/table_test.rb:
     comments: 
     reviewed_by: josh.pencheon

data/lib/ndr_import/file/base.rb

@@ -1,3 +1,4 @@
+require 'active_support/core_ext/hash'
 require 'ndr_support/safe_file'
 require 'ndr_import/csv_library'
 require_relative 'registry'
@@ -11,7 +12,7 @@ module NdrImport
       def initialize(filename, format, options = {})
         @filename = filename
         @format = format
-        @options = options
+        @options = options.stringify_keys
 
         validate_filename_is_safe_and_readable!
       end

data/lib/ndr_import/file/docx.rb

@@ -17,7 +17,7 @@ module NdrImport
       def rows(&block)
         return enum_for(:rows) unless block
 
-        send(@options.key?(:file_password) ? :decrypted_path : :unencrypted_path) do |path|
+        send(@options['file_password'] ? :decrypted_path : :unencrypted_path) do |path|
           doc = ::Docx::Document.open(path)
 
           doc.paragraphs.each do |p|
@@ -32,8 +32,9 @@ module NdrImport
 
       # This method returns the path to the temporary, decrypted file
       def decrypted_path
-        Tempfile.create(['decrypted', '.docx']) do |file|
-          file.write(decrypted_file_string(@filename, @options[:file_password]))
+        file_string = decrypted_file_string(@filename, @options['file_password'])
+        Tempfile.create(['decrypted', '.docx'], encoding: file_string.encoding) do |file|
+          file.write(file_string)
           file.close
 
           yield file.path

data/lib/ndr_import/file/excel.rb

@@ -91,8 +91,8 @@ module NdrImport
         when '.xls'
           Roo::Excel.new(SafeFile.safepath_to_string(path))
         when '.xlsx'
-          if @options.key?(:file_password)
-            Roo::Excelx.new(StringIO.new(decrypted_file_string(path, @options[:file_password])))
+          if @options['file_password']
+            Roo::Excelx.new(StringIO.new(decrypted_file_string(path, @options['file_password'])))
           else
             Roo::Excelx.new(SafeFile.safepath_to_string(path))
           end

data/lib/ndr_import/file/xml.rb

@@ -1,5 +1,6 @@
 require 'ndr_support/safe_file'
 require 'ndr_import/helpers/file/xml'
+require 'ndr_import/helpers/file/xml_streaming'
 require_relative 'registry'
 
 module NdrImport
@@ -9,6 +10,7 @@ module NdrImport
     # This class is a xml file handler that returns a single table.
     class Xml < Base
       include NdrImport::Helpers::File::Xml
+      include NdrImport::Helpers::File::XmlStreaming
 
       private
 
@@ -16,9 +18,14 @@ module NdrImport
       def rows(&block)
         return enum_for(:rows) unless block
 
-        doc = read_xml_file(@filename)
+        xpath = @options['xml_record_xpath']
 
-        doc.xpath(@options['xml_record_xpath']).each(&block)
+        if @options['slurp']
+          doc = read_xml_file(@filename)
+          doc.xpath(xpath).each(&block)
+        else
+          each_node(@filename, xpath, &block)
+        end
       rescue StandardError => e
         raise("#{SafeFile.basename(@filename)} [#{e.class}: #{e.message}]")
       end

data/lib/ndr_import/helpers/file/xml.rb

@@ -15,10 +15,13 @@ module NdrImport
 
           require 'nokogiri'
 
-          Nokogiri::XML(ensure_utf8! file_data).tap do |doc|
-            doc.encoding = 'UTF-8'
-            emulate_strict_mode_fatal_check!(doc)
+          doc = Nokogiri::XML((ensure_utf8! file_data)) do |config|
+            config.huge
           end
+          doc.encoding = 'UTF-8'
+          emulate_strict_mode_fatal_check!(doc)
+
+          doc
         end
 
         # Nokogiri can use give a `STRICT` parse option to libxml, but our friendly

data/lib/ndr_import/helpers/file/xml_streaming.rb

@@ -0,0 +1,183 @@
+require 'shellwords'
+
+require 'ndr_support/safe_file'
+require 'ndr_support/utf8_encoding'
+
+module NdrImport
+  module Helpers
+    module File
+      # This mixin adds XML streaming functionality, to support more performant handling
+      # of large files by Nokogiri. Uses the `XML::Reader` API, and maintains a temporary
+      # DOM as the XML is streamed to allow XPath querying from the root node.
+      #
+      # If the system has `iconv` available, will attempt to verify the encoding of the
+      # file being read externally, so it can be streamed in to Ruby. Otherwise, will load
+      # the raw data in to check the encoding, but still stream it through Nokogiri's parser.
+      module XmlStreaming
+        # Base error for all streaming-specific issues.
+        class Error < StandardError; end
+
+        # Raised if nested tags are accounted which the streaming approach cannnot handle.
+        class NestingError < Error
+          def initialize(node)
+            super <<~STR
+              Element '#{node.name}' was found nested inside another of the same type.
+              This is not accessible, and a known limitation of XmlStreaming.
+            STR
+          end
+        end
+
+        # Object to track state as the XML is iterated over, and detect
+        # when an element of interest is entered.
+        class Cursor
+          # wrapper to hold a representation of each element we descent into:
+          StackItem = Struct.new(:name, :attrs, :empty)
+
+          def initialize(xpath)
+            @xpath = xpath
+            @stack = []
+            @match_depth = nil
+          end
+
+          # Has this cursor already passed inside a similar node?
+          def in?(node)
+            @stack.detect { |item| item.name == node.name }
+          end
+
+          def enter(node)
+            @stack.push StackItem.new(node.name, node.attributes, node.empty_element?)
+          end
+
+          def leave(_node)
+            @stack.pop
+            @match_depth = nil if @match_depth && @stack.length < @match_depth
+          end
+
+          # Does the element that the cursor is currently on match what
+          # is being looked for?
+          def matches?
+            # Can't match again if we're inside a match already:
+            return false if @matched_depth
+
+            match = current_stack_match?
+
+            # "empty element" matches are yielded immediately, without
+            # tagging the stack as having matched, because there won't
+            # be an equivalent closing tag to end the match with later.
+            if in_empty_element?
+              @stack.pop
+            elsif match
+              @match_depth = @stack.length
+            end
+
+            match
+          end
+
+          private
+
+          def in_empty_element?
+            @stack.last.empty
+          end
+
+          # Does the current state of the stack mean we've met the xpath
+          # criteria? Must be an exact match, not just matching a parent
+          # element in the DOM.
+          def current_stack_match?
+            parent_stack = @stack[0..-2]
+
+            return false unless dom_stubs[@stack].at_xpath(@xpath)
+
+            parent_stack.empty? || !dom_stubs[parent_stack].at_xpath(@xpath)
+          end
+
+          # A cached collection of DOM fragments, to represent the structure
+          # necessary to use xpath to descend into the main document's DOM.
+          def dom_stubs
+            @dom_stubs ||= Hash.new do |hash, items|
+              hash[items.dup] = Nokogiri::XML::Builder.new do |dom|
+                add_items_to_dom(dom, items.dup)
+              end.doc
+            end
+          end
+
+          # Helper to recursively build XML fragment.
+          def add_items_to_dom(dom, items)
+            item = items.shift
+            dom.send(item.name, item.attrs) do
+              add_items_to_dom(dom, items) if items.any?
+            end
+          end
+        end
+
+        include UTF8Encoding
+
+        # Streams the contents of the given `safe_path`, and yields
+        # each element matching `xpath` as they're found.
+        #
+        # In the case of dodgy encoding, may fall back to slurping the
+        # file, but will still use stream parsing for XML.
+        def each_node(safe_path, xpath, &block)
+          return enum_for(:each_node, safe_path, xpath) unless block
+
+          require 'nokogiri'
+
+          with_encoding_check(safe_path) do |stream, encoding|
+            stream_xml_nodes(stream, xpath, encoding, &block)
+          end
+        end
+
+        private
+
+        # We need to ensure the raw data is UTF8 before we start streaming
+        # it with nokogiri. If we can do an external check, great. Otherwise,
+        # we need to slurp and convert the raw data before presenting it.
+        def with_encoding_check(safe_path)
+          forced_encoding = nil
+
+          stream = ::File.open(SafeFile.safepath_to_string(safe_path))
+
+          unless external_utf8_check?(safe_path)
+            stream = StringIO.new ensure_utf8!(stream.read)
+            forced_encoding = 'UTF8'
+          end
+
+          yield stream, forced_encoding
+        end
+
+        # Use iconv, if available, to check raw data encoding:
+        def external_utf8_check?(safe_path)
+          iconv = system('command -v iconv > /dev/null 2>&1')
+          return false unless iconv
+
+          path = SafeFile.safepath_to_string(safe_path)
+          system("iconv -f UTF-8 #{Shellwords.escape(path)} > /dev/null 2>&1")
+        end
+
+        def stream_xml_nodes(io, node_xpath, encoding = nil)
+          # Track nesting as the cursor moves through the document:
+          cursor = Cursor.new(node_xpath)
+
+          # If markup isn't well-formed, try to work around it:
+          options = Nokogiri::XML::ParseOptions::RECOVER
+          reader  = Nokogiri::XML::Reader(io, nil, encoding, options)
+
+          reader.each do |node|
+            case node.node_type
+            when Nokogiri::XML::Reader::TYPE_ELEMENT # "opening tag"
+              raise NestingError, node if cursor.in?(node)
+
+              cursor.enter(node)
+              next unless cursor.matches?
+
+              # The xpath matched - construct a DOM fragment to yield back:
+              element = Nokogiri::XML(node.outer_xml).at("./#{node.name}")
+              yield element
+            when Nokogiri::XML::Reader::TYPE_END_ELEMENT # "closing tag"
+              cursor.leave(node)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ndr_import/mapper.rb

@@ -277,7 +277,7 @@ module NdrImport::Mapper
   end
 
   def read_docx(stream)
-    Tempfile.create(encoding: 'ascii-8bit') do |tempfile|
+    Tempfile.create(encoding: stream.external_encoding) do |tempfile|
       tempfile.write(stream.read)
 
       docx = ::Docx::Document.open(tempfile.path)

data/lib/ndr_import/non_tabular/column_mapping.rb

@@ -6,7 +6,7 @@ module NdrImport
     # the logic associated with finding matching lines of source data and subsequently
     # capturing arrays of values within them.
     class ColumnMapping
-      attr_accessor :name, :cell_mapping, :lines, :capture, :join
+      attr_accessor :name, :cell_mapping, :lines, :capture, :join, :preserve_blank_lines
 
       def initialize(column_mapping)
         @name         = column_mapping['rawtext_name'] ||
@@ -18,6 +18,7 @@ module NdrImport
 
         @lines = @cell_mapping['lines']
         @join  = @cell_mapping['join']
+        @preserve_blank_lines = @cell_mapping['preserve_blank_lines']
       end
 
       # This method returns the range of matching source data lines. If the range is a

data/lib/ndr_import/non_tabular/record.rb

@@ -74,7 +74,10 @@ module NdrImport
           begin
             matches = get_matches(column_mapping)
             # Join the non-blank lines together and add to the array of cells
-            cells << matches.select { |value| !value.blank? }.join(column_mapping.join || '')
+            lines = matches.select do |value|
+              column_mapping.preserve_blank_lines ? value : value.present?
+            end
+            cells << lines.join(column_mapping.join || '')
           rescue RegexpRange::PatternMatchError
             cells << nil
           end

data/lib/ndr_import/universal_importer_helper.rb

@@ -1,3 +1,5 @@
+require 'shellwords'
+
 require 'ndr_import/file/registry'
 
 module NdrImport
@@ -38,7 +40,8 @@ module NdrImport
           'col_sep'          => table_mapping.try(:delimiter),
           'file_password'    => table_mapping.try(:file_password),
           'liberal_parsing'  => table_mapping.try(:liberal_parsing),
-          'xml_record_xpath' => table_mapping.try(:xml_record_xpath)
+          'xml_record_xpath' => table_mapping.try(:xml_record_xpath),
+          'slurp'            => table_mapping.try(:slurp)
         }
 
         tables = NdrImport::File::Registry.tables(filename, table_mapping.try(:format), options)

data/lib/ndr_import/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 # This stores the current version of the NdrImport gem
 module NdrImport
-  VERSION = '8.5.1'.freeze
+  VERSION = '9.0.2'.freeze
 end

data/ndr_import.gemspec

@@ -15,7 +15,7 @@ Gem::Specification.new do |spec|
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
   spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
-    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(docs|test|spec|features)/}) }
   end
   spec.files         -= %w[.travis.yml] # Not needed in the gem
   spec.bindir        = 'exe'
@@ -26,7 +26,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'activesupport', '>= 5.0', '< 7'
   spec.add_dependency 'ndr_support', '>= 5.3.2', '< 6'
 
-  spec.add_dependency 'rubyzip', '~> 1.2', '>= 1.2.2'
+  spec.add_dependency 'rubyzip', '~> 2.0'
   spec.add_dependency 'roo', '~> 2.0'
 
   spec.add_dependency 'docx', '~> 0.3'
@@ -36,15 +36,15 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'pdf-reader', '~> 2.1'
   spec.add_dependency 'roo-xls'
   spec.add_dependency 'seven_zip_ruby', '~> 1.2'
-  spec.add_dependency 'spreadsheet', '1.0.3'
+  spec.add_dependency 'spreadsheet', '1.2.6'
 
   spec.required_ruby_version = '>= 2.5'
 
   spec.add_development_dependency 'bundler'
-  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rake', '~> 12.3', '>= 12.3.3'
   spec.add_development_dependency 'minitest'
   spec.add_development_dependency 'mocha'
-  spec.add_development_dependency 'ndr_dev_support', '~> 3.1', '>= 3.1.3'
+  spec.add_development_dependency 'ndr_dev_support', '>= 3.1.3'
   spec.add_development_dependency 'guard'
   spec.add_development_dependency 'guard-rubocop'
   spec.add_development_dependency 'guard-test'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ndr_import
 version: !ruby/object:Gem::Version
-  version: 8.5.1
+  version: 9.0.2
 platform: ruby
 authors:
 - NCRS Development Team
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-05-15 00:00:00.000000000 Z
+date: 2020-08-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -70,20 +70,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.2.2
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.2.2
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: roo
   requirement: !ruby/object:Gem::Requirement
@@ -208,14 +202,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.3
+        version: 1.2.6
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.3
+        version: 1.2.6
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -236,14 +230,20 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 12.3.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 12.3.3
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -276,9 +276,6 @@ dependencies:
   name: ndr_dev_support
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.1'
     - - ">="
       - !ruby/object:Gem::Version
         version: 3.1.3
@@ -286,9 +283,6 @@ dependencies:
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.1'
     - - ">="
       - !ruby/object:Gem::Version
         version: 3.1.3
@@ -413,6 +407,7 @@ files:
 - lib/ndr_import/helpers/file/pdf.rb
 - lib/ndr_import/helpers/file/word.rb
 - lib/ndr_import/helpers/file/xml.rb
+- lib/ndr_import/helpers/file/xml_streaming.rb
 - lib/ndr_import/helpers/file/zip.rb
 - lib/ndr_import/mapper.rb
 - lib/ndr_import/mapping_error.rb