ndr_import 8.5.2 → 9.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 66af1dbe18a160875b514f4b7c7bf259b6b43f0565d45beeccb87fd647a5983e
-  data.tar.gz: 0c2a1a0d00ab0371ef9b9d845487cd8e0b67137816b02cefbce7d554c4df6a32
+  metadata.gz: 2237d88e9dee0141fe297fd2eb0d2e76f5434e9bb791fb667505193f30263014
+  data.tar.gz: 52746c16790c3da92e64ac14ed7d77786a41c5457ddc0b8190f52d4414e4c6d7
 SHA512:
-  metadata.gz: 93e749531fc2834866ecf83a16231b233b2800eab53a8ea1d663fa9525fc014a9f9a77b6fd93b43100fe34bbd2e435e3c77e6b4b5ce538927646ceb39c6551c1
-  data.tar.gz: d04b1fb034f37f6129213a1615bd7db360690ed1da9e9028990dc0bd70aeff0ac9211a2028912d89700c67907e03453d6b9cf2868b67194a450b9a14f48ca4e1
+  metadata.gz: 8b8793a45dea035b2abf04a44c20d2ec4fc90bcfef9c5dec4415fab26abdba2ff218b7d2996a165f3447f86046dfd558bfb363e21221f147fec3db9c7c83668c
+  data.tar.gz: eedb044185f1bc7e843e91e634fdc8b88cae9c026344d92e3cdc1046979873d843d476c7a0f7ebfb3c631901b47c9109b6826696124c56b07082ba4a6842af9c

data/.gitignore

@@ -5,7 +5,7 @@
 /gemfiles/Gemfile.*.lock
 /_yardoc/
 /coverage/
-/doc/
+/docs/_site/
 /pkg/
 /spec/reports/
 /tmp/

data/CHANGELOG.md

@@ -1,6 +1,35 @@
 ## [Unreleased]
 *no unreleased changes*
 
+## 9.0.3 / 2021-01-04
+### Fixed
+* Address issue importing multiple files against the same table (#54)
+
+### Changed
+* ensure keyword args are properly splatted for ruby 2.7
+
+### Added
+* Ruby 2.7 to travis matrix
+
+## 9.0.2 / 2020-08-14
+### Changed
+* Configure Nokogiri with HUGE for large xml files
+
+## 9.0.1 / 2020-03-26
+### Fixed
+* bumps to `nokogiri` / `spreadsheet` / `rubyzip` dependencies
+
+## 9.0.0 / 2019-07-31
+### Changed
+* `File::Xml` will now stream XML files by default. Use `slurp: true` for the old behaviour. (#43)
+
+### Added
+* Add `XmlStreaming` helper, for more performant handling of large XML documents with Nokogiri. (#43)
+
+## 8.6.0 / 2019-06-07
+### Added
+* Allow conditional preservation of blank lines when joining lines in non-tabular data (#41)
+
 ## 8.5.2 / 2019-05-17
 ### Fixed
 * Fixed issue with `file_password` option key as a String or Symbol

data/README.md

@@ -1,4 +1,4 @@
-# NdrImport [![Build Status](https://travis-ci.org/PublicHealthEngland/ndr_import.svg?branch=master)](https://travis-ci.org/PublicHealthEngland/ndr_import) [![Gem Version](https://badge.fury.io/rb/ndr_import.svg)](https://badge.fury.io/rb/ndr_import)
+# NdrImport [![Build Status](https://travis-ci.org/PublicHealthEngland/ndr_import.svg?branch=master)](https://travis-ci.org/PublicHealthEngland/ndr_import) [![Gem Version](https://badge.fury.io/rb/ndr_import.svg)](https://badge.fury.io/rb/ndr_import) [![Documentation](https://img.shields.io/badge/ndr_import-docs-blue.svg)](https://www.rubydoc.info/gems/ndr_import)
 
 This is the Public Health England (PHE) National Disease Registers (NDR) Import ETL ruby gem, providing:
 

data/code_safety.yml

@@ -3,7 +3,7 @@ file safety:
   ".gitignore":
     comments: whole file re-reviewed
     reviewed_by: josh.pencheon
-    safe_revision: 3ef51291c413fd5772d61a8394359146a02ae628
+    safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
   ".hound.yml":
     comments: 
     reviewed_by: timgentry
@@ -14,12 +14,12 @@ file safety:
     safe_revision: b09e268ff9c8349b914aa1b7ba888e1d39f97e4a
   ".travis.yml":
     comments: 
-    reviewed_by: josh.pencheon
-    safe_revision: d3d9a987befeecb122a448d8d06e66d74da13fb5
+    reviewed_by: ollietulloch
+    safe_revision: ed8513d290796b9cb9554c4ebe6e87ac21fbbeae
   CHANGELOG.md:
     comments: 
-    reviewed_by: josh.pencheon
-    safe_revision: bd60b56117e65f106d7cb02e0c5772b9e5ec470e
+    reviewed_by: ollietulloch
+    safe_revision: 92d2d9d56f17c143709e25560d7620aaf147008f
   CODE_OF_CONDUCT.md:
     comments: 
     reviewed_by: timgentry
@@ -39,7 +39,7 @@ file safety:
   README.md:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: e1d967c10059e8c635452838c3f3dd2b969d9ae4
+    safe_revision: 1bc459db8970dde36e9b240b6dd08cca629664e3
   Rakefile:
     comments: 
     reviewed_by: josh.pencheon
@@ -52,6 +52,70 @@ file safety:
     comments: 
     reviewed_by: josh.pencheon
     safe_revision: e1d967c10059e8c635452838c3f3dd2b969d9ae4
+  docs/Gemfile:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
+  docs/Gemfile.lock:
+    comments: 
+    reviewed_by: ollietulloch
+    safe_revision: 6f274715bb341c3070190f04f67af9500b510580
+  docs/_config.yml:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
+  docs/_includes/footer.html:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
+  docs/_includes/header.html:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
+  docs/capturing-data.md:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
+  docs/date-formats.md:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: fa21d6d967bf132800b456b585795beec80b08a3
+  docs/getting-started.md:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: fa21d6d967bf132800b456b585795beec80b08a3
+  docs/identifying-and-splitting-records.md:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
+  docs/inbuilt-cleaning-methods.md:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 694b57ce14e0709fc4d31a1357f8416e98f5de91
+  docs/index.md:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
+  docs/local-code-transformation-in-yaml-mappings.md:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
+  docs/non-tabular-mappings.md:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
+  docs/priority-field-mapping.md:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
+  docs/standard-yaml-mappings.md:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
+  docs/yaml-mapping-user-guide.md:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
   exe/pdf_acro_form_to_yaml:
     comments: 
     reviewed_by: josh.pencheon
@@ -82,8 +146,8 @@ file safety:
     safe_revision: 24d6449fd0612552f132dfbf4cada2ae28d0469e
   lib/ndr_import/csv_library.rb:
     comments: 
-    reviewed_by: josh.pencheon
-    safe_revision: be12e57519d3737e8d3901d7b01485c6995708dd
+    reviewed_by: ollietulloch
+    safe_revision: acc72173b81702d55fe3aff78054425609a4d339
   lib/ndr_import/file/acro_form.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -139,7 +203,7 @@ file safety:
   lib/ndr_import/file/xml.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 4ab72f84201c2d5f0147b7dfd041f488f6ff0422
+    safe_revision: ae75fb49baf028ac8ce08e4bedcd3625ff3ff0cd
   lib/ndr_import/file/zip.rb:
     comments: 
     reviewed_by: timgentry
@@ -150,8 +214,8 @@ file safety:
     safe_revision: dfc958d44b6c58355445fa395db08a62213ee709
   lib/ndr_import/helpers/file/delimited.rb:
     comments: 
-    reviewed_by: josh.pencheon
-    safe_revision: 607c0668f1fffd70d181bc1a31c4f56eed5f6189
+    reviewed_by: ollietulloch
+    safe_revision: acc72173b81702d55fe3aff78054425609a4d339
   lib/ndr_import/helpers/file/excel.rb:
     comments: 
     reviewed_by: joshpencheon
@@ -166,8 +230,13 @@ file safety:
     safe_revision: 45da71ebd3acbc0fe53755bcd75483ba17cb6924
   lib/ndr_import/helpers/file/xml.rb:
     comments: 
+    reviewed_by: ollietulloch
+    safe_revision: 4d337bd233f7e60cf9d363c92400f21269a28da7
+  lib/ndr_import/helpers/file/xml_streaming.rb:
+    comments: uses SafePath and Shellwords when accessing filesystem, or making system
+      calls
     reviewed_by: josh.pencheon
-    safe_revision: d2245268ec6a0e4f60c521d171a820f299632c4f
+    safe_revision: 55e502bb4445cb8b985e530e8eb26d92b574ded9
   lib/ndr_import/helpers/file/zip.rb:
     comments: 
     reviewed_by: timgentry
@@ -187,7 +256,7 @@ file safety:
   lib/ndr_import/non_tabular/column_mapping.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: f216687d0bba7f2940f74a3353a32be3c900c194
+    safe_revision: bb44ade56a2151706eede2c31142440ccf49e6f6
   lib/ndr_import/non_tabular/line.rb:
     comments: 
     reviewed_by: timgentry
@@ -198,8 +267,8 @@ file safety:
     safe_revision: cf382902508a21a95b80ac4582fbbd117164e80e
   lib/ndr_import/non_tabular/record.rb:
     comments: 
-    reviewed_by: timgentry
-    safe_revision: 63b1e8c275bd63377309d9b124176f39f75576a9
+    reviewed_by: josh.pencheon
+    safe_revision: bb44ade56a2151706eede2c31142440ccf49e6f6
   lib/ndr_import/non_tabular/table.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -222,16 +291,16 @@ file safety:
     safe_revision: a69d4a57ddcf13cdc13c27bd2eb91a395fa7ea36
   lib/ndr_import/universal_importer_helper.rb:
     comments: 
-    reviewed_by: josh.pencheon
-    safe_revision: a69d4a57ddcf13cdc13c27bd2eb91a395fa7ea36
+    reviewed_by: ollietulloch
+    safe_revision: 830de0f8cb139c5f61525652b424423935cfc7ac
   lib/ndr_import/unmapped_data_error.rb:
     comments: 
     reviewed_by: josh.pencheon
     safe_revision: 5cd2cd0b3a1e254d30d4acc28c6731825a1f84f5
   lib/ndr_import/version.rb:
     comments: another check?
-    reviewed_by: josh.pencheon
-    safe_revision: bd60b56117e65f106d7cb02e0c5772b9e5ec470e
+    reviewed_by: ollietulloch
+    safe_revision: 92d2d9d56f17c143709e25560d7620aaf147008f
   lib/ndr_import/xml/table.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -239,7 +308,7 @@ file safety:
   ndr_import.gemspec:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: d3d9a987befeecb122a448d8d06e66d74da13fb5
+    safe_revision: 95e6ee9997d06471fe6f2f169c3c701471086371
   test/file/acro_form_test.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -283,7 +352,7 @@ file safety:
   test/file/xml_test.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 4ab72f84201c2d5f0147b7dfd041f488f6ff0422
+    safe_revision: ae75fb49baf028ac8ce08e4bedcd3625ff3ff0cd
   test/file/zip_test.rb:
     comments: 
     reviewed_by: timgentry
@@ -308,6 +377,10 @@ file safety:
     comments: 
     reviewed_by: timgentry
     safe_revision: 9abdd6ced1d0c90ce8dd88abee4eb6472c7ff0d6
+  test/helpers/file/xml_streaming_test.rb:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: ae75fb49baf028ac8ce08e4bedcd3625ff3ff0cd
   test/helpers/file/xml_test.rb:
     comments: 
     reviewed_by: timgentry
@@ -330,8 +403,8 @@ file safety:
     safe_revision: a69d4a57ddcf13cdc13c27bd2eb91a395fa7ea36
   test/non_tabular_file_helper_test.rb:
     comments: 
-    reviewed_by: timgentry
-    safe_revision: cf382902508a21a95b80ac4582fbbd117164e80e
+    reviewed_by: josh.pencheon
+    safe_revision: bb44ade56a2151706eede2c31142440ccf49e6f6
   test/pdf_form/table_test.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -356,6 +429,10 @@ file safety:
     comments: 
     reviewed_by: timgentry
     safe_revision: dab4b8a3e4b29d85eccd971e79936982d888cffd
+  test/resources/claims_utf16be_but_isnt.xml:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: ae75fb49baf028ac8ce08e4bedcd3625ff3ff0cd
   test/resources/filesystem_paths.yml:
     comments: 
     reviewed_by: timgentry
@@ -492,6 +569,10 @@ file safety:
     comments: 
     reviewed_by: timgentry
     safe_revision: 31fb1935f4578729d8786eea41cf0ce0a19be1cd
+  test/resources/two_files_single_table_mapping.zip:
+    comments: 
+    reviewed_by: ollietulloch
+    safe_revision: 830de0f8cb139c5f61525652b424423935cfc7ac
   test/resources/txt_file_xls_extension.xls:
     comments: 
     reviewed_by: timgentry
@@ -542,8 +623,8 @@ file safety:
     safe_revision: 93ccee82fc2165d1ca2d9b03d146ae03e769ea96
   test/universal_importer_helper_test.rb:
     comments: 
-    reviewed_by: josh.pencheon
-    safe_revision: c3183e522bce50008df576ceb47fe4761ab8f966
+    reviewed_by: ollietulloch
+    safe_revision: 830de0f8cb139c5f61525652b424423935cfc7ac
   test/xml/table_test.rb:
     comments: 
     reviewed_by: josh.pencheon

data/lib/ndr_import/csv_library.rb

@@ -12,8 +12,8 @@ class << CSVLibrary
 
   # Ensure that we can pass "mode" straight through the underlying IO object
   def foreach(path, **options, &block)
-    return to_enum(__method__, path, options) unless block
-    open(path, options.delete(:mode) || 'r', options) do |csv|
+    return to_enum(__method__, path, **options) unless block
+    open(path, options.delete(:mode) || 'r', **options) do |csv|
       csv.each(&block)
     end
   end

data/lib/ndr_import/file/xml.rb

@@ -1,5 +1,6 @@
 require 'ndr_support/safe_file'
 require 'ndr_import/helpers/file/xml'
+require 'ndr_import/helpers/file/xml_streaming'
 require_relative 'registry'
 
 module NdrImport
@@ -9,6 +10,7 @@ module NdrImport
     # This class is a xml file handler that returns a single table.
     class Xml < Base
       include NdrImport::Helpers::File::Xml
+      include NdrImport::Helpers::File::XmlStreaming
 
       private
 
@@ -16,9 +18,14 @@ module NdrImport
       def rows(&block)
         return enum_for(:rows) unless block
 
-        doc = read_xml_file(@filename)
+        xpath = @options['xml_record_xpath']
 
-        doc.xpath(@options['xml_record_xpath']).each(&block)
+        if @options['slurp']
+          doc = read_xml_file(@filename)
+          doc.xpath(xpath).each(&block)
+        else
+          each_node(@filename, xpath, &block)
+        end
       rescue StandardError => e
         raise("#{SafeFile.basename(@filename)} [#{e.class}: #{e.message}]")
       end

data/lib/ndr_import/helpers/file/delimited.rb

@@ -36,7 +36,7 @@ module NdrImport
 
           # By now, we know `encodings` should let us read the whole
           # file succesfully; if there are problems, we should crash.
-          CSVLibrary.foreach(safe_path, encodings) do |line|
+          CSVLibrary.foreach(safe_path, **encodings) do |line|
             yield line.map(&:to_s)
           end
         end
@@ -73,7 +73,7 @@ module NdrImport
 
               row_num = 0
               # Iterate through the file; if we reach the end, this encoding worked:
-              CSVLibrary.foreach(safe_path, options) { |_line| row_num += 1 }
+              CSVLibrary.foreach(safe_path, **options) { |_line| row_num += 1 }
               return options
             rescue ArgumentError => e
               next if e.message =~ /invalid byte sequence/ # This encoding didn't work

data/lib/ndr_import/helpers/file/xml.rb

@@ -15,10 +15,13 @@ module NdrImport
 
           require 'nokogiri'
 
-          Nokogiri::XML(ensure_utf8! file_data).tap do |doc|
-            doc.encoding = 'UTF-8'
-            emulate_strict_mode_fatal_check!(doc)
+          doc = Nokogiri::XML((ensure_utf8! file_data)) do |config|
+            config.huge
           end
+          doc.encoding = 'UTF-8'
+          emulate_strict_mode_fatal_check!(doc)
+
+          doc
         end
 
         # Nokogiri can use give a `STRICT` parse option to libxml, but our friendly

data/lib/ndr_import/helpers/file/xml_streaming.rb

@@ -0,0 +1,183 @@
+require 'shellwords'
+
+require 'ndr_support/safe_file'
+require 'ndr_support/utf8_encoding'
+
+module NdrImport
+  module Helpers
+    module File
+      # This mixin adds XML streaming functionality, to support more performant handling
+      # of large files by Nokogiri. Uses the `XML::Reader` API, and maintains a temporary
+      # DOM as the XML is streamed to allow XPath querying from the root node.
+      #
+      # If the system has `iconv` available, will attempt to verify the encoding of the
+      # file being read externally, so it can be streamed in to Ruby. Otherwise, will load
+      # the raw data in to check the encoding, but still stream it through Nokogiri's parser.
+      module XmlStreaming
+        # Base error for all streaming-specific issues.
+        class Error < StandardError; end
+
+        # Raised if nested tags are accounted which the streaming approach cannnot handle.
+        class NestingError < Error
+          def initialize(node)
+            super <<~STR
+              Element '#{node.name}' was found nested inside another of the same type.
+              This is not accessible, and a known limitation of XmlStreaming.
+            STR
+          end
+        end
+
+        # Object to track state as the XML is iterated over, and detect
+        # when an element of interest is entered.
+        class Cursor
+          # wrapper to hold a representation of each element we descent into:
+          StackItem = Struct.new(:name, :attrs, :empty)
+
+          def initialize(xpath)
+            @xpath = xpath
+            @stack = []
+            @match_depth = nil
+          end
+
+          # Has this cursor already passed inside a similar node?
+          def in?(node)
+            @stack.detect { |item| item.name == node.name }
+          end
+
+          def enter(node)
+            @stack.push StackItem.new(node.name, node.attributes, node.empty_element?)
+          end
+
+          def leave(_node)
+            @stack.pop
+            @match_depth = nil if @match_depth && @stack.length < @match_depth
+          end
+
+          # Does the element that the cursor is currently on match what
+          # is being looked for?
+          def matches?
+            # Can't match again if we're inside a match already:
+            return false if @matched_depth
+
+            match = current_stack_match?
+
+            # "empty element" matches are yielded immediately, without
+            # tagging the stack as having matched, because there won't
+            # be an equivalent closing tag to end the match with later.
+            if in_empty_element?
+              @stack.pop
+            elsif match
+              @match_depth = @stack.length
+            end
+
+            match
+          end
+
+          private
+
+          def in_empty_element?
+            @stack.last.empty
+          end
+
+          # Does the current state of the stack mean we've met the xpath
+          # criteria? Must be an exact match, not just matching a parent
+          # element in the DOM.
+          def current_stack_match?
+            parent_stack = @stack[0..-2]
+
+            return false unless dom_stubs[@stack].at_xpath(@xpath)
+
+            parent_stack.empty? || !dom_stubs[parent_stack].at_xpath(@xpath)
+          end
+
+          # A cached collection of DOM fragments, to represent the structure
+          # necessary to use xpath to descend into the main document's DOM.
+          def dom_stubs
+            @dom_stubs ||= Hash.new do |hash, items|
+              hash[items.dup] = Nokogiri::XML::Builder.new do |dom|
+                add_items_to_dom(dom, items.dup)
+              end.doc
+            end
+          end
+
+          # Helper to recursively build XML fragment.
+          def add_items_to_dom(dom, items)
+            item = items.shift
+            dom.send(item.name, item.attrs) do
+              add_items_to_dom(dom, items) if items.any?
+            end
+          end
+        end
+
+        include UTF8Encoding
+
+        # Streams the contents of the given `safe_path`, and yields
+        # each element matching `xpath` as they're found.
+        #
+        # In the case of dodgy encoding, may fall back to slurping the
+        # file, but will still use stream parsing for XML.
+        def each_node(safe_path, xpath, &block)
+          return enum_for(:each_node, safe_path, xpath) unless block
+
+          require 'nokogiri'
+
+          with_encoding_check(safe_path) do |stream, encoding|
+            stream_xml_nodes(stream, xpath, encoding, &block)
+          end
+        end
+
+        private
+
+        # We need to ensure the raw data is UTF8 before we start streaming
+        # it with nokogiri. If we can do an external check, great. Otherwise,
+        # we need to slurp and convert the raw data before presenting it.
+        def with_encoding_check(safe_path)
+          forced_encoding = nil
+
+          stream = ::File.open(SafeFile.safepath_to_string(safe_path))
+
+          unless external_utf8_check?(safe_path)
+            stream = StringIO.new ensure_utf8!(stream.read)
+            forced_encoding = 'UTF8'
+          end
+
+          yield stream, forced_encoding
+        end
+
+        # Use iconv, if available, to check raw data encoding:
+        def external_utf8_check?(safe_path)
+          iconv = system('command -v iconv > /dev/null 2>&1')
+          return false unless iconv
+
+          path = SafeFile.safepath_to_string(safe_path)
+          system("iconv -f UTF-8 #{Shellwords.escape(path)} > /dev/null 2>&1")
+        end
+
+        def stream_xml_nodes(io, node_xpath, encoding = nil)
+          # Track nesting as the cursor moves through the document:
+          cursor = Cursor.new(node_xpath)
+
+          # If markup isn't well-formed, try to work around it:
+          options = Nokogiri::XML::ParseOptions::RECOVER
+          reader  = Nokogiri::XML::Reader(io, nil, encoding, options)
+
+          reader.each do |node|
+            case node.node_type
+            when Nokogiri::XML::Reader::TYPE_ELEMENT # "opening tag"
+              raise NestingError, node if cursor.in?(node)
+
+              cursor.enter(node)
+              next unless cursor.matches?
+
+              # The xpath matched - construct a DOM fragment to yield back:
+              element = Nokogiri::XML(node.outer_xml).at("./#{node.name}")
+              yield element
+            when Nokogiri::XML::Reader::TYPE_END_ELEMENT # "closing tag"
+              cursor.leave(node)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ndr_import/non_tabular/column_mapping.rb

@@ -6,7 +6,7 @@ module NdrImport
     # the logic associated with finding matching lines of source data and subsequently
     # capturing arrays of values within them.
     class ColumnMapping
-      attr_accessor :name, :cell_mapping, :lines, :capture, :join
+      attr_accessor :name, :cell_mapping, :lines, :capture, :join, :preserve_blank_lines
 
       def initialize(column_mapping)
         @name         = column_mapping['rawtext_name'] ||
@@ -18,6 +18,7 @@ module NdrImport
 
         @lines = @cell_mapping['lines']
         @join  = @cell_mapping['join']
+        @preserve_blank_lines = @cell_mapping['preserve_blank_lines']
       end
 
       # This method returns the range of matching source data lines. If the range is a

data/lib/ndr_import/non_tabular/record.rb

@@ -74,7 +74,10 @@ module NdrImport
           begin
             matches = get_matches(column_mapping)
             # Join the non-blank lines together and add to the array of cells
-            cells << matches.select { |value| !value.blank? }.join(column_mapping.join || '')
+            lines = matches.select do |value|
+              column_mapping.preserve_blank_lines ? value : value.present?
+            end
+            cells << lines.join(column_mapping.join || '')
           rescue RegexpRange::PatternMatchError
             cells << nil
           end

data/lib/ndr_import/universal_importer_helper.rb

@@ -1,3 +1,5 @@
+require 'shellwords'
+
 require 'ndr_import/file/registry'
 
 module NdrImport
@@ -5,11 +7,31 @@ module NdrImport
   # complexity of enumerating over files and tables (which should be universally useful).
   # It is assumed that the host module/class defines `unzip_path`.
   module UniversalImporterHelper
+    # Helper class to allow multiple source enumerators to contribute to one overall table.
+    class TableEnumProxy
+      include Enumerable
+
+      def initialize
+        @table_enums = []
+      end
+
+      def add_table_enum(table_enum)
+        @table_enums << table_enum
+      end
+
+      def each(&block)
+        return enum_for(:each) unless block
+
+        @table_enums.each { |table_enum| table_enum.each(&block) }
+      end
+    end
+
     def table_enumerators(filename)
       table_enumerators = {}
+      table_enumerators = Hash.new { |hash, key| hash[key] = TableEnumProxy.new }
 
       extract(filename).each do |table, rows|
-        table_enumerators[table.canonical_name] = table.transform(rows)
+        table_enumerators[table.canonical_name].add_table_enum table.transform(rows)
       end
 
       table_enumerators
@@ -38,7 +60,8 @@ module NdrImport
           'col_sep'          => table_mapping.try(:delimiter),
           'file_password'    => table_mapping.try(:file_password),
           'liberal_parsing'  => table_mapping.try(:liberal_parsing),
-          'xml_record_xpath' => table_mapping.try(:xml_record_xpath)
+          'xml_record_xpath' => table_mapping.try(:xml_record_xpath),
+          'slurp'            => table_mapping.try(:slurp)
         }
 
         tables = NdrImport::File::Registry.tables(filename, table_mapping.try(:format), options)

data/lib/ndr_import/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 # This stores the current version of the NdrImport gem
 module NdrImport
-  VERSION = '8.5.2'.freeze
+  VERSION = '9.0.3'.freeze
 end

data/ndr_import.gemspec

@@ -15,7 +15,7 @@ Gem::Specification.new do |spec|
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
   spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
-    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(docs|test|spec|features)/}) }
   end
   spec.files         -= %w[.travis.yml] # Not needed in the gem
   spec.bindir        = 'exe'
@@ -26,7 +26,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'activesupport', '>= 5.0', '< 7'
   spec.add_dependency 'ndr_support', '>= 5.3.2', '< 6'
 
-  spec.add_dependency 'rubyzip', '~> 1.2', '>= 1.2.2'
+  spec.add_dependency 'rubyzip', '~> 2.0'
   spec.add_dependency 'roo', '~> 2.0'
 
   spec.add_dependency 'docx', '~> 0.3'
@@ -36,15 +36,15 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'pdf-reader', '~> 2.1'
   spec.add_dependency 'roo-xls'
   spec.add_dependency 'seven_zip_ruby', '~> 1.2'
-  spec.add_dependency 'spreadsheet', '1.0.3'
+  spec.add_dependency 'spreadsheet', '1.2.6'
 
   spec.required_ruby_version = '>= 2.5'
 
   spec.add_development_dependency 'bundler'
-  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rake', '~> 12.3', '>= 12.3.3'
   spec.add_development_dependency 'minitest'
   spec.add_development_dependency 'mocha'
-  spec.add_development_dependency 'ndr_dev_support', '~> 3.1', '>= 3.1.3'
+  spec.add_development_dependency 'ndr_dev_support', '>= 3.1.3'
   spec.add_development_dependency 'guard'
   spec.add_development_dependency 'guard-rubocop'
   spec.add_development_dependency 'guard-test'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ndr_import
 version: !ruby/object:Gem::Version
-  version: 8.5.2
+  version: 9.0.3
 platform: ruby
 authors:
 - NCRS Development Team
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-05-17 00:00:00.000000000 Z
+date: 2021-01-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -70,20 +70,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.2.2
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.2.2
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: roo
   requirement: !ruby/object:Gem::Requirement
@@ -208,14 +202,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.3
+        version: 1.2.6
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.3
+        version: 1.2.6
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -236,14 +230,20 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 12.3.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 12.3.3
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -276,9 +276,6 @@ dependencies:
   name: ndr_dev_support
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.1'
     - - ">="
       - !ruby/object:Gem::Version
         version: 3.1.3
@@ -286,9 +283,6 @@ dependencies:
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.1'
     - - ">="
       - !ruby/object:Gem::Version
         version: 3.1.3
@@ -413,6 +407,7 @@ files:
 - lib/ndr_import/helpers/file/pdf.rb
 - lib/ndr_import/helpers/file/word.rb
 - lib/ndr_import/helpers/file/xml.rb
+- lib/ndr_import/helpers/file/xml_streaming.rb
 - lib/ndr_import/helpers/file/zip.rb
 - lib/ndr_import/mapper.rb
 - lib/ndr_import/mapping_error.rb