ndr_import 8.6.0 → 9.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ffddef09a58ca08e5d491449ef48cd829dfd045bf6f38b5289cff240859940e0
-  data.tar.gz: 30540a5af117b4ae57e002794f26df18544a99b07c7fdfb95a2ad552e9efa7b6
+  metadata.gz: 500566629ada1bfbab0def117060fec471bf2fab2c320a112b05a97a8474ee95
+  data.tar.gz: 572dc475c3704f2f749e9de11e031829e593d1e2a6fa24a6755342f2f9bdbaae
 SHA512:
-  metadata.gz: 2ff3840cb513c4d0253c9c40db709563bf7bd8d62777ed49358e9a67a5304c03b7f18a966bb894099a4f7f9a64933bc11151048e29ce50989a791732cceee74c
-  data.tar.gz: de437a4e4e62c77e93050df6c0d50c486d2faca290830acb1d719ddadc0b2165bd5405a11a38db4bd0ca438432b082fddafb42edaf41b09a98efcdfd79c63f55
+  metadata.gz: 8c8491043c7e58d0ca4953c58ff12914c1dd8deb587c67a9cbd488e8420b26f9e8ea55f7b23cbea68bf02956c3215602932dee5ffd23506d4017ac997378b835
+  data.tar.gz: 03c24de311a201a871f752773f0d43c38e2c9e49d31585759c0b009cc76adbf6668350d0bc7c6c2746f9b84998bd384055b3fa47806fc242a3fa49fdb3ed4851

data/CHANGELOG.md

@@ -1,6 +1,13 @@
 ## [Unreleased]
 *no unreleased changes*
 
+## 9.0.0 / 2019-07-31
+### Changed
+* `File::Xml` will now stream XML files by default. Use `slurp: true` for the old behaviour. (#43)
+
+### Added
+* Add `XmlStreaming` helper, for more performant handling of large XML documents with Nokogiri. (#43)
+
 ## 8.6.0 / 2019-06-07
 ### Added
 * Allow conditional preservation of blank lines when joining lines in non-tabular data (#41)

data/code_safety.yml

@@ -19,7 +19,7 @@ file safety:
   CHANGELOG.md:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 53c17f14ee19a9ea83d044508093147daa32ba3d
+    safe_revision: 0a4e05d45ee65e25edc36de84d3c450cc15dc3ed
   CODE_OF_CONDUCT.md:
     comments: 
     reviewed_by: timgentry
@@ -139,7 +139,7 @@ file safety:
   lib/ndr_import/file/xml.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 4ab72f84201c2d5f0147b7dfd041f488f6ff0422
+    safe_revision: ae75fb49baf028ac8ce08e4bedcd3625ff3ff0cd
   lib/ndr_import/file/zip.rb:
     comments: 
     reviewed_by: timgentry
@@ -168,6 +168,11 @@ file safety:
     comments: 
     reviewed_by: josh.pencheon
     safe_revision: d2245268ec6a0e4f60c521d171a820f299632c4f
+  lib/ndr_import/helpers/file/xml_streaming.rb:
+    comments: uses SafePath and Shellwords when accessing filesystem, or making system
+      calls
+    reviewed_by: josh.pencheon
+    safe_revision: ae75fb49baf028ac8ce08e4bedcd3625ff3ff0cd
   lib/ndr_import/helpers/file/zip.rb:
     comments: 
     reviewed_by: timgentry
@@ -223,7 +228,7 @@ file safety:
   lib/ndr_import/universal_importer_helper.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: a69d4a57ddcf13cdc13c27bd2eb91a395fa7ea36
+    safe_revision: ae75fb49baf028ac8ce08e4bedcd3625ff3ff0cd
   lib/ndr_import/unmapped_data_error.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -231,7 +236,7 @@ file safety:
   lib/ndr_import/version.rb:
     comments: another check?
     reviewed_by: josh.pencheon
-    safe_revision: 53c17f14ee19a9ea83d044508093147daa32ba3d
+    safe_revision: 0a4e05d45ee65e25edc36de84d3c450cc15dc3ed
   lib/ndr_import/xml/table.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -239,7 +244,7 @@ file safety:
   ndr_import.gemspec:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: d3d9a987befeecb122a448d8d06e66d74da13fb5
+    safe_revision: ae75fb49baf028ac8ce08e4bedcd3625ff3ff0cd
   test/file/acro_form_test.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -283,7 +288,7 @@ file safety:
   test/file/xml_test.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 4ab72f84201c2d5f0147b7dfd041f488f6ff0422
+    safe_revision: ae75fb49baf028ac8ce08e4bedcd3625ff3ff0cd
   test/file/zip_test.rb:
     comments: 
     reviewed_by: timgentry
@@ -308,6 +313,10 @@ file safety:
     comments: 
     reviewed_by: timgentry
     safe_revision: 9abdd6ced1d0c90ce8dd88abee4eb6472c7ff0d6
+  test/helpers/file/xml_streaming_test.rb:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: ae75fb49baf028ac8ce08e4bedcd3625ff3ff0cd
   test/helpers/file/xml_test.rb:
     comments: 
     reviewed_by: timgentry
@@ -356,6 +365,10 @@ file safety:
     comments: 
     reviewed_by: timgentry
     safe_revision: dab4b8a3e4b29d85eccd971e79936982d888cffd
+  test/resources/claims_utf16be_but_isnt.xml:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: ae75fb49baf028ac8ce08e4bedcd3625ff3ff0cd
   test/resources/filesystem_paths.yml:
     comments: 
     reviewed_by: timgentry

data/lib/ndr_import/file/xml.rb

@@ -1,5 +1,6 @@
 require 'ndr_support/safe_file'
 require 'ndr_import/helpers/file/xml'
+require 'ndr_import/helpers/file/xml_streaming'
 require_relative 'registry'
 
 module NdrImport
@@ -9,6 +10,7 @@ module NdrImport
     # This class is a xml file handler that returns a single table.
     class Xml < Base
       include NdrImport::Helpers::File::Xml
+      include NdrImport::Helpers::File::XmlStreaming
 
       private
 
@@ -16,9 +18,14 @@ module NdrImport
       def rows(&block)
         return enum_for(:rows) unless block
 
-        doc = read_xml_file(@filename)
+        xpath = @options['xml_record_xpath']
 
-        doc.xpath(@options['xml_record_xpath']).each(&block)
+        if @options['slurp']
+          doc = read_xml_file(@filename)
+          doc.xpath(xpath).each(&block)
+        else
+          each_node(@filename, xpath, &block)
+        end
       rescue StandardError => e
         raise("#{SafeFile.basename(@filename)} [#{e.class}: #{e.message}]")
       end

data/lib/ndr_import/helpers/file/xml_streaming.rb

@@ -0,0 +1,181 @@
+require 'ndr_support/safe_file'
+require 'ndr_support/utf8_encoding'
+
+module NdrImport
+  module Helpers
+    module File
+      # This mixin adds XML streaming functionality, to support more performant handling
+      # of large files by Nokogiri. Uses the `XML::Reader` API, and maintains a temporary
+      # DOM as the XML is streamed to allow XPath querying from the root node.
+      #
+      # If the system has `iconv` available, will attempt to verify the encoding of the
+      # file being read externally, so it can be streamed in to Ruby. Otherwise, will load
+      # the raw data in to check the encoding, but still stream it through Nokogiri's parser.
+      module XmlStreaming
+        # Base error for all streaming-specific issues.
+        class Error < StandardError; end
+
+        # Raised if nested tags are accounted which the streaming approach cannnot handle.
+        class NestingError < Error
+          def initialize(node)
+            super <<~STR
+              Element '#{node.name}' was found nested inside another of the same type.
+              This is not accessible, and a known limitation of XmlStreaming.
+            STR
+          end
+        end
+
+        # Object to track state as the XML is iterated over, and detect
+        # when an element of interest is entered.
+        class Cursor
+          # wrapper to hold a representation of each element we descent into:
+          StackItem = Struct.new(:name, :attrs, :empty)
+
+          def initialize(xpath)
+            @xpath = xpath
+            @stack = []
+            @match_depth = nil
+          end
+
+          # Has this cursor already passed inside a similar node?
+          def in?(node)
+            @stack.detect { |item| item.name == node.name }
+          end
+
+          def enter(node)
+            @stack.push StackItem.new(node.name, node.attributes, node.empty_element?)
+          end
+
+          def leave(_node)
+            @stack.pop
+            @match_depth = nil if @match_depth && @stack.length < @match_depth
+          end
+
+          # Does the element that the cursor is currently on match what
+          # is being looked for?
+          def matches?
+            # Can't match again if we're inside a match already:
+            return false if @matched_depth
+
+            match = current_stack_match?
+
+            # "empty element" matches are yielded immediately, without
+            # tagging the stack as having matched, because there won't
+            # be an equivalent closing tag to end the match with later.
+            if in_empty_element?
+              @stack.pop
+            elsif match
+              @match_depth = @stack.length
+            end
+
+            match
+          end
+
+          private
+
+          def in_empty_element?
+            @stack.last.empty
+          end
+
+          # Does the current state of the stack mean we've met the xpath
+          # criteria? Must be an exact match, not just matching a parent
+          # element in the DOM.
+          def current_stack_match?
+            parent_stack = @stack[0..-2]
+
+            return false unless dom_stubs[@stack].at_xpath(@xpath)
+
+            parent_stack.empty? || !dom_stubs[parent_stack].at_xpath(@xpath)
+          end
+
+          # A cached collection of DOM fragments, to represent the structure
+          # necessary to use xpath to descend into the main document's DOM.
+          def dom_stubs
+            @dom_stubs ||= Hash.new do |hash, items|
+              hash[items.dup] = Nokogiri::XML::Builder.new do |dom|
+                add_items_to_dom(dom, items.dup)
+              end.doc
+            end
+          end
+
+          # Helper to recursively build XML fragment.
+          def add_items_to_dom(dom, items)
+            item = items.shift
+            dom.send(item.name, item.attrs) do
+              add_items_to_dom(dom, items) if items.any?
+            end
+          end
+        end
+
+        include UTF8Encoding
+
+        # Streams the contents of the given `safe_path`, and yields
+        # each element matching `xpath` as they're found.
+        #
+        # In the case of dodgy encoding, may fall back to slurping the
+        # file, but will still use stream parsing for XML.
+        def each_node(safe_path, xpath, &block)
+          return enum_for(:each_node, safe_path, xpath) unless block
+
+          require 'nokogiri'
+
+          with_encoding_check(safe_path) do |stream, encoding|
+            stream_xml_nodes(stream, xpath, encoding, &block)
+          end
+        end
+
+        private
+
+        # We need to ensure the raw data is UTF8 before we start streaming
+        # it with nokogiri. If we can do an external check, great. Otherwise,
+        # we need to slurp and convert the raw data before presenting it.
+        def with_encoding_check(safe_path)
+          forced_encoding = nil
+
+          stream = ::File.open(SafeFile.safepath_to_string(safe_path))
+
+          unless external_utf8_check?(safe_path)
+            stream = StringIO.new ensure_utf8!(stream.read)
+            forced_encoding = 'UTF8'
+          end
+
+          yield stream, forced_encoding
+        end
+
+        # Use iconv, if available, to check raw data encoding:
+        def external_utf8_check?(safe_path)
+          iconv = system('command -v iconv > /dev/null 2>&1')
+          return false unless iconv
+
+          path = SafeFile.safepath_to_string(safe_path)
+          system("iconv -f UTF-8 #{Shellwords.escape(path)} > /dev/null 2>&1")
+        end
+
+        def stream_xml_nodes(io, node_xpath, encoding = nil)
+          # Track nesting as the cursor moves through the document:
+          cursor = Cursor.new(node_xpath)
+
+          # If markup isn't well-formed, try to work around it:
+          options = Nokogiri::XML::ParseOptions::RECOVER
+          reader  = Nokogiri::XML::Reader(io, nil, encoding, options)
+
+          reader.each do |node|
+            case node.node_type
+            when Nokogiri::XML::Reader::TYPE_ELEMENT # "opening tag"
+              raise NestingError, node if cursor.in?(node)
+
+              cursor.enter(node)
+              next unless cursor.matches?
+
+              # The xpath matched - construct a DOM fragment to yield back:
+              element = Nokogiri::XML(node.outer_xml).at("./#{node.name}")
+              yield element
+            when Nokogiri::XML::Reader::TYPE_END_ELEMENT # "closing tag"
+              cursor.leave(node)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ndr_import/universal_importer_helper.rb

@@ -38,7 +38,8 @@ module NdrImport
           'col_sep'          => table_mapping.try(:delimiter),
           'file_password'    => table_mapping.try(:file_password),
           'liberal_parsing'  => table_mapping.try(:liberal_parsing),
-          'xml_record_xpath' => table_mapping.try(:xml_record_xpath)
+          'xml_record_xpath' => table_mapping.try(:xml_record_xpath),
+          'slurp'            => table_mapping.try(:slurp)
         }
 
         tables = NdrImport::File::Registry.tables(filename, table_mapping.try(:format), options)

data/lib/ndr_import/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 # This stores the current version of the NdrImport gem
 module NdrImport
-  VERSION = '8.6.0'.freeze
+  VERSION = '9.0.0'.freeze
 end

data/ndr_import.gemspec

@@ -44,7 +44,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rake', '~> 10.0'
   spec.add_development_dependency 'minitest'
   spec.add_development_dependency 'mocha'
-  spec.add_development_dependency 'ndr_dev_support', '~> 3.1', '>= 3.1.3'
+  spec.add_development_dependency 'ndr_dev_support', '>= 3.1.3'
   spec.add_development_dependency 'guard'
   spec.add_development_dependency 'guard-rubocop'
   spec.add_development_dependency 'guard-test'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ndr_import
 version: !ruby/object:Gem::Version
-  version: 8.6.0
+  version: 9.0.0
 platform: ruby
 authors:
 - NCRS Development Team
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-06-07 00:00:00.000000000 Z
+date: 2019-07-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -276,9 +276,6 @@ dependencies:
   name: ndr_dev_support
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.1'
     - - ">="
       - !ruby/object:Gem::Version
         version: 3.1.3
@@ -286,9 +283,6 @@ dependencies:
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.1'
     - - ">="
       - !ruby/object:Gem::Version
         version: 3.1.3
@@ -413,6 +407,7 @@ files:
 - lib/ndr_import/helpers/file/pdf.rb
 - lib/ndr_import/helpers/file/word.rb
 - lib/ndr_import/helpers/file/xml.rb
+- lib/ndr_import/helpers/file/xml_streaming.rb
 - lib/ndr_import/helpers/file/zip.rb
 - lib/ndr_import/mapper.rb
 - lib/ndr_import/mapping_error.rb