ndr_import 11.0.2 → 11.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ab2194b39d6226b9c8883f89ef30045c714fe276cdd5312f0b53f643cebbc003
-  data.tar.gz: 54433f513c6a6fb546f2f1b217d711214e44e1c30f488d1d00f9a67f41b94c0b
+  metadata.gz: e2869dc7651ab267d05332b78b6e18827ec8a2b2cf971ec023fba442df836f87
+  data.tar.gz: b8cc3ca305d12f282ad566b65d304af7d1ac527ced8f0bda7171cda5e3baeeac
 SHA512:
-  metadata.gz: 6c60e8a7a9abd7e30ff754898e6c332f150133488ab5837cdf030bff483db6b3eda6196bb42a2bb42099104690d2621c1b9ad39580313ff82c66f813043f2843
-  data.tar.gz: 4094561f3f994b0a82ee974f5eba00f6e70fdf219c311534ab070dc5da8f861a4dbe73d67842db2231766a18d0a1721cd13e9a8e46a9910987e450f388f06369
+  metadata.gz: 607860deb2f9a9f35491b96c280e21070f123c8a70de6c2796db308aeb5dff7b4ee9a65416ff0708d9ea0a587d749a2f756d59c6446df55b78ac3eee0ebf3b63
+  data.tar.gz: 6513f63649016ec5025b3b7f000f76b57be3db1c6670c066b632f9fbb2198bbf2c700668575026bf903f44f866ed3377a99c93686e2eac94036b666254b4abc8

data/CHANGELOG.md

@@ -1,6 +1,14 @@
 ## [Unreleased]
 =======
-* no unreleased changes*
+* no unreleased changes *
+
+## 11.2.0 / 2024-04-10
+### Added
+* XML file/table metadata storage
+
+## 11.1.0 / 2024-03-07
+### Added
+* XML table should not expect column mappings for empty nodes/elements
 
 ## 11.0.2 / 2024-02-06
 ### Fixed

data/lib/ndr_import/file/base.rb

@@ -9,6 +9,8 @@ module NdrImport
   module File
     # All common base file handler logic is defined here.
     class Base
+      attr_accessor :file_metadata
+
       def initialize(filename, format, options = {})
         @filename = filename
         @format = format
@@ -32,10 +34,10 @@ module NdrImport
         yield @filename
       end
 
-      # This method iterates over the tables in the given file and yields with two arguments:
-      # a tablename and a row enumerator (for that table). For a spreadsheet it may yield for
-      # every worksheet in the file and for a CSV file it will only yield once (the entire
-      # file is one table).
+      # This method iterates over the tables in the given file and yields with three arguments:
+      # a tablename, a row enumerator (for that table) and any file metadata.
+      # For a spreadsheet it may yield for  every worksheet in the file and for a CSV file it
+      # will only yield once (the entire file is one table).
       #
       # As single table files are in the majority, the Base implementation is defined for
       # single table handlers and you will only need to implement the rows iterator. If your
@@ -45,7 +47,7 @@ module NdrImport
       def tables
         return enum_for(:tables) unless block_given?
 
-        yield nil, rows
+        yield nil, rows, file_metadata
       end
 
       private

data/lib/ndr_import/file/xml.rb

@@ -16,18 +16,73 @@ module NdrImport
         super
 
         @pattern_match_xpath = @options['pattern_match_record_xpath']
+        @xml_file_metadata   = @options['xml_file_metadata']
+        @options['slurp'] ? prepare_slurped_file : prepare_streamed_file
       end
 
       private
 
+      def prepare_slurped_file
+        @doc = read_xml_file(@filename)
+        slurp_metadata_values
+      end
+
+      def prepare_streamed_file
+        with_encoding_check(@filename) do |stream, encoding|
+          @stream   = stream
+          @encoding = encoding
+        end
+        stream_metadata_values
+      end
+
+      def slurp_metadata_values
+        return unless @xml_file_metadata.is_a?(Hash)
+
+        self.file_metadata = @xml_file_metadata.transform_values do |xpath|
+          @doc.xpath(xpath).inner_text
+        end
+      end
+
+      def stream_metadata_values
+        return unless @xml_file_metadata.is_a?(Hash)
+
+        self.file_metadata = @xml_file_metadata.transform_values.with_index do |xpath, index|
+          # Ensure we're at the start of the stream each time
+          @stream.rewind unless index.zero?
+
+          metadata_from_stream(xpath)
+        end
+      end
+
+      def metadata_from_stream(xpath)
+        cursor = Cursor.new(xpath, false)
+
+        # If markup isn't well-formed, try to work around it:
+        options = Nokogiri::XML::ParseOptions::RECOVER
+        reader  = Nokogiri::XML::Reader(@stream, nil, @encoding, options)
+
+        reader.each do |node|
+          case node.node_type
+          when Nokogiri::XML::Reader::TYPE_ELEMENT # "opening tag"
+            raise NestingError, node if cursor.in?(node)
+
+            cursor.enter(node)
+            return cursor.inner_text if cursor.send(:current_stack_match?)
+          when Nokogiri::XML::Reader::TYPE_END_ELEMENT # "closing tag"
+            cursor.leave(node)
+          end
+        end
+      end
+
       # Iterate through the file, yielding each 'xml_record_xpath' element in turn.
       def rows(&block)
         return enum_for(:rows) unless block
 
         if @options['slurp']
-          record_elements(read_xml_file(@filename)).each(&block)
+          record_elements.each(&block)
         else
-          each_node(@filename, xml_record_xpath, @pattern_match_xpath, &block)
+          @stream.rewind
+          each_node(@stream, @encoding, xml_record_xpath, @pattern_match_xpath, &block)
         end
       end
 
@@ -35,13 +90,13 @@ module NdrImport
         @pattern_match_xpath ? @options['xml_record_xpath'] : "*/#{@options['xml_record_xpath']}"
       end
 
-      def record_elements(doc)
+      def record_elements
         if @pattern_match_xpath
-          doc.root.children.find_all do |element|
+          @doc.root.children.find_all do |element|
             element.name =~ Regexp.new(@options['xml_record_xpath'])
           end
         else
-          doc.root.xpath(@options['xml_record_xpath'])
+          @doc.root.xpath(@options['xml_record_xpath'])
         end
       end
     end

data/lib/ndr_import/helpers/file/xml_streaming.rb

@@ -78,6 +78,10 @@ module NdrImport
             match
           end
 
+          def inner_text
+            dom_stubs[@stack].xpath(@xpath)&.inner_text
+          end
+
           private
 
           def in_empty_element?
@@ -134,21 +138,18 @@ module NdrImport
 
         include UTF8Encoding
 
-        # Streams the contents of the given `safe_path`, and yields
-        # each element matching `xpath` as they're found.
+        # Yields each element matching `xpath` from `stream` as they're found.
         #
         # In the case of dodgy encoding, may fall back to slurping the
         # file, but will still use stream parsing for XML.
         #
         # Optionally pattern match the xpath
-        def each_node(safe_path, xpath, pattern_match_xpath = nil, &block)
-          return enum_for(:each_node, safe_path, xpath, pattern_match_xpath) unless block
+        def each_node(stream, encoding, xpath, pattern_match_xpath = nil, &block)
+          return enum_for(:each_node, stream, encoding, xpath, pattern_match_xpath) unless block
 
           require 'nokogiri'
 
-          with_encoding_check(safe_path) do |stream, encoding|
-            stream_xml_nodes(stream, xpath, pattern_match_xpath, encoding, &block)
-          end
+          stream_xml_nodes(stream, xpath, pattern_match_xpath, encoding, &block)
         end
 
         private

data/lib/ndr_import/table.rb

@@ -21,7 +21,7 @@ module NdrImport
     end
 
     attr_reader(*all_valid_options)
-    attr_accessor :notifier
+    attr_accessor :notifier, :table_metadata
 
     def initialize(options = {})
       options.stringify_keys! if options.is_a?(Hash)

data/lib/ndr_import/universal_importer_helper.rb

@@ -59,7 +59,8 @@ module NdrImport
                     'xml_record_xpath'           => table_mapping.try(:xml_record_xpath),
                     'slurp'                      => table_mapping.try(:slurp),
                     'yield_xml_record'           => table_mapping.try(:yield_xml_record),
-                    'pattern_match_record_xpath' => table_mapping.try(:pattern_match_record_xpath) }
+                    'pattern_match_record_xpath' => table_mapping.try(:pattern_match_record_xpath),
+                    'xml_file_metadata'          => table_mapping.try(:xml_file_metadata) }
 
         tables = NdrImport::File::Registry.tables(filename, table_mapping.try(:format), options)
         yield_tables_and_their_content(filename, tables, &block)
@@ -71,12 +72,12 @@ module NdrImport
     def yield_tables_and_their_content(filename, tables, &block)
       return enum_for(:yield_tables_and_their_content, filename, tables) unless block_given?
 
-      tables.each do |tablename, table_content|
+      tables.each do |tablename, table_content, file_metadata|
         mapping = get_table_mapping(filename, tablename)
         next if mapping.nil?
 
         mapping.notifier = get_notifier(record_total(filename, table_content))
-
+        mapping.table_metadata = file_metadata || {}
         yield(mapping, table_content)
       end
     end

data/lib/ndr_import/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 # This stores the current version of the NdrImport gem
 module NdrImport
-  VERSION = '11.0.2'
+  VERSION = '11.2.0'
 end

data/lib/ndr_import/xml/table.rb

@@ -10,7 +10,8 @@ module NdrImport
       require 'ndr_import/xml/column_mapping'
       require 'ndr_import/xml/masked_mappings'
 
-      XML_OPTIONS = %w[pattern_match_record_xpath xml_record_xpath yield_xml_record].freeze
+      XML_OPTIONS = %w[pattern_match_record_xpath xml_file_metadata xml_record_xpath
+                       yield_xml_record].freeze
 
       def self.all_valid_options
         super - %w[delimiter header_lines footer_lines] + XML_OPTIONS
@@ -169,6 +170,8 @@ module NdrImport
         xpaths = []
 
         line.xpath('.//*[not(child::*)]').each do |node|
+          next unless populated?(node)
+
           xpath = node.path.sub("#{line.path}/", '')
           if node.attributes.any?
             node.attributes.each_key { |key| xpaths << "#{xpath}/@#{key}" }
@@ -179,6 +182,12 @@ module NdrImport
         xpaths
       end
 
+      def populated?(node)
+        node.element_children.empty? &&
+          !node.is_a?(Nokogiri::XML::Comment) && !node.text? && !node.cdata? &&
+          !(node.attributes.empty? && node.content.strip.blank?)
+      end
+
       def build_xpath_from(column)
         column_name = column_name_from(column)
         column['xml_cell'].presence ? relative_path_from(column, column_name) : column_name

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ndr_import
 version: !ruby/object:Gem::Version
-  version: 11.0.2
+  version: 11.2.0
 platform: ruby
 authors:
 - NCRS Development Team
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-02-06 00:00:00.000000000 Z
+date: 2024-04-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -465,7 +465,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.33
+rubygems_version: 3.2.3
 signing_key:
 specification_version: 4
 summary: NDR Import