ndr_import 11.1.0 → 11.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1acf9db0301b6da991e890e74db10573ff666dd732eda4b58ba90e03370bb32a
-  data.tar.gz: eba9da2d2d9f5fa98c3a7f8745086f3945c363844f4d1f8fc1dc6c38ab4bcb5b
+  metadata.gz: e2869dc7651ab267d05332b78b6e18827ec8a2b2cf971ec023fba442df836f87
+  data.tar.gz: b8cc3ca305d12f282ad566b65d304af7d1ac527ced8f0bda7171cda5e3baeeac
 SHA512:
-  metadata.gz: 8509511d4f54f36344e09d380da53f168f8fa94efd4eec2d73e29c80811bd644d853bb82a835bf7ddf2820202b8abad28cbe0d265de3557e2af8adca4f12b178
-  data.tar.gz: 384d8365d4e6ba741bf6b76a1ecfe1bf1bb92c6fd0e737489d2d3a13047d9e557b883c339ff2f89f372071140a071bcbedfdd303b588dda79d58870bab0be2e6
+  metadata.gz: 607860deb2f9a9f35491b96c280e21070f123c8a70de6c2796db308aeb5dff7b4ee9a65416ff0708d9ea0a587d749a2f756d59c6446df55b78ac3eee0ebf3b63
+  data.tar.gz: 6513f63649016ec5025b3b7f000f76b57be3db1c6670c066b632f9fbb2198bbf2c700668575026bf903f44f866ed3377a99c93686e2eac94036b666254b4abc8

data/CHANGELOG.md

@@ -2,6 +2,10 @@
 =======
 * no unreleased changes *
 
+## 11.2.0 / 2024-04-10
+### Added
+* XML file/table metadata storage
+
 ## 11.1.0 / 2024-03-07
 ### Added
 * XML table should not expect column mappings for empty nodes/elements

data/lib/ndr_import/file/base.rb

@@ -9,6 +9,8 @@ module NdrImport
   module File
     # All common base file handler logic is defined here.
     class Base
+      attr_accessor :file_metadata
+
       def initialize(filename, format, options = {})
         @filename = filename
         @format = format
@@ -32,10 +34,10 @@ module NdrImport
         yield @filename
       end
 
-      # This method iterates over the tables in the given file and yields with two arguments:
-      # a tablename and a row enumerator (for that table). For a spreadsheet it may yield for
-      # every worksheet in the file and for a CSV file it will only yield once (the entire
-      # file is one table).
+      # This method iterates over the tables in the given file and yields with three arguments:
+      # a tablename, a row enumerator (for that table) and any file metadata.
+      # For a spreadsheet it may yield for  every worksheet in the file and for a CSV file it
+      # will only yield once (the entire file is one table).
       #
       # As single table files are in the majority, the Base implementation is defined for
       # single table handlers and you will only need to implement the rows iterator. If your
@@ -45,7 +47,7 @@ module NdrImport
       def tables
         return enum_for(:tables) unless block_given?
 
-        yield nil, rows
+        yield nil, rows, file_metadata
       end
 
       private

data/lib/ndr_import/file/xml.rb

@@ -16,18 +16,73 @@ module NdrImport
         super
 
         @pattern_match_xpath = @options['pattern_match_record_xpath']
+        @xml_file_metadata   = @options['xml_file_metadata']
+        @options['slurp'] ? prepare_slurped_file : prepare_streamed_file
       end
 
       private
 
+      def prepare_slurped_file
+        @doc = read_xml_file(@filename)
+        slurp_metadata_values
+      end
+
+      def prepare_streamed_file
+        with_encoding_check(@filename) do |stream, encoding|
+          @stream   = stream
+          @encoding = encoding
+        end
+        stream_metadata_values
+      end
+
+      def slurp_metadata_values
+        return unless @xml_file_metadata.is_a?(Hash)
+
+        self.file_metadata = @xml_file_metadata.transform_values do |xpath|
+          @doc.xpath(xpath).inner_text
+        end
+      end
+
+      def stream_metadata_values
+        return unless @xml_file_metadata.is_a?(Hash)
+
+        self.file_metadata = @xml_file_metadata.transform_values.with_index do |xpath, index|
+          # Ensure we're at the start of the stream each time
+          @stream.rewind unless index.zero?
+
+          metadata_from_stream(xpath)
+        end
+      end
+
+      def metadata_from_stream(xpath)
+        cursor = Cursor.new(xpath, false)
+
+        # If markup isn't well-formed, try to work around it:
+        options = Nokogiri::XML::ParseOptions::RECOVER
+        reader  = Nokogiri::XML::Reader(@stream, nil, @encoding, options)
+
+        reader.each do |node|
+          case node.node_type
+          when Nokogiri::XML::Reader::TYPE_ELEMENT # "opening tag"
+            raise NestingError, node if cursor.in?(node)
+
+            cursor.enter(node)
+            return cursor.inner_text if cursor.send(:current_stack_match?)
+          when Nokogiri::XML::Reader::TYPE_END_ELEMENT # "closing tag"
+            cursor.leave(node)
+          end
+        end
+      end
+
       # Iterate through the file, yielding each 'xml_record_xpath' element in turn.
       def rows(&block)
         return enum_for(:rows) unless block
 
         if @options['slurp']
-          record_elements(read_xml_file(@filename)).each(&block)
+          record_elements.each(&block)
         else
-          each_node(@filename, xml_record_xpath, @pattern_match_xpath, &block)
+          @stream.rewind
+          each_node(@stream, @encoding, xml_record_xpath, @pattern_match_xpath, &block)
         end
       end
 
@@ -35,13 +90,13 @@ module NdrImport
         @pattern_match_xpath ? @options['xml_record_xpath'] : "*/#{@options['xml_record_xpath']}"
       end
 
-      def record_elements(doc)
+      def record_elements
         if @pattern_match_xpath
-          doc.root.children.find_all do |element|
+          @doc.root.children.find_all do |element|
             element.name =~ Regexp.new(@options['xml_record_xpath'])
           end
         else
-          doc.root.xpath(@options['xml_record_xpath'])
+          @doc.root.xpath(@options['xml_record_xpath'])
         end
       end
     end

data/lib/ndr_import/helpers/file/xml_streaming.rb

@@ -78,6 +78,10 @@ module NdrImport
             match
           end
 
+          def inner_text
+            dom_stubs[@stack].xpath(@xpath)&.inner_text
+          end
+
           private
 
           def in_empty_element?
@@ -134,21 +138,18 @@ module NdrImport
 
         include UTF8Encoding
 
-        # Streams the contents of the given `safe_path`, and yields
-        # each element matching `xpath` as they're found.
+        # Yields each element matching `xpath` from `stream` as they're found.
         #
         # In the case of dodgy encoding, may fall back to slurping the
         # file, but will still use stream parsing for XML.
         #
         # Optionally pattern match the xpath
-        def each_node(safe_path, xpath, pattern_match_xpath = nil, &block)
-          return enum_for(:each_node, safe_path, xpath, pattern_match_xpath) unless block
+        def each_node(stream, encoding, xpath, pattern_match_xpath = nil, &block)
+          return enum_for(:each_node, stream, encoding, xpath, pattern_match_xpath) unless block
 
           require 'nokogiri'
 
-          with_encoding_check(safe_path) do |stream, encoding|
-            stream_xml_nodes(stream, xpath, pattern_match_xpath, encoding, &block)
-          end
+          stream_xml_nodes(stream, xpath, pattern_match_xpath, encoding, &block)
         end
 
         private

data/lib/ndr_import/table.rb

@@ -21,7 +21,7 @@ module NdrImport
     end
 
     attr_reader(*all_valid_options)
-    attr_accessor :notifier
+    attr_accessor :notifier, :table_metadata
 
     def initialize(options = {})
       options.stringify_keys! if options.is_a?(Hash)

data/lib/ndr_import/universal_importer_helper.rb

@@ -59,7 +59,8 @@ module NdrImport
                     'xml_record_xpath'           => table_mapping.try(:xml_record_xpath),
                     'slurp'                      => table_mapping.try(:slurp),
                     'yield_xml_record'           => table_mapping.try(:yield_xml_record),
-                    'pattern_match_record_xpath' => table_mapping.try(:pattern_match_record_xpath) }
+                    'pattern_match_record_xpath' => table_mapping.try(:pattern_match_record_xpath),
+                    'xml_file_metadata'          => table_mapping.try(:xml_file_metadata) }
 
         tables = NdrImport::File::Registry.tables(filename, table_mapping.try(:format), options)
         yield_tables_and_their_content(filename, tables, &block)
@@ -71,12 +72,12 @@ module NdrImport
     def yield_tables_and_their_content(filename, tables, &block)
       return enum_for(:yield_tables_and_their_content, filename, tables) unless block_given?
 
-      tables.each do |tablename, table_content|
+      tables.each do |tablename, table_content, file_metadata|
         mapping = get_table_mapping(filename, tablename)
         next if mapping.nil?
 
         mapping.notifier = get_notifier(record_total(filename, table_content))
-
+        mapping.table_metadata = file_metadata || {}
         yield(mapping, table_content)
       end
     end

data/lib/ndr_import/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 # This stores the current version of the NdrImport gem
 module NdrImport
-  VERSION = '11.1.0'
+  VERSION = '11.2.0'
 end

data/lib/ndr_import/xml/table.rb

@@ -10,7 +10,8 @@ module NdrImport
       require 'ndr_import/xml/column_mapping'
       require 'ndr_import/xml/masked_mappings'
 
-      XML_OPTIONS = %w[pattern_match_record_xpath xml_record_xpath yield_xml_record].freeze
+      XML_OPTIONS = %w[pattern_match_record_xpath xml_file_metadata xml_record_xpath
+                       yield_xml_record].freeze
 
       def self.all_valid_options
         super - %w[delimiter header_lines footer_lines] + XML_OPTIONS

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ndr_import
 version: !ruby/object:Gem::Version
-  version: 11.1.0
+  version: 11.2.0
 platform: ruby
 authors:
 - NCRS Development Team
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-03-07 00:00:00.000000000 Z
+date: 2024-04-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel