ndr_import 6.2.0 → 6.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 33380c2e5d1abc43cb56a040c8b6255ec6315bb9860506c173dcd573df777f11
-  data.tar.gz: fe9a51d153f9f64cdeaf872198f06887ba0c0becf9877e9a0f6cbb511160ae10
+  metadata.gz: 9ae1ff625cdd7352f4b3306f9a49905b2563557b487762611dfd0391735177ee
+  data.tar.gz: a7bdc413db721c0f813b9fd7ef862beff3cf6a9a76f042eef15d141b2526f5a7
 SHA512:
-  metadata.gz: e11257edc7d89f30f05943c9eb992f7c3660d66a15100092639840768843456e15a8b3babe16194f04387bd890f399fab58238842276e1a8c40209cfc127ddf4
-  data.tar.gz: 27f159c69d12780b967caa9c5223f8ab91a33fa6cd21b62ebaa3811ab7a8d115d388fc084a8e2501663cbb24d81a97ae4cf090bd377ae4ccd1bbdb617e0ea9ab
+  metadata.gz: 3d572905a49329c295a9ab701de99a4f82a9165a399b05ca6f80e48218e2c603fddca14768d4aad2e9139ac07cdbaa6c5ba1f1851527cc50f7f6f4b45ed2392e
+  data.tar.gz: 3414e9efa2b16b60e9d532075fca1df095bdcab20516738243fc9ed9ac5b30540b3ad1f9139feaedfd3f5c3182c049e2deb5f5cd878a67360bdefaeffd9c6452

data/code_safety.yml

@@ -15,7 +15,7 @@ file safety:
   ".travis.yml":
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 661b5b8e71572bba28aa92fb95aa218e3f8444f4
+    safe_revision: 71979e0a602ca5a0ce415c194f10add9959f0116
   CODE_OF_CONDUCT.md:
     comments: 
     reviewed_by: timgentry
@@ -58,8 +58,8 @@ file safety:
     safe_revision: c158dc783b84cab31380708e76e3812544cc1c2f
   lib/ndr_import.rb:
     comments: 
-    reviewed_by: ollie.tulloch
-    safe_revision: dfc958d44b6c58355445fa395db08a62213ee709
+    reviewed_by: josh.pencheon
+    safe_revision: 4ab72f84201c2d5f0147b7dfd041f488f6ff0422
   lib/ndr_import/csv_library.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -67,7 +67,7 @@ file safety:
   lib/ndr_import/file/all.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 1b66cfcbb61dfac93c44889ca0ced5836101c20c
+    safe_revision: 4ab72f84201c2d5f0147b7dfd041f488f6ff0422
   lib/ndr_import/file/base.rb:
     comments: 
     reviewed_by: timgentry
@@ -75,7 +75,7 @@ file safety:
   lib/ndr_import/file/delimited.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 902f5326d85372d9632de9869d6f56fc02b83a10
+    safe_revision: 71979e0a602ca5a0ce415c194f10add9959f0116
   lib/ndr_import/file/docx.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -104,6 +104,10 @@ file safety:
     comments: 
     reviewed_by: timgentry
     safe_revision: c88000b32401b5ae9ef7f5878a9b630506ab5a94
+  lib/ndr_import/file/xml.rb:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 4ab72f84201c2d5f0147b7dfd041f488f6ff0422
   lib/ndr_import/file/zip.rb:
     comments: 
     reviewed_by: timgentry
@@ -139,7 +143,7 @@ file safety:
   lib/ndr_import/mapper.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: fbd4e1bfda8acd9ae026601dc4ecdabca5be4bc7
+    safe_revision: 4ab72f84201c2d5f0147b7dfd041f488f6ff0422
   lib/ndr_import/mapping_error.rb:
     comments: 
     reviewed_by: timgentry
@@ -163,7 +167,7 @@ file safety:
   lib/ndr_import/non_tabular/table.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 337bf56e39f0f08cf7593b03867bb2da48630663
+    safe_revision: 71979e0a602ca5a0ce415c194f10add9959f0116
   lib/ndr_import/non_tabular_file_helper.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -175,19 +179,23 @@ file safety:
   lib/ndr_import/table.rb:
     comments: uses File.basename
     reviewed_by: josh.pencheon
-    safe_revision: 902f5326d85372d9632de9869d6f56fc02b83a10
+    safe_revision: 71979e0a602ca5a0ce415c194f10add9959f0116
   lib/ndr_import/universal_importer_helper.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 902f5326d85372d9632de9869d6f56fc02b83a10
+    safe_revision: 71979e0a602ca5a0ce415c194f10add9959f0116
   lib/ndr_import/version.rb:
     comments: another check?
     reviewed_by: josh.pencheon
-    safe_revision: b85d90430543f238706d569bacd750ee50cb5493
+    safe_revision: 0b1ab7c810d0fa46d153238d69627c07f56d1efa
+  lib/ndr_import/xml/table.rb:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 4ab72f84201c2d5f0147b7dfd041f488f6ff0422
   ndr_import.gemspec:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 1b66cfcbb61dfac93c44889ca0ced5836101c20c
+    safe_revision: 71979e0a602ca5a0ce415c194f10add9959f0116
   test/file/base_test.rb:
     comments: 
     reviewed_by: timgentry
@@ -195,7 +203,7 @@ file safety:
   test/file/delimited_test.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 902f5326d85372d9632de9869d6f56fc02b83a10
+    safe_revision: ab9f926a53e84f8ffa826abdd967abee455c89df
   test/file/docx_test.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -211,7 +219,7 @@ file safety:
   test/file/registry_test.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 1b66cfcbb61dfac93c44889ca0ced5836101c20c
+    safe_revision: 4ab72f84201c2d5f0147b7dfd041f488f6ff0422
   test/file/text_test.rb:
     comments: 
     reviewed_by: timgentry
@@ -220,6 +228,10 @@ file safety:
     comments: 
     reviewed_by: josh.pencheon
     safe_revision: 1b66cfcbb61dfac93c44889ca0ced5836101c20c
+  test/file/xml_test.rb:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 4ab72f84201c2d5f0147b7dfd041f488f6ff0422
   test/file/zip_test.rb:
     comments: 
     reviewed_by: timgentry
@@ -255,7 +267,7 @@ file safety:
   test/mapper_test.rb:
     comments: exposes Mapper internals to test them
     reviewed_by: josh.pencheon
-    safe_revision: 3cea78767a73fd0d0ae64a4af3f07389e45349e9
+    safe_revision: 4ab72f84201c2d5f0147b7dfd041f488f6ff0422
   test/non_tabular/mapping_test.rb:
     comments: 
     reviewed_by: timgentry
@@ -340,10 +352,18 @@ file safety:
     comments: 
     reviewed_by: josh.pencheon
     safe_revision: 902f5326d85372d9632de9869d6f56fc02b83a10
+  test/resources/malformed.csv:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 71979e0a602ca5a0ce415c194f10add9959f0116
   test/resources/malformed.xml:
     comments: 
     reviewed_by: timgentry
     safe_revision: 137170d443ea6bcc0afb18f62202c285ae6501eb
+  test/resources/malformed_pipe.csv:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 71979e0a602ca5a0ce415c194f10add9959f0116
   test/resources/normal.csv:
     comments: 
     reviewed_by: timgentry
@@ -376,6 +396,10 @@ file safety:
     comments: 
     reviewed_by: josh.pencheon
     safe_revision: 902f5326d85372d9632de9869d6f56fc02b83a10
+  test/resources/sample.xml:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 4ab72f84201c2d5f0147b7dfd041f488f6ff0422
   test/resources/sample_xls.xls:
     comments: 
     reviewed_by: timgentry
@@ -448,3 +472,7 @@ file safety:
     comments: 
     reviewed_by: josh.pencheon
     safe_revision: 902f5326d85372d9632de9869d6f56fc02b83a10
+  test/xml/table_test.rb:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 4ab72f84201c2d5f0147b7dfd041f488f6ff0422

data/lib/ndr_import.rb

@@ -6,6 +6,7 @@ require 'ndr_import/non_tabular_file_helper'
 require 'ndr_import/table'
 require 'ndr_import/non_tabular/table'
 require 'ndr_import/fixed_width/table'
+require 'ndr_import/xml/table'
 
 module NdrImport
   def self.root

data/lib/ndr_import/file/all.rb

@@ -6,5 +6,5 @@ require_relative 'pdf'
 require_relative 'text'
 require_relative 'unregistered_filetype'
 require_relative 'word'
-# # require_relative 'xml'
+require_relative 'xml'
 require_relative 'zip'

data/lib/ndr_import/file/delimited.rb

@@ -52,8 +52,9 @@ module NdrImport
           begin
             col_sep = @options['col_sep']
             options = {
-              :col_sep  => (col_sep || ',').force_encoding(delimiter_encoding),
-              :mode     => access_mode
+              col_sep:         (col_sep || ',').force_encoding(delimiter_encoding),
+              mode:            access_mode,
+              liberal_parsing: @options['liberal_parsing'].presence
             }
 
             row_num = 0
@@ -76,7 +77,7 @@ module NdrImport
 
         # We tried them all, and none worked:
         unless successful_options
-          fail "None of the encodings #{supported_encodings.values.inspect} were successful!"
+          raise "None of the encodings #{supported_encodings.values.inspect} were successful!"
         end
 
         successful_options

data/lib/ndr_import/file/xml.rb

@@ -0,0 +1,29 @@
+require 'ndr_support/safe_file'
+require 'ndr_import/helpers/file/xml'
+require_relative 'registry'
+
+module NdrImport
+  # This is one of a collection of file handlers that deal with individual formats of data.
+  # They can be instantiated directly or via the factory method Registry.tables
+  module File
+    # This class is a xml file handler that returns a single table.
+    class Xml < Base
+      include NdrImport::Helpers::File::Xml
+
+      private
+
+      # Iterate through the file, yielding each 'xml_record_xpath' element in turn.
+      def rows(&block)
+        return enum_for(:rows) unless block
+
+        doc = read_xml_file(@filename)
+
+        doc.xpath(@options['xml_record_xpath']).each(&block)
+      rescue StandardError => e
+        raise("#{SafeFile.basename(@filename)} [#{e.class}: #{e.message}]")
+      end
+    end
+    # Not all xml files may want to be registered, so 'xml' is not registered by design.
+    Registry.register(Xml, 'xml_table')
+  end
+end

data/lib/ndr_import/mapper.rb

@@ -231,13 +231,14 @@ module NdrImport::Mapper
   #
   # would base64 decode a word document and then 'decode' the word document into plain text
   def decode_raw_value(raw_value, encoding)
+    return raw_value if raw_value.blank?
     case encoding
     when :base64
       Base64.decode64(raw_value)
     when :word_doc
       read_word_stream(StringIO.new(raw_value, 'r'))
     else
-      fail "Cannot decode: #{encoding}"
+      raise "Cannot decode: #{encoding}"
     end
   end
 

data/lib/ndr_import/non_tabular/table.rb

@@ -16,12 +16,15 @@ module NdrImport
 
       include UTF8Encoding
 
-      NON_TABULAR_OPTIONS = %w(capture_end_line capture_start_line start_line_pattern
+      TABULAR_ONLY_OPTIONS = %w[delimiter liberal_parsing tablename_pattern
+                                header_lines footer_lines xml_record_xpath].freeze
+
+      NON_TABULAR_OPTIONS = %w[capture_end_line capture_start_line start_line_pattern
                                end_line_pattern remove_lines start_in_a_record
-                               end_in_a_record).freeze
+                               end_in_a_record].freeze
 
       def self.all_valid_options
-        super - %w(delimiter tablename_pattern header_lines footer_lines) + NON_TABULAR_OPTIONS
+        super - TABULAR_ONLY_OPTIONS + NON_TABULAR_OPTIONS
       end
 
       attr_reader(*NON_TABULAR_OPTIONS)

data/lib/ndr_import/table.rb

@@ -10,8 +10,8 @@ module NdrImport
     include NdrImport::Mapper
 
     def self.all_valid_options
-      %w[canonical_name delimiter filename_pattern tablename_pattern header_lines footer_lines
-         format klass columns]
+      %w[canonical_name delimiter liberal_parsing filename_pattern tablename_pattern header_lines
+         footer_lines format klass columns xml_record_xpath]
     end
 
     def all_valid_options

data/lib/ndr_import/universal_importer_helper.rb

@@ -32,10 +32,14 @@ module NdrImport
         # now at the individual file level, can we find the table mapping?
         table_mapping = get_table_mapping(filename, nil)
 
-        tables = NdrImport::File::Registry.tables(filename,
-                                                  table_mapping.try(:format),
-                                                  'unzip_path' => unzip_path,
-                                                  'col_sep' => table_mapping.try(:delimiter))
+        options = {
+          'unzip_path'       => unzip_path,
+          'col_sep'          => table_mapping.try(:delimiter),
+          'liberal_parsing'  => table_mapping.try(:liberal_parsing),
+          'xml_record_xpath' => table_mapping.try(:xml_record_xpath)
+        }
+
+        tables = NdrImport::File::Registry.tables(filename, table_mapping.try(:format), options)
         yield_tables_and_their_content(filename, tables, &block)
       end
     end

data/lib/ndr_import/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 # This stores the current version of the NdrImport gem
 module NdrImport
-  VERSION = '6.2.0'.freeze
+  VERSION = '6.3.0'.freeze
 end

data/lib/ndr_import/xml/table.rb

@@ -0,0 +1,90 @@
+require 'ndr_import/table'
+
+module NdrImport
+  module Xml
+    # This class maintains the state of a xml table mapping and encapsulates
+    # the logic required to transform a table of data into "records". Particular
+    # attention has been made to use enumerables throughout to help with the
+    # transformation of large quantities of data.
+    class Table < ::NdrImport::Table
+      def self.all_valid_options
+        super - %w[delimiter header_lines footer_lines]
+      end
+
+      def header_lines
+        0
+      end
+
+      def footer_lines
+        0
+      end
+
+      # This method transforms an incoming line (element) of xml data by applying
+      # each of the klass masked mappings to the line and yielding the klass
+      # and fields for each mapped klass.
+      def transform_line(line, index)
+        return enum_for(:transform_line, line, index) unless block_given?
+
+        raise 'Not an Nokogiri::XML::Element!' unless line.is_a? Nokogiri::XML::Element
+
+        validate_column_mappings(line)
+
+        xml_line = column_xpaths.map { |column_xpath| line.xpath(column_xpath).inner_text }
+
+        masked_mappings.each do |klass, klass_mappings|
+          fields = mapped_line(xml_line, klass_mappings)
+          next if fields[:skip].to_s == 'true'.freeze
+          yield(klass, fields, index)
+        end
+      end
+
+      private
+
+      # Ensure every leaf is accounted for in the column mappings
+      def validate_column_mappings(line)
+        missing_nodes = mappable_xpaths_from(line) - column_xpaths
+        raise "Unmapped data! #{missing_nodes}" unless missing_nodes.empty?
+      end
+
+      def column_name_from(column)
+        column[Strings::COLUMN] || column[Strings::STANDARD_MAPPING]
+      end
+
+      def column_xpaths
+        @column_xpaths ||= columns.map { |column| build_xpath_from(column) }
+      end
+
+      def mappable_xpaths_from(line)
+        xpaths = []
+
+        line.xpath('.//*[not(child::*)]').each do |node|
+          xpath = node.path.sub(line.path + '/', '')
+          xpaths << xpath
+          node.attributes.each_key { |key| xpaths << "#{xpath}/@#{key}" }
+        end
+        xpaths
+      end
+
+      def build_xpath_from(column)
+        column_name = column_name_from(column)
+        column['xml_cell'].presence ? relative_path_from(column, column_name) : column_name
+      end
+
+      def relative_path_from(column, colum_name)
+        xml_cell      = column['xml_cell']
+        relative_path = xml_cell['relative_path'].presence ? xml_cell['relative_path'] : nil
+        attribute     = xml_cell['attribute'].presence ? '@' + xml_cell['attribute'] : nil
+
+        if relative_path && attribute
+          relative_path + '/' + colum_name + '/' + attribute
+        elsif relative_path
+          relative_path + '/' + colum_name
+        elsif attribute
+          colum_name + '/' + attribute
+        else
+          colum_name
+        end
+      end
+    end
+  end
+end

data/ndr_import.gemspec

@@ -33,7 +33,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'roo-xls'
   spec.add_dependency 'spreadsheet', '1.0.3'
 
-  spec.required_ruby_version = '>= 2.2'
+  spec.required_ruby_version = '>= 2.4'
 
   spec.add_development_dependency 'bundler', '~> 1.7'
   spec.add_development_dependency 'rake', '~> 10.0'

data/test/file/delimited_test.rb

@@ -33,6 +33,21 @@ module NdrImport
         end
       end
 
+      test 'should read malformed pipe correctly' do
+        file_path = @permanent_test_files.join('malformed_pipe.csv')
+        handler = NdrImport::File::Delimited.new(file_path, 'delimited', 'col_sep' => '|',
+                                                                         'liberal_parsing' => 'true')
+        handler.tables.each do |tablename, sheet|
+          assert_nil tablename
+          sheet = sheet.to_a
+          assert_equal(('A'..'Z').to_a, sheet[0])
+          assert_equal ['1'] * 26, sheet[1]
+          expected_row = ['2'] * 25
+          expected_row << '2"malformed"'
+          assert_equal expected_row, sheet[2].sort
+        end
+      end
+
       test 'should read thorn correctly' do
         file_path = @permanent_test_files.join('normal_thorn.csv')
         handler = NdrImport::File::Delimited.new(file_path, 'delimited', 'col_sep' => "\xfe")
@@ -99,6 +114,47 @@ module NdrImport
         assert_equal ['2'] * 26, rows[2]
       end
 
+      test 'should read malformed delimited txt' do
+        rows = []
+        file_path = @permanent_test_files.join('malformed.csv')
+        handler = NdrImport::File::Delimited.new(file_path, 'csv', 'col_sep' => nil,
+                                                                   'liberal_parsing' => 'true')
+        handler.tables.each do |tablename, sheet|
+          assert_nil tablename
+          assert_instance_of Enumerator, sheet
+          sheet.each do |row|
+            rows << row
+          end
+        end
+
+        assert_equal(('A'..'Z').to_a, rows[0])
+        assert_equal ['1'] * 26, rows[1]
+        expected_row = ['2'] * 25
+        expected_row << '2"malformed"'
+        assert_equal expected_row, rows[2].sort
+      end
+
+      test 'should fail to read malformed delimited txt without liberal_parsing' do
+        rows_yielded = []
+        exception    = assert_raises(CSVLibrary::MalformedCSVError) do
+          file_path = @permanent_test_files.join('malformed.csv')
+          handler = NdrImport::File::Delimited.new(file_path, 'csv')
+
+          handler.tables.each do |tablename, sheet|
+            assert_nil tablename
+            assert_instance_of Enumerator, sheet
+            sheet.each do |row|
+              rows_yielded << row
+            end
+          end
+        end
+
+        assert rows_yielded.empty?, 'no rows should have been yielded'
+
+        msg = 'Invalid CSV format on row 3 of malformed.csv. Original: Illegal quoting in line 3.'
+        assert_equal msg, exception.message
+      end
+
       test 'should read line-by-line' do
         rows = []
         file_path = @permanent_test_files.join('normal.csv')

data/test/file/registry_test.rb

@@ -12,7 +12,7 @@ module NdrImport
 
       test 'Registry.handlers' do
         assert_instance_of Hash, NdrImport::File::Registry.handlers
-        assert_equal %w[csv delimited doc docx nontabular pdf text txt xls xlsx zip],
+        assert_equal %w[csv delimited doc docx nontabular pdf text txt xls xlsx xml_table zip],
                      NdrImport::File::Registry.handlers.keys.sort
       end
 

data/test/file/xml_test.rb

@@ -0,0 +1,21 @@
+require 'test_helper'
+require 'ndr_import/file/xml'
+
+module NdrImport
+  module File
+    # Xml file handler tests
+    class XmlTest < ActiveSupport::TestCase
+      def setup
+        @permanent_test_files = SafePath.new('permanent_test_files')
+      end
+
+      test 'should return enum of xml elements' do
+        file_path = @permanent_test_files.join('sample.xml')
+        handler   = NdrImport::File::Xml.new(file_path, nil, 'xml_record_xpath' => 'root/record')
+        rows      = handler.send(:rows)
+        assert rows.is_a? Enumerator
+        assert(rows.all? { |row| row.is_a? Nokogiri::XML::Element })
+      end
+    end
+  end
+end

data/test/mapper_test.rb

@@ -635,6 +635,11 @@ class MapperTest < ActiveSupport::TestCase
     assert_equal 'Hello world, this is a word document', file_content
   end
 
+  test 'should handle blank values when attempting to decode_raw_value' do
+    text_content = TestMapper.new.send(:decode_raw_value, '', :word_doc)
+    assert_equal '', text_content
+  end
+
   test 'should raise unknown encoding exception' do
     assert_raise(RuntimeError) do
       TestMapper.new.mapped_line(['A'], invalid_decode_mapping)

data/test/resources/malformed.csv

@@ -0,0 +1,3 @@
+A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S,T,U,V,W,X,Y,Z
+1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1
+2,2,2,2,2,2,2,2"malformed",2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2

data/test/resources/malformed_pipe.csv

@@ -0,0 +1,3 @@
+A|B|C|D|E|F|G|H|I|J|K|L|M|N|O|P|Q|R|S|T|U|V|W|X|Y|Z
+1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1
+2|2|2|2|2|2|2|2|2|2|2|2|2|2|2|2|2"malformed"|2|2|2|2|2|2|2|2|2

data/test/resources/sample.xml

@@ -0,0 +1,34 @@
+<root>
+  <record>
+    <no_relative_path value="A value"/>
+    <no_path_or_att>Another value</no_path_or_att>
+    <demographics>
+      <demographics_1>AAA</demographics_1>
+      <address>
+        <address_line1>Address</address_line1>
+        <address_line1>Address 2</address_line1>
+      </address>
+      <demographics_2 code="03">Inner text</demographics_2>
+    </demographics>
+    <pathology>
+      <pathology_date_1>2018-01-01</pathology_date_1>
+      <pathology_date_2 />
+    </pathology>
+  </record>
+  <record>
+    <demographics>
+      <address>
+        <address_line1>Address</address_line1>
+        <address_line1>Address 2</address_line1>
+      </address>
+      <demographics_2 code="03">Inner text</demographics_2>
+      <demographics_1>AAA</demographics_1>
+    </demographics>
+    <no_path_or_att><![CDATA[Another value]]></no_path_or_att>
+    <pathology>
+      <pathology_date_1>2018-01-01</pathology_date_1>
+      <pathology_date_2 />
+    </pathology>
+    <no_relative_path value="A value"/>
+  </record>
+</root>

data/test/xml/table_test.rb

@@ -0,0 +1,90 @@
+require 'test_helper'
+
+# This tests the NdrImport::Xml::Table mapping class
+module Xml
+  class TableTest < ActiveSupport::TestCase
+    def setup
+      file_path = SafePath.new('permanent_test_files').join('sample.xml')
+      handler   = NdrImport::File::Xml.new(file_path, nil, 'xml_record_xpath' => 'root/record')
+
+      @element_lines = handler.send(:rows)
+    end
+
+    def test_should_transform_xml_element_lines
+      table = NdrImport::Xml::Table.new(klass: 'SomeTestKlass', columns: xml_column_mapping)
+
+      expected_data = ['SomeTestKlass', { rawtext: {
+        'no_relative_path'            => 'A value',
+        'no_relative_path_inner_text' => '',
+        'no_path_or_att'              => 'Another value',
+        'demographics_1'              => 'AAA',
+        'demographics_2'              => '03',
+        'demographics_2_inner_text'   => 'Inner text',
+        'address1'                    => 'Address',
+        'address2'                    => 'Address 2',
+        'pathology_date_1'            => '2018-01-01',
+        'pathology_date_2'            => '',
+        'should_be_blank'             => ''
+      } }, 1]
+
+      transformed_data = table.transform(@element_lines)
+      assert_equal 2, transformed_data.count
+
+      transformed_data.each do |klass, fields, _index|
+        assert_equal expected_data[0], klass
+        assert_equal expected_data[1], fields
+      end
+    end
+
+    def test_should_fail_with_unmappped_nodes
+      table = NdrImport::Xml::Table.new(klass: 'SomeTestKlass', columns: partial_xml_column_mapping)
+
+      exception = assert_raises(RuntimeError) { table.transform(@element_lines).to_a }
+      assert exception.message.starts_with? 'sample.xml [RuntimeError: Unmapped data!'
+    end
+
+    private
+
+    def xml_column_mapping
+      [
+        { 'column' => 'no_relative_path',
+          'xml_cell' => { 'relative_path' => '', 'attribute' => 'value' } },
+        { 'column' => 'no_relative_path', 'rawtext_name' => 'no_relative_path_inner_text',
+          'xml_cell' => { 'relative_path' => '' } },
+        { 'column' => 'no_path_or_att',
+          'xml_cell' => { 'relative_path' => '', 'attribute' => '' } },
+        { 'column' => 'demographics_1',
+          'xml_cell' => { 'relative_path' => 'demographics' } },
+        { 'column' => 'demographics_2',
+          'xml_cell' => { 'relative_path' => 'demographics', 'attribute' => 'code' } },
+        { 'column' => 'demographics_2', 'rawtext_name' => 'demographics_2_inner_text',
+          'xml_cell' => { 'relative_path' => 'demographics' } },
+        { 'column' => 'address_line1[1]', 'rawtext_name' => 'address1',
+          'xml_cell' => { 'relative_path' => 'demographics/address' } },
+        { 'column' => 'address_line1[2]', 'rawtext_name' => 'address2',
+          'xml_cell' => { 'relative_path' => 'demographics/address' } },
+        { 'column' => 'pathology_date_1',
+          'xml_cell' => { 'relative_path' => 'pathology' } },
+        { 'column' => 'pathology_date_2',
+          'xml_cell' => { 'relative_path' => 'pathology' } },
+        { 'column' => 'should_be_blank',
+          'xml_cell' => { 'relative_path' => 'not_present' } }
+      ]
+    end
+
+    def partial_xml_column_mapping
+      [
+        { 'column' => 'no_relative_path',
+          'xml_cell' => { 'relative_path' => '', 'attribute' => 'value' } },
+        { 'column' => 'no_path_or_att',
+          'xml_cell' => { 'relative_path' => '', 'attribute' => '' } },
+        { 'column' => 'demographics_1',
+          'xml_cell' => { 'relative_path' => 'demographics' } },
+        { 'column' => 'demographics_2',
+          'xml_cell' => { 'relative_path' => 'demographics', 'attribute' => 'code' } },
+        { 'column' => 'address_line1',
+          'xml_cell' => { 'relative_path' => 'demographics/address' } }
+      ]
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ndr_import
 version: !ruby/object:Gem::Version
-  version: 6.2.0
+  version: 6.3.0
 platform: ruby
 authors:
 - NCRS Development Team
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-09-24 00:00:00.000000000 Z
+date: 2018-10-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -352,6 +352,7 @@ files:
 - lib/ndr_import/file/text.rb
 - lib/ndr_import/file/unregistered_filetype.rb
 - lib/ndr_import/file/word.rb
+- lib/ndr_import/file/xml.rb
 - lib/ndr_import/file/zip.rb
 - lib/ndr_import/fixed_width/table.rb
 - lib/ndr_import/helpers/file/delimited.rb
@@ -372,6 +373,7 @@ files:
 - lib/ndr_import/table.rb
 - lib/ndr_import/universal_importer_helper.rb
 - lib/ndr_import/version.rb
+- lib/ndr_import/xml/table.rb
 - ndr_import.gemspec
 - test/file/base_test.rb
 - test/file/delimited_test.rb
@@ -381,6 +383,7 @@ files:
 - test/file/registry_test.rb
 - test/file/text_test.rb
 - test/file/word_test.rb
+- test/file/xml_test.rb
 - test/file/zip_test.rb
 - test/fixed_width/table_test.rb
 - test/helpers/file/delimited_test.rb
@@ -411,7 +414,9 @@ files:
 - test/resources/hello_world.txt
 - test/resources/high_ascii_delimited.txt
 - test/resources/high_ascii_delimited_example_two.txt
+- test/resources/malformed.csv
 - test/resources/malformed.xml
+- test/resources/malformed_pipe.csv
 - test/resources/normal.csv
 - test/resources/normal.csv.zip
 - test/resources/normal_pipe.csv
@@ -420,6 +425,7 @@ files:
 - test/resources/not_a_word_file.doc
 - test/resources/not_a_word_file.docx
 - test/resources/not_sign_delimited.txt
+- test/resources/sample.xml
 - test/resources/sample_xls.xls
 - test/resources/sample_xlsx.xlsx
 - test/resources/sheet_streaming.xls
@@ -438,6 +444,7 @@ files:
 - test/table_test.rb
 - test/test_helper.rb
 - test/universal_importer_helper_test.rb
+- test/xml/table_test.rb
 homepage: https://github.com/PublicHealthEngland/ndr_import
 licenses:
 - MIT
@@ -450,7 +457,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.2'
+      version: '2.4'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -471,6 +478,7 @@ test_files:
 - test/file/registry_test.rb
 - test/file/text_test.rb
 - test/file/word_test.rb
+- test/file/xml_test.rb
 - test/file/zip_test.rb
 - test/fixed_width/table_test.rb
 - test/helpers/file/delimited_test.rb
@@ -501,7 +509,9 @@ test_files:
 - test/resources/hello_world.txt
 - test/resources/high_ascii_delimited.txt
 - test/resources/high_ascii_delimited_example_two.txt
+- test/resources/malformed.csv
 - test/resources/malformed.xml
+- test/resources/malformed_pipe.csv
 - test/resources/normal.csv
 - test/resources/normal.csv.zip
 - test/resources/normal_pipe.csv
@@ -510,6 +520,7 @@ test_files:
 - test/resources/not_a_word_file.doc
 - test/resources/not_a_word_file.docx
 - test/resources/not_sign_delimited.txt
+- test/resources/sample.xml
 - test/resources/sample_xls.xls
 - test/resources/sample_xlsx.xlsx
 - test/resources/sheet_streaming.xls
@@ -528,3 +539,4 @@ test_files:
 - test/table_test.rb
 - test/test_helper.rb
 - test/universal_importer_helper_test.rb
+- test/xml/table_test.rb