ndr_import 8.5.0 → 8.5.1

data/test/pdf_form/table_test.rb

@@ -1,119 +0,0 @@
-require 'test_helper'
-
-# This tests the NdrImport::PdfForm::Table mapping class
-module PdfForm
-  class TableTest < ActiveSupport::TestCase
-    def setup
-      @form_data = [{
-        'address2'        => 'Address 2',
-        'should_be_blank' => '',
-        'date_2'          => '2018-12-01',
-        'date_1'          => '2018-01-01',
-        'address1'        => 'Address'
-      }]
-    end
-
-    def test_should_transform_pdf_form_hash
-      table = NdrImport::PdfForm::Table.new(klass: 'SomeTestKlass',
-                                            columns: pdf_form_column_mapping)
-
-      assert_equal 'acroform', table.format
-
-      transformed_data = table.transform(@form_data)
-      assert_equal 1, transformed_data.count
-
-      expected_data = ['SomeTestKlass', { rawtext: {
-        'address1'          => 'Address',
-        'address2'          => 'Address 2',
-        'missing_from_data' => '',
-        'date_1'            => '2018-01-01',
-        'date_2'            => '2018-12-01',
-        'should_be_blank'   => ''
-      } }, 0]
-
-      klass, fields, index = *transformed_data.first
-      assert_equal expected_data[0],  klass
-      assert_equal expected_data[1],  fields
-      assert_equal expected_data[-1], index
-    end
-
-    def test_should_transform_mutli_klass_pdf_form_hash
-      table = NdrImport::PdfForm::Table.new(columns: multi_klass_pdf_form_column_mapping)
-
-      expected_data = [
-        ['TestKlassOne',
-         { rawtext:
-           { 'address1'          => 'Address',
-             'address2'          => 'Address 2',
-             'missing_from_data' => '' } },
-         0],
-        ['TestKlassTwo',
-         { rawtext:
-           { 'address1'        => 'Address',
-             'date_1'          => '2018-01-01',
-             'date_2'          => '2018-12-01',
-             'should_be_blank' => '' } },
-         0]
-      ]
-
-      transformed_data = table.transform(@form_data).to_a
-      assert_equal 2, transformed_data.count
-
-      expected_data.each_with_index do |expected, index|
-        transformed = transformed_data[index]
-        assert_equal expected, transformed
-      end
-    end
-
-    def test_should_fail_with_unmappped_form_data
-      table = NdrImport::PdfForm::Table.new(klass: 'SomeTestKlass',
-                                            columns: partial_pdf_form_column_mapping)
-
-      exception = assert_raises(NdrImport::UnmappedDataError) { table.transform(@form_data).to_a }
-      assert exception.message == 'Unmapped data: address2 and date_1'
-    end
-
-    def test_should_not_be_valid_with_bespoke_format
-      exception = assert_raises(ArgumentError) { NdrImport::PdfForm::Table.new(format: 'a_format') }
-      exception.message == 'Unrecognised options: ["format"]'
-    end
-
-    private
-
-    def pdf_form_column_mapping
-      [
-        { 'column' => 'address1' },
-        { 'column' => 'address2' },
-        { 'column' => 'missing_from_data' },
-        { 'column' => 'date_1' },
-        { 'column' => 'date_2' },
-        { 'column' => 'should_be_blank' }
-      ]
-    end
-
-    def multi_klass_pdf_form_column_mapping
-      [
-        { 'column' => 'address1',
-          'klass'  => %w[TestKlassOne TestKlassTwo] },
-        { 'column' => 'address2',
-          'klass'  => 'TestKlassOne' },
-        { 'column' => 'missing_from_data',
-          'klass'  => 'TestKlassOne' },
-        { 'column' => 'date_1',
-          'klass'  => 'TestKlassTwo' },
-        { 'column' => 'date_2',
-          'klass'  => 'TestKlassTwo' },
-        { 'column' => 'should_be_blank',
-          'klass'  => 'TestKlassTwo' }
-      ]
-    end
-
-    def partial_pdf_form_column_mapping
-      [
-        { 'column' => 'address1' },
-        { 'column' => 'date_2' },
-        { 'column' => 'should_be_blank' }
-      ]
-    end
-  end
-end

data/test/readme_test.rb

@@ -1,53 +0,0 @@
-# encoding: UTF-8
-require 'test_helper'
-
-# This tests the README page example
-class ReadmeTest < ActiveSupport::TestCase
-  test 'readme example' do
-    require 'ndr_import/non_tabular/table'
-    require 'ndr_import/file/registry'
-
-    unzip_path = SafePath.new('test_space_rw')
-    source_file = SafePath.new('permanent_test_files').join('flat_file.pdf')
-    options = { 'unzip_path' => unzip_path }
-
-    table = NdrImport::NonTabular::Table.new(
-      'start_in_a_record' => false,
-      'end_in_a_record' => false,
-      'klass' => 'SomeTestKlass',
-      'start_line_pattern' => /\A------\z/,
-      'remove_lines' => { 'footer' => [/\A== Page \d+ of \d+ ==\z/i] },
-      'columns' => [
-        {
-          'column' => 'one',
-          'non_tabular_cell' => { 'lines' => Range.new(0, -1, true), 'capture' => /^(.*)$/i }
-        }
-      ]
-    )
-
-    # Use the Registry to enumerate over the files and their tables
-    files = NdrImport::File::Registry.files(source_file, options)
-    files.each do |filename|
-      tables = NdrImport::File::Registry.tables(filename, nil, options)
-      tables.each do |_tablename, table_content|
-        # Use the NonTabular::Table to tabulate the "table" contents
-        table.transform(table_content).each do |_klass, _fields, _index|
-          # Your code goes here
-        end
-
-        # Now we test the example
-        results = []
-        table.transform(table_content).each do |_klass, fields, _index|
-          results << fields[:rawtext]['one']
-        end
-        assert table.is_a?(NdrImport::NonTabular::Table)
-        assert_equal 4, results.count
-        assert results.first.start_with?('1')
-        assert results.last.start_with?('4')
-        assert results.any? { |result| result =~ /This is captured/ }
-        refute results.any? { |result| result =~ /This is never captured/ }
-        refute results.any? { |result| result =~ /== Page/ }
-      end
-    end
-  end
-end

data/test/resources/bomd.csv

@@ -1,3 +0,0 @@
-A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S,T,U,V,W,X,Y,Z
-1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1
-2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2

data/test/resources/broken.csv

@@ -1,3 +0,0 @@
-I,really,think
-"this","is "not" "
-going,to,work

data/test/resources/filesystem_paths.yml

@@ -1,26 +0,0 @@
-<% require 'tmpdir' %>
-# This allows us different filesystem paths for different platforms
---- 
-test_files: &test_files
-  root: <%= Dir.mktmpdir %>
-
-? !ruby/regexp /.*/
-: 
-  test_space_r:
-    <<: *test_files
-    prms:
-      - r
-  test_space_w:
-    <<: *test_files
-    prms:
-      - w
-  test_space_rw:
-    <<: *test_files
-    prms:
-      - r
-      - w
-  permanent_test_files:
-    root: <%= NdrImport.root + '/test/resources' %>
-    prms:
-      - r
-      - w

data/test/resources/flat_file.txt

@@ -1,27 +0,0 @@
-0
-This is never captured
-------
-1
-== Page 1 of 5 ==
-Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut
-labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris
-nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit
-esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt
-== Page 2 of 5 ==
-in culpa qui officia deserunt mollit anim id est laborum.
-------
-2
-This is captured
-Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
-== Page 3 of 5 ==
-------
-3
-Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.
-------
-== Page 4 of 5 ==
-4
-Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
-------
--1
-== Page 5 of 5 ==
-This is never captured

data/test/resources/flat_file.yml

@@ -1,20 +0,0 @@
---- !ruby/object:NdrImport::NonTabular::Table
-# canonical_name: somename
-# filename_pattern:
-# format: pipe
-klass: SomeTestKlass
-start_line_pattern: !ruby/regexp /\A------\z/
-# end_line_pattern:
-start_in_a_record: false
-end_in_a_record: false
-remove_lines:
-  footer:
-  - !ruby/regexp /\A== Page \d+ of \d+ ==\z/i
-columns:
-- column: one
-  non_tabular_cell:
-    lines: !ruby/range
-      begin: 0
-      end: -1
-      excl: true
-    capture: !ruby/regexp /^(.*)$/i

data/test/resources/hello_utf8.txt

@@ -1,2 +0,0 @@
-Hello world
-This is a thorny þ issue!

data/test/resources/hello_windows.txt

@@ -1,2 +0,0 @@
-Hello windows world
-This is a thorny � issue!

data/test/resources/hello_world.txt

@@ -1,2 +0,0 @@
-Hello world,
-this is a text document

data/test/resources/high_ascii_delimited.txt

@@ -1,2 +0,0 @@
-345465�1234567890�Dr Bob��BLOGGS�JOE�M�05 NOV 1990�1 noddy street, anytown�CB22 3AD�IP�234534654�42�25 Jun 2015�14253�WWJ�SMITH, John�SGH�AA�LAB���Made discvery|�Dr Josh|
-345465�1234567890�Dr Bob��BLOGGS�JOE�M�05 NOV 1990�1 noddy street, anytown�CB22 3AD�IP�234534654�42�25 Jun 2015�14253�WWJ�SMITH, John�SGH�AA�LAB���Made discvery|�Dr Josh|

data/test/resources/high_ascii_delimited_example_two.txt

@@ -1,3 +0,0 @@
-A�B�C�D�E�F�G�H�I�J�K�L�M�N�O�P�Q�R�S�T�U�V�W�X�Y�Z
-1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1
-2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2

data/test/resources/malformed.csv

@@ -1,3 +0,0 @@
-A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S,T,U,V,W,X,Y,Z
-1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1
-2,2,2,2,2,2,2,2"malformed",2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2

data/test/resources/malformed.xml

@@ -1,6 +0,0 @@
-<?xml version="1.0"?>
-<root>
-  <note><![CDATA[
-    This is 
 a note!
-  ]]></note>
-</root>

data/test/resources/malformed_pipe.csv

@@ -1,3 +0,0 @@
-A|B|C|D|E|F|G|H|I|J|K|L|M|N|O|P|Q|R|S|T|U|V|W|X|Y|Z
-1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1
-2|2|2|2|2|2|2|2|2|2|2|2|2|2|2|2|2"malformed"|2|2|2|2|2|2|2|2|2

data/test/resources/normal.csv

@@ -1,3 +0,0 @@
-A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S,T,U,V,W,X,Y,Z
-1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1
-2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2

data/test/resources/normal_pipe.csv

@@ -1,3 +0,0 @@
-A|B|C|D|E|F|G|H|I|J|K|L|M|N|O|P|Q|R|S|T|U|V|W|X|Y|Z
-1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1
-2|2|2|2|2|2|2|2|2|2|2|2|2|2|2|2|2|2|2|2|2|2|2|2|2|2

data/test/resources/normal_thorn.csv

@@ -1,3 +0,0 @@
-A�B�C�D�E�F�G�H�I�J�K�L�M�N�O�P�Q�R�S�T�U�V�W�X�Y�Z
-1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1
-2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2

data/test/resources/not_sign_delimited.txt

@@ -1,3 +0,0 @@
-one¬two¬three
-some¬data¬here
-more¬data¬here

data/test/resources/sample.xml

@@ -1,34 +0,0 @@
-<root>
-  <record>
-    <no_relative_path value="A value"/>
-    <no_path_or_att>Another value</no_path_or_att>
-    <demographics>
-      <demographics_1>AAA</demographics_1>
-      <address>
-        <address_line1>Address</address_line1>
-        <address_line1>Address 2</address_line1>
-      </address>
-      <demographics_2 code="03">Inner text</demographics_2>
-    </demographics>
-    <pathology>
-      <pathology_date_1>2018-01-01</pathology_date_1>
-      <pathology_date_2 />
-    </pathology>
-  </record>
-  <record>
-    <demographics>
-      <address>
-        <address_line1>Address</address_line1>
-        <address_line1>Address 2</address_line1>
-      </address>
-      <demographics_2 code="03">Inner text</demographics_2>
-      <demographics_1>AAA</demographics_1>
-    </demographics>
-    <no_path_or_att><![CDATA[Another value]]></no_path_or_att>
-    <pathology>
-      <pathology_date_1>2018-01-01</pathology_date_1>
-      <pathology_date_2 />
-    </pathology>
-    <no_relative_path value="A value"/>
-  </record>
-</root>

data/test/resources/standard_mappings.yml

@@ -1,39 +0,0 @@
----
-surname:
-  column: surname
-  rawtext_name: surname
-  mappings:
-  - field: surname
-    clean: :name
-previoussurname:
-  column: previoussurname
-  rawtext_name: previoussurname
-  mappings:
-  - field: previoussurname
-    clean: :name
-forenames:
-  column: forenames
-  rawtext_name: forenames
-  mappings:
-  - field: forenames
-    clean: :name
-sex:
-  column: sex
-  rawtext_name: sex
-  mappings:
-  - field: sex
-    clean: :sex
-nhsnumber:
-  column: nhsnumber
-  rawtext_name: nhsnumber
-  mappings:
-  - field: nhsnumber
-    clean: :nhsnumber
-postcode:
-  column: postcode
-  rawtext_name: postcode
-  mappings:
-  - field: postcode
-    clean: :postcode
-test:
-  column: standard_mapping_column_name

data/test/resources/txt_file_xls_extension.xls

@@ -1 +0,0 @@
-Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

data/test/resources/txt_file_xlsx_extension.xlsx

@@ -1 +0,0 @@
-Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.