ndr_import 3.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +15 -0
- data/.gitignore +14 -0
- data/.rubocop.yml +27 -0
- data/.ruby-version +1 -0
- data/.travis.yml +22 -0
- data/CODE_OF_CONDUCT.md +13 -0
- data/Gemfile +4 -0
- data/Guardfile +16 -0
- data/LICENSE.txt +21 -0
- data/README.md +69 -0
- data/Rakefile +13 -0
- data/code_safety.yml +374 -0
- data/gemfiles/Gemfile.rails32 +5 -0
- data/gemfiles/Gemfile.rails32.lock +142 -0
- data/gemfiles/Gemfile.rails41 +5 -0
- data/gemfiles/Gemfile.rails41.lock +145 -0
- data/gemfiles/Gemfile.rails42 +5 -0
- data/gemfiles/Gemfile.rails42.lock +145 -0
- data/lib/ndr_import.rb +13 -0
- data/lib/ndr_import/csv_library.rb +40 -0
- data/lib/ndr_import/file/all.rb +8 -0
- data/lib/ndr_import/file/base.rb +76 -0
- data/lib/ndr_import/file/delimited.rb +86 -0
- data/lib/ndr_import/file/excel.rb +131 -0
- data/lib/ndr_import/file/pdf.rb +38 -0
- data/lib/ndr_import/file/registry.rb +50 -0
- data/lib/ndr_import/file/text.rb +52 -0
- data/lib/ndr_import/file/word.rb +30 -0
- data/lib/ndr_import/file/zip.rb +67 -0
- data/lib/ndr_import/helpers/file/delimited.rb +105 -0
- data/lib/ndr_import/helpers/file/excel.rb +181 -0
- data/lib/ndr_import/helpers/file/pdf.rb +29 -0
- data/lib/ndr_import/helpers/file/word.rb +27 -0
- data/lib/ndr_import/helpers/file/xml.rb +45 -0
- data/lib/ndr_import/helpers/file/zip.rb +44 -0
- data/lib/ndr_import/mapper.rb +220 -0
- data/lib/ndr_import/mapping_error.rb +5 -0
- data/lib/ndr_import/non_tabular/column_mapping.rb +73 -0
- data/lib/ndr_import/non_tabular/line.rb +46 -0
- data/lib/ndr_import/non_tabular/mapping.rb +35 -0
- data/lib/ndr_import/non_tabular/record.rb +99 -0
- data/lib/ndr_import/non_tabular/table.rb +193 -0
- data/lib/ndr_import/non_tabular_file_helper.rb +160 -0
- data/lib/ndr_import/standard_mappings.rb +23 -0
- data/lib/ndr_import/table.rb +179 -0
- data/lib/ndr_import/version.rb +4 -0
- data/ndr_import.gemspec +44 -0
- data/test/file/base_test.rb +54 -0
- data/test/file/delimited_test.rb +143 -0
- data/test/file/excel_test.rb +85 -0
- data/test/file/pdf_test.rb +35 -0
- data/test/file/registry_test.rb +60 -0
- data/test/file/text_test.rb +92 -0
- data/test/file/word_test.rb +35 -0
- data/test/file/zip_test.rb +47 -0
- data/test/helpers/file/delimited_test.rb +113 -0
- data/test/helpers/file/excel_test.rb +97 -0
- data/test/helpers/file/pdf_test.rb +26 -0
- data/test/helpers/file/word_test.rb +26 -0
- data/test/helpers/file/xml_test.rb +131 -0
- data/test/helpers/file/zip_test.rb +75 -0
- data/test/mapper_test.rb +551 -0
- data/test/non_tabular/mapping_test.rb +36 -0
- data/test/non_tabular/table_test.rb +510 -0
- data/test/non_tabular_file_helper_test.rb +501 -0
- data/test/readme_test.rb +53 -0
- data/test/resources/bomd.csv +3 -0
- data/test/resources/broken.csv +3 -0
- data/test/resources/filesystem_paths.yml +26 -0
- data/test/resources/flat_file.pdf +0 -0
- data/test/resources/flat_file.txt +27 -0
- data/test/resources/flat_file.yml +20 -0
- data/test/resources/hello_utf16be.txt +0 -0
- data/test/resources/hello_utf16le.txt +0 -0
- data/test/resources/hello_utf8.txt +2 -0
- data/test/resources/hello_windows.txt +2 -0
- data/test/resources/hello_world.doc +0 -0
- data/test/resources/hello_world.pdf +0 -0
- data/test/resources/hello_world.txt +2 -0
- data/test/resources/high_ascii_delimited.txt +2 -0
- data/test/resources/malformed.xml +6 -0
- data/test/resources/normal.csv +3 -0
- data/test/resources/normal.csv.zip +0 -0
- data/test/resources/normal_pipe.csv +3 -0
- data/test/resources/normal_thorn.csv +3 -0
- data/test/resources/not_a_pdf.pdf +0 -0
- data/test/resources/not_a_word_file.doc +0 -0
- data/test/resources/sample_xls.xls +0 -0
- data/test/resources/sample_xlsx.xlsx +0 -0
- data/test/resources/standard_mappings.yml +39 -0
- data/test/resources/txt_file_xls_extension.xls +1 -0
- data/test/resources/txt_file_xlsx_extension.xlsx +1 -0
- data/test/resources/utf-16be_xml.xml +0 -0
- data/test/resources/utf-16be_xml_with_declaration.xml +0 -0
- data/test/resources/utf-16le_xml.xml +0 -0
- data/test/resources/utf-8_xml.xml +9 -0
- data/test/resources/windows-1252_xml.xml +9 -0
- data/test/resources/windows.csv +5 -0
- data/test/resources/xlsx_file_xls_extension.xls +0 -0
- data/test/standard_mappings_test.rb +22 -0
- data/test/table_test.rb +288 -0
- data/test/test_helper.rb +13 -0
- metadata +443 -0
data/test/readme_test.rb
ADDED
@@ -0,0 +1,53 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
require 'test_helper'
|
3
|
+
|
4
|
+
# This tests the README page example
|
5
|
+
class ReadmeTest < ActiveSupport::TestCase
|
6
|
+
test 'readme example' do
|
7
|
+
require 'ndr_import/non_tabular/table'
|
8
|
+
require 'ndr_import/file/registry'
|
9
|
+
|
10
|
+
unzip_path = SafePath.new('test_space_rw')
|
11
|
+
source_file = SafePath.new('permanent_test_files').join('flat_file.pdf')
|
12
|
+
options = { 'unzip_path' => unzip_path }
|
13
|
+
|
14
|
+
table = NdrImport::NonTabular::Table.new(
|
15
|
+
'start_in_a_record' => false,
|
16
|
+
'end_in_a_record' => false,
|
17
|
+
'klass' => 'SomeTestKlass',
|
18
|
+
'start_line_pattern' => /\A------\z/,
|
19
|
+
'remove_lines' => { 'footer' => [/\A== Page \d+ of \d+ ==\z/i] },
|
20
|
+
'columns' => [
|
21
|
+
{
|
22
|
+
'column' => 'one',
|
23
|
+
'non_tabular_cell' => { 'lines' => Range.new(0, -1, true), 'capture' => /^(.*)$/i }
|
24
|
+
}
|
25
|
+
]
|
26
|
+
)
|
27
|
+
|
28
|
+
# Use the Registry to enumerate over the files and their tables
|
29
|
+
files = NdrImport::File::Registry.files(source_file, options)
|
30
|
+
files.each do |filename|
|
31
|
+
tables = NdrImport::File::Registry.tables(filename, nil, options)
|
32
|
+
tables.each do |_tablename, table_content|
|
33
|
+
# Use the NonTabular::Table to tabulate the "table" contents
|
34
|
+
table.transform(table_content).each do |_klass, _fields, _index|
|
35
|
+
# Your code goes here
|
36
|
+
end
|
37
|
+
|
38
|
+
# Now we test the example
|
39
|
+
results = []
|
40
|
+
table.transform(table_content).each do |_klass, fields, _index|
|
41
|
+
results << fields[:rawtext]['one']
|
42
|
+
end
|
43
|
+
assert table.is_a?(NdrImport::NonTabular::Table)
|
44
|
+
assert_equal 4, results.count
|
45
|
+
assert results.first.start_with?('1')
|
46
|
+
assert results.last.start_with?('4')
|
47
|
+
assert results.any? { |result| result =~ /This is captured/ }
|
48
|
+
refute results.any? { |result| result =~ /This is never captured/ }
|
49
|
+
refute results.any? { |result| result =~ /== Page/ }
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
<% require 'tmpdir' %>
|
2
|
+
# This allows us different filesystem paths for different platforms
|
3
|
+
---
|
4
|
+
test_files: &test_files
|
5
|
+
root: <%= Dir.mktmpdir %>
|
6
|
+
|
7
|
+
? !ruby/regexp /.*/
|
8
|
+
:
|
9
|
+
test_space_r:
|
10
|
+
<<: *test_files
|
11
|
+
prms:
|
12
|
+
- r
|
13
|
+
test_space_w:
|
14
|
+
<<: *test_files
|
15
|
+
prms:
|
16
|
+
- w
|
17
|
+
test_space_rw:
|
18
|
+
<<: *test_files
|
19
|
+
prms:
|
20
|
+
- r
|
21
|
+
- w
|
22
|
+
permanent_test_files:
|
23
|
+
root: <%= NdrImport.root + '/test/resources' %>
|
24
|
+
prms:
|
25
|
+
- r
|
26
|
+
- w
|
Binary file
|
@@ -0,0 +1,27 @@
|
|
1
|
+
0
|
2
|
+
This is never captured
|
3
|
+
------
|
4
|
+
1
|
5
|
+
== Page 1 of 5 ==
|
6
|
+
Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut
|
7
|
+
labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris
|
8
|
+
nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit
|
9
|
+
esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt
|
10
|
+
== Page 2 of 5 ==
|
11
|
+
in culpa qui officia deserunt mollit anim id est laborum.
|
12
|
+
------
|
13
|
+
2
|
14
|
+
This is captured
|
15
|
+
Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
|
16
|
+
== Page 3 of 5 ==
|
17
|
+
------
|
18
|
+
3
|
19
|
+
Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.
|
20
|
+
------
|
21
|
+
== Page 4 of 5 ==
|
22
|
+
4
|
23
|
+
Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
|
24
|
+
------
|
25
|
+
-1
|
26
|
+
== Page 5 of 5 ==
|
27
|
+
This is never captured
|
@@ -0,0 +1,20 @@
|
|
1
|
+
--- !ruby/object:NdrImport::NonTabular::Table
|
2
|
+
# canonical_name: somename
|
3
|
+
# filename_pattern:
|
4
|
+
# format: pipe
|
5
|
+
klass: SomeTestKlass
|
6
|
+
start_line_pattern: !ruby/regexp /\A------\z/
|
7
|
+
# end_line_pattern:
|
8
|
+
start_in_a_record: false
|
9
|
+
end_in_a_record: false
|
10
|
+
remove_lines:
|
11
|
+
footer:
|
12
|
+
- !ruby/regexp /\A== Page \d+ of \d+ ==\z/i
|
13
|
+
columns:
|
14
|
+
- column: one
|
15
|
+
non_tabular_cell:
|
16
|
+
lines: !ruby/range
|
17
|
+
begin: 0
|
18
|
+
end: -1
|
19
|
+
excl: true
|
20
|
+
capture: !ruby/regexp /^(.*)$/i
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
@@ -0,0 +1,2 @@
|
|
1
|
+
345465�1234567890�Dr Bob��BLOGGS�JOE�M�05 NOV 1990�1 noddy street, anytown�CB22 3AD�IP�234534654�42�25 Jun 2015�14253�WWJ�SMITH, John�SGH�AA�LAB���Made discvery|�Dr Josh|
|
2
|
+
345465�1234567890�Dr Bob��BLOGGS�JOE�M�05 NOV 1990�1 noddy street, anytown�CB22 3AD�IP�234534654�42�25 Jun 2015�14253�WWJ�SMITH, John�SGH�AA�LAB���Made discvery|�Dr Josh|
|
Binary file
|
File without changes
|
File without changes
|
Binary file
|
Binary file
|
@@ -0,0 +1,39 @@
|
|
1
|
+
---
|
2
|
+
surname:
|
3
|
+
column: surname
|
4
|
+
rawtext_name: surname
|
5
|
+
mappings:
|
6
|
+
- field: surname
|
7
|
+
clean: :name
|
8
|
+
previoussurname:
|
9
|
+
column: previoussurname
|
10
|
+
rawtext_name: previoussurname
|
11
|
+
mappings:
|
12
|
+
- field: previoussurname
|
13
|
+
clean: :name
|
14
|
+
forenames:
|
15
|
+
column: forenames
|
16
|
+
rawtext_name: forenames
|
17
|
+
mappings:
|
18
|
+
- field: forenames
|
19
|
+
clean: :name
|
20
|
+
sex:
|
21
|
+
column: sex
|
22
|
+
rawtext_name: sex
|
23
|
+
mappings:
|
24
|
+
- field: sex
|
25
|
+
clean: :sex
|
26
|
+
nhsnumber:
|
27
|
+
column: nhsnumber
|
28
|
+
rawtext_name: nhsnumber
|
29
|
+
mappings:
|
30
|
+
- field: nhsnumber
|
31
|
+
clean: :nhsnumber
|
32
|
+
postcode:
|
33
|
+
column: postcode
|
34
|
+
rawtext_name: postcode
|
35
|
+
mappings:
|
36
|
+
- field: postcode
|
37
|
+
clean: :postcode
|
38
|
+
test:
|
39
|
+
column: standard_mapping_column_name
|
@@ -0,0 +1 @@
|
|
1
|
+
Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
|
@@ -0,0 +1 @@
|
|
1
|
+
Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
|
Binary file
|
Binary file
|
Binary file
|
@@ -0,0 +1,5 @@
|
|
1
|
+
NHS NUMBER�,PERSON BIRTH DATE,POSTCODE OF USUAL ADDRESS (AT DIAGNOSIS),PERSON FAMILY NAME,PERSON GIVEN NAME,PATIENT USUAL ADDRESS (AT DIAGNOSIS),ORGANISATION CODE (CODE OF PROVIDER),MULTIDISCIPLINARY TEAM DISCUSSION DATE (CANCER),CONSULTANT CODE (FIRST SEEN),DATE OF DIAGNOSIS (CLINICALLY AGREED),PERFORMANCE STATUS (ADULT) ,ASA SCORE (PROSTATE),SOURCE OF REFERRAL FOR OUT-PATIENTS ,PSA (DIAGNOSIS),PROSTATE BIOPSY TECHNIQUE,GLEASON GRADE (PRIMARY),GLEASON GRADE (SECONDARY),GLEASON GRADE (TERTIARY),MULTIPARAMETRIC MRI PERFORMED,T CATEGORY (FINAL PRE-TREATMENT),N CATEGORY (FINAL PRE-TREATMENT),M CATEGORY (FINAL PRE-TREATMENT),PERINEURAL INVASION,"NUMBER OF POSITIVE CORES
|
2
|
+
|
3
|
+
",TOTAL NUMBER,GREATEST PERCENTAGE OF CANCER IN SINGLE MOST INVOLVED CORE,"SPECIALIST REFERRAL
|
4
|
+
APPOINTMENT
|
5
|
+
",ORGANISATION SITE CODE (PROVIDER TREATMENT START DATE (CANCER) ,CONSULTANT CODE (TREATMENT),TYPE OF RADICAL PROSTATECTOMY (ACTUAL),PROCEDURE DATE,PROCEDURE � NERVE SPARING,T CATEGORY (PATHOLOGICAL),N CATEGORY (PATHOLOGICAL),ORGAN CONFINED,SEMINAL VESICLES INVASION,RADICAL PROSTATECTOMY MARGIN STATUS,NUMBER OF NODES EXAMINED,NUMBER OF NODES POSITIVE,PLANNED RADIOTHERAPY INTENT (PROSTATE),PLANNED RADIOTHERAPY TYPE,PLANNED TYPE OF IMAGE-GUIDANCE FOR EXTERNAL BEAM RADIOTHERAPY,PLANNED RADIOTHERAPY FIELD,PLANNED BRACHYTHERAPY TYPE,PLANNED BRACHYTHERAPY TOTAL DOSE,PLANNED BRACHYTHERAPY TOTAL FRACTIONS,PLANNED DURATION OF NEOADJUVANT ANDROGEN DEPRIVATION THERAPY,PLANNED TOTAL DURATION OF ADJUVANT ANDROGEN DEPRIVATION THERAPY
|
Binary file
|
@@ -0,0 +1,22 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
require 'test_helper'
|
3
|
+
|
4
|
+
# This tests the StandardMappings configuration class
|
5
|
+
class StandardMappingsTest < ActiveSupport::TestCase
|
6
|
+
test 'should not raise exception on reconfiguring NdrImport::StandardMappings' do
|
7
|
+
NdrImport::StandardMappings.mappings =
|
8
|
+
YAML.load_file(SafePath.new('permanent_test_files').join('standard_mappings.yml'))
|
9
|
+
end
|
10
|
+
|
11
|
+
test 'should raise exception on setting non-hash standard mappings' do
|
12
|
+
assert_raise ArgumentError do
|
13
|
+
NdrImport::StandardMappings.mappings = true
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
test 'should return mappings' do
|
18
|
+
safe_path = SafePath.new('permanent_test_files').join('standard_mappings.yml')
|
19
|
+
assert_instance_of Hash, NdrImport::StandardMappings.mappings
|
20
|
+
assert_equal YAML.load_file(safe_path), NdrImport::StandardMappings.mappings
|
21
|
+
end
|
22
|
+
end
|
data/test/table_test.rb
ADDED
@@ -0,0 +1,288 @@
|
|
1
|
+
require 'test_helper'
|
2
|
+
|
3
|
+
# This tests the NdrImport::Table mapping class
|
4
|
+
class TableTest < ActiveSupport::TestCase
|
5
|
+
def test_deserialize_table
|
6
|
+
table = simple_deserialized_table
|
7
|
+
assert_instance_of NdrImport::Table, table
|
8
|
+
assert_equal 2, table.header_lines
|
9
|
+
assert_equal 1, table.footer_lines
|
10
|
+
assert_equal 'pipe', table.format
|
11
|
+
assert_equal 'SomeTestKlass', table.klass
|
12
|
+
assert_equal 'somename', table.canonical_name
|
13
|
+
assert_equal [{ 'column' => 'one' }, { 'column' => 'two' }, { 'column' => 'three' }],
|
14
|
+
table.columns
|
15
|
+
end
|
16
|
+
|
17
|
+
def test_initialize
|
18
|
+
table = NdrImport::Table.new(:header_lines => 2, :footer_lines => 1,
|
19
|
+
:format => 'pipe', :klass => 'SomeTestKlass',
|
20
|
+
:columns => [{ 'column' => 'one' }, { 'column' => 'two' }])
|
21
|
+
assert_instance_of NdrImport::Table, table
|
22
|
+
assert_equal 2, table.header_lines
|
23
|
+
assert_equal 1, table.footer_lines
|
24
|
+
assert_equal 'pipe', table.format
|
25
|
+
assert_equal 'SomeTestKlass', table.klass
|
26
|
+
assert_equal [{ 'column' => 'one' }, { 'column' => 'two' }], table.columns
|
27
|
+
end
|
28
|
+
|
29
|
+
def test_should_raise_error_on_invalid_initialization
|
30
|
+
# incorrect parameter type
|
31
|
+
assert_raises ArgumentError do
|
32
|
+
NdrImport::Table.new([])
|
33
|
+
end
|
34
|
+
# invalid option
|
35
|
+
assert_raises ArgumentError do
|
36
|
+
NdrImport::Table.new(:potato => true)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def test_match_with_no_patterns
|
41
|
+
table = NdrImport::Table.new
|
42
|
+
assert table.match('example.csv', nil)
|
43
|
+
assert table.match('example.xslx', 'Sheet1')
|
44
|
+
end
|
45
|
+
|
46
|
+
def test_match_with_only_filename_pattern
|
47
|
+
table = NdrImport::Table.new(:filename_pattern => /\.(csv|xlsx)\z/i)
|
48
|
+
assert table.match('example.csv', nil)
|
49
|
+
assert table.match('example.xlsx', 'Sheet1')
|
50
|
+
|
51
|
+
table = NdrImport::Table.new(:filename_pattern => /\Ademo\.(csv|xlsx)\z/i)
|
52
|
+
refute table.match('example.csv', nil)
|
53
|
+
refute table.match('example.xlsx', 'Sheet1')
|
54
|
+
end
|
55
|
+
|
56
|
+
def test_match_with_both_patterns
|
57
|
+
table = NdrImport::Table.new(:filename_pattern => /\.xlsx\z/i,
|
58
|
+
:tablename_pattern => /\Asheet1\z/i)
|
59
|
+
assert table.match('example.xlsx', 'Sheet1')
|
60
|
+
refute table.match('example.xlsx', 'Sheet2')
|
61
|
+
end
|
62
|
+
|
63
|
+
def test_transform
|
64
|
+
lines = [%w(ONE TWO), %w(CARROT POTATO), %w(BACON SAUSAGE)].each
|
65
|
+
table = NdrImport::Table.new(:header_lines => 1, :footer_lines => 0,
|
66
|
+
:klass => 'SomeTestKlass',
|
67
|
+
:columns => [{ 'column' => 'one' }, { 'column' => 'two' }])
|
68
|
+
|
69
|
+
output = []
|
70
|
+
table.transform(lines).each do |klass, fields, index|
|
71
|
+
output << [klass, fields, index]
|
72
|
+
end
|
73
|
+
|
74
|
+
expected_output = [
|
75
|
+
['SomeTestKlass', { :rawtext => { 'one' => 'CARROT', 'two' => 'POTATO' } }, 1],
|
76
|
+
['SomeTestKlass', { :rawtext => { 'one' => 'BACON', 'two' => 'SAUSAGE' } }, 2]
|
77
|
+
]
|
78
|
+
assert_equal expected_output, output
|
79
|
+
end
|
80
|
+
|
81
|
+
def test_process_line
|
82
|
+
# No header row, process the first line
|
83
|
+
table = NdrImport::Table.new(:header_lines => 0, :footer_lines => 0,
|
84
|
+
:klass => 'SomeTestKlass',
|
85
|
+
:columns => [{ 'column' => 'one' }, { 'column' => 'two' }])
|
86
|
+
|
87
|
+
output = []
|
88
|
+
table.process_line(%w(CARROT POTATO)).each do |klass, fields, index|
|
89
|
+
output << [klass, fields, index]
|
90
|
+
end
|
91
|
+
|
92
|
+
expected_output = [
|
93
|
+
['SomeTestKlass', { :rawtext => { 'one' => 'CARROT', 'two' => 'POTATO' } }, 0]
|
94
|
+
]
|
95
|
+
assert_equal expected_output.sort, output.sort
|
96
|
+
|
97
|
+
# One header row, don't process the first line
|
98
|
+
table = NdrImport::Table.new(:header_lines => 1, :footer_lines => 0,
|
99
|
+
:klass => 'SomeTestKlass',
|
100
|
+
:columns => [{ 'column' => 'one' }, { 'column' => 'two' }])
|
101
|
+
|
102
|
+
output = []
|
103
|
+
table.process_line(%w(CARROT POTATO)).each do |klass, fields, index|
|
104
|
+
output << [klass, fields, index]
|
105
|
+
end
|
106
|
+
|
107
|
+
assert_equal [], output
|
108
|
+
end
|
109
|
+
|
110
|
+
def test_transform_line
|
111
|
+
table = NdrImport::Table.new(:header_lines => 2, :footer_lines => 1,
|
112
|
+
:columns => column_level_klass_mapping)
|
113
|
+
enum = table.transform_line(%w(CARROT POTATO PEA), 7)
|
114
|
+
assert_instance_of Enumerator, enum
|
115
|
+
|
116
|
+
output = []
|
117
|
+
enum.each do |klass, fields, index|
|
118
|
+
output << [klass, fields, index]
|
119
|
+
end
|
120
|
+
|
121
|
+
expected_output = [
|
122
|
+
['SomeTestKlass', { :rawtext => { 'one' => 'CARROT', 'two' => 'POTATO' } }, 7],
|
123
|
+
['SomeOtherKlass', { :rawtext => { 'two' => 'POTATO', 'three' => 'PEA' } }, 7]
|
124
|
+
]
|
125
|
+
assert_equal expected_output.sort, output.sort
|
126
|
+
end
|
127
|
+
|
128
|
+
def test_encode_with
|
129
|
+
# encode_with(coder)
|
130
|
+
end
|
131
|
+
|
132
|
+
def test_skip_footer_lines
|
133
|
+
table = simple_deserialized_table
|
134
|
+
lines = (1..10).each
|
135
|
+
assert_equal((1..7).to_a, table.send(:skip_footer_lines, lines, 3).to_a)
|
136
|
+
assert_equal((1..10).to_a, table.send(:skip_footer_lines, lines, 0).to_a)
|
137
|
+
end
|
138
|
+
|
139
|
+
def test_masked_mappings
|
140
|
+
# table level
|
141
|
+
table = simple_deserialized_table
|
142
|
+
table_level_klass_masked_mappings = {
|
143
|
+
'SomeTestKlass' => [{ 'column' => 'one' }, { 'column' => 'two' }, { 'column' => 'three' }]
|
144
|
+
}
|
145
|
+
assert_equal table_level_klass_masked_mappings, table.send(:masked_mappings)
|
146
|
+
|
147
|
+
# column level
|
148
|
+
table = NdrImport::Table.new(:header_lines => 2, :footer_lines => 1,
|
149
|
+
:columns => column_level_klass_mapping)
|
150
|
+
|
151
|
+
assert_equal column_level_klass_masked_mappings, table.send(:masked_mappings)
|
152
|
+
end
|
153
|
+
|
154
|
+
def test_column_level_klass_masked_mappings
|
155
|
+
table = NdrImport::Table.new(:header_lines => 2, :footer_lines => 1,
|
156
|
+
:columns => column_level_klass_mapping)
|
157
|
+
|
158
|
+
assert_equal column_level_klass_masked_mappings,
|
159
|
+
table.send(:column_level_klass_masked_mappings)
|
160
|
+
end
|
161
|
+
|
162
|
+
def test_ensure_mappings_define_klass
|
163
|
+
table = NdrImport::Table.new(:header_lines => 2, :footer_lines => 1, :columns => [
|
164
|
+
{ 'column' => 'one', 'klass' => 'SomeTestKlass' },
|
165
|
+
{ 'column' => 'two' }
|
166
|
+
])
|
167
|
+
assert_raise(RuntimeError) { table.send(:ensure_mappings_define_klass) }
|
168
|
+
|
169
|
+
table = NdrImport::Table.new(:header_lines => 2, :footer_lines => 1, :columns => [
|
170
|
+
{ 'column' => 'one', 'klass' => 'SomeTestKlass' },
|
171
|
+
{ 'column' => 'two', 'klass' => 'SomeOtherKlass' }
|
172
|
+
])
|
173
|
+
table.send(:ensure_mappings_define_klass)
|
174
|
+
end
|
175
|
+
|
176
|
+
def test_mask_mappings_by_klass
|
177
|
+
table = NdrImport::Table.new(:header_lines => 2, :footer_lines => 1,
|
178
|
+
:columns => column_level_klass_mapping)
|
179
|
+
some_test_klass_mapping = [
|
180
|
+
{ 'column' => 'one', 'klass' => 'SomeTestKlass' },
|
181
|
+
{ 'column' => 'two', 'klass' => %w(SomeTestKlass SomeOtherKlass) },
|
182
|
+
{ 'do_not_capture' => true }
|
183
|
+
]
|
184
|
+
assert_equal some_test_klass_mapping,
|
185
|
+
table.send(:mask_mappings_by_klass, 'SomeTestKlass')
|
186
|
+
|
187
|
+
some_other_klass_mapping = [
|
188
|
+
{ 'do_not_capture' => true },
|
189
|
+
{ 'column' => 'two', 'klass' => %w(SomeTestKlass SomeOtherKlass) },
|
190
|
+
{ 'column' => 'three', 'klass' => 'SomeOtherKlass' }
|
191
|
+
]
|
192
|
+
assert_equal some_other_klass_mapping,
|
193
|
+
table.send(:mask_mappings_by_klass, 'SomeOtherKlass')
|
194
|
+
end
|
195
|
+
|
196
|
+
def test_valid_single_line_header
|
197
|
+
lines = [
|
198
|
+
%w(ONE TWO),
|
199
|
+
%w(CARROT POTATO),
|
200
|
+
%w(BACON SAUSAGE)
|
201
|
+
].each
|
202
|
+
|
203
|
+
table = NdrImport::Table.new(:header_lines => 1, :footer_lines => 0,
|
204
|
+
:klass => 'SomeTestKlass',
|
205
|
+
:columns => [{ 'column' => 'one' }, { 'column' => 'two' }])
|
206
|
+
|
207
|
+
output = []
|
208
|
+
table.transform(lines).each do |klass, fields, index|
|
209
|
+
output << [klass, fields, index]
|
210
|
+
end
|
211
|
+
|
212
|
+
expected_output = [
|
213
|
+
['SomeTestKlass', { :rawtext => { 'one' => 'CARROT', 'two' => 'POTATO' } }, 1],
|
214
|
+
['SomeTestKlass', { :rawtext => { 'one' => 'BACON', 'two' => 'SAUSAGE' } }, 2]
|
215
|
+
]
|
216
|
+
assert table.header_valid?
|
217
|
+
assert_equal expected_output, output
|
218
|
+
end
|
219
|
+
|
220
|
+
def test_valid_multi_line_header
|
221
|
+
lines = [
|
222
|
+
%w(NOTHEADING1 NOTHEADING2),
|
223
|
+
%w(ONE TWO),
|
224
|
+
%w(DEFINITELYNOTHEADING1 DEFINITELYNOTHEADING2),
|
225
|
+
%w(CARROT POTATO),
|
226
|
+
%w(BACON SAUSAGE)
|
227
|
+
].each
|
228
|
+
|
229
|
+
table = NdrImport::Table.new(:header_lines => 3, :footer_lines => 0,
|
230
|
+
:klass => 'SomeTestKlass',
|
231
|
+
:columns => [{ 'column' => 'one' }, { 'column' => 'two' }])
|
232
|
+
|
233
|
+
output = []
|
234
|
+
table.transform(lines).each do |klass, fields, index|
|
235
|
+
output << [klass, fields, index]
|
236
|
+
end
|
237
|
+
|
238
|
+
expected_output = [
|
239
|
+
['SomeTestKlass', { :rawtext => { 'one' => 'CARROT', 'two' => 'POTATO' } }, 3],
|
240
|
+
['SomeTestKlass', { :rawtext => { 'one' => 'BACON', 'two' => 'SAUSAGE' } }, 4]
|
241
|
+
]
|
242
|
+
assert table.header_valid?
|
243
|
+
assert_equal expected_output, output
|
244
|
+
end
|
245
|
+
|
246
|
+
private
|
247
|
+
|
248
|
+
def simple_deserialized_table
|
249
|
+
Psych.load <<YML
|
250
|
+
--- !ruby/object:NdrImport::Table
|
251
|
+
canonical_name: somename
|
252
|
+
# filename_pattern: !ruby/regexp //
|
253
|
+
header_lines: 2
|
254
|
+
footer_lines: 1
|
255
|
+
format: pipe
|
256
|
+
klass: SomeTestKlass
|
257
|
+
# non_tabular_row:
|
258
|
+
# ...
|
259
|
+
columns:
|
260
|
+
- column: one
|
261
|
+
- column: two
|
262
|
+
- column: three
|
263
|
+
YML
|
264
|
+
end
|
265
|
+
|
266
|
+
def column_level_klass_mapping
|
267
|
+
[
|
268
|
+
{ 'column' => 'one', 'klass' => 'SomeTestKlass' },
|
269
|
+
{ 'column' => 'two', 'klass' => %w(SomeTestKlass SomeOtherKlass) },
|
270
|
+
{ 'column' => 'three', 'klass' => 'SomeOtherKlass' }
|
271
|
+
]
|
272
|
+
end
|
273
|
+
|
274
|
+
def column_level_klass_masked_mappings
|
275
|
+
{
|
276
|
+
'SomeTestKlass' => [
|
277
|
+
{ 'column' => 'one', 'klass' => 'SomeTestKlass' },
|
278
|
+
{ 'column' => 'two', 'klass' => %w(SomeTestKlass SomeOtherKlass) },
|
279
|
+
{ 'do_not_capture' => true }
|
280
|
+
],
|
281
|
+
'SomeOtherKlass' => [
|
282
|
+
{ 'do_not_capture' => true },
|
283
|
+
{ 'column' => 'two', 'klass' => %w(SomeTestKlass SomeOtherKlass) },
|
284
|
+
{ 'column' => 'three', 'klass' => 'SomeOtherKlass' }
|
285
|
+
]
|
286
|
+
}
|
287
|
+
end
|
288
|
+
end
|