ndr_import 3.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (103) hide show
  1. checksums.yaml +15 -0
  2. data/.gitignore +14 -0
  3. data/.rubocop.yml +27 -0
  4. data/.ruby-version +1 -0
  5. data/.travis.yml +22 -0
  6. data/CODE_OF_CONDUCT.md +13 -0
  7. data/Gemfile +4 -0
  8. data/Guardfile +16 -0
  9. data/LICENSE.txt +21 -0
  10. data/README.md +69 -0
  11. data/Rakefile +13 -0
  12. data/code_safety.yml +374 -0
  13. data/gemfiles/Gemfile.rails32 +5 -0
  14. data/gemfiles/Gemfile.rails32.lock +142 -0
  15. data/gemfiles/Gemfile.rails41 +5 -0
  16. data/gemfiles/Gemfile.rails41.lock +145 -0
  17. data/gemfiles/Gemfile.rails42 +5 -0
  18. data/gemfiles/Gemfile.rails42.lock +145 -0
  19. data/lib/ndr_import.rb +13 -0
  20. data/lib/ndr_import/csv_library.rb +40 -0
  21. data/lib/ndr_import/file/all.rb +8 -0
  22. data/lib/ndr_import/file/base.rb +76 -0
  23. data/lib/ndr_import/file/delimited.rb +86 -0
  24. data/lib/ndr_import/file/excel.rb +131 -0
  25. data/lib/ndr_import/file/pdf.rb +38 -0
  26. data/lib/ndr_import/file/registry.rb +50 -0
  27. data/lib/ndr_import/file/text.rb +52 -0
  28. data/lib/ndr_import/file/word.rb +30 -0
  29. data/lib/ndr_import/file/zip.rb +67 -0
  30. data/lib/ndr_import/helpers/file/delimited.rb +105 -0
  31. data/lib/ndr_import/helpers/file/excel.rb +181 -0
  32. data/lib/ndr_import/helpers/file/pdf.rb +29 -0
  33. data/lib/ndr_import/helpers/file/word.rb +27 -0
  34. data/lib/ndr_import/helpers/file/xml.rb +45 -0
  35. data/lib/ndr_import/helpers/file/zip.rb +44 -0
  36. data/lib/ndr_import/mapper.rb +220 -0
  37. data/lib/ndr_import/mapping_error.rb +5 -0
  38. data/lib/ndr_import/non_tabular/column_mapping.rb +73 -0
  39. data/lib/ndr_import/non_tabular/line.rb +46 -0
  40. data/lib/ndr_import/non_tabular/mapping.rb +35 -0
  41. data/lib/ndr_import/non_tabular/record.rb +99 -0
  42. data/lib/ndr_import/non_tabular/table.rb +193 -0
  43. data/lib/ndr_import/non_tabular_file_helper.rb +160 -0
  44. data/lib/ndr_import/standard_mappings.rb +23 -0
  45. data/lib/ndr_import/table.rb +179 -0
  46. data/lib/ndr_import/version.rb +4 -0
  47. data/ndr_import.gemspec +44 -0
  48. data/test/file/base_test.rb +54 -0
  49. data/test/file/delimited_test.rb +143 -0
  50. data/test/file/excel_test.rb +85 -0
  51. data/test/file/pdf_test.rb +35 -0
  52. data/test/file/registry_test.rb +60 -0
  53. data/test/file/text_test.rb +92 -0
  54. data/test/file/word_test.rb +35 -0
  55. data/test/file/zip_test.rb +47 -0
  56. data/test/helpers/file/delimited_test.rb +113 -0
  57. data/test/helpers/file/excel_test.rb +97 -0
  58. data/test/helpers/file/pdf_test.rb +26 -0
  59. data/test/helpers/file/word_test.rb +26 -0
  60. data/test/helpers/file/xml_test.rb +131 -0
  61. data/test/helpers/file/zip_test.rb +75 -0
  62. data/test/mapper_test.rb +551 -0
  63. data/test/non_tabular/mapping_test.rb +36 -0
  64. data/test/non_tabular/table_test.rb +510 -0
  65. data/test/non_tabular_file_helper_test.rb +501 -0
  66. data/test/readme_test.rb +53 -0
  67. data/test/resources/bomd.csv +3 -0
  68. data/test/resources/broken.csv +3 -0
  69. data/test/resources/filesystem_paths.yml +26 -0
  70. data/test/resources/flat_file.pdf +0 -0
  71. data/test/resources/flat_file.txt +27 -0
  72. data/test/resources/flat_file.yml +20 -0
  73. data/test/resources/hello_utf16be.txt +0 -0
  74. data/test/resources/hello_utf16le.txt +0 -0
  75. data/test/resources/hello_utf8.txt +2 -0
  76. data/test/resources/hello_windows.txt +2 -0
  77. data/test/resources/hello_world.doc +0 -0
  78. data/test/resources/hello_world.pdf +0 -0
  79. data/test/resources/hello_world.txt +2 -0
  80. data/test/resources/high_ascii_delimited.txt +2 -0
  81. data/test/resources/malformed.xml +6 -0
  82. data/test/resources/normal.csv +3 -0
  83. data/test/resources/normal.csv.zip +0 -0
  84. data/test/resources/normal_pipe.csv +3 -0
  85. data/test/resources/normal_thorn.csv +3 -0
  86. data/test/resources/not_a_pdf.pdf +0 -0
  87. data/test/resources/not_a_word_file.doc +0 -0
  88. data/test/resources/sample_xls.xls +0 -0
  89. data/test/resources/sample_xlsx.xlsx +0 -0
  90. data/test/resources/standard_mappings.yml +39 -0
  91. data/test/resources/txt_file_xls_extension.xls +1 -0
  92. data/test/resources/txt_file_xlsx_extension.xlsx +1 -0
  93. data/test/resources/utf-16be_xml.xml +0 -0
  94. data/test/resources/utf-16be_xml_with_declaration.xml +0 -0
  95. data/test/resources/utf-16le_xml.xml +0 -0
  96. data/test/resources/utf-8_xml.xml +9 -0
  97. data/test/resources/windows-1252_xml.xml +9 -0
  98. data/test/resources/windows.csv +5 -0
  99. data/test/resources/xlsx_file_xls_extension.xls +0 -0
  100. data/test/standard_mappings_test.rb +22 -0
  101. data/test/table_test.rb +288 -0
  102. data/test/test_helper.rb +13 -0
  103. metadata +443 -0
@@ -0,0 +1,53 @@
1
+ # encoding: UTF-8
2
+ require 'test_helper'
3
+
4
+ # This tests the README page example
5
+ class ReadmeTest < ActiveSupport::TestCase
6
+ test 'readme example' do
7
+ require 'ndr_import/non_tabular/table'
8
+ require 'ndr_import/file/registry'
9
+
10
+ unzip_path = SafePath.new('test_space_rw')
11
+ source_file = SafePath.new('permanent_test_files').join('flat_file.pdf')
12
+ options = { 'unzip_path' => unzip_path }
13
+
14
+ table = NdrImport::NonTabular::Table.new(
15
+ 'start_in_a_record' => false,
16
+ 'end_in_a_record' => false,
17
+ 'klass' => 'SomeTestKlass',
18
+ 'start_line_pattern' => /\A------\z/,
19
+ 'remove_lines' => { 'footer' => [/\A== Page \d+ of \d+ ==\z/i] },
20
+ 'columns' => [
21
+ {
22
+ 'column' => 'one',
23
+ 'non_tabular_cell' => { 'lines' => Range.new(0, -1, true), 'capture' => /^(.*)$/i }
24
+ }
25
+ ]
26
+ )
27
+
28
+ # Use the Registry to enumerate over the files and their tables
29
+ files = NdrImport::File::Registry.files(source_file, options)
30
+ files.each do |filename|
31
+ tables = NdrImport::File::Registry.tables(filename, nil, options)
32
+ tables.each do |_tablename, table_content|
33
+ # Use the NonTabular::Table to tabulate the "table" contents
34
+ table.transform(table_content).each do |_klass, _fields, _index|
35
+ # Your code goes here
36
+ end
37
+
38
+ # Now we test the example
39
+ results = []
40
+ table.transform(table_content).each do |_klass, fields, _index|
41
+ results << fields[:rawtext]['one']
42
+ end
43
+ assert table.is_a?(NdrImport::NonTabular::Table)
44
+ assert_equal 4, results.count
45
+ assert results.first.start_with?('1')
46
+ assert results.last.start_with?('4')
47
+ assert results.any? { |result| result =~ /This is captured/ }
48
+ refute results.any? { |result| result =~ /This is never captured/ }
49
+ refute results.any? { |result| result =~ /== Page/ }
50
+ end
51
+ end
52
+ end
53
+ end
@@ -0,0 +1,3 @@
1
+ A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S,T,U,V,W,X,Y,Z
2
+ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1
3
+ 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2
@@ -0,0 +1,3 @@
1
+ I,really,think
2
+ "this","is "not" "
3
+ going,to,work
@@ -0,0 +1,26 @@
1
+ <% require 'tmpdir' %>
2
+ # This allows us different filesystem paths for different platforms
3
+ ---
4
+ test_files: &test_files
5
+ root: <%= Dir.mktmpdir %>
6
+
7
+ ? !ruby/regexp /.*/
8
+ :
9
+ test_space_r:
10
+ <<: *test_files
11
+ prms:
12
+ - r
13
+ test_space_w:
14
+ <<: *test_files
15
+ prms:
16
+ - w
17
+ test_space_rw:
18
+ <<: *test_files
19
+ prms:
20
+ - r
21
+ - w
22
+ permanent_test_files:
23
+ root: <%= NdrImport.root + '/test/resources' %>
24
+ prms:
25
+ - r
26
+ - w
Binary file
@@ -0,0 +1,27 @@
1
+ 0
2
+ This is never captured
3
+ ------
4
+ 1
5
+ == Page 1 of 5 ==
6
+ Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut
7
+ labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris
8
+ nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit
9
+ esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt
10
+ == Page 2 of 5 ==
11
+ in culpa qui officia deserunt mollit anim id est laborum.
12
+ ------
13
+ 2
14
+ This is captured
15
+ Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
16
+ == Page 3 of 5 ==
17
+ ------
18
+ 3
19
+ Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.
20
+ ------
21
+ == Page 4 of 5 ==
22
+ 4
23
+ Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
24
+ ------
25
+ -1
26
+ == Page 5 of 5 ==
27
+ This is never captured
@@ -0,0 +1,20 @@
1
+ --- !ruby/object:NdrImport::NonTabular::Table
2
+ # canonical_name: somename
3
+ # filename_pattern:
4
+ # format: pipe
5
+ klass: SomeTestKlass
6
+ start_line_pattern: !ruby/regexp /\A------\z/
7
+ # end_line_pattern:
8
+ start_in_a_record: false
9
+ end_in_a_record: false
10
+ remove_lines:
11
+ footer:
12
+ - !ruby/regexp /\A== Page \d+ of \d+ ==\z/i
13
+ columns:
14
+ - column: one
15
+ non_tabular_cell:
16
+ lines: !ruby/range
17
+ begin: 0
18
+ end: -1
19
+ excl: true
20
+ capture: !ruby/regexp /^(.*)$/i
Binary file
Binary file
@@ -0,0 +1,2 @@
1
+ Hello world
2
+ This is a thorny þ issue!
@@ -0,0 +1,2 @@
1
+ Hello windows world
2
+ This is a thorny � issue!
Binary file
Binary file
@@ -0,0 +1,2 @@
1
+ Hello world,
2
+ this is a text document
@@ -0,0 +1,2 @@
1
+ 345465�1234567890�Dr Bob��BLOGGS�JOE�M�05 NOV 1990�1 noddy street, anytown�CB22 3AD�IP�234534654�42�25 Jun 2015�14253�WWJ�SMITH, John�SGH�AA�LAB���Made discvery|�Dr Josh|
2
+ 345465�1234567890�Dr Bob��BLOGGS�JOE�M�05 NOV 1990�1 noddy street, anytown�CB22 3AD�IP�234534654�42�25 Jun 2015�14253�WWJ�SMITH, John�SGH�AA�LAB���Made discvery|�Dr Josh|
@@ -0,0 +1,6 @@
1
+ <?xml version="1.0"?>
2
+ <root>
3
+ <note><![CDATA[
4
+ This is  a note!
5
+ ]]></note>
6
+ </root>
@@ -0,0 +1,3 @@
1
+ A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S,T,U,V,W,X,Y,Z
2
+ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1
3
+ 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2
Binary file
@@ -0,0 +1,3 @@
1
+ A|B|C|D|E|F|G|H|I|J|K|L|M|N|O|P|Q|R|S|T|U|V|W|X|Y|Z
2
+ 1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1
3
+ 2|2|2|2|2|2|2|2|2|2|2|2|2|2|2|2|2|2|2|2|2|2|2|2|2|2
@@ -0,0 +1,3 @@
1
+ A�B�C�D�E�F�G�H�I�J�K�L�M�N�O�P�Q�R�S�T�U�V�W�X�Y�Z
2
+ 1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1
3
+ 2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2
File without changes
File without changes
Binary file
Binary file
@@ -0,0 +1,39 @@
1
+ ---
2
+ surname:
3
+ column: surname
4
+ rawtext_name: surname
5
+ mappings:
6
+ - field: surname
7
+ clean: :name
8
+ previoussurname:
9
+ column: previoussurname
10
+ rawtext_name: previoussurname
11
+ mappings:
12
+ - field: previoussurname
13
+ clean: :name
14
+ forenames:
15
+ column: forenames
16
+ rawtext_name: forenames
17
+ mappings:
18
+ - field: forenames
19
+ clean: :name
20
+ sex:
21
+ column: sex
22
+ rawtext_name: sex
23
+ mappings:
24
+ - field: sex
25
+ clean: :sex
26
+ nhsnumber:
27
+ column: nhsnumber
28
+ rawtext_name: nhsnumber
29
+ mappings:
30
+ - field: nhsnumber
31
+ clean: :nhsnumber
32
+ postcode:
33
+ column: postcode
34
+ rawtext_name: postcode
35
+ mappings:
36
+ - field: postcode
37
+ clean: :postcode
38
+ test:
39
+ column: standard_mapping_column_name
@@ -0,0 +1 @@
1
+ Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
@@ -0,0 +1 @@
1
+ Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
Binary file
Binary file
@@ -0,0 +1,9 @@
1
+ <?xml version="1.0"?>
2
+ <root>
3
+ <note id="alpha">
4
+ <letter>α</letter>
5
+ </note>
6
+ <note id="beta">
7
+ <letter>β</letter>
8
+ </note>
9
+ </root>
@@ -0,0 +1,9 @@
1
+ <?xml version="1.0"?>
2
+ <root>
3
+ <note id="apostrophe">
4
+ <letter>�</letter>
5
+ </note>
6
+ <note id="dash">
7
+ <letter>�</letter>
8
+ </note>
9
+ </root>
@@ -0,0 +1,5 @@
1
+ NHS NUMBER�,PERSON BIRTH DATE,POSTCODE OF USUAL ADDRESS (AT DIAGNOSIS),PERSON FAMILY NAME,PERSON GIVEN NAME,PATIENT USUAL ADDRESS (AT DIAGNOSIS),ORGANISATION CODE (CODE OF PROVIDER),MULTIDISCIPLINARY TEAM DISCUSSION DATE (CANCER),CONSULTANT CODE (FIRST SEEN),DATE OF DIAGNOSIS (CLINICALLY AGREED),PERFORMANCE STATUS (ADULT) ,ASA SCORE (PROSTATE),SOURCE OF REFERRAL FOR OUT-PATIENTS ,PSA (DIAGNOSIS),PROSTATE BIOPSY TECHNIQUE,GLEASON GRADE (PRIMARY),GLEASON GRADE (SECONDARY),GLEASON GRADE (TERTIARY),MULTIPARAMETRIC MRI PERFORMED,T CATEGORY (FINAL PRE-TREATMENT),N CATEGORY (FINAL PRE-TREATMENT),M CATEGORY (FINAL PRE-TREATMENT),PERINEURAL INVASION,"NUMBER OF POSITIVE CORES
2
+
3
+ ",TOTAL NUMBER,GREATEST PERCENTAGE OF CANCER IN SINGLE MOST INVOLVED CORE,"SPECIALIST REFERRAL
4
+ APPOINTMENT
5
+ ",ORGANISATION SITE CODE (PROVIDER TREATMENT START DATE (CANCER) ,CONSULTANT CODE (TREATMENT),TYPE OF RADICAL PROSTATECTOMY (ACTUAL),PROCEDURE DATE,PROCEDURE � NERVE SPARING,T CATEGORY (PATHOLOGICAL),N CATEGORY (PATHOLOGICAL),ORGAN CONFINED,SEMINAL VESICLES INVASION,RADICAL PROSTATECTOMY MARGIN STATUS,NUMBER OF NODES EXAMINED,NUMBER OF NODES POSITIVE,PLANNED RADIOTHERAPY INTENT (PROSTATE),PLANNED RADIOTHERAPY TYPE,PLANNED TYPE OF IMAGE-GUIDANCE FOR EXTERNAL BEAM RADIOTHERAPY,PLANNED RADIOTHERAPY FIELD,PLANNED BRACHYTHERAPY TYPE,PLANNED BRACHYTHERAPY TOTAL DOSE,PLANNED BRACHYTHERAPY TOTAL FRACTIONS,PLANNED DURATION OF NEOADJUVANT ANDROGEN DEPRIVATION THERAPY,PLANNED TOTAL DURATION OF ADJUVANT ANDROGEN DEPRIVATION THERAPY
@@ -0,0 +1,22 @@
1
+ # encoding: UTF-8
2
+ require 'test_helper'
3
+
4
+ # This tests the StandardMappings configuration class
5
+ class StandardMappingsTest < ActiveSupport::TestCase
6
+ test 'should not raise exception on reconfiguring NdrImport::StandardMappings' do
7
+ NdrImport::StandardMappings.mappings =
8
+ YAML.load_file(SafePath.new('permanent_test_files').join('standard_mappings.yml'))
9
+ end
10
+
11
+ test 'should raise exception on setting non-hash standard mappings' do
12
+ assert_raise ArgumentError do
13
+ NdrImport::StandardMappings.mappings = true
14
+ end
15
+ end
16
+
17
+ test 'should return mappings' do
18
+ safe_path = SafePath.new('permanent_test_files').join('standard_mappings.yml')
19
+ assert_instance_of Hash, NdrImport::StandardMappings.mappings
20
+ assert_equal YAML.load_file(safe_path), NdrImport::StandardMappings.mappings
21
+ end
22
+ end
@@ -0,0 +1,288 @@
1
+ require 'test_helper'
2
+
3
+ # This tests the NdrImport::Table mapping class
4
+ class TableTest < ActiveSupport::TestCase
5
+ def test_deserialize_table
6
+ table = simple_deserialized_table
7
+ assert_instance_of NdrImport::Table, table
8
+ assert_equal 2, table.header_lines
9
+ assert_equal 1, table.footer_lines
10
+ assert_equal 'pipe', table.format
11
+ assert_equal 'SomeTestKlass', table.klass
12
+ assert_equal 'somename', table.canonical_name
13
+ assert_equal [{ 'column' => 'one' }, { 'column' => 'two' }, { 'column' => 'three' }],
14
+ table.columns
15
+ end
16
+
17
+ def test_initialize
18
+ table = NdrImport::Table.new(:header_lines => 2, :footer_lines => 1,
19
+ :format => 'pipe', :klass => 'SomeTestKlass',
20
+ :columns => [{ 'column' => 'one' }, { 'column' => 'two' }])
21
+ assert_instance_of NdrImport::Table, table
22
+ assert_equal 2, table.header_lines
23
+ assert_equal 1, table.footer_lines
24
+ assert_equal 'pipe', table.format
25
+ assert_equal 'SomeTestKlass', table.klass
26
+ assert_equal [{ 'column' => 'one' }, { 'column' => 'two' }], table.columns
27
+ end
28
+
29
+ def test_should_raise_error_on_invalid_initialization
30
+ # incorrect parameter type
31
+ assert_raises ArgumentError do
32
+ NdrImport::Table.new([])
33
+ end
34
+ # invalid option
35
+ assert_raises ArgumentError do
36
+ NdrImport::Table.new(:potato => true)
37
+ end
38
+ end
39
+
40
+ def test_match_with_no_patterns
41
+ table = NdrImport::Table.new
42
+ assert table.match('example.csv', nil)
43
+ assert table.match('example.xslx', 'Sheet1')
44
+ end
45
+
46
+ def test_match_with_only_filename_pattern
47
+ table = NdrImport::Table.new(:filename_pattern => /\.(csv|xlsx)\z/i)
48
+ assert table.match('example.csv', nil)
49
+ assert table.match('example.xlsx', 'Sheet1')
50
+
51
+ table = NdrImport::Table.new(:filename_pattern => /\Ademo\.(csv|xlsx)\z/i)
52
+ refute table.match('example.csv', nil)
53
+ refute table.match('example.xlsx', 'Sheet1')
54
+ end
55
+
56
+ def test_match_with_both_patterns
57
+ table = NdrImport::Table.new(:filename_pattern => /\.xlsx\z/i,
58
+ :tablename_pattern => /\Asheet1\z/i)
59
+ assert table.match('example.xlsx', 'Sheet1')
60
+ refute table.match('example.xlsx', 'Sheet2')
61
+ end
62
+
63
+ def test_transform
64
+ lines = [%w(ONE TWO), %w(CARROT POTATO), %w(BACON SAUSAGE)].each
65
+ table = NdrImport::Table.new(:header_lines => 1, :footer_lines => 0,
66
+ :klass => 'SomeTestKlass',
67
+ :columns => [{ 'column' => 'one' }, { 'column' => 'two' }])
68
+
69
+ output = []
70
+ table.transform(lines).each do |klass, fields, index|
71
+ output << [klass, fields, index]
72
+ end
73
+
74
+ expected_output = [
75
+ ['SomeTestKlass', { :rawtext => { 'one' => 'CARROT', 'two' => 'POTATO' } }, 1],
76
+ ['SomeTestKlass', { :rawtext => { 'one' => 'BACON', 'two' => 'SAUSAGE' } }, 2]
77
+ ]
78
+ assert_equal expected_output, output
79
+ end
80
+
81
+ def test_process_line
82
+ # No header row, process the first line
83
+ table = NdrImport::Table.new(:header_lines => 0, :footer_lines => 0,
84
+ :klass => 'SomeTestKlass',
85
+ :columns => [{ 'column' => 'one' }, { 'column' => 'two' }])
86
+
87
+ output = []
88
+ table.process_line(%w(CARROT POTATO)).each do |klass, fields, index|
89
+ output << [klass, fields, index]
90
+ end
91
+
92
+ expected_output = [
93
+ ['SomeTestKlass', { :rawtext => { 'one' => 'CARROT', 'two' => 'POTATO' } }, 0]
94
+ ]
95
+ assert_equal expected_output.sort, output.sort
96
+
97
+ # One header row, don't process the first line
98
+ table = NdrImport::Table.new(:header_lines => 1, :footer_lines => 0,
99
+ :klass => 'SomeTestKlass',
100
+ :columns => [{ 'column' => 'one' }, { 'column' => 'two' }])
101
+
102
+ output = []
103
+ table.process_line(%w(CARROT POTATO)).each do |klass, fields, index|
104
+ output << [klass, fields, index]
105
+ end
106
+
107
+ assert_equal [], output
108
+ end
109
+
110
+ def test_transform_line
111
+ table = NdrImport::Table.new(:header_lines => 2, :footer_lines => 1,
112
+ :columns => column_level_klass_mapping)
113
+ enum = table.transform_line(%w(CARROT POTATO PEA), 7)
114
+ assert_instance_of Enumerator, enum
115
+
116
+ output = []
117
+ enum.each do |klass, fields, index|
118
+ output << [klass, fields, index]
119
+ end
120
+
121
+ expected_output = [
122
+ ['SomeTestKlass', { :rawtext => { 'one' => 'CARROT', 'two' => 'POTATO' } }, 7],
123
+ ['SomeOtherKlass', { :rawtext => { 'two' => 'POTATO', 'three' => 'PEA' } }, 7]
124
+ ]
125
+ assert_equal expected_output.sort, output.sort
126
+ end
127
+
128
+ def test_encode_with
129
+ # encode_with(coder)
130
+ end
131
+
132
+ def test_skip_footer_lines
133
+ table = simple_deserialized_table
134
+ lines = (1..10).each
135
+ assert_equal((1..7).to_a, table.send(:skip_footer_lines, lines, 3).to_a)
136
+ assert_equal((1..10).to_a, table.send(:skip_footer_lines, lines, 0).to_a)
137
+ end
138
+
139
+ def test_masked_mappings
140
+ # table level
141
+ table = simple_deserialized_table
142
+ table_level_klass_masked_mappings = {
143
+ 'SomeTestKlass' => [{ 'column' => 'one' }, { 'column' => 'two' }, { 'column' => 'three' }]
144
+ }
145
+ assert_equal table_level_klass_masked_mappings, table.send(:masked_mappings)
146
+
147
+ # column level
148
+ table = NdrImport::Table.new(:header_lines => 2, :footer_lines => 1,
149
+ :columns => column_level_klass_mapping)
150
+
151
+ assert_equal column_level_klass_masked_mappings, table.send(:masked_mappings)
152
+ end
153
+
154
+ def test_column_level_klass_masked_mappings
155
+ table = NdrImport::Table.new(:header_lines => 2, :footer_lines => 1,
156
+ :columns => column_level_klass_mapping)
157
+
158
+ assert_equal column_level_klass_masked_mappings,
159
+ table.send(:column_level_klass_masked_mappings)
160
+ end
161
+
162
+ def test_ensure_mappings_define_klass
163
+ table = NdrImport::Table.new(:header_lines => 2, :footer_lines => 1, :columns => [
164
+ { 'column' => 'one', 'klass' => 'SomeTestKlass' },
165
+ { 'column' => 'two' }
166
+ ])
167
+ assert_raise(RuntimeError) { table.send(:ensure_mappings_define_klass) }
168
+
169
+ table = NdrImport::Table.new(:header_lines => 2, :footer_lines => 1, :columns => [
170
+ { 'column' => 'one', 'klass' => 'SomeTestKlass' },
171
+ { 'column' => 'two', 'klass' => 'SomeOtherKlass' }
172
+ ])
173
+ table.send(:ensure_mappings_define_klass)
174
+ end
175
+
176
+ def test_mask_mappings_by_klass
177
+ table = NdrImport::Table.new(:header_lines => 2, :footer_lines => 1,
178
+ :columns => column_level_klass_mapping)
179
+ some_test_klass_mapping = [
180
+ { 'column' => 'one', 'klass' => 'SomeTestKlass' },
181
+ { 'column' => 'two', 'klass' => %w(SomeTestKlass SomeOtherKlass) },
182
+ { 'do_not_capture' => true }
183
+ ]
184
+ assert_equal some_test_klass_mapping,
185
+ table.send(:mask_mappings_by_klass, 'SomeTestKlass')
186
+
187
+ some_other_klass_mapping = [
188
+ { 'do_not_capture' => true },
189
+ { 'column' => 'two', 'klass' => %w(SomeTestKlass SomeOtherKlass) },
190
+ { 'column' => 'three', 'klass' => 'SomeOtherKlass' }
191
+ ]
192
+ assert_equal some_other_klass_mapping,
193
+ table.send(:mask_mappings_by_klass, 'SomeOtherKlass')
194
+ end
195
+
196
+ def test_valid_single_line_header
197
+ lines = [
198
+ %w(ONE TWO),
199
+ %w(CARROT POTATO),
200
+ %w(BACON SAUSAGE)
201
+ ].each
202
+
203
+ table = NdrImport::Table.new(:header_lines => 1, :footer_lines => 0,
204
+ :klass => 'SomeTestKlass',
205
+ :columns => [{ 'column' => 'one' }, { 'column' => 'two' }])
206
+
207
+ output = []
208
+ table.transform(lines).each do |klass, fields, index|
209
+ output << [klass, fields, index]
210
+ end
211
+
212
+ expected_output = [
213
+ ['SomeTestKlass', { :rawtext => { 'one' => 'CARROT', 'two' => 'POTATO' } }, 1],
214
+ ['SomeTestKlass', { :rawtext => { 'one' => 'BACON', 'two' => 'SAUSAGE' } }, 2]
215
+ ]
216
+ assert table.header_valid?
217
+ assert_equal expected_output, output
218
+ end
219
+
220
+ def test_valid_multi_line_header
221
+ lines = [
222
+ %w(NOTHEADING1 NOTHEADING2),
223
+ %w(ONE TWO),
224
+ %w(DEFINITELYNOTHEADING1 DEFINITELYNOTHEADING2),
225
+ %w(CARROT POTATO),
226
+ %w(BACON SAUSAGE)
227
+ ].each
228
+
229
+ table = NdrImport::Table.new(:header_lines => 3, :footer_lines => 0,
230
+ :klass => 'SomeTestKlass',
231
+ :columns => [{ 'column' => 'one' }, { 'column' => 'two' }])
232
+
233
+ output = []
234
+ table.transform(lines).each do |klass, fields, index|
235
+ output << [klass, fields, index]
236
+ end
237
+
238
+ expected_output = [
239
+ ['SomeTestKlass', { :rawtext => { 'one' => 'CARROT', 'two' => 'POTATO' } }, 3],
240
+ ['SomeTestKlass', { :rawtext => { 'one' => 'BACON', 'two' => 'SAUSAGE' } }, 4]
241
+ ]
242
+ assert table.header_valid?
243
+ assert_equal expected_output, output
244
+ end
245
+
246
+ private
247
+
248
+ def simple_deserialized_table
249
+ Psych.load <<YML
250
+ --- !ruby/object:NdrImport::Table
251
+ canonical_name: somename
252
+ # filename_pattern: !ruby/regexp //
253
+ header_lines: 2
254
+ footer_lines: 1
255
+ format: pipe
256
+ klass: SomeTestKlass
257
+ # non_tabular_row:
258
+ # ...
259
+ columns:
260
+ - column: one
261
+ - column: two
262
+ - column: three
263
+ YML
264
+ end
265
+
266
+ def column_level_klass_mapping
267
+ [
268
+ { 'column' => 'one', 'klass' => 'SomeTestKlass' },
269
+ { 'column' => 'two', 'klass' => %w(SomeTestKlass SomeOtherKlass) },
270
+ { 'column' => 'three', 'klass' => 'SomeOtherKlass' }
271
+ ]
272
+ end
273
+
274
+ def column_level_klass_masked_mappings
275
+ {
276
+ 'SomeTestKlass' => [
277
+ { 'column' => 'one', 'klass' => 'SomeTestKlass' },
278
+ { 'column' => 'two', 'klass' => %w(SomeTestKlass SomeOtherKlass) },
279
+ { 'do_not_capture' => true }
280
+ ],
281
+ 'SomeOtherKlass' => [
282
+ { 'do_not_capture' => true },
283
+ { 'column' => 'two', 'klass' => %w(SomeTestKlass SomeOtherKlass) },
284
+ { 'column' => 'three', 'klass' => 'SomeOtherKlass' }
285
+ ]
286
+ }
287
+ end
288
+ end