ndr_import 8.5.0 → 8.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +7 -0
- data/Gemfile +0 -3
- data/README.md +6 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/code_safety.yml +27 -11
- data/exe/pdf_acro_form_to_yaml +23 -0
- data/exe/pdf_to_text +28 -0
- data/exe/word_to_text +26 -0
- data/gemfiles/Gemfile.rails52 +0 -3
- data/gemfiles/Gemfile.rails60 +5 -0
- data/lib/ndr_import/version.rb +1 -1
- data/ndr_import.gemspec +9 -7
- metadata +23 -164
- data/gemfiles/Gemfile.rails50 +0 -8
- data/gemfiles/Gemfile.rails51 +0 -9
- data/test/file/acro_form_test.rb +0 -39
- data/test/file/base_test.rb +0 -54
- data/test/file/delimited_test.rb +0 -233
- data/test/file/docx_test.rb +0 -53
- data/test/file/excel_test.rb +0 -124
- data/test/file/pdf_test.rb +0 -36
- data/test/file/registry_test.rb +0 -62
- data/test/file/seven_zip_test.rb +0 -59
- data/test/file/text_test.rb +0 -92
- data/test/file/word_test.rb +0 -35
- data/test/file/xml_test.rb +0 -21
- data/test/file/zip_test.rb +0 -47
- data/test/fixed_width/table_test.rb +0 -35
- data/test/helpers/file/delimited_test.rb +0 -105
- data/test/helpers/file/excel_test.rb +0 -82
- data/test/helpers/file/pdf_test.rb +0 -27
- data/test/helpers/file/word_test.rb +0 -26
- data/test/helpers/file/xml_test.rb +0 -131
- data/test/helpers/file/zip_test.rb +0 -75
- data/test/mapper_test.rb +0 -676
- data/test/non_tabular/mapping_test.rb +0 -36
- data/test/non_tabular/table_test.rb +0 -590
- data/test/non_tabular_file_helper_test.rb +0 -501
- data/test/pdf_form/table_test.rb +0 -119
- data/test/readme_test.rb +0 -53
- data/test/resources/acro_form.pdf +0 -0
- data/test/resources/blank_tab_test.xlsx +0 -0
- data/test/resources/bomd.csv +0 -3
- data/test/resources/broken.csv +0 -3
- data/test/resources/filesystem_paths.yml +0 -26
- data/test/resources/flat_file.pdf +0 -0
- data/test/resources/flat_file.txt +0 -27
- data/test/resources/flat_file.yml +0 -20
- data/test/resources/hello_utf16be.txt +0 -0
- data/test/resources/hello_utf16le.txt +0 -0
- data/test/resources/hello_utf8.txt +0 -2
- data/test/resources/hello_windows.txt +0 -2
- data/test/resources/hello_world.doc +0 -0
- data/test/resources/hello_world.docx +0 -0
- data/test/resources/hello_world.pdf +0 -0
- data/test/resources/hello_world.txt +0 -2
- data/test/resources/high_ascii_delimited.txt +0 -2
- data/test/resources/high_ascii_delimited_example_two.txt +0 -3
- data/test/resources/malformed.csv +0 -3
- data/test/resources/malformed.xml +0 -6
- data/test/resources/malformed_pipe.csv +0 -3
- data/test/resources/normal.7z +0 -0
- data/test/resources/normal.csv +0 -3
- data/test/resources/normal.csv.zip +0 -0
- data/test/resources/normal_pipe.csv +0 -3
- data/test/resources/normal_thorn.csv +0 -3
- data/test/resources/not_a_pdf.pdf +0 -0
- data/test/resources/not_a_word_file.doc +0 -0
- data/test/resources/not_a_word_file.docx +0 -0
- data/test/resources/not_sign_delimited.txt +0 -3
- data/test/resources/password_protected_hello_world.docx +0 -0
- data/test/resources/password_protected_sample_xlsx.xlsx +0 -0
- data/test/resources/sample.xml +0 -34
- data/test/resources/sample_xls.xls +0 -0
- data/test/resources/sample_xlsx.xlsx +0 -0
- data/test/resources/sheet_streaming.xls +0 -0
- data/test/resources/sheet_streaming.xlsx +0 -0
- data/test/resources/standard_mappings.yml +0 -39
- data/test/resources/txt_file_xls_extension.xls +0 -1
- data/test/resources/txt_file_xlsx_extension.xlsx +0 -1
- data/test/resources/utf-16be_xml.xml +0 -0
- data/test/resources/utf-16be_xml_with_declaration.xml +0 -0
- data/test/resources/utf-16le_xml.xml +0 -0
- data/test/resources/utf-8_xml.xml +0 -9
- data/test/resources/windows-1252_xml.xml +0 -9
- data/test/resources/windows.csv +0 -5
- data/test/resources/xlsx_file_xls_extension.xls +0 -0
- data/test/standard_mappings_test.rb +0 -22
- data/test/table_test.rb +0 -545
- data/test/test_helper.rb +0 -35
- data/test/universal_importer_helper_test.rb +0 -86
- data/test/xml/table_test.rb +0 -90
data/test/resources/windows.csv
DELETED
@@ -1,5 +0,0 @@
|
|
1
|
-
NHS NUMBER�,PERSON BIRTH DATE,POSTCODE OF USUAL ADDRESS (AT DIAGNOSIS),PERSON FAMILY NAME,PERSON GIVEN NAME,PATIENT USUAL ADDRESS (AT DIAGNOSIS),ORGANISATION CODE (CODE OF PROVIDER),MULTIDISCIPLINARY TEAM DISCUSSION DATE (CANCER),CONSULTANT CODE (FIRST SEEN),DATE OF DIAGNOSIS (CLINICALLY AGREED),PERFORMANCE STATUS (ADULT) ,ASA SCORE (PROSTATE),SOURCE OF REFERRAL FOR OUT-PATIENTS ,PSA (DIAGNOSIS),PROSTATE BIOPSY TECHNIQUE,GLEASON GRADE (PRIMARY),GLEASON GRADE (SECONDARY),GLEASON GRADE (TERTIARY),MULTIPARAMETRIC MRI PERFORMED,T CATEGORY (FINAL PRE-TREATMENT),N CATEGORY (FINAL PRE-TREATMENT),M CATEGORY (FINAL PRE-TREATMENT),PERINEURAL INVASION,"NUMBER OF POSITIVE CORES
|
2
|
-
|
3
|
-
",TOTAL NUMBER,GREATEST PERCENTAGE OF CANCER IN SINGLE MOST INVOLVED CORE,"SPECIALIST REFERRAL
|
4
|
-
APPOINTMENT
|
5
|
-
",ORGANISATION SITE CODE (PROVIDER TREATMENT START DATE (CANCER) ,CONSULTANT CODE (TREATMENT),TYPE OF RADICAL PROSTATECTOMY (ACTUAL),PROCEDURE DATE,PROCEDURE � NERVE SPARING,T CATEGORY (PATHOLOGICAL),N CATEGORY (PATHOLOGICAL),ORGAN CONFINED,SEMINAL VESICLES INVASION,RADICAL PROSTATECTOMY MARGIN STATUS,NUMBER OF NODES EXAMINED,NUMBER OF NODES POSITIVE,PLANNED RADIOTHERAPY INTENT (PROSTATE),PLANNED RADIOTHERAPY TYPE,PLANNED TYPE OF IMAGE-GUIDANCE FOR EXTERNAL BEAM RADIOTHERAPY,PLANNED RADIOTHERAPY FIELD,PLANNED BRACHYTHERAPY TYPE,PLANNED BRACHYTHERAPY TOTAL DOSE,PLANNED BRACHYTHERAPY TOTAL FRACTIONS,PLANNED DURATION OF NEOADJUVANT ANDROGEN DEPRIVATION THERAPY,PLANNED TOTAL DURATION OF ADJUVANT ANDROGEN DEPRIVATION THERAPY
|
Binary file
|
@@ -1,22 +0,0 @@
|
|
1
|
-
# encoding: UTF-8
|
2
|
-
require 'test_helper'
|
3
|
-
|
4
|
-
# This tests the StandardMappings configuration class
|
5
|
-
class StandardMappingsTest < ActiveSupport::TestCase
|
6
|
-
test 'should not raise exception on reconfiguring NdrImport::StandardMappings' do
|
7
|
-
NdrImport::StandardMappings.mappings =
|
8
|
-
YAML.load_file(SafePath.new('permanent_test_files').join('standard_mappings.yml'))
|
9
|
-
end
|
10
|
-
|
11
|
-
test 'should raise exception on setting non-hash standard mappings' do
|
12
|
-
assert_raise ArgumentError do
|
13
|
-
NdrImport::StandardMappings.mappings = true
|
14
|
-
end
|
15
|
-
end
|
16
|
-
|
17
|
-
test 'should return mappings' do
|
18
|
-
safe_path = SafePath.new('permanent_test_files').join('standard_mappings.yml')
|
19
|
-
assert_instance_of Hash, NdrImport::StandardMappings.mappings
|
20
|
-
assert_equal YAML.load_file(safe_path), NdrImport::StandardMappings.mappings
|
21
|
-
end
|
22
|
-
end
|
data/test/table_test.rb
DELETED
@@ -1,545 +0,0 @@
|
|
1
|
-
require 'test_helper'
|
2
|
-
|
3
|
-
class TestNoCoderTable < NdrImport::NonTabular::Table
|
4
|
-
undef_method :encode_with
|
5
|
-
end
|
6
|
-
|
7
|
-
# This tests the NdrImport::Table mapping class
|
8
|
-
class TableTest < ActiveSupport::TestCase
|
9
|
-
def test_deserialize_table
|
10
|
-
table = simple_deserialized_table
|
11
|
-
assert_instance_of NdrImport::Table, table
|
12
|
-
assert_equal 2, table.header_lines
|
13
|
-
assert_equal 1, table.footer_lines
|
14
|
-
assert_equal 'pipe', table.format
|
15
|
-
assert_equal 'SomeTestKlass', table.klass
|
16
|
-
assert_equal 'somename', table.canonical_name
|
17
|
-
assert_equal [{ 'column' => 'one' }, { 'column' => 'two' }, { 'column' => 'three' }],
|
18
|
-
table.columns
|
19
|
-
end
|
20
|
-
|
21
|
-
def test_initialize
|
22
|
-
table = NdrImport::Table.new(header_lines: 2, file_password: 'leek', footer_lines: 1,
|
23
|
-
format: 'pipe', klass: 'SomeTestKlass',
|
24
|
-
columns: [{ 'column' => 'one' }, { 'column' => 'two' }])
|
25
|
-
assert_instance_of NdrImport::Table, table
|
26
|
-
assert_equal 2, table.header_lines
|
27
|
-
assert_equal 1, table.footer_lines
|
28
|
-
assert_equal 'pipe', table.format
|
29
|
-
assert_equal 'SomeTestKlass', table.klass
|
30
|
-
assert_equal 'leek', table.file_password
|
31
|
-
assert_equal [{ 'column' => 'one' }, { 'column' => 'two' }], table.columns
|
32
|
-
end
|
33
|
-
|
34
|
-
def test_should_raise_error_on_invalid_initialization
|
35
|
-
# incorrect parameter type
|
36
|
-
assert_raises ArgumentError do
|
37
|
-
NdrImport::Table.new([])
|
38
|
-
end
|
39
|
-
# invalid option
|
40
|
-
assert_raises ArgumentError do
|
41
|
-
NdrImport::Table.new(:potato => true)
|
42
|
-
end
|
43
|
-
end
|
44
|
-
|
45
|
-
def test_match_with_no_patterns
|
46
|
-
table = NdrImport::Table.new
|
47
|
-
assert table.match('example.csv', nil)
|
48
|
-
assert table.match('example.xslx', 'Sheet1')
|
49
|
-
end
|
50
|
-
|
51
|
-
def test_match_with_only_filename_pattern
|
52
|
-
table = NdrImport::Table.new(:filename_pattern => /\.(csv|xlsx)\z/i)
|
53
|
-
assert table.match('example.csv', nil)
|
54
|
-
assert table.match('example.xlsx', 'Sheet1')
|
55
|
-
|
56
|
-
table = NdrImport::Table.new(:filename_pattern => /\Ademo\.(csv|xlsx)\z/i)
|
57
|
-
refute table.match('example.csv', nil)
|
58
|
-
refute table.match('example.xlsx', 'Sheet1')
|
59
|
-
end
|
60
|
-
|
61
|
-
def test_match_with_both_patterns
|
62
|
-
table = NdrImport::Table.new(:filename_pattern => /\.xlsx\z/i,
|
63
|
-
:tablename_pattern => /\Asheet1\z/i)
|
64
|
-
assert table.match('example.xlsx', 'Sheet1')
|
65
|
-
refute table.match('example.xlsx', 'Sheet2')
|
66
|
-
end
|
67
|
-
|
68
|
-
def test_transform
|
69
|
-
lines = [%w(ONE TWO), %w(CARROT POTATO), %w(BACON SAUSAGE)].each
|
70
|
-
table = NdrImport::Table.new(:header_lines => 1, :footer_lines => 0,
|
71
|
-
:klass => 'SomeTestKlass',
|
72
|
-
:columns => [{ 'column' => 'one' }, { 'column' => 'two' }])
|
73
|
-
|
74
|
-
output = []
|
75
|
-
table.transform(lines).each do |klass, fields, index|
|
76
|
-
output << [klass, fields, index]
|
77
|
-
end
|
78
|
-
|
79
|
-
expected_output = [
|
80
|
-
['SomeTestKlass', { :rawtext => { 'one' => 'CARROT', 'two' => 'POTATO' } }, 1],
|
81
|
-
['SomeTestKlass', { :rawtext => { 'one' => 'BACON', 'two' => 'SAUSAGE' } }, 2]
|
82
|
-
]
|
83
|
-
assert_equal expected_output, output
|
84
|
-
end
|
85
|
-
|
86
|
-
def test_process_line
|
87
|
-
# No header row, process the first line
|
88
|
-
table = NdrImport::Table.new(:header_lines => 0, :footer_lines => 0,
|
89
|
-
:klass => 'SomeTestKlass',
|
90
|
-
:columns => [{ 'column' => 'one' }, { 'column' => 'two' }])
|
91
|
-
|
92
|
-
output = []
|
93
|
-
table.process_line(%w(CARROT POTATO)).each do |klass, fields, index|
|
94
|
-
output << [klass, fields, index]
|
95
|
-
end
|
96
|
-
|
97
|
-
expected_output = [
|
98
|
-
['SomeTestKlass', { :rawtext => { 'one' => 'CARROT', 'two' => 'POTATO' } }, 0]
|
99
|
-
]
|
100
|
-
assert_equal expected_output.sort, output.sort
|
101
|
-
|
102
|
-
# One header row, don't process the first line
|
103
|
-
table = NdrImport::Table.new(:header_lines => 1, :footer_lines => 0,
|
104
|
-
:klass => 'SomeTestKlass',
|
105
|
-
:columns => [{ 'column' => 'one' }, { 'column' => 'two' }])
|
106
|
-
|
107
|
-
output = []
|
108
|
-
table.process_line(%w(ONE TWO)).each { |*stuff| output << stuff }
|
109
|
-
table.process_line(%w(CARROT POTATO)).each { |*stuff| output << stuff }
|
110
|
-
|
111
|
-
expected_output = [
|
112
|
-
['SomeTestKlass', { :rawtext => { 'one' => 'CARROT', 'two' => 'POTATO' } }, 1]
|
113
|
-
]
|
114
|
-
assert_equal expected_output, output
|
115
|
-
end
|
116
|
-
|
117
|
-
def test_process_line_with_unsatisifed_header
|
118
|
-
table = NdrImport::Table.new(:header_lines => 1, :footer_lines => 0,
|
119
|
-
:klass => 'SomeTestKlass',
|
120
|
-
:columns => [{ 'column' => 'one' }, { 'column' => 'two' }])
|
121
|
-
|
122
|
-
exception = assert_raises(RuntimeError) { table.process_line(%w(ONE THREE)).to_a }
|
123
|
-
assert_equal 'Header is not valid! missing: ["two"] unexpected: ["three"]', exception.message
|
124
|
-
end
|
125
|
-
|
126
|
-
def test_transform_line
|
127
|
-
table = NdrImport::Table.new(:header_lines => 2, :footer_lines => 1,
|
128
|
-
:columns => column_level_klass_mapping)
|
129
|
-
enum = table.transform_line(%w(CARROT POTATO PEA), 7)
|
130
|
-
assert_instance_of Enumerator, enum
|
131
|
-
|
132
|
-
output = []
|
133
|
-
enum.each do |klass, fields, index|
|
134
|
-
output << [klass, fields, index]
|
135
|
-
end
|
136
|
-
|
137
|
-
expected_output = [
|
138
|
-
['SomeTestKlass', { :rawtext => { 'one' => 'CARROT', 'two' => 'POTATO' } }, 7],
|
139
|
-
['SomeOtherKlass', { :rawtext => { 'two' => 'POTATO', 'three' => 'PEA' } }, 7]
|
140
|
-
]
|
141
|
-
assert_equal expected_output.sort, output.sort
|
142
|
-
end
|
143
|
-
|
144
|
-
def test_transform_line_row_identifier_index
|
145
|
-
table = NdrImport::Table.new(:header_lines => 2, :footer_lines => 1, :row_identifier => :index,
|
146
|
-
:columns => column_level_klass_mapping)
|
147
|
-
enum = table.transform_line(%w(CARROT POTATO PEA), 7)
|
148
|
-
assert_instance_of Enumerator, enum
|
149
|
-
|
150
|
-
output = []
|
151
|
-
enum.each do |klass, fields, index|
|
152
|
-
output << [klass, fields, index]
|
153
|
-
end
|
154
|
-
|
155
|
-
expected_output = [
|
156
|
-
['SomeTestKlass', { :rawtext => { 'one' => 'CARROT', 'two' => 'POTATO' }, 'row_identifier' => 7 }, 7],
|
157
|
-
['SomeOtherKlass', { :rawtext => { 'two' => 'POTATO', 'three' => 'PEA' }, 'row_identifier' => 7 }, 7]
|
158
|
-
]
|
159
|
-
assert_equal expected_output.sort, output.sort
|
160
|
-
end
|
161
|
-
|
162
|
-
def test_transform_line_row_identifier_uuid
|
163
|
-
table = NdrImport::Table.new(:header_lines => 2, :footer_lines => 1, :row_identifier => :uuid,
|
164
|
-
:columns => column_level_klass_mapping)
|
165
|
-
enum = table.transform_line(%w(CARROT POTATO PEA), 7)
|
166
|
-
assert_instance_of Enumerator, enum
|
167
|
-
|
168
|
-
output = []
|
169
|
-
enum.each do |klass, fields, _index|
|
170
|
-
output << [klass, fields['row_identifier']]
|
171
|
-
end
|
172
|
-
identifiers = output.map(&:last)
|
173
|
-
|
174
|
-
assert_equal 2, identifiers.count
|
175
|
-
assert_equal 2, identifiers.compact.count
|
176
|
-
assert_equal 1, identifiers.uniq.count
|
177
|
-
assert_equal 36, identifiers.first.length
|
178
|
-
end
|
179
|
-
|
180
|
-
def test_encode_with
|
181
|
-
table = NdrImport::Table.new
|
182
|
-
assert table.instance_variables.include?(:@row_index)
|
183
|
-
refute table.class.all_valid_options.include?('row_index')
|
184
|
-
assert_nil table.columns
|
185
|
-
|
186
|
-
coder = {}
|
187
|
-
table.encode_with(coder)
|
188
|
-
assert coder.key?('columns')
|
189
|
-
|
190
|
-
yaml_output = table.to_yaml
|
191
|
-
assert yaml_output.include?('columns')
|
192
|
-
refute yaml_output.include?('row_index')
|
193
|
-
assert YAML.load(yaml_output).is_a?(NdrImport::Table)
|
194
|
-
end
|
195
|
-
|
196
|
-
def test_encode_with_compare
|
197
|
-
table_options = {
|
198
|
-
columns: %w[a b],
|
199
|
-
klass: 'SomeKlass',
|
200
|
-
start_line_pattern: 'TODO',
|
201
|
-
end_line_pattern: 'TODO'
|
202
|
-
}
|
203
|
-
no_coder_table = TestNoCoderTable.new(table_options)
|
204
|
-
ndr_table = NdrImport::NonTabular::Table.new(table_options)
|
205
|
-
|
206
|
-
assert no_coder_table.is_a?(NdrImport::Table)
|
207
|
-
assert ndr_table.is_a?(NdrImport::Table)
|
208
|
-
assert no_coder_table.is_a?(NdrImport::NonTabular::Table)
|
209
|
-
assert ndr_table.is_a?(NdrImport::NonTabular::Table)
|
210
|
-
|
211
|
-
refute no_coder_table.respond_to?(:encode_with)
|
212
|
-
assert ndr_table.respond_to?(:encode_with)
|
213
|
-
|
214
|
-
no_coder_table_yaml_order = get_yaml_mapping_order(no_coder_table.to_yaml)
|
215
|
-
ndr_table_yaml_order = get_yaml_mapping_order(ndr_table.to_yaml)
|
216
|
-
|
217
|
-
# no_coder_table_yaml_order => ["klass", "columns", "start_line_pattern", "end_line_pattern", "row_index"]
|
218
|
-
# ndr_table_yaml_order => ["klass", "start_line_pattern", "end_line_pattern", "columns"]
|
219
|
-
|
220
|
-
assert no_coder_table_yaml_order.include?('row_index')
|
221
|
-
refute ndr_table_yaml_order.include?('row_index')
|
222
|
-
|
223
|
-
refute no_coder_table_yaml_order.last == 'columns'
|
224
|
-
assert ndr_table_yaml_order.last == 'columns'
|
225
|
-
|
226
|
-
# test objects deserialized from yaml mappings
|
227
|
-
deserialized_no_coder_table_yaml = YAML.load(no_coder_table.to_yaml)
|
228
|
-
deserialized_ndr_table_yaml = YAML.load(ndr_table.to_yaml)
|
229
|
-
|
230
|
-
assert deserialized_no_coder_table_yaml.is_a?(NdrImport::NonTabular::Table)
|
231
|
-
assert deserialized_ndr_table_yaml.is_a?(NdrImport::NonTabular::Table)
|
232
|
-
|
233
|
-
assert_nil deserialized_no_coder_table_yaml.filename_pattern
|
234
|
-
assert_equal deserialized_no_coder_table_yaml.klass, no_coder_table.klass
|
235
|
-
assert_equal deserialized_no_coder_table_yaml.start_line_pattern, no_coder_table.start_line_pattern
|
236
|
-
assert_equal deserialized_no_coder_table_yaml.columns, no_coder_table.columns
|
237
|
-
|
238
|
-
assert_nil deserialized_ndr_table_yaml.filename_pattern
|
239
|
-
assert_equal deserialized_ndr_table_yaml.klass, ndr_table.klass
|
240
|
-
assert_equal deserialized_ndr_table_yaml.start_line_pattern, ndr_table.start_line_pattern
|
241
|
-
assert_equal deserialized_ndr_table_yaml.columns, ndr_table.columns
|
242
|
-
end
|
243
|
-
|
244
|
-
def test_skip_footer_lines
|
245
|
-
table = simple_deserialized_table
|
246
|
-
lines = (1..10).each
|
247
|
-
assert_equal((1..7).to_a, table.send(:skip_footer_lines, lines, 3).to_a)
|
248
|
-
assert_equal((1..10).to_a, table.send(:skip_footer_lines, lines, 0).to_a)
|
249
|
-
end
|
250
|
-
|
251
|
-
def test_masked_mappings
|
252
|
-
# table level
|
253
|
-
table = simple_deserialized_table
|
254
|
-
table_level_klass_masked_mappings = {
|
255
|
-
'SomeTestKlass' => [{ 'column' => 'one' }, { 'column' => 'two' }, { 'column' => 'three' }]
|
256
|
-
}
|
257
|
-
assert_equal table_level_klass_masked_mappings, table.send(:masked_mappings)
|
258
|
-
|
259
|
-
# column level
|
260
|
-
table = NdrImport::Table.new(:header_lines => 2, :footer_lines => 1,
|
261
|
-
:columns => column_level_klass_mapping)
|
262
|
-
|
263
|
-
assert_equal column_level_klass_masked_mappings, table.send(:masked_mappings)
|
264
|
-
end
|
265
|
-
|
266
|
-
def test_column_level_klass_masked_mappings
|
267
|
-
table = NdrImport::Table.new(:header_lines => 2, :footer_lines => 1,
|
268
|
-
:columns => column_level_klass_mapping)
|
269
|
-
|
270
|
-
assert_equal column_level_klass_masked_mappings,
|
271
|
-
table.send(:column_level_klass_masked_mappings)
|
272
|
-
end
|
273
|
-
|
274
|
-
def test_ensure_mappings_define_klass
|
275
|
-
table = NdrImport::Table.new(:header_lines => 2, :footer_lines => 1, :columns => [
|
276
|
-
{ 'column' => 'one', 'klass' => 'SomeTestKlass' },
|
277
|
-
{ 'column' => 'two' }
|
278
|
-
])
|
279
|
-
assert_raise(RuntimeError) { table.send(:ensure_mappings_define_klass) }
|
280
|
-
|
281
|
-
table = NdrImport::Table.new(:header_lines => 2, :footer_lines => 1, :columns => [
|
282
|
-
{ 'column' => 'one', 'klass' => 'SomeTestKlass' },
|
283
|
-
{ 'column' => 'two', 'klass' => 'SomeOtherKlass' }
|
284
|
-
])
|
285
|
-
table.send(:ensure_mappings_define_klass)
|
286
|
-
end
|
287
|
-
|
288
|
-
def test_mask_mappings_by_klass
|
289
|
-
table1 = NdrImport::Table.new(:header_lines => 2, :footer_lines => 1,
|
290
|
-
:columns => column_level_klass_mapping)
|
291
|
-
|
292
|
-
some_test_klass_mapping = [
|
293
|
-
{ 'column' => 'one', 'klass' => 'SomeTestKlass' },
|
294
|
-
{ 'column' => 'two', 'klass' => %w(SomeTestKlass SomeOtherKlass) },
|
295
|
-
{ 'do_not_capture' => true }
|
296
|
-
]
|
297
|
-
assert_equal some_test_klass_mapping,
|
298
|
-
table1.send(:mask_mappings_by_klass, 'SomeTestKlass')
|
299
|
-
|
300
|
-
some_other_klass_mapping = [
|
301
|
-
{ 'do_not_capture' => true },
|
302
|
-
{ 'column' => 'two', 'klass' => %w(SomeTestKlass SomeOtherKlass) },
|
303
|
-
{ 'column' => 'three', 'klass' => 'SomeOtherKlass' }
|
304
|
-
]
|
305
|
-
assert_equal some_other_klass_mapping,
|
306
|
-
table1.send(:mask_mappings_by_klass, 'SomeOtherKlass')
|
307
|
-
|
308
|
-
table2 = NdrImport::Table.new(:header_lines => 2, :footer_lines => 1,
|
309
|
-
:columns => column_level_klass_mapping_embedded_klasses)
|
310
|
-
|
311
|
-
some_test_klass_mapping_embedded_klasses = [
|
312
|
-
{ 'column' => 'one', 'klass' => 'SomeTestKlass' },
|
313
|
-
{ 'column' => 'two', 'klass' => [['SomeTestKlass'], 'SomeOtherKlass'] },
|
314
|
-
{ 'do_not_capture' => true }
|
315
|
-
]
|
316
|
-
assert_equal some_test_klass_mapping_embedded_klasses,
|
317
|
-
table2.send(:mask_mappings_by_klass, 'SomeTestKlass')
|
318
|
-
|
319
|
-
some_other_klass_mapping_embedded_klasses = [
|
320
|
-
{ 'do_not_capture' => true },
|
321
|
-
{ 'column' => 'two', 'klass' => [['SomeTestKlass'], 'SomeOtherKlass'] },
|
322
|
-
{ 'column' => 'three', 'klass' => 'SomeOtherKlass' }
|
323
|
-
]
|
324
|
-
assert_equal some_other_klass_mapping_embedded_klasses,
|
325
|
-
table2.send(:mask_mappings_by_klass, 'SomeOtherKlass')
|
326
|
-
end
|
327
|
-
|
328
|
-
def test_valid_single_line_header
|
329
|
-
lines = [
|
330
|
-
%w(ONE TWO),
|
331
|
-
%w(CARROT POTATO),
|
332
|
-
%w(BACON SAUSAGE)
|
333
|
-
].each
|
334
|
-
|
335
|
-
table = NdrImport::Table.new(:header_lines => 1, :footer_lines => 0,
|
336
|
-
:klass => 'SomeTestKlass',
|
337
|
-
:columns => [{ 'column' => 'one' }, { 'column' => 'two' }])
|
338
|
-
|
339
|
-
output = []
|
340
|
-
table.transform(lines).each do |klass, fields, index|
|
341
|
-
output << [klass, fields, index]
|
342
|
-
end
|
343
|
-
|
344
|
-
expected_output = [
|
345
|
-
['SomeTestKlass', { :rawtext => { 'one' => 'CARROT', 'two' => 'POTATO' } }, 1],
|
346
|
-
['SomeTestKlass', { :rawtext => { 'one' => 'BACON', 'two' => 'SAUSAGE' } }, 2]
|
347
|
-
]
|
348
|
-
assert table.header_valid?
|
349
|
-
assert_equal expected_output, output
|
350
|
-
end
|
351
|
-
|
352
|
-
def test_valid_multi_line_header
|
353
|
-
lines = [
|
354
|
-
%w(NOTHEADING1 NOTHEADING2),
|
355
|
-
%w(ONE TWO),
|
356
|
-
%w(DEFINITELYNOTHEADING1 DEFINITELYNOTHEADING2),
|
357
|
-
%w(CARROT POTATO),
|
358
|
-
%w(BACON SAUSAGE)
|
359
|
-
].each
|
360
|
-
|
361
|
-
table = NdrImport::Table.new(:header_lines => 3, :footer_lines => 0,
|
362
|
-
:klass => 'SomeTestKlass',
|
363
|
-
:columns => [{ 'column' => 'one' }, { 'column' => 'two' }])
|
364
|
-
|
365
|
-
output = []
|
366
|
-
table.transform(lines).each do |klass, fields, index|
|
367
|
-
output << [klass, fields, index]
|
368
|
-
end
|
369
|
-
|
370
|
-
expected_output = [
|
371
|
-
['SomeTestKlass', { :rawtext => { 'one' => 'CARROT', 'two' => 'POTATO' } }, 3],
|
372
|
-
['SomeTestKlass', { :rawtext => { 'one' => 'BACON', 'two' => 'SAUSAGE' } }, 4]
|
373
|
-
]
|
374
|
-
assert table.header_valid?
|
375
|
-
assert_equal expected_output, output
|
376
|
-
end
|
377
|
-
|
378
|
-
def test_varying_header_line_lengths_with_valid_header_row_including_nils
|
379
|
-
lines = [
|
380
|
-
[nil] << 'RIGHTALIGN1' << 'RIGHTALIGN2',
|
381
|
-
%w(ONE TWO),
|
382
|
-
%w(LEFTALIGN) << nil,
|
383
|
-
%w(CENTRE1) << nil << 'CENTRE2',
|
384
|
-
%w(UNO DOS)
|
385
|
-
].each
|
386
|
-
|
387
|
-
table = NdrImport::Table.new(:header_lines => 4, :footer_lines => 0,
|
388
|
-
:klass => 'SomeTestKlass',
|
389
|
-
:columns => [{ 'column' => 'one' }, { 'column' => 'two' }])
|
390
|
-
|
391
|
-
output = []
|
392
|
-
table.transform(lines).each do |klass, fields, index|
|
393
|
-
output << [klass, fields, index]
|
394
|
-
end
|
395
|
-
|
396
|
-
assert table.header_valid?
|
397
|
-
|
398
|
-
expected_output = [['SomeTestKlass', { :rawtext => { 'one' => 'UNO', 'two' => 'DOS' } }, 4]]
|
399
|
-
assert_equal expected_output, output
|
400
|
-
end
|
401
|
-
|
402
|
-
def test_varying_header_line_lengths_with_valid_header_row
|
403
|
-
lines = [
|
404
|
-
%w(NOTHEADING1 NOTHEADING2 UHOH3 UHOH4),
|
405
|
-
%w(ONE TWO),
|
406
|
-
%w(DEFINITELYNOTHEADING1 DEFINITELYNOTHEADING2),
|
407
|
-
%w(UNO DOS)
|
408
|
-
].each
|
409
|
-
|
410
|
-
table = NdrImport::Table.new(:header_lines => 3, :footer_lines => 0,
|
411
|
-
:klass => 'SomeTestKlass',
|
412
|
-
:columns => [{ 'column' => 'one' }, { 'column' => 'two' }])
|
413
|
-
|
414
|
-
output = []
|
415
|
-
table.transform(lines).each do |klass, fields, index|
|
416
|
-
output << [klass, fields, index]
|
417
|
-
end
|
418
|
-
|
419
|
-
assert table.header_valid?
|
420
|
-
|
421
|
-
expected_output = [['SomeTestKlass', { :rawtext => { 'one' => 'UNO', 'two' => 'DOS' } }, 3]]
|
422
|
-
assert_equal expected_output, output
|
423
|
-
end
|
424
|
-
|
425
|
-
def test_varying_header_line_lengths_without_valid_header_row
|
426
|
-
lines = [
|
427
|
-
%w(NOTHEADING1 NOTHEADING2 UHOH3 UHOH4),
|
428
|
-
%w(ONE TWO NOPE),
|
429
|
-
%w(NOT_HERE OR_HERE),
|
430
|
-
%w(UNO DOS)
|
431
|
-
].each
|
432
|
-
|
433
|
-
table = NdrImport::Table.new(:header_lines => 3, :footer_lines => 0,
|
434
|
-
:klass => 'SomeTestKlass',
|
435
|
-
:columns => [{ 'column' => 'one' }, { 'column' => 'two' }])
|
436
|
-
|
437
|
-
exception = assert_raises(RuntimeError) { table.transform(lines).to_a }
|
438
|
-
|
439
|
-
assert_match(/Header is not valid!/, exception.message)
|
440
|
-
assert_match(/missing: \["one", "two"\]/, exception.message)
|
441
|
-
assert_match(/unexpected: \["not_here", "or_here"\]/, exception.message)
|
442
|
-
end
|
443
|
-
|
444
|
-
def test_jumbled_header
|
445
|
-
lines = [
|
446
|
-
%w(NOTHEADING1 NOTHEADING2 NOTHEADING3),
|
447
|
-
%w(ONE THREE TWO),
|
448
|
-
%w(DATA ROW HERE)
|
449
|
-
].each
|
450
|
-
|
451
|
-
table = NdrImport::Table.new(
|
452
|
-
:header_lines => 2,
|
453
|
-
:footer_lines => 0,
|
454
|
-
:klass => 'SomeTestKlass',
|
455
|
-
:columns => [
|
456
|
-
{ 'column' => 'one' },
|
457
|
-
{ 'column' => 'two' },
|
458
|
-
{ 'column' => 'three' }
|
459
|
-
]
|
460
|
-
)
|
461
|
-
|
462
|
-
exception = assert_raises(RuntimeError) { table.transform(lines).to_a }
|
463
|
-
assert_equal('Header is not valid! (out of order)', exception.message)
|
464
|
-
end
|
465
|
-
|
466
|
-
def test_wrong_header_names
|
467
|
-
lines = [
|
468
|
-
%w(NOTHEADING1 NOTHEADING2 NOTHEADING3),
|
469
|
-
%w(FUN TWO TREE),
|
470
|
-
%w(DATA ROW HERE)
|
471
|
-
].each
|
472
|
-
|
473
|
-
table = NdrImport::Table.new(
|
474
|
-
:header_lines => 2,
|
475
|
-
:footer_lines => 0,
|
476
|
-
:klass => 'SomeTestKlass',
|
477
|
-
:columns => [
|
478
|
-
{ 'column' => 'one' },
|
479
|
-
{ 'column' => 'two' },
|
480
|
-
{ 'column' => 'three' }
|
481
|
-
]
|
482
|
-
)
|
483
|
-
|
484
|
-
exception = assert_raises(RuntimeError) { table.transform(lines).to_a }
|
485
|
-
assert_equal('Header is not valid! missing: ' \
|
486
|
-
'["one", "three"] unexpected: ["fun", "tree"]', exception.message)
|
487
|
-
end
|
488
|
-
|
489
|
-
private
|
490
|
-
|
491
|
-
def simple_deserialized_table
|
492
|
-
Psych.load <<YML
|
493
|
-
--- !ruby/object:NdrImport::Table
|
494
|
-
canonical_name: somename
|
495
|
-
# filename_pattern: !ruby/regexp //
|
496
|
-
header_lines: 2
|
497
|
-
footer_lines: 1
|
498
|
-
format: pipe
|
499
|
-
klass: SomeTestKlass
|
500
|
-
# non_tabular_row:
|
501
|
-
# ...
|
502
|
-
columns:
|
503
|
-
- column: one
|
504
|
-
- column: two
|
505
|
-
- column: three
|
506
|
-
YML
|
507
|
-
end
|
508
|
-
|
509
|
-
def column_level_klass_mapping
|
510
|
-
[
|
511
|
-
{ 'column' => 'one', 'klass' => 'SomeTestKlass' },
|
512
|
-
{ 'column' => 'two', 'klass' => %w(SomeTestKlass SomeOtherKlass) },
|
513
|
-
{ 'column' => 'three', 'klass' => 'SomeOtherKlass' }
|
514
|
-
]
|
515
|
-
end
|
516
|
-
|
517
|
-
def column_level_klass_mapping_embedded_klasses
|
518
|
-
[
|
519
|
-
{ 'column' => 'one', 'klass' => 'SomeTestKlass' },
|
520
|
-
{ 'column' => 'two', 'klass' => [['SomeTestKlass'], 'SomeOtherKlass'] },
|
521
|
-
{ 'column' => 'three', 'klass' => 'SomeOtherKlass' }
|
522
|
-
]
|
523
|
-
end
|
524
|
-
|
525
|
-
def column_level_klass_masked_mappings
|
526
|
-
{
|
527
|
-
'SomeTestKlass' => [
|
528
|
-
{ 'column' => 'one', 'klass' => 'SomeTestKlass' },
|
529
|
-
{ 'column' => 'two', 'klass' => %w(SomeTestKlass SomeOtherKlass) },
|
530
|
-
{ 'do_not_capture' => true }
|
531
|
-
],
|
532
|
-
'SomeOtherKlass' => [
|
533
|
-
{ 'do_not_capture' => true },
|
534
|
-
{ 'column' => 'two', 'klass' => %w(SomeTestKlass SomeOtherKlass) },
|
535
|
-
{ 'column' => 'three', 'klass' => 'SomeOtherKlass' }
|
536
|
-
]
|
537
|
-
}
|
538
|
-
end
|
539
|
-
|
540
|
-
def get_yaml_mapping_order(yaml_mapping)
|
541
|
-
yaml_mapping.split("\n").
|
542
|
-
delete_if { |line| /-+/.match(line) }.
|
543
|
-
map { |line| /(.*):/.match(line)[1].to_s }
|
544
|
-
end
|
545
|
-
end
|