ndr_import 8.6.0 → 9.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/CODEOWNERS +2 -0
- data/.github/workflows/lint.yml +23 -0
- data/.github/workflows/test.yml +72 -0
- data/.gitignore +1 -1
- data/CHANGELOG.md +34 -0
- data/README.md +1 -2
- data/code_safety.yml +126 -29
- data/lib/ndr_import/csv_library.rb +25 -3
- data/lib/ndr_import/file/excel.rb +10 -12
- data/lib/ndr_import/file/xml.rb +9 -2
- data/lib/ndr_import/helpers/file/delimited.rb +7 -8
- data/lib/ndr_import/helpers/file/xml.rb +6 -3
- data/lib/ndr_import/helpers/file/xml_streaming.rb +183 -0
- data/lib/ndr_import/universal_importer_helper.rb +25 -3
- data/lib/ndr_import/version.rb +1 -1
- data/ndr_import.gemspec +5 -5
- metadata +18 -20
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: aaf3c826acb51f4d579fb956e9606b5b86e120f3bd561db762d72081f27a1098
|
|
4
|
+
data.tar.gz: 2b11ddf7dc9b748b4a1ac9cc91b47c47b99677e4bde990c44d1ec20e76efbfc0
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 3faaf744255693f04425b6e5ed1ec7198e912d2854b3f482a39d9bc53fe3859ed3bb6ef7855cdfd03959042a530f3a2f528239548676c7159c9029d69b6c160d
|
|
7
|
+
data.tar.gz: 1aa9d1895d7f874499823c4b87f1874e025b376f239030f3baf40bab5f5ed3ed09f80721205e271446893a9070652df3f26fceb48b59a9eb86ac121fdbfd92c6
|
data/.github/CODEOWNERS
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
name: Lint
|
|
2
|
+
|
|
3
|
+
on: [pull_request]
|
|
4
|
+
|
|
5
|
+
jobs:
|
|
6
|
+
rubocop:
|
|
7
|
+
name: RuboCop
|
|
8
|
+
|
|
9
|
+
runs-on: ubuntu-latest
|
|
10
|
+
|
|
11
|
+
steps:
|
|
12
|
+
- uses: actions/checkout@v2
|
|
13
|
+
with:
|
|
14
|
+
fetch-depth: 0 # fetch everything
|
|
15
|
+
- name: Set up Ruby
|
|
16
|
+
uses: ruby/setup-ruby@v1
|
|
17
|
+
with:
|
|
18
|
+
ruby-version: 3.0
|
|
19
|
+
- name: Install dependencies
|
|
20
|
+
run: bundle install
|
|
21
|
+
- name: Run RuboCop against BASE..HEAD changes
|
|
22
|
+
run: bundle exec rake rubocop:diff origin/${GITHUB_BASE_REF#*/}
|
|
23
|
+
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
name: Test
|
|
2
|
+
|
|
3
|
+
on: [push]
|
|
4
|
+
|
|
5
|
+
jobs:
|
|
6
|
+
test:
|
|
7
|
+
strategy:
|
|
8
|
+
fail-fast: false
|
|
9
|
+
matrix:
|
|
10
|
+
ruby-version:
|
|
11
|
+
- 2.6
|
|
12
|
+
- 2.7
|
|
13
|
+
- 3.0
|
|
14
|
+
gemfile:
|
|
15
|
+
- gemfiles/Gemfile.rails52
|
|
16
|
+
- gemfiles/Gemfile.rails60
|
|
17
|
+
|
|
18
|
+
name: Ruby ${{ matrix.ruby-version }} / Bundle ${{ matrix.gemfile }}
|
|
19
|
+
|
|
20
|
+
runs-on: ubuntu-latest
|
|
21
|
+
|
|
22
|
+
env:
|
|
23
|
+
BUNDLE_GEMFILE: ${{ matrix.gemfile }}
|
|
24
|
+
|
|
25
|
+
steps:
|
|
26
|
+
- uses: actions/checkout@v2
|
|
27
|
+
- name: Set up Ruby
|
|
28
|
+
uses: ruby/setup-ruby@v1
|
|
29
|
+
with:
|
|
30
|
+
ruby-version: ${{ matrix.ruby-version }}
|
|
31
|
+
- name: Install dependencies
|
|
32
|
+
run: bundle install
|
|
33
|
+
- name: Run tests
|
|
34
|
+
run: bundle exec rake
|
|
35
|
+
|
|
36
|
+
# A utility job upon which Branch Protection can depend,
|
|
37
|
+
# thus remaining agnostic of the matrix.
|
|
38
|
+
test_matrix:
|
|
39
|
+
if: ${{ always() }}
|
|
40
|
+
runs-on: ubuntu-latest
|
|
41
|
+
name: Matrix
|
|
42
|
+
needs: test
|
|
43
|
+
steps:
|
|
44
|
+
- name: Check build matrix status
|
|
45
|
+
if: ${{ needs.test.result != 'success' }}
|
|
46
|
+
run: exit 1
|
|
47
|
+
|
|
48
|
+
notify:
|
|
49
|
+
# Run only on master, but regardless of whether tests past:
|
|
50
|
+
if: ${{ always() && github.ref == 'refs/heads/master' }}
|
|
51
|
+
|
|
52
|
+
needs: test_matrix
|
|
53
|
+
|
|
54
|
+
runs-on: ubuntu-latest
|
|
55
|
+
|
|
56
|
+
steps:
|
|
57
|
+
- uses: 8398a7/action-slack@v3
|
|
58
|
+
with:
|
|
59
|
+
status: custom
|
|
60
|
+
fields: workflow,commit,author
|
|
61
|
+
custom_payload: |
|
|
62
|
+
{
|
|
63
|
+
channel: 'C7FQWGDHP',
|
|
64
|
+
username: 'CI – ' + '${{ github.repository }}'.split('/')[1],
|
|
65
|
+
icon_emoji: ':hammer_and_wrench:',
|
|
66
|
+
attachments: [{
|
|
67
|
+
color: '${{ needs.test_matrix.result }}' === 'success' ? 'good' : '${{ needs.test_matrix.result }}' === 'failure' ? 'danger' : 'warning',
|
|
68
|
+
text: `${process.env.AS_WORKFLOW} against \`${{ github.ref }}\` (${process.env.AS_COMMIT}) for ${{ github.actor }} resulted in *${{ needs.test_matrix.result }}*.`
|
|
69
|
+
}]
|
|
70
|
+
}
|
|
71
|
+
env:
|
|
72
|
+
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
|
data/.gitignore
CHANGED
data/CHANGELOG.md
CHANGED
|
@@ -1,6 +1,40 @@
|
|
|
1
1
|
## [Unreleased]
|
|
2
2
|
*no unreleased changes*
|
|
3
3
|
|
|
4
|
+
## 9.1.0 / 2021-02-01
|
|
5
|
+
### Added
|
|
6
|
+
* `CSVLibrary` is now deprecated.
|
|
7
|
+
* Handle xlsm files
|
|
8
|
+
|
|
9
|
+
### Fixed
|
|
10
|
+
* Fix `CSVLibrary.foreach` on Ruby 3.0
|
|
11
|
+
* Updated jekyll bundle
|
|
12
|
+
|
|
13
|
+
## 9.0.3 / 2021-01-04
|
|
14
|
+
### Fixed
|
|
15
|
+
* Address issue importing multiple files against the same table (#54)
|
|
16
|
+
|
|
17
|
+
### Changed
|
|
18
|
+
* ensure keyword args are properly splatted for ruby 2.7
|
|
19
|
+
|
|
20
|
+
### Added
|
|
21
|
+
* Ruby 2.7 to travis matrix
|
|
22
|
+
|
|
23
|
+
## 9.0.2 / 2020-08-14
|
|
24
|
+
### Changed
|
|
25
|
+
* Configure Nokogiri with HUGE for large xml files
|
|
26
|
+
|
|
27
|
+
## 9.0.1 / 2020-03-26
|
|
28
|
+
### Fixed
|
|
29
|
+
* bumps to `nokogiri` / `spreadsheet` / `rubyzip` dependencies
|
|
30
|
+
|
|
31
|
+
## 9.0.0 / 2019-07-31
|
|
32
|
+
### Changed
|
|
33
|
+
* `File::Xml` will now stream XML files by default. Use `slurp: true` for the old behaviour. (#43)
|
|
34
|
+
|
|
35
|
+
### Added
|
|
36
|
+
* Add `XmlStreaming` helper, for more performant handling of large XML documents with Nokogiri. (#43)
|
|
37
|
+
|
|
4
38
|
## 8.6.0 / 2019-06-07
|
|
5
39
|
### Added
|
|
6
40
|
* Allow conditional preservation of blank lines when joining lines in non-tabular data (#41)
|
data/README.md
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
# NdrImport [](https://github.com/publichealthengland/ndr_import/actions?query=workflow%3Atest) [](https://rubygems.org/gems/ndr_import) [](https://www.rubydoc.info/gems/ndr_import)
|
|
3
2
|
This is the Public Health England (PHE) National Disease Registers (NDR) Import ETL ruby gem, providing:
|
|
4
3
|
|
|
5
4
|
1. file import handlers for *extracting* data from delimited files (csv, pipe, tab, thorn), .xls(x) spreadsheets, .doc(x) word documents, PDF, PDF AcroForms, XML, 7-Zip and Zip files.
|
data/code_safety.yml
CHANGED
|
@@ -1,9 +1,21 @@
|
|
|
1
1
|
---
|
|
2
2
|
file safety:
|
|
3
|
+
".github/CODEOWNERS":
|
|
4
|
+
comments:
|
|
5
|
+
reviewed_by: ollietulloch
|
|
6
|
+
safe_revision: b64ff21375dcde2b8fefe622ee9861f0fea21487
|
|
7
|
+
".github/workflows/lint.yml":
|
|
8
|
+
comments:
|
|
9
|
+
reviewed_by: ollietulloch
|
|
10
|
+
safe_revision: b64ff21375dcde2b8fefe622ee9861f0fea21487
|
|
11
|
+
".github/workflows/test.yml":
|
|
12
|
+
comments:
|
|
13
|
+
reviewed_by: ollietulloch
|
|
14
|
+
safe_revision: b64ff21375dcde2b8fefe622ee9861f0fea21487
|
|
3
15
|
".gitignore":
|
|
4
16
|
comments: whole file re-reviewed
|
|
5
17
|
reviewed_by: josh.pencheon
|
|
6
|
-
safe_revision:
|
|
18
|
+
safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
|
|
7
19
|
".hound.yml":
|
|
8
20
|
comments:
|
|
9
21
|
reviewed_by: timgentry
|
|
@@ -12,14 +24,10 @@ file safety:
|
|
|
12
24
|
comments:
|
|
13
25
|
reviewed_by: josh.pencheon
|
|
14
26
|
safe_revision: b09e268ff9c8349b914aa1b7ba888e1d39f97e4a
|
|
15
|
-
".travis.yml":
|
|
16
|
-
comments:
|
|
17
|
-
reviewed_by: josh.pencheon
|
|
18
|
-
safe_revision: d3d9a987befeecb122a448d8d06e66d74da13fb5
|
|
19
27
|
CHANGELOG.md:
|
|
20
28
|
comments:
|
|
21
|
-
reviewed_by:
|
|
22
|
-
safe_revision:
|
|
29
|
+
reviewed_by: ollietulloch
|
|
30
|
+
safe_revision: d88ded7c260da37200610e4f0b204a4ea2e481f9
|
|
23
31
|
CODE_OF_CONDUCT.md:
|
|
24
32
|
comments:
|
|
25
33
|
reviewed_by: timgentry
|
|
@@ -38,8 +46,8 @@ file safety:
|
|
|
38
46
|
safe_revision: 5d185a0aeba6a9cd2ff5e59efadcaeec9be45d8b
|
|
39
47
|
README.md:
|
|
40
48
|
comments:
|
|
41
|
-
reviewed_by:
|
|
42
|
-
safe_revision:
|
|
49
|
+
reviewed_by: ollietulloch
|
|
50
|
+
safe_revision: b64ff21375dcde2b8fefe622ee9861f0fea21487
|
|
43
51
|
Rakefile:
|
|
44
52
|
comments:
|
|
45
53
|
reviewed_by: josh.pencheon
|
|
@@ -52,6 +60,70 @@ file safety:
|
|
|
52
60
|
comments:
|
|
53
61
|
reviewed_by: josh.pencheon
|
|
54
62
|
safe_revision: e1d967c10059e8c635452838c3f3dd2b969d9ae4
|
|
63
|
+
docs/Gemfile:
|
|
64
|
+
comments:
|
|
65
|
+
reviewed_by: josh.pencheon
|
|
66
|
+
safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
|
|
67
|
+
docs/Gemfile.lock:
|
|
68
|
+
comments:
|
|
69
|
+
reviewed_by: ollietulloch
|
|
70
|
+
safe_revision: ea0149c7739676463a252ffd9fbe4af238762b2b
|
|
71
|
+
docs/_config.yml:
|
|
72
|
+
comments:
|
|
73
|
+
reviewed_by: josh.pencheon
|
|
74
|
+
safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
|
|
75
|
+
docs/_includes/footer.html:
|
|
76
|
+
comments:
|
|
77
|
+
reviewed_by: josh.pencheon
|
|
78
|
+
safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
|
|
79
|
+
docs/_includes/header.html:
|
|
80
|
+
comments:
|
|
81
|
+
reviewed_by: josh.pencheon
|
|
82
|
+
safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
|
|
83
|
+
docs/capturing-data.md:
|
|
84
|
+
comments:
|
|
85
|
+
reviewed_by: josh.pencheon
|
|
86
|
+
safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
|
|
87
|
+
docs/date-formats.md:
|
|
88
|
+
comments:
|
|
89
|
+
reviewed_by: josh.pencheon
|
|
90
|
+
safe_revision: fa21d6d967bf132800b456b585795beec80b08a3
|
|
91
|
+
docs/getting-started.md:
|
|
92
|
+
comments:
|
|
93
|
+
reviewed_by: josh.pencheon
|
|
94
|
+
safe_revision: fa21d6d967bf132800b456b585795beec80b08a3
|
|
95
|
+
docs/identifying-and-splitting-records.md:
|
|
96
|
+
comments:
|
|
97
|
+
reviewed_by: josh.pencheon
|
|
98
|
+
safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
|
|
99
|
+
docs/inbuilt-cleaning-methods.md:
|
|
100
|
+
comments:
|
|
101
|
+
reviewed_by: josh.pencheon
|
|
102
|
+
safe_revision: 694b57ce14e0709fc4d31a1357f8416e98f5de91
|
|
103
|
+
docs/index.md:
|
|
104
|
+
comments:
|
|
105
|
+
reviewed_by: josh.pencheon
|
|
106
|
+
safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
|
|
107
|
+
docs/local-code-transformation-in-yaml-mappings.md:
|
|
108
|
+
comments:
|
|
109
|
+
reviewed_by: josh.pencheon
|
|
110
|
+
safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
|
|
111
|
+
docs/non-tabular-mappings.md:
|
|
112
|
+
comments:
|
|
113
|
+
reviewed_by: josh.pencheon
|
|
114
|
+
safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
|
|
115
|
+
docs/priority-field-mapping.md:
|
|
116
|
+
comments:
|
|
117
|
+
reviewed_by: josh.pencheon
|
|
118
|
+
safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
|
|
119
|
+
docs/standard-yaml-mappings.md:
|
|
120
|
+
comments:
|
|
121
|
+
reviewed_by: josh.pencheon
|
|
122
|
+
safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
|
|
123
|
+
docs/yaml-mapping-user-guide.md:
|
|
124
|
+
comments:
|
|
125
|
+
reviewed_by: josh.pencheon
|
|
126
|
+
safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
|
|
55
127
|
exe/pdf_acro_form_to_yaml:
|
|
56
128
|
comments:
|
|
57
129
|
reviewed_by: josh.pencheon
|
|
@@ -82,8 +154,8 @@ file safety:
|
|
|
82
154
|
safe_revision: 24d6449fd0612552f132dfbf4cada2ae28d0469e
|
|
83
155
|
lib/ndr_import/csv_library.rb:
|
|
84
156
|
comments:
|
|
85
|
-
reviewed_by:
|
|
86
|
-
safe_revision:
|
|
157
|
+
reviewed_by: ollietulloch
|
|
158
|
+
safe_revision: 6b8668967dbd42d7893a0fa5f0aa1ec1c11227e1
|
|
87
159
|
lib/ndr_import/file/acro_form.rb:
|
|
88
160
|
comments:
|
|
89
161
|
reviewed_by: josh.pencheon
|
|
@@ -106,8 +178,8 @@ file safety:
|
|
|
106
178
|
safe_revision: 897f8b648d633368cf2001d17ab89c06a12d445b
|
|
107
179
|
lib/ndr_import/file/excel.rb:
|
|
108
180
|
comments:
|
|
109
|
-
reviewed_by:
|
|
110
|
-
safe_revision:
|
|
181
|
+
reviewed_by: ollietulloch
|
|
182
|
+
safe_revision: 37482c79448bea80033f6f69d97584df330c9861
|
|
111
183
|
lib/ndr_import/file/office_file_helper.rb:
|
|
112
184
|
comments:
|
|
113
185
|
reviewed_by: josh.pencheon
|
|
@@ -139,7 +211,7 @@ file safety:
|
|
|
139
211
|
lib/ndr_import/file/xml.rb:
|
|
140
212
|
comments:
|
|
141
213
|
reviewed_by: josh.pencheon
|
|
142
|
-
safe_revision:
|
|
214
|
+
safe_revision: ae75fb49baf028ac8ce08e4bedcd3625ff3ff0cd
|
|
143
215
|
lib/ndr_import/file/zip.rb:
|
|
144
216
|
comments:
|
|
145
217
|
reviewed_by: timgentry
|
|
@@ -150,8 +222,8 @@ file safety:
|
|
|
150
222
|
safe_revision: dfc958d44b6c58355445fa395db08a62213ee709
|
|
151
223
|
lib/ndr_import/helpers/file/delimited.rb:
|
|
152
224
|
comments:
|
|
153
|
-
reviewed_by:
|
|
154
|
-
safe_revision:
|
|
225
|
+
reviewed_by: ollietulloch
|
|
226
|
+
safe_revision: 4a5cc1d362c632fc1f9242c69982fbce33557e17
|
|
155
227
|
lib/ndr_import/helpers/file/excel.rb:
|
|
156
228
|
comments:
|
|
157
229
|
reviewed_by: joshpencheon
|
|
@@ -166,8 +238,13 @@ file safety:
|
|
|
166
238
|
safe_revision: 45da71ebd3acbc0fe53755bcd75483ba17cb6924
|
|
167
239
|
lib/ndr_import/helpers/file/xml.rb:
|
|
168
240
|
comments:
|
|
241
|
+
reviewed_by: ollietulloch
|
|
242
|
+
safe_revision: 4d337bd233f7e60cf9d363c92400f21269a28da7
|
|
243
|
+
lib/ndr_import/helpers/file/xml_streaming.rb:
|
|
244
|
+
comments: uses SafePath and Shellwords when accessing filesystem, or making system
|
|
245
|
+
calls
|
|
169
246
|
reviewed_by: josh.pencheon
|
|
170
|
-
safe_revision:
|
|
247
|
+
safe_revision: 55e502bb4445cb8b985e530e8eb26d92b574ded9
|
|
171
248
|
lib/ndr_import/helpers/file/zip.rb:
|
|
172
249
|
comments:
|
|
173
250
|
reviewed_by: timgentry
|
|
@@ -222,16 +299,16 @@ file safety:
|
|
|
222
299
|
safe_revision: a69d4a57ddcf13cdc13c27bd2eb91a395fa7ea36
|
|
223
300
|
lib/ndr_import/universal_importer_helper.rb:
|
|
224
301
|
comments:
|
|
225
|
-
reviewed_by:
|
|
226
|
-
safe_revision:
|
|
302
|
+
reviewed_by: ollietulloch
|
|
303
|
+
safe_revision: ee2e74e4ceda4ff48cbda6872a6bdf0874212c21
|
|
227
304
|
lib/ndr_import/unmapped_data_error.rb:
|
|
228
305
|
comments:
|
|
229
306
|
reviewed_by: josh.pencheon
|
|
230
307
|
safe_revision: 5cd2cd0b3a1e254d30d4acc28c6731825a1f84f5
|
|
231
308
|
lib/ndr_import/version.rb:
|
|
232
309
|
comments: another check?
|
|
233
|
-
reviewed_by:
|
|
234
|
-
safe_revision:
|
|
310
|
+
reviewed_by: ollietulloch
|
|
311
|
+
safe_revision: d88ded7c260da37200610e4f0b204a4ea2e481f9
|
|
235
312
|
lib/ndr_import/xml/table.rb:
|
|
236
313
|
comments:
|
|
237
314
|
reviewed_by: josh.pencheon
|
|
@@ -239,7 +316,11 @@ file safety:
|
|
|
239
316
|
ndr_import.gemspec:
|
|
240
317
|
comments:
|
|
241
318
|
reviewed_by: josh.pencheon
|
|
242
|
-
safe_revision:
|
|
319
|
+
safe_revision: 95e6ee9997d06471fe6f2f169c3c701471086371
|
|
320
|
+
test/csv_library_test.rb:
|
|
321
|
+
comments:
|
|
322
|
+
reviewed_by: ollietulloch
|
|
323
|
+
safe_revision: 6b8668967dbd42d7893a0fa5f0aa1ec1c11227e1
|
|
243
324
|
test/file/acro_form_test.rb:
|
|
244
325
|
comments:
|
|
245
326
|
reviewed_by: josh.pencheon
|
|
@@ -258,16 +339,16 @@ file safety:
|
|
|
258
339
|
safe_revision: a69d4a57ddcf13cdc13c27bd2eb91a395fa7ea36
|
|
259
340
|
test/file/excel_test.rb:
|
|
260
341
|
comments:
|
|
261
|
-
reviewed_by:
|
|
262
|
-
safe_revision:
|
|
342
|
+
reviewed_by: ollietulloch
|
|
343
|
+
safe_revision: 85a080deaa93e4220ad1bf566f29cbdac9b31c0f
|
|
263
344
|
test/file/pdf_test.rb:
|
|
264
345
|
comments:
|
|
265
346
|
reviewed_by: josh.pencheon
|
|
266
347
|
safe_revision: cb24ed3ea8116730d07f74546cd6fed0738b171d
|
|
267
348
|
test/file/registry_test.rb:
|
|
268
349
|
comments:
|
|
269
|
-
reviewed_by:
|
|
270
|
-
safe_revision:
|
|
350
|
+
reviewed_by: ollietulloch
|
|
351
|
+
safe_revision: 85a080deaa93e4220ad1bf566f29cbdac9b31c0f
|
|
271
352
|
test/file/seven_zip_test.rb:
|
|
272
353
|
comments:
|
|
273
354
|
reviewed_by: josh.pencheon
|
|
@@ -283,7 +364,7 @@ file safety:
|
|
|
283
364
|
test/file/xml_test.rb:
|
|
284
365
|
comments:
|
|
285
366
|
reviewed_by: josh.pencheon
|
|
286
|
-
safe_revision:
|
|
367
|
+
safe_revision: ae75fb49baf028ac8ce08e4bedcd3625ff3ff0cd
|
|
287
368
|
test/file/zip_test.rb:
|
|
288
369
|
comments:
|
|
289
370
|
reviewed_by: timgentry
|
|
@@ -308,6 +389,10 @@ file safety:
|
|
|
308
389
|
comments:
|
|
309
390
|
reviewed_by: timgentry
|
|
310
391
|
safe_revision: 9abdd6ced1d0c90ce8dd88abee4eb6472c7ff0d6
|
|
392
|
+
test/helpers/file/xml_streaming_test.rb:
|
|
393
|
+
comments:
|
|
394
|
+
reviewed_by: josh.pencheon
|
|
395
|
+
safe_revision: ae75fb49baf028ac8ce08e4bedcd3625ff3ff0cd
|
|
311
396
|
test/helpers/file/xml_test.rb:
|
|
312
397
|
comments:
|
|
313
398
|
reviewed_by: timgentry
|
|
@@ -356,6 +441,10 @@ file safety:
|
|
|
356
441
|
comments:
|
|
357
442
|
reviewed_by: timgentry
|
|
358
443
|
safe_revision: dab4b8a3e4b29d85eccd971e79936982d888cffd
|
|
444
|
+
test/resources/claims_utf16be_but_isnt.xml:
|
|
445
|
+
comments:
|
|
446
|
+
reviewed_by: josh.pencheon
|
|
447
|
+
safe_revision: ae75fb49baf028ac8ce08e4bedcd3625ff3ff0cd
|
|
359
448
|
test/resources/filesystem_paths.yml:
|
|
360
449
|
comments:
|
|
361
450
|
reviewed_by: timgentry
|
|
@@ -476,6 +565,10 @@ file safety:
|
|
|
476
565
|
comments:
|
|
477
566
|
reviewed_by: timgentry
|
|
478
567
|
safe_revision: 8c30f89f0562ab120769c166d4e93ff839c055f7
|
|
568
|
+
test/resources/sample_xlsm.xlsm:
|
|
569
|
+
comments:
|
|
570
|
+
reviewed_by: ollietulloch
|
|
571
|
+
safe_revision: 85a080deaa93e4220ad1bf566f29cbdac9b31c0f
|
|
479
572
|
test/resources/sample_xlsx.xlsx:
|
|
480
573
|
comments:
|
|
481
574
|
reviewed_by: timgentry
|
|
@@ -492,6 +585,10 @@ file safety:
|
|
|
492
585
|
comments:
|
|
493
586
|
reviewed_by: timgentry
|
|
494
587
|
safe_revision: 31fb1935f4578729d8786eea41cf0ce0a19be1cd
|
|
588
|
+
test/resources/two_files_single_table_mapping.zip:
|
|
589
|
+
comments:
|
|
590
|
+
reviewed_by: ollietulloch
|
|
591
|
+
safe_revision: 830de0f8cb139c5f61525652b424423935cfc7ac
|
|
495
592
|
test/resources/txt_file_xls_extension.xls:
|
|
496
593
|
comments:
|
|
497
594
|
reviewed_by: timgentry
|
|
@@ -542,8 +639,8 @@ file safety:
|
|
|
542
639
|
safe_revision: 93ccee82fc2165d1ca2d9b03d146ae03e769ea96
|
|
543
640
|
test/universal_importer_helper_test.rb:
|
|
544
641
|
comments:
|
|
545
|
-
reviewed_by:
|
|
546
|
-
safe_revision:
|
|
642
|
+
reviewed_by: ollietulloch
|
|
643
|
+
safe_revision: 830de0f8cb139c5f61525652b424423935cfc7ac
|
|
547
644
|
test/xml/table_test.rb:
|
|
548
645
|
comments:
|
|
549
646
|
reviewed_by: josh.pencheon
|
|
@@ -1,38 +1,60 @@
|
|
|
1
1
|
# This file allows us to choose the CSV library we want to use.
|
|
2
2
|
|
|
3
3
|
require 'csv'
|
|
4
|
+
require 'active_support/deprecation'
|
|
5
|
+
|
|
4
6
|
# Using relevant core CSV library.
|
|
5
|
-
CSVLibrary
|
|
7
|
+
class CSVLibrary < CSV; end
|
|
6
8
|
|
|
7
9
|
class << CSVLibrary
|
|
8
10
|
# Is the library we're using FasterCSV?
|
|
9
11
|
def fastercsv?
|
|
12
|
+
deprecate('if you desparately want fastercsv, please use it explicitly')
|
|
10
13
|
not self.const_defined?(:Reader)
|
|
11
14
|
end
|
|
12
15
|
|
|
13
16
|
# Ensure that we can pass "mode" straight through the underlying IO object
|
|
17
|
+
#
|
|
18
|
+
# Note: this could likely be refactored now, as upstream support for something
|
|
19
|
+
# very similar was added:
|
|
20
|
+
#
|
|
21
|
+
# https://github.com/ruby/csv/commit/b4edaf2cf1aa36f5c6264c07514b66739b87ceee
|
|
22
|
+
#
|
|
14
23
|
def foreach(path, **options, &block)
|
|
15
|
-
|
|
16
|
-
|
|
24
|
+
deprecate('CSV#foreach exists, with an optional `mode` argument')
|
|
25
|
+
return to_enum(__method__, path, **options) unless block
|
|
26
|
+
open(path, options.delete(:mode) || 'r', **options) do |csv|
|
|
17
27
|
csv.each(&block)
|
|
18
28
|
end
|
|
19
29
|
end
|
|
20
30
|
|
|
21
31
|
def write_csv_to_string(data)
|
|
32
|
+
deprecate('write_csv_to_string -> generate')
|
|
22
33
|
self.generate do |csv|
|
|
23
34
|
data.each { |line| csv << line }
|
|
24
35
|
end
|
|
25
36
|
end
|
|
26
37
|
|
|
27
38
|
def write_csv_to_file(data, filepath, mode = 'w')
|
|
39
|
+
deprecate('write_csv_to_file -> open')
|
|
28
40
|
self.open(filepath, mode) do |csv|
|
|
29
41
|
data.each { |line| csv << line }
|
|
30
42
|
end
|
|
31
43
|
end
|
|
32
44
|
|
|
33
45
|
def read_csv_from_file(filepath)
|
|
46
|
+
deprecate('read_csv_from_file -> read')
|
|
34
47
|
self.read(filepath)
|
|
35
48
|
end
|
|
49
|
+
|
|
50
|
+
private
|
|
51
|
+
|
|
52
|
+
def deprecate(additional_message = nil)
|
|
53
|
+
ActiveSupport::Deprecation.warn(<<~MESSAGE)
|
|
54
|
+
CSVLibrary is deprecated, and will be removed in a future version of ndr_import.
|
|
55
|
+
Please use standard functionality provided by Ruby's CSV library (#{additional_message}).
|
|
56
|
+
MESSAGE
|
|
57
|
+
end
|
|
36
58
|
end
|
|
37
59
|
|
|
38
60
|
# Forward port CSV::Cell, as it is sometimes
|
|
@@ -90,14 +90,14 @@ module NdrImport
|
|
|
90
90
|
case SafeFile.extname(path).downcase
|
|
91
91
|
when '.xls'
|
|
92
92
|
Roo::Excel.new(SafeFile.safepath_to_string(path))
|
|
93
|
-
when '.xlsx'
|
|
93
|
+
when '.xlsm', '.xlsx'
|
|
94
94
|
if @options['file_password']
|
|
95
95
|
Roo::Excelx.new(StringIO.new(decrypted_file_string(path, @options['file_password'])))
|
|
96
96
|
else
|
|
97
97
|
Roo::Excelx.new(SafeFile.safepath_to_string(path))
|
|
98
98
|
end
|
|
99
99
|
else
|
|
100
|
-
|
|
100
|
+
raise "Received file path with unexpected extension #{SafeFile.extname(path)}"
|
|
101
101
|
end
|
|
102
102
|
rescue Ole::Storage::FormatError => e
|
|
103
103
|
# TODO: Do we need to remove the new_file after using it?
|
|
@@ -105,16 +105,14 @@ module NdrImport
|
|
|
105
105
|
# try to load the .xls file as an .xlsx file, useful for sources like USOM
|
|
106
106
|
# roo check file extensions in file_type_check (GenericSpreadsheet),
|
|
107
107
|
# so we create a duplicate file in xlsx extension
|
|
108
|
-
|
|
109
|
-
new_file_name = SafeFile.basename(path).gsub(/(.*)\.xls$/, '\1_amend.xlsx')
|
|
110
|
-
new_file_path = SafeFile.dirname(path).join(new_file_name)
|
|
111
|
-
copy_file(path, new_file_path)
|
|
108
|
+
raise e.message unless /(.*)\.xls$/.match(path)
|
|
112
109
|
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
110
|
+
new_file_name = SafeFile.basename(path).gsub(/(.*)\.xls$/, '\1_amend.xlsx')
|
|
111
|
+
new_file_path = SafeFile.dirname(path).join(new_file_name)
|
|
112
|
+
copy_file(path, new_file_path)
|
|
113
|
+
|
|
114
|
+
load_workbook(new_file_path)
|
|
115
|
+
rescue RuntimeError, ::Zip::Error => e
|
|
118
116
|
raise ["Unable to read the file '#{path}'", e.message].join('; ')
|
|
119
117
|
end
|
|
120
118
|
|
|
@@ -133,6 +131,6 @@ module NdrImport
|
|
|
133
131
|
end
|
|
134
132
|
end
|
|
135
133
|
|
|
136
|
-
Registry.register(Excel, 'xls', 'xlsx')
|
|
134
|
+
Registry.register(Excel, 'xls', 'xlsm', 'xlsx')
|
|
137
135
|
end
|
|
138
136
|
end
|
data/lib/ndr_import/file/xml.rb
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
require 'ndr_support/safe_file'
|
|
2
2
|
require 'ndr_import/helpers/file/xml'
|
|
3
|
+
require 'ndr_import/helpers/file/xml_streaming'
|
|
3
4
|
require_relative 'registry'
|
|
4
5
|
|
|
5
6
|
module NdrImport
|
|
@@ -9,6 +10,7 @@ module NdrImport
|
|
|
9
10
|
# This class is a xml file handler that returns a single table.
|
|
10
11
|
class Xml < Base
|
|
11
12
|
include NdrImport::Helpers::File::Xml
|
|
13
|
+
include NdrImport::Helpers::File::XmlStreaming
|
|
12
14
|
|
|
13
15
|
private
|
|
14
16
|
|
|
@@ -16,9 +18,14 @@ module NdrImport
|
|
|
16
18
|
def rows(&block)
|
|
17
19
|
return enum_for(:rows) unless block
|
|
18
20
|
|
|
19
|
-
|
|
21
|
+
xpath = @options['xml_record_xpath']
|
|
20
22
|
|
|
21
|
-
|
|
23
|
+
if @options['slurp']
|
|
24
|
+
doc = read_xml_file(@filename)
|
|
25
|
+
doc.xpath(xpath).each(&block)
|
|
26
|
+
else
|
|
27
|
+
each_node(@filename, xpath, &block)
|
|
28
|
+
end
|
|
22
29
|
rescue StandardError => e
|
|
23
30
|
raise("#{SafeFile.basename(@filename)} [#{e.class}: #{e.message}]")
|
|
24
31
|
end
|
|
@@ -32,11 +32,11 @@ module NdrImport
|
|
|
32
32
|
return enum_for(:delimited_rows, path, col_sep, liberal) unless block_given?
|
|
33
33
|
|
|
34
34
|
safe_path = SafeFile.safepath_to_string(path)
|
|
35
|
-
|
|
35
|
+
options = determine_encodings!(safe_path, col_sep, liberal)
|
|
36
36
|
|
|
37
|
-
# By now, we know `
|
|
37
|
+
# By now, we know `options` should let us read the whole
|
|
38
38
|
# file succesfully; if there are problems, we should crash.
|
|
39
|
-
|
|
39
|
+
CSV.foreach(safe_path, options.delete(:mode), **options) do |line|
|
|
40
40
|
yield line.map(&:to_s)
|
|
41
41
|
end
|
|
42
42
|
end
|
|
@@ -46,7 +46,7 @@ module NdrImport
|
|
|
46
46
|
# Derive the source encoding by trying all supported encodings.
|
|
47
47
|
# Returns first set of working options, or raises if none could be found.
|
|
48
48
|
def determine_encodings!(safe_path, col_sep, liberal)
|
|
49
|
-
# delimiter encoding => #
|
|
49
|
+
# delimiter encoding => # CSV encoding string
|
|
50
50
|
supported_encodings = {
|
|
51
51
|
'UTF-8' => 'r:bom|utf-8',
|
|
52
52
|
'Windows-1252' => 'r:windows-1252:utf-8'
|
|
@@ -67,14 +67,13 @@ module NdrImport
|
|
|
67
67
|
begin
|
|
68
68
|
options = {
|
|
69
69
|
col_sep: (col_sep || ',').force_encoding(delimiter_encoding),
|
|
70
|
-
liberal_parsing: liberal
|
|
71
|
-
mode: access_mode
|
|
70
|
+
liberal_parsing: liberal
|
|
72
71
|
}
|
|
73
72
|
|
|
74
73
|
row_num = 0
|
|
75
74
|
# Iterate through the file; if we reach the end, this encoding worked:
|
|
76
|
-
|
|
77
|
-
return options
|
|
75
|
+
CSV.foreach(safe_path, access_mode, **options) { |_line| row_num += 1 }
|
|
76
|
+
return options.merge(mode: access_mode)
|
|
78
77
|
rescue ArgumentError => e
|
|
79
78
|
next if e.message =~ /invalid byte sequence/ # This encoding didn't work
|
|
80
79
|
raise(e)
|
|
@@ -15,10 +15,13 @@ module NdrImport
|
|
|
15
15
|
|
|
16
16
|
require 'nokogiri'
|
|
17
17
|
|
|
18
|
-
Nokogiri::XML(ensure_utf8! file_data)
|
|
19
|
-
|
|
20
|
-
emulate_strict_mode_fatal_check!(doc)
|
|
18
|
+
doc = Nokogiri::XML((ensure_utf8! file_data)) do |config|
|
|
19
|
+
config.huge
|
|
21
20
|
end
|
|
21
|
+
doc.encoding = 'UTF-8'
|
|
22
|
+
emulate_strict_mode_fatal_check!(doc)
|
|
23
|
+
|
|
24
|
+
doc
|
|
22
25
|
end
|
|
23
26
|
|
|
24
27
|
# Nokogiri can use give a `STRICT` parse option to libxml, but our friendly
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
require 'shellwords'
|
|
2
|
+
|
|
3
|
+
require 'ndr_support/safe_file'
|
|
4
|
+
require 'ndr_support/utf8_encoding'
|
|
5
|
+
|
|
6
|
+
module NdrImport
|
|
7
|
+
module Helpers
|
|
8
|
+
module File
|
|
9
|
+
# This mixin adds XML streaming functionality, to support more performant handling
|
|
10
|
+
# of large files by Nokogiri. Uses the `XML::Reader` API, and maintains a temporary
|
|
11
|
+
# DOM as the XML is streamed to allow XPath querying from the root node.
|
|
12
|
+
#
|
|
13
|
+
# If the system has `iconv` available, will attempt to verify the encoding of the
|
|
14
|
+
# file being read externally, so it can be streamed in to Ruby. Otherwise, will load
|
|
15
|
+
# the raw data in to check the encoding, but still stream it through Nokogiri's parser.
|
|
16
|
+
module XmlStreaming
|
|
17
|
+
# Base error for all streaming-specific issues.
|
|
18
|
+
class Error < StandardError; end
|
|
19
|
+
|
|
20
|
+
# Raised if nested tags are accounted which the streaming approach cannnot handle.
|
|
21
|
+
class NestingError < Error
|
|
22
|
+
def initialize(node)
|
|
23
|
+
super <<~STR
|
|
24
|
+
Element '#{node.name}' was found nested inside another of the same type.
|
|
25
|
+
This is not accessible, and a known limitation of XmlStreaming.
|
|
26
|
+
STR
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# Object to track state as the XML is iterated over, and detect
|
|
31
|
+
# when an element of interest is entered.
|
|
32
|
+
class Cursor
|
|
33
|
+
# wrapper to hold a representation of each element we descent into:
|
|
34
|
+
StackItem = Struct.new(:name, :attrs, :empty)
|
|
35
|
+
|
|
36
|
+
def initialize(xpath)
|
|
37
|
+
@xpath = xpath
|
|
38
|
+
@stack = []
|
|
39
|
+
@match_depth = nil
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# Has this cursor already passed inside a similar node?
|
|
43
|
+
def in?(node)
|
|
44
|
+
@stack.detect { |item| item.name == node.name }
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def enter(node)
|
|
48
|
+
@stack.push StackItem.new(node.name, node.attributes, node.empty_element?)
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def leave(_node)
|
|
52
|
+
@stack.pop
|
|
53
|
+
@match_depth = nil if @match_depth && @stack.length < @match_depth
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# Does the element that the cursor is currently on match what
|
|
57
|
+
# is being looked for?
|
|
58
|
+
def matches?
|
|
59
|
+
# Can't match again if we're inside a match already:
|
|
60
|
+
return false if @matched_depth
|
|
61
|
+
|
|
62
|
+
match = current_stack_match?
|
|
63
|
+
|
|
64
|
+
# "empty element" matches are yielded immediately, without
|
|
65
|
+
# tagging the stack as having matched, because there won't
|
|
66
|
+
# be an equivalent closing tag to end the match with later.
|
|
67
|
+
if in_empty_element?
|
|
68
|
+
@stack.pop
|
|
69
|
+
elsif match
|
|
70
|
+
@match_depth = @stack.length
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
match
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
private
|
|
77
|
+
|
|
78
|
+
def in_empty_element?
|
|
79
|
+
@stack.last.empty
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# Does the current state of the stack mean we've met the xpath
|
|
83
|
+
# criteria? Must be an exact match, not just matching a parent
|
|
84
|
+
# element in the DOM.
|
|
85
|
+
def current_stack_match?
|
|
86
|
+
parent_stack = @stack[0..-2]
|
|
87
|
+
|
|
88
|
+
return false unless dom_stubs[@stack].at_xpath(@xpath)
|
|
89
|
+
|
|
90
|
+
parent_stack.empty? || !dom_stubs[parent_stack].at_xpath(@xpath)
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# A cached collection of DOM fragments, to represent the structure
|
|
94
|
+
# necessary to use xpath to descend into the main document's DOM.
|
|
95
|
+
def dom_stubs
|
|
96
|
+
@dom_stubs ||= Hash.new do |hash, items|
|
|
97
|
+
hash[items.dup] = Nokogiri::XML::Builder.new do |dom|
|
|
98
|
+
add_items_to_dom(dom, items.dup)
|
|
99
|
+
end.doc
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# Helper to recursively build XML fragment.
|
|
104
|
+
def add_items_to_dom(dom, items)
|
|
105
|
+
item = items.shift
|
|
106
|
+
dom.send(item.name, item.attrs) do
|
|
107
|
+
add_items_to_dom(dom, items) if items.any?
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
include UTF8Encoding
|
|
113
|
+
|
|
114
|
+
# Streams the contents of the given `safe_path`, and yields
|
|
115
|
+
# each element matching `xpath` as they're found.
|
|
116
|
+
#
|
|
117
|
+
# In the case of dodgy encoding, may fall back to slurping the
|
|
118
|
+
# file, but will still use stream parsing for XML.
|
|
119
|
+
def each_node(safe_path, xpath, &block)
|
|
120
|
+
return enum_for(:each_node, safe_path, xpath) unless block
|
|
121
|
+
|
|
122
|
+
require 'nokogiri'
|
|
123
|
+
|
|
124
|
+
with_encoding_check(safe_path) do |stream, encoding|
|
|
125
|
+
stream_xml_nodes(stream, xpath, encoding, &block)
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
private
|
|
130
|
+
|
|
131
|
+
# We need to ensure the raw data is UTF8 before we start streaming
|
|
132
|
+
# it with nokogiri. If we can do an external check, great. Otherwise,
|
|
133
|
+
# we need to slurp and convert the raw data before presenting it.
|
|
134
|
+
def with_encoding_check(safe_path)
|
|
135
|
+
forced_encoding = nil
|
|
136
|
+
|
|
137
|
+
stream = ::File.open(SafeFile.safepath_to_string(safe_path))
|
|
138
|
+
|
|
139
|
+
unless external_utf8_check?(safe_path)
|
|
140
|
+
stream = StringIO.new ensure_utf8!(stream.read)
|
|
141
|
+
forced_encoding = 'UTF8'
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
yield stream, forced_encoding
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
# Use iconv, if available, to check raw data encoding:
|
|
148
|
+
def external_utf8_check?(safe_path)
|
|
149
|
+
iconv = system('command -v iconv > /dev/null 2>&1')
|
|
150
|
+
return false unless iconv
|
|
151
|
+
|
|
152
|
+
path = SafeFile.safepath_to_string(safe_path)
|
|
153
|
+
system("iconv -f UTF-8 #{Shellwords.escape(path)} > /dev/null 2>&1")
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
def stream_xml_nodes(io, node_xpath, encoding = nil)
|
|
157
|
+
# Track nesting as the cursor moves through the document:
|
|
158
|
+
cursor = Cursor.new(node_xpath)
|
|
159
|
+
|
|
160
|
+
# If markup isn't well-formed, try to work around it:
|
|
161
|
+
options = Nokogiri::XML::ParseOptions::RECOVER
|
|
162
|
+
reader = Nokogiri::XML::Reader(io, nil, encoding, options)
|
|
163
|
+
|
|
164
|
+
reader.each do |node|
|
|
165
|
+
case node.node_type
|
|
166
|
+
when Nokogiri::XML::Reader::TYPE_ELEMENT # "opening tag"
|
|
167
|
+
raise NestingError, node if cursor.in?(node)
|
|
168
|
+
|
|
169
|
+
cursor.enter(node)
|
|
170
|
+
next unless cursor.matches?
|
|
171
|
+
|
|
172
|
+
# The xpath matched - construct a DOM fragment to yield back:
|
|
173
|
+
element = Nokogiri::XML(node.outer_xml).at("./#{node.name}")
|
|
174
|
+
yield element
|
|
175
|
+
when Nokogiri::XML::Reader::TYPE_END_ELEMENT # "closing tag"
|
|
176
|
+
cursor.leave(node)
|
|
177
|
+
end
|
|
178
|
+
end
|
|
179
|
+
end
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
end
|
|
183
|
+
end
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
require 'shellwords'
|
|
2
|
+
|
|
1
3
|
require 'ndr_import/file/registry'
|
|
2
4
|
|
|
3
5
|
module NdrImport
|
|
@@ -5,11 +7,30 @@ module NdrImport
|
|
|
5
7
|
# complexity of enumerating over files and tables (which should be universally useful).
|
|
6
8
|
# It is assumed that the host module/class defines `unzip_path`.
|
|
7
9
|
module UniversalImporterHelper
|
|
10
|
+
# Helper class to allow multiple source enumerators to contribute to one overall table.
|
|
11
|
+
class TableEnumProxy
|
|
12
|
+
include Enumerable
|
|
13
|
+
|
|
14
|
+
def initialize
|
|
15
|
+
@table_enums = []
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def add_table_enum(table_enum)
|
|
19
|
+
@table_enums << table_enum
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def each(&block)
|
|
23
|
+
return enum_for(:each) unless block
|
|
24
|
+
|
|
25
|
+
@table_enums.each { |table_enum| table_enum.each(&block) }
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
|
|
8
29
|
def table_enumerators(filename)
|
|
9
|
-
table_enumerators = {}
|
|
30
|
+
table_enumerators = Hash.new { |hash, key| hash[key] = TableEnumProxy.new }
|
|
10
31
|
|
|
11
32
|
extract(filename).each do |table, rows|
|
|
12
|
-
table_enumerators[table.canonical_name]
|
|
33
|
+
table_enumerators[table.canonical_name].add_table_enum table.transform(rows)
|
|
13
34
|
end
|
|
14
35
|
|
|
15
36
|
table_enumerators
|
|
@@ -38,7 +59,8 @@ module NdrImport
|
|
|
38
59
|
'col_sep' => table_mapping.try(:delimiter),
|
|
39
60
|
'file_password' => table_mapping.try(:file_password),
|
|
40
61
|
'liberal_parsing' => table_mapping.try(:liberal_parsing),
|
|
41
|
-
'xml_record_xpath' => table_mapping.try(:xml_record_xpath)
|
|
62
|
+
'xml_record_xpath' => table_mapping.try(:xml_record_xpath),
|
|
63
|
+
'slurp' => table_mapping.try(:slurp)
|
|
42
64
|
}
|
|
43
65
|
|
|
44
66
|
tables = NdrImport::File::Registry.tables(filename, table_mapping.try(:format), options)
|
data/lib/ndr_import/version.rb
CHANGED
data/ndr_import.gemspec
CHANGED
|
@@ -15,7 +15,7 @@ Gem::Specification.new do |spec|
|
|
|
15
15
|
# Specify which files should be added to the gem when it is released.
|
|
16
16
|
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
|
17
17
|
spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do
|
|
18
|
-
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
|
18
|
+
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(docs|test|spec|features)/}) }
|
|
19
19
|
end
|
|
20
20
|
spec.files -= %w[.travis.yml] # Not needed in the gem
|
|
21
21
|
spec.bindir = 'exe'
|
|
@@ -26,7 +26,7 @@ Gem::Specification.new do |spec|
|
|
|
26
26
|
spec.add_dependency 'activesupport', '>= 5.0', '< 7'
|
|
27
27
|
spec.add_dependency 'ndr_support', '>= 5.3.2', '< 6'
|
|
28
28
|
|
|
29
|
-
spec.add_dependency 'rubyzip', '~>
|
|
29
|
+
spec.add_dependency 'rubyzip', '~> 2.0'
|
|
30
30
|
spec.add_dependency 'roo', '~> 2.0'
|
|
31
31
|
|
|
32
32
|
spec.add_dependency 'docx', '~> 0.3'
|
|
@@ -36,15 +36,15 @@ Gem::Specification.new do |spec|
|
|
|
36
36
|
spec.add_dependency 'pdf-reader', '~> 2.1'
|
|
37
37
|
spec.add_dependency 'roo-xls'
|
|
38
38
|
spec.add_dependency 'seven_zip_ruby', '~> 1.2'
|
|
39
|
-
spec.add_dependency 'spreadsheet', '1.
|
|
39
|
+
spec.add_dependency 'spreadsheet', '1.2.6'
|
|
40
40
|
|
|
41
41
|
spec.required_ruby_version = '>= 2.5'
|
|
42
42
|
|
|
43
43
|
spec.add_development_dependency 'bundler'
|
|
44
|
-
spec.add_development_dependency 'rake', '~>
|
|
44
|
+
spec.add_development_dependency 'rake', '~> 12.3', '>= 12.3.3'
|
|
45
45
|
spec.add_development_dependency 'minitest'
|
|
46
46
|
spec.add_development_dependency 'mocha'
|
|
47
|
-
spec.add_development_dependency 'ndr_dev_support', '
|
|
47
|
+
spec.add_development_dependency 'ndr_dev_support', '>= 3.1.3'
|
|
48
48
|
spec.add_development_dependency 'guard'
|
|
49
49
|
spec.add_development_dependency 'guard-rubocop'
|
|
50
50
|
spec.add_development_dependency 'guard-test'
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: ndr_import
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version:
|
|
4
|
+
version: 9.1.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- NCRS Development Team
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2021-02-01 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activemodel
|
|
@@ -70,20 +70,14 @@ dependencies:
|
|
|
70
70
|
requirements:
|
|
71
71
|
- - "~>"
|
|
72
72
|
- !ruby/object:Gem::Version
|
|
73
|
-
version: '
|
|
74
|
-
- - ">="
|
|
75
|
-
- !ruby/object:Gem::Version
|
|
76
|
-
version: 1.2.2
|
|
73
|
+
version: '2.0'
|
|
77
74
|
type: :runtime
|
|
78
75
|
prerelease: false
|
|
79
76
|
version_requirements: !ruby/object:Gem::Requirement
|
|
80
77
|
requirements:
|
|
81
78
|
- - "~>"
|
|
82
79
|
- !ruby/object:Gem::Version
|
|
83
|
-
version: '
|
|
84
|
-
- - ">="
|
|
85
|
-
- !ruby/object:Gem::Version
|
|
86
|
-
version: 1.2.2
|
|
80
|
+
version: '2.0'
|
|
87
81
|
- !ruby/object:Gem::Dependency
|
|
88
82
|
name: roo
|
|
89
83
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -208,14 +202,14 @@ dependencies:
|
|
|
208
202
|
requirements:
|
|
209
203
|
- - '='
|
|
210
204
|
- !ruby/object:Gem::Version
|
|
211
|
-
version: 1.
|
|
205
|
+
version: 1.2.6
|
|
212
206
|
type: :runtime
|
|
213
207
|
prerelease: false
|
|
214
208
|
version_requirements: !ruby/object:Gem::Requirement
|
|
215
209
|
requirements:
|
|
216
210
|
- - '='
|
|
217
211
|
- !ruby/object:Gem::Version
|
|
218
|
-
version: 1.
|
|
212
|
+
version: 1.2.6
|
|
219
213
|
- !ruby/object:Gem::Dependency
|
|
220
214
|
name: bundler
|
|
221
215
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -236,14 +230,20 @@ dependencies:
|
|
|
236
230
|
requirements:
|
|
237
231
|
- - "~>"
|
|
238
232
|
- !ruby/object:Gem::Version
|
|
239
|
-
version: '
|
|
233
|
+
version: '12.3'
|
|
234
|
+
- - ">="
|
|
235
|
+
- !ruby/object:Gem::Version
|
|
236
|
+
version: 12.3.3
|
|
240
237
|
type: :development
|
|
241
238
|
prerelease: false
|
|
242
239
|
version_requirements: !ruby/object:Gem::Requirement
|
|
243
240
|
requirements:
|
|
244
241
|
- - "~>"
|
|
245
242
|
- !ruby/object:Gem::Version
|
|
246
|
-
version: '
|
|
243
|
+
version: '12.3'
|
|
244
|
+
- - ">="
|
|
245
|
+
- !ruby/object:Gem::Version
|
|
246
|
+
version: 12.3.3
|
|
247
247
|
- !ruby/object:Gem::Dependency
|
|
248
248
|
name: minitest
|
|
249
249
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -276,9 +276,6 @@ dependencies:
|
|
|
276
276
|
name: ndr_dev_support
|
|
277
277
|
requirement: !ruby/object:Gem::Requirement
|
|
278
278
|
requirements:
|
|
279
|
-
- - "~>"
|
|
280
|
-
- !ruby/object:Gem::Version
|
|
281
|
-
version: '3.1'
|
|
282
279
|
- - ">="
|
|
283
280
|
- !ruby/object:Gem::Version
|
|
284
281
|
version: 3.1.3
|
|
@@ -286,9 +283,6 @@ dependencies:
|
|
|
286
283
|
prerelease: false
|
|
287
284
|
version_requirements: !ruby/object:Gem::Requirement
|
|
288
285
|
requirements:
|
|
289
|
-
- - "~>"
|
|
290
|
-
- !ruby/object:Gem::Version
|
|
291
|
-
version: '3.1'
|
|
292
286
|
- - ">="
|
|
293
287
|
- !ruby/object:Gem::Version
|
|
294
288
|
version: 3.1.3
|
|
@@ -371,6 +365,9 @@ executables:
|
|
|
371
365
|
extensions: []
|
|
372
366
|
extra_rdoc_files: []
|
|
373
367
|
files:
|
|
368
|
+
- ".github/CODEOWNERS"
|
|
369
|
+
- ".github/workflows/lint.yml"
|
|
370
|
+
- ".github/workflows/test.yml"
|
|
374
371
|
- ".gitignore"
|
|
375
372
|
- ".hound.yml"
|
|
376
373
|
- ".rubocop.yml"
|
|
@@ -413,6 +410,7 @@ files:
|
|
|
413
410
|
- lib/ndr_import/helpers/file/pdf.rb
|
|
414
411
|
- lib/ndr_import/helpers/file/word.rb
|
|
415
412
|
- lib/ndr_import/helpers/file/xml.rb
|
|
413
|
+
- lib/ndr_import/helpers/file/xml_streaming.rb
|
|
416
414
|
- lib/ndr_import/helpers/file/zip.rb
|
|
417
415
|
- lib/ndr_import/mapper.rb
|
|
418
416
|
- lib/ndr_import/mapping_error.rb
|