ndr_import 8.6.0 → 9.1.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: ffddef09a58ca08e5d491449ef48cd829dfd045bf6f38b5289cff240859940e0
4
- data.tar.gz: 30540a5af117b4ae57e002794f26df18544a99b07c7fdfb95a2ad552e9efa7b6
3
+ metadata.gz: aaf3c826acb51f4d579fb956e9606b5b86e120f3bd561db762d72081f27a1098
4
+ data.tar.gz: 2b11ddf7dc9b748b4a1ac9cc91b47c47b99677e4bde990c44d1ec20e76efbfc0
5
5
  SHA512:
6
- metadata.gz: 2ff3840cb513c4d0253c9c40db709563bf7bd8d62777ed49358e9a67a5304c03b7f18a966bb894099a4f7f9a64933bc11151048e29ce50989a791732cceee74c
7
- data.tar.gz: de437a4e4e62c77e93050df6c0d50c486d2faca290830acb1d719ddadc0b2165bd5405a11a38db4bd0ca438432b082fddafb42edaf41b09a98efcdfd79c63f55
6
+ metadata.gz: 3faaf744255693f04425b6e5ed1ec7198e912d2854b3f482a39d9bc53fe3859ed3bb6ef7855cdfd03959042a530f3a2f528239548676c7159c9029d69b6c160d
7
+ data.tar.gz: 1aa9d1895d7f874499823c4b87f1874e025b376f239030f3baf40bab5f5ed3ed09f80721205e271446893a9070652df3f26fceb48b59a9eb86ac121fdbfd92c6
@@ -0,0 +1,2 @@
1
+ # Admins should have oversight of the version:
2
+ lib/ndr_import/version.rb @publichealthengland/ndr-admins
@@ -0,0 +1,23 @@
1
+ name: Lint
2
+
3
+ on: [pull_request]
4
+
5
+ jobs:
6
+ rubocop:
7
+ name: RuboCop
8
+
9
+ runs-on: ubuntu-latest
10
+
11
+ steps:
12
+ - uses: actions/checkout@v2
13
+ with:
14
+ fetch-depth: 0 # fetch everything
15
+ - name: Set up Ruby
16
+ uses: ruby/setup-ruby@v1
17
+ with:
18
+ ruby-version: 3.0
19
+ - name: Install dependencies
20
+ run: bundle install
21
+ - name: Run RuboCop against BASE..HEAD changes
22
+ run: bundle exec rake rubocop:diff origin/${GITHUB_BASE_REF#*/}
23
+
@@ -0,0 +1,72 @@
1
+ name: Test
2
+
3
+ on: [push]
4
+
5
+ jobs:
6
+ test:
7
+ strategy:
8
+ fail-fast: false
9
+ matrix:
10
+ ruby-version:
11
+ - 2.6
12
+ - 2.7
13
+ - 3.0
14
+ gemfile:
15
+ - gemfiles/Gemfile.rails52
16
+ - gemfiles/Gemfile.rails60
17
+
18
+ name: Ruby ${{ matrix.ruby-version }} / Bundle ${{ matrix.gemfile }}
19
+
20
+ runs-on: ubuntu-latest
21
+
22
+ env:
23
+ BUNDLE_GEMFILE: ${{ matrix.gemfile }}
24
+
25
+ steps:
26
+ - uses: actions/checkout@v2
27
+ - name: Set up Ruby
28
+ uses: ruby/setup-ruby@v1
29
+ with:
30
+ ruby-version: ${{ matrix.ruby-version }}
31
+ - name: Install dependencies
32
+ run: bundle install
33
+ - name: Run tests
34
+ run: bundle exec rake
35
+
36
+ # A utility job upon which Branch Protection can depend,
37
+ # thus remaining agnostic of the matrix.
38
+ test_matrix:
39
+ if: ${{ always() }}
40
+ runs-on: ubuntu-latest
41
+ name: Matrix
42
+ needs: test
43
+ steps:
44
+ - name: Check build matrix status
45
+ if: ${{ needs.test.result != 'success' }}
46
+ run: exit 1
47
+
48
+ notify:
49
+ # Run only on master, but regardless of whether tests past:
50
+ if: ${{ always() && github.ref == 'refs/heads/master' }}
51
+
52
+ needs: test_matrix
53
+
54
+ runs-on: ubuntu-latest
55
+
56
+ steps:
57
+ - uses: 8398a7/action-slack@v3
58
+ with:
59
+ status: custom
60
+ fields: workflow,commit,author
61
+ custom_payload: |
62
+ {
63
+ channel: 'C7FQWGDHP',
64
+ username: 'CI – ' + '${{ github.repository }}'.split('/')[1],
65
+ icon_emoji: ':hammer_and_wrench:',
66
+ attachments: [{
67
+ color: '${{ needs.test_matrix.result }}' === 'success' ? 'good' : '${{ needs.test_matrix.result }}' === 'failure' ? 'danger' : 'warning',
68
+ text: `${process.env.AS_WORKFLOW} against \`${{ github.ref }}\` (${process.env.AS_COMMIT}) for ${{ github.actor }} resulted in *${{ needs.test_matrix.result }}*.`
69
+ }]
70
+ }
71
+ env:
72
+ SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
data/.gitignore CHANGED
@@ -5,7 +5,7 @@
5
5
  /gemfiles/Gemfile.*.lock
6
6
  /_yardoc/
7
7
  /coverage/
8
- /doc/
8
+ /docs/_site/
9
9
  /pkg/
10
10
  /spec/reports/
11
11
  /tmp/
data/CHANGELOG.md CHANGED
@@ -1,6 +1,40 @@
1
1
  ## [Unreleased]
2
2
  *no unreleased changes*
3
3
 
4
+ ## 9.1.0 / 2021-02-01
5
+ ### Added
6
+ * `CSVLibrary` is now deprecated.
7
+ * Handle xlsm files
8
+
9
+ ### Fixed
10
+ * Fix `CSVLibrary.foreach` on Ruby 3.0
11
+ * Updated jekyll bundle
12
+
13
+ ## 9.0.3 / 2021-01-04
14
+ ### Fixed
15
+ * Address issue importing multiple files against the same table (#54)
16
+
17
+ ### Changed
18
+ * ensure keyword args are properly splatted for ruby 2.7
19
+
20
+ ### Added
21
+ * Ruby 2.7 to travis matrix
22
+
23
+ ## 9.0.2 / 2020-08-14
24
+ ### Changed
25
+ * Configure Nokogiri with HUGE for large xml files
26
+
27
+ ## 9.0.1 / 2020-03-26
28
+ ### Fixed
29
+ * bumps to `nokogiri` / `spreadsheet` / `rubyzip` dependencies
30
+
31
+ ## 9.0.0 / 2019-07-31
32
+ ### Changed
33
+ * `File::Xml` will now stream XML files by default. Use `slurp: true` for the old behaviour. (#43)
34
+
35
+ ### Added
36
+ * Add `XmlStreaming` helper, for more performant handling of large XML documents with Nokogiri. (#43)
37
+
4
38
  ## 8.6.0 / 2019-06-07
5
39
  ### Added
6
40
  * Allow conditional preservation of blank lines when joining lines in non-tabular data (#41)
data/README.md CHANGED
@@ -1,5 +1,4 @@
1
- # NdrImport [![Build Status](https://travis-ci.org/PublicHealthEngland/ndr_import.svg?branch=master)](https://travis-ci.org/PublicHealthEngland/ndr_import) [![Gem Version](https://badge.fury.io/rb/ndr_import.svg)](https://badge.fury.io/rb/ndr_import)
2
-
1
+ # NdrImport [![Build Status](https://github.com/publichealthengland/ndr_import/workflows/Test/badge.svg)](https://github.com/publichealthengland/ndr_import/actions?query=workflow%3Atest) [![Gem Version](https://badge.fury.io/rb/ndr_import.svg)](https://rubygems.org/gems/ndr_import) [![Documentation](https://img.shields.io/badge/ndr_import-docs-blue.svg)](https://www.rubydoc.info/gems/ndr_import)
3
2
  This is the Public Health England (PHE) National Disease Registers (NDR) Import ETL ruby gem, providing:
4
3
 
5
4
  1. file import handlers for *extracting* data from delimited files (csv, pipe, tab, thorn), .xls(x) spreadsheets, .doc(x) word documents, PDF, PDF AcroForms, XML, 7-Zip and Zip files.
data/code_safety.yml CHANGED
@@ -1,9 +1,21 @@
1
1
  ---
2
2
  file safety:
3
+ ".github/CODEOWNERS":
4
+ comments:
5
+ reviewed_by: ollietulloch
6
+ safe_revision: b64ff21375dcde2b8fefe622ee9861f0fea21487
7
+ ".github/workflows/lint.yml":
8
+ comments:
9
+ reviewed_by: ollietulloch
10
+ safe_revision: b64ff21375dcde2b8fefe622ee9861f0fea21487
11
+ ".github/workflows/test.yml":
12
+ comments:
13
+ reviewed_by: ollietulloch
14
+ safe_revision: b64ff21375dcde2b8fefe622ee9861f0fea21487
3
15
  ".gitignore":
4
16
  comments: whole file re-reviewed
5
17
  reviewed_by: josh.pencheon
6
- safe_revision: 3ef51291c413fd5772d61a8394359146a02ae628
18
+ safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
7
19
  ".hound.yml":
8
20
  comments:
9
21
  reviewed_by: timgentry
@@ -12,14 +24,10 @@ file safety:
12
24
  comments:
13
25
  reviewed_by: josh.pencheon
14
26
  safe_revision: b09e268ff9c8349b914aa1b7ba888e1d39f97e4a
15
- ".travis.yml":
16
- comments:
17
- reviewed_by: josh.pencheon
18
- safe_revision: d3d9a987befeecb122a448d8d06e66d74da13fb5
19
27
  CHANGELOG.md:
20
28
  comments:
21
- reviewed_by: josh.pencheon
22
- safe_revision: 53c17f14ee19a9ea83d044508093147daa32ba3d
29
+ reviewed_by: ollietulloch
30
+ safe_revision: d88ded7c260da37200610e4f0b204a4ea2e481f9
23
31
  CODE_OF_CONDUCT.md:
24
32
  comments:
25
33
  reviewed_by: timgentry
@@ -38,8 +46,8 @@ file safety:
38
46
  safe_revision: 5d185a0aeba6a9cd2ff5e59efadcaeec9be45d8b
39
47
  README.md:
40
48
  comments:
41
- reviewed_by: josh.pencheon
42
- safe_revision: e1d967c10059e8c635452838c3f3dd2b969d9ae4
49
+ reviewed_by: ollietulloch
50
+ safe_revision: b64ff21375dcde2b8fefe622ee9861f0fea21487
43
51
  Rakefile:
44
52
  comments:
45
53
  reviewed_by: josh.pencheon
@@ -52,6 +60,70 @@ file safety:
52
60
  comments:
53
61
  reviewed_by: josh.pencheon
54
62
  safe_revision: e1d967c10059e8c635452838c3f3dd2b969d9ae4
63
+ docs/Gemfile:
64
+ comments:
65
+ reviewed_by: josh.pencheon
66
+ safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
67
+ docs/Gemfile.lock:
68
+ comments:
69
+ reviewed_by: ollietulloch
70
+ safe_revision: ea0149c7739676463a252ffd9fbe4af238762b2b
71
+ docs/_config.yml:
72
+ comments:
73
+ reviewed_by: josh.pencheon
74
+ safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
75
+ docs/_includes/footer.html:
76
+ comments:
77
+ reviewed_by: josh.pencheon
78
+ safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
79
+ docs/_includes/header.html:
80
+ comments:
81
+ reviewed_by: josh.pencheon
82
+ safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
83
+ docs/capturing-data.md:
84
+ comments:
85
+ reviewed_by: josh.pencheon
86
+ safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
87
+ docs/date-formats.md:
88
+ comments:
89
+ reviewed_by: josh.pencheon
90
+ safe_revision: fa21d6d967bf132800b456b585795beec80b08a3
91
+ docs/getting-started.md:
92
+ comments:
93
+ reviewed_by: josh.pencheon
94
+ safe_revision: fa21d6d967bf132800b456b585795beec80b08a3
95
+ docs/identifying-and-splitting-records.md:
96
+ comments:
97
+ reviewed_by: josh.pencheon
98
+ safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
99
+ docs/inbuilt-cleaning-methods.md:
100
+ comments:
101
+ reviewed_by: josh.pencheon
102
+ safe_revision: 694b57ce14e0709fc4d31a1357f8416e98f5de91
103
+ docs/index.md:
104
+ comments:
105
+ reviewed_by: josh.pencheon
106
+ safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
107
+ docs/local-code-transformation-in-yaml-mappings.md:
108
+ comments:
109
+ reviewed_by: josh.pencheon
110
+ safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
111
+ docs/non-tabular-mappings.md:
112
+ comments:
113
+ reviewed_by: josh.pencheon
114
+ safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
115
+ docs/priority-field-mapping.md:
116
+ comments:
117
+ reviewed_by: josh.pencheon
118
+ safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
119
+ docs/standard-yaml-mappings.md:
120
+ comments:
121
+ reviewed_by: josh.pencheon
122
+ safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
123
+ docs/yaml-mapping-user-guide.md:
124
+ comments:
125
+ reviewed_by: josh.pencheon
126
+ safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
55
127
  exe/pdf_acro_form_to_yaml:
56
128
  comments:
57
129
  reviewed_by: josh.pencheon
@@ -82,8 +154,8 @@ file safety:
82
154
  safe_revision: 24d6449fd0612552f132dfbf4cada2ae28d0469e
83
155
  lib/ndr_import/csv_library.rb:
84
156
  comments:
85
- reviewed_by: josh.pencheon
86
- safe_revision: be12e57519d3737e8d3901d7b01485c6995708dd
157
+ reviewed_by: ollietulloch
158
+ safe_revision: 6b8668967dbd42d7893a0fa5f0aa1ec1c11227e1
87
159
  lib/ndr_import/file/acro_form.rb:
88
160
  comments:
89
161
  reviewed_by: josh.pencheon
@@ -106,8 +178,8 @@ file safety:
106
178
  safe_revision: 897f8b648d633368cf2001d17ab89c06a12d445b
107
179
  lib/ndr_import/file/excel.rb:
108
180
  comments:
109
- reviewed_by: josh.pencheon
110
- safe_revision: c3183e522bce50008df576ceb47fe4761ab8f966
181
+ reviewed_by: ollietulloch
182
+ safe_revision: 37482c79448bea80033f6f69d97584df330c9861
111
183
  lib/ndr_import/file/office_file_helper.rb:
112
184
  comments:
113
185
  reviewed_by: josh.pencheon
@@ -139,7 +211,7 @@ file safety:
139
211
  lib/ndr_import/file/xml.rb:
140
212
  comments:
141
213
  reviewed_by: josh.pencheon
142
- safe_revision: 4ab72f84201c2d5f0147b7dfd041f488f6ff0422
214
+ safe_revision: ae75fb49baf028ac8ce08e4bedcd3625ff3ff0cd
143
215
  lib/ndr_import/file/zip.rb:
144
216
  comments:
145
217
  reviewed_by: timgentry
@@ -150,8 +222,8 @@ file safety:
150
222
  safe_revision: dfc958d44b6c58355445fa395db08a62213ee709
151
223
  lib/ndr_import/helpers/file/delimited.rb:
152
224
  comments:
153
- reviewed_by: josh.pencheon
154
- safe_revision: 607c0668f1fffd70d181bc1a31c4f56eed5f6189
225
+ reviewed_by: ollietulloch
226
+ safe_revision: 4a5cc1d362c632fc1f9242c69982fbce33557e17
155
227
  lib/ndr_import/helpers/file/excel.rb:
156
228
  comments:
157
229
  reviewed_by: joshpencheon
@@ -166,8 +238,13 @@ file safety:
166
238
  safe_revision: 45da71ebd3acbc0fe53755bcd75483ba17cb6924
167
239
  lib/ndr_import/helpers/file/xml.rb:
168
240
  comments:
241
+ reviewed_by: ollietulloch
242
+ safe_revision: 4d337bd233f7e60cf9d363c92400f21269a28da7
243
+ lib/ndr_import/helpers/file/xml_streaming.rb:
244
+ comments: uses SafePath and Shellwords when accessing filesystem, or making system
245
+ calls
169
246
  reviewed_by: josh.pencheon
170
- safe_revision: d2245268ec6a0e4f60c521d171a820f299632c4f
247
+ safe_revision: 55e502bb4445cb8b985e530e8eb26d92b574ded9
171
248
  lib/ndr_import/helpers/file/zip.rb:
172
249
  comments:
173
250
  reviewed_by: timgentry
@@ -222,16 +299,16 @@ file safety:
222
299
  safe_revision: a69d4a57ddcf13cdc13c27bd2eb91a395fa7ea36
223
300
  lib/ndr_import/universal_importer_helper.rb:
224
301
  comments:
225
- reviewed_by: josh.pencheon
226
- safe_revision: a69d4a57ddcf13cdc13c27bd2eb91a395fa7ea36
302
+ reviewed_by: ollietulloch
303
+ safe_revision: ee2e74e4ceda4ff48cbda6872a6bdf0874212c21
227
304
  lib/ndr_import/unmapped_data_error.rb:
228
305
  comments:
229
306
  reviewed_by: josh.pencheon
230
307
  safe_revision: 5cd2cd0b3a1e254d30d4acc28c6731825a1f84f5
231
308
  lib/ndr_import/version.rb:
232
309
  comments: another check?
233
- reviewed_by: josh.pencheon
234
- safe_revision: 53c17f14ee19a9ea83d044508093147daa32ba3d
310
+ reviewed_by: ollietulloch
311
+ safe_revision: d88ded7c260da37200610e4f0b204a4ea2e481f9
235
312
  lib/ndr_import/xml/table.rb:
236
313
  comments:
237
314
  reviewed_by: josh.pencheon
@@ -239,7 +316,11 @@ file safety:
239
316
  ndr_import.gemspec:
240
317
  comments:
241
318
  reviewed_by: josh.pencheon
242
- safe_revision: d3d9a987befeecb122a448d8d06e66d74da13fb5
319
+ safe_revision: 95e6ee9997d06471fe6f2f169c3c701471086371
320
+ test/csv_library_test.rb:
321
+ comments:
322
+ reviewed_by: ollietulloch
323
+ safe_revision: 6b8668967dbd42d7893a0fa5f0aa1ec1c11227e1
243
324
  test/file/acro_form_test.rb:
244
325
  comments:
245
326
  reviewed_by: josh.pencheon
@@ -258,16 +339,16 @@ file safety:
258
339
  safe_revision: a69d4a57ddcf13cdc13c27bd2eb91a395fa7ea36
259
340
  test/file/excel_test.rb:
260
341
  comments:
261
- reviewed_by: josh.pencheon
262
- safe_revision: a69d4a57ddcf13cdc13c27bd2eb91a395fa7ea36
342
+ reviewed_by: ollietulloch
343
+ safe_revision: 85a080deaa93e4220ad1bf566f29cbdac9b31c0f
263
344
  test/file/pdf_test.rb:
264
345
  comments:
265
346
  reviewed_by: josh.pencheon
266
347
  safe_revision: cb24ed3ea8116730d07f74546cd6fed0738b171d
267
348
  test/file/registry_test.rb:
268
349
  comments:
269
- reviewed_by: josh.pencheon
270
- safe_revision: 5cd2cd0b3a1e254d30d4acc28c6731825a1f84f5
350
+ reviewed_by: ollietulloch
351
+ safe_revision: 85a080deaa93e4220ad1bf566f29cbdac9b31c0f
271
352
  test/file/seven_zip_test.rb:
272
353
  comments:
273
354
  reviewed_by: josh.pencheon
@@ -283,7 +364,7 @@ file safety:
283
364
  test/file/xml_test.rb:
284
365
  comments:
285
366
  reviewed_by: josh.pencheon
286
- safe_revision: 4ab72f84201c2d5f0147b7dfd041f488f6ff0422
367
+ safe_revision: ae75fb49baf028ac8ce08e4bedcd3625ff3ff0cd
287
368
  test/file/zip_test.rb:
288
369
  comments:
289
370
  reviewed_by: timgentry
@@ -308,6 +389,10 @@ file safety:
308
389
  comments:
309
390
  reviewed_by: timgentry
310
391
  safe_revision: 9abdd6ced1d0c90ce8dd88abee4eb6472c7ff0d6
392
+ test/helpers/file/xml_streaming_test.rb:
393
+ comments:
394
+ reviewed_by: josh.pencheon
395
+ safe_revision: ae75fb49baf028ac8ce08e4bedcd3625ff3ff0cd
311
396
  test/helpers/file/xml_test.rb:
312
397
  comments:
313
398
  reviewed_by: timgentry
@@ -356,6 +441,10 @@ file safety:
356
441
  comments:
357
442
  reviewed_by: timgentry
358
443
  safe_revision: dab4b8a3e4b29d85eccd971e79936982d888cffd
444
+ test/resources/claims_utf16be_but_isnt.xml:
445
+ comments:
446
+ reviewed_by: josh.pencheon
447
+ safe_revision: ae75fb49baf028ac8ce08e4bedcd3625ff3ff0cd
359
448
  test/resources/filesystem_paths.yml:
360
449
  comments:
361
450
  reviewed_by: timgentry
@@ -476,6 +565,10 @@ file safety:
476
565
  comments:
477
566
  reviewed_by: timgentry
478
567
  safe_revision: 8c30f89f0562ab120769c166d4e93ff839c055f7
568
+ test/resources/sample_xlsm.xlsm:
569
+ comments:
570
+ reviewed_by: ollietulloch
571
+ safe_revision: 85a080deaa93e4220ad1bf566f29cbdac9b31c0f
479
572
  test/resources/sample_xlsx.xlsx:
480
573
  comments:
481
574
  reviewed_by: timgentry
@@ -492,6 +585,10 @@ file safety:
492
585
  comments:
493
586
  reviewed_by: timgentry
494
587
  safe_revision: 31fb1935f4578729d8786eea41cf0ce0a19be1cd
588
+ test/resources/two_files_single_table_mapping.zip:
589
+ comments:
590
+ reviewed_by: ollietulloch
591
+ safe_revision: 830de0f8cb139c5f61525652b424423935cfc7ac
495
592
  test/resources/txt_file_xls_extension.xls:
496
593
  comments:
497
594
  reviewed_by: timgentry
@@ -542,8 +639,8 @@ file safety:
542
639
  safe_revision: 93ccee82fc2165d1ca2d9b03d146ae03e769ea96
543
640
  test/universal_importer_helper_test.rb:
544
641
  comments:
545
- reviewed_by: josh.pencheon
546
- safe_revision: c3183e522bce50008df576ceb47fe4761ab8f966
642
+ reviewed_by: ollietulloch
643
+ safe_revision: 830de0f8cb139c5f61525652b424423935cfc7ac
547
644
  test/xml/table_test.rb:
548
645
  comments:
549
646
  reviewed_by: josh.pencheon
@@ -1,38 +1,60 @@
1
1
  # This file allows us to choose the CSV library we want to use.
2
2
 
3
3
  require 'csv'
4
+ require 'active_support/deprecation'
5
+
4
6
  # Using relevant core CSV library.
5
- CSVLibrary = CSV
7
+ class CSVLibrary < CSV; end
6
8
 
7
9
  class << CSVLibrary
8
10
  # Is the library we're using FasterCSV?
9
11
  def fastercsv?
12
+ deprecate('if you desparately want fastercsv, please use it explicitly')
10
13
  not self.const_defined?(:Reader)
11
14
  end
12
15
 
13
16
  # Ensure that we can pass "mode" straight through the underlying IO object
17
+ #
18
+ # Note: this could likely be refactored now, as upstream support for something
19
+ # very similar was added:
20
+ #
21
+ # https://github.com/ruby/csv/commit/b4edaf2cf1aa36f5c6264c07514b66739b87ceee
22
+ #
14
23
  def foreach(path, **options, &block)
15
- return to_enum(__method__, path, options) unless block
16
- open(path, options.delete(:mode) || 'r', options) do |csv|
24
+ deprecate('CSV#foreach exists, with an optional `mode` argument')
25
+ return to_enum(__method__, path, **options) unless block
26
+ open(path, options.delete(:mode) || 'r', **options) do |csv|
17
27
  csv.each(&block)
18
28
  end
19
29
  end
20
30
 
21
31
  def write_csv_to_string(data)
32
+ deprecate('write_csv_to_string -> generate')
22
33
  self.generate do |csv|
23
34
  data.each { |line| csv << line }
24
35
  end
25
36
  end
26
37
 
27
38
  def write_csv_to_file(data, filepath, mode = 'w')
39
+ deprecate('write_csv_to_file -> open')
28
40
  self.open(filepath, mode) do |csv|
29
41
  data.each { |line| csv << line }
30
42
  end
31
43
  end
32
44
 
33
45
  def read_csv_from_file(filepath)
46
+ deprecate('read_csv_from_file -> read')
34
47
  self.read(filepath)
35
48
  end
49
+
50
+ private
51
+
52
+ def deprecate(additional_message = nil)
53
+ ActiveSupport::Deprecation.warn(<<~MESSAGE)
54
+ CSVLibrary is deprecated, and will be removed in a future version of ndr_import.
55
+ Please use standard functionality provided by Ruby's CSV library (#{additional_message}).
56
+ MESSAGE
57
+ end
36
58
  end
37
59
 
38
60
  # Forward port CSV::Cell, as it is sometimes
@@ -90,14 +90,14 @@ module NdrImport
90
90
  case SafeFile.extname(path).downcase
91
91
  when '.xls'
92
92
  Roo::Excel.new(SafeFile.safepath_to_string(path))
93
- when '.xlsx'
93
+ when '.xlsm', '.xlsx'
94
94
  if @options['file_password']
95
95
  Roo::Excelx.new(StringIO.new(decrypted_file_string(path, @options['file_password'])))
96
96
  else
97
97
  Roo::Excelx.new(SafeFile.safepath_to_string(path))
98
98
  end
99
99
  else
100
- fail "Received file path with unexpected extension #{SafeFile.extname(path)}"
100
+ raise "Received file path with unexpected extension #{SafeFile.extname(path)}"
101
101
  end
102
102
  rescue Ole::Storage::FormatError => e
103
103
  # TODO: Do we need to remove the new_file after using it?
@@ -105,16 +105,14 @@ module NdrImport
105
105
  # try to load the .xls file as an .xlsx file, useful for sources like USOM
106
106
  # roo check file extensions in file_type_check (GenericSpreadsheet),
107
107
  # so we create a duplicate file in xlsx extension
108
- if /(.*)\.xls$/.match(path)
109
- new_file_name = SafeFile.basename(path).gsub(/(.*)\.xls$/, '\1_amend.xlsx')
110
- new_file_path = SafeFile.dirname(path).join(new_file_name)
111
- copy_file(path, new_file_path)
108
+ raise e.message unless /(.*)\.xls$/.match(path)
112
109
 
113
- load_workbook(new_file_path)
114
- else
115
- raise e.message
116
- end
117
- rescue => e
110
+ new_file_name = SafeFile.basename(path).gsub(/(.*)\.xls$/, '\1_amend.xlsx')
111
+ new_file_path = SafeFile.dirname(path).join(new_file_name)
112
+ copy_file(path, new_file_path)
113
+
114
+ load_workbook(new_file_path)
115
+ rescue RuntimeError, ::Zip::Error => e
118
116
  raise ["Unable to read the file '#{path}'", e.message].join('; ')
119
117
  end
120
118
 
@@ -133,6 +131,6 @@ module NdrImport
133
131
  end
134
132
  end
135
133
 
136
- Registry.register(Excel, 'xls', 'xlsx')
134
+ Registry.register(Excel, 'xls', 'xlsm', 'xlsx')
137
135
  end
138
136
  end
@@ -1,5 +1,6 @@
1
1
  require 'ndr_support/safe_file'
2
2
  require 'ndr_import/helpers/file/xml'
3
+ require 'ndr_import/helpers/file/xml_streaming'
3
4
  require_relative 'registry'
4
5
 
5
6
  module NdrImport
@@ -9,6 +10,7 @@ module NdrImport
9
10
  # This class is a xml file handler that returns a single table.
10
11
  class Xml < Base
11
12
  include NdrImport::Helpers::File::Xml
13
+ include NdrImport::Helpers::File::XmlStreaming
12
14
 
13
15
  private
14
16
 
@@ -16,9 +18,14 @@ module NdrImport
16
18
  def rows(&block)
17
19
  return enum_for(:rows) unless block
18
20
 
19
- doc = read_xml_file(@filename)
21
+ xpath = @options['xml_record_xpath']
20
22
 
21
- doc.xpath(@options['xml_record_xpath']).each(&block)
23
+ if @options['slurp']
24
+ doc = read_xml_file(@filename)
25
+ doc.xpath(xpath).each(&block)
26
+ else
27
+ each_node(@filename, xpath, &block)
28
+ end
22
29
  rescue StandardError => e
23
30
  raise("#{SafeFile.basename(@filename)} [#{e.class}: #{e.message}]")
24
31
  end
@@ -32,11 +32,11 @@ module NdrImport
32
32
  return enum_for(:delimited_rows, path, col_sep, liberal) unless block_given?
33
33
 
34
34
  safe_path = SafeFile.safepath_to_string(path)
35
- encodings = determine_encodings!(safe_path, col_sep, liberal)
35
+ options = determine_encodings!(safe_path, col_sep, liberal)
36
36
 
37
- # By now, we know `encodings` should let us read the whole
37
+ # By now, we know `options` should let us read the whole
38
38
  # file succesfully; if there are problems, we should crash.
39
- CSVLibrary.foreach(safe_path, encodings) do |line|
39
+ CSV.foreach(safe_path, options.delete(:mode), **options) do |line|
40
40
  yield line.map(&:to_s)
41
41
  end
42
42
  end
@@ -46,7 +46,7 @@ module NdrImport
46
46
  # Derive the source encoding by trying all supported encodings.
47
47
  # Returns first set of working options, or raises if none could be found.
48
48
  def determine_encodings!(safe_path, col_sep, liberal)
49
- # delimiter encoding => # FasterCSV encoding string
49
+ # delimiter encoding => # CSV encoding string
50
50
  supported_encodings = {
51
51
  'UTF-8' => 'r:bom|utf-8',
52
52
  'Windows-1252' => 'r:windows-1252:utf-8'
@@ -67,14 +67,13 @@ module NdrImport
67
67
  begin
68
68
  options = {
69
69
  col_sep: (col_sep || ',').force_encoding(delimiter_encoding),
70
- liberal_parsing: liberal,
71
- mode: access_mode
70
+ liberal_parsing: liberal
72
71
  }
73
72
 
74
73
  row_num = 0
75
74
  # Iterate through the file; if we reach the end, this encoding worked:
76
- CSVLibrary.foreach(safe_path, options) { |_line| row_num += 1 }
77
- return options
75
+ CSV.foreach(safe_path, access_mode, **options) { |_line| row_num += 1 }
76
+ return options.merge(mode: access_mode)
78
77
  rescue ArgumentError => e
79
78
  next if e.message =~ /invalid byte sequence/ # This encoding didn't work
80
79
  raise(e)
@@ -15,10 +15,13 @@ module NdrImport
15
15
 
16
16
  require 'nokogiri'
17
17
 
18
- Nokogiri::XML(ensure_utf8! file_data).tap do |doc|
19
- doc.encoding = 'UTF-8'
20
- emulate_strict_mode_fatal_check!(doc)
18
+ doc = Nokogiri::XML((ensure_utf8! file_data)) do |config|
19
+ config.huge
21
20
  end
21
+ doc.encoding = 'UTF-8'
22
+ emulate_strict_mode_fatal_check!(doc)
23
+
24
+ doc
22
25
  end
23
26
 
24
27
  # Nokogiri can use give a `STRICT` parse option to libxml, but our friendly
@@ -0,0 +1,183 @@
1
+ require 'shellwords'
2
+
3
+ require 'ndr_support/safe_file'
4
+ require 'ndr_support/utf8_encoding'
5
+
6
+ module NdrImport
7
+ module Helpers
8
+ module File
9
+ # This mixin adds XML streaming functionality, to support more performant handling
10
+ # of large files by Nokogiri. Uses the `XML::Reader` API, and maintains a temporary
11
+ # DOM as the XML is streamed to allow XPath querying from the root node.
12
+ #
13
+ # If the system has `iconv` available, will attempt to verify the encoding of the
14
+ # file being read externally, so it can be streamed in to Ruby. Otherwise, will load
15
+ # the raw data in to check the encoding, but still stream it through Nokogiri's parser.
16
+ module XmlStreaming
17
+ # Base error for all streaming-specific issues.
18
+ class Error < StandardError; end
19
+
20
+ # Raised if nested tags are accounted which the streaming approach cannnot handle.
21
+ class NestingError < Error
22
+ def initialize(node)
23
+ super <<~STR
24
+ Element '#{node.name}' was found nested inside another of the same type.
25
+ This is not accessible, and a known limitation of XmlStreaming.
26
+ STR
27
+ end
28
+ end
29
+
30
+ # Object to track state as the XML is iterated over, and detect
31
+ # when an element of interest is entered.
32
+ class Cursor
33
+ # wrapper to hold a representation of each element we descent into:
34
+ StackItem = Struct.new(:name, :attrs, :empty)
35
+
36
+ def initialize(xpath)
37
+ @xpath = xpath
38
+ @stack = []
39
+ @match_depth = nil
40
+ end
41
+
42
+ # Has this cursor already passed inside a similar node?
43
+ def in?(node)
44
+ @stack.detect { |item| item.name == node.name }
45
+ end
46
+
47
+ def enter(node)
48
+ @stack.push StackItem.new(node.name, node.attributes, node.empty_element?)
49
+ end
50
+
51
+ def leave(_node)
52
+ @stack.pop
53
+ @match_depth = nil if @match_depth && @stack.length < @match_depth
54
+ end
55
+
56
+ # Does the element that the cursor is currently on match what
57
+ # is being looked for?
58
+ def matches?
59
+ # Can't match again if we're inside a match already:
60
+ return false if @matched_depth
61
+
62
+ match = current_stack_match?
63
+
64
+ # "empty element" matches are yielded immediately, without
65
+ # tagging the stack as having matched, because there won't
66
+ # be an equivalent closing tag to end the match with later.
67
+ if in_empty_element?
68
+ @stack.pop
69
+ elsif match
70
+ @match_depth = @stack.length
71
+ end
72
+
73
+ match
74
+ end
75
+
76
+ private
77
+
78
+ def in_empty_element?
79
+ @stack.last.empty
80
+ end
81
+
82
+ # Does the current state of the stack mean we've met the xpath
83
+ # criteria? Must be an exact match, not just matching a parent
84
+ # element in the DOM.
85
+ def current_stack_match?
86
+ parent_stack = @stack[0..-2]
87
+
88
+ return false unless dom_stubs[@stack].at_xpath(@xpath)
89
+
90
+ parent_stack.empty? || !dom_stubs[parent_stack].at_xpath(@xpath)
91
+ end
92
+
93
+ # A cached collection of DOM fragments, to represent the structure
94
+ # necessary to use xpath to descend into the main document's DOM.
95
+ def dom_stubs
96
+ @dom_stubs ||= Hash.new do |hash, items|
97
+ hash[items.dup] = Nokogiri::XML::Builder.new do |dom|
98
+ add_items_to_dom(dom, items.dup)
99
+ end.doc
100
+ end
101
+ end
102
+
103
+ # Helper to recursively build XML fragment.
104
+ def add_items_to_dom(dom, items)
105
+ item = items.shift
106
+ dom.send(item.name, item.attrs) do
107
+ add_items_to_dom(dom, items) if items.any?
108
+ end
109
+ end
110
+ end
111
+
112
+ include UTF8Encoding
113
+
114
+ # Streams the contents of the given `safe_path`, and yields
115
+ # each element matching `xpath` as they're found.
116
+ #
117
+ # In the case of dodgy encoding, may fall back to slurping the
118
+ # file, but will still use stream parsing for XML.
119
+ def each_node(safe_path, xpath, &block)
120
+ return enum_for(:each_node, safe_path, xpath) unless block
121
+
122
+ require 'nokogiri'
123
+
124
+ with_encoding_check(safe_path) do |stream, encoding|
125
+ stream_xml_nodes(stream, xpath, encoding, &block)
126
+ end
127
+ end
128
+
129
+ private
130
+
131
+ # We need to ensure the raw data is UTF8 before we start streaming
132
+ # it with nokogiri. If we can do an external check, great. Otherwise,
133
+ # we need to slurp and convert the raw data before presenting it.
134
+ def with_encoding_check(safe_path)
135
+ forced_encoding = nil
136
+
137
+ stream = ::File.open(SafeFile.safepath_to_string(safe_path))
138
+
139
+ unless external_utf8_check?(safe_path)
140
+ stream = StringIO.new ensure_utf8!(stream.read)
141
+ forced_encoding = 'UTF8'
142
+ end
143
+
144
+ yield stream, forced_encoding
145
+ end
146
+
147
+ # Use iconv, if available, to check raw data encoding:
148
+ def external_utf8_check?(safe_path)
149
+ iconv = system('command -v iconv > /dev/null 2>&1')
150
+ return false unless iconv
151
+
152
+ path = SafeFile.safepath_to_string(safe_path)
153
+ system("iconv -f UTF-8 #{Shellwords.escape(path)} > /dev/null 2>&1")
154
+ end
155
+
156
+ def stream_xml_nodes(io, node_xpath, encoding = nil)
157
+ # Track nesting as the cursor moves through the document:
158
+ cursor = Cursor.new(node_xpath)
159
+
160
+ # If markup isn't well-formed, try to work around it:
161
+ options = Nokogiri::XML::ParseOptions::RECOVER
162
+ reader = Nokogiri::XML::Reader(io, nil, encoding, options)
163
+
164
+ reader.each do |node|
165
+ case node.node_type
166
+ when Nokogiri::XML::Reader::TYPE_ELEMENT # "opening tag"
167
+ raise NestingError, node if cursor.in?(node)
168
+
169
+ cursor.enter(node)
170
+ next unless cursor.matches?
171
+
172
+ # The xpath matched - construct a DOM fragment to yield back:
173
+ element = Nokogiri::XML(node.outer_xml).at("./#{node.name}")
174
+ yield element
175
+ when Nokogiri::XML::Reader::TYPE_END_ELEMENT # "closing tag"
176
+ cursor.leave(node)
177
+ end
178
+ end
179
+ end
180
+ end
181
+ end
182
+ end
183
+ end
@@ -1,3 +1,5 @@
1
+ require 'shellwords'
2
+
1
3
  require 'ndr_import/file/registry'
2
4
 
3
5
  module NdrImport
@@ -5,11 +7,30 @@ module NdrImport
5
7
  # complexity of enumerating over files and tables (which should be universally useful).
6
8
  # It is assumed that the host module/class defines `unzip_path`.
7
9
  module UniversalImporterHelper
10
+ # Helper class to allow multiple source enumerators to contribute to one overall table.
11
+ class TableEnumProxy
12
+ include Enumerable
13
+
14
+ def initialize
15
+ @table_enums = []
16
+ end
17
+
18
+ def add_table_enum(table_enum)
19
+ @table_enums << table_enum
20
+ end
21
+
22
+ def each(&block)
23
+ return enum_for(:each) unless block
24
+
25
+ @table_enums.each { |table_enum| table_enum.each(&block) }
26
+ end
27
+ end
28
+
8
29
  def table_enumerators(filename)
9
- table_enumerators = {}
30
+ table_enumerators = Hash.new { |hash, key| hash[key] = TableEnumProxy.new }
10
31
 
11
32
  extract(filename).each do |table, rows|
12
- table_enumerators[table.canonical_name] = table.transform(rows)
33
+ table_enumerators[table.canonical_name].add_table_enum table.transform(rows)
13
34
  end
14
35
 
15
36
  table_enumerators
@@ -38,7 +59,8 @@ module NdrImport
38
59
  'col_sep' => table_mapping.try(:delimiter),
39
60
  'file_password' => table_mapping.try(:file_password),
40
61
  'liberal_parsing' => table_mapping.try(:liberal_parsing),
41
- 'xml_record_xpath' => table_mapping.try(:xml_record_xpath)
62
+ 'xml_record_xpath' => table_mapping.try(:xml_record_xpath),
63
+ 'slurp' => table_mapping.try(:slurp)
42
64
  }
43
65
 
44
66
  tables = NdrImport::File::Registry.tables(filename, table_mapping.try(:format), options)
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
  # This stores the current version of the NdrImport gem
3
3
  module NdrImport
4
- VERSION = '8.6.0'.freeze
4
+ VERSION = '9.1.0'.freeze
5
5
  end
data/ndr_import.gemspec CHANGED
@@ -15,7 +15,7 @@ Gem::Specification.new do |spec|
15
15
  # Specify which files should be added to the gem when it is released.
16
16
  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
17
17
  spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do
18
- `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
18
+ `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(docs|test|spec|features)/}) }
19
19
  end
20
20
  spec.files -= %w[.travis.yml] # Not needed in the gem
21
21
  spec.bindir = 'exe'
@@ -26,7 +26,7 @@ Gem::Specification.new do |spec|
26
26
  spec.add_dependency 'activesupport', '>= 5.0', '< 7'
27
27
  spec.add_dependency 'ndr_support', '>= 5.3.2', '< 6'
28
28
 
29
- spec.add_dependency 'rubyzip', '~> 1.2', '>= 1.2.2'
29
+ spec.add_dependency 'rubyzip', '~> 2.0'
30
30
  spec.add_dependency 'roo', '~> 2.0'
31
31
 
32
32
  spec.add_dependency 'docx', '~> 0.3'
@@ -36,15 +36,15 @@ Gem::Specification.new do |spec|
36
36
  spec.add_dependency 'pdf-reader', '~> 2.1'
37
37
  spec.add_dependency 'roo-xls'
38
38
  spec.add_dependency 'seven_zip_ruby', '~> 1.2'
39
- spec.add_dependency 'spreadsheet', '1.0.3'
39
+ spec.add_dependency 'spreadsheet', '1.2.6'
40
40
 
41
41
  spec.required_ruby_version = '>= 2.5'
42
42
 
43
43
  spec.add_development_dependency 'bundler'
44
- spec.add_development_dependency 'rake', '~> 10.0'
44
+ spec.add_development_dependency 'rake', '~> 12.3', '>= 12.3.3'
45
45
  spec.add_development_dependency 'minitest'
46
46
  spec.add_development_dependency 'mocha'
47
- spec.add_development_dependency 'ndr_dev_support', '~> 3.1', '>= 3.1.3'
47
+ spec.add_development_dependency 'ndr_dev_support', '>= 3.1.3'
48
48
  spec.add_development_dependency 'guard'
49
49
  spec.add_development_dependency 'guard-rubocop'
50
50
  spec.add_development_dependency 'guard-test'
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: ndr_import
3
3
  version: !ruby/object:Gem::Version
4
- version: 8.6.0
4
+ version: 9.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - NCRS Development Team
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2019-06-07 00:00:00.000000000 Z
11
+ date: 2021-02-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activemodel
@@ -70,20 +70,14 @@ dependencies:
70
70
  requirements:
71
71
  - - "~>"
72
72
  - !ruby/object:Gem::Version
73
- version: '1.2'
74
- - - ">="
75
- - !ruby/object:Gem::Version
76
- version: 1.2.2
73
+ version: '2.0'
77
74
  type: :runtime
78
75
  prerelease: false
79
76
  version_requirements: !ruby/object:Gem::Requirement
80
77
  requirements:
81
78
  - - "~>"
82
79
  - !ruby/object:Gem::Version
83
- version: '1.2'
84
- - - ">="
85
- - !ruby/object:Gem::Version
86
- version: 1.2.2
80
+ version: '2.0'
87
81
  - !ruby/object:Gem::Dependency
88
82
  name: roo
89
83
  requirement: !ruby/object:Gem::Requirement
@@ -208,14 +202,14 @@ dependencies:
208
202
  requirements:
209
203
  - - '='
210
204
  - !ruby/object:Gem::Version
211
- version: 1.0.3
205
+ version: 1.2.6
212
206
  type: :runtime
213
207
  prerelease: false
214
208
  version_requirements: !ruby/object:Gem::Requirement
215
209
  requirements:
216
210
  - - '='
217
211
  - !ruby/object:Gem::Version
218
- version: 1.0.3
212
+ version: 1.2.6
219
213
  - !ruby/object:Gem::Dependency
220
214
  name: bundler
221
215
  requirement: !ruby/object:Gem::Requirement
@@ -236,14 +230,20 @@ dependencies:
236
230
  requirements:
237
231
  - - "~>"
238
232
  - !ruby/object:Gem::Version
239
- version: '10.0'
233
+ version: '12.3'
234
+ - - ">="
235
+ - !ruby/object:Gem::Version
236
+ version: 12.3.3
240
237
  type: :development
241
238
  prerelease: false
242
239
  version_requirements: !ruby/object:Gem::Requirement
243
240
  requirements:
244
241
  - - "~>"
245
242
  - !ruby/object:Gem::Version
246
- version: '10.0'
243
+ version: '12.3'
244
+ - - ">="
245
+ - !ruby/object:Gem::Version
246
+ version: 12.3.3
247
247
  - !ruby/object:Gem::Dependency
248
248
  name: minitest
249
249
  requirement: !ruby/object:Gem::Requirement
@@ -276,9 +276,6 @@ dependencies:
276
276
  name: ndr_dev_support
277
277
  requirement: !ruby/object:Gem::Requirement
278
278
  requirements:
279
- - - "~>"
280
- - !ruby/object:Gem::Version
281
- version: '3.1'
282
279
  - - ">="
283
280
  - !ruby/object:Gem::Version
284
281
  version: 3.1.3
@@ -286,9 +283,6 @@ dependencies:
286
283
  prerelease: false
287
284
  version_requirements: !ruby/object:Gem::Requirement
288
285
  requirements:
289
- - - "~>"
290
- - !ruby/object:Gem::Version
291
- version: '3.1'
292
286
  - - ">="
293
287
  - !ruby/object:Gem::Version
294
288
  version: 3.1.3
@@ -371,6 +365,9 @@ executables:
371
365
  extensions: []
372
366
  extra_rdoc_files: []
373
367
  files:
368
+ - ".github/CODEOWNERS"
369
+ - ".github/workflows/lint.yml"
370
+ - ".github/workflows/test.yml"
374
371
  - ".gitignore"
375
372
  - ".hound.yml"
376
373
  - ".rubocop.yml"
@@ -413,6 +410,7 @@ files:
413
410
  - lib/ndr_import/helpers/file/pdf.rb
414
411
  - lib/ndr_import/helpers/file/word.rb
415
412
  - lib/ndr_import/helpers/file/xml.rb
413
+ - lib/ndr_import/helpers/file/xml_streaming.rb
416
414
  - lib/ndr_import/helpers/file/zip.rb
417
415
  - lib/ndr_import/mapper.rb
418
416
  - lib/ndr_import/mapping_error.rb