ndr_import 10.1.2 → 10.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +13 -0
- data/LICENSE.txt +1 -1
- data/README.md +5 -5
- data/lib/ndr_import/avro/table.rb +49 -0
- data/lib/ndr_import/file/all.rb +1 -0
- data/lib/ndr_import/file/avro.rb +34 -0
- data/lib/ndr_import/file/json_lines.rb +55 -0
- data/lib/ndr_import/table.rb +1 -1
- data/lib/ndr_import/version.rb +1 -1
- data/lib/ndr_import.rb +1 -0
- metadata +29 -32
- data/.github/CODEOWNERS +0 -2
- data/.github/workflows/lint.yml +0 -23
- data/.github/workflows/test.yml +0 -79
- data/.gitignore +0 -16
- data/.hound.yml +0 -10
- data/.rubocop.yml +0 -1
- data/Gemfile +0 -4
- data/Guardfile +0 -16
- data/bin/console +0 -14
- data/bin/setup +0 -8
- data/code_safety.yml +0 -671
- data/gemfiles/Gemfile.rails52 +0 -7
- data/gemfiles/Gemfile.rails60 +0 -5
- data/ndr_import.gemspec +0 -53
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: aaca9cf96e7433b7889c004f769997288ea1cec77322754e965b97a545ffb0ab
|
4
|
+
data.tar.gz: 4a1e456f8766f2ea3422b0b712b2ca4c420873248e66fc77b1db8559470a1247
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: '081ce6d8bce5dde04dca97a68057897a093bc90cca23f7336415330d92dcc599b96b606e9626358731c6638f0c3c1e41b7e763b2c03488cb76eeaa5c5c7a2cd8'
|
7
|
+
data.tar.gz: 802ba8017a16cc843196004854c3a82d161e76bdd4c2f61a5f93e34018dae9090212dd67c4e503a46c5ee78d6ed2da2d7e6f17192e62e4abefbd2382389d1cce
|
data/CHANGELOG.md
CHANGED
@@ -1,6 +1,19 @@
|
|
1
1
|
## [Unreleased]
|
2
2
|
*no unreleased changes*
|
3
3
|
|
4
|
+
## 10.2.0 / 2023-05-16
|
5
|
+
* avro file support
|
6
|
+
* allow storage of `significant_mapped_fields` in `Table`
|
7
|
+
|
8
|
+
## 10.1.3 / 2022-12-08
|
9
|
+
### Added
|
10
|
+
* JSON Lines file parser
|
11
|
+
* Test against Rails 6.1
|
12
|
+
* Support Ruby 3.1, Rails 7.0
|
13
|
+
|
14
|
+
### Changed
|
15
|
+
* Update Ruby/Rails supported versions. 2.7/6.0 is now minimum
|
16
|
+
|
4
17
|
## 10.1.2 / 2021-09-28
|
5
18
|
### Fixed
|
6
19
|
* Bump to `nokogiri` to address CVE-2021-41098
|
data/LICENSE.txt
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
The MIT License (MIT)
|
2
2
|
|
3
|
-
Copyright (c) 2011-
|
3
|
+
Copyright (c) 2011-2022 NHS Digital
|
4
4
|
|
5
5
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
6
|
of this software and associated documentation files (the "Software"), to deal
|
data/README.md
CHANGED
@@ -1,7 +1,7 @@
|
|
1
|
-
# NdrImport [![Build Status](https://github.com/
|
2
|
-
This is the
|
1
|
+
# NdrImport [![Build Status](https://github.com/NHSDigital/ndr_import/workflows/Test/badge.svg)](https://github.com/NHSDigital/ndr_import/actions?query=workflow%3Atest) [![Gem Version](https://badge.fury.io/rb/ndr_import.svg)](https://rubygems.org/gems/ndr_import) [![Documentation](https://img.shields.io/badge/ndr_import-docs-blue.svg)](https://www.rubydoc.info/gems/ndr_import)
|
2
|
+
This is the NHS Digital (NHSD) National Disease Registers (NDR) Import ETL ruby gem, providing:
|
3
3
|
|
4
|
-
1. file import handlers for *extracting* data from delimited files (csv, pipe, tab, thorn), .xls(x) spreadsheets, .doc(x) word documents, PDF, PDF AcroForms, XML, 7-Zip and
|
4
|
+
1. file import handlers for *extracting* data from delimited files (csv, pipe, tab, thorn), JSON Lines, .xls(x) spreadsheets, .doc(x) word documents, PDF, PDF AcroForms, XML, 7-Zip, Zip and avro files.
|
5
5
|
2. table mappers for *transforming* tabular and non-tabular data into key value pairs grouped by a common "klass".
|
6
6
|
|
7
7
|
## Installation
|
@@ -49,7 +49,7 @@ end
|
|
49
49
|
|
50
50
|
See `test/readme_test.rb` for a more complete working example.
|
51
51
|
|
52
|
-
More information on the workings of the mapper are available in the [wiki](https://github.com/
|
52
|
+
More information on the workings of the mapper are available in the [wiki](https://github.com/NHSDigital/ndr_import/wiki).
|
53
53
|
|
54
54
|
## Development
|
55
55
|
|
@@ -59,7 +59,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
|
|
59
59
|
|
60
60
|
## Contributing
|
61
61
|
|
62
|
-
1. Fork it ( https://github.com/
|
62
|
+
1. Fork it ( https://github.com/NHSDigital/ndr_import/fork )
|
63
63
|
2. Create your feature branch (`git checkout -b my-new-feature`)
|
64
64
|
3. Commit your changes (`git commit -am 'Add some feature'`)
|
65
65
|
4. Push to the branch (`git push origin my-new-feature`)
|
@@ -0,0 +1,49 @@
|
|
1
|
+
require 'ndr_import/table'
|
2
|
+
|
3
|
+
module NdrImport
|
4
|
+
module Avro
|
5
|
+
# Syntatic sugar to ensure `header_lines` and `footer_lines` are 1 and 0 respectively.
|
6
|
+
# All other Table logic is inherited from `NdrImport::Table`
|
7
|
+
class Table < ::NdrImport::Table
|
8
|
+
# Scaffold an `NdrImport::Avro::Table` instance from avro schema file
|
9
|
+
def self.from_schema(safe_path)
|
10
|
+
raise SecurityError, "#{safe_path} is not a SafePath" unless safe_path.is_a? SafePath
|
11
|
+
|
12
|
+
table_columns = columns_from(::Avro::Schema.parse(::File.open(safe_path)))
|
13
|
+
file_name = SafeFile.basename(safe_path).sub(/\.avsc\z/, '.avro')
|
14
|
+
|
15
|
+
new(filename_pattern: "/#{file_name}\\z/",
|
16
|
+
klass: 'ExampleKlass',
|
17
|
+
columns: table_columns)
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.all_valid_options
|
21
|
+
super - %w[delimiter header_lines footer_lines]
|
22
|
+
end
|
23
|
+
|
24
|
+
def header_lines
|
25
|
+
1
|
26
|
+
end
|
27
|
+
|
28
|
+
def footer_lines
|
29
|
+
0
|
30
|
+
end
|
31
|
+
|
32
|
+
def self.columns_from(schema)
|
33
|
+
schema.fields.map do |field|
|
34
|
+
column = { column: field.name }
|
35
|
+
column[:mappings] = { field: field.name, daysafter: '1970-01-01' } if date_field?(field)
|
36
|
+
|
37
|
+
column
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def self.date_field?(field)
|
42
|
+
field.type.schemas.any? { |schema| schema.logical_type == 'date' }
|
43
|
+
end
|
44
|
+
|
45
|
+
private_class_method :columns_from
|
46
|
+
private_class_method :date_field?
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
data/lib/ndr_import/file/all.rb
CHANGED
@@ -0,0 +1,34 @@
|
|
1
|
+
require 'avro'
|
2
|
+
require 'ndr_support/safe_file'
|
3
|
+
require_relative 'registry'
|
4
|
+
|
5
|
+
module NdrImport
|
6
|
+
# This is one of a collection of file handlers that deal with individual formats of data.
|
7
|
+
# They can be instantiated directly or via the factory method Registry.tables
|
8
|
+
module File
|
9
|
+
# This class is an avro file handler that returns a single table.
|
10
|
+
class Avro < Base
|
11
|
+
private
|
12
|
+
|
13
|
+
def rows(&block)
|
14
|
+
return enum_for(:rows) unless block
|
15
|
+
|
16
|
+
# Create an instance of DatumReader
|
17
|
+
reader = ::Avro::IO::DatumReader.new
|
18
|
+
# Open @filename in read mode
|
19
|
+
file = ::File.open(@filename, 'rb')
|
20
|
+
# Equivalent to DataFileReader instance creation in Java
|
21
|
+
dr = ::Avro::DataFile::Reader.new(file, reader)
|
22
|
+
|
23
|
+
dr.each_with_index do |avro_row, i|
|
24
|
+
# Ensure the first row is always the "header"
|
25
|
+
yield(avro_row.keys) if i.zero?
|
26
|
+
yield(avro_row.values.map(&:to_s))
|
27
|
+
end
|
28
|
+
rescue StandardError => e
|
29
|
+
raise("#{SafeFile.basename(@filename)} [#{e.class}: #{e.message}]")
|
30
|
+
end
|
31
|
+
end
|
32
|
+
Registry.register(Avro, 'avro')
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
require 'json'
|
2
|
+
require 'ndr_support/safe_file'
|
3
|
+
require 'ndr_support/utf8_encoding'
|
4
|
+
require_relative 'registry'
|
5
|
+
|
6
|
+
module NdrImport
|
7
|
+
# This is one of a collection of file handlers that deal with individual formats of data.
|
8
|
+
# They can be instantiated directly or via the factory method Registry.tables
|
9
|
+
module File
|
10
|
+
# This class is a JSON Lines file handler that returns a single "table" of JSON object "rows".
|
11
|
+
class JsonLines < Base
|
12
|
+
include UTF8Encoding
|
13
|
+
|
14
|
+
private
|
15
|
+
|
16
|
+
def rows(&block)
|
17
|
+
return enum_for(:rows) unless block
|
18
|
+
|
19
|
+
# Encoding:
|
20
|
+
# As we're going to be yielding the parsed json lines of the file as it is streamed
|
21
|
+
# (rather than slurped in advance), we need to know which encoding / mode
|
22
|
+
# is going to work in advance.
|
23
|
+
#
|
24
|
+
path = SafeFile.safepath_to_string(@filename)
|
25
|
+
mode = read_mode_for(path)
|
26
|
+
|
27
|
+
# SECURE: TG 25 Oct 2021 SafeFile.safepath_to_string ensures that the path is SafePath.
|
28
|
+
::File.new(path, mode).each { |line| block.call JSON.parse(ensure_utf8!(line).chomp) }
|
29
|
+
rescue StandardError => e
|
30
|
+
raise "Failed to read #{SafeFile.basename(@filename)} as text [#{e.class}: #{e.message}]"
|
31
|
+
end
|
32
|
+
|
33
|
+
# TODO: In Ruby 2.0+, a mode of "rb:bom|utf-16:utf-8" seemed to fix all cases,
|
34
|
+
# but this doesn't work on Ruby 1.9.3, which we are currently still supporting.
|
35
|
+
# Therefore, we have to test multiple modes in advance, hence #read_mode_for.
|
36
|
+
def read_mode_for(trusted_path)
|
37
|
+
# These are the read modes we will try, in order:
|
38
|
+
modes = ['rb:utf-16:utf-8', 'r:utf-8']
|
39
|
+
|
40
|
+
begin
|
41
|
+
::File.new(trusted_path, modes.first).each do |_line|
|
42
|
+
# We're just making sure this doesn't raise an error
|
43
|
+
end
|
44
|
+
rescue Encoding::InvalidByteSequenceError
|
45
|
+
modes.shift # That one didn't work...
|
46
|
+
retry if modes.any?
|
47
|
+
end
|
48
|
+
|
49
|
+
modes.first || raise('Unable to determine working stream encoding!')
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
Registry.register(JsonLines, 'jsonl')
|
54
|
+
end
|
55
|
+
end
|
data/lib/ndr_import/table.rb
CHANGED
@@ -12,7 +12,7 @@ module NdrImport
|
|
12
12
|
def self.all_valid_options
|
13
13
|
%w[canonical_name delimiter liberal_parsing filename_pattern file_password last_data_column
|
14
14
|
tablename_pattern header_lines footer_lines format klass columns xml_record_xpath slurp
|
15
|
-
row_identifier]
|
15
|
+
row_identifier significant_mapped_fields]
|
16
16
|
end
|
17
17
|
|
18
18
|
def all_valid_options
|
data/lib/ndr_import/version.rb
CHANGED
data/lib/ndr_import.rb
CHANGED
@@ -9,6 +9,7 @@ require 'ndr_import/non_tabular/table'
|
|
9
9
|
require 'ndr_import/fixed_width/table'
|
10
10
|
require 'ndr_import/xml/table'
|
11
11
|
require 'ndr_import/pdf_form/table'
|
12
|
+
require 'ndr_import/avro/table'
|
12
13
|
require 'ndr_import/unmapped_data_error'
|
13
14
|
require 'ndr_import/acroform_reader'
|
14
15
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ndr_import
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 10.
|
4
|
+
version: 10.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- NCRS Development Team
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2023-05-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activemodel
|
@@ -30,20 +30,20 @@ dependencies:
|
|
30
30
|
requirements:
|
31
31
|
- - ">="
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: '
|
33
|
+
version: '6.0'
|
34
34
|
- - "<"
|
35
35
|
- !ruby/object:Gem::Version
|
36
|
-
version: '7'
|
36
|
+
version: '7.1'
|
37
37
|
type: :runtime
|
38
38
|
prerelease: false
|
39
39
|
version_requirements: !ruby/object:Gem::Requirement
|
40
40
|
requirements:
|
41
41
|
- - ">="
|
42
42
|
- !ruby/object:Gem::Version
|
43
|
-
version: '
|
43
|
+
version: '6.0'
|
44
44
|
- - "<"
|
45
45
|
- !ruby/object:Gem::Version
|
46
|
-
version: '7'
|
46
|
+
version: '7.1'
|
47
47
|
- !ruby/object:Gem::Dependency
|
48
48
|
name: ndr_support
|
49
49
|
requirement: !ruby/object:Gem::Requirement
|
@@ -92,6 +92,20 @@ dependencies:
|
|
92
92
|
- - "~>"
|
93
93
|
- !ruby/object:Gem::Version
|
94
94
|
version: '2.0'
|
95
|
+
- !ruby/object:Gem::Dependency
|
96
|
+
name: avro
|
97
|
+
requirement: !ruby/object:Gem::Requirement
|
98
|
+
requirements:
|
99
|
+
- - "~>"
|
100
|
+
- !ruby/object:Gem::Version
|
101
|
+
version: 1.11.0
|
102
|
+
type: :runtime
|
103
|
+
prerelease: false
|
104
|
+
version_requirements: !ruby/object:Gem::Requirement
|
105
|
+
requirements:
|
106
|
+
- - "~>"
|
107
|
+
- !ruby/object:Gem::Version
|
108
|
+
version: 1.11.0
|
95
109
|
- !ruby/object:Gem::Dependency
|
96
110
|
name: docx
|
97
111
|
requirement: !ruby/object:Gem::Requirement
|
@@ -228,9 +242,6 @@ dependencies:
|
|
228
242
|
name: rake
|
229
243
|
requirement: !ruby/object:Gem::Requirement
|
230
244
|
requirements:
|
231
|
-
- - "~>"
|
232
|
-
- !ruby/object:Gem::Version
|
233
|
-
version: '12.3'
|
234
245
|
- - ">="
|
235
246
|
- !ruby/object:Gem::Version
|
236
247
|
version: 12.3.3
|
@@ -238,9 +249,6 @@ dependencies:
|
|
238
249
|
prerelease: false
|
239
250
|
version_requirements: !ruby/object:Gem::Requirement
|
240
251
|
requirements:
|
241
|
-
- - "~>"
|
242
|
-
- !ruby/object:Gem::Version
|
243
|
-
version: '12.3'
|
244
252
|
- - ">="
|
245
253
|
- !ruby/object:Gem::Version
|
246
254
|
version: 12.3.3
|
@@ -365,36 +373,26 @@ executables:
|
|
365
373
|
extensions: []
|
366
374
|
extra_rdoc_files: []
|
367
375
|
files:
|
368
|
-
- ".github/CODEOWNERS"
|
369
|
-
- ".github/workflows/lint.yml"
|
370
|
-
- ".github/workflows/test.yml"
|
371
|
-
- ".gitignore"
|
372
|
-
- ".hound.yml"
|
373
|
-
- ".rubocop.yml"
|
374
376
|
- CHANGELOG.md
|
375
377
|
- CODE_OF_CONDUCT.md
|
376
|
-
- Gemfile
|
377
|
-
- Guardfile
|
378
378
|
- LICENSE.txt
|
379
379
|
- README.md
|
380
380
|
- Rakefile
|
381
|
-
- bin/console
|
382
|
-
- bin/setup
|
383
|
-
- code_safety.yml
|
384
381
|
- exe/pdf_acro_form_to_yaml
|
385
382
|
- exe/pdf_to_text
|
386
383
|
- exe/word_to_text
|
387
|
-
- gemfiles/Gemfile.rails52
|
388
|
-
- gemfiles/Gemfile.rails60
|
389
384
|
- lib/ndr_import.rb
|
390
385
|
- lib/ndr_import/acroform_reader.rb
|
386
|
+
- lib/ndr_import/avro/table.rb
|
391
387
|
- lib/ndr_import/csv_library.rb
|
392
388
|
- lib/ndr_import/file/acro_form.rb
|
393
389
|
- lib/ndr_import/file/all.rb
|
390
|
+
- lib/ndr_import/file/avro.rb
|
394
391
|
- lib/ndr_import/file/base.rb
|
395
392
|
- lib/ndr_import/file/delimited.rb
|
396
393
|
- lib/ndr_import/file/docx.rb
|
397
394
|
- lib/ndr_import/file/excel.rb
|
395
|
+
- lib/ndr_import/file/json_lines.rb
|
398
396
|
- lib/ndr_import/file/office_file_helper.rb
|
399
397
|
- lib/ndr_import/file/pdf.rb
|
400
398
|
- lib/ndr_import/file/registry.rb
|
@@ -429,12 +427,11 @@ files:
|
|
429
427
|
- lib/ndr_import/version.rb
|
430
428
|
- lib/ndr_import/xml/control_char_escaper.rb
|
431
429
|
- lib/ndr_import/xml/table.rb
|
432
|
-
|
433
|
-
homepage: https://github.com/PublicHealthEngland/ndr_import
|
430
|
+
homepage: https://github.com/NHSDigital/ndr_import
|
434
431
|
licenses:
|
435
432
|
- MIT
|
436
433
|
metadata: {}
|
437
|
-
post_install_message:
|
434
|
+
post_install_message:
|
438
435
|
rdoc_options: []
|
439
436
|
require_paths:
|
440
437
|
- lib
|
@@ -442,15 +439,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
442
439
|
requirements:
|
443
440
|
- - ">="
|
444
441
|
- !ruby/object:Gem::Version
|
445
|
-
version: '2.
|
442
|
+
version: '2.7'
|
446
443
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
447
444
|
requirements:
|
448
445
|
- - ">="
|
449
446
|
- !ruby/object:Gem::Version
|
450
447
|
version: '0'
|
451
448
|
requirements: []
|
452
|
-
rubygems_version: 3.
|
453
|
-
signing_key:
|
449
|
+
rubygems_version: 3.2.33
|
450
|
+
signing_key:
|
454
451
|
specification_version: 4
|
455
452
|
summary: NDR Import
|
456
453
|
test_files: []
|
data/.github/CODEOWNERS
DELETED
data/.github/workflows/lint.yml
DELETED
@@ -1,23 +0,0 @@
|
|
1
|
-
name: Lint
|
2
|
-
|
3
|
-
on: [pull_request]
|
4
|
-
|
5
|
-
jobs:
|
6
|
-
rubocop:
|
7
|
-
name: RuboCop
|
8
|
-
|
9
|
-
runs-on: ubuntu-latest
|
10
|
-
|
11
|
-
steps:
|
12
|
-
- uses: actions/checkout@v2
|
13
|
-
with:
|
14
|
-
fetch-depth: 0 # fetch everything
|
15
|
-
- name: Set up Ruby
|
16
|
-
uses: ruby/setup-ruby@v1
|
17
|
-
with:
|
18
|
-
ruby-version: 3.0
|
19
|
-
- name: Install dependencies
|
20
|
-
run: bundle install
|
21
|
-
- name: Run RuboCop against BASE..HEAD changes
|
22
|
-
run: bundle exec rake rubocop:diff origin/${GITHUB_BASE_REF#*/}
|
23
|
-
|
data/.github/workflows/test.yml
DELETED
@@ -1,79 +0,0 @@
|
|
1
|
-
name: Test
|
2
|
-
|
3
|
-
on:
|
4
|
-
# Run on all master branch commits
|
5
|
-
push:
|
6
|
-
branches:
|
7
|
-
- master
|
8
|
-
|
9
|
-
# Run against all PRs (from the main repo, or forks)
|
10
|
-
pull_request:
|
11
|
-
|
12
|
-
jobs:
|
13
|
-
test:
|
14
|
-
strategy:
|
15
|
-
fail-fast: false
|
16
|
-
matrix:
|
17
|
-
ruby-version:
|
18
|
-
- 2.6
|
19
|
-
- 2.7
|
20
|
-
- 3.0
|
21
|
-
gemfile:
|
22
|
-
- gemfiles/Gemfile.rails52
|
23
|
-
- gemfiles/Gemfile.rails60
|
24
|
-
|
25
|
-
name: Ruby ${{ matrix.ruby-version }} / Bundle ${{ matrix.gemfile }}
|
26
|
-
|
27
|
-
runs-on: ubuntu-latest
|
28
|
-
|
29
|
-
env:
|
30
|
-
BUNDLE_GEMFILE: ${{ matrix.gemfile }}
|
31
|
-
|
32
|
-
steps:
|
33
|
-
- uses: actions/checkout@v2
|
34
|
-
- name: Set up Ruby
|
35
|
-
uses: ruby/setup-ruby@v1
|
36
|
-
with:
|
37
|
-
ruby-version: ${{ matrix.ruby-version }}
|
38
|
-
- name: Install dependencies
|
39
|
-
run: bundle install
|
40
|
-
- name: Run tests
|
41
|
-
run: bundle exec rake
|
42
|
-
|
43
|
-
# A utility job upon which Branch Protection can depend,
|
44
|
-
# thus remaining agnostic of the matrix.
|
45
|
-
test_matrix:
|
46
|
-
if: ${{ always() }}
|
47
|
-
runs-on: ubuntu-latest
|
48
|
-
name: Matrix
|
49
|
-
needs: test
|
50
|
-
steps:
|
51
|
-
- name: Check build matrix status
|
52
|
-
if: ${{ needs.test.result != 'success' }}
|
53
|
-
run: exit 1
|
54
|
-
|
55
|
-
notify:
|
56
|
-
# Run only on master, but regardless of whether tests past:
|
57
|
-
if: ${{ always() && github.ref == 'refs/heads/master' }}
|
58
|
-
|
59
|
-
needs: test_matrix
|
60
|
-
|
61
|
-
runs-on: ubuntu-latest
|
62
|
-
|
63
|
-
steps:
|
64
|
-
- uses: 8398a7/action-slack@v3
|
65
|
-
with:
|
66
|
-
status: custom
|
67
|
-
fields: workflow,commit,author
|
68
|
-
custom_payload: |
|
69
|
-
{
|
70
|
-
channel: 'C7FQWGDHP',
|
71
|
-
username: 'CI – ' + '${{ github.repository }}'.split('/')[1],
|
72
|
-
icon_emoji: ':hammer_and_wrench:',
|
73
|
-
attachments: [{
|
74
|
-
color: '${{ needs.test_matrix.result }}' === 'success' ? 'good' : '${{ needs.test_matrix.result }}' === 'failure' ? 'danger' : 'warning',
|
75
|
-
text: `${process.env.AS_WORKFLOW} against \`${{ github.ref }}\` (${process.env.AS_COMMIT}) for ${{ github.actor }} resulted in *${{ needs.test_matrix.result }}*.`
|
76
|
-
}]
|
77
|
-
}
|
78
|
-
env:
|
79
|
-
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
|
data/.gitignore
DELETED
@@ -1,16 +0,0 @@
|
|
1
|
-
/.rubocop-https---raw-githubusercontent-com-PublicHealthEngland-ndr-dev-support-master--rubocop-yml
|
2
|
-
/.bundle/
|
3
|
-
/.yardoc
|
4
|
-
/Gemfile.lock
|
5
|
-
/gemfiles/Gemfile.*.lock
|
6
|
-
/_yardoc/
|
7
|
-
/coverage/
|
8
|
-
/docs/_site/
|
9
|
-
/pkg/
|
10
|
-
/spec/reports/
|
11
|
-
/tmp/
|
12
|
-
*.bundle
|
13
|
-
*.so
|
14
|
-
*.o
|
15
|
-
*.a
|
16
|
-
mkmf.log
|
data/.hound.yml
DELETED
data/.rubocop.yml
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
inherit_from: 'https://raw.githubusercontent.com/PublicHealthEngland/ndr_dev_support/master/.rubocop.yml'
|
data/Gemfile
DELETED
data/Guardfile
DELETED
@@ -1,16 +0,0 @@
|
|
1
|
-
# A sample Guardfile
|
2
|
-
# More info at https://github.com/guard/guard#readme
|
3
|
-
|
4
|
-
# automatically check Ruby code style with Rubocop when files are modified
|
5
|
-
guard :rubocop, :all_on_start => false, :keep_failed => false do
|
6
|
-
watch(/.+\.rb$/)
|
7
|
-
watch(%r{(?:.+/)?\.rubocop\.yml$}) { |m| File.dirname(m[0]) }
|
8
|
-
end
|
9
|
-
|
10
|
-
guard :test do
|
11
|
-
watch(%r{^test/.+_test\.rb$})
|
12
|
-
watch('test/test_helper.rb') { 'test' }
|
13
|
-
|
14
|
-
# Non-rails
|
15
|
-
watch(%r{^lib/ndr_import/(.+)\.rb$}) { |m| "test/#{m[1]}_test.rb" }
|
16
|
-
end
|
data/bin/console
DELETED
@@ -1,14 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
|
3
|
-
require 'bundler/setup'
|
4
|
-
require 'ndr_import'
|
5
|
-
|
6
|
-
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
-
# with your gem easier. You can also use a different console, if you like.
|
8
|
-
|
9
|
-
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
-
# require "pry"
|
11
|
-
# Pry.start
|
12
|
-
|
13
|
-
require 'irb'
|
14
|
-
IRB.start(__FILE__)
|
data/bin/setup
DELETED
data/code_safety.yml
DELETED
@@ -1,671 +0,0 @@
|
|
1
|
-
---
|
2
|
-
file safety:
|
3
|
-
".github/CODEOWNERS":
|
4
|
-
comments:
|
5
|
-
reviewed_by: ollietulloch
|
6
|
-
safe_revision: b64ff21375dcde2b8fefe622ee9861f0fea21487
|
7
|
-
".github/workflows/lint.yml":
|
8
|
-
comments:
|
9
|
-
reviewed_by: ollietulloch
|
10
|
-
safe_revision: b64ff21375dcde2b8fefe622ee9861f0fea21487
|
11
|
-
".github/workflows/test.yml":
|
12
|
-
comments:
|
13
|
-
reviewed_by: ollietulllch
|
14
|
-
safe_revision: c3dd24e8abefe61f04fa9d3bb71ec9d0ac109bbe
|
15
|
-
".gitignore":
|
16
|
-
comments: whole file re-reviewed
|
17
|
-
reviewed_by: josh.pencheon
|
18
|
-
safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
|
19
|
-
".hound.yml":
|
20
|
-
comments:
|
21
|
-
reviewed_by: timgentry
|
22
|
-
safe_revision: df3c0be0e655f4fc95c86d4a57bda6eef1fb8955
|
23
|
-
".rubocop.yml":
|
24
|
-
comments:
|
25
|
-
reviewed_by: josh.pencheon
|
26
|
-
safe_revision: b09e268ff9c8349b914aa1b7ba888e1d39f97e4a
|
27
|
-
CHANGELOG.md:
|
28
|
-
comments:
|
29
|
-
reviewed_by: ollietulloch
|
30
|
-
safe_revision: e938689d115b75074313541ec9d6b2bc60475add
|
31
|
-
CODE_OF_CONDUCT.md:
|
32
|
-
comments:
|
33
|
-
reviewed_by: timgentry
|
34
|
-
safe_revision: 5d185a0aeba6a9cd2ff5e59efadcaeec9be45d8b
|
35
|
-
Gemfile:
|
36
|
-
comments:
|
37
|
-
reviewed_by: josh.pencheon
|
38
|
-
safe_revision: 13a1fae896c9d2b3fbe99a71d52045476f25b425
|
39
|
-
Guardfile:
|
40
|
-
comments:
|
41
|
-
reviewed_by: timgentry
|
42
|
-
safe_revision: fb1f4a97327f426cc73a6ebdc4f1089caae98e93
|
43
|
-
LICENSE.txt:
|
44
|
-
comments:
|
45
|
-
reviewed_by: timgentry
|
46
|
-
safe_revision: 5d185a0aeba6a9cd2ff5e59efadcaeec9be45d8b
|
47
|
-
README.md:
|
48
|
-
comments:
|
49
|
-
reviewed_by: ollietulloch
|
50
|
-
safe_revision: b64ff21375dcde2b8fefe622ee9861f0fea21487
|
51
|
-
Rakefile:
|
52
|
-
comments:
|
53
|
-
reviewed_by: josh.pencheon
|
54
|
-
safe_revision: 5a7f26cecaabab20f4e666776f9166dcc3fa6bfe
|
55
|
-
bin/console:
|
56
|
-
comments:
|
57
|
-
reviewed_by: josh.pencheon
|
58
|
-
safe_revision: e1d967c10059e8c635452838c3f3dd2b969d9ae4
|
59
|
-
bin/setup:
|
60
|
-
comments:
|
61
|
-
reviewed_by: josh.pencheon
|
62
|
-
safe_revision: e1d967c10059e8c635452838c3f3dd2b969d9ae4
|
63
|
-
docs/Gemfile:
|
64
|
-
comments:
|
65
|
-
reviewed_by: josh.pencheon
|
66
|
-
safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
|
67
|
-
docs/Gemfile.lock:
|
68
|
-
comments:
|
69
|
-
reviewed_by: ollietulloch
|
70
|
-
safe_revision: 75877ddc14e99a10e5e751b7034964a9c7a9d1ef
|
71
|
-
docs/_config.yml:
|
72
|
-
comments:
|
73
|
-
reviewed_by: josh.pencheon
|
74
|
-
safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
|
75
|
-
docs/_includes/footer.html:
|
76
|
-
comments:
|
77
|
-
reviewed_by: josh.pencheon
|
78
|
-
safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
|
79
|
-
docs/_includes/header.html:
|
80
|
-
comments:
|
81
|
-
reviewed_by: josh.pencheon
|
82
|
-
safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
|
83
|
-
docs/capturing-data.md:
|
84
|
-
comments:
|
85
|
-
reviewed_by: josh.pencheon
|
86
|
-
safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
|
87
|
-
docs/date-formats.md:
|
88
|
-
comments:
|
89
|
-
reviewed_by: josh.pencheon
|
90
|
-
safe_revision: fa21d6d967bf132800b456b585795beec80b08a3
|
91
|
-
docs/getting-started.md:
|
92
|
-
comments:
|
93
|
-
reviewed_by: josh.pencheon
|
94
|
-
safe_revision: fa21d6d967bf132800b456b585795beec80b08a3
|
95
|
-
docs/identifying-and-splitting-records.md:
|
96
|
-
comments:
|
97
|
-
reviewed_by: josh.pencheon
|
98
|
-
safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
|
99
|
-
docs/inbuilt-cleaning-methods.md:
|
100
|
-
comments:
|
101
|
-
reviewed_by: josh.pencheon
|
102
|
-
safe_revision: 694b57ce14e0709fc4d31a1357f8416e98f5de91
|
103
|
-
docs/index.md:
|
104
|
-
comments:
|
105
|
-
reviewed_by: josh.pencheon
|
106
|
-
safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
|
107
|
-
docs/local-code-transformation-in-yaml-mappings.md:
|
108
|
-
comments:
|
109
|
-
reviewed_by: josh.pencheon
|
110
|
-
safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
|
111
|
-
docs/non-tabular-mappings.md:
|
112
|
-
comments:
|
113
|
-
reviewed_by: josh.pencheon
|
114
|
-
safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
|
115
|
-
docs/priority-field-mapping.md:
|
116
|
-
comments:
|
117
|
-
reviewed_by: josh.pencheon
|
118
|
-
safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
|
119
|
-
docs/standard-yaml-mappings.md:
|
120
|
-
comments:
|
121
|
-
reviewed_by: josh.pencheon
|
122
|
-
safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
|
123
|
-
docs/yaml-mapping-user-guide.md:
|
124
|
-
comments:
|
125
|
-
reviewed_by: josh.pencheon
|
126
|
-
safe_revision: 02aaf91b116c510a7c16f2b6f2389736b2742f49
|
127
|
-
exe/pdf_acro_form_to_yaml:
|
128
|
-
comments:
|
129
|
-
reviewed_by: josh.pencheon
|
130
|
-
safe_revision: e1d967c10059e8c635452838c3f3dd2b969d9ae4
|
131
|
-
exe/pdf_to_text:
|
132
|
-
comments:
|
133
|
-
reviewed_by: josh.pencheon
|
134
|
-
safe_revision: e1d967c10059e8c635452838c3f3dd2b969d9ae4
|
135
|
-
exe/word_to_text:
|
136
|
-
comments:
|
137
|
-
reviewed_by: josh.pencheon
|
138
|
-
safe_revision: e1d967c10059e8c635452838c3f3dd2b969d9ae4
|
139
|
-
gemfiles/Gemfile.rails52:
|
140
|
-
comments:
|
141
|
-
reviewed_by: josh.pencheon
|
142
|
-
safe_revision: 13a1fae896c9d2b3fbe99a71d52045476f25b425
|
143
|
-
gemfiles/Gemfile.rails60:
|
144
|
-
comments:
|
145
|
-
reviewed_by: josh.pencheon
|
146
|
-
safe_revision: d3d9a987befeecb122a448d8d06e66d74da13fb5
|
147
|
-
lib/ndr_import.rb:
|
148
|
-
comments:
|
149
|
-
reviewed_by: josh.pencheon
|
150
|
-
safe_revision: 24d6449fd0612552f132dfbf4cada2ae28d0469e
|
151
|
-
lib/ndr_import/acroform_reader.rb:
|
152
|
-
comments:
|
153
|
-
reviewed_by: josh.pencheon
|
154
|
-
safe_revision: 24d6449fd0612552f132dfbf4cada2ae28d0469e
|
155
|
-
lib/ndr_import/csv_library.rb:
|
156
|
-
comments:
|
157
|
-
reviewed_by: ollietulloch
|
158
|
-
safe_revision: 6b8668967dbd42d7893a0fa5f0aa1ec1c11227e1
|
159
|
-
lib/ndr_import/file/acro_form.rb:
|
160
|
-
comments:
|
161
|
-
reviewed_by: josh.pencheon
|
162
|
-
safe_revision: 8a6ea666616c5b5d7c93cdf5aa019e8fc69d19e1
|
163
|
-
lib/ndr_import/file/all.rb:
|
164
|
-
comments:
|
165
|
-
reviewed_by: josh.pencheon
|
166
|
-
safe_revision: 5cd2cd0b3a1e254d30d4acc28c6731825a1f84f5
|
167
|
-
lib/ndr_import/file/base.rb:
|
168
|
-
comments:
|
169
|
-
reviewed_by: josh.pencheon
|
170
|
-
safe_revision: c3183e522bce50008df576ceb47fe4761ab8f966
|
171
|
-
lib/ndr_import/file/delimited.rb:
|
172
|
-
comments:
|
173
|
-
reviewed_by: josh.pencheon
|
174
|
-
safe_revision: 607c0668f1fffd70d181bc1a31c4f56eed5f6189
|
175
|
-
lib/ndr_import/file/docx.rb:
|
176
|
-
comments:
|
177
|
-
reviewed_by: josh.pencheon
|
178
|
-
safe_revision: 897f8b648d633368cf2001d17ab89c06a12d445b
|
179
|
-
lib/ndr_import/file/excel.rb:
|
180
|
-
comments:
|
181
|
-
reviewed_by: ollietulloch
|
182
|
-
safe_revision: 37482c79448bea80033f6f69d97584df330c9861
|
183
|
-
lib/ndr_import/file/office_file_helper.rb:
|
184
|
-
comments:
|
185
|
-
reviewed_by: josh.pencheon
|
186
|
-
safe_revision: a69d4a57ddcf13cdc13c27bd2eb91a395fa7ea36
|
187
|
-
lib/ndr_import/file/pdf.rb:
|
188
|
-
comments:
|
189
|
-
reviewed_by: timgentry
|
190
|
-
safe_revision: c88000b32401b5ae9ef7f5878a9b630506ab5a94
|
191
|
-
lib/ndr_import/file/registry.rb:
|
192
|
-
comments:
|
193
|
-
reviewed_by: josh.pencheon
|
194
|
-
safe_revision: 2104514689a1a1286195fff18144a8cecb93048b
|
195
|
-
lib/ndr_import/file/seven_zip.rb:
|
196
|
-
comments:
|
197
|
-
reviewed_by: josh.pencheon
|
198
|
-
safe_revision: f2feb0c430947839183d7223e60e7c29b2c0f846
|
199
|
-
lib/ndr_import/file/text.rb:
|
200
|
-
comments:
|
201
|
-
reviewed_by: josh.pencheon
|
202
|
-
safe_revision: dfe367e64c6e0ff80495cd7989fc50311d5b258f
|
203
|
-
lib/ndr_import/file/unregistered_filetype.rb:
|
204
|
-
comments:
|
205
|
-
reviewed_by: josh.pencheon
|
206
|
-
safe_revision: 2104514689a1a1286195fff18144a8cecb93048b
|
207
|
-
lib/ndr_import/file/word.rb:
|
208
|
-
comments:
|
209
|
-
reviewed_by: timgentry
|
210
|
-
safe_revision: c88000b32401b5ae9ef7f5878a9b630506ab5a94
|
211
|
-
lib/ndr_import/file/xml.rb:
|
212
|
-
comments:
|
213
|
-
reviewed_by: josh.pencheon
|
214
|
-
safe_revision: ae75fb49baf028ac8ce08e4bedcd3625ff3ff0cd
|
215
|
-
lib/ndr_import/file/zip.rb:
|
216
|
-
comments:
|
217
|
-
reviewed_by: timgentry
|
218
|
-
safe_revision: c88000b32401b5ae9ef7f5878a9b630506ab5a94
|
219
|
-
lib/ndr_import/fixed_width/table.rb:
|
220
|
-
comments:
|
221
|
-
reviewed_by: ollie.tulloch
|
222
|
-
safe_revision: dfc958d44b6c58355445fa395db08a62213ee709
|
223
|
-
lib/ndr_import/helpers/file/delimited.rb:
|
224
|
-
comments:
|
225
|
-
reviewed_by: ollietulloch
|
226
|
-
safe_revision: 4a5cc1d362c632fc1f9242c69982fbce33557e17
|
227
|
-
lib/ndr_import/helpers/file/excel.rb:
|
228
|
-
comments:
|
229
|
-
reviewed_by: joshpencheon
|
230
|
-
safe_revision: 06fd16643d02957ce856728e4911143c8e0178dc
|
231
|
-
lib/ndr_import/helpers/file/pdf.rb:
|
232
|
-
comments:
|
233
|
-
reviewed_by: timgentry
|
234
|
-
safe_revision: 3757fece40192a335c2d7d9290f2e4e9eed43619
|
235
|
-
lib/ndr_import/helpers/file/word.rb:
|
236
|
-
comments:
|
237
|
-
reviewed_by: timgentry
|
238
|
-
safe_revision: 45da71ebd3acbc0fe53755bcd75483ba17cb6924
|
239
|
-
lib/ndr_import/helpers/file/xml.rb:
|
240
|
-
comments:
|
241
|
-
reviewed_by: josh.pencheon
|
242
|
-
safe_revision: 9a6cc769abce5f9bfa5b4f8bd5cda52dfe18b12b
|
243
|
-
lib/ndr_import/helpers/file/xml_streaming.rb:
|
244
|
-
comments: uses SafePath and Shellwords when accessing filesystem, or making system
|
245
|
-
calls
|
246
|
-
reviewed_by: josh.pencheon
|
247
|
-
safe_revision: 55e502bb4445cb8b985e530e8eb26d92b574ded9
|
248
|
-
lib/ndr_import/helpers/file/zip.rb:
|
249
|
-
comments:
|
250
|
-
reviewed_by: timgentry
|
251
|
-
safe_revision: 6c6f204fab2f4232d81cb76aa523c26b0c490ae7
|
252
|
-
lib/ndr_import/mapper.rb:
|
253
|
-
comments: Writes to a Tempfile, but cleans up. Ruby will respect TMP_DIR
|
254
|
-
reviewed_by: josh.pencheon
|
255
|
-
safe_revision: 897f8b648d633368cf2001d17ab89c06a12d445b
|
256
|
-
lib/ndr_import/mapping_error.rb:
|
257
|
-
comments:
|
258
|
-
reviewed_by: timgentry
|
259
|
-
safe_revision: 96ccc535bb9f933081dfc73bf0442b32b6bdce1d
|
260
|
-
lib/ndr_import/missing_field_error.rb:
|
261
|
-
comments:
|
262
|
-
reviewed_by: josh.pencheon
|
263
|
-
safe_revision: eca44583e9989159b45e90021dd1c65228447180
|
264
|
-
lib/ndr_import/non_tabular/column_mapping.rb:
|
265
|
-
comments:
|
266
|
-
reviewed_by: josh.pencheon
|
267
|
-
safe_revision: bb44ade56a2151706eede2c31142440ccf49e6f6
|
268
|
-
lib/ndr_import/non_tabular/line.rb:
|
269
|
-
comments:
|
270
|
-
reviewed_by: timgentry
|
271
|
-
safe_revision: ac30f66578ab380649be800a4426d917ddbcb329
|
272
|
-
lib/ndr_import/non_tabular/mapping.rb:
|
273
|
-
comments:
|
274
|
-
reviewed_by: timgentry
|
275
|
-
safe_revision: cf382902508a21a95b80ac4582fbbd117164e80e
|
276
|
-
lib/ndr_import/non_tabular/record.rb:
|
277
|
-
comments:
|
278
|
-
reviewed_by: josh.pencheon
|
279
|
-
safe_revision: bb44ade56a2151706eede2c31142440ccf49e6f6
|
280
|
-
lib/ndr_import/non_tabular/table.rb:
|
281
|
-
comments:
|
282
|
-
reviewed_by: ollietulloch
|
283
|
-
safe_revision: 66cff59af2f078152f7459c436d51b57cb93f28e
|
284
|
-
lib/ndr_import/non_tabular_file_helper.rb:
|
285
|
-
comments:
|
286
|
-
reviewed_by: josh.pencheon
|
287
|
-
safe_revision: ac30f66578ab380649be800a4426d917ddbcb329
|
288
|
-
lib/ndr_import/pdf_form/table.rb:
|
289
|
-
comments:
|
290
|
-
reviewed_by: josh.pencheon
|
291
|
-
safe_revision: 5fd247eeb13a3f1356ab2d76ac9fabf9e19d5d36
|
292
|
-
lib/ndr_import/standard_mappings.rb:
|
293
|
-
comments:
|
294
|
-
reviewed_by: josh.pencheon
|
295
|
-
safe_revision: 3c7f827d17aacbf7b811eea67e27553f3b039070
|
296
|
-
lib/ndr_import/table.rb:
|
297
|
-
comments: uses File.basename
|
298
|
-
reviewed_by: ollietulloch
|
299
|
-
safe_revision: 66cff59af2f078152f7459c436d51b57cb93f28e
|
300
|
-
lib/ndr_import/universal_importer_helper.rb:
|
301
|
-
comments:
|
302
|
-
reviewed_by: josh.pencheon
|
303
|
-
safe_revision: 85869d99ae93252b7f3ef2d0a4db817c88d35c9e
|
304
|
-
lib/ndr_import/unmapped_data_error.rb:
|
305
|
-
comments:
|
306
|
-
reviewed_by: josh.pencheon
|
307
|
-
safe_revision: 5cd2cd0b3a1e254d30d4acc28c6731825a1f84f5
|
308
|
-
lib/ndr_import/version.rb:
|
309
|
-
comments: another check?
|
310
|
-
reviewed_by: ollietulloch
|
311
|
-
safe_revision: da0aed5e8a8659c5f70254f6d76264f8e780d835
|
312
|
-
lib/ndr_import/xml/control_char_escaper.rb:
|
313
|
-
comments:
|
314
|
-
reviewed_by: josh.pencheon
|
315
|
-
safe_revision: 9a6cc769abce5f9bfa5b4f8bd5cda52dfe18b12b
|
316
|
-
lib/ndr_import/xml/table.rb:
|
317
|
-
comments:
|
318
|
-
reviewed_by: josh.pencheon
|
319
|
-
safe_revision: 4ab72f84201c2d5f0147b7dfd041f488f6ff0422
|
320
|
-
ndr_import.gemspec:
|
321
|
-
comments:
|
322
|
-
reviewed_by: ollietulloch
|
323
|
-
safe_revision: 3b437f0ca271fa962121edecd4559017c2446a3a
|
324
|
-
test/csv_library_test.rb:
|
325
|
-
comments:
|
326
|
-
reviewed_by: ollietulloch
|
327
|
-
safe_revision: 6b8668967dbd42d7893a0fa5f0aa1ec1c11227e1
|
328
|
-
test/file/acro_form_test.rb:
|
329
|
-
comments:
|
330
|
-
reviewed_by: josh.pencheon
|
331
|
-
safe_revision: 5cd2cd0b3a1e254d30d4acc28c6731825a1f84f5
|
332
|
-
test/file/base_test.rb:
|
333
|
-
comments:
|
334
|
-
reviewed_by: timgentry
|
335
|
-
safe_revision: a8451809db86e0e023b0d93826114a33ff896c4f
|
336
|
-
test/file/delimited_test.rb:
|
337
|
-
comments:
|
338
|
-
reviewed_by: josh.pencheon
|
339
|
-
safe_revision: 93ccee82fc2165d1ca2d9b03d146ae03e769ea96
|
340
|
-
test/file/docx_test.rb:
|
341
|
-
comments:
|
342
|
-
reviewed_by: josh.pencheon
|
343
|
-
safe_revision: a69d4a57ddcf13cdc13c27bd2eb91a395fa7ea36
|
344
|
-
test/file/excel_test.rb:
|
345
|
-
comments:
|
346
|
-
reviewed_by: ollietulloch
|
347
|
-
safe_revision: 85a080deaa93e4220ad1bf566f29cbdac9b31c0f
|
348
|
-
test/file/pdf_test.rb:
|
349
|
-
comments:
|
350
|
-
reviewed_by: josh.pencheon
|
351
|
-
safe_revision: cb24ed3ea8116730d07f74546cd6fed0738b171d
|
352
|
-
test/file/registry_test.rb:
|
353
|
-
comments:
|
354
|
-
reviewed_by: ollietulloch
|
355
|
-
safe_revision: 85a080deaa93e4220ad1bf566f29cbdac9b31c0f
|
356
|
-
test/file/seven_zip_test.rb:
|
357
|
-
comments:
|
358
|
-
reviewed_by: josh.pencheon
|
359
|
-
safe_revision: f2feb0c430947839183d7223e60e7c29b2c0f846
|
360
|
-
test/file/text_test.rb:
|
361
|
-
comments:
|
362
|
-
reviewed_by: timgentry
|
363
|
-
safe_revision: 3107f711805b6d3b89d32ec923178425aa600dac
|
364
|
-
test/file/word_test.rb:
|
365
|
-
comments:
|
366
|
-
reviewed_by: josh.pencheon
|
367
|
-
safe_revision: 1b66cfcbb61dfac93c44889ca0ced5836101c20c
|
368
|
-
test/file/xml_test.rb:
|
369
|
-
comments:
|
370
|
-
reviewed_by: josh.pencheon
|
371
|
-
safe_revision: ae75fb49baf028ac8ce08e4bedcd3625ff3ff0cd
|
372
|
-
test/file/zip_test.rb:
|
373
|
-
comments:
|
374
|
-
reviewed_by: timgentry
|
375
|
-
safe_revision: c88000b32401b5ae9ef7f5878a9b630506ab5a94
|
376
|
-
test/fixed_width/table_test.rb:
|
377
|
-
comments:
|
378
|
-
reviewed_by: ollie.tulloch
|
379
|
-
safe_revision: dfc958d44b6c58355445fa395db08a62213ee709
|
380
|
-
test/helpers/file/delimited_test.rb:
|
381
|
-
comments:
|
382
|
-
reviewed_by: josh.pencheon
|
383
|
-
safe_revision: 93ccee82fc2165d1ca2d9b03d146ae03e769ea96
|
384
|
-
test/helpers/file/excel_test.rb:
|
385
|
-
comments:
|
386
|
-
reviewed_by: joshpencheon
|
387
|
-
safe_revision: 377c060bd1e012d6e162a4ab8923a3609aacdf57
|
388
|
-
test/helpers/file/pdf_test.rb:
|
389
|
-
comments:
|
390
|
-
reviewed_by: josh.pencheon
|
391
|
-
safe_revision: cb24ed3ea8116730d07f74546cd6fed0738b171d
|
392
|
-
test/helpers/file/word_test.rb:
|
393
|
-
comments:
|
394
|
-
reviewed_by: timgentry
|
395
|
-
safe_revision: 9abdd6ced1d0c90ce8dd88abee4eb6472c7ff0d6
|
396
|
-
test/helpers/file/xml_streaming_test.rb:
|
397
|
-
comments:
|
398
|
-
reviewed_by: josh.pencheon
|
399
|
-
safe_revision: ae75fb49baf028ac8ce08e4bedcd3625ff3ff0cd
|
400
|
-
test/helpers/file/xml_test.rb:
|
401
|
-
comments:
|
402
|
-
reviewed_by: josh.pencheon
|
403
|
-
safe_revision: 9a6cc769abce5f9bfa5b4f8bd5cda52dfe18b12b
|
404
|
-
test/helpers/file/zip_test.rb:
|
405
|
-
comments:
|
406
|
-
reviewed_by: josh.pencheon
|
407
|
-
safe_revision: 9abdd6ced1d0c90ce8dd88abee4eb6472c7ff0d6
|
408
|
-
test/mapper_test.rb:
|
409
|
-
comments: exposes Mapper internals to test them
|
410
|
-
reviewed_by: josh.pencheon
|
411
|
-
safe_revision: eca44583e9989159b45e90021dd1c65228447180
|
412
|
-
test/non_tabular/mapping_test.rb:
|
413
|
-
comments:
|
414
|
-
reviewed_by: timgentry
|
415
|
-
safe_revision: 63b1e8c275bd63377309d9b124176f39f75576a9
|
416
|
-
test/non_tabular/table_test.rb:
|
417
|
-
comments:
|
418
|
-
reviewed_by: josh.pencheon
|
419
|
-
safe_revision: a69d4a57ddcf13cdc13c27bd2eb91a395fa7ea36
|
420
|
-
test/non_tabular_file_helper_test.rb:
|
421
|
-
comments:
|
422
|
-
reviewed_by: josh.pencheon
|
423
|
-
safe_revision: bb44ade56a2151706eede2c31142440ccf49e6f6
|
424
|
-
test/pdf_form/table_test.rb:
|
425
|
-
comments:
|
426
|
-
reviewed_by: josh.pencheon
|
427
|
-
safe_revision: 5cd2cd0b3a1e254d30d4acc28c6731825a1f84f5
|
428
|
-
test/readme_test.rb:
|
429
|
-
comments:
|
430
|
-
reviewed_by: timgentry
|
431
|
-
safe_revision: cf382902508a21a95b80ac4582fbbd117164e80e
|
432
|
-
test/resources/acro_form.pdf:
|
433
|
-
comments:
|
434
|
-
reviewed_by: josh.pencheon
|
435
|
-
safe_revision: 5cd2cd0b3a1e254d30d4acc28c6731825a1f84f5
|
436
|
-
test/resources/blank_tab_test.xlsx:
|
437
|
-
comments: reviewed contents
|
438
|
-
reviewed_by: joshpencheon
|
439
|
-
safe_revision: 80ce15cc5ca15019400d4cd24c3353191bec00cc
|
440
|
-
test/resources/bomd.csv:
|
441
|
-
comments:
|
442
|
-
reviewed_by: timgentry
|
443
|
-
safe_revision: f755c6960182f7dd460c18866cccfdf09178e860
|
444
|
-
test/resources/broken.csv:
|
445
|
-
comments:
|
446
|
-
reviewed_by: timgentry
|
447
|
-
safe_revision: dab4b8a3e4b29d85eccd971e79936982d888cffd
|
448
|
-
test/resources/claims_utf16be_but_isnt.xml:
|
449
|
-
comments:
|
450
|
-
reviewed_by: josh.pencheon
|
451
|
-
safe_revision: ae75fb49baf028ac8ce08e4bedcd3625ff3ff0cd
|
452
|
-
test/resources/filesystem_paths.yml:
|
453
|
-
comments:
|
454
|
-
reviewed_by: timgentry
|
455
|
-
safe_revision: a8451809db86e0e023b0d93826114a33ff896c4f
|
456
|
-
test/resources/flat_file.pdf:
|
457
|
-
comments:
|
458
|
-
reviewed_by: timgentry
|
459
|
-
safe_revision: aba37ca7167f4e43a912e267a13b0c24a8737731
|
460
|
-
test/resources/flat_file.txt:
|
461
|
-
comments:
|
462
|
-
reviewed_by: timgentry
|
463
|
-
safe_revision: 63b1e8c275bd63377309d9b124176f39f75576a9
|
464
|
-
test/resources/flat_file.yml:
|
465
|
-
comments:
|
466
|
-
reviewed_by: timgentry
|
467
|
-
safe_revision: cf382902508a21a95b80ac4582fbbd117164e80e
|
468
|
-
test/resources/hello_utf16be.txt:
|
469
|
-
comments:
|
470
|
-
reviewed_by: timgentry
|
471
|
-
safe_revision: 3107f711805b6d3b89d32ec923178425aa600dac
|
472
|
-
test/resources/hello_utf16le.txt:
|
473
|
-
comments:
|
474
|
-
reviewed_by: timgentry
|
475
|
-
safe_revision: 3107f711805b6d3b89d32ec923178425aa600dac
|
476
|
-
test/resources/hello_utf8.txt:
|
477
|
-
comments:
|
478
|
-
reviewed_by: timgentry
|
479
|
-
safe_revision: 3961826eb16a30598381c424b95897a2f1e8c40a
|
480
|
-
test/resources/hello_windows.txt:
|
481
|
-
comments:
|
482
|
-
reviewed_by: timgentry
|
483
|
-
safe_revision: 3961826eb16a30598381c424b95897a2f1e8c40a
|
484
|
-
test/resources/hello_world.doc:
|
485
|
-
comments:
|
486
|
-
reviewed_by: timgentry
|
487
|
-
safe_revision: 45da71ebd3acbc0fe53755bcd75483ba17cb6924
|
488
|
-
test/resources/hello_world.docx:
|
489
|
-
comments:
|
490
|
-
reviewed_by: josh.pencheon
|
491
|
-
safe_revision: 1b66cfcbb61dfac93c44889ca0ced5836101c20c
|
492
|
-
test/resources/hello_world.pdf:
|
493
|
-
comments:
|
494
|
-
reviewed_by: josh.pencheon
|
495
|
-
safe_revision: f82e6839f7e3f9932a6f92242c03d94234fc1d82
|
496
|
-
test/resources/hello_world.txt:
|
497
|
-
comments:
|
498
|
-
reviewed_by: timgentry
|
499
|
-
safe_revision: c88000b32401b5ae9ef7f5878a9b630506ab5a94
|
500
|
-
test/resources/high_ascii_delimited.txt:
|
501
|
-
comments:
|
502
|
-
reviewed_by: timgentry
|
503
|
-
safe_revision: e13005efa08f7168c1c65c60ce97057287ace534
|
504
|
-
test/resources/high_ascii_delimited_example_two.txt:
|
505
|
-
comments:
|
506
|
-
reviewed_by: josh.pencheon
|
507
|
-
safe_revision: 902f5326d85372d9632de9869d6f56fc02b83a10
|
508
|
-
test/resources/malformed.csv:
|
509
|
-
comments:
|
510
|
-
reviewed_by: josh.pencheon
|
511
|
-
safe_revision: 71979e0a602ca5a0ce415c194f10add9959f0116
|
512
|
-
test/resources/malformed.xml:
|
513
|
-
comments:
|
514
|
-
reviewed_by: joshpencheon
|
515
|
-
safe_revision: 3947f13e0cbd17f449eba292ad343eeb82116fe9
|
516
|
-
test/resources/malformed_pipe.csv:
|
517
|
-
comments:
|
518
|
-
reviewed_by: josh.pencheon
|
519
|
-
safe_revision: 71979e0a602ca5a0ce415c194f10add9959f0116
|
520
|
-
test/resources/normal.7z:
|
521
|
-
comments:
|
522
|
-
reviewed_by: josh.pencheon
|
523
|
-
safe_revision: f2feb0c430947839183d7223e60e7c29b2c0f846
|
524
|
-
test/resources/normal.csv:
|
525
|
-
comments:
|
526
|
-
reviewed_by: timgentry
|
527
|
-
safe_revision: f755c6960182f7dd460c18866cccfdf09178e860
|
528
|
-
test/resources/normal.csv.zip:
|
529
|
-
comments:
|
530
|
-
reviewed_by: timgentry
|
531
|
-
safe_revision: c88000b32401b5ae9ef7f5878a9b630506ab5a94
|
532
|
-
test/resources/normal_pipe.csv:
|
533
|
-
comments:
|
534
|
-
reviewed_by: timgentry
|
535
|
-
safe_revision: c88000b32401b5ae9ef7f5878a9b630506ab5a94
|
536
|
-
test/resources/normal_thorn.csv:
|
537
|
-
comments:
|
538
|
-
reviewed_by: timgentry
|
539
|
-
safe_revision: c88000b32401b5ae9ef7f5878a9b630506ab5a94
|
540
|
-
test/resources/not_a_pdf.pdf:
|
541
|
-
comments:
|
542
|
-
reviewed_by: timgentry
|
543
|
-
safe_revision: ed2360bd226dd6babb7683f2f888ba9547ff7e53
|
544
|
-
test/resources/not_a_word_file.doc:
|
545
|
-
comments:
|
546
|
-
reviewed_by: timgentry
|
547
|
-
safe_revision: 45da71ebd3acbc0fe53755bcd75483ba17cb6924
|
548
|
-
test/resources/not_a_word_file.docx:
|
549
|
-
comments:
|
550
|
-
reviewed_by: josh.pencheon
|
551
|
-
safe_revision: 1b66cfcbb61dfac93c44889ca0ced5836101c20c
|
552
|
-
test/resources/not_sign_delimited.txt:
|
553
|
-
comments:
|
554
|
-
reviewed_by: josh.pencheon
|
555
|
-
safe_revision: 902f5326d85372d9632de9869d6f56fc02b83a10
|
556
|
-
test/resources/password_protected_hello_world.docx:
|
557
|
-
comments:
|
558
|
-
reviewed_by: josh.pencheon
|
559
|
-
safe_revision: a69d4a57ddcf13cdc13c27bd2eb91a395fa7ea36
|
560
|
-
test/resources/password_protected_sample_xlsx.xlsx:
|
561
|
-
comments:
|
562
|
-
reviewed_by: josh.pencheon
|
563
|
-
safe_revision: a69d4a57ddcf13cdc13c27bd2eb91a395fa7ea36
|
564
|
-
test/resources/sample.xml:
|
565
|
-
comments:
|
566
|
-
reviewed_by: josh.pencheon
|
567
|
-
safe_revision: 4ab72f84201c2d5f0147b7dfd041f488f6ff0422
|
568
|
-
test/resources/sample_xls.xls:
|
569
|
-
comments:
|
570
|
-
reviewed_by: timgentry
|
571
|
-
safe_revision: 8c30f89f0562ab120769c166d4e93ff839c055f7
|
572
|
-
test/resources/sample_xlsm.xlsm:
|
573
|
-
comments:
|
574
|
-
reviewed_by: ollietulloch
|
575
|
-
safe_revision: 85a080deaa93e4220ad1bf566f29cbdac9b31c0f
|
576
|
-
test/resources/sample_xlsx.xlsx:
|
577
|
-
comments:
|
578
|
-
reviewed_by: timgentry
|
579
|
-
safe_revision: 8c30f89f0562ab120769c166d4e93ff839c055f7
|
580
|
-
test/resources/sheet_streaming.xls:
|
581
|
-
comments:
|
582
|
-
reviewed_by: joshpencheon
|
583
|
-
safe_revision: 334d53bb9e8698bba3d92b4dbd71b36faacae629
|
584
|
-
test/resources/sheet_streaming.xlsx:
|
585
|
-
comments:
|
586
|
-
reviewed_by: joshpencheon
|
587
|
-
safe_revision: 334d53bb9e8698bba3d92b4dbd71b36faacae629
|
588
|
-
test/resources/standard_mappings.yml:
|
589
|
-
comments:
|
590
|
-
reviewed_by: timgentry
|
591
|
-
safe_revision: 31fb1935f4578729d8786eea41cf0ce0a19be1cd
|
592
|
-
test/resources/two_files_single_table_mapping.zip:
|
593
|
-
comments:
|
594
|
-
reviewed_by: ollietulloch
|
595
|
-
safe_revision: 830de0f8cb139c5f61525652b424423935cfc7ac
|
596
|
-
test/resources/txt_file_xls_extension.xls:
|
597
|
-
comments:
|
598
|
-
reviewed_by: timgentry
|
599
|
-
safe_revision: c88000b32401b5ae9ef7f5878a9b630506ab5a94
|
600
|
-
test/resources/txt_file_xlsx_extension.xlsx:
|
601
|
-
comments:
|
602
|
-
reviewed_by: timgentry
|
603
|
-
safe_revision: c88000b32401b5ae9ef7f5878a9b630506ab5a94
|
604
|
-
test/resources/utf-16be_xml.xml:
|
605
|
-
comments:
|
606
|
-
reviewed_by: timgentry
|
607
|
-
safe_revision: 19b1f3d2c2da501cf26ab2b6953a44199871f68a
|
608
|
-
test/resources/utf-16be_xml_with_declaration.xml:
|
609
|
-
comments:
|
610
|
-
reviewed_by: timgentry
|
611
|
-
safe_revision: 39be14a276d30d9b711451a882874a1db41bf9f2
|
612
|
-
test/resources/utf-16le_xml.xml:
|
613
|
-
comments:
|
614
|
-
reviewed_by: timgentry
|
615
|
-
safe_revision: 19b1f3d2c2da501cf26ab2b6953a44199871f68a
|
616
|
-
test/resources/utf-8_xml.xml:
|
617
|
-
comments:
|
618
|
-
reviewed_by: timgentry
|
619
|
-
safe_revision: 19b1f3d2c2da501cf26ab2b6953a44199871f68a
|
620
|
-
test/resources/windows-1252_xml.xml:
|
621
|
-
comments:
|
622
|
-
reviewed_by: timgentry
|
623
|
-
safe_revision: 19b1f3d2c2da501cf26ab2b6953a44199871f68a
|
624
|
-
test/resources/windows.csv:
|
625
|
-
comments:
|
626
|
-
reviewed_by: timgentry
|
627
|
-
safe_revision: f755c6960182f7dd460c18866cccfdf09178e860
|
628
|
-
test/resources/with-control-char-references-in-cdata.xml:
|
629
|
-
comments:
|
630
|
-
reviewed_by: josh.pencheon
|
631
|
-
safe_revision: 9a6cc769abce5f9bfa5b4f8bd5cda52dfe18b12b
|
632
|
-
test/resources/with-control-char-references.xml:
|
633
|
-
comments:
|
634
|
-
reviewed_by: josh.pencheon
|
635
|
-
safe_revision: 9a6cc769abce5f9bfa5b4f8bd5cda52dfe18b12b
|
636
|
-
test/resources/with-control-chars.xml:
|
637
|
-
comments:
|
638
|
-
reviewed_by: joshpencheon
|
639
|
-
safe_revision: 3947f13e0cbd17f449eba292ad343eeb82116fe9
|
640
|
-
test/resources/with-non-control-char-references.xml:
|
641
|
-
comments:
|
642
|
-
reviewed_by: josh.pencheon
|
643
|
-
safe_revision: 9a6cc769abce5f9bfa5b4f8bd5cda52dfe18b12b
|
644
|
-
test/resources/xlsx_file_xls_extension.xls:
|
645
|
-
comments:
|
646
|
-
reviewed_by: timgentry
|
647
|
-
safe_revision: 8c30f89f0562ab120769c166d4e93ff839c055f7
|
648
|
-
test/standard_mappings_test.rb:
|
649
|
-
comments:
|
650
|
-
reviewed_by: timgentry
|
651
|
-
safe_revision: 3c7f827d17aacbf7b811eea67e27553f3b039070
|
652
|
-
test/table_test.rb:
|
653
|
-
comments:
|
654
|
-
reviewed_by: josh.pencheon
|
655
|
-
safe_revision: 3cf7473181f7f835b3dfe7822f6833d751805eaf
|
656
|
-
test/test_helper.rb:
|
657
|
-
comments:
|
658
|
-
reviewed_by: josh.pencheon
|
659
|
-
safe_revision: 93ccee82fc2165d1ca2d9b03d146ae03e769ea96
|
660
|
-
test/universal_importer_helper_test.rb:
|
661
|
-
comments:
|
662
|
-
reviewed_by: josh.pencheon
|
663
|
-
safe_revision: 85869d99ae93252b7f3ef2d0a4db817c88d35c9e
|
664
|
-
test/xml/control_char_escaper_test.rb:
|
665
|
-
comments:
|
666
|
-
reviewed_by: josh.pencheon
|
667
|
-
safe_revision: 9a6cc769abce5f9bfa5b4f8bd5cda52dfe18b12b
|
668
|
-
test/xml/table_test.rb:
|
669
|
-
comments:
|
670
|
-
reviewed_by: ollietulloch
|
671
|
-
safe_revision: 66cff59af2f078152f7459c436d51b57cb93f28e
|
data/gemfiles/Gemfile.rails52
DELETED
data/gemfiles/Gemfile.rails60
DELETED
data/ndr_import.gemspec
DELETED
@@ -1,53 +0,0 @@
|
|
1
|
-
lib = File.expand_path('../lib', __FILE__)
|
2
|
-
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
3
|
-
require 'ndr_import/version'
|
4
|
-
|
5
|
-
Gem::Specification.new do |spec|
|
6
|
-
spec.name = 'ndr_import'
|
7
|
-
spec.version = NdrImport::VERSION
|
8
|
-
spec.authors = ['NCRS Development Team']
|
9
|
-
spec.email = []
|
10
|
-
spec.summary = 'NDR Import'
|
11
|
-
spec.description = 'NDR ETL Importer'
|
12
|
-
spec.homepage = 'https://github.com/PublicHealthEngland/ndr_import'
|
13
|
-
spec.license = 'MIT'
|
14
|
-
|
15
|
-
# Specify which files should be added to the gem when it is released.
|
16
|
-
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
17
|
-
spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do
|
18
|
-
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(docs|test|spec|features)/}) }
|
19
|
-
end
|
20
|
-
spec.files -= %w[.travis.yml] # Not needed in the gem
|
21
|
-
spec.bindir = 'exe'
|
22
|
-
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
23
|
-
spec.require_paths = ['lib']
|
24
|
-
|
25
|
-
spec.add_dependency 'activemodel'
|
26
|
-
spec.add_dependency 'activesupport', '>= 5.0', '< 7'
|
27
|
-
spec.add_dependency 'ndr_support', '>= 5.3.2', '< 6'
|
28
|
-
|
29
|
-
spec.add_dependency 'rubyzip', '~> 2.0'
|
30
|
-
spec.add_dependency 'roo', '~> 2.0'
|
31
|
-
|
32
|
-
spec.add_dependency 'docx', '~> 0.3'
|
33
|
-
spec.add_dependency 'msworddoc-extractor', '0.2.0'
|
34
|
-
spec.add_dependency 'nokogiri', '~> 1.8', '>= 1.8.5'
|
35
|
-
spec.add_dependency 'ooxml_decrypt'
|
36
|
-
spec.add_dependency 'pdf-reader', '~> 2.1'
|
37
|
-
spec.add_dependency 'roo-xls'
|
38
|
-
spec.add_dependency 'seven_zip_ruby', '~> 1.3'
|
39
|
-
spec.add_dependency 'spreadsheet', '1.2.6'
|
40
|
-
|
41
|
-
spec.required_ruby_version = '>= 2.5'
|
42
|
-
|
43
|
-
spec.add_development_dependency 'bundler'
|
44
|
-
spec.add_development_dependency 'rake', '~> 12.3', '>= 12.3.3'
|
45
|
-
spec.add_development_dependency 'minitest'
|
46
|
-
spec.add_development_dependency 'mocha'
|
47
|
-
spec.add_development_dependency 'ndr_dev_support', '>= 3.1.3'
|
48
|
-
spec.add_development_dependency 'guard'
|
49
|
-
spec.add_development_dependency 'guard-rubocop'
|
50
|
-
spec.add_development_dependency 'guard-test'
|
51
|
-
spec.add_development_dependency 'terminal-notifier-guard' if RUBY_PLATFORM =~ /darwin/
|
52
|
-
spec.add_development_dependency 'simplecov'
|
53
|
-
end
|