ndr_import 9.0.3 → 9.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2237d88e9dee0141fe297fd2eb0d2e76f5434e9bb791fb667505193f30263014
-  data.tar.gz: 52746c16790c3da92e64ac14ed7d77786a41c5457ddc0b8190f52d4414e4c6d7
+  metadata.gz: aaf3c826acb51f4d579fb956e9606b5b86e120f3bd561db762d72081f27a1098
+  data.tar.gz: 2b11ddf7dc9b748b4a1ac9cc91b47c47b99677e4bde990c44d1ec20e76efbfc0
 SHA512:
-  metadata.gz: 8b8793a45dea035b2abf04a44c20d2ec4fc90bcfef9c5dec4415fab26abdba2ff218b7d2996a165f3447f86046dfd558bfb363e21221f147fec3db9c7c83668c
-  data.tar.gz: eedb044185f1bc7e843e91e634fdc8b88cae9c026344d92e3cdc1046979873d843d476c7a0f7ebfb3c631901b47c9109b6826696124c56b07082ba4a6842af9c
+  metadata.gz: 3faaf744255693f04425b6e5ed1ec7198e912d2854b3f482a39d9bc53fe3859ed3bb6ef7855cdfd03959042a530f3a2f528239548676c7159c9029d69b6c160d
+  data.tar.gz: 1aa9d1895d7f874499823c4b87f1874e025b376f239030f3baf40bab5f5ed3ed09f80721205e271446893a9070652df3f26fceb48b59a9eb86ac121fdbfd92c6

data/.github/CODEOWNERS

@@ -0,0 +1,2 @@
+# Admins should have oversight of the version:
+lib/ndr_import/version.rb      @publichealthengland/ndr-admins

data/.github/workflows/lint.yml

@@ -0,0 +1,23 @@
+name: Lint
+
+on: [pull_request]
+
+jobs:
+  rubocop:
+    name: RuboCop
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0 # fetch everything
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: 3.0
+    - name: Install dependencies
+      run: bundle install
+    - name: Run RuboCop against BASE..HEAD changes
+      run: bundle exec rake rubocop:diff origin/${GITHUB_BASE_REF#*/}
+

data/.github/workflows/test.yml

@@ -0,0 +1,72 @@
+name: Test
+
+on: [push]
+
+jobs:
+  test:
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby-version:
+          - 2.6
+          - 2.7
+          - 3.0
+        gemfile:
+          - gemfiles/Gemfile.rails52
+          - gemfiles/Gemfile.rails60
+
+    name: Ruby ${{ matrix.ruby-version }} / Bundle ${{ matrix.gemfile }}
+
+    runs-on: ubuntu-latest
+
+    env:
+      BUNDLE_GEMFILE: ${{ matrix.gemfile }}
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby-version }}
+    - name: Install dependencies
+      run: bundle install
+    - name: Run tests
+      run: bundle exec rake
+
+  # A utility job upon which Branch Protection can depend,
+  # thus remaining agnostic of the matrix.
+  test_matrix:
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    name: Matrix
+    needs: test
+    steps:
+    - name: Check build matrix status
+      if: ${{ needs.test.result != 'success' }}
+      run: exit 1
+
+  notify:
+    # Run only on master, but regardless of whether tests past:
+    if: ${{ always() && github.ref == 'refs/heads/master' }}
+
+    needs: test_matrix
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: 8398a7/action-slack@v3
+      with:
+        status: custom
+        fields: workflow,commit,author
+        custom_payload: |
+          {
+            channel: 'C7FQWGDHP',
+            username: 'CI – ' + '${{ github.repository }}'.split('/')[1],
+            icon_emoji: ':hammer_and_wrench:',
+            attachments: [{
+              color: '${{ needs.test_matrix.result }}' === 'success' ? 'good' : '${{ needs.test_matrix.result }}' === 'failure' ? 'danger' : 'warning',
+              text: `${process.env.AS_WORKFLOW} against \`${{ github.ref }}\` (${process.env.AS_COMMIT}) for ${{ github.actor }} resulted in *${{ needs.test_matrix.result }}*.`
+            }]
+          }
+      env:
+        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

data/CHANGELOG.md

@@ -1,6 +1,15 @@
 ## [Unreleased]
 *no unreleased changes*
 
+## 9.1.0 / 2021-02-01
+### Added
+* `CSVLibrary` is now deprecated.
+* Handle xlsm files
+
+### Fixed
+* Fix `CSVLibrary.foreach` on Ruby 3.0
+* Updated jekyll bundle
+
 ## 9.0.3 / 2021-01-04
 ### Fixed
 * Address issue importing multiple files against the same table (#54)

data/README.md

@@ -1,5 +1,4 @@
-# NdrImport [![Build Status](https://travis-ci.org/PublicHealthEngland/ndr_import.svg?branch=master)](https://travis-ci.org/PublicHealthEngland/ndr_import) [![Gem Version](https://badge.fury.io/rb/ndr_import.svg)](https://badge.fury.io/rb/ndr_import) [![Documentation](https://img.shields.io/badge/ndr_import-docs-blue.svg)](https://www.rubydoc.info/gems/ndr_import)
-
+# NdrImport [![Build Status](https://github.com/publichealthengland/ndr_import/workflows/Test/badge.svg)](https://github.com/publichealthengland/ndr_import/actions?query=workflow%3Atest) [![Gem Version](https://badge.fury.io/rb/ndr_import.svg)](https://rubygems.org/gems/ndr_import) [![Documentation](https://img.shields.io/badge/ndr_import-docs-blue.svg)](https://www.rubydoc.info/gems/ndr_import)
 This is the Public Health England (PHE) National Disease Registers (NDR) Import ETL ruby gem, providing:
 
 1. file import handlers for *extracting* data from delimited files (csv, pipe, tab, thorn), .xls(x) spreadsheets, .doc(x) word documents, PDF, PDF AcroForms, XML, 7-Zip and Zip files.

data/code_safety.yml

@@ -1,5 +1,17 @@
 ---
 file safety:
+  ".github/CODEOWNERS":
+    comments: 
+    reviewed_by: ollietulloch
+    safe_revision: b64ff21375dcde2b8fefe622ee9861f0fea21487
+  ".github/workflows/lint.yml":
+    comments: 
+    reviewed_by: ollietulloch
+    safe_revision: b64ff21375dcde2b8fefe622ee9861f0fea21487
+  ".github/workflows/test.yml":
+    comments: 
+    reviewed_by: ollietulloch
+    safe_revision: b64ff21375dcde2b8fefe622ee9861f0fea21487
   ".gitignore":
     comments: whole file re-reviewed
     reviewed_by: josh.pencheon
@@ -12,14 +24,10 @@ file safety:
     comments: 
     reviewed_by: josh.pencheon
     safe_revision: b09e268ff9c8349b914aa1b7ba888e1d39f97e4a
-  ".travis.yml":
-    comments: 
-    reviewed_by: ollietulloch
-    safe_revision: ed8513d290796b9cb9554c4ebe6e87ac21fbbeae
   CHANGELOG.md:
     comments: 
     reviewed_by: ollietulloch
-    safe_revision: 92d2d9d56f17c143709e25560d7620aaf147008f
+    safe_revision: d88ded7c260da37200610e4f0b204a4ea2e481f9
   CODE_OF_CONDUCT.md:
     comments: 
     reviewed_by: timgentry
@@ -38,8 +46,8 @@ file safety:
     safe_revision: 5d185a0aeba6a9cd2ff5e59efadcaeec9be45d8b
   README.md:
     comments: 
-    reviewed_by: josh.pencheon
-    safe_revision: 1bc459db8970dde36e9b240b6dd08cca629664e3
+    reviewed_by: ollietulloch
+    safe_revision: b64ff21375dcde2b8fefe622ee9861f0fea21487
   Rakefile:
     comments: 
     reviewed_by: josh.pencheon
@@ -59,7 +67,7 @@ file safety:
   docs/Gemfile.lock:
     comments: 
     reviewed_by: ollietulloch
-    safe_revision: 6f274715bb341c3070190f04f67af9500b510580
+    safe_revision: ea0149c7739676463a252ffd9fbe4af238762b2b
   docs/_config.yml:
     comments: 
     reviewed_by: josh.pencheon
@@ -147,7 +155,7 @@ file safety:
   lib/ndr_import/csv_library.rb:
     comments: 
     reviewed_by: ollietulloch
-    safe_revision: acc72173b81702d55fe3aff78054425609a4d339
+    safe_revision: 6b8668967dbd42d7893a0fa5f0aa1ec1c11227e1
   lib/ndr_import/file/acro_form.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -170,8 +178,8 @@ file safety:
     safe_revision: 897f8b648d633368cf2001d17ab89c06a12d445b
   lib/ndr_import/file/excel.rb:
     comments: 
-    reviewed_by: josh.pencheon
-    safe_revision: c3183e522bce50008df576ceb47fe4761ab8f966
+    reviewed_by: ollietulloch
+    safe_revision: 37482c79448bea80033f6f69d97584df330c9861
   lib/ndr_import/file/office_file_helper.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -215,7 +223,7 @@ file safety:
   lib/ndr_import/helpers/file/delimited.rb:
     comments: 
     reviewed_by: ollietulloch
-    safe_revision: acc72173b81702d55fe3aff78054425609a4d339
+    safe_revision: 4a5cc1d362c632fc1f9242c69982fbce33557e17
   lib/ndr_import/helpers/file/excel.rb:
     comments: 
     reviewed_by: joshpencheon
@@ -292,7 +300,7 @@ file safety:
   lib/ndr_import/universal_importer_helper.rb:
     comments: 
     reviewed_by: ollietulloch
-    safe_revision: 830de0f8cb139c5f61525652b424423935cfc7ac
+    safe_revision: ee2e74e4ceda4ff48cbda6872a6bdf0874212c21
   lib/ndr_import/unmapped_data_error.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -300,7 +308,7 @@ file safety:
   lib/ndr_import/version.rb:
     comments: another check?
     reviewed_by: ollietulloch
-    safe_revision: 92d2d9d56f17c143709e25560d7620aaf147008f
+    safe_revision: d88ded7c260da37200610e4f0b204a4ea2e481f9
   lib/ndr_import/xml/table.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -309,6 +317,10 @@ file safety:
     comments: 
     reviewed_by: josh.pencheon
     safe_revision: 95e6ee9997d06471fe6f2f169c3c701471086371
+  test/csv_library_test.rb:
+    comments: 
+    reviewed_by: ollietulloch
+    safe_revision: 6b8668967dbd42d7893a0fa5f0aa1ec1c11227e1
   test/file/acro_form_test.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -327,16 +339,16 @@ file safety:
     safe_revision: a69d4a57ddcf13cdc13c27bd2eb91a395fa7ea36
   test/file/excel_test.rb:
     comments: 
-    reviewed_by: josh.pencheon
-    safe_revision: a69d4a57ddcf13cdc13c27bd2eb91a395fa7ea36
+    reviewed_by: ollietulloch
+    safe_revision: 85a080deaa93e4220ad1bf566f29cbdac9b31c0f
   test/file/pdf_test.rb:
     comments: 
     reviewed_by: josh.pencheon
     safe_revision: cb24ed3ea8116730d07f74546cd6fed0738b171d
   test/file/registry_test.rb:
     comments: 
-    reviewed_by: josh.pencheon
-    safe_revision: 5cd2cd0b3a1e254d30d4acc28c6731825a1f84f5
+    reviewed_by: ollietulloch
+    safe_revision: 85a080deaa93e4220ad1bf566f29cbdac9b31c0f
   test/file/seven_zip_test.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -553,6 +565,10 @@ file safety:
     comments: 
     reviewed_by: timgentry
     safe_revision: 8c30f89f0562ab120769c166d4e93ff839c055f7
+  test/resources/sample_xlsm.xlsm:
+    comments: 
+    reviewed_by: ollietulloch
+    safe_revision: 85a080deaa93e4220ad1bf566f29cbdac9b31c0f
   test/resources/sample_xlsx.xlsx:
     comments: 
     reviewed_by: timgentry

data/lib/ndr_import/csv_library.rb

@@ -1,17 +1,27 @@
 # This file allows us to choose the CSV library we want to use.
 
 require 'csv'
+require 'active_support/deprecation'
+
 # Using relevant core CSV library.
-CSVLibrary = CSV
+class CSVLibrary < CSV; end
 
 class << CSVLibrary
   # Is the library we're using FasterCSV?
   def fastercsv?
+    deprecate('if you desparately want fastercsv, please use it explicitly')
     not self.const_defined?(:Reader)
   end
 
   # Ensure that we can pass "mode" straight through the underlying IO object
+  #
+  # Note: this could likely be refactored now, as upstream support for something
+  #       very similar was added:
+  #
+  #       https://github.com/ruby/csv/commit/b4edaf2cf1aa36f5c6264c07514b66739b87ceee
+  #
   def foreach(path, **options, &block)
+    deprecate('CSV#foreach exists, with an optional `mode` argument')
     return to_enum(__method__, path, **options) unless block
     open(path, options.delete(:mode) || 'r', **options) do |csv|
       csv.each(&block)
@@ -19,20 +29,32 @@ class << CSVLibrary
   end
 
   def write_csv_to_string(data)
+    deprecate('write_csv_to_string -> generate')
     self.generate do |csv|
       data.each { |line| csv << line }
     end
   end
 
   def write_csv_to_file(data, filepath, mode = 'w')
+    deprecate('write_csv_to_file -> open')
     self.open(filepath, mode) do |csv|
       data.each { |line| csv << line }
     end
   end
 
   def read_csv_from_file(filepath)
+    deprecate('read_csv_from_file -> read')
     self.read(filepath)
   end
+
+  private
+
+  def deprecate(additional_message = nil)
+    ActiveSupport::Deprecation.warn(<<~MESSAGE)
+      CSVLibrary is deprecated, and will be removed in a future version of ndr_import.
+      Please use standard functionality provided by Ruby's CSV library (#{additional_message}).
+    MESSAGE
+  end
 end
 
 # Forward port CSV::Cell, as it is sometimes

data/lib/ndr_import/file/excel.rb

@@ -90,14 +90,14 @@ module NdrImport
         case SafeFile.extname(path).downcase
         when '.xls'
           Roo::Excel.new(SafeFile.safepath_to_string(path))
-        when '.xlsx'
+        when '.xlsm', '.xlsx'
           if @options['file_password']
             Roo::Excelx.new(StringIO.new(decrypted_file_string(path, @options['file_password'])))
           else
             Roo::Excelx.new(SafeFile.safepath_to_string(path))
           end
         else
-          fail "Received file path with unexpected extension #{SafeFile.extname(path)}"
+          raise "Received file path with unexpected extension #{SafeFile.extname(path)}"
         end
       rescue Ole::Storage::FormatError => e
         # TODO: Do we need to remove the new_file after using it?
@@ -105,16 +105,14 @@ module NdrImport
         # try to load the .xls file as an .xlsx file, useful for sources like USOM
         # roo check file extensions in file_type_check (GenericSpreadsheet),
         # so we create a duplicate file in xlsx extension
-        if /(.*)\.xls$/.match(path)
-          new_file_name = SafeFile.basename(path).gsub(/(.*)\.xls$/, '\1_amend.xlsx')
-          new_file_path = SafeFile.dirname(path).join(new_file_name)
-          copy_file(path, new_file_path)
+        raise e.message unless /(.*)\.xls$/.match(path)
 
-          load_workbook(new_file_path)
-        else
-          raise e.message
-        end
-      rescue => e
+        new_file_name = SafeFile.basename(path).gsub(/(.*)\.xls$/, '\1_amend.xlsx')
+        new_file_path = SafeFile.dirname(path).join(new_file_name)
+        copy_file(path, new_file_path)
+
+        load_workbook(new_file_path)
+      rescue RuntimeError, ::Zip::Error => e
         raise ["Unable to read the file '#{path}'", e.message].join('; ')
       end
 
@@ -133,6 +131,6 @@ module NdrImport
       end
     end
 
-    Registry.register(Excel, 'xls', 'xlsx')
+    Registry.register(Excel, 'xls', 'xlsm', 'xlsx')
   end
 end

data/lib/ndr_import/helpers/file/delimited.rb

@@ -32,11 +32,11 @@ module NdrImport
           return enum_for(:delimited_rows, path, col_sep, liberal) unless block_given?
 
           safe_path = SafeFile.safepath_to_string(path)
-          encodings = determine_encodings!(safe_path, col_sep, liberal)
+          options = determine_encodings!(safe_path, col_sep, liberal)
 
-          # By now, we know `encodings` should let us read the whole
+          # By now, we know `options` should let us read the whole
           # file succesfully; if there are problems, we should crash.
-          CSVLibrary.foreach(safe_path, **encodings) do |line|
+          CSV.foreach(safe_path, options.delete(:mode), **options) do |line|
             yield line.map(&:to_s)
           end
         end
@@ -46,7 +46,7 @@ module NdrImport
         # Derive the source encoding by trying all supported encodings.
         # Returns first set of working options, or raises if none could be found.
         def determine_encodings!(safe_path, col_sep, liberal)
-          # delimiter encoding => # FasterCSV encoding string
+          # delimiter encoding => # CSV encoding string
           supported_encodings = {
             'UTF-8'        => 'r:bom|utf-8',
             'Windows-1252' => 'r:windows-1252:utf-8'
@@ -67,14 +67,13 @@ module NdrImport
             begin
               options = {
                 col_sep: (col_sep || ',').force_encoding(delimiter_encoding),
-                liberal_parsing: liberal,
-                mode: access_mode
+                liberal_parsing: liberal
               }
 
               row_num = 0
               # Iterate through the file; if we reach the end, this encoding worked:
-              CSVLibrary.foreach(safe_path, **options) { |_line| row_num += 1 }
-              return options
+              CSV.foreach(safe_path, access_mode, **options) { |_line| row_num += 1 }
+              return options.merge(mode: access_mode)
             rescue ArgumentError => e
               next if e.message =~ /invalid byte sequence/ # This encoding didn't work
               raise(e)

data/lib/ndr_import/universal_importer_helper.rb

@@ -27,7 +27,6 @@ module NdrImport
     end
 
     def table_enumerators(filename)
-      table_enumerators = {}
       table_enumerators = Hash.new { |hash, key| hash[key] = TableEnumProxy.new }
 
       extract(filename).each do |table, rows|

data/lib/ndr_import/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 # This stores the current version of the NdrImport gem
 module NdrImport
-  VERSION = '9.0.3'.freeze
+  VERSION = '9.1.0'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ndr_import
 version: !ruby/object:Gem::Version
-  version: 9.0.3
+  version: 9.1.0
 platform: ruby
 authors:
 - NCRS Development Team
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-01-04 00:00:00.000000000 Z
+date: 2021-02-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -365,6 +365,9 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/CODEOWNERS"
+- ".github/workflows/lint.yml"
+- ".github/workflows/test.yml"
 - ".gitignore"
 - ".hound.yml"
 - ".rubocop.yml"