ndr_import 11.6.0 → 11.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0f7ea78a292f3c2ee622bda06d341329c97af0dd0edd36c039ee349a08b74284
-  data.tar.gz: 1dae1db120f401a1cfb85b5d243563fd9ba26f290b313d561d4b6b3327b732b8
+  metadata.gz: c16e2b4fa7494365c5c55d5b8947641a3396cd58bf53ae6914525554cdf565a1
+  data.tar.gz: ca00a216b0bf6542e03777c60b26a41b2824c3902da8b3bee2bd12369b3e0197
 SHA512:
-  metadata.gz: bb20b9455c14347bd9ce894858693f94f3c7d8d3fa2c2eac390bb948011db5ae1001513ce6654cca1ca2bdbad40cf5dbd5e6c3bb491def911893159447f7fbe9
-  data.tar.gz: 5683e42c10156785a8e58f13269250f59f90c81ca46613840e405f6f7e6c80a7f203e3ccdb4b49fef0eeb1d300394f5120464cb5d5bd193950505f6f36365281
+  metadata.gz: b0fa0f74a4c2a1cf3d036a26aa60e186b56531fb3d4328c68ead64b6d8fc81a046f7eca633f5e9acd149121debae3ecf67f83d52f7a56ad316da46f1b963a9ff
+  data.tar.gz: e9f9a6adb560d9d5fce254e9a33a869dfeb57e4121b2ecb8a6b4d501b9b4bc965021805e912eb7dad37f899bf3eeb0340367f1232a595580cfcef6bef1960a32

data/CHANGELOG.md

@@ -1,6 +1,23 @@
 ## [Unreleased]
 * no unreleased changes
 
+## 11.8.0 / 2026-11-03
+### Added
+* Support Gzip/gz
+
+### Changed
+* Removed deprecated `CSVLibrary`
+* Minimum `rubyzip` version is now 2.4
+
+### Fixed
+* Drop support for Rails 7.1, 7.2
+* Fix bug in VCF metadata
+* Fix bug where table columns are mutated by a regexp column
+
+## 11.7.0 / 2026-09-24
+### Added
+* Added the ability to define a virtual :filename column
+
 ## 11.6.0 / 2026-09-08
 ### Fixed
 * `NdrImport::File::SevenZip` was extracting files one directory level too deep

data/README.md

@@ -1,7 +1,7 @@
 # NdrImport [![Build Status](https://github.com/NHSDigital/ndr_import/workflows/Test/badge.svg)](https://github.com/NHSDigital/ndr_import/actions?query=workflow%3Atest) [![Gem Version](https://badge.fury.io/rb/ndr_import.svg)](https://rubygems.org/gems/ndr_import) [![Documentation](https://img.shields.io/badge/ndr_import-docs-blue.svg)](https://www.rubydoc.info/gems/ndr_import)
 This is the NHS Digital (NHSD) National Disease Registers (NDR) Import ETL ruby gem, providing:
 
-1. file import handlers for *extracting* data from delimited files (csv, pipe, tab, thorn), JSON Lines, .xls(x) spreadsheets, .doc(x) word documents, PDF, PDF AcroForms, XML, 7-Zip, Zip, avro and VCF files.
+1. file import handlers for *extracting* data from delimited files (csv, pipe, tab, thorn), JSON Lines, .xls(x) spreadsheets, .doc(x) word documents, PDF, PDF AcroForms, XML, 7-Zip, Zip, gzip, avro and VCF files.
 2. table mappers for *transforming* tabular and non-tabular data into key value pairs grouped by a common "klass".
 
 ## Installation

data/lib/ndr_import/csv_cell.rb

@@ -0,0 +1,13 @@
+require 'csv'
+
+# Forward port CSV::Cell, as it is sometimes
+# serialised in YAML. :-(
+class CSV::Cell < String
+  def initialize(data = '', is_null = false)
+    super(is_null ? '' : data)
+  end
+
+  def data
+    to_s
+  end
+end

data/lib/ndr_import/file/all.rb

@@ -3,6 +3,7 @@ require_relative 'acro_form'
 require_relative 'avro'
 require_relative 'delimited'
 require_relative 'docx'
+require_relative 'gzip'
 require_relative 'excel'
 require_relative 'pdf'
 require_relative 'seven_zip'

data/lib/ndr_import/file/base.rb

@@ -1,6 +1,6 @@
 require 'active_support/core_ext/hash'
 require 'ndr_support/safe_file'
-require 'ndr_import/csv_library'
+require 'ndr_import/csv_cell'
 require_relative 'registry'
 
 module NdrImport

data/lib/ndr_import/file/delimited.rb

@@ -1,5 +1,5 @@
 require 'ndr_support/safe_file'
-require 'ndr_import/csv_library'
+require 'ndr_import/csv_cell'
 require 'ndr_import/helpers/file/delimited'
 require_relative 'registry'
 

data/lib/ndr_import/file/gzip.rb

@@ -0,0 +1,66 @@
+require 'securerandom'
+require 'zlib'
+require 'ndr_support/safe_file'
+require_relative 'registry'
+
+module NdrImport
+  # This is one of a collection of file handlers that deal with individual formats of data.
+  # They can be instantiated directly or via the factory method Registry.tables
+  module File
+    # This class is a gzip file handler that returns tables from the decompressed file. It also
+    # handles bgzip files (e.g. .vcf.gz), which are a series of gzip members.
+    class Gzip < Base
+      def initialize(filename, format, options = {})
+        super
+        @unzip_path = options['unzip_path']
+
+        validate_unzip_path_is_safe!
+      end
+
+      def files(&block)
+        fail 'Not allowed in external environment' if defined?(::Rails) && ::Rails.env.external?
+
+        return enum_for(:files) unless block
+
+        destination = unique_destination
+        FileUtils.mkdir_p(SafeFile.safepath_to_string(destination))
+
+        # e.g. sample.vcf.gz is decompressed to sample.vcf
+        basename = ::File.basename(SafeFile.safepath_to_string(@filename), '.gz')
+        decompressed_filename = destination.join(basename)
+        decompress(decompressed_filename)
+
+        Registry.files(decompressed_filename, @options).each(&block)
+      end
+
+      # Gzip files produce files, never tables.
+      def tables
+        fail 'Gzip#tables should never be called'
+      end
+
+      private
+
+      # A new folder to decompress into. The time alone isn't unique: two decompressions in the
+      # same millisecond (e.g. consecutive tests, or two imports) would share a folder and collide.
+      def unique_destination
+        @unzip_path.join("#{Time.current.strftime('%H%M%S%L')}_#{SecureRandom.hex(4)}")
+      end
+
+      # zcat reads every gzip member. GzipReader#read stops after the first, which would silently
+      # read only the first block of a bgzip file.
+      def decompress(decompressed_filename)
+        ::File.open(SafeFile.safepath_to_string(@filename), 'rb') do |input|
+          ::File.open(SafeFile.safepath_to_string(decompressed_filename), 'wb') do |output|
+            Zlib::GzipReader.zcat(input) { |chunk| output.write(chunk) }
+          end
+        end
+      end
+
+      def validate_unzip_path_is_safe!
+        SafeFile.safepath_to_string(@unzip_path)
+      end
+    end
+
+    Registry.register(Gzip, 'gz')
+  end
+end

data/lib/ndr_import/file/zip.rb

@@ -1,3 +1,4 @@
+require 'securerandom'
 require 'zip'
 require 'ndr_support/safe_file'
 require_relative 'registry'
@@ -21,7 +22,7 @@ module NdrImport
 
         return enum_for(:files) unless block
 
-        destination = @unzip_path.join(Time.current.strftime('%H%M%S%L'))
+        destination = unique_destination
         FileUtils.mkdir_p(SafeFile.safepath_to_string(destination))
 
         ::Zip::File.open(SafeFile.safepath_to_string(@filename)) do |zipfile|
@@ -36,6 +37,12 @@ module NdrImport
 
       private
 
+      # A new folder to extract into. The time alone isn't unique: two extractions in the same
+      # millisecond (e.g. consecutive tests, or two imports) would share a folder and collide.
+      def unique_destination
+        @unzip_path.join("#{Time.current.strftime('%H%M%S%L')}_#{SecureRandom.hex(4)}")
+      end
+
       # Unzip the zip file entry and enumerate over it
       def unzip_entries(zipfile, destination, &block)
         zipfile.entries.each do |entry|

data/lib/ndr_import/helpers/file/delimited.rb

@@ -1,5 +1,5 @@
 require 'ndr_support/safe_file'
-require 'ndr_import/csv_library'
+require 'ndr_import/csv_cell'
 
 module NdrImport
   module Helpers
@@ -81,7 +81,7 @@ module NdrImport
             next if e.message =~ /invalid multibyte character/ # This encoding didn't work
 
             raise(e)
-          rescue CSVLibrary::MalformedCSVError => e
+          rescue CSV::MalformedCSVError => e
             next if e.message =~ /Invalid byte sequence/ # This encoding didn't work
 
             raise malformed_csv_error(e, col_sep, row_num + 1, safe_path)

data/lib/ndr_import/table.rb

@@ -50,6 +50,7 @@ module NdrImport
       @row_index = 0
       @header_valid = false
       @header_best_guess = nil
+      reset_regexp_columns
       @notifier.try(:started)
 
       last_col = last_column_to_transform
@@ -80,13 +81,19 @@ module NdrImport
       @notifier.try(:processed, @row_index)
     end
 
-    # Update 'column' values expressed as a regular expression
+    # Update 'column' values expressed as a regular expression to the header they match in this
+    # file. The patterns themselves are kept (see regexp_column_patterns), so the next file is
+    # matched afresh.
     def mutate_regexp_columns(line)
-      @columns.each_with_index do |column, index|
-        next unless column['column'].is_a? Regexp
+      return if regexp_column_patterns.empty?
 
-        column['column'] = line[index] if line[index].match? column['column']
+      @columns = @columns.each_with_index.map do |column, index|
+        pattern = regexp_column_patterns[index]
+        next column unless pattern && line[index].to_s.match?(pattern)
+
+        column.merge('column' => line[index])
       end
+      @masked_mappings = nil
     end
 
     # This method transforms an incoming line of data by applying each of the klass masked
@@ -120,11 +127,40 @@ module NdrImport
         value = send(option)
         coder[option] = value if value
       end
-      coder['columns'] = @columns
+      coder['columns'] = columns_with_regexp_patterns
     end
 
     private
 
+    # The 'column' values given as regular expressions, by column index, captured before any
+    # are replaced with a matched header. Memoised lazily rather than in initialize, so it also
+    # works for tables loaded from YAML, which don't call initialize.
+    def regexp_column_patterns
+      @regexp_column_patterns ||=
+        Array(@columns).each_with_index.with_object({}) do |(column, index), patterns|
+          patterns[index] = column['column'] if column && column['column'].is_a?(Regexp)
+        end
+    end
+
+    # Puts the original regular expressions back, so a new file is matched against them
+    def reset_regexp_columns
+      return if regexp_column_patterns.empty?
+
+      @columns         = columns_with_regexp_patterns
+      @masked_mappings = nil
+    end
+
+    # The columns, with any 'column' values given as regular expressions restored. Unchanged
+    # (including nil, for a table with no columns) when there are none.
+    def columns_with_regexp_patterns
+      return @columns if regexp_column_patterns.empty?
+
+      Array(@columns).each_with_index.map do |column, index|
+        pattern = regexp_column_patterns[index]
+        pattern ? column.merge('column' => pattern) : column
+      end
+    end
+
     # This method uses a buffer to not yield the last <buffer_size> iterations of an enumerable.
     # We use it to skip footer lines (without having to convert the enumerable to an array).
     def skip_footer_lines(lines, buffer_size)
@@ -237,7 +273,10 @@ module NdrImport
 
     # returns the column names as we expect to receive them
     def column_names(column_mappings)
-      column_mappings.map { |c| (c['column'] || c['standard_mapping']).try(:downcase) }
+      column_mappings.map do |c|
+        name = c['column'] || c['standard_mapping']
+        name.to_s.downcase unless name.is_a?(Regexp)
+      end
     end
 
     # If specified in the mapping, stop transforming data at a given index (column)

data/lib/ndr_import/universal_importer_helper.rb

@@ -83,7 +83,28 @@ module NdrImport
         # Merge file_metadata (from VCF or XML) with directly assigned table_metadata
         # to form `table_metadata`
         mapping.table_metadata = (mapping.table_metadata || {}).merge(file_metadata || {})
-        yield(mapping, table_content)
+
+        augmented_table_content = filename_augmented_rows(mapping, table_content, filename)
+
+        yield(mapping, augmented_table_content)
+      end
+    end
+
+    def filename_augmented_rows(table, rows, filename)
+      return enum_for(:filename_augmented_rows, table, rows, filename) unless block_given?
+
+      filename_column_position = table.columns.index { |c| c['column'] == :filename }
+
+      rows.each.with_index do |row, i|
+        if filename_column_position
+          if i < table.header_lines
+            row.insert(filename_column_position, :filename)
+          else
+            row.insert(filename_column_position, ::File.basename(filename))
+          end
+        end
+
+        yield row
       end
     end
 

data/lib/ndr_import/vcf/table.rb

@@ -5,10 +5,14 @@ module NdrImport
     # Syntatic sugar to ensure `header_lines` and `footer_lines` are 1 and 0 respectively.
     # All other Table logic is inherited from `NdrImport::Table`
     class Table < ::NdrImport::Table
+      VCF_OPTIONS = %w[vcf_file_metadata].freeze
+
       def self.all_valid_options
-        super - %w[delimiter header_lines footer_lines] + %w[vcf_file_metadata]
+        super - %w[delimiter header_lines footer_lines] + VCF_OPTIONS
       end
 
+      attr_reader(*VCF_OPTIONS)
+
       def header_lines
         1
       end

data/lib/ndr_import/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 # This stores the current version of the NdrImport gem
 module NdrImport
-  VERSION = '11.6.0'
+  VERSION = '11.8.0'
 end

data/lib/ndr_import.rb

@@ -1,5 +1,5 @@
 require 'ndr_import/version'
-require 'ndr_import/csv_library'
+require 'ndr_import/csv_cell'
 require 'ndr_import/mapping_error'
 require 'ndr_import/missing_field_error'
 require 'ndr_import/mapper'

metadata

@@ -1,13 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ndr_import
 version: !ruby/object:Gem::Version
-  version: 11.6.0
+  version: 11.8.0
 platform: ruby
 authors:
 - NCRS Development Team
+autorequire:
 bindir: exe
 cert_chain: []
-date: 1980-01-02 00:00:00.000000000 Z
+date: 2026-11-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -29,7 +30,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '7.1'
+        version: '8.0'
     - - "<"
       - !ruby/object:Gem::Version
         version: '8.2'
@@ -39,7 +40,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '7.1'
+        version: '8.0'
     - - "<"
       - !ruby/object:Gem::Version
         version: '8.2'
@@ -89,7 +90,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '2.4'
     - - "<"
       - !ruby/object:Gem::Version
         version: '4'
@@ -99,7 +100,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '2.4'
     - - "<"
       - !ruby/object:Gem::Version
         version: '4'
@@ -107,16 +108,22 @@ dependencies:
   name: avro
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.11.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.13'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.11.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.13'
 - !ruby/object:Gem::Dependency
   name: bio-vcf
   requirement: !ruby/object:Gem::Requirement
@@ -284,21 +291,21 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: guard
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 12.3.3
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 12.3.3
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: minitest
+  name: guard-rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -312,7 +319,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: mocha
+  name: guard-test
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -326,21 +333,21 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: ndr_dev_support
+  name: minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.1.3
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.1.3
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: guard
+  name: mocha
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -354,35 +361,35 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: guard-rubocop
+  name: ndr_dev_support
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.1.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.1.3
 - !ruby/object:Gem::Dependency
-  name: guard-test
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 12.3.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 12.3.3
 - !ruby/object:Gem::Dependency
-  name: terminal-notifier-guard
+  name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -396,7 +403,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: simplecov
+  name: terminal-notifier-guard
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -430,7 +437,7 @@ files:
 - lib/ndr_import.rb
 - lib/ndr_import/acroform_reader.rb
 - lib/ndr_import/avro/table.rb
-- lib/ndr_import/csv_library.rb
+- lib/ndr_import/csv_cell.rb
 - lib/ndr_import/file/acro_form.rb
 - lib/ndr_import/file/all.rb
 - lib/ndr_import/file/avro.rb
@@ -438,6 +445,7 @@ files:
 - lib/ndr_import/file/delimited.rb
 - lib/ndr_import/file/docx.rb
 - lib/ndr_import/file/excel.rb
+- lib/ndr_import/file/gzip.rb
 - lib/ndr_import/file/json_lines.rb
 - lib/ndr_import/file/office_file_helper.rb
 - lib/ndr_import/file/pdf.rb
@@ -482,6 +490,7 @@ homepage: https://github.com/NHSDigital/ndr_import
 licenses:
 - MIT
 metadata: {}
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -496,7 +505,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.9
+rubygems_version: 3.5.22
+signing_key:
 specification_version: 4
 summary: NDR Import
 test_files: []

data/lib/ndr_import/csv_library.rb

@@ -1,74 +0,0 @@
-# This file allows us to choose the CSV library we want to use.
-
-require 'csv'
-require 'active_support/deprecation'
-
-# Using relevant core CSV library.
-class CSVLibrary < CSV; end
-
-class << CSVLibrary
-  # Is the library we're using FasterCSV?
-  def fastercsv?
-    deprecate('if you desparately want fastercsv, please use it explicitly')
-    not self.const_defined?(:Reader)
-  end
-
-  # Ensure that we can pass "mode" straight through the underlying IO object
-  #
-  # Note: this could likely be refactored now, as upstream support for something
-  #       very similar was added:
-  #
-  #       https://github.com/ruby/csv/commit/b4edaf2cf1aa36f5c6264c07514b66739b87ceee
-  #
-  def foreach(path, **options, &block)
-    deprecate('CSV#foreach exists, with an optional `mode` argument')
-    return to_enum(__method__, path, **options) unless block
-    open(path, options.delete(:mode) || 'r', **options) do |csv|
-      csv.each(&block)
-    end
-  end
-
-  def write_csv_to_string(data)
-    deprecate('write_csv_to_string -> generate')
-    self.generate do |csv|
-      data.each { |line| csv << line }
-    end
-  end
-
-  def write_csv_to_file(data, filepath, mode = 'w')
-    deprecate('write_csv_to_file -> open')
-    self.open(filepath, mode) do |csv|
-      data.each { |line| csv << line }
-    end
-  end
-
-  def read_csv_from_file(filepath)
-    deprecate('read_csv_from_file -> read')
-    self.read(filepath)
-  end
-
-  private
-
-  def deprecator
-    @deprecator ||= ActiveSupport::Deprecation.new(NdrImport::VERSION, 'NdrImport')
-  end
-
-  def deprecate(additional_message = nil)
-    deprecator.warn(<<~MESSAGE)
-      CSVLibrary is deprecated, and will be removed in a future version of ndr_import.
-      Please use standard functionality provided by Ruby's CSV library (#{additional_message}).
-    MESSAGE
-  end
-end
-
-# Forward port CSV::Cell, as it is sometimes
-# serialised in YAML. :-(
-class CSV::Cell < String
-  def initialize(data = '', is_null = false)
-    super(is_null ? '' : data)
-  end
-
-  def data
-    to_s
-  end
-end