ndr_import 10.1.3 → 10.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: adf4468c5093398277008f08e3b4ff01619b0b39152dd7a604cccc863e7229f2
-  data.tar.gz: aa3225ff8384c7bc42f33251c2943060382b9c27dae6243fbe70c18796597fbf
+  metadata.gz: 877346774d65eccb73f913081d75a48345a92b9e4c11e6f87702c9e0d59ebc3d
+  data.tar.gz: e59f23137e7568ce5e6eb1e4734b9046aed8f8fa93f392a9f2d82091013c3c04
 SHA512:
-  metadata.gz: d437ffcdf6733d4d62859f3450f47a7067bc7155c93404c14b883b5b2cb45dd661d555146c6317d87be640ad7033b7d214c5be6842cd32009e8462a398105249
-  data.tar.gz: f7d10bf3ba7c1e16c75cb05585ff70d88dd56b00305b389752b5d5dec329d88f313454494307025b53cb02e1164c10a4328286b8c55220cdf506f5cf7bf4e094
+  metadata.gz: 5bd15b2baf53b654a9f2be306d45dd3a509c298427790c139431f9f77df84b14f6506d625620b6781e9d44315fb8ada40307865a059c2ae3e78945293065dc9f
+  data.tar.gz: 06070cd9f9d5311835a523f8f926c938e0c65561c863d8bc972a1250a278ccf06991013773c31f877173873b600bbee40e6ce9faa6d3da167dd93a4fc11b4965

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 ## [Unreleased]
-* No unreleased changes
+=======
+*no unreleased changes*
+
+## 10.3.0 / 2023-09-07
+### Added
+* VCF file support
+* Support Ruby 3.2. Drop support for Ruby 2.7, Rails 6.0
+
+## 10.2.0 / 2023-05-16
+* avro file support
+* allow storage of `significant_mapped_fields` in `Table`
 
 ## 10.1.3 / 2022-12-08
 ### Added

data/README.md

@@ -1,7 +1,7 @@
 # NdrImport [![Build Status](https://github.com/NHSDigital/ndr_import/workflows/Test/badge.svg)](https://github.com/NHSDigital/ndr_import/actions?query=workflow%3Atest) [![Gem Version](https://badge.fury.io/rb/ndr_import.svg)](https://rubygems.org/gems/ndr_import) [![Documentation](https://img.shields.io/badge/ndr_import-docs-blue.svg)](https://www.rubydoc.info/gems/ndr_import)
 This is the NHS Digital (NHSD) National Disease Registers (NDR) Import ETL ruby gem, providing:
 
-1. file import handlers for *extracting* data from delimited files (csv, pipe, tab, thorn), JSON Lines, .xls(x) spreadsheets, .doc(x) word documents, PDF, PDF AcroForms, XML, 7-Zip and Zip files.
+1. file import handlers for *extracting* data from delimited files (csv, pipe, tab, thorn), JSON Lines, .xls(x) spreadsheets, .doc(x) word documents, PDF, PDF AcroForms, XML, 7-Zip, Zip, avro and VCF files.
 2. table mappers for *transforming* tabular and non-tabular data into key value pairs grouped by a common "klass".
 
 ## Installation

data/lib/ndr_import/avro/table.rb

@@ -0,0 +1,49 @@
+require 'ndr_import/table'
+
+module NdrImport
+  module Avro
+    # Syntatic sugar to ensure `header_lines` and `footer_lines` are 1 and 0 respectively.
+    # All other Table logic is inherited from `NdrImport::Table`
+    class Table < ::NdrImport::Table
+      # Scaffold an `NdrImport::Avro::Table` instance from avro schema file
+      def self.from_schema(safe_path)
+        raise SecurityError, "#{safe_path} is not a SafePath" unless safe_path.is_a? SafePath
+
+        table_columns = columns_from(::Avro::Schema.parse(::File.open(safe_path)))
+        file_name     = SafeFile.basename(safe_path).sub(/\.avsc\z/, '.avro')
+
+        new(filename_pattern: "/#{file_name}\\z/",
+            klass: 'ExampleKlass',
+            columns: table_columns)
+      end
+
+      def self.all_valid_options
+        super - %w[delimiter header_lines footer_lines]
+      end
+
+      def header_lines
+        1
+      end
+
+      def footer_lines
+        0
+      end
+
+      def self.columns_from(schema)
+        schema.fields.map do |field|
+          column = { column: field.name }
+          column[:mappings] = { field: field.name, daysafter: '1970-01-01' } if date_field?(field)
+
+          column
+        end
+      end
+
+      def self.date_field?(field)
+        field.type.schemas.any? { |schema| schema.logical_type == 'date' }
+      end
+
+      private_class_method :columns_from
+      private_class_method :date_field?
+    end
+  end
+end

data/lib/ndr_import/file/all.rb

@@ -1,5 +1,6 @@
 require_relative 'base'
 require_relative 'acro_form'
+require_relative 'avro'
 require_relative 'delimited'
 require_relative 'docx'
 require_relative 'excel'
@@ -7,6 +8,7 @@ require_relative 'pdf'
 require_relative 'seven_zip'
 require_relative 'text'
 require_relative 'unregistered_filetype'
+require_relative 'vcf'
 require_relative 'word'
 require_relative 'xml'
 require_relative 'zip'

data/lib/ndr_import/file/avro.rb

@@ -0,0 +1,34 @@
+require 'avro'
+require 'ndr_support/safe_file'
+require_relative 'registry'
+
+module NdrImport
+  # This is one of a collection of file handlers that deal with individual formats of data.
+  # They can be instantiated directly or via the factory method Registry.tables
+  module File
+    # This class is an avro file handler that returns a single table.
+    class Avro < Base
+      private
+
+      def rows(&block)
+        return enum_for(:rows) unless block
+
+        # Create an instance of DatumReader
+        reader = ::Avro::IO::DatumReader.new
+        # Open @filename in read mode
+        file   = ::File.open(@filename, 'rb')
+        # Equivalent to DataFileReader instance creation in Java
+        dr     = ::Avro::DataFile::Reader.new(file, reader)
+
+        dr.each_with_index do |avro_row, i|
+          # Ensure the first row is always the "header"
+          yield(avro_row.keys) if i.zero?
+          yield(avro_row.values.map(&:to_s))
+        end
+      rescue StandardError => e
+        raise("#{SafeFile.basename(@filename)} [#{e.class}: #{e.message}]")
+      end
+    end
+    Registry.register(Avro, 'avro')
+  end
+end

data/lib/ndr_import/file/vcf.rb

@@ -0,0 +1,25 @@
+require 'bio-vcf/vcfline'
+require 'ndr_support/safe_file'
+require_relative 'registry'
+
+module NdrImport
+  # This is one of a collection of file handlers that deal with individual formats of data.
+  # They can be instantiated directly or via the factory method Registry.tables
+  module File
+    # This class is a vcf file handler that returns a single table.
+    class Vcf < Base
+      private
+
+      def rows(&block)
+        return enum_for(:rows) unless block
+
+        ::File.read(@filename).each_line do |line|
+          next if line =~ /^##/
+
+          yield BioVcf::VcfLine.parse(line)
+        end
+      end
+    end
+    Registry.register(Vcf, 'vcf')
+  end
+end

data/lib/ndr_import/table.rb

@@ -12,7 +12,7 @@ module NdrImport
     def self.all_valid_options
       %w[canonical_name delimiter liberal_parsing filename_pattern file_password last_data_column
          tablename_pattern header_lines footer_lines format klass columns xml_record_xpath slurp
-         row_identifier]
+         row_identifier significant_mapped_fields]
     end
 
     def all_valid_options

data/lib/ndr_import/vcf/table.rb

@@ -0,0 +1,21 @@
+require 'ndr_import/table'
+
+module NdrImport
+  module Vcf
+    # Syntatic sugar to ensure `header_lines` and `footer_lines` are 1 and 0 respectively.
+    # All other Table logic is inherited from `NdrImport::Table`
+    class Table < ::NdrImport::Table
+      def self.all_valid_options
+        super - %w[delimiter header_lines footer_lines xml_record_xpath]
+      end
+
+      def header_lines
+        1
+      end
+
+      def footer_lines
+        0
+      end
+    end
+  end
+end

data/lib/ndr_import/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 # This stores the current version of the NdrImport gem
 module NdrImport
-  VERSION = '10.1.3'
+  VERSION = '10.3.0'
 end

data/lib/ndr_import.rb

@@ -9,6 +9,8 @@ require 'ndr_import/non_tabular/table'
 require 'ndr_import/fixed_width/table'
 require 'ndr_import/xml/table'
 require 'ndr_import/pdf_form/table'
+require 'ndr_import/avro/table'
+require 'ndr_import/vcf/table'
 require 'ndr_import/unmapped_data_error'
 require 'ndr_import/acroform_reader'
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ndr_import
 version: !ruby/object:Gem::Version
-  version: 10.1.3
+  version: 10.3.0
 platform: ruby
 authors:
 - NCRS Development Team
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-12-08 00:00:00.000000000 Z
+date: 2023-09-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -30,7 +30,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '6.0'
+        version: '6.1'
     - - "<"
       - !ruby/object:Gem::Version
         version: '7.1'
@@ -40,7 +40,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '6.0'
+        version: '6.1'
     - - "<"
       - !ruby/object:Gem::Version
         version: '7.1'
@@ -92,6 +92,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: avro
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.11.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.11.0
+- !ruby/object:Gem::Dependency
+  name: bio-vcf
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.5
 - !ruby/object:Gem::Dependency
   name: docx
   requirement: !ruby/object:Gem::Requirement
@@ -369,9 +397,11 @@ files:
 - exe/word_to_text
 - lib/ndr_import.rb
 - lib/ndr_import/acroform_reader.rb
+- lib/ndr_import/avro/table.rb
 - lib/ndr_import/csv_library.rb
 - lib/ndr_import/file/acro_form.rb
 - lib/ndr_import/file/all.rb
+- lib/ndr_import/file/avro.rb
 - lib/ndr_import/file/base.rb
 - lib/ndr_import/file/delimited.rb
 - lib/ndr_import/file/docx.rb
@@ -383,6 +413,7 @@ files:
 - lib/ndr_import/file/seven_zip.rb
 - lib/ndr_import/file/text.rb
 - lib/ndr_import/file/unregistered_filetype.rb
+- lib/ndr_import/file/vcf.rb
 - lib/ndr_import/file/word.rb
 - lib/ndr_import/file/xml.rb
 - lib/ndr_import/file/zip.rb
@@ -408,6 +439,7 @@ files:
 - lib/ndr_import/table.rb
 - lib/ndr_import/universal_importer_helper.rb
 - lib/ndr_import/unmapped_data_error.rb
+- lib/ndr_import/vcf/table.rb
 - lib/ndr_import/version.rb
 - lib/ndr_import/xml/control_char_escaper.rb
 - lib/ndr_import/xml/table.rb
@@ -423,14 +455,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.7'
+      version: '3.0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.26
+rubygems_version: 3.4.10
 signing_key:
 specification_version: 4
 summary: NDR Import