ndr_import 10.1.3 → 10.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: adf4468c5093398277008f08e3b4ff01619b0b39152dd7a604cccc863e7229f2
-  data.tar.gz: aa3225ff8384c7bc42f33251c2943060382b9c27dae6243fbe70c18796597fbf
+  metadata.gz: aaca9cf96e7433b7889c004f769997288ea1cec77322754e965b97a545ffb0ab
+  data.tar.gz: 4a1e456f8766f2ea3422b0b712b2ca4c420873248e66fc77b1db8559470a1247
 SHA512:
-  metadata.gz: d437ffcdf6733d4d62859f3450f47a7067bc7155c93404c14b883b5b2cb45dd661d555146c6317d87be640ad7033b7d214c5be6842cd32009e8462a398105249
-  data.tar.gz: f7d10bf3ba7c1e16c75cb05585ff70d88dd56b00305b389752b5d5dec329d88f313454494307025b53cb02e1164c10a4328286b8c55220cdf506f5cf7bf4e094
+  metadata.gz: '081ce6d8bce5dde04dca97a68057897a093bc90cca23f7336415330d92dcc599b96b606e9626358731c6638f0c3c1e41b7e763b2c03488cb76eeaa5c5c7a2cd8'
+  data.tar.gz: 802ba8017a16cc843196004854c3a82d161e76bdd4c2f61a5f93e34018dae9090212dd67c4e503a46c5ee78d6ed2da2d7e6f17192e62e4abefbd2382389d1cce

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 ## [Unreleased]
-* No unreleased changes
+*no unreleased changes*
+
+## 10.2.0 / 2023-05-16
+* avro file support
+* allow storage of `significant_mapped_fields` in `Table`
 
 ## 10.1.3 / 2022-12-08
 ### Added

data/README.md

@@ -1,7 +1,7 @@
 # NdrImport [![Build Status](https://github.com/NHSDigital/ndr_import/workflows/Test/badge.svg)](https://github.com/NHSDigital/ndr_import/actions?query=workflow%3Atest) [![Gem Version](https://badge.fury.io/rb/ndr_import.svg)](https://rubygems.org/gems/ndr_import) [![Documentation](https://img.shields.io/badge/ndr_import-docs-blue.svg)](https://www.rubydoc.info/gems/ndr_import)
 This is the NHS Digital (NHSD) National Disease Registers (NDR) Import ETL ruby gem, providing:
 
-1. file import handlers for *extracting* data from delimited files (csv, pipe, tab, thorn), JSON Lines, .xls(x) spreadsheets, .doc(x) word documents, PDF, PDF AcroForms, XML, 7-Zip and Zip files.
+1. file import handlers for *extracting* data from delimited files (csv, pipe, tab, thorn), JSON Lines, .xls(x) spreadsheets, .doc(x) word documents, PDF, PDF AcroForms, XML, 7-Zip, Zip and avro files.
 2. table mappers for *transforming* tabular and non-tabular data into key value pairs grouped by a common "klass".
 
 ## Installation

data/lib/ndr_import/avro/table.rb

@@ -0,0 +1,49 @@
+require 'ndr_import/table'
+
+module NdrImport
+  module Avro
+    # Syntatic sugar to ensure `header_lines` and `footer_lines` are 1 and 0 respectively.
+    # All other Table logic is inherited from `NdrImport::Table`
+    class Table < ::NdrImport::Table
+      # Scaffold an `NdrImport::Avro::Table` instance from avro schema file
+      def self.from_schema(safe_path)
+        raise SecurityError, "#{safe_path} is not a SafePath" unless safe_path.is_a? SafePath
+
+        table_columns = columns_from(::Avro::Schema.parse(::File.open(safe_path)))
+        file_name     = SafeFile.basename(safe_path).sub(/\.avsc\z/, '.avro')
+
+        new(filename_pattern: "/#{file_name}\\z/",
+            klass: 'ExampleKlass',
+            columns: table_columns)
+      end
+
+      def self.all_valid_options
+        super - %w[delimiter header_lines footer_lines]
+      end
+
+      def header_lines
+        1
+      end
+
+      def footer_lines
+        0
+      end
+
+      def self.columns_from(schema)
+        schema.fields.map do |field|
+          column = { column: field.name }
+          column[:mappings] = { field: field.name, daysafter: '1970-01-01' } if date_field?(field)
+
+          column
+        end
+      end
+
+      def self.date_field?(field)
+        field.type.schemas.any? { |schema| schema.logical_type == 'date' }
+      end
+
+      private_class_method :columns_from
+      private_class_method :date_field?
+    end
+  end
+end

data/lib/ndr_import/file/all.rb

@@ -1,5 +1,6 @@
 require_relative 'base'
 require_relative 'acro_form'
+require_relative 'avro'
 require_relative 'delimited'
 require_relative 'docx'
 require_relative 'excel'

data/lib/ndr_import/file/avro.rb

@@ -0,0 +1,34 @@
+require 'avro'
+require 'ndr_support/safe_file'
+require_relative 'registry'
+
+module NdrImport
+  # This is one of a collection of file handlers that deal with individual formats of data.
+  # They can be instantiated directly or via the factory method Registry.tables
+  module File
+    # This class is an avro file handler that returns a single table.
+    class Avro < Base
+      private
+
+      def rows(&block)
+        return enum_for(:rows) unless block
+
+        # Create an instance of DatumReader
+        reader = ::Avro::IO::DatumReader.new
+        # Open @filename in read mode
+        file   = ::File.open(@filename, 'rb')
+        # Equivalent to DataFileReader instance creation in Java
+        dr     = ::Avro::DataFile::Reader.new(file, reader)
+
+        dr.each_with_index do |avro_row, i|
+          # Ensure the first row is always the "header"
+          yield(avro_row.keys) if i.zero?
+          yield(avro_row.values.map(&:to_s))
+        end
+      rescue StandardError => e
+        raise("#{SafeFile.basename(@filename)} [#{e.class}: #{e.message}]")
+      end
+    end
+    Registry.register(Avro, 'avro')
+  end
+end

data/lib/ndr_import/table.rb

@@ -12,7 +12,7 @@ module NdrImport
     def self.all_valid_options
       %w[canonical_name delimiter liberal_parsing filename_pattern file_password last_data_column
          tablename_pattern header_lines footer_lines format klass columns xml_record_xpath slurp
-         row_identifier]
+         row_identifier significant_mapped_fields]
     end
 
     def all_valid_options

data/lib/ndr_import/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 # This stores the current version of the NdrImport gem
 module NdrImport
-  VERSION = '10.1.3'
+  VERSION = '10.2.0'
 end

data/lib/ndr_import.rb

@@ -9,6 +9,7 @@ require 'ndr_import/non_tabular/table'
 require 'ndr_import/fixed_width/table'
 require 'ndr_import/xml/table'
 require 'ndr_import/pdf_form/table'
+require 'ndr_import/avro/table'
 require 'ndr_import/unmapped_data_error'
 require 'ndr_import/acroform_reader'
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ndr_import
 version: !ruby/object:Gem::Version
-  version: 10.1.3
+  version: 10.2.0
 platform: ruby
 authors:
 - NCRS Development Team
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-12-08 00:00:00.000000000 Z
+date: 2023-05-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -92,6 +92,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: avro
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.11.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.11.0
 - !ruby/object:Gem::Dependency
   name: docx
   requirement: !ruby/object:Gem::Requirement
@@ -369,9 +383,11 @@ files:
 - exe/word_to_text
 - lib/ndr_import.rb
 - lib/ndr_import/acroform_reader.rb
+- lib/ndr_import/avro/table.rb
 - lib/ndr_import/csv_library.rb
 - lib/ndr_import/file/acro_form.rb
 - lib/ndr_import/file/all.rb
+- lib/ndr_import/file/avro.rb
 - lib/ndr_import/file/base.rb
 - lib/ndr_import/file/delimited.rb
 - lib/ndr_import/file/docx.rb
@@ -430,7 +446,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.26
+rubygems_version: 3.2.33
 signing_key:
 specification_version: 4
 summary: NDR Import