ndr_import 11.2.1 → 11.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 77f2f2adb4de01a7dca5f2e004976a0f96e8d486f4bf573e24543c1a27b894dd
-  data.tar.gz: eb83c8b95408ee9761b513c9e9309a6b565fd937e7cfba585088bd3d7e7b2573
+  metadata.gz: 81d7c1189af2322610f0e094c3b469dc187a2738d4e97edfd0a03a2f5bec079a
+  data.tar.gz: 5a2553b6bbe30e6b74142395abde887d02298a268e893adb49674effb6047e31
 SHA512:
-  metadata.gz: 6dcfa45678b041cbe5c298b43c935aa2dacc47da9421cfbc6d6f8e646049f89326fb7a7a41e6e5ea283a1114ce367e7cff92d77dd4b83db56775e1d69a93fc6e
-  data.tar.gz: 2443a1c7b4ab1f1c8fcc105cf7c3d29b1885f996e41ad55b65f0ea74d64fbe43042d50a2bb3943a5563374aa8bdc0dd3d47e066226f010922f1fd7e69c167641
+  metadata.gz: c7ce3014831428ab2e0c151af76aa45b0b6f04f6c61232d170a2c88952b3a20c4e8a7e48631cf0f6776da137371d709dc3de8cbe25781083861f218676c80def
+  data.tar.gz: 76839142c01dcab91c21e69d6f96b36de761f89e71621ad4f6a86c9e11becf029b358e30aabbc628590763f6b66f018189c587fe1f9b139858bdaf4df7d1d6f1

data/CHANGELOG.md

@@ -1,6 +1,20 @@
 ## [Unreleased]
 * no unreleased changes *
 
+## 11.3.1/ 2025-09-02
+### Enhancement
+* Performance improvements for XML mask mappings
+
+## 11.3.0/ 2025-02-11
+### Fixed
+* Fix CSV parsing bug
+
+### Added
+* Column zipping functionality *
+* Capturing Column name *
+* Regular expression column names *
+* VCF file/table metadata storage *
+
 ## 11.2.1 / 2024-11-18
 ### Fixed
 * Support Ruby 3.2 and 3.3, Rails 7.1, 7.2 and 8.0

data/lib/ndr_import/file/vcf.rb

@@ -8,13 +8,38 @@ module NdrImport
   module File
     # This class is a vcf file handler that returns a single table.
     class Vcf < Base
+      attr_accessor :vcf_file_metadata
+
+      def initialize(*)
+        super
+
+        @vcf_file_metadata = @options['vcf_file_metadata']
+        assign_file_metadata
+      end
+
       private
 
+      def assign_file_metadata
+        return unless vcf_file_metadata.is_a?(Hash)
+
+        file_metadata_hash = {}
+
+        ::File.read(@filename).each_line do |line|
+          next unless line.match?(/^##/)
+
+          vcf_file_metadata.each do |attribute, pattern|
+            file_metadata_hash[attribute] = line.match(pattern)[1].presence if line.match? pattern
+          end
+        end
+
+        self.file_metadata = file_metadata_hash
+      end
+
       def rows(&block)
         return enum_for(:rows) unless block
 
         ::File.read(@filename).each_line do |line|
-          next if line =~ /^##/
+          next if line.match?(/^##/)
 
           yield BioVcf::VcfLine.parse(line)
         end

data/lib/ndr_import/helpers/file/delimited.rb

@@ -28,7 +28,7 @@ module NdrImport
         end
 
         # Iterate through the file line by line, yielding each one in turn.
-        def delimited_rows(path, col_sep = nil, liberal = false)
+        def delimited_rows(path, col_sep = nil, liberal = false) # rubocop:disable Style/OptionalBooleanParameter
           return enum_for(:delimited_rows, path, col_sep, liberal) unless block_given?
 
           safe_path = SafeFile.safepath_to_string(path)
@@ -36,7 +36,7 @@ module NdrImport
 
           # By now, we know `options` should let us read the whole
           # file succesfully; if there are problems, we should crash.
-          CSV.foreach(safe_path, options.delete(:mode), **options) do |line|
+          CSV.foreach(safe_path, options[:mode], **options.except(:mode)) do |line|
             yield line.map(&:to_s)
           end
         end

data/lib/ndr_import/mapper.rb

@@ -30,6 +30,9 @@ module NdrImport::Mapper
     STANDARD_MAPPING = 'standard_mapping'.freeze
     UNPACK_PATTERN   = 'unpack_pattern'.freeze
     VALIDATES        = 'validates'.freeze
+    ZIP_ORDER        = 'zip_order'.freeze
+    SPLIT_CHAR       = 'split_char'.freeze
+    MAP_COLUMNAME_TO = 'map_columname_to'.freeze
   end
 
   private
@@ -118,10 +121,18 @@ module NdrImport::Mapper
       # Store the raw column value
       rawtext[rawtext_column_name] = raw_value
 
+      # If configured, store the column name in the given field
+      if column_mapping[Strings::MAP_COLUMNAME_TO].present?
+        data[column_mapping[Strings::MAP_COLUMNAME_TO]] ||= {}
+        data[column_mapping[Strings::MAP_COLUMNAME_TO]][:values] = [column_mapping['column']]
+        rawtext[column_mapping[Strings::MAP_COLUMNAME_TO]] = column_mapping['column']
+      end
+
       next unless column_mapping.key?(Strings::MAPPINGS)
+
       column_mapping[Strings::MAPPINGS].each do |field_mapping|
         # create a duplicate of the raw value we can manipulate
-        original_value = raw_value ? raw_value.dup : nil
+        original_value = raw_value&.dup
 
         replace_before_mapping(original_value, field_mapping)
         value = mapped_value(original_value, field_mapping)
@@ -137,7 +148,8 @@ module NdrImport::Mapper
 
         data[field] ||= {}
         data[field][:values] ||= [] # "better" values come earlier
-        data[field][:compact]  = true unless data[field].key?(:compact)
+        data[field][:zipped_values] ||= []
+        data[field][:compact] = true unless data[field].key?(:compact)
 
         if field_mapping[Strings::ORDER]
           data[field][:join] ||= field_mapping[Strings::JOIN]
@@ -148,6 +160,9 @@ module NdrImport::Mapper
           data[field][:values][field_mapping[Strings::ORDER] - 1] = value
         elsif field_mapping[Strings::PRIORITY]
           data[field][:values][field_mapping[Strings::PRIORITY]] = value
+        elsif field_zippable?(field_mapping, data[field])
+          data[field][:split_char] ||= field_mapping[Strings::SPLIT_CHAR]
+          data[field][:zipped_values][field_mapping[Strings::ZIP_ORDER] - 1] = value
         else
           data[field][:values].unshift(value) # new "best" value
         end
@@ -160,6 +175,7 @@ module NdrImport::Mapper
     # and one to many, for cross-populating
     data.each do |field, field_data|
       values = field_data[:values]
+      zipped_values = field_data[:zipped_values]
 
       attributes[field] =
         if field_data.key?(:join)
@@ -167,6 +183,9 @@ module NdrImport::Mapper
           values = values.map(&:presence)
           values.compact! if field_data[:compact]
           values.join(field_data[:join])
+        elsif zipped_values.present?
+          values = zipped_values.map { |value| value.split(field_data[:split_char]) }
+          values.first.zip(*values[1..])
         else
           values.detect(&:present?)
         end
@@ -176,6 +195,12 @@ module NdrImport::Mapper
     attributes
   end
 
+  def field_zippable?(field_mapping, data_field)
+    return false if field_mapping[Strings::ZIP_ORDER].blank?
+
+    data_field[:split_char].present? || field_mapping[Strings::SPLIT_CHAR].present?
+  end
+
   def mapped_value(original_value, field_mapping)
     if field_mapping.include?(Strings::FORMAT)
       begin

data/lib/ndr_import/table.rb

@@ -66,6 +66,7 @@ module NdrImport
       return enum_for(:process_line, line) unless block
 
       if @row_index < header_lines
+        mutate_regexp_columns(line)
         consume_header_line(line, @columns)
       else
         transform_line(line, @row_index, &block)
@@ -79,6 +80,15 @@ module NdrImport
       @notifier.try(:processed, @row_index)
     end
 
+    # Update 'column' values expressed as a regular expression
+    def mutate_regexp_columns(line)
+      @columns.each_with_index do |column, index|
+        next unless column['column'].is_a? Regexp
+
+        column['column'] = line[index] if line[index].match? column['column']
+      end
+    end
+
     # This method transforms an incoming line of data by applying each of the klass masked
     # mappings to the line and yielding the klass and fields for each mapped klass.
     def transform_line(line, index)
@@ -227,7 +237,7 @@ module NdrImport
 
     # returns the column names as we expect to receive them
     def column_names(column_mappings)
-      column_mappings.map { |c| (c['column'] || c['standard_mapping']).downcase }
+      column_mappings.map { |c| (c['column'] || c['standard_mapping']).try(:downcase) }
     end
 
     # If specified in the mapping, stop transforming data at a given index (column)

data/lib/ndr_import/universal_importer_helper.rb

@@ -51,22 +51,25 @@ module NdrImport
       NdrImport::File::Registry.files(source_file, 'unzip_path' => unzip_path).each do |filename|
         # now at the individual file level, can we find the table mapping?
         table_mapping = get_table_mapping(filename, nil)
-
-        options = { 'unzip_path'                 => unzip_path,
-                    'col_sep'                    => table_mapping.try(:delimiter),
-                    'file_password'              => table_mapping.try(:file_password),
-                    'liberal_parsing'            => table_mapping.try(:liberal_parsing),
-                    'xml_record_xpath'           => table_mapping.try(:xml_record_xpath),
-                    'slurp'                      => table_mapping.try(:slurp),
-                    'yield_xml_record'           => table_mapping.try(:yield_xml_record),
-                    'pattern_match_record_xpath' => table_mapping.try(:pattern_match_record_xpath),
-                    'xml_file_metadata'          => table_mapping.try(:xml_file_metadata) }
+        options       = table_options_from(table_mapping).merge { 'unzip_path' => unzip_path }
 
         tables = NdrImport::File::Registry.tables(filename, table_mapping.try(:format), options)
         yield_tables_and_their_content(filename, tables, &block)
       end
     end
 
+    def table_options_from(table_mapping)
+      { 'col_sep'                    => table_mapping.try(:delimiter),
+        'file_password'              => table_mapping.try(:file_password),
+        'liberal_parsing'            => table_mapping.try(:liberal_parsing),
+        'xml_record_xpath'           => table_mapping.try(:xml_record_xpath),
+        'slurp'                      => table_mapping.try(:slurp),
+        'yield_xml_record'           => table_mapping.try(:yield_xml_record),
+        'pattern_match_record_xpath' => table_mapping.try(:pattern_match_record_xpath),
+        'xml_file_metadata'          => table_mapping.try(:xml_file_metadata),
+        'vcf_file_metadata'          => table_mapping.try(:vcf_file_metadata) }
+    end
+
     # This method does the table row yielding for the extract method, setting the notifier
     # so that we can monitor progress
     def yield_tables_and_their_content(filename, tables, &block)

data/lib/ndr_import/vcf/table.rb

@@ -6,7 +6,7 @@ module NdrImport
     # All other Table logic is inherited from `NdrImport::Table`
     class Table < ::NdrImport::Table
       def self.all_valid_options
-        super - %w[delimiter header_lines footer_lines]
+        super - %w[delimiter header_lines footer_lines] + %w[vcf_file_metadata]
       end
 
       def header_lines

data/lib/ndr_import/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 # This stores the current version of the NdrImport gem
 module NdrImport
-  VERSION = '11.2.1'
+  VERSION = '11.3.1'
 end

data/lib/ndr_import/xml/masked_mappings.rb

@@ -4,70 +4,156 @@ module NdrImport
     # Overriding the NdrImport::Table method to avoid memoizing. This by design, column mappings
     # can change if new mappings are added on the fly where repeating sections are present
     class MaskedMappings
-      attr_accessor :klass, :augmented_columns
+      DO_NOT_CAPTURE_MAPPING = { 'do_not_capture' => true }.freeze
+      KLASS_KEY              = 'klass'.freeze
+      COLUMN_KEY             = 'column'.freeze
+      STANDARD_MAPPING_KEY   = 'standard_mapping'.freeze
+      DO_NOT_CAPTURE_KEY     = 'do_not_capture'.freeze
+      XML_CELL_KEY           = 'xml_cell'.freeze
+      KEEP_KLASS_KEY         = 'keep_klass'.freeze
+
+      # Pre-compiled regex for numbered variants
+      NUMBERED_VARIANT_PATTERN = /#\d+\z/
+
+      attr_reader :klass, :augmented_columns
 
       def initialize(klass, augmented_columns)
         @klass             = klass
         @augmented_columns = augmented_columns
+        @column_count      = augmented_columns.size
+        @has_klass         = !klass.nil?
+
+        freeze
       end
 
       def call
-        return { klass => augmented_columns } if klass.present?
-
-        masked_mappings = column_level_klass_masked_mappings
-
-        augmented_masked_mappings = masked_mappings
-        # Remove any masked klasses where additional columns mappings
-        # have been added for repeated sections
-        # e.g. SomeTestKlass column mappings are not needed if SomeTestKlass#1
-        # have been added
-        masked_mappings.each do |masked_key, columns|
-          # There may be occasions where the e.g. SomeTestKlass should be kept,
-          # This can be flagged in the one the klass's column mappings
-          next if columns.any? { |column| column.dig('xml_cell', 'keep_klass') }
-
-          if masked_mappings.keys.any? { |key| key =~ /\A#{masked_key}#\d+\z/ }
-            augmented_masked_mappings.delete(masked_key)
+        return { @klass => @augmented_columns } if @has_klass
+
+        masked_mappings = build_masked_mappings
+        remove_superseded_base_klasses(masked_mappings)
+      end
+
+      private
+
+      def build_masked_mappings
+        # Pre-validate and extract all klasses in one pass
+        all_klasses_set, klassless_column_names = extract_klasses_and_validate
+
+        raise "Missing klass for column(s): #{klassless_column_names.join(', ')}" unless klassless_column_names.empty?
+
+        all_klasses_array = all_klasses_set.to_a
+
+        # Pre-allocate result hash with exact size
+        result = Hash.new(all_klasses_array.size)
+
+        all_klasses_array.each do |current_klass|
+          result[current_klass] = mask_mappings_for_klass(current_klass)
+        end
+
+        result
+      end
+
+      def extract_klasses_and_validate
+        klasses_set            = Set.new
+        klassless_column_names = []
+
+        @augmented_columns.each do |mapping|
+          mapping_klass = mapping[KLASS_KEY]
+
+          if mapping_klass.nil?
+            # Only collect klassless mappings that aren't marked as do_not_capture
+            klassless_column_names << column_name_from(mapping) unless mapping[DO_NOT_CAPTURE_KEY]
+          elsif mapping_klass.is_a?(Array)
+            klasses_set.merge(mapping_klass.compact)
+          else
+            klasses_set.add(mapping_klass)
           end
         end
 
-        augmented_masked_mappings
+        [klasses_set, klassless_column_names]
       end
 
-      private
+      def column_name_from(mapping)
+        mapping[COLUMN_KEY] || mapping[STANDARD_MAPPING_KEY]
+      end
 
-      # This method duplicates the mappings and applies a do_not_capture mask to those that do not
-      # relate to this klass, returning the masked mappings
-      def mask_mappings_by_klass(klass)
-        augmented_columns.deep_dup.map do |mapping|
-          Array(mapping['klass']).flatten.include?(klass) ? mapping : { 'do_not_capture' => true }
+      def mask_mappings_for_klass(target_klass)
+        # Pre-allocate array with exact size
+        result = Array.new(@column_count)
+
+        # Single pass with index tracking
+        @augmented_columns.each_with_index do |mapping, index|
+          result[index] = if mapping_applies_to_klass?(mapping, target_klass)
+                            mapping.deep_dup
+                          else
+                            DO_NOT_CAPTURE_MAPPING
+                          end
+        end
+
+        result
+      end
+
+      def mapping_applies_to_klass?(mapping, target_klass)
+        mapping_klass = mapping[KLASS_KEY]
+        return false unless mapping_klass
+
+        # Optimized type checking and inclusion
+        case mapping_klass
+        when Array
+          mapping_klass.include?(target_klass)
+        when String
+          mapping_klass == target_klass
+        else
+          false
         end
       end
 
-      def column_level_klass_masked_mappings
-        ensure_mappings_define_klass
+      def remove_superseded_base_klasses(masked_mappings)
+        return masked_mappings if masked_mappings.size <= 1
+
+        # Pre-build numbered variants lookup for O(1) access
+        numbered_klasses = build_numbered_klasses_lookup(masked_mappings.keys)
+        return masked_mappings if numbered_klasses.empty?
+
+        klasses_to_keep = compute_klasses_to_keep(masked_mappings)
 
-        # Loop through each klass
-        masked_mappings = {}
-        augmented_columns.pluck('klass').flatten.compact.uniq.each do |klass|
-          # Do not capture fields that relate to other klasses
-          masked_mappings[klass] = mask_mappings_by_klass(klass)
+        masked_mappings.select do |klass, _columns|
+          klasses_to_keep.include?(klass) || numbered_klasses.exclude?(klass)
         end
-        masked_mappings
       end
 
-      # This method ensures that every column mapping defines a klass (unless it is a column that
-      # we do not capture). It is only used where a table level klass is not defined.
-      def ensure_mappings_define_klass
-        klassless_mappings = augmented_columns.
-                             select { |mapping| mapping.nil? || mapping['klass'].nil? }.
-                             reject { |mapping| mapping['do_not_capture'] }.
-                             map { |mapping| mapping['column'] || mapping['standard_mapping'] }
+      def build_numbered_klasses_lookup(klass_keys)
+        numbered_klasses = Set.new
 
-        return if klassless_mappings.empty?
+        klass_keys.each do |key|
+          next unless key.match?(NUMBERED_VARIANT_PATTERN)
+
+          # Extract base klass name (everything before #)
+          base_klass = key.split(NUMBERED_VARIANT_PATTERN, 2).first
+          numbered_klasses.add(base_klass)
+        end
+
+        numbered_klasses
+      end
+
+      def compute_klasses_to_keep(masked_mappings)
+        klasses_to_keep = Set.new
+
+        masked_mappings.each do |klass, columns|
+          klasses_to_keep.add(klass) if should_keep_base_klass?(columns)
+        end
+
+        klasses_to_keep
+      end
+
+      def should_keep_base_klass?(columns)
+        # Fast iteration with early termination
+        columns.each do |column|
+          xml_cell = column[XML_CELL_KEY]
+          return true if xml_cell && xml_cell[KEEP_KLASS_KEY]
+        end
 
-        # All column mappings for the single item file require a klass definition.
-        raise "Missing klass for column(s): #{klassless_mappings.to_sentence}"
+        false
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ndr_import
 version: !ruby/object:Gem::Version
-  version: 11.2.1
+  version: 11.3.1
 platform: ruby
 authors:
 - NCRS Development Team
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-11-18 00:00:00.000000000 Z
+date: 2025-09-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -479,7 +479,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.27
+rubygems_version: 3.4.19
 signing_key:
 specification_version: 4
 summary: NDR Import