ndr_import 11.2.1 → 11.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 77f2f2adb4de01a7dca5f2e004976a0f96e8d486f4bf573e24543c1a27b894dd
-  data.tar.gz: eb83c8b95408ee9761b513c9e9309a6b565fd937e7cfba585088bd3d7e7b2573
+  metadata.gz: 3dd3ec74b568c0429492ded5083edf0eaf31a0a6c7849945f7fa69dbe0e43a49
+  data.tar.gz: 0f845a90d21901350122a6d41437c8ebe53e1eaa49694af76b38811210280804
 SHA512:
-  metadata.gz: 6dcfa45678b041cbe5c298b43c935aa2dacc47da9421cfbc6d6f8e646049f89326fb7a7a41e6e5ea283a1114ce367e7cff92d77dd4b83db56775e1d69a93fc6e
-  data.tar.gz: 2443a1c7b4ab1f1c8fcc105cf7c3d29b1885f996e41ad55b65f0ea74d64fbe43042d50a2bb3943a5563374aa8bdc0dd3d47e066226f010922f1fd7e69c167641
+  metadata.gz: 26668128f4fa8a7165d50ee39e729f5d5a63b704c3db41a5cab1be8df02ea38bdeb7cd69494ef62656c22c7b8bd4aea2c94ee839019accff59a28f2b1452065c
+  data.tar.gz: 6b09bbb5ae408a1ca9d79e74e6c1de3f2b7a1d3597e67a02207e20dc83e18a24bdd8e6f2117f56d781cff855dd14495a686ecfb4589dc5af4388ffc13a8f3d03

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 ## [Unreleased]
-* no unreleased changes *
+* no unreleased changes
+
+## 11.3.0/ 2025-02-11
+### Fixed
+* Fix CSV parsing bug
+
+### Added
+* Column zipping functionality *
+* Capturing Column name *
+* Regular expression column names *
+* VCF file/table metadata storage *
 
 ## 11.2.1 / 2024-11-18
 ### Fixed

data/lib/ndr_import/file/vcf.rb

@@ -8,13 +8,38 @@ module NdrImport
   module File
     # This class is a vcf file handler that returns a single table.
     class Vcf < Base
+      attr_accessor :vcf_file_metadata
+
+      def initialize(*)
+        super
+
+        @vcf_file_metadata = @options['vcf_file_metadata']
+        assign_file_metadata
+      end
+
       private
 
+      def assign_file_metadata
+        return unless vcf_file_metadata.is_a?(Hash)
+
+        file_metadata_hash = {}
+
+        ::File.read(@filename).each_line do |line|
+          next unless line.match?(/^##/)
+
+          vcf_file_metadata.each do |attribute, pattern|
+            file_metadata_hash[attribute] = line.match(pattern)[1].presence if line.match? pattern
+          end
+        end
+
+        self.file_metadata = file_metadata_hash
+      end
+
       def rows(&block)
         return enum_for(:rows) unless block
 
         ::File.read(@filename).each_line do |line|
-          next if line =~ /^##/
+          next if line.match?(/^##/)
 
           yield BioVcf::VcfLine.parse(line)
         end

data/lib/ndr_import/helpers/file/delimited.rb

@@ -28,7 +28,7 @@ module NdrImport
         end
 
         # Iterate through the file line by line, yielding each one in turn.
-        def delimited_rows(path, col_sep = nil, liberal = false)
+        def delimited_rows(path, col_sep = nil, liberal = false) # rubocop:disable Style/OptionalBooleanParameter
           return enum_for(:delimited_rows, path, col_sep, liberal) unless block_given?
 
           safe_path = SafeFile.safepath_to_string(path)
@@ -36,7 +36,7 @@ module NdrImport
 
           # By now, we know `options` should let us read the whole
           # file succesfully; if there are problems, we should crash.
-          CSV.foreach(safe_path, options.delete(:mode), **options) do |line|
+          CSV.foreach(safe_path, options[:mode], **options.except(:mode)) do |line|
             yield line.map(&:to_s)
           end
         end

data/lib/ndr_import/mapper.rb

@@ -30,6 +30,9 @@ module NdrImport::Mapper
     STANDARD_MAPPING = 'standard_mapping'.freeze
     UNPACK_PATTERN   = 'unpack_pattern'.freeze
     VALIDATES        = 'validates'.freeze
+    ZIP_ORDER        = 'zip_order'.freeze
+    SPLIT_CHAR       = 'split_char'.freeze
+    MAP_COLUMNAME_TO = 'map_columname_to'.freeze
   end
 
   private
@@ -118,10 +121,18 @@ module NdrImport::Mapper
       # Store the raw column value
       rawtext[rawtext_column_name] = raw_value
 
+      # If configured, store the column name in the given field
+      if column_mapping[Strings::MAP_COLUMNAME_TO].present?
+        data[column_mapping[Strings::MAP_COLUMNAME_TO]] ||= {}
+        data[column_mapping[Strings::MAP_COLUMNAME_TO]][:values] = [column_mapping['column']]
+        rawtext[column_mapping[Strings::MAP_COLUMNAME_TO]] = column_mapping['column']
+      end
+
       next unless column_mapping.key?(Strings::MAPPINGS)
+
       column_mapping[Strings::MAPPINGS].each do |field_mapping|
         # create a duplicate of the raw value we can manipulate
-        original_value = raw_value ? raw_value.dup : nil
+        original_value = raw_value&.dup
 
         replace_before_mapping(original_value, field_mapping)
         value = mapped_value(original_value, field_mapping)
@@ -137,7 +148,8 @@ module NdrImport::Mapper
 
         data[field] ||= {}
         data[field][:values] ||= [] # "better" values come earlier
-        data[field][:compact]  = true unless data[field].key?(:compact)
+        data[field][:zipped_values] ||= []
+        data[field][:compact] = true unless data[field].key?(:compact)
 
         if field_mapping[Strings::ORDER]
           data[field][:join] ||= field_mapping[Strings::JOIN]
@@ -148,6 +160,9 @@ module NdrImport::Mapper
           data[field][:values][field_mapping[Strings::ORDER] - 1] = value
         elsif field_mapping[Strings::PRIORITY]
           data[field][:values][field_mapping[Strings::PRIORITY]] = value
+        elsif field_zippable?(field_mapping, data[field])
+          data[field][:split_char] ||= field_mapping[Strings::SPLIT_CHAR]
+          data[field][:zipped_values][field_mapping[Strings::ZIP_ORDER] - 1] = value
         else
           data[field][:values].unshift(value) # new "best" value
         end
@@ -160,6 +175,7 @@ module NdrImport::Mapper
     # and one to many, for cross-populating
     data.each do |field, field_data|
       values = field_data[:values]
+      zipped_values = field_data[:zipped_values]
 
       attributes[field] =
         if field_data.key?(:join)
@@ -167,6 +183,9 @@ module NdrImport::Mapper
           values = values.map(&:presence)
           values.compact! if field_data[:compact]
           values.join(field_data[:join])
+        elsif zipped_values.present?
+          values = zipped_values.map { |value| value.split(field_data[:split_char]) }
+          values.first.zip(*values[1..])
         else
           values.detect(&:present?)
         end
@@ -176,6 +195,12 @@ module NdrImport::Mapper
     attributes
   end
 
+  def field_zippable?(field_mapping, data_field)
+    return false if field_mapping[Strings::ZIP_ORDER].blank?
+
+    data_field[:split_char].present? || field_mapping[Strings::SPLIT_CHAR].present?
+  end
+
   def mapped_value(original_value, field_mapping)
     if field_mapping.include?(Strings::FORMAT)
       begin

data/lib/ndr_import/table.rb

@@ -66,6 +66,7 @@ module NdrImport
       return enum_for(:process_line, line) unless block
 
       if @row_index < header_lines
+        mutate_regexp_columns(line)
         consume_header_line(line, @columns)
       else
         transform_line(line, @row_index, &block)
@@ -79,6 +80,15 @@ module NdrImport
       @notifier.try(:processed, @row_index)
     end
 
+    # Update 'column' values expressed as a regular expression
+    def mutate_regexp_columns(line)
+      @columns.each_with_index do |column, index|
+        next unless column['column'].is_a? Regexp
+
+        column['column'] = line[index] if line[index].match? column['column']
+      end
+    end
+
     # This method transforms an incoming line of data by applying each of the klass masked
     # mappings to the line and yielding the klass and fields for each mapped klass.
     def transform_line(line, index)
@@ -227,7 +237,7 @@ module NdrImport
 
     # returns the column names as we expect to receive them
     def column_names(column_mappings)
-      column_mappings.map { |c| (c['column'] || c['standard_mapping']).downcase }
+      column_mappings.map { |c| (c['column'] || c['standard_mapping']).try(:downcase) }
     end
 
     # If specified in the mapping, stop transforming data at a given index (column)

data/lib/ndr_import/universal_importer_helper.rb

@@ -51,22 +51,25 @@ module NdrImport
       NdrImport::File::Registry.files(source_file, 'unzip_path' => unzip_path).each do |filename|
         # now at the individual file level, can we find the table mapping?
         table_mapping = get_table_mapping(filename, nil)
-
-        options = { 'unzip_path'                 => unzip_path,
-                    'col_sep'                    => table_mapping.try(:delimiter),
-                    'file_password'              => table_mapping.try(:file_password),
-                    'liberal_parsing'            => table_mapping.try(:liberal_parsing),
-                    'xml_record_xpath'           => table_mapping.try(:xml_record_xpath),
-                    'slurp'                      => table_mapping.try(:slurp),
-                    'yield_xml_record'           => table_mapping.try(:yield_xml_record),
-                    'pattern_match_record_xpath' => table_mapping.try(:pattern_match_record_xpath),
-                    'xml_file_metadata'          => table_mapping.try(:xml_file_metadata) }
+        options       = table_options_from(table_mapping).merge { 'unzip_path' => unzip_path }
 
         tables = NdrImport::File::Registry.tables(filename, table_mapping.try(:format), options)
         yield_tables_and_their_content(filename, tables, &block)
       end
     end
 
+    def table_options_from(table_mapping)
+      { 'col_sep'                    => table_mapping.try(:delimiter),
+        'file_password'              => table_mapping.try(:file_password),
+        'liberal_parsing'            => table_mapping.try(:liberal_parsing),
+        'xml_record_xpath'           => table_mapping.try(:xml_record_xpath),
+        'slurp'                      => table_mapping.try(:slurp),
+        'yield_xml_record'           => table_mapping.try(:yield_xml_record),
+        'pattern_match_record_xpath' => table_mapping.try(:pattern_match_record_xpath),
+        'xml_file_metadata'          => table_mapping.try(:xml_file_metadata),
+        'vcf_file_metadata'          => table_mapping.try(:vcf_file_metadata) }
+    end
+
     # This method does the table row yielding for the extract method, setting the notifier
     # so that we can monitor progress
     def yield_tables_and_their_content(filename, tables, &block)

data/lib/ndr_import/vcf/table.rb

@@ -6,7 +6,7 @@ module NdrImport
     # All other Table logic is inherited from `NdrImport::Table`
     class Table < ::NdrImport::Table
       def self.all_valid_options
-        super - %w[delimiter header_lines footer_lines]
+        super - %w[delimiter header_lines footer_lines] + %w[vcf_file_metadata]
       end
 
       def header_lines

data/lib/ndr_import/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 # This stores the current version of the NdrImport gem
 module NdrImport
-  VERSION = '11.2.1'
+  VERSION = '11.3.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ndr_import
 version: !ruby/object:Gem::Version
-  version: 11.2.1
+  version: 11.3.0
 platform: ruby
 authors:
 - NCRS Development Team
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-11-18 00:00:00.000000000 Z
+date: 2025-02-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -479,7 +479,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.27
+rubygems_version: 3.2.3
 signing_key:
 specification_version: 4
 summary: NDR Import