ds-convert 0.1.1

data/lib/ds/util/csv_validator.rb

@@ -0,0 +1,209 @@
+# frozen_string_literal: true
+
+module DS
+  module Util
+    class CsvValidator
+
+      ERROR_UNBALANCED_SUBFIELDS     = 'Row has subfields of different lengths'
+      ERROR_BLANK_SUBFIELDS          = 'Row has blank subfields'
+      ERROR_MISSING_REQUIRED_COLUMNS = "CSV is missing required column(s)"
+      ERROR_TRAILING_WHITESPACE      = 'Row contains trailing whitespace'
+
+      # split on pipes that are not escaped with '\'
+      PIPE_SPLIT_REGEXP     = %r{(?<!\\)\|}
+      # split on pipes and semicolons that are not escaped with '\'
+      PIPE_SEMICOLON_REGEXP = %r{(?<!\\)[;|]}
+
+
+      # Validates all rows of data against a set of required columns, balanced columns, and nested columns.
+      #
+      # @param rows [Array<Hash,CSV::Row>] The rows of data to be validated.
+      # @param required_columns [Array<Symbol>] The required columns for each row.
+      # @param balanced_columns [Hash<Symbol, Array<Symbol>>] A hash of groups of balanced columns.
+      # @param nested_columns [Hash<Symbol, Array<Symbol>>] A hash of nested columns.
+      # @param allow_blank [Boolean] Whether to allow blank subfields in balanced columns.
+      # @return [Array<String>] An array of error messages, if any.
+      def self.validate_all_rows rows, required_columns: [], balanced_columns: {}, nested_columns: {}, allow_blank: false
+        errors = validate_required_columns(rows.first, row_num: 1, required_columns: required_columns)
+        return errors unless errors.blank?
+        rows.each_with_index do |row, row_num|
+          errors += validate_row(
+            row, row_num: row_num + 1,
+            required_columns: required_columns,
+            balanced_columns: balanced_columns,
+            nested_columns: nested_columns,
+            allow_blank: allow_blank
+          )
+        end
+        errors
+      end
+
+      # Validates a row of data against a set of required columns and balanced columns.
+      #
+      #   # validate a CSV row for required columns and balanced columns
+      #   # columns a and b are required,
+      #   # columns a and b, and c and d are balanced
+      #   # balanced_columns keys are used as labels for the error messages
+      #   required_columns = [:a, :b]
+      #   balanced_columns = { group1: [:a, :b], group2: [:c: :d] }
+      #   csv_validator.validate(row, required_columns: required_columns, balanced_columns: balanced_columns)
+      #
+      # @param row [Hash,CSV::Row] The row of data to be validated.
+      # @param required_columns [Array<Symbol>] The required columns for the row.
+      # @param balanced_columns [Hash<Symbol, Array<Symbol>>] a hash of groups of balanced columns; see example above
+      # @param allow_blank [Boolean] Whether to allow blank subfields in balanced columns
+      # @return [Array<String>] An array of error messages, if any.
+      def self.validate_row row, row_num:, required_columns: [], balanced_columns: {}, nested_columns: {}, allow_blank: false
+        errors = []
+        errors += validate_required_columns(row, row_num: row_num, required_columns: required_columns)
+        return errors unless errors.blank?
+        errors += validate_balanced_columns(row, row_num: row_num, balanced_columns: balanced_columns, allow_blank: allow_blank)
+        errors += validate_whitespace(row, row_num: row_num, nested_columns: nested_columns)
+        errors
+      end
+
+      # Validates the presence of required columns in a given row of data.
+      #
+      # @param row [Hash, CSV::Row] The row of data to be validated.
+      # @param required_columns [Array<Symbol>] The required columns for the row.
+      # @return [Array<String>] An array of error messages, if any; otherwise, an empty array.
+      def self.validate_required_columns row, row_num:, required_columns:
+        return [] if required_columns.blank?
+        missing = required_columns - row.to_h.keys
+        return [] if missing.empty?
+        ["#{ERROR_MISSING_REQUIRED_COLUMNS}: #{missing.map(&:inspect).join(', ')} row #{row_num}"]
+      end
+
+
+      # Validates the balanced columns in a given row of data.
+      #
+      # +balanced_columns+ is a hash of groups of balanced columns.
+      #
+      # @param row [Hash] The row of data to be validated.
+      # @param balanced_columns [Hash<Symbol, Array<Symbol>>] A hash of groups of balanced columns.
+      # @param allow_blank [Boolean] Whether to allow blank subfields in balanced columns.
+      # @return [Array<String>] An array of error messages, if any; otherwise, an empty array.
+      #
+      # @example
+      #     # row has unbalanced columns :a and :b
+      #     row = { a: 'a', b: 'b|b', c: 'c', d: 'd' }
+      #     balanced_columns = { group1: [:a, :b] }
+      #     csv_validator.validate_balanced_columns(
+      #         row, balanced_columns: balanced_columns
+      #     )  # => ["Row has subfields of different lengths: group: :group1, sizes: [1, 2], row: [\"a\", \"b|b\"]"]
+      def self.validate_balanced_columns row, row_num:, balanced_columns: {}, allow_blank: false
+        return [] if balanced_columns.blank?
+        errors = []
+        balanced_columns.each { |group, columns|
+          values = columns.map { |column| row[column.to_s] || row[column.to_sym] }
+          errors += validate_row_splits(group: group, row_num: row_num, row_values: values, allow_blank: allow_blank)
+        }
+      errors
+      end
+
+      # Maximum number of subfields to allow in a row; this number is
+      # arbitrarily set to 100,000 to ensure all trailing empty
+      # values are included in the array output by split.
+      MAX_SPLITS = 100000
+
+
+      ##
+      # Return an error if each value in +row_values+ has the same number of subfields
+      # **and** none of the subfields are blank; otherwise, return +nil+.
+      #
+      # If +allow_blank+ is +true+, ignore blanks, only check for balanced
+      # subfields.
+      #
+      # Note: It is always allowed for every value to be blank (empty string).
+      # When row values are +nil+ they are treated as empty strings.
+      # Blank values are treated a single values
+      #
+      # So:
+      #
+      #   [ 'a|b|c', '1|2|3' ]   # => valid, return []
+      #   [ '', '' ]             # => valid, return []
+      #   [ 'a', '']             # => valid, return []
+      #   [ 'a|b|c', '1|2' ]     # => not valid, return ERROR_UNBALANCED_SUBFIELDS
+      #   [ 'a|b', '']           # => not valid, return ERROR_UNBALANCED_SUBFIELDS
+      #   [ 'a||c', '1|2|3' ]    # => not valid, return ERROR_BLANK_SUBFIELDS
+      #   [ 'a||c', '1|2|3' ]    # => valid if allow_blank == true, return []
+      #
+      #
+      # @param [Array<String>] row_values an array of strings from one or more columns
+      # @param [String] separators a list of allowed subfield separators; e.g., ';', '|', ';|'
+      # @param [Boolean] allow_blank whether any of the subfields may be blank
+      # @return [Array<String>] the row errors, or [] if there are no errors
+      def self.validate_row_splits row_values: [], row_num:, separators: '|;', allow_blank: false, group: nil
+        errors = []
+        return errors if row_values.all? { |val| val.blank? }
+        # Input array is an array of two or more strings that must split into
+        # equal numbers of subfields.
+        #
+        #   ['a|bc', '1|2|3'] => [['a', 'b', 'c'],
+        #                         ['1', '2', '3']]
+        #   ['a|b|c', '1|2']  => [['a', 'b', 'c'],
+        #                         ['1' '2']]
+        #
+        # Count the subfields and make sure there's an equal number in each field
+        #
+        #    ['a|bc', '1|2|3'] => # 3 subfields each; => valid
+        #    ['a|b|c', '1|2']  => # 2 and 3 subfields; => not valid
+        splits = row_values.map { |v|
+          v.to_s.split %r{[#{Regexp.escape separators}]}, MAX_SPLITS
+        }
+
+        # all sizes should 0 or 1; or there should be only one
+        # subfield length
+        sizes = splits.map { |vals| vals.size }
+        if sizes.all? { |size| [0,1].include? size }
+          return errors
+        elsif sizes.uniq.size > 1
+          errors << "#{ERROR_UNBALANCED_SUBFIELDS}: group: #{group.inspect}, sizes: #{sizes.inspect}, row: #{row_values.inspect} (row #{row_num})"
+        end
+
+        # return true if we don't have check for blanks
+        return errors if allow_blank
+
+        # return an error if any of the subfields are blank
+        if splits.flatten.any? &:blank?
+          errors << "#{ERROR_BLANK_SUBFIELDS}: group: #{group.inspect}, row: #{row_values.inspect} (row #{row_num})"
+        end
+        errors
+      end
+
+      # Validates a row of data for trailing whitespace. Returns an
+      # error for each column that contains trailing whitespace.
+      #
+      # Nested columns is a hash with column names as keys and group
+      # names as values; e.g.,
+      #
+      #     nested_columns = {
+      #       "subject_label" => :subjects,
+      #       "subject" => :subjects,
+      #       "genre_label" => :genres
+      #       "genre" => :genres
+      #     }
+      #
+      # @param row [Hash] The row of data to be validated.
+      # @param nested_columns [Array<Symbol>] A hash of nested columns.
+      # @return [Array<String>] An array of error messages, if any.
+      def self.validate_whitespace row, row_num:, nested_columns: []
+        errors = []
+
+        row.each do |column, value|
+          # Assume all columns can have subfields delimited by pipes;
+          # some columns are "nested"; that is, they can be be further
+          # subdivided by semicolons. Select the regexp for the
+          # subfield type
+          split_chars = nested_columns.include?(column) ? PIPE_SEMICOLON_REGEXP : PIPE_SPLIT_REGEXP
+          if value.to_s.split(split_chars).any? { |sub| sub =~ %r{\s+$} }
+            errors << "#{ERROR_TRAILING_WHITESPACE}: column #{column.inspect}, value: #{value.inspect} (row #{row_num})"
+          end
+        end
+
+        errors
+      end
+
+    end
+  end
+end

data/lib/ds/util/csv_writer.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module DS
+  module Util
+    class CSVWriter
+      attr_reader :headers
+      attr_reader :outfile
+
+      def initialize outfile:, headers: []
+       @headers = headers
+        @outfile = outfile
+      end
+
+      def write rows=nil, &block
+        if block_given?
+          _write_with_block &block
+        elsif rows.is_a? Enumerable
+          _write_all rows
+        else
+          raise ""
+        end
+      end
+
+      private
+      def _write_with_block
+        CSV.open outfile, 'w+', headers: true do |csv|
+          csv << headers
+          yield csv
+        end
+      end
+
+      def _write_all rows
+        CSV.open outfile, 'w+', headers: true do |csv|
+          csv << headers
+          rows.each do |row|
+            csv << row
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ds/util/strings.rb

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+module DS
+  module Util
+    module Strings
+
+      ##
+      # This method calls
+      #
+      #  - +convert_mets_superscript+
+      #  - +remove_brackets+
+      #  - +fix_double_periods+
+      #  - +escape_pipes+
+      #  - +normalize_string+
+      #
+      # If +terminator+ is non-nil, the method removes any trailing
+      # punctuation and whitespace and appends +terminator+.
+      #
+      # Set +terminator+ to +``+ (empty string) to remove trailing
+      # punctuation.
+      #
+      # @param [String] string the string to clean
+      # @param [String] terminator the terminator to use, if any
+      # @param [Boolean] force use exact termination with +terminator+
+      # @return [String] the cleaned string
+      def clean_string string, terminator: nil, force: false
+        normal = normalize_string(
+          escape_pipes(
+            fix_double_periods(
+              remove_brackets(
+                convert_mets_superscript(string.to_s)
+              )
+            )
+          )
+        )
+
+        return normal if terminator.nil?
+
+        cleaned = terminate normal, terminator: terminator, force: force
+        # keep cleaning until no changes are made
+        return clean_string cleaned unless cleaned == string
+        cleaned
+      end
+
+      # TERMINAL_PUNCT_REGEX matches strings terminated by any of +.,;:?!+
+      TERMINAL_PUNCT_REGEX =  %r{\s*([.,;:!]+)("?)$}
+
+      # ELLIPSIS_REGEX matches strings terminated by +...+
+      ELLIPSIS_REGEX       = %r{\.\.\."?$}
+
+      # ABBREV_REGEX matches values like 'N.T.', 'O.T.'
+      ABBREV_REGEX         = %r{\W[A-Z]\.$}
+
+      # Final ? regex
+      FINAL_QUESTION_REGEX = %r{\s*\?(\s*"?\s*)$}
+      ##
+      # Add termination to string if it lacks terminal punctuation.
+      # Terminal punctuation is one of
+      #
+      #     . , ; : ? !
+      #
+      # When +:terminator+ is +''+ or +nil+, trailing punctuation is*always*
+      # removed.
+      #
+      # Strings ending with ellipsis, '...' or '..."' are returned unaltered. This
+      # behavior cannot be overridden with `:force`.
+      #
+      # @param [String] str the string to terminate
+      # @param [String] terminator the terminator to use; default: +.+
+      # @param [Boolean] force use exact termination with +terminator+
+      # @return [String]
+      def terminate str, terminator: '.', force: false
+        str.strip!
+        # DE 2022.08.12 Note the \s* to match and replace whitespace before
+        #     punctuation; this addresses a bug where some strings were returned
+        #     with trailing whitespace: 'value :' => 'value '
+        # TODO: Refactor? Two functions: strip_punctuation(), terminate() ??
+
+        # don't strip ellipses
+        return str if str =~ ELLIPSIS_REGEX
+        # don't strip final periods for strings like "N.T."
+        return str if str =~ ABBREV_REGEX
+
+        # don't strip final question marks
+        return str if str =~ FINAL_QUESTION_REGEX
+
+        # if :terminator is '' or nil, remove any terminal punctuation
+        return str.sub TERMINAL_PUNCT_REGEX, '\2' if terminator.blank?
+
+        # str is already terminated
+        return str if str.end_with? terminator
+        return str if str.end_with? %Q{#{terminator}"}
+
+        # if string ends with '?', don't add terminator
+        return str if str.end_with? '?'
+
+        # str lacks terminal punctuation; add it;
+        #  \\1 => keep final '"' (double-quote)
+        return str.sub %r{("?)$}, "#{terminator}\\1" if str !~ TERMINAL_PUNCT_REGEX
+        # str has to have exact terminal punctuation
+        #  \\1 => keep final '"' (double-quote)
+        return str.sub TERMINAL_PUNCT_REGEX, "#{terminator}\\2" if force
+        # string has some terminal punctuation; return it
+        str
+      end
+
+      ##
+      # Strip and replace all sequences of white space with single
+      # spaces and apply Unicode normalization. NFC normalization is
+      # used for all strings except URLs, to which NFKC normalization
+      # is applied. See RFC 3987:
+      #
+      # https://datatracker.ietf.org/doc/html/rfc3987#section-5.3.2.2
+      #
+      # @param [String] value the string to normalize
+      # @return [String] the normalized string
+      def normalize_string value
+        form = is_url?(value) ? :nfkc : :nfc
+        escape_pipes(
+          clean_white_space(
+            unicode_normalize(value, form)
+          )
+        )
+      end
+
+      ##
+      # converts encoded DS 1.0 encoded superscripts to parenthetical
+      # values; e.g., 'XVI#^4/4#' is converted to 'XVI(4/4)'
+      def convert_mets_superscript value
+        value.to_s.gsub(%r{#\^([^#]+)#}, '(\1)')
+      end
+
+      ##
+      # Escape pipe characters in source strings so split operations
+      # can avoid splitting on them.
+      def escape_pipes value
+        value.gsub('|', '\|')
+      end
+
+      def clean_white_space value
+        value.to_s.strip.gsub(%r{\s+}, ' ')
+      end
+
+      ##
+      # Return the string using unicode normalization form +form+.
+      # Use +NFC+ normalization by default. NFC normalization is
+      # recommended best practice. See
+      #
+      # https://www.honeybadger.io/blog/ruby-unicode-normalization/
+      #
+      # In short: NFC should be used for most strings, but NFKC for
+      # URLs. See RFC 3987:
+      #
+      # https://datatracker.ietf.org/doc/html/rfc3987#section-5.3.2.2
+      #
+      # Wikibase uses NFC normalization:
+      #
+      # https://doc.wikimedia.org/Wikibase/REL1_28/php/classWikibase_1_1Repo_1_1Parsers_1_1WikibaseStringValueNormalizer.html
+      #
+      # @param [String] value the string to normalize
+      # @param [Symbol] form the normalization form: +:nfc+, +:nfkc+.
+      #       +:nfd+, or +:nfkd+; default: +:nfc+
+      # @return [String] the normalized string
+      def unicode_normalize value, form = :nfc
+        value.to_s.unicode_normalize form
+      end
+
+      def remove_brackets value
+        value.to_s.strip.delete_prefix('[').delete_suffix(']')
+      end
+
+      ##
+      # Replace any sequence of two '..' with a single period.
+      # Ellipses, that is, sequences of three periods '...', are
+      # ignored.
+      #
+      #     fix_double_periods('....')       #  => "...."
+      #     fix_double_periods('.. ..')      #  => ". ."
+      #     fix_double_periods('... ..')     #  => "... ."
+      #     fix_double_periods('... a..')    #  => "... a."
+      #     fix_double_periods('a... a..')   #  => "a... a."
+      #
+      # @param [String] value the string to process
+      # @return [String]
+      def fix_double_periods value
+        value.to_s.gsub(%r{(?<!\.)\.\.(?!\.)}, '.')
+      end
+
+      def is_url? value
+        value.to_s =~ URI::regexp
+      end
+    end
+  end
+end

data/lib/ds/util.rb

@@ -0,0 +1,37 @@
+require 'nokogiri'
+
+require_relative 'util/strings'
+require_relative 'util/cache'
+require_relative 'util/csv_writer'
+require_relative 'util//csv_validator'
+
+module DS
+  module Util
+
+    extend DS::Util::Strings
+    ##
+    # Open and parse each XML file in +files+, optionally stripping namespaces
+    # from the parsed XML, running block on each XML document:
+    #
+    #   data = []
+    #   process_xml files, remove_namespaces: true do |xml|
+    #     data << xml.xpath('//some/path/text()').text
+    #   end
+    #
+    # @yield [xml, data] yields a Nokogiri XML document and the array of data
+    #         to populate the CSV; you must know the format of each item
+    #         in the ++data++ array
+    #
+    # @param files [Enumerable<String>] XML files to process
+    # @param remove_namespaces [Boolean] whether strip namespaces from parsed XML
+    # @yieldparam xml [Nokogiri::XML::Document] the parsed document
+    def process_xml files, remove_namespaces: false, &block
+      files.each do |in_xml|
+        # may be reading file list from STDIN; remove any trailing \r or \n
+        xml = File.open(in_xml.chomp) { |f| Nokogiri::XML f }
+        xml.remove_namespaces! if remove_namespaces
+        yield xml
+      end
+    end
+  end
+end

data/lib/ds/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module DS
+  VERSION = "0.1.1"
+end