ds-convert 0.1.1

data/lib/ds/recon.rb

@@ -0,0 +1,292 @@
+require_relative 'util'
+require_relative 'recon/source_enumerator'
+require_relative 'recon/ds_csv_enumerator'
+require_relative 'recon/marc_xml_enumerator'
+require_relative 'recon/tei_xml_enumerator'
+require_relative 'recon/ds_mets_xml_enumerator'
+require_relative 'recon/url_lookup'
+require_relative 'recon/type/recon_type'
+require_relative 'recon/type/genres'
+require_relative 'recon/type/languages'
+require_relative 'recon/type/materials'
+require_relative 'recon/type/names'
+require_relative 'recon/type/places'
+require_relative 'recon/type/subjects'
+require_relative 'recon/type/splits'
+require_relative 'recon/type/named_subjects'
+require_relative 'recon/type/all_subjects'
+require_relative 'recon/type/titles'
+require_relative 'recon/recon_builder'
+require_relative 'recon/recon_manager'
+require_relative 'constants'
+require 'logger'
+require 'csv'
+require 'ostruct'
+
+# The DS Recon module contains classes and methods for working with DS
+# recon data dictionaries.
+#
+# The classes in this module manage and support all the following:
+#
+# - The loading recon data dictionary CSV files for recon lookups
+# - The generation of recon CSV files from import sources
+# - The addition of recon data to import CSVs
+#
+# The key modules and classes in the Recon module are:
+#
+# - {Recon} -- validation and loading of recon data dictionary CSVs; data dictionary lookups; retrieval and updates of the DS data git repository, which includes the data dictionary CSVs
+# - {Recon::Type} -- recon type configurations used for lookups, extractions, and column mappings
+# - {Recon::ReconManager} -- the main interface for the Recon module; used to build and write recon CSVs
+# - {Recon::ReconBuilder} -- used by the Recon::Manager to build recon values hashes by extracting DS::Extractor::BaseTerm instances from source records and performing lookups
+# - {Recon::SourceEnumerator} instances -- used by Recon::ReconBuilder to iterate over source records
+#
+# @example
+#     require 'ds'
+#     # write the places.csv file for a set of MARC XML files
+#     files = Dir['source/files/*.xml']
+#     recon_manager = Recon::ReconManager.new(
+#       source_type: 'marc-xml',
+#       out_dir: 'path/to/dir',
+#       files: files
+#     )
+#     recon_type = Recon.find_recon_type :places
+#     recon_manager.write_csv recon_type # => 'path/to/dir/places.csv'
+#
+#
+module Recon
+
+  ERROR_UNBALANCED_SUBFIELDS = 'Row has unmatched subfields'
+  ERROR_BLANK_SUBFIELDS = 'Row has blank subfields'
+  ERROR_MISSING_REQUIRED_COLUMNS = "CSV is missing required column(s)"
+  ERROR_CSV_FILE_NOT_FOUND = 'Recon CSV file cannot be found'
+
+
+  RECON_SETS = %i{
+    genres
+    languages
+    materials
+    named-subjects
+    names
+    places
+    subjects
+    titles
+  }
+
+
+  RECON_TYPES_MAP = {
+    :genres => Recon::Type::Genres,
+    :languages => Recon::Type::Languages,
+    :materials => Recon::Type::Materials,
+    :'all-subjects' => Recon::Type::Subjects,
+    :'named-subjects' => Recon::Type::NamedSubjects,
+    :names => Recon::Type::Names,
+    :places => Recon::Type::Places,
+    :subjects => Recon::Type::Subjects,
+    :titles => Recon::Type::Titles,
+    :splits => Recon::Type::Splits
+  }.freeze
+
+  RECON_TYPES = RECON_TYPES_MAP.values.freeze
+
+  RECON_VALIDATION_SETS = RECON_TYPES.map(&:set_name).freeze
+
+
+  # For the recon data dictionary with +set_name+, find the value in the +column+ with the
+  # key +values.join('$$')+.
+  #
+  # @param [String] set_name the name of the set to look up
+  # @param [String] values the lookup key values
+  # @param [Symbol] column the column value to retrieve
+  # @return [Object, nil] the value found in the specified column, or nil if not found
+  def self.lookup_single set_name, key_values:, column:
+    recon_set = find_set set_name
+    key = build_key key_values
+    return recon_set.dig key, column if recon_set.include? key
+
+    # try a key with a "cleaned" string
+    alt_key = build_alt_key key
+    recon_set.dig(alt_key, column)
+  end
+
+  # Return the set of terms for the given set name.
+  #
+  # @param set_name [String] the name of the set
+  # @return [Hash<String, Struct>, nil] the set of terms, with keys as values and term structs as values
+  def self.find_set set_name
+    @@reconciliations ||= {}
+    @@reconciliations[set_name] ||= load_set set_name
+  end
+
+  # The path to the DS data git repository
+  #
+  # @return [String] the path to the DS data git repository
+  def self.git_repo
+    File.join Settings.recon.local_dir, Settings.recon.git_local_name
+  end
+
+  # Finds the +config/settings.yml+ configuration for the given set name.
+  #
+  # @param set_name [String] the name of the set
+  # @return [Hash<String, Object>] the configuration for the set name, or nil if not found
+  # @raise [DSError] if the set name is not found
+  def self.find_set_config name
+    config = Settings.recon.sets.find { |s| s.name == name }
+    raise DSError, "Unknown set name: #{name.inspect}" unless config
+    config
+  end
+
+  # Finds the reconciliation type configuration for the given set name.
+  #
+  # @param set_name [String] the name of the set
+  # @return [Recon::Type::ReconType, nil] the configuration for the set name, or nil if not found
+  def self.find_recon_type set_name
+    return RECON_TYPES_MAP[set_name.to_sym] if RECON_TYPES_MAP.key? set_name.to_sym
+
+    raise "Unknown recon set_name: #{set_name.inspect}"
+  end
+
+  # Returns an array of paths to the CSV files for the given set name
+  #
+  # @param set_name [String] the name of the set
+  # @return [Array<String>] an array of paths to the CSV files
+  def self.csv_files set_name
+    set_config = find_set_config set_name
+    repo_paths = [set_config['repo_path']].flatten # ensure repo_path is an array
+    repo_paths.map { |path| File.join Recon.git_repo, path }
+  end
+
+  # Return and return the reconciliation data dictionary for the given set name.
+  #
+  # The hash keys are the concatenated key values for the reconciliation type (e.g., {Recon::Type::Names.get_key_values})
+  #
+  # @param set_name [String] the name of the set
+  # @return [Hash<String, Struct>] the reconciliation data
+  def self.load_set set_name
+    set_config = find_set_config set_name
+    recon_type = find_recon_type set_name
+    raise "No configured set found for: '#{set_name}'" unless set_config
+
+    data = {}
+    params = {
+      recon_type:    recon_type,
+      data:          data
+    }
+
+    # Path may be a single value or an array. Make sure it's an array.
+    csv_files(set_name).each do |csv_file|
+      params[:csv_file] = csv_file
+      validate! set_name, params[:csv_file]
+      read_csv **params
+    end
+
+    add_alt_keys data
+    data
+  end
+
+  # Read one CSV file and add the reconciliation data to the data hash
+  # and return the updated data hash.
+  #
+  # @param csv_file [String] the path to the CSV file
+  # @param recon_type [Recon::Type::ReconType] the reconciliation type
+  # @param data [Hash<String, Struct>] the reconciliation data
+  # @return [Hash<String, Struct>] the updated reconciliation data
+  def self.read_csv csv_file:, recon_type:, data:
+    CSV.foreach csv_file, headers: true do |row|
+      row = row.to_h.symbolize_keys
+      next if recon_type.lookup_values(row).blank?
+      struct    = OpenStruct.new row.to_h
+      key       = build_key recon_type.get_key_values row
+      data[key] = struct
+    end
+    data
+  end
+
+  # Validate an input or output recon CSV file for the given set name.
+  #
+  # Validates each row the CSV file for required column headers and
+  # values and for balanced columns. Required headers and values are
+  # defined in the Recon::Type::ReconType class, by
+  # {Recon::Type::ReconType.recon_csv_headers} and
+  # {Recon::Type::ReconType.balanced_headers}.
+  #
+  # @param set_name [String] the name of the set
+  # @param csv_file [String] the path to the CSV file
+  # @return [Array<String>] an array of errors
+  def self.validate set_name, csv_file
+    return unless RECON_VALIDATION_SETS.include? set_name
+    return "#{ERROR_CSV_FILE_NOT_FOUND}: '#{csv_file}'" unless File.exist? csv_file
+
+    recon_type = Recon.find_recon_type set_name
+    row_num    = 0
+    CSV.readlines(csv_file, headers: true).map(&:to_h).filter_map { |row|
+      row.symbolize_keys!
+      error = validate_row recon_type, row, row_num+=1
+      error if error.present?
+    }
+  end
+
+  # Invoke {Recon.validate} for the input or output recon CSV file
+  # and raise an exception if there is an error.
+  #
+  # @param set_name [String] the name of the set
+  # @param csv_file [String] the path to the CSV file
+  # @return [void]
+  # @raise [DSError] if there is an error
+  def self.validate! set_name, csv_file
+    error = validate set_name, csv_file
+    return unless error.present?
+
+    raise DSError, "Error validating #{set_name} recon CSV #{csv_file}:\n#{error}"
+  end
+
+  # Validate one row of a CSV file for required column headers and values
+  # and for balanced columns.
+  #
+  # @param recon_type [Recon::Type::ReconType] the reconciliation type
+  # @param row [Hash] the row of data
+  # @param row_num [Integer] the row number used in error messages
+  # @return [Array<String>] an array of errors
+  def self.validate_row recon_type, row, row_num
+    errors = DS::Util::CsvValidator.validate_required_columns(row, required_columns: recon_type.recon_csv_headers, row_num: row_num)
+    raise DSError.new errors.join("\n") unless errors.blank?
+    DS::Util::CsvValidator.validate_balanced_columns(
+      row, balanced_columns: recon_type.balanced_columns, row_num: row_num
+    )
+  end
+
+  # Builds an alt key from key, splitting it into an array of values,
+  # invoking DS::Util::clean_string on each value and rejoining the
+  # cleaned values separated by '$$'.
+  #
+  # @param key [String] the key to be included in the alt key
+  # @return [String] the built alt key
+  def self.add_alt_keys data
+    data.keys.each do |key|
+      alt_key = build_alt_key key
+      next if data.include? alt_key
+      data[alt_key] = data[key]
+    end
+  end
+
+  # Builds an alt key from key, splitting it into an array of values,
+  # invoking DS::Util::clean_string on each value and rejoining the
+  # cleaned values separated by '$$'.
+  #
+  # @param key [String] the key to be included in the alt key
+  # @return [String] the built alt key
+  def self.build_alt_key key
+    key.split('$$').map { |v|
+      DS::Util.clean_string v, terminator: ''
+    }.join '$$'
+  end
+
+  # Builds a key by concatenating the normalized Unicode representation of +values+,
+  # separated by '$$', and converts it to lowercase.
+  #
+  # @param values [Array<String>] the values to be included in the key
+  # @param subset [String] the subset to be included in the key
+  # @return [String] the built key
+  def self.build_key values
+    DS::Util.unicode_normalize values.select(&:present?).join('$$').downcase
+  end
+end

data/lib/ds/source/base_source.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module DS
+  module Source
+    ##
+    # Encapsulates methods for caching and opening source files.
+    #
+    # This class includes the DS::Source::SourceCache module, but does
+    # not implement the +open_source+ method. Concrete subclasses of
+    # {DS::Source::BaseSource} must implement +open_source+.
+    #
+    class BaseSource
+      include DS::Source::SourceCache
+
+      def source_type
+        self.class::TYPE
+      end
+
+      # Loads the specified source path.
+      #
+      # @param source_path [String] The path to the source file.
+      # @return [Object] The parsed source file; e.g, Nokogiri::XML::Document or CSV::Table
+      def load_source source_path
+        find_or_open_source source_path
+      end
+
+      def to_s
+        "#{self.class_name}: source_type #{source_type}"
+      end
+    end
+  end
+end

data/lib/ds/source/ds_csv.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module DS
+  module Source
+    class DSCSV < BaseSource
+
+      TYPE = DS::Constants::DS_CSV
+
+      # Opens a CSV file at the specified `source_file_path` and returns a CSV object.
+      #
+      # @param source_file_path [String] The path to the CSV file.
+      # @return [CSV] A CSV object representing the opened CSV file.
+      def open_source source_file_path
+        CSV.open(source_file_path, 'r', headers: true)
+      end
+    end
+  end
+end

data/lib/ds/source/ds_mets_xml.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module DS
+  module Source
+    class DSMetsXML < BaseSource
+
+      TYPE = DS::Constants::DS_METS
+
+      # Opens a METS XML file at the given path and returns it as a Nokogiri::XML object.
+      #
+      # Namespaces are *not* removed from the document.
+      #
+      # @param source_file_path [String] the path to the source file
+      # @return [Nokogiri::XML::Document] the contents of the source file as a Nokogiri::XML object
+      def open_source source_file_path
+        File.open(source_file_path) { |f| Nokogiri::XML f }
+      end
+    end
+  end
+end

data/lib/ds/source/marc_xml.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module DS
+  module Source
+    class MarcXML < BaseSource
+
+      TYPE = DS::Constants::MARC_XML
+
+      # Opens a MARC XML source file at the given path and returns a Nokogiri::XML object representing the record.
+      #
+      # NB: Namespaces are stripped from the document.
+      #
+      # @param source_file_path [String] the path to the source file
+      # @return [Nokogiri::XML::Document] the MARC XML record
+      def open_source source_file_path
+        xml = File.open(source_file_path) { |f| Nokogiri::XML f }
+        xml.remove_namespaces!
+        xml
+      end
+    end
+  end
+end

data/lib/ds/source/source_cache.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module DS
+  module Source
+
+    ##
+    # This module provides methods for caching and opening source files.
+    # It is used by the DS::Mapper::BaseMapper class.
+    #
+    # It makes available a +#find_or_open_source+ method that can be used
+    # by the including class to open or retrieve a parse source file
+    # from the cache.
+    #
+    # Including classes must implement the +open_source+ method.
+    #
+    # The file +path+ is used as the cache key.
+    #
+    # The initial cache size is the value of DS::Util::Cache::DEFAULT_MAX_SIZE.
+    #
+    # Cache max size can be set and retrieved using the +max_cache_size+ and +max_cache_size=+ methods. #
+    module SourceCache
+
+      # Finds or opens a source file at the given path.
+      #
+      # @param source_path [String] the path to the source file
+      # @return [Object] the contents of the source file
+      def find_or_open_source source_path
+        return cache.get_item source_path if cache.include? source_path
+        source = open_source source_path
+        cache.add source_path, source
+        source
+      end
+
+      # Opens a source file at the given path.
+      #
+      # @param source_path [String] the path to the source file
+      # @return [Object] the contents of the source file
+      # @raise [NotImplementedError] unless implemented by including class
+      def open_source source_path
+        raise NotImplementedError
+      end
+
+      # Returns the cache object.
+      #
+      # This method lazily initializes the cache object if it is not already initialized.
+      # The cache object is an instance of the DS::Util::Cache class.
+      #
+      # @return [DS::Util::Cache] the cache object
+      def cache
+        @cache ||= DS::Util::Cache.new
+      end
+
+      # Sets the maximum cache size.
+      #
+      # @param size [Integer] the maximum number of items to store in the cache
+      # @return [void]
+      def max_cache_size= size
+        cache.max_size = size
+      end
+
+      # Returns the maximum cache size.
+      #
+      # @return [Integer] the maximum number of items to store in the cache
+      def max_cache_size
+        cache.max_size
+      end
+    end
+  end
+end

data/lib/ds/source/tei_xml.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module DS
+  module Source
+    class TeiXML < BaseSource
+
+      TYPE = DS::Constants::TEI_XML
+
+      # Opens a TEI XML file at the given path and returns it as a Nokogiri::XML object.
+      #
+      # NB: Namespaces are stripped from the document.
+      #
+      # @param source_file_path [String] the path to the source file
+      # @return [Nokogiri::XML::Document] the contents of the source file as a Nokogiri::XML object
+      def open_source source_file_path
+        xml = File.open(source_file_path) { |f| Nokogiri::XML f }
+        xml.remove_namespaces!
+        xml
+      end
+    end
+  end
+end

data/lib/ds/source.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require_relative 'source/source_cache'
+require_relative 'source/base_source'
+require_relative 'source/marc_xml'
+require_relative 'source/tei_xml'
+require_relative 'source/ds_mets_xml'
+require_relative 'source/ds_csv'
+
+module DS
+  # DS Source module classes encapsulates the loading of source files.
+  # They are used by DS::Mapper classes and DS::Manifest id validator
+  # classes.
+  #
+  # A primary function of the DS::Source classes is to manage
+  # caching of source files, which may be expensive to load and parse; e.g.,
+  # MARC XML or CSV files with a large number of records.
+  module Source
+  end
+end

data/lib/ds/util/cache.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+module DS
+  module Util
+    class Cache
+      DEFAULT_MAX_SIZE = 10
+      UNLIMITED_SIZE = Float::INFINITY
+
+      attr_accessor :max_size
+      attr_reader :items
+      attr_reader :keys
+
+      # Initializes a new instance of the class with the specified maximum size.
+      #
+      # @param max_size [Integer] (DEFAULT_MAX_SIZE) the maximum size of the cache
+      # @return [void]
+      def initialize max_size: DEFAULT_MAX_SIZE
+        @max_size = max_size
+        @items    = {}
+      end
+
+      # Adds an item to the cache if it is not already present, or if the cache is not limited and the item is not already present.
+      #
+      # @param key [Object] the key used to identify the item in the cache
+      # @param item [Object] the item to be added to the cache
+      # @return [Object] the item that was added to the cache
+      def get_or_add key, item
+        add(key, item) unless include? key && unlimited?
+        get_item key
+      end
+
+      # Adds an item to the cache if it is not already present, or if the cache is not limited and the item is not already present.
+      #
+      # @param key [Object] the key used to identify the item in the cache
+      # @param item [Object] the item to be added to the cache
+      # @return [Object] the item that was added to the cache
+      def add key, item
+        delete_item key
+        items[key] = item
+        cleanup
+        item
+      end
+
+      # Checks if the given key is present in the cache.
+      #
+      # @param key [Object] the key to check for in the cache
+      # @return [Boolean] true if the key is present in the cache, false otherwise
+       def include? key
+        keys.include? key
+      end
+
+      # Checks if the cache is unlimited.
+      #
+      # @return [Boolean] true if the cache is unlimited, false otherwise
+      def unlimited?
+        max_size == UNLIMITED_SIZE
+      end
+
+      # Retrieves an item from the cache using the specified key.
+      #
+      # @param key [Object] The key used to identify the item in the cache.
+      # @return [Object] The item associated with the specified key, or nil if the key is not present in the cache.
+      def get_item key
+        items[key]
+      end
+
+      # Retrieves an item from the cache using the specified key.
+      #
+      # @param key [Object] The key used to identify the item in the cache.
+      # @return [Object] The item associated with the specified key, or nil if the key is not present in the cache.
+      def [](key)
+        get_item key
+      end
+
+      # Returns an array of all the keys in the cache.
+      #
+      # @return [Array<Object>] An array of keys.
+      def keys
+        items.keys
+      end
+
+      # Returns the number of items in the cache.
+      #
+      # @return [Integer] The number of items in the cache.
+      def size
+        keys.size
+      end
+
+      # Deletes an item from the cache using the specified key.
+      #
+      # @param key [Object] The key used to identify the item in the cache.
+      # @return [void]
+      def delete_item key
+        items.delete key
+      end
+
+      # Cleanup the cache by removing items until the size is less than or equal to the maximum size.
+      #
+      # This method does not take any parameters.
+      #
+      # @return [void]
+      def cleanup
+        return if size < max_size
+        return if keys.blank? # don't allow an infinite loop
+        while size > max_size
+          delete_item keys.first
+        end
+      end
+    end
+  end
+end