ds-convert 0.1.1

data/lib/ds/manifest/manifest_validator.rb

@@ -0,0 +1,256 @@
+# frozen_string_literal: true
+
+require 'csv'
+require 'uri'
+require 'date'
+require_relative './constants'
+
+module DS
+  module Manifest
+    ##
+    # Validate a DS input manifest.
+    #
+    # Validation does the following:
+    #
+    #   - Confirms all required columns are present
+    #   - Confirms all all required values are present
+    #   - Confirms all column values are the correct type
+    #   - Confirms all listed input files are present
+    #   - Confirms all listed input files match the record
+    #     identifier provided in the manifest
+    #
+    # @todo Add test for live URLs
+    class ManifestValidator
+      include DS::Manifest::Constants
+
+      attr_reader :manifest
+      attr_reader :source_dir
+      attr_reader :errors
+
+      URI_REGEXP = URI::DEFAULT_PARSER.make_regexp %w{http https}
+      QID_REGEXP = %r{\AQ\d+\z}
+
+      ##
+      # @param [DS::Manifest] manifest DS::Manifest instance
+      # @return [DS::ManifestValidator]
+      def initialize manifest
+        @manifest = manifest
+        @errors   = []
+        @id_validators = {}
+      end
+
+      ##
+      # @return [boolean] true if the manifest is valid
+      def valid?
+        return false unless validate_columns
+        return false unless validate_required_values
+        return false unless validate_ids_unique
+        return false unless validate_data_types
+        return false unless validate_files_exist
+        return false unless validate_records_present
+        true
+      end
+
+      ##
+      # @return [boolean] true if all required columns are present
+      def validate_columns
+        found_columns = manifest.headers
+        diff          = MANIFEST_COLUMNS - found_columns
+        return true if diff.blank?
+        add_error "Manifest missing required columns: #{diff.join ', '}" if diff.present?
+        false
+      end
+
+      ##
+      # @return [boolean] true if all require values present
+      def validate_required_values
+        is_valid = true
+        manifest.each_with_index do |row, ndx|
+          REQUIRED_VALUES.each do |col|
+            if row[col].blank?
+              add_error "Required value missing in row: #{ndx+1}, col.: #{col}"
+              is_valid = false
+            end
+          end
+        end
+        is_valid
+      end
+
+      ##
+      # @return [boolean] true if all data types are valid
+      def validate_data_types
+        is_valid = true
+        manifest.each_with_index do |entry, row_num|
+          is_valid = false unless validate_urls entry, row_num
+          is_valid = false unless validate_qids entry, row_num
+          is_valid = false unless validate_dates entry, row_num
+        end
+        is_valid
+      end
+
+      ##
+      # @return [boolean] true if all list input files are present
+      def validate_files_exist
+        is_valid = true
+        manifest.each_with_index do |entry, row_num|
+          file_path = File.join manifest.source_dir, entry.filename
+          unless File.exist? file_path
+            is_valid = false
+            add_error "Source file not found row: #{row_num+1}; source directory: #{source_dir}; file: #{entry.filename}"
+          end
+        end
+        is_valid
+      end
+
+      # Validates the uniqueness of all IDs in the manifest.
+      #
+      # This method collects the count of all IDs in the manifest and selects those with a count greater than 1.
+      # It then iterates over the multiples and adds an error for each duplicate ID found.
+      #
+      # Returns:
+      # - `true` if no duplicate IDs are found.
+      # - `false` if duplicate IDs are found.
+      def validate_ids_unique
+        # collect the count of all ids and select those with a count > 1
+        multiples = manifest.inject({}) { |h, id|
+          h[id] ||= 0; h[id] += 1; h
+        }.filter_map { |id, count|
+          [id, count] if count > 1
+        }
+
+        return true if multiples.blank?
+
+        multiples.each do |id, count|
+          add_error "Duplicate ID found in manifest: ID '#{id}' found in #{count} rows"
+        end
+        false
+      end
+
+      ##
+      # @return [boolean] true if all +holding_institution_institutional_id+
+      #     values match source file
+      def validate_records_present
+        is_valid = true
+        manifest.each_with_index do |entry, row_num|
+          file_path = File.join manifest.source_dir, entry.filename
+
+          inst_id      = entry.institutional_id
+          id_validator = get_id_validator entry.source_type
+          found        = id_validator.valid? file_path, inst_id, entry.institutional_id_location_in_source
+
+          unless found
+            is_valid = false
+            id_validator.errors.each { |error| add_error error }
+          end
+        end
+        is_valid
+      end
+
+      # Handles the error when the count of records found for a given `inst_id` and `location_in_source` is
+      # 0 or more than 1.
+      #
+      # @param count [Integer] the number of records found for the given `inst_id` and `location_in_source`
+      # @param inst_id [String] the identifier of the record
+      # @param location_in_source [String] the location in the source where the record is found
+      # @return [nil]
+      def handle_count_error count, inst_id, location_in_source
+        return if count == 1
+
+        if count > 1
+          add_error "ERROR: Multiple records (#{count}) found for id: #{inst_id} (location: #{location_in_source})"
+        elsif count == 0
+          add_error "ERROR: No records found for id: #{inst_id} (location: #{location_in_source})"
+        end
+        nil
+      end
+
+      ####################################
+      # Type validations
+      ####################################
+      def validate_source_type entry, row_num
+        is_valid = true
+
+        unless source_types.include? entry.source_type
+          add_error "Invalid source type in row: #{row_num+1}; expected one of #{VALID_SOURCE_TYPES.join ', '}; got: '#{entry.source_type}'"
+          is_valid = false
+        end
+        is_valid
+      end
+
+      def validate_urls entry, row_num
+        is_valid = true
+        URI_COLUMNS.each do |col|
+          if entry[col].present? && entry[col].to_s !~ URI_REGEXP
+            add_error "Invalid URL in row: #{row_num+1}; col.: #{col}: '#{entry[col]}'"
+            is_valid = false
+          end
+        end
+        is_valid
+      end
+
+      def validate_qids entry, row_num
+        is_valid = true
+        QID_COLUMNS.each do |col|
+          unless entry[col].to_s =~ QID_REGEXP
+            is_valid = false
+            add_error "Invalid QID in row: #{row_num+1}; col.: #{col}: '#{entry[col]}'"
+          end
+        end
+        is_valid
+      end
+
+      def validate_dates entry, row_num
+        is_valid = true
+        DATE_TIME_COLUMNS.each do |col|
+          next if entry[col].blank?
+          begin
+            Date.parse entry[col]
+          rescue Date::Error
+            is_valid = false
+            add_error "Invalid date in row: #{row_num+1}, col.: #{col}: '#{entry[col]}'"
+          end
+        end
+        is_valid
+      end
+
+      # Adds an error message to the list of errors.
+      #
+      # @param message [String] the error message to add
+      # @return [void]
+      def add_error message
+        @errors << message
+      end
+
+      # Checks if there are any errors in the errors collection.
+      #
+      # @return [Boolean] true if there are errors, false otherwise
+      def has_errors?
+        errors.any?
+      end
+
+      # Retrieves the appropriate ID validator for the given source type.
+      #
+      # @param source_type [Symbol] the type of the source
+      # @return [DS::Manifest::BaseIdValidator] the ID validator for the source type
+      # @raise [NotImplementedError] if the source type is not implemented
+      def get_id_validator source_type
+        case source_type
+        when DS::Constants::MARC_XML
+          @id_validators[source_type] ||= SimpleXmlIdValidator.new(DS::Source::MarcXML.new)
+        when DS::Constants::DS_METS
+          @id_validators[source_type] ||= SimpleXmlIdValidator.new(DS::Source::DSMetsXML.new)
+        when DS::Constants::TEI_XML
+          @id_validators[source_type] ||= SimpleXmlIdValidator.new(DS::Source::TeiXML.new)
+        when DS::Constants::DS_CSV
+          @id_validators[source_type] ||= DsCsvIdValidator.new(DS::Source::DSCSV.new)
+        else
+          raise NotImplementedError, "validate_ids not implemented for: #{source_type}"
+        end
+      end
+
+      def source_types
+        VALID_SOURCE_TYPES
+      end
+    end
+  end
+end

data/lib/ds/manifest/simple_xml_id_validator.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module DS
+  module Manifest
+    class SimpleXmlIdValidator < BaseIdValidator
+
+      attr_accessor :namespaces
+
+      def initialize source, namespaces = {}
+        @namespaces = namespaces.present? ? namespaces : DS::Constants::XML_NAMESPACES
+        super source
+      end
+
+      # Locates a record in the XML document based on the given source path, ID, and ID location.
+      #
+      #  +id_location+ should be a template XPath expression that
+      #     returns one or more records, for example:
+      #
+      #     "//record[controlfield[@tag='001'] = 'ID_PLACEHOLDER']"
+      #
+      # The string 'ID_PLACEHOLDER' must be in the template.It will
+      # be replaced with the ID of the record to locate.
+      #
+      # @param source_path [String] the path to the XML source file
+      # @param id [String] the ID of the record to locate
+      # @param id_location [String] the XPath expression to locate the record
+      # @return [Nokogiri::XML::NodeSet] the located record(s)
+      def locate_record source_path, id, id_location
+        locator = DS::Extractor::XmlRecordLocator.new namespaces: namespaces
+        xml = source.load_source source_path
+        locator.locate_record xml, id, id_location
+      end
+
+      def try_locate_record xml, xpath, namespaces: nil
+        xml.xpath xpath, namespaces
+      rescue Nokogiri::XML::XPath::SyntaxError => e
+        raise unless e.message =~ /undefined namespace prefix/i
+        []
+      end
+    end
+  end
+end

data/lib/ds/manifest.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+
+require_relative 'manifest/constants'
+require_relative 'manifest/entry'
+require_relative 'manifest/manifest'
+require_relative 'manifest/base_id_validator'
+require_relative 'manifest/simple_xml_id_validator'
+require_relative 'manifest/ds_csv_id_validator'
+require_relative 'manifest/manifest_validator'
+
+module DS
+  ##
+  # {DS::Manifest} comprises classes and a module (DS::Manifest::Constants)
+  # for encapsulating and validating a DS delivery manifest CSV.
+  #
+  # The manifest CSV provides all information needed to ingest a set of
+  # source records. This information is detailed in
+  # {DS::Manifest::Manifest} and {DS::Manifest::Entry}.
+  #
+  # The {DS::Manifest::ManifestValidator} validates
+  # that the Manifest is completed and well-formed, and that all records
+  # can be found the specified source diretory.
+  #
+  # The valid {DS::Manifest::Manifest} is used by {DS::Converter::Converter}
+  # to orchestrate mapping of source record data for the creation of
+  # the DS import CSV.
+  module Manifest
+  end
+end

data/lib/ds/mapper/base_mapper.rb

@@ -0,0 +1,221 @@
+# frozen_string_literal: true
+
+module DS
+  module Mapper
+    # DS::Mapper::BaseMapper abstract Mapper class. Implementing classes
+    # map DS sources records to CSV rows.
+    #
+    # Implementing classes must implement:
+    #
+    # - `#extract_record`
+    # - `#map_record`
+    #
+    class BaseMapper
+      attr_reader :timestamp
+      attr_reader :source_dir
+      attr_reader :source
+      attr_reader :recon_builder
+
+      PLACES_COLUMN_MAP = {
+        production_place_as_recorded: :place_as_recorded,
+        production_place_ds_qid:      :ds_qid
+      }
+
+      TITLES_COLUMN_MAP = {
+        title_as_recorded:         :title_as_recorded,
+        title_as_recorded_agr:     :title_as_recorded_agr,
+        uniform_title_as_recorded: :uniform_title_as_recorded,
+        uniform_title_agr:         :uniform_title_as_recorded_agr,
+        standard_title_ds_qid:     :ds_qid
+      }
+
+      GENRES_COLUMN_MAP = {
+        genre_as_recorded: :genre_as_recorded,
+        genre_ds_qid:      :ds_qid
+      }
+
+      SUBJECTS_COLUMN_MAP = {
+        subject_as_recorded: :subject_as_recorded,
+        subject_ds_qid:      :ds_qid,
+      }
+
+      AUTHORS_COLUMN_MAP = {
+        author_as_recorded:     :name_as_recorded,
+        author_as_recorded_agr: :name_agr,
+        author_ds_qid:          :ds_qid
+
+      }
+
+      ARTISTS_COLUMN_MAP = {
+        artist_as_recorded:     :name_as_recorded,
+        artist_as_recorded_agr: :name_agr,
+        artist_ds_qid:          :ds_qid
+      }
+
+      SCRIBES_COLUMN_MAP = {
+        scribe_as_recorded:     :name_as_recorded,
+        scribe_as_recorded_agr: :name_agr,
+        scribe_ds_qid:          :ds_qid
+      }
+
+      ASSOCIATED_AGENT_COLUMN_MAP = {
+        associated_agent_as_recorded:     :name_as_recorded,
+        associated_agent_as_recorded_agr: :name_agr,
+        associated_agent_ds_qid:          :ds_qid
+      }
+
+      LANGUAGE_COLUMN_MAP = {
+        language_as_recorded: :language_as_recorded,
+        language_ds_qid:      :ds_qid
+      }
+
+      FORMER_OWNER_COLUMN_MAP = {
+        former_owner_as_recorded:     :name_as_recorded,
+        former_owner_as_recorded_agr: :name_agr,
+        former_owner_ds_qid:          :ds_qid
+      }
+
+      MATERIAL_COLUMN_MAP = {
+        material_as_recorded: :material_as_recorded,
+        material_ds_qid:      :ds_qid
+      }
+
+      # Maps recon type to column map
+      RECON_TYPE_COLUMN_MAP = {
+        Recon::Type::Places           => PLACES_COLUMN_MAP,
+        Recon::Type::Titles           => TITLES_COLUMN_MAP,
+        Recon::Type::Genres           => GENRES_COLUMN_MAP,
+        Recon::Type::AllSubjects      => SUBJECTS_COLUMN_MAP,
+        Recon::Type::Authors          => AUTHORS_COLUMN_MAP,
+        Recon::Type::Artists          => ARTISTS_COLUMN_MAP,
+        Recon::Type::Scribes          => SCRIBES_COLUMN_MAP,
+        Recon::Type::AssociatedAgents => ASSOCIATED_AGENT_COLUMN_MAP,
+        Recon::Type::Languages        => LANGUAGE_COLUMN_MAP,
+        Recon::Type::FormerOwners     => FORMER_OWNER_COLUMN_MAP,
+        Recon::Type::Materials        => MATERIAL_COLUMN_MAP,
+      }.freeze
+
+      # Initializes a new instance of the class.
+      #
+      # @param source_dir [String] the directory where the source files are located
+      # @param timestamp [Time] the timestamp of the source files
+      # @param source [DS::Source::BaseSource] the source object
+      # @return [void]
+      def initialize source_dir:, timestamp:, source:
+        @recon_builder = Recon::ReconBuilder.new source_type: source.source_type, files: [], out_dir: []
+        @source        = source
+        @source_dir    = source_dir
+        @timestamp     = timestamp
+      end
+
+      def to_s # :nodoc:
+        "#{self.class.name}: source_dir: #{source_dir}, timestamp: #{timestamp}, source: #{source}"
+      end
+
+      # Extracts a record from the source for the given manifest entry.
+      #
+      # @param [DS::Manifest::Entry] entry the entry representing one row in a manifest
+      # @return [Object] the extracted record; e.g., a Nokogiri::XML::Node or CSV::Row
+      # @raise [NotImplementedError] if the method is not implemented in a subclass
+      def extract_record entry
+        raise NotImplementedError
+      end
+
+      # Maps a source record for the given manifest entry.
+      #
+      # @param [DS::Manifest::Entry] entry the entry representing one row in a manifest
+      # @return [Hash<Symbol, String>] the mapped record
+      # @raise [NotImplementedError] if the method is not implemented in a subclass
+      def map_record entry
+        raise NotImplementedError
+      end
+
+      # Builds term strings based on the given recons and column mapping.
+      #
+      #
+      # @example
+      #  recons = [
+      #      { as_recorded: 'Brown, Jamie', authorized_label: 'Jamie Brown' },
+      #      { as_recorded: 'Hendrix, Morgan', authorized_label: 'Morgan Hendrix' }
+      #  ]
+      #  column_map = { author_as_recorded: :as_recorded, author_label: :authorized_label }
+      #  build_term_strings(recons, column_map)
+      #  # => { 'author_as_recorded' => 'Brown, Jamie| Hendrix, Morgan', :author_label => 'Jamie Brown|Morgan Hendrix', 'author_label' => 'Jamie Brown|Morgan Hendrix' }
+      #
+      # @param [Array<Hash>] recons the recons to build term strings from
+      # @param [Hash] column_map a mapping of import CSV columns to recon keys
+      # @return [Hash] a hash with import CSV columns as keys and corresponding term strings as values
+      def build_term_strings recons, column_map
+        column_map.inject({}) do |hash, (import_csv_col, recon_key)|
+          hash[import_csv_col] = build_term_string recons, recon_key
+          hash
+        end
+      end
+
+      # Creates an import CSV hash for the given record for all recon
+      # term types, using the given extractor. The extractor is one
+      # of
+      #
+      #   DS::Extractor::MarcXml
+      #   DS::Extractor::TeiXml
+      #   DS::Extractor::DsCsvExtractor
+      #   DS::Extractor::DsMetsXml
+      #
+      # The following recon term types are mapped for all
+      # records/extractors:
+      #
+      #   Recon::Type::Places
+      #   Recon::Type::Titles
+      #   Recon::Type::Genres
+      #   Recon::Type::Subjects
+      #   Recon::Type::Authors
+      #   Recon::Type::Artists
+      #   Recon::Type::Scribes
+      #   Recon::Type::AssociatedAgents
+      #   Recon::Type::Languages
+      #   Recon::Type::FormerOwners
+      #   Recon::Type::Materials
+      #
+      # Column mappings are defined in DS::Mapper::RECON_TYPE_COLUMN_MAP
+      #
+      # @param [DS::Extractor::MarcXml, DS::Extractor::TeiXml, DS::Extractor::DsCsvExtractor, DS::Extractor::DsMetsXml] extractor the extractor object
+      # @param [Nokogiri::XML::Node, CSV::Row] record the record to extract terms from
+      # @return [Hash<Symbol, String>] a hash of terms mapped to import CSV columns
+      def build_term_maps extractor, record
+        RECON_TYPE_COLUMN_MAP.inject({}) { |hash, (recon_type, column_map)|
+          terms = recon_type.method_name.flat_map { |method| extractor.send(method, record) }
+          hash.update map_terms terms, recon_type, column_map
+        }
+      end
+
+      # Builds a term string by concatenating the values of the given reconstructions
+      # corresponding to the specified recon key, separated by '|'.
+      #
+      # @example
+      #   recons = [
+      #       { as_recorded: 'Brown, Jamie', authorized_label: 'Jamie Brown' },
+      #       { as_recorded: 'Hendrix, Morgan', authorized_label: 'Morgan Hendrix' }
+      #   ]
+      #   build_term_string(recons, :as_recorded) # => 'Brown, Jamie|Hendrix, Morgan'
+      #
+      # @param recons [Array<Hash>] The array of recons hashes
+      # @param recon_key [String, Symbol] The key used to access the values in each recon hash
+      # @return [String] The concatenated term string.
+      def build_term_string recons, recon_key
+        recons.map { |recon| recon[recon_key.to_sym] }.join('|')
+      end
+
+      # Maps the given terms using the given recon type and column mapping.
+      #
+      # @param terms [Array<DS::Extractor::BaseTerm>] an array of terms to map
+      # @param recon_type [Recon::Type::ReconType] a recon type configuration
+      # @param column_map [Hash] a mapping of import CSV columns to recon keys
+      # @return [Hash<Symbol,String>] an hash of mapped terms; e.g., { :author_as_recorded => 'Brown, Jamie|Hendrix, Morgan', :author_label => 'Jamie Brown|Morgan Hendrix', ... } for an array of mapped terms
+      def map_terms terms, recon_type, column_map
+        recons = recon_builder.build_all_recons terms, recon_type
+        term_strings = build_term_strings recons, column_map
+        term_strings
+      end
+    end
+  end
+end

data/lib/ds/mapper/ds_csv_mapper.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module DS
+  module Mapper
+
+    class DSCSVMapper < DS::Mapper::BaseMapper
+
+
+      def initialize(source_dir:, timestamp:)
+        super(
+          source_dir: source_dir,
+          timestamp:  timestamp,
+          source:     DS::Source::DSCSV.new,
+        )
+      end
+
+      def map_record entry
+        record                             = extract_record entry
+        source_type                        = 'ds-csv'
+        source_file                        = entry.filename
+        ds_id                              = entry.ds_id
+        date_added                         = ''
+        date_last_updated                  = ''
+        dated                              = entry.dated?
+        cataloging_convention              = DS::Extractor::DsCsvExtractor.extract_cataloging_convention(record)
+        holding_institution_ds_qid         = entry.institution_ds_qid
+        holding_institution_as_recorded    = entry.institution_wikidata_label
+        holding_institution_id_number      = entry.institutional_id
+        holding_institution_shelfmark      = entry.call_number
+        link_to_holding_institution_record = entry.link_to_institutional_record
+        iiif_manifest                      = entry.iiif_manifest_url
+        production_date_as_recorded        = DS::Extractor::DsCsvExtractor.extract_production_date_as_recorded(record).join '|'
+        production_date                    = DS::Extractor::DsCsvExtractor.extract_date_range(record, range_sep: '^').join '|'
+        century                            = DS.transform_dates_to_centuries production_date
+        century_aat                        = DS.transform_centuries_to_aat century
+        physical_description               = DS::Extractor::DsCsvExtractor.extract_physical_description(record).join '|'
+        note                               = DS::Extractor::DsCsvExtractor.extract_notes(record).join '|'
+        data_processed_at                  = timestamp
+        data_source_modified               = entry.record_last_updated
+        acknowledgments                    = DS::Extractor::DsCsvExtractor.extract_acknowledgments(record).join '|'
+
+        {
+          ds_id:                              ds_id,
+          date_added:                         date_added,
+          date_last_updated:                  date_last_updated,
+          dated:                              dated,
+          source_type:                        source_type,
+          cataloging_convention:              cataloging_convention,
+          holding_institution_ds_qid:         holding_institution_ds_qid,
+          holding_institution_as_recorded:    holding_institution_as_recorded,
+          holding_institution_id_number:      holding_institution_id_number,
+          holding_institution_shelfmark:      holding_institution_shelfmark,
+          link_to_holding_institution_record: link_to_holding_institution_record,
+          iiif_manifest:                      iiif_manifest,
+          production_date:                    production_date,
+          century:                            century,
+          century_aat:                        century_aat,
+          production_date_as_recorded:        production_date_as_recorded,
+          physical_description:               physical_description,
+          note:                               note,
+          data_processed_at:                  data_processed_at,
+          data_source_modified:               data_source_modified,
+          source_file:                        source_file,
+          acknowledgments:                    acknowledgments
+        }.update build_term_maps DS::Extractor::DsCsvExtractor, record
+      end
+
+      def extract_record entry
+        locator = DS::Extractor::CsvRecordLocator.new
+        csv = source.load_source File.join(source_dir, entry.filename)
+        locator.locate_record(csv, entry.institutional_id, entry.institutional_id_location_in_source).first
+      end
+
+
+    end
+  end
+end