ds-convert 0.1.1

data/lib/ds/extractor/title.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module DS
+  module Extractor
+    class Title < BaseTerm
+      attr_accessor :vernacular
+      attr_accessor :title_type
+      attr_accessor :uniform_title
+      attr_accessor :uniform_title_vernacular
+
+      # Initializes a new Title object.
+      #
+      # Parameters:
+      # - as_recorded: the title as recorded
+      # - vernacular: the vernacular title (default is nil)
+      # - uniform_title: the uniform title (default is nil)
+      # - uniform_title_vernacular: the vernacular uniform title (default is nil)
+      #
+      # Returns:
+      # - A new Title object
+      def initialize as_recorded:, vernacular: nil, uniform_title: nil, uniform_title_vernacular: nil
+        @vernacular               = vernacular
+        @uniform_title            = uniform_title
+        @uniform_title_vernacular = uniform_title_vernacular
+        super(as_recorded: as_recorded)
+      end
+
+      # Returns an array containing the title as recorded, vernacular title, uniform title, and vernacular uniform title.
+      #
+      # @return [Array] the title as an array
+      def to_a
+        # title_type is not included
+        [as_recorded, vernacular, uniform_title, uniform_title_vernacular]
+      end
+
+      # Returns a hash representation of the title object.
+      #
+      # Keys are :as_recorded, :title_as_recorded_agr, :uniform_title_as_recorded, :uniform_title_as_recorded_agr
+      #
+      # @return [Hash] the title as a hash
+      def to_h
+        {
+          title_as_recorded: as_recorded,
+          as_recorded: as_recorded,
+          title_as_recorded_agr: vernacular,
+          uniform_title_as_recorded: uniform_title,
+          uniform_title_as_recorded_agr: uniform_title_vernacular
+        }
+      end
+    end
+  end
+end

data/lib/ds/extractor/xml_record_locator.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module DS
+  module Extractor
+    class XmlRecordLocator < DS::Extractor::BaseRecordLocator
+
+      attr_accessor :namespaces
+
+      def initialize namespaces: DS::Constants::XML_NAMESPACES
+        @namespaces = namespaces
+        super()
+      end
+
+      def locate_record xml, id, id_location
+        xpath = id_location.gsub(/ID_PLACEHOLDER/, id)
+        # try with namespaces
+        record = try_locate_record xml, xpath, namespaces: namespaces
+        return record if record.present?
+
+        # try without providing namespaces
+        record = try_locate_record xml, xpath
+        return record if record.present?
+
+        # strip namespaces and try one last time
+        xml.remove_namespaces!
+        try_locate_record xml, xpath
+      end
+
+      def try_locate_record xml, xpath, namespaces: nil
+        xml.xpath xpath, namespaces
+      rescue Nokogiri::XML::XPath::SyntaxError => e
+        add_error e.message
+        raise unless e.message =~ /undefined namespace prefix/i
+        []
+      end
+    end
+  end
+end

data/lib/ds/extractor.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require_relative 'extractor/base_term'
+require_relative 'extractor/genre'
+require_relative 'extractor/material'
+require_relative 'extractor/name'
+require_relative 'extractor/place'
+require_relative 'extractor/subject'
+require_relative 'extractor/title'
+require_relative 'extractor/language'
+require_relative 'extractor/base_record_locator'
+require_relative 'extractor/xml_record_locator'
+require_relative 'extractor/csv_record_locator'
+
+module DS
+  # Module for DS Extractor classes, which are responsible for extracting
+  # import CSV rows from source records.
+  #
+  # Extractors are used by {DS::Mapper::BaseMapper} instances to extract
+  # data from a source records and by {Recon::ReconBuilder} instances
+  # to extract data from DS data sources for recon CSVs.
+  module Extractor
+  end
+end

data/lib/ds/institutions.rb

@@ -0,0 +1,55 @@
+module DS
+  ##
+  # Class for access configure institutions. Values from
+  # `config/institutions.yml`
+  #
+  # File contents look like:
+  #
+  #     ---
+  #     institutions:
+  #       Q814779:
+  #         - Beinecke Rare Book & Manuscript Library
+  #         - beinecke
+  #       Q995265:
+  #         - Bryn Mawr College
+  #         - brynmawr
+  #
+  # +DS.configure!+ must be invoked before this class is accessed.
+  module Institutions
+    @@names_to_qids = nil
+
+    ##
+    # Return the contents of `config/institutions.yml' as hash with the
+    # institution names as keys and the Wikidata QIDs as values.
+    #
+    # @return [Hash]
+    def self.names_to_qids
+      @@names_to_qids ||= Settings.institutions.inject({}) do |h, qid_names|
+        qid = qid_names.first.to_s
+        qid_names.last.inject(h) { |j, name| j.merge(name => qid) }
+      end
+    end
+
+    ##
+    # Return the QID for the give institution name/alias.
+    #
+    # @param [String] inst_alias a name of the institution
+    # @return [String] the institution Wikidata QID
+    def self.find_qid inst_alias
+      # try without changes; and then normalize
+      names_to_qids[inst_alias] or
+        names_to_qids[inst_alias.to_s.strip] or
+        names_to_qids[inst_alias.to_s.strip.downcase]
+    end
+
+    ##
+    # Return the preferred name of the institution for the given alias.
+    #
+    # @param [String] inst_alias a name of the institution
+    # @return [String] the first list name of the institution
+    def self.preferred_name inst_alias
+      qid = find_qid inst_alias
+      Settings.institutions[qid.to_sym].first
+    end
+  end
+end

data/lib/ds/manifest/base_id_validator.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module DS
+  module Manifest
+    ##
+    # A {DS::Manifest::BaseIdValidator} is a base class for a
+    # cacheable ID validator for sources. The validator is responsible
+    # for opening and caching source files and dtermining that one
+    # record is found for each source +id+ at the specified
+    # +id_location+ in the parsed source.
+    #
+    # The motivation for this class is to handle ID validation for
+    # source types that can have multiple records per source file,
+    # saving the time required to parse the source file for each check.
+    #
+    # Concrete subclasses of {DS::Manifest::BaseIdValidator} must implement
+    #
+    #   - +#locate_record+, required this class
+    #
+    class BaseIdValidator
+
+      attr_reader :errors
+      attr_reader :source
+
+      ##
+      # Create a new ID Validator
+      #
+      # @param source [DS::Source::BaseSource] the source to validate
+      # @return [void]
+      def initialize source
+        @source = source
+        @errors = []
+      end
+
+      # Checks if the given file path, id, and id location are valid.
+      #
+      # @param file_path [String] The path to the file.
+      # @param id [String] The id to check.
+      # @param id_location [String] The location of the id.
+      # @return [Boolean] Returns true if the records size is equal to 1, false otherwise.
+      def valid? file_path, id, id_location
+        records = locate_record file_path, id, id_location
+
+        return true if records.size == 1
+        handle_count_error records.size, id, id_location
+        false
+      end
+
+      # Locates a record based on the given source path, ID, and ID location.
+      #
+      # @param source_path [String] the path to the source file
+      # @param id [String] the ID of the record
+      # @param id_location [String] the location of the ID within the record
+      # @raise [NotImplementedError] this method is not implemented and should be overridden
+      # @return [Array<Object>] an array of objects for each record
+      def locate_record source_path, id, id_location
+        raise NotImplementedError
+      end
+
+      def handle_count_error count, inst_id, location_in_source
+        return if count == 1
+
+        if count > 1
+          add_error "ERROR: Multiple records (#{count}) found for id: #{inst_id} (location: #{location_in_source})"
+        elsif count == 0
+          add_error "ERROR: No records found for id: #{inst_id} (location: #{location_in_source})"
+        end
+        nil
+      end
+
+      def add_error message
+        @errors << message
+      end
+    end
+  end
+end

data/lib/ds/manifest/constants.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+module DS
+  module Manifest
+    module Constants
+      include DS
+
+      INSTITUTION_DS_QID = 'holding_institution_ds_qid'
+      FILENAME                            = 'filename'
+      INSTITUTION_WIKIDATA_LABEL          = 'holding_institution_wikidata_label'
+      SOURCE_TYPE                         = 'source_data_type'
+      DS_ID                               = 'ds_id'
+      DATED                               = 'dated'
+      INSTITUTIONAL_ID                    = 'holding_institution_institutional_id'
+      INSTITUTIONAL_ID_LOCATION_IN_SOURCE = 'institutional_id_location_in_source'
+      RECORD_LAST_UPDATED                 = 'record_last_updated'
+      CALL_NUMBER                         = 'call_number'
+      TITLE                               = 'title'
+      IIIF_MANIFEST_URL                   = 'iiif_manifest_url'
+      LINK_TO_INSTITUTIONAL_RECORD        = 'link_to_institutional_record'
+      MANIFEST_GENERATED_AT               = 'manifest_generated_at'
+
+      MANIFEST_COLUMNS = [
+        INSTITUTION_DS_QID,
+        INSTITUTION_WIKIDATA_LABEL,
+        FILENAME,
+        SOURCE_TYPE,
+        DS_ID,
+        DATED,
+        INSTITUTIONAL_ID,
+        INSTITUTIONAL_ID_LOCATION_IN_SOURCE,
+        RECORD_LAST_UPDATED,
+        CALL_NUMBER,
+        TITLE,
+        IIIF_MANIFEST_URL,
+        LINK_TO_INSTITUTIONAL_RECORD,
+        MANIFEST_GENERATED_AT
+      ].freeze
+
+      REQUIRED_VALUES = [
+        INSTITUTION_DS_QID,
+        FILENAME,
+        INSTITUTION_WIKIDATA_LABEL,
+        SOURCE_TYPE,
+        INSTITUTIONAL_ID,
+        INSTITUTIONAL_ID_LOCATION_IN_SOURCE,
+        RECORD_LAST_UPDATED,
+        CALL_NUMBER,
+        MANIFEST_GENERATED_AT
+      ].freeze
+
+      URI_COLUMNS = [
+        LINK_TO_INSTITUTIONAL_RECORD,
+        IIIF_MANIFEST_URL
+      ].freeze
+
+      QID_COLUMNS = [
+        INSTITUTION_DS_QID
+      ].freeze
+
+      DATE_TIME_COLUMNS = [
+        RECORD_LAST_UPDATED,
+        MANIFEST_GENERATED_AT
+      ].freeze
+
+    end
+  end
+end

data/lib/ds/manifest/ds_csv_id_validator.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module DS
+  module Manifest
+    class DsCsvIdValidator < BaseIdValidator
+
+      def locate_record source_path, id, id_location
+        locator = DS::Extractor::CsvRecordLocator.new
+        csv = source.load_source source_path
+        csv.rewind
+        locator.locate_record csv, id, id_location
+      end
+    end
+  end
+end

data/lib/ds/manifest/entry.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+require_relative './constants'
+module DS
+  module Manifest
+    ##
+    # The manifest Entry provides information to validate delivered data
+    # and to drive the data extraction process. Specifically, each
+    # line of the manifest:
+    #
+    # 1. Provides information often not present in standard a location
+    #    in the source record, like shelfmark, source type (MARC XML,
+    #    TEI XML, etc.), link to a IIIF manifest, and link to the
+    #    institution's record in an OPAC or on the institution's
+    #    website
+    #
+    # 2. Gives the file name for the record present in the delivered
+    #    set of records
+    #
+    # 3. Provides information needed to validate the source record:
+    #    its presence in the delivered data, correspondence of the
+    #    source file(s) to identifying information, etc.)
+    #
+    #
+    class Entry
+      include DS::Manifest::Constants
+
+      attr_reader :row
+      attr_reader :manifest
+      ##
+      # @param [CSV::Row] row a manifest CSV row
+      # @param [DS::Manifest::Manifest] manifest the parent manifest
+      def initialize row, manifest
+        @row = row
+        @manifest = manifest
+      end
+
+      def [] key
+        row[key]
+      end
+
+      def institution_ds_qid
+        row[INSTITUTION_DS_QID]
+      end
+
+      # FILENAME                            = 'filename'
+      def filename
+        row[FILENAME]
+      end
+
+      def institution_wikidata_label
+        row[INSTITUTION_WIKIDATA_LABEL]
+      end
+
+      def source_type
+        row[SOURCE_TYPE]
+      end
+      # DS_ID                               = 'ds_id'
+      def ds_id
+        row[DS_ID]
+      end
+
+      # DATED                               = 'dated'
+      def dated
+        row[DATED]
+      end
+      # INSTITUTIONAL_ID                    = 'holding_institution_institutional_id'
+      def institutional_id
+        row[INSTITUTIONAL_ID]
+      end
+
+      # INSTITUTIONAL_ID_LOCATION_IN_SOURCE = 'institutional_id_location_in_source'
+      def institutional_id_location_in_source
+        row[INSTITUTIONAL_ID_LOCATION_IN_SOURCE]
+      end
+      # RECORD_LAST_UPDATED                 = 'record_last_updated'
+      def record_last_updated
+        row[RECORD_LAST_UPDATED]
+      end
+      #
+      # CALL_NUMBER                         = 'call_number'
+      def call_number
+        row[CALL_NUMBER]
+      end
+      # TITLE                               = 'title'
+      def title
+        row[TITLE]
+      end
+
+      # IIIF_MANIFEST_URL                   = 'iiif_manifest_url'
+      def iiif_manifest_url
+        return '' unless row[IIIF_MANIFEST_URL]
+        # there may be multiple manifests; split & join with pipes
+        row[IIIF_MANIFEST_URL].split(/[ ;|]+/).join('|')
+      end
+      # LINK_TO_INSTITUTIONAL_RECORD        = 'link_to_institutional_record'
+      def link_to_institutional_record
+        row[LINK_TO_INSTITUTIONAL_RECORD]
+      end
+      # MANIFEST_GENERATED_AT               = 'manifest_generated_at'
+      def manifest_generated_at
+        row[MANIFEST_GENERATED_AT]
+      end
+
+      def manifest_path
+        manifest.present? && manifest.path
+      end
+
+      def dated?
+        dated.to_s.strip.downcase == 'true'
+      end
+
+      def to_h
+        {
+          institution_ds_qid:           institution_ds_qid,
+          institution_wikidata_label:   institution_wikidata_label,
+          ds_id:                        ds_id,
+          call_number:                  call_number,
+          institutional_id:             institutional_id,
+          title:                        title,
+          link_to_institutional_record: link_to_institutional_record,
+          iiif_manifest_url:            iiif_manifest_url,
+          record_last_updated:          record_last_updated,
+          source_type:                  source_type,
+          filename:                     filename,
+          dated:                        dated?,
+          manifest_generated_at:        manifest_generated_at,
+        }
+      end
+
+    end
+  end
+end

data/lib/ds/manifest/manifest.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require 'csv'
+
+module DS
+  module Manifest
+    ##
+    # A Manifest is a collection of {DS::Manifest::Entry} instances.
+    #
+    # This class loads a manifest CSV and provides an {#each} method
+    # to iterate over the entries in the manifest.
+    #
+    class Manifest
+      include Enumerable
+
+      attr_reader :csv_path
+      attr_reader :source_dir
+
+      ##
+      # Create a new Manifest instance. If +dir+ is not provided,
+      # directory containing source files must the same as the
+      # +manifest_csv+ directory.
+      #
+      # @param [String] csv_path manifest CSV path
+      # @param [String] dir optional path to the directory containing the
+      #   source file(s); if
+      # @return [DS::Manifest::Manifest] a new Manifest instance
+      def initialize csv_path, dir = nil
+        @csv_path = csv_path
+        @source_dir = get_source_dir dir
+      end
+
+      ##
+      # The headers from the parsed Manifest CSV.
+      # @return [Array<String>]
+      def headers
+        csv.first.headers
+      end
+
+      ##
+      # @yield [DS::Manifest::Entry] entry representation of the manifest row
+      def each &block
+        csv.each do |row|
+          yield DS::Manifest::Entry.new row, self
+        end
+      end
+
+      ##
+      # Return the String path of the directory expected to contain
+      # the source records. If +dir+ is present, return
+      # +dir+; otherwise, return the directory of the manifest
+      # CSV.
+      #
+      # @param [String] dir a source directory path or +nil+
+      # @return [String] the directory containing source files
+      def get_source_dir dir
+        return dir if dir.present?
+        File.dirname csv.path
+      end
+
+      ##
+      # Return a CSV::Table for +manifest_csv+. Determine +manifest_csv+
+      # type and return the value (if a CSV::Table) or return the parsed
+      # value as appropriate.
+      #
+      # @return [CSV::Table] the parsed manifest
+      def csv
+        @csv ||= CSV.open csv_path, 'r', headers: true
+        @csv.rewind
+        @csv
+      end
+    end
+  end
+end