ds-convert 0.1.1

data/exe/ds-recon

@@ -0,0 +1,275 @@
+#!/usr/bin/env ruby
+
+require 'thor'
+require 'csv'
+require_relative '../lib/ds/cli'
+
+class ReconCLI < DS::CLI
+
+  def self.exit_on_failure?
+    true
+  end
+
+  RECON_SETS = Settings.recon.sets.map { |set| set[:name] }.freeze
+  # We don't validate the aggregate subjects, named subjects set
+
+  class_option :directory, banner: 'PATH', desc: "Output directory [ignored by recon-update, validate]", aliases: '-o', default: '.'
+  class_option :'source-type',
+               banner: 'TYPE',
+               desc: "One of #{DS::VALID_SOURCE_TYPES.join(', ')}; REQUIRED except for recon-update, validate",
+               enum: DS::VALID_SOURCE_TYPES,
+               aliases: '-t',
+               required: (ARGV.size > 0 && ! %w{recon-update validate}.include?(ARGV[0]))
+  class_option :verbose, desc: "Print full error messages", aliases: '-v', type: :boolean, default: false
+
+  desc "validate [RECON_CSV]", "Validate RECON CSV structure and data"
+  long_desc <<~LONGDESC
+    Validate the recon CSV using set configuration 'config/recon.yml'
+
+    -t/-source-type option is NOT REQUIRED
+
+    ds-recon validate -s names path/to/names.csv
+  LONGDESC
+  option :'recon-set', banner: 'SET', desc: "One of #{Recon::RECON_VALIDATION_SETS.join ', '}; REQUIRED", aliases: '-s', required: true
+  def validate *files
+    set_name = options[:'recon-set']
+    recon_csvs = files.present? ? files : Recon.csv_files(set_name.to_sym)
+    errors = recon_csvs.flat_map { |csv| Recon.validate set_name, csv }.compact
+
+    if errors.empty?
+      puts "SUCCESS no errors found for: #{recon_csvs.join ', '}"
+    else
+      puts "ERRORS validating #{recon_csvs.join ', '}:#{$/}#{$/}#{errors.join $/}" unless errors.empty?
+    end
+
+    true
+  end
+
+
+  desc "names FILES", "Extract names from one or more FILEs and write to CSV"
+  long_desc <<-LONGDESC
+    Extract names from one or more FILEs.
+
+    Use '-' to read a list of files from standard input:
+
+      cat list_of_files | recon names -t mets -
+  LONGDESC
+  def names *files
+    # TODO: Add role column to names
+    return false unless validate_args files
+
+    out_csv = write_csv_for :names, files
+    puts "Wrote: #{out_csv}"
+
+    true
+  end
+
+  desc "places FILES", "Extract place names from one or more FILEs and write to CSV"
+  long_desc <<-LONGDESC
+    Extract place names from one or more FILEs.
+
+    Use '-' to read a list of files from standard input:
+
+      cat list_of_files | recon places -t mets -
+  LONGDESC
+  def places *files
+    return false unless validate_args files
+    invoke :recon_update
+
+    out_csv = write_csv_for :places, files
+    puts "Wrote: #{out_csv}"
+
+    true
+  end
+
+  desc "subjects FILES", "Extract subjects from one or more FILEs and write to CSV"
+  long_desc <<-LONGDESC
+  Extract LC subjects from one or more FILEs.
+
+  Use '-' to read a list of files from standard input:
+
+      cat list_of_files | recon subjects -t marc -
+
+  By default MARC fields 650, 651 are extracted, but fields 600, 610 are extracted if `--named-subjects` is given.
+
+  NOTE: Not implemented for mets or tei source types.
+
+LONGDESC
+  option :'named-subjects', desc: 'Extract named subjects', aliases: '-n', type: :boolean, default: false
+  def subjects *files
+
+    return false unless validate_args files
+    invoke :recon_update
+
+    out_csv = write_csv_for :subjects, files
+    puts "Wrote: #{out_csv}"
+
+    true
+  end
+
+  desc "genres FILES", "Extract genre terms from one or more FILEs and write to CSV"
+  long_desc <<-LONGDESC
+  Extract genre terms from one or more FILEs.
+
+  Use '-' to read a list of files from standard input:
+
+      cat list_of_files | recon genres -t marc -
+
+  NOTE: Not implemented for mets or tei source types.
+
+  LONGDESC
+  def genres *files
+    return false unless validate_args files
+    invoke :recon_update
+
+    out_csv = write_csv_for :genres, files
+    puts "Wrote: #{out_csv}"
+
+    true
+  end
+
+  desc "materials FILES", "Extract materials terms from one or more FILEs and write to CSV"
+  long_desc <<-LONGDESC
+  Extract materials terms from one or more FILEs.
+
+  Use '-' to read a list of files from standard input:
+
+      cat list_of_files | recon materials -t mets -
+  LONGDESC
+  def materials *files
+    return false unless validate_args files
+    invoke :recon_update
+
+    out_csv = write_csv_for :materials, files
+    puts "Wrote: #{out_csv}"
+
+    true
+  end
+
+  desc "languages FILES", "Extract languages from one or more FILEs and write to CSV"
+  long_desc <<-LONGDESC
+  Extract languages from one or more files.
+
+  Use '-' to read a list of files from standard input:
+
+      cat list_of_files | recon languages -t mets -
+  LONGDESC
+  def languages *files
+    return false unless validate_args files
+    invoke :recon_update
+
+    out_csv = write_csv_for :languages, files
+    puts "Wrote: #{out_csv}"
+
+    true
+  end
+
+  desc "titles FILES", "Extract titles from one or more FILEs and write to CSV"
+  long_desc <<-LONGDESC
+  Extract titles from one or more files.
+
+  Use '-' to read a list of files from standard input:
+
+      cat list_of_files | recon titles -t mets -
+  LONGDESC
+  def titles *files
+    return false unless validate_args files
+    invoke :recon_update
+
+    out_csv = write_csv_for :titles, files
+    puts "Wrote: #{out_csv}"
+
+    true
+  end
+
+  desc 'write-all FILES', 'Extract recons and write to CSV for all recon types'
+  def write_all *files
+    return false unless validate_args files
+    invoke :recon_update
+
+    recon_manager = build_recon_manager files
+    Recon::RECON_TYPES.each do |recon_type|
+      out_csv = recon_manager.write_csv recon_type
+      puts "Wrote: #{out_csv}"
+    end
+    true
+  end
+
+  desc "splits FILES", "Extract long lines"
+  long_desc <<-LONGDESC
+  Extract strings longer than 400 characters
+
+  Use '-' to read a list of files from standard input:
+
+      cat list_of_files | recon splits -t mets -
+  LONGDESC
+  def splits *files
+    return false unless validate_args files
+
+    out_csv = out_file 'splits', options
+    begin
+      invoke :recon_update
+
+      case options[:'source-type']
+      when 'marc'
+        data = Splits.from_marc select_input files
+      when 'mets'
+        data = Splits.from_mets select_input files
+      when 'tei'
+        data = Splits.from_tei select_input files
+      else
+        raise NotImplementedError, "No method to process splits for source type: '#{options[:source_type]}'"
+      end
+
+      validate! data unless options[:skip_validation]
+      write_csv out_csv, Splits::CSV_HEADERS, data
+
+    rescue NotImplementedError, StandardError
+      STDERR.puts $!.backtrace if options[:verbose]
+      abort "#{$!}"
+    end
+
+    true
+  end
+
+  protected
+
+  # Extract recon values from files for given recon type, and write to CSV.
+  #
+  # @param recon_type_name [String] the name of the recon type
+  # @param files [Array<String>] the list of files to process
+  # @return [String] the path of the output CSV file
+  def write_csv_for recon_type_name, files
+    build_recon_manager = build_recon_manager files
+    recon_type = Recon.find_recon_type recon_type_name
+    out_file = build_recon_manager.write_csv recon_type
+    out_file
+  end
+
+  # Builds a ReconManager object with the given files.
+  #
+  # @param files [Array<String>] the list of files to process
+  # @return [Recon::ReconManager] the ReconManager object
+  def build_recon_manager files
+    Recon::ReconManager.new(
+      source_type: options[:'source-type'],
+      out_dir: options[:directory],
+      files: files
+    )
+  end
+
+  # Ensures that the list of files is not empty and that they exist.
+  #
+  # @param files [Array<String>] the list of files to validate
+  # @return [Boolean] true if the arguments are valid, false otherwise
+  def validate_args files
+    return true if read_from_stdin? files
+    cannot_find = files.reject { |f| File.exist?(f) }
+    return true if cannot_find.empty?
+
+    puts "Can't find input file(s): #{cannot_find.join '; ' }"
+    false
+  end
+end
+
+ReconCLI.start ARGV

data/exe/ds-validate-csv

@@ -0,0 +1,40 @@
+#!/usr/bin/env ruby
+
+require 'csv'
+require 'optparse'
+
+require_relative '../lib/ds/csv_util'
+
+##
+# Check output CSV values for trailing whitespace.
+#
+# Trailing whitespace is not permitted in Wikibase values.
+parser = OptionParser.new do |opts|
+
+  opts.banner = <<EOF
+Usage: #{File.basename __FILE__} CSV [CSV...]
+
+Check output CSV values for trailing whitespace. 
+
+Trailing whitespace is not permitted in Wikibase values.
+
+EOF
+  help_help = 'Prints this help'
+  opts.on "-h", "--help", help_help do
+    puts opts
+    exit
+  end
+end
+
+parser.parse!
+
+abort 'Please provide a CSV' if ARGV.empty?
+
+ARGV.each do |csv|
+  rows     = CSV.readlines(csv, headers: true).map &:to_h
+  is_valid = DS::CSVUtil.validate rows
+
+  flash = is_valid ? 'SUCCESS!' :'ERROR! '
+  result = is_valid ? 'CSV is valid' : 'CSV is NOT valid'
+  puts "#{flash} #{result} -- '#{csv}'"
+end

data/exe/marc-mrc-to-xml.rb

@@ -0,0 +1,80 @@
+#!/usr/bin/env ruby
+
+######
+# Script to convert UPenn Marc XML to DS 2.0 CSV format.
+#
+
+
+require 'marc'
+require 'csv'
+require 'optionparser'
+require 'rexml'
+require_relative '../lib/ds'
+
+options = {}
+options[:directory] = '.'
+options[:prefix]    = ''
+options[:encoding]  = 'UTF-8'
+
+OptionParser.new do |opts|
+
+  opts.banner = <<EOF
+Usage: #{File.basename __FILE__} [options] MRC [MRC ..]
+
+Convert MARC MRC to MARC XML.
+
+EOF
+
+  p_help = "String prefix for file name; e.g., 'inst-marc-' [default: '#{options[:prefix]}']"
+  opts.on('-p PREFIX', '--prefix=PREFIX', p_help) do |prefix|
+    options[:prefix] = prefix
+  end
+
+  d_help = "Path of an directory to output XML files to [default: '#{options[:directory]}']"
+  opts.on('-d PATH', '--directory=PATH', d_help) do |directory|
+    options[:directory] = directory
+  end
+
+  e_help = "Encoding of the incoming MARC MRC/DAT file: [default: #{options[:encoding]}]"
+  opts.on('-e ENCODING', '--marc-encoding=ENCODING', e_help) do |encoding|
+    options[:encoding] = encoding
+  end
+
+  l_help = "List encodings available on this computer; WARNING: long list"
+  opts.on('-l', '--list-encodings', l_help) do
+    puts "Known encodings: "
+    puts Encoding.list.map { |enc| enc.names.join ', ' }
+    exit
+  end
+
+  h_help = <<~EOF
+Prints this help
+
+Note on encodings: Legacy MARC files can use MARC-8. If the default fails, 
+try that. To see a list of available encodings (a long list), use the -l option.
+EOF
+  opts.on("-h", "--help", h_help) do
+    puts opts
+    exit
+  end
+end.parse!
+
+marc_mrc = ARGV.dup
+
+abort 'Please provide an input MRC' if marc_mrc.empty?
+cannot_find = marc_mrc.reject { |f| File.exist?(f) }
+abort "Can't find input MRC: #{cannot_find.join '; ' }" unless cannot_find.empty?
+abort "Cannot find output path: #{options[:directory]}" unless File.directory? options[:directory]
+
+marc_mrc.each do |mrc|
+  reader = MARC::Reader.new(mrc, :external_encoding => "UTF-8")
+  reader.each_with_index do |record,ndx|
+    base     = sprintf '%s%03d.xml', options[:prefix], ndx
+    marc_out = File.join options[:directory], base
+    # Use REXML::Document to format and preserve whitespace in outputs
+    doc = REXML::Document.new(record.to_xml.to_s)
+    # transitive: true prevents the formatter compressing whitespace
+    doc.write(output: File.open(marc_out, 'w+'), indent: 2, transitive: true)
+    puts "Wrote #{marc_out}"
+  end
+end

data/lib/ds/cli.rb

@@ -0,0 +1,102 @@
+require 'thor'
+require 'colorize'
+require_relative '../ds'
+
+module DS
+  class CLI < Thor
+    include Recon
+    include ActiveSupport::NumberHelper
+
+    DS.env = (ENV['DS_ENV'].present? ? ENV['DS_ENV'] : 'production')
+    DS.configure!
+
+    class_option :'skip-recon-update', desc: "Skip CSV update from git [ignored by recon-update, validate]", aliases: '-G', type: :boolean, default: false
+    class_option :'skip-validation', desc: "Skip validation of CSV values [same as SKIP_OUTPUT_VALIDATION=true, ignored by recon-update, validate]", aliases: '-V', type: :boolean, default: false
+    class_option :verbose, desc: "Be chatty; print stacktraces; overrides --quiet", aliases: '-v', type: :boolean, default: false
+    class_option :quiet, desc: "Don't print messages", aliases: '-q', type: :boolean, default: false
+
+
+    desc "recon-update", "Update Recon CSVs from git"
+    long_desc <<-LONGDESC
+    Update Recon CSVs from #{Settings.recon.git_repo}.
+
+    NOTE: This command ignores all options, including `--skip-recon-update`; set
+          the SKIP_RECON_UPDATE environment variable to override.
+
+    LONGDESC
+    def recon_update(*args)
+      # allow any args so this command can be invoked by any other
+      if skip_git? options
+        print_message(options, verbose_only: true) { <<~EOF.squish }
+          WARNING: SKIP_RECON_UPDATE or 
+          --skip-recon-update set; skipping git pull
+        EOF
+        return
+      end
+      STDOUT.print "Updating Recon CSVs from #{Settings.recon.git_repo}..."
+      Recon::ReconData.update!
+      STDOUT.puts "done."
+    end
+
+    ##
+    # Needed to return a non-zero exit code on failure. See:
+    #
+    # https://github.com/rails/thor/wiki/Making-An-Executable
+    def self.exit_on_failure?
+      true
+    end
+
+    protected
+    def skip_git? options
+      return true if options[:'skip-recon-update']
+      return true if ENV['SKIP_RECON_UPDATE']
+      false
+    end
+
+    ##
+    # See if the user has signaled input is coming from STDIN
+    #
+    # @param files [Enumerable<String>] the file list from ARGV
+    #     (by way of Thor)
+    # @return [Boolean]
+    def read_from_stdin? files
+      files == ['-']
+    end
+
+    ##
+    # @param [String] msg the message to print
+    # @param [Hash] options the command options; the following are used here:
+    # @option options [Boolean] :verbose whether to print all messages
+    # @option option [Boolean] :quiet suppress all messages (except errors); overrides +:verbose+
+    # @param [Boolean] verbose_only print message only if +:verbose+ is true
+    def print_message options, verbose_only: false, &msg
+      return if options[:quiet]
+      # if +verbose_only+ is true, return unless +options[:verbose]+ is true
+      return if verbose_only && ! options[:verbose]
+
+      puts yield
+    end
+
+    ##
+    # Return the input to read from based on whether input is stdin.
+    # If `read_from_stdin?` returns true, return +ARGF+; otherwise,
+    # return +files+.
+    #
+    # @param files [Enumerable<String>] the file list from ARGV
+    #     (by way of Thor)
+    # @return [Enumerable] +files+ or +ARGF+
+    def select_input files
+      return files unless read_from_stdin? files
+      ARGV.clear
+      ARGF
+    end
+
+    def validate! rows
+      return if options[:'skip-validation']
+      return if ENV['SKIP_OUTPUT_VALIDATION']
+      return if CSVUtil.validate rows
+
+      raise StandardError, "Validation errors found in output"
+    end
+  end
+end

data/lib/ds/constants.rb

@@ -0,0 +1,166 @@
+module DS
+  module Constants
+    HEADINGS = %i{
+ds_id
+date_added
+date_last_updated
+source_type
+cataloging_convention
+holding_institution_ds_qid
+holding_institution_as_recorded
+holding_institution_id_number
+holding_institution_shelfmark
+link_to_holding_institution_record
+iiif_manifest
+production_place_as_recorded
+production_place_ds_qid
+production_date_as_recorded
+production_date
+century
+century_aat
+dated
+title_as_recorded
+title_as_recorded_agr
+uniform_title_as_recorded
+uniform_title_agr
+standard_title_ds_qid
+genre_as_recorded
+genre_ds_qid
+subject_as_recorded
+subject_ds_qid
+author_as_recorded
+author_as_recorded_agr
+author_ds_qid
+artist_as_recorded
+artist_as_recorded_agr
+artist_ds_qid
+scribe_as_recorded
+scribe_as_recorded_agr
+scribe_ds_qid
+associated_agent_as_recorded
+associated_agent_as_recorded_agr
+associated_agent_ds_qid
+former_owner_as_recorded
+former_owner_as_recorded_agr
+former_owner_ds_qid
+language_as_recorded
+language_ds_qid
+material_as_recorded
+material_ds_qid
+physical_description
+note
+acknowledgments
+data_processed_at
+data_source_modified
+source_file
+}
+
+
+    NESTED_COLUMNS = %i{ subject subject_label genre genre_label production_place production_place_label language language_label }
+    # Institutions dependent on DS and their DS IDs
+    # Some institutions have more than one collection
+    #
+    # conception    15
+    # csl           12, 9
+    # cuny           5
+    # grolier       24
+    # gts           23
+    # indiana       40
+    # kansas        30
+    # nelsonatkins  46
+    # nyu           25
+    # providence    28
+    # rutgers        6
+    # ucb            1, 8, 11
+    # wellesley     50
+
+
+
+    INSTITUTION_DS_IDS = {
+      1  => 'ucb',
+      2  => 'harvard',
+      3  => 'fordham',
+      4  => 'freelib',
+      5  => 'cuny',
+      6  => 'rutgers',
+      7  => 'ucd',
+      8  => 'ucb',
+      9  => 'csl',
+      10 => 'ucr',
+      11 => 'ucb',
+      12 => 'csl',
+      13 => 'sfu',
+      14 => 'notredame',
+      15 => 'conception',
+      16 => 'columbia',
+      17 => 'columbia',
+      18 => 'columbia',
+      19 => 'columbia',
+      20 => 'columbia',
+      21 => 'columbia',
+      22 => 'columbia',
+      23 => 'gts',
+      24 => 'grolier',
+      25 => 'nyu',
+      26 => 'oberlin',
+      27 => 'penn',
+      28 => 'providence',
+      29 => 'rome',
+      30 => 'kansas',
+      31 => 'jhopkins',
+      32 => 'jhopkins',
+      33 => 'jhopkins',
+      34 => 'jhopkins',
+      35 => 'walters',
+      36 => 'pittsburgh',
+      37 => 'txaustin',
+      38 => 'uvm',
+      39 => 'jtsa',
+      40 => 'indiana',
+      41 => 'nypl',
+      42 => 'nypl',
+      43 => 'huntington',
+      44 => 'slu',
+      45 => 'missouri',
+      46 => 'nelsonatkins',
+      47 => 'beinecke',
+      48 => 'smith',
+      50 => 'wellesley',
+      52 => 'tufts'
+    }.freeze
+
+    TRAILING_PUNCTUATION_RE = %r{[,.:!?;[:space:]]+$}
+
+    MAX_WIKIBASE_FIELD_LENGTH = 400
+
+    INSTITUTIONS = INSTITUTION_DS_IDS.values.uniq.freeze
+
+    MARC_XML = 'marc-xml'
+    TEI_XML  = 'tei-xml'
+    DS_CSV   = 'ds-csv'
+    DS_METS  = 'ds-mets-xml'
+
+    # source type list of all type names and normalized names; i.e.,
+    # lower case names stripped of all whitespace and non-word characters
+    VALID_SOURCE_TYPES = [
+      MARC_XML,
+      TEI_XML,
+      DS_CSV,
+      DS_METS
+    ].freeze
+
+    XML_NAMESPACES = {
+      marc:  'http://www.loc.gov/MARC21/slim',
+      mets:  'http://www.loc.gov/METS/',
+      mods:  'http://www.loc.gov/mods/v3',
+      rts:   'http://cosimo.stanford.edu/sdr/metsrights/',
+      mix:   'http://www.loc.gov/mix/v10',
+      xlink: 'http://www.w3.org/1999/xlink',
+      xsi:   'http://www.w3.org/2001/XMLSchema-instance',
+      xs:    'http://www.w3.org/2001/XMLSchema',
+      xd:    'http://www.oxygenxml.com/ns/doc/xsl',
+      tei:   'http://www.tei-c.org/ns/1.0'
+    }
+
+  end
+end