arclight 0.0.1 → 1.6.2

data/lib/arclight/traject/ead2_config.rb

@@ -0,0 +1,316 @@
+# frozen_string_literal: true
+
+require 'logger'
+require 'traject'
+require 'traject/nokogiri_reader'
+require 'traject_plus'
+require 'traject_plus/macros'
+require 'arclight/level_label'
+require 'arclight/normalized_id'
+require 'arclight/normalized_date'
+require 'arclight/normalized_title'
+require 'active_model/conversion' ## Needed for Arclight::Repository
+require 'active_support/core_ext/array/wrap'
+require 'arclight/digital_object'
+require 'arclight/year_range'
+require 'arclight/repository'
+require 'arclight/traject/nokogiri_namespaceless_reader'
+
+# rubocop:disable Style/MixinUsage
+extend TrajectPlus::Macros
+# rubocop:enable Style/MixinUsage
+
+NAME_ELEMENTS = %w[corpname famname name persname].freeze
+
+SEARCHABLE_NOTES_FIELDS = %w[
+  accessrestrict
+  accruals
+  altformavail
+  appraisal
+  arrangement
+  bibliography
+  bioghist
+  custodhist
+  fileplan
+  note
+  odd
+  originalsloc
+  otherfindaid
+  phystech
+  prefercite
+  processinfo
+  relatedmaterial
+  scopecontent
+  separatedmaterial
+  userestrict
+].freeze
+
+DID_SEARCHABLE_NOTES_FIELDS = %w[
+  abstract
+  materialspec
+  physloc
+  note
+].freeze
+
+settings do
+  provide 'component_traject_config', File.join(__dir__, 'ead2_component_config.rb')
+  provide 'id_normalizer', 'Arclight::NormalizedId'
+  provide 'date_normalizer', 'Arclight::NormalizedDate'
+  provide 'title_normalizer', 'Arclight::NormalizedTitle'
+  provide 'reader_class_name', 'Arclight::Traject::NokogiriNamespacelessReader'
+  provide 'solr_writer.commit_on_close', 'true'
+  provide 'repository', ENV.fetch('REPOSITORY_ID', nil)
+  provide 'logger', Logger.new($stderr)
+end
+
+each_record do |_record, context|
+  next unless settings['repository']
+
+  context.clipboard[:repository] = Arclight::Repository.find_by(
+    slug: settings['repository']
+  ).name
+end
+
+# ==================
+# Top level document
+#
+# NOTE: All fields should be stored in Solr
+# ==================
+
+to_field 'id' do |record, accumulator|
+  id = record.at_xpath('/ead/eadheader/eadid')&.text
+  unitid = record.at_xpath('/ead/archdesc/did/unitid')&.text
+  title = record.at_xpath('/ead/archdesc/did/unittitle')&.text
+  repository = settings['repository']
+  accumulator << settings['id_normalizer'].constantize.new(id, unitid: unitid, title: title, repository: repository).to_s
+end
+
+to_field 'title_filing_ssi', extract_xpath('/ead/eadheader/filedesc/titlestmt/titleproper[@type="filing"]')
+to_field 'title_ssm', extract_xpath('/ead/archdesc/did/unittitle')
+to_field 'title_tesim', extract_xpath('/ead/archdesc/did/unittitle')
+to_field 'ead_ssi', extract_xpath('/ead/eadheader/eadid')
+
+to_field 'unitdate_ssm', extract_xpath('/ead/archdesc/did/unitdate')
+to_field 'unitdate_bulk_ssim', extract_xpath('/ead/archdesc/did/unitdate[@type="bulk"]')
+to_field 'unitdate_inclusive_ssm', extract_xpath('/ead/archdesc/did/unitdate[@type="inclusive"]')
+to_field 'unitdate_other_ssim', extract_xpath('/ead/archdesc/did/unitdate[not(@type)]')
+
+# All top-level docs treated as 'collection' for routing / display purposes
+to_field 'level_ssm' do |_record, accumulator|
+  accumulator << 'collection'
+end
+
+# Keep the original top-level archdesc/@level for Level facet in addition to 'Collection'
+to_field 'level_ssim' do |record, accumulator|
+  level = record.at_xpath('/ead/archdesc').attribute('level')&.value
+  other_level = record.at_xpath('/ead/archdesc').attribute('otherlevel')&.value
+
+  accumulator << Arclight::LevelLabel.new(level, other_level).to_s
+  accumulator << 'Collection' unless level == 'collection'
+end
+
+to_field 'unitid_ssm', extract_xpath('/ead/archdesc/did/unitid')
+to_field 'unitid_tesim', extract_xpath('/ead/archdesc/did/unitid')
+
+to_field 'normalized_date_ssm' do |_record, accumulator, context|
+  accumulator << settings['date_normalizer'].constantize.new(
+    context.output_hash['unitdate_inclusive_ssm'],
+    context.output_hash['unitdate_bulk_ssim'],
+    context.output_hash['unitdate_other_ssim']
+  ).to_s
+end
+
+to_field 'normalized_title_ssm' do |_record, accumulator, context|
+  title = context.output_hash['title_ssm']&.first
+  date = context.output_hash['normalized_date_ssm']&.first
+  accumulator << settings['title_normalizer'].constantize.new(title, date).to_s
+end
+
+to_field 'collection_title_tesim' do |_record, accumulator, context|
+  accumulator.concat context.output_hash.fetch('normalized_title_ssm', [])
+end
+
+to_field 'collection_ssim' do |_record, accumulator, context|
+  accumulator.concat context.output_hash.fetch('normalized_title_ssm', [])
+end
+
+to_field 'repository_ssm' do |_record, accumulator, context|
+  accumulator << context.clipboard[:repository]
+end
+
+to_field 'repository_ssim' do |_record, accumulator, context|
+  accumulator << context.clipboard[:repository]
+end
+
+to_field 'geogname_ssm', extract_xpath('/ead/archdesc/controlaccess/geogname')
+to_field 'geogname_ssim', extract_xpath('/ead/archdesc/controlaccess/geogname')
+
+to_field 'creator_ssm', extract_xpath('/ead/archdesc/did/origination')
+to_field 'creator_ssim', extract_xpath('/ead/archdesc/did/origination')
+to_field 'creator_sort' do |record, accumulator|
+  accumulator << record.xpath('/ead/archdesc/did/origination').map { |c| c.text.strip }.join(', ')
+end
+
+to_field 'creator_persname_ssim', extract_xpath('/ead/archdesc/did/origination/persname')
+to_field 'creator_corpname_ssim', extract_xpath('/ead/archdesc/did/origination/corpname')
+to_field 'creator_famname_ssim', extract_xpath('/ead/archdesc/did/origination/famname')
+
+to_field 'creators_ssim' do |_record, accumulator, context|
+  accumulator.concat context.output_hash['creator_persname_ssim'] if context.output_hash['creator_persname_ssim']
+  accumulator.concat context.output_hash['creator_corpname_ssim'] if context.output_hash['creator_corpname_ssim']
+  accumulator.concat context.output_hash['creator_famname_ssim'] if context.output_hash['creator_famname_ssim']
+end
+
+to_field 'places_ssim', extract_xpath('/ead/archdesc/controlaccess/geogname')
+
+to_field 'access_terms_ssm', extract_xpath('/ead/archdesc/userestrict/*[local-name()!="head"]')
+
+to_field 'acqinfo_ssim', extract_xpath('/ead/archdesc/acqinfo/*[local-name()!="head"]')
+to_field 'acqinfo_ssim', extract_xpath('/ead/archdesc/descgrp/acqinfo/*[local-name()!="head"]')
+
+to_field 'access_subjects_ssim', extract_xpath('/ead/archdesc/controlaccess', to_text: false) do |_record, accumulator|
+  accumulator.map! do |element|
+    %w[subject function occupation genreform].map do |selector|
+      element.xpath(".//#{selector}").map(&:text)
+    end
+  end.flatten!
+end
+
+to_field 'access_subjects_ssm' do |_record, accumulator, context|
+  accumulator.concat Array.wrap(context.output_hash['access_subjects_ssim'])
+end
+
+to_field 'has_online_content_ssim', extract_xpath('.//dao') do |_record, accumulator|
+  accumulator.replace([accumulator.any?])
+end
+
+to_field 'digital_objects_ssm', extract_xpath('/ead/archdesc/did/dao|/ead/archdesc/dao', to_text: false) do |_record, accumulator|
+  accumulator.map! do |dao|
+    label = dao.attributes['title']&.value ||
+            dao.xpath('daodesc/p')&.text
+    href = (dao.attributes['href'] || dao.attributes['xlink:href'])&.value
+    Arclight::DigitalObject.new(label: label, href: href).to_json
+  end
+end
+
+# This accumulates direct text from a physdesc, ignoring child elements handled elsewhere
+to_field 'physdesc_tesim', extract_xpath('/ead/archdesc/did/physdesc', to_text: false) do |_record, accumulator|
+  accumulator.map! do |element|
+    physdesc = []
+    element.children.map do |child|
+      next if child.instance_of?(Nokogiri::XML::Element)
+
+      physdesc << child.text&.strip unless child.text&.strip&.empty?
+    end.flatten
+    physdesc.join(' ') unless physdesc.empty?
+  end
+end
+
+to_field 'extent_ssm' do |record, accumulator|
+  physdescs = record.xpath('/ead/archdesc/did/physdesc')
+  extents_per_physdesc = physdescs.map do |physdesc|
+    extents = physdesc.xpath('./extent').map { |e| e.text.strip }
+    # Join extents within the same physdesc with an empty string
+    extents.join(' ') unless extents.empty?
+  end
+
+  # Add each physdesc separately to the accumulator
+  accumulator.concat(extents_per_physdesc)
+end
+
+to_field 'extent_tesim' do |_record, accumulator, context|
+  accumulator.concat context.output_hash['extent_ssm'] || []
+end
+
+to_field 'physfacet_tesim', extract_xpath('/ead/archdesc/did/physdesc/physfacet')
+to_field 'dimensions_tesim', extract_xpath('/ead/archdesc/did/physdesc/dimensions')
+
+to_field 'genreform_ssim', extract_xpath('/ead/archdesc/controlaccess/genreform')
+
+to_field 'date_range_isim', extract_xpath('/ead/archdesc/did/unitdate/@normal', to_text: false) do |_record, accumulator|
+  range = Arclight::YearRange.new
+  next range.years if accumulator.blank?
+
+  ranges = accumulator.map(&:to_s)
+  range << range.parse_ranges(ranges)
+  accumulator.replace range.years
+end
+
+to_field 'indexes_html_tesm', extract_xpath('/ead/archdesc/index', to_text: false)
+to_field 'indexes_tesim', extract_xpath('/ead/archdesc/index')
+
+SEARCHABLE_NOTES_FIELDS.map do |selector|
+  to_field "#{selector}_html_tesm", extract_xpath("/ead/archdesc/#{selector}/*[local-name()!='head']", to_text: false) do |_record, accumulator|
+    accumulator.map!(&:to_html)
+  end
+  to_field "#{selector}_heading_ssm", extract_xpath("/ead/archdesc/#{selector}/head") unless selector == 'prefercite'
+  to_field "#{selector}_tesim", extract_xpath("/ead/archdesc/#{selector}/*[local-name()!='head']")
+end
+
+DID_SEARCHABLE_NOTES_FIELDS.map do |selector|
+  to_field "#{selector}_html_tesm", extract_xpath("/ead/archdesc/did/#{selector}", to_text: false)
+  to_field "#{selector}_tesim", extract_xpath("/ead/archdesc/did/#{selector}")
+end
+
+NAME_ELEMENTS.map do |selector|
+  to_field 'names_coll_ssim', extract_xpath("/ead/archdesc/controlaccess/#{selector}")
+  to_field 'names_ssim', extract_xpath("//#{selector}"), unique
+  to_field "#{selector}_ssim", extract_xpath("//#{selector}"), unique
+end
+
+to_field 'language_ssim', extract_xpath('/ead/archdesc/did/langmaterial')
+
+to_field 'descrules_ssm', extract_xpath('/ead/eadheader/profiledesc/descrules')
+
+# count all descendant components from the top-level
+to_field 'total_component_count_is', first_only do |record, accumulator|
+  accumulator << record.xpath('//c|//c01|//c02|//c03|//c04|//c05|//c06|//c07|//c08|//c09|//c10|//c11|//c12').count
+end
+
+# count all digital objects from the top-level
+to_field 'online_item_count_is', first_only do |record, accumulator|
+  accumulator << record.xpath('//dao').count
+end
+
+to_field 'component_level_isim' do |_record, accumulator, _context|
+  accumulator << 0
+end
+
+to_field 'sort_isi' do |_record, accumulator, _context|
+  accumulator << 0
+end
+
+# =============================
+# Each component child document
+# <c> <c01> <c12>
+# =============================
+
+to_field 'components' do |record, accumulator, context|
+  child_components = record.xpath("/ead/archdesc/dsc/c|#{('/ead/archdesc/dsc/c01'..'/ead/archdesc/dsc/c12').to_a.join('|')}")
+  next unless child_components.any?
+
+  counter = Class.new do
+    def increment
+      @counter ||= 0
+      @counter += 1
+    end
+  end.new
+
+  component_indexer = Traject::Indexer::NokogiriIndexer.new.tap do |i|
+    i.settings do
+      provide :parent, context
+      provide :root, context
+      provide :counter, counter
+      provide :depth, 1
+      provide :logger, context.settings[:logger]
+      provide :component_traject_config, context.settings[:component_traject_config]
+    end
+
+    i.load_config_file(context.settings[:component_traject_config])
+  end
+
+  child_components.each do |child_component|
+    output = component_indexer.map_record(child_component)
+    accumulator << output if output.keys.any?
+  end
+end

data/lib/arclight/traject/nokogiri_namespaceless_reader.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Arclight
+  module Traject
+    # Provides a Traject Reader for XML Documents which removes the namespaces
+    class NokogiriNamespacelessReader < ::Traject::NokogiriReader
+      # Overrides the #each method (which is used for iterating through each Document)
+      # @param args
+      # @see ::Traject::NokogiriReader#each
+      # @see Enumerable#each
+      def each(*args)
+        return to_enum(:each, *args) unless block_given?
+
+        super do |doc|
+          new_doc = doc.dup
+          new_doc.remove_namespaces!
+          yield new_doc
+        end
+      end
+    end
+  end
+end

data/lib/arclight/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Arclight
-  VERSION = '0.0.1'
+  VERSION = '1.6.2'
 end

data/lib/arclight/year_range.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+module Arclight
+  # A range of years that handles gaps, such as [1999, 2000, 2002].
+  # Primary usage is:
+  # ```
+  # range = YearRange.new('1999/2004')
+  # range.years => [1999, 2000, 2001, 2002, 2003, 2004]
+  # range.to_s => '1999-2004'
+  # range << range.parse_ranges(['2010/2010'])
+  # range.years => [1999, 2000, 2001, 2002, 2003, 2004, 2010]
+  # range.to_s => '1999-2004, 2010'
+  # ```
+  class YearRange
+    attr_accessor :years
+
+    # @param [Array<String>] `dates` in the form YYYY/YYYY
+    def initialize(dates = nil)
+      @years = []
+      self << parse_ranges(dates) if dates.present?
+    end
+
+    # @return [String] a concise, human-readable version of the year range, including gaps
+    def to_s
+      return if years.empty?
+      return to_s_for_streak(years) unless gaps?
+
+      to_s_with_gaps
+    end
+
+    # @param [Array<Integer>] `other` the set of years to add
+    def <<(other)
+      return self if other.blank?
+
+      @years |= other # will remove duplicates
+      @years.sort!
+      self
+    end
+
+    # @param [String] `dates` in the form YYYY/YYYY
+    # @return [Array<Integer>] the set of years in the given range
+    def parse_range(dates)
+      return if dates.blank?
+
+      start_year, end_year = dates.split('/').map { |date| to_year_from_iso8601(date) }
+      return [start_year] if end_year.blank?
+      raise ArgumentError, "Range is too large: #{dates}" if (end_year - start_year) > 1000
+      raise ArgumentError, "Range is inverted: #{dates}" unless start_year <= end_year
+
+      (start_year..end_year).to_a
+    end
+
+    # @param [Array<String>] `dates` in the form YYYY/YYYY
+    # @return [Array<Integer>] the set of years in the given range
+    def parse_ranges(dates)
+      dates.map { |date| parse_range(date) }.flatten.sort.uniq
+    end
+
+    # @param [String] `date` a date in one of these formats:
+    #                        YYYY, YYYY-MM, YYYY-MM-DD, and YYYYMMDD
+    def to_year_from_iso8601(date)
+      return if date.blank?
+
+      date.split('-').first[0..3].to_i # Time.parse doesn't work here
+    end
+
+    # @return [Boolean] are there gaps between the years, such as 1999, 2000, 2002?
+    def gaps?
+      return false if years.blank?
+
+      (years.min..years.max).to_a != years
+    end
+
+    private
+
+    # Deals with making a human-readable range for years with 1 or more gaps.
+    # It involves detection of streaks between the gaps.
+    # @return [String] 1999-2000, 2002 for 1999, 2000, 2002
+    # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+    def to_s_with_gaps
+      raise ArgumentError if years.blank? || years.length < 2
+
+      results = []
+      streak = [years[0]]
+      i = streak.first
+      years[1..].each do |j|
+        i += 1
+        if i == j
+          streak << j
+        else # we have a gap
+          results << if streak.length == 1
+                       streak.first.to_s
+                     else
+                       to_s_for_streak(streak)
+                     end
+          streak = [j]
+          i = j
+        end
+      end
+      results << to_s_for_streak(streak)
+      results.join(', ')
+    end
+    # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
+
+    def to_s_for_streak(streak)
+      return streak.min.to_s if streak.min == streak.max
+
+      streak.minmax.map(&:to_s).join('-')
+    end
+  end
+end

data/lib/arclight.rb

@@ -1,5 +1,15 @@
+# frozen_string_literal: true
+
 require 'arclight/version'
+require 'arclight/engine'
+require 'arclight/repository'
+require 'arclight/year_range'
 
+# :nodoc:
 module Arclight
-  # Your code goes here...
+  autoload :Routes, 'arclight/routes'
+
+  def self.deprecation
+    @deprecation ||= ActiveSupport::Deprecation.new('2.0', 'Arclight')
+  end
 end

data/lib/generators/arclight/install_generator.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+require 'rails/generators'
+
+module Arclight
+  ##
+  # Arclight install generator
+  class Install < Rails::Generators::Base # rubocop:disable Metrics/ClassLength
+    source_root File.expand_path('templates', __dir__)
+
+    class_option :test, type: :boolean, default: false, aliases: '-t', desc: 'Indicates that app will be installed in a test environment'
+
+    def create_blacklight_catalog
+      remove_file 'app/controllers/catalog_controller.rb'
+      copy_file 'catalog_controller.rb', 'app/controllers/catalog_controller.rb'
+    end
+
+    def include_arclight_solrdocument
+      inject_into_file 'app/models/solr_document.rb', after: 'include Blacklight::Solr::Document' do
+        "\n include Arclight::SolrDocument"
+      end
+    end
+
+    def install_blacklight_locale_picker
+      gem 'blacklight-locale_picker'
+
+      Bundler.with_unbundled_env do
+        run 'bundle install'
+      end
+
+      generate 'blacklight:locale_picker:install'
+
+      inject_into_file 'app/helpers/application_helper.rb', after: 'include Blacklight::LocalePicker::LocaleHelper' do
+        "\n\n  def additional_locale_routing_scopes\n    [blacklight, arclight_engine]\n  end"
+      end
+    end
+
+    def add_custom_routes
+      inject_into_file 'config/routes.rb', after: "mount Blacklight::Engine => '/'" do
+        "\n  mount Arclight::Engine => '/'\n"
+      end
+
+      gsub_file 'config/routes.rb', 'root to: "catalog#index"', 'root to: "arclight/repositories#index"'
+    end
+
+    def add_frontend
+      if ENV['CI']
+        run "yarn add file:#{Arclight::Engine.root}"
+      elsif options[:test]
+        run 'yarn link arclight'
+
+      # If a branch was specified (e.g. you are running a template.rb build
+      # against a test branch), use the latest version available on npm
+      elsif ENV['BRANCH']
+        run 'yarn add arclight@latest'
+
+      # Otherwise, pick the version from npm that matches the Arclight
+      # gem version
+      else
+        run "yarn add arclight@#{arclight_yarn_version}"
+      end
+    end
+
+    def add_stylesheets
+      copy_file 'arclight.scss', 'app/assets/stylesheets/arclight.scss'
+      append_to_file 'app/assets/stylesheets/application.bootstrap.scss' do
+        <<~CONTENT
+          @import "arclight";
+        CONTENT
+      end
+    end
+
+    def add_arclight_search_behavior
+      inject_into_file 'app/models/search_builder.rb', after: 'include Blacklight::Solr::SearchBuilderBehavior' do
+        "\n  include Arclight::SearchBehavior"
+      end
+    end
+
+    def solr_config
+      directory '../../../../solr', 'solr', force: true
+    end
+
+    def add_repository_config
+      copy_file 'config/repositories.yml' unless File.exist?('config/repositories.yml')
+    end
+
+    def add_download_config
+      copy_file 'config/downloads.yml' unless File.exist?('config/downloads.yml')
+    end
+
+    def add_i18n_config
+      copy_file 'config/locales/arclight.en.yml'
+    end
+
+    def modify_blacklight_yml
+      gsub_file 'config/locales/blacklight.en.yml', "application_name: 'Blacklight'", "application_name: 'Arclight'"
+    end
+
+    def assets
+      if using_importmap?
+        pin_javascript_dependencies
+        import_arclight_javascript
+      else
+        install_javascript_dependencies
+      end
+    end
+
+    def inject_arclight_routes
+      inject_into_file 'config/routes.rb',
+                       "  concern :hierarchy, Arclight::Routes::Hierarchy.new\n",
+                       after: /concern :exportable.*\n/
+      inject_into_file 'config/routes.rb',
+                       "  concerns :hierarchy\n",
+                       after: /resources :solr_documents.*\n/
+    end
+
+    private
+
+    def root
+      @root ||= Pathname(destination_root)
+    end
+
+    def using_importmap?
+      @using_importmap ||= root.join('config/importmap.rb').exist?
+    end
+
+    # This is the last step because any failure here wouldn't prevent the other steps from running
+    def install_javascript_dependencies
+      inject_into_file 'app/assets/javascripts/application.js', after: '//= require blacklight/blacklight' do
+        "\n// Required by Arclight" \
+          "\n//= require arclight/arclight"
+      end
+    end
+
+    def pin_javascript_dependencies
+      say 'Arclight Importmap asset generation'
+
+      append_to_file 'config/importmap.rb', <<~RUBY
+        pin "arclight", to: "arclight/arclight.js"
+        # TODO: We may be able to move these to a single importmap for arclight.
+        pin "arclight/oembed_controller", to: "arclight/oembed_controller.js"
+        pin "arclight/truncate_controller", to: "arclight/truncate_controller.js"
+      RUBY
+    end
+
+    def import_arclight_javascript
+      append_to_file 'app/javascript/application.js', "\nimport \"arclight\""
+    end
+
+    def package_yarn_version(package_name, requested_version)
+      versions = JSON.parse(`yarn info #{package_name} versions --json`)['data']
+      exact_match = versions.find { |v| v == requested_version }
+      return exact_match if exact_match
+
+      major_version = Gem::Version.new(requested_version).segments.first
+      "^#{major_version}"
+    end
+
+    def arclight_yarn_version
+      package_yarn_version('arclight', Arclight::VERSION)
+    end
+  end
+end

data/lib/generators/arclight/templates/arclight.scss

@@ -0,0 +1,6 @@
+:root {
+  --bs-breadcrumb-divider: "»";
+}
+
+// Arclight, imported from arclight frontend package
+@import 'arclight/app/assets/stylesheets/arclight/application';