digital_scriptorium 0.1.2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 26bfe7a6e13db8a90e6d3fb40f2c134383d954503e4d81d7eeff75ae7f0e83ab
-  data.tar.gz: 11f3be8b35b9279d1d5b217d277c8117a2b2b00b5b52190f6a978ee734a9e275
+  metadata.gz: 90d97605a47a87aec5fb8dc41385f9ce55e415d8d96bd51bc91f0436b8d5ec07
+  data.tar.gz: 873ab9fbb3d1fb275419ec753c36929f1a806fe40a9023aaa3279b2684871218
 SHA512:
-  metadata.gz: d00ac9d13e72ce1213158ccd6d0aa063c620f8a58637b706d1869b41e63641c687b16f1cdc5d3508c253406b4dd08d9bf714ac52af15e139cc560f9ecac92f65
-  data.tar.gz: 02d25955d26867e0b2560df2118f2bc634822a8c2a62a9cf6adb3c49bf48a189515add16fc1e25e27f462459b49a033ee1f0a16413c496fa186f5dd644966a5f
+  metadata.gz: 0142d571dd96cd21270a782c0327bf0798af63e00d904eb71263d7a43fb8eb69e4fbe2854ad75dcf4b4bed7b56148e81a4a0be1862bff0e76168572126b94532
+  data.tar.gz: e51efd7f188fcc8bc29e846d50b6fceeb6a61ec00ccb1217807ed124303060290a6f6890980b25ab54adae11283896319af3ba0af63b6ca1f4d890a7bf2243e4

data/.rubocop.yml

@@ -1,30 +1,4 @@
-require:
-  - rubocop-rake
-  - rubocop-rspec
+inherit_from: .rubocop_todo.yml
 
-AllCops:
-  TargetRubyVersion: 3.0
-  NewCops: enable
-Layout/LineLength:
-  Exclude:
-    - spec/**/*
-Metrics/AbcSize:
-  Enabled: false
-Metrics/BlockLength:
-  Enabled: false
-Metrics/CyclomaticComplexity:
-  Enabled: false
-Metrics/MethodLength:
-  Enabled: false
-Metrics/ModuleLength:
-  Enabled: false
-Metrics/PerceivedComplexity:
-  Enabled: false
-RSpec/ExampleLength:
-  Enabled: false
-RSpec/MultipleExpectations:
-  Enabled: false
-RSpec/MultipleMemoizedHelpers:
-  Enabled: false
-Style/SafeNavigationChainLength:
-  Enabled: false
+inherit_gem:
+  upennlib-rubocop: upennlib_rubocop_defaults.yml

data/.rubocop_todo.yml

@@ -0,0 +1,12 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2025-01-17 01:39:58 UTC using RuboCop version 1.70.0.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 2
+# Configuration parameters: CountComments, CountAsOne.
+Metrics/ModuleLength:
+  Max: 137

data/lib/digital_scriptorium/ds_item.rb

@@ -10,19 +10,23 @@ module DigitalScriptorium
     end
 
     def ds_id
-      claim_by_property_id(PropertyId::DS_ID)&.data_value # P1
+      claims_by_property_id(PropertyId::DS_ID)&.first&.data_value # P1
     end
 
-    def holding_id
-      claim_by_property_id(PropertyId::MANUSCRIPT_HOLDING)&.entity_id_value # P2
+    def holding_ids
+      claims_by_property_id(PropertyId::MANUSCRIPT_HOLDING)&.map(&:entity_id_value) # P2
     end
 
     def described_manuscript_id
-      claim_by_property_id(PropertyId::DESCRIBED_MANUSCRIPT)&.entity_id_value # P3
+      claims_by_property_id(PropertyId::DESCRIBED_MANUSCRIPT)&.first&.entity_id_value # P3
+    end
+
+    def holding_status
+      claims_by_property_id(PropertyId::HOLDING_STATUS)&.first&.entity_id_value # P6
     end
 
     def iiif_manifest
-      claim_by_property_id(PropertyId::IIIF_MANIFEST)&.entity_id_value # P41
+      claims_by_property_id(PropertyId::IIIF_MANIFEST)&.first&.entity_id_value # P41
     end
 
     def core_model_item?

data/lib/digital_scriptorium/ds_meta.rb

@@ -3,15 +3,32 @@
 module DigitalScriptorium
   # Represents a meta record consisting of a manuscript, its holding information, and metadata record.
   class DsMeta
+    include ItemId
+    include PropertyId
+
     attr_reader :holding, :manuscript, :record
 
     def initialize(record, export_hash)
       manuscript = export_hash[record.described_manuscript_id]
-      holding = export_hash[manuscript.holding_id]
+      current_holdings = current_holdings(manuscript, export_hash)
+
+      if current_holdings.size != 1
+        raise "Manuscripts must have exactly 1 current holding, found #{current_holdings.size}"
+      end
 
-      @holding = holding
+      @holding = current_holdings.first
       @manuscript = manuscript
       @record = record
     end
+
+    def current?(holding)
+      holding.holding_status == HOLDING_STATUS_CURRENT
+    end
+
+    def current_holdings(manuscript, export_hash)
+      manuscript.holding_ids
+                .map { |id| export_hash[id] }
+                .filter { |holding| current?(holding) }
+    end
   end
 end

data/lib/digital_scriptorium/item_id.rb

@@ -5,9 +5,10 @@ require 'set'
 module DigitalScriptorium
   # Constants for core model item IDs.
   module ItemId
-    MANUSCRIPT = 'Q1'
-    HOLDING    = 'Q2'
-    RECORD     = 'Q3'
+    MANUSCRIPT             = 'Q1'
+    HOLDING                = 'Q2'
+    RECORD                 = 'Q3'
+    HOLDING_STATUS_CURRENT = 'Q4'
 
     CORE_MODEL_ITEMS = Set[MANUSCRIPT, HOLDING, RECORD]
   end

data/lib/digital_scriptorium/manuscript.rb

@@ -6,11 +6,11 @@ module DigitalScriptorium
     include PropertyId
 
     def ds_id
-      claim_by_property_id(DS_ID).data_value # P1
+      claims_by_property_id(DS_ID)&.first&.data_value # P1
     end
 
-    def holding_id
-      claim_by_property_id(MANUSCRIPT_HOLDING).entity_id_value # P2
+    def holding_ids
+      claims_by_property_id(MANUSCRIPT_HOLDING)&.map(&:entity_id_value) # P2
     end
   end
 end

data/lib/digital_scriptorium/record.rb

@@ -6,7 +6,7 @@ module DigitalScriptorium
     include PropertyId
 
     def described_manuscript_id
-      claim_by_property_id(DESCRIBED_MANUSCRIPT).entity_id_value # P3
+      claims_by_property_id(DESCRIBED_MANUSCRIPT)&.first&.entity_id_value # P3
     end
 
     def title_as_recorded_claims

data/lib/digital_scriptorium/transformers/acknowledgements_claim_transformer.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module DigitalScriptorium
+  # Transformer for acknowledgements (P33) claims.
+  class AcknowledgementsClaimTransformer < BaseClaimTransformer
+    def initialize(claim, _, **kwargs)
+      super(claim, **kwargs)
+    end
+
+    def display_values
+      [display_value(claim.data_value)]
+    end
+  end
+end

data/lib/digital_scriptorium/transformers/base_claim_transformer.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module DigitalScriptorium
+  # Base transformer class providing a common interface for all transformers.
+  class BaseClaimTransformer
+    attr_reader :claim, :prefix
+
+    def initialize(claim, **kwargs)
+      @claim = claim
+      @prefix = kwargs[:prefix]
+    end
+
+    def display_values
+      []
+    end
+
+    def search_values
+      []
+    end
+
+    def facet_values
+      []
+    end
+
+    def display_value(recorded_value, in_original_script = nil, linked_terms = [])
+      value = { 'recorded_value' => recorded_value }
+      value['original_script'] = in_original_script if in_original_script
+      value['linked_terms'] = linked_terms if linked_terms.any?
+      value.to_json
+    end
+
+    def solr_props
+      solr_props = {}
+      solr_props["#{prefix}_display"] = display_values if display_values.any?
+      solr_props["#{prefix}_search"] = search_values if search_values.any?
+      solr_props["#{prefix}_facet"] = facet_values if facet_values.any?
+      solr_props
+    end
+  end
+end

data/lib/digital_scriptorium/transformers/date_claim_transformer.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require 'time'
+
+module DigitalScriptorium
+  # Transformer for production date (P23) claims.
+  class DateClaimTransformer < QualifiedClaimTransformer
+    include PropertyId
+
+    def solr_props
+      super.merge(meta_props).merge(int_props)
+    end
+
+    def meta_props
+      {
+        'date_meta' => [claim.data_value]
+      }
+    end
+
+    def int_props
+      return {} unless claim.qualifiers_by_property_id? CENTURY
+
+      {
+        'century_int' => [century_int]
+      }
+    end
+
+    def linked_term_for(authority)
+      {
+        'label' => authority.label('en'),
+        'facet_field' => 'century_int',
+        'facet_value' => century_int,
+        'source_url' => external_uri(authority) || wikidata_uri(authority)
+      }.compact
+    end
+
+    def century_int
+      parse_year(time_value_from_qualifier(CENTURY))
+    end
+
+    def time_value_from_qualifier(property_id)
+      claim.qualifiers_by_property_id(property_id)&.first&.time_value
+    end
+
+    # Wikibase date format "resembling ISO 8601": +YYYY-MM-DDT00:00:00Z
+    # https://www.wikidata.org/wiki/Help:Dates#Time_datatype
+    def parse_year(date)
+      Time.iso8601(date[1..]).year
+    end
+  end
+end

data/lib/digital_scriptorium/transformers/dated_claim_transformer.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module DigitalScriptorium
+  # Transformer for dated? (P26) claims.
+  class DatedClaimTransformer < BaseClaimTransformer
+    attr_reader :export_hash
+
+    def initialize(claim, export_hash, **kwargs)
+      super(claim, **kwargs)
+      @export_hash = export_hash
+    end
+
+    def facet_values
+      [export_hash[claim.entity_id_value]&.label('en')].compact
+    end
+  end
+end

data/lib/digital_scriptorium/transformers/iiif_manifest_claim_transformer.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module DigitalScriptorium
+  # Transformer for IIIF Manifest (P41) claims.
+  class IiifManifestClaimTransformer < LinkClaimTransformer
+    def solr_props
+      super.merge({ 'images_facet' => ['Yes'] })
+    end
+  end
+end

data/lib/digital_scriptorium/transformers/link_claim_transformer.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module DigitalScriptorium
+  # Transformer for extracting links from relevant Digital Scriptorium claims.
+  class LinkClaimTransformer < BaseClaimTransformer
+    def initialize(claim, _, **kwargs)
+      super(claim, **kwargs)
+    end
+
+    def solr_props
+      super.merge({ "#{prefix}_link" => [claim.data_value] })
+    end
+  end
+end

data/lib/digital_scriptorium/transformers/name_claim_transformer.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module DigitalScriptorium
+  # Transformer for associated name (P14) claims.
+  # NOTE: Name claims produce fields prefixes derived from the value of their role (P15) qualifiers
+  # (owner, author, scribe, artist, agent).
+  class NameClaimTransformer < QualifiedClaimTransformerWithFacetFallback
+    include PropertyId
+
+    def initialize(claim, export_hash, **kwargs)
+      super(claim, export_hash, prefix: role_prefix(claim, export_hash), authority_id: kwargs[:authority_id])
+    end
+
+    def role_prefix(claim, export_hash)
+      role_entity_id = claim.qualifiers_by_property_id(ROLE_IN_AUTHORITY_FILE)&.first&.entity_id_value
+      raise 'Missing role qualifier for name claim' unless role_entity_id
+
+      role_item = export_hash[role_entity_id]
+      role_label = role_item.label('en')
+      role_label.split.last.downcase
+    end
+  end
+end

data/lib/digital_scriptorium/transformers/note_claim_transformer.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module DigitalScriptorium
+  # Transformer for acknowledgements (P32) claims.
+  class NoteClaimTransformer < BaseClaimTransformer
+    def initialize(claim, _, **kwargs)
+      super(claim, **kwargs)
+    end
+
+    def display_values
+      [display_value(claim.data_value)]
+    end
+
+    def search_values
+      [claim.data_value]
+    end
+  end
+end

data/lib/digital_scriptorium/transformers/physical_description_claim_transformer.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module DigitalScriptorium
+  # Transformer for physical description (P29) claims.
+  class PhysicalDescriptionClaimTransformer < BaseClaimTransformer
+    def initialize(claim, _, **kwargs)
+      super(claim, **kwargs)
+    end
+
+    def display_values
+      [display_value(claim.data_value)]
+    end
+
+    def search_values
+      [claim.data_value]
+    end
+  end
+end

data/lib/digital_scriptorium/transformers/qualified_claim_transformer.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module DigitalScriptorium
+  # Transformer for converting qualified claims of Digital Scriptorium items into Solr fields.
+  class QualifiedClaimTransformer < BaseClaimTransformer
+    include PropertyId
+
+    attr_reader :export_hash, :authority_id
+
+    def initialize(claim, export_hash, **kwargs)
+      super(claim, **kwargs)
+      @export_hash = export_hash
+      @authority_id = kwargs[:authority_id]
+    end
+
+    def display_values
+      [display_value(main_snak_value, in_original_script, linked_terms)]
+    end
+
+    def search_values
+      [main_snak_value, in_original_script, linked_term_labels].flatten.compact.uniq
+    end
+
+    def facet_values
+      linked_term_labels
+    end
+
+    def in_original_script
+      claim.qualifiers_by_property_id(IN_ORIGINAL_SCRIPT)&.first&.data_value&.value
+    end
+
+    def external_uri(authority)
+      authority.claims_by_property_id(EXTERNAL_URI)&.first&.data_value
+    end
+
+    def wikidata_id(authority)
+      authority.claims_by_property_id(WIKIDATA_QID)&.first&.data_value
+    end
+
+    def wikidata_uri(authority)
+      wikidata_id(authority) && "https://www.wikidata.org/wiki/#{wikidata_id(authority)}"
+    end
+
+    def linked_term_for(authority)
+      {
+        'label' => authority.label('en'),
+        'source_url' => external_uri(authority) || wikidata_uri(authority)
+      }.compact
+    end
+
+    def linked_terms
+      @linked_terms ||= begin
+        linked_terms = []
+
+        claim.qualifiers_by_property_id(authority_id)&.each do |qualifier|
+          authority_file_item_id = qualifier.entity_id_value
+          authority = export_hash[authority_file_item_id]
+          linked_terms << linked_term_for(authority) if authority
+        end
+
+        linked_terms.uniq
+      end
+    end
+
+    def linked_term_labels
+      @linked_term_labels ||= linked_terms.map { |term| term['label'] }.uniq
+    end
+
+    def main_snak_value
+      if claim.value_type? WikibaseRepresentable::Model::EntityIdValue
+        entity_id = claim.entity_id_value
+        referenced_item = export_hash[entity_id]
+        referenced_item.label('en')
+      else
+        claim.data_value
+      end
+    end
+  end
+end

data/lib/digital_scriptorium/transformers/qualified_claim_transformer_with_facet_fallback.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module DigitalScriptorium
+  # Transformer for converting qualified claims of Digital Scriptorium items into Solr fields
+  # with a fallback to the value as-recorded for the facet field.
+  class QualifiedClaimTransformerWithFacetFallback < QualifiedClaimTransformer
+    def facet_values
+      super.any? ? super : [claim.data_value]
+    end
+  end
+end

data/lib/digital_scriptorium/transformers/shelfmark_claim_transformer.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module DigitalScriptorium
+  # Transformer for shelfmark (P8) claims.
+  class ShelfmarkClaimTransformer < BaseClaimTransformer
+    def initialize(claim, _, **kwargs)
+      super(claim, **kwargs)
+    end
+
+    def display_values
+      [display_value(claim.data_value)]
+    end
+
+    def search_values
+      [claim.data_value]
+    end
+  end
+end

data/lib/digital_scriptorium/transformers/uniform_title_claim_transformer.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module DigitalScriptorium
+  # Transformer for uniform title (P12) claims.
+  class UniformTitleClaimTransformer < BaseClaimTransformer
+    def initialize(claim, _, **kwargs)
+      super(claim, **kwargs)
+    end
+
+    def search_values
+      [claim.data_value]
+    end
+  end
+end

data/lib/digital_scriptorium/transformers.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+require_relative 'transformers/base_claim_transformer'
+require_relative 'transformers/link_claim_transformer'
+require_relative 'transformers/qualified_claim_transformer'
+require_relative 'transformers/qualified_claim_transformer_with_facet_fallback'
+
+require_relative 'transformers/acknowledgements_claim_transformer'
+require_relative 'transformers/date_claim_transformer'
+require_relative 'transformers/dated_claim_transformer'
+require_relative 'transformers/iiif_manifest_claim_transformer'
+require_relative 'transformers/name_claim_transformer'
+require_relative 'transformers/note_claim_transformer'
+require_relative 'transformers/physical_description_claim_transformer'
+require_relative 'transformers/shelfmark_claim_transformer'
+require_relative 'transformers/uniform_title_claim_transformer'
+
+module DigitalScriptorium
+  # Factory for creating claim transformers
+  module Transformers
+    include PropertyId
+
+    TRANSFORMERS = {
+      HOLDING_INSTITUTION_AS_RECORDED => QualifiedClaimTransformer,
+      SHELFMARK                       => ShelfmarkClaimTransformer,
+      LINK_TO_INSTITUTIONAL_RECORD    => LinkClaimTransformer,
+      TITLE_AS_RECORDED               => QualifiedClaimTransformerWithFacetFallback,
+      UNIFORM_TITLE_AS_RECORDED       => UniformTitleClaimTransformer,
+      ASSOCIATED_NAME_AS_RECORDED     => NameClaimTransformer,
+      GENRE_AS_RECORDED               => QualifiedClaimTransformerWithFacetFallback,
+      SUBJECT_AS_RECORDED             => QualifiedClaimTransformerWithFacetFallback,
+      LANGUAGE_AS_RECORDED            => QualifiedClaimTransformer,
+      PRODUCTION_DATE_AS_RECORDED     => DateClaimTransformer,
+      DATED                           => DatedClaimTransformer,
+      PRODUCTION_PLACE_AS_RECORDED    => QualifiedClaimTransformer,
+      PHYSICAL_DESCRIPTION            => PhysicalDescriptionClaimTransformer,
+      MATERIAL_AS_RECORDED            => QualifiedClaimTransformer,
+      NOTE                            => NoteClaimTransformer,
+      ACKNOWLEDGEMENTS                => AcknowledgementsClaimTransformer,
+      IIIF_MANIFEST                   => IiifManifestClaimTransformer
+    }.freeze
+
+    AUTHORITY_IDS = {
+      HOLDING_INSTITUTION_AS_RECORDED => HOLDING_INSTITUTION_IN_AUTHORITY_FILE,
+      TITLE_AS_RECORDED               => STANDARD_TITLE,
+      ASSOCIATED_NAME_AS_RECORDED     => NAME_IN_AUTHORITY_FILE,
+      GENRE_AS_RECORDED               => TERM_IN_AUTHORITY_FILE,
+      SUBJECT_AS_RECORDED             => TERM_IN_AUTHORITY_FILE,
+      LANGUAGE_AS_RECORDED            => LANGUAGE_IN_AUTHORITY_FILE,
+      PRODUCTION_DATE_AS_RECORDED     => PRODUCTION_CENTURY_IN_AUTHORITY_FILE,
+      PRODUCTION_PLACE_AS_RECORDED    => PLACE_IN_AUTHORITY_FILE,
+      MATERIAL_AS_RECORDED            => MATERIAL_IN_AUTHORITY_FILE
+    }.freeze
+
+    PREFIXES = {
+      HOLDING_INSTITUTION_AS_RECORDED => 'institution',
+      SHELFMARK                       => 'shelfmark',
+      LINK_TO_INSTITUTIONAL_RECORD    => 'institutional_record',
+      TITLE_AS_RECORDED               => 'title',
+      UNIFORM_TITLE_AS_RECORDED       => 'uniform_title',
+      ASSOCIATED_NAME_AS_RECORDED     => 'name',
+      GENRE_AS_RECORDED               => 'term',
+      SUBJECT_AS_RECORDED             => 'term',
+      LANGUAGE_AS_RECORDED            => 'language',
+      PRODUCTION_DATE_AS_RECORDED     => 'date',
+      DATED                           => 'dated',
+      PRODUCTION_PLACE_AS_RECORDED    => 'place',
+      PHYSICAL_DESCRIPTION            => 'physical_description',
+      MATERIAL_AS_RECORDED            => 'material',
+      NOTE                            => 'note',
+      ACKNOWLEDGEMENTS                => 'acknowledgements',
+      IIIF_MANIFEST                   => 'iiif_manifest'
+    }.freeze
+
+    def self.defined?(property_id)
+      TRANSFORMERS.include?(property_id)
+    end
+
+    def self.transformer(property_id)
+      TRANSFORMERS[property_id]
+    end
+
+    def self.authority_id(property_id)
+      AUTHORITY_IDS[property_id]
+    end
+
+    def self.prefix(property_id)
+      PREFIXES[property_id]
+    end
+
+    def self.create(property_id, claim, export_hash)
+      transformer_class = TRANSFORMERS[property_id]
+      authority_id = AUTHORITY_IDS[property_id]
+      prefix = PREFIXES[property_id]
+      return unless transformer_class && prefix
+
+      transformer_class.new(claim, export_hash, prefix: prefix, authority_id: authority_id)
+    end
+  end
+end

data/lib/digital_scriptorium/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DigitalScriptorium
-  VERSION = '0.1.2'
+  VERSION = '0.2.0'
 end

data/lib/digital_scriptorium.rb

@@ -12,6 +12,4 @@ require 'digital_scriptorium/record'
 require 'digital_scriptorium/export'
 require 'digital_scriptorium/export_representer'
 
-require 'digital_scriptorium/claim_transformer'
-require 'digital_scriptorium/date_claim_transformer'
-require 'digital_scriptorium/name_claim_transformer'
+require 'digital_scriptorium/transformers'

data/wikibase_to_solr_new.rb

@@ -2,10 +2,11 @@
 
 require 'digital_scriptorium'
 require 'json'
+require 'logging'
 require 'optparse'
+require 'set'
 require 'time'
 require 'tty-spinner'
-require 'yaml'
 require 'zlib'
 
 dir = File.dirname __FILE__
@@ -15,7 +16,9 @@ output_file = File.expand_path 'solr_import.json', dir
 config_file = File.expand_path 'property_config.yml', dir
 pretty_print = false
 
-OptionParser.new do |opts|
+logger = Logging.logger($stdout)
+
+OptionParser.new { |opts|
   opts.banner = 'Usage: wikibase_to_solr.rb [options]'
 
   opts.on('-i', '--in FILE', 'The file path to the gzipped Wikibase JSON export file.') do |f|
@@ -33,7 +36,7 @@ OptionParser.new do |opts|
   opts.on('-p', '--pretty-print', 'Whether to pretty-print the JSON output.') do
     pretty_print = true
   end
-end.parse!
+}.parse!
 
 def merge(solr_item, new_props)
   solr_item.merge(new_props) do |_, old_val, new_val|
@@ -41,21 +44,23 @@ def merge(solr_item, new_props)
   end
 end
 
-def merge_transformed_fields(solr_item, claim, export_hash, property_config)
-  if claim.property_id == DigitalScriptorium::PropertyId::ASSOCIATED_NAME_AS_RECORDED
-    merge(solr_item, DigitalScriptorium::NameClaimTransformer.transform(claim, export_hash))
-  elsif claim.property_id == DigitalScriptorium::PropertyId::PRODUCTION_DATE_AS_RECORDED
-    merge(solr_item,
-          DigitalScriptorium::DateClaimTransformer.transform(claim, export_hash, property_config))
-  else
-    merge(solr_item,
-          DigitalScriptorium::ClaimTransformer.transform(claim, export_hash, property_config))
-  end
+def base_solr_item(meta)
+  ds_id = meta.manuscript.ds_id
+  {
+    'qid_meta' => [meta.holding.id, meta.manuscript.id, meta.record.id],
+    'id' => [ds_id],
+    'id_display' => [JSON.generate(recorded_value: ds_id)],
+    'id_search' => [ds_id]
+  }
 end
 
-start_time = Time.now
+def record?(entity)
+  entity.is_a?(DigitalScriptorium::DsItem) &&
+    entity.claims_by_property_id?(DigitalScriptorium::PropertyId::INSTANCE_OF) &&
+    entity.record?
+end
 
-config = YAML.load_file(config_file)
+start_time = Time.now.utc
 
 loading_spinner = TTY::Spinner.new('[:spinner] Loading export data', hide_cursor: true)
 loading_spinner.auto_spin
@@ -64,7 +69,7 @@ export_json = Zlib::GzipReader.open(input_file).read
 export_hash = DigitalScriptorium::ExportRepresenter.new(DigitalScriptorium::Export.new)
                                                    .from_json(export_json)
                                                    .to_hash
-loaded_time = Time.now
+loaded_time = Time.now.utc
 loading_spinner.success("(#{format('%0.02f', loaded_time - start_time)}s)")
 
 item_count = 0
@@ -76,19 +81,22 @@ File.open(output_file, 'w') do |file|
   file << "\n" if pretty_print
 
   export_hash.each_with_index do |(_, entity), idx|
-    next unless entity.is_a?(DigitalScriptorium::DsItem) &&
-                entity.claims_by_property_id?(DigitalScriptorium::PropertyId::INSTANCE_OF) &&
-                entity.record?
+    next unless record?(entity)
 
     meta = DigitalScriptorium::DsMeta.new(entity, export_hash)
-    solr_item = { 'qid_meta' => [meta.holding.id, meta.manuscript.id, meta.record.id] }
+    solr_item = base_solr_item(meta)
 
     [meta.holding, meta.manuscript, meta.record].each do |item|
       item.claims.each do |property_id, claims|
         claims.each do |claim|
-          next unless (property_config = config[property_id])
-
-          solr_item = merge_transformed_fields(solr_item, claim, export_hash, property_config)
+          next unless DigitalScriptorium::Transformers.defined? property_id
+
+          begin
+            transformer = DigitalScriptorium::Transformers.create property_id, claim, export_hash
+            solr_item = merge solr_item, transformer.solr_props
+          rescue StandardError => e
+            logger.error "Error processing #{property_id} claim for item #{item.id}: #{e}"
+          end
         end
       end
     end
@@ -103,6 +111,6 @@ File.open(output_file, 'w') do |file|
   file << ']'
 end
 
-finish_time = Time.now
+finish_time = Time.now.utc
 generating_spinner.success("(#{format('%0.02f', finish_time - loaded_time)}s)")
 puts "Generated #{item_count} Solr documents in #{format('%0.02f', finish_time - start_time)} seconds"

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: digital_scriptorium
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
 platform: ruby
 authors:
 - Michael Holloway
 bindir: exe
 cert_chain: []
-date: 2025-01-08 00:00:00.000000000 Z
+date: 2025-01-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json
@@ -65,6 +65,62 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.2'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.13'
+- !ruby/object:Gem::Dependency
+  name: upennlib-rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
 email:
 - michael@mdholloway.org
 executables: []
@@ -74,13 +130,12 @@ files:
 - ".ignore"
 - ".rspec"
 - ".rubocop.yml"
+- ".rubocop_todo.yml"
 - LICENSE.txt
 - README.md
 - Rakefile
 - doc/overview.md
 - lib/digital_scriptorium.rb
-- lib/digital_scriptorium/claim_transformer.rb
-- lib/digital_scriptorium/date_claim_transformer.rb
 - lib/digital_scriptorium/ds_item.rb
 - lib/digital_scriptorium/ds_meta.rb
 - lib/digital_scriptorium/export.rb
@@ -88,11 +143,23 @@ files:
 - lib/digital_scriptorium/holding.rb
 - lib/digital_scriptorium/item_id.rb
 - lib/digital_scriptorium/manuscript.rb
-- lib/digital_scriptorium/name_claim_transformer.rb
 - lib/digital_scriptorium/property_id.rb
 - lib/digital_scriptorium/record.rb
+- lib/digital_scriptorium/transformers.rb
+- lib/digital_scriptorium/transformers/acknowledgements_claim_transformer.rb
+- lib/digital_scriptorium/transformers/base_claim_transformer.rb
+- lib/digital_scriptorium/transformers/date_claim_transformer.rb
+- lib/digital_scriptorium/transformers/dated_claim_transformer.rb
+- lib/digital_scriptorium/transformers/iiif_manifest_claim_transformer.rb
+- lib/digital_scriptorium/transformers/link_claim_transformer.rb
+- lib/digital_scriptorium/transformers/name_claim_transformer.rb
+- lib/digital_scriptorium/transformers/note_claim_transformer.rb
+- lib/digital_scriptorium/transformers/physical_description_claim_transformer.rb
+- lib/digital_scriptorium/transformers/qualified_claim_transformer.rb
+- lib/digital_scriptorium/transformers/qualified_claim_transformer_with_facet_fallback.rb
+- lib/digital_scriptorium/transformers/shelfmark_claim_transformer.rb
+- lib/digital_scriptorium/transformers/uniform_title_claim_transformer.rb
 - lib/digital_scriptorium/version.rb
-- property_config.yml
 - sig/digital_scriptorium.rbs
 - wikibase_to_solr_new.rb
 homepage: https://github.com/mdholloway/digital_scriptorium

data/lib/digital_scriptorium/claim_transformer.rb

@@ -1,82 +0,0 @@
-# frozen_string_literal: true
-
-require 'wikibase_representable'
-
-module DigitalScriptorium
-  # Transformer for converting claims of Digital Scriptorium items into Solr fields.
-  class ClaimTransformer
-    include PropertyId
-    include WikibaseRepresentable::Model
-
-    def self.transform(claim, export_hash, config)
-      solr_props = {}
-
-      prefix = config['prefix']
-      requested_fields = config['fields']
-      authority_property_id = config['authority']
-
-      value = primary_value_from_claim(claim, export_hash)
-
-      solr_props['id'] = [value] if requested_fields.include? 'id'
-      solr_props["#{prefix}_meta"] = [value] if requested_fields.include? 'meta'
-
-      unless authority_property_id && claim.qualifiers_by_property_id?(authority_property_id)
-        solr_props["#{prefix}_display"] = [{ 'PV' => value }.to_json] if requested_fields.include? 'display'
-        solr_props["#{prefix}_search"] = [value] if requested_fields.include? 'search'
-        solr_props["#{prefix}_facet"] = [value] if requested_fields.include? 'facet'
-
-        solr_props['images_facet'] = ['Yes'] if value && claim.property_id == IIIF_MANIFEST
-        solr_props["#{prefix}_link"] = [value] if requested_fields.include? 'link'
-
-        return solr_props
-      end
-
-      display_entries = []
-      search_entries = [value]
-      facets = []
-
-      claim.qualifiers_by_property_id(authority_property_id).each do |qualifier|
-        display_props = { 'PV' => value }
-
-        authority_id = qualifier.entity_id_value
-        authority = export_hash[authority_id]
-
-        if authority
-          label = authority.label('en')
-
-          display_props['QL'] = label
-          search_entries << label
-          facets << label
-
-          external_uri = authority.claim_by_property_id(EXTERNAL_URI)&.data_value
-          wikidata_id = authority.claim_by_property_id(WIKIDATA_QID)&.data_value
-          wikidata_uri = wikidata_id && "https://www.wikidata.org/wiki/#{wikidata_id}"
-
-          # Only one or the other of these seem to exist for a given item in practice.
-          display_props['QU'] = external_uri if external_uri
-          display_props['QU'] = wikidata_uri if wikidata_uri
-        end
-
-        display_entries << display_props.to_json
-      end
-
-      solr_props["#{prefix}_display"] = display_entries.uniq if requested_fields.include? 'display'
-      solr_props["#{prefix}_search"] = search_entries.uniq if requested_fields.include? 'search'
-      solr_props["#{prefix}_facet"] = facets.uniq if requested_fields.include? 'facet'
-
-      solr_props
-    end
-
-    def self.primary_value_from_claim(claim, export_hash)
-      if claim.value_type? EntityIdValue
-        entity_id = claim.entity_id_value
-        referenced_item = export_hash[entity_id]
-        referenced_item.label('en')
-      elsif claim.value_type? TimeValue
-        claim.time_value
-      else
-        claim.data_value
-      end
-    end
-  end
-end

data/lib/digital_scriptorium/date_claim_transformer.rb

@@ -1,25 +0,0 @@
-# frozen_string_literal: true
-
-require 'time'
-
-module DigitalScriptorium
-  # Special-purpose transformer for date (P23) claims
-  class DateClaimTransformer
-    include PropertyId
-
-    def self.transform(claim, export_hash, config)
-      solr_props = ClaimTransformer.transform(claim, export_hash, config)
-      return solr_props unless claim.qualifiers
-
-      century = claim.qualifier_by_property_id(CENTURY).time_value
-      earliest = claim.qualifier_by_property_id(EARLIEST_DATE).time_value
-      latest = claim.qualifier_by_property_id(LATEST_DATE).time_value
-
-      solr_props['century_int'] = [Time.parse(century).year] unless century.nil?
-      solr_props['earliest_int'] = [Time.parse(earliest).year] unless earliest.nil?
-      solr_props['latest_int'] = [Time.parse(latest).year] unless latest.nil?
-
-      solr_props
-    end
-  end
-end

data/lib/digital_scriptorium/name_claim_transformer.rb

@@ -1,61 +0,0 @@
-# frozen_string_literal: true
-
-module DigitalScriptorium
-  # Special-purpose transformer for name (P14) claims
-  class NameClaimTransformer
-    include PropertyId
-
-    def self.transform(claim, export_hash)
-      return {} unless claim.qualifiers_by_property_id? ROLE_IN_AUTHORITY_FILE
-
-      role_entity_id = claim.qualifier_by_property_id(ROLE_IN_AUTHORITY_FILE).entity_id_value
-      role_item = export_hash[role_entity_id]
-      role_label = role_item.label('en')
-      prefix = role_label.downcase.split.last
-
-      recorded_name = claim.data_value
-      display_names = { 'PV' => recorded_name }
-      search_names = [recorded_name]
-
-      name_in_original_script = claim.qualifier_by_property_id(IN_ORIGINAL_SCRIPT)&.data_value&.value
-      display_names['AGR'] = name_in_original_script if name_in_original_script
-      search_names << name_in_original_script if name_in_original_script
-
-      unless claim.qualifiers_by_property_id? NAME_IN_AUTHORITY_FILE
-        return {
-          "#{prefix}_display" => [display_names.to_json],
-          "#{prefix}_search" => search_names,
-          "#{prefix}_facet" => [recorded_name]
-        }
-      end
-
-      display_entries = []
-      facets = []
-
-      claim.qualifiers_by_property_id(NAME_IN_AUTHORITY_FILE).each do |qualifier|
-        display_names_for_qualifier = { 'PV' => recorded_name }
-        display_names_for_qualifier['AGR'] = name_in_original_script if name_in_original_script
-
-        name_entity_id = qualifier.entity_id_value
-        name_item = export_hash[name_entity_id]
-        name_label = name_item.label('en')
-
-        display_names_for_qualifier['QL'] = name_label
-        search_names << name_label
-        facets << name_label
-
-        wikidata_id = name_item.claim_by_property_id(WIKIDATA_QID).data_value
-        wikidata_url = "https://www.wikidata.org/wiki/#{wikidata_id}"
-        display_names_for_qualifier['QU'] = wikidata_url if wikidata_url
-
-        display_entries << display_names_for_qualifier.to_json
-      end
-
-      {
-        "#{prefix}_display" => display_entries.uniq,
-        "#{prefix}_search" => search_names.uniq,
-        "#{prefix}_facet" => facets.uniq
-      }
-    end
-  end
-end

data/property_config.yml

@@ -1,106 +0,0 @@
-P1:
-  prefix: id
-  fields:
-    - id
-    - display
-    - search
-P5:
-  prefix: institution
-  fields:
-    - display
-    - search
-    - facet
-  authority: P4
-P6:
-  prefix: holding_status
-  fields:
-    - display
-P8:
-  prefix: shelfmark
-  fields:
-    - display
-    - search
-P9:
-  prefix: institutional_record
-  fields:
-    - link
-P10:
-  prefix: title
-  fields:
-    - display
-    - search
-    - facet
-  authority: P11
-P12:
-  prefix: uniform_title
-  fields:
-    - search
-# NOTE: P14 can translate to any of a few different Solr fields based on the value of the
-# associated role (P15) qualifier, and is handled in its own dedicated processing method
-P14:
-  prefix: associated_name
-  fields: []
-P18:
-  prefix: term
-  fields:
-    - display
-    - search
-    - facet
-  authority: P20
-P19:
-  prefix: term
-  fields:
-    - display
-    - search
-    - facet
-  authority: P20
-P21:
-  prefix: language
-  fields:
-    - display
-    - search
-    - facet
-  authority: P22
-P23:
-  prefix: date
-  fields:
-    - meta
-    - display
-    - search
-    - facet
-  authority: P24
-P26:
-  prefix: dated
-  fields:
-    - display
-    - facet
-P27:
-  prefix: place
-  fields:
-    - display
-    - search
-    - facet
-  authority: P28
-P29:
-  prefix: physical_description
-  fields:
-    - display
-    - search
-P30:
-  prefix: material
-  fields:
-    - facet
-  authority: P31
-P32:
-  prefix: note
-  fields:
-    - display
-    - search
-P33:
-  prefix: acknowledgements
-  fields:
-    - display
-P41:
-  prefix: iiif_manifest
-  fields:
-    - link