digital_scriptorium 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: '085e39f712849a9c7e65b62b1f9a7715830b60a87adad5d67607f42f74535304'
+  data.tar.gz: 8739348215acc0f8df17b17155df21ed641ffff4845d4d72a6ab3c9652e9cfca
+SHA512:
+  metadata.gz: f8d519fa3b4246f3c182738d4ad42567046dbf0d691a3f07958809669ace36b25d2f4697b1671f46bff0fae97c9692d389b108696c334252708377a68320906a
+  data.tar.gz: 50f9861fb347aee4fc8a556ffc013009c23b84ac631c17fc705e4466852ea63a617c06aae5718431df908035d258928615b64a2bdf45cbaa16e46efc684d90d1

data/.ignore

@@ -0,0 +1 @@
+*.json

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,30 @@
+require:
+  - rubocop-rake
+  - rubocop-rspec
+
+AllCops:
+  TargetRubyVersion: 3.0
+  NewCops: enable
+Layout/LineLength:
+  Exclude:
+    - spec/**/*
+Metrics/AbcSize:
+  Enabled: false
+Metrics/BlockLength:
+  Enabled: false
+Metrics/CyclomaticComplexity:
+  Enabled: false
+Metrics/MethodLength:
+  Enabled: false
+Metrics/ModuleLength:
+  Enabled: false
+Metrics/PerceivedComplexity:
+  Enabled: false
+RSpec/ExampleLength:
+  Enabled: false
+RSpec/MultipleExpectations:
+  Enabled: false
+RSpec/MultipleMemoizedHelpers:
+  Enabled: false
+Style/SafeNavigationChainLength:
+  Enabled: false

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2024 Michael Holloway
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,33 @@
+# DigitalScriptorium
+
+This gem provides code to support the transformation of Digital Scriptorium Wikibase data exports into collections of Apache Solr records that can be searched using [DS Catalog](https://search.digital-scriptorium.org/).
+
+See [here](doc/overview.md) for a technical overview of the logic for transforming Wikibase items in the export to Solr records.
+
+## Installation
+
+Install the gem and add to the application's Gemfile by executing:
+
+```bash
+bundle add digital_scriptorium
+```
+
+If bundler is not being used to manage dependencies, install the gem by executing:
+
+```bash
+gem install digital_scriptorium
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/mdholloway/digital_scriptorium.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+require 'rubocop/rake_task'
+
+RuboCop::RakeTask.new
+
+task default: %i[spec rubocop]

data/doc/overview.md

@@ -0,0 +1,13 @@
+# How it works
+
+For a general description of the Wikibase data model, see [Wikibase/DataModel](https://www.mediawiki.org/wiki/Wikibase/DataModel) on mediawiki.org.
+
+The Digital Scriptorium Wikibase data export is a JSON-formatted array of Wikibase entities. The bulk of the entities in the export consist of triplets that together form a meta-record consisting of one each of the DS Catalog core model types: manuscipts, holdings, and records. The export also contains entities representing property definitions and authoritative references to common topics.
+
+The [ExportRepresenter](../lib/digital_scriptorium/export_representer.rb) class can be used to deserialize an export in its entirety. The resulting [Export](../lib/digital_scriptorium/export.rb) object is essentially an array of Item and Property objects. Entities in the export are modeled using domain-specific classes provided by the [wikibase_representable](https://rubygems.org/gems/wikibase_representable) gem, such as Items, Properties, Statements (also known as Claims), and Snaks, which represent the primary claim of any statement as well as any qualifiers. Convenience methods are also provided to facilitate extracting data values.
+
+The conversion script [wikibase_to_solr_new.rb](../wikibase_to_solr_new.rb) proceeds by deserializing the export and converting the resulting array of Wikibase objects to a hash keyed by entity ID. It then iterates over the elements of the hash. When it finds a record item based on the value of its instance-of (P16) claim, it retrieves the linked manuscript item, as well as the holding item linked in turn to the manuscript item, from the export hash by entity ID. It then iterates over the claims attached to manuscript, holding, and record in turn, extracting the Solr fields requested based on the property ID that is the subject of the claim and adding them to the Solr record to be produced for the meta-record. Claims for most properties are transformed to Solr fields using a generic algorithm implemented in [ClaimTransformer](../lib/digital_scriptorium/claim_transformer.rb). Name and date claims require some special handling, and are handled in dedicated claim transformer classes ([NameClaimTransformer](../lib/digital_scriptorium/name_claim_transformer.rb) and [DateClaimTransformer](../lib/digital_scriptorium/date_claim_transformer.rb) respectively). After all claims from the manuscript, holding, and record have been processed, the resulting Solr record is written to the output file.
+
+The specific Solr fields produced for each claim are controlled by the configuration file [property_config.yml](../property_config.yml). This file also defines the prefix (representing the property name) to be attached to each field for a given property, and whether a claim based on the property might have a related authority qualifier. 
+
+The script was written so as not to rely on the structure of the export file beyond that it will be a JSON array of Wikibase entities, with records linked to manuscripts and manuscripts linked to holdings by P3 (described manuscript) and P2 (holding) claims respectively.

data/lib/digital_scriptorium/claim_transformer.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require 'wikibase_representable'
+
+module DigitalScriptorium
+  # Transformer for converting claims of Digital Scriptorium items into Solr fields.
+  class ClaimTransformer
+    include PropertyId
+    include WikibaseRepresentable::Model
+
+    def self.transform(claim, export_hash, config)
+      solr_props = {}
+
+      prefix = config['prefix']
+      requested_fields = config['fields']
+      authority_property_id = config['authority']
+
+      value = primary_value_from_claim(claim, export_hash)
+
+      solr_props['id'] = [value] if requested_fields.include? 'id'
+      solr_props["#{prefix}_meta"] = [value] if requested_fields.include? 'meta'
+
+      display_props = { 'PV' => value }
+
+      if authority_property_id && claim.qualifiers_by_property_id?(authority_property_id)
+        authority_id = claim.qualifier_by_property_id(authority_property_id).entity_id_value
+        authority = export_hash[authority_id]
+
+        if authority
+          label = authority.label('en')
+          display_props['QL'] = label
+
+          external_uri = authority.claim_by_property_id(EXTERNAL_URI)&.data_value
+
+          wikidata_id = authority.claim_by_property_id(WIKIDATA_QID)&.data_value
+          wikidata_uri = wikidata_id && "https://www.wikidata.org/wiki/#{wikidata_id}"
+
+          # Only one or the other of these seem to exist for a given item in practice.
+          display_props['QU'] = external_uri if external_uri
+          display_props['QU'] = wikidata_uri if wikidata_uri
+
+          solr_props["#{config['prefix']}_display"] = [display_props.to_json] if config['fields'].include? 'display'
+          solr_props["#{config['prefix']}_search"] = [value, label].uniq if config['fields'].include? 'search'
+          solr_props["#{config['prefix']}_facet"] = [label] if config['fields'].include? 'facet'
+
+          return solr_props
+        end
+      end
+
+      solr_props["#{config['prefix']}_display"] = [display_props.to_json] if config['fields'].include? 'display'
+      solr_props["#{config['prefix']}_search"] = [value] if config['fields'].include? 'search'
+      solr_props["#{config['prefix']}_facet"] = [value] if config['fields'].include? 'facet'
+
+      solr_props['images_facet'] = ['Yes'] if value && claim.property_id == IIIF_MANIFEST
+      solr_props["#{config['prefix']}_link"] = [value] if config['fields'].include? 'link'
+
+      solr_props
+    end
+
+    def self.primary_value_from_claim(claim, export_hash)
+      if claim.value_type? EntityIdValue
+        entity_id = claim.entity_id_value
+        referenced_item = export_hash[entity_id]
+        referenced_item.label('en')
+      elsif claim.value_type? TimeValue
+        claim.time_value
+      else
+        claim.data_value
+      end
+    end
+  end
+end

data/lib/digital_scriptorium/date_claim_transformer.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require 'time'
+
+module DigitalScriptorium
+  # Special-purpose transformer for date (P23) claims
+  class DateClaimTransformer
+    include PropertyId
+
+    def self.transform(claim, export_hash, config)
+      solr_props = ClaimTransformer.transform(claim, export_hash, config)
+      return solr_props unless claim.qualifiers
+
+      century = claim.qualifier_by_property_id(CENTURY).time_value
+      earliest = claim.qualifier_by_property_id(EARLIEST_DATE).time_value
+      latest = claim.qualifier_by_property_id(LATEST_DATE).time_value
+
+      solr_props['century_int'] = [Time.parse(century).year] unless century.nil?
+      solr_props['earliest_int'] = [Time.parse(earliest).year] unless earliest.nil?
+      solr_props['latest_int'] = [Time.parse(latest).year] unless latest.nil?
+
+      solr_props
+    end
+  end
+end

data/lib/digital_scriptorium/ds_item.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require 'wikibase_representable'
+
+module DigitalScriptorium
+  # Represents a Digital Scriptorium item
+  class DsItem < WikibaseRepresentable::Model::Item
+    def instance_of_claims
+      claims_by_property_id PropertyId::INSTANCE_OF # P16
+    end
+
+    def ds_id
+      claim_by_property_id(PropertyId::DS_ID)&.data_value # P1
+    end
+
+    def holding_id
+      claim_by_property_id(PropertyId::MANUSCRIPT_HOLDING)&.entity_id_value # P2
+    end
+
+    def described_manuscript_id
+      claim_by_property_id(PropertyId::DESCRIBED_MANUSCRIPT)&.entity_id_value # P3
+    end
+
+    def iiif_manifest
+      claim_by_property_id(PropertyId::IIIF_MANIFEST)&.entity_id_value # P41
+    end
+
+    def core_model_item?
+      instance_of_claims.any? { |claim| ItemId::CORE_MODEL_ITEMS.include? claim.entity_id_value }
+    end
+
+    def manuscript?
+      instance_of_claims.any? { |claim| claim.entity_id_value == ItemId::MANUSCRIPT }
+    end
+
+    def holding?
+      instance_of_claims.any? { |claim| claim.entity_id_value == ItemId::HOLDING }
+    end
+
+    def record?
+      instance_of_claims.any? { |claim| claim.entity_id_value == ItemId::RECORD }
+    end
+  end
+end

data/lib/digital_scriptorium/ds_meta.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module DigitalScriptorium
+  # Represents a meta record consisting of a manuscript, its holding information, and metadata record.
+  class DsMeta
+    attr_reader :holding, :manuscript, :record
+
+    def initialize(record, export_hash)
+      manuscript = export_hash[record.described_manuscript_id]
+      holding = export_hash[manuscript.holding_id]
+
+      @holding = holding
+      @manuscript = manuscript
+      @record = record
+    end
+  end
+end

data/lib/digital_scriptorium/export.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module DigitalScriptorium
+  # Simple model class representing a Wikibase JSON export.
+  # Provides a to_hash method to facilitate entity lookups by ID.
+  class Export < Array
+    def to_hash
+      hash = {}
+      each do |el|
+        hash[el.id] = el
+      end
+      hash
+    end
+  end
+end

data/lib/digital_scriptorium/export_representer.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require 'representable/json/collection'
+require 'wikibase_representable'
+
+module DigitalScriptorium
+  # Representer class for deserializing Wikibase data exports from JSON.
+  class ExportRepresenter < Representable::Decorator
+    include Representable::JSON::Collection
+    include WikibaseRepresentable::Model
+    include WikibaseRepresentable::Representers
+
+    items decorator: lambda { |input:, **|
+      input.type == Item::ENTITY_TYPE ? ItemRepresenter : PropertyRepresenter
+    }, class: lambda { |input:, **|
+      input['type'] == Item::ENTITY_TYPE ? DsItem : Property
+    }
+  end
+end

data/lib/digital_scriptorium/holding.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module DigitalScriptorium
+  # An item representing a Digital Scriptorium holding (instance of Q2)
+  class Holding < DsItem
+    def institution_as_recorded_claims
+      claims_by_property_id HOLDING_INSTITUTION_AS_RECORDED # P5
+    end
+
+    def status_claims
+      claims_by_property_id HOLDING_STATUS # P6
+    end
+
+    def institutional_id_claims
+      claims_by_property_id INSTITUTIONAL_ID # P7
+    end
+
+    def shelfmark_claims
+      claims_by_property_id SHELFMARK # P8
+    end
+
+    def link_to_institutional_record_claims
+      claims_by_property_id LINK_TO_INSTITUTIONAL_RECORD # P9
+    end
+
+    def start_time_claims
+      claims_by_property_id START_TIME # P38
+    end
+
+    def end_time_claims
+      claims_by_property_id END_TIME # P39
+    end
+  end
+end

data/lib/digital_scriptorium/item_id.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module DigitalScriptorium
+  # Constants for core model item IDs.
+  module ItemId
+    MANUSCRIPT = 'Q1'
+    HOLDING    = 'Q2'
+    RECORD     = 'Q3'
+
+    CORE_MODEL_ITEMS = Set[MANUSCRIPT, HOLDING, RECORD]
+  end
+end

data/lib/digital_scriptorium/manuscript.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module DigitalScriptorium
+  # An item representing a Digital Scriptorium manuscript (instance of Q1)
+  class Manuscript < DsItem
+    include PropertyId
+
+    def ds_id
+      claim_by_property_id(DS_ID).data_value # P1
+    end
+
+    def holding_id
+      claim_by_property_id(MANUSCRIPT_HOLDING).entity_id_value # P2
+    end
+  end
+end

data/lib/digital_scriptorium/name_claim_transformer.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module DigitalScriptorium
+  # Special-purpose transformer for name (P14) claims
+  class NameClaimTransformer
+    include PropertyId
+
+    def self.transform(claim, export_hash)
+      return {} unless claim.qualifiers_by_property_id? ROLE_IN_AUTHORITY_FILE
+
+      role_entity_id = claim.qualifier_by_property_id(ROLE_IN_AUTHORITY_FILE).entity_id_value
+      role_item = export_hash[role_entity_id]
+      role_label = role_item.label('en')
+      prefix = role_label.downcase.split.last
+
+      recorded_name = claim.data_value
+      display_names = { 'PV' => recorded_name }
+      search_names = [recorded_name]
+
+      name_in_original_script = claim.qualifier_by_property_id(IN_ORIGINAL_SCRIPT)&.data_value&.value
+      display_names['AGR'] = name_in_original_script if name_in_original_script
+      search_names << name_in_original_script if name_in_original_script
+
+      unless claim.qualifiers_by_property_id? NAME_IN_AUTHORITY_FILE
+        return {
+          "#{prefix}_display" => [display_names.to_json],
+          "#{prefix}_search" => search_names,
+          "#{prefix}_facet" => [recorded_name]
+        }
+      end
+
+      name_entity_id = claim.qualifier_by_property_id(NAME_IN_AUTHORITY_FILE).entity_id_value
+      name_item = export_hash[name_entity_id]
+      name_label = name_item.label('en')
+
+      wikidata_id = name_item.claim_by_property_id(WIKIDATA_QID).data_value
+      wikidata_url = "https://www.wikidata.org/wiki/#{wikidata_id}"
+
+      search_names << name_label
+      display_names['QL'] = name_label
+      display_names['QU'] = wikidata_url if wikidata_url
+
+      {
+        "#{prefix}_display" => [display_names.to_json],
+        "#{prefix}_search" => search_names,
+        "#{prefix}_facet" => [name_label]
+      }
+    end
+  end
+end

data/lib/digital_scriptorium/property_id.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module DigitalScriptorium
+  # Constants for Digital Scriptorium Wikibase property IDs
+  module PropertyId
+    DS_ID                                 = 'P1'
+    MANUSCRIPT_HOLDING                    = 'P2'
+    DESCRIBED_MANUSCRIPT                  = 'P3'
+    HOLDING_INSTITUTION_IN_AUTHORITY_FILE = 'P4'
+    HOLDING_INSTITUTION_AS_RECORDED       = 'P5' # qualifiers: P4
+    HOLDING_STATUS                        = 'P6'
+    INSTITUTIONAL_ID                      = 'P7'
+    SHELFMARK                             = 'P8'
+    LINK_TO_INSTITUTIONAL_RECORD          = 'P9'
+    TITLE_AS_RECORDED                     = 'P10' # qualifiers: P11, P13
+    STANDARD_TITLE                        = 'P11'
+    UNIFORM_TITLE_AS_RECORDED             = 'P12'
+    IN_ORIGINAL_SCRIPT                    = 'P13'
+    ASSOCIATED_NAME_AS_RECORDED           = 'P14' # qualifiers: P15, P17
+    ROLE_IN_AUTHORITY_FILE                = 'P15'
+    INSTANCE_OF                           = 'P16'
+    NAME_IN_AUTHORITY_FILE                = 'P17'
+    GENRE_AS_RECORDED                     = 'P18' # qualifiers: P20
+    SUBJECT_AS_RECORDED                   = 'P19' # qualifiers: P20
+    TERM_IN_AUTHORITY_FILE                = 'P20'
+    LANGUAGE_AS_RECORDED                  = 'P21' # qualifiers: P22
+    LANGUAGE_IN_AUTHORITY_FILE            = 'P22'
+    PRODUCTION_DATE_AS_RECORDED           = 'P23' # qualifiers: P25, P24, P37, P36
+    PRODUCTION_CENTURY_IN_AUTHORITY_FILE  = 'P24'
+    CENTURY                               = 'P25'
+    DATED                                 = 'P26'
+    PRODUCTION_PLACE_AS_RECORDED          = 'P27' # qualifiers: P28
+    PLACE_IN_AUTHORITY_FILE               = 'P28'
+    PHYSICAL_DESCRIPTION                  = 'P29'
+    MATERIAL_AS_RECORDED                  = 'P30' # qualifiers: P31
+    MATERIAL_IN_AUTHORITY_FILE            = 'P31'
+    NOTE                                  = 'P32'
+    ACKNOWLEDGEMENTS                      = 'P33'
+    DATE_ADDED                            = 'P34'
+    DATE_LAST_UPDATED                     = 'P35'
+    LATEST_DATE                           = 'P36'
+    EARLIEST_DATE                         = 'P37'
+    START_TIME                            = 'P38'
+    END_TIME                              = 'P39'
+    EXTERNAL_IDENTIFIER                   = 'P40'
+    IIIF_MANIFEST                         = 'P41'
+    WIKIDATA_QID                          = 'P42'
+    VIAF_ID                               = 'P43'
+    EXTERNAL_URI                          = 'P44'
+    EQUIVALENT_PROPERTY                   = 'P45'
+    FORMATTER_URL                         = 'P46'
+    SUBCLASS_OF                           = 'P47'
+  end
+end

data/lib/digital_scriptorium/record.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module DigitalScriptorium
+  # An item representing a Digital Scriptorium record (instance of Q3)
+  class Record < DsItem
+    include PropertyId
+
+    def described_manuscript_id
+      claim_by_property_id(DESCRIBED_MANUSCRIPT).entity_id_value # P3
+    end
+
+    def title_as_recorded_claims
+      claims_by_property_id TITLE_AS_RECORDED # P10
+    end
+
+    def uniform_title_as_recorded_claims
+      claims_by_property_id UNIFORM_TITLE_AS_RECORDED # P12
+    end
+
+    def associated_name_as_recorded_claims
+      claims_by_property_id ASSOCIATED_NAME_AS_RECORDED # P14
+    end
+
+    def genre_as_recorded_claims
+      claims_by_property_id GENRE_AS_RECORDED # P18
+    end
+
+    def language_as_recorded_claims
+      claims_by_property_id LANGUAGE_AS_RECORDED # P21
+    end
+
+    def production_date_as_recorded_claims
+      claims_by_property_id PRODUCTION_DATE_AS_RECORDED # P23
+    end
+
+    def dated_claims
+      claims_by_property_id DATED # P26
+    end
+
+    def production_place_as_recorded_claims
+      claims_by_property_id PRODUCTION_PLACE_AS_RECORDED # P27
+    end
+
+    def physical_description_claims
+      claims_by_property_id PHYSICAL_DESCRIPTION # P29
+    end
+
+    def material_as_recorded_claims
+      claims_by_property_id MATERIAL_AS_RECORDED # P30
+    end
+
+    def note_claims
+      claims_by_property_id NOTE # P32
+    end
+
+    def acknowledgements_claims
+      claims_by_property_id ACKNOWLEDGEMENTS # P33
+    end
+
+    def date_added_claims
+      claims_by_property_id DATE_ADDED # P34
+    end
+
+    def date_last_updated_claims
+      claims_by_property_id DATE_LAST_UPDATED # P35
+    end
+
+    def iiif_manifest_claims
+      claims_by_property_id IIIF_MANIFEST # P41
+    end
+  end
+end

data/lib/digital_scriptorium/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module DigitalScriptorium
+  VERSION = '0.1.0'
+end

data/lib/digital_scriptorium.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require 'digital_scriptorium/item_id'
+require 'digital_scriptorium/property_id'
+
+require 'digital_scriptorium/ds_item'
+require 'digital_scriptorium/ds_meta'
+require 'digital_scriptorium/holding'
+require 'digital_scriptorium/manuscript'
+require 'digital_scriptorium/record'
+
+require 'digital_scriptorium/export'
+require 'digital_scriptorium/export_representer'
+
+require 'digital_scriptorium/claim_transformer'
+require 'digital_scriptorium/date_claim_transformer'
+require 'digital_scriptorium/name_claim_transformer'

data/property_config.yml

@@ -0,0 +1,106 @@
+P1:
+  prefix: id
+  fields:
+    - id
+    - display
+    - search
+P5:
+  prefix: institution
+  fields:
+    - display
+    - search
+    - facet
+  authority: P4
+P6:
+  prefix: holding_status
+  fields:
+    - display
+P8:
+  prefix: shelfmark
+  fields:
+    - display
+    - search
+P9:
+  prefix: institutional_record
+  fields:
+    - link
+P10:
+  prefix: title
+  fields:
+    - display
+    - search
+    - facet
+  authority: P11
+P12:
+  prefix: uniform_title
+  fields:
+    - search
+# NOTE: P14 can translate to any of a few different Solr fields based on the value of the
+# associated role (P15) qualifier, and is handled in its own dedicated processing method
+P14:
+  prefix: associated_name
+  fields: []
+P18:
+  prefix: term
+  fields:
+    - display
+    - search
+    - facet
+  authority: P20
+P19:
+  prefix: term
+  fields:
+    - display
+    - search
+    - facet
+  authority: P20
+P21:
+  prefix: language
+  fields:
+    - display
+    - search
+    - facet
+  authority: P22
+P23:
+  prefix: date
+  fields:
+    - meta
+    - display
+    - search
+    - facet
+  authority: P24
+P26:
+  prefix: dated
+  fields:
+    - display
+    - facet
+P27:
+  prefix: place
+  fields:
+    - display
+    - search
+    - facet
+  authority: P28
+P29:
+  prefix: physical_description
+  fields:
+    - display
+    - search
+P30:
+  prefix: material
+  fields:
+    - facet
+  authority: P31
+P32:
+  prefix: note
+  fields:
+    - display
+    - search
+P33:
+  prefix: acknowledgements
+  fields:
+    - display
+P41:
+  prefix: iiif_manifest
+  fields:
+    - link

data/sig/digital_scriptorium.rbs

@@ -0,0 +1,4 @@
+module DigitalScriptorium
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end

data/wikibase_to_solr_new.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require 'digital_scriptorium'
+require 'json'
+require 'optparse'
+require 'time'
+require 'tty-spinner'
+require 'yaml'
+require 'zlib'
+
+dir = File.dirname __FILE__
+
+input_file = File.expand_path 'wikibase_export.json.gz', dir
+output_file = File.expand_path 'solr_import.json', dir
+config_file = File.expand_path 'property_config.yml', dir
+pretty_print = false
+
+OptionParser.new do |opts|
+  opts.banner = 'Usage: wikibase_to_solr.rb [options]'
+
+  opts.on('-i', '--in FILE', 'The file path to the gzipped Wikibase JSON export file.') do |f|
+    input_file = File.expand_path f, dir
+  end
+
+  opts.on('-o', '--out FILE', 'The file path to output the formatted Solr JSON file.') do |f|
+    output_file = File.expand_path f, dir
+  end
+
+  opts.on('-c', '--config FILE', 'The file path to the property configuration file.') do |f|
+    config_file = File.expand_path f, dir
+  end
+
+  opts.on('-p', '--pretty-print', 'Whether to pretty-print the JSON output.') do
+    pretty_print = true
+  end
+end.parse!
+
+def merge(solr_item, new_props)
+  solr_item.merge(new_props) do |_, old_val, new_val|
+    old_val.nil? ? new_val : (old_val + new_val).uniq
+  end
+end
+
+def merge_transformed_fields(solr_item, claim, export_hash, property_config)
+  if claim.property_id == DigitalScriptorium::PropertyId::ASSOCIATED_NAME_AS_RECORDED
+    merge(solr_item, DigitalScriptorium::NameClaimTransformer.transform(claim, export_hash))
+  elsif claim.property_id == DigitalScriptorium::PropertyId::PRODUCTION_DATE_AS_RECORDED
+    merge(solr_item,
+          DigitalScriptorium::DateClaimTransformer.transform(claim, export_hash, property_config))
+  else
+    merge(solr_item,
+          DigitalScriptorium::ClaimTransformer.transform(claim, export_hash, property_config))
+  end
+end
+
+start_time = Time.now
+
+config = YAML.load_file(config_file)
+
+loading_spinner = TTY::Spinner.new('[:spinner] Loading export data', hide_cursor: true)
+loading_spinner.auto_spin
+
+export_json = Zlib::GzipReader.open(input_file).read
+export_hash = DigitalScriptorium::ExportRepresenter.new(DigitalScriptorium::Export.new)
+                                                   .from_json(export_json)
+                                                   .to_hash
+loaded_time = Time.now
+loading_spinner.success("(#{format('%0.02f', loaded_time - start_time)}s)")
+
+item_count = 0
+generating_spinner = TTY::Spinner.new('[:spinner] Generating Solr documents', hide_cursor: true)
+generating_spinner.auto_spin
+
+File.open(output_file, 'w') do |file|
+  file << '['
+  file << "\n" if pretty_print
+
+  export_hash.each_with_index do |(_, entity), idx|
+    next unless entity.is_a?(DigitalScriptorium::DsItem) &&
+                entity.claims_by_property_id?(DigitalScriptorium::PropertyId::INSTANCE_OF) &&
+                entity.record?
+
+    meta = DigitalScriptorium::DsMeta.new(entity, export_hash)
+    solr_item = { 'qid_meta' => [meta.holding.id, meta.manuscript.id, meta.record.id] }
+
+    [meta.holding, meta.manuscript, meta.record].each do |item|
+      item.claims.each do |property_id, claims|
+        claims.each do |claim|
+          next unless (property_config = config[property_id])
+
+          solr_item = merge_transformed_fields(solr_item, claim, export_hash, property_config)
+        end
+      end
+    end
+
+    file << (pretty_print ? JSON.pretty_generate(solr_item) : JSON.generate(solr_item))
+    file << ',' if idx < export_hash.size - 1
+    file << "\n" if pretty_print
+
+    item_count += 1
+  end
+
+  file << ']'
+end
+
+finish_time = Time.now
+generating_spinner.success("(#{format('%0.02f', finish_time - loaded_time)}s)")
+puts "Generated #{item_count} Solr documents in #{format('%0.02f', finish_time - start_time)} seconds"

metadata

@@ -0,0 +1,121 @@
+--- !ruby/object:Gem::Specification
+name: digital_scriptorium
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Michael Holloway
+bindir: exe
+cert_chain: []
+date: 2025-01-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: multi_json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+- !ruby/object:Gem::Dependency
+  name: representable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+- !ruby/object:Gem::Dependency
+  name: tty-spinner
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: wikibase_representable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+email:
+- michael@mdholloway.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".ignore"
+- ".rspec"
+- ".rubocop.yml"
+- LICENSE.txt
+- README.md
+- Rakefile
+- doc/overview.md
+- lib/digital_scriptorium.rb
+- lib/digital_scriptorium/claim_transformer.rb
+- lib/digital_scriptorium/date_claim_transformer.rb
+- lib/digital_scriptorium/ds_item.rb
+- lib/digital_scriptorium/ds_meta.rb
+- lib/digital_scriptorium/export.rb
+- lib/digital_scriptorium/export_representer.rb
+- lib/digital_scriptorium/holding.rb
+- lib/digital_scriptorium/item_id.rb
+- lib/digital_scriptorium/manuscript.rb
+- lib/digital_scriptorium/name_claim_transformer.rb
+- lib/digital_scriptorium/property_id.rb
+- lib/digital_scriptorium/record.rb
+- lib/digital_scriptorium/version.rb
+- property_config.yml
+- sig/digital_scriptorium.rbs
+- wikibase_to_solr_new.rb
+homepage: https://github.com/mdholloway/digital_scriptorium
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/mdholloway/digital_scriptorium
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.2
+specification_version: 4
+summary: Supporting code for the Digital Scriptorium DS Catalog 2.0 project
+test_files: []