dor_indexing 1.0.0

data/lib/dor_indexing/builders/topic_builder.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+class DorIndexing
+  module Builders
+    # Builds the topic fields for a solr document
+    class TopicBuilder
+      # @param [Array] subjects
+      # @param [String] filter can either be 'topic' or 'name'
+      def self.build(subjects, filter:, remove_trailing_punctuation: false)
+        new(filter:, remove_trailing_punctuation:).build(subjects)
+      end
+
+      def initialize(filter:, remove_trailing_punctuation:)
+        @filter = filter
+        @remove_trailing_punctuation = remove_trailing_punctuation
+      end
+
+      def build(subjects)
+        topics(subjects).flat_map { |topic| flat_topic(topic) }.compact.uniq
+      end
+
+      private
+
+      attr_reader :filter
+
+      def remove_trailing_punctuation?
+        @remove_trailing_punctuation
+      end
+
+      # Filter the subjects we are interested in>
+      # Handles:
+      # parallelValue that contain structuredValue and the parallelValue has the type AND
+      # parallelValue that contain structuredValue each with their own type AND
+      # parallelValue that has a type conferred to the child AND
+      # structuredValue that contains structuredValue where the type can be at the higher or lower level.
+      def topics(subjects)
+        (
+          subjects.flat_map { |subject| basic_value(subject) } +
+          subjects.flat_map { |subject| structured_values(subject) } +
+          parallel_subjects(subjects)
+        ).compact
+      end
+
+      def parallel_subjects(subjects)
+        parallels = subjects.select(&:parallelValue)
+        parallels.flat_map { |subject| parallel_with_type(subject, subject.type) if subject.type } +
+          parallels.flat_map { |subject| topics(subject.parallelValue) unless subject.type }
+      end
+
+      def flat_topic(value)
+        if value.parallelValue.present?
+          value.parallelValue.flat_map { |topic| flat_topic(topic) }
+        elsif remove_trailing_punctuation?
+          # comma, semicolon, and backslash are dropped
+          Array(value.value&.sub(/[ ,;\\]+$/, ''))
+        else
+          Array(value.value)
+        end
+      end
+
+      def parallel_with_type(item, type_from_parent)
+        return unless type_matches_filter?(type_from_parent)
+
+        item
+      end
+
+      def basic_value(subject)
+        return create_fullname(subject) if filter == 'name' && subject.type == 'person'
+        return create_title(subject) if filter == 'name' && subject.type == 'title'
+
+        subject if type_matches_filter?(subject.type)
+      end
+
+      def structured_values(subject)
+        selected = Array(subject.structuredValue).select { |child| type_matches_filter?(child.type) }
+
+        topics(selected)
+      end
+
+      def create_title(title)
+        titles = Cocina::Models::Builders::TitleBuilder.build([title], strategy: :all, add_punctuation: false)
+        titles.map { |value| Cocina::Models::DescriptiveValue.new(value:) }
+      end
+
+      def create_fullname(name)
+        names = NameBuilder.build([name], strategy: :all)
+        names.map { |value| Cocina::Models::DescriptiveValue.new(value:) }
+      end
+
+      def type_matches_filter?(type)
+        (filter == 'name' && %w[person organization title occupation].include?(type)) ||
+          type == filter
+      end
+    end
+  end
+end

data/lib/dor_indexing/cocina_repository.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+class DorIndexing
+  # Interface for retrieving Cocina objects.
+  # In DSA, the concrete implementation backs this with CocinaObjectStore.
+  # In DIA, the concrete implementation backs this with Dor Services Client.
+  class CocinaRepository
+    class RepositoryError < StandardError; end
+
+    # @param [String] druid
+    # @return [Cocina::Models::DROWithMetadata,Cocina::Models::CollectionWithMetadata,Cocina::Models::AdminPolicyWithMetadata]
+    # @raise [RepositoryError] if the object is not found or other error occurs
+    def find(druid)
+      raise NotImplementedError
+    end
+
+    # @param [String] druid
+    # @return [Array<String>] administrative tags
+    # @raise [RepositoryError] if the object is not found or other error occurs
+    def administrative_tags(druid)
+      raise NotImplementedError
+    end
+  end
+end

data/lib/dor_indexing/indexers/administrative_tag_indexer.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+class DorIndexing
+  module Indexers
+    # Index administrative tags for an object.
+    # NOTE: Most of this code was extracted from the dor-services gem:
+    #       https://github.com/sul-dlss/dor-services/blob/v9.0.0/lib/dor/datastreams/identity_metadata_ds.rb#L196-L218
+    class AdministrativeTagIndexer
+      TAG_PART_DELIMITER = ' : '
+      SPECIAL_TAG_TYPES_TO_INDEX = ['Project', 'Registered By'].freeze
+
+      attr_reader :id
+
+      def initialize(id:, administrative_tags:, **)
+        @id = id
+        @administrative_tags = administrative_tags
+      end
+
+      # @return [Hash] the partial solr document for administrative tags
+      # rubocop:disable Metrics/MethodLength
+      # rubocop:disable Metrics/AbcSize
+      # rubocop:disable Metrics/CyclomaticComplexity
+      def to_solr
+        solr_doc = {
+          'tag_ssim' => [],
+          'tag_text_unstemmed_im' => [],
+          'exploded_nonproject_tag_ssim' => []
+        }
+        administrative_tags.each do |tag|
+          tag_prefix, rest = tag.split(TAG_PART_DELIMITER, 2)
+          prefix = tag_prefix.downcase.strip.gsub(/\s/, '_')
+
+          solr_doc['tag_ssim'] << tag # for facet and display
+          solr_doc['tag_text_unstemmed_im'] << tag # for search
+
+          solr_doc['exploded_nonproject_tag_ssim'] += exploded_tags_from(tag) unless prefix == 'project'
+
+          next if SPECIAL_TAG_TYPES_TO_INDEX.exclude?(tag_prefix) || rest.nil?
+
+          (solr_doc["#{prefix}_tag_ssim"] ||= []) << rest.strip
+
+          if prefix == 'project'
+            solr_doc['exploded_project_tag_ssim'] ||= []
+            solr_doc['exploded_project_tag_ssim'] += exploded_tags_from(rest.strip)
+          end
+        end
+        solr_doc
+      end
+      # rubocop:enable Metrics/MethodLength
+      # rubocop:enable Metrics/AbcSize
+      # rubocop:enable Metrics/CyclomaticComplexity
+
+      private
+
+      attr_reader :administrative_tags
+
+      # solrize each possible prefix for the tag, inclusive of the full tag.
+      # e.g., for a tag such as "A : B : C", this will solrize to an _ssim field
+      # that contains ["A",  "A : B",  "A : B : C"].
+      def exploded_tags_from(tag)
+        tag_parts = tag.split(TAG_PART_DELIMITER)
+
+        1.upto(tag_parts.count).map do |i|
+          tag_parts.take(i).join(TAG_PART_DELIMITER)
+        end
+      end
+    end
+  end
+end

data/lib/dor_indexing/indexers/collection_title_indexer.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class DorIndexing
+  module Indexers
+    # Indexes the collection title
+    class CollectionTitleIndexer
+      attr_reader :cocina, :parent_collections
+
+      def initialize(cocina:, parent_collections:, **)
+        @cocina = cocina
+        @parent_collections = parent_collections
+      end
+
+      # @return [Hash] the partial solr document for identifiable concerns
+      def to_solr
+        {}.tap do |solr_doc|
+          parent_collections.each do |related_obj|
+            coll_title = Cocina::Models::Builders::TitleBuilder.build(related_obj.description.title)
+
+            # create/append collection_title_tesim and collection_title_ssim
+            ::Solrizer.insert_field(solr_doc, 'collection_title', coll_title, :stored_searchable, :symbol)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dor_indexing/indexers/composite_indexer.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+class DorIndexing
+  module Indexers
+    # Borrowed from https://github.com/samvera/valkyrie/blob/master/lib/valkyrie/persistence/solr/composite_indexer.rb
+    class CompositeIndexer
+      attr_reader :indexers
+
+      def initialize(*indexers)
+        @indexers = indexers
+      end
+
+      def new(**)
+        Instance.new(indexers, **)
+      end
+
+      # Instance for a composite indexer
+      class Instance
+        attr_reader :indexers
+
+        def initialize(indexers, **)
+          @indexers = indexers.map do |i|
+            i.new(**)
+          rescue ArgumentError => e
+            raise ArgumentError, "Unable to initialize #{i}. #{e.message}"
+          end
+        end
+
+        # @return [Hash] the merged solr document for all the sub-indexers
+        def to_solr
+          indexers.map(&:to_solr).inject({}, &:merge)
+        end
+      end
+    end
+  end
+end

data/lib/dor_indexing/indexers/content_metadata_indexer.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+class DorIndexing
+  module Indexers
+    # Indexes the content metadata
+    class ContentMetadataIndexer
+      attr_reader :cocina
+
+      def initialize(cocina:, **)
+        @cocina = cocina
+      end
+
+      # @return [Hash] the partial solr document for contentMetadata
+      def to_solr
+        {
+          'content_type_ssim' => type(cocina.type),
+          'content_file_mimetypes_ssim' => files.map(&:hasMimeType).uniq,
+          'content_file_count_itsi' => files.size,
+          'shelved_content_file_count_itsi' => shelved_files.size,
+          'resource_count_itsi' => file_sets.size,
+          'preserved_size_dbtsi' => preserved_size, # double (trie) to support very large sizes
+          'content_file_roles_ssim' => files.filter_map(&:use),
+          # first_shelved_image is neither indexed nor multiple
+          'first_shelved_image_ss' => first_shelved_image
+        }
+      end
+
+      private
+
+      def first_shelved_image
+        shelved_files.find { |file| file.filename.end_with?('jp2') }&.filename
+      end
+
+      def shelved_files
+        files.select { |file| file.administrative.shelve }
+      end
+
+      def preserved_size
+        files.select { |file| file.administrative.sdrPreserve }
+             .filter_map(&:size).sum # filter out missing size
+      end
+
+      def files
+        @files ||= file_sets.flat_map { |fs| fs.structural.contains }
+      end
+
+      def file_sets
+        @file_sets ||= Array(cocina.structural&.contains)
+      end
+
+      TYPES = {
+        Cocina::Models::ObjectType.image => 'image',
+        Cocina::Models::ObjectType.manuscript => 'image',
+        Cocina::Models::ObjectType.book => 'book',
+        Cocina::Models::ObjectType.map => 'map',
+        Cocina::Models::ObjectType.three_dimensional => '3d',
+        Cocina::Models::ObjectType.media => 'media',
+        Cocina::Models::ObjectType.webarchive_seed => 'webarchive-seed',
+        Cocina::Models::ObjectType.webarchive_binary => 'webarchive-binary',
+        Cocina::Models::ObjectType.geo => 'geo',
+        Cocina::Models::ObjectType.document => 'document'
+      }.freeze
+
+      def type(object_type)
+        TYPES.fetch(object_type, 'file')
+      end
+    end
+  end
+end

data/lib/dor_indexing/indexers/data_indexer.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+class DorIndexing
+  module Indexers
+    # Indexing provided by ActiveFedora
+    class DataIndexer
+      attr_reader :cocina, :workflow_client
+
+      def initialize(cocina:, workflow_client:, **)
+        @cocina = cocina
+        @workflow_client = workflow_client
+      end
+
+      # rubocop:disable Metrics/AbcSize
+      # rubocop:disable Metrics/MethodLength
+      def to_solr
+        {}.tap do |solr_doc|
+          solr_doc[:id] = cocina.externalIdentifier
+          solr_doc['current_version_isi'] = cocina.version # Argo Facet field "Version"
+          solr_doc['obj_label_tesim'] = cocina.label
+
+          solr_doc['modified_latest_dttsi'] = modified_latest
+          solr_doc['created_at_dttsi'] = created_at
+
+          # is_member_of_collection_ssim is used by dor-services-app for querying for members of a
+          # collection and it is a facet in Argo
+          solr_doc['is_member_of_collection_ssim'] = legacy_collections
+          solr_doc['is_governed_by_ssim'] = legacy_apo # Argo facet
+
+          # Used so that DSA can generate public XML whereas a constituent can find the virtual object it is part of.
+          solr_doc['has_constituents_ssim'] = virtual_object_constituents
+        end.merge(DorIndexing::WorkflowFields.for(druid: cocina.externalIdentifier, version: cocina.version, workflow_client:))
+           .transform_keys(&:to_s)
+      end
+      # rubocop:enable Metrics/AbcSize
+      # rubocop:enable Metrics/MethodLength
+
+      def modified_latest
+        cocina.modified.to_datetime.strftime('%FT%TZ')
+      end
+
+      def created_at
+        cocina.created.to_datetime.strftime('%FT%TZ')
+      end
+
+      def legacy_collections
+        case cocina.type
+        when Cocina::Models::ObjectType.admin_policy, Cocina::Models::ObjectType.collection
+          []
+        else
+          Array(cocina.structural&.isMemberOf).map { |col_id| "info:fedora/#{col_id}" }
+        end
+      end
+
+      def virtual_object_constituents
+        return unless cocina.dro?
+
+        Array(cocina.structural&.hasMemberOrders).first&.members
+      end
+
+      def legacy_apo
+        "info:fedora/#{cocina.administrative.hasAdminPolicy}"
+      end
+    end
+  end
+end

data/lib/dor_indexing/indexers/default_object_rights_indexer.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+class DorIndexing
+  module Indexers
+    # Indexes the default object rights
+    class DefaultObjectRightsIndexer
+      attr_reader :cocina
+
+      def initialize(cocina:, **)
+        @cocina = cocina
+      end
+
+      # @return [Hash] the partial solr document for defaultObjectRights
+      def to_solr
+        return {} unless cocina.administrative.accessTemplate
+
+        {
+          'use_statement_ssim' => use_statement,
+          'copyright_ssim' => copyright,
+          'rights_descriptions_ssim' => 'dark',
+          'default_rights_descriptions_ssim' => Cocina::Models::Builders::RightsDescriptionBuilder.build(cocina)
+        }
+      end
+
+      private
+
+      def use_statement
+        cocina.administrative.accessTemplate.useAndReproductionStatement
+      end
+
+      def copyright
+        cocina.administrative.accessTemplate.copyright
+      end
+    end
+  end
+end

data/lib/dor_indexing/indexers/descriptive_metadata_indexer.rb

@@ -0,0 +1,226 @@
+# frozen_string_literal: true
+
+require 'stanford-mods'
+
+class DorIndexing
+  module Indexers
+    # rubocop:disable Metrics/ClassLength
+    # Indexes the descriptive metadata
+    class DescriptiveMetadataIndexer
+      attr_reader :cocina, :stanford_mods_record
+
+      def initialize(cocina:, **)
+        @cocina = cocina
+        mods_ng = Cocina::Models::Mapping::ToMods::Description.transform(cocina.description, cocina.externalIdentifier)
+        @stanford_mods_record = Stanford::Mods::Record.new.from_nk_node(mods_ng.root)
+      end
+
+      # @return [Hash] the partial solr document for descriptive metadata
+      # rubocop:disable Metrics/MethodLength
+      # rubocop:disable Metrics/AbcSize
+      def to_solr
+        {
+          # title
+          'sw_display_title_tesim' => title,
+          # contributor
+          'author_text_nostem_im' => author_primary, # primary author tokenized but not stemmed
+          'sw_author_tesim' => author_primary, # used for author display in Argo
+          'contributor_text_nostem_im' => author_all, # author names should be tokenized but not stemmed
+          'contributor_orcids_ssim' => orcids,
+          # topic
+          'topic_ssim' => stanford_mods_record.topic_facet&.uniq,
+          'topic_tesim' => stemmable_topics,
+          # publication
+          'originInfo_date_created_tesim' => creation_date,
+          'originInfo_publisher_tesim' => publisher_name,
+          'originInfo_place_placeTerm_tesim' => event_place, # do we want this?
+          'sw_pub_date_facet_ssi' => stanford_mods_record.pub_year_int.to_s, # SW Date facet
+
+          'metadata_format_ssim' => 'mods', # no longer used? https://github.com/search?q=org%3Asul-dlss+metadata_format_ssim&type=code
+
+          # SW facets plus a friend facet
+          'sw_format_ssim' => sw_format, # SW Resource Type facet
+          'mods_typeOfResource_ssim' => resource_type, # MODS Resource Type facet
+          'sw_genre_ssim' => stanford_mods_record.sw_genre, # SW Genre facet
+          'sw_language_ssim' => stanford_mods_record.sw_language_facet, # SW Language facet
+          'sw_subject_temporal_ssim' => stanford_mods_record.era_facet, # SW Era facet
+          'sw_subject_geographic_ssim' => subject_geographic, # SW Region facet
+
+          # all the descriptive data that we want to search on, with different flavors for better recall and precision
+          'descriptive_tiv' => all_search_text, # ICU tokenized, ICU folded
+          'descriptive_text_nostem_i' => all_search_text, # whitespace tokenized, ICU folded, word delimited
+          'descriptive_teiv' => all_search_text # ICU tokenized, ICU folded, minimal stemming
+        }.select { |_k, v| v.present? }
+      end
+      # rubocop:enable Metrics/MethodLength
+      # rubocop:enable Metrics/AbcSize
+
+      private
+
+      def subject_temporal
+        DorIndexing::Builders::TemporalBuilder.build(subjects)
+      end
+
+      def subject_geographic
+        DorIndexing::Builders::GeographicBuilder.build(subjects)
+      end
+
+      def subjects
+        @subjects ||= Array(cocina.description.subject)
+      end
+
+      def author_primary
+        author_builder.build_primary
+      end
+
+      def author_all
+        author_builder.build_all
+      end
+
+      def author_builder
+        @author_builder ||= DorIndexing::Builders::AuthorBuilder.new(Array(cocina.description.contributor))
+      end
+
+      def orcids
+        DorIndexing::Builders::OrcidBuilder.build(Array(cocina.description.contributor))
+      end
+
+      def title
+        Cocina::Models::Builders::TitleBuilder.build(cocina.description.title)
+      end
+
+      def forms
+        @forms ||= Array(cocina.description.form)
+      end
+
+      def resource_type
+        @resource_type ||= forms.select do |form|
+          form.source&.value == 'MODS resource types' &&
+            %w[collection manuscript].exclude?(form.value)
+        end.map(&:value)
+      end
+
+      # See https://github.com/sul-dlss/stanford-mods/blob/master/lib/stanford-mods/searchworks.rb#L244
+      FORMAT = {
+        'cartographic' => 'Map',
+        'manuscript' => 'Archive/Manuscript',
+        'mixed material' => 'Archive/Manuscript',
+        'moving image' => 'Video',
+        'notated music' => 'Music score',
+        'software, multimedia' => 'Software/Multimedia',
+        'sound recording-musical' => 'Music recording',
+        'sound recording-nonmusical' => 'Sound recording',
+        'sound recording' => 'Sound recording',
+        'still image' => 'Image',
+        'three dimensional object' => 'Object',
+        'text' => 'Book'
+      }.freeze
+
+      # rubocop:disable Metrics/CyclomaticComplexity
+      # rubocop:disable Metrics/PerceivedComplexity
+      # rubocop:disable Metrics/AbcSize
+      def sw_format
+        return ['Map'] if resource_type?('software, multimedia') && resource_type?('cartographic')
+        return ['Dataset'] if resource_type?('software, multimedia') && genre?('dataset')
+        return ['Archived website'] if resource_type?('text') && genre?('archived website')
+        return ['Book'] if resource_type?('text') && issuance?('monographic')
+        return ['Journal/Periodical'] if resource_type?('text') && (issuance?('continuing') || issuance?('serial') || frequency?)
+
+        resource_type_formats = flat_forms_for('resource type').map { |form| FORMAT[form.value&.downcase] }.uniq.compact
+        resource_type_formats.delete('Book') if resource_type_formats.include?('Archive/Manuscript')
+
+        return resource_type_formats if resource_type_formats == ['Book']
+
+        genre_formats = flat_forms_for('genre').map { |form| form.value&.capitalize }.uniq
+
+        (resource_type_formats + genre_formats).presence
+      end
+      # rubocop:enable Metrics/CyclomaticComplexity
+      # rubocop:enable Metrics/PerceivedComplexity
+      # rubocop:enable Metrics/AbcSize
+
+      def resource_type?(type)
+        flat_forms_for('resource type').any? { |form| form.value == type }
+      end
+
+      def genre?(genre)
+        flat_forms_for('genre').any? { |form| form.value == genre }
+      end
+
+      def issuance?(issuance)
+        flat_event_notes.any? { |note| note.type == 'issuance' && note.value == issuance }
+      end
+
+      def frequency?
+        flat_event_notes.any? { |note| note.type == 'frequency' }
+      end
+
+      def flat_forms_for(type)
+        forms.flat_map do |form|
+          if form.type == type
+            flat_value(form)
+          else
+            flat_value(form).select { |form_value| form_value.type == type }
+          end
+        end
+      end
+
+      def flat_event_notes
+        @flat_event_notes ||= events.flat_map { |event| flat_event(event) }.flat_map do |event|
+          Array(event.note).flat_map do |note|
+            flat_value(note)
+          end
+        end
+      end
+
+      def pub_year
+        DorIndexing::Selectors::PubYearSelector.build(events)
+      end
+
+      def creation_date
+        @creation_date ||= DorIndexing::Builders::EventDateBuilder.build(creation_event, 'creation')
+      end
+
+      def event_place
+        place_event = events.find { |event| event.type == 'publication' } || events.first
+        DorIndexing::Builders::EventPlaceBuilder.build(place_event)
+      end
+
+      def publisher_name
+        publish_events = events.map { |event| event.parallelEvent&.first || event }
+        return if publish_events.blank?
+
+        DorIndexing::Builders::PublisherNameBuilder.build(publish_events)
+      end
+
+      def stemmable_topics
+        DorIndexing::Builders::TopicBuilder.build(Array(cocina.description.subject), filter: 'topic')
+      end
+
+      def publication_event
+        @publication_event ||= DorIndexing::Selectors::EventSelector.select(events, 'publication')
+      end
+
+      def creation_event
+        @creation_event ||= DorIndexing::Selectors::EventSelector.select(events, 'creation')
+      end
+
+      def events
+        @events ||= Array(cocina.description.event).compact
+      end
+
+      def flat_event(event)
+        event.parallelEvent.presence || Array(event)
+      end
+
+      def flat_value(value)
+        value.parallelValue.presence || value.groupedValue.presence || value.structuredValue.presence || Array(value)
+      end
+
+      def all_search_text
+        @all_search_text ||= DorIndexing::Builders::AllSearchTextBuilder.build(cocina.description)
+      end
+    end
+    # rubocop:enable Metrics/ClassLength
+  end
+end

data/lib/dor_indexing/indexers/embargo_metadata_indexer.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+class DorIndexing
+  module Indexers
+    # Indexes the embargo metadata
+    class EmbargoMetadataIndexer
+      attr_reader :cocina
+
+      def initialize(cocina:, **)
+        @cocina = cocina
+      end
+
+      # These fields are used by the EmbargoReleaseService in dor-services-app
+      # @return [Hash] the partial solr document for embargoMetadata
+      def to_solr
+        {}.tap do |solr_doc|
+          embargo_release_date = embargo_release_date(cocina)
+          if embargo_release_date.present?
+            solr_doc['embargo_status_ssim'] = ['embargoed']
+            solr_doc['embargo_release_dtsim'] = [embargo_release_date.utc.iso8601]
+          end
+        end
+      end
+
+      private
+
+      def embargo_release_date(cocina)
+        cocina.access.embargo.releaseDate if cocina.access.embargo&.releaseDate.present?
+      end
+    end
+  end
+end