dor-services 2.2.4

data/lib/dor/provenance_metadata_service.rb

@@ -0,0 +1,65 @@
+require 'nokogiri'
+require 'date'
+require 'time'
+
+module Dor
+  class ProvenanceMetadataService
+
+    def self.add_provenance(dor_item, workflow_id, event_text)
+      druid = dor_item.pid
+      # workflow_xml = get_workflow_xml(druid, workflow_id)
+      workflow_provenance = create_workflow_provenance(druid, workflow_id, event_text)
+      dsname = 'provenanceMetadata'
+      if dor_item.datastream_names.include?(dsname)
+        ds = dor_item.datastreams[dsname]
+        old_provenance = ds.content
+        ds.ng_xml = update_provenance(old_provenance, workflow_provenance)
+      else
+        ds = dor_item.datastreams[dsname]
+        ds.label = 'Provenance Metadata'
+        ds.ng_xml = workflow_provenance
+      end
+      ds.save
+    end
+
+    # not used
+    def self.get_workflow_xml(druid, workflow_id)
+      Dor::WorkflowService.get_workflow_xml('dor', druid, workflow_id)
+    end
+
+    # @return [String]
+    def self.create_workflow_provenance(druid, workflow_id, event_text)
+      builder = Nokogiri::XML::Builder.new do |xml|
+        xml.provenanceMetadata(:objectId => druid) {
+          xml.agent(:name => 'DOR') {
+            xml.what(:object => druid) {
+              xml.event(:who => "DOR-#{workflow_id}", :when => Time.new.iso8601) {
+                xml.text(event_text)
+              }
+            }
+          }
+        }
+      end
+      builder.doc
+    end
+
+  # Reformat the XML
+  # @param[String]
+  def self.parse_xml_remove_blank_nodes(old_provenance)
+    # http://blog.slashpoundbang.com/post/1454850669/how-to-pretty-print-xml-with-nokogiri
+    Nokogiri::XML(old_provenance) { |x| x.noblanks }
+  end
+
+      # Append new stanzas in the contentMetadata for the googleMETS and technicalMetadata files
+  # @param[String, Hash<Symbol,String>]
+  def self.update_provenance(old_provenance, workflow_provenance)
+    pm_xml = Nokogiri::XML(old_provenance)
+    builder = Nokogiri::XML::Builder.with(pm_xml.at 'provenanceMetadata') do |xml|
+      xml << workflow_provenance.xpath('/provenanceMetadata/agent').first.to_xml
+    end
+    parse_xml_remove_blank_nodes(builder.to_xml)
+  end
+
+  end
+
+end

data/lib/dor/registration_service.rb

@@ -0,0 +1,87 @@
+require 'active_fedora'
+require 'uuidtools'
+require 'xml_models/foxml'
+require 'xml_models/identity_metadata/identity_metadata'
+
+require 'dor/search_service'
+
+module Dor
+  
+  class RegistrationService
+    
+    class << self
+      def register_object(params = {})
+        [:object_type, :label].each do |required_param|
+          unless params[required_param]
+            raise Dor::ParameterError, "#{required_param.inspect} must be specified in call to #{self.name}.register_object"
+          end
+        end
+
+        object_type = params[:object_type]
+        content_model = params[:content_model]
+        admin_policy = params[:admin_policy]
+        label = params[:label]
+        source_id = params[:source_id] || {}
+        other_ids = params[:other_ids] || {}
+        tags = params[:tags] || []
+        parent = params[:parent]
+        pid = nil
+        if params[:pid]
+          pid = params[:pid]
+          existing_pid = SearchService.query_by_id(pid).first
+          unless existing_pid.nil?
+            raise Dor::DuplicateIdError.new(existing_pid), "An object with the PID #{pid} has already been registered."
+          end
+        else
+          pid = Dor::SuriService.mint_id
+        end
+
+        if (other_ids.has_key?(:uuid) or other_ids.has_key?('uuid')) == false
+          other_ids[:uuid] = UUIDTools::UUID.timestamp_create.to_s
+        end
+
+        apo_object = Dor::AdminPolicyObject.load_instance(admin_policy)
+        adm_xml = apo_object.datastreams['administrativeMetadata'].ng_xml
+        agreement_id = adm_xml.at('/administrativeMetadata/registration/agreementId/text()').to_s
+        
+        idmd = IdentityMetadata.new
+
+        unless source_id.empty?
+          source_name = source_id.keys.first
+          source_value = source_id[source_name]
+          existing_pid = SearchService.query_by_id("#{source_name}:#{source_value}").first
+          unless existing_pid.nil?
+            raise Dor::DuplicateIdError.new(existing_pid), "An object with the source ID '#{source_name}:#{source_value}' has already been registered."
+          end
+          idmd.sourceId.source = source_name
+          idmd.sourceId.value = source_value
+        end
+        
+        idmd.objectId = pid
+        idmd.objectCreators << 'DOR'
+        idmd.objectLabels << label
+        idmd.objectTypes << object_type
+        idmd.adminPolicy = admin_policy
+        idmd.agreementId = agreement_id
+        other_ids.each_pair { |name,value| idmd.add_identifier(name,value) }
+        tags.each { |tag| idmd.add_tag(tag) }
+    
+        foxml = Foxml.new(pid, label, content_model, idmd.to_xml, parent)
+        foxml.admin_policy_object = admin_policy
+        rdf = foxml.xml.at('//rdf:Description', { 'rdf' => 'http://www.w3.org/1999/02/22-rdf-syntax-ns#' })
+        rels = adm_xml.xpath('/administrativeMetadata/relationships/*')
+        rels.each { |rel| rdf.add_child(rel.clone) }
+    
+        repo = ActiveFedora.fedora
+        http_response = repo.ingest(foxml.to_xml(:undent_datastreams => true))
+        result = {
+          :response => http_response,
+          :pid => pid
+        }
+        return(result)
+      end
+    end
+    
+  end
+
+end

data/lib/dor/rsolr.rb

@@ -0,0 +1,27 @@
+require 'rest-client'
+
+module Dor
+  
+  class RSolrConnection
+
+    def execute client, request_context
+      $stderr.puts request_context.inspect
+      resource = RestClient::Resource.new(
+        request_context[:uri].to_s,
+        :ssl_client_cert  =>  OpenSSL::X509::Certificate.new(File.read(Dor::Config.fedora.cert_file)),
+        :ssl_client_key   =>  OpenSSL::PKey::RSA.new(File.read(Dor::Config.fedora.key_file), Dor::Config.fedora.key_pass)
+      )
+      result = {}
+      resource.send(request_context[:method]) { |response, request, result, &block|
+        result = {
+          :status => response.net_http_res.code.to_i,
+          :headers => response.net_http_res.to_hash,
+          :body => response.net_http_res.body
+        }
+      }
+      result
+    end
+
+  end
+  
+end

data/lib/dor/sdr_ingest_service.rb

@@ -0,0 +1,117 @@
+require 'lyber-utils'
+
+module Dor
+  class SdrIngestService
+
+    Config.declare(:sdr) do
+  	  local_workspace_root '/dor/workspace'
+      local_export_home '/dor/export'
+      datastreams do
+        contentMetadata 'required'
+        descMetadata 'required'
+        identityMetadata 'required'
+        provenanceMetadata 'required'
+        relationshipMetadata 'required'
+        technicalMetadata 'required'
+        rightsMetadata 'optional'
+        sourceMetadata 'optional'
+      end
+    end
+
+    # Some boilerplace entries for the bagit metadata file
+    METADATA_INFO =  {
+        'Source-Organization' => 'Stanford University Libraries',
+        'Stanford-Content-Metadata'  =>  'data/metadata/contentMetadata.xml',
+        'Stanford-Identity-Metadata'  =>  'data/metadata/identityMetadata.xml',
+        'Stanford-Provenance-Metadata'  =>  'data/metadata/provenanceMetadata.xml'
+    }
+
+    # Create a bagit object and fill it with content
+    # Then tar it
+    # @param [LyberCore::Robots::WorkItem]
+    def self.transfer(dor_item, agreement_id)
+      druid = dor_item.pid
+      content_dir = Druid.new(druid).path(Config.sdr.local_workspace_root)
+
+      # Create the bag
+      bag_dir = File.join(Config.sdr.local_export_home, druid)
+      export_bag = LyberUtils::BagitBag.new(bag_dir)
+
+      # Fill the bag
+      export_bag.add_content_files(content_dir, use_links=true)
+      add_metadata_datastreams(dor_item, export_bag)
+      export_bag.write_metadata_info(metadata_info(druid, agreement_id))
+      export_bag.write_manifests()
+      export_bag.validate()
+
+      unless LyberUtils::FileUtilities.tar_object(bag_dir)
+        raise 'Unable to tar the bag'
+      end
+
+      # Now bootstrap SDR workflow queue to start SDR robots
+      Dor::WorkflowService.create_workflow('sdr', druid, 'sdrIngestWF', read_sdr_workflow_xml(), {:create_ds => false})
+    end
+
+    # Read in the XML file needed to initialize the SDR workflow
+    # @return [String]
+    def self.read_sdr_workflow_xml()
+      return IO.read(File.join("#{ROBOT_ROOT}", "config", "workflows", "sdrIngestWF", "sdrIngestWF.xml"))
+    end
+
+    # For each of the metadata files or datastreams, create a file in in the bag's data/metadata folder
+    # @param[String, LyberUtils::BagitBag]
+    def self.add_metadata_datastreams(dor_item, export_bag)
+      Config.sdr.datastreams.to_hash.each_pair do |ds_name, required|
+        # ds_name in this context is a symbol, so convert it to a string
+        filename = "#{ds_name.to_s}.xml"
+        metadata_string = self.get_datastream_content(dor_item, ds_name.to_s, required)
+        self.export_metadata_string(metadata_string, filename, export_bag) unless metadata_string.nil?
+      end
+    end
+
+    # create a link to a metadata file in the bag's data/metadata folder
+    def self.export_metadata_file(metadata_dir, filename, bag)
+      metadata_file = File.join(metadata_dir, filename)
+      bag_metadata_dir = File.join(bag.bag_dir, 'data', 'metadata')
+      if (File.exist?(metadata_file))
+        bag_file = File.join(bag_metadata_dir, filename)
+        File.link(metadata_file, bag_file)
+        return true
+      else
+        return false
+      end
+    end
+
+    # return the content of the specied datastream if it exists
+    # if non-existant, return nil or raise exception depending on value of required
+    def self.get_datastream_content(dor_item, ds_name, required)
+      ds = (ds_name == 'relationshipMetadata' ? 'RELS-EXT' : ds_name)
+      if dor_item.datastreams_in_fedora.keys.include?(ds)
+        return dor_item.datastreams[ds].content
+      elsif (required == 'optional')
+        return nil
+      else
+        raise "required datastream #{ds_name} not found in DOR"
+      end
+    end
+
+    # create a file in the bag's data/metadata folder containing the metadata string
+    # @param[String, String, LyberUtils::BagitBag]
+    def self.export_metadata_string(metadata_string, filename, bag)
+      bag.add_metadata_file_from_string(metadata_string, filename)
+    end
+
+    # merge item-specific data into the standard hash of metadata information
+    # @param [String, String]
+    def self.metadata_info(druid, agreement_id)
+       item_info = {
+              'External-Identifier' =>  druid,
+              'Stanford-Agreement-ID'  =>  agreement_id
+      }
+      merged_info = item_info.merge(METADATA_INFO)
+      return merged_info
+    end
+
+  end
+
+end

data/lib/dor/search_service.rb

@@ -0,0 +1,86 @@
+require 'json'
+require 'active_support/core_ext'
+
+module Dor
+  
+  class SearchService
+
+    RISEARCH_TEMPLATE = "select $object from <#ri> where $object <dc:identifier> '%s'"
+
+    Config.declare(:gsearch) { 
+      url nil 
+      instance_eval do
+        def client
+          RestClient::Resource.new(
+            self.url,
+            :ssl_client_cert  =>  OpenSSL::X509::Certificate.new(File.read(Config.fedora.cert_file)),
+            :ssl_client_key   =>  OpenSSL::PKey::RSA.new(File.read(Config.fedora.key_file), Config.fedora.key_pass)
+          )
+        end
+      end
+    }
+    
+    class << self
+      
+      def reindex(*pids)
+        fedora_client = Config.fedora.client
+        solr_client = Config.gsearch.client
+        xsl = Nokogiri::XSLT(File.read(File.expand_path('../../gsearch/demoFoxmlToSolr.xslt', __FILE__)))
+        pids.in_groups_of(20, false) do |group|
+          doc = Nokogiri::XML('<update/>')
+          group.each do |pid|
+            begin
+              foxml = Dor::Base.get_foxml(pid,true)
+              doc.root.add_child(xsl.transform(foxml).root)
+            rescue RestClient::ResourceNotFound
+              doc.root.add_child("<delete><id>#{pid}</id></delete>")
+            end
+          end
+          yield group if block_given?
+          solr_client['update'].post(doc.to_xml, :content_type => 'application/xml')
+        end
+        pids
+      end
+
+      def risearch(query, opts = {})
+        client = Config.fedora.client['risearch']
+        client.options[:timeout] = opts.delete(:timeout)
+        query_params = {
+          :type => 'tuples',
+          :lang => 'itql',
+          :format => 'CSV',
+          :limit => '1000',
+          :stream => 'on',
+          :query => query
+        }.merge(opts)
+        result = client.post(query_params)
+        result.split(/\n/)[1..-1].collect { |pid| pid.chomp.sub(/^info:fedora\//,'') }
+      end
+      
+      def gsearch(params)
+        client = Config.gsearch.client
+        query_params = params.merge(:wt => 'json')
+        query_string = query_params.collect { |k,v| 
+          if v.is_a?(Array)
+            v.collect { |vv| "#{k}=#{URI.encode(vv.to_s)}" }.join('&')
+          else
+            "#{k}=#{URI.encode(v.to_s)}" 
+          end
+        }.join('&')
+        result = JSON.parse(client["select?#{query_string}"].get)
+      end
+      
+      def query_by_id(id)
+        if id.is_a?(Hash) # Single valued: { :google => 'STANFORD_0123456789' }
+          id = id.collect { |*v| v.join(':') }.first
+        elsif id.is_a?(Array) # Two values: [ 'google', 'STANFORD_0123456789' ]
+          id = id.join(':')
+        end
+        self.risearch(RISEARCH_TEMPLATE % id)
+      end
+
+    end
+    
+  end
+  
+end

data/lib/dor/suri_service.rb

@@ -0,0 +1,37 @@
+require 'rest-client'
+require 'active_fedora'
+
+module Dor
+  class SuriService
+    
+    Config.declare(:suri) do
+  	  url nil
+  	  user nil
+  	  pass nil
+  	  id_namespace 'druid'
+  	  mint_ids false
+    end
+    
+    # If Dor::Config.suri.mint_ids is set to true, then this method
+    # returns Config.suri.id_namespace:id_from_suri
+    # Throws an exception if there were any problems
+    def self.mint_id
+      unless(Config.suri.mint_ids)
+        return Fedora::Repository.instance.nextid
+      end
+      
+      #Post with no body
+      resource = RestClient::Resource.new("#{Config.suri.url}/suri2/namespaces/#{Config.suri.id_namespace}/identifiers",
+                                :user => Config.suri.user, :password => Config.suri.pass)
+      id = resource.post('').chomp
+
+      return "#{Config.suri.id_namespace}:#{id.strip}"
+
+#    rescue Exception => e
+#      Rails.logger.error("Unable to mint id from suri: #{e.to_s}")
+#      raise e
+    end
+  
+    
+  end
+end

data/lib/dor/tei2dc.xslt

@@ -0,0 +1,102 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+    xmlns:tei="http://www.tei-c.org/ns/1.0"
+    xmlns:dc="http://purl.org/dc/elements/1.1/"
+    xmlns:dcterms="http://purl.org/dc/terms/"
+    xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" 
+    exclude-result-prefixes="tei xsl"
+    version="1.0">
+    
+    <xsl:output xml:space="default" indent="yes"/>
+
+    <xsl:template match="/">
+        <oai_dc:dc xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd">
+            <xsl:apply-templates/>
+        </oai_dc:dc>
+    </xsl:template>
+
+    <xsl:template match="tei:teiHeader">
+        <xsl:apply-templates/>
+    </xsl:template>
+    
+    <xsl:template match="tei:fileDesc/tei:titleStmt/tei:title">
+        <xsl:call-template name="ptext"><xsl:with-param name="element-name">dc:title</xsl:with-param></xsl:call-template>
+    </xsl:template>
+    
+    <xsl:template match="tei:fileDesc/tei:titleStmt/tei:author">
+        <xsl:call-template name="ptext"><xsl:with-param name="element-name">dc:creator</xsl:with-param></xsl:call-template>
+    </xsl:template>
+    
+    <xsl:template match="tei:profileDesc/tei:textClass/tei:keywords/tei:list/tei:item">
+        <xsl:call-template name="ptext"><xsl:with-param name="element-name">dc:subject</xsl:with-param></xsl:call-template>
+    </xsl:template>
+    
+    <xsl:template match="tei:encodingDesc/tei:refsDecl|tei:encodingDesc/tei:projectDesc|tei:encodingDesc/tei:editorialDesc">
+        <xsl:call-template name="ptext"><xsl:with-param name="element-name">dc:description</xsl:with-param></xsl:call-template>
+    </xsl:template>
+    
+    <xsl:template match="tei:fileDesc/tei:publicationStmt/tei:publisher/tei:publisher|tei:fileDesc/tei:publicationStmt/tei:publisher/tei:pubPlace">
+        <xsl:call-template name="ptext"><xsl:with-param name="element-name">dc:publisher</xsl:with-param></xsl:call-template>
+    </xsl:template>
+    
+    <xsl:template match="tei:fileDesc/tei:titleStmt/tei:editor|tei:fileDesc/tei:titleStmt/tei:funder|tei:fileDesc/tei:titleStmt/tei:sponsor|tei:fileDesc/tei:titleStmt/tei:principle">
+        <xsl:call-template name="ptext"><xsl:with-param name="element-name">dc:contributor</xsl:with-param></xsl:call-template>
+    </xsl:template>
+    
+    <xsl:template match="tei:fileDesc/tei:publicationStmt/tei:date">
+        <xsl:call-template name="ptext"><xsl:with-param name="element-name">dc:date</xsl:with-param></xsl:call-template>
+    </xsl:template>
+    
+    <xsl:template match="tei:extent/tei:seg[@type='size']">
+        <xsl:call-template name="ptext"><xsl:with-param name="element-name">dcterms:extent</xsl:with-param></xsl:call-template>
+    </xsl:template>
+    
+    <xsl:template match="tei:fileDesc/tei:publicationStmt/tei:idno[@type='ARK']">
+        <xsl:call-template name="ptext"><xsl:with-param name="element-name">dc:identifier</xsl:with-param></xsl:call-template>
+    </xsl:template>
+    
+    <xsl:template match="tei:sourceDesc/tei:bibful/tei:publicationStmt/tei:publisher|tei:sourceDesc/tei:bibful/tei:publicationStmt/tei:pubPlace|tei:sourceDesc/tei:bibful/tei:publicationStmt/tei:date|tei:sourceDesc/tei:bibl">
+        <xsl:call-template name="ptext"><xsl:with-param name="element-name">dc:source</xsl:with-param></xsl:call-template>
+    </xsl:template>
+    
+    <xsl:template match="tei:profileDesc/tei:langUsage/tei:language">
+        <xsl:call-template name="ptext"><xsl:with-param name="element-name">dc:language</xsl:with-param></xsl:call-template>
+    </xsl:template>
+    
+    <xsl:template match="tei:fileDesc/tei:seriesStmt/tei:title">
+        <xsl:call-template name="ptext"><xsl:with-param name="element-name">dc:relation</xsl:with-param></xsl:call-template>
+    </xsl:template>
+    
+    <xsl:template match="tei:fileDesc/tei:publicationStmt/tei:availability">
+        <xsl:call-template name="ptext"><xsl:with-param name="element-name">dcterms:accessRights</xsl:with-param></xsl:call-template>
+    </xsl:template>
+    
+    <xsl:template match="tei:notesStmt/tei:note[@type='summary']">
+        <xsl:call-template name="ptext"><xsl:with-param name="element-name">dcterms:abstract</xsl:with-param></xsl:call-template>
+    </xsl:template>
+
+<!-- 
+    <xsl:template match="tei:notesStmt/tei:note[not(@type)]">
+        <xsl:call-template name="ptext"><xsl:with-param name="element-name">dcterms:note</xsl:with-param></xsl:call-template>
+    </xsl:template>
+-->
+    
+    <xsl:template name="ptext">
+        <xsl:param name="element-name"/>
+        <xsl:variable name="text">
+            <xsl:for-each select=".|./tei:p">
+                <xsl:variable name="t" select="normalize-space(./text())"/>
+                <xsl:if test="string-length($t) &gt; 0">
+                    <xsl:value-of select="$t"/>
+                </xsl:if>
+            </xsl:for-each>
+        </xsl:variable>
+        <xsl:if test="string-length($text) &gt; 0">
+            <xsl:element name="{$element-name}"><xsl:value-of select="$text"/></xsl:element>
+        </xsl:if>
+    </xsl:template>
+    
+    <xsl:template match="text()|@*"/>
+    
+</xsl:stylesheet>