dor-services 2.2.4 → 4.4.10

data/lib/dor/migrations/identifiable/fix_model_assertions.rb

@@ -0,0 +1,13 @@
+Dor::Identifiable.on_upgrade '3.6.1', 'Assert correct models' do |obj|
+  applied = false
+  if obj.relationships.any? { |r| r.predicate.to_s == 'info:fedora/fedora-system:def/model#hasModel' && r.object.to_s == 'info:fedora/hydra:commonMetadata' }
+    obj.remove_relationship :has_model, 'info:fedora/hydra:commonMetadata'
+    applied = true
+  end
+  
+  unless obj.relationships.predicates.any? { |p| p.to_s == 'info:fedora/fedora-system:def/model#hasModel' }
+    obj.assert_content_model
+    applied = true
+  end
+  applied
+end

data/lib/dor/migrations/identifiable/record_remediation.rb

@@ -0,0 +1,18 @@
+Dor::Identifiable.on_upgrade '3.6.1', 'Record Remediation Version' do |obj|
+  version_tag = obj.identityMetadata.find_by_terms(:tag).find { |e| e.text =~ /Remediated By\s*:\s*(.+)/ }
+  add_tag = false
+  if version_tag
+    current_version = Gem::Version.new($1)
+    if current_version < Gem::Version.new(Dor::VERSION)
+      version_tag.remove
+      add_tag = true
+    end
+  else
+    add_tag = true
+  end
+  
+  if add_tag
+    obj.identityMetadata.add_value :tag, "Remediated By : #{Dor::VERSION}"
+  end
+  add_tag
+end

data/lib/dor/migrations/identifiable/uriify_augmented_contentlocation_refs.rb

@@ -0,0 +1,18 @@
+Dor::Identifiable.on_upgrade '3.14.8', 'Fix up invalid URIs in content-augmented datastreams' do |obj|
+  bad_content_location_uri = begin
+    URI.parse(obj.send("content-augmented").dsLocation)
+    false
+  rescue URI::InvalidURIError
+    true
+  rescue 
+    false
+  end
+
+  next unless bad_content_location_uri
+
+  parts = obj.send("content-augmented").dsLocation.split('/')
+  parts[parts.length - 1] = URI.escape(parts.last)
+  obj.send("content-augmented").dsLocation = parts.join('/')
+
+  obj.send("content-augmented").save
+end

data/lib/dor/migrations/identifiable/uriify_contentlocation_refs.rb

@@ -0,0 +1,18 @@
+Dor::Identifiable.on_upgrade '3.11.6', 'Fix up invalid URIs in objects' do |obj|
+  bad_content_location_uri = begin
+    URI.parse(obj.content.dsLocation)
+    false
+  rescue URI::InvalidURIError
+    true
+  rescue 
+    false
+  end
+
+  next unless bad_content_location_uri
+
+  parts = obj.content.dsLocation.split('/')
+  parts[parts.length - 1] = URI.escape(parts.last)
+  obj.content.dsLocation = parts.join('/')
+
+  obj.content.save
+end

data/lib/dor/migrations/processable/unify_workflows.rb

@@ -0,0 +1,17 @@
+Dor::Processable.on_upgrade '3.5.0', 'Replace individual *WF datastreams with unified workflows datastream' do |obj|
+  run = false
+  # Touch workflows datastream to ensure it gets saved
+  if obj.workflows.new?
+    obj.workflows.content
+    run = true
+  end
+      
+  # Remove individual *WF datastreams
+  obj.datastreams.each_pair do |dsid,ds| 
+    if ds.controlGroup == 'E' and dsid =~ /WF$/
+      ds.delete 
+      run = true
+    end
+  end
+  run
+end

data/lib/dor/migrations/versionable/add_missing_version_md.rb

@@ -0,0 +1,9 @@
+Dor::Versionable.on_upgrade '3.12.2', 'Add missing versionMetadata' do |obj|
+  run = false
+  vm = obj.datastreams['versionMetadata']
+  if(vm.content.nil? || vm.content.strip == '' || vm.new?) # We do not have a versionMetadata ds
+    vm.content = vm.ng_xml.to_s
+    run = true
+  end
+  run
+end

data/lib/dor/models/admin_policy_object.rb

@@ -0,0 +1,16 @@
+module Dor
+  class AdminPolicyObject < ::ActiveFedora::Base
+    include Identifiable
+    include Governable
+    include Editable
+    include Describable
+    include Processable
+    include Versionable
+
+    has_many :things, :property => :is_governed_by, :inbound => :true, :class_name => "ActiveFedora::Base"
+    has_object_type 'adminPolicy'
+    has_metadata :name => "administrativeMetadata", :type => Dor::AdministrativeMetadataDS, :label => 'Administrative Metadata'
+    has_metadata :name => "roleMetadata", :type => Dor::RoleMetadataDS, :label => 'Role Metadata'
+    has_metadata :name => "defaultObjectRights", :type => Dor::DefaultObjectRightsDS, :label => 'Default Object Rights'
+  end
+end

data/lib/dor/models/assembleable.rb

@@ -0,0 +1,14 @@
+module Dor
+  module Assembleable
+    
+    def initialize_workspace(source=nil)
+      if(source.nil?)
+        druid = DruidTools::Druid.new(self.pid,Config.stacks.local_workspace_root)
+        druid.mkdir
+      else
+        druid = DruidTools::Druid.new(self.pid, Config.stacks.local_workspace_root)
+        druid.mkdir_with_final_link(source)
+      end
+    end
+  end
+end

data/lib/dor/models/collection.rb

@@ -0,0 +1,14 @@
+module Dor
+  class Collection < ::ActiveFedora::Base
+    include Identifiable
+    include Processable
+    include Governable
+    include Describable
+    include Publishable
+    include Versionable
+    include Discoverable
+
+    has_many :members, :property => :is_member_of_collection, :inbound => true, :class_name => "ActiveFedora::Base"
+    has_object_type 'collection'
+  end
+end

data/lib/dor/models/contentable.rb

@@ -0,0 +1,227 @@
+module Dor
+  module Contentable
+    extend ActiveSupport::Concern
+
+    #add a file to a resource, not to be confused with add a resource to an object
+    def add_file file, resource, file_name, mime_type=nil,publish='no', shelve='no', preserve='no'
+      contentMD=self.datastreams['contentMetadata']
+      xml=contentMD.ng_xml
+      #make sure the resource exists
+      if xml.search('//resource[@id=\''+resource+'\']').length == 0
+        raise 'resource doesnt exist.'
+      end
+      sftp=Net::SFTP.start(Config.content.content_server,Config.content.content_user,:auth_methods=>['publickey'])
+      druid_tools=DruidTools::Druid.new(self.pid,Config.content.content_base_dir)
+      location=druid_tools.path(file_name)
+      oldlocation=location.gsub('/'+self.pid.gsub('druid:',''),'')
+      md5=Digest::MD5.file(file.path).hexdigest
+      sha1=Digest::SHA1.file(file.path).hexdigest
+      size=File.size?(file.path)
+      #update contentmd
+      file_hash={:name=>file_name,:md5 => md5, :publish=>publish, :shelve=> shelve, :preserve => preserve, :size=>size.to_s, :sha1=>sha1, :mime_type => mime_type}
+      begin
+        request=sftp.stat!(location.gsub(file_name,''))
+        begin
+          request=sftp.stat!(location)
+          raise 'The file '+file_name+' already exists!'
+        rescue Net::SFTP::StatusException
+          sftp.upload!(file.path,location)
+          self.contentMetadata.add_file file_hash,resource
+        end
+      rescue Net::SFTP::StatusException
+        #the directory layout doesnt match the new style, so use the old style.
+        begin
+          request=sftp.stat!(oldlocation)
+          raise 'The file '+file_name+' already exists!'
+        rescue Net::SFTP::StatusException
+          #the file doesnt already exist, which is good. Add it
+          sftp.upload!(file.path,oldlocation)
+          self.contentMetadata.add_file file_hash,resource
+        end
+      end
+      #can only arrive at this point if a non status exception occurred.
+    end
+
+    def replace_file file,file_name
+      sftp=Net::SFTP.start(Config.content.content_server,Config.content.content_user,:auth_methods=>['publickey'])
+      item=Dor::Item.find(self.pid)
+      druid_tools=DruidTools::Druid.new(self.pid,Config.content.content_base_dir)
+      location=druid_tools.path(file_name)
+      oldlocation=location.gsub('/'+self.pid.gsub('druid:',''),'')
+
+      md5=Digest::MD5.file(file.path).hexdigest
+      sha1=Digest::SHA1.file(file.path).hexdigest
+      size=File.size?(file.path)
+      #update contentmd
+      file_hash={:name=>file_name,:md5 => md5, :size=>size.to_s, :sha1=>sha1}
+      begin
+        request=sftp.stat!(location)
+        sftp.upload!(file.path,location)
+        #this doesnt allow renaming files
+        item.contentMetadata.update_file(file_hash, file_name)
+      rescue
+        sftp.upload!(file.path,oldlocation)
+        item.contentMetadata.update_file(file_hash, file_name)
+      end
+    end
+
+    def get_preserved_file file, version
+      preservation_server=Config.content.sdr_server+'/sdr/objects/'+self.pid+"/content/"
+      file=URI.encode(file)
+      add=preservation_server+file+"?version="+version
+      uri = URI(add)
+      req = Net::HTTP::Get.new(uri.request_uri)
+      req.basic_auth Config.content.sdr_user, Config.content.sdr_pass
+      res = Net::HTTP.start(uri.hostname, uri.port, :use_ssl => uri.scheme == 'https') {|http|
+        http.request(req)
+      }
+    end
+
+    def get_file file
+      druid_tools=DruidTools::Druid.new(self.pid,Config.content.content_base_dir)
+      location=druid_tools.path(file)
+      oldlocation=location.gsub('/'+file,'').gsub('/'+self.pid.gsub('druid:',''),'')+'/'+file
+      sftp=Net::SFTP.start(Config.content.content_server,Config.content.content_user,:auth_methods=>['publickey'])
+      begin
+        data=sftp.download!(location)
+      rescue
+        data=sftp.download!(oldlocation)
+      end
+    end
+    def remove_file filename
+      druid_tools=DruidTools::Druid.new(self.pid,Config.content.content_base_dir)
+      location=druid_tools.path(filename)
+      oldlocation=location.gsub('/'+self.pid.gsub('druid:',''),'')
+      sftp=Net::SFTP.start(Config.content.content_server,Config.content.content_user,:auth_methods=>['publickey'])
+      begin
+        data=sftp.remove!(location)
+      rescue
+        #if the file doesnt exist, that is ok, not all files will be present in the workspace
+        begin
+          data=sftp.remove!(oldlocation)
+        rescue Net::SFTP::StatusException
+        end
+      end
+      self.contentMetadata.remove_file filename
+    end
+    def rename_file old_name, new_name
+      druid_tools=DruidTools::Druid.new(self.pid,Config.content.content_base_dir)
+      location=druid_tools.path(old_name)
+      oldlocation=location.gsub('/'+self.pid.gsub('druid:',''),'')
+      sftp=Net::SFTP.start(Config.content.content_server,Config.content.content_user,:auth_methods=>['publickey'])
+      begin
+        data=sftp.rename!(location,location.gsub(old_name,new_name))
+      rescue
+        data=sftp.rename!(oldlocation,oldlocation.gsub(old_name,new_name))
+      end
+      self.contentMetadata.rename_file(old_name, new_name)
+    end
+    def remove_resource resource_name
+      #run delete for all of the files in the resource
+      xml=self.contentMetadata.ng_xml
+      files=xml.search('//resource[@id=\''+resource_name+'\']/file').each do |file|
+        self.remove_file(file['id'])
+      end
+      #remove the resource record from the metadata and renumber the resource sequence
+      self.contentMetadata.remove_resource resource_name
+    end
+    #list files in the workspace
+    def list_files
+      filename='none'
+      files=[]
+      sftp=Net::SFTP.start(Config.content.content_server,Config.content.content_user,:auth_methods=>['publickey'])
+      druid_tools=DruidTools::Druid.new(self.pid,Config.content.content_base_dir)
+      location=druid_tools.path(filename).gsub(filename,'')
+      oldlocation=location.gsub('/'+self.pid.gsub('druid:',''),'')
+      begin
+        sftp.dir.entries(location, "*") do |file|
+          files<<file.name
+        end
+      rescue
+        begin
+          sftp.dir.glob(oldlocation, "*") do |file|
+            files<<file.name
+          end
+        rescue Net::SFTP::StatusException
+          return files
+        end
+      end
+      return files
+    end
+
+    # Appends contentMetadata file resources from the source objects to this object
+    # @param [Array<String>] source_obj_pids ids of the secondary objects that will get their contentMetadata merged into this one
+    # @param [Logger] logger optional logger to record warnings.  Otherwise, warnings get sent to STDOUT
+    def copy_file_resources source_obj_pids
+      primary_cm = contentMetadata.ng_xml
+      base_id = primary_cm.at_xpath('/contentMetadata/@objectId').value
+      max_sequence = primary_cm.at_xpath('/contentMetadata/resource[last()]/@sequence').value.to_i
+
+      source_obj_pids.each do |src_pid|
+        source_obj = Dor::Item.find src_pid
+        source_cm = source_obj.contentMetadata.ng_xml
+
+        # Copy the resources from each source object
+        source_cm.xpath('/contentMetadata/resource').each do |old_resource|
+          max_sequence += 1
+          resource_copy = old_resource.clone
+          resource_copy['sequence'] = "#{max_sequence}"
+
+          # Append sequence number to each secondary filename, then
+          # look for filename collisions with the primary object
+          resource_copy.xpath('file').each do |secondary_file|
+            secondary_file['id'] = new_secondary_file_name(secondary_file['id'], max_sequence)
+
+            if primary_cm.at_xpath("//file[@id = '#{secondary_file["id"]}']")
+              raise Dor::Exception.new "File '#{secondary_file['id']}' from secondary object #{src_pid} already exist in primary object: #{self.pid}"
+            end
+          end
+
+          if old_resource['type']
+            resource_copy['id'] = "#{old_resource['type']}_#{max_sequence}"
+          else
+            resource_copy['id'] = "#{base_id}_#{max_sequence}"
+          end
+
+          lbl = old_resource.at_xpath 'label'
+          if lbl && lbl.text =~ /^(.*)\s+\d+$/
+            resource_copy.at_xpath('label').content = "#{$1} #{max_sequence}"
+          end
+
+          primary_cm.at_xpath('/contentMetadata/resource[last()]').add_next_sibling resource_copy
+          attr_node = primary_cm.create_element 'attr', src_pid, :name => 'mergedFromPid'
+          resource_copy.first_element_child.add_previous_sibling attr_node
+          attr_node = primary_cm.create_element 'attr', old_resource['id'], :name => 'mergedFromResource'
+          resource_copy.first_element_child.add_previous_sibling attr_node
+        end
+      end
+      self.contentMetadata.content_will_change!
+    end
+
+    def new_secondary_file_name old_name, sequence_num
+      if old_name =~ /^(.*)\.(.*)$/
+        return "#{$1}_#{sequence_num}.#{$2}"
+      else
+        return "#{old_name}_#{sequence_num}"
+      end
+    end
+
+    # Clears RELS-EXT relationships, sets the isGovernedBy relationship to the SDR Graveyard APO
+    # @param [String] tag optional String of text that is concatenated to the identityMetadata/tag "Decomissioned : "
+    def decomission tag = nil
+      # remove isMemberOf and isMemberOfCollection relationships
+      clear_relationship :is_member_of
+      clear_relationship :is_member_of_collection
+      # remove isGovernedBy relationship
+      clear_relationship :is_governed_by
+      # add isGovernedBy to graveyard APO druid:sw909tc7852
+      # SEARCH BY dc title for 'SDR Graveyard'
+      add_relationship :is_governed_by, ActiveFedora::Base.find(Dor::SearchService.sdr_graveyard_apo_druid)
+      # eliminate contentMetadata. set it to <contentMetadata/> ?
+      contentMetadata.content = '<contentMetadata/>'
+      # eliminate rightsMetadata. set it to <rightsMetadata/> ?
+      rightsMetadata.content = '<rightsMetadata/>'
+      add_tag "Decommissioned : #{tag}"
+    end
+  end
+end

data/lib/dor/models/describable.rb

@@ -0,0 +1,194 @@
+module Dor
+	module Describable
+		extend ActiveSupport::Concern
+
+		DESC_MD_FORMATS = {
+			"http://www.tei-c.org/ns/1.0" => 'tei',
+			"http://www.loc.gov/mods/v3" =>	 'mods'
+		}
+		class CrosswalkError < Exception; end
+		
+		included do
+			has_metadata :name => "descMetadata", :type => Dor::DescMetadataDS, :label => 'Descriptive Metadata', :control_group => 'M'
+		end
+
+		def fetch_descMetadata_datastream
+			candidates = self.datastreams['identityMetadata'].otherId.collect { |oid| oid.to_s }
+			metadata_id = Dor::MetadataService.resolvable(candidates).first
+			unless metadata_id.nil?
+				return Dor::MetadataService.fetch(metadata_id.to_s)
+			else
+				return nil
+			end
+		end
+
+		def build_descMetadata_datastream(ds)
+			content = fetch_descMetadata_datastream
+			unless content.nil?
+				ds.dsLabel = 'Descriptive Metadata'
+				ds.ng_xml = Nokogiri::XML(content)
+				ds.ng_xml.normalize_text!
+				ds.content = ds.ng_xml.to_xml
+			end
+		end
+		
+		# Generates Dublin Core from the MODS in the descMetadata datastream using the LoC mods2dc stylesheet
+		#		Should not be used for the Fedora DC datastream
+		# @raise [Exception] Raises an Exception if the generated DC is empty or has no children
+		def generate_dublin_core
+			format = self.metadata_format
+			if format.nil?
+				raise CrosswalkError, "Unknown descMetadata namespace: #{metadata_namespace.inspect}"
+			end
+			xslt = Nokogiri::XSLT(File.new(File.expand_path(File.dirname(__FILE__) + "/#{format}2dc.xslt")) )
+			dc_doc = xslt.transform(Nokogiri::XML(self.add_collection_reference))
+			# Remove empty nodes
+			dc_doc.xpath('/oai_dc:dc/*[count(text()) = 0]').remove
+			if(dc_doc.root.nil? || dc_doc.root.children.size == 0)
+				raise CrosswalkError, "Dor::Item#generate_dublin_core produced incorrect xml:\n#{dc_doc.to_xml}"
+			end
+			dc_doc
+		end
+		#returns the desc metadata a relatedItem with information about the collection this object belongs to for use in published mods and mods to DC conversion
+	  def add_collection_reference
+	    if not self.methods.include? :public_relationships
+        return self.descMetadata.ng_xml.to_s
+      end
+	    relationships=self.public_relationships
+	    xml=Nokogiri::XML(self.descMetadata.ng_xml.to_s)
+	    
+	    collections=relationships.search('//rdf:RDF/rdf:Description/fedora:isMemberOfCollection','fedora' => 'info:fedora/fedora-system:def/relations-external#', 'rdf' => 'http://www.w3.org/1999/02/22-rdf-syntax-ns#' 	)
+	    #if there is an existing relatedItem node with type=host and a child typeOfResource @collection=yes dont add anything
+	    existing_node=xml.search('//mods:relatedItem/mods:typeOfResource[@collection=\'yes\']', 'mods' => 'http://www.loc.gov/mods/v3')
+      if(existing_node.length>0)
+        return xml.to_s
+      end
+      collections.each do |collection_node|
+        druid=collection_node['rdf:resource']
+        druid=druid.gsub('info:fedora/','')
+        collection_obj=Dor::Item.find(druid)
+        collection_title = Dor::Describable.get_collection_title(collection_obj)
+        node=xml.search('//mods:mods', 'mods' => 'http://www.loc.gov/mods/v3')
+        node=node.first
+        related_item_node=Nokogiri::XML::Node.new('relatedItem',xml)
+        related_item_node['type']='host'
+        title_info_node=Nokogiri::XML::Node.new('titleInfo',xml)
+        title_node=Nokogiri::XML::Node.new('title',xml)
+        title_node.content=collection_title
+        type_node=Nokogiri::XML::Node.new('typeOfResource',xml)
+        type_node['collection'] = 'yes'
+        node.add_child(related_item_node)
+        related_item_node.add_child(title_info_node)
+        title_info_node.add_child(title_node)
+        related_item_node.add_child(type_node)
+      end
+      Nokogiri::XML(xml.to_s) {|x| x.noblanks }.to_s
+    end
+		def metadata_namespace
+			desc_md = self.datastreams['descMetadata'].ng_xml
+			if desc_md.nil? or desc_md.root.nil? or desc_md.root.namespace.nil?
+				return nil 
+			else
+				return desc_md.root.namespace.href
+			end
+		end
+		
+		def metadata_format
+			DESC_MD_FORMATS[metadata_namespace]
+		end
+		
+		def to_solr(solr_doc=Hash.new, *args)
+			super solr_doc, *args
+			add_solr_value(solr_doc, "metadata_format", self.metadata_format, :string, [:searchable, :facetable])
+			begin
+				dc_doc = self.generate_dublin_core
+				dc_doc.xpath('/oai_dc:dc/*').each do |node|
+					add_solr_value(solr_doc, "public_dc_#{node.name}", node.text, :string, [:searchable])
+				end
+				creator=''
+				dc_doc.xpath('//dc:creator').each do |node|
+				  creator=node.text
+				end
+				title=''
+				dc_doc.xpath('//dc:title').each do |node|
+				  title=node.text
+				end
+        creator_title=creator+title
+        add_solr_value(solr_doc, 'creator_title', creator_title , :string, [:sortable])
+			rescue CrosswalkError => e
+				ActiveFedora.logger.warn "Cannot index #{self.pid}.descMetadata: #{e.message}"
+			end
+			solr_doc
+		end
+		def update_title(new_title)
+				if not update_simple_field('mods:mods/mods:titleInfo/mods:title',new_title)
+					raise 'Descriptive metadata has no title to update!'
+				end
+			end
+			def add_identifier(type, value)
+				ds_xml=self.descMetadata.ng_xml
+				ds_xml.search('//mods:mods','mods' => 'http://www.loc.gov/mods/v3').each do |node|
+				new_node=Nokogiri::XML::Node.new('identifier',ds_xml) #this ends up being mods:identifier without having to specify the namespace
+				new_node['type']=type
+				new_node.content=value
+				node.add_child(new_node)
+				end
+			end
+			def delete_identifier(type,value=nil)
+				
+				ds_xml=self.descMetadata.ng_xml
+				ds_xml.search('//mods:identifier','mods' => 'http://www.loc.gov/mods/v3').each do |node|	
+					if node.content == value or value==nil
+						node.remove
+						return true 
+					end
+				end
+				return false
+			end
+			
+			def set_desc_metadata_using_label(force=false)	
+				ds=self.descMetadata
+				unless force or ds.new?#22 is the length of <?xml version="1.0"?>
+					raise 'Cannot proceed, there is already content in the descriptive metadata datastream.'+ds.content.to_s
+				end
+				label=self.label
+				builder = Nokogiri::XML::Builder.new { |xml|
+				xml.mods( 'xmlns' => 'http://www.loc.gov/mods/v3', 'xmlns:xsi' => 'http://www.w3.org/2001/XMLSchema-instance',:version => '3.3', "xsi:schemaLocation" => 'http://www.loc.gov/mods/v3 http://www.loc.gov/standards/mods/v3/mods-3-3.xsd'){
+					xml.titleInfo{
+						xml.title label
+						}
+					}
+				} 
+				self.descMetadata.content=builder.to_xml
+			end
+			
+			def self.get_collection_title(obj)
+			  xml=obj.descMetadata.ng_xml
+			  preferred_citation=xml.search('//mods:mods/mods:note[@type=\'preferredCitation\']','mods' => 'http://www.loc.gov/mods/v3')
+			  title=''
+			  if preferred_citation.length == 1
+			    title=preferred_citation.first.content
+		    else
+		      title=xml.search('//mods:mods/mods:titleInfo/mods:title','mods' => 'http://www.loc.gov/mods/v3').first.content
+		      subtitle=xml.search('//mods:mods/mods:titleInfo/mods:subTitle','mods' => 'http://www.loc.gov/mods/v3')
+		      if(subtitle.length==1)
+		        title+=' ('+subtitle.first.content+')'
+	        end
+	      end
+	      title
+		  end
+			
+			private
+			#generic updater useful for updating things like title or subtitle which can only have a single occurance and must be present
+			def update_simple_field(field,new_val)
+				ds_xml=self.descMetadata.ng_xml
+				ds_xml.search('//'+field,'mods' => 'http://www.loc.gov/mods/v3').each do |node|
+					node.content=new_val
+					return true
+				end
+				return false
+			end
+			
+		
+	end
+end