dor-services 2.2.4 → 4.4.10

data/lib/dor/services/sdr_ingest_service.rb

@@ -0,0 +1,181 @@
+require 'rubygems'
+require 'lyber-utils'
+require 'moab_stanford'
+require 'dor-services'
+
+module Dor
+  class SdrIngestService
+
+    # @param [Dor::Item] dor_item The representation of the digital object
+    # @param [String] agreement_id  depreciated, included for backward compatability with common-accessoning
+    # @return [void] Create the moab manifests, export data to a BagIt bag, kick off the SDR ingest workflow
+    def self.transfer(dor_item, agreement_id=nil)
+      druid = dor_item.pid
+      workspace = DruidTools::Druid.new(druid,Dor::Config.sdr.local_workspace_root)
+      signature_catalog = get_signature_catalog(druid)
+      new_version_id = signature_catalog.version_id + 1
+      metadata_dir = extract_datastreams(dor_item, workspace)
+      verify_version_metadata(metadata_dir, new_version_id)
+      version_inventory = get_version_inventory(metadata_dir, druid, new_version_id)
+      version_addtions = signature_catalog.version_additions(version_inventory)
+      content_addtions = version_addtions.group('content')
+      if content_addtions.nil? or content_addtions.files.empty?
+        content_dir = nil
+      else
+        new_file_list = content_addtions.path_list
+        content_dir = workspace.find_filelist_parent('content',new_file_list)
+      end
+      content_group = version_inventory.group('content')
+      unless content_group.nil? or content_group.files.empty?
+        signature_catalog.normalize_group_signatures(content_group, content_dir)
+      end
+      # export the bag (in tar format)
+      bag_dir = Pathname(Dor::Config.sdr.local_export_home).join(druid.sub('druid:',''))
+      bagger = Moab::Bagger.new(version_inventory, signature_catalog, bag_dir)
+      bagger.reset_bag
+      bagger.create_bag_inventory(:depositor)
+      bagger.deposit_group('content', content_dir)
+      bagger.deposit_group('metadata', metadata_dir)
+      bagger.create_tagfiles
+      verify_bag_structure(bag_dir)
+      # Now bootstrap SDR workflow. but do not create the workflows datastream
+      dor_item.initialize_workflow('sdrIngestWF', 'sdr', false)
+    rescue Exception => e
+      raise LyberCore::Exceptions::ItemError.new(druid, "Export failure", e)
+    end
+
+    # @param [String] druid The object identifier
+    # @return [Moab::SignatureCatalog] the catalog of all files previously ingested
+    def self.get_signature_catalog(druid)
+      sdr_client = Dor::Config.sdr.rest_client
+      url = "objects/#{druid}/manifest/signatureCatalog.xml"
+      response = sdr_client[url].get
+      Moab::SignatureCatalog.parse(response)
+    rescue
+      Moab::SignatureCatalog.new(:digital_object_id => druid, :version_id => 0)
+    end
+
+    # @param [Dor::Item] dor_item The representation of the digital object
+    # @param [DruidTools::Druid] workspace The representation of the item's work area
+    # @return [Pathname] Pull all the datastreams specified in the configuration file
+    #   into the workspace's metadata directory, overwriting existing file if present
+    def self.extract_datastreams(dor_item, workspace)
+      metadata_dir = Pathname.new(workspace.path('metadata',create=true))
+      Config.sdr.datastreams.to_hash.each_pair do |ds_name, required|
+        ds_name = ds_name.to_s
+        metadata_file = metadata_dir.join("#{ds_name}.xml")
+        metadata_string = self.get_datastream_content(dor_item, ds_name, required)
+        metadata_file.open('w') { |f| f << metadata_string } if metadata_string
+      end
+      metadata_dir
+    end
+
+    # @param [Dor::Item] dor_item The representation of the digital object
+    # @param [String] ds_name The name of the desired Fedora datastream
+    # @param [String] required Enumeration: one of ['required', 'optional']
+    # @return [String] return the xml text of the specified datastream if it exists.
+    #   If not found, return nil unless it is a required datastream in which case raise exception
+    def self.get_datastream_content(dor_item, ds_name, required)
+      ds = (ds_name == 'relationshipMetadata' ? 'RELS-EXT' : ds_name)
+      if dor_item.datastreams.keys.include?(ds) and not dor_item.datastreams[ds].new?
+        return dor_item.datastreams[ds].content
+      elsif (required == 'optional')
+        return nil
+      else
+        raise "required datastream #{ds_name} not found in DOR"
+      end
+    end
+
+    # @param [Pathname] metadata_dir the location of the metadata directory in the workspace
+    # @param [Integer] expected the version identifer expected to be used in the versionMetadata
+    def self.verify_version_metadata(metadata_dir, expected)
+      vmfile = metadata_dir.join("versionMetadata.xml")
+      verify_version_id(vmfile, expected, vmfile_version_id(vmfile))
+      true
+    end
+
+    # @param [Pathname] pathname The location of the file containing a version number
+    # @param [Integer] expected The version number that should be in the file
+    # @param [Integer] found The version number that is actually in the file
+    def self.verify_version_id(pathname, expected, found)
+      raise "Version mismatch in #{pathname}, expected #{expected}, found #{found}" unless (expected == found)
+      true
+    end
+
+    # @param [Pathname] pathname the location of the versionMetadata file
+    # @return [Integer] the versionId found in the last version element, or nil if missing
+    def self.vmfile_version_id(pathname)
+      verify_pathname(pathname)
+      doc = Nokogiri::XML(File.open(pathname.to_s))
+      nodeset = doc.xpath("/versionMetadata/version")
+      version_id = nodeset.last['versionId']
+      version_id.nil? ? nil : version_id.to_i
+    end
+
+    # @param [Pathname] metadata_dir The location of the the object's metadata files
+    # @param [String] druid The object identifier
+    # @param [Integer] version_id The version number
+    # @return [Moab::FileInventory] Generate and return a version inventory for the object
+    def self.get_version_inventory(metadata_dir, druid, version_id)
+      version_inventory = get_content_inventory(metadata_dir, druid, version_id)
+      version_inventory.groups << get_metadata_file_group(metadata_dir)
+      version_inventory
+    end
+
+    # @param [Pathname] metadata_dir The location of the the object's metadata files
+    # @param [String] druid The object identifier
+    # @param [Integer] version_id The version number
+    # @return [Moab::FileInventory] Parse the contentMetadata
+    #   and generate a new version inventory object containing a content group
+    def self.get_content_inventory(metadata_dir, druid, version_id)
+      content_metadata = get_content_metadata(metadata_dir)
+      if content_metadata
+        Stanford::ContentInventory.new.inventory_from_cm(content_metadata, druid, subset='preserve', version_id)
+      else
+        FileInventory.new(:type=>"version",:digital_object_id=>druid, :version_id=>version_id)
+      end
+    end
+
+    # @param [Pathname] metadata_dir The location of the the object's metadata files
+    # @return [String] Return the contents of the contentMetadata.xml file from the content directory
+    def self.get_content_metadata(metadata_dir)
+      content_metadata_pathname = metadata_dir.join('contentMetadata.xml')
+      if content_metadata_pathname.exist?
+        content_metadata_pathname.read
+      else
+        nil
+      end
+    end
+
+    # @param [Pathname] metadata_dir The location of the the object's metadata files
+    # @return [Moab::FileGroup] Traverse the metadata directory and generate a metadata group
+    def self.get_metadata_file_group(metadata_dir)
+      file_group = FileGroup.new(:group_id=>'metadata').group_from_directory(metadata_dir)
+      file_group
+    end
+
+    # @param [Pathname] bag_dir the location of the bag to be verified
+    # @return [Boolean] true if all required files exist, raises exception if not
+    def self.verify_bag_structure(bag_dir)
+      verify_pathname(bag_dir)
+      verify_pathname(bag_dir.join('data'))
+      verify_pathname(bag_dir.join('bagit.txt'))
+      verify_pathname(bag_dir.join('bag-info.txt'))
+      verify_pathname(bag_dir.join('manifest-sha256.txt'))
+      verify_pathname(bag_dir.join('tagmanifest-sha256.txt'))
+      verify_pathname(bag_dir.join('versionAdditions.xml'))
+      verify_pathname(bag_dir.join('versionInventory.xml'))
+      verify_pathname(bag_dir.join('data','metadata','versionMetadata.xml'))
+      true
+    end
+
+    # @param [Pathname] pathname The file whose existence should be verified
+    # @return [Boolean] true if file exists, raises exception if not
+    def self.verify_pathname(pathname)
+      raise "#{pathname.basename} not found at #{pathname}" unless pathname.exist?
+      true
+    end
+
+  end
+
+end

data/lib/dor/services/search_service.rb

@@ -0,0 +1,131 @@
+require 'json'
+require 'active_support/core_ext'
+
+module Dor
+
+  class SearchService
+
+    include Solrizer::FieldNameMapper
+    RISEARCH_TEMPLATE = "select $object from <#ri> where $object <dc:identifier> '%s'"
+    @@index_version = nil
+
+    class << self
+
+      def index_version
+        if @@index_version.nil?
+          xsl_doc = Nokogiri::XML(File.read(File.expand_path('../../../gsearch/demoFoxmlToSolr.xslt',__FILE__)))
+          @@index_version = xsl_doc.at_xpath('/xsl:stylesheet/xsl:variable[@name="INDEXVERSION"]/text()').to_s
+        end
+        @@index_version
+      end
+
+      def reindex(*pids)
+        client = Config.gsearch.rest_client
+        pids.in_groups_of(20, false) do |group|
+          group.each { |pid| client["?operation=updateIndex&action=fromPid&value=#{pid}"].get }
+          yield group if block_given?
+        end
+        pids
+      end
+
+      def risearch(query, opts = {})
+        client = Config.fedora.client['risearch']
+        client.options[:timeout] = opts.delete(:timeout)
+        query_params = {
+          :type => 'tuples',
+          :lang => 'itql',
+          :format => 'CSV',
+          :limit => '1000',
+          :stream => 'on',
+          :query => query
+        }.merge(opts)
+        result = client.post(query_params)
+        result.split(/\n/)[1..-1].collect { |pid| pid.chomp.sub(/^info:fedora\//,'') }
+      end
+
+      def iterate_over_pids(opts = {}, &block)
+        opts[:query] ||= "select $object from <#ri> where $object <info:fedora/fedora-system:def/model#label> $label"
+        opts[:in_groups_of] ||= 100
+        opts[:mode] ||= :single
+        start = 0
+        pids = Dor::SearchService.risearch("#{opts[:query]} limit #{opts[:in_groups_of]} offset #{start}")
+        while pids.present?
+          if opts[:mode] == :single
+            pids.each { |pid| yield pid }
+          else
+            yield pids
+          end
+          start += pids.length
+          pids = Dor::SearchService.risearch("#{opts[:query]} limit #{opts[:in_groups_of]} offset #{start}")
+        end
+      end
+
+      def gsearch(params)
+        client = Config.gsearch.client
+        query_params = params.merge(:wt => 'json')
+        query_string = query_params.collect { |k,v|
+          if v.is_a?(Array)
+            v.collect { |vv| "#{k}=#{URI.encode(vv.to_s)}" }.join('&')
+          else
+            "#{k}=#{URI.encode(v.to_s)}"
+          end
+        }.join('&')
+        result = JSON.parse(client["select?#{query_string}"].get)
+      end
+
+      def query query, args={}
+        params = args.merge({ :q => query })
+        params[:start] ||= 0
+        resp = solr.find params
+        if block_given?
+          cont = true
+          while cont and resp.docs.length > 0
+            cont = yield(resp)
+            params[:rows] ||= resp.docs.length
+            params[:start] += params[:rows]
+            resp = solr.find params
+          end
+        else
+          return resp
+        end
+      end
+
+      def query_by_id(id)
+        if id.is_a?(Hash) # Single valued: { :google => 'STANFORD_0123456789' }
+          id = id.collect { |*v| v.join(':') }.first
+        elsif id.is_a?(Array) # Two values: [ 'google', 'STANFORD_0123456789' ]
+          id = id.join(':')
+        end
+        q = %{#{solr_name 'identifier', :string}:"#{id}"}
+        result = []
+        resp = query(q, :fl => 'id', :rows => 1000) do |resp|
+          result += resp.docs.collect { |doc| doc['id'] }
+          true
+        end
+        result
+      end
+
+      def solr
+        @@solr ||= ActiveFedora.solr.conn.is_a?(RSolr::Client) ? ActiveFedora.solr.conn : Dor::Config.make_solr_connection
+      end
+
+      # @return String druid of the SDR Graveyard APO
+      #   nil if APO does not exist in the currently configured environment
+      def sdr_graveyard_apo_druid
+        @@sdr_graveyard_apo ||= find_sdr_graveyard_apo_druid
+      end
+
+      def find_sdr_graveyard_apo_druid
+        r = Dor::SearchService.query('dc_title_t:"SDR Graveyard"', :fl => 'id')
+        if r.docs.empty?
+          nil
+        else
+          r.docs.first[:id]
+        end
+      end
+
+    end
+
+  end
+
+end

data/lib/dor/services/suri_service.rb

@@ -0,0 +1,32 @@
+require 'rest-client'
+require 'active_fedora'
+
+module Dor
+  class SuriService
+    # If Dor::Config.suri.mint_ids is set to true, then this method
+    # returns Config.suri.id_namespace:id_from_suri
+    # Throws an exception if there were any problems
+    def self.mint_id quantity=nil
+      want_array = quantity.is_a?(Numeric)
+      quantity = 1 if quantity.nil?
+      ids = []
+      if Config.suri.mint_ids
+        #Post with no body
+        resource = RestClient::Resource.new("#{Config.suri.url}/suri2/namespaces/#{Config.suri.id_namespace}",
+                                  :user => Config.suri.user, :password => Config.suri.pass)
+        ids = resource["identifiers?quantity=#{quantity}"].post('').chomp.split(/\n/).collect { |id| "#{Config.suri.id_namespace}:#{id.strip}" }
+      else
+        repo = ActiveFedora::Base.respond_to?(:connection_for_pid) ? ActiveFedora::Base.connection_for_pid(0) : ActiveFedora.fedora.connection
+        resp = Nokogiri::XML(repo.next_pid :numPIDs => quantity)
+        ids = resp.xpath('/pidList/pid').collect { |node| node.text }
+      end
+      return want_array ? ids : ids.first
+
+#    rescue Exception => e
+#      Rails.logger.error("Unable to mint id from suri: #{e.to_s}")
+#      raise e
+    end
+  
+    
+  end
+end

data/lib/dor/services/technical_metadata_service.rb

@@ -0,0 +1,226 @@
+require 'rubygems'
+require 'moab_stanford'
+require 'jhove_service'
+require 'dor-services'
+
+module Dor
+
+  class TechnicalMetadataService
+
+    # @param [Dor::Item] dor_item The DOR item being processed by the technical metadata robot
+    # @return [Boolean] True if technical metadata is correctly added or updated
+    def self.add_update_technical_metadata(dor_item)
+      test_jhove_service
+      druid = dor_item.pid
+      content_group_diff = get_content_group_diff(dor_item)
+      deltas = get_file_deltas(content_group_diff)
+      new_files = get_new_files(deltas)
+      old_techmd = get_old_technical_metadata(dor_item)
+      new_techmd = get_new_technical_metadata(druid, new_files)
+      if old_techmd.nil?
+        # this is version 1 or previous technical metadata was not saved
+        final_techmd = new_techmd
+      elsif content_group_diff.difference_count == 0
+        # there have been no changes to content files from previous version
+        return true
+      else
+        merged_nodes = merge_file_nodes(old_techmd, new_techmd, deltas)
+        final_techmd = build_technical_metadata(druid,merged_nodes)
+      end
+      ds = dor_item.datastreams["technicalMetadata"]
+      ds.dsLabel = 'Technical Metadata'
+      ds.content = final_techmd
+      ds.save
+      true
+    end
+
+    # @return [Boolean] Make sure that the jhove-service gem is loaded
+    def self.test_jhove_service
+      unless defined? ::JhoveService
+        begin
+          require 'jhove_service'
+        rescue LoadError => e
+          puts e.inspect
+          raise "jhove-service dependency gem was not found.  Please add it to your Gemfile and run bundle install"
+        end
+      end
+    end
+
+    # @param [Dor::Item] dor_item The DOR item being processed by the technical metadata robot
+    # @return [FileGroupDifference] The differences between two versions of a group of files
+    def self.get_content_group_diff(dor_item)
+      inventory_diff_xml = dor_item.get_content_diff('all')
+      inventory_diff = Moab::FileInventoryDifference.parse(inventory_diff_xml)
+      content_group_diff = inventory_diff.group_difference("content")
+      content_group_diff
+    end
+
+    # @param [FileGroupDifference] content_group_diff
+    # @return [Hash<Symbol,Array>] Sets of filenames grouped by change type for use in performing file or metadata operations
+    def self.get_file_deltas(content_group_diff)
+      deltas = content_group_diff.file_deltas
+      deltas
+    end
+
+    # @param [Hash<Symbol,Array>] deltas Sets of filenames grouped by change type for use in performing file or metadata operations
+    # @return [Array<String>] The list of filenames for files that are either added or modifed since the previous version
+    def self.get_new_files(deltas)
+      deltas[:added] + deltas[:modified]
+    end
+
+    # @param [Dor::Item] dor_item The DOR item being processed by the technical metadata robot
+    # @return [String] The technicalMetadata datastream from the previous version of the digital object
+    def self.get_old_technical_metadata(dor_item)
+      sdr_techmd = get_sdr_technical_metadata(dor_item.pid)
+      return sdr_techmd unless sdr_techmd.nil?
+      get_dor_technical_metadata(dor_item)
+    end
+
+    # @param [String] druid The identifier of the digital object being processed by the technical metadata robot
+    # @return [String] The technicalMetadata datastream from the previous version of the digital object (fetched from SDR storage)
+    #   The data is updated to the latest format.
+    def self.get_sdr_technical_metadata(druid)
+      begin
+        sdr_techmd = get_sdr_metadata(druid, "technicalMetadata")
+      rescue RestClient::ResourceNotFound => e
+        return nil
+      end
+      if sdr_techmd =~ /<technicalMetadata/
+        return sdr_techmd
+      elsif sdr_techmd =~ /<jhove/
+        return ::JhoveService.new.upgrade_technical_metadata(sdr_techmd)
+      else
+        return nil
+      end
+    end
+
+    # @param [Dor::Item] dor_item The DOR item being processed by the technical metadata robot
+    # @return [String] The technicalMetadata datastream from the previous version of the digital object (fetched from DOR fedora).
+    #   The data is updated to the latest format.
+    def self.get_dor_technical_metadata(dor_item)
+      ds = "technicalMetadata"
+      if dor_item.datastreams.keys.include?(ds) and not dor_item.datastreams[ds].new?
+        dor_techmd = dor_item.datastreams[ds].content
+      else
+        return nil
+      end
+      if dor_techmd =~ /<technicalMetadata/
+        return dor_techmd
+      elsif dor_techmd =~ /<jhove/
+        return ::JhoveService.new.upgrade_technical_metadata(dor_techmd)
+      else
+        return nil
+      end
+    end
+
+    # @param [String] druid The identifier of the digital object being processed by the technical metadata robot
+    # @param [String] dsname The identifier of the metadata datastream
+    # @return [String] The datastream contents from the previous version of the digital object (fetched from SDR storage)
+    def self.get_sdr_metadata(druid, dsname)
+      sdr_client = Dor::Config.sdr.rest_client
+      url = "objects/#{druid}/metadata/#{dsname}.xml"
+      response = sdr_client[url].get
+      response
+    end
+
+    # @param [DruidTools::Druid] druid_tool A wrapper class for the druid identifier.  Used to generate paths
+    # @param [Array<String>] new_files The list of filenames for files that are either added or modifed since the previous version
+    # @return [String] The technicalMetadata datastream for the new files of the new digital object version
+    def self.get_new_technical_metadata(druid, new_files)
+      return nil if new_files.nil? or new_files.empty?
+      workspace = DruidTools::Druid.new(druid, Dor::Config.sdr.local_workspace_root)
+      content_dir = workspace.find_filelist_parent('content',new_files)
+      temp_dir = workspace.temp_dir
+      jhove_service = ::JhoveService.new(temp_dir)
+      jhove_service.digital_object_id=druid
+      fileset_file = write_fileset(temp_dir, new_files)
+      jhove_output_file = jhove_service.run_jhove(content_dir, fileset_file)
+      tech_md_file = jhove_service.create_technical_metadata(jhove_output_file)
+      IO.read(tech_md_file)
+    end
+
+    # @param [Pathname]  temp_dir  The pathname of the temp folder in the object's workspace area
+    # @param [Object] new_files [Array<String>] The list of filenames for files that are either added or modifed since the previous version
+    # @return [Pathname] Save the new_files list to a text file and return that file's name
+    def self.write_fileset(temp_dir, new_files)
+      fileset_pathname = Pathname(temp_dir).join('jhove_fileset.txt')
+      fileset_pathname.open('w') {|f| f.puts(new_files) }
+      fileset_pathname
+    end
+
+    # @param [String] old_techmd The technicalMetadata datastream from the previous version of the digital object
+    # @param [String] new_techmd The technicalMetadata datastream for the new files of the new digital object version
+    # @param [Array<String>] deltas The list of filenames for files that are either added or modifed since the previous version
+    # @return [Hash<String,Nokogiri::XML::Node>] The complete set of technicalMetadata nodes for the digital object, indexed by filename
+    def self.merge_file_nodes(old_techmd, new_techmd, deltas)
+      old_file_nodes = get_file_nodes(old_techmd)
+      new_file_nodes = get_file_nodes(new_techmd)
+      merged_nodes = Hash.new
+      deltas[:identical].each do |path|
+        merged_nodes[path] = old_file_nodes[path]
+      end
+      deltas[:modified].each do |path|
+        merged_nodes[path] = new_file_nodes[path]
+      end
+      deltas[:added].each do |path|
+        merged_nodes[path] = new_file_nodes[path]
+      end
+      deltas[:renamed].each do |oldpath,newpath|
+        clone = old_file_nodes[oldpath].clone
+        clone.sub!(/<file\s*id.*?["'].*?["'].*?>/, "<file id='#{newpath}'>")
+        merged_nodes[newpath] = clone
+      end
+      deltas[:copyadded].each do |oldpath,newpath|
+        clone = old_file_nodes[oldpath].clone
+        clone.sub!(/<file\s*id.*?["'].*?["'].*?>/, "<file id='#{newpath}'>")
+        merged_nodes[newpath] = clone
+      end
+      merged_nodes
+    end
+
+    # @param [String] technical_metadata A technicalMetadata datastream contents
+    # @return [Hash<String,Nokogiri::XML::Node>] The set of nodes from a technicalMetadata datastream , indexed by filename
+    def self.get_file_nodes(technical_metadata)
+      file_hash = Hash.new
+      return file_hash if technical_metadata.nil?
+      current_file = Array.new
+      path = nil
+      in_file = false
+      technical_metadata.each_line do |line|
+        if line =~ /^\s*<file.*["'](.*?)["']/
+          current_file << line
+          path = $1
+          in_file = true
+        elsif line =~ /^\s*<\/file>/
+          current_file << line
+          file_hash[path] = current_file.join
+          current_file = Array.new
+          path = nil
+          in_file = false
+        elsif in_file
+          current_file << line
+        end
+      end
+      file_hash
+    end
+
+    # @param [String] druid The identifier of the digital object being processed by the technical metadata robot
+    # @param [Hash<String,Nokogiri::XML::Node>] merged_nodes The complete set of technicalMetadata nodes for the digital object, indexed by filename
+    # @return [String] The finalized technicalMetadata datastream contents for the new object version
+    def self.build_technical_metadata(druid, merged_nodes)
+      techmd_root = <<-EOF
+<technicalMetadata objectId='#{druid}' datetime='#{Time.now.utc.iso8601}'
+    xmlns:jhove='http://hul.harvard.edu/ois/xml/ns/jhove'
+    xmlns:mix='http://www.loc.gov/mix/v10'
+    xmlns:textmd='info:lc/xmlns/textMD-v3'>
+      EOF
+      doc = techmd_root
+      merged_nodes.keys.sort.each {|path| doc << merged_nodes[path] }
+      doc << "</technicalMetadata>"
+      doc
+    end
+
+  end
+
+end
+