dor-services 6.8.0 → 7.0.0

data/lib/dor/services/reset_workspace_service.rb

@@ -1,27 +0,0 @@
-# frozen_string_literal: true
-
-module Dor
-  # Rename the druid trees  at the end of the accessionWF in order to be cleaned/deleted later.
-  class ResetWorkspaceService
-    def self.reset_workspace_druid_tree(druid, version, workspace_root)
-      druid_tree_path = DruidTools::Druid.new(druid, workspace_root).pathname.to_s
-
-      raise "The archived directory #{druid_tree_path}_v#{version} already existed." if File.exist?("#{druid_tree_path}_v#{version}")
-
-      if File.exist?(druid_tree_path)
-        FileUtils.mv(druid_tree_path, "#{druid_tree_path}_v#{version}")
-      end # Else is a truncated tree where we shouldn't do anything
-    end
-
-    def self.reset_export_bag(druid, version, export_root)
-      id = druid.split(':').last
-      bag_dir = File.join(export_root, id)
-
-      raise "The archived bag #{bag_dir}_v#{version} already existed." if File.exist?("#{bag_dir}_v#{version}")
-
-      FileUtils.mv(bag_dir, "#{bag_dir}_v#{version}") if File.exist?(bag_dir)
-
-      FileUtils.mv("#{bag_dir}.tar", "#{bag_dir}_v#{version}.tar") if File.exist?("#{bag_dir}.tar")
-    end
-  end
-end

data/lib/dor/services/sdr_ingest_service.rb

@@ -1,172 +0,0 @@
-# frozen_string_literal: true
-
-require 'moab/stanford'
-
-module Dor
-  # Note: This should probably live in common-accessioning robot sdr-ingest-transfer
-  #  as that is the only robot that uses it.  See also preservable concern.
-  class SdrIngestService
-    # @param [Dor::Item] dor_item The representation of the digital object
-    # @param [String] _agreement_id deprecated, included for backward compatability with common-accessoning
-    # @return [void] Create the Moab/bag manifests for new version, export data to BagIt bag, kick off the SDR preservation workflow
-    def self.transfer(dor_item, _agreement_id = nil)
-      druid = dor_item.pid
-      workspace = DruidTools::Druid.new(druid, Dor::Config.sdr.local_workspace_root)
-      signature_catalog = get_signature_catalog(druid)
-      new_version_id = signature_catalog.version_id + 1
-      metadata_dir = extract_datastreams(dor_item, workspace)
-      verify_version_metadata(metadata_dir, new_version_id)
-      version_inventory = get_version_inventory(metadata_dir, druid, new_version_id)
-      version_addtions = signature_catalog.version_additions(version_inventory)
-      content_addtions = version_addtions.group('content')
-      if content_addtions.nil? || content_addtions.files.empty?
-        content_dir = nil
-      else
-        new_file_list = content_addtions.path_list
-        content_dir = workspace.find_filelist_parent('content', new_file_list)
-      end
-      content_group = version_inventory.group('content')
-      signature_catalog.normalize_group_signatures(content_group, content_dir) unless content_group.nil? || content_group.files.empty?
-      # export the bag (in tar format)
-      bag_dir = Pathname(Dor::Config.sdr.local_export_home).join(druid.sub('druid:', ''))
-      bagger = Moab::Bagger.new(version_inventory, signature_catalog, bag_dir)
-      bagger.reset_bag
-      bagger.create_bag_inventory(:depositor)
-      bagger.deposit_group('content', content_dir)
-      bagger.deposit_group('metadata', metadata_dir)
-      bagger.create_tagfiles
-      verify_bag_structure(bag_dir)
-      # start SDR preservation workflow (but do not create the workflows datastream)
-      CreateWorkflowService.create_workflow(dor_item, name: 'preservationIngestWF', create_ds: false)
-    rescue Exception => e
-      raise Dor::Exception, "Error exporting new object version to bag: #{e.message}"
-    end
-
-    # Note: the following methods should probably all be private
-
-    # @param [String] druid The object identifier
-    # @return [Moab::SignatureCatalog] the catalog of all files previously ingested
-    def self.get_signature_catalog(druid)
-      Dor::Services::Client.object(druid).sdr.signature_catalog
-    end
-
-    # @param [Dor::Item] dor_item The representation of the digital object
-    # @param [DruidTools::Druid] workspace The representation of the item's work area
-    # @return [Pathname] Pull all the datastreams specified in the configuration file
-    #   into the workspace's metadata directory, overwriting existing file if present
-    def self.extract_datastreams(dor_item, workspace)
-      metadata_dir = Pathname.new(workspace.path('metadata', true))
-      Config.sdr.datastreams.to_hash.each_pair do |ds_name, required|
-        ds_name = ds_name.to_s
-        metadata_file = metadata_dir.join("#{ds_name}.xml")
-        metadata_string = get_datastream_content(dor_item, ds_name, required)
-        metadata_file.open('w') { |f| f << metadata_string } if metadata_string
-      end
-      metadata_dir
-    end
-
-    # @param [Dor::Item] dor_item The representation of the digital object
-    # @param [String] ds_name The name of the desired Fedora datastream
-    # @param [String] required Enumeration: one of ['required', 'optional']
-    # @return [String] return the xml text of the specified datastream if it exists.
-    #   If not found, return nil unless it is a required datastream in which case raise exception
-    def self.get_datastream_content(dor_item, ds_name, required)
-      ds = (ds_name == 'relationshipMetadata' ? 'RELS-EXT' : ds_name)
-      if dor_item.datastreams.key?(ds) && !dor_item.datastreams[ds].new?
-        return dor_item.datastreams[ds].content
-      elsif required == 'optional'
-        return nil
-      else
-        raise "required datastream #{ds_name} not found in DOR"
-      end
-    end
-
-    # @param [Pathname] metadata_dir the location of the metadata directory in the workspace
-    # @param [Integer] expected the version identifer expected to be used in the versionMetadata
-    def self.verify_version_metadata(metadata_dir, expected)
-      vmfile = metadata_dir.join('versionMetadata.xml')
-      verify_version_id(vmfile, expected, vmfile_version_id(vmfile))
-      true
-    end
-
-    # @param [Pathname] pathname The location of the file containing a version number
-    # @param [Integer] expected The version number that should be in the file
-    # @param [Integer] found The version number that is actually in the file
-    def self.verify_version_id(pathname, expected, found)
-      raise "Version mismatch in #{pathname}, expected #{expected}, found #{found}" unless expected == found
-
-      true
-    end
-
-    # @param [Pathname] pathname the location of the versionMetadata file
-    # @return [Integer] the versionId found in the last version element, or nil if missing
-    def self.vmfile_version_id(pathname)
-      verify_pathname(pathname)
-      doc = Nokogiri::XML(File.open(pathname.to_s))
-      nodeset = doc.xpath('/versionMetadata/version')
-      version_id = nodeset.last['versionId']
-      version_id.nil? ? nil : version_id.to_i
-    end
-
-    # @param [Pathname] metadata_dir The location of the the object's metadata files
-    # @param [String] druid The object identifier
-    # @param [Integer] version_id The version number
-    # @return [Moab::FileInventory] Generate and return a version inventory for the object
-    def self.get_version_inventory(metadata_dir, druid, version_id)
-      version_inventory = get_content_inventory(metadata_dir, druid, version_id)
-      version_inventory.groups << get_metadata_file_group(metadata_dir)
-      version_inventory
-    end
-
-    # @param [Pathname] metadata_dir The location of the the object's metadata files
-    # @param [String] druid The object identifier
-    # @param [Integer] version_id The version number
-    # @return [Moab::FileInventory] Parse the contentMetadata
-    #   and generate a new version inventory object containing a content group
-    def self.get_content_inventory(metadata_dir, druid, version_id)
-      content_metadata = get_content_metadata(metadata_dir)
-      if content_metadata
-        Stanford::ContentInventory.new.inventory_from_cm(content_metadata, druid, 'preserve', version_id)
-      else
-        Moab::FileInventory.new(type: 'version', digital_object_id: druid, version_id: version_id)
-      end
-    end
-
-    # @param [Pathname] metadata_dir The location of the the object's metadata files
-    # @return [String] Return the contents of the contentMetadata.xml file from the content directory
-    def self.get_content_metadata(metadata_dir)
-      content_metadata_pathname = metadata_dir.join('contentMetadata.xml')
-      content_metadata_pathname.read if content_metadata_pathname.exist?
-    end
-
-    # @param [Pathname] metadata_dir The location of the the object's metadata files
-    # @return [Moab::FileGroup] Traverse the metadata directory and generate a metadata group
-    def self.get_metadata_file_group(metadata_dir)
-      file_group = Moab::FileGroup.new(group_id: 'metadata').group_from_directory(metadata_dir)
-      file_group
-    end
-
-    # @param [Pathname] bag_dir the location of the bag to be verified
-    # @return [Boolean] true if all required files exist, raises exception if not
-    def self.verify_bag_structure(bag_dir)
-      verify_pathname(bag_dir)
-      verify_pathname(bag_dir.join('data'))
-      verify_pathname(bag_dir.join('bagit.txt'))
-      verify_pathname(bag_dir.join('bag-info.txt'))
-      verify_pathname(bag_dir.join('manifest-sha256.txt'))
-      verify_pathname(bag_dir.join('tagmanifest-sha256.txt'))
-      verify_pathname(bag_dir.join('versionAdditions.xml'))
-      verify_pathname(bag_dir.join('versionInventory.xml'))
-      verify_pathname(bag_dir.join('data', 'metadata', 'versionMetadata.xml'))
-      true
-    end
-
-    # @param [Pathname] pathname The file whose existence should be verified
-    # @return [Boolean] true if file exists, raises exception if not
-    def self.verify_pathname(pathname)
-      raise "#{pathname.basename} not found at #{pathname}" unless pathname.exist?
-
-      true
-    end
-  end
-end

data/lib/dor/services/secondary_file_name_service.rb

@@ -1,10 +0,0 @@
-# frozen_string_literal: true
-
-module Dor
-  # Merges contentMetadata from several objects into one.
-  class SecondaryFileNameService
-    def self.create(old_name, sequence_num)
-      old_name =~ /^(.*)\.(.*)$/ ? "#{Regexp.last_match(1)}_#{sequence_num}.#{Regexp.last_match(2)}" : "#{old_name}_#{sequence_num}"
-    end
-  end
-end

data/lib/dor/services/shelving_service.rb

@@ -1,69 +0,0 @@
-# frozen_string_literal: true
-
-module Dor
-  # Push file changes for shelve-able files into the stacks
-  class ShelvingService
-    def self.shelve(work)
-      new(work).shelve
-    end
-
-    def initialize(work)
-      @work = work
-    end
-
-    def shelve
-      # retrieve the differences between the current contentMetadata and the previously ingested version
-      diff = shelve_diff
-      stacks_object_pathname = stacks_location
-      # determine the location of the object's files in the stacks area
-      stacks_druid = DruidTools::StacksDruid.new work.id, stacks_object_pathname
-      stacks_object_pathname = Pathname(stacks_druid.path)
-      # determine the location of the object's content files in the workspace area
-      workspace_druid = DruidTools::Druid.new(work.id, Config.stacks.local_workspace_root)
-      workspace_content_pathname = workspace_content_dir(diff, workspace_druid)
-      # delete, rename, or copy files to the stacks area
-      DigitalStacksService.remove_from_stacks(stacks_object_pathname, diff)
-      DigitalStacksService.rename_in_stacks(stacks_object_pathname, diff)
-      DigitalStacksService.shelve_to_stacks(workspace_content_pathname, stacks_object_pathname, diff)
-    end
-
-    private
-
-    attr_reader :work
-
-    # retrieve the differences between the current contentMetadata and the previously ingested version
-    # (filtering to select only the files that should be shelved to stacks)
-    def shelve_diff
-      raise Dor::ParameterError, 'Missing Dor::Config.stacks.local_workspace_root' if Config.stacks.local_workspace_root.nil?
-      raise Dor::Exception, 'Missing contentMetadata datastream' if work.contentMetadata.nil?
-
-      client = Dor::Services::Client.object(work.pid).sdr
-      current_content = work.contentMetadata.content
-      inventory_diff = client.content_diff(current_content: current_content, subset: 'shelve')
-      inventory_diff.group_difference('content')
-    end
-
-    # Find the location of the object's content files in the workspace area
-    # @param [Moab::FileGroupDifference] content_diff The differences between the current contentMetadata and the previously ingested version
-    # @param [DruidTools::Druid] workspace_druid the location of the object's files in the workspace area
-    # @return [Pathname] The location of the object's content files in the workspace area
-    def workspace_content_dir(content_diff, workspace_druid)
-      deltas = content_diff.file_deltas
-      filelist = deltas[:modified] + deltas[:added] + deltas[:copyadded].collect { |_old, new| new }
-      return nil if filelist.empty?
-
-      Pathname(workspace_druid.find_filelist_parent('content', filelist))
-    end
-
-    # get the stack location based on the contentMetadata stacks attribute
-    # or using the default value from the config file if it doesn't exist
-    def stacks_location
-      return Config.stacks.local_stacks_root unless work.contentMetadata&.stacks.present?
-
-      location = work.contentMetadata.stacks[0]
-      return location if location.start_with? '/' # Absolute stacks path
-
-      raise "stacks attribute for item: #{work.id} contentMetadata should start with /. The current value is #{location}"
-    end
-  end
-end

data/lib/dor/services/technical_metadata_service.rb

@@ -1,232 +0,0 @@
-# frozen_string_literal: true
-
-require 'rubygems'
-require 'moab/stanford'
-require 'jhove_service'
-require 'dor-services'
-
-module Dor
-  # Extracts technical metadata from files using JHOVE
-  # If this is a new version it gets the old technicalMetadata datastream by
-  # making an API call to sdr-services-app (via dor-services-app) and
-  # only overwrites/adds parts for the files that were changed or added.
-  # This allows us to avoid re-staging files that have not changed.
-  # Switching to a more granular data model that has file metadata separate from
-  # the Work metadata will allow us to simplify this greatly.
-  class TechnicalMetadataService
-    # @param [Dor::Item] dor_item The DOR item being processed by the technical metadata robot
-    # @return [Boolean] True if technical metadata is correctly added or updated
-    def self.add_update_technical_metadata(dor_item)
-      test_jhove_service
-      druid = dor_item.pid
-      content_group_diff = get_content_group_diff(dor_item)
-      deltas = get_file_deltas(content_group_diff)
-      new_files = get_new_files(deltas)
-      old_techmd = get_old_technical_metadata(dor_item)
-      new_techmd = get_new_technical_metadata(druid, new_files)
-      if old_techmd.nil?
-        # this is version 1 or previous technical metadata was not saved
-        final_techmd = new_techmd
-      elsif content_group_diff.difference_count == 0
-        # there have been no changes to content files from previous version
-        return true
-      else
-        merged_nodes = merge_file_nodes(old_techmd, new_techmd, deltas)
-        final_techmd = build_technical_metadata(druid, merged_nodes)
-      end
-      ds = dor_item.datastreams['technicalMetadata']
-      ds.dsLabel = 'Technical Metadata'
-      ds.content = final_techmd
-      ds.save
-      true
-    end
-
-    # @return [Boolean] Make sure that the jhove-service gem is loaded
-    def self.test_jhove_service
-      unless defined? ::JhoveService
-        begin
-          require 'jhove_service'
-        rescue LoadError => e
-          puts e.inspect
-          raise 'jhove-service dependency gem was not found.  Please add it to your Gemfile and run bundle install'
-        end
-      end
-    end
-    private_class_method :test_jhove_service
-
-    # @param [Dor::Item] dor_item The DOR item being processed by the technical metadata robot
-    # @return [FileGroupDifference] The differences between two versions of a group of files
-    def self.get_content_group_diff(dor_item)
-      return Moab::FileGroupDifference.new if dor_item.contentMetadata.nil?
-      raise Dor::ParameterError, 'Missing Dor::Config.stacks.local_workspace_root' if Config.stacks.local_workspace_root.nil?
-
-      client = Dor::Services::Client.object(dor_item.pid).sdr
-      current_content = dor_item.contentMetadata.content
-      inventory_diff = client.content_diff(current_content: current_content)
-      inventory_diff.group_difference('content')
-    end
-    private_class_method :get_content_group_diff
-
-    # @param [FileGroupDifference] content_group_diff
-    # @return [Hash<Symbol,Array>] Sets of filenames grouped by change type for use in performing file or metadata operations
-    def self.get_file_deltas(content_group_diff)
-      content_group_diff.file_deltas
-    end
-    private_class_method :get_file_deltas
-
-    # @param [Hash<Symbol,Array>] deltas Sets of filenames grouped by change type for use in performing file or metadata operations
-    # @return [Array<String>] The list of filenames for files that are either added or modifed since the previous version
-    def self.get_new_files(deltas)
-      deltas[:added] + deltas[:modified]
-    end
-    private_class_method :get_new_files
-
-    # @param [Dor::Item] dor_item The DOR item being processed by the technical metadata robot
-    # @return [String] The technicalMetadata datastream from the previous version of the digital object
-    def self.get_old_technical_metadata(dor_item)
-      sdr_techmd = get_sdr_technical_metadata(dor_item.pid)
-      return sdr_techmd unless sdr_techmd.nil?
-
-      get_dor_technical_metadata(dor_item)
-    end
-    private_class_method :get_old_technical_metadata
-
-    # @param [String] druid The identifier of the digital object being processed by the technical metadata robot
-    # @return [String] The technicalMetadata datastream from the previous version of the digital object (fetched from SDR storage)
-    #   The data is updated to the latest format.
-    def self.get_sdr_technical_metadata(druid)
-      sdr_techmd = get_sdr_metadata(druid, 'technicalMetadata')
-      return sdr_techmd if sdr_techmd =~ /<technicalMetadata/
-      return ::JhoveService.new.upgrade_technical_metadata(sdr_techmd) if sdr_techmd =~ /<jhove/
-
-      nil
-    end
-    private_class_method :get_sdr_technical_metadata
-
-    # @param [Dor::Item] dor_item The DOR item being processed by the technical metadata robot
-    # @return [String] The technicalMetadata datastream from the previous version of the digital object (fetched from DOR fedora).
-    #   The data is updated to the latest format.
-    def self.get_dor_technical_metadata(dor_item)
-      ds = 'technicalMetadata'
-      return nil unless dor_item.datastreams.key?(ds) && !dor_item.datastreams[ds].new?
-
-      dor_techmd = dor_item.datastreams[ds].content
-      return dor_techmd if dor_techmd =~ /<technicalMetadata/
-      return ::JhoveService.new.upgrade_technical_metadata(dor_techmd) if dor_techmd =~ /<jhove/
-
-      nil
-    end
-    private_class_method :get_dor_technical_metadata
-
-    # @param [String] druid The identifier of the digital object being processed by the technical metadata robot
-    # @param [String] dsname The identifier of the metadata datastream
-    # @return [String] The datastream contents from the previous version of the digital object (fetched from SDR storage)
-    def self.get_sdr_metadata(druid, dsname)
-      Dor::Services::Client.object(druid).sdr.metadata(datastream: dsname)
-    end
-    private_class_method :get_sdr_metadata
-
-    # @param [DruidTools::Druid] druid A wrapper class for the druid identifier.  Used to generate paths
-    # @param [Array<String>] new_files The list of filenames for files that are either added or modifed since the previous version
-    # @return [String] The technicalMetadata datastream for the new files of the new digital object version
-    def self.get_new_technical_metadata(druid, new_files)
-      return nil if new_files.nil? || new_files.empty?
-
-      workspace = DruidTools::Druid.new(druid, Dor::Config.sdr.local_workspace_root)
-      content_dir = workspace.find_filelist_parent('content', new_files)
-      temp_dir = workspace.temp_dir
-      jhove_service = ::JhoveService.new(temp_dir)
-      jhove_service.digital_object_id = druid
-      fileset_file = write_fileset(temp_dir, new_files)
-      jhove_output_file = jhove_service.run_jhove(content_dir, fileset_file)
-      tech_md_file = jhove_service.create_technical_metadata(jhove_output_file)
-      IO.read(tech_md_file)
-    end
-    private_class_method :get_new_technical_metadata
-
-    # @param [Pathname]  temp_dir  The pathname of the temp folder in the object's workspace area
-    # @param [Object] new_files [Array<String>] The list of filenames for files that are either added or modifed since the previous version
-    # @return [Pathname] Save the new_files list to a text file and return that file's name
-    def self.write_fileset(temp_dir, new_files)
-      fileset_pathname = Pathname(temp_dir).join('jhove_fileset.txt')
-      fileset_pathname.open('w') { |f| f.puts(new_files) }
-      fileset_pathname
-    end
-    private_class_method :write_fileset
-
-    # @param [String] old_techmd The technicalMetadata datastream from the previous version of the digital object
-    # @param [String] new_techmd The technicalMetadata datastream for the new files of the new digital object version
-    # @param [Array<String>] deltas The list of filenames for files that are either added or modifed since the previous version
-    # @return [Hash<String,Nokogiri::XML::Node>] The complete set of technicalMetadata nodes for the digital object, indexed by filename
-    def self.merge_file_nodes(old_techmd, new_techmd, deltas)
-      old_file_nodes = get_file_nodes(old_techmd)
-      new_file_nodes = get_file_nodes(new_techmd)
-      merged_nodes = {}
-      deltas[:identical].each do |path|
-        merged_nodes[path] = old_file_nodes[path]
-      end
-      deltas[:modified].each do |path|
-        merged_nodes[path] = new_file_nodes[path]
-      end
-      deltas[:added].each do |path|
-        merged_nodes[path] = new_file_nodes[path]
-      end
-      deltas[:renamed].each do |oldpath, newpath|
-        clone = old_file_nodes[oldpath].clone
-        clone.sub!(/<file\s*id.*?["'].*?["'].*?>/, "<file id='#{newpath}'>")
-        merged_nodes[newpath] = clone
-      end
-      deltas[:copyadded].each do |oldpath, newpath|
-        clone = old_file_nodes[oldpath].clone
-        clone.sub!(/<file\s*id.*?["'].*?["'].*?>/, "<file id='#{newpath}'>")
-        merged_nodes[newpath] = clone
-      end
-      merged_nodes
-    end
-    private_class_method :merge_file_nodes
-
-    # @param [String] technical_metadata A technicalMetadata datastream contents
-    # @return [Hash<String,Nokogiri::XML::Node>] The set of nodes from a technicalMetadata datastream, indexed by filename
-    def self.get_file_nodes(technical_metadata)
-      file_hash = {}
-      return file_hash if technical_metadata.nil?
-
-      current_file = []
-      path = nil
-      in_file = false
-      technical_metadata.each_line do |line|
-        if line =~ /^\s*<file.*["'](.*?)["']/
-          current_file << line
-          path = $1
-          in_file = true
-        elsif line =~ /^\s*<\/file>/
-          current_file << line
-          file_hash[path] = current_file.join
-          current_file = []
-          path = nil
-          in_file = false
-        elsif in_file
-          current_file << line
-        end
-      end
-      file_hash
-    end
-    private_class_method :get_file_nodes
-
-    # @param [String] druid The identifier of the digital object being processed by the technical metadata robot
-    # @param [Hash<String,Nokogiri::XML::Node>] merged_nodes The complete set of technicalMetadata nodes for the digital object, indexed by filename
-    # @return [String] The finalized technicalMetadata datastream contents for the new object version
-    def self.build_technical_metadata(druid, merged_nodes)
-      techmd_root = +<<~EOF
-        <technicalMetadata objectId='#{druid}' datetime='#{Time.now.utc.iso8601}'
-            xmlns:jhove='http://hul.harvard.edu/ois/xml/ns/jhove'
-            xmlns:mix='http://www.loc.gov/mix/v10'
-            xmlns:textmd='info:lc/xmlns/textMD-v3'>
-      EOF
-      doc = techmd_root
-      merged_nodes.keys.sort.each { |path| doc << merged_nodes[path] }
-      doc + '</technicalMetadata>'
-    end
-    private_class_method :build_technical_metadata
-  end
-end