longleaf 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a5f659d06bd9b1094ed6dd0e10d108282bacde2522e4f07f5a2118dcbffb030e
-  data.tar.gz: '0368bb1228372a7b617aa148844f8cc88fbd453b3a704aa0c66d2d1b0abd21ef'
+  metadata.gz: 71c3c9e78a968f153c457d2a59ae17e8d45b4f6e905deeb787172844cee3733f
+  data.tar.gz: d354dba30939cdf1917b09fa7fc874654135f311dd1d5c60453cb42fe2b2a736
 SHA512:
-  metadata.gz: 12917813fba8f517585d9e20d22044fce229451b5a22b10edd44f3accaea807a4049d867980fb8b301202f6d0bd4f047b0a06df78d46d054e8e7cc4eeeda9cb6
-  data.tar.gz: fe2f94a2f0bdbe5b2095c63275adca26a5c0f8a6d8dd013bdfa9a1465073085f80a8f2e32b0d5aa7b53c19cce58e48bb1fa9b16479db9b3bcd0af4c649128188
+  metadata.gz: af2433108df902de025137d67c4b3d089b8766f7856d10d5d93d6bd449b82227c7cf3e987e50199c83a0869056842cb34d55f710102fa8a654282170c215882a
+  data.tar.gz: 93deda52a5f3de34ebc29baef3f531db099ea76521389dc223f33c2922da0e1fb81d775c8178370ade67ea397761553bf9ec34c03c12eec1e0a77d12174e07de

data/.circleci/config.yml

@@ -78,7 +78,7 @@ jobs:
 
   test_with_ruby_latest:
     docker:
-      - image: circleci/ruby:latest
+      - image: circleci/ruby:2.7
         environment:
           BUNDLER_VERSION: 2.1.4
 

data/lib/longleaf/candidates/file_selector.rb

@@ -1,4 +1,5 @@
 require 'longleaf/logging'
+require 'longleaf/candidates/physical_path_provider'
 
 module Longleaf
   # Selects and allows for iteration over files which match a provided set of selection criteria
@@ -10,7 +11,8 @@ module Longleaf
     attr_reader :specificity
 
     # May only provide either file_paths or storage_locations
-    def initialize(file_paths: nil, storage_locations: nil, app_config:)
+    def initialize(file_paths: nil, storage_locations: nil, physical_provider: Longleaf::PhysicalPathProvider.new,
+           app_config:)
       if nil_or_empty?(file_paths) && nil_or_empty?(storage_locations)
         raise ArgumentError.new("Must provide either file paths or storage locations")
       end
@@ -36,6 +38,7 @@ module Longleaf
       end
       # The set of storage locations to select file paths from
       @storage_locations = storage_locations
+      @physical_provider = physical_provider
       # Validate that the selected storage locations are known
       if @storage_locations.nil?
         @specificity = SPECIFICITY_PATH
@@ -63,7 +66,7 @@ module Longleaf
       @target_paths
     end
 
-    # Get the next file path for this selector.
+    # Get the next logical file path for this selector.
     # @return [String] an absolute path to the next file targeted by this selector,
     # or nil if no more files selected
     def next_path
@@ -80,9 +83,17 @@ module Longleaf
       path = @paths.pop
       until path.nil? do
         @app_config.location_manager.verify_path_in_location(path)
+        physical_path = @physical_provider.get_physical_path(path)
+        separate_logical = physical_path != path
+        if separate_logical
+          @app_config.location_manager.verify_path_in_location(physical_path)
+        end
 
-        if File.exist?(path)
-          if File.directory?(path)
+        if File.exist?(physical_path)
+          if File.directory?(physical_path)
+            if separate_logical
+              raise InvalidStoragePathError.new("Cannot specify physical path to a directory: #{physical_path}")
+            end
             logger.debug("Expanding directory #{path}")
             # For a directory, add all children to file_paths
             Dir.entries(path).sort.reverse_each do |child|
@@ -93,7 +104,7 @@ module Longleaf
             return path
           end
         else
-          raise InvalidStoragePathError.new("File #{path} does not exist.")
+          raise InvalidStoragePathError.new("File #{physical_path} does not exist.")
         end
 
         # Returned path was not a suitable file, try the next path

data/lib/longleaf/candidates/manifest_digest_provider.rb

@@ -1,7 +1,7 @@
 module Longleaf
   # Provides digests for files from a manifest
   class ManifestDigestProvider
-    # @param hash which maps file paths to hashs of digests
+    # @param digests_mapping hash which maps file paths to hashs of digests
     def initialize(digests_mapping)
       @digests_mapping = digests_mapping
     end

data/lib/longleaf/candidates/physical_path_provider.rb

@@ -0,0 +1,17 @@
+module Longleaf
+  # Provides physical paths for logical paths from a mapping
+  class PhysicalPathProvider
+    # @param phys_mapping hash with logical paths as keys, physical paths as values
+    def initialize(phys_mapping = Hash.new)
+      @phys_mapping = phys_mapping
+    end
+
+    # @param logical_path [String] logical path of file
+    # @return physical path of the file
+    def get_physical_path(logical_path)
+      # return the logical path itself if no physical path is mapped
+      return logical_path unless @phys_mapping.key?(logical_path)
+      @phys_mapping[logical_path]
+    end
+  end
+end

data/lib/longleaf/cli.rb

@@ -65,12 +65,12 @@ module Longleaf
               :aliases => "-f",
               :required => false,
               :desc => 'File or files to perform this operation on. If multiple files are provided, they must be comma separated.' })
+
     add_shared_option(
-        :location, :file_selection, {
+        :location, :registered_selection, {
               :aliases => "-s",
               :required => false,
               :desc => 'Name or comma separated names of storage locations to perform this operation over.' })
-
     add_shared_option(
         :from_list, :registered_selection, {
               :aliases => "-l",
@@ -115,7 +115,21 @@ module Longleaf
           ...
           md5:
           <digest> <path>
-          ...})
+          ...
+          
+          To provide separate logical and physical paths, add a physical path column:
+          '-m sha1:@-'
+          Where the content in STDIN adheres to the format:
+          <digest> <logical path> <physical path>
+          ...
+        })
+    method_option(:physical_path,
+        :aliases => "-p",
+        :required => false,
+        :desc => %q{Comma separated list of physical paths of files to register. Only needed
+          if the physical and logical paths of the files differ, otherwise they will be assumed to be the same.
+          Only applicable when used with the -f option, and only for individual files, not directories.
+          Must be provided in the same order as the logical paths.})
     method_option(:force,
         :type => :boolean,
         :default => false,
@@ -132,11 +146,12 @@ module Longleaf
 
       app_config_manager = load_application_config(options)
 
-      file_selector, digest_provider = SelectionOptionsParser.parse_registration_selection_options(
-          options, app_config_manager)
+      file_selector, digest_provider, physical_provider = SelectionOptionsParser
+          .parse_registration_selection_options(options, app_config_manager)
 
       command = RegisterCommand.new(app_config_manager)
-      exit command.execute(file_selector: file_selector, force: options[:force], digest_provider: digest_provider)
+      exit command.execute(file_selector: file_selector, force: options[:force], digest_provider: digest_provider,
+           physical_provider: physical_provider)
     end
 
     desc "deregister", "Deregister files with Longleaf"

data/lib/longleaf/commands/register_command.rb

@@ -16,9 +16,10 @@ module Longleaf
     # Execute the register command on the given parameters
     # @param file_selector [FileSelector] selector for files to register
     # @param force [Boolean] force flag
-    # @param digest_provider [DigestProvider] object which provides digests for files being registered
+    # @param digest_provider [ManifestDigestProvider] object which provides digests for files being registered
+    # @param physical_provider [PhysicalPathProvider] object which provides physical paths for files being registered
     # @return [Integer] status code
-    def execute(file_selector:, force: false, digest_provider: nil)
+    def execute(file_selector:, force: false, digest_provider: nil, physical_provider: nil)
       start_time = Time.now
       logger.info('Performing register command')
       begin
@@ -29,7 +30,8 @@ module Longleaf
 
           storage_location = @app_manager.location_manager.get_location_by_path(f_path)
 
-          file_rec = FileRecord.new(f_path, storage_location)
+          phys_path = physical_provider.get_physical_path(f_path)
+          file_rec = FileRecord.new(f_path, storage_location, nil, phys_path)
 
           register_event = RegisterEvent.new(file_rec: file_rec, force: force, app_manager: @app_manager,
               digest_provider: digest_provider)

data/lib/longleaf/events/preserve_event.rb

@@ -27,12 +27,13 @@ module Longleaf
       service_manager = @app_manager.service_manager
       md_rec = @file_rec.metadata_record
       f_path = @file_rec.path
+      phys_path = @file_rec.physical_path
 
-      logger.info("Performing preserve event on #{@file_rec.path}")
+      logger.info("Performing preserve event on #{f_path}")
 
       needs_persist = false
       begin
-        if !File.exist?(f_path)
+        if !File.exist?(phys_path)
           # Need to persist metadata to avoid repeating processing of this file too soon.
           needs_persist = true
           record_failure(EventNames::PRESERVE, f_path, "File is registered but missing.")

data/lib/longleaf/events/register_event.rb

@@ -70,10 +70,17 @@ module Longleaf
     private
     def populate_file_properties
       md_rec = @file_rec.metadata_record
+      physical_path = @file_rec.physical_path
 
       # Set file properties
-      md_rec.last_modified = File.mtime(@file_rec.path).utc.iso8601(3)
-      md_rec.file_size = File.size(@file_rec.path)
+      md_rec.last_modified = File.mtime(physical_path).utc.iso8601(3)
+      md_rec.file_size = File.size(physical_path)
+
+      if physical_path != @file_rec.path
+        md_rec.physical_path = physical_path
+      else
+        md_rec.physical_path = nil
+      end
     end
 
     # Copy a subset of properties from an existing metadata record to the new record

data/lib/longleaf/helpers/selection_options_parser.rb

@@ -1,6 +1,7 @@
 require 'longleaf/candidates/file_selector'
 require 'longleaf/candidates/registered_file_selector'
 require 'longleaf/candidates/manifest_digest_provider'
+require 'longleaf/candidates/physical_path_provider'
 require 'longleaf/candidates/single_digest_provider'
 
 module Longleaf
@@ -18,8 +19,11 @@ module Longleaf
           options, :file, :manifest, :location)
 
       if !options[:manifest].nil?
-        digests_mapping = self.manifests_to_digest_mapping(options[:manifest])
-        selector = FileSelector.new(file_paths: digests_mapping.keys, app_config: app_config_manager)
+        digests_mapping, logical_phys_mapping = self.parse_manifest(options[:manifest])
+        physical_provider = PhysicalPathProvider.new(logical_phys_mapping)
+        selector = FileSelector.new(file_paths: digests_mapping.keys,
+             physical_provider: physical_provider,
+             app_config: app_config_manager)
         digest_provider = ManifestDigestProvider.new(digests_mapping)
       elsif !options[:file].nil?
         if options[:checksums]
@@ -35,18 +39,28 @@ module Longleaf
           end
         end
 
-        file_paths = options[:file].split(/\s*,\s*/)
-        selector = FileSelector.new(file_paths: file_paths, app_config: app_config_manager)
-      elsif !options[:location].nil?
-        storage_locations = options[:location].split(/\s*,\s*/)
-        selector = FileSelector.new(storage_locations: storage_locations, app_config: app_config_manager)
-        digest_provider = SingleDigestProvider.new(nil)
+        file_paths = self.split_quoted(options[:file], "\\s*,\\s*")
+        if !options[:physical_path].nil?
+          physical_paths = self.split_quoted(options[:physical_path], "\\s*,\\s*")
+          if physical_paths.length != file_paths.length
+            logger.failure("Invalid physical paths parameter, number of paths did not match number of logical paths")
+            exit 1
+          end
+          logical_phys_mapping = Hash[file_paths.zip physical_paths]
+          physical_provider = PhysicalPathProvider.new(logical_phys_mapping)
+        else
+          physical_provider = PhysicalPathProvider.new
+        end
+        
+        selector = FileSelector.new(file_paths: file_paths,
+             physical_provider: physical_provider,
+             app_config: app_config_manager)
       else
         logger.failure("Must provide one of the following file selection options: -f, l, or -m")
         exit 1
       end
 
-      [selector, digest_provider]
+      [selector, digest_provider, physical_provider]
     end
 
     def self.there_can_be_only_one(failure_msg, options, *names)
@@ -69,7 +83,7 @@ module Longleaf
     #.      <manifest_path> OR @-
     # @return a hash containing the aggregated contents of the provided manifests. The keys are
     #    paths to manifested files. The values are hashes, mapping digest algorithms to digest values.
-    def self.manifests_to_digest_mapping(manifest_vals)
+    def self.parse_manifest(manifest_vals)
       alg_manifest_pairs = []
       # interpret option inputs into a list of algorithms to manifest sources
       manifest_vals.each do |manifest_val|
@@ -87,6 +101,7 @@ module Longleaf
 
       # read the provided manifests to build a mapping from file uri to all supplied digests
       digests_mapping = Hash.new { |h,k| h[k] = Hash.new }
+      logical_phys_mapping = Hash.new
       alg_manifest_pairs.each do |mpair|
         source_stream = nil
         # Determine if reading from a manifest file or stdin
@@ -111,17 +126,28 @@ module Longleaf
             if current_alg.nil?
               self.fail("Manifest with unknown checksums encountered, an algorithm must be specified")
             end
-            entry_parts = line.split(' ', 2)
-            if entry_parts.length != 2
+            entry_parts = self.split_quoted(line)
+            if entry_parts.length != 2 && entry_parts.length != 3
               self.fail("Invalid manifest entry: #{line}")
             end
 
             digests_mapping[entry_parts[1]][current_alg] = entry_parts[0]
+            if (entry_parts.length == 3)
+              logical_phys_mapping[entry_parts[1]] = entry_parts[2]
+            end
           end
         end
       end
 
-      digests_mapping
+      [digests_mapping, logical_phys_mapping]
+    end
+    
+    # Splits a string of quoted or unquoted tokens separated by spaces
+    # @param
+    def self.split_quoted(text, delimiter = "\\s+", limit = -1)
+      text.split(/#{delimiter}(?=(?:[^'"]|'[^']*'|"[^"]*")*$)/, limit)
+          .select {|s| not s.empty? }
+          .map {|s| s.gsub(/(^ +)|( +$)|(^["']+)|(["']+$)/, '')}
     end
 
     # Parses the provided options to create a selector for registered files

data/lib/longleaf/logging/redirecting_logger.rb

@@ -65,6 +65,10 @@ module Longleaf
       def unknown(progname = nil, &block)
         @stderr_log.unknown(progname, &block)
       end
+      
+      def <<(msg)
+        @stderr_log << msg
+      end
 
       # Logs a success message to STDOUT, as well as STDERR at info level.
       #

data/lib/longleaf/models/file_record.rb

@@ -7,13 +7,16 @@ module Longleaf
 
     # @param file_path [String] path to the file
     # @param storage_location [StorageLocation] storage location containing the file
-    def initialize(file_path, storage_location, metadata_record = nil)
+    # @param metadata_record [MetadataRecord] metadata record for this file object. Optional.
+    # @param physical_path [String] physical path where the file is located. Defaults to the file_path.
+    def initialize(file_path, storage_location, metadata_record = nil, physical_path = nil)
       raise ArgumentError.new("FileRecord requires a path") if file_path.nil?
       raise ArgumentError.new("FileRecord requires a storage_location") if storage_location.nil?
 
       @path = file_path
       @storage_location = storage_location
       @metadata_record = metadata_record
+      @physical_path = physical_path
     end
 
     # @return [String] path for the metadata file for this file
@@ -21,6 +24,17 @@ module Longleaf
       @metadata_path = @storage_location.get_metadata_path_for(path) if @metadata_path.nil?
       @metadata_path
     end
+    
+    def physical_path
+      if @physical_path.nil?
+        if @metadata_record.nil? || @metadata_record.physical_path.nil?
+          @physical_path = @path
+        else
+          @physical_path = @metadata_record.physical_path
+        end
+      end
+      @physical_path
+    end
 
     def metadata_present?
       File.exist?(metadata_path)

data/lib/longleaf/models/md_fields.rb

@@ -9,6 +9,7 @@ module Longleaf
 
     LAST_MODIFIED = 'last-modified'
     FILE_SIZE = 'size'
+    PHYSICAL_PATH = 'physical-path'
 
     CHECKSUMS = 'checksums'
 

data/lib/longleaf/models/metadata_record.rb

@@ -10,6 +10,7 @@ module Longleaf
     attr_reader :checksums
     attr_reader :properties
     attr_accessor :file_size, :last_modified
+    attr_accessor :physical_path
 
     # @param properties [Hash] initial data properties for this record
     # @param services [Hash] initial service property tree
@@ -18,8 +19,9 @@ module Longleaf
     # @param checksums [Hash] hash of checksum values
     # @param file_size [Integer] size of file in bytes
     # @param last_modified [String] iso8601 representation of the last modified date of file
+    # @param physical_path [String] physical path where the file is located
     def initialize(properties: nil, services: nil, deregistered: nil, registered: nil, checksums: nil,
-          file_size: nil, last_modified: nil)
+          file_size: nil, last_modified: nil, physical_path: nil)
       @properties = properties || Hash.new
       @registered = registered
       @deregistered = deregistered
@@ -28,6 +30,7 @@ module Longleaf
       @services = services || Hash.new
       @file_size = file_size
       @last_modified = last_modified
+      @physical_path = physical_path
     end
 
     # @return [Boolean] true if the record is deregistered

data/lib/longleaf/models/s3_storage_location.rb

@@ -1,6 +1,7 @@
 require 'longleaf/models/storage_location'
 require 'longleaf/models/storage_types'
 require 'longleaf/helpers/s3_uri_helper'
+require 'longleaf/logging'
 require 'uri'
 require 'aws-sdk-s3'
 
@@ -12,6 +13,7 @@ module Longleaf
   # https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html#constructor_details
 
   class S3StorageLocation < StorageLocation
+    include Longleaf::Logging
 
     IS_URI_REGEX = /\A#{URI::regexp}\z/
 
@@ -38,6 +40,9 @@ module Longleaf
         # Clone options and convert keys to symbols
         @client_options = Hash[custom_options.map { |(k,v)| [k.to_sym,v] } ]
       end
+      @client_options[:logger] = logger
+      @client_options[:log_level] = :debug if @client_options[:log_level].nil?
+      
       # If no region directly configured, use region from path
       if !@client_options.key?(:region)
         region = S3UriHelper.extract_region(@path)

data/lib/longleaf/preservation_services/file_check_service.rb

@@ -23,22 +23,23 @@ module Longleaf
     # @raise [PreservationServiceError] if the file system information does not match the stored details
     def perform(file_rec, event)
       file_path = file_rec.path
+      phys_path = file_rec.physical_path
       md_rec = file_rec.metadata_record
 
       logger.debug("Performing file information check of #{file_path}")
 
-      if !File.exist?(file_path)
-        raise PreservationServiceError.new("File does not exist: #{file_path}")
+      if !File.exist?(phys_path)
+        raise PreservationServiceError.new("File does not exist: #{phys_path}")
       end
 
-      file_size = File.size(file_rec.path)
+      file_size = File.size(phys_path)
       if file_size != md_rec.file_size
-        raise PreservationServiceError.new("File size for #{file_path} does not match the expected value: registered = #{md_rec.file_size} bytes, actual = #{file_size} bytes")
+        raise PreservationServiceError.new("File size for #{phys_path} does not match the expected value: registered = #{md_rec.file_size} bytes, actual = #{file_size} bytes")
       end
 
-      last_modified = File.mtime(file_rec.path).utc.iso8601(3)
+      last_modified = File.mtime(phys_path).utc.iso8601(3)
       if last_modified != md_rec.last_modified
-        raise PreservationServiceError.new("Last modified timestamp for #{file_path} does not match the expected value: registered = #{md_rec.last_modified}, actual = #{last_modified}")
+        raise PreservationServiceError.new("Last modified timestamp for #{phys_path} does not match the expected value: registered = #{md_rec.last_modified}, actual = #{last_modified}")
       end
     end
 

data/lib/longleaf/preservation_services/fixity_check_service.rb

@@ -63,6 +63,7 @@ module Longleaf
     # @raise [ChecksumMismatchError] if the checksum on record does not match the generated checksum
     def perform(file_rec, event)
       path = file_rec.path
+      phys_path = file_rec.physical_path
       md_rec = file_rec.metadata_record
 
       # Get the list of existing checksums for the file and normalize algorithm names
@@ -89,19 +90,19 @@ module Longleaf
         end
 
         digest = DigestHelper::start_digest(alg)
-        digest.file(path)
+        digest.file(phys_path)
         generated_digest = digest.hexdigest
 
         # Store the missing checksum if using the 'generate' behavior
         if existing_digest.nil? && @absent_digest_behavior == GENERATE_IF_ABSENT
           md_rec.checksums[alg] = generated_digest
-          logger.info("Generated and stored digest using algorithm '#{alg}' for file #{path}")
+          logger.info("Generated and stored digest using algorithm '#{alg}' for file #{phys_path}")
         else
           # Compare the new digest to the one on record
           if existing_digest == generated_digest
-            logger.info("Fixity check using algorithm '#{alg}' succeeded for file #{path}")
+            logger.info("Fixity check using algorithm '#{alg}' succeeded for file #{phys_path}")
           else
-            raise ChecksumMismatchError.new("Fixity check using algorithm '#{alg}' failed for file #{path}: expected '#{existing_digest}', calculated '#{generated_digest}.'")
+            raise ChecksumMismatchError.new("Fixity check using algorithm '#{alg}' failed for file #{phys_path}: expected '#{existing_digest}', calculated '#{generated_digest}.'")
           end
         end
       end

data/lib/longleaf/preservation_services/rsync_replication_service.rb

@@ -51,9 +51,6 @@ module Longleaf
             + " rsync_options may not include the following: #{DISALLOWED_OPTIONS.join(' ')}")
       end
 
-      # Add -R (--relative) in to command options to ensure full path gets replicated
-      @options = @options + " -R"
-
       # Set and validate the replica collision policy
       @collision_policy = @service_def.properties[SF::COLLISION_PROPERTY] || SF::DEFAULT_COLLISION_POLICY
       if !SF::VALID_COLLISION_POLICIES.include?(@collision_policy)
@@ -101,17 +98,36 @@ module Longleaf
         else
           dest_path = destination
         end
-
+        
+        logical_physical_same = file_rec.path == file_rec.physical_path
         # Determine the path to the file being replicated relative to its storage location
         rel_path = file_rec.storage_location.relativize(file_rec.path)
-        # source path with . so that rsync will only create destination directories starting from that point
-        source_path = File.join(file_rec.storage_location.path, "./#{rel_path}")
+        
+        options = @options
+        if logical_physical_same
+          options = options + " -R"
+          # source path with . so that rsync will only create destination directories starting from that point
+          source_path = File.join(file_rec.storage_location.path, "./#{rel_path}")
+        else
+          options = options + " --no-relative"
+          source_path = file_rec.physical_path
+          dest_path = File.join(dest_path, rel_path)
+          if (dest_is_storage_loc && destination.is_a?(Longleaf::FilesystemStorageLocation)) || !dest_is_storage_loc
+            # Fill in missing parent directories, as rsync cannot do so when specifying a different source and dest filename
+            dirname = File.dirname(dest_path)
+            logger.debug("Creating parent dirs #{dirname} for #{file_rec.path}")
+            FileUtils.mkdir_p(dirname)
+          else
+            raise PreservationServiceError.new(
+                "Destination #{destination.name} does not currently support separate physical and logical paths")
+          end
+        end
 
         # Check that the destination is available because attempting to write
         verify_destination_available(destination, file_rec)
 
-        logger.debug("Invoking rsync with command: #{@command} \"#{source_path}\" \"#{dest_path}\" #{@options}")
-        stdout, stderr, status = Open3.capture3("#{@command} \"#{source_path}\" \"#{dest_path}\" #{@options}")
+        logger.debug("Invoking rsync with command: #{@command} \"#{source_path}\" \"#{dest_path}\" #{options}")
+        stdout, stderr, status = Open3.capture3("#{@command} \"#{source_path}\" \"#{dest_path}\" #{options}")
         raise PreservationServiceError.new("Failed to replicate #{file_rec.path} to #{dest_path}: #{stderr}") \
             unless status.success?
 

data/lib/longleaf/preservation_services/s3_replication_service.rb

@@ -83,8 +83,6 @@ module Longleaf
       # Determine the path to the file being replicated relative to its storage location
       rel_path = file_rec.storage_location.relativize(file_rec.path)
 
-      content_md5 = get_content_md5(file_rec)
-
       @destinations.each do |destination|
         # Check that the destination is available before attempting to write
         verify_destination_available(destination, file_rec)
@@ -92,7 +90,7 @@ module Longleaf
         rel_to_bucket = destination.relative_to_bucket_path(rel_path)
         file_obj = destination.s3_bucket.object(rel_to_bucket)
         begin
-          file_obj.upload_file(file_rec.path, { :content_md5 => content_md5 })
+          file_obj.upload_file(file_rec.physical_path)
         rescue Aws::S3::Errors::BadDigest => e
           raise ChecksumMismatchError.new("Transfer to bucket '#{destination.s3_bucket.name}' failed, " \
               + "MD5 provided did not match the received content for #{file_rec.path}")
@@ -129,15 +127,5 @@ module Longleaf
             + e.message)
       end
     end
-
-    def get_content_md5(file_rec)
-      md_rec = file_rec.metadata_record
-      if md_rec.checksums.key?('md5')
-        # base 64 encode the digest, as is required by the Content-Md5 header
-        [[md_rec.checksums['md5']].pack("H*")].pack("m0")
-      else
-        nil
-      end
-    end
   end
 end

data/lib/longleaf/services/metadata_deserializer.rb

@@ -32,6 +32,7 @@ module Longleaf
       checksums = data.delete(MDFields::CHECKSUMS)
       file_size = data.delete(MDFields::FILE_SIZE)
       last_modified = data.delete(MDFields::LAST_MODIFIED)
+      physical_path = data.delete(MDFields::PHYSICAL_PATH)
 
       services = md[MDF::SERVICES]
       service_records = Hash.new
@@ -57,7 +58,8 @@ module Longleaf
                          deregistered: deregistered,
                          checksums: checksums,
                          file_size: file_size,
-                         last_modified: last_modified)
+                         last_modified: last_modified,
+                         physical_path: physical_path)
     end
 
     # Load configuration a yaml encoded configuration file

data/lib/longleaf/services/metadata_serializer.rb

@@ -52,6 +52,7 @@ module Longleaf
       data[MDF::CHECKSUMS] = metadata.checksums unless metadata.checksums && metadata.checksums.empty?
       data[MDF::FILE_SIZE] = metadata.file_size unless metadata.file_size.nil?
       data[MDF::LAST_MODIFIED] = metadata.last_modified if metadata.last_modified
+      data[MDF::PHYSICAL_PATH] = metadata.physical_path if metadata.physical_path
 
       props[MDF::DATA] = data
 

data/lib/longleaf/specs/metadata_builder.rb

@@ -29,6 +29,11 @@ module Longleaf
       @checksums[alg] = value
       self
     end
+    
+    def with_physical_path(phys_path)
+      @physical_path = phys_path
+      self
+    end
 
     def with_service(name, timestamp: ServiceDateHelper::formatted_timestamp, run_needed: false, properties: nil,
           failure_timestamp: nil)
@@ -56,7 +61,8 @@ module Longleaf
           registered: @registered,
           checksums: @checksums,
           file_size: @file_size,
-          last_modified: @last_modified)
+          last_modified: @last_modified,
+          physical_path: @physical_path)
     end
 
     # Add the generated metadata record to the given file record

data/lib/longleaf/version.rb

@@ -1,3 +1,3 @@
 module Longleaf
-  VERSION = "1.0.0"
+  VERSION = "1.1.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: longleaf
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Ben Pennell
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-10-14 00:00:00.000000000 Z
+date: 2021-03-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -372,6 +372,7 @@ files:
 - lib/longleaf.rb
 - lib/longleaf/candidates/file_selector.rb
 - lib/longleaf/candidates/manifest_digest_provider.rb
+- lib/longleaf/candidates/physical_path_provider.rb
 - lib/longleaf/candidates/registered_file_selector.rb
 - lib/longleaf/candidates/service_candidate_filesystem_iterator.rb
 - lib/longleaf/candidates/service_candidate_index_iterator.rb
@@ -449,7 +450,7 @@ licenses:
 - Apache-2.0
 metadata:
   source_code_uri: https://github.com/UNC-Libraries/longleaf-preservation
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -464,8 +465,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: Longleaf preservation services tool
 test_files: []