longleaf 0.1.0 → 1.1.1

data/lib/longleaf/models/service_definition.rb

@@ -1,21 +1,22 @@
 require_relative 'service_fields'
 
-# Definition of a preservation service
 module Longleaf
+  # Definition of a configured preservation service
   class ServiceDefinition
     attr_reader :name
-    attr_reader :work_script
+    attr_reader :work_script, :work_class
     attr_reader :frequency, :delay
     attr_reader :properties
-    
-    def initialize(name:, work_script:, frequency: nil, delay: nil, properties: Hash.new)
+
+    def initialize(name:, work_script:, work_class: nil, frequency: nil, delay: nil, properties: Hash.new)
       raise ArgumentError.new("Parameters name and work_script are required") unless name && work_script
-      
+
       @properties = properties
       @name = name
       @work_script = work_script
+      @work_class = work_class
       @frequency = frequency
       @delay = delay
     end
   end
-end
+end

data/lib/longleaf/models/service_fields.rb

@@ -1,10 +1,16 @@
 module Longleaf
+  # Constants for common configuration fields for preservation service definitions
   class ServiceFields
     WORK_SCRIPT = 'work_script'
+    WORK_CLASS = 'work_class'
     FREQUENCY = 'frequency'
     DELAY = 'delay'
-    
+
     REPLICATE_TO = 'to'
     DIGEST_ALGORITHMS = 'algorithms'
+
+    COLLISION_PROPERTY = "replica_collision_policy"
+    DEFAULT_COLLISION_POLICY = "replace"
+    VALID_COLLISION_POLICIES = ["replace"]
   end
 end

data/lib/longleaf/models/service_record.rb

@@ -1,27 +1,31 @@
-# Record for an individual service in a file's metadata record.
 module Longleaf
+  # Record for an individual service in a file's metadata record.
   class ServiceRecord
     attr_reader :properties
     attr_accessor :stale_replicas, :timestamp, :run_needed
-    
+    attr_accessor :failure_timestamp
+
     # @param properties [Hash] initial properties for this service record
+    # @param stale_replicas [Boolean] whether there are any stale replicas from this service
+    # @param timestamp [String] timestamp when this service last ran or was initialized
+    # @param run_needed [Boolean] flag indicating that this service should be run at the next available opportunity
     def initialize(properties: Hash.new, stale_replicas: false, timestamp: nil, run_needed: false)
       raise ArgumentError.new("Service properties must be a hash") if properties.class != Hash
-      
+
       @properties = properties
       @timestamp = timestamp
       @stale_replicas = stale_replicas
       @run_needed = run_needed
     end
-    
+
     # @return the value of a service property identified by key
     def [](key)
       @properties[key]
     end
-    
+
     # set the value of a service property identified by key
     def []=(key, value)
       @properties[key] = value
     end
   end
-end
+end

data/lib/longleaf/models/storage_location.rb

@@ -1,19 +1,25 @@
-require 'longleaf/services/metadata_serializer'
+require 'longleaf/models/app_fields'
 
 module Longleaf
+  # Representation of a configured storage location
   class StorageLocation
+    AF ||= Longleaf::AppFields
+
     attr_reader :name
     attr_reader :path
-    attr_reader :metadata_path
-    
-    def initialize(name:, path:, metadata_path:)
-      raise ArgumentError.new("Parameters name, path and metadata_path are required") unless name && path && metadata_path
-      
-      @path = path
+    attr_reader :metadata_location
+
+    # @param name [String] the name of this storage location
+    # @param config [Hash] hash containing the configuration options for this location
+    # @param md_loc [MetadataLocation] metadata location associated with this storage location
+    def initialize(name, config, md_loc)
+      raise ArgumentError.new("Config parameter is required") unless config
+      @path = config[AF::LOCATION_PATH]
       @name = name
-      @metadata_path = metadata_path
+      raise ArgumentError.new("Parameters name, path and metadata location are required") unless @name && @path && md_loc
+      @metadata_location = md_loc
     end
-    
+
     # Get the path for the metadata file for the given file path located in this storage location.
     # @param file_path [String] path of the file
     # @raise [ArgumentError] if the file_path is not provided or is not in this storage location.
@@ -22,16 +28,15 @@ module Longleaf
       raise ArgumentError.new("Provided file path is not contained by storage location #{@name}: #{file_path}") \
           unless file_path.start_with?(@path)
 
-      file_path.sub(/^#{@path}/, metadata_path) + MetadataSerializer::metadata_suffix
+      rel_file_path = relativize(file_path)
+
+      @metadata_location.metadata_path_for(rel_file_path)
     end
-    
-    # Checks that the path and metadata path defined in this location are available
-    # @raise [StorageLocationUnavailableError] if the storage location is not available
-    def available?
-      raise StorageLocationUnavailableError.new("Path does not exist or is not a directory: #{@path}")\
-          unless Dir.exist?(@path)
-      raise StorageLocationUnavailableError.new("Metadata path does not exist or is not a directory: #{@metadata_path}")\
-          unless Dir.exist?(@metadata_path)
+
+    # @param [String] path to check
+    # @return true if the file path is contained by the path for this location
+    def contains?(file_path)
+      file_path.start_with?(@path)
     end
   end
-end
+end

data/lib/longleaf/models/storage_types.rb

@@ -0,0 +1,9 @@
+module Longleaf
+  # Storage type constants
+  class StorageTypes
+    FILESYSTEM_STORAGE_TYPE = 'filesystem'
+    S3_STORAGE_TYPE = 's3'
+
+    DEFAULT_STORAGE_TYPE = FILESYSTEM_STORAGE_TYPE
+  end
+end

data/lib/longleaf/models/system_config_fields.rb

@@ -0,0 +1,9 @@
+module Longleaf
+  # System configuration field names
+  class SystemConfigFields
+    MD_INDEX = 'index'
+    MD_INDEX_ADAPTER = 'adapter'
+    MD_INDEX_CONNECTION = 'connection'
+    MD_INDEX_PAGE_SIZE = 'page_size'
+  end
+end

data/lib/longleaf/preservation_services/file_check_service.rb

@@ -0,0 +1,59 @@
+require 'longleaf/events/event_names'
+require 'longleaf/logging'
+
+module Longleaf
+  # Preservation service which validates a file using current filesystem information compared against the
+  # last registered details for that file. Checks using file name, size and last modified timestamp.
+  class FileCheckService
+    include Longleaf::Logging
+
+    # Initialize a FileCheckService from the given service definition
+    #
+    # @param service_def [ServiceDefinition] the configuration for this service
+    # @param app_manager [ApplicationConfigManager] manager for configured storage locations
+    def initialize(service_def, app_manager)
+      @service_def = service_def
+      @app_manager = app_manager
+    end
+
+    # Perform file information check.
+    #
+    # @param file_rec [FileRecord] record representing the file to perform the service on.
+    # @param event [String] name of the event this service is being invoked by.
+    # @raise [PreservationServiceError] if the file system information does not match the stored details
+    def perform(file_rec, event)
+      file_path = file_rec.path
+      phys_path = file_rec.physical_path
+      md_rec = file_rec.metadata_record
+
+      logger.debug("Performing file information check of #{file_path}")
+
+      if !File.exist?(phys_path)
+        raise PreservationServiceError.new("File does not exist: #{phys_path}")
+      end
+
+      file_size = File.size(phys_path)
+      if file_size != md_rec.file_size
+        raise PreservationServiceError.new("File size for #{phys_path} does not match the expected value: registered = #{md_rec.file_size} bytes, actual = #{file_size} bytes")
+      end
+
+      last_modified = File.mtime(phys_path).utc.iso8601(3)
+      if last_modified != md_rec.last_modified
+        raise PreservationServiceError.new("Last modified timestamp for #{phys_path} does not match the expected value: registered = #{md_rec.last_modified}, actual = #{last_modified}")
+      end
+    end
+
+    # Determine if this service is applicable for the provided event, given the configured service definition
+    #
+    # @param event [String] name of the event
+    # @return [Boolean] returns true if this service is applicable for the provided event
+    def is_applicable?(event)
+      case event
+      when EventNames::PRESERVE
+        true
+      else
+        false
+      end
+    end
+  end
+end

data/lib/longleaf/preservation_services/fixity_check_service.rb

@@ -0,0 +1,124 @@
+require 'longleaf/events/event_names'
+require 'longleaf/models/service_fields'
+require 'longleaf/logging'
+require 'longleaf/helpers/digest_helper'
+require 'set'
+
+module Longleaf
+  # Preservation service which performs one or more fixity checks on a file based on the configured list
+  # of digest algorithms. It currently supports 'md5', 'sha1', 'sha2', 'sha256', 'sha384', 'sha512' and 'rmd160'.
+  #
+  # If the service encounters a file which is missing any of the digest algorithms the service is configured
+  # to check, the outcome may be controlled with the 'absent_digest' property via the following values:
+  #   * 'fail' - the service will raise a ChecksumMismatchError for the missing algorithm. This is the default.
+  #   * 'ignore' - the service will skip calculating any algorithms not already present for the file.
+  #   * 'generate' - the service will generate and store any missing digests from the set of configured algorithms.
+  class FixityCheckService
+    include Longleaf::Logging
+
+    SUPPORTED_ALGORITHMS = ['md5', 'sha1', 'sha2', 'sha256', 'sha384', 'sha512', 'rmd160']
+
+    # service configuration property indicating how to handle situations where a file does not
+    # have a digest for one of the expected algorithms on record.
+    ABSENT_DIGEST_PROPERTY = 'absent_digest'
+    FAIL_IF_ABSENT = 'fail'
+    GENERATE_IF_ABSENT = 'generate'
+    IGNORE_IF_ABSENT = 'ignore'
+    ABSENT_DIGEST_OPTIONS = [FAIL_IF_ABSENT, GENERATE_IF_ABSENT, IGNORE_IF_ABSENT]
+
+    # Initialize a FixityCheckService from the given service definition
+    #
+    # @param service_def [ServiceDefinition] the configuration for this service
+    # @param app_manager [ApplicationConfigManager] manager for configured storage locations
+    def initialize(service_def, app_manager)
+      @service_def = service_def
+      @absent_digest_behavior = @service_def.properties[ABSENT_DIGEST_PROPERTY] || FAIL_IF_ABSENT
+      unless ABSENT_DIGEST_OPTIONS.include?(@absent_digest_behavior)
+        raise ArgumentError.new("Invalid option '#{@absent_digest_behavior}' for property #{ABSENT_DIGEST_PROPERTY} in service #{service_def.name}")
+      end
+
+      service_algs = service_def.properties[ServiceFields::DIGEST_ALGORITHMS]
+      if service_algs.nil? || service_algs.empty?
+        raise ArgumentError.new("FixityCheckService from definition #{service_def.name} requires a list of one or more digest algorithms")
+      end
+
+      service_algs = [service_algs] if service_algs.is_a?(String)
+
+      # Store the list of digest algorithms to verify, using normalized algorithm names.
+      @digest_algs = Set.new
+      service_algs.each do |alg|
+        normalized_alg = alg.downcase.delete('-')
+        if SUPPORTED_ALGORITHMS.include?(normalized_alg)
+          @digest_algs << normalized_alg
+        else
+          raise ArgumentError.new("Unsupported checksum algorithm '#{alg}' in definition #{service_def.name}. Supported algorithms are: #{SUPPORTED_ALGORITHMS}")
+        end
+      end
+    end
+
+    # Perform all configured fixity checks on the provided file
+    #
+    # @param file_rec [FileRecord] record representing the file to perform the service on.
+    # @param event [String] name of the event this service is being invoked by.
+    # @raise [ChecksumMismatchError] if the checksum on record does not match the generated checksum
+    def perform(file_rec, event)
+      path = file_rec.path
+      phys_path = file_rec.physical_path
+      md_rec = file_rec.metadata_record
+
+      # Get the list of existing checksums for the file and normalize algorithm names
+      file_digests = Hash.new
+      md_rec.checksums&.each do |alg, digest|
+        normalized_alg = alg.downcase.delete('-')
+        if @digest_algs.include?(normalized_alg)
+          file_digests[normalized_alg] = digest
+        else
+          logger.debug("Metadata for file #{path} contains unexpected '#{alg}' digest, it will be ignored.")
+        end
+      end
+
+      @digest_algs.each do |alg|
+        existing_digest = file_digests[alg]
+
+        if existing_digest.nil?
+          if @absent_digest_behavior == FAIL_IF_ABSENT
+            raise ChecksumMismatchError.new("Fixity check using algorithm '#{alg}' failed for file #{path}: no existing digest of type '#{alg}' on record.")
+          elsif @absent_digest_behavior == IGNORE_IF_ABSENT
+            logger.debug("Skipping check of algorithm '#{alg}' for file #{path}: no digest on record.")
+            next
+          end
+        end
+
+        digest = DigestHelper::start_digest(alg)
+        digest.file(phys_path)
+        generated_digest = digest.hexdigest
+
+        # Store the missing checksum if using the 'generate' behavior
+        if existing_digest.nil? && @absent_digest_behavior == GENERATE_IF_ABSENT
+          md_rec.checksums[alg] = generated_digest
+          logger.info("Generated and stored digest using algorithm '#{alg}' for file #{phys_path}")
+        else
+          # Compare the new digest to the one on record
+          if existing_digest == generated_digest
+            logger.info("Fixity check using algorithm '#{alg}' succeeded for file #{phys_path}")
+          else
+            raise ChecksumMismatchError.new("Fixity check using algorithm '#{alg}' failed for file #{phys_path}: expected '#{existing_digest}', calculated '#{generated_digest}.'")
+          end
+        end
+      end
+    end
+
+    # Determine if this service is applicable for the provided event, given the configured service definition
+    #
+    # @param event [String] name of the event
+    # @return [Boolean] returns true if this service is applicable for the provided event
+    def is_applicable?(event)
+      case event
+      when EventNames::PRESERVE
+        true
+      else
+        false
+      end
+    end
+  end
+end

data/lib/longleaf/preservation_services/rsync_replication_service.rb

@@ -0,0 +1,198 @@
+require 'longleaf/events/event_names'
+require 'longleaf/logging'
+require 'longleaf/errors'
+require 'longleaf/models/file_record'
+require 'longleaf/models/service_fields'
+require 'longleaf/events/register_event'
+require 'longleaf/candidates/single_digest_provider'
+require 'open3'
+
+module Longleaf
+  # Preservation service which performs replication of a file to one or more destinations using rsync.
+  #
+  # The service definition must contain one or more destinations, specified with the "to" property.
+  # These destinations must be either a known storage location name, a remote path, or absolute path.
+  #
+  # Optional service configuration properties:
+  # * replica_collision_policy = specifies the desired outcome if the service attempts to replicate
+  #     a file which already exists at a destination. Default: "replace".
+  # * rsync_command = the command to invoke in order to execute rsync. Default: "rsync"
+  # * rsync_options = additional parameters that will be passed along to rsync. Cannot include options
+  #     which change the target of the command or prevent its execution, such as "files-from", "dry-run",
+  #     "help", etc. Command will always include "-R". Default "-a".
+  class RsyncReplicationService
+    include Longleaf::Logging
+    SF ||= Longleaf::ServiceFields
+
+    RSYNC_COMMAND_PROPERTY = "rsync_command"
+    DEFAULT_COMMAND = "rsync"
+
+    RSYNC_OPTIONS_PROPERTY = "rsync_options"
+    DEFAULT_OPTIONS = "-a"
+    DISALLOWED_OPTIONS = ["files-from", "n", "dry-run", "exclude", "exclude-from", "cvs-exclude",
+       "h", "help", "f", "F", "filter"]
+
+    attr_reader :command, :options, :collision_policy
+
+    # Initialize a RsyncReplicationService from the given service definition
+    #
+    # @param service_def [ServiceDefinition] the configuration for this service
+    # @param app_manager [ApplicationConfigManager] the application configuration
+    def initialize(service_def, app_manager)
+      @service_def = service_def
+      @app_manager = app_manager
+
+      @command = @service_def.properties[RSYNC_COMMAND_PROPERTY] || DEFAULT_COMMAND
+
+      # Validate rsync parameters
+      @options = @service_def.properties[RSYNC_OPTIONS_PROPERTY] || DEFAULT_OPTIONS
+      if contains_disallowed_option?(@options)
+        raise ArgumentError.new("Service #{service_def.name} specifies a disallowed rsync paramter," \
+            + " rsync_options may not include the following: #{DISALLOWED_OPTIONS.join(' ')}")
+      end
+
+      # Set and validate the replica collision policy
+      @collision_policy = @service_def.properties[SF::COLLISION_PROPERTY] || SF::DEFAULT_COLLISION_POLICY
+      if !SF::VALID_COLLISION_POLICIES.include?(@collision_policy)
+        raise ArgumentError.new("Service #{service_def.name} received invalid #{SF::COLLISION_PROPERTY}" \
+            + " value #{@collision_policy}")
+      end
+
+      # Store and validate destinations
+      replicate_to = @service_def.properties[SF::REPLICATE_TO]
+      if replicate_to.nil? || replicate_to.empty?
+        raise ArgumentError.new("Service #{service_def.name} must provide one or more replication destinations.")
+      end
+      replicate_to = [replicate_to] if replicate_to.is_a?(String)
+
+      loc_manager = app_manager.location_manager
+      # Build list of destinations, translating to storage locations when relevant
+      @destinations = Array.new
+      replicate_to.each do |dest|
+        # Assume that if destination contains a : or / it is a path rather than storage location
+        if dest =~ /[:\/]/
+          @destinations << dest
+        else
+          if loc_manager.locations.key?(dest)
+            @destinations << loc_manager.locations[dest]
+          else
+            raise ArgumentError.new("Service #{service_def.name} specifies unknown storage location '#{dest}'" \
+                + " as a replication destination")
+          end
+        end
+      end
+    end
+
+    # During a replication event, perform replication of the specified file to all configured destinations
+    # as necessary.
+    #
+    # @param file_rec [FileRecord] record representing the file to perform the service on.
+    # @param event [String] name of the event this service is being invoked by.
+    # @raise [PreservationServiceError] if the rsync replication fails
+    def perform(file_rec, event)
+      @destinations.each do |destination|
+        dest_is_storage_loc = destination.is_a?(Longleaf::StorageLocation)
+
+        if dest_is_storage_loc
+          dest_path = destination.path
+        else
+          dest_path = destination
+        end
+        
+        logical_physical_same = file_rec.path == file_rec.physical_path
+        # Determine the path to the file being replicated relative to its storage location
+        rel_path = file_rec.storage_location.relativize(file_rec.path)
+        
+        options = @options
+        if logical_physical_same
+          options = options + " -R"
+          # source path with . so that rsync will only create destination directories starting from that point
+          source_path = File.join(file_rec.storage_location.path, "./#{rel_path}")
+        else
+          options = options + " --no-relative"
+          source_path = file_rec.physical_path
+          dest_path = File.join(dest_path, rel_path)
+          if (dest_is_storage_loc && destination.is_a?(Longleaf::FilesystemStorageLocation)) || !dest_is_storage_loc
+            # Fill in missing parent directories, as rsync cannot do so when specifying a different source and dest filename
+            dirname = File.dirname(dest_path)
+            logger.debug("Creating parent dirs #{dirname} for #{file_rec.path}")
+            FileUtils.mkdir_p(dirname)
+          else
+            raise PreservationServiceError.new(
+                "Destination #{destination.name} does not currently support separate physical and logical paths")
+          end
+        end
+
+        # Check that the destination is available because attempting to write
+        verify_destination_available(destination, file_rec)
+
+        logger.debug("Invoking rsync with command: #{@command} \"#{source_path}\" \"#{dest_path}\" #{options}")
+        stdout, stderr, status = Open3.capture3("#{@command} \"#{source_path}\" \"#{dest_path}\" #{options}")
+        raise PreservationServiceError.new("Failed to replicate #{file_rec.path} to #{dest_path}: #{stderr}") \
+            unless status.success?
+
+        logger.info("Replicated #{file_rec.path} to destination #{dest_path}")
+
+        # For destinations which are storage locations, register the replica with longleaf
+        if dest_is_storage_loc
+          register_replica(destination, rel_path, file_rec)
+        end
+      end
+    end
+
+    # Determine if this service is applicable for the provided event, given the configured service definition
+    #
+    # @param event [String] name of the event
+    # @return [Boolean] returns true if this service is applicable for the provided event
+    def is_applicable?(event)
+      case event
+      when EventNames::PRESERVE
+        true
+      else
+        false
+      end
+    end
+
+    private
+    def contains_disallowed_option?(options)
+      DISALLOWED_OPTIONS.each do |disallowed|
+        if disallowed.length == 1
+          if options =~ /(\A| )-[a-zA-Z0-9]*#{disallowed}[a-zA-Z0-9]*( |=|\z)/
+            return true
+          end
+        else
+          if options =~ /(\A| )--#{disallowed}( |=|\z)/
+            return true
+          end
+        end
+      end
+
+      false
+    end
+
+    def verify_destination_available(destination, file_rec)
+      if destination.is_a?(Longleaf::StorageLocation)
+        begin
+          destination.available?
+        rescue StorageLocationUnavailableError => e
+          raise StorageLocationUnavailableError.new("Cannot replicate #{file_rec.path} to destination #{destination.name}: " \
+              + e.message)
+        end
+      elsif destination.start_with?("/")
+        raise StorageLocationUnavailableError.new("Cannot replicate #{file_rec.path} to destination" \
+            + " #{destination}, path does not exist.") unless Dir.exist?(destination)
+      end
+    end
+
+    def register_replica(destination, rel_path, file_rec)
+      dest_file_path = File.join(destination.path, rel_path)
+      dest_file_rec = FileRecord.new(dest_file_path, destination)
+
+      register_event = RegisterEvent.new(file_rec: dest_file_rec,
+          app_manager: @app_manager,
+          force: true,
+          digest_provider: SingleDigestProvider.new(file_rec.metadata_record.checksums))
+      register_event.perform
+    end
+  end
+end