longleaf 0.1.0 → 0.2.0.pre.1

data/lib/longleaf/helpers/digest_helper.rb

@@ -0,0 +1,50 @@
+require 'longleaf/errors'
+require 'digest'
+
+module Longleaf
+  # Helper methods for generating digests
+  class DigestHelper
+    KNOWN_DIGESTS ||= ['md5', 'sha1', 'sha2', 'sha256', 'sha384', 'sha512', 'rmd160']
+    
+    # @param algs Either a string containing one or an array containing zero or more digest
+    #    algorithm names.
+    # @raise [InvalidDigestAlgorithmError] thrown if any of the digest algorithms listed are not
+    #    known to the system.
+    def self.validate_algorithms(algs)
+      return if algs.nil?
+      if algs.is_a?(String)
+        unless KNOWN_DIGESTS.include?(algs)
+          raise InvalidDigestAlgorithmError.new("Unknown digest algorithm #{algs}")
+        end
+      else
+        unknown = algs.select { |alg| !KNOWN_DIGESTS.include?(alg) }
+        unless unknown.empty?
+          raise InvalidDigestAlgorithmError.new("Unknown digest algorithm(s): #{unknown.to_s}")
+        end
+      end
+    end
+    
+    # Get a Digest class for the specified algorithm
+    # @param alg [String] name of the digest algorithm
+    # @return [Digest] A digest class for the requested algorithm
+    # @raise [InvalidDigestAlgorithmError] if an unknown digest algorithm is requested
+    def self.start_digest(alg)
+      case alg
+      when 'md5'
+        return Digest::MD5.new
+      when 'sha1'
+        return Digest::SHA1.new
+      when 'sha2', 'sha256'
+        return Digest::SHA2.new
+      when 'sha384'
+        return Digest::SHA2.new(384)
+      when 'sha512'
+        return Digest::SHA2.new(512)
+      when 'rmd160'
+        return Digest::RMD160.new
+      else
+        raise InvalidDigestAlgorithmError.new("Cannot produce digest for unknown algorithm '#{alg}'.")
+      end
+    end
+  end
+end

data/lib/longleaf/helpers/service_date_helper.rb

@@ -0,0 +1,51 @@
+require 'time'
+
+module Longleaf
+  # Helper methods for interacting with dates/timestamps on services
+  class ServiceDateHelper
+    
+    # Adds the amount of time from modifier to the provided timestamp
+    # @param timestamp [String] ISO-8601 timestamp string
+    # @param modifier [String] amount of time to add to the timestamp. It must follow the syntax
+    # "<quantity> <time unit>", where quantity must be a positive whole number and time unit
+    # must be second, minute, hour, day, week, month or year (unit may be plural).
+    # Any info after a comma will be ignored.
+    # @return [String] the original timestamp in ISO-8601 format with the provided amount of time added.
+    def self.add_to_timestamp(timestamp, modifier)
+      if modifier =~ /^(\d+) *(second|minute|hour|day|week|month|year)s?(,.*)?/
+        value = $1.to_i
+        unit = $2
+      else
+        raise ArgumentError.new("Cannot parse time modifier #{modifier}")
+      end
+      
+      datetime = Time.iso8601(timestamp)
+      case unit
+      when 'second'
+        unit_modifier = 1
+      when 'minute'
+        unit_modifier = 60
+      when 'hour'
+        unit_modifier = 3600
+      when 'day'
+        unit_modifier = 24 * 3600
+      when 'week'
+        unit_modifier = 7 * 24 * 3600
+      when 'month'
+        unit_modifier = 30 * 24 * 3600
+      when 'year'
+        unit_modifier = 365 * 24 * 3600
+      end
+      
+      modified_time = datetime + (value * unit_modifier)
+      modified_time.iso8601
+    end
+    
+    # Get a timestamp in the format expected for service timestamps.
+    # @param timestamp [Time] the time to format. Defaults to now.
+    # @return [String] the time formatted as iso8601
+    def self.formatted_timestamp(timestamp = Time.now)
+      timestamp.iso8601.to_s
+    end
+  end
+end

data/lib/longleaf/logging.rb

@@ -1,6 +1,7 @@
 require 'longleaf/logging/redirecting_logger'
 
 module Longleaf
+  # Module for access logging within longleaf
   module Logging
     # Get the main logger for longleaf
     def logger

data/lib/longleaf/logging/redirecting_logger.rb

@@ -1,14 +1,14 @@
 require 'logger'
 
-# Logger which directs messages to stdout and/or stderr, depending on the nature of the message.
-# Status logging, which includes standard logger methods, goes to STDERR.
-# Operation success and failure messages go to STDOUT, and to STDERR at info level.
 module Longleaf
   module Logging
+    # Logger which directs messages to stdout and/or stderr, depending on the nature of the message.
+    # Status logging, which includes standard logger methods, goes to STDERR.
+    # Operation success and failure messages go to STDOUT, and to STDERR at info level.
     class RedirectingLogger
-      # @param failure_only [Boolean] If set to true, only failure messages will be output to STDOUT
+      # @param [Boolean] failure_only If set to true, only failure messages will be output to STDOUT
       # @param log_level [String] logger level used for output to STDERR
-      # @param log_format [Strfailure_onlying] format string for log entries to STDERR. There are 4 variables available
+      # @param log_format [String] format string for log entries to STDERR. There are 4 variables available
       #    for inclusion in the output: severity, datetime, progname, msg. Variables must be wrapped in %{}.
       # @param datetime_format [String] datetime formatting string used for logger dates appearing in STDERR.
       def initialize(failure_only: false, log_level: 'WARN', log_format: nil, datetime_format: nil)
@@ -67,7 +67,8 @@ module Longleaf
       end
     
       # Logs a success message to STDOUT, as well as STDERR at info level.
-      # @param eventOrMessage [String] name of the preservation event which succeeded, 
+      # 
+      # @param [String] eventOrMessage name of the preservation event which succeeded, 
       #    or the message to output if it is the only parameter. Required.
       # @param file_name [String] file name which is the subject of this message.
       # @param message [String] descriptive message to accompany this output
@@ -90,6 +91,7 @@ module Longleaf
       
         @stderr_log.info(text)
         @stderr_log.error("#{error.message}") unless error.nil?
+        @stderr_log.error("#{error.backtrace.to_s}") unless error.nil? || error.backtrace.nil?
       end
     
       # Logs an outcome message to STDOUT, as well as STDERR at info level.
@@ -97,7 +99,7 @@ module Longleaf
       #
       # @param outcome [String] The status of the outcome. Required.
       # @param eventOrMessage [String] name of the preservation event which was successful, 
-      #    or the message to output if it is the only parameter. Required.
+      #     or the message to output if it is the only parameter. Required.
       # @param file_name [String] file name which is the subject of this message.
       # @param message [String] descriptive message to accompany this output
       # @param service [String] name of the service which executed.
@@ -108,7 +110,6 @@ module Longleaf
         @stderr_log.info(text)
       end
     
-      # FAILURE verify[cdr_fixity_check] /path/to/file: Something terrible
       private
       def outcome_text(outcome, eventOrMessage, file_name = nil, message = nil, service = nil, error = nil)
         message_only = file_name.nil? && message.nil? && error.nil?

data/lib/longleaf/models/app_fields.rb

@@ -1,4 +1,5 @@
 module Longleaf
+  # Application configuration field names
   class AppFields
     LOCATIONS = 'locations'
     SERVICES = 'services'
@@ -6,5 +7,6 @@ module Longleaf
     
     LOCATION_PATH = 'path'
     METADATA_PATH = 'metadata_path'
+    METADATA_DIGESTS = 'metadata_digests'
   end
 end

data/lib/longleaf/models/file_record.rb

@@ -1,5 +1,5 @@
-# Record for an individual file and its associated information 
 module Longleaf
+  # Record for an individual file and its associated information
   class FileRecord
     
     attr_accessor :metadata_record
@@ -7,13 +7,14 @@ module Longleaf
     attr_reader :path
     
     # @param file_path [String] path to the file
-    # @param storage_location [Longleaf::StorageLocation] storage location containing the file
-    def initialize(file_path, storage_location)
+    # @param storage_location [StorageLocation] storage location containing the file
+    def initialize(file_path, storage_location, metadata_record = nil)
       raise ArgumentError.new("FileRecord requires a path") if file_path.nil?
       raise ArgumentError.new("FileRecord requires a storage_location") if storage_location.nil?
       
       @path = file_path
       @storage_location = storage_location
+      @metadata_record = metadata_record
     end
     
     # @return [String] path for the metadata file for this file
@@ -21,5 +22,9 @@ module Longleaf
       @metadata_path = @storage_location.get_metadata_path_for(path) if @metadata_path.nil?
       @metadata_path
     end
+    
+    def metadata_present?
+      File.exist?(metadata_path)
+    end
   end
 end

data/lib/longleaf/models/md_fields.rb

@@ -1,4 +1,5 @@
 module Longleaf
+  # File metadata fields
   class MDFields
     DATA = 'data'
     SERVICES = 'services'

data/lib/longleaf/models/metadata_record.rb

@@ -1,10 +1,11 @@
 require_relative 'md_fields'
 require_relative 'service_record'
 
-# Metadata record for a single file
 module Longleaf
+  # Metadata record for a single file
   class MetadataRecord
-    attr_reader :deregistered, :registered
+    attr_reader :registered
+    attr_accessor :deregistered
     attr_reader :checksums
     attr_reader :properties
     attr_accessor :file_size, :last_modified
@@ -35,14 +36,25 @@ module Longleaf
     # Adds a service to this record
     #
     # @param name [String] identifier for the service being added
-    # @param service_properties [ServiceRecord] properties for populating the new service
-    def add_service(name, service = Longleaf::ServiceRecord.new)
+    # @param service [ServiceRecord] properties for populating the new service
+    # @return [ServiceRecord] the service added
+    def add_service(name, service = ServiceRecord.new)
       raise ArgumentError.new("Value must be a ServiceRecord object when adding a service") unless service.class == Longleaf::ServiceRecord
       raise IndexError.new("Service with name '#{name}' already exists") if @services.key?(name)
       
       @services[name] = service
     end
     
+    # Updates details of service record as if the service had been executed.
+    # @param service_name [String] name of the service run
+    # @return [ServiceRecord] the service record updated
+    def update_service_as_performed(service_name)
+      service_rec = service(service_name) || add_service(service_name)
+      service_rec.run_needed = false
+      service_rec.timestamp = ServiceDateHelper.formatted_timestamp
+      service_rec
+    end
+    
     # @param name [String] name identifier of the service to retrieve
     # @return [ServiceRecord] the ServiceRecord for the service identified by name, or nil
     def service(name)

data/lib/longleaf/models/service_definition.rb

@@ -1,19 +1,20 @@
 require_relative 'service_fields'
 
-# Definition of a preservation service
 module Longleaf
+  # Definition of a configured preservation service
   class ServiceDefinition
     attr_reader :name
-    attr_reader :work_script
+    attr_reader :work_script, :work_class
     attr_reader :frequency, :delay
     attr_reader :properties
     
-    def initialize(name:, work_script:, frequency: nil, delay: nil, properties: Hash.new)
+    def initialize(name:, work_script:, work_class: nil, frequency: nil, delay: nil, properties: Hash.new)
       raise ArgumentError.new("Parameters name and work_script are required") unless name && work_script
       
       @properties = properties
       @name = name
       @work_script = work_script
+      @work_class = work_class
       @frequency = frequency
       @delay = delay
     end

data/lib/longleaf/models/service_fields.rb

@@ -1,6 +1,8 @@
 module Longleaf
+  # Constants for common configuration fields for preservation service definitions
   class ServiceFields
     WORK_SCRIPT = 'work_script'
+    WORK_CLASS = 'work_class'
     FREQUENCY = 'frequency'
     DELAY = 'delay'
     

data/lib/longleaf/models/service_record.rb

@@ -1,10 +1,13 @@
-# Record for an individual service in a file's metadata record.
 module Longleaf
+  # Record for an individual service in a file's metadata record.
   class ServiceRecord
     attr_reader :properties
     attr_accessor :stale_replicas, :timestamp, :run_needed
     
     # @param properties [Hash] initial properties for this service record
+    # @param stale_replicas [Boolean] whether there are any stale replicas from this service
+    # @param timestamp [String] timestamp when this service last ran or was initialized
+    # @param run_needed [Boolean] flag indicating that this service should be run at the next available opportunity
     def initialize(properties: Hash.new, stale_replicas: false, timestamp: nil, run_needed: false)
       raise ArgumentError.new("Service properties must be a hash") if properties.class != Hash
       

data/lib/longleaf/models/storage_location.rb

@@ -1,17 +1,34 @@
 require 'longleaf/services/metadata_serializer'
 
 module Longleaf
+  # Representation of a configured storage location
   class StorageLocation
     attr_reader :name
     attr_reader :path
     attr_reader :metadata_path
+    attr_reader :metadata_digests
     
-    def initialize(name:, path:, metadata_path:)
+    # @param name [String] the name of this storage location
+    # @param path [String] absolute path where the storage location is located
+    # @param metadata_path [String] absolute path where the metadata for files in this location will be stored.
+    # @param metadata_digests list of digest algorithms to use for metadata file digests in this location.
+    def initialize(name:, path:, metadata_path:, metadata_digests: [])
       raise ArgumentError.new("Parameters name, path and metadata_path are required") unless name && path && metadata_path
       
       @path = path
+      @path += '/' unless @path.end_with?('/')
       @name = name
       @metadata_path = metadata_path
+      @metadata_path += '/' unless @metadata_path.end_with?('/')
+      
+      if metadata_digests.nil?
+        @metadata_digests = []
+      elsif metadata_digests.is_a?(String)
+        @metadata_digests = [metadata_digests.downcase]
+      else
+        @metadata_digests = metadata_digests.map(&:downcase)
+      end
+      DigestHelper::validate_algorithms(@metadata_digests)
     end
     
     # Get the path for the metadata file for the given file path located in this storage location.

data/lib/longleaf/preservation_services/fixity_check_service.rb

@@ -0,0 +1,121 @@
+require 'longleaf/events/event_names'
+require 'longleaf/models/service_fields'
+require 'longleaf/logging'
+require 'longleaf/helpers/digest_helper'
+require 'set'
+
+module Longleaf
+  # Preservation service which performs one or more fixity checks on a file based on the configured list
+  # of digest algorithms. It currently supports 'md5', 'sha1', 'sha2', 'sha256', 'sha384', 'sha512' and 'rmd160'.
+  #
+  # If the service encounters a file which is missing any of the digest algorithms the service is configured
+  # to check, the outcome may be controlled with the 'absent_digest' property via the following values:
+  #   * 'fail' - the service will raise a ChecksumMismatchError for the missing algorithm. This is the default.
+  #   * 'ignore' - the service will skip calculating any algorithms not already present for the file.
+  #   * 'generate' - the service will generate and store any missing digests from the set of configured algorithms.
+  class FixityCheckService
+    include Longleaf::Logging
+    
+    SUPPORTED_ALGORITHMS = ['md5', 'sha1', 'sha2', 'sha256', 'sha384', 'sha512', 'rmd160']
+    
+    # service configuration property indicating how to handle situations where a file does not
+    # have a digest for one of the expected algorithms on record.
+    ABSENT_DIGEST_PROPERTY = 'absent_digest'
+    FAIL_IF_ABSENT = 'fail'
+    GENERATE_IF_ABSENT = 'generate'
+    IGNORE_IF_ABSENT = 'ignore'
+    ABSENT_DIGEST_OPTIONS = [FAIL_IF_ABSENT, GENERATE_IF_ABSENT, IGNORE_IF_ABSENT]
+    
+    # Initialize a FixityCheckService from the given service definition
+    #
+    # @param service_def [ServiceDefinition] the configuration for this service
+    # @param app_manager [ApplicationConfigManager] manager for configured storage locations
+    def initialize(service_def, app_manager)
+      @service_def = service_def
+      @absent_digest_behavior = @service_def.properties[ABSENT_DIGEST_PROPERTY] || FAIL_IF_ABSENT
+      unless ABSENT_DIGEST_OPTIONS.include?(@absent_digest_behavior)
+        raise ArgumentError.new("Invalid option '#{@absent_digest_behavior}' for property #{ABSENT_DIGEST_PROPERTY} in service #{service_def.name}")
+      end
+      
+      service_algs = service_def.properties[ServiceFields::DIGEST_ALGORITHMS]
+      if service_algs.nil? || service_algs.empty?
+        raise ArgumentError.new("FixityCheckService from definition #{service_def.name} requires a list of one or more digest algorithms")
+      end
+      
+      # Store the list of digest algorithms to verify, using normalized algorithm names.
+      @digest_algs = Set.new
+      service_algs.each do |alg|
+        normalized_alg = alg.downcase.delete('-')
+        if SUPPORTED_ALGORITHMS.include?(normalized_alg)
+          @digest_algs << normalized_alg
+        else
+          raise ArgumentError.new("Unsupported checksum algorithm '#{alg}' in definition #{service_def.name}. Supported algorithms are: #{SUPPORTED_ALGORITHMS.to_s}")
+        end
+      end
+    end
+    
+    # Perform all configured fixity checks on the provided file
+    #
+    # @param file_rec [FileRecord] record representing the file to perform the service on.
+    # @param event [String] name of the event this service is being invoked by.
+    # @raise [ChecksumMismatchError] if the checksum on record does not match the generated checksum
+    def perform(file_rec, event)
+      path = file_rec.path
+      md_rec = file_rec.metadata_record
+      
+      # Get the list of existing checksums for the file and normalize algorithm names
+      file_digests = Hash.new
+      md_rec.checksums&.each do |alg, digest|
+        normalized_alg = alg.downcase.delete('-')
+        if @digest_algs.include?(normalized_alg)
+          file_digests[normalized_alg] = digest
+        else
+          logger.debug("Metadata for file #{path} contains unexpected '#{alg}' digest, it will be ignored.")
+        end
+      end
+      
+      @digest_algs.each do |alg|
+        existing_digest = file_digests[alg]
+        
+        if existing_digest.nil?
+          if @absent_digest_behavior == FAIL_IF_ABSENT
+            raise ChecksumMismatchError.new("Fixity check using algorithm '#{alg}' failed for file #{path}: no existing digest of type '#{alg}' on record.")
+          elsif @absent_digest_behavior == IGNORE_IF_ABSENT
+            logger.debug("Skipping check of algorithm '#{alg}' for file #{path}: no digest on record.")
+            next
+          end
+        end
+        
+        digest = DigestHelper::start_digest(alg)
+        digest.file(path)
+        generated_digest = digest.hexdigest
+        
+        # Store the missing checksum if using the 'generate' behavior
+        if existing_digest.nil? && @absent_digest_behavior == GENERATE_IF_ABSENT
+          md_rec.checksums[alg] = generated_digest
+          logger.info("Generated and stored digest using algorithm '#{alg}' for file #{path}")
+        else
+          # Compare the new digest to the one on record
+          if existing_digest == generated_digest
+            logger.info("Fixity check using algorithm '#{alg}' succeeded for file #{path}")
+          else
+            raise ChecksumMismatchError.new("Fixity check using algorithm '#{alg}' failed for file #{path}: expected '#{existing_digest}', calculated '#{generated_digest}.'")
+          end
+        end
+      end
+    end
+    
+    # Determine if this service is applicable for the provided event, given the configured service definition
+    #
+    # @param event [String] name of the event
+    # @return [Boolean] returns true if this service is applicable for the provided event
+    def is_applicable?(event)
+      case event
+      when EventNames::PRESERVE
+        true
+      else
+        false
+      end
+    end
+  end
+end