longleaf 0.2.0.pre.1 → 0.3.0

data/lib/longleaf/logging.rb

@@ -7,16 +7,16 @@ module Longleaf
     def logger
       Logging.logger
     end
-    
+
     # Get the main logger for longleaf
     def self.logger
       @logger ||= RedirectingLogger.new
     end
-  
+
     def initialize_logger(failure_only, log_level, log_format, datetime_format)
       Logging.initialize_logger(failure_only, log_level, log_format, datetime_format)
     end
-  
+
     def self.initialize_logger(failure_only, log_level, log_format, datetime_format)
       @logger = RedirectingLogger.new(failure_only: failure_only,
           log_level: log_level,
@@ -24,4 +24,4 @@ module Longleaf
           datetime_format: datetime_format)
     end
   end
-end
+end

data/lib/longleaf/logging/redirecting_logger.rb

@@ -25,12 +25,12 @@ module Longleaf
           @stderr_log.formatter = proc do |severity, datetime, progname, msg|
             # Make sure the format ends with a newline
             @log_format = @log_format + "\n" unless @log_format.end_with?("\n")
-        
+
             formatted_date = @stderr_log.datetime_format.nil? ? datetime : datetime.strftime(datetime_format)
             @log_format % { :severity => severity, :datetime => formatted_date, :progname => progname, :msg => msg }
           end
         end
-      
+
         @stdout_log = Logger.new($stdout)
         @stdout_log.formatter = proc do |severity, datetime, progname, msg|
           "#{msg}\n"
@@ -41,34 +41,34 @@ module Longleaf
           @stdout_log.level = 'info'
         end
       end
-    
+
       def debug(progname = nil, &block)
         @stderr_log.debug(progname, &block)
       end
-    
+
       def info(progname = nil, &block)
         @stderr_log.info(progname, &block)
       end
-    
+
       def warn(progname = nil, &block)
         @stderr_log.warn(progname, &block)
       end
-    
+
       def error(progname = nil, &block)
         @stderr_log.error(progname, &block)
       end
-    
+
       def fatal(progname = nil, &block)
         @stderr_log.fatal(progname, &block)
       end
-    
+
       def unknown(progname = nil, &block)
         @stderr_log.unknown(progname, &block)
       end
-    
+
       # Logs a success message to STDOUT, as well as STDERR at info level.
-      # 
-      # @param [String] eventOrMessage name of the preservation event which succeeded, 
+      #
+      # @param [String] eventOrMessage name of the preservation event which succeeded,
       #    or the message to output if it is the only parameter. Required.
       # @param file_name [String] file name which is the subject of this message.
       # @param message [String] descriptive message to accompany this output
@@ -76,10 +76,10 @@ module Longleaf
       def success(eventOrMessage, file_name = nil, message = nil, service = nil)
         outcome('SUCCESS', eventOrMessage, file_name, message, service)
       end
-    
+
       # Logs a failure message to STDOUT, as well as STDERR at info level.
       # If an error was provided, it is logged to STDERR at error level.
-      # @param eventOrMessage [String] name of the preservation event which failed, 
+      # @param eventOrMessage [String] name of the preservation event which failed,
       #    or the message to output if it is the only parameter.
       # @param file_name [String] file name which is the subject of this message.
       # @param message [String] descriptive message to accompany this output
@@ -88,17 +88,17 @@ module Longleaf
       def failure(eventOrMessage, file_name = nil, message = nil, service = nil, error: nil)
         text = outcome_text('FAILURE', eventOrMessage, file_name, message, service, error)
         @stdout_log.warn(text)
-      
+
         @stderr_log.info(text)
         @stderr_log.error("#{error.message}") unless error.nil?
-        @stderr_log.error("#{error.backtrace.to_s}") unless error.nil? || error.backtrace.nil?
+        @stderr_log.error("#{error.backtrace}") unless error.nil? || error.backtrace.nil?
       end
-    
+
       # Logs an outcome message to STDOUT, as well as STDERR at info level.
       # If file_name and message are nil, eventOrMessage will be used as the message.
       #
       # @param outcome [String] The status of the outcome. Required.
-      # @param eventOrMessage [String] name of the preservation event which was successful, 
+      # @param eventOrMessage [String] name of the preservation event which was successful,
       #     or the message to output if it is the only parameter. Required.
       # @param file_name [String] file name which is the subject of this message.
       # @param message [String] descriptive message to accompany this output
@@ -109,13 +109,13 @@ module Longleaf
         @stdout_log.info(text)
         @stderr_log.info(text)
       end
-    
+
       private
       def outcome_text(outcome, eventOrMessage, file_name = nil, message = nil, service = nil, error = nil)
         message_only = file_name.nil? && message.nil? && error.nil?
-        
+
         text = "#{outcome}"
-      
+
         if message_only
           text << ": #{eventOrMessage}"
         else

data/lib/longleaf/models/app_fields.rb

@@ -4,7 +4,8 @@ module Longleaf
     LOCATIONS = 'locations'
     SERVICES = 'services'
     SERVICE_MAPPINGS = 'service_mappings'
-    
+    SYSTEM = 'system'
+
     LOCATION_PATH = 'path'
     METADATA_PATH = 'metadata_path'
     METADATA_DIGESTS = 'metadata_digests'

data/lib/longleaf/models/file_record.rb

@@ -1,30 +1,34 @@
 module Longleaf
   # Record for an individual file and its associated information
   class FileRecord
-    
     attr_accessor :metadata_record
     attr_reader :storage_location
     attr_reader :path
-    
+
     # @param file_path [String] path to the file
     # @param storage_location [StorageLocation] storage location containing the file
     def initialize(file_path, storage_location, metadata_record = nil)
       raise ArgumentError.new("FileRecord requires a path") if file_path.nil?
       raise ArgumentError.new("FileRecord requires a storage_location") if storage_location.nil?
-      
+
       @path = file_path
       @storage_location = storage_location
       @metadata_record = metadata_record
     end
-    
+
     # @return [String] path for the metadata file for this file
     def metadata_path
       @metadata_path = @storage_location.get_metadata_path_for(path) if @metadata_path.nil?
       @metadata_path
     end
-    
+
     def metadata_present?
       File.exist?(metadata_path)
     end
+
+    def ==(other_obj)
+      return false unless other_obj.is_a?(FileRecord)
+      path == other_obj.path
+    end
   end
-end
+end

data/lib/longleaf/models/md_fields.rb

@@ -3,7 +3,7 @@ module Longleaf
   class MDFields
     DATA = 'data'
     SERVICES = 'services'
-    
+
     REGISTERED_TIMESTAMP = 'registered'
     DEREGISTERED_TIMESTAMP = 'deregistered'
 

data/lib/longleaf/models/metadata_record.rb

@@ -9,7 +9,7 @@ module Longleaf
     attr_reader :checksums
     attr_reader :properties
     attr_accessor :file_size, :last_modified
-    
+
     # @param properties [Hash] initial data properties for this record
     # @param services [Hash] initial service property tree
     # @param deregistered [String] deregistered timestamp
@@ -17,22 +17,22 @@ module Longleaf
     # @param checksums [Hash] hash of checksum values
     # @param file_size [Integer] size of file in bytes
     # @param last_modified [String] iso8601 representation of the last modified date of file
-    def initialize(properties: Hash.new, services: Hash.new, deregistered: nil, registered: nil, checksums: Hash.new,
+    def initialize(properties: nil, services: nil, deregistered: nil, registered: nil, checksums: nil,
           file_size: nil, last_modified: nil)
-      @properties = properties
+      @properties = properties || Hash.new
       @registered = registered
       @deregistered = deregistered
-      @checksums = checksums
-      @services = services
+      @checksums = checksums || Hash.new
+      @services = services || Hash.new
       @file_size = file_size
       @last_modified = last_modified
     end
-    
+
     # @return [Boolean] true if the record is deregistered
     def deregistered?
       !@deregistered.nil?
     end
-    
+
     # Adds a service to this record
     #
     # @param name [String] identifier for the service being added
@@ -41,10 +41,10 @@ module Longleaf
     def add_service(name, service = ServiceRecord.new)
       raise ArgumentError.new("Value must be a ServiceRecord object when adding a service") unless service.class == Longleaf::ServiceRecord
       raise IndexError.new("Service with name '#{name}' already exists") if @services.key?(name)
-      
+
       @services[name] = service
     end
-    
+
     # Updates details of service record as if the service had been executed.
     # @param service_name [String] name of the service run
     # @return [ServiceRecord] the service record updated
@@ -54,16 +54,26 @@ module Longleaf
       service_rec.timestamp = ServiceDateHelper.formatted_timestamp
       service_rec
     end
-    
+
+    # Updates details of service record as if the service had encountered a
+    # failure during execution.
+    # @param service_name [String] name of the service run
+    # @return [ServiceRecord] the service record updated
+    def update_service_as_failed(service_name)
+      service_rec = service(service_name) || add_service(service_name)
+      service_rec.failure_timestamp = ServiceDateHelper.formatted_timestamp
+      service_rec
+    end
+
     # @param name [String] name identifier of the service to retrieve
     # @return [ServiceRecord] the ServiceRecord for the service identified by name, or nil
     def service(name)
       @services[name]
     end
-    
+
     # @return [Array<String>] a list of name identifiers for services registered to this record
     def list_services
       @services.keys
     end
   end
-end
+end

data/lib/longleaf/models/service_definition.rb

@@ -7,10 +7,10 @@ module Longleaf
     attr_reader :work_script, :work_class
     attr_reader :frequency, :delay
     attr_reader :properties
-    
+
     def initialize(name:, work_script:, work_class: nil, frequency: nil, delay: nil, properties: Hash.new)
       raise ArgumentError.new("Parameters name and work_script are required") unless name && work_script
-      
+
       @properties = properties
       @name = name
       @work_script = work_script
@@ -19,4 +19,4 @@ module Longleaf
       @delay = delay
     end
   end
-end
+end

data/lib/longleaf/models/service_fields.rb

@@ -5,7 +5,7 @@ module Longleaf
     WORK_CLASS = 'work_class'
     FREQUENCY = 'frequency'
     DELAY = 'delay'
-    
+
     REPLICATE_TO = 'to'
     DIGEST_ALGORITHMS = 'algorithms'
   end

data/lib/longleaf/models/service_record.rb

@@ -3,28 +3,29 @@ module Longleaf
   class ServiceRecord
     attr_reader :properties
     attr_accessor :stale_replicas, :timestamp, :run_needed
-    
+    attr_accessor :failure_timestamp
+
     # @param properties [Hash] initial properties for this service record
     # @param stale_replicas [Boolean] whether there are any stale replicas from this service
     # @param timestamp [String] timestamp when this service last ran or was initialized
     # @param run_needed [Boolean] flag indicating that this service should be run at the next available opportunity
     def initialize(properties: Hash.new, stale_replicas: false, timestamp: nil, run_needed: false)
       raise ArgumentError.new("Service properties must be a hash") if properties.class != Hash
-      
+
       @properties = properties
       @timestamp = timestamp
       @stale_replicas = stale_replicas
       @run_needed = run_needed
     end
-    
+
     # @return the value of a service property identified by key
     def [](key)
       @properties[key]
     end
-    
+
     # set the value of a service property identified by key
     def []=(key, value)
       @properties[key] = value
     end
   end
-end
+end

data/lib/longleaf/models/storage_location.rb

@@ -7,20 +7,20 @@ module Longleaf
     attr_reader :path
     attr_reader :metadata_path
     attr_reader :metadata_digests
-    
+
     # @param name [String] the name of this storage location
     # @param path [String] absolute path where the storage location is located
     # @param metadata_path [String] absolute path where the metadata for files in this location will be stored.
     # @param metadata_digests list of digest algorithms to use for metadata file digests in this location.
     def initialize(name:, path:, metadata_path:, metadata_digests: [])
       raise ArgumentError.new("Parameters name, path and metadata_path are required") unless name && path && metadata_path
-      
+
       @path = path
       @path += '/' unless @path.end_with?('/')
       @name = name
       @metadata_path = metadata_path
       @metadata_path += '/' unless @metadata_path.end_with?('/')
-      
+
       if metadata_digests.nil?
         @metadata_digests = []
       elsif metadata_digests.is_a?(String)
@@ -30,7 +30,7 @@ module Longleaf
       end
       DigestHelper::validate_algorithms(@metadata_digests)
     end
-    
+
     # Get the path for the metadata file for the given file path located in this storage location.
     # @param file_path [String] path of the file
     # @raise [ArgumentError] if the file_path is not provided or is not in this storage location.
@@ -39,9 +39,28 @@ module Longleaf
       raise ArgumentError.new("Provided file path is not contained by storage location #{@name}: #{file_path}") \
           unless file_path.start_with?(@path)
 
-      file_path.sub(/^#{@path}/, metadata_path) + MetadataSerializer::metadata_suffix
+      md_path = file_path.sub(/^#{@path}/, @metadata_path)
+      # If the file_path is to a file, then add metadata suffix.
+      if md_path.end_with?('/')
+        md_path
+      else
+        md_path + MetadataSerializer::metadata_suffix
+      end
+    end
+
+    # Get the metadata path for the provided file path located in this storage location.
+    # @param md_path [String] metadata file path
+    # @raise [ArgumentError] if the md_path is not in this storage location
+    # @return [String] the path for the file associated with this metadata
+    def get_path_from_metadata_path(md_path)
+      raise ArgumentError.new("A file_path parameter is required") if md_path.nil? || md_path.empty?
+      raise ArgumentError.new("Provided metadata path is not contained by storage location #{@name}: #{md_path}") \
+          unless md_path&.start_with?(@metadata_path)
+
+      file_path = md_path.sub(/^#{@metadata_path}/, @path)
+      file_path.sub(/#{MetadataSerializer::metadata_suffix}$/, '')
     end
-    
+
     # Checks that the path and metadata path defined in this location are available
     # @raise [StorageLocationUnavailableError] if the storage location is not available
     def available?
@@ -51,4 +70,4 @@ module Longleaf
           unless Dir.exist?(@metadata_path)
     end
   end
-end
+end

data/lib/longleaf/models/system_config_fields.rb

@@ -0,0 +1,9 @@
+module Longleaf
+  # System configuration field names
+  class SystemConfigFields
+    MD_INDEX = 'index'
+    MD_INDEX_ADAPTER = 'adapter'
+    MD_INDEX_CONNECTION = 'connection'
+    MD_INDEX_PAGE_SIZE = 'page_size'
+  end
+end

data/lib/longleaf/preservation_services/file_check_service.rb

@@ -0,0 +1,58 @@
+require 'longleaf/events/event_names'
+require 'longleaf/logging'
+
+module Longleaf
+  # Preservation service which validates a file using current filesystem information compared against the
+  # last registered details for that file. Checks using file name, size and last modified timestamp.
+  class FileCheckService
+    include Longleaf::Logging
+
+    # Initialize a FileCheckService from the given service definition
+    #
+    # @param service_def [ServiceDefinition] the configuration for this service
+    # @param app_manager [ApplicationConfigManager] manager for configured storage locations
+    def initialize(service_def, app_manager)
+      @service_def = service_def
+      @app_manager = app_manager
+    end
+
+    # Perform file information check.
+    #
+    # @param file_rec [FileRecord] record representing the file to perform the service on.
+    # @param event [String] name of the event this service is being invoked by.
+    # @raise [PreservationServiceError] if the file system information does not match the stored details
+    def perform(file_rec, event)
+      file_path = file_rec.path
+      md_rec = file_rec.metadata_record
+
+      logger.debug("Performing file information check of #{file_path}")
+
+      if !File.exist?(file_path)
+        raise PreservationServiceError.new("File does not exist: #{file_path}")
+      end
+
+      file_size = File.size(file_rec.path)
+      if file_size != md_rec.file_size
+        raise PreservationServiceError.new("File size for #{file_path} does not match the expected value: registered = #{md_rec.file_size} bytes, actual = #{file_size} bytes")
+      end
+
+      last_modified = File.mtime(file_rec.path).utc.iso8601(3)
+      if last_modified != md_rec.last_modified
+        raise PreservationServiceError.new("Last modified timestamp for #{file_path} does not match the expected value: registered = #{md_rec.last_modified}, actual = #{last_modified}")
+      end
+    end
+
+    # Determine if this service is applicable for the provided event, given the configured service definition
+    #
+    # @param event [String] name of the event
+    # @return [Boolean] returns true if this service is applicable for the provided event
+    def is_applicable?(event)
+      case event
+      when EventNames::PRESERVE
+        true
+      else
+        false
+      end
+    end
+  end
+end