longleaf 0.1.0.pre.2 → 1.0.0

data/lib/longleaf/logging.rb

@@ -1,21 +1,22 @@
 require 'longleaf/logging/redirecting_logger'
 
 module Longleaf
+  # Module for access logging within longleaf
   module Logging
     # Get the main logger for longleaf
     def logger
       Logging.logger
     end
-    
+
     # Get the main logger for longleaf
     def self.logger
       @logger ||= RedirectingLogger.new
     end
-  
+
     def initialize_logger(failure_only, log_level, log_format, datetime_format)
       Logging.initialize_logger(failure_only, log_level, log_format, datetime_format)
     end
-  
+
     def self.initialize_logger(failure_only, log_level, log_format, datetime_format)
       @logger = RedirectingLogger.new(failure_only: failure_only,
           log_level: log_level,
@@ -23,4 +24,4 @@ module Longleaf
           datetime_format: datetime_format)
     end
   end
-end
+end

data/lib/longleaf/logging/redirecting_logger.rb

@@ -1,14 +1,14 @@
 require 'logger'
 
-# Logger which directs messages to stdout and/or stderr, depending on the nature of the message.
-# Status logging, which includes standard logger methods, goes to STDERR.
-# Operation success and failure messages go to STDOUT, and to STDERR at info level.
 module Longleaf
   module Logging
+    # Logger which directs messages to stdout and/or stderr, depending on the nature of the message.
+    # Status logging, which includes standard logger methods, goes to STDERR.
+    # Operation success and failure messages go to STDOUT, and to STDERR at info level.
     class RedirectingLogger
-      # @param failure_only [Boolean] If set to true, only failure messages will be output to STDOUT
+      # @param [Boolean] failure_only If set to true, only failure messages will be output to STDOUT
       # @param log_level [String] logger level used for output to STDERR
-      # @param log_format [Strfailure_onlying] format string for log entries to STDERR. There are 4 variables available
+      # @param log_format [String] format string for log entries to STDERR. There are 4 variables available
       #    for inclusion in the output: severity, datetime, progname, msg. Variables must be wrapped in %{}.
       # @param datetime_format [String] datetime formatting string used for logger dates appearing in STDERR.
       def initialize(failure_only: false, log_level: 'WARN', log_format: nil, datetime_format: nil)
@@ -25,12 +25,12 @@ module Longleaf
           @stderr_log.formatter = proc do |severity, datetime, progname, msg|
             # Make sure the format ends with a newline
             @log_format = @log_format + "\n" unless @log_format.end_with?("\n")
-        
+
             formatted_date = @stderr_log.datetime_format.nil? ? datetime : datetime.strftime(datetime_format)
             @log_format % { :severity => severity, :datetime => formatted_date, :progname => progname, :msg => msg }
           end
         end
-      
+
         @stdout_log = Logger.new($stdout)
         @stdout_log.formatter = proc do |severity, datetime, progname, msg|
           "#{msg}\n"
@@ -41,33 +41,34 @@ module Longleaf
           @stdout_log.level = 'info'
         end
       end
-    
+
       def debug(progname = nil, &block)
         @stderr_log.debug(progname, &block)
       end
-    
+
       def info(progname = nil, &block)
         @stderr_log.info(progname, &block)
       end
-    
+
       def warn(progname = nil, &block)
         @stderr_log.warn(progname, &block)
       end
-    
+
       def error(progname = nil, &block)
         @stderr_log.error(progname, &block)
       end
-    
+
       def fatal(progname = nil, &block)
         @stderr_log.fatal(progname, &block)
       end
-    
+
       def unknown(progname = nil, &block)
         @stderr_log.unknown(progname, &block)
       end
-    
+
       # Logs a success message to STDOUT, as well as STDERR at info level.
-      # @param eventOrMessage [String] name of the preservation event which succeeded, 
+      #
+      # @param [String] eventOrMessage name of the preservation event which succeeded,
       #    or the message to output if it is the only parameter. Required.
       # @param file_name [String] file name which is the subject of this message.
       # @param message [String] descriptive message to accompany this output
@@ -75,10 +76,10 @@ module Longleaf
       def success(eventOrMessage, file_name = nil, message = nil, service = nil)
         outcome('SUCCESS', eventOrMessage, file_name, message, service)
       end
-    
+
       # Logs a failure message to STDOUT, as well as STDERR at info level.
       # If an error was provided, it is logged to STDERR at error level.
-      # @param eventOrMessage [String] name of the preservation event which failed, 
+      # @param eventOrMessage [String] name of the preservation event which failed,
       #    or the message to output if it is the only parameter.
       # @param file_name [String] file name which is the subject of this message.
       # @param message [String] descriptive message to accompany this output
@@ -87,17 +88,18 @@ module Longleaf
       def failure(eventOrMessage, file_name = nil, message = nil, service = nil, error: nil)
         text = outcome_text('FAILURE', eventOrMessage, file_name, message, service, error)
         @stdout_log.warn(text)
-      
+
         @stderr_log.info(text)
         @stderr_log.error("#{error.message}") unless error.nil?
+        @stderr_log.error("#{error.backtrace}") unless error.nil? || error.backtrace.nil?
       end
-    
+
       # Logs an outcome message to STDOUT, as well as STDERR at info level.
       # If file_name and message are nil, eventOrMessage will be used as the message.
       #
       # @param outcome [String] The status of the outcome. Required.
-      # @param eventOrMessage [String] name of the preservation event which was successful, 
-      #    or the message to output if it is the only parameter. Required.
+      # @param eventOrMessage [String] name of the preservation event which was successful,
+      #     or the message to output if it is the only parameter. Required.
       # @param file_name [String] file name which is the subject of this message.
       # @param message [String] descriptive message to accompany this output
       # @param service [String] name of the service which executed.
@@ -107,14 +109,13 @@ module Longleaf
         @stdout_log.info(text)
         @stderr_log.info(text)
       end
-    
-      # FAILURE verify[cdr_fixity_check] /path/to/file: Something terrible
+
       private
       def outcome_text(outcome, eventOrMessage, file_name = nil, message = nil, service = nil, error = nil)
         message_only = file_name.nil? && message.nil? && error.nil?
-        
+
         text = "#{outcome}"
-      
+
         if message_only
           text << ": #{eventOrMessage}"
         else

data/lib/longleaf/models/app_fields.rb

@@ -1,10 +1,15 @@
 module Longleaf
+  # Application configuration field names
   class AppFields
     LOCATIONS = 'locations'
     SERVICES = 'services'
     SERVICE_MAPPINGS = 'service_mappings'
-    
+    SYSTEM = 'system'
+
     LOCATION_PATH = 'path'
-    METADATA_PATH = 'metadata_path'
+    METADATA_CONFIG = 'metadata'
+    METADATA_DIGESTS = 'digests'
+
+    STORAGE_TYPE = 'type'
   end
 end

data/lib/longleaf/models/file_record.rb

@@ -1,25 +1,34 @@
-# Record for an individual file and its associated information 
 module Longleaf
+  # Record for an individual file and its associated information
   class FileRecord
-    
     attr_accessor :metadata_record
     attr_reader :storage_location
     attr_reader :path
-    
+
     # @param file_path [String] path to the file
-    # @param storage_location [Longleaf::StorageLocation] storage location containing the file
-    def initialize(file_path, storage_location)
+    # @param storage_location [StorageLocation] storage location containing the file
+    def initialize(file_path, storage_location, metadata_record = nil)
       raise ArgumentError.new("FileRecord requires a path") if file_path.nil?
       raise ArgumentError.new("FileRecord requires a storage_location") if storage_location.nil?
-      
+
       @path = file_path
       @storage_location = storage_location
+      @metadata_record = metadata_record
     end
-    
+
     # @return [String] path for the metadata file for this file
     def metadata_path
       @metadata_path = @storage_location.get_metadata_path_for(path) if @metadata_path.nil?
       @metadata_path
     end
+
+    def metadata_present?
+      File.exist?(metadata_path)
+    end
+
+    def ==(other_obj)
+      return false unless other_obj.is_a?(FileRecord)
+      path == other_obj.path
+    end
   end
-end
+end

data/lib/longleaf/models/filesystem_metadata_location.rb

@@ -0,0 +1,56 @@
+require 'longleaf/services/metadata_serializer'
+require 'longleaf/models/metadata_location'
+require 'longleaf/models/storage_types'
+
+module Longleaf
+  # A filesystem based location in which metadata associated with registered files is stored.
+  class FilesystemMetadataLocation < MetadataLocation
+    AF ||= Longleaf::AppFields
+
+    def initialize(config)
+      super(config)
+    end
+
+    # @return the storage type for this location
+    def type
+      StorageTypes::FILESYSTEM_STORAGE_TYPE
+    end
+
+    # Get the absolute path for the metadata file for the given file path located in this storage location.
+    # @param file_path [String] path of the file relative its storage location
+    # @return absolute path to the metadata
+    # @raise [ArgumentError] if the file_path is not provided.
+    def metadata_path_for(file_path)
+      raise ArgumentError.new("A file_path parameter is required") if file_path.nil?
+      raise ArgumentError.new("File path must be relative") if Pathname.new(file_path).absolute?
+
+      md_path = File.join(@path, file_path)
+      # If the file_path is to a file, then add metadata suffix.
+      if md_path.end_with?('/')
+        md_path
+      else
+        md_path + MetadataSerializer::metadata_suffix
+      end
+    end
+
+    # Get the metadata path relative to this location
+    # @param md_path [String] metadata file path
+    # @return the metadata path relative to this location
+    # @raise [ArgumentError] if the metadata path is not contained by this location
+    def relativize(md_path)
+      return md_path if Pathname.new(md_path).relative?
+
+      raise ArgumentError.new("Metadata path must be contained by this location") if !md_path.start_with?(@path)
+
+      md_path.sub(@path, "")
+    end
+
+
+    # Checks that the path defined in this metadata location are available
+    # @raise [StorageLocationUnavailableError] if the metadata location is not available
+    def available?
+      raise StorageLocationUnavailableError.new("Metadata path does not exist or is not a directory: #{@path}")\
+          unless Dir.exist?(@path)
+    end
+  end
+end

data/lib/longleaf/models/filesystem_storage_location.rb

@@ -0,0 +1,52 @@
+require 'longleaf/models/storage_location'
+require 'longleaf/models/storage_types'
+
+module Longleaf
+  # A storage location in a local filesystem
+  class FilesystemStorageLocation < StorageLocation
+    # @param name [String] the name of this storage location
+    # @param config [Hash] hash containing the configuration options for this location
+    # @param md_loc [MetadataLocation] metadata location associated with this storage location
+    def initialize(name, config, md_loc)
+      super(name, config, md_loc)
+      @path += File::SEPARATOR unless @path.end_with?(File::SEPARATOR)
+    end
+
+    # @return the storage type for this location
+    def type
+      StorageTypes::FILESYSTEM_STORAGE_TYPE
+    end
+
+    # Get that absolute path to the file associated with the provided metadata path
+    # @param md_path [String] metadata file path
+    # @raise [ArgumentError] if the md_path is not in this storage location
+    # @return [String] the path for the file associated with this metadata
+    def get_path_from_metadata_path(md_path)
+      raise ArgumentError.new("A file_path parameter is required") if md_path.nil? || md_path.empty?
+
+      rel_path = @metadata_location.relative_file_path_for(md_path)
+
+      File.join(@path, rel_path)
+    end
+
+    # Checks that the path and metadata path defined in this location are available
+    # @raise [StorageLocationUnavailableError] if the storage location is not available
+    def available?
+      raise StorageLocationUnavailableError.new("Path does not exist or is not a directory: #{@path}")\
+          unless Dir.exist?(@path)
+      @metadata_location.available?
+    end
+
+    # Get the file path relative to this location
+    # @param file_path [String] file path
+    # @return the file path relative to this location
+    # @raise [ArgumentError] if the file path is not contained by this location
+    def relativize(file_path)
+      return file_path if Pathname.new(file_path).relative?
+
+      raise ArgumentError.new("Metadata path must be contained by this location") if !file_path.start_with?(@path)
+
+      file_path.sub(@path, "")
+    end
+  end
+end

data/lib/longleaf/models/md_fields.rb

@@ -1,8 +1,9 @@
 module Longleaf
+  # File metadata fields
   class MDFields
     DATA = 'data'
     SERVICES = 'services'
-    
+
     REGISTERED_TIMESTAMP = 'registered'
     DEREGISTERED_TIMESTAMP = 'deregistered'
 

data/lib/longleaf/models/metadata_location.rb

@@ -0,0 +1,47 @@
+require 'longleaf/models/app_fields'
+
+module Longleaf
+  # A location in which metadata associated with registered files is stored.
+  class MetadataLocation
+    AF ||= Longleaf::AppFields
+
+    attr_reader :path
+    attr_reader :digests
+
+    def initialize(config)
+      raise ArgumentError.new("Config parameter is required") unless config
+      @path = config[AF::LOCATION_PATH]
+      raise ArgumentError.new("Parameter path is required") unless @path
+      @path += '/' unless @path.end_with?('/')
+
+      digests = config[AF::METADATA_DIGESTS]
+      if digests.nil?
+        @digests = []
+      elsif digests.is_a?(String)
+        @digests = [digests.downcase]
+      else
+        @digests = digests.map(&:downcase)
+      end
+      DigestHelper::validate_algorithms(@digests)
+    end
+
+    # Transforms the given metadata path into a relative storage location path
+    # @param md_path [String] path of the metadata file or directory to compute file path for.
+    # @return
+    def relative_file_path_for(md_path)
+      rel_md_path = relativize(md_path)
+
+      if rel_md_path.end_with?(MetadataSerializer::metadata_suffix)
+        rel_md_path[0..-MetadataSerializer::metadata_suffix.length - 1]
+      else
+        rel_md_path
+      end
+    end
+
+    # @param [String] metadata path to check
+    # @return true if the metadata path is contained by the path for this location
+    def contains?(md_path)
+      md_path.start_with?(@path)
+    end
+  end
+end

data/lib/longleaf/models/metadata_record.rb

@@ -1,14 +1,16 @@
 require_relative 'md_fields'
 require_relative 'service_record'
+require 'longleaf/helpers/case_insensitive_hash'
 
-# Metadata record for a single file
 module Longleaf
+  # Metadata record for a single file
   class MetadataRecord
-    attr_reader :deregistered, :registered
+    attr_reader :registered
+    attr_accessor :deregistered
     attr_reader :checksums
     attr_reader :properties
     attr_accessor :file_size, :last_modified
-    
+
     # @param properties [Hash] initial data properties for this record
     # @param services [Hash] initial service property tree
     # @param deregistered [String] deregistered timestamp
@@ -16,42 +18,64 @@ module Longleaf
     # @param checksums [Hash] hash of checksum values
     # @param file_size [Integer] size of file in bytes
     # @param last_modified [String] iso8601 representation of the last modified date of file
-    def initialize(properties: Hash.new, services: Hash.new, deregistered: nil, registered: nil, checksums: Hash.new,
+    def initialize(properties: nil, services: nil, deregistered: nil, registered: nil, checksums: nil,
           file_size: nil, last_modified: nil)
-      @properties = properties
+      @properties = properties || Hash.new
       @registered = registered
       @deregistered = deregistered
-      @checksums = checksums
-      @services = services
+      @checksums = CaseInsensitiveHash.new
+      @checksums.merge!(checksums) unless checksums.nil?
+      @services = services || Hash.new
       @file_size = file_size
       @last_modified = last_modified
     end
-    
+
     # @return [Boolean] true if the record is deregistered
     def deregistered?
       !@deregistered.nil?
     end
-    
+
     # Adds a service to this record
     #
     # @param name [String] identifier for the service being added
-    # @param service_properties [ServiceRecord] properties for populating the new service
-    def add_service(name, service = Longleaf::ServiceRecord.new)
+    # @param service [ServiceRecord] properties for populating the new service
+    # @return [ServiceRecord] the service added
+    def add_service(name, service = ServiceRecord.new)
       raise ArgumentError.new("Value must be a ServiceRecord object when adding a service") unless service.class == Longleaf::ServiceRecord
       raise IndexError.new("Service with name '#{name}' already exists") if @services.key?(name)
-      
+
       @services[name] = service
     end
-    
+
+    # Updates details of service record as if the service had been executed.
+    # @param service_name [String] name of the service run
+    # @return [ServiceRecord] the service record updated
+    def update_service_as_performed(service_name)
+      service_rec = service(service_name) || add_service(service_name)
+      service_rec.run_needed = false
+      service_rec.timestamp = ServiceDateHelper.formatted_timestamp
+      service_rec
+    end
+
+    # Updates details of service record as if the service had encountered a
+    # failure during execution.
+    # @param service_name [String] name of the service run
+    # @return [ServiceRecord] the service record updated
+    def update_service_as_failed(service_name)
+      service_rec = service(service_name) || add_service(service_name)
+      service_rec.failure_timestamp = ServiceDateHelper.formatted_timestamp
+      service_rec
+    end
+
     # @param name [String] name identifier of the service to retrieve
     # @return [ServiceRecord] the ServiceRecord for the service identified by name, or nil
     def service(name)
       @services[name]
     end
-    
+
     # @return [Array<String>] a list of name identifiers for services registered to this record
     def list_services
       @services.keys
     end
   end
-end
+end