longleaf 0.1.0 → 0.2.0.pre.1

data/lib/longleaf/commands/preserve_command.rb

@@ -0,0 +1,45 @@
+require 'longleaf/errors'
+require 'longleaf/events/event_status_tracking'
+require 'longleaf/events/preserve_event'
+require 'longleaf/services/application_config_deserializer'
+require 'longleaf/candidates/file_selector'
+require 'longleaf/candidates/service_candidate_locator'
+require 'longleaf/events/event_names'
+require 'longleaf/logging'
+
+module Longleaf
+  # Command for preserving files
+  class PreserveCommand
+    include Longleaf::Logging
+    include Longleaf::EventStatusTracking
+    
+    def initialize(app_manager)
+      @app_manager = app_manager
+    end
+    
+    # Execute the preserve command on the given parameters
+    # @param file_selector [FileSelector] selector for files to preserve
+    # @param force [Boolean] force flag
+    # @return [Integer] status code
+    def execute(file_selector:, force: false)
+      begin
+        # Perform preserve events on each of the file paths provided
+        candidate_locator = ServiceCandidateLocator.new(@app_manager)
+        candidate_it = candidate_locator.candidate_iterator(file_selector, EventNames::PRESERVE, force)
+        candidate_it.each do |file_rec|
+          f_path = file_rec.path
+          
+          logger.debug("Selected candidate #{file_rec.path} for a preserve event")
+          preserve_event = PreserveEvent.new(file_rec: file_rec, force: force, app_manager: @app_manager)
+          track_status(preserve_event.perform)
+        end
+      rescue LongleafError => e
+        record_failure(EventNames::PRESERVE, nil, e.message)
+      rescue => err
+        record_failure(EventNames::PRESERVE, error: err)
+      end
+      
+      return_status
+    end
+  end
+end

data/lib/longleaf/commands/register_command.rb

@@ -1,56 +1,42 @@
 require 'longleaf/services/application_config_deserializer'
 require 'longleaf/events/register_event'
 require 'longleaf/models/file_record'
-require 'longleaf/commands/abstract_command'
+require 'longleaf/events/event_names'
+require 'longleaf/events/event_status_tracking'
 
-# Command for registering files with longleaf
 module Longleaf
-  class RegisterCommand < AbstractCommand
+  # Command for registering files with longleaf
+  class RegisterCommand
+    include Longleaf::EventStatusTracking
     
-    def initialize(config_path)
-      @config_path = config_path
+    def initialize(app_manager)
+      @app_manager = app_manager
     end
 
     # Execute the register command on the given parameters
-    def execute(file_paths: nil, force: false, checksums: nil)
-      if file_paths.nil? || file_paths.empty?
-        record_failure("Must provide one or more file paths to register")
-        return return_status
-      end
-      
+    # @param file_selector [FileSelector] selector for files to register
+    # @param force [Boolean] force flag
+    # @param checksums [Array] array of checksums
+    # @return [Integer] status code
+    def execute(file_selector:, force: false, checksums: nil)
       begin
-        # Retrieve the application configuration
-        app_manager = Longleaf::ApplicationConfigDeserializer.deserialize(@config_path)
-        
         # Perform register events on each of the file paths provided
-        file_paths.each do |f_path|
-          begin
-            storage_location = app_manager.location_manager.get_location_by_path(f_path)
-            if storage_location.nil?
-              raise InvalidStoragePathError.new(
-                  "Unable to register '#{f_path}', it does not belong to any registered storage locations.")
-            end
+        loop do
+          f_path = file_selector.next_path
+          break if f_path.nil?
           
-            raise InvalidStoragePathError.new("Unable to register '#{f_path}', file does not exist or is unreachable.") \
-                unless File.file?(f_path)
+          storage_location = @app_manager.location_manager.get_location_by_path(f_path)
+        
+          file_rec = FileRecord.new(f_path, storage_location)
           
-            file_rec = FileRecord.new(f_path, storage_location)
-            
-            register_event = RegisterEvent.new(file_rec: file_rec, force: force, app_manager: app_manager,
-                checksums: checksums)
-            register_event.perform
-            
-            record_success(RegisterEvent::EVENT_NAME, f_path)
-          rescue RegistrationError => err
-            record_failure(RegisterEvent::EVENT_NAME, f_path, err.message)
-          rescue InvalidStoragePathError => err
-            record_failure(RegisterEvent::EVENT_NAME, f_path, err.message)
-          end
+          register_event = RegisterEvent.new(file_rec: file_rec, force: force, app_manager: @app_manager,
+              checksums: checksums)
+          track_status(register_event.perform)
         end
-      rescue ConfigurationError => err
-        record_failure("Failed to load application configuration due to the following issue:\n#{err.message}")
+      rescue InvalidStoragePathError, StorageLocationUnavailableError => err
+        record_failure(EventNames::REGISTER, nil, err.message)
       rescue => err
-        record_failure(RegisterEvent::EVENT_NAME, error: err)
+        record_failure(EventNames::REGISTER, error: err)
       end
       
       return_status

data/lib/longleaf/commands/validate_config_command.rb

@@ -1,12 +1,16 @@
 require 'longleaf/services/application_config_deserializer'
-require 'longleaf/commands/abstract_command'
+require 'longleaf/events/event_status_tracking'
 
 module Longleaf
-  class ValidateConfigCommand < AbstractCommand
+  # Command for validating an application configuration file
+  class ValidateConfigCommand
+    include Longleaf::EventStatusTracking
+    
     def initialize(config_path)
       @config_path = config_path
     end
     
+    # Execute the validate command on the specified configuration yml file
     def execute
       begin
         app_config_manager = Longleaf::ApplicationConfigDeserializer.deserialize(@config_path)

data/lib/longleaf/commands/validate_metadata_command.rb

@@ -0,0 +1,49 @@
+require 'longleaf/services/application_config_deserializer'
+require 'longleaf/models/file_record'
+require 'longleaf/events/event_status_tracking'
+require 'longleaf/errors'
+
+module Longleaf
+  # Command for validating file metadata longleaf
+  class ValidateMetadataCommand
+    include Longleaf::EventStatusTracking
+    
+    def initialize(app_manager)
+      @app_manager = app_manager
+    end
+
+    # Execute the validation command
+    # @param file_selector [FileSelector] selector for files to register
+    # @return [Integer] status code
+    def execute(file_selector:)
+      begin
+        # Perform metadata validation on each of the file paths provided
+        loop do
+          f_path = file_selector.next_path
+          break if f_path.nil?
+          
+          storage_location = @app_manager.location_manager.get_location_by_path(f_path)
+          
+          begin
+            file_rec = FileRecord.new(f_path, storage_location)
+            unless file_rec.metadata_present?
+              raise MetadataError.new("Cannot validate metadata for #{f_path}, file is not registered.")
+            end
+          
+            MetadataDeserializer::deserialize(file_path: file_rec.metadata_path,
+                digest_algs: storage_location.metadata_digests)
+            record_success("Metadata for file passed validation: #{f_path}")
+          rescue LongleafError => err
+            record_failure(err.message)
+          end
+        end
+      rescue InvalidStoragePathError, StorageLocationUnavailableError => err
+        record_failure(err.message)
+      rescue => err
+        record_failure("Encountered error while validating metadata files", error: err)
+      end
+      
+      return_status
+    end
+  end
+end

data/lib/longleaf/errors.rb

@@ -1,15 +1,34 @@
 module Longleaf
+  # General Longleaf error
   class LongleafError < StandardError; end
   
+  # Invalid application configuration error
   class ConfigurationError < LongleafError; end
   
+  # Invalid storage path error
   class InvalidStoragePathError < LongleafError; end
   
+  # Metadata does not meet requirements error
   class MetadataError < LongleafError; end
   
+  # Unavailable storage location error
   class StorageLocationUnavailableError < LongleafError; end
   
+  # Error related to executing a preservation event
   class EventError < LongleafError; end
   
+  # Error while attempting to perform a registration event
   class RegistrationError < EventError; end
+  
+  # Error while attempting to perform a deregistration event
+  class DeregistrationError < EventError; end
+  
+  # Error while performing a preservation service
+  class PreservationServiceError < LongleafError; end
+  
+  # Fixity check failure error
+  class ChecksumMismatchError < PreservationServiceError; end
+  
+  # Error indicating an unknown or invalid digest algorithm was specified
+  class InvalidDigestAlgorithmError < LongleafError; end
 end

data/lib/longleaf/events/deregister_event.rb

@@ -0,0 +1,55 @@
+require 'longleaf/errors'
+require 'longleaf/events/event_names'
+require 'longleaf/events/event_status_tracking'
+require 'longleaf/services/metadata_serializer'
+
+module Longleaf
+  # Event to deregister a file from longleaf
+  class DeregisterEvent
+    include Longleaf::EventStatusTracking
+    
+    # @param file_rec [FileRecord] file record
+    # @param app_manager [ApplicationConfigManager] the application configuration
+    # @param force [boolean] if true, then already deregistered files will be deregistered again
+    def initialize(file_rec:, app_manager:, force: false)
+      raise ArgumentError.new('Must provide a file_rec parameter') if file_rec.nil?
+      raise ArgumentError.new('Parameter file_rec must be a FileRecord') \
+          unless file_rec.is_a?(FileRecord)
+      raise ArgumentError.new('Must provide an ApplicationConfigManager') if app_manager.nil?
+      raise ArgumentError.new('Parameter app_manager must be an ApplicationConfigManager') \
+          unless app_manager.is_a?(ApplicationConfigManager)
+      
+      @app_manager = app_manager
+      @file_rec = file_rec
+      @force = force
+    end
+    
+    # Perform a deregistration event on the given file record
+    # @raise DeregistrationError if a file cannot be deregistered 
+    def perform
+      begin
+        md_rec = @file_rec.metadata_record
+        
+        # Only need to deregister a deregistered file if the force flag is provided
+        if md_rec.deregistered? && !@force
+          raise DeregistrationError.new("Unable to deregister '#{@file_rec.path}', it is already deregistered.")
+        end
+        
+        md_rec.deregistered = Time.now.utc.iso8601
+        
+        # persist the metadata out to file
+        MetadataSerializer::write(metadata: md_rec,
+            file_path: @file_rec.metadata_path,
+            digest_algs: @file_rec.storage_location.metadata_digests)
+        
+        record_success(EventNames::DEREGISTER, @file_rec.path)
+      rescue DeregistrationError => err
+        record_failure(EventNames::DEREGISTER, @file_rec.path, err.message)
+      rescue InvalidStoragePathError => err
+        record_failure(EventNames::DEREGISTER, @file_rec.path, err.message)
+      end
+      
+      return_status
+    end
+  end
+end

data/lib/longleaf/events/event_names.rb

@@ -0,0 +1,9 @@
+module Longleaf
+  # Constants for preservation event names
+  class EventNames
+    REGISTER = 'register'
+    PRESERVE = 'preserve'
+    CLEANUP = 'cleanup'
+    DEREGISTER = 'deregister'
+  end
+end

data/lib/longleaf/events/event_status_tracking.rb

@@ -0,0 +1,59 @@
+require 'longleaf/logging'
+
+module Longleaf
+  # Helper methods for tracking and recording the overall outcome of a preservation event.
+  module EventStatusTracking
+    include Longleaf::Logging
+    
+    # Record a successful operation to the output and the overall status of this event.
+    # @param args [Array] arguments to pass to logger
+    def record_success(*args)
+      logger.success(*args)
+      track_success
+    end
+    
+    # Update the status of this action with a success outcome.
+    def track_success
+      if @return_status.nil? || @return_status == 0
+        @return_status = 0
+      else
+        @return_status = 2
+      end
+    end
+    
+    # Record a failed operation to the output and the overall status of this event.
+    # @param args [Array] arguments to pass to logger
+    def record_failure(*args)
+      logger.failure(*args)
+      track_failure
+    end
+    
+    # Update the status of this action with a failure outcome.
+    def track_failure
+      if @return_status.nil? || @return_status == 1
+        @return_status = 1
+      else
+        @return_status = 2
+      end
+    end
+    
+    # Update the status of this action with the provided outcome status number.
+    # @param status [Integer] outcome status
+    def track_status(status)
+      if status == 2
+        @return_status = 2
+      elsif status == 0
+        track_success
+      elsif status == 1
+        track_failure
+      end
+    end
+    
+    # @return [Integer] the return status for this event, where 0 indicates success,
+    # 1 indicates failure, and 2 indicates partial failure
+    def return_status
+      @return_status = 0 if @return_status.nil?
+      @return_status
+    end
+  end
+end

data/lib/longleaf/events/preserve_event.rb

@@ -0,0 +1,71 @@
+require 'longleaf/services/service_manager'
+require 'longleaf/events/event_names'
+require 'longleaf/events/event_status_tracking'
+require 'longleaf/logging'
+
+module Longleaf
+  # Verify event for a single file
+  class PreserveEvent
+    include Longleaf::Logging
+    include Longleaf::EventStatusTracking
+    
+    # @param file_rec [FileRecord] file record
+    # @param app_manager [ApplicationConfigManager] the application configuration
+    # @param force [boolean] if true, then services run regardless of whether they are flagged as needed
+    def initialize(file_rec:, app_manager:, force: false)
+      raise ArgumentError.new('Must provide a file_rec parameter') if file_rec.nil?
+      raise ArgumentError.new('Must provide an ApplicationConfigManager') if app_manager.nil?
+      
+      @app_manager = app_manager
+      @file_rec = file_rec
+      @force = force
+    end
+    
+    # Perform a preserve event on the given file, updating its metadata record if any services were executed.
+    def perform
+      storage_loc = @file_rec.storage_location
+      service_manager = @app_manager.service_manager
+      md_rec = @file_rec.metadata_record
+      f_path = @file_rec.path
+      
+      logger.info("Performing preserve event on #{@file_rec.path}")
+      
+      service_performed = false
+      begin
+        # get the list of services applicable to this location and event
+        service_manager.list_services(location: storage_loc.name, event: EventNames::PRESERVE).each do |service_name|
+          # Skip over this service if it does not need to be run, unless force flag active
+          unless @force || service_manager.service_needed?(service_name, md_rec)
+            logger.debug("Service #{service_name} not needed for file '#{@file_rec.path}', skipping")
+            next
+          end
+          
+          begin
+            logger.info("Performing preserve service #{service_name} for #{@file_rec.path}")
+            # execute the service
+            service_manager.perform_service(service_name, @file_rec, EventNames::PRESERVE)
+            
+            # record the outcome
+            @file_rec.metadata_record.update_service_as_performed(service_name)
+            service_performed = true
+            record_success(EventNames::PRESERVE, f_path, nil, service_name)
+          rescue PreservationServiceError => e
+            record_failure(EventNames::PRESERVE, f_path, e.message, service_name)
+          rescue StandardError => e
+            record_failure(EventNames::PRESERVE, f_path, nil, service_name, error: e)
+            return return_status
+          end
+        end
+      ensure
+        # persist the metadata out to file if any services were executed
+        if service_performed
+          MetadataSerializer::write(metadata: @file_rec.metadata_record,
+              file_path: @file_rec.metadata_path,
+              digest_algs: storage_loc.metadata_digests)
+        end
+      end
+      
+      return_status
+    end
+  end
+end

data/lib/longleaf/events/register_event.rb

@@ -1,13 +1,15 @@
 require 'longleaf/errors'
+require 'longleaf/events/event_names'
+require 'longleaf/events/event_status_tracking'
 require 'longleaf/models/metadata_record'
 require 'longleaf/services/metadata_deserializer'
 require 'longleaf/services/metadata_serializer'
 require 'time'
 
-# Event to register a file with longleaf
 module Longleaf
+  # Event to register a file with longleaf
   class RegisterEvent
-    EVENT_NAME = 'register'
+    include Longleaf::EventStatusTracking
     
     # @param file_rec [FileRecord] file record
     # @param app_manager [ApplicationConfigManager] the application configuration
@@ -27,31 +29,42 @@ module Longleaf
     end
     
     # Perform a registration event on the given file
-    # @raises RegistrationError if a file cannot be registered 
+    # @raise RegistrationError if a file cannot be registered 
     def perform
-      metadata_exists = File.file?(@file_rec.metadata_path)
-      # If the file's metadata exists, only need to register it if the force flag is provided
-      if metadata_exists && !@force
-        raise RegistrationError.new("Unable to register '#{@file_rec.path}', it is already registered.")
-      end
+      begin
+        # Only need to re-register file if the force flag is provided
+        if @file_rec.metadata_present? && !@force
+          raise RegistrationError.new("Unable to register '#{@file_rec.path}', it is already registered.")
+        end
       
-      # create metadata record
-      md_rec = MetadataRecord.new(registered: Time.now.utc.iso8601)
-      @file_rec.metadata_record = md_rec
+        # create metadata record
+        md_rec = MetadataRecord.new(registered: Time.now.utc.iso8601)
+        @file_rec.metadata_record = md_rec
       
-      # retain significant details from former record
-      if metadata_exists
-        retain_existing_properties
-      end
+        # retain significant details from former record
+        if @file_rec.metadata_present?
+          retain_existing_properties
+        end
       
-      populate_file_properties
+        populate_file_properties
       
-      md_rec.checksums.merge!(@checksums) unless @checksums.nil?
+        md_rec.checksums.merge!(@checksums) unless @checksums.nil?
       
-      populate_services
+        populate_services
       
-      # persist the metadata out to file
-      MetadataSerializer::write(metadata: md_rec, file_path: @file_rec.metadata_path)
+        # persist the metadata out to file
+        MetadataSerializer::write(metadata: md_rec,
+            file_path: @file_rec.metadata_path,
+            digest_algs: @file_rec.storage_location.metadata_digests)
+        
+        record_success(EventNames::REGISTER, @file_rec.path)
+      rescue RegistrationError => err
+        record_failure(EventNames::REGISTER, @file_rec.path, err.message)
+      rescue InvalidStoragePathError => err
+        record_failure(EventNames::REGISTER, @file_rec.path, err.message)
+      end
+      
+      return_status
     end
     
     private
@@ -67,20 +80,18 @@ module Longleaf
       md_rec = @file_rec.metadata_record
       
       service_manager = @app_manager.service_manager
-      definitions = service_manager.list_service_definitions(location: @file_rec.storage_location.name)
+      service_names = service_manager.list_services(location: @file_rec.storage_location.name)
       
       # Add service section
-      definitions.each do |serv_def|
-        serv_name = serv_def.name
-        md_rec.add_service(serv_name)
-      end
+      service_names.each { |serv_name| md_rec.add_service(serv_name) }
     end
     
     # Copy a subset of properties from an existing metadata record to the new record
     def retain_existing_properties
       md_rec = @file_rec.metadata_record
       
-      old_md = MetadataDeserializer.deserialize(file_path: @file_rec.metadata_path)
+      old_md = MetadataDeserializer.deserialize(file_path: @file_rec.metadata_path,
+              digest_algs: @file_rec.storage_location.metadata_digests)
       # Copy custom properties
       old_md.properties.each { |name, value| md_rec.properties[name] = value }
       # Copy stale-replicas flag per service