longleaf 0.2.0.pre.1 → 0.3.0

data/lib/longleaf/commands/register_command.rb

@@ -8,7 +8,7 @@ module Longleaf
   # Command for registering files with longleaf
   class RegisterCommand
     include Longleaf::EventStatusTracking
-    
+
     def initialize(app_manager)
       @app_manager = app_manager
     end
@@ -19,16 +19,18 @@ module Longleaf
     # @param checksums [Array] array of checksums
     # @return [Integer] status code
     def execute(file_selector:, force: false, checksums: nil)
+      start_time = Time.now
+      logger.info('Performing register command')
       begin
         # Perform register events on each of the file paths provided
         loop do
           f_path = file_selector.next_path
           break if f_path.nil?
-          
+
           storage_location = @app_manager.location_manager.get_location_by_path(f_path)
-        
+
           file_rec = FileRecord.new(f_path, storage_location)
-          
+
           register_event = RegisterEvent.new(file_rec: file_rec, force: force, app_manager: @app_manager,
               checksums: checksums)
           track_status(register_event.perform)
@@ -38,8 +40,9 @@ module Longleaf
       rescue => err
         record_failure(EventNames::REGISTER, error: err)
       end
-      
+
+      logger.info("Completed register command in #{Time.now - start_time}s")
       return_status
     end
   end
-end
+end

data/lib/longleaf/commands/reindex_command.rb

@@ -0,0 +1,92 @@
+require 'longleaf/errors'
+require 'longleaf/events/event_status_tracking'
+require 'longleaf/logging'
+
+module Longleaf
+  # Command for reindexing metadata
+  class ReindexCommand
+    include Longleaf::Logging
+    include Longleaf::EventStatusTracking
+
+    def initialize(app_manager)
+      @app_manager = app_manager
+      @index_manager = @app_manager.index_manager
+    end
+
+    # Execute the reindex command
+    # @param only_if_stale [boolean] if true, then the reindex command will perform no operation unless the index is stale.
+    # @return [Integer] status code
+    def execute(only_if_stale: false)
+      if !@index_manager.using_index?
+        record_failure("Cannot perform reindex, no index is configured")
+        return return_status
+      end
+
+      if only_if_stale && !@index_manager.index_stale?
+        record_success("Index is not stale, performing no action")
+        return return_status
+      end
+
+      start_time = Time.now
+      logger.info('Performing full reindex')
+      results = nil
+      begin
+        start_time = Time.now.utc
+
+        selector = all_storage_locations_selector
+
+        # Repopulate the index
+        results = index_all(selector)
+
+        # List and then clear all files which were not reindexed
+        @index_manager.each_registered_path(selector, older_than: start_time) do |file_path|
+          logger.warn("Clearing '#{file_path}' from index, file is no longer present.")
+        end
+        @index_manager.clear_index(start_time)
+
+        # Update the state of the index to indicate it has been reindexed
+        @index_manager.update_index_state
+      rescue => err
+        record_failure("Encountered error while reindexing", error: err)
+      end
+
+      if results['fail'] > 0
+        record_success("Completed reindexing, #{results['success']} successful, #{results['fail']} failed.")
+      else
+        record_success("Completed reindexing, #{results['success']} successful.")
+      end
+
+      logger.info("Completed full reindex in #{Time.now - start_time}s")
+      return_status
+    end
+
+    private
+    def index_all(selector)
+      count = 0
+      failures = 0
+
+      selector.each do |file_path|
+        begin
+          storage_loc = @app_manager.location_manager.get_location_by_path(file_path)
+          file_rec = FileRecord.new(file_path, storage_loc)
+
+          @app_manager.md_manager.load(file_rec)
+          @index_manager.index(file_rec)
+
+          record_success("Reindexed #{file_rec.path}")
+          count += 1
+        rescue LongleafError => err
+          record_failure("Failed to reindex #{file_rec.path}: #{err.message}")
+          failures += 1
+        end
+      end
+      {'success' => count, 'fail' => failures}
+    end
+
+    def all_storage_locations_selector
+      storage_loc_names = @app_manager.location_manager.locations.keys
+
+      RegisteredFileSelector.new(storage_locations: storage_loc_names, app_config: @app_manager)
+    end
+  end
+end

data/lib/longleaf/commands/validate_config_command.rb

@@ -5,29 +5,50 @@ module Longleaf
   # Command for validating an application configuration file
   class ValidateConfigCommand
     include Longleaf::EventStatusTracking
-    
+
     def initialize(config_path)
       @config_path = config_path
     end
-    
+
     # Execute the validate command on the specified configuration yml file
     def execute
+      start_time = Time.now
+      logger.info('Performing validate configuration command')
       begin
         app_config_manager = Longleaf::ApplicationConfigDeserializer.deserialize(@config_path)
-        
+
         location_manager = app_config_manager.location_manager
         location_manager.locations.each do |name, location|
           location.available?
         end
-        
+
+        validate_services(app_config_manager.service_manager)
+
         record_success("Application configuration passed validation: #{@config_path}")
       rescue Longleaf::ConfigurationError, Longleaf::StorageLocationUnavailableError => err
         record_failure("Application configuration invalid due to the following issue:\n#{err.message}")
       rescue => err
         record_failure("Failed to validate application configuration", error: err)
       end
-      
+
+      logger.info("Completed validate configuration command in #{Time.now - start_time}s")
       return_status
     end
+
+    private
+    # Verify that all defined services are valid and may be instantiated with the given configuration,
+    # according to internal expectations.
+    # @raise ConfigurationError if any services may not be instantiated
+    def validate_services(service_manager)
+      def_manager = service_manager.definition_manager
+
+      def_manager.services.each do |service_name, service_def|
+        begin
+          service_manager.service(service_name)
+        rescue => e
+          raise ConfigurationError.new(e.message)
+        end
+      end
+    end
   end
-end
+end

data/lib/longleaf/commands/validate_metadata_command.rb

@@ -7,7 +7,7 @@ module Longleaf
   # Command for validating file metadata longleaf
   class ValidateMetadataCommand
     include Longleaf::EventStatusTracking
-    
+
     def initialize(app_manager)
       @app_manager = app_manager
     end
@@ -16,34 +16,36 @@ module Longleaf
     # @param file_selector [FileSelector] selector for files to register
     # @return [Integer] status code
     def execute(file_selector:)
+      start_time = Time.now
+      logger.info('Performing validate metadata command')
       begin
         # Perform metadata validation on each of the file paths provided
         loop do
           f_path = file_selector.next_path
           break if f_path.nil?
-          
+
           storage_location = @app_manager.location_manager.get_location_by_path(f_path)
-          
+
           begin
             file_rec = FileRecord.new(f_path, storage_location)
             unless file_rec.metadata_present?
               raise MetadataError.new("Cannot validate metadata for #{f_path}, file is not registered.")
             end
-          
-            MetadataDeserializer::deserialize(file_path: file_rec.metadata_path,
-                digest_algs: storage_location.metadata_digests)
+
+            @app_manager.md_manager.load(file_rec)
             record_success("Metadata for file passed validation: #{f_path}")
           rescue LongleafError => err
             record_failure(err.message)
           end
         end
-      rescue InvalidStoragePathError, StorageLocationUnavailableError => err
+      rescue RegistrationError, InvalidStoragePathError, StorageLocationUnavailableError => err
         record_failure(err.message)
       rescue => err
         record_failure("Encountered error while validating metadata files", error: err)
       end
-      
+
+      logger.info("Completed validate metadata command in #{Time.now - start_time}s")
       return_status
     end
   end
-end
+end

data/lib/longleaf/errors.rb

@@ -1,34 +1,34 @@
 module Longleaf
   # General Longleaf error
   class LongleafError < StandardError; end
-  
+
   # Invalid application configuration error
   class ConfigurationError < LongleafError; end
-  
+
   # Invalid storage path error
   class InvalidStoragePathError < LongleafError; end
-  
+
   # Metadata does not meet requirements error
   class MetadataError < LongleafError; end
-  
+
   # Unavailable storage location error
   class StorageLocationUnavailableError < LongleafError; end
-  
+
   # Error related to executing a preservation event
   class EventError < LongleafError; end
-  
-  # Error while attempting to perform a registration event
+
+  # Error with the registration state of a file or while attempting to perform a registration event
   class RegistrationError < EventError; end
-  
+
   # Error while attempting to perform a deregistration event
   class DeregistrationError < EventError; end
-  
+
   # Error while performing a preservation service
   class PreservationServiceError < LongleafError; end
-  
+
   # Fixity check failure error
   class ChecksumMismatchError < PreservationServiceError; end
-  
+
   # Error indicating an unknown or invalid digest algorithm was specified
   class InvalidDigestAlgorithmError < LongleafError; end
-end
+end

data/lib/longleaf/events/deregister_event.rb

@@ -7,7 +7,7 @@ module Longleaf
   # Event to deregister a file from longleaf
   class DeregisterEvent
     include Longleaf::EventStatusTracking
-    
+
     # @param file_rec [FileRecord] file record
     # @param app_manager [ApplicationConfigManager] the application configuration
     # @param force [boolean] if true, then already deregistered files will be deregistered again
@@ -18,38 +18,36 @@ module Longleaf
       raise ArgumentError.new('Must provide an ApplicationConfigManager') if app_manager.nil?
       raise ArgumentError.new('Parameter app_manager must be an ApplicationConfigManager') \
           unless app_manager.is_a?(ApplicationConfigManager)
-      
+
       @app_manager = app_manager
       @file_rec = file_rec
       @force = force
     end
-    
+
     # Perform a deregistration event on the given file record
-    # @raise DeregistrationError if a file cannot be deregistered 
+    # @raise DeregistrationError if a file cannot be deregistered
     def perform
       begin
         md_rec = @file_rec.metadata_record
-        
+
         # Only need to deregister a deregistered file if the force flag is provided
         if md_rec.deregistered? && !@force
           raise DeregistrationError.new("Unable to deregister '#{@file_rec.path}', it is already deregistered.")
         end
-        
-        md_rec.deregistered = Time.now.utc.iso8601
-        
-        # persist the metadata out to file
-        MetadataSerializer::write(metadata: md_rec,
-            file_path: @file_rec.metadata_path,
-            digest_algs: @file_rec.storage_location.metadata_digests)
-        
+
+        md_rec.deregistered = Time.now.utc.iso8601(3)
+
+        # persist the metadata
+        @app_manager.md_manager.persist(@file_rec)
+
         record_success(EventNames::DEREGISTER, @file_rec.path)
       rescue DeregistrationError => err
         record_failure(EventNames::DEREGISTER, @file_rec.path, err.message)
       rescue InvalidStoragePathError => err
         record_failure(EventNames::DEREGISTER, @file_rec.path, err.message)
       end
-      
+
       return_status
     end
   end
-end
+end

data/lib/longleaf/events/event_status_tracking.rb

@@ -4,14 +4,14 @@ module Longleaf
   # Helper methods for tracking and recording the overall outcome of a preservation event.
   module EventStatusTracking
     include Longleaf::Logging
-    
+
     # Record a successful operation to the output and the overall status of this event.
     # @param args [Array] arguments to pass to logger
     def record_success(*args)
       logger.success(*args)
       track_success
     end
-    
+
     # Update the status of this action with a success outcome.
     def track_success
       if @return_status.nil? || @return_status == 0
@@ -20,14 +20,14 @@ module Longleaf
         @return_status = 2
       end
     end
-    
+
     # Record a failed operation to the output and the overall status of this event.
     # @param args [Array] arguments to pass to logger
     def record_failure(*args)
       logger.failure(*args)
       track_failure
     end
-    
+
     # Update the status of this action with a failure outcome.
     def track_failure
       if @return_status.nil? || @return_status == 1
@@ -36,7 +36,7 @@ module Longleaf
         @return_status = 2
       end
     end
-    
+
     # Update the status of this action with the provided outcome status number.
     # @param status [Integer] outcome status
     def track_status(status)
@@ -48,7 +48,7 @@ module Longleaf
         track_failure
       end
     end
-    
+
     # @return [Integer] the return status for this event, where 0 indicates success,
     # 1 indicates failure, and 2 indicates partial failure
     def return_status
@@ -56,4 +56,4 @@ module Longleaf
       @return_status
     end
   end
-end
+end

data/lib/longleaf/events/preserve_event.rb

@@ -8,30 +8,37 @@ module Longleaf
   class PreserveEvent
     include Longleaf::Logging
     include Longleaf::EventStatusTracking
-    
+
     # @param file_rec [FileRecord] file record
     # @param app_manager [ApplicationConfigManager] the application configuration
     # @param force [boolean] if true, then services run regardless of whether they are flagged as needed
     def initialize(file_rec:, app_manager:, force: false)
       raise ArgumentError.new('Must provide a file_rec parameter') if file_rec.nil?
       raise ArgumentError.new('Must provide an ApplicationConfigManager') if app_manager.nil?
-      
+
       @app_manager = app_manager
       @file_rec = file_rec
       @force = force
     end
-    
+
     # Perform a preserve event on the given file, updating its metadata record if any services were executed.
     def perform
       storage_loc = @file_rec.storage_location
       service_manager = @app_manager.service_manager
       md_rec = @file_rec.metadata_record
       f_path = @file_rec.path
-      
+
       logger.info("Performing preserve event on #{@file_rec.path}")
-      
-      service_performed = false
+
+      needs_persist = false
       begin
+        if !File.exist?(f_path)
+          # Need to persist metadata to avoid repeating processing of this file too soon.
+          needs_persist = true
+          record_failure(EventNames::PRESERVE, f_path, "File is registered but missing.")
+          return return_status
+        end
+
         # get the list of services applicable to this location and event
         service_manager.list_services(location: storage_loc.name, event: EventNames::PRESERVE).each do |service_name|
           # Skip over this service if it does not need to be run, unless force flag active
@@ -39,32 +46,35 @@ module Longleaf
             logger.debug("Service #{service_name} not needed for file '#{@file_rec.path}', skipping")
             next
           end
-          
+
           begin
             logger.info("Performing preserve service #{service_name} for #{@file_rec.path}")
+            needs_persist = true
             # execute the service
             service_manager.perform_service(service_name, @file_rec, EventNames::PRESERVE)
-            
+
             # record the outcome
             @file_rec.metadata_record.update_service_as_performed(service_name)
-            service_performed = true
             record_success(EventNames::PRESERVE, f_path, nil, service_name)
           rescue PreservationServiceError => e
+            @file_rec.metadata_record.update_service_as_failed(service_name)
             record_failure(EventNames::PRESERVE, f_path, e.message, service_name)
+          rescue StorageLocationUnavailableError => e
+            raise e
           rescue StandardError => e
+            @file_rec.metadata_record.update_service_as_failed(service_name)
             record_failure(EventNames::PRESERVE, f_path, nil, service_name, error: e)
             return return_status
           end
         end
       ensure
         # persist the metadata out to file if any services were executed
-        if service_performed
-          MetadataSerializer::write(metadata: @file_rec.metadata_record,
-              file_path: @file_rec.metadata_path,
-              digest_algs: storage_loc.metadata_digests)
+        if needs_persist
+          # persist the metadata
+          @app_manager.md_manager.persist(@file_rec)
         end
       end
-      
+
       return_status
     end
   end