longleaf 0.1.0.pre.3 → 1.1.0

data/lib/longleaf/commands/deregister_command.rb

@@ -0,0 +1,51 @@
+require 'longleaf/services/application_config_deserializer'
+require 'longleaf/events/deregister_event'
+require 'longleaf/models/file_record'
+require 'longleaf/events/event_names'
+require 'longleaf/events/event_status_tracking'
+
+module Longleaf
+  # Command for deregistering files with longleaf
+  class DeregisterCommand
+    include Longleaf::EventStatusTracking
+
+    def initialize(app_manager)
+      @app_manager = app_manager
+    end
+
+    # Execute the deregister command on the given parameters
+    # @param file_selector [FileSelector] selector for files to deregister
+    # @param force [Boolean] force flag
+    # @return [Integer] status code
+    def execute(file_selector:, force: false)
+      start_time = Time.now
+      logger.info('Performing deregister command')
+      begin
+        # Perform deregister events on each of the file paths provided
+        loop do
+          f_path = file_selector.next_path
+          break if f_path.nil?
+
+          storage_location = @app_manager.location_manager.get_location_by_path(f_path)
+
+          file_rec = FileRecord.new(f_path, storage_location)
+          unless file_rec.metadata_present?
+            raise DeregistrationError.new("Cannot deregister #{f_path}, file is not registered.")
+          end
+
+          @app_manager.md_manager.load(file_rec)
+
+          event = DeregisterEvent.new(file_rec: file_rec, force: force, app_manager: @app_manager)
+          track_status(event.perform)
+        end
+      rescue RegistrationError, DeregistrationError, InvalidStoragePathError, StorageLocationUnavailableError => err
+        record_failure(EventNames::DEREGISTER, nil, err.message)
+      rescue => err
+        record_failure(EventNames::DEREGISTER, error: err)
+      end
+
+      logger.info("Completed deregister command in #{Time.now - start_time}s")
+      return_status
+    end
+  end
+end

data/lib/longleaf/commands/preserve_command.rb

@@ -0,0 +1,50 @@
+require 'longleaf/errors'
+require 'longleaf/events/event_status_tracking'
+require 'longleaf/events/preserve_event'
+require 'longleaf/services/application_config_deserializer'
+require 'longleaf/candidates/file_selector'
+require 'longleaf/candidates/service_candidate_locator'
+require 'longleaf/events/event_names'
+require 'longleaf/logging'
+
+module Longleaf
+  # Command for preserving files
+  class PreserveCommand
+    include Longleaf::Logging
+    include Longleaf::EventStatusTracking
+
+    def initialize(app_manager)
+      @app_manager = app_manager
+    end
+
+    # Execute the preserve command on the given parameters
+    # @param file_selector [FileSelector] selector for files to preserve
+    # @param force [Boolean] force flag
+    # @return [Integer] status code
+    def execute(file_selector:, force: false)
+      start_time = Time.now
+      logger.info('Performing preserve command')
+      begin
+        # Perform preserve events on each of the file paths provided
+        candidate_locator = ServiceCandidateLocator.new(@app_manager)
+        candidate_it = candidate_locator.candidate_iterator(file_selector, EventNames::PRESERVE, force)
+        candidate_it.each do |file_rec|
+          begin
+            logger.debug("Selected candidate #{file_rec.path} for a preserve event")
+            preserve_event = PreserveEvent.new(file_rec: file_rec, force: force, app_manager: @app_manager)
+            track_status(preserve_event.perform)
+          rescue InvalidStoragePathError => e
+            record_failure(EventNames::PRESERVE, nil, e.message)
+          end
+        end
+      rescue LongleafError => e
+        record_failure(EventNames::PRESERVE, nil, e.message)
+      rescue => err
+        record_failure(EventNames::PRESERVE, error: err)
+      end
+
+      logger.info("Completed preserve command in #{Time.now - start_time}s")
+      return_status
+    end
+  end
+end

data/lib/longleaf/commands/register_command.rb

@@ -1,59 +1,50 @@
 require 'longleaf/services/application_config_deserializer'
 require 'longleaf/events/register_event'
 require 'longleaf/models/file_record'
-require 'longleaf/commands/abstract_command'
+require 'longleaf/events/event_names'
+require 'longleaf/events/event_status_tracking'
 
-# Command for registering files with longleaf
 module Longleaf
-  class RegisterCommand < AbstractCommand
-    
-    def initialize(config_path)
-      @config_path = config_path
+  # Command for registering files with longleaf
+  class RegisterCommand
+    include Longleaf::EventStatusTracking
+
+    def initialize(app_manager)
+      @app_manager = app_manager
     end
 
     # Execute the register command on the given parameters
-    def execute(file_paths: nil, force: false, checksums: nil)
-      if file_paths.nil? || file_paths.empty?
-        record_failure("Must provide one or more file paths to register")
-        return return_status
-      end
-      
+    # @param file_selector [FileSelector] selector for files to register
+    # @param force [Boolean] force flag
+    # @param digest_provider [ManifestDigestProvider] object which provides digests for files being registered
+    # @param physical_provider [PhysicalPathProvider] object which provides physical paths for files being registered
+    # @return [Integer] status code
+    def execute(file_selector:, force: false, digest_provider: nil, physical_provider: nil)
+      start_time = Time.now
+      logger.info('Performing register command')
       begin
-        # Retrieve the application configuration
-        app_manager = Longleaf::ApplicationConfigDeserializer.deserialize(@config_path)
-        
         # Perform register events on each of the file paths provided
-        file_paths.each do |f_path|
-          begin
-            storage_location = app_manager.location_manager.get_location_by_path(f_path)
-            if storage_location.nil?
-              raise InvalidStoragePathError.new(
-                  "Unable to register '#{f_path}', it does not belong to any registered storage locations.")
-            end
-          
-            raise InvalidStoragePathError.new("Unable to register '#{f_path}', file does not exist or is unreachable.") \
-                unless File.file?(f_path)
-          
-            file_rec = FileRecord.new(f_path, storage_location)
-            
-            register_event = RegisterEvent.new(file_rec: file_rec, force: force, app_manager: app_manager,
-                checksums: checksums)
-            register_event.perform
-            
-            record_success(RegisterEvent::EVENT_NAME, f_path)
-          rescue RegistrationError => err
-            record_failure(RegisterEvent::EVENT_NAME, f_path, err.message)
-          rescue InvalidStoragePathError => err
-            record_failure(RegisterEvent::EVENT_NAME, f_path, err.message)
-          end
+        loop do
+          f_path = file_selector.next_path
+          break if f_path.nil?
+
+          storage_location = @app_manager.location_manager.get_location_by_path(f_path)
+
+          phys_path = physical_provider.get_physical_path(f_path)
+          file_rec = FileRecord.new(f_path, storage_location, nil, phys_path)
+
+          register_event = RegisterEvent.new(file_rec: file_rec, force: force, app_manager: @app_manager,
+              digest_provider: digest_provider)
+          track_status(register_event.perform)
         end
-      rescue ConfigurationError => err
-        record_failure("Failed to load application configuration due to the following issue:\n#{err.message}")
+      rescue InvalidStoragePathError, StorageLocationUnavailableError => err
+        record_failure(EventNames::REGISTER, nil, err.message)
       rescue => err
-        record_failure(RegisterEvent::EVENT_NAME, error: err)
+        record_failure(EventNames::REGISTER, error: err)
       end
-      
+
+      logger.info("Completed register command in #{Time.now - start_time}s")
       return_status
     end
   end
-end
+end

data/lib/longleaf/commands/reindex_command.rb

@@ -0,0 +1,92 @@
+require 'longleaf/errors'
+require 'longleaf/events/event_status_tracking'
+require 'longleaf/logging'
+
+module Longleaf
+  # Command for reindexing metadata
+  class ReindexCommand
+    include Longleaf::Logging
+    include Longleaf::EventStatusTracking
+
+    def initialize(app_manager)
+      @app_manager = app_manager
+      @index_manager = @app_manager.index_manager
+    end
+
+    # Execute the reindex command
+    # @param only_if_stale [boolean] if true, then the reindex command will perform no operation unless the index is stale.
+    # @return [Integer] status code
+    def execute(only_if_stale: false)
+      if !@index_manager.using_index?
+        record_failure("Cannot perform reindex, no index is configured")
+        return return_status
+      end
+
+      if only_if_stale && !@index_manager.index_stale?
+        record_success("Index is not stale, performing no action")
+        return return_status
+      end
+
+      start_time = Time.now
+      logger.info('Performing full reindex')
+      results = nil
+      begin
+        start_time = Time.now.utc
+
+        selector = all_storage_locations_selector
+
+        # Repopulate the index
+        results = index_all(selector)
+
+        # List and then clear all files which were not reindexed
+        @index_manager.each_registered_path(selector, older_than: start_time) do |file_path|
+          logger.warn("Clearing '#{file_path}' from index, file is no longer present.")
+        end
+        @index_manager.clear_index(start_time)
+
+        # Update the state of the index to indicate it has been reindexed
+        @index_manager.update_index_state
+      rescue => err
+        record_failure("Encountered error while reindexing", error: err)
+      end
+
+      if results['fail'] > 0
+        record_success("Completed reindexing, #{results['success']} successful, #{results['fail']} failed.")
+      else
+        record_success("Completed reindexing, #{results['success']} successful.")
+      end
+
+      logger.info("Completed full reindex in #{Time.now - start_time}s")
+      return_status
+    end
+
+    private
+    def index_all(selector)
+      count = 0
+      failures = 0
+
+      selector.each do |file_path|
+        begin
+          storage_loc = @app_manager.location_manager.get_location_by_path(file_path)
+          file_rec = FileRecord.new(file_path, storage_loc)
+
+          @app_manager.md_manager.load(file_rec)
+          @index_manager.index(file_rec)
+
+          record_success("Reindexed #{file_rec.path}")
+          count += 1
+        rescue LongleafError => err
+          record_failure("Failed to reindex #{file_rec.path}: #{err.message}")
+          failures += 1
+        end
+      end
+      {'success' => count, 'fail' => failures}
+    end
+
+    def all_storage_locations_selector
+      storage_loc_names = @app_manager.location_manager.locations.keys
+
+      RegisteredFileSelector.new(storage_locations: storage_loc_names, app_config: @app_manager)
+    end
+  end
+end

data/lib/longleaf/commands/validate_config_command.rb

@@ -1,29 +1,54 @@
 require 'longleaf/services/application_config_deserializer'
-require 'longleaf/commands/abstract_command'
+require 'longleaf/events/event_status_tracking'
 
 module Longleaf
-  class ValidateConfigCommand < AbstractCommand
+  # Command for validating an application configuration file
+  class ValidateConfigCommand
+    include Longleaf::EventStatusTracking
+
     def initialize(config_path)
       @config_path = config_path
     end
-    
+
+    # Execute the validate command on the specified configuration yml file
     def execute
+      start_time = Time.now
+      logger.info('Performing validate configuration command')
       begin
         app_config_manager = Longleaf::ApplicationConfigDeserializer.deserialize(@config_path)
-        
+
         location_manager = app_config_manager.location_manager
         location_manager.locations.each do |name, location|
           location.available?
         end
-        
+
+        validate_services(app_config_manager.service_manager)
+
         record_success("Application configuration passed validation: #{@config_path}")
       rescue Longleaf::ConfigurationError, Longleaf::StorageLocationUnavailableError => err
-        record_failure("Application configuration invalid due to the following issue:\n#{err.message}")
+        record_failure("Application configuration invalid due to the following issue(s):\n#{err.message}")
       rescue => err
         record_failure("Failed to validate application configuration", error: err)
       end
-      
+
+      logger.info("Completed validate configuration command in #{Time.now - start_time}s")
       return_status
     end
+
+    private
+    # Verify that all defined services are valid and may be instantiated with the given configuration,
+    # according to internal expectations.
+    # @raise ConfigurationError if any services may not be instantiated
+    def validate_services(service_manager)
+      def_manager = service_manager.definition_manager
+
+      def_manager.services.each do |service_name, service_def|
+        begin
+          service_manager.service(service_name)
+        rescue => e
+          raise ConfigurationError.new(e.message)
+        end
+      end
+    end
   end
-end
+end

data/lib/longleaf/commands/validate_metadata_command.rb

@@ -0,0 +1,51 @@
+require 'longleaf/services/application_config_deserializer'
+require 'longleaf/models/file_record'
+require 'longleaf/events/event_status_tracking'
+require 'longleaf/errors'
+
+module Longleaf
+  # Command for validating file metadata longleaf
+  class ValidateMetadataCommand
+    include Longleaf::EventStatusTracking
+
+    def initialize(app_manager)
+      @app_manager = app_manager
+    end
+
+    # Execute the validation command
+    # @param file_selector [FileSelector] selector for files to register
+    # @return [Integer] status code
+    def execute(file_selector:)
+      start_time = Time.now
+      logger.info('Performing validate metadata command')
+      begin
+        # Perform metadata validation on each of the file paths provided
+        loop do
+          f_path = file_selector.next_path
+          break if f_path.nil?
+
+          storage_location = @app_manager.location_manager.get_location_by_path(f_path)
+
+          begin
+            file_rec = FileRecord.new(f_path, storage_location)
+            unless file_rec.metadata_present?
+              raise MetadataError.new("Cannot validate metadata for #{f_path}, file is not registered.")
+            end
+
+            @app_manager.md_manager.load(file_rec)
+            record_success("Metadata for file passed validation: #{f_path}")
+          rescue LongleafError => err
+            record_failure(err.message)
+          end
+        end
+      rescue RegistrationError, InvalidStoragePathError, StorageLocationUnavailableError => err
+        record_failure(err.message)
+      rescue => err
+        record_failure("Encountered error while validating metadata files", error: err)
+      end
+
+      logger.info("Completed validate metadata command in #{Time.now - start_time}s")
+      return_status
+    end
+  end
+end

data/lib/longleaf/errors.rb

@@ -1,15 +1,34 @@
 module Longleaf
+  # General Longleaf error
   class LongleafError < StandardError; end
-  
+
+  # Invalid application configuration error
   class ConfigurationError < LongleafError; end
-  
+
+  # Invalid storage path error
   class InvalidStoragePathError < LongleafError; end
-  
+
+  # Metadata does not meet requirements error
   class MetadataError < LongleafError; end
-  
+
+  # Unavailable storage location error
   class StorageLocationUnavailableError < LongleafError; end
-  
+
+  # Error related to executing a preservation event
   class EventError < LongleafError; end
-  
+
+  # Error with the registration state of a file or while attempting to perform a registration event
   class RegistrationError < EventError; end
-end
+
+  # Error while attempting to perform a deregistration event
+  class DeregistrationError < EventError; end
+
+  # Error while performing a preservation service
+  class PreservationServiceError < LongleafError; end
+
+  # Fixity check failure error
+  class ChecksumMismatchError < PreservationServiceError; end
+
+  # Error indicating an unknown or invalid digest algorithm was specified
+  class InvalidDigestAlgorithmError < LongleafError; end
+end