longleaf 0.2.0.pre.1 → 0.3.0

data/lib/longleaf/candidates/service_candidate_locator.rb

@@ -1,4 +1,5 @@
 require 'longleaf/candidates/service_candidate_filesystem_iterator'
+require 'longleaf/candidates/service_candidate_index_iterator'
 
 module Longleaf
   # Service which locates files that have services which need to be performed on them.
@@ -6,13 +7,17 @@ module Longleaf
     def initialize(app_config)
       @app_config = app_config
     end
-    
+
     # Get a iterator of the candidates matching the given FileSelector which need services run.
     # @param file_selector [FileSelector] selector identifying the files to pull candidates from.
     # @return an iterator
     def candidate_iterator(file_selector, event, force = false)
-      # Get filesystem based implementation
-      ServiceCandidateFilesystemIterator.new(file_selector, event, @app_config, force)
+      if @app_config.index_manager.using_index?
+        ServiceCandidateIndexIterator.new(file_selector, event, @app_config, force)
+      else
+        # Get filesystem based implementation
+        ServiceCandidateFilesystemIterator.new(file_selector, event, @app_config, force)
+      end
     end
   end
-end
+end

data/lib/longleaf/cli.rb

@@ -2,55 +2,101 @@ require 'thor'
 require 'yaml'
 require 'longleaf/logging'
 require 'longleaf/errors'
+require 'longleaf/version'
 require 'longleaf/commands/deregister_command'
 require 'longleaf/commands/validate_config_command'
 require 'longleaf/commands/validate_metadata_command'
 require 'longleaf/commands/register_command'
+require 'longleaf/commands/reindex_command'
 require 'longleaf/commands/preserve_command'
 require 'longleaf/candidates/file_selector'
+require 'longleaf/candidates/registered_file_selector'
 
 module Longleaf
   # Main commandline interface setup for Longleaf using Thor.
   class CLI < Thor
     include Longleaf::Logging
-    
-    class_option(:config, :aliases => "-c",
-        :default => ENV['LONGLEAF_CFG'],
-        :required => true,
-        :desc => 'Absolute path to the application configuration used for this command. By default, the value of the environment variable LONGLEAF_CFG is used.')
-    class_option(:load_path, :aliases => "-I",
-        :desc => 'Specify comma seperated directories to add to the $LOAD_PATH, which can be used to specify additional paths from which to load preservation services.')
-    # Logging/output options
-    class_option(:failure_only,
-        :type => :boolean,
-        :default => false,
-        :desc => 'Only display failure messages to STDOUT.')
-    class_option(:log_level,
-        :default => 'WARN',
-        :desc => 'Level of logging to send to STDERR, following standard ruby Logger levels. This includes: DEBUG, INFO, WARN, ERROR, FATAL, UNKNOWN. Default is WARN.')
-    class_option(:log_format,
-        :desc => 'Format to use for log information sent to STDERR. Can contain the following parameters, which must be wrapped in %{}: severity, datetime, progname, msg. Default is "%{severity} [%{datetime}]: %{msg}"')
-    class_option(:log_datetime,
-        :desc => 'Format to use for timestamps used in logging to STDERR, following strftime syntax.')
-    
+
+    # Register a shared method option in a shared option group
+    def self.add_shared_option(name, group, options = {})
+      @shared_groups = {} if @shared_groups.nil?
+      @shared_groups[group] = {} if @shared_groups[group].nil?
+      @shared_groups[group][name] = options
+    end
+
+    # Add all of the shared options in the specified group as method options
+    def self.shared_options_group(group_name)
+      @shared_groups[group_name].each do |opt_name, opt|
+        option opt_name, opt
+      end
+    end
+
+    # Config options
+    add_shared_option(
+        :config, :common, {
+              :aliases => "-c",
+              :default => ENV['LONGLEAF_CFG'],
+              :required => false,
+              :desc => 'Path to the application configuration used for this command. By default, the value of the environment variable LONGLEAF_CFG is used. A config path is required for most commands.' })
+    add_shared_option(
+        :load_path, :common, {
+              :aliases => "-I",
+              :desc => 'Specify comma seperated directories to add to the $LOAD_PATH, which can be used to specify additional paths from which to load preservation services.' })
+
+    # Logging options
+    add_shared_option(
+        :failure_only, :common, {
+              :type => :boolean,
+              :default => false,
+              :desc => 'Only display failure messages to STDOUT.' })
+    add_shared_option(
+        :log_level, :common, {
+              :default => 'WARN',
+              :desc => 'Level of logging to send to STDERR, following standard ruby Logger levels. This includes: DEBUG, INFO, WARN, ERROR, FATAL, UNKNOWN.' })
+    add_shared_option(
+        :log_format, :common, {
+              :desc => 'Format to use for log information sent to STDERR. Can contain the following parameters, which must be wrapped in %{}: severity, datetime, progname, msg. Default is "%{severity} [%{datetime}]: %{msg}"' })
+    add_shared_option(
+        :log_datetime, :common, {
+              :desc => 'Format to use for timestamps used in logging to STDERR, following strftime syntax.' })
+
+    # File selection options
+    add_shared_option(
+        :file, :file_selection, {
+              :aliases => "-f",
+              :required => false,
+              :desc => 'File or files to perform this operation on. If multiple files are provided, they must be comma separated.' })
+    add_shared_option(
+        :location, :file_selection, {
+              :aliases => "-s",
+              :required => false,
+              :desc => 'Name or comma separated names of storage locations to perform this operation over.' })
+
+    # Commands
+    map %w[--version] => :__print_version
+    desc "--version", "Prints the Longleaf version number."
+    def __print_version
+      puts "longleaf version #{Longleaf::VERSION}"
+    end
+
     desc "register", "Register files with Longleaf"
-    method_option(:file, :aliases => "-f", 
-        :required => true,
-        :desc => 'File or files to register. Paths must be absolute. If multiple files are provided, they must be comma separated.')
+    shared_options_group(:file_selection)
     method_option(:force,
-        :type => :boolean, 
+        :type => :boolean,
         :default => false,
         :desc => 'Force the registration of already registered files.')
     method_option(:checksums,
-        :desc => %q{Checksums for the submitted file. Each checksum must be prefaced with an algorithm prefix. Multiple checksums must be comma separated. If multiple files were submitted, they will be provided with the same checksums. For example: 
+        :desc => %q{Checksums for the submitted file. Each checksum must be prefaced with an algorithm prefix. Multiple checksums must be comma separated. If multiple files were submitted, they will be provided with the same checksums. For example:
           '--checksums "md5:d8e8fca2dc0f896fd7cb4cb0031ba249,sha1:4e1243bd22c66e76c2ba9eddc1f91394e57f9f83"'})
+    shared_options_group(:common)
     # Register event command
     def register
+      verify_config_provided(options)
       setup_logger(options)
-      
-      config_path = options[:config]
-      app_config_manager = load_application_config(options[:config])
-      file_selector = create_file_selector(options[:file], nil, app_config_manager)
+
+      app_config_manager = load_application_config(options)
+
+      file_selector = create_file_selector(options, app_config_manager)
       if options[:checksums]
         checksums = options[:checksums]
         # validate checksum list format, must a comma delimited list of prefix:checksums
@@ -62,78 +108,106 @@ module Longleaf
           exit 1
         end
       end
-      
+
       command = RegisterCommand.new(app_config_manager)
       exit command.execute(file_selector: file_selector, force: options[:force], checksums: checksums)
     end
-    
+
     desc "deregister", "Deregister files with Longleaf"
-    method_option(:file, :aliases => "-f", 
-        :required => true,
-        :desc => 'File or files to deregister. Paths must be absolute. If multiple files are provided, they must be comma separated.')
+    shared_options_group(:file_selection)
     method_option(:force,
-        :type => :boolean, 
+        :type => :boolean,
         :default => false,
         :desc => 'Force the deregistration of already deregistered files.')
+    shared_options_group(:common)
     # Deregister event command
     def deregister
+      verify_config_provided(options)
       setup_logger(options)
-      
-      config_path = options[:config]
-      app_config_manager = load_application_config(options[:config])
-      file_selector = create_file_selector(options[:file], nil, app_config_manager)
-      
+
+      app_config_manager = load_application_config(options)
+      file_selector = create_registered_selector(options, app_config_manager)
+
       command = DeregisterCommand.new(app_config_manager)
       exit command.execute(file_selector: file_selector, force: options[:force])
     end
-    
+
     desc "preserve", "Perform preservation services on files with Longleaf"
-    method_option(:file, :aliases => "-f", 
-        :required => false,
-        :desc => 'File or files to preserve. Paths must be absolute. If multiple files are provided, they must be comma separated.')
-    method_option(:location, :aliases => "-s",
-        :required => false,
-        :desc => 'Name or comma separated names of storage locations to preserve.')
+    shared_options_group(:file_selection)
     method_option(:force,
-        :type => :boolean, 
+        :type => :boolean,
         :default => false,
         :desc => 'Force the execution of preservation services, disregarding scheduling information.')
+    shared_options_group(:common)
     def preserve
+      verify_config_provided(options)
       setup_logger(options)
-      
+
       extend_load_path(options[:load_path])
-      app_config_manager = load_application_config(options[:config])
-      file_selector = create_file_selector(options[:file], options[:location], app_config_manager)
-      
+      app_config_manager = load_application_config(options)
+      file_selector = create_registered_selector(options, app_config_manager)
+
       command = PreserveCommand.new(app_config_manager)
       exit command.execute(file_selector: file_selector, force: options[:force])
     end
-    
+
     desc "validate_config", "Validate an application configuration file, provided using --config."
+    shared_options_group(:common)
     # Application configuration validation command
     def validate_config
+      verify_config_provided(options)
       setup_logger(options)
-      
+      extend_load_path(options[:load_path])
+
       exit Longleaf::ValidateConfigCommand.new(options[:config]).execute
     end
-    
+
     desc "validate_metadata", "Validate metadata files."
-    method_option(:file, :aliases => "-f", 
-        :required => false,
-        :desc => 'File or files of which to validate the metadata. Paths must be absolute. If multiple files are provided, they must be comma separated.')
-    method_option(:location, :aliases => "-s",
-        :required => false,
-        :desc => 'Name or comma separated names of storage locations to validate.')
+    shared_options_group(:file_selection)
+    shared_options_group(:common)
     # File metadata validation command
     def validate_metadata
+      verify_config_provided(options)
       setup_logger(options)
-      
-      app_config_manager = load_application_config(options[:config])
-      file_selector = create_file_selector(options[:file], options[:location], app_config_manager)
-      
+
+      app_config_manager = load_application_config(options)
+      file_selector = create_registered_selector(options, app_config_manager)
+
       exit Longleaf::ValidateMetadataCommand.new(app_config_manager).execute(file_selector: file_selector)
     end
-    
+
+    desc "setup_index", "Sets up the structure of the metadata index, if one is configured using the system configuration file provided using the --system_config option. Some index types may require additional steps to be taken by an administrator before hand, such as creating users and databases."
+    shared_options_group(:common)
+    def setup_index
+      verify_config_provided(options)
+      setup_logger(options)
+
+      app_config_manager = load_application_config(options)
+
+      if app_config_manager.index_manager.using_index?
+        app_config_manager.index_manager.setup_index
+        logger.success("Setup of index complete")
+        exit 0
+      else
+        logger.failure("No index configured, unable to perform setup.")
+        exit 1
+      end
+    end
+
+    desc "reindex", "Perform a full reindex of file metadata stored within the configured storage locations."
+    method_option(:if_stale,
+        :type => :boolean,
+        :default => false,
+        :desc => 'Only perform the reindex if the index is known to be stale, generally after an config file change.')
+    shared_options_group(:common)
+    def reindex
+      verify_config_provided(options)
+      setup_logger(options)
+      app_config_manager = load_application_config(options)
+
+      exit Longleaf::ReindexCommand.new(app_config_manager).execute(only_if_stale: options[:if_stale])
+    end
+
     no_commands do
       def setup_logger(options)
         initialize_logger(options[:failure_only],
@@ -141,34 +215,42 @@ module Longleaf
             options[:log_format],
             options[:log_datetime])
       end
-      
-      def load_application_config(config_path)
+
+      def load_application_config(options)
         begin
-          app_manager = ApplicationConfigDeserializer.deserialize(config_path)
+          app_manager = ApplicationConfigDeserializer.deserialize(options[:config])
         rescue ConfigurationError => err
           logger.failure("Failed to load application configuration due to the following issue:\n#{err.message}")
           exit 1
         end
       end
-      
-      def create_file_selector(file_paths, storage_locations, app_config_manager)
-        file_paths = file_paths&.split(/\s*,\s*/)
-        storage_locations = storage_locations&.split(/\s*,\s*/)
-        
+
+      def verify_config_provided(options)
+        if options[:config].nil? || options[:config].empty?
+          raise "No value provided for required options '--config'"
+        end
+      end
+
+      def create_file_selector(options, app_config_manager, selector_class: FileSelector)
+        file_paths = options[:file]&.split(/\s*,\s*/)
+        storage_locations = options[:location]&.split(/\s*,\s*/)
+
         begin
-          FileSelector.new(file_paths: file_paths, storage_locations: storage_locations, app_config: app_config_manager)
+          selector_class.new(file_paths: file_paths, storage_locations: storage_locations, app_config: app_config_manager)
         rescue ArgumentError => e
           logger.failure(e.message)
           exit 1
         end
       end
-      
+
+      def create_registered_selector(options, app_config_manager)
+        create_file_selector(options, app_config_manager, selector_class: RegisteredFileSelector)
+      end
+
       def extend_load_path(load_paths)
         load_paths = load_paths&.split(/\s*,\s*/)
-        if !load_paths.nil?
-          load_paths.each { |path| $LOAD_PATH.unshift(path) }
-        end
+        load_paths&.each { |path| $LOAD_PATH.unshift(path) }
       end
     end
   end
-end
+end

data/lib/longleaf/commands/deregister_command.rb

@@ -3,13 +3,12 @@ require 'longleaf/events/deregister_event'
 require 'longleaf/models/file_record'
 require 'longleaf/events/event_names'
 require 'longleaf/events/event_status_tracking'
-require 'longleaf/services/metadata_deserializer'
 
 module Longleaf
   # Command for deregistering files with longleaf
   class DeregisterCommand
     include Longleaf::EventStatusTracking
-    
+
     def initialize(app_manager)
       @app_manager = app_manager
     end
@@ -19,32 +18,34 @@ module Longleaf
     # @param force [Boolean] force flag
     # @return [Integer] status code
     def execute(file_selector:, force: false)
+      start_time = Time.now
+      logger.info('Performing deregister command')
       begin
         # Perform deregister events on each of the file paths provided
         loop do
           f_path = file_selector.next_path
           break if f_path.nil?
-          
+
           storage_location = @app_manager.location_manager.get_location_by_path(f_path)
-        
+
           file_rec = FileRecord.new(f_path, storage_location)
           unless file_rec.metadata_present?
             raise DeregistrationError.new("Cannot deregister #{f_path}, file is not registered.")
           end
-          
-          file_rec.metadata_record = MetadataDeserializer.deserialize(file_path: file_rec.metadata_path,
-              digest_algs: storage_location.metadata_digests)
-          
+
+          @app_manager.md_manager.load(file_rec)
+
           event = DeregisterEvent.new(file_rec: file_rec, force: force, app_manager: @app_manager)
           track_status(event.perform)
         end
-      rescue DeregistrationError, InvalidStoragePathError, StorageLocationUnavailableError => err
+      rescue RegistrationError, DeregistrationError, InvalidStoragePathError, StorageLocationUnavailableError => err
         record_failure(EventNames::DEREGISTER, nil, err.message)
       rescue => err
         record_failure(EventNames::DEREGISTER, error: err)
       end
-      
+
+      logger.info("Completed deregister command in #{Time.now - start_time}s")
       return_status
     end
   end
-end
+end

data/lib/longleaf/commands/preserve_command.rb

@@ -12,33 +12,38 @@ module Longleaf
   class PreserveCommand
     include Longleaf::Logging
     include Longleaf::EventStatusTracking
-    
+
     def initialize(app_manager)
       @app_manager = app_manager
     end
-    
+
     # Execute the preserve command on the given parameters
     # @param file_selector [FileSelector] selector for files to preserve
     # @param force [Boolean] force flag
     # @return [Integer] status code
     def execute(file_selector:, force: false)
+      start_time = Time.now
+      logger.info('Performing preserve command')
       begin
         # Perform preserve events on each of the file paths provided
         candidate_locator = ServiceCandidateLocator.new(@app_manager)
         candidate_it = candidate_locator.candidate_iterator(file_selector, EventNames::PRESERVE, force)
         candidate_it.each do |file_rec|
-          f_path = file_rec.path
-          
-          logger.debug("Selected candidate #{file_rec.path} for a preserve event")
-          preserve_event = PreserveEvent.new(file_rec: file_rec, force: force, app_manager: @app_manager)
-          track_status(preserve_event.perform)
+          begin
+            logger.debug("Selected candidate #{file_rec.path} for a preserve event")
+            preserve_event = PreserveEvent.new(file_rec: file_rec, force: force, app_manager: @app_manager)
+            track_status(preserve_event.perform)
+          rescue InvalidStoragePathError => e
+            record_failure(EventNames::PRESERVE, nil, e.message)
+          end
         end
       rescue LongleafError => e
         record_failure(EventNames::PRESERVE, nil, e.message)
       rescue => err
         record_failure(EventNames::PRESERVE, error: err)
       end
-      
+
+      logger.info("Completed preserve command in #{Time.now - start_time}s")
       return_status
     end
   end