longleaf 0.2.0.pre.1 → 0.3.0

data/lib/longleaf/preservation_services/fixity_check_service.rb

@@ -15,9 +15,9 @@ module Longleaf
   #   * 'generate' - the service will generate and store any missing digests from the set of configured algorithms.
   class FixityCheckService
     include Longleaf::Logging
-    
+
     SUPPORTED_ALGORITHMS = ['md5', 'sha1', 'sha2', 'sha256', 'sha384', 'sha512', 'rmd160']
-    
+
     # service configuration property indicating how to handle situations where a file does not
     # have a digest for one of the expected algorithms on record.
     ABSENT_DIGEST_PROPERTY = 'absent_digest'
@@ -25,7 +25,7 @@ module Longleaf
     GENERATE_IF_ABSENT = 'generate'
     IGNORE_IF_ABSENT = 'ignore'
     ABSENT_DIGEST_OPTIONS = [FAIL_IF_ABSENT, GENERATE_IF_ABSENT, IGNORE_IF_ABSENT]
-    
+
     # Initialize a FixityCheckService from the given service definition
     #
     # @param service_def [ServiceDefinition] the configuration for this service
@@ -36,12 +36,14 @@ module Longleaf
       unless ABSENT_DIGEST_OPTIONS.include?(@absent_digest_behavior)
         raise ArgumentError.new("Invalid option '#{@absent_digest_behavior}' for property #{ABSENT_DIGEST_PROPERTY} in service #{service_def.name}")
       end
-      
+
       service_algs = service_def.properties[ServiceFields::DIGEST_ALGORITHMS]
       if service_algs.nil? || service_algs.empty?
         raise ArgumentError.new("FixityCheckService from definition #{service_def.name} requires a list of one or more digest algorithms")
       end
-      
+
+      service_algs = [service_algs] if service_algs.is_a?(String)
+
       # Store the list of digest algorithms to verify, using normalized algorithm names.
       @digest_algs = Set.new
       service_algs.each do |alg|
@@ -49,11 +51,11 @@ module Longleaf
         if SUPPORTED_ALGORITHMS.include?(normalized_alg)
           @digest_algs << normalized_alg
         else
-          raise ArgumentError.new("Unsupported checksum algorithm '#{alg}' in definition #{service_def.name}. Supported algorithms are: #{SUPPORTED_ALGORITHMS.to_s}")
+          raise ArgumentError.new("Unsupported checksum algorithm '#{alg}' in definition #{service_def.name}. Supported algorithms are: #{SUPPORTED_ALGORITHMS}")
         end
       end
     end
-    
+
     # Perform all configured fixity checks on the provided file
     #
     # @param file_rec [FileRecord] record representing the file to perform the service on.
@@ -62,7 +64,7 @@ module Longleaf
     def perform(file_rec, event)
       path = file_rec.path
       md_rec = file_rec.metadata_record
-      
+
       # Get the list of existing checksums for the file and normalize algorithm names
       file_digests = Hash.new
       md_rec.checksums&.each do |alg, digest|
@@ -73,10 +75,10 @@ module Longleaf
           logger.debug("Metadata for file #{path} contains unexpected '#{alg}' digest, it will be ignored.")
         end
       end
-      
+
       @digest_algs.each do |alg|
         existing_digest = file_digests[alg]
-        
+
         if existing_digest.nil?
           if @absent_digest_behavior == FAIL_IF_ABSENT
             raise ChecksumMismatchError.new("Fixity check using algorithm '#{alg}' failed for file #{path}: no existing digest of type '#{alg}' on record.")
@@ -85,11 +87,11 @@ module Longleaf
             next
           end
         end
-        
+
         digest = DigestHelper::start_digest(alg)
         digest.file(path)
         generated_digest = digest.hexdigest
-        
+
         # Store the missing checksum if using the 'generate' behavior
         if existing_digest.nil? && @absent_digest_behavior == GENERATE_IF_ABSENT
           md_rec.checksums[alg] = generated_digest
@@ -104,7 +106,7 @@ module Longleaf
         end
       end
     end
-    
+
     # Determine if this service is applicable for the provided event, given the configured service definition
     #
     # @param event [String] name of the event
@@ -118,4 +120,4 @@ module Longleaf
       end
     end
   end
-end
+end

data/lib/longleaf/preservation_services/rsync_replication_service.rb

@@ -17,58 +17,59 @@ module Longleaf
   #     a file which already exists at a destination. Default: "replace".
   # * rsync_command = the command to invoke in order to execute rsync. Default: "rsync"
   # * rsync_options = additional parameters that will be passed along to rsync. Cannot include options
-  #     which change the target of the command or prevent its execution, such as "files-from", "dry-run", 
+  #     which change the target of the command or prevent its execution, such as "files-from", "dry-run",
   #     "help", etc. Command will always include "-R". Default "-a".
   class RsyncReplicationService
     include Longleaf::Logging
-    
+
     COLLISION_PROPERTY = "replica_collision_policy"
     DEFAULT_COLLISION_POLICY = "replace"
     VALID_COLLISION_POLICIES = ["replace"]
-    
+
     RSYNC_COMMAND_PROPERTY = "rsync_command"
     DEFAULT_COMMAND = "rsync"
-    
+
     RSYNC_OPTIONS_PROPERTY = "rsync_options"
     DEFAULT_OPTIONS = "-a"
     DISALLOWED_OPTIONS = ["files-from", "n", "dry-run", "exclude", "exclude-from", "cvs-exclude",
        "h", "help", "f", "F", "filter"]
-       
+
     attr_reader :command, :options, :collision_policy
-    
+
     # Initialize a RsyncReplicationService from the given service definition
     #
     # @param service_def [ServiceDefinition] the configuration for this service
-    # @param location_manager [StorageLocationManager] manager for configured storage locations
+    # @param app_manager [ApplicationConfigManager] the application configuration
     def initialize(service_def, app_manager)
       @service_def = service_def
       @app_manager = app_manager
-      
+
       @command = @service_def.properties[RSYNC_COMMAND_PROPERTY] || DEFAULT_COMMAND
-      
+
       # Validate rsync parameters
       @options = @service_def.properties[RSYNC_OPTIONS_PROPERTY] || DEFAULT_OPTIONS
       if contains_disallowed_option?(@options)
         raise ArgumentError.new("Service #{service_def.name} specifies a disallowed rsync paramter," \
-            + " rsync_options may not include the following: #{DISALLOWED_OPTIONS.join(" ")}")
+            + " rsync_options may not include the following: #{DISALLOWED_OPTIONS.join(' ')}")
       end
-      
+
       # Add -R (--relative) in to command options to ensure full path gets replicated
       @options = @options + " -R"
-      
+
       # Set and validate the replica collision policy
       @collision_policy = @service_def.properties[COLLISION_PROPERTY] || DEFAULT_COLLISION_POLICY
       if !VALID_COLLISION_POLICIES.include?(@collision_policy)
         raise ArgumentError.new("Service #{service_def.name} received invalid #{COLLISION_PROPERTY}" \
             + " value #{collision_policy}")
       end
-      
+
       # Store and validate destinations
       replicate_to = @service_def.properties[ServiceFields::REPLICATE_TO]
       if replicate_to.nil? || replicate_to.empty?
         raise ArgumentError.new("Service #{service_def.name} must provide one or more replication destinations.")
       end
-      
+      replicate_to = [replicate_to] if replicate_to.is_a?(String)
+
       loc_manager = app_manager.location_manager
       # Build list of destinations, translating to storage locations when relevant
       @destinations = Array.new
@@ -86,7 +87,7 @@ module Longleaf
         end
       end
     end
-    
+
     # During a replication event, perform replication of the specified file to all configured destinations
     # as necessary.
     #
@@ -96,35 +97,35 @@ module Longleaf
     def perform(file_rec, event)
       @destinations.each do |destination|
         dest_is_storage_loc = destination.is_a?(Longleaf::StorageLocation)
-        
+
         if dest_is_storage_loc
           dest_path = destination.path
         else
           dest_path = destination
         end
-        
+
         # Determine the path to the file being replicated relative to its storage location
         rel_path = file_rec.path.sub(/\A#{file_rec.storage_location.path}/, "")
         # source path with . so that rsync will only create destination directories starting from that point
         source_path = File.join(file_rec.storage_location.path, "./#{rel_path}")
-        
+
         # Check that the destination is available because attempting to write
         verify_destination_available(destination, file_rec)
-        
+
         logger.debug("Invoking rsync with command: #{@command} \"#{source_path}\" \"#{dest_path}\" #{@options}")
         stdout, stderr, status = Open3.capture3("#{@command} \"#{source_path}\" \"#{dest_path}\" #{@options}")
         raise PreservationServiceError.new("Failed to replicate #{file_rec.path} to #{dest_path}: #{stderr}") \
             unless status.success?
-              
+
         logger.info("Replicated #{file_rec.path} to destination #{dest_path}")
-        
+
         # For destinations which are storage locations, register the replica with longleaf
         if dest_is_storage_loc
           register_replica(destination, rel_path, file_rec)
         end
       end
     end
-    
+
     # Determine if this service is applicable for the provided event, given the configured service definition
     #
     # @param event [String] name of the event
@@ -137,7 +138,7 @@ module Longleaf
         false
       end
     end
-    
+
     private
     def contains_disallowed_option?(options)
       DISALLOWED_OPTIONS.each do |disallowed|
@@ -151,33 +152,33 @@ module Longleaf
           end
         end
       end
-      
+
       false
     end
-    
+
     def verify_destination_available(destination, file_rec)
       if destination.is_a?(Longleaf::StorageLocation)
         begin
           destination.available?
         rescue StorageLocationUnavailableError => e
-          raise StorageLocationUnavailableError.new("Cannot replicate #{file_rec.path} to destination" \
-              + " storage location #{destination.name}, it is unavailable.") unless destination.available?
+          raise StorageLocationUnavailableError.new("Cannot replicate #{file_rec.path} to destination #{destination.name}: " \
+              + e.message)
         end
       elsif destination.start_with?("/")
         raise StorageLocationUnavailableError.new("Cannot replicate #{file_rec.path} to destination" \
             + " #{destination}, path does not exist.") unless Dir.exist?(destination)
       end
     end
-    
+
     def register_replica(destination, rel_path, file_rec)
       dest_file_path = File.join(destination.path, rel_path)
       dest_file_rec = FileRecord.new(dest_file_path, destination)
-      
+
       register_event = RegisterEvent.new(file_rec: dest_file_rec,
           app_manager: @app_manager,
-          force: true, 
+          force: true,
           checksums: file_rec.metadata_record.checksums)
       register_event.perform
     end
   end
-end
+end

data/lib/longleaf/services/application_config_deserializer.rb

@@ -1,44 +1,81 @@
 require 'longleaf/services/application_config_validator'
 require 'longleaf/services/application_config_manager'
+require 'digest/md5'
+require 'pathname'
 
 module Longleaf
   # Deserializer for application configuration files
   class ApplicationConfigDeserializer
-    
+    AF ||= Longleaf::AppFields
+
     # Deserializes a valid application configuration file as a ApplicationConfigManager option
-    # @param config_path [String] file path to the application configuration file
+    # @param config_path [String] file path to the service and storage mapping configuration file
     # @param format [String] encoding format of the config file
     # return [ApplicationConfigManager] manager for the loaded configuration
     def self.deserialize(config_path, format: 'yaml')
-      config = load(config_path, format: format)
-      
+      content = load_config_file(config_path)
+      config = load(content, format)
+
+      config_md5 = Digest::MD5.hexdigest(content)
+
+      make_paths_absolute(config_path, config)
       Longleaf::ApplicationConfigValidator.validate(config)
-      Longleaf::ApplicationConfigManager.new(config)
+      Longleaf::ApplicationConfigManager.new(config, config_md5)
+    end
+
+    def self.load_config_file(config_path)
+      begin
+        File.read(config_path)
+      rescue Errno::ENOENT
+        raise Longleaf::ConfigurationError.new(
+            "Configuration file #{config_path} does not exist.")
+      end
     end
-    
+
     # Deserialize a configuration file into a hash
-    # @param config_path [String] file path to the application configuration file
+    # @param content [String] the contents of the application configuration file
     # @param format [String] encoding format of the config file
     # return [Hash] hash containing the configuration
-    def self.load(config_path, format: 'yaml')
+    def self.load(content, format)
       case format
       when 'yaml'
-        from_yaml(config_path)
+        from_yaml(content)
       else
-        raise ArgumentError.new('Invalid deserialization format #{format} specified')
+        raise ArgumentError.new("Invalid deserialization format #{format} specified")
       end
     end
-    
-    private
-    def self.from_yaml(config_path)
+
+    def self.from_yaml(content)
       begin
-        YAML.load_file(config_path)
-      rescue Errno::ENOENT => err
-        raise Longleaf::ConfigurationError.new(
-            "Configuration file #{config_path} does not exist.")
+        YAML.safe_load(content, [], [], true)
       rescue => err
         raise Longleaf::ConfigurationError.new(err)
       end
     end
+
+    def self.make_paths_absolute(config_path, config)
+      base_pathname = Pathname.new(config_path).expand_path.parent
+
+      config[AF::LOCATIONS].each do |name, properties|
+        properties[AF::LOCATION_PATH] = absolution(base_pathname, properties[AF::LOCATION_PATH])
+
+        properties[AF::METADATA_PATH] = absolution(base_pathname, properties[AF::METADATA_PATH])
+      end
+    end
+
+    def self.absolution(base_pathname, file_path)
+      if file_path.nil?
+        nil
+      else
+        path = Pathname.new(file_path)
+        if path.absolute?
+          path = path.expand_path.to_s
+        else
+          path = (base_pathname + path).to_s
+        end
+      end
+    end
+
+    private_class_method :load_config_file
   end
-end
+end

data/lib/longleaf/services/application_config_manager.rb

@@ -5,22 +5,34 @@ require_relative 'service_definition_manager'
 require_relative 'service_mapping_validator'
 require_relative 'service_mapping_manager'
 require_relative 'service_manager'
+require_relative 'metadata_persistence_manager'
+require 'longleaf/indexing/index_manager'
+require 'longleaf/models/app_fields'
 
 module Longleaf
   # Manager which loads and provides access to the configuration of the application
   class ApplicationConfigManager
+    attr_reader :config_md5
     attr_reader :service_manager
     attr_reader :location_manager
-    
-    def initialize(config)
+    attr_reader :index_manager
+    attr_reader :md_manager
+
+    def initialize(config, config_md5 = nil)
+      @config_md5 = config_md5
+
       @location_manager = Longleaf::StorageLocationManager.new(config)
-      
+
       definition_manager = Longleaf::ServiceDefinitionManager.new(config)
       mapping_manager = Longleaf::ServiceMappingManager.new(config)
       @service_manager = Longleaf::ServiceManager.new(
           definition_manager: definition_manager,
           mapping_manager: mapping_manager,
           app_manager: self)
+
+      sys_config = config[AppFields::SYSTEM]
+      @index_manager = IndexManager.new(sys_config, self)
+      @md_manager = MetadataPersistenceManager.new(@index_manager)
     end
   end
-end
+end

data/lib/longleaf/services/application_config_validator.rb

@@ -5,7 +5,6 @@ require_relative 'service_mapping_validator'
 module Longleaf
   # Validator for Longleaf application configuration
   class ApplicationConfigValidator
-    
     # Validates the application configuration provided. Will raise ConfigurationError
     # if any portion of the configuration is not syntactically or semantically valid.
     # @param config [Hash] application configuration
@@ -15,4 +14,4 @@ module Longleaf
       Longleaf::ServiceMappingValidator::validate_config(config)
     end
   end
-end
+end

data/lib/longleaf/services/configuration_validator.rb

@@ -1,9 +1,11 @@
 module Longleaf
   # Abstract configuration validator class
   class ConfigurationValidator
-    protected
-    def self.assert(fail_message, assertion_passed)
-      raise ConfigurationError.new(fail_message) unless assertion_passed
+    class << self
+      protected
+      def assert(fail_message, assertion_passed)
+        raise ConfigurationError.new(fail_message) unless assertion_passed
+      end
     end
   end
-end
+end

data/lib/longleaf/services/metadata_deserializer.rb

@@ -9,7 +9,7 @@ module Longleaf
   class MetadataDeserializer
     extend Longleaf::Logging
     MDF ||= MDFields
-    
+
     # Deserialize a file into a MetadataRecord object
     #
     # @param file_path [String] path of the file to read. Required.
@@ -19,13 +19,13 @@ module Longleaf
       when 'yaml'
         md = from_yaml(file_path, digest_algs)
       else
-        raise ArgumentError.new('Invalid deserialization format #{format} specified')
+        raise ArgumentError.new("Invalid deserialization format #{format} specified")
       end
-      
+
       if !md || !md.is_a?(Hash) || !md.key?(MDF::DATA) || !md.key?(MDF::SERVICES)
         raise Longleaf::MetadataError.new("Invalid metadata file, did not contain data or services fields: #{file_path}")
       end
-      
+
       data = Hash.new.merge(md[MDF::DATA])
       # Extract reserved properties for submission as separate parameters
       registered = data.delete(MDFields::REGISTERED_TIMESTAMP)
@@ -33,50 +33,52 @@ module Longleaf
       checksums = data.delete(MDFields::CHECKSUMS)
       file_size = data.delete(MDFields::FILE_SIZE)
       last_modified = data.delete(MDFields::LAST_MODIFIED)
-      
+
       services = md[MDF::SERVICES]
       service_records = Hash.new
-      unless services.nil?
-        services.each do |name, props|
-          raise Longleaf::MetadataError.new("Value of service #{name} must be a hash") unless props.class == Hash
-          
-          service_props = Hash.new.merge(props)
-          
-          stale_replicas = service_props.delete(MDFields::STALE_REPLICAS)
-          timestamp = service_props.delete(MDFields::SERVICE_TIMESTAMP)
-          run_needed = service_props.delete(MDFields::RUN_NEEDED)
-          
-          service_records[name] = ServiceRecord.new(
-              properties: service_props,
-              stale_replicas: stale_replicas,
-              timestamp: timestamp,
-              run_needed: run_needed)
-        end
+      services&.each do |name, props|
+        raise Longleaf::MetadataError.new("Value of service #{name} must be a hash") unless props.class == Hash
+
+        service_props = Hash.new.merge(props)
+
+        stale_replicas = service_props.delete(MDFields::STALE_REPLICAS)
+        timestamp = service_props.delete(MDFields::SERVICE_TIMESTAMP)
+        run_needed = service_props.delete(MDFields::RUN_NEEDED)
+
+        service_records[name] = ServiceRecord.new(
+          properties: service_props,
+          stale_replicas: stale_replicas,
+          timestamp: timestamp,
+          run_needed: run_needed)
       end
-      
+
       MetadataRecord.new(properties: data,
-          services: service_records,
-          registered: registered,
-          deregistered: deregistered,
-          checksums: checksums,
-          file_size: file_size,
-          last_modified: last_modified)
+                         services: service_records,
+                         registered: registered,
+                         deregistered: deregistered,
+                         checksums: checksums,
+                         file_size: file_size,
+                         last_modified: last_modified)
     end
-    
+
     # Load configuration a yaml encoded configuration file
     def self.from_yaml(file_path, digest_algs)
       File.open(file_path, 'r:bom|utf-8') do |f|
         contents = f.read
-        
+
         verify_digests(file_path, contents, digest_algs)
-        
-        YAML.load(contents)
+
+        begin
+          YAML.safe_load(contents, [], [], true)
+        rescue => err
+          raise Longleaf::MetadataError.new("Failed to parse metadata file #{file_path}: #{err.message}")
+        end
       end
     end
-    
+
     def self.verify_digests(file_path, contents, digest_algs)
       return if digest_algs.nil? || digest_algs.empty?
-      
+
       digest_algs.each do |alg|
         if file_path.respond_to?(:path)
           path = file_path.path
@@ -88,18 +90,18 @@ module Longleaf
           logger.warn("Missing expected #{alg} digest for #{path}")
           next
         end
-        
+
         digest = DigestHelper::start_digest(alg)
         result = digest.hexdigest(contents)
         existing_digest = IO.read(digest_path)
-        
+
         if result == existing_digest
           logger.info("Metadata fixity check using algorithm '#{alg}' succeeded for file #{path}")
         else
           raise ChecksumMismatchError.new("Metadata digest of type #{alg} did not match the contents of #{path}:" \
-              + " expected #{existing_digest}, calculated #{result}")
+                                          + " expected #{existing_digest}, calculated #{result}")
         end
       end
     end
   end
-end
+end