longleaf 0.1.0 → 0.2.0.pre.1

data/lib/longleaf/preservation_services/rsync_replication_service.rb

@@ -0,0 +1,183 @@
+require 'longleaf/events/event_names'
+require 'longleaf/logging'
+require 'longleaf/errors'
+require 'longleaf/models/file_record'
+require 'longleaf/models/service_fields'
+require 'longleaf/events/register_event'
+require 'open3'
+
+module Longleaf
+  # Preservation service which performs replication of a file to one or more destinations using rsync.
+  #
+  # The service definition must contain one or more destinations, specified with the "to" property.
+  # These destinations must be either a known storage location name, a remote path, or absolute path.
+  #
+  # Optional service configuration properties:
+  # * replica_collision_policy = specifies the desired outcome if the service attempts to replicate
+  #     a file which already exists at a destination. Default: "replace".
+  # * rsync_command = the command to invoke in order to execute rsync. Default: "rsync"
+  # * rsync_options = additional parameters that will be passed along to rsync. Cannot include options
+  #     which change the target of the command or prevent its execution, such as "files-from", "dry-run", 
+  #     "help", etc. Command will always include "-R". Default "-a".
+  class RsyncReplicationService
+    include Longleaf::Logging
+    
+    COLLISION_PROPERTY = "replica_collision_policy"
+    DEFAULT_COLLISION_POLICY = "replace"
+    VALID_COLLISION_POLICIES = ["replace"]
+    
+    RSYNC_COMMAND_PROPERTY = "rsync_command"
+    DEFAULT_COMMAND = "rsync"
+    
+    RSYNC_OPTIONS_PROPERTY = "rsync_options"
+    DEFAULT_OPTIONS = "-a"
+    DISALLOWED_OPTIONS = ["files-from", "n", "dry-run", "exclude", "exclude-from", "cvs-exclude",
+       "h", "help", "f", "F", "filter"]
+       
+    attr_reader :command, :options, :collision_policy
+    
+    # Initialize a RsyncReplicationService from the given service definition
+    #
+    # @param service_def [ServiceDefinition] the configuration for this service
+    # @param location_manager [StorageLocationManager] manager for configured storage locations
+    def initialize(service_def, app_manager)
+      @service_def = service_def
+      @app_manager = app_manager
+      
+      @command = @service_def.properties[RSYNC_COMMAND_PROPERTY] || DEFAULT_COMMAND
+      
+      # Validate rsync parameters
+      @options = @service_def.properties[RSYNC_OPTIONS_PROPERTY] || DEFAULT_OPTIONS
+      if contains_disallowed_option?(@options)
+        raise ArgumentError.new("Service #{service_def.name} specifies a disallowed rsync paramter," \
+            + " rsync_options may not include the following: #{DISALLOWED_OPTIONS.join(" ")}")
+      end
+      
+      # Add -R (--relative) in to command options to ensure full path gets replicated
+      @options = @options + " -R"
+      
+      # Set and validate the replica collision policy
+      @collision_policy = @service_def.properties[COLLISION_PROPERTY] || DEFAULT_COLLISION_POLICY
+      if !VALID_COLLISION_POLICIES.include?(@collision_policy)
+        raise ArgumentError.new("Service #{service_def.name} received invalid #{COLLISION_PROPERTY}" \
+            + " value #{collision_policy}")
+      end
+      
+      # Store and validate destinations
+      replicate_to = @service_def.properties[ServiceFields::REPLICATE_TO]
+      if replicate_to.nil? || replicate_to.empty?
+        raise ArgumentError.new("Service #{service_def.name} must provide one or more replication destinations.")
+      end
+      
+      loc_manager = app_manager.location_manager
+      # Build list of destinations, translating to storage locations when relevant
+      @destinations = Array.new
+      replicate_to.each do |dest|
+        # Assume that if destination contains a : or / it is a path rather than storage location
+        if dest =~ /[:\/]/
+          @destinations << dest
+        else
+          if loc_manager.locations.key?(dest)
+            @destinations << loc_manager.locations[dest]
+          else
+            raise ArgumentError.new("Service #{service_def.name} specifies unknown storage location '#{dest}'" \
+                + " as a replication destination")
+          end
+        end
+      end
+    end
+    
+    # During a replication event, perform replication of the specified file to all configured destinations
+    # as necessary.
+    #
+    # @param file_rec [FileRecord] record representing the file to perform the service on.
+    # @param event [String] name of the event this service is being invoked by.
+    # @raise [PreservationServiceError] if the rsync replication fails
+    def perform(file_rec, event)
+      @destinations.each do |destination|
+        dest_is_storage_loc = destination.is_a?(Longleaf::StorageLocation)
+        
+        if dest_is_storage_loc
+          dest_path = destination.path
+        else
+          dest_path = destination
+        end
+        
+        # Determine the path to the file being replicated relative to its storage location
+        rel_path = file_rec.path.sub(/\A#{file_rec.storage_location.path}/, "")
+        # source path with . so that rsync will only create destination directories starting from that point
+        source_path = File.join(file_rec.storage_location.path, "./#{rel_path}")
+        
+        # Check that the destination is available because attempting to write
+        verify_destination_available(destination, file_rec)
+        
+        logger.debug("Invoking rsync with command: #{@command} \"#{source_path}\" \"#{dest_path}\" #{@options}")
+        stdout, stderr, status = Open3.capture3("#{@command} \"#{source_path}\" \"#{dest_path}\" #{@options}")
+        raise PreservationServiceError.new("Failed to replicate #{file_rec.path} to #{dest_path}: #{stderr}") \
+            unless status.success?
+              
+        logger.info("Replicated #{file_rec.path} to destination #{dest_path}")
+        
+        # For destinations which are storage locations, register the replica with longleaf
+        if dest_is_storage_loc
+          register_replica(destination, rel_path, file_rec)
+        end
+      end
+    end
+    
+    # Determine if this service is applicable for the provided event, given the configured service definition
+    #
+    # @param event [String] name of the event
+    # @return [Boolean] returns true if this service is applicable for the provided event
+    def is_applicable?(event)
+      case event
+      when EventNames::PRESERVE
+        true
+      else
+        false
+      end
+    end
+    
+    private
+    def contains_disallowed_option?(options)
+      DISALLOWED_OPTIONS.each do |disallowed|
+        if disallowed.length == 1
+          if options =~ /(\A| )-[a-zA-Z0-9]*#{disallowed}[a-zA-Z0-9]*( |=|\z)/
+            return true
+          end
+        else
+          if options =~ /(\A| )--#{disallowed}( |=|\z)/
+            return true
+          end
+        end
+      end
+      
+      false
+    end
+    
+    def verify_destination_available(destination, file_rec)
+      if destination.is_a?(Longleaf::StorageLocation)
+        begin
+          destination.available?
+        rescue StorageLocationUnavailableError => e
+          raise StorageLocationUnavailableError.new("Cannot replicate #{file_rec.path} to destination" \
+              + " storage location #{destination.name}, it is unavailable.") unless destination.available?
+        end
+      elsif destination.start_with?("/")
+        raise StorageLocationUnavailableError.new("Cannot replicate #{file_rec.path} to destination" \
+            + " #{destination}, path does not exist.") unless Dir.exist?(destination)
+      end
+    end
+    
+    def register_replica(destination, rel_path, file_rec)
+      dest_file_path = File.join(destination.path, rel_path)
+      dest_file_rec = FileRecord.new(dest_file_path, destination)
+      
+      register_event = RegisterEvent.new(file_rec: dest_file_rec,
+          app_manager: @app_manager,
+          force: true, 
+          checksums: file_rec.metadata_record.checksums)
+      register_event.perform
+    end
+  end
+end

data/lib/longleaf/services/application_config_deserializer.rb

@@ -1,14 +1,14 @@
 require 'longleaf/services/application_config_validator'
 require 'longleaf/services/application_config_manager'
 
-# Deserializer for application configuration files
 module Longleaf
+  # Deserializer for application configuration files
   class ApplicationConfigDeserializer
     
     # Deserializes a valid application configuration file as a ApplicationConfigManager option
     # @param config_path [String] file path to the application configuration file
     # @param format [String] encoding format of the config file
-    # return [Longleaf::ApplicationConfigManager] manager for the loaded configuration
+    # return [ApplicationConfigManager] manager for the loaded configuration
     def self.deserialize(config_path, format: 'yaml')
       config = load(config_path, format: format)
       
@@ -35,11 +35,9 @@ module Longleaf
         YAML.load_file(config_path)
       rescue Errno::ENOENT => err
         raise Longleaf::ConfigurationError.new(
-            "Cannot load application configuration, file #{config_path} does not exist.")
+            "Configuration file #{config_path} does not exist.")
       rescue => err
-        raise Longleaf::ConfigurationError.new(
-            %Q(Failed to load application configuration due to the following reason:
-            #{err.message}))
+        raise Longleaf::ConfigurationError.new(err)
       end
     end
   end

data/lib/longleaf/services/application_config_manager.rb

@@ -6,8 +6,8 @@ require_relative 'service_mapping_validator'
 require_relative 'service_mapping_manager'
 require_relative 'service_manager'
 
-# Manager which loads and provides access to the configuration of the application
 module Longleaf
+  # Manager which loads and provides access to the configuration of the application
   class ApplicationConfigManager
     attr_reader :service_manager
     attr_reader :location_manager
@@ -18,7 +18,9 @@ module Longleaf
       definition_manager = Longleaf::ServiceDefinitionManager.new(config)
       mapping_manager = Longleaf::ServiceMappingManager.new(config)
       @service_manager = Longleaf::ServiceManager.new(
-          definition_manager: definition_manager, mapping_manager: mapping_manager)
+          definition_manager: definition_manager,
+          mapping_manager: mapping_manager,
+          app_manager: self)
     end
   end
 end

data/lib/longleaf/services/application_config_validator.rb

@@ -2,8 +2,8 @@ require_relative 'storage_location_validator'
 require_relative 'service_definition_validator'
 require_relative 'service_mapping_validator'
 
-# Validator for Longleaf application configuration
 module Longleaf
+  # Validator for Longleaf application configuration
   class ApplicationConfigValidator
     
     # Validates the application configuration provided. Will raise ConfigurationError

data/lib/longleaf/services/configuration_validator.rb

@@ -1,4 +1,5 @@
 module Longleaf
+  # Abstract configuration validator class
   class ConfigurationValidator
     protected
     def self.assert(fail_message, assertion_passed)

data/lib/longleaf/services/metadata_deserializer.rb

@@ -1,26 +1,28 @@
 require 'yaml'
-require_relative '../models/metadata_record'
-require_relative '../models/md_fields'
-require_relative '../errors'
+require 'longleaf/models/metadata_record'
+require 'longleaf/models/md_fields'
+require 'longleaf/errors'
+require 'longleaf/logging'
 
-# Service which deserializes metadata files into MetadataRecord objects
 module Longleaf
+  # Service which deserializes metadata files into MetadataRecord objects
   class MetadataDeserializer
-    MDF = Longleaf::MDFields
+    extend Longleaf::Logging
+    MDF ||= MDFields
     
     # Deserialize a file into a MetadataRecord object
     #
     # @param file_path [String] path of the file to read. Required.
     # @param format [String] format the file is stored in. Default is 'yaml'.
-    def self.deserialize(file_path:, format: 'yaml')
+    def self.deserialize(file_path:, format: 'yaml', digest_algs: [])
       case format
       when 'yaml'
-        md = from_yaml(file_path)
+        md = from_yaml(file_path, digest_algs)
       else
         raise ArgumentError.new('Invalid deserialization format #{format} specified')
       end
       
-      if !md || !md.key?(MDF::DATA) || !md.key?(MDF::SERVICES)
+      if !md || !md.is_a?(Hash) || !md.key?(MDF::DATA) || !md.key?(MDF::SERVICES)
         raise Longleaf::MetadataError.new("Invalid metadata file, did not contain data or services fields: #{file_path}")
       end
       
@@ -61,8 +63,43 @@ module Longleaf
           last_modified: last_modified)
     end
     
-    def self.from_yaml(file_path)
-      YAML.load_file(file_path)
+    # Load configuration a yaml encoded configuration file
+    def self.from_yaml(file_path, digest_algs)
+      File.open(file_path, 'r:bom|utf-8') do |f|
+        contents = f.read
+        
+        verify_digests(file_path, contents, digest_algs)
+        
+        YAML.load(contents)
+      end
+    end
+    
+    def self.verify_digests(file_path, contents, digest_algs)
+      return if digest_algs.nil? || digest_algs.empty?
+      
+      digest_algs.each do |alg|
+        if file_path.respond_to?(:path)
+          path = file_path.path
+        else
+          path = file_path
+        end
+        digest_path = "#{path}.#{alg}"
+        unless File.exist?(digest_path)
+          logger.warn("Missing expected #{alg} digest for #{path}")
+          next
+        end
+        
+        digest = DigestHelper::start_digest(alg)
+        result = digest.hexdigest(contents)
+        existing_digest = IO.read(digest_path)
+        
+        if result == existing_digest
+          logger.info("Metadata fixity check using algorithm '#{alg}' succeeded for file #{path}")
+        else
+          raise ChecksumMismatchError.new("Metadata digest of type #{alg} did not match the contents of #{path}:" \
+              + " expected #{existing_digest}, calculated #{result}")
+        end
+      end
     end
   end
 end

data/lib/longleaf/services/metadata_serializer.rb

@@ -1,20 +1,27 @@
 require 'yaml'
-require_relative '../models/metadata_record'
-require_relative '../models/md_fields'
+require 'longleaf/models/metadata_record'
+require 'longleaf/models/md_fields'
+require 'longleaf/helpers/digest_helper'
+require 'longleaf/errors'
+require 'longleaf/logging'
+require 'pathname'
 
-# Service which serializes MetadataRecord objects
 module Longleaf
+  # Service which serializes MetadataRecord objects
   class MetadataSerializer
-    MDF = Longleaf::MDFields
+    extend Longleaf::Logging
+    MDF ||= MDFields
     
     # Serialize the contents of the provided metadata record to the specified path
     #
     # @param metadata [MetadataRecord] metadata record to serialize. Required.
     # @param file_path [String] path to write the file to. Required.
     # @param format [String] format to serialize the metadata in. Default is 'yaml'.
-    def self.write(metadata:, file_path:, format: 'yaml')
+    # @param digest_algs [Array] if provided, sidecar digest files for the metadata file
+    #    will be generated for each algorithm.
+    def self.write(metadata:, file_path:, format: 'yaml', digest_algs: [])
       raise ArgumentError.new('metadata parameter must be a MetadataRecord') \
-          unless metadata.class == Longleaf::MetadataRecord
+          unless metadata.class == MetadataRecord
       
       case format
       when 'yaml'
@@ -23,7 +30,12 @@ module Longleaf
         raise ArgumentError.new('Invalid serialization format #{format} specified')
       end
       
+      # Fill in parent directories if they do not exist
+      parent_dir = Pathname(file_path).parent
+      parent_dir.mkpath unless parent_dir.exist?
+      
       File.write(file_path, content)
+      write_digests(file_path, content, digest_algs)
     end
     
     # @param metadata [MetadataRecord] metadata record to transform
@@ -33,6 +45,8 @@ module Longleaf
       props.to_yaml
     end
     
+    # Create a hash representation of the given MetadataRecord file
+    # @param metadata [MetadataRecord] metadata record to transform into a hash
     def self.to_hash(metadata)
       props = Hash.new
       
@@ -59,6 +73,9 @@ module Longleaf
       props
     end
     
+    # @param format [String] encoding format used for metadata file
+    # @return [String] the suffix used to indicate that a file is a metadata file in the provided encoding
+    # @raise [ArgumentError] raised if the provided format is not a supported metadata encoding format
     def self.metadata_suffix(format: 'yaml')
       case format
       when 'yaml'
@@ -67,5 +84,24 @@ module Longleaf
         raise ArgumentError.new('Invalid serialization format #{format} specified')
       end
     end
+    
+    private
+    def self.write_digests(file_path, content, digests)
+      return if digests.nil? || digests.empty?
+      
+      digests.each do |alg|
+        digest_class = DigestHelper::start_digest(alg)
+        result = digest_class.hexdigest(content)
+        if file_path.respond_to?(:path)
+          digest_path = "#{file_path.path}.#{alg}"
+        else
+          digest_path = "#{file_path}.#{alg}"
+        end
+        
+        File.write(digest_path, result)
+        
+        self.logger.debug("Generated #{alg} digest for metadata file #{file_path}: #{result}")
+      end
+    end
   end
 end

data/lib/longleaf/services/service_class_cache.rb

@@ -0,0 +1,112 @@
+require 'pathname'
+
+module Longleaf
+  # Cache for loading and retrieving preservation service classes
+  class ServiceClassCache
+    STD_PRESERVATION_SERVICE_PATH = 'longleaf/preservation_services/'
+    
+    def initialize(app_manager)
+      @app_manager = app_manager
+      # Cache storing per service definition instances of service classes
+      @service_instance_cache = Hash.new
+      # Cache storing per script path class of service
+      @class_cache = Hash.new
+    end
+    
+    # Returns an instance of the preversation service defined for the provided service definition,
+    # based on the work_script and work_class properties provided.
+    #
+    # @param service_def [ServiceDefinition] definition of service to instantiate
+    # @return [PreservationService] Instance of the preservation service class for the definition.
+    def service_instance(service_def)
+      service_name = service_def.name
+      # Return the cached instance of the service
+      if @service_instance_cache.key?(service_name)
+        return @service_instance_cache[service_name]
+      end
+      
+      clazz = service_class(service_def)
+      # Cache and return the class instance
+      @service_instance_cache[service_name] = clazz.new(service_def, @app_manager)
+    end
+    
+    # Load and return the PreservationService class assigned to the provided service definition,
+    # based on the work_script and work_class properties provided.
+    #
+    # @param service_def [ServiceDefinition] definition of service to retrieve class for
+    # @return [Class] class of work_script
+    def service_class(service_def)
+      service_name = service_def.name
+      work_script = service_def.work_script
+      
+      if work_script.include?('/')
+        expanded_path = Pathname.new(work_script).expand_path.to_s
+        if !from_permitted_path?(expanded_path)
+          raise ConfigurationError.new("Unable to load work_script for service #{service_name}, #{work_script} is not in a known library path.")
+        end
+        
+        last_slash_index = work_script.rindex('/')
+        script_path = work_script[0..last_slash_index]
+        script_name = work_script[(last_slash_index + 1)..-1]
+      else
+        script_path = STD_PRESERVATION_SERVICE_PATH
+        script_name = work_script
+      end
+      
+      # Strip off the extension
+      script_name.sub!('.rb', '')
+      
+      require_path = File.join(script_path, script_name)
+      # Return the cached Class if this path has been encountered before
+      if @class_cache.key?(require_path)
+        return @class_cache[require_path]
+      end
+      
+      # Load the script
+      begin
+        require require_path
+      rescue LoadError => e
+        raise ConfigurationError.new("Failed to load work_script '#{script_name}' for service #{service_name}")
+      end
+      
+      # Generate the class name, either configured or from file naming convention if possible
+      if service_def.work_class
+        class_name = service_def.work_class
+      else
+        class_name = script_name.split('_').map(&:capitalize).join
+        # Assume the longleaf module for classes in the standard path
+        class_name = 'Longleaf::' + class_name if script_path == STD_PRESERVATION_SERVICE_PATH
+      end
+      
+      begin
+        class_constant = constantize(class_name)
+        # cache the class for this work_script and return it
+        @class_cache[require_path] = class_constant
+      rescue NameError
+        raise ConfigurationError.new("Failed to load work_script '#{script_name}' for service #{service_name}, class name #{class_name} was not found.")
+      end
+    end
+    
+    private
+    # Borrowed from sidekiq implementation
+    def constantize(str)
+      names = str.split('::')
+      names.shift if names.empty? || names.first.empty?
+
+      names.inject(Object) do |constant, name|
+        # the false flag limits search for name to under the constant namespace
+        #   which mimics Rails' behaviour
+        constant.const_defined?(name, false) ? constant.const_get(name, false) : constant.const_missing(name)
+      end
+    end
+    
+    def from_permitted_path?(script_path)
+      $LOAD_PATH.each do |lib_path|
+        if script_path.start_with?(lib_path)
+          return true
+        end
+      end
+      false
+    end
+  end
+end