longleaf 0.1.0.pre.3 → 1.1.0

data/lib/longleaf/preservation_services/s3_replication_service.rb

@@ -0,0 +1,131 @@
+require 'longleaf/events/event_names'
+require 'longleaf/logging'
+require 'longleaf/errors'
+require 'longleaf/models/file_record'
+require 'longleaf/models/service_fields'
+require 'longleaf/events/register_event'
+require 'longleaf/models/storage_types'
+require 'aws-sdk-s3'
+
+module Longleaf
+  # Preservation service which performs replication of a file to one or more s3 destinations.
+  #
+  # The service definition must contain one or more destinations, specified with the "to" property.
+  # These destinations must be either a known s3 storage location. The s3 client configuration
+  # is controlled by the storage location.
+  #
+  # Optional service configuration properties:
+  # * replica_collision_policy = specifies the desired outcome if the service attempts to replicate
+  #     a file which already exists at a destination. Default: "replace".
+  class S3ReplicationService
+    include Longleaf::Logging
+    ST ||= Longleaf::StorageTypes
+    SF ||= Longleaf::ServiceFields
+
+    attr_reader :collision_policy
+
+    # Initialize a S3ReplicationService from the given service definition
+    #
+    # @param service_def [ServiceDefinition] the configuration for this service
+    # @param app_manager [ApplicationConfigManager] the application configuration
+    def initialize(service_def, app_manager)
+      @service_def = service_def
+      @app_manager = app_manager
+
+      # Set and validate the replica collision policy
+      @collision_policy = @service_def.properties[SF::COLLISION_PROPERTY] || SF::DEFAULT_COLLISION_POLICY
+      if !SF::VALID_COLLISION_POLICIES.include?(@collision_policy)
+        raise ArgumentError.new("Service #{service_def.name} received invalid #{SF::COLLISION_PROPERTY}" \
+            + " value #{@collision_policy}")
+      end
+
+      # Store and validate destinations
+      replicate_to = @service_def.properties[SF::REPLICATE_TO]
+      if replicate_to.nil? || replicate_to.empty?
+        raise ArgumentError.new("Service #{service_def.name} must provide one or more replication destinations.")
+      end
+      replicate_to = [replicate_to] if replicate_to.is_a?(String)
+
+      loc_manager = app_manager.location_manager
+      # Build list of destinations, translating to storage locations when relevant
+      @destinations = Array.new
+      replicate_to.each do |dest|
+        if loc_manager.locations.key?(dest)
+          location = loc_manager.locations[dest]
+          if location.type != ST::S3_STORAGE_TYPE
+            raise ArgumentError.new(
+                "Service #{service_def.name} specifies destination #{dest} which is not of type 's3'")
+          end
+          @destinations << loc_manager.locations[dest]
+        else
+          raise ArgumentError.new("Service #{service_def.name} specifies unknown storage location '#{dest}'" \
+              + " as a replication destination")
+        end
+      end
+    end
+
+    # During a replication event, perform replication of the specified file to all configured destinations
+    # as necessary.
+    #
+    # @param file_rec [FileRecord] record representing the file to perform the service on.
+    # @param event [String] name of the event this service is being invoked by.
+    # @raise [PreservationServiceError] if the rsync replication fails
+    def perform(file_rec, event)
+      if file_rec.storage_location.type == ST::FILESYSTEM_STORAGE_TYPE
+        replicate_from_fs(file_rec)
+      else
+        raise PreservationServiceError.new("Replication from storage location of type " \
+            + "#{file_rec.storage_location.type} to s3 is not supported")
+      end
+    end
+
+    def replicate_from_fs(file_rec)
+      # Determine the path to the file being replicated relative to its storage location
+      rel_path = file_rec.storage_location.relativize(file_rec.path)
+
+      @destinations.each do |destination|
+        # Check that the destination is available before attempting to write
+        verify_destination_available(destination, file_rec)
+
+        rel_to_bucket = destination.relative_to_bucket_path(rel_path)
+        file_obj = destination.s3_bucket.object(rel_to_bucket)
+        begin
+          file_obj.upload_file(file_rec.physical_path)
+        rescue Aws::S3::Errors::BadDigest => e
+          raise ChecksumMismatchError.new("Transfer to bucket '#{destination.s3_bucket.name}' failed, " \
+              + "MD5 provided did not match the received content for #{file_rec.path}")
+        rescue Aws::Errors::ServiceError => e
+          raise PreservationServiceError.new("Failed to transfer #{file_rec.path} to bucket " \
+              + "'#{destination.s3_bucket.name}': #{e.message}")
+        end
+
+        logger.info("Replicated #{file_rec.path} to destination #{file_obj.public_url}")
+
+        # TODO register file in destination
+      end
+    end
+
+    # Determine if this service is applicable for the provided event, given the configured service definition
+    #
+    # @param event [String] name of the event
+    # @return [Boolean] returns true if this service is applicable for the provided event
+    def is_applicable?(event)
+      case event
+      when EventNames::PRESERVE
+        true
+      else
+        false
+      end
+    end
+
+    private
+    def verify_destination_available(destination, file_rec)
+      begin
+        destination.available?
+      rescue StorageLocationUnavailableError => e
+        raise StorageLocationUnavailableError.new("Cannot replicate #{file_rec.path} to destination #{destination.name}: " \
+            + e.message)
+      end
+    end
+  end
+end

data/lib/longleaf/services/application_config_deserializer.rb

@@ -1,44 +1,103 @@
 require 'longleaf/services/application_config_validator'
 require 'longleaf/services/application_config_manager'
+require 'digest/md5'
+require 'pathname'
 
-# Deserializer for application configuration files
 module Longleaf
+  # Deserializer for application configuration files
   class ApplicationConfigDeserializer
-    
+    AF ||= Longleaf::AppFields
+
     # Deserializes a valid application configuration file as a ApplicationConfigManager option
-    # @param config_path [String] file path to the application configuration file
+    # @param config_path [String] file path to the service and storage mapping configuration file
     # @param format [String] encoding format of the config file
-    # return [Longleaf::ApplicationConfigManager] manager for the loaded configuration
+    # return [ApplicationConfigManager] manager for the loaded configuration
     def self.deserialize(config_path, format: 'yaml')
-      config = load(config_path, format: format)
-      
-      Longleaf::ApplicationConfigValidator.validate(config)
-      Longleaf::ApplicationConfigManager.new(config)
+      content = load_config_file(config_path)
+      config = load(content, format)
+
+      config_md5 = Digest::MD5.hexdigest(content)
+
+      make_paths_absolute(config_path, config)
+
+      ApplicationConfigValidator.new(config).validate_config.raise_if_invalid
+      ApplicationConfigManager.new(config, config_md5)
     end
-    
+
+    def self.load_config_file(config_path)
+      begin
+        File.read(config_path)
+      rescue Errno::ENOENT
+        raise Longleaf::ConfigurationError.new(
+            "Configuration file #{config_path} does not exist.")
+      end
+    end
+
     # Deserialize a configuration file into a hash
-    # @param config_path [String] file path to the application configuration file
+    # @param content [String] the contents of the application configuration file
     # @param format [String] encoding format of the config file
     # return [Hash] hash containing the configuration
-    def self.load(config_path, format: 'yaml')
+    def self.load(content, format)
       case format
       when 'yaml'
-        from_yaml(config_path)
+        from_yaml(content)
       else
-        raise ArgumentError.new('Invalid deserialization format #{format} specified')
+        raise ArgumentError.new("Invalid deserialization format #{format} specified")
       end
     end
-    
-    private
-    def self.from_yaml(config_path)
+
+    def self.from_yaml(content)
       begin
-        YAML.load_file(config_path)
-      rescue Errno::ENOENT => err
-        raise Longleaf::ConfigurationError.new(
-            "Configuration file #{config_path} does not exist.")
+        YAML.safe_load(content, [], [], true)
       rescue => err
         raise Longleaf::ConfigurationError.new(err)
       end
     end
+
+    def self.make_paths_absolute(config_path, config)
+      base_pathname = Pathname.new(config_path).expand_path.parent
+
+      config[AF::LOCATIONS].each do |name, properties|
+        properties[AF::LOCATION_PATH] = make_file_paths_absolute(base_pathname, properties)
+
+        # Resolve single field metadata location into expanded form
+        md_config = properties[AF::METADATA_CONFIG]
+        if md_config.nil?
+          next
+        end
+        if md_config.is_a?(String)
+          md_config = { AF::LOCATION => m_config }
+        end
+        md_config[AF::LOCATION_PATH] = make_file_paths_absolute(base_pathname, md_config)
+      end
+    end
+
+    def self.make_file_paths_absolute(base_pathname, properties)
+      path = properties[AF::LOCATION_PATH]
+      return nil if path.nil?
+
+      uri = URI(path)
+
+      if uri.scheme.nil? || uri.scheme.casecmp("file") == 0
+        absolution(base_pathname, uri.path)
+      else
+        path
+      end
+    end
+
+    def self.absolution(base_pathname, file_path)
+      if file_path.nil?
+        nil
+      else
+        path = Pathname.new(file_path)
+        if path.absolute?
+          path = path.expand_path.to_s
+        else
+          path = (base_pathname + path).to_s
+        end
+      end
+    end
+
+    private_class_method :load_config_file
   end
-end
+end

data/lib/longleaf/services/application_config_manager.rb

@@ -5,20 +5,34 @@ require_relative 'service_definition_manager'
 require_relative 'service_mapping_validator'
 require_relative 'service_mapping_manager'
 require_relative 'service_manager'
+require_relative 'metadata_persistence_manager'
+require 'longleaf/indexing/index_manager'
+require 'longleaf/models/app_fields'
 
-# Manager which loads and provides access to the configuration of the application
 module Longleaf
+  # Manager which loads and provides access to the configuration of the application
   class ApplicationConfigManager
+    attr_reader :config_md5
     attr_reader :service_manager
     attr_reader :location_manager
-    
-    def initialize(config)
+    attr_reader :index_manager
+    attr_reader :md_manager
+
+    def initialize(config, config_md5 = nil)
+      @config_md5 = config_md5
+
       @location_manager = Longleaf::StorageLocationManager.new(config)
-      
+
       definition_manager = Longleaf::ServiceDefinitionManager.new(config)
       mapping_manager = Longleaf::ServiceMappingManager.new(config)
       @service_manager = Longleaf::ServiceManager.new(
-          definition_manager: definition_manager, mapping_manager: mapping_manager)
+          definition_manager: definition_manager,
+          mapping_manager: mapping_manager,
+          app_manager: self)
+
+      sys_config = config[AppFields::SYSTEM]
+      @index_manager = IndexManager.new(sys_config, self)
+      @md_manager = MetadataPersistenceManager.new(@index_manager)
     end
   end
-end
+end

data/lib/longleaf/services/application_config_validator.rb

@@ -2,17 +2,27 @@ require_relative 'storage_location_validator'
 require_relative 'service_definition_validator'
 require_relative 'service_mapping_validator'
 
-# Validator for Longleaf application configuration
 module Longleaf
-  class ApplicationConfigValidator
-    
+  # Validator for Longleaf application configuration
+  class ApplicationConfigValidator < ConfigurationValidator
+    # @param config [Hash] hash containing the application configuration
+    def initialize(config)
+      super(config)
+    end
+
+    protected
     # Validates the application configuration provided. Will raise ConfigurationError
     # if any portion of the configuration is not syntactically or semantically valid.
-    # @param config [Hash] application configuration
-    def self.validate(config)
-      Longleaf::StorageLocationValidator::validate_config(config)
-      Longleaf::ServiceDefinitionValidator::validate_config(config)
-      Longleaf::ServiceMappingValidator::validate_config(config)
+    def validate
+      loc_result = StorageLocationValidator.new(@config).validate_config
+      defs_result = ServiceDefinitionValidator.new(@config).validate_config
+      mapping_result = ServiceMappingValidator.new(@config).validate_config
+
+      @result.errors.concat(loc_result.errors) unless loc_result.valid?
+      @result.errors.concat(defs_result.errors) unless defs_result.valid?
+      @result.errors.concat(mapping_result.errors) unless mapping_result.valid?
+
+      @result
     end
   end
-end
+end

data/lib/longleaf/services/configuration_validator.rb

@@ -1,8 +1,71 @@
 module Longleaf
+  # Abstract configuration validator class
   class ConfigurationValidator
-    protected
-    def self.assert(fail_message, assertion_passed)
-      raise ConfigurationError.new(fail_message) unless assertion_passed
+    attr_reader :result
+
+    def initialize(config)
+      @result = ConfigurationValidationResult.new
+      @config = config
+    end
+
+    # Verify that the provided configuration is valid
+    # @return [ConfigurationValidationResult] the result of the validation
+    def validate_config
+      register_on_failure { validate }
+
+      @result
+    end
+
+    # Asserts that the given conditional is true, raising a ConfigurationError if it is not.
+    def assert(fail_message, assertion_passed)
+      fail(fail_message) unless assertion_passed
+    end
+
+    # Indicate that validation has failed, throwing a Configuration error with the given message
+    def fail(fail_message)
+      raise ConfigurationError.new(fail_message)
+    end
+
+    # Registers an error to the result for this validator
+    def register_error(error)
+      if error.is_a?(StandardError)
+        @result.register_error(error.msg)
+      else
+        @result.register_error(error)
+      end
+    end
+
+    # Performs the provided block. If the block produces a ConfigurationError, the error
+    # is swallowed and registered to the result
+    def register_on_failure
+      begin
+        yield
+      rescue ConfigurationError => err
+        register_error(err.message)
+      end
+    end
+  end
+
+  class ConfigurationValidationResult
+    attr_reader :errors
+
+    def initialize
+      @errors = Array.new
+    end
+
+    # Register an error with this validation result
+    def register_error(error_message)
+      @errors << error_message
+    end
+
+    # @return [boolean] true if validation produced not errors
+    def valid?
+      @errors.length == 0
+    end
+
+    # @raise [ConfigurationError] if the result is not valid, which lists all failures
+    def raise_if_invalid
+      raise ConfigurationError.new(@errors.join("\n")) unless valid?
     end
   end
-end
+end

data/lib/longleaf/services/filesystem_location_validator.rb

@@ -0,0 +1,16 @@
+require 'pathname'
+require 'longleaf/errors'
+
+module Longleaf
+  # Validates the configuration of a filesystem based location
+  class FilesystemLocationValidator
+
+    def self.validate(p_validator, name, path_prop, section_name, path)
+      base_msg = "Storage location '#{name}' specifies invalid #{section_name} '#{path_prop}' property: "
+      p_validator.assert(base_msg + 'Path must not be empty', !path.nil? && !path.to_s.strip.empty?)
+      p_validator.assert(base_msg + 'Path must not contain any relative modifiers (/..)', !path.include?('/..'))
+      p_validator.assert(base_msg + 'Path must be absolute', Pathname.new(path).absolute?)
+      p_validator.assert(base_msg + 'Path does not exist', Dir.exist?(path))
+    end
+  end
+end

data/lib/longleaf/services/metadata_deserializer.rb

@@ -1,29 +1,30 @@
 require 'yaml'
-require_relative '../models/metadata_record'
-require_relative '../models/md_fields'
-require_relative '../errors'
+require 'longleaf/models/metadata_record'
+require 'longleaf/models/md_fields'
+require 'longleaf/services/metadata_validator'
+require 'longleaf/errors'
+require 'longleaf/logging'
 
-# Service which deserializes metadata files into MetadataRecord objects
 module Longleaf
+  # Service which deserializes metadata files into MetadataRecord objects
   class MetadataDeserializer
-    MDF = Longleaf::MDFields
-    
+    extend Longleaf::Logging
+    MDF ||= MDFields
+
     # Deserialize a file into a MetadataRecord object
     #
     # @param file_path [String] path of the file to read. Required.
     # @param format [String] format the file is stored in. Default is 'yaml'.
-    def self.deserialize(file_path:, format: 'yaml')
+    def self.deserialize(file_path:, format: 'yaml', digest_algs: [])
+      file_path = file_path.path if file_path.is_a?(File)
+
       case format
       when 'yaml'
-        md = from_yaml(file_path)
+        md = from_yaml(file_path, digest_algs)
       else
-        raise ArgumentError.new('Invalid deserialization format #{format} specified')
+        raise ArgumentError.new("Invalid deserialization format #{format} specified")
       end
-      
-      if !md || !md.key?(MDF::DATA) || !md.key?(MDF::SERVICES)
-        raise Longleaf::MetadataError.new("Invalid metadata file, did not contain data or services fields: #{file_path}")
-      end
-      
+
       data = Hash.new.merge(md[MDF::DATA])
       # Extract reserved properties for submission as separate parameters
       registered = data.delete(MDFields::REGISTERED_TIMESTAMP)
@@ -31,38 +32,110 @@ module Longleaf
       checksums = data.delete(MDFields::CHECKSUMS)
       file_size = data.delete(MDFields::FILE_SIZE)
       last_modified = data.delete(MDFields::LAST_MODIFIED)
-      
+      physical_path = data.delete(MDFields::PHYSICAL_PATH)
+
       services = md[MDF::SERVICES]
       service_records = Hash.new
-      unless services.nil?
-        services.each do |name, props|
-          raise Longleaf::MetadataError.new("Value of service #{name} must be a hash") unless props.class == Hash
-          
-          service_props = Hash.new.merge(props)
-          
-          stale_replicas = service_props.delete(MDFields::STALE_REPLICAS)
-          timestamp = service_props.delete(MDFields::SERVICE_TIMESTAMP)
-          run_needed = service_props.delete(MDFields::RUN_NEEDED)
-          
-          service_records[name] = ServiceRecord.new(
-              properties: service_props,
-              stale_replicas: stale_replicas,
-              timestamp: timestamp,
-              run_needed: run_needed)
-        end
+      services&.each do |name, props|
+        raise MetadataError.new("Value of service #{name} must be a hash") unless props.class == Hash
+
+        service_props = Hash.new.merge(props)
+
+        stale_replicas = service_props.delete(MDFields::STALE_REPLICAS)
+        timestamp = service_props.delete(MDFields::SERVICE_TIMESTAMP)
+        run_needed = service_props.delete(MDFields::RUN_NEEDED)
+
+        service_records[name] = ServiceRecord.new(
+          properties: service_props,
+          stale_replicas: stale_replicas,
+          timestamp: timestamp,
+          run_needed: run_needed)
       end
-      
+
       MetadataRecord.new(properties: data,
-          services: service_records,
-          registered: registered,
-          deregistered: deregistered,
-          checksums: checksums,
-          file_size: file_size,
-          last_modified: last_modified)
+                         services: service_records,
+                         registered: registered,
+                         deregistered: deregistered,
+                         checksums: checksums,
+                         file_size: file_size,
+                         last_modified: last_modified,
+                         physical_path: physical_path)
     end
-    
-    def self.from_yaml(file_path)
-      YAML.load_file(file_path)
+
+    # Load configuration a yaml encoded configuration file
+    def self.from_yaml(file_path, digest_algs)
+      File.open(file_path, 'r:bom|utf-8') do |f|
+        contents = f.read
+
+        checksum_error = nil
+        begin
+          verify_digests(file_path, contents, digest_algs)
+        rescue ChecksumMismatchError => err
+          # Hold onto the checksum error, in case we can identify the underlying cause
+          checksum_error = err
+        end
+
+        begin
+          md = nil
+          begin
+            md = YAML.safe_load(contents, [], [], true)
+          rescue => err
+            raise MetadataError.new("Failed to parse metadata file #{file_path}: #{err.message}")
+          end
+
+          validation_result = MetadataValidator.new(md).validate_config
+          if !validation_result.valid?
+            if checksum_error.nil?
+              raise MetadataError.new("Invalid metadata file #{file_path.to_s}:\n#{validation_result.errors.join("\n")}")
+            else
+              raise MetadataError.new(validation_result.errors.join("\n"))
+            end
+          end
+
+          # Either return the valid metadata, or raise the checksum error as is
+          if checksum_error.nil?
+            md
+          else
+            raise checksum_error
+          end
+        rescue MetadataError => err
+          if checksum_error.nil?
+            raise err
+          else
+            # Add underlying cause from the metadata error to the checksum mismatch error
+            msg = checksum_error.message + "\nWith related issue(s):\n#{err.message}"
+            raise ChecksumMismatchError.new(msg)
+          end
+        end
+      end
+    end
+
+    def self.verify_digests(file_path, contents, digest_algs)
+      return if digest_algs.nil? || digest_algs.empty?
+
+      digest_algs.each do |alg|
+        if file_path.respond_to?(:path)
+          path = file_path.path
+        else
+          path = file_path
+        end
+        digest_path = "#{path}.#{alg}"
+        unless File.exist?(digest_path)
+          logger.warn("Missing expected #{alg} digest for #{path}")
+          next
+        end
+
+        digest = DigestHelper::start_digest(alg)
+        result = digest.hexdigest(contents)
+        existing_digest = IO.read(digest_path)
+
+        if result == existing_digest
+          logger.info("Metadata fixity check using algorithm '#{alg}' succeeded for file #{path}")
+        else
+          raise ChecksumMismatchError.new("Metadata digest of type #{alg} did not match the contents of #{path}:" \
+                                          + " expected #{existing_digest}, calculated #{result}")
+        end
+      end
     end
   end
-end
+end