longleaf 0.1.0.pre.2 → 1.0.0

data/lib/longleaf/preservation_services/rsync_replication_service.rb

@@ -0,0 +1,182 @@
+require 'longleaf/events/event_names'
+require 'longleaf/logging'
+require 'longleaf/errors'
+require 'longleaf/models/file_record'
+require 'longleaf/models/service_fields'
+require 'longleaf/events/register_event'
+require 'longleaf/candidates/single_digest_provider'
+require 'open3'
+
+module Longleaf
+  # Preservation service which performs replication of a file to one or more destinations using rsync.
+  #
+  # The service definition must contain one or more destinations, specified with the "to" property.
+  # These destinations must be either a known storage location name, a remote path, or absolute path.
+  #
+  # Optional service configuration properties:
+  # * replica_collision_policy = specifies the desired outcome if the service attempts to replicate
+  #     a file which already exists at a destination. Default: "replace".
+  # * rsync_command = the command to invoke in order to execute rsync. Default: "rsync"
+  # * rsync_options = additional parameters that will be passed along to rsync. Cannot include options
+  #     which change the target of the command or prevent its execution, such as "files-from", "dry-run",
+  #     "help", etc. Command will always include "-R". Default "-a".
+  class RsyncReplicationService
+    include Longleaf::Logging
+    SF ||= Longleaf::ServiceFields
+
+    RSYNC_COMMAND_PROPERTY = "rsync_command"
+    DEFAULT_COMMAND = "rsync"
+
+    RSYNC_OPTIONS_PROPERTY = "rsync_options"
+    DEFAULT_OPTIONS = "-a"
+    DISALLOWED_OPTIONS = ["files-from", "n", "dry-run", "exclude", "exclude-from", "cvs-exclude",
+       "h", "help", "f", "F", "filter"]
+
+    attr_reader :command, :options, :collision_policy
+
+    # Initialize a RsyncReplicationService from the given service definition
+    #
+    # @param service_def [ServiceDefinition] the configuration for this service
+    # @param app_manager [ApplicationConfigManager] the application configuration
+    def initialize(service_def, app_manager)
+      @service_def = service_def
+      @app_manager = app_manager
+
+      @command = @service_def.properties[RSYNC_COMMAND_PROPERTY] || DEFAULT_COMMAND
+
+      # Validate rsync parameters
+      @options = @service_def.properties[RSYNC_OPTIONS_PROPERTY] || DEFAULT_OPTIONS
+      if contains_disallowed_option?(@options)
+        raise ArgumentError.new("Service #{service_def.name} specifies a disallowed rsync paramter," \
+            + " rsync_options may not include the following: #{DISALLOWED_OPTIONS.join(' ')}")
+      end
+
+      # Add -R (--relative) in to command options to ensure full path gets replicated
+      @options = @options + " -R"
+
+      # Set and validate the replica collision policy
+      @collision_policy = @service_def.properties[SF::COLLISION_PROPERTY] || SF::DEFAULT_COLLISION_POLICY
+      if !SF::VALID_COLLISION_POLICIES.include?(@collision_policy)
+        raise ArgumentError.new("Service #{service_def.name} received invalid #{SF::COLLISION_PROPERTY}" \
+            + " value #{@collision_policy}")
+      end
+
+      # Store and validate destinations
+      replicate_to = @service_def.properties[SF::REPLICATE_TO]
+      if replicate_to.nil? || replicate_to.empty?
+        raise ArgumentError.new("Service #{service_def.name} must provide one or more replication destinations.")
+      end
+      replicate_to = [replicate_to] if replicate_to.is_a?(String)
+
+      loc_manager = app_manager.location_manager
+      # Build list of destinations, translating to storage locations when relevant
+      @destinations = Array.new
+      replicate_to.each do |dest|
+        # Assume that if destination contains a : or / it is a path rather than storage location
+        if dest =~ /[:\/]/
+          @destinations << dest
+        else
+          if loc_manager.locations.key?(dest)
+            @destinations << loc_manager.locations[dest]
+          else
+            raise ArgumentError.new("Service #{service_def.name} specifies unknown storage location '#{dest}'" \
+                + " as a replication destination")
+          end
+        end
+      end
+    end
+
+    # During a replication event, perform replication of the specified file to all configured destinations
+    # as necessary.
+    #
+    # @param file_rec [FileRecord] record representing the file to perform the service on.
+    # @param event [String] name of the event this service is being invoked by.
+    # @raise [PreservationServiceError] if the rsync replication fails
+    def perform(file_rec, event)
+      @destinations.each do |destination|
+        dest_is_storage_loc = destination.is_a?(Longleaf::StorageLocation)
+
+        if dest_is_storage_loc
+          dest_path = destination.path
+        else
+          dest_path = destination
+        end
+
+        # Determine the path to the file being replicated relative to its storage location
+        rel_path = file_rec.storage_location.relativize(file_rec.path)
+        # source path with . so that rsync will only create destination directories starting from that point
+        source_path = File.join(file_rec.storage_location.path, "./#{rel_path}")
+
+        # Check that the destination is available because attempting to write
+        verify_destination_available(destination, file_rec)
+
+        logger.debug("Invoking rsync with command: #{@command} \"#{source_path}\" \"#{dest_path}\" #{@options}")
+        stdout, stderr, status = Open3.capture3("#{@command} \"#{source_path}\" \"#{dest_path}\" #{@options}")
+        raise PreservationServiceError.new("Failed to replicate #{file_rec.path} to #{dest_path}: #{stderr}") \
+            unless status.success?
+
+        logger.info("Replicated #{file_rec.path} to destination #{dest_path}")
+
+        # For destinations which are storage locations, register the replica with longleaf
+        if dest_is_storage_loc
+          register_replica(destination, rel_path, file_rec)
+        end
+      end
+    end
+
+    # Determine if this service is applicable for the provided event, given the configured service definition
+    #
+    # @param event [String] name of the event
+    # @return [Boolean] returns true if this service is applicable for the provided event
+    def is_applicable?(event)
+      case event
+      when EventNames::PRESERVE
+        true
+      else
+        false
+      end
+    end
+
+    private
+    def contains_disallowed_option?(options)
+      DISALLOWED_OPTIONS.each do |disallowed|
+        if disallowed.length == 1
+          if options =~ /(\A| )-[a-zA-Z0-9]*#{disallowed}[a-zA-Z0-9]*( |=|\z)/
+            return true
+          end
+        else
+          if options =~ /(\A| )--#{disallowed}( |=|\z)/
+            return true
+          end
+        end
+      end
+
+      false
+    end
+
+    def verify_destination_available(destination, file_rec)
+      if destination.is_a?(Longleaf::StorageLocation)
+        begin
+          destination.available?
+        rescue StorageLocationUnavailableError => e
+          raise StorageLocationUnavailableError.new("Cannot replicate #{file_rec.path} to destination #{destination.name}: " \
+              + e.message)
+        end
+      elsif destination.start_with?("/")
+        raise StorageLocationUnavailableError.new("Cannot replicate #{file_rec.path} to destination" \
+            + " #{destination}, path does not exist.") unless Dir.exist?(destination)
+      end
+    end
+
+    def register_replica(destination, rel_path, file_rec)
+      dest_file_path = File.join(destination.path, rel_path)
+      dest_file_rec = FileRecord.new(dest_file_path, destination)
+
+      register_event = RegisterEvent.new(file_rec: dest_file_rec,
+          app_manager: @app_manager,
+          force: true,
+          digest_provider: SingleDigestProvider.new(file_rec.metadata_record.checksums))
+      register_event.perform
+    end
+  end
+end

data/lib/longleaf/preservation_services/s3_replication_service.rb

@@ -0,0 +1,143 @@
+require 'longleaf/events/event_names'
+require 'longleaf/logging'
+require 'longleaf/errors'
+require 'longleaf/models/file_record'
+require 'longleaf/models/service_fields'
+require 'longleaf/events/register_event'
+require 'longleaf/models/storage_types'
+require 'aws-sdk-s3'
+
+module Longleaf
+  # Preservation service which performs replication of a file to one or more s3 destinations.
+  #
+  # The service definition must contain one or more destinations, specified with the "to" property.
+  # These destinations must be either a known s3 storage location. The s3 client configuration
+  # is controlled by the storage location.
+  #
+  # Optional service configuration properties:
+  # * replica_collision_policy = specifies the desired outcome if the service attempts to replicate
+  #     a file which already exists at a destination. Default: "replace".
+  class S3ReplicationService
+    include Longleaf::Logging
+    ST ||= Longleaf::StorageTypes
+    SF ||= Longleaf::ServiceFields
+
+    attr_reader :collision_policy
+
+    # Initialize a S3ReplicationService from the given service definition
+    #
+    # @param service_def [ServiceDefinition] the configuration for this service
+    # @param app_manager [ApplicationConfigManager] the application configuration
+    def initialize(service_def, app_manager)
+      @service_def = service_def
+      @app_manager = app_manager
+
+      # Set and validate the replica collision policy
+      @collision_policy = @service_def.properties[SF::COLLISION_PROPERTY] || SF::DEFAULT_COLLISION_POLICY
+      if !SF::VALID_COLLISION_POLICIES.include?(@collision_policy)
+        raise ArgumentError.new("Service #{service_def.name} received invalid #{SF::COLLISION_PROPERTY}" \
+            + " value #{@collision_policy}")
+      end
+
+      # Store and validate destinations
+      replicate_to = @service_def.properties[SF::REPLICATE_TO]
+      if replicate_to.nil? || replicate_to.empty?
+        raise ArgumentError.new("Service #{service_def.name} must provide one or more replication destinations.")
+      end
+      replicate_to = [replicate_to] if replicate_to.is_a?(String)
+
+      loc_manager = app_manager.location_manager
+      # Build list of destinations, translating to storage locations when relevant
+      @destinations = Array.new
+      replicate_to.each do |dest|
+        if loc_manager.locations.key?(dest)
+          location = loc_manager.locations[dest]
+          if location.type != ST::S3_STORAGE_TYPE
+            raise ArgumentError.new(
+                "Service #{service_def.name} specifies destination #{dest} which is not of type 's3'")
+          end
+          @destinations << loc_manager.locations[dest]
+        else
+          raise ArgumentError.new("Service #{service_def.name} specifies unknown storage location '#{dest}'" \
+              + " as a replication destination")
+        end
+      end
+    end
+
+    # During a replication event, perform replication of the specified file to all configured destinations
+    # as necessary.
+    #
+    # @param file_rec [FileRecord] record representing the file to perform the service on.
+    # @param event [String] name of the event this service is being invoked by.
+    # @raise [PreservationServiceError] if the rsync replication fails
+    def perform(file_rec, event)
+      if file_rec.storage_location.type == ST::FILESYSTEM_STORAGE_TYPE
+        replicate_from_fs(file_rec)
+      else
+        raise PreservationServiceError.new("Replication from storage location of type " \
+            + "#{file_rec.storage_location.type} to s3 is not supported")
+      end
+    end
+
+    def replicate_from_fs(file_rec)
+      # Determine the path to the file being replicated relative to its storage location
+      rel_path = file_rec.storage_location.relativize(file_rec.path)
+
+      content_md5 = get_content_md5(file_rec)
+
+      @destinations.each do |destination|
+        # Check that the destination is available before attempting to write
+        verify_destination_available(destination, file_rec)
+
+        rel_to_bucket = destination.relative_to_bucket_path(rel_path)
+        file_obj = destination.s3_bucket.object(rel_to_bucket)
+        begin
+          file_obj.upload_file(file_rec.path, { :content_md5 => content_md5 })
+        rescue Aws::S3::Errors::BadDigest => e
+          raise ChecksumMismatchError.new("Transfer to bucket '#{destination.s3_bucket.name}' failed, " \
+              + "MD5 provided did not match the received content for #{file_rec.path}")
+        rescue Aws::Errors::ServiceError => e
+          raise PreservationServiceError.new("Failed to transfer #{file_rec.path} to bucket " \
+              + "'#{destination.s3_bucket.name}': #{e.message}")
+        end
+
+        logger.info("Replicated #{file_rec.path} to destination #{file_obj.public_url}")
+
+        # TODO register file in destination
+      end
+    end
+
+    # Determine if this service is applicable for the provided event, given the configured service definition
+    #
+    # @param event [String] name of the event
+    # @return [Boolean] returns true if this service is applicable for the provided event
+    def is_applicable?(event)
+      case event
+      when EventNames::PRESERVE
+        true
+      else
+        false
+      end
+    end
+
+    private
+    def verify_destination_available(destination, file_rec)
+      begin
+        destination.available?
+      rescue StorageLocationUnavailableError => e
+        raise StorageLocationUnavailableError.new("Cannot replicate #{file_rec.path} to destination #{destination.name}: " \
+            + e.message)
+      end
+    end
+
+    def get_content_md5(file_rec)
+      md_rec = file_rec.metadata_record
+      if md_rec.checksums.key?('md5')
+        # base 64 encode the digest, as is required by the Content-Md5 header
+        [[md_rec.checksums['md5']].pack("H*")].pack("m0")
+      else
+        nil
+      end
+    end
+  end
+end

data/lib/longleaf/services/application_config_deserializer.rb

@@ -1,46 +1,103 @@
 require 'longleaf/services/application_config_validator'
 require 'longleaf/services/application_config_manager'
+require 'digest/md5'
+require 'pathname'
 
-# Deserializer for application configuration files
 module Longleaf
+  # Deserializer for application configuration files
   class ApplicationConfigDeserializer
-    
+    AF ||= Longleaf::AppFields
+
     # Deserializes a valid application configuration file as a ApplicationConfigManager option
-    # @param config_path [String] file path to the application configuration file
+    # @param config_path [String] file path to the service and storage mapping configuration file
     # @param format [String] encoding format of the config file
-    # return [Longleaf::ApplicationConfigManager] manager for the loaded configuration
+    # return [ApplicationConfigManager] manager for the loaded configuration
     def self.deserialize(config_path, format: 'yaml')
-      config = load(config_path, format: format)
-      
-      Longleaf::ApplicationConfigValidator.validate(config)
-      Longleaf::ApplicationConfigManager.new(config)
+      content = load_config_file(config_path)
+      config = load(content, format)
+
+      config_md5 = Digest::MD5.hexdigest(content)
+
+      make_paths_absolute(config_path, config)
+
+      ApplicationConfigValidator.new(config).validate_config.raise_if_invalid
+      ApplicationConfigManager.new(config, config_md5)
     end
-    
+
+    def self.load_config_file(config_path)
+      begin
+        File.read(config_path)
+      rescue Errno::ENOENT
+        raise Longleaf::ConfigurationError.new(
+            "Configuration file #{config_path} does not exist.")
+      end
+    end
+
     # Deserialize a configuration file into a hash
-    # @param config_path [String] file path to the application configuration file
+    # @param content [String] the contents of the application configuration file
     # @param format [String] encoding format of the config file
     # return [Hash] hash containing the configuration
-    def self.load(config_path, format: 'yaml')
+    def self.load(content, format)
       case format
       when 'yaml'
-        from_yaml(config_path)
+        from_yaml(content)
       else
-        raise ArgumentError.new('Invalid deserialization format #{format} specified')
+        raise ArgumentError.new("Invalid deserialization format #{format} specified")
       end
     end
-    
-    private
-    def self.from_yaml(config_path)
+
+    def self.from_yaml(content)
       begin
-        YAML.load_file(config_path)
-      rescue Errno::ENOENT => err
-        raise Longleaf::ConfigurationError.new(
-            "Cannot load application configuration, file #{config_path} does not exist.")
+        YAML.safe_load(content, [], [], true)
       rescue => err
-        raise Longleaf::ConfigurationError.new(
-            %Q(Failed to load application configuration due to the following reason:
-            #{err.message}))
+        raise Longleaf::ConfigurationError.new(err)
       end
     end
+
+    def self.make_paths_absolute(config_path, config)
+      base_pathname = Pathname.new(config_path).expand_path.parent
+
+      config[AF::LOCATIONS].each do |name, properties|
+        properties[AF::LOCATION_PATH] = make_file_paths_absolute(base_pathname, properties)
+
+        # Resolve single field metadata location into expanded form
+        md_config = properties[AF::METADATA_CONFIG]
+        if md_config.nil?
+          next
+        end
+        if md_config.is_a?(String)
+          md_config = { AF::LOCATION => m_config }
+        end
+        md_config[AF::LOCATION_PATH] = make_file_paths_absolute(base_pathname, md_config)
+      end
+    end
+
+    def self.make_file_paths_absolute(base_pathname, properties)
+      path = properties[AF::LOCATION_PATH]
+      return nil if path.nil?
+
+      uri = URI(path)
+
+      if uri.scheme.nil? || uri.scheme.casecmp("file") == 0
+        absolution(base_pathname, uri.path)
+      else
+        path
+      end
+    end
+
+    def self.absolution(base_pathname, file_path)
+      if file_path.nil?
+        nil
+      else
+        path = Pathname.new(file_path)
+        if path.absolute?
+          path = path.expand_path.to_s
+        else
+          path = (base_pathname + path).to_s
+        end
+      end
+    end
+
+    private_class_method :load_config_file
   end
-end
+end