longleaf 0.3.0 → 1.0.0

data/lib/longleaf/models/s3_storage_location.rb

@@ -0,0 +1,133 @@
+require 'longleaf/models/storage_location'
+require 'longleaf/models/storage_types'
+require 'longleaf/helpers/s3_uri_helper'
+require 'uri'
+require 'aws-sdk-s3'
+
+module Longleaf
+  # A storage location in a s3 bucket
+  #
+  # Optionally, the location configuration may include an "options" sub-hash in order to provide
+  # any of the s3 client options specified in Client initializer:
+  # https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html#constructor_details
+
+  class S3StorageLocation < StorageLocation
+
+    IS_URI_REGEX = /\A#{URI::regexp}\z/
+
+    CLIENT_OPTIONS_FIELD = 'options'
+
+    # @param name [String] the name of this storage location
+    # @param config [Hash] hash containing the configuration options for this location
+    # @param md_loc [MetadataLocation] metadata location associated with this storage location
+    def initialize(name, config, md_loc)
+      super(name, config, md_loc)
+
+      @bucket_name = S3UriHelper.extract_bucket(@path)
+      if @bucket_name.nil?
+        raise ArgumentError.new("Unable to identify bucket for location #{@name} from path #{@path}")
+      end
+
+      # Force path to always end with a slash
+      @path += '/' unless @path.end_with?('/')
+
+      custom_options = config[CLIENT_OPTIONS_FIELD]
+      if custom_options.nil?
+        @client_options = Hash.new
+      else
+        # Clone options and convert keys to symbols
+        @client_options = Hash[custom_options.map { |(k,v)| [k.to_sym,v] } ]
+      end
+      # If no region directly configured, use region from path
+      if !@client_options.key?(:region)
+        region = S3UriHelper.extract_region(@path)
+        @client_options[:region] = region unless region.nil?
+      end
+      
+      @subpath_prefix = S3UriHelper.extract_path(@path)
+    end
+
+    # @return the storage type for this location
+    def type
+      StorageTypes::S3_STORAGE_TYPE
+    end
+
+    # Get that absolute path to the file associated with the provided metadata path
+    # @param md_path [String] metadata file path
+    # @raise [ArgumentError] if the md_path is not in this storage location
+    # @return [String] the path for the file associated with this metadata
+    def get_path_from_metadata_path(md_path)
+      raise ArgumentError.new("A file_path parameter is required") if md_path.nil? || md_path.empty?
+
+      rel_path = @metadata_location.relative_file_path_for(md_path)
+
+      URI.join(@path, rel_path).to_s
+    end
+
+    # Checks that the path and metadata path defined in this location are available
+    # @raise [StorageLocationUnavailableError] if the storage location is not available
+    def available?
+      begin
+        s3_client().head_bucket({ bucket: @bucket_name, use_accelerate_endpoint: false })
+      rescue StandardError => e
+        raise StorageLocationUnavailableError.new("Destination bucket #{@bucket_name} does not exist " \
+            + "or is not accessible: #{e.message}")
+      end
+      @metadata_location.available?
+    end
+
+    # Get the file path relative to this location
+    # @param file_path [String] file path
+    # @return the file path relative to this location
+    # @raise [ArgumentError] if the file path is not contained by this location
+    def relativize(file_path)
+      raise ArgumentError.new("Must provide a non-nil path to relativize") if file_path.nil?
+
+      if file_path.start_with?(@path)
+        file_path[@path.length..-1]
+      else
+        if file_path =~ IS_URI_REGEX
+          raise ArgumentError.new("Path #{file_path} is not contained by #{@name}")
+        else
+          # path already relative
+          file_path
+        end
+      end
+    end
+    
+    # Prefixes the provided path with the query path portion of the location's path
+    # after the bucket uri, used to place relative paths into the same sub-URL of a bucket.
+    # For example:
+    # Given a location with 'path' http://example.s3-amazonaws.com/env/test/
+    # Where rel_path = 'path/to/text.txt'
+    # The result would be 'env/test/path/to/text.txt'
+    # @param rel_path relative path to work with
+    # @return the given relative path prefixed with the path portion of the storage location path
+    def relative_to_bucket_path(rel_path)
+      raise ArgumentError.new("Must provide a non-nil path") if rel_path.nil?
+      
+      if @subpath_prefix.nil?
+        return rel_path
+      end
+      
+      @subpath_prefix + rel_path
+    end
+
+    # @return the bucket used by this storage location
+    def s3_bucket
+      if @bucket.nil?
+        @s3 = Aws::S3::Resource.new(client: s3_client())
+        @bucket = @s3.bucket(@bucket_name)
+      end
+      @bucket
+    end
+
+    # @return the s3 client used by this storage locatio
+    def s3_client
+      if @client.nil?
+        @client = Aws::S3::Client.new(**@client_options)
+      end
+      @client
+    end
+  end
+end

data/lib/longleaf/models/service_fields.rb

@@ -8,5 +8,9 @@ module Longleaf
 
     REPLICATE_TO = 'to'
     DIGEST_ALGORITHMS = 'algorithms'
+
+    COLLISION_PROPERTY = "replica_collision_policy"
+    DEFAULT_COLLISION_POLICY = "replace"
+    VALID_COLLISION_POLICIES = ["replace"]
   end
 end

data/lib/longleaf/models/storage_location.rb

@@ -1,34 +1,23 @@
-require 'longleaf/services/metadata_serializer'
+require 'longleaf/models/app_fields'
 
 module Longleaf
   # Representation of a configured storage location
   class StorageLocation
+    AF ||= Longleaf::AppFields
+
     attr_reader :name
     attr_reader :path
-    attr_reader :metadata_path
-    attr_reader :metadata_digests
+    attr_reader :metadata_location
 
     # @param name [String] the name of this storage location
-    # @param path [String] absolute path where the storage location is located
-    # @param metadata_path [String] absolute path where the metadata for files in this location will be stored.
-    # @param metadata_digests list of digest algorithms to use for metadata file digests in this location.
-    def initialize(name:, path:, metadata_path:, metadata_digests: [])
-      raise ArgumentError.new("Parameters name, path and metadata_path are required") unless name && path && metadata_path
-
-      @path = path
-      @path += '/' unless @path.end_with?('/')
+    # @param config [Hash] hash containing the configuration options for this location
+    # @param md_loc [MetadataLocation] metadata location associated with this storage location
+    def initialize(name, config, md_loc)
+      raise ArgumentError.new("Config parameter is required") unless config
+      @path = config[AF::LOCATION_PATH]
       @name = name
-      @metadata_path = metadata_path
-      @metadata_path += '/' unless @metadata_path.end_with?('/')
-
-      if metadata_digests.nil?
-        @metadata_digests = []
-      elsif metadata_digests.is_a?(String)
-        @metadata_digests = [metadata_digests.downcase]
-      else
-        @metadata_digests = metadata_digests.map(&:downcase)
-      end
-      DigestHelper::validate_algorithms(@metadata_digests)
+      raise ArgumentError.new("Parameters name, path and metadata location are required") unless @name && @path && md_loc
+      @metadata_location = md_loc
     end
 
     # Get the path for the metadata file for the given file path located in this storage location.
@@ -39,35 +28,15 @@ module Longleaf
       raise ArgumentError.new("Provided file path is not contained by storage location #{@name}: #{file_path}") \
           unless file_path.start_with?(@path)
 
-      md_path = file_path.sub(/^#{@path}/, @metadata_path)
-      # If the file_path is to a file, then add metadata suffix.
-      if md_path.end_with?('/')
-        md_path
-      else
-        md_path + MetadataSerializer::metadata_suffix
-      end
-    end
-
-    # Get the metadata path for the provided file path located in this storage location.
-    # @param md_path [String] metadata file path
-    # @raise [ArgumentError] if the md_path is not in this storage location
-    # @return [String] the path for the file associated with this metadata
-    def get_path_from_metadata_path(md_path)
-      raise ArgumentError.new("A file_path parameter is required") if md_path.nil? || md_path.empty?
-      raise ArgumentError.new("Provided metadata path is not contained by storage location #{@name}: #{md_path}") \
-          unless md_path&.start_with?(@metadata_path)
+      rel_file_path = relativize(file_path)
 
-      file_path = md_path.sub(/^#{@metadata_path}/, @path)
-      file_path.sub(/#{MetadataSerializer::metadata_suffix}$/, '')
+      @metadata_location.metadata_path_for(rel_file_path)
     end
 
-    # Checks that the path and metadata path defined in this location are available
-    # @raise [StorageLocationUnavailableError] if the storage location is not available
-    def available?
-      raise StorageLocationUnavailableError.new("Path does not exist or is not a directory: #{@path}")\
-          unless Dir.exist?(@path)
-      raise StorageLocationUnavailableError.new("Metadata path does not exist or is not a directory: #{@metadata_path}")\
-          unless Dir.exist?(@metadata_path)
+    # @param [String] path to check
+    # @return true if the file path is contained by the path for this location
+    def contains?(file_path)
+      file_path.start_with?(@path)
     end
   end
 end

data/lib/longleaf/models/storage_types.rb

@@ -0,0 +1,9 @@
+module Longleaf
+  # Storage type constants
+  class StorageTypes
+    FILESYSTEM_STORAGE_TYPE = 'filesystem'
+    S3_STORAGE_TYPE = 's3'
+
+    DEFAULT_STORAGE_TYPE = FILESYSTEM_STORAGE_TYPE
+  end
+end

data/lib/longleaf/preservation_services/rsync_replication_service.rb

@@ -4,6 +4,7 @@ require 'longleaf/errors'
 require 'longleaf/models/file_record'
 require 'longleaf/models/service_fields'
 require 'longleaf/events/register_event'
+require 'longleaf/candidates/single_digest_provider'
 require 'open3'
 
 module Longleaf
@@ -21,10 +22,7 @@ module Longleaf
   #     "help", etc. Command will always include "-R". Default "-a".
   class RsyncReplicationService
     include Longleaf::Logging
-
-    COLLISION_PROPERTY = "replica_collision_policy"
-    DEFAULT_COLLISION_POLICY = "replace"
-    VALID_COLLISION_POLICIES = ["replace"]
+    SF ||= Longleaf::ServiceFields
 
     RSYNC_COMMAND_PROPERTY = "rsync_command"
     DEFAULT_COMMAND = "rsync"
@@ -57,14 +55,14 @@ module Longleaf
       @options = @options + " -R"
 
       # Set and validate the replica collision policy
-      @collision_policy = @service_def.properties[COLLISION_PROPERTY] || DEFAULT_COLLISION_POLICY
-      if !VALID_COLLISION_POLICIES.include?(@collision_policy)
-        raise ArgumentError.new("Service #{service_def.name} received invalid #{COLLISION_PROPERTY}" \
-            + " value #{collision_policy}")
+      @collision_policy = @service_def.properties[SF::COLLISION_PROPERTY] || SF::DEFAULT_COLLISION_POLICY
+      if !SF::VALID_COLLISION_POLICIES.include?(@collision_policy)
+        raise ArgumentError.new("Service #{service_def.name} received invalid #{SF::COLLISION_PROPERTY}" \
+            + " value #{@collision_policy}")
       end
 
       # Store and validate destinations
-      replicate_to = @service_def.properties[ServiceFields::REPLICATE_TO]
+      replicate_to = @service_def.properties[SF::REPLICATE_TO]
       if replicate_to.nil? || replicate_to.empty?
         raise ArgumentError.new("Service #{service_def.name} must provide one or more replication destinations.")
       end
@@ -105,7 +103,7 @@ module Longleaf
         end
 
         # Determine the path to the file being replicated relative to its storage location
-        rel_path = file_rec.path.sub(/\A#{file_rec.storage_location.path}/, "")
+        rel_path = file_rec.storage_location.relativize(file_rec.path)
         # source path with . so that rsync will only create destination directories starting from that point
         source_path = File.join(file_rec.storage_location.path, "./#{rel_path}")
 
@@ -177,7 +175,7 @@ module Longleaf
       register_event = RegisterEvent.new(file_rec: dest_file_rec,
           app_manager: @app_manager,
           force: true,
-          checksums: file_rec.metadata_record.checksums)
+          digest_provider: SingleDigestProvider.new(file_rec.metadata_record.checksums))
       register_event.perform
     end
   end

data/lib/longleaf/preservation_services/s3_replication_service.rb

@@ -0,0 +1,143 @@
+require 'longleaf/events/event_names'
+require 'longleaf/logging'
+require 'longleaf/errors'
+require 'longleaf/models/file_record'
+require 'longleaf/models/service_fields'
+require 'longleaf/events/register_event'
+require 'longleaf/models/storage_types'
+require 'aws-sdk-s3'
+
+module Longleaf
+  # Preservation service which performs replication of a file to one or more s3 destinations.
+  #
+  # The service definition must contain one or more destinations, specified with the "to" property.
+  # These destinations must be either a known s3 storage location. The s3 client configuration
+  # is controlled by the storage location.
+  #
+  # Optional service configuration properties:
+  # * replica_collision_policy = specifies the desired outcome if the service attempts to replicate
+  #     a file which already exists at a destination. Default: "replace".
+  class S3ReplicationService
+    include Longleaf::Logging
+    ST ||= Longleaf::StorageTypes
+    SF ||= Longleaf::ServiceFields
+
+    attr_reader :collision_policy
+
+    # Initialize a S3ReplicationService from the given service definition
+    #
+    # @param service_def [ServiceDefinition] the configuration for this service
+    # @param app_manager [ApplicationConfigManager] the application configuration
+    def initialize(service_def, app_manager)
+      @service_def = service_def
+      @app_manager = app_manager
+
+      # Set and validate the replica collision policy
+      @collision_policy = @service_def.properties[SF::COLLISION_PROPERTY] || SF::DEFAULT_COLLISION_POLICY
+      if !SF::VALID_COLLISION_POLICIES.include?(@collision_policy)
+        raise ArgumentError.new("Service #{service_def.name} received invalid #{SF::COLLISION_PROPERTY}" \
+            + " value #{@collision_policy}")
+      end
+
+      # Store and validate destinations
+      replicate_to = @service_def.properties[SF::REPLICATE_TO]
+      if replicate_to.nil? || replicate_to.empty?
+        raise ArgumentError.new("Service #{service_def.name} must provide one or more replication destinations.")
+      end
+      replicate_to = [replicate_to] if replicate_to.is_a?(String)
+
+      loc_manager = app_manager.location_manager
+      # Build list of destinations, translating to storage locations when relevant
+      @destinations = Array.new
+      replicate_to.each do |dest|
+        if loc_manager.locations.key?(dest)
+          location = loc_manager.locations[dest]
+          if location.type != ST::S3_STORAGE_TYPE
+            raise ArgumentError.new(
+                "Service #{service_def.name} specifies destination #{dest} which is not of type 's3'")
+          end
+          @destinations << loc_manager.locations[dest]
+        else
+          raise ArgumentError.new("Service #{service_def.name} specifies unknown storage location '#{dest}'" \
+              + " as a replication destination")
+        end
+      end
+    end
+
+    # During a replication event, perform replication of the specified file to all configured destinations
+    # as necessary.
+    #
+    # @param file_rec [FileRecord] record representing the file to perform the service on.
+    # @param event [String] name of the event this service is being invoked by.
+    # @raise [PreservationServiceError] if the rsync replication fails
+    def perform(file_rec, event)
+      if file_rec.storage_location.type == ST::FILESYSTEM_STORAGE_TYPE
+        replicate_from_fs(file_rec)
+      else
+        raise PreservationServiceError.new("Replication from storage location of type " \
+            + "#{file_rec.storage_location.type} to s3 is not supported")
+      end
+    end
+
+    def replicate_from_fs(file_rec)
+      # Determine the path to the file being replicated relative to its storage location
+      rel_path = file_rec.storage_location.relativize(file_rec.path)
+
+      content_md5 = get_content_md5(file_rec)
+
+      @destinations.each do |destination|
+        # Check that the destination is available before attempting to write
+        verify_destination_available(destination, file_rec)
+
+        rel_to_bucket = destination.relative_to_bucket_path(rel_path)
+        file_obj = destination.s3_bucket.object(rel_to_bucket)
+        begin
+          file_obj.upload_file(file_rec.path, { :content_md5 => content_md5 })
+        rescue Aws::S3::Errors::BadDigest => e
+          raise ChecksumMismatchError.new("Transfer to bucket '#{destination.s3_bucket.name}' failed, " \
+              + "MD5 provided did not match the received content for #{file_rec.path}")
+        rescue Aws::Errors::ServiceError => e
+          raise PreservationServiceError.new("Failed to transfer #{file_rec.path} to bucket " \
+              + "'#{destination.s3_bucket.name}': #{e.message}")
+        end
+
+        logger.info("Replicated #{file_rec.path} to destination #{file_obj.public_url}")
+
+        # TODO register file in destination
+      end
+    end
+
+    # Determine if this service is applicable for the provided event, given the configured service definition
+    #
+    # @param event [String] name of the event
+    # @return [Boolean] returns true if this service is applicable for the provided event
+    def is_applicable?(event)
+      case event
+      when EventNames::PRESERVE
+        true
+      else
+        false
+      end
+    end
+
+    private
+    def verify_destination_available(destination, file_rec)
+      begin
+        destination.available?
+      rescue StorageLocationUnavailableError => e
+        raise StorageLocationUnavailableError.new("Cannot replicate #{file_rec.path} to destination #{destination.name}: " \
+            + e.message)
+      end
+    end
+
+    def get_content_md5(file_rec)
+      md_rec = file_rec.metadata_record
+      if md_rec.checksums.key?('md5')
+        # base 64 encode the digest, as is required by the Content-Md5 header
+        [[md_rec.checksums['md5']].pack("H*")].pack("m0")
+      else
+        nil
+      end
+    end
+  end
+end

data/lib/longleaf/services/application_config_deserializer.rb

@@ -19,8 +19,9 @@ module Longleaf
       config_md5 = Digest::MD5.hexdigest(content)
 
       make_paths_absolute(config_path, config)
-      Longleaf::ApplicationConfigValidator.validate(config)
-      Longleaf::ApplicationConfigManager.new(config, config_md5)
+
+      ApplicationConfigValidator.new(config).validate_config.raise_if_invalid
+      ApplicationConfigManager.new(config, config_md5)
     end
 
     def self.load_config_file(config_path)
@@ -57,9 +58,30 @@ module Longleaf
       base_pathname = Pathname.new(config_path).expand_path.parent
 
       config[AF::LOCATIONS].each do |name, properties|
-        properties[AF::LOCATION_PATH] = absolution(base_pathname, properties[AF::LOCATION_PATH])
+        properties[AF::LOCATION_PATH] = make_file_paths_absolute(base_pathname, properties)
 
-        properties[AF::METADATA_PATH] = absolution(base_pathname, properties[AF::METADATA_PATH])
+        # Resolve single field metadata location into expanded form
+        md_config = properties[AF::METADATA_CONFIG]
+        if md_config.nil?
+          next
+        end
+        if md_config.is_a?(String)
+          md_config = { AF::LOCATION => m_config }
+        end
+        md_config[AF::LOCATION_PATH] = make_file_paths_absolute(base_pathname, md_config)
+      end
+    end
+
+    def self.make_file_paths_absolute(base_pathname, properties)
+      path = properties[AF::LOCATION_PATH]
+      return nil if path.nil?
+
+      uri = URI(path)
+
+      if uri.scheme.nil? || uri.scheme.casecmp("file") == 0
+        absolution(base_pathname, uri.path)
+      else
+        path
       end
     end