longleaf 0.1.0.pre.2 → 1.0.0

data/lib/longleaf/models/s3_storage_location.rb

@@ -0,0 +1,133 @@
+require 'longleaf/models/storage_location'
+require 'longleaf/models/storage_types'
+require 'longleaf/helpers/s3_uri_helper'
+require 'uri'
+require 'aws-sdk-s3'
+
+module Longleaf
+  # A storage location in a s3 bucket
+  #
+  # Optionally, the location configuration may include an "options" sub-hash in order to provide
+  # any of the s3 client options specified in Client initializer:
+  # https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html#constructor_details
+
+  class S3StorageLocation < StorageLocation
+
+    IS_URI_REGEX = /\A#{URI::regexp}\z/
+
+    CLIENT_OPTIONS_FIELD = 'options'
+
+    # @param name [String] the name of this storage location
+    # @param config [Hash] hash containing the configuration options for this location
+    # @param md_loc [MetadataLocation] metadata location associated with this storage location
+    def initialize(name, config, md_loc)
+      super(name, config, md_loc)
+
+      @bucket_name = S3UriHelper.extract_bucket(@path)
+      if @bucket_name.nil?
+        raise ArgumentError.new("Unable to identify bucket for location #{@name} from path #{@path}")
+      end
+
+      # Force path to always end with a slash
+      @path += '/' unless @path.end_with?('/')
+
+      custom_options = config[CLIENT_OPTIONS_FIELD]
+      if custom_options.nil?
+        @client_options = Hash.new
+      else
+        # Clone options and convert keys to symbols
+        @client_options = Hash[custom_options.map { |(k,v)| [k.to_sym,v] } ]
+      end
+      # If no region directly configured, use region from path
+      if !@client_options.key?(:region)
+        region = S3UriHelper.extract_region(@path)
+        @client_options[:region] = region unless region.nil?
+      end
+      
+      @subpath_prefix = S3UriHelper.extract_path(@path)
+    end
+
+    # @return the storage type for this location
+    def type
+      StorageTypes::S3_STORAGE_TYPE
+    end
+
+    # Get that absolute path to the file associated with the provided metadata path
+    # @param md_path [String] metadata file path
+    # @raise [ArgumentError] if the md_path is not in this storage location
+    # @return [String] the path for the file associated with this metadata
+    def get_path_from_metadata_path(md_path)
+      raise ArgumentError.new("A file_path parameter is required") if md_path.nil? || md_path.empty?
+
+      rel_path = @metadata_location.relative_file_path_for(md_path)
+
+      URI.join(@path, rel_path).to_s
+    end
+
+    # Checks that the path and metadata path defined in this location are available
+    # @raise [StorageLocationUnavailableError] if the storage location is not available
+    def available?
+      begin
+        s3_client().head_bucket({ bucket: @bucket_name, use_accelerate_endpoint: false })
+      rescue StandardError => e
+        raise StorageLocationUnavailableError.new("Destination bucket #{@bucket_name} does not exist " \
+            + "or is not accessible: #{e.message}")
+      end
+      @metadata_location.available?
+    end
+
+    # Get the file path relative to this location
+    # @param file_path [String] file path
+    # @return the file path relative to this location
+    # @raise [ArgumentError] if the file path is not contained by this location
+    def relativize(file_path)
+      raise ArgumentError.new("Must provide a non-nil path to relativize") if file_path.nil?
+
+      if file_path.start_with?(@path)
+        file_path[@path.length..-1]
+      else
+        if file_path =~ IS_URI_REGEX
+          raise ArgumentError.new("Path #{file_path} is not contained by #{@name}")
+        else
+          # path already relative
+          file_path
+        end
+      end
+    end
+    
+    # Prefixes the provided path with the query path portion of the location's path
+    # after the bucket uri, used to place relative paths into the same sub-URL of a bucket.
+    # For example:
+    # Given a location with 'path' http://example.s3-amazonaws.com/env/test/
+    # Where rel_path = 'path/to/text.txt'
+    # The result would be 'env/test/path/to/text.txt'
+    # @param rel_path relative path to work with
+    # @return the given relative path prefixed with the path portion of the storage location path
+    def relative_to_bucket_path(rel_path)
+      raise ArgumentError.new("Must provide a non-nil path") if rel_path.nil?
+      
+      if @subpath_prefix.nil?
+        return rel_path
+      end
+      
+      @subpath_prefix + rel_path
+    end
+
+    # @return the bucket used by this storage location
+    def s3_bucket
+      if @bucket.nil?
+        @s3 = Aws::S3::Resource.new(client: s3_client())
+        @bucket = @s3.bucket(@bucket_name)
+      end
+      @bucket
+    end
+
+    # @return the s3 client used by this storage locatio
+    def s3_client
+      if @client.nil?
+        @client = Aws::S3::Client.new(**@client_options)
+      end
+      @client
+    end
+  end
+end

data/lib/longleaf/models/service_definition.rb

@@ -1,21 +1,22 @@
 require_relative 'service_fields'
 
-# Definition of a preservation service
 module Longleaf
+  # Definition of a configured preservation service
   class ServiceDefinition
     attr_reader :name
-    attr_reader :work_script
+    attr_reader :work_script, :work_class
     attr_reader :frequency, :delay
     attr_reader :properties
-    
-    def initialize(name:, work_script:, frequency: nil, delay: nil, properties: Hash.new)
+
+    def initialize(name:, work_script:, work_class: nil, frequency: nil, delay: nil, properties: Hash.new)
       raise ArgumentError.new("Parameters name and work_script are required") unless name && work_script
-      
+
       @properties = properties
       @name = name
       @work_script = work_script
+      @work_class = work_class
       @frequency = frequency
       @delay = delay
     end
   end
-end
+end

data/lib/longleaf/models/service_fields.rb

@@ -1,10 +1,16 @@
 module Longleaf
+  # Constants for common configuration fields for preservation service definitions
   class ServiceFields
     WORK_SCRIPT = 'work_script'
+    WORK_CLASS = 'work_class'
     FREQUENCY = 'frequency'
     DELAY = 'delay'
-    
+
     REPLICATE_TO = 'to'
     DIGEST_ALGORITHMS = 'algorithms'
+
+    COLLISION_PROPERTY = "replica_collision_policy"
+    DEFAULT_COLLISION_POLICY = "replace"
+    VALID_COLLISION_POLICIES = ["replace"]
   end
 end

data/lib/longleaf/models/service_record.rb

@@ -1,27 +1,31 @@
-# Record for an individual service in a file's metadata record.
 module Longleaf
+  # Record for an individual service in a file's metadata record.
   class ServiceRecord
     attr_reader :properties
     attr_accessor :stale_replicas, :timestamp, :run_needed
-    
+    attr_accessor :failure_timestamp
+
     # @param properties [Hash] initial properties for this service record
+    # @param stale_replicas [Boolean] whether there are any stale replicas from this service
+    # @param timestamp [String] timestamp when this service last ran or was initialized
+    # @param run_needed [Boolean] flag indicating that this service should be run at the next available opportunity
     def initialize(properties: Hash.new, stale_replicas: false, timestamp: nil, run_needed: false)
       raise ArgumentError.new("Service properties must be a hash") if properties.class != Hash
-      
+
       @properties = properties
       @timestamp = timestamp
       @stale_replicas = stale_replicas
       @run_needed = run_needed
     end
-    
+
     # @return the value of a service property identified by key
     def [](key)
       @properties[key]
     end
-    
+
     # set the value of a service property identified by key
     def []=(key, value)
       @properties[key] = value
     end
   end
-end
+end

data/lib/longleaf/models/storage_location.rb

@@ -1,19 +1,25 @@
-require 'longleaf/services/metadata_serializer'
+require 'longleaf/models/app_fields'
 
 module Longleaf
+  # Representation of a configured storage location
   class StorageLocation
+    AF ||= Longleaf::AppFields
+
     attr_reader :name
     attr_reader :path
-    attr_reader :metadata_path
-    
-    def initialize(name:, path:, metadata_path:)
-      raise ArgumentError.new("Parameters name, path and metadata_path are required") unless name && path && metadata_path
-      
-      @path = path
+    attr_reader :metadata_location
+
+    # @param name [String] the name of this storage location
+    # @param config [Hash] hash containing the configuration options for this location
+    # @param md_loc [MetadataLocation] metadata location associated with this storage location
+    def initialize(name, config, md_loc)
+      raise ArgumentError.new("Config parameter is required") unless config
+      @path = config[AF::LOCATION_PATH]
       @name = name
-      @metadata_path = metadata_path
+      raise ArgumentError.new("Parameters name, path and metadata location are required") unless @name && @path && md_loc
+      @metadata_location = md_loc
     end
-    
+
     # Get the path for the metadata file for the given file path located in this storage location.
     # @param file_path [String] path of the file
     # @raise [ArgumentError] if the file_path is not provided or is not in this storage location.
@@ -22,16 +28,15 @@ module Longleaf
       raise ArgumentError.new("Provided file path is not contained by storage location #{@name}: #{file_path}") \
           unless file_path.start_with?(@path)
 
-      file_path.sub(/^#{@path}/, metadata_path) + MetadataSerializer::metadata_suffix
+      rel_file_path = relativize(file_path)
+
+      @metadata_location.metadata_path_for(rel_file_path)
     end
-    
-    # Checks that the path and metadata path defined in this location are available
-    # @raise [StorageLocationUnavailableError] if the storage location is not available
-    def available?
-      raise StorageLocationUnavailableError.new("Path does not exist or is not a directory: #{@path}")\
-          unless Dir.exist?(@path)
-      raise StorageLocationUnavailableError.new("Metadata path does not exist or is not a directory: #{@metadata_path}")\
-          unless Dir.exist?(@metadata_path)
+
+    # @param [String] path to check
+    # @return true if the file path is contained by the path for this location
+    def contains?(file_path)
+      file_path.start_with?(@path)
     end
   end
-end
+end

data/lib/longleaf/models/storage_types.rb

@@ -0,0 +1,9 @@
+module Longleaf
+  # Storage type constants
+  class StorageTypes
+    FILESYSTEM_STORAGE_TYPE = 'filesystem'
+    S3_STORAGE_TYPE = 's3'
+
+    DEFAULT_STORAGE_TYPE = FILESYSTEM_STORAGE_TYPE
+  end
+end

data/lib/longleaf/models/system_config_fields.rb

@@ -0,0 +1,9 @@
+module Longleaf
+  # System configuration field names
+  class SystemConfigFields
+    MD_INDEX = 'index'
+    MD_INDEX_ADAPTER = 'adapter'
+    MD_INDEX_CONNECTION = 'connection'
+    MD_INDEX_PAGE_SIZE = 'page_size'
+  end
+end

data/lib/longleaf/preservation_services/file_check_service.rb

@@ -0,0 +1,58 @@
+require 'longleaf/events/event_names'
+require 'longleaf/logging'
+
+module Longleaf
+  # Preservation service which validates a file using current filesystem information compared against the
+  # last registered details for that file. Checks using file name, size and last modified timestamp.
+  class FileCheckService
+    include Longleaf::Logging
+
+    # Initialize a FileCheckService from the given service definition
+    #
+    # @param service_def [ServiceDefinition] the configuration for this service
+    # @param app_manager [ApplicationConfigManager] manager for configured storage locations
+    def initialize(service_def, app_manager)
+      @service_def = service_def
+      @app_manager = app_manager
+    end
+
+    # Perform file information check.
+    #
+    # @param file_rec [FileRecord] record representing the file to perform the service on.
+    # @param event [String] name of the event this service is being invoked by.
+    # @raise [PreservationServiceError] if the file system information does not match the stored details
+    def perform(file_rec, event)
+      file_path = file_rec.path
+      md_rec = file_rec.metadata_record
+
+      logger.debug("Performing file information check of #{file_path}")
+
+      if !File.exist?(file_path)
+        raise PreservationServiceError.new("File does not exist: #{file_path}")
+      end
+
+      file_size = File.size(file_rec.path)
+      if file_size != md_rec.file_size
+        raise PreservationServiceError.new("File size for #{file_path} does not match the expected value: registered = #{md_rec.file_size} bytes, actual = #{file_size} bytes")
+      end
+
+      last_modified = File.mtime(file_rec.path).utc.iso8601(3)
+      if last_modified != md_rec.last_modified
+        raise PreservationServiceError.new("Last modified timestamp for #{file_path} does not match the expected value: registered = #{md_rec.last_modified}, actual = #{last_modified}")
+      end
+    end
+
+    # Determine if this service is applicable for the provided event, given the configured service definition
+    #
+    # @param event [String] name of the event
+    # @return [Boolean] returns true if this service is applicable for the provided event
+    def is_applicable?(event)
+      case event
+      when EventNames::PRESERVE
+        true
+      else
+        false
+      end
+    end
+  end
+end

data/lib/longleaf/preservation_services/fixity_check_service.rb

@@ -0,0 +1,123 @@
+require 'longleaf/events/event_names'
+require 'longleaf/models/service_fields'
+require 'longleaf/logging'
+require 'longleaf/helpers/digest_helper'
+require 'set'
+
+module Longleaf
+  # Preservation service which performs one or more fixity checks on a file based on the configured list
+  # of digest algorithms. It currently supports 'md5', 'sha1', 'sha2', 'sha256', 'sha384', 'sha512' and 'rmd160'.
+  #
+  # If the service encounters a file which is missing any of the digest algorithms the service is configured
+  # to check, the outcome may be controlled with the 'absent_digest' property via the following values:
+  #   * 'fail' - the service will raise a ChecksumMismatchError for the missing algorithm. This is the default.
+  #   * 'ignore' - the service will skip calculating any algorithms not already present for the file.
+  #   * 'generate' - the service will generate and store any missing digests from the set of configured algorithms.
+  class FixityCheckService
+    include Longleaf::Logging
+
+    SUPPORTED_ALGORITHMS = ['md5', 'sha1', 'sha2', 'sha256', 'sha384', 'sha512', 'rmd160']
+
+    # service configuration property indicating how to handle situations where a file does not
+    # have a digest for one of the expected algorithms on record.
+    ABSENT_DIGEST_PROPERTY = 'absent_digest'
+    FAIL_IF_ABSENT = 'fail'
+    GENERATE_IF_ABSENT = 'generate'
+    IGNORE_IF_ABSENT = 'ignore'
+    ABSENT_DIGEST_OPTIONS = [FAIL_IF_ABSENT, GENERATE_IF_ABSENT, IGNORE_IF_ABSENT]
+
+    # Initialize a FixityCheckService from the given service definition
+    #
+    # @param service_def [ServiceDefinition] the configuration for this service
+    # @param app_manager [ApplicationConfigManager] manager for configured storage locations
+    def initialize(service_def, app_manager)
+      @service_def = service_def
+      @absent_digest_behavior = @service_def.properties[ABSENT_DIGEST_PROPERTY] || FAIL_IF_ABSENT
+      unless ABSENT_DIGEST_OPTIONS.include?(@absent_digest_behavior)
+        raise ArgumentError.new("Invalid option '#{@absent_digest_behavior}' for property #{ABSENT_DIGEST_PROPERTY} in service #{service_def.name}")
+      end
+
+      service_algs = service_def.properties[ServiceFields::DIGEST_ALGORITHMS]
+      if service_algs.nil? || service_algs.empty?
+        raise ArgumentError.new("FixityCheckService from definition #{service_def.name} requires a list of one or more digest algorithms")
+      end
+
+      service_algs = [service_algs] if service_algs.is_a?(String)
+
+      # Store the list of digest algorithms to verify, using normalized algorithm names.
+      @digest_algs = Set.new
+      service_algs.each do |alg|
+        normalized_alg = alg.downcase.delete('-')
+        if SUPPORTED_ALGORITHMS.include?(normalized_alg)
+          @digest_algs << normalized_alg
+        else
+          raise ArgumentError.new("Unsupported checksum algorithm '#{alg}' in definition #{service_def.name}. Supported algorithms are: #{SUPPORTED_ALGORITHMS}")
+        end
+      end
+    end
+
+    # Perform all configured fixity checks on the provided file
+    #
+    # @param file_rec [FileRecord] record representing the file to perform the service on.
+    # @param event [String] name of the event this service is being invoked by.
+    # @raise [ChecksumMismatchError] if the checksum on record does not match the generated checksum
+    def perform(file_rec, event)
+      path = file_rec.path
+      md_rec = file_rec.metadata_record
+
+      # Get the list of existing checksums for the file and normalize algorithm names
+      file_digests = Hash.new
+      md_rec.checksums&.each do |alg, digest|
+        normalized_alg = alg.downcase.delete('-')
+        if @digest_algs.include?(normalized_alg)
+          file_digests[normalized_alg] = digest
+        else
+          logger.debug("Metadata for file #{path} contains unexpected '#{alg}' digest, it will be ignored.")
+        end
+      end
+
+      @digest_algs.each do |alg|
+        existing_digest = file_digests[alg]
+
+        if existing_digest.nil?
+          if @absent_digest_behavior == FAIL_IF_ABSENT
+            raise ChecksumMismatchError.new("Fixity check using algorithm '#{alg}' failed for file #{path}: no existing digest of type '#{alg}' on record.")
+          elsif @absent_digest_behavior == IGNORE_IF_ABSENT
+            logger.debug("Skipping check of algorithm '#{alg}' for file #{path}: no digest on record.")
+            next
+          end
+        end
+
+        digest = DigestHelper::start_digest(alg)
+        digest.file(path)
+        generated_digest = digest.hexdigest
+
+        # Store the missing checksum if using the 'generate' behavior
+        if existing_digest.nil? && @absent_digest_behavior == GENERATE_IF_ABSENT
+          md_rec.checksums[alg] = generated_digest
+          logger.info("Generated and stored digest using algorithm '#{alg}' for file #{path}")
+        else
+          # Compare the new digest to the one on record
+          if existing_digest == generated_digest
+            logger.info("Fixity check using algorithm '#{alg}' succeeded for file #{path}")
+          else
+            raise ChecksumMismatchError.new("Fixity check using algorithm '#{alg}' failed for file #{path}: expected '#{existing_digest}', calculated '#{generated_digest}.'")
+          end
+        end
+      end
+    end
+
+    # Determine if this service is applicable for the provided event, given the configured service definition
+    #
+    # @param event [String] name of the event
+    # @return [Boolean] returns true if this service is applicable for the provided event
+    def is_applicable?(event)
+      case event
+      when EventNames::PRESERVE
+        true
+      else
+        false
+      end
+    end
+  end
+end