longleaf 0.1.0.pre.2

data/lib/longleaf/models/metadata_record.rb

@@ -0,0 +1,57 @@
+require_relative 'md_fields'
+require_relative 'service_record'
+
+# Metadata record for a single file
+module Longleaf
+  class MetadataRecord
+    attr_reader :deregistered, :registered
+    attr_reader :checksums
+    attr_reader :properties
+    attr_accessor :file_size, :last_modified
+    
+    # @param properties [Hash] initial data properties for this record
+    # @param services [Hash] initial service property tree
+    # @param deregistered [String] deregistered timestamp
+    # @param registered [String] registered timestamp
+    # @param checksums [Hash] hash of checksum values
+    # @param file_size [Integer] size of file in bytes
+    # @param last_modified [String] iso8601 representation of the last modified date of file
+    def initialize(properties: Hash.new, services: Hash.new, deregistered: nil, registered: nil, checksums: Hash.new,
+          file_size: nil, last_modified: nil)
+      @properties = properties
+      @registered = registered
+      @deregistered = deregistered
+      @checksums = checksums
+      @services = services
+      @file_size = file_size
+      @last_modified = last_modified
+    end
+    
+    # @return [Boolean] true if the record is deregistered
+    def deregistered?
+      !@deregistered.nil?
+    end
+    
+    # Adds a service to this record
+    #
+    # @param name [String] identifier for the service being added
+    # @param service_properties [ServiceRecord] properties for populating the new service
+    def add_service(name, service = Longleaf::ServiceRecord.new)
+      raise ArgumentError.new("Value must be a ServiceRecord object when adding a service") unless service.class == Longleaf::ServiceRecord
+      raise IndexError.new("Service with name '#{name}' already exists") if @services.key?(name)
+      
+      @services[name] = service
+    end
+    
+    # @param name [String] name identifier of the service to retrieve
+    # @return [ServiceRecord] the ServiceRecord for the service identified by name, or nil
+    def service(name)
+      @services[name]
+    end
+    
+    # @return [Array<String>] a list of name identifiers for services registered to this record
+    def list_services
+      @services.keys
+    end
+  end
+end

data/lib/longleaf/models/service_definition.rb

@@ -0,0 +1,21 @@
+require_relative 'service_fields'
+
+# Definition of a preservation service
+module Longleaf
+  class ServiceDefinition
+    attr_reader :name
+    attr_reader :work_script
+    attr_reader :frequency, :delay
+    attr_reader :properties
+    
+    def initialize(name:, work_script:, frequency: nil, delay: nil, properties: Hash.new)
+      raise ArgumentError.new("Parameters name and work_script are required") unless name && work_script
+      
+      @properties = properties
+      @name = name
+      @work_script = work_script
+      @frequency = frequency
+      @delay = delay
+    end
+  end
+end

data/lib/longleaf/models/service_fields.rb

@@ -0,0 +1,10 @@
+module Longleaf
+  class ServiceFields
+    WORK_SCRIPT = 'work_script'
+    FREQUENCY = 'frequency'
+    DELAY = 'delay'
+    
+    REPLICATE_TO = 'to'
+    DIGEST_ALGORITHMS = 'algorithms'
+  end
+end

data/lib/longleaf/models/service_record.rb

@@ -0,0 +1,27 @@
+# Record for an individual service in a file's metadata record.
+module Longleaf
+  class ServiceRecord
+    attr_reader :properties
+    attr_accessor :stale_replicas, :timestamp, :run_needed
+    
+    # @param properties [Hash] initial properties for this service record
+    def initialize(properties: Hash.new, stale_replicas: false, timestamp: nil, run_needed: false)
+      raise ArgumentError.new("Service properties must be a hash") if properties.class != Hash
+      
+      @properties = properties
+      @timestamp = timestamp
+      @stale_replicas = stale_replicas
+      @run_needed = run_needed
+    end
+    
+    # @return the value of a service property identified by key
+    def [](key)
+      @properties[key]
+    end
+    
+    # set the value of a service property identified by key
+    def []=(key, value)
+      @properties[key] = value
+    end
+  end
+end

data/lib/longleaf/models/storage_location.rb

@@ -0,0 +1,37 @@
+require 'longleaf/services/metadata_serializer'
+
+module Longleaf
+  class StorageLocation
+    attr_reader :name
+    attr_reader :path
+    attr_reader :metadata_path
+    
+    def initialize(name:, path:, metadata_path:)
+      raise ArgumentError.new("Parameters name, path and metadata_path are required") unless name && path && metadata_path
+      
+      @path = path
+      @name = name
+      @metadata_path = metadata_path
+    end
+    
+    # Get the path for the metadata file for the given file path located in this storage location.
+    # @param file_path [String] path of the file
+    # @raise [ArgumentError] if the file_path is not provided or is not in this storage location.
+    def get_metadata_path_for(file_path)
+      raise ArgumentError.new("A file_path parameter is required") if file_path.nil? || file_path.empty?
+      raise ArgumentError.new("Provided file path is not contained by storage location #{@name}: #{file_path}") \
+          unless file_path.start_with?(@path)
+
+      file_path.sub(/^#{@path}/, metadata_path) + MetadataSerializer::metadata_suffix
+    end
+    
+    # Checks that the path and metadata path defined in this location are available
+    # @raise [StorageLocationUnavailableError] if the storage location is not available
+    def available?
+      raise StorageLocationUnavailableError.new("Path does not exist or is not a directory: #{@path}")\
+          unless Dir.exist?(@path)
+      raise StorageLocationUnavailableError.new("Metadata path does not exist or is not a directory: #{@metadata_path}")\
+          unless Dir.exist?(@metadata_path)
+    end
+  end
+end

data/lib/longleaf/services/application_config_deserializer.rb

@@ -0,0 +1,46 @@
+require 'longleaf/services/application_config_validator'
+require 'longleaf/services/application_config_manager'
+
+# Deserializer for application configuration files
+module Longleaf
+  class ApplicationConfigDeserializer
+    
+    # Deserializes a valid application configuration file as a ApplicationConfigManager option
+    # @param config_path [String] file path to the application configuration file
+    # @param format [String] encoding format of the config file
+    # return [Longleaf::ApplicationConfigManager] manager for the loaded configuration
+    def self.deserialize(config_path, format: 'yaml')
+      config = load(config_path, format: format)
+      
+      Longleaf::ApplicationConfigValidator.validate(config)
+      Longleaf::ApplicationConfigManager.new(config)
+    end
+    
+    # Deserialize a configuration file into a hash
+    # @param config_path [String] file path to the application configuration file
+    # @param format [String] encoding format of the config file
+    # return [Hash] hash containing the configuration
+    def self.load(config_path, format: 'yaml')
+      case format
+      when 'yaml'
+        from_yaml(config_path)
+      else
+        raise ArgumentError.new('Invalid deserialization format #{format} specified')
+      end
+    end
+    
+    private
+    def self.from_yaml(config_path)
+      begin
+        YAML.load_file(config_path)
+      rescue Errno::ENOENT => err
+        raise Longleaf::ConfigurationError.new(
+            "Cannot load application configuration, file #{config_path} does not exist.")
+      rescue => err
+        raise Longleaf::ConfigurationError.new(
+            %Q(Failed to load application configuration due to the following reason:
+            #{err.message}))
+      end
+    end
+  end
+end

data/lib/longleaf/services/application_config_manager.rb

@@ -0,0 +1,24 @@
+require_relative 'storage_location_validator'
+require_relative 'storage_location_manager'
+require_relative 'service_definition_validator'
+require_relative 'service_definition_manager'
+require_relative 'service_mapping_validator'
+require_relative 'service_mapping_manager'
+require_relative 'service_manager'
+
+# Manager which loads and provides access to the configuration of the application
+module Longleaf
+  class ApplicationConfigManager
+    attr_reader :service_manager
+    attr_reader :location_manager
+    
+    def initialize(config)
+      @location_manager = Longleaf::StorageLocationManager.new(config)
+      
+      definition_manager = Longleaf::ServiceDefinitionManager.new(config)
+      mapping_manager = Longleaf::ServiceMappingManager.new(config)
+      @service_manager = Longleaf::ServiceManager.new(
+          definition_manager: definition_manager, mapping_manager: mapping_manager)
+    end
+  end
+end

data/lib/longleaf/services/application_config_validator.rb

@@ -0,0 +1,18 @@
+require_relative 'storage_location_validator'
+require_relative 'service_definition_validator'
+require_relative 'service_mapping_validator'
+
+# Validator for Longleaf application configuration
+module Longleaf
+  class ApplicationConfigValidator
+    
+    # Validates the application configuration provided. Will raise ConfigurationError
+    # if any portion of the configuration is not syntactically or semantically valid.
+    # @param config [Hash] application configuration
+    def self.validate(config)
+      Longleaf::StorageLocationValidator::validate_config(config)
+      Longleaf::ServiceDefinitionValidator::validate_config(config)
+      Longleaf::ServiceMappingValidator::validate_config(config)
+    end
+  end
+end

data/lib/longleaf/services/configuration_validator.rb

@@ -0,0 +1,8 @@
+module Longleaf
+  class ConfigurationValidator
+    protected
+    def self.assert(fail_message, assertion_passed)
+      raise ConfigurationError.new(fail_message) unless assertion_passed
+    end
+  end
+end

data/lib/longleaf/services/metadata_deserializer.rb

@@ -0,0 +1,68 @@
+require 'yaml'
+require_relative '../models/metadata_record'
+require_relative '../models/md_fields'
+require_relative '../errors'
+
+# Service which deserializes metadata files into MetadataRecord objects
+module Longleaf
+  class MetadataDeserializer
+    MDF = Longleaf::MDFields
+    
+    # Deserialize a file into a MetadataRecord object
+    #
+    # @param file_path [String] path of the file to read. Required.
+    # @param format [String] format the file is stored in. Default is 'yaml'.
+    def self.deserialize(file_path:, format: 'yaml')
+      case format
+      when 'yaml'
+        md = from_yaml(file_path)
+      else
+        raise ArgumentError.new('Invalid deserialization format #{format} specified')
+      end
+      
+      if !md || !md.key?(MDF::DATA) || !md.key?(MDF::SERVICES)
+        raise Longleaf::MetadataError.new("Invalid metadata file, did not contain data or services fields: #{file_path}")
+      end
+      
+      data = Hash.new.merge(md[MDF::DATA])
+      # Extract reserved properties for submission as separate parameters
+      registered = data.delete(MDFields::REGISTERED_TIMESTAMP)
+      deregistered = data.delete(MDFields::DEREGISTERED_TIMESTAMP)
+      checksums = data.delete(MDFields::CHECKSUMS)
+      file_size = data.delete(MDFields::FILE_SIZE)
+      last_modified = data.delete(MDFields::LAST_MODIFIED)
+      
+      services = md[MDF::SERVICES]
+      service_records = Hash.new
+      unless services.nil?
+        services.each do |name, props|
+          raise Longleaf::MetadataError.new("Value of service #{name} must be a hash") unless props.class == Hash
+          
+          service_props = Hash.new.merge(props)
+          
+          stale_replicas = service_props.delete(MDFields::STALE_REPLICAS)
+          timestamp = service_props.delete(MDFields::SERVICE_TIMESTAMP)
+          run_needed = service_props.delete(MDFields::RUN_NEEDED)
+          
+          service_records[name] = ServiceRecord.new(
+              properties: service_props,
+              stale_replicas: stale_replicas,
+              timestamp: timestamp,
+              run_needed: run_needed)
+        end
+      end
+      
+      MetadataRecord.new(properties: data,
+          services: service_records,
+          registered: registered,
+          deregistered: deregistered,
+          checksums: checksums,
+          file_size: file_size,
+          last_modified: last_modified)
+    end
+    
+    def self.from_yaml(file_path)
+      YAML.load_file(file_path)
+    end
+  end
+end

data/lib/longleaf/services/metadata_serializer.rb

@@ -0,0 +1,76 @@
+require 'yaml'
+require 'longleaf/models/metadata_record'
+require 'longleaf/models/md_fields'
+require 'pathname'
+
+# Service which serializes MetadataRecord objects
+module Longleaf
+  class MetadataSerializer
+    MDF = Longleaf::MDFields
+    
+    # Serialize the contents of the provided metadata record to the specified path
+    #
+    # @param metadata [MetadataRecord] metadata record to serialize. Required.
+    # @param file_path [String] path to write the file to. Required.
+    # @param format [String] format to serialize the metadata in. Default is 'yaml'.
+    def self.write(metadata:, file_path:, format: 'yaml')
+      raise ArgumentError.new('metadata parameter must be a MetadataRecord') \
+          unless metadata.class == Longleaf::MetadataRecord
+      
+      case format
+      when 'yaml'
+        content = to_yaml(metadata)
+      else
+        raise ArgumentError.new('Invalid serialization format #{format} specified')
+      end
+      
+      # Fill in parent directories if they do not exist
+      parent_dir = Pathname(file_path).parent
+      parent_dir.mkpath unless parent_dir.exist?
+      
+      File.write(file_path, content)
+    end
+    
+    # @param metadata [MetadataRecord] metadata record to transform
+    # @return [String] a yaml representation of the provided MetadataRecord
+    def self.to_yaml(metadata)
+      props = to_hash(metadata)
+      props.to_yaml
+    end
+    
+    def self.to_hash(metadata)
+      props = Hash.new
+      
+      data = Hash.new.merge(metadata.properties)
+      data[MDF::REGISTERED_TIMESTAMP] = metadata.registered if metadata.registered
+      data[MDF::DEREGISTERED_TIMESTAMP] = metadata.deregistered if metadata.deregistered
+      data[MDF::CHECKSUMS] = metadata.checksums unless metadata.checksums&.empty?
+      data[MDF::FILE_SIZE] = metadata.file_size unless metadata.file_size.nil?
+      data[MDF::LAST_MODIFIED] = metadata.last_modified if metadata.last_modified
+      
+      props[MDF::DATA] = data
+      
+      services = Hash.new
+      metadata.list_services.each do |name|
+        service = metadata.service(name)
+        service[MDF::STALE_REPLICAS] = service.stale_replicas if service.stale_replicas
+        service[MDF::SERVICE_TIMESTAMP] = service.timestamp unless service.timestamp.nil?
+        service[MDF::RUN_NEEDED] = service.run_needed if service.run_needed
+        services[name] = service.properties
+      end
+      
+      props[MDF::SERVICES] = services
+      
+      props
+    end
+    
+    def self.metadata_suffix(format: 'yaml')
+      case format
+      when 'yaml'
+        '-llmd.yaml'
+      else
+        raise ArgumentError.new('Invalid serialization format #{format} specified')
+      end
+    end
+  end
+end

data/lib/longleaf/services/service_definition_manager.rb

@@ -0,0 +1,36 @@
+require_relative '../models/app_fields'
+require_relative '../models/service_definition'
+
+# Manager which loads and provides access to Longleaf::ServiceDefinition objects
+module Longleaf
+  class ServiceDefinitionManager
+    SF ||= Longleaf::ServiceFields
+    AF ||= Longleaf::AppFields
+    
+    attr_reader :services
+    
+    def initialize(config)
+      raise ArgumentError.new("Configuration must be provided") if config.nil? || config.empty?
+
+      services_config = config[AF::SERVICES]
+      raise ArgumentError.new("Services configuration must be provided") if services_config.nil?
+      
+      @services = Hash.new
+      config[AF::SERVICES].each do |name, properties|
+        work_script = properties.delete(SF::WORK_SCRIPT)
+        frequency = properties.delete(SF::FREQUENCY)
+        delay = properties.delete(SF::DELAY)
+        service = Longleaf::ServiceDefinition.new(
+            name: name,
+            work_script: work_script,
+            frequency: frequency,
+            delay: delay,
+            properties: properties)
+        
+        @services[name] = service
+      end
+      @services.freeze
+    end
+    
+  end
+end

data/lib/longleaf/services/service_definition_validator.rb

@@ -0,0 +1,32 @@
+require 'pathname'
+require_relative '../models/service_fields'
+require_relative '../models/app_fields'
+require_relative '../errors'
+require_relative 'configuration_validator'
+
+# Validates application configuration of service definitions
+module Longleaf
+  class ServiceDefinitionValidator < ConfigurationValidator
+    SF ||= Longleaf::ServiceFields
+    AF ||= Longleaf::AppFields
+    
+    # Validates configuration to ensure that it is syntactically correct and does not violate 
+    # schema requirements.
+    # @param config [Hash] hash containing the application configuration
+    def self.validate_config(config)
+      assert("Configuration must be a hash, but a #{config.class} was provided", config.class == Hash)
+      assert("Configuration must contain a root '#{AF::SERVICES}' key", config.key?(AF::SERVICES))
+      services = config[AF::SERVICES]
+      assert("'#{AF::SERVICES}' must be a hash of services", services.class == Hash)
+      
+      existing_paths = Array.new
+      services.each do |name, properties|
+        assert("Name of service definition must be a string, but was of type #{name.class}", name.instance_of?(String))
+        assert("Service definition '#{name}' must be a hash, but a #{properties.class} was provided", properties.is_a?(Hash))
+        
+        work_script = properties[SF::WORK_SCRIPT]
+        assert("Service definition '#{name}' must specify a '#{SF::WORK_SCRIPT}' property", !work_script.nil? && !work_script.empty?)
+      end
+    end
+  end
+end

data/lib/longleaf/services/service_manager.rb

@@ -0,0 +1,21 @@
+# Manager which provides preservation service definitions based on their mappings
+module Longleaf
+  class ServiceManager
+    
+    def initialize(definition_manager:, mapping_manager:)
+      raise ArgumentError.new('Service definition manager required') if definition_manager.nil?
+      raise ArgumentError.new('Service mappings manager required') if mapping_manager.nil?
+      @definition_manager = definition_manager
+      @mapping_manager = mapping_manager
+    end
+    
+    # Gets a list of ServiceDefinition objects which match the given criteria
+    # @param location [String] name of the location to lookup
+    # @return [Array] a list of ServiceDefinition objects associated with the location,
+    #    or an empty list if no services match the criteria
+    def list_service_definitions(location: nil)
+      service_names = @mapping_manager.list_services(location)
+      service_names.collect { |name| @definition_manager.services[name] }
+    end
+  end
+end

data/lib/longleaf/services/service_mapping_manager.rb

@@ -0,0 +1,47 @@
+require_relative '../models/app_fields'
+require_relative '../models/service_definition'
+
+# Manager which loads and provides access to location to service mappings
+module Longleaf
+  class ServiceMappingManager
+    AF ||= Longleaf::AppFields
+    
+    def initialize(config)
+      raise ArgumentError.new("Configuration must be provided") if config.nil? || config.empty?
+
+      mappings_config = config[AF::SERVICE_MAPPINGS]
+      raise ArgumentError.new("Service mappings configuration must be provided") if mappings_config.nil?
+      
+      @loc_to_services = Hash.new
+      
+      mappings_config.each do |mapping|
+        locations = mapping[AF::LOCATIONS]
+        services = mapping[AF::SERVICES]
+        
+        locations = [locations] if locations.is_a?(String)
+        services = [services] if services.is_a?(String)
+        
+        locations.each do |loc_name|
+          @loc_to_services[loc_name] = Array.new unless @loc_to_services.key?(loc_name)
+          
+          service_set = @loc_to_services[loc_name]
+          if services.is_a?(String)
+            service_set.push(services)
+          else
+            service_set.concat(services)
+          end
+        end
+      end
+      
+      @loc_to_services.each { |loc, services| services.uniq! }
+      @loc_to_services.freeze
+    end
+    
+    # Gets a list of service names associated with the given location
+    # @param loc_name [String] name of the location to lookup
+    # @return [Array] a list of service names associated with the location
+    def list_services(loc_name)
+      @loc_to_services[loc_name] || []
+    end
+  end
+end

data/lib/longleaf/services/service_mapping_validator.rb

@@ -0,0 +1,49 @@
+require 'pathname'
+require_relative '../models/service_fields'
+require_relative '../models/app_fields'
+require_relative '../errors'
+require_relative 'configuration_validator'
+
+# Validates application configuration of service to location mappings
+module Longleaf
+  class ServiceMappingValidator < ConfigurationValidator
+    AF ||= Longleaf::AppFields
+    
+    # Validates service mapping configuration to ensure that it is syntactically and referentially correct.
+    # @param config [Hash] hash containing the application configuration
+    def self.validate_config(config)
+      
+      assert("Configuration must be a hash, but a #{config.class} was provided", config.class == Hash)
+      assert("Configuration must contain a root '#{AF::SERVICE_MAPPINGS}' key", config.key?(AF::SERVICE_MAPPINGS))
+      mappings = config[AF::SERVICE_MAPPINGS]
+      return if mappings.nil? || mappings.empty?
+      assert("'#{AF::SERVICE_MAPPINGS}' must be an array of mappings", mappings.is_a?(Array))
+      
+      service_names = config[AF::SERVICES].keys
+      location_names = config[AF::LOCATIONS].keys
+      
+      existing_paths = Array.new
+      mappings.each do |mapping|
+        assert("Mapping must be a hash, but received #{mapping.inspect} instead", mapping.is_a?(Hash))
+        
+        validate_mapping_field(AF::LOCATIONS, mapping, location_names)
+        validate_mapping_field(AF::SERVICES, mapping, service_names)
+      end
+    end
+    
+    private
+    def self.validate_mapping_field(field, mapping, valid_values)
+      assert("Mapping must contain a '#{field}' field", mapping.key?(field))
+      field_values = mapping[field]
+      assert("Mapping '#{field}' field must be either a string or an array, but received '#{field_values.inspect}' instead",
+          field_values.is_a?(Array) || field_values.is_a?(String))
+      assert("Mapping must specify one or more value in the '#{field}' field", !field_values.empty?)
+      
+      check_values = field_values.is_a?(String) ? [field_values] : field_values
+      check_values.each do |value|
+        assert("Mapping '#{field}' specifies value '#{value}', but no #{field} with that name exist",
+            valid_values.include?(value))
+      end
+    end
+  end
+end

data/lib/longleaf/services/storage_location_manager.rb

@@ -0,0 +1,37 @@
+require_relative '../models/app_fields'
+require_relative '../models/storage_location'
+
+# Manager which loads and provides access to Longleaf::StorageLocation objects
+module Longleaf
+  class StorageLocationManager
+    AF ||= Longleaf::AppFields
+    
+    attr_reader :locations
+    
+    def initialize(config)
+      raise ArgumentError.new("Configuration must be provided") if config&.empty?
+
+      @locations = Hash.new
+      config[AF::LOCATIONS].each do |name, properties|
+        path = properties[AF::LOCATION_PATH]
+        md_path = properties[AF::METADATA_PATH]
+        location = Longleaf::StorageLocation.new(name: name, path: path, metadata_path: md_path)
+        
+        @locations[name] = location
+      end
+      @locations.freeze
+    end
+    
+    # Get the StorageLocation object which should contain the given path
+    # @return [Longleaf::StorageLocation] location containing the given path
+    #    or nil if the path is not contained by a registered location.
+    def get_location_by_path(path)
+      raise ArgumentError.new("Path parameter is required") if path.nil? || path.empty?
+      @locations.each do |name, location|
+        return location if path.start_with?(location.path) 
+      end
+      
+      nil
+    end
+  end
+end