longleaf 0.1.0.pre.2 → 1.0.0

data/lib/longleaf/services/metadata_validator.rb

@@ -0,0 +1,76 @@
+require 'pathname'
+require 'longleaf/models/md_fields'
+require 'longleaf/errors'
+require_relative 'configuration_validator'
+
+module Longleaf
+  # Validator for file metadata
+  class MetadataValidator < ConfigurationValidator
+    MDF ||= MDFields
+
+    # @param config [Hash] hash containing the application configuration
+    def initialize(config)
+      super(config)
+    end
+
+    protected
+    # Validates the provided metadata for a file to ensure that it is syntactically correct and field types
+    # are validate.
+    def validate
+      assert("Metadata must be a hash, but a #{@config.class} was provided", @config.class == Hash)
+      assert("Metadata must contain a '#{MDF::DATA}' key", @config.key?(MDF::DATA))
+      assert("Metadata must contain a '#{MDF::SERVICES}' key", @config.key?(MDF::SERVICES))
+
+      data = @config[MDF::DATA]
+      register_on_failure { validate_date_field(data, MDF::REGISTERED_TIMESTAMP) }
+      register_on_failure { validate_date_field(data, MDF::DEREGISTERED_TIMESTAMP, required: false) }
+      register_on_failure { validate_date_field(data, MDF::LAST_MODIFIED) }
+
+      register_on_failure { validate_positive_integer(data, MDF::FILE_SIZE) }
+
+      checksums = data[MDF::CHECKSUMS]
+      register_on_failure do
+        if !checksums.nil? && !checksums.is_a?(Hash)
+          fail("Field '#{MDF::CHECKSUMS}' must be a map of algorithms to digests, but was a #{checksums.class}")
+        end
+      end
+
+      # Ensure that any service timestamps present are valid dates
+      services = @config[MDF::SERVICES]
+      services.each do |service_name, service_rec|
+        register_on_failure { validate_date_field(service_rec, MDF::SERVICE_TIMESTAMP, required: false) }
+      end
+    end
+
+    def validate_date_field(section, field_key, required: true)
+      field_val = section[field_key]
+
+      if field_val
+        begin
+          Time.iso8601(section[field_key])
+        rescue ArgumentError
+          fail("Field '#{field_key}' must be a valid ISO8601 timestamp, but contained value '#{section[field_key]}'")
+        end
+      elsif required
+        fail("Metadata must contain a '#{field_key}' field")
+      end
+    end
+
+    def validate_positive_integer(section, field_key, required: true)
+      field_val = section[field_key]
+
+      if field_val
+        begin
+          val = field_val.is_a?(Integer) ? field_val : Integer(field_val, 10)
+          if val < 0
+            fail("Field '#{field_key}' must be a positive integer")
+          end
+        rescue ArgumentError => err
+          fail("Field '#{field_key}' must be a positive integer")
+        end
+      elsif required
+        fail("Metadata must contain a '#{field_key}' field")
+      end
+    end
+  end
+end

data/lib/longleaf/services/s3_location_validator.rb

@@ -0,0 +1,19 @@
+require 'pathname'
+require 'longleaf/errors'
+require 'longleaf/helpers/s3_uri_helper'
+
+module Longleaf
+  # Validates the configuration of a s3 based location
+  class S3LocationValidator
+    def self.validate(p_validator, name, path_prop, section_name, path)
+      base_msg = "Storage location '#{name}' specifies invalid #{section_name} '#{path_prop}' property: "
+      p_validator.assert(base_msg + 'Path must not be empty', !path.nil? && !path.to_s.strip.empty?)
+      begin
+        bucket_name = S3UriHelper.extract_bucket(path)
+        p_validator.assert(base_msg + 'Path must specify a bucket', !bucket_name.nil?)
+      rescue ArgumentError => e
+        p_validator.fail(base_msg + e.message)
+      end
+    end
+  end
+end

data/lib/longleaf/services/service_class_cache.rb

@@ -0,0 +1,112 @@
+require 'pathname'
+
+module Longleaf
+  # Cache for loading and retrieving preservation service classes
+  class ServiceClassCache
+    STD_PRESERVATION_SERVICE_PATH = 'longleaf/preservation_services/'
+
+    def initialize(app_manager)
+      @app_manager = app_manager
+      # Cache storing per service definition instances of service classes
+      @service_instance_cache = Hash.new
+      # Cache storing per script path class of service
+      @class_cache = Hash.new
+    end
+
+    # Returns an instance of the preversation service defined for the provided service definition,
+    # based on the work_script and work_class properties provided.
+    #
+    # @param service_def [ServiceDefinition] definition of service to instantiate
+    # @return [PreservationService] Instance of the preservation service class for the definition.
+    def service_instance(service_def)
+      service_name = service_def.name
+      # Return the cached instance of the service
+      if @service_instance_cache.key?(service_name)
+        return @service_instance_cache[service_name]
+      end
+
+      clazz = service_class(service_def)
+      # Cache and return the class instance
+      @service_instance_cache[service_name] = clazz.new(service_def, @app_manager)
+    end
+
+    # Load and return the PreservationService class assigned to the provided service definition,
+    # based on the work_script and work_class properties provided.
+    #
+    # @param service_def [ServiceDefinition] definition of service to retrieve class for
+    # @return [Class] class of work_script
+    def service_class(service_def)
+      service_name = service_def.name
+      work_script = service_def.work_script
+
+      if work_script.include?('/')
+        expanded_path = Pathname.new(work_script).expand_path.to_s
+        if !from_permitted_path?(expanded_path)
+          raise ConfigurationError.new("Unable to load work_script for service #{service_name}, #{work_script} is not in a known library path.")
+        end
+
+        last_slash_index = work_script.rindex('/')
+        script_path = work_script[0..last_slash_index]
+        script_name = work_script[(last_slash_index + 1)..-1]
+      else
+        script_path = STD_PRESERVATION_SERVICE_PATH
+        script_name = work_script
+      end
+
+      # Strip off the extension
+      script_name.sub!('.rb', '')
+
+      require_path = File.join(script_path, script_name)
+      # Return the cached Class if this path has been encountered before
+      if @class_cache.key?(require_path)
+        return @class_cache[require_path]
+      end
+
+      # Load the script
+      begin
+        require require_path
+      rescue LoadError
+        raise ConfigurationError.new("Failed to load work_script '#{script_name}' for service #{service_name}")
+      end
+
+      # Generate the class name, either configured or from file naming convention if possible
+      if service_def.work_class
+        class_name = service_def.work_class
+      else
+        class_name = script_name.split('_').map(&:capitalize).join
+        # Assume the longleaf module for classes in the standard path
+        class_name = 'Longleaf::' + class_name if script_path == STD_PRESERVATION_SERVICE_PATH
+      end
+
+      begin
+        class_constant = constantize(class_name)
+        # cache the class for this work_script and return it
+        @class_cache[require_path] = class_constant
+      rescue NameError
+        raise ConfigurationError.new("Failed to load work_script '#{script_name}' for service #{service_name}, class name #{class_name} was not found.")
+      end
+    end
+
+    private
+    # Borrowed from sidekiq implementation
+    def constantize(str)
+      names = str.split('::')
+      names.shift if names.empty? || names.first.empty?
+
+      names.inject(Object) do |constant, name|
+        # the false flag limits search for name to under the constant namespace
+        #   which mimics Rails' behaviour
+        constant.const_defined?(name, false) ? constant.const_get(name, false) : constant.const_missing(name)
+      end
+    end
+
+    def from_permitted_path?(script_path)
+      $LOAD_PATH.each do |lib_path|
+        if script_path.start_with?(lib_path)
+          return true
+        end
+      end
+      false
+    end
+  end
+end

data/lib/longleaf/services/service_definition_manager.rb

@@ -1,36 +1,39 @@
 require_relative '../models/app_fields'
 require_relative '../models/service_definition'
 
-# Manager which loads and provides access to Longleaf::ServiceDefinition objects
 module Longleaf
+  # Manager which loads and provides access to Longleaf::ServiceDefinition objects
   class ServiceDefinitionManager
     SF ||= Longleaf::ServiceFields
     AF ||= Longleaf::AppFields
-    
+
+    # Hash containing the set of configured services, represented as {ServiceDefinition} objects
     attr_reader :services
-    
+
+    # @param config [Hash] hash representation of the application configuration
     def initialize(config)
       raise ArgumentError.new("Configuration must be provided") if config.nil? || config.empty?
 
       services_config = config[AF::SERVICES]
       raise ArgumentError.new("Services configuration must be provided") if services_config.nil?
-      
+
       @services = Hash.new
       config[AF::SERVICES].each do |name, properties|
         work_script = properties.delete(SF::WORK_SCRIPT)
+        work_class = properties.delete(SF::WORK_CLASS)
         frequency = properties.delete(SF::FREQUENCY)
         delay = properties.delete(SF::DELAY)
         service = Longleaf::ServiceDefinition.new(
             name: name,
             work_script: work_script,
+            work_class: work_class,
             frequency: frequency,
             delay: delay,
             properties: properties)
-        
+
         @services[name] = service
       end
       @services.freeze
     end
-    
   end
-end
+end

data/lib/longleaf/services/service_definition_validator.rb

@@ -1,32 +1,39 @@
 require 'pathname'
-require_relative '../models/service_fields'
-require_relative '../models/app_fields'
-require_relative '../errors'
+require 'longleaf/models/service_fields'
+require 'longleaf/models/app_fields'
+require 'longleaf/errors'
 require_relative 'configuration_validator'
 
-# Validates application configuration of service definitions
 module Longleaf
+  # Validates application configuration of service definitions
   class ServiceDefinitionValidator < ConfigurationValidator
     SF ||= Longleaf::ServiceFields
     AF ||= Longleaf::AppFields
-    
-    # Validates configuration to ensure that it is syntactically correct and does not violate 
+
+    # @param config [Hash] hash containing the application configuration
+    def initialize(config)
+      super(config)
+    end
+
+    protected
+    # Validates configuration to ensure that it is syntactically correct and does not violate
     # schema requirements.
     # @param config [Hash] hash containing the application configuration
-    def self.validate_config(config)
-      assert("Configuration must be a hash, but a #{config.class} was provided", config.class == Hash)
-      assert("Configuration must contain a root '#{AF::SERVICES}' key", config.key?(AF::SERVICES))
-      services = config[AF::SERVICES]
+    def validate
+      assert("Configuration must be a hash, but a #{@config.class} was provided", @config.class == Hash)
+      assert("Configuration must contain a root '#{AF::SERVICES}' key", @config.key?(AF::SERVICES))
+      services = @config[AF::SERVICES]
       assert("'#{AF::SERVICES}' must be a hash of services", services.class == Hash)
-      
-      existing_paths = Array.new
+
       services.each do |name, properties|
-        assert("Name of service definition must be a string, but was of type #{name.class}", name.instance_of?(String))
-        assert("Service definition '#{name}' must be a hash, but a #{properties.class} was provided", properties.is_a?(Hash))
-        
-        work_script = properties[SF::WORK_SCRIPT]
-        assert("Service definition '#{name}' must specify a '#{SF::WORK_SCRIPT}' property", !work_script.nil? && !work_script.empty?)
+        register_on_failure do
+          assert("Name of service definition must be a string, but was of type #{name.class}", name.instance_of?(String))
+          assert("Service definition '#{name}' must be a hash, but a #{properties.class} was provided", properties.is_a?(Hash))
+
+          work_script = properties[SF::WORK_SCRIPT]
+          assert("Service definition '#{name}' must specify a '#{SF::WORK_SCRIPT}' property", !work_script.nil? && !work_script.empty?)
+        end
       end
     end
   end
-end
+end

data/lib/longleaf/services/service_manager.rb

@@ -1,21 +1,96 @@
-# Manager which provides preservation service definitions based on their mappings
+require 'longleaf/helpers/service_date_helper'
+require 'longleaf/services/service_class_cache'
+
 module Longleaf
+  # Manager which provides preservation service definitions based on their mappings
   class ServiceManager
-    
-    def initialize(definition_manager:, mapping_manager:)
+    attr_reader :definition_manager
+    attr_reader :mapping_manager
+
+    # @param definition_manager [ServiceDefinitionManager] the service definition manager
+    # @param mapping_manager [ServiceMappingManager] the mapping of services to locations
+    # @param app_manager [ApplicationConfigManager] manager for storage locations
+    def initialize(definition_manager:, mapping_manager:, app_manager:)
       raise ArgumentError.new('Service definition manager required') if definition_manager.nil?
       raise ArgumentError.new('Service mappings manager required') if mapping_manager.nil?
+      raise ArgumentError.new('Storage location manager required') if app_manager.nil?
       @definition_manager = definition_manager
       @mapping_manager = mapping_manager
+      @app_manager = app_manager
+      @service_class_cache = ServiceClassCache.new(app_manager)
     end
-    
-    # Gets a list of ServiceDefinition objects which match the given criteria
-    # @param location [String] name of the location to lookup
-    # @return [Array] a list of ServiceDefinition objects associated with the location,
-    #    or an empty list if no services match the criteria
-    def list_service_definitions(location: nil)
+
+    # Return a service instance instance for provided service name.
+    # @param service_name [String] name of the service
+    # @return Preservation service class for the provided name
+    # @raise ArgumentError if service_name does not reference an existing service
+    def service(service_name)
+      raise ArgumentError.new('Service name is required') if service_name.nil? || service_name.empty?
+      raise ArgumentError.new("No service with name #{service_name}") unless @definition_manager.services.key?(service_name)
+      definition = @definition_manager.services[service_name]
+      @service_class_cache.service_instance(definition)
+    end
+
+    # List the names of services which are applicable to the given criteria
+    # @param location [String] name of the locations to lookup
+    # @param event [String] name of the preservation event taking place
+    # @return [Array] a list of service names which match the provided criteria
+    def list_services(location: nil, event: nil)
       service_names = @mapping_manager.list_services(location)
-      service_names.collect { |name| @definition_manager.services[name] }
+      if !event.nil?
+        # Filter service names down by event
+        service_names.select { |name| applicable_for_event?(name, event) }
+      else
+        service_names
+      end
+    end
+
+    # List definitions for services which are applicable to the given criteria
+    # @param location [String] name of the locations to lookup
+    # @param event [String] name of the preservation event taking place
+    # @return [Array] List of service definitions which match the provided criteria
+    def list_service_definitions(location: nil, event: nil)
+      names = list_services(location: location, event: event)
+      names.map { |name| @definition_manager.services[name] }
+    end
+
+    # Determines if a service is applicable for a specific preservation event
+    # @param service_name [String] name of the service being evaluated
+    # @param event [String] name of the event to check against
+    # @return [Boolean] true if the service is applicable for the event
+    def applicable_for_event?(service_name, event)
+      service(service_name).is_applicable?(event)
+    end
+
+    # Determine if a service should run for a particular file based on the service's definition and
+    # the file's service related metadata.
+    # @param service_name [String] name of the service being evaluated
+    # @param md_rec [MetadataRecord] metadata record for the file being evaluated
+    # @return [Boolean] true if the service should be run.
+    def service_needed?(service_name, md_rec)
+      service_rec = md_rec.service(service_name)
+      return true if !service_rec.nil? && service_rec.run_needed
+
+      definition = @definition_manager.services[service_name]
+
+      next_run = ServiceDateHelper.next_run_needed(md_rec, definition)
+
+      return false if next_run.nil?
+
+      # If next run timestamp has passed then service is needed
+      now = ServiceDateHelper.formatted_timestamp
+      now >= next_run
+    end
+
+    # Perform the specified service on the file record, in the context of the specified event
+    # @param service_name [String] name of the service
+    # @param file_rec [FileRecord] file record to perform service upon
+    # @param event [String] name of the event service is being performed within.
+    def perform_service(service_name, file_rec, event)
+      definition = @definition_manager.services[service_name]
+
+      service = @service_class_cache.service_instance(definition)
+      service.perform(file_rec, event)
     end
   end
-end
+end

data/lib/longleaf/services/service_mapping_manager.rb

@@ -1,29 +1,30 @@
-require_relative '../models/app_fields'
-require_relative '../models/service_definition'
+require 'longleaf/models/app_fields'
+require 'longleaf/models/service_definition'
 
-# Manager which loads and provides access to location to service mappings
 module Longleaf
+  # Manager which loads and provides access to location to service mappings
   class ServiceMappingManager
     AF ||= Longleaf::AppFields
-    
+
+    # @param config [Hash] has representation of the application configuration
     def initialize(config)
       raise ArgumentError.new("Configuration must be provided") if config.nil? || config.empty?
 
       mappings_config = config[AF::SERVICE_MAPPINGS]
       raise ArgumentError.new("Service mappings configuration must be provided") if mappings_config.nil?
-      
+
       @loc_to_services = Hash.new
-      
+
       mappings_config.each do |mapping|
         locations = mapping[AF::LOCATIONS]
         services = mapping[AF::SERVICES]
-        
+
         locations = [locations] if locations.is_a?(String)
         services = [services] if services.is_a?(String)
-        
+
         locations.each do |loc_name|
           @loc_to_services[loc_name] = Array.new unless @loc_to_services.key?(loc_name)
-          
+
           service_set = @loc_to_services[loc_name]
           if services.is_a?(String)
             service_set.push(services)
@@ -32,11 +33,11 @@ module Longleaf
           end
         end
       end
-      
+
       @loc_to_services.each { |loc, services| services.uniq! }
       @loc_to_services.freeze
     end
-    
+
     # Gets a list of service names associated with the given location
     # @param loc_name [String] name of the location to lookup
     # @return [Array] a list of service names associated with the location
@@ -44,4 +45,4 @@ module Longleaf
       @loc_to_services[loc_name] || []
     end
   end
-end
+end