longleaf 0.2.0.pre.1 → 0.3.0

data/lib/longleaf/services/metadata_persistence_manager.rb

@@ -0,0 +1,46 @@
+require 'longleaf/services/metadata_serializer'
+require 'longleaf/errors'
+
+module Longleaf
+  # Handles the persistence of metadata records
+  class MetadataPersistenceManager
+    # Initialize the MetadataPersistenceManager
+    # @param index_manager [IndexManager] system config manager
+    def initialize(index_manager)
+      @index_manager = index_manager
+    end
+
+    # Persist the metadata for the provided file record to all configured destinations.
+    # This may include to disk as well as to an index.
+    # @param file_rec [FileRecord] file record
+    def persist(file_rec)
+      if file_rec.metadata_record.nil?
+        raise MetadataError.new("No metadata record provided, cannot persist metadata for #{file_rec.path}")
+      end
+
+      MetadataSerializer::write(metadata: file_rec.metadata_record,
+          file_path: file_rec.metadata_path,
+          digest_algs: file_rec.storage_location.metadata_digests)
+
+      index(file_rec)
+    end
+
+    # Index metadata for the provided file record
+    # @param file_rec [FileRecord] file record
+    def index(file_rec)
+      if @index_manager.using_index?
+        @index_manager.index(file_rec)
+      end
+    end
+
+    # Load the metadata record for the provided file record
+    # @param file_rec [FileRecord] file record
+    # @return [MetadataRecord] the metadata record for the file record
+    def load(file_rec)
+      md_rec = MetadataDeserializer.deserialize(file_path: file_rec.metadata_path,
+                  digest_algs: file_rec.storage_location.metadata_digests)
+      file_rec.metadata_record = md_rec
+      md_rec
+    end
+  end
+end

data/lib/longleaf/services/metadata_serializer.rb

@@ -11,7 +11,7 @@ module Longleaf
   class MetadataSerializer
     extend Longleaf::Logging
     MDF ||= MDFields
-    
+
     # Serialize the contents of the provided metadata record to the specified path
     #
     # @param metadata [MetadataRecord] metadata record to serialize. Required.
@@ -22,57 +22,57 @@ module Longleaf
     def self.write(metadata:, file_path:, format: 'yaml', digest_algs: [])
       raise ArgumentError.new('metadata parameter must be a MetadataRecord') \
           unless metadata.class == MetadataRecord
-      
+
       case format
       when 'yaml'
         content = to_yaml(metadata)
       else
-        raise ArgumentError.new('Invalid serialization format #{format} specified')
+        raise ArgumentError.new("Invalid serialization format #{format} specified")
       end
-      
+
       # Fill in parent directories if they do not exist
       parent_dir = Pathname(file_path).parent
       parent_dir.mkpath unless parent_dir.exist?
-      
+
       File.write(file_path, content)
       write_digests(file_path, content, digest_algs)
     end
-    
+
     # @param metadata [MetadataRecord] metadata record to transform
     # @return [String] a yaml representation of the provided MetadataRecord
     def self.to_yaml(metadata)
       props = to_hash(metadata)
       props.to_yaml
     end
-    
+
     # Create a hash representation of the given MetadataRecord file
     # @param metadata [MetadataRecord] metadata record to transform into a hash
     def self.to_hash(metadata)
       props = Hash.new
-      
+
       data = Hash.new.merge(metadata.properties)
       data[MDF::REGISTERED_TIMESTAMP] = metadata.registered if metadata.registered
       data[MDF::DEREGISTERED_TIMESTAMP] = metadata.deregistered if metadata.deregistered
-      data[MDF::CHECKSUMS] = metadata.checksums unless metadata.checksums&.empty?
+      data[MDF::CHECKSUMS] = metadata.checksums unless metadata.checksums && metadata.checksums.empty?
       data[MDF::FILE_SIZE] = metadata.file_size unless metadata.file_size.nil?
       data[MDF::LAST_MODIFIED] = metadata.last_modified if metadata.last_modified
-      
+
       props[MDF::DATA] = data
-      
+
       services = Hash.new
       metadata.list_services.each do |name|
         service = metadata.service(name)
         service[MDF::STALE_REPLICAS] = service.stale_replicas if service.stale_replicas
         service[MDF::SERVICE_TIMESTAMP] = service.timestamp unless service.timestamp.nil?
         service[MDF::RUN_NEEDED] = service.run_needed if service.run_needed
-        services[name] = service.properties
+        services[name] = service.properties unless service.properties.empty?
       end
-      
+
       props[MDF::SERVICES] = services
-      
+
       props
     end
-    
+
     # @param format [String] encoding format used for metadata file
     # @return [String] the suffix used to indicate that a file is a metadata file in the provided encoding
     # @raise [ArgumentError] raised if the provided format is not a supported metadata encoding format
@@ -81,14 +81,13 @@ module Longleaf
       when 'yaml'
         '-llmd.yaml'
       else
-        raise ArgumentError.new('Invalid serialization format #{format} specified')
+        raise ArgumentError.new("Invalid serialization format #{format} specified")
       end
     end
-    
-    private
+
     def self.write_digests(file_path, content, digests)
       return if digests.nil? || digests.empty?
-      
+
       digests.each do |alg|
         digest_class = DigestHelper::start_digest(alg)
         result = digest_class.hexdigest(content)
@@ -97,11 +96,13 @@ module Longleaf
         else
           digest_path = "#{file_path}.#{alg}"
         end
-        
+
         File.write(digest_path, result)
-        
+
         self.logger.debug("Generated #{alg} digest for metadata file #{file_path}: #{result}")
       end
     end
+
+    private_class_method :write_digests
   end
-end
+end

data/lib/longleaf/services/service_class_cache.rb

@@ -4,7 +4,7 @@ module Longleaf
   # Cache for loading and retrieving preservation service classes
   class ServiceClassCache
     STD_PRESERVATION_SERVICE_PATH = 'longleaf/preservation_services/'
-    
+
     def initialize(app_manager)
       @app_manager = app_manager
       # Cache storing per service definition instances of service classes
@@ -12,7 +12,7 @@ module Longleaf
       # Cache storing per script path class of service
       @class_cache = Hash.new
     end
-    
+
     # Returns an instance of the preversation service defined for the provided service definition,
     # based on the work_script and work_class properties provided.
     #
@@ -24,12 +24,12 @@ module Longleaf
       if @service_instance_cache.key?(service_name)
         return @service_instance_cache[service_name]
       end
-      
+
       clazz = service_class(service_def)
       # Cache and return the class instance
       @service_instance_cache[service_name] = clazz.new(service_def, @app_manager)
     end
-    
+
     # Load and return the PreservationService class assigned to the provided service definition,
     # based on the work_script and work_class properties provided.
     #
@@ -38,13 +38,13 @@ module Longleaf
     def service_class(service_def)
       service_name = service_def.name
       work_script = service_def.work_script
-      
+
       if work_script.include?('/')
         expanded_path = Pathname.new(work_script).expand_path.to_s
         if !from_permitted_path?(expanded_path)
           raise ConfigurationError.new("Unable to load work_script for service #{service_name}, #{work_script} is not in a known library path.")
         end
-        
+
         last_slash_index = work_script.rindex('/')
         script_path = work_script[0..last_slash_index]
         script_name = work_script[(last_slash_index + 1)..-1]
@@ -52,23 +52,23 @@ module Longleaf
         script_path = STD_PRESERVATION_SERVICE_PATH
         script_name = work_script
       end
-      
+
       # Strip off the extension
       script_name.sub!('.rb', '')
-      
+
       require_path = File.join(script_path, script_name)
       # Return the cached Class if this path has been encountered before
       if @class_cache.key?(require_path)
         return @class_cache[require_path]
       end
-      
+
       # Load the script
       begin
         require require_path
-      rescue LoadError => e
+      rescue LoadError
         raise ConfigurationError.new("Failed to load work_script '#{script_name}' for service #{service_name}")
       end
-      
+
       # Generate the class name, either configured or from file naming convention if possible
       if service_def.work_class
         class_name = service_def.work_class
@@ -77,7 +77,7 @@ module Longleaf
         # Assume the longleaf module for classes in the standard path
         class_name = 'Longleaf::' + class_name if script_path == STD_PRESERVATION_SERVICE_PATH
       end
-      
+
       begin
         class_constant = constantize(class_name)
         # cache the class for this work_script and return it
@@ -86,7 +86,7 @@ module Longleaf
         raise ConfigurationError.new("Failed to load work_script '#{script_name}' for service #{service_name}, class name #{class_name} was not found.")
       end
     end
-    
+
     private
     # Borrowed from sidekiq implementation
     def constantize(str)
@@ -99,7 +99,7 @@ module Longleaf
         constant.const_defined?(name, false) ? constant.const_get(name, false) : constant.const_missing(name)
       end
     end
-    
+
     def from_permitted_path?(script_path)
       $LOAD_PATH.each do |lib_path|
         if script_path.start_with?(lib_path)
@@ -109,4 +109,4 @@ module Longleaf
       false
     end
   end
-end
+end

data/lib/longleaf/services/service_definition_manager.rb

@@ -6,17 +6,17 @@ module Longleaf
   class ServiceDefinitionManager
     SF ||= Longleaf::ServiceFields
     AF ||= Longleaf::AppFields
-    
+
     # Hash containing the set of configured services, represented as {ServiceDefinition} objects
     attr_reader :services
-    
+
     # @param config [Hash] hash representation of the application configuration
     def initialize(config)
       raise ArgumentError.new("Configuration must be provided") if config.nil? || config.empty?
 
       services_config = config[AF::SERVICES]
       raise ArgumentError.new("Services configuration must be provided") if services_config.nil?
-      
+
       @services = Hash.new
       config[AF::SERVICES].each do |name, properties|
         work_script = properties.delete(SF::WORK_SCRIPT)
@@ -30,11 +30,10 @@ module Longleaf
             frequency: frequency,
             delay: delay,
             properties: properties)
-        
+
         @services[name] = service
       end
       @services.freeze
     end
-    
   end
-end
+end

data/lib/longleaf/services/service_definition_validator.rb

@@ -9,8 +9,8 @@ module Longleaf
   class ServiceDefinitionValidator < ConfigurationValidator
     SF ||= Longleaf::ServiceFields
     AF ||= Longleaf::AppFields
-    
-    # Validates configuration to ensure that it is syntactically correct and does not violate 
+
+    # Validates configuration to ensure that it is syntactically correct and does not violate
     # schema requirements.
     # @param config [Hash] hash containing the application configuration
     def self.validate_config(config)
@@ -18,15 +18,14 @@ module Longleaf
       assert("Configuration must contain a root '#{AF::SERVICES}' key", config.key?(AF::SERVICES))
       services = config[AF::SERVICES]
       assert("'#{AF::SERVICES}' must be a hash of services", services.class == Hash)
-      
-      existing_paths = Array.new
+
       services.each do |name, properties|
         assert("Name of service definition must be a string, but was of type #{name.class}", name.instance_of?(String))
         assert("Service definition '#{name}' must be a hash, but a #{properties.class} was provided", properties.is_a?(Hash))
-        
+
         work_script = properties[SF::WORK_SCRIPT]
         assert("Service definition '#{name}' must specify a '#{SF::WORK_SCRIPT}' property", !work_script.nil? && !work_script.empty?)
       end
     end
   end
-end
+end

data/lib/longleaf/services/service_manager.rb

@@ -4,6 +4,9 @@ require 'longleaf/services/service_class_cache'
 module Longleaf
   # Manager which provides preservation service definitions based on their mappings
   class ServiceManager
+    attr_reader :definition_manager
+    attr_reader :mapping_manager
+
     # @param definition_manager [ServiceDefinitionManager] the service definition manager
     # @param mapping_manager [ServiceMappingManager] the mapping of services to locations
     # @param app_manager [ApplicationConfigManager] manager for storage locations
@@ -16,7 +19,18 @@ module Longleaf
       @app_manager = app_manager
       @service_class_cache = ServiceClassCache.new(app_manager)
     end
-    
+
+    # Return a service instance instance for provided service name.
+    # @param service_name [String] name of the service
+    # @return Preservation service class for the provided name
+    # @raise ArgumentError if service_name does not reference an existing service
+    def service(service_name)
+      raise ArgumentError.new('Service name is required') if service_name.nil? || service_name.empty?
+      raise ArgumentError.new("No service with name #{service_name}") unless @definition_manager.services.key?(service_name)
+      definition = @definition_manager.services[service_name]
+      @service_class_cache.service_instance(definition)
+    end
+
     # List the names of services which are applicable to the given criteria
     # @param location [String] name of the locations to lookup
     # @param event [String] name of the preservation event taking place
@@ -25,23 +39,29 @@ module Longleaf
       service_names = @mapping_manager.list_services(location)
       if !event.nil?
         # Filter service names down by event
-        service_names.select{ |name| applicable_for_event?(name, event) }
+        service_names.select { |name| applicable_for_event?(name, event) }
       else
         service_names
       end
     end
-    
+
+    # List definitions for services which are applicable to the given criteria
+    # @param location [String] name of the locations to lookup
+    # @param event [String] name of the preservation event taking place
+    # @return [Array] List of service definitions which match the provided criteria
+    def list_service_definitions(location: nil, event: nil)
+      names = list_services(location: location, event: event)
+      names.map { |name| @definition_manager.services[name] }
+    end
+
     # Determines if a service is applicable for a specific preservation event
     # @param service_name [String] name of the service being evaluated
     # @param event [String] name of the event to check against
     # @return [Boolean] true if the service is applicable for the event
     def applicable_for_event?(service_name, event)
-      definition = @definition_manager.services[service_name]
-      service = @service_class_cache.service_instance(definition)
-      
-      service.is_applicable?(event)
+      service(service_name).is_applicable?(event)
     end
-    
+
     # Determine if a service should run for a particular file based on the service's definition and
     # the file's service related metadata.
     # @param service_name [String] name of the service being evaluated
@@ -51,34 +71,34 @@ module Longleaf
       # If service not recorded for file, then it is needed
       present_services = md_rec.list_services
       return true unless present_services.include?(service_name)
-      
+
       service_rec = md_rec.service(service_name)
-      
+
       return true if service_rec.run_needed
       return true if service_rec.timestamp.nil?
-      
+
       definition = @definition_manager.services[service_name]
-      
+
       # Check if the amount of time defined in frequency has passed since the service timestamp
       frequency = definition.frequency
       unless frequency.nil?
         service_timestamp = service_rec.timestamp
         now = ServiceDateHelper.formatted_timestamp
-        
+
         return true if now > ServiceDateHelper.add_to_timestamp(service_timestamp, frequency)
       end
       false
     end
-    
+
     # Perform the specified service on the file record, in the context of the specified event
     # @param service_name [String] name of the service
     # @param file_rec [FileRecord] file record to perform service upon
-    # @param event_name [String] name of the event service is being performed within.
+    # @param event [String] name of the event service is being performed within.
     def perform_service(service_name, file_rec, event)
       definition = @definition_manager.services[service_name]
-      
+
       service = @service_class_cache.service_instance(definition)
       service.perform(file_rec, event)
     end
   end
-end
+end

data/lib/longleaf/services/service_mapping_manager.rb

@@ -5,26 +5,26 @@ module Longleaf
   # Manager which loads and provides access to location to service mappings
   class ServiceMappingManager
     AF ||= Longleaf::AppFields
-    
+
     # @param config [Hash] has representation of the application configuration
     def initialize(config)
       raise ArgumentError.new("Configuration must be provided") if config.nil? || config.empty?
 
       mappings_config = config[AF::SERVICE_MAPPINGS]
       raise ArgumentError.new("Service mappings configuration must be provided") if mappings_config.nil?
-      
+
       @loc_to_services = Hash.new
-      
+
       mappings_config.each do |mapping|
         locations = mapping[AF::LOCATIONS]
         services = mapping[AF::SERVICES]
-        
+
         locations = [locations] if locations.is_a?(String)
         services = [services] if services.is_a?(String)
-        
+
         locations.each do |loc_name|
           @loc_to_services[loc_name] = Array.new unless @loc_to_services.key?(loc_name)
-          
+
           service_set = @loc_to_services[loc_name]
           if services.is_a?(String)
             service_set.push(services)
@@ -33,11 +33,11 @@ module Longleaf
           end
         end
       end
-      
+
       @loc_to_services.each { |loc, services| services.uniq! }
       @loc_to_services.freeze
     end
-    
+
     # Gets a list of service names associated with the given location
     # @param loc_name [String] name of the location to lookup
     # @return [Array] a list of service names associated with the location
@@ -45,4 +45,4 @@ module Longleaf
       @loc_to_services[loc_name] || []
     end
   end
-end
+end