longleaf 0.1.0.pre.3 → 1.1.0

data/lib/longleaf/services/metadata_persistence_manager.rb

@@ -0,0 +1,47 @@
+require 'longleaf/services/metadata_serializer'
+require 'longleaf/services/metadata_deserializer'
+require 'longleaf/errors'
+
+module Longleaf
+  # Handles the persistence of metadata records
+  class MetadataPersistenceManager
+    # Initialize the MetadataPersistenceManager
+    # @param index_manager [IndexManager] system config manager
+    def initialize(index_manager)
+      @index_manager = index_manager
+    end
+
+    # Persist the metadata for the provided file record to all configured destinations.
+    # This may include to disk as well as to an index.
+    # @param file_rec [FileRecord] file record
+    def persist(file_rec)
+      if file_rec.metadata_record.nil?
+        raise MetadataError.new("No metadata record provided, cannot persist metadata for #{file_rec.path}")
+      end
+
+      MetadataSerializer::write(metadata: file_rec.metadata_record,
+          file_path: file_rec.metadata_path,
+          digest_algs: file_rec.storage_location.metadata_location.digests)
+
+      index(file_rec)
+    end
+
+    # Index metadata for the provided file record
+    # @param file_rec [FileRecord] file record
+    def index(file_rec)
+      if @index_manager.using_index?
+        @index_manager.index(file_rec)
+      end
+    end
+
+    # Load the metadata record for the provided file record
+    # @param file_rec [FileRecord] file record
+    # @return [MetadataRecord] the metadata record for the file record
+    def load(file_rec)
+      md_rec = MetadataDeserializer.deserialize(file_path: file_rec.metadata_path,
+                  digest_algs: file_rec.storage_location.metadata_location.digests)
+      file_rec.metadata_record = md_rec
+      md_rec
+    end
+  end
+end

data/lib/longleaf/services/metadata_serializer.rb

@@ -1,76 +1,190 @@
 require 'yaml'
 require 'longleaf/models/metadata_record'
 require 'longleaf/models/md_fields'
+require 'longleaf/helpers/digest_helper'
+require 'longleaf/errors'
+require 'longleaf/logging'
 require 'pathname'
+require "tempfile"
 
-# Service which serializes MetadataRecord objects
 module Longleaf
+  # Service which serializes MetadataRecord objects
   class MetadataSerializer
-    MDF = Longleaf::MDFields
-    
+    extend Longleaf::Logging
+    MDF ||= MDFields
+
     # Serialize the contents of the provided metadata record to the specified path
     #
     # @param metadata [MetadataRecord] metadata record to serialize. Required.
     # @param file_path [String] path to write the file to. Required.
     # @param format [String] format to serialize the metadata in. Default is 'yaml'.
-    def self.write(metadata:, file_path:, format: 'yaml')
+    # @param digest_algs [Array] if provided, sidecar digest files for the metadata file
+    #    will be generated for each algorithm.
+    def self.write(metadata:, file_path:, format: 'yaml', digest_algs: [])
       raise ArgumentError.new('metadata parameter must be a MetadataRecord') \
-          unless metadata.class == Longleaf::MetadataRecord
-      
+          unless metadata.class == MetadataRecord
+
       case format
       when 'yaml'
         content = to_yaml(metadata)
       else
-        raise ArgumentError.new('Invalid serialization format #{format} specified')
+        raise ArgumentError.new("Invalid serialization format #{format} specified")
       end
-      
-      # Fill in parent directories if they do not exist
-      parent_dir = Pathname(file_path).parent
-      parent_dir.mkpath unless parent_dir.exist?
-      
-      File.write(file_path, content)
+
+      atomic_write(file_path, content, digest_algs)
     end
-    
+
     # @param metadata [MetadataRecord] metadata record to transform
     # @return [String] a yaml representation of the provided MetadataRecord
     def self.to_yaml(metadata)
       props = to_hash(metadata)
       props.to_yaml
     end
-    
+
+    # Create a hash representation of the given MetadataRecord file
+    # @param metadata [MetadataRecord] metadata record to transform into a hash
     def self.to_hash(metadata)
       props = Hash.new
-      
+
       data = Hash.new.merge(metadata.properties)
       data[MDF::REGISTERED_TIMESTAMP] = metadata.registered if metadata.registered
       data[MDF::DEREGISTERED_TIMESTAMP] = metadata.deregistered if metadata.deregistered
-      data[MDF::CHECKSUMS] = metadata.checksums unless metadata.checksums&.empty?
+      data[MDF::CHECKSUMS] = metadata.checksums unless metadata.checksums && metadata.checksums.empty?
       data[MDF::FILE_SIZE] = metadata.file_size unless metadata.file_size.nil?
       data[MDF::LAST_MODIFIED] = metadata.last_modified if metadata.last_modified
-      
+      data[MDF::PHYSICAL_PATH] = metadata.physical_path if metadata.physical_path
+
       props[MDF::DATA] = data
-      
+
       services = Hash.new
       metadata.list_services.each do |name|
         service = metadata.service(name)
         service[MDF::STALE_REPLICAS] = service.stale_replicas if service.stale_replicas
         service[MDF::SERVICE_TIMESTAMP] = service.timestamp unless service.timestamp.nil?
         service[MDF::RUN_NEEDED] = service.run_needed if service.run_needed
-        services[name] = service.properties
+        services[name] = service.properties unless service.properties.empty?
       end
-      
+
       props[MDF::SERVICES] = services
-      
+
       props
     end
-    
+
+    # @param format [String] encoding format used for metadata file
+    # @return [String] the suffix used to indicate that a file is a metadata file in the provided encoding
+    # @raise [ArgumentError] raised if the provided format is not a supported metadata encoding format
     def self.metadata_suffix(format: 'yaml')
       case format
       when 'yaml'
         '-llmd.yaml'
       else
-        raise ArgumentError.new('Invalid serialization format #{format} specified')
+        raise ArgumentError.new("Invalid serialization format #{format} specified")
+      end
+    end
+
+    # Safely writes the new metadata file and its digests.
+    # It does so by first writing the content and its digests to temp files,
+    # then making the temp files the current version of the file.
+    # Attempts to clean up new data in the case of failure.
+    def self.atomic_write(file_path, content, digest_algs)
+      # Fill in parent directories if they do not exist
+      parent_dir = Pathname(file_path).parent
+      parent_dir.mkpath unless parent_dir.exist?
+
+      file_path = file_path.path if file_path.respond_to?(:path)
+
+      # If file does not already exist, then simply write it
+      if !File.exist?(file_path)
+        File.write(file_path, content)
+        write_digests(file_path, content, digest_algs)
+        return
+      end
+
+      # Updating file, use safe atomic write
+      File.open(file_path) do |original_file|
+        original_file.flock(File::LOCK_EX)
+
+        base_name = File.basename(file_path)
+        Tempfile.open(base_name, parent_dir) do |temp_file|
+          begin
+            # Write content to temp file
+            temp_file.write(content)
+            temp_file.close
+
+            temp_path = temp_file.path
+
+            # Set permissions of new file to match old if it exists
+            old_stat = File.stat(file_path)
+            set_perms(temp_path, old_stat)
+
+            begin
+              digest_paths = write_digests(temp_path, content, digest_algs)
+
+              File.rename(temp_path, file_path)
+            rescue => e
+              cleanup_digests(temp_path)
+              raise e
+            end
+          rescue => e
+            temp_file.delete
+            raise e
+          end
+
+          # Cleanup all existing digest files, in case the set of algorithms has changed
+          cleanup_digests(file_path)
+          # Move new digests into place
+          digest_paths.each do |digest_path|
+            File.rename(digest_path, digest_path.sub(temp_path, file_path))
+          end
+        end
+      end
+    end
+
+    def self.set_perms(file_path, stat_info)
+      if stat_info
+        # Set correct permissions on new file
+        begin
+          File.chown(stat_info.uid, stat_info.gid, file_path)
+          # This operation will affect filesystem ACL's
+          File.chmod(stat_info.mode, file_path)
+        rescue Errno::EPERM, Errno::EACCES
+          # Changing file ownership failed, moving on.
+          return false
+        end
       end
+      true
     end
+
+    # Deletes all known digest files for the provided file path
+    def self.cleanup_digests(file_path)
+      DigestHelper::KNOWN_DIGESTS.each do |alg|
+        digest_path = "#{file_path}.#{alg}"
+        File.delete(digest_path) if File.exist?(digest_path)
+      end
+    end
+
+    def self.write_digests(file_path, content, digests)
+      return [] if digests.nil? || digests.empty?
+
+      digest_paths = Array.new
+
+      digests.each do |alg|
+        digest_class = DigestHelper::start_digest(alg)
+        result = digest_class.hexdigest(content)
+        digest_path = "#{file_path}.#{alg}"
+
+        File.write(digest_path, result)
+
+        digest_paths.push(digest_path)
+
+        self.logger.debug("Generated #{alg} digest for metadata file #{file_path}: #{digest_path} #{result}")
+      end
+
+      digest_paths
+    end
+
+    private_class_method :cleanup_digests
+    private_class_method :write_digests
+    private_class_method :atomic_write
   end
-end
+end

data/lib/longleaf/services/metadata_validator.rb

@@ -0,0 +1,76 @@
+require 'pathname'
+require 'longleaf/models/md_fields'
+require 'longleaf/errors'
+require_relative 'configuration_validator'
+
+module Longleaf
+  # Validator for file metadata
+  class MetadataValidator < ConfigurationValidator
+    MDF ||= MDFields
+
+    # @param config [Hash] hash containing the application configuration
+    def initialize(config)
+      super(config)
+    end
+
+    protected
+    # Validates the provided metadata for a file to ensure that it is syntactically correct and field types
+    # are validate.
+    def validate
+      assert("Metadata must be a hash, but a #{@config.class} was provided", @config.class == Hash)
+      assert("Metadata must contain a '#{MDF::DATA}' key", @config.key?(MDF::DATA))
+      assert("Metadata must contain a '#{MDF::SERVICES}' key", @config.key?(MDF::SERVICES))
+
+      data = @config[MDF::DATA]
+      register_on_failure { validate_date_field(data, MDF::REGISTERED_TIMESTAMP) }
+      register_on_failure { validate_date_field(data, MDF::DEREGISTERED_TIMESTAMP, required: false) }
+      register_on_failure { validate_date_field(data, MDF::LAST_MODIFIED) }
+
+      register_on_failure { validate_positive_integer(data, MDF::FILE_SIZE) }
+
+      checksums = data[MDF::CHECKSUMS]
+      register_on_failure do
+        if !checksums.nil? && !checksums.is_a?(Hash)
+          fail("Field '#{MDF::CHECKSUMS}' must be a map of algorithms to digests, but was a #{checksums.class}")
+        end
+      end
+
+      # Ensure that any service timestamps present are valid dates
+      services = @config[MDF::SERVICES]
+      services.each do |service_name, service_rec|
+        register_on_failure { validate_date_field(service_rec, MDF::SERVICE_TIMESTAMP, required: false) }
+      end
+    end
+
+    def validate_date_field(section, field_key, required: true)
+      field_val = section[field_key]
+
+      if field_val
+        begin
+          Time.iso8601(section[field_key])
+        rescue ArgumentError
+          fail("Field '#{field_key}' must be a valid ISO8601 timestamp, but contained value '#{section[field_key]}'")
+        end
+      elsif required
+        fail("Metadata must contain a '#{field_key}' field")
+      end
+    end
+
+    def validate_positive_integer(section, field_key, required: true)
+      field_val = section[field_key]
+
+      if field_val
+        begin
+          val = field_val.is_a?(Integer) ? field_val : Integer(field_val, 10)
+          if val < 0
+            fail("Field '#{field_key}' must be a positive integer")
+          end
+        rescue ArgumentError => err
+          fail("Field '#{field_key}' must be a positive integer")
+        end
+      elsif required
+        fail("Metadata must contain a '#{field_key}' field")
+      end
+    end
+  end
+end

data/lib/longleaf/services/s3_location_validator.rb

@@ -0,0 +1,19 @@
+require 'pathname'
+require 'longleaf/errors'
+require 'longleaf/helpers/s3_uri_helper'
+
+module Longleaf
+  # Validates the configuration of a s3 based location
+  class S3LocationValidator
+    def self.validate(p_validator, name, path_prop, section_name, path)
+      base_msg = "Storage location '#{name}' specifies invalid #{section_name} '#{path_prop}' property: "
+      p_validator.assert(base_msg + 'Path must not be empty', !path.nil? && !path.to_s.strip.empty?)
+      begin
+        bucket_name = S3UriHelper.extract_bucket(path)
+        p_validator.assert(base_msg + 'Path must specify a bucket', !bucket_name.nil?)
+      rescue ArgumentError => e
+        p_validator.fail(base_msg + e.message)
+      end
+    end
+  end
+end

data/lib/longleaf/services/service_class_cache.rb

@@ -0,0 +1,112 @@
+require 'pathname'
+
+module Longleaf
+  # Cache for loading and retrieving preservation service classes
+  class ServiceClassCache
+    STD_PRESERVATION_SERVICE_PATH = 'longleaf/preservation_services/'
+
+    def initialize(app_manager)
+      @app_manager = app_manager
+      # Cache storing per service definition instances of service classes
+      @service_instance_cache = Hash.new
+      # Cache storing per script path class of service
+      @class_cache = Hash.new
+    end
+
+    # Returns an instance of the preversation service defined for the provided service definition,
+    # based on the work_script and work_class properties provided.
+    #
+    # @param service_def [ServiceDefinition] definition of service to instantiate
+    # @return [PreservationService] Instance of the preservation service class for the definition.
+    def service_instance(service_def)
+      service_name = service_def.name
+      # Return the cached instance of the service
+      if @service_instance_cache.key?(service_name)
+        return @service_instance_cache[service_name]
+      end
+
+      clazz = service_class(service_def)
+      # Cache and return the class instance
+      @service_instance_cache[service_name] = clazz.new(service_def, @app_manager)
+    end
+
+    # Load and return the PreservationService class assigned to the provided service definition,
+    # based on the work_script and work_class properties provided.
+    #
+    # @param service_def [ServiceDefinition] definition of service to retrieve class for
+    # @return [Class] class of work_script
+    def service_class(service_def)
+      service_name = service_def.name
+      work_script = service_def.work_script
+
+      if work_script.include?('/')
+        expanded_path = Pathname.new(work_script).expand_path.to_s
+        if !from_permitted_path?(expanded_path)
+          raise ConfigurationError.new("Unable to load work_script for service #{service_name}, #{work_script} is not in a known library path.")
+        end
+
+        last_slash_index = work_script.rindex('/')
+        script_path = work_script[0..last_slash_index]
+        script_name = work_script[(last_slash_index + 1)..-1]
+      else
+        script_path = STD_PRESERVATION_SERVICE_PATH
+        script_name = work_script
+      end
+
+      # Strip off the extension
+      script_name.sub!('.rb', '')
+
+      require_path = File.join(script_path, script_name)
+      # Return the cached Class if this path has been encountered before
+      if @class_cache.key?(require_path)
+        return @class_cache[require_path]
+      end
+
+      # Load the script
+      begin
+        require require_path
+      rescue LoadError
+        raise ConfigurationError.new("Failed to load work_script '#{script_name}' for service #{service_name}")
+      end
+
+      # Generate the class name, either configured or from file naming convention if possible
+      if service_def.work_class
+        class_name = service_def.work_class
+      else
+        class_name = script_name.split('_').map(&:capitalize).join
+        # Assume the longleaf module for classes in the standard path
+        class_name = 'Longleaf::' + class_name if script_path == STD_PRESERVATION_SERVICE_PATH
+      end
+
+      begin
+        class_constant = constantize(class_name)
+        # cache the class for this work_script and return it
+        @class_cache[require_path] = class_constant
+      rescue NameError
+        raise ConfigurationError.new("Failed to load work_script '#{script_name}' for service #{service_name}, class name #{class_name} was not found.")
+      end
+    end
+
+    private
+    # Borrowed from sidekiq implementation
+    def constantize(str)
+      names = str.split('::')
+      names.shift if names.empty? || names.first.empty?
+
+      names.inject(Object) do |constant, name|
+        # the false flag limits search for name to under the constant namespace
+        #   which mimics Rails' behaviour
+        constant.const_defined?(name, false) ? constant.const_get(name, false) : constant.const_missing(name)
+      end
+    end
+
+    def from_permitted_path?(script_path)
+      $LOAD_PATH.each do |lib_path|
+        if script_path.start_with?(lib_path)
+          return true
+        end
+      end
+      false
+    end
+  end
+end