longleaf 0.3.0 → 1.0.0

data/lib/longleaf/services/application_config_validator.rb

@@ -4,14 +4,25 @@ require_relative 'service_mapping_validator'
 
 module Longleaf
   # Validator for Longleaf application configuration
-  class ApplicationConfigValidator
+  class ApplicationConfigValidator < ConfigurationValidator
+    # @param config [Hash] hash containing the application configuration
+    def initialize(config)
+      super(config)
+    end
+
+    protected
     # Validates the application configuration provided. Will raise ConfigurationError
     # if any portion of the configuration is not syntactically or semantically valid.
-    # @param config [Hash] application configuration
-    def self.validate(config)
-      Longleaf::StorageLocationValidator::validate_config(config)
-      Longleaf::ServiceDefinitionValidator::validate_config(config)
-      Longleaf::ServiceMappingValidator::validate_config(config)
+    def validate
+      loc_result = StorageLocationValidator.new(@config).validate_config
+      defs_result = ServiceDefinitionValidator.new(@config).validate_config
+      mapping_result = ServiceMappingValidator.new(@config).validate_config
+
+      @result.errors.concat(loc_result.errors) unless loc_result.valid?
+      @result.errors.concat(defs_result.errors) unless defs_result.valid?
+      @result.errors.concat(mapping_result.errors) unless mapping_result.valid?
+
+      @result
     end
   end
 end

data/lib/longleaf/services/configuration_validator.rb

@@ -1,11 +1,71 @@
 module Longleaf
   # Abstract configuration validator class
   class ConfigurationValidator
-    class << self
-      protected
-      def assert(fail_message, assertion_passed)
-        raise ConfigurationError.new(fail_message) unless assertion_passed
+    attr_reader :result
+
+    def initialize(config)
+      @result = ConfigurationValidationResult.new
+      @config = config
+    end
+
+    # Verify that the provided configuration is valid
+    # @return [ConfigurationValidationResult] the result of the validation
+    def validate_config
+      register_on_failure { validate }
+
+      @result
+    end
+
+    # Asserts that the given conditional is true, raising a ConfigurationError if it is not.
+    def assert(fail_message, assertion_passed)
+      fail(fail_message) unless assertion_passed
+    end
+
+    # Indicate that validation has failed, throwing a Configuration error with the given message
+    def fail(fail_message)
+      raise ConfigurationError.new(fail_message)
+    end
+
+    # Registers an error to the result for this validator
+    def register_error(error)
+      if error.is_a?(StandardError)
+        @result.register_error(error.msg)
+      else
+        @result.register_error(error)
+      end
+    end
+
+    # Performs the provided block. If the block produces a ConfigurationError, the error
+    # is swallowed and registered to the result
+    def register_on_failure
+      begin
+        yield
+      rescue ConfigurationError => err
+        register_error(err.message)
       end
     end
   end
+
+  class ConfigurationValidationResult
+    attr_reader :errors
+
+    def initialize
+      @errors = Array.new
+    end
+
+    # Register an error with this validation result
+    def register_error(error_message)
+      @errors << error_message
+    end
+
+    # @return [boolean] true if validation produced not errors
+    def valid?
+      @errors.length == 0
+    end
+
+    # @raise [ConfigurationError] if the result is not valid, which lists all failures
+    def raise_if_invalid
+      raise ConfigurationError.new(@errors.join("\n")) unless valid?
+    end
+  end
 end

data/lib/longleaf/services/filesystem_location_validator.rb

@@ -0,0 +1,16 @@
+require 'pathname'
+require 'longleaf/errors'
+
+module Longleaf
+  # Validates the configuration of a filesystem based location
+  class FilesystemLocationValidator
+
+    def self.validate(p_validator, name, path_prop, section_name, path)
+      base_msg = "Storage location '#{name}' specifies invalid #{section_name} '#{path_prop}' property: "
+      p_validator.assert(base_msg + 'Path must not be empty', !path.nil? && !path.to_s.strip.empty?)
+      p_validator.assert(base_msg + 'Path must not contain any relative modifiers (/..)', !path.include?('/..'))
+      p_validator.assert(base_msg + 'Path must be absolute', Pathname.new(path).absolute?)
+      p_validator.assert(base_msg + 'Path does not exist', Dir.exist?(path))
+    end
+  end
+end

data/lib/longleaf/services/metadata_deserializer.rb

@@ -1,6 +1,7 @@
 require 'yaml'
 require 'longleaf/models/metadata_record'
 require 'longleaf/models/md_fields'
+require 'longleaf/services/metadata_validator'
 require 'longleaf/errors'
 require 'longleaf/logging'
 
@@ -15,6 +16,8 @@ module Longleaf
     # @param file_path [String] path of the file to read. Required.
     # @param format [String] format the file is stored in. Default is 'yaml'.
     def self.deserialize(file_path:, format: 'yaml', digest_algs: [])
+      file_path = file_path.path if file_path.is_a?(File)
+
       case format
       when 'yaml'
         md = from_yaml(file_path, digest_algs)
@@ -22,10 +25,6 @@ module Longleaf
         raise ArgumentError.new("Invalid deserialization format #{format} specified")
       end
 
-      if !md || !md.is_a?(Hash) || !md.key?(MDF::DATA) || !md.key?(MDF::SERVICES)
-        raise Longleaf::MetadataError.new("Invalid metadata file, did not contain data or services fields: #{file_path}")
-      end
-
       data = Hash.new.merge(md[MDF::DATA])
       # Extract reserved properties for submission as separate parameters
       registered = data.delete(MDFields::REGISTERED_TIMESTAMP)
@@ -37,7 +36,7 @@ module Longleaf
       services = md[MDF::SERVICES]
       service_records = Hash.new
       services&.each do |name, props|
-        raise Longleaf::MetadataError.new("Value of service #{name} must be a hash") unless props.class == Hash
+        raise MetadataError.new("Value of service #{name} must be a hash") unless props.class == Hash
 
         service_props = Hash.new.merge(props)
 
@@ -66,12 +65,45 @@ module Longleaf
       File.open(file_path, 'r:bom|utf-8') do |f|
         contents = f.read
 
-        verify_digests(file_path, contents, digest_algs)
+        checksum_error = nil
+        begin
+          verify_digests(file_path, contents, digest_algs)
+        rescue ChecksumMismatchError => err
+          # Hold onto the checksum error, in case we can identify the underlying cause
+          checksum_error = err
+        end
 
         begin
-          YAML.safe_load(contents, [], [], true)
-        rescue => err
-          raise Longleaf::MetadataError.new("Failed to parse metadata file #{file_path}: #{err.message}")
+          md = nil
+          begin
+            md = YAML.safe_load(contents, [], [], true)
+          rescue => err
+            raise MetadataError.new("Failed to parse metadata file #{file_path}: #{err.message}")
+          end
+
+          validation_result = MetadataValidator.new(md).validate_config
+          if !validation_result.valid?
+            if checksum_error.nil?
+              raise MetadataError.new("Invalid metadata file #{file_path.to_s}:\n#{validation_result.errors.join("\n")}")
+            else
+              raise MetadataError.new(validation_result.errors.join("\n"))
+            end
+          end
+
+          # Either return the valid metadata, or raise the checksum error as is
+          if checksum_error.nil?
+            md
+          else
+            raise checksum_error
+          end
+        rescue MetadataError => err
+          if checksum_error.nil?
+            raise err
+          else
+            # Add underlying cause from the metadata error to the checksum mismatch error
+            msg = checksum_error.message + "\nWith related issue(s):\n#{err.message}"
+            raise ChecksumMismatchError.new(msg)
+          end
         end
       end
     end

data/lib/longleaf/services/metadata_persistence_manager.rb

@@ -1,4 +1,5 @@
 require 'longleaf/services/metadata_serializer'
+require 'longleaf/services/metadata_deserializer'
 require 'longleaf/errors'
 
 module Longleaf
@@ -20,7 +21,7 @@ module Longleaf
 
       MetadataSerializer::write(metadata: file_rec.metadata_record,
           file_path: file_rec.metadata_path,
-          digest_algs: file_rec.storage_location.metadata_digests)
+          digest_algs: file_rec.storage_location.metadata_location.digests)
 
       index(file_rec)
     end
@@ -38,7 +39,7 @@ module Longleaf
     # @return [MetadataRecord] the metadata record for the file record
     def load(file_rec)
       md_rec = MetadataDeserializer.deserialize(file_path: file_rec.metadata_path,
-                  digest_algs: file_rec.storage_location.metadata_digests)
+                  digest_algs: file_rec.storage_location.metadata_location.digests)
       file_rec.metadata_record = md_rec
       md_rec
     end

data/lib/longleaf/services/metadata_serializer.rb

@@ -5,6 +5,7 @@ require 'longleaf/helpers/digest_helper'
 require 'longleaf/errors'
 require 'longleaf/logging'
 require 'pathname'
+require "tempfile"
 
 module Longleaf
   # Service which serializes MetadataRecord objects
@@ -30,12 +31,7 @@ module Longleaf
         raise ArgumentError.new("Invalid serialization format #{format} specified")
       end
 
-      # Fill in parent directories if they do not exist
-      parent_dir = Pathname(file_path).parent
-      parent_dir.mkpath unless parent_dir.exist?
-
-      File.write(file_path, content)
-      write_digests(file_path, content, digest_algs)
+      atomic_write(file_path, content, digest_algs)
     end
 
     # @param metadata [MetadataRecord] metadata record to transform
@@ -85,24 +81,109 @@ module Longleaf
       end
     end
 
+    # Safely writes the new metadata file and its digests.
+    # It does so by first writing the content and its digests to temp files,
+    # then making the temp files the current version of the file.
+    # Attempts to clean up new data in the case of failure.
+    def self.atomic_write(file_path, content, digest_algs)
+      # Fill in parent directories if they do not exist
+      parent_dir = Pathname(file_path).parent
+      parent_dir.mkpath unless parent_dir.exist?
+
+      file_path = file_path.path if file_path.respond_to?(:path)
+
+      # If file does not already exist, then simply write it
+      if !File.exist?(file_path)
+        File.write(file_path, content)
+        write_digests(file_path, content, digest_algs)
+        return
+      end
+
+      # Updating file, use safe atomic write
+      File.open(file_path) do |original_file|
+        original_file.flock(File::LOCK_EX)
+
+        base_name = File.basename(file_path)
+        Tempfile.open(base_name, parent_dir) do |temp_file|
+          begin
+            # Write content to temp file
+            temp_file.write(content)
+            temp_file.close
+
+            temp_path = temp_file.path
+
+            # Set permissions of new file to match old if it exists
+            old_stat = File.stat(file_path)
+            set_perms(temp_path, old_stat)
+
+            begin
+              digest_paths = write_digests(temp_path, content, digest_algs)
+
+              File.rename(temp_path, file_path)
+            rescue => e
+              cleanup_digests(temp_path)
+              raise e
+            end
+          rescue => e
+            temp_file.delete
+            raise e
+          end
+
+          # Cleanup all existing digest files, in case the set of algorithms has changed
+          cleanup_digests(file_path)
+          # Move new digests into place
+          digest_paths.each do |digest_path|
+            File.rename(digest_path, digest_path.sub(temp_path, file_path))
+          end
+        end
+      end
+    end
+
+    def self.set_perms(file_path, stat_info)
+      if stat_info
+        # Set correct permissions on new file
+        begin
+          File.chown(stat_info.uid, stat_info.gid, file_path)
+          # This operation will affect filesystem ACL's
+          File.chmod(stat_info.mode, file_path)
+        rescue Errno::EPERM, Errno::EACCES
+          # Changing file ownership failed, moving on.
+          return false
+        end
+      end
+      true
+    end
+
+    # Deletes all known digest files for the provided file path
+    def self.cleanup_digests(file_path)
+      DigestHelper::KNOWN_DIGESTS.each do |alg|
+        digest_path = "#{file_path}.#{alg}"
+        File.delete(digest_path) if File.exist?(digest_path)
+      end
+    end
+
     def self.write_digests(file_path, content, digests)
-      return if digests.nil? || digests.empty?
+      return [] if digests.nil? || digests.empty?
+
+      digest_paths = Array.new
 
       digests.each do |alg|
         digest_class = DigestHelper::start_digest(alg)
         result = digest_class.hexdigest(content)
-        if file_path.respond_to?(:path)
-          digest_path = "#{file_path.path}.#{alg}"
-        else
-          digest_path = "#{file_path}.#{alg}"
-        end
+        digest_path = "#{file_path}.#{alg}"
 
         File.write(digest_path, result)
 
-        self.logger.debug("Generated #{alg} digest for metadata file #{file_path}: #{result}")
+        digest_paths.push(digest_path)
+
+        self.logger.debug("Generated #{alg} digest for metadata file #{file_path}: #{digest_path} #{result}")
       end
+
+      digest_paths
     end
 
+    private_class_method :cleanup_digests
     private_class_method :write_digests
+    private_class_method :atomic_write
   end
 end

data/lib/longleaf/services/metadata_validator.rb

@@ -0,0 +1,76 @@
+require 'pathname'
+require 'longleaf/models/md_fields'
+require 'longleaf/errors'
+require_relative 'configuration_validator'
+
+module Longleaf
+  # Validator for file metadata
+  class MetadataValidator < ConfigurationValidator
+    MDF ||= MDFields
+
+    # @param config [Hash] hash containing the application configuration
+    def initialize(config)
+      super(config)
+    end
+
+    protected
+    # Validates the provided metadata for a file to ensure that it is syntactically correct and field types
+    # are validate.
+    def validate
+      assert("Metadata must be a hash, but a #{@config.class} was provided", @config.class == Hash)
+      assert("Metadata must contain a '#{MDF::DATA}' key", @config.key?(MDF::DATA))
+      assert("Metadata must contain a '#{MDF::SERVICES}' key", @config.key?(MDF::SERVICES))
+
+      data = @config[MDF::DATA]
+      register_on_failure { validate_date_field(data, MDF::REGISTERED_TIMESTAMP) }
+      register_on_failure { validate_date_field(data, MDF::DEREGISTERED_TIMESTAMP, required: false) }
+      register_on_failure { validate_date_field(data, MDF::LAST_MODIFIED) }
+
+      register_on_failure { validate_positive_integer(data, MDF::FILE_SIZE) }
+
+      checksums = data[MDF::CHECKSUMS]
+      register_on_failure do
+        if !checksums.nil? && !checksums.is_a?(Hash)
+          fail("Field '#{MDF::CHECKSUMS}' must be a map of algorithms to digests, but was a #{checksums.class}")
+        end
+      end
+
+      # Ensure that any service timestamps present are valid dates
+      services = @config[MDF::SERVICES]
+      services.each do |service_name, service_rec|
+        register_on_failure { validate_date_field(service_rec, MDF::SERVICE_TIMESTAMP, required: false) }
+      end
+    end
+
+    def validate_date_field(section, field_key, required: true)
+      field_val = section[field_key]
+
+      if field_val
+        begin
+          Time.iso8601(section[field_key])
+        rescue ArgumentError
+          fail("Field '#{field_key}' must be a valid ISO8601 timestamp, but contained value '#{section[field_key]}'")
+        end
+      elsif required
+        fail("Metadata must contain a '#{field_key}' field")
+      end
+    end
+
+    def validate_positive_integer(section, field_key, required: true)
+      field_val = section[field_key]
+
+      if field_val
+        begin
+          val = field_val.is_a?(Integer) ? field_val : Integer(field_val, 10)
+          if val < 0
+            fail("Field '#{field_key}' must be a positive integer")
+          end
+        rescue ArgumentError => err
+          fail("Field '#{field_key}' must be a positive integer")
+        end
+      elsif required
+        fail("Metadata must contain a '#{field_key}' field")
+      end
+    end
+  end
+end