longleaf 0.1.0.pre.2 → 1.0.0

data/lib/longleaf/services/application_config_manager.rb

@@ -5,20 +5,34 @@ require_relative 'service_definition_manager'
 require_relative 'service_mapping_validator'
 require_relative 'service_mapping_manager'
 require_relative 'service_manager'
+require_relative 'metadata_persistence_manager'
+require 'longleaf/indexing/index_manager'
+require 'longleaf/models/app_fields'
 
-# Manager which loads and provides access to the configuration of the application
 module Longleaf
+  # Manager which loads and provides access to the configuration of the application
   class ApplicationConfigManager
+    attr_reader :config_md5
     attr_reader :service_manager
     attr_reader :location_manager
-    
-    def initialize(config)
+    attr_reader :index_manager
+    attr_reader :md_manager
+
+    def initialize(config, config_md5 = nil)
+      @config_md5 = config_md5
+
       @location_manager = Longleaf::StorageLocationManager.new(config)
-      
+
       definition_manager = Longleaf::ServiceDefinitionManager.new(config)
       mapping_manager = Longleaf::ServiceMappingManager.new(config)
       @service_manager = Longleaf::ServiceManager.new(
-          definition_manager: definition_manager, mapping_manager: mapping_manager)
+          definition_manager: definition_manager,
+          mapping_manager: mapping_manager,
+          app_manager: self)
+
+      sys_config = config[AppFields::SYSTEM]
+      @index_manager = IndexManager.new(sys_config, self)
+      @md_manager = MetadataPersistenceManager.new(@index_manager)
     end
   end
-end
+end

data/lib/longleaf/services/application_config_validator.rb

@@ -2,17 +2,27 @@ require_relative 'storage_location_validator'
 require_relative 'service_definition_validator'
 require_relative 'service_mapping_validator'
 
-# Validator for Longleaf application configuration
 module Longleaf
-  class ApplicationConfigValidator
-    
+  # Validator for Longleaf application configuration
+  class ApplicationConfigValidator < ConfigurationValidator
+    # @param config [Hash] hash containing the application configuration
+    def initialize(config)
+      super(config)
+    end
+
+    protected
     # Validates the application configuration provided. Will raise ConfigurationError
     # if any portion of the configuration is not syntactically or semantically valid.
-    # @param config [Hash] application configuration
-    def self.validate(config)
-      Longleaf::StorageLocationValidator::validate_config(config)
-      Longleaf::ServiceDefinitionValidator::validate_config(config)
-      Longleaf::ServiceMappingValidator::validate_config(config)
+    def validate
+      loc_result = StorageLocationValidator.new(@config).validate_config
+      defs_result = ServiceDefinitionValidator.new(@config).validate_config
+      mapping_result = ServiceMappingValidator.new(@config).validate_config
+
+      @result.errors.concat(loc_result.errors) unless loc_result.valid?
+      @result.errors.concat(defs_result.errors) unless defs_result.valid?
+      @result.errors.concat(mapping_result.errors) unless mapping_result.valid?
+
+      @result
     end
   end
-end
+end

data/lib/longleaf/services/configuration_validator.rb

@@ -1,8 +1,71 @@
 module Longleaf
+  # Abstract configuration validator class
   class ConfigurationValidator
-    protected
-    def self.assert(fail_message, assertion_passed)
-      raise ConfigurationError.new(fail_message) unless assertion_passed
+    attr_reader :result
+
+    def initialize(config)
+      @result = ConfigurationValidationResult.new
+      @config = config
+    end
+
+    # Verify that the provided configuration is valid
+    # @return [ConfigurationValidationResult] the result of the validation
+    def validate_config
+      register_on_failure { validate }
+
+      @result
+    end
+
+    # Asserts that the given conditional is true, raising a ConfigurationError if it is not.
+    def assert(fail_message, assertion_passed)
+      fail(fail_message) unless assertion_passed
+    end
+
+    # Indicate that validation has failed, throwing a Configuration error with the given message
+    def fail(fail_message)
+      raise ConfigurationError.new(fail_message)
+    end
+
+    # Registers an error to the result for this validator
+    def register_error(error)
+      if error.is_a?(StandardError)
+        @result.register_error(error.msg)
+      else
+        @result.register_error(error)
+      end
+    end
+
+    # Performs the provided block. If the block produces a ConfigurationError, the error
+    # is swallowed and registered to the result
+    def register_on_failure
+      begin
+        yield
+      rescue ConfigurationError => err
+        register_error(err.message)
+      end
+    end
+  end
+
+  class ConfigurationValidationResult
+    attr_reader :errors
+
+    def initialize
+      @errors = Array.new
+    end
+
+    # Register an error with this validation result
+    def register_error(error_message)
+      @errors << error_message
+    end
+
+    # @return [boolean] true if validation produced not errors
+    def valid?
+      @errors.length == 0
+    end
+
+    # @raise [ConfigurationError] if the result is not valid, which lists all failures
+    def raise_if_invalid
+      raise ConfigurationError.new(@errors.join("\n")) unless valid?
     end
   end
-end
+end

data/lib/longleaf/services/filesystem_location_validator.rb

@@ -0,0 +1,16 @@
+require 'pathname'
+require 'longleaf/errors'
+
+module Longleaf
+  # Validates the configuration of a filesystem based location
+  class FilesystemLocationValidator
+
+    def self.validate(p_validator, name, path_prop, section_name, path)
+      base_msg = "Storage location '#{name}' specifies invalid #{section_name} '#{path_prop}' property: "
+      p_validator.assert(base_msg + 'Path must not be empty', !path.nil? && !path.to_s.strip.empty?)
+      p_validator.assert(base_msg + 'Path must not contain any relative modifiers (/..)', !path.include?('/..'))
+      p_validator.assert(base_msg + 'Path must be absolute', Pathname.new(path).absolute?)
+      p_validator.assert(base_msg + 'Path does not exist', Dir.exist?(path))
+    end
+  end
+end

data/lib/longleaf/services/metadata_deserializer.rb

@@ -1,29 +1,30 @@
 require 'yaml'
-require_relative '../models/metadata_record'
-require_relative '../models/md_fields'
-require_relative '../errors'
+require 'longleaf/models/metadata_record'
+require 'longleaf/models/md_fields'
+require 'longleaf/services/metadata_validator'
+require 'longleaf/errors'
+require 'longleaf/logging'
 
-# Service which deserializes metadata files into MetadataRecord objects
 module Longleaf
+  # Service which deserializes metadata files into MetadataRecord objects
   class MetadataDeserializer
-    MDF = Longleaf::MDFields
-    
+    extend Longleaf::Logging
+    MDF ||= MDFields
+
     # Deserialize a file into a MetadataRecord object
     #
     # @param file_path [String] path of the file to read. Required.
     # @param format [String] format the file is stored in. Default is 'yaml'.
-    def self.deserialize(file_path:, format: 'yaml')
+    def self.deserialize(file_path:, format: 'yaml', digest_algs: [])
+      file_path = file_path.path if file_path.is_a?(File)
+
       case format
       when 'yaml'
-        md = from_yaml(file_path)
+        md = from_yaml(file_path, digest_algs)
       else
-        raise ArgumentError.new('Invalid deserialization format #{format} specified')
+        raise ArgumentError.new("Invalid deserialization format #{format} specified")
       end
-      
-      if !md || !md.key?(MDF::DATA) || !md.key?(MDF::SERVICES)
-        raise Longleaf::MetadataError.new("Invalid metadata file, did not contain data or services fields: #{file_path}")
-      end
-      
+
       data = Hash.new.merge(md[MDF::DATA])
       # Extract reserved properties for submission as separate parameters
       registered = data.delete(MDFields::REGISTERED_TIMESTAMP)
@@ -31,38 +32,108 @@ module Longleaf
       checksums = data.delete(MDFields::CHECKSUMS)
       file_size = data.delete(MDFields::FILE_SIZE)
       last_modified = data.delete(MDFields::LAST_MODIFIED)
-      
+
       services = md[MDF::SERVICES]
       service_records = Hash.new
-      unless services.nil?
-        services.each do |name, props|
-          raise Longleaf::MetadataError.new("Value of service #{name} must be a hash") unless props.class == Hash
-          
-          service_props = Hash.new.merge(props)
-          
-          stale_replicas = service_props.delete(MDFields::STALE_REPLICAS)
-          timestamp = service_props.delete(MDFields::SERVICE_TIMESTAMP)
-          run_needed = service_props.delete(MDFields::RUN_NEEDED)
-          
-          service_records[name] = ServiceRecord.new(
-              properties: service_props,
-              stale_replicas: stale_replicas,
-              timestamp: timestamp,
-              run_needed: run_needed)
-        end
+      services&.each do |name, props|
+        raise MetadataError.new("Value of service #{name} must be a hash") unless props.class == Hash
+
+        service_props = Hash.new.merge(props)
+
+        stale_replicas = service_props.delete(MDFields::STALE_REPLICAS)
+        timestamp = service_props.delete(MDFields::SERVICE_TIMESTAMP)
+        run_needed = service_props.delete(MDFields::RUN_NEEDED)
+
+        service_records[name] = ServiceRecord.new(
+          properties: service_props,
+          stale_replicas: stale_replicas,
+          timestamp: timestamp,
+          run_needed: run_needed)
       end
-      
+
       MetadataRecord.new(properties: data,
-          services: service_records,
-          registered: registered,
-          deregistered: deregistered,
-          checksums: checksums,
-          file_size: file_size,
-          last_modified: last_modified)
+                         services: service_records,
+                         registered: registered,
+                         deregistered: deregistered,
+                         checksums: checksums,
+                         file_size: file_size,
+                         last_modified: last_modified)
     end
-    
-    def self.from_yaml(file_path)
-      YAML.load_file(file_path)
+
+    # Load configuration a yaml encoded configuration file
+    def self.from_yaml(file_path, digest_algs)
+      File.open(file_path, 'r:bom|utf-8') do |f|
+        contents = f.read
+
+        checksum_error = nil
+        begin
+          verify_digests(file_path, contents, digest_algs)
+        rescue ChecksumMismatchError => err
+          # Hold onto the checksum error, in case we can identify the underlying cause
+          checksum_error = err
+        end
+
+        begin
+          md = nil
+          begin
+            md = YAML.safe_load(contents, [], [], true)
+          rescue => err
+            raise MetadataError.new("Failed to parse metadata file #{file_path}: #{err.message}")
+          end
+
+          validation_result = MetadataValidator.new(md).validate_config
+          if !validation_result.valid?
+            if checksum_error.nil?
+              raise MetadataError.new("Invalid metadata file #{file_path.to_s}:\n#{validation_result.errors.join("\n")}")
+            else
+              raise MetadataError.new(validation_result.errors.join("\n"))
+            end
+          end
+
+          # Either return the valid metadata, or raise the checksum error as is
+          if checksum_error.nil?
+            md
+          else
+            raise checksum_error
+          end
+        rescue MetadataError => err
+          if checksum_error.nil?
+            raise err
+          else
+            # Add underlying cause from the metadata error to the checksum mismatch error
+            msg = checksum_error.message + "\nWith related issue(s):\n#{err.message}"
+            raise ChecksumMismatchError.new(msg)
+          end
+        end
+      end
+    end
+
+    def self.verify_digests(file_path, contents, digest_algs)
+      return if digest_algs.nil? || digest_algs.empty?
+
+      digest_algs.each do |alg|
+        if file_path.respond_to?(:path)
+          path = file_path.path
+        else
+          path = file_path
+        end
+        digest_path = "#{path}.#{alg}"
+        unless File.exist?(digest_path)
+          logger.warn("Missing expected #{alg} digest for #{path}")
+          next
+        end
+
+        digest = DigestHelper::start_digest(alg)
+        result = digest.hexdigest(contents)
+        existing_digest = IO.read(digest_path)
+
+        if result == existing_digest
+          logger.info("Metadata fixity check using algorithm '#{alg}' succeeded for file #{path}")
+        else
+          raise ChecksumMismatchError.new("Metadata digest of type #{alg} did not match the contents of #{path}:" \
+                                          + " expected #{existing_digest}, calculated #{result}")
+        end
+      end
     end
   end
-end
+end

data/lib/longleaf/services/metadata_persistence_manager.rb

@@ -0,0 +1,47 @@
+require 'longleaf/services/metadata_serializer'
+require 'longleaf/services/metadata_deserializer'
+require 'longleaf/errors'
+
+module Longleaf
+  # Handles the persistence of metadata records
+  class MetadataPersistenceManager
+    # Initialize the MetadataPersistenceManager
+    # @param index_manager [IndexManager] system config manager
+    def initialize(index_manager)
+      @index_manager = index_manager
+    end
+
+    # Persist the metadata for the provided file record to all configured destinations.
+    # This may include to disk as well as to an index.
+    # @param file_rec [FileRecord] file record
+    def persist(file_rec)
+      if file_rec.metadata_record.nil?
+        raise MetadataError.new("No metadata record provided, cannot persist metadata for #{file_rec.path}")
+      end
+
+      MetadataSerializer::write(metadata: file_rec.metadata_record,
+          file_path: file_rec.metadata_path,
+          digest_algs: file_rec.storage_location.metadata_location.digests)
+
+      index(file_rec)
+    end
+
+    # Index metadata for the provided file record
+    # @param file_rec [FileRecord] file record
+    def index(file_rec)
+      if @index_manager.using_index?
+        @index_manager.index(file_rec)
+      end
+    end
+
+    # Load the metadata record for the provided file record
+    # @param file_rec [FileRecord] file record
+    # @return [MetadataRecord] the metadata record for the file record
+    def load(file_rec)
+      md_rec = MetadataDeserializer.deserialize(file_path: file_rec.metadata_path,
+                  digest_algs: file_rec.storage_location.metadata_location.digests)
+      file_rec.metadata_record = md_rec
+      md_rec
+    end
+  end
+end

data/lib/longleaf/services/metadata_serializer.rb

@@ -1,76 +1,189 @@
 require 'yaml'
 require 'longleaf/models/metadata_record'
 require 'longleaf/models/md_fields'
+require 'longleaf/helpers/digest_helper'
+require 'longleaf/errors'
+require 'longleaf/logging'
 require 'pathname'
+require "tempfile"
 
-# Service which serializes MetadataRecord objects
 module Longleaf
+  # Service which serializes MetadataRecord objects
   class MetadataSerializer
-    MDF = Longleaf::MDFields
-    
+    extend Longleaf::Logging
+    MDF ||= MDFields
+
     # Serialize the contents of the provided metadata record to the specified path
     #
     # @param metadata [MetadataRecord] metadata record to serialize. Required.
     # @param file_path [String] path to write the file to. Required.
     # @param format [String] format to serialize the metadata in. Default is 'yaml'.
-    def self.write(metadata:, file_path:, format: 'yaml')
+    # @param digest_algs [Array] if provided, sidecar digest files for the metadata file
+    #    will be generated for each algorithm.
+    def self.write(metadata:, file_path:, format: 'yaml', digest_algs: [])
       raise ArgumentError.new('metadata parameter must be a MetadataRecord') \
-          unless metadata.class == Longleaf::MetadataRecord
-      
+          unless metadata.class == MetadataRecord
+
       case format
       when 'yaml'
         content = to_yaml(metadata)
       else
-        raise ArgumentError.new('Invalid serialization format #{format} specified')
+        raise ArgumentError.new("Invalid serialization format #{format} specified")
       end
-      
-      # Fill in parent directories if they do not exist
-      parent_dir = Pathname(file_path).parent
-      parent_dir.mkpath unless parent_dir.exist?
-      
-      File.write(file_path, content)
+
+      atomic_write(file_path, content, digest_algs)
     end
-    
+
     # @param metadata [MetadataRecord] metadata record to transform
     # @return [String] a yaml representation of the provided MetadataRecord
     def self.to_yaml(metadata)
       props = to_hash(metadata)
       props.to_yaml
     end
-    
+
+    # Create a hash representation of the given MetadataRecord file
+    # @param metadata [MetadataRecord] metadata record to transform into a hash
     def self.to_hash(metadata)
       props = Hash.new
-      
+
       data = Hash.new.merge(metadata.properties)
       data[MDF::REGISTERED_TIMESTAMP] = metadata.registered if metadata.registered
       data[MDF::DEREGISTERED_TIMESTAMP] = metadata.deregistered if metadata.deregistered
-      data[MDF::CHECKSUMS] = metadata.checksums unless metadata.checksums&.empty?
+      data[MDF::CHECKSUMS] = metadata.checksums unless metadata.checksums && metadata.checksums.empty?
       data[MDF::FILE_SIZE] = metadata.file_size unless metadata.file_size.nil?
       data[MDF::LAST_MODIFIED] = metadata.last_modified if metadata.last_modified
-      
+
       props[MDF::DATA] = data
-      
+
       services = Hash.new
       metadata.list_services.each do |name|
         service = metadata.service(name)
         service[MDF::STALE_REPLICAS] = service.stale_replicas if service.stale_replicas
         service[MDF::SERVICE_TIMESTAMP] = service.timestamp unless service.timestamp.nil?
         service[MDF::RUN_NEEDED] = service.run_needed if service.run_needed
-        services[name] = service.properties
+        services[name] = service.properties unless service.properties.empty?
       end
-      
+
       props[MDF::SERVICES] = services
-      
+
       props
     end
-    
+
+    # @param format [String] encoding format used for metadata file
+    # @return [String] the suffix used to indicate that a file is a metadata file in the provided encoding
+    # @raise [ArgumentError] raised if the provided format is not a supported metadata encoding format
     def self.metadata_suffix(format: 'yaml')
       case format
       when 'yaml'
         '-llmd.yaml'
       else
-        raise ArgumentError.new('Invalid serialization format #{format} specified')
+        raise ArgumentError.new("Invalid serialization format #{format} specified")
+      end
+    end
+
+    # Safely writes the new metadata file and its digests.
+    # It does so by first writing the content and its digests to temp files,
+    # then making the temp files the current version of the file.
+    # Attempts to clean up new data in the case of failure.
+    def self.atomic_write(file_path, content, digest_algs)
+      # Fill in parent directories if they do not exist
+      parent_dir = Pathname(file_path).parent
+      parent_dir.mkpath unless parent_dir.exist?
+
+      file_path = file_path.path if file_path.respond_to?(:path)
+
+      # If file does not already exist, then simply write it
+      if !File.exist?(file_path)
+        File.write(file_path, content)
+        write_digests(file_path, content, digest_algs)
+        return
+      end
+
+      # Updating file, use safe atomic write
+      File.open(file_path) do |original_file|
+        original_file.flock(File::LOCK_EX)
+
+        base_name = File.basename(file_path)
+        Tempfile.open(base_name, parent_dir) do |temp_file|
+          begin
+            # Write content to temp file
+            temp_file.write(content)
+            temp_file.close
+
+            temp_path = temp_file.path
+
+            # Set permissions of new file to match old if it exists
+            old_stat = File.stat(file_path)
+            set_perms(temp_path, old_stat)
+
+            begin
+              digest_paths = write_digests(temp_path, content, digest_algs)
+
+              File.rename(temp_path, file_path)
+            rescue => e
+              cleanup_digests(temp_path)
+              raise e
+            end
+          rescue => e
+            temp_file.delete
+            raise e
+          end
+
+          # Cleanup all existing digest files, in case the set of algorithms has changed
+          cleanup_digests(file_path)
+          # Move new digests into place
+          digest_paths.each do |digest_path|
+            File.rename(digest_path, digest_path.sub(temp_path, file_path))
+          end
+        end
+      end
+    end
+
+    def self.set_perms(file_path, stat_info)
+      if stat_info
+        # Set correct permissions on new file
+        begin
+          File.chown(stat_info.uid, stat_info.gid, file_path)
+          # This operation will affect filesystem ACL's
+          File.chmod(stat_info.mode, file_path)
+        rescue Errno::EPERM, Errno::EACCES
+          # Changing file ownership failed, moving on.
+          return false
+        end
       end
+      true
     end
+
+    # Deletes all known digest files for the provided file path
+    def self.cleanup_digests(file_path)
+      DigestHelper::KNOWN_DIGESTS.each do |alg|
+        digest_path = "#{file_path}.#{alg}"
+        File.delete(digest_path) if File.exist?(digest_path)
+      end
+    end
+
+    def self.write_digests(file_path, content, digests)
+      return [] if digests.nil? || digests.empty?
+
+      digest_paths = Array.new
+
+      digests.each do |alg|
+        digest_class = DigestHelper::start_digest(alg)
+        result = digest_class.hexdigest(content)
+        digest_path = "#{file_path}.#{alg}"
+
+        File.write(digest_path, result)
+
+        digest_paths.push(digest_path)
+
+        self.logger.debug("Generated #{alg} digest for metadata file #{file_path}: #{digest_path} #{result}")
+      end
+
+      digest_paths
+    end
+
+    private_class_method :cleanup_digests
+    private_class_method :write_digests
+    private_class_method :atomic_write
   end
-end
+end