longleaf 0.1.0.pre.2 → 1.0.0

data/lib/longleaf/services/service_mapping_validator.rb

@@ -1,49 +1,59 @@
 require 'pathname'
-require_relative '../models/service_fields'
-require_relative '../models/app_fields'
-require_relative '../errors'
+require 'longleaf/models/service_fields'
+require 'longleaf/models/app_fields'
+require 'longleaf/errors'
 require_relative 'configuration_validator'
 
-# Validates application configuration of service to location mappings
 module Longleaf
+  # Validates application configuration of service to location mappings
   class ServiceMappingValidator < ConfigurationValidator
     AF ||= Longleaf::AppFields
-    
+
+    # @param config [Hash] hash containing the application configuration
+    def initialize(config)
+      super(config)
+    end
+
+    protected
     # Validates service mapping configuration to ensure that it is syntactically and referentially correct.
     # @param config [Hash] hash containing the application configuration
-    def self.validate_config(config)
-      
-      assert("Configuration must be a hash, but a #{config.class} was provided", config.class == Hash)
-      assert("Configuration must contain a root '#{AF::SERVICE_MAPPINGS}' key", config.key?(AF::SERVICE_MAPPINGS))
-      mappings = config[AF::SERVICE_MAPPINGS]
+    def validate
+      assert("Configuration must be a hash, but a #{@config.class} was provided", @config.class == Hash)
+      assert("Configuration must contain a root '#{AF::SERVICE_MAPPINGS}' key", @config.key?(AF::SERVICE_MAPPINGS))
+      mappings = @config[AF::SERVICE_MAPPINGS]
       return if mappings.nil? || mappings.empty?
       assert("'#{AF::SERVICE_MAPPINGS}' must be an array of mappings", mappings.is_a?(Array))
-      
-      service_names = config[AF::SERVICES].keys
-      location_names = config[AF::LOCATIONS].keys
-      
-      existing_paths = Array.new
+
+      service_names = @config[AF::SERVICES].keys
+      location_names = @config[AF::LOCATIONS].keys
+
       mappings.each do |mapping|
-        assert("Mapping must be a hash, but received #{mapping.inspect} instead", mapping.is_a?(Hash))
-        
-        validate_mapping_field(AF::LOCATIONS, mapping, location_names)
-        validate_mapping_field(AF::SERVICES, mapping, service_names)
+        register_on_failure do
+          assert("Mapping must be a hash, but received #{mapping.inspect} instead", mapping.is_a?(Hash))
+
+          register_on_failure { validate_mapping_field(AF::LOCATIONS, mapping, location_names) }
+          register_on_failure { validate_mapping_field(AF::SERVICES, mapping, service_names) }
+        end
       end
+
+      @result
     end
-    
+
     private
-    def self.validate_mapping_field(field, mapping, valid_values)
+    def validate_mapping_field(field, mapping, valid_values)
       assert("Mapping must contain a '#{field}' field", mapping.key?(field))
       field_values = mapping[field]
-      assert("Mapping '#{field}' field must be either a string or an array, but received '#{field_values.inspect}' instead",
+      assert("Mapping '#{field}' field must be either a string or an array, but received '#{field_values.class}' instead",
           field_values.is_a?(Array) || field_values.is_a?(String))
       assert("Mapping must specify one or more value in the '#{field}' field", !field_values.empty?)
-      
+
       check_values = field_values.is_a?(String) ? [field_values] : field_values
       check_values.each do |value|
-        assert("Mapping '#{field}' specifies value '#{value}', but no #{field} with that name exist",
-            valid_values.include?(value))
+        register_on_failure do
+          assert("Mapping specifies value '#{value}', but no #{field} with that name exist",
+              valid_values.include?(value))
+        end
       end
     end
   end
-end
+end

data/lib/longleaf/services/storage_location_manager.rb

@@ -1,37 +1,98 @@
-require_relative '../models/app_fields'
-require_relative '../models/storage_location'
+require 'longleaf/models/app_fields'
+require 'longleaf/models/storage_types'
+require 'longleaf/models/filesystem_storage_location'
+require 'longleaf/models/s3_storage_location'
+require 'longleaf/models/filesystem_metadata_location'
+require 'longleaf/errors'
 
-# Manager which loads and provides access to Longleaf::StorageLocation objects
 module Longleaf
+  # Manager which loads and provides access to {StorageLocation} objects
   class StorageLocationManager
     AF ||= Longleaf::AppFields
-    
+    ST ||= Longleaf::StorageTypes
+
+    # Hash mapping storage location names to {StorageLocation} objects
     attr_reader :locations
-    
+    # Mapping of storage types to storage location classes
+    @@storage_type_mappings = {
+        ST::FILESYSTEM_STORAGE_TYPE => Longleaf::FilesystemStorageLocation,
+        ST::S3_STORAGE_TYPE => Longleaf::S3StorageLocation
+      }
+    @@metadata_type_mappings = { ST::FILESYSTEM_STORAGE_TYPE => Longleaf::FilesystemMetadataLocation }
+
+    # @param config [Hash] has representation of the application configuration
     def initialize(config)
       raise ArgumentError.new("Configuration must be provided") if config&.empty?
 
       @locations = Hash.new
       config[AF::LOCATIONS].each do |name, properties|
-        path = properties[AF::LOCATION_PATH]
-        md_path = properties[AF::METADATA_PATH]
-        location = Longleaf::StorageLocation.new(name: name, path: path, metadata_path: md_path)
-        
-        @locations[name] = location
+        md_loc = instantiate_metadata_location(properties)
+
+        @locations[name] = instantiate_storage_location(name, properties, md_loc)
       end
       @locations.freeze
     end
-    
-    # Get the StorageLocation object which should contain the given path
+
+    # Get the {StorageLocation} object which should contain the given path
     # @return [Longleaf::StorageLocation] location containing the given path
     #    or nil if the path is not contained by a registered location.
     def get_location_by_path(path)
       raise ArgumentError.new("Path parameter is required") if path.nil? || path.empty?
       @locations.each do |name, location|
-        return location if path.start_with?(location.path) 
+        return location if location.contains?(path)
+      end
+
+      nil
+    end
+
+    # Get the {StorageLocation} object which should contain the given metadata path
+    # @return [Longleaf::StorageLocation] location containing the given metadata path
+    #    or nil if the path is not contained by a registered location.
+    def get_location_by_metadata_path(md_path)
+      raise ArgumentError.new("Metadata path parameter is required") if md_path.nil? || md_path.empty?
+      @locations.each do |name, location|
+        return location if location.metadata_location.contains?(md_path)
       end
-      
+
       nil
     end
+
+    # Raises a {StorageLocationUnavailableError} if the given path is not in a known storage location,
+    #    or if it is not within the expected location if provided
+    # @param path [String] file path
+    # @param expected_loc [String] name of the storage location which path should be contained by
+    # @raise [StorageLocationUnavailableError] if the path is not in a known/expected storage location
+    # @return [StorageLocation] the storage location which contains path, if it was within one.
+    def verify_path_in_location(path, expected_loc = nil)
+      location = get_location_by_path(path)
+      if location.nil?
+        raise StorageLocationUnavailableError.new("Path #{path} is not from a known storage location.")
+      elsif !expected_loc.nil? && expected_loc != location.name
+        raise StorageLocationUnavailableError.new("Path #{path} is not contained by storage location #{expected_loc}.")
+      end
+      location
+    end
+
+    private
+    def instantiate_metadata_location(loc_properties)
+      m_config = loc_properties[AF::METADATA_CONFIG]
+      m_type = m_config[AF::STORAGE_TYPE]
+      m_type = ST::FILESYSTEM_STORAGE_TYPE if m_type.nil?
+
+      m_class = @@metadata_type_mappings[m_type]
+      raise ArgumentError.new("Unknown metadata location type #{m_type}") if m_class.nil?
+
+      m_class.new(m_config)
+    end
+
+    def instantiate_storage_location(name, properties, md_loc)
+      s_type = properties[AF::STORAGE_TYPE]
+      s_type = ST::FILESYSTEM_STORAGE_TYPE if s_type.nil?
+
+      s_class = @@storage_type_mappings[s_type]
+      raise ArgumentError.new("Unknown storage location type #{s_type}") if s_class.nil?
+
+      s_class.new(name, properties, md_loc)
+    end
   end
-end
+end

data/lib/longleaf/services/storage_location_validator.rb

@@ -1,60 +1,74 @@
 require 'pathname'
-require 'longleaf/models/storage_location'
 require 'longleaf/models/app_fields'
+require 'longleaf/models/storage_types'
 require 'longleaf/errors'
 require_relative 'configuration_validator'
-require 'longleaf/services/storage_path_validator'
+require 'longleaf/services/filesystem_location_validator'
+require 'longleaf/services/s3_location_validator'
 
-# Validates application configuration of storage locations
 module Longleaf
+  # Validates application configuration of storage locations
   class StorageLocationValidator < ConfigurationValidator
     AF ||= Longleaf::AppFields
-    
-    # Validates configuration to ensure that it is syntactically correct and does not violate 
-    # schema and uniqueness requirements.
+    ST ||= Longleaf::StorageTypes
+
+    @@storage_type_mappings = {
+        ST::FILESYSTEM_STORAGE_TYPE => Longleaf::FilesystemLocationValidator,
+        ST::S3_STORAGE_TYPE => Longleaf::S3LocationValidator
+      }
+
     # @param config [Hash] hash containing the application configuration
-    def self.validate_config(config)
-      assert("Configuration must be a hash, but a #{config.class} was provided", config.class == Hash)
-      assert("Configuration must contain a root '#{AF::LOCATIONS}' key", config.key?(AF::LOCATIONS))
-      locations = config[AF::LOCATIONS]
+    def initialize(config)
+      super(config)
+      @existing_paths = Array.new
+    end
+
+    protected
+    # Validates configuration to ensure that it is syntactically correct and does not violate
+    # schema and uniqueness requirements.
+    def validate
+      assert("Configuration must be a hash, but a #{@config.class} was provided", @config.class == Hash)
+      assert("Configuration must contain a root '#{AF::LOCATIONS}' key", @config.key?(AF::LOCATIONS))
+      locations = @config[AF::LOCATIONS]
       assert("'#{AF::LOCATIONS}' must be a hash of locations", locations.class == Hash)
-      
-      existing_paths = Array.new
+
       locations.each do |name, properties|
-        assert("Name of storage location must be a string, but was of type #{name.class}", name.instance_of?(String))
-        assert("Storage location '#{name}' must be a hash, but a #{properties.class} was provided", properties.is_a?(Hash))
-        
-        assert_path_property_valid(name, AF::LOCATION_PATH, properties, existing_paths)
-        assert_path_property_valid(name, AF::METADATA_PATH, properties, existing_paths)
+        register_on_failure do
+          assert("Name of storage location must be a string, but was of type #{name.class}", name.instance_of?(String))
+          assert("Storage location '#{name}' must be a hash, but a #{properties.class} was provided", properties.is_a?(Hash))
+
+          register_on_failure { assert_path_property_valid(name, AF::LOCATION_PATH, properties, 'location') }
+
+          assert("Metadata location must be present for location '#{name}'", properties.key?(AF::METADATA_CONFIG))
+          assert_path_property_valid(name, AF::LOCATION_PATH, properties[AF::METADATA_CONFIG], 'metadata')
+        end
       end
+
+      @result
     end
-    
+
     private
-    def self.assert_path_property_valid(name, path_prop, properties, existing_paths)
+    def assert_path_property_valid(name, path_prop, properties, section_name)
       path = properties[path_prop]
-      begin
-        StoragePathValidator::validate(path)
-      rescue InvalidStoragePathError => err
-        raise ConfigurationError.new(
-            "Storage location '#{name}' specifies invalid '#{path_prop}' property: #{err.message}")
-      end
-      assert("Storage location '#{name}' must specify a '#{path_prop}' property", !path.nil? && !path.empty?)
-      assert("Storage location '#{name}' must specify an absolute path for property '#{path_prop}'",
-          Pathname.new(path).absolute? && !path.include?('/..'))
+
+      storage_type = properties[AF::STORAGE_TYPE] || ST::DEFAULT_STORAGE_TYPE
+      type_validator = @@storage_type_mappings[storage_type]
+      type_validator.validate(self, name, path_prop, section_name, path)
+
       # Ensure paths have trailing slash to avoid matching on partial directory names
       path += '/' unless path.end_with?('/')
       # Verify that the (metadata_)path property's value is not inside of another storage location or vice versa
-      existing_paths.each do |existing|
+      @existing_paths.each do |existing|
         if existing.start_with?(path) || path.start_with?(existing)
-          msg = "Location '#{name}' defines property #{path_prop} with value '#{path}'" \
+          msg = "Location '#{name}' defines property #{section_name} #{path_prop} with value '#{path}'" \
                 " which overlaps with another configured path '#{existing}'." \
-                " Storage locations must not define #{AF::LOCATION_PATH} or #{AF::METADATA_PATH}" \
+                " Storage locations must not define #{AF::LOCATION_PATH}" \
                 " properties which are contained by another location property"
-          raise ConfigurationError.new(msg)
+          fail(msg)
         end
       end
-      
-      existing_paths << path
+
+      @existing_paths << path
     end
   end
-end
+end

data/lib/longleaf/specs/config_builder.rb

@@ -1,19 +1,19 @@
-require_relative '../models/app_fields'
-require_relative '../models/service_fields'
+require 'longleaf/models/app_fields'
+require 'longleaf/models/service_fields'
 require 'yaml'
 
-# Test helper for constructing application configuration hashes
 module Longleaf
+  # Test helper for constructing application configuration hashes
   class ConfigBuilder
     AF ||= Longleaf::AppFields
     SF ||= Longleaf::ServiceFields
-    
+
     attr_accessor :config
-  
+
     def initialize
       @config = Hash.new
     end
-  
+
     # Add a root 'locations' field to the config
     # @param locations [Hash] value for the locations fields. Default is {}
     # @return this builder
@@ -21,22 +21,30 @@ module Longleaf
       @config[AF::LOCATIONS] = locations
       self
     end
-  
+
     # Add a 'location' to the config
     # @param name [String] name of the location
     # @param path [String] value for the 'path' field
     # @param md_path [String] value for the 'metadata_path' field
     # @return this builder
-    def with_location(name:, path: '/file/path/', md_path: '/metadata/path/')
+    def with_location(name:, path: '/file/path/', s_type: nil, md_path: '/metadata/path/', md_type: nil, md_digests: nil)
       @config[AF::LOCATIONS] = Hash.new unless @config.key?(AF::LOCATIONS)
-      
+
       location = {}
       @config[AF::LOCATIONS][name] = location
       location[AF::LOCATION_PATH] = path unless path.nil?
-      location[AF::METADATA_PATH] = md_path unless md_path.nil?
+      location[AF::STORAGE_TYPE] = s_type unless s_type.nil?
+
+      if !md_path.nil?
+        md_loc = { AF::LOCATION_PATH => md_path }
+        location[AF::METADATA_CONFIG] = md_loc
+
+        md_loc[AF::METADATA_DIGESTS] = md_digests unless md_digests.nil?
+        md_loc[AF::STORAGE_TYPE] = md_type unless md_type.nil?
+      end
       self
     end
-    
+
     # Add a root 'services' field to the config
     # @param services [Hash] value for the services field. Default is {}
     # @return this builder
@@ -44,26 +52,29 @@ module Longleaf
       @config[AF::SERVICES] = services
       self
     end
-    
+
     # Add a 'service' to the config
     # @param name [String] name of the service
     # @param work_script [String] value for the 'work_script' field
+    # @param work_class [String] value for the 'work_class' field
     # @param frequency [String] value for the 'frequency' field
     # @param delay [String] value for the 'delay' field
     # @param properties [Hash] hash of additional properties to include in the service
     # @return this builder
-    def with_service(name:, work_script: 'some_pres_service.rb', frequency: nil, delay: nil, properties: nil)
+    def with_service(name:, work_script: 'some_pres_service.rb', work_class: nil,
+         frequency: nil, delay: nil, properties: nil)
       @config[AF::SERVICES] = Hash.new unless @config.key?(AF::SERVICES)
-      
+
       service = {}
       service[SF::WORK_SCRIPT] = work_script
+      service[SF::WORK_CLASS] = work_class
       service[SF::FREQUENCY] = frequency unless frequency.nil?
       service[SF::DELAY] = delay unless delay.nil?
       service = service.merge(properties) unless properties.nil?
       @config[AF::SERVICES][name] = service
       self
     end
-    
+
     # Adds a 'service_mappings' section to the config
     # @param mappings [Object] the mappings config
     # @return this builder
@@ -71,32 +82,45 @@ module Longleaf
       @config[AF::SERVICE_MAPPINGS] = mappings
       self
     end
-    
+
     # Add a mapping from one or more services to one or more location
     # @param loc_names [Object] one or more location names. Can be a string or array.
     # @param service_names [Object] one or more service names. Can be a string or array.
     def map_services(loc_names, service_names)
       @config[AF::SERVICE_MAPPINGS] = Array.new unless @config.key?(AF::SERVICE_MAPPINGS)
-      
+
       mapping = Hash.new
       mapping[AF::LOCATIONS] = loc_names unless loc_names.nil?
       mapping[AF::SERVICES] = service_names unless service_names.nil?
       @config[AF::SERVICE_MAPPINGS].push(mapping)
       self
     end
-    
+
+    # Add a system config section to the config
+    def with_system(sys_config)
+      @config[AF::SYSTEM] = sys_config
+      self
+    end
+
     # @return the constructed configuration
     def get
       @config
     end
-    
+
     # Writes the configuration from this builder into a temporary file
     # @return the file path of the config file
-    def write_to_yaml_file
-      Tempfile.open('config') do |f|
+    def write_to_yaml_file(dest_path = nil)
+      if dest_path.nil?
+        file = Tempfile.new('config')
+        file.close
+        dest_path = file.path
+        # deleting temp file but reusing file name. This is to avoid premature garbage collection.
+        file.unlink
+      end
+      File.open(dest_path, 'w') do |f|
         f.write(@config.to_yaml)
-        return f.path
       end
+      dest_path
     end
   end
-end
+end