longleaf 0.2.0.pre.1 → 0.3.0

data/lib/longleaf/services/service_mapping_validator.rb

@@ -8,42 +8,41 @@ module Longleaf
   # Validates application configuration of service to location mappings
   class ServiceMappingValidator < ConfigurationValidator
     AF ||= Longleaf::AppFields
-    
+
     # Validates service mapping configuration to ensure that it is syntactically and referentially correct.
     # @param config [Hash] hash containing the application configuration
     def self.validate_config(config)
-      
       assert("Configuration must be a hash, but a #{config.class} was provided", config.class == Hash)
       assert("Configuration must contain a root '#{AF::SERVICE_MAPPINGS}' key", config.key?(AF::SERVICE_MAPPINGS))
       mappings = config[AF::SERVICE_MAPPINGS]
       return if mappings.nil? || mappings.empty?
       assert("'#{AF::SERVICE_MAPPINGS}' must be an array of mappings", mappings.is_a?(Array))
-      
+
       service_names = config[AF::SERVICES].keys
       location_names = config[AF::LOCATIONS].keys
-      
-      existing_paths = Array.new
+
       mappings.each do |mapping|
         assert("Mapping must be a hash, but received #{mapping.inspect} instead", mapping.is_a?(Hash))
-        
+
         validate_mapping_field(AF::LOCATIONS, mapping, location_names)
         validate_mapping_field(AF::SERVICES, mapping, service_names)
       end
     end
-    
-    private
+
     def self.validate_mapping_field(field, mapping, valid_values)
       assert("Mapping must contain a '#{field}' field", mapping.key?(field))
       field_values = mapping[field]
       assert("Mapping '#{field}' field must be either a string or an array, but received '#{field_values.inspect}' instead",
           field_values.is_a?(Array) || field_values.is_a?(String))
       assert("Mapping must specify one or more value in the '#{field}' field", !field_values.empty?)
-      
+
       check_values = field_values.is_a?(String) ? [field_values] : field_values
       check_values.each do |value|
         assert("Mapping '#{field}' specifies value '#{value}', but no #{field} with that name exist",
             valid_values.include?(value))
       end
     end
+
+    private_class_method :validate_mapping_field
   end
-end
+end

data/lib/longleaf/services/storage_location_manager.rb

@@ -6,10 +6,10 @@ module Longleaf
   # Manager which loads and provides access to {StorageLocation} objects
   class StorageLocationManager
     AF ||= Longleaf::AppFields
-    
+
     # Hash mapping storage location names to {StorageLocation} objects
     attr_reader :locations
-    
+
     # @param config [Hash] has representation of the application configuration
     def initialize(config)
       raise ArgumentError.new("Configuration must be provided") if config&.empty?
@@ -23,29 +23,42 @@ module Longleaf
             path: path,
             metadata_path: md_path,
             metadata_digests: md_digests)
-        
+
         @locations[name] = location
       end
       @locations.freeze
     end
-    
+
     # Get the {StorageLocation} object which should contain the given path
     # @return [Longleaf::StorageLocation] location containing the given path
     #    or nil if the path is not contained by a registered location.
     def get_location_by_path(path)
       raise ArgumentError.new("Path parameter is required") if path.nil? || path.empty?
       @locations.each do |name, location|
-        return location if path.start_with?(location.path) 
+        return location if path.start_with?(location.path)
       end
-      
+
       nil
     end
-    
+
+    # Get the {StorageLocation} object which should contain the given metadata path
+    # @return [Longleaf::StorageLocation] location containing the given metadata path
+    #    or nil if the path is not contained by a registered location.
+    def get_location_by_metadata_path(md_path)
+      raise ArgumentError.new("Metadata path parameter is required") if md_path.nil? || md_path.empty?
+      @locations.each do |name, location|
+        return location if md_path.start_with?(location.metadata_path)
+      end
+
+      nil
+    end
+
     # Raises a {StorageLocationUnavailableError} if the given path is not in a known storage location,
     #    or if it is not within the expected location if provided
     # @param path [String] file path
     # @param expected_loc [String] name of the storage location which path should be contained by
     # @raise [StorageLocationUnavailableError] if the path is not in a known/expected storage location
+    # @return [StorageLocation] the storage location which contains path, if it was within one.
     def verify_path_in_location(path, expected_loc = nil)
       location = get_location_by_path(path)
       if location.nil?
@@ -53,6 +66,7 @@ module Longleaf
       elsif !expected_loc.nil? && expected_loc != location.name
         raise StorageLocationUnavailableError.new("Path #{path} is not contained by storage location #{expected_loc}.")
       end
+      location
     end
   end
-end
+end

data/lib/longleaf/services/storage_location_validator.rb

@@ -9,8 +9,8 @@ module Longleaf
   # Validates application configuration of storage locations
   class StorageLocationValidator < ConfigurationValidator
     AF ||= Longleaf::AppFields
-    
-    # Validates configuration to ensure that it is syntactically correct and does not violate 
+
+    # Validates configuration to ensure that it is syntactically correct and does not violate
     # schema and uniqueness requirements.
     # @param config [Hash] hash containing the application configuration
     def self.validate_config(config)
@@ -18,18 +18,17 @@ module Longleaf
       assert("Configuration must contain a root '#{AF::LOCATIONS}' key", config.key?(AF::LOCATIONS))
       locations = config[AF::LOCATIONS]
       assert("'#{AF::LOCATIONS}' must be a hash of locations", locations.class == Hash)
-      
+
       existing_paths = Array.new
       locations.each do |name, properties|
         assert("Name of storage location must be a string, but was of type #{name.class}", name.instance_of?(String))
         assert("Storage location '#{name}' must be a hash, but a #{properties.class} was provided", properties.is_a?(Hash))
-        
+
         assert_path_property_valid(name, AF::LOCATION_PATH, properties, existing_paths)
         assert_path_property_valid(name, AF::METADATA_PATH, properties, existing_paths)
       end
     end
-    
-    private
+
     def self.assert_path_property_valid(name, path_prop, properties, existing_paths)
       path = properties[path_prop]
       begin
@@ -41,6 +40,8 @@ module Longleaf
       assert("Storage location '#{name}' must specify a '#{path_prop}' property", !path.nil? && !path.empty?)
       assert("Storage location '#{name}' must specify an absolute path for property '#{path_prop}'",
           Pathname.new(path).absolute? && !path.include?('/..'))
+      assert("Storage location '#{name}' specifies a '#{path_prop}' directory which does not exist", Dir.exist?(path))
+
       # Ensure paths have trailing slash to avoid matching on partial directory names
       path += '/' unless path.end_with?('/')
       # Verify that the (metadata_)path property's value is not inside of another storage location or vice versa
@@ -53,8 +54,10 @@ module Longleaf
           raise ConfigurationError.new(msg)
         end
       end
-      
+
       existing_paths << path
     end
+
+    private_class_method :assert_path_property_valid
   end
-end
+end

data/lib/longleaf/services/storage_path_validator.rb

@@ -13,4 +13,4 @@ module Longleaf
           if path.include?('/..')
     end
   end
-end
+end

data/lib/longleaf/specs/config_builder.rb

@@ -7,13 +7,13 @@ module Longleaf
   class ConfigBuilder
     AF ||= Longleaf::AppFields
     SF ||= Longleaf::ServiceFields
-    
+
     attr_accessor :config
-  
+
     def initialize
       @config = Hash.new
     end
-  
+
     # Add a root 'locations' field to the config
     # @param locations [Hash] value for the locations fields. Default is {}
     # @return this builder
@@ -21,7 +21,7 @@ module Longleaf
       @config[AF::LOCATIONS] = locations
       self
     end
-  
+
     # Add a 'location' to the config
     # @param name [String] name of the location
     # @param path [String] value for the 'path' field
@@ -29,7 +29,7 @@ module Longleaf
     # @return this builder
     def with_location(name:, path: '/file/path/', md_path: '/metadata/path/', md_digests: nil)
       @config[AF::LOCATIONS] = Hash.new unless @config.key?(AF::LOCATIONS)
-      
+
       location = {}
       @config[AF::LOCATIONS][name] = location
       location[AF::LOCATION_PATH] = path unless path.nil?
@@ -37,7 +37,7 @@ module Longleaf
       location[AF::METADATA_DIGESTS] = md_digests unless md_digests.nil?
       self
     end
-    
+
     # Add a root 'services' field to the config
     # @param services [Hash] value for the services field. Default is {}
     # @return this builder
@@ -45,7 +45,7 @@ module Longleaf
       @config[AF::SERVICES] = services
       self
     end
-    
+
     # Add a 'service' to the config
     # @param name [String] name of the service
     # @param work_script [String] value for the 'work_script' field
@@ -57,7 +57,7 @@ module Longleaf
     def with_service(name:, work_script: 'some_pres_service.rb', work_class: nil,
          frequency: nil, delay: nil, properties: nil)
       @config[AF::SERVICES] = Hash.new unless @config.key?(AF::SERVICES)
-      
+
       service = {}
       service[SF::WORK_SCRIPT] = work_script
       service[SF::WORK_CLASS] = work_class
@@ -67,7 +67,7 @@ module Longleaf
       @config[AF::SERVICES][name] = service
       self
     end
-    
+
     # Adds a 'service_mappings' section to the config
     # @param mappings [Object] the mappings config
     # @return this builder
@@ -75,32 +75,45 @@ module Longleaf
       @config[AF::SERVICE_MAPPINGS] = mappings
       self
     end
-    
+
     # Add a mapping from one or more services to one or more location
     # @param loc_names [Object] one or more location names. Can be a string or array.
     # @param service_names [Object] one or more service names. Can be a string or array.
     def map_services(loc_names, service_names)
       @config[AF::SERVICE_MAPPINGS] = Array.new unless @config.key?(AF::SERVICE_MAPPINGS)
-      
+
       mapping = Hash.new
       mapping[AF::LOCATIONS] = loc_names unless loc_names.nil?
       mapping[AF::SERVICES] = service_names unless service_names.nil?
       @config[AF::SERVICE_MAPPINGS].push(mapping)
       self
     end
-    
+
+    # Add a system config section to the config
+    def with_system(sys_config)
+      @config[AF::SYSTEM] = sys_config
+      self
+    end
+
     # @return the constructed configuration
     def get
       @config
     end
-    
+
     # Writes the configuration from this builder into a temporary file
     # @return the file path of the config file
-    def write_to_yaml_file
-      Tempfile.open('config') do |f|
+    def write_to_yaml_file(dest_path = nil)
+      if dest_path.nil?
+        file = Tempfile.new('config')
+        file.close
+        dest_path = file.path
+        # deleting temp file but reusing file name. This is to avoid premature garbage collection.
+        file.unlink
+      end
+      File.open(dest_path, 'w') do |f|
         f.write(@config.to_yaml)
-        return f.path
       end
+      dest_path
     end
   end
-end
+end

data/lib/longleaf/specs/custom_matchers.rb

@@ -6,4 +6,4 @@ module Longleaf
       actual.path == expected
     end
   end
-end
+end

data/lib/longleaf/specs/file_helpers.rb

@@ -7,7 +7,7 @@ module Longleaf
     def make_test_dir(parent: nil, name: 'dir')
       FileHelpers.make_test_dir(parent: parent, name: name)
     end
-    
+
     def self.make_test_dir(parent: nil, name: 'dir')
       if parent.nil?
         Dir.mktmpdir(name)
@@ -17,14 +17,15 @@ module Longleaf
         path
       end
     end
-    
-    def create_test_file(dir: nil, name: 'test_file', content: 'content')
+
+    def create_test_file(dir: nil, name: nil, content: 'content')
       FileHelpers.create_test_file(dir: dir, name: name, content: content)
     end
-    
-    def self.create_test_file(dir: nil, name: 'test_file', content: 'content')
-      if dir.nil?
-        file = Tempfile.create(name)
+
+    def self.create_test_file(dir: nil, name: nil, content: 'content')
+      if dir.nil? || name.nil?
+        name = 'test_file' if name.nil?
+        file = Tempfile.create(name, dir)
         file << content
         file.close
         return file.path
@@ -34,16 +35,16 @@ module Longleaf
         path
       end
     end
-    
-    def create_work_class(lib_dir, class_name, file_name, module_name = nil, is_applicable: true, perform: "")
-      FileHelpers.create_work_class(lib_dir, class_name, file_name, module_name, is_applicable: is_applicable,
-         perform: perform)
+
+    def create_work_class(lib_dir, class_name, file_name, module_name = nil, is_applicable: true, init_body: "", perform: "")
+      FileHelpers.create_work_class(lib_dir, class_name, file_name, module_name, is_applicable, init_body, perform)
     end
-    
-    def self.create_work_class(lib_dir, class_name, file_name, module_name = nil, is_applicable: true, perform: "")
+
+    def self.create_work_class(lib_dir, class_name, file_name, module_name, is_applicable, init_body, perform)
       class_contents = %Q(
         class #{class_name}
           def initialize(service_def, app_manager)
+            #{init_body}
           end
           def perform(file_rec, event)
             #{perform}
@@ -57,4 +58,4 @@ module Longleaf
       create_test_file(dir: lib_dir, name: file_name, content: class_contents)
     end
   end
-end
+end

data/lib/longleaf/specs/metadata_builder.rb

@@ -0,0 +1,91 @@
+require 'longleaf/models/md_fields'
+require 'longleaf/helpers/service_date_helper'
+require 'yaml'
+
+module Longleaf
+  # Test helper for constructing file metadata records
+  class MetadataBuilder
+    MF ||= Longleaf::MDFields
+
+    def initialize(file_path: nil, registered: ServiceDateHelper::formatted_timestamp)
+      @data = Hash.new
+      @services = Hash.new
+
+      unless file_path.nil?
+        @last_modified = File.mtime(file_path).utc.iso8601(3)
+        @file_size = File.size(file_path)
+      end
+
+      @registered = registered
+    end
+
+    def deregistered(timestamp = ServiceDateHelper::formatted_timestamp)
+      @deregistered = timestamp
+      self
+    end
+
+    def with_checksum(alg, value)
+      @checksums = Hash.new unless @data.key?(MF::CHECKSUMS)
+      @checksums[alg] = value
+      self
+    end
+
+    def with_service(name, timestamp: ServiceDateHelper::formatted_timestamp, run_needed: false, properties: nil,
+          failure_timestamp: nil)
+      timestamp = format_timestamp(timestamp)
+      failure_timestamp = format_timestamp(failure_timestamp) unless failure_timestamp.nil?
+
+      @services[name] = ServiceRecord.new(
+          properties: properties.nil? ? Hash.new : nil,
+          timestamp: timestamp,
+          run_needed: run_needed)
+      @services[name].failure_timestamp = failure_timestamp
+      self
+    end
+
+    def with_properties(properties)
+      @properties = properties
+    end
+
+    # @return the constructed metadata record
+    def get_metadata_record
+      MetadataRecord.new(properties: @properties,
+          services: @services,
+          deregistered: @deregistered,
+          registered: @registered,
+          checksums: @checksums,
+          file_size: @file_size,
+          last_modified: @last_modified)
+    end
+
+    # Add the generated metadata record to the given file record
+    def register_to(file_rec)
+      file_rec.metadata_record = get_metadata_record
+      self
+    end
+
+    # Writes the metadata record from this builder into a temporary file, or if a file
+    # record is provided, then to the expected metadata path for the record, and assigns
+    # the result as the metadata record for the file record.
+    # @return the file path of the config file
+    def write_to_yaml_file(file_rec: nil)
+      md_path = nil
+      if file_rec.nil?
+        md_path = TempFile.new(['metadata', 'yml']).path
+      else
+        md_path = file_rec.metadata_path
+      end
+
+      md_rec = get_metadata_record
+      MetadataSerializer::write(metadata: md_rec, file_path: md_path)
+      file_rec.metadata_record = md_rec
+
+      md_path
+    end
+
+    private
+    def format_timestamp(timestamp)
+      timestamp.kind_of?(Time) ? ServiceDateHelper::formatted_timestamp(timestamp) : timestamp
+    end
+  end
+end