longleaf 0.1.0 → 1.1.1

data/lib/longleaf/services/storage_location_validator.rb

@@ -1,60 +1,74 @@
 require 'pathname'
-require 'longleaf/models/storage_location'
 require 'longleaf/models/app_fields'
+require 'longleaf/models/storage_types'
 require 'longleaf/errors'
 require_relative 'configuration_validator'
-require 'longleaf/services/storage_path_validator'
+require 'longleaf/services/filesystem_location_validator'
+require 'longleaf/services/s3_location_validator'
 
-# Validates application configuration of storage locations
 module Longleaf
+  # Validates application configuration of storage locations
   class StorageLocationValidator < ConfigurationValidator
     AF ||= Longleaf::AppFields
-    
-    # Validates configuration to ensure that it is syntactically correct and does not violate 
-    # schema and uniqueness requirements.
+    ST ||= Longleaf::StorageTypes
+
+    @@storage_type_mappings = {
+        ST::FILESYSTEM_STORAGE_TYPE => Longleaf::FilesystemLocationValidator,
+        ST::S3_STORAGE_TYPE => Longleaf::S3LocationValidator
+      }
+
     # @param config [Hash] hash containing the application configuration
-    def self.validate_config(config)
-      assert("Configuration must be a hash, but a #{config.class} was provided", config.class == Hash)
-      assert("Configuration must contain a root '#{AF::LOCATIONS}' key", config.key?(AF::LOCATIONS))
-      locations = config[AF::LOCATIONS]
+    def initialize(config)
+      super(config)
+      @existing_paths = Array.new
+    end
+
+    protected
+    # Validates configuration to ensure that it is syntactically correct and does not violate
+    # schema and uniqueness requirements.
+    def validate
+      assert("Configuration must be a hash, but a #{@config.class} was provided", @config.class == Hash)
+      assert("Configuration must contain a root '#{AF::LOCATIONS}' key", @config.key?(AF::LOCATIONS))
+      locations = @config[AF::LOCATIONS]
       assert("'#{AF::LOCATIONS}' must be a hash of locations", locations.class == Hash)
-      
-      existing_paths = Array.new
+
       locations.each do |name, properties|
-        assert("Name of storage location must be a string, but was of type #{name.class}", name.instance_of?(String))
-        assert("Storage location '#{name}' must be a hash, but a #{properties.class} was provided", properties.is_a?(Hash))
-        
-        assert_path_property_valid(name, AF::LOCATION_PATH, properties, existing_paths)
-        assert_path_property_valid(name, AF::METADATA_PATH, properties, existing_paths)
+        register_on_failure do
+          assert("Name of storage location must be a string, but was of type #{name.class}", name.instance_of?(String))
+          assert("Storage location '#{name}' must be a hash, but a #{properties.class} was provided", properties.is_a?(Hash))
+
+          register_on_failure { assert_path_property_valid(name, AF::LOCATION_PATH, properties, 'location') }
+
+          assert("Metadata location must be present for location '#{name}'", properties.key?(AF::METADATA_CONFIG))
+          assert_path_property_valid(name, AF::LOCATION_PATH, properties[AF::METADATA_CONFIG], 'metadata')
+        end
       end
+
+      @result
     end
-    
+
     private
-    def self.assert_path_property_valid(name, path_prop, properties, existing_paths)
+    def assert_path_property_valid(name, path_prop, properties, section_name)
       path = properties[path_prop]
-      begin
-        StoragePathValidator::validate(path)
-      rescue InvalidStoragePathError => err
-        raise ConfigurationError.new(
-            "Storage location '#{name}' specifies invalid '#{path_prop}' property: #{err.message}")
-      end
-      assert("Storage location '#{name}' must specify a '#{path_prop}' property", !path.nil? && !path.empty?)
-      assert("Storage location '#{name}' must specify an absolute path for property '#{path_prop}'",
-          Pathname.new(path).absolute? && !path.include?('/..'))
+
+      storage_type = properties[AF::STORAGE_TYPE] || ST::DEFAULT_STORAGE_TYPE
+      type_validator = @@storage_type_mappings[storage_type]
+      type_validator.validate(self, name, path_prop, section_name, path)
+
       # Ensure paths have trailing slash to avoid matching on partial directory names
       path += '/' unless path.end_with?('/')
       # Verify that the (metadata_)path property's value is not inside of another storage location or vice versa
-      existing_paths.each do |existing|
+      @existing_paths.each do |existing|
         if existing.start_with?(path) || path.start_with?(existing)
-          msg = "Location '#{name}' defines property #{path_prop} with value '#{path}'" \
+          msg = "Location '#{name}' defines property #{section_name} #{path_prop} with value '#{path}'" \
                 " which overlaps with another configured path '#{existing}'." \
-                " Storage locations must not define #{AF::LOCATION_PATH} or #{AF::METADATA_PATH}" \
+                " Storage locations must not define #{AF::LOCATION_PATH}" \
                 " properties which are contained by another location property"
-          raise ConfigurationError.new(msg)
+          fail(msg)
         end
       end
-      
-      existing_paths << path
+
+      @existing_paths << path
     end
   end
-end
+end

data/lib/longleaf/specs/config_builder.rb

@@ -1,19 +1,19 @@
-require_relative '../models/app_fields'
-require_relative '../models/service_fields'
+require 'longleaf/models/app_fields'
+require 'longleaf/models/service_fields'
 require 'yaml'
 
-# Test helper for constructing application configuration hashes
 module Longleaf
+  # Test helper for constructing application configuration hashes
   class ConfigBuilder
     AF ||= Longleaf::AppFields
     SF ||= Longleaf::ServiceFields
-    
+
     attr_accessor :config
-  
+
     def initialize
       @config = Hash.new
     end
-  
+
     # Add a root 'locations' field to the config
     # @param locations [Hash] value for the locations fields. Default is {}
     # @return this builder
@@ -21,22 +21,30 @@ module Longleaf
       @config[AF::LOCATIONS] = locations
       self
     end
-  
+
     # Add a 'location' to the config
     # @param name [String] name of the location
     # @param path [String] value for the 'path' field
     # @param md_path [String] value for the 'metadata_path' field
     # @return this builder
-    def with_location(name:, path: '/file/path/', md_path: '/metadata/path/')
+    def with_location(name:, path: '/file/path/', s_type: nil, md_path: '/metadata/path/', md_type: nil, md_digests: nil)
       @config[AF::LOCATIONS] = Hash.new unless @config.key?(AF::LOCATIONS)
-      
+
       location = {}
       @config[AF::LOCATIONS][name] = location
       location[AF::LOCATION_PATH] = path unless path.nil?
-      location[AF::METADATA_PATH] = md_path unless md_path.nil?
+      location[AF::STORAGE_TYPE] = s_type unless s_type.nil?
+
+      if !md_path.nil?
+        md_loc = { AF::LOCATION_PATH => md_path }
+        location[AF::METADATA_CONFIG] = md_loc
+
+        md_loc[AF::METADATA_DIGESTS] = md_digests unless md_digests.nil?
+        md_loc[AF::STORAGE_TYPE] = md_type unless md_type.nil?
+      end
       self
     end
-    
+
     # Add a root 'services' field to the config
     # @param services [Hash] value for the services field. Default is {}
     # @return this builder
@@ -44,26 +52,29 @@ module Longleaf
       @config[AF::SERVICES] = services
       self
     end
-    
+
     # Add a 'service' to the config
     # @param name [String] name of the service
     # @param work_script [String] value for the 'work_script' field
+    # @param work_class [String] value for the 'work_class' field
     # @param frequency [String] value for the 'frequency' field
     # @param delay [String] value for the 'delay' field
     # @param properties [Hash] hash of additional properties to include in the service
     # @return this builder
-    def with_service(name:, work_script: 'some_pres_service.rb', frequency: nil, delay: nil, properties: nil)
+    def with_service(name:, work_script: 'some_pres_service.rb', work_class: nil,
+         frequency: nil, delay: nil, properties: nil)
       @config[AF::SERVICES] = Hash.new unless @config.key?(AF::SERVICES)
-      
+
       service = {}
       service[SF::WORK_SCRIPT] = work_script
+      service[SF::WORK_CLASS] = work_class
       service[SF::FREQUENCY] = frequency unless frequency.nil?
       service[SF::DELAY] = delay unless delay.nil?
       service = service.merge(properties) unless properties.nil?
       @config[AF::SERVICES][name] = service
       self
     end
-    
+
     # Adds a 'service_mappings' section to the config
     # @param mappings [Object] the mappings config
     # @return this builder
@@ -71,32 +82,45 @@ module Longleaf
       @config[AF::SERVICE_MAPPINGS] = mappings
       self
     end
-    
+
     # Add a mapping from one or more services to one or more location
     # @param loc_names [Object] one or more location names. Can be a string or array.
     # @param service_names [Object] one or more service names. Can be a string or array.
     def map_services(loc_names, service_names)
       @config[AF::SERVICE_MAPPINGS] = Array.new unless @config.key?(AF::SERVICE_MAPPINGS)
-      
+
       mapping = Hash.new
       mapping[AF::LOCATIONS] = loc_names unless loc_names.nil?
       mapping[AF::SERVICES] = service_names unless service_names.nil?
       @config[AF::SERVICE_MAPPINGS].push(mapping)
       self
     end
-    
+
+    # Add a system config section to the config
+    def with_system(sys_config)
+      @config[AF::SYSTEM] = sys_config
+      self
+    end
+
     # @return the constructed configuration
     def get
       @config
     end
-    
+
     # Writes the configuration from this builder into a temporary file
     # @return the file path of the config file
-    def write_to_yaml_file
-      Tempfile.open('config') do |f|
+    def write_to_yaml_file(dest_path = nil)
+      if dest_path.nil?
+        file = Tempfile.new('config')
+        file.close
+        dest_path = file.path
+        # deleting temp file but reusing file name. This is to avoid premature garbage collection.
+        file.unlink
+      end
+      File.open(dest_path, 'w') do |f|
         f.write(@config.to_yaml)
-        return f.path
       end
+      dest_path
     end
   end
-end
+end

data/lib/longleaf/specs/config_validator_helpers.rb

@@ -0,0 +1,16 @@
+module Longleaf
+  module ConfigValidatorHelpers
+    def fails_validation_with_error(validator, *error_messages)
+      result = validator.validate_config
+      expect(result.valid?).to be false
+      error_messages.each do |error_message|
+        expect(result.errors).to include(error_message)
+      end
+    end
+
+    def passes_validation(validator)
+      result = validator.validate_config
+      expect(result.valid?).to eq(true), "expected validation to pass, but received errors:\n#{result.errors&.join("\n")}"
+    end
+  end
+end

data/lib/longleaf/specs/custom_matchers.rb

@@ -0,0 +1,9 @@
+module Longleaf
+  # Match if the parameter is a {FileRecord} with the expected path
+  RSpec::Matchers.define :be_file_record_for do |expected|
+    match do |actual|
+      return false if actual.nil? || !actual.is_a?(Longleaf::FileRecord)
+      actual.path == expected
+    end
+  end
+end

data/lib/longleaf/specs/file_helpers.rb

@@ -0,0 +1,61 @@
+require 'tempfile'
+require 'tmpdir'
+
+module Longleaf
+  # Test helper methods for creating test files
+  module FileHelpers
+    def make_test_dir(parent: nil, name: 'dir')
+      FileHelpers.make_test_dir(parent: parent, name: name)
+    end
+
+    def self.make_test_dir(parent: nil, name: 'dir')
+      if parent.nil?
+        Dir.mktmpdir(name)
+      else
+        path = File.join(parent, name)
+        Dir.mkdir(path)
+        path
+      end
+    end
+
+    def create_test_file(dir: nil, name: nil, content: 'content')
+      FileHelpers.create_test_file(dir: dir, name: name, content: content)
+    end
+
+    def self.create_test_file(dir: nil, name: nil, content: 'content')
+      if dir.nil? || name.nil?
+        name = 'test_file' if name.nil?
+        file = Tempfile.create(name, dir)
+        file << content
+        file.close
+        return file.path
+      else
+        path = File.join(dir, name)
+        File.open(path, 'w') { |f| f.write(content) }
+        path
+      end
+    end
+
+    def create_work_class(lib_dir, class_name, file_name, module_name = nil, is_applicable: true, init_body: "", perform: "")
+      FileHelpers.create_work_class(lib_dir, class_name, file_name, module_name, is_applicable, init_body, perform)
+    end
+
+    def self.create_work_class(lib_dir, class_name, file_name, module_name, is_applicable, init_body, perform)
+      class_contents = %Q(
+        class #{class_name}
+          def initialize(service_def, app_manager)
+            #{init_body}
+          end
+          def perform(file_rec, event)
+            #{perform}
+          end
+          def is_applicable?(event)
+            #{is_applicable}
+          end
+        end
+      )
+      class_contents = "module #{module_name}\n#{class_contents}\nend" unless module_name.nil?
+      create_test_file(dir: lib_dir, name: file_name, content: class_contents)
+    end
+  end
+end

data/lib/longleaf/specs/metadata_builder.rb

@@ -0,0 +1,98 @@
+require 'longleaf/models/md_fields'
+require 'longleaf/helpers/service_date_helper'
+require 'yaml'
+
+module Longleaf
+  # Test helper for constructing file metadata records
+  class MetadataBuilder
+    MF ||= Longleaf::MDFields
+
+    def initialize(file_path: nil, registered: ServiceDateHelper::formatted_timestamp)
+      @data = Hash.new
+      @services = Hash.new
+
+      unless file_path.nil?
+        @last_modified = File.mtime(file_path).utc.iso8601(3)
+        @file_size = File.size(file_path)
+      end
+
+      @registered = registered
+    end
+
+    def deregistered(timestamp = ServiceDateHelper::formatted_timestamp)
+      @deregistered = timestamp
+      self
+    end
+
+    def with_checksum(alg, value)
+      @checksums = Hash.new unless @data.key?(MF::CHECKSUMS)
+      @checksums[alg] = value
+      self
+    end
+    
+    def with_physical_path(phys_path)
+      @physical_path = phys_path
+      self
+    end
+
+    def with_service(name, timestamp: ServiceDateHelper::formatted_timestamp, run_needed: false, properties: nil,
+          failure_timestamp: nil)
+      timestamp = format_timestamp(timestamp)
+      failure_timestamp = format_timestamp(failure_timestamp) unless failure_timestamp.nil?
+
+      @services[name] = ServiceRecord.new(
+          properties: properties.nil? ? Hash.new : nil,
+          timestamp: timestamp,
+          run_needed: run_needed)
+      @services[name].failure_timestamp = failure_timestamp
+      self
+    end
+
+    def with_properties(properties)
+      @properties = properties
+      self
+    end
+
+    # @return the constructed metadata record
+    def get_metadata_record
+      MetadataRecord.new(properties: @properties,
+          services: @services,
+          deregistered: @deregistered,
+          registered: @registered,
+          checksums: @checksums,
+          file_size: @file_size,
+          last_modified: @last_modified,
+          physical_path: @physical_path)
+    end
+
+    # Add the generated metadata record to the given file record
+    def register_to(file_rec)
+      file_rec.metadata_record = get_metadata_record
+      self
+    end
+
+    # Writes the metadata record from this builder into a temporary file, or if a file
+    # record is provided, then to the expected metadata path for the record, and assigns
+    # the result as the metadata record for the file record.
+    # @return the file path of the config file
+    def write_to_yaml_file(file_rec: nil)
+      md_path = nil
+      if file_rec.nil?
+        md_path = TempFile.new(['metadata', 'yml']).path
+      else
+        md_path = file_rec.metadata_path
+      end
+
+      md_rec = get_metadata_record
+      MetadataSerializer::write(metadata: md_rec, file_path: md_path)
+      file_rec.metadata_record = md_rec
+
+      md_path
+    end
+
+    private
+    def format_timestamp(timestamp)
+      timestamp.kind_of?(Time) ? ServiceDateHelper::formatted_timestamp(timestamp) : timestamp
+    end
+  end
+end