longleaf 0.1.0 → 0.2.0.pre.1

data/lib/longleaf/services/service_definition_manager.rb

@@ -1,14 +1,16 @@
 require_relative '../models/app_fields'
 require_relative '../models/service_definition'
 
-# Manager which loads and provides access to Longleaf::ServiceDefinition objects
 module Longleaf
+  # Manager which loads and provides access to Longleaf::ServiceDefinition objects
   class ServiceDefinitionManager
     SF ||= Longleaf::ServiceFields
     AF ||= Longleaf::AppFields
     
+    # Hash containing the set of configured services, represented as {ServiceDefinition} objects
     attr_reader :services
     
+    # @param config [Hash] hash representation of the application configuration
     def initialize(config)
       raise ArgumentError.new("Configuration must be provided") if config.nil? || config.empty?
 
@@ -18,11 +20,13 @@ module Longleaf
       @services = Hash.new
       config[AF::SERVICES].each do |name, properties|
         work_script = properties.delete(SF::WORK_SCRIPT)
+        work_class = properties.delete(SF::WORK_CLASS)
         frequency = properties.delete(SF::FREQUENCY)
         delay = properties.delete(SF::DELAY)
         service = Longleaf::ServiceDefinition.new(
             name: name,
             work_script: work_script,
+            work_class: work_class,
             frequency: frequency,
             delay: delay,
             properties: properties)

data/lib/longleaf/services/service_definition_validator.rb

@@ -1,11 +1,11 @@
 require 'pathname'
-require_relative '../models/service_fields'
-require_relative '../models/app_fields'
-require_relative '../errors'
+require 'longleaf/models/service_fields'
+require 'longleaf/models/app_fields'
+require 'longleaf/errors'
 require_relative 'configuration_validator'
 
-# Validates application configuration of service definitions
 module Longleaf
+  # Validates application configuration of service definitions
   class ServiceDefinitionValidator < ConfigurationValidator
     SF ||= Longleaf::ServiceFields
     AF ||= Longleaf::AppFields

data/lib/longleaf/services/service_manager.rb

@@ -1,21 +1,84 @@
-# Manager which provides preservation service definitions based on their mappings
+require 'longleaf/helpers/service_date_helper'
+require 'longleaf/services/service_class_cache'
+
 module Longleaf
+  # Manager which provides preservation service definitions based on their mappings
   class ServiceManager
-    
-    def initialize(definition_manager:, mapping_manager:)
+    # @param definition_manager [ServiceDefinitionManager] the service definition manager
+    # @param mapping_manager [ServiceMappingManager] the mapping of services to locations
+    # @param app_manager [ApplicationConfigManager] manager for storage locations
+    def initialize(definition_manager:, mapping_manager:, app_manager:)
       raise ArgumentError.new('Service definition manager required') if definition_manager.nil?
       raise ArgumentError.new('Service mappings manager required') if mapping_manager.nil?
+      raise ArgumentError.new('Storage location manager required') if app_manager.nil?
       @definition_manager = definition_manager
       @mapping_manager = mapping_manager
+      @app_manager = app_manager
+      @service_class_cache = ServiceClassCache.new(app_manager)
     end
     
-    # Gets a list of ServiceDefinition objects which match the given criteria
-    # @param location [String] name of the location to lookup
-    # @return [Array] a list of ServiceDefinition objects associated with the location,
-    #    or an empty list if no services match the criteria
-    def list_service_definitions(location: nil)
+    # List the names of services which are applicable to the given criteria
+    # @param location [String] name of the locations to lookup
+    # @param event [String] name of the preservation event taking place
+    # @return [Array] a list of service names which match the provided criteria
+    def list_services(location: nil, event: nil)
       service_names = @mapping_manager.list_services(location)
-      service_names.collect { |name| @definition_manager.services[name] }
+      if !event.nil?
+        # Filter service names down by event
+        service_names.select{ |name| applicable_for_event?(name, event) }
+      else
+        service_names
+      end
+    end
+    
+    # Determines if a service is applicable for a specific preservation event
+    # @param service_name [String] name of the service being evaluated
+    # @param event [String] name of the event to check against
+    # @return [Boolean] true if the service is applicable for the event
+    def applicable_for_event?(service_name, event)
+      definition = @definition_manager.services[service_name]
+      service = @service_class_cache.service_instance(definition)
+      
+      service.is_applicable?(event)
+    end
+    
+    # Determine if a service should run for a particular file based on the service's definition and
+    # the file's service related metadata.
+    # @param service_name [String] name of the service being evaluated
+    # @param md_rec [MetadataRecord] metadata record for the file being evaluated
+    # @return [Boolean] true if the service should be run.
+    def service_needed?(service_name, md_rec)
+      # If service not recorded for file, then it is needed
+      present_services = md_rec.list_services
+      return true unless present_services.include?(service_name)
+      
+      service_rec = md_rec.service(service_name)
+      
+      return true if service_rec.run_needed
+      return true if service_rec.timestamp.nil?
+      
+      definition = @definition_manager.services[service_name]
+      
+      # Check if the amount of time defined in frequency has passed since the service timestamp
+      frequency = definition.frequency
+      unless frequency.nil?
+        service_timestamp = service_rec.timestamp
+        now = ServiceDateHelper.formatted_timestamp
+        
+        return true if now > ServiceDateHelper.add_to_timestamp(service_timestamp, frequency)
+      end
+      false
+    end
+    
+    # Perform the specified service on the file record, in the context of the specified event
+    # @param service_name [String] name of the service
+    # @param file_rec [FileRecord] file record to perform service upon
+    # @param event_name [String] name of the event service is being performed within.
+    def perform_service(service_name, file_rec, event)
+      definition = @definition_manager.services[service_name]
+      
+      service = @service_class_cache.service_instance(definition)
+      service.perform(file_rec, event)
     end
   end
 end

data/lib/longleaf/services/service_mapping_manager.rb

@@ -1,11 +1,12 @@
-require_relative '../models/app_fields'
-require_relative '../models/service_definition'
+require 'longleaf/models/app_fields'
+require 'longleaf/models/service_definition'
 
-# Manager which loads and provides access to location to service mappings
 module Longleaf
+  # Manager which loads and provides access to location to service mappings
   class ServiceMappingManager
     AF ||= Longleaf::AppFields
     
+    # @param config [Hash] has representation of the application configuration
     def initialize(config)
       raise ArgumentError.new("Configuration must be provided") if config.nil? || config.empty?
 

data/lib/longleaf/services/service_mapping_validator.rb

@@ -1,11 +1,11 @@
 require 'pathname'
-require_relative '../models/service_fields'
-require_relative '../models/app_fields'
-require_relative '../errors'
+require 'longleaf/models/service_fields'
+require 'longleaf/models/app_fields'
+require 'longleaf/errors'
 require_relative 'configuration_validator'
 
-# Validates application configuration of service to location mappings
 module Longleaf
+  # Validates application configuration of service to location mappings
   class ServiceMappingValidator < ConfigurationValidator
     AF ||= Longleaf::AppFields
     

data/lib/longleaf/services/storage_location_manager.rb

@@ -1,13 +1,16 @@
-require_relative '../models/app_fields'
-require_relative '../models/storage_location'
+require 'longleaf/models/app_fields'
+require 'longleaf/models/storage_location'
+require 'longleaf/errors'
 
-# Manager which loads and provides access to Longleaf::StorageLocation objects
 module Longleaf
+  # Manager which loads and provides access to {StorageLocation} objects
   class StorageLocationManager
     AF ||= Longleaf::AppFields
     
+    # Hash mapping storage location names to {StorageLocation} objects
     attr_reader :locations
     
+    # @param config [Hash] has representation of the application configuration
     def initialize(config)
       raise ArgumentError.new("Configuration must be provided") if config&.empty?
 
@@ -15,14 +18,18 @@ module Longleaf
       config[AF::LOCATIONS].each do |name, properties|
         path = properties[AF::LOCATION_PATH]
         md_path = properties[AF::METADATA_PATH]
-        location = Longleaf::StorageLocation.new(name: name, path: path, metadata_path: md_path)
+        md_digests = properties[AF::METADATA_DIGESTS]
+        location = Longleaf::StorageLocation.new(name: name,
+            path: path,
+            metadata_path: md_path,
+            metadata_digests: md_digests)
         
         @locations[name] = location
       end
       @locations.freeze
     end
     
-    # Get the StorageLocation object which should contain the given path
+    # Get the {StorageLocation} object which should contain the given path
     # @return [Longleaf::StorageLocation] location containing the given path
     #    or nil if the path is not contained by a registered location.
     def get_location_by_path(path)
@@ -33,5 +40,19 @@ module Longleaf
       
       nil
     end
+    
+    # Raises a {StorageLocationUnavailableError} if the given path is not in a known storage location,
+    #    or if it is not within the expected location if provided
+    # @param path [String] file path
+    # @param expected_loc [String] name of the storage location which path should be contained by
+    # @raise [StorageLocationUnavailableError] if the path is not in a known/expected storage location
+    def verify_path_in_location(path, expected_loc = nil)
+      location = get_location_by_path(path)
+      if location.nil?
+        raise StorageLocationUnavailableError.new("Path #{path} is not from a known storage location.")
+      elsif !expected_loc.nil? && expected_loc != location.name
+        raise StorageLocationUnavailableError.new("Path #{path} is not contained by storage location #{expected_loc}.")
+      end
+    end
   end
 end

data/lib/longleaf/services/storage_location_validator.rb

@@ -5,8 +5,8 @@ require 'longleaf/errors'
 require_relative 'configuration_validator'
 require 'longleaf/services/storage_path_validator'
 
-# Validates application configuration of storage locations
 module Longleaf
+  # Validates application configuration of storage locations
   class StorageLocationValidator < ConfigurationValidator
     AF ||= Longleaf::AppFields
     

data/lib/longleaf/services/storage_path_validator.rb

@@ -1,11 +1,11 @@
 require 'longleaf/errors'
 
-# Validator for storage paths
 module Longleaf
+  # Validator for storage paths
   class StoragePathValidator
     # Checks that the given path is a syntactically valid storage path
     # @param path [String] file storage path to validate
-    # @raises [InvalidStoragePathError]
+    # @raise [InvalidStoragePathError]
     def self.validate(path)
       raise InvalidStoragePathError.new("Path must not be empty") if path.to_s.strip.empty?
       raise InvalidStoragePathError.new("Path must be absolute") unless Pathname.new(path).absolute?

data/lib/longleaf/specs/config_builder.rb

@@ -1,9 +1,9 @@
-require_relative '../models/app_fields'
-require_relative '../models/service_fields'
+require 'longleaf/models/app_fields'
+require 'longleaf/models/service_fields'
 require 'yaml'
 
-# Test helper for constructing application configuration hashes
 module Longleaf
+  # Test helper for constructing application configuration hashes
   class ConfigBuilder
     AF ||= Longleaf::AppFields
     SF ||= Longleaf::ServiceFields
@@ -27,13 +27,14 @@ module Longleaf
     # @param path [String] value for the 'path' field
     # @param md_path [String] value for the 'metadata_path' field
     # @return this builder
-    def with_location(name:, path: '/file/path/', md_path: '/metadata/path/')
+    def with_location(name:, path: '/file/path/', md_path: '/metadata/path/', md_digests: nil)
       @config[AF::LOCATIONS] = Hash.new unless @config.key?(AF::LOCATIONS)
       
       location = {}
       @config[AF::LOCATIONS][name] = location
       location[AF::LOCATION_PATH] = path unless path.nil?
       location[AF::METADATA_PATH] = md_path unless md_path.nil?
+      location[AF::METADATA_DIGESTS] = md_digests unless md_digests.nil?
       self
     end
     
@@ -48,15 +49,18 @@ module Longleaf
     # Add a 'service' to the config
     # @param name [String] name of the service
     # @param work_script [String] value for the 'work_script' field
+    # @param work_class [String] value for the 'work_class' field
     # @param frequency [String] value for the 'frequency' field
     # @param delay [String] value for the 'delay' field
     # @param properties [Hash] hash of additional properties to include in the service
     # @return this builder
-    def with_service(name:, work_script: 'some_pres_service.rb', frequency: nil, delay: nil, properties: nil)
+    def with_service(name:, work_script: 'some_pres_service.rb', work_class: nil,
+         frequency: nil, delay: nil, properties: nil)
       @config[AF::SERVICES] = Hash.new unless @config.key?(AF::SERVICES)
       
       service = {}
       service[SF::WORK_SCRIPT] = work_script
+      service[SF::WORK_CLASS] = work_class
       service[SF::FREQUENCY] = frequency unless frequency.nil?
       service[SF::DELAY] = delay unless delay.nil?
       service = service.merge(properties) unless properties.nil?

data/lib/longleaf/specs/custom_matchers.rb

@@ -0,0 +1,9 @@
+module Longleaf
+  # Match if the parameter is a {FileRecord} with the expected path
+  RSpec::Matchers.define :be_file_record_for do |expected|
+    match do |actual|
+      return false if actual.nil? || !actual.is_a?(Longleaf::FileRecord)
+      actual.path == expected
+    end
+  end
+end

data/lib/longleaf/specs/file_helpers.rb

@@ -0,0 +1,60 @@
+require 'tempfile'
+require 'tmpdir'
+
+module Longleaf
+  # Test helper methods for creating test files
+  module FileHelpers
+    def make_test_dir(parent: nil, name: 'dir')
+      FileHelpers.make_test_dir(parent: parent, name: name)
+    end
+    
+    def self.make_test_dir(parent: nil, name: 'dir')
+      if parent.nil?
+        Dir.mktmpdir(name)
+      else
+        path = File.join(parent, name)
+        Dir.mkdir(path)
+        path
+      end
+    end
+    
+    def create_test_file(dir: nil, name: 'test_file', content: 'content')
+      FileHelpers.create_test_file(dir: dir, name: name, content: content)
+    end
+    
+    def self.create_test_file(dir: nil, name: 'test_file', content: 'content')
+      if dir.nil?
+        file = Tempfile.create(name)
+        file << content
+        file.close
+        return file.path
+      else
+        path = File.join(dir, name)
+        File.open(path, 'w') { |f| f.write(content) }
+        path
+      end
+    end
+    
+    def create_work_class(lib_dir, class_name, file_name, module_name = nil, is_applicable: true, perform: "")
+      FileHelpers.create_work_class(lib_dir, class_name, file_name, module_name, is_applicable: is_applicable,
+         perform: perform)
+    end
+    
+    def self.create_work_class(lib_dir, class_name, file_name, module_name = nil, is_applicable: true, perform: "")
+      class_contents = %Q(
+        class #{class_name}
+          def initialize(service_def, app_manager)
+          end
+          def perform(file_rec, event)
+            #{perform}
+          end
+          def is_applicable?(event)
+            #{is_applicable}
+          end
+        end
+      )
+      class_contents = "module #{module_name}\n#{class_contents}\nend" unless module_name.nil?
+      create_test_file(dir: lib_dir, name: file_name, content: class_contents)
+    end
+  end
+end

data/lib/longleaf/version.rb

@@ -1,3 +1,3 @@
 module Longleaf
-  VERSION = "0.1.0"
+  VERSION = "0.2.0.pre.1"
 end

data/longleaf.gemspec

@@ -31,4 +31,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rspec", "~> 3.6.0"
   spec.add_development_dependency "factory_bot", "~> 4.0"
   spec.add_development_dependency "aruba", "~> 0.14.0"
+  spec.add_development_dependency "bixby", ">= 2.0.0.pre.beta1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: longleaf
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0.pre.1
 platform: ruby
 authors:
 - bbpennel
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-10-17 00:00:00.000000000 Z
+date: 2019-03-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -108,6 +108,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.14.0
+- !ruby/object:Gem::Dependency
+  name: bixby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.0.pre.beta1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.0.pre.beta1
 description: Provides a framework for performing preservation services over sets of
   files.
 email:
@@ -117,8 +131,11 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".editorconfig"
 - ".gitignore"
 - ".rspec"
+- ".rubocop.yml"
+- ".rubocop_todo.yml"
 - ".travis.yml"
 - Gemfile
 - LICENSE.txt
@@ -128,12 +145,23 @@ files:
 - bin/setup
 - exe/longleaf
 - lib/longleaf.rb
+- lib/longleaf/candidates/file_selector.rb
+- lib/longleaf/candidates/service_candidate_filesystem_iterator.rb
+- lib/longleaf/candidates/service_candidate_locator.rb
 - lib/longleaf/cli.rb
-- lib/longleaf/commands/abstract_command.rb
+- lib/longleaf/commands/deregister_command.rb
+- lib/longleaf/commands/preserve_command.rb
 - lib/longleaf/commands/register_command.rb
 - lib/longleaf/commands/validate_config_command.rb
+- lib/longleaf/commands/validate_metadata_command.rb
 - lib/longleaf/errors.rb
+- lib/longleaf/events/deregister_event.rb
+- lib/longleaf/events/event_names.rb
+- lib/longleaf/events/event_status_tracking.rb
+- lib/longleaf/events/preserve_event.rb
 - lib/longleaf/events/register_event.rb
+- lib/longleaf/helpers/digest_helper.rb
+- lib/longleaf/helpers/service_date_helper.rb
 - lib/longleaf/logging.rb
 - lib/longleaf/logging/redirecting_logger.rb
 - lib/longleaf/models/app_fields.rb
@@ -144,12 +172,15 @@ files:
 - lib/longleaf/models/service_fields.rb
 - lib/longleaf/models/service_record.rb
 - lib/longleaf/models/storage_location.rb
+- lib/longleaf/preservation_services/fixity_check_service.rb
+- lib/longleaf/preservation_services/rsync_replication_service.rb
 - lib/longleaf/services/application_config_deserializer.rb
 - lib/longleaf/services/application_config_manager.rb
 - lib/longleaf/services/application_config_validator.rb
 - lib/longleaf/services/configuration_validator.rb
 - lib/longleaf/services/metadata_deserializer.rb
 - lib/longleaf/services/metadata_serializer.rb
+- lib/longleaf/services/service_class_cache.rb
 - lib/longleaf/services/service_definition_manager.rb
 - lib/longleaf/services/service_definition_validator.rb
 - lib/longleaf/services/service_manager.rb
@@ -159,6 +190,8 @@ files:
 - lib/longleaf/services/storage_location_validator.rb
 - lib/longleaf/services/storage_path_validator.rb
 - lib/longleaf/specs/config_builder.rb
+- lib/longleaf/specs/custom_matchers.rb
+- lib/longleaf/specs/file_helpers.rb
 - lib/longleaf/version.rb
 - longleaf.gemspec
 homepage: https://github.com/UNC-Libraries/
@@ -176,12 +209,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.2
 signing_key: 
 specification_version: 4
 summary: Longleaf preservation services tool