longleaf 0.1.0 → 1.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.circleci/config.yml +94 -0
- data/.editorconfig +13 -0
- data/.gitignore +4 -1
- data/.rubocop.yml +44 -0
- data/.rubocop_todo.yml +834 -0
- data/.yardopts +1 -0
- data/Gemfile +16 -1
- data/README.md +98 -12
- data/Rakefile +6 -0
- data/bin/setup +16 -1
- data/docs/aboutlongleaf.md +28 -0
- data/docs/extra.css +32 -0
- data/docs/img/change-file.png +0 -0
- data/docs/img/ll-example-preserved.png +0 -0
- data/docs/index.md +19 -0
- data/docs/install.md +66 -0
- data/docs/ll-example/config-example-relative.yml +33 -0
- data/docs/ll-example/files-dir/LLexample-PDF.pdf +0 -0
- data/docs/ll-example/files-dir/LLexample-TOCHANGE.txt +15 -0
- data/docs/ll-example/files-dir/LLexample-tokeep.txt +10 -0
- data/docs/ll-example/metadata-dir/.gitkeep +0 -0
- data/docs/ll-example/replica-files/.gitkeep +0 -0
- data/docs/ll-example/replica-metadata/.gitkeep +0 -0
- data/docs/quickstart.md +270 -0
- data/docs/rdocs/Longleaf.html +135 -0
- data/docs/rdocs/Longleaf/AppFields.html +178 -0
- data/docs/rdocs/Longleaf/ApplicationConfigDeserializer.html +631 -0
- data/docs/rdocs/Longleaf/ApplicationConfigManager.html +610 -0
- data/docs/rdocs/Longleaf/ApplicationConfigValidator.html +238 -0
- data/docs/rdocs/Longleaf/CLI.html +909 -0
- data/docs/rdocs/Longleaf/ChecksumMismatchError.html +151 -0
- data/docs/rdocs/Longleaf/ConfigBuilder.html +1339 -0
- data/docs/rdocs/Longleaf/ConfigurationError.html +143 -0
- data/docs/rdocs/Longleaf/ConfigurationValidator.html +227 -0
- data/docs/rdocs/Longleaf/DeregisterCommand.html +420 -0
- data/docs/rdocs/Longleaf/DeregisterEvent.html +453 -0
- data/docs/rdocs/Longleaf/DeregistrationError.html +151 -0
- data/docs/rdocs/Longleaf/DigestHelper.html +419 -0
- data/docs/rdocs/Longleaf/EventError.html +147 -0
- data/docs/rdocs/Longleaf/EventNames.html +163 -0
- data/docs/rdocs/Longleaf/EventStatusTracking.html +656 -0
- data/docs/rdocs/Longleaf/FileCheckService.html +540 -0
- data/docs/rdocs/Longleaf/FileHelpers.html +520 -0
- data/docs/rdocs/Longleaf/FileRecord.html +716 -0
- data/docs/rdocs/Longleaf/FileSelector.html +901 -0
- data/docs/rdocs/Longleaf/FixityCheckService.html +691 -0
- data/docs/rdocs/Longleaf/IndexManager.html +1155 -0
- data/docs/rdocs/Longleaf/InvalidDigestAlgorithmError.html +143 -0
- data/docs/rdocs/Longleaf/InvalidStoragePathError.html +143 -0
- data/docs/rdocs/Longleaf/Logging.html +405 -0
- data/docs/rdocs/Longleaf/Logging/RedirectingLogger.html +1213 -0
- data/docs/rdocs/Longleaf/LongleafError.html +139 -0
- data/docs/rdocs/Longleaf/MDFields.html +193 -0
- data/docs/rdocs/Longleaf/MetadataBuilder.html +787 -0
- data/docs/rdocs/Longleaf/MetadataDeserializer.html +537 -0
- data/docs/rdocs/Longleaf/MetadataError.html +143 -0
- data/docs/rdocs/Longleaf/MetadataPersistenceManager.html +539 -0
- data/docs/rdocs/Longleaf/MetadataRecord.html +1411 -0
- data/docs/rdocs/Longleaf/MetadataSerializer.html +786 -0
- data/docs/rdocs/Longleaf/PreservationServiceError.html +147 -0
- data/docs/rdocs/Longleaf/PreserveCommand.html +410 -0
- data/docs/rdocs/Longleaf/PreserveEvent.html +491 -0
- data/docs/rdocs/Longleaf/RegisterCommand.html +428 -0
- data/docs/rdocs/Longleaf/RegisterEvent.html +628 -0
- data/docs/rdocs/Longleaf/RegisteredFileSelector.html +446 -0
- data/docs/rdocs/Longleaf/RegistrationError.html +151 -0
- data/docs/rdocs/Longleaf/ReindexCommand.html +576 -0
- data/docs/rdocs/Longleaf/RsyncReplicationService.html +1180 -0
- data/docs/rdocs/Longleaf/SequelIndexDriver.html +1978 -0
- data/docs/rdocs/Longleaf/ServiceCandidateFilesystemIterator.html +572 -0
- data/docs/rdocs/Longleaf/ServiceCandidateIndexIterator.html +532 -0
- data/docs/rdocs/Longleaf/ServiceCandidateLocator.html +333 -0
- data/docs/rdocs/Longleaf/ServiceClassCache.html +725 -0
- data/docs/rdocs/Longleaf/ServiceDateHelper.html +425 -0
- data/docs/rdocs/Longleaf/ServiceDefinition.html +683 -0
- data/docs/rdocs/Longleaf/ServiceDefinitionManager.html +371 -0
- data/docs/rdocs/Longleaf/ServiceDefinitionValidator.html +269 -0
- data/docs/rdocs/Longleaf/ServiceFields.html +173 -0
- data/docs/rdocs/Longleaf/ServiceManager.html +1229 -0
- data/docs/rdocs/Longleaf/ServiceMappingManager.html +410 -0
- data/docs/rdocs/Longleaf/ServiceMappingValidator.html +347 -0
- data/docs/rdocs/Longleaf/ServiceRecord.html +821 -0
- data/docs/rdocs/Longleaf/StorageLocation.html +985 -0
- data/docs/rdocs/Longleaf/StorageLocationManager.html +729 -0
- data/docs/rdocs/Longleaf/StorageLocationUnavailableError.html +143 -0
- data/docs/rdocs/Longleaf/StorageLocationValidator.html +373 -0
- data/docs/rdocs/Longleaf/StoragePathValidator.html +253 -0
- data/docs/rdocs/Longleaf/SystemConfigBuilder.html +441 -0
- data/docs/rdocs/Longleaf/SystemConfigFields.html +163 -0
- data/docs/rdocs/Longleaf/ValidateConfigCommand.html +451 -0
- data/docs/rdocs/Longleaf/ValidateMetadataCommand.html +408 -0
- data/docs/rdocs/_index.html +660 -0
- data/docs/rdocs/class_list.html +51 -0
- data/docs/rdocs/css/common.css +1 -0
- data/docs/rdocs/css/full_list.css +58 -0
- data/docs/rdocs/css/style.css +496 -0
- data/docs/rdocs/file.README.html +165 -0
- data/docs/rdocs/file_list.html +56 -0
- data/docs/rdocs/frames.html +17 -0
- data/docs/rdocs/index.html +165 -0
- data/docs/rdocs/js/app.js +303 -0
- data/docs/rdocs/js/full_list.js +216 -0
- data/docs/rdocs/js/jquery.js +4 -0
- data/docs/rdocs/method_list.html +2051 -0
- data/docs/rdocs/top-level-namespace.html +110 -0
- data/lib/longleaf/candidates/file_selector.rb +150 -0
- data/lib/longleaf/candidates/manifest_digest_provider.rb +17 -0
- data/lib/longleaf/candidates/physical_path_provider.rb +17 -0
- data/lib/longleaf/candidates/registered_file_selector.rb +67 -0
- data/lib/longleaf/candidates/service_candidate_filesystem_iterator.rb +93 -0
- data/lib/longleaf/candidates/service_candidate_index_iterator.rb +84 -0
- data/lib/longleaf/candidates/service_candidate_locator.rb +23 -0
- data/lib/longleaf/candidates/single_digest_provider.rb +13 -0
- data/lib/longleaf/cli.rb +252 -46
- data/lib/longleaf/commands/deregister_command.rb +51 -0
- data/lib/longleaf/commands/preserve_command.rb +50 -0
- data/lib/longleaf/commands/register_command.rb +34 -43
- data/lib/longleaf/commands/reindex_command.rb +92 -0
- data/lib/longleaf/commands/validate_config_command.rb +33 -8
- data/lib/longleaf/commands/validate_metadata_command.rb +51 -0
- data/lib/longleaf/errors.rb +26 -7
- data/lib/longleaf/events/deregister_event.rb +53 -0
- data/lib/longleaf/events/event_names.rb +9 -0
- data/lib/longleaf/events/event_status_tracking.rb +59 -0
- data/lib/longleaf/events/preserve_event.rb +82 -0
- data/lib/longleaf/events/register_event.rb +59 -51
- data/lib/longleaf/helpers/case_insensitive_hash.rb +38 -0
- data/lib/longleaf/helpers/digest_helper.rb +56 -0
- data/lib/longleaf/helpers/s3_uri_helper.rb +86 -0
- data/lib/longleaf/helpers/selection_options_parser.rb +215 -0
- data/lib/longleaf/helpers/service_date_helper.rb +78 -0
- data/lib/longleaf/indexing/index_manager.rb +101 -0
- data/lib/longleaf/indexing/sequel_index_driver.rb +306 -0
- data/lib/longleaf/logging.rb +5 -4
- data/lib/longleaf/logging/redirecting_logger.rb +30 -25
- data/lib/longleaf/models/app_fields.rb +7 -2
- data/lib/longleaf/models/file_record.rb +31 -8
- data/lib/longleaf/models/filesystem_metadata_location.rb +56 -0
- data/lib/longleaf/models/filesystem_storage_location.rb +52 -0
- data/lib/longleaf/models/md_fields.rb +3 -1
- data/lib/longleaf/models/metadata_location.rb +47 -0
- data/lib/longleaf/models/metadata_record.rb +43 -16
- data/lib/longleaf/models/s3_storage_location.rb +138 -0
- data/lib/longleaf/models/service_definition.rb +7 -6
- data/lib/longleaf/models/service_fields.rb +7 -1
- data/lib/longleaf/models/service_record.rb +10 -6
- data/lib/longleaf/models/storage_location.rb +24 -19
- data/lib/longleaf/models/storage_types.rb +9 -0
- data/lib/longleaf/models/system_config_fields.rb +9 -0
- data/lib/longleaf/preservation_services/file_check_service.rb +59 -0
- data/lib/longleaf/preservation_services/fixity_check_service.rb +124 -0
- data/lib/longleaf/preservation_services/rsync_replication_service.rb +198 -0
- data/lib/longleaf/preservation_services/s3_replication_service.rb +131 -0
- data/lib/longleaf/services/application_config_deserializer.rb +81 -24
- data/lib/longleaf/services/application_config_manager.rb +20 -6
- data/lib/longleaf/services/application_config_validator.rb +19 -9
- data/lib/longleaf/services/configuration_validator.rb +67 -4
- data/lib/longleaf/services/filesystem_location_validator.rb +16 -0
- data/lib/longleaf/services/metadata_deserializer.rb +115 -42
- data/lib/longleaf/services/metadata_persistence_manager.rb +47 -0
- data/lib/longleaf/services/metadata_serializer.rb +156 -23
- data/lib/longleaf/services/metadata_validator.rb +76 -0
- data/lib/longleaf/services/s3_location_validator.rb +19 -0
- data/lib/longleaf/services/service_class_cache.rb +112 -0
- data/lib/longleaf/services/service_definition_manager.rb +10 -7
- data/lib/longleaf/services/service_definition_validator.rb +25 -18
- data/lib/longleaf/services/service_manager.rb +86 -11
- data/lib/longleaf/services/service_mapping_manager.rb +13 -12
- data/lib/longleaf/services/service_mapping_validator.rb +36 -26
- data/lib/longleaf/services/storage_location_manager.rb +76 -15
- data/lib/longleaf/services/storage_location_validator.rb +49 -35
- data/lib/longleaf/specs/config_builder.rb +47 -23
- data/lib/longleaf/specs/config_validator_helpers.rb +16 -0
- data/lib/longleaf/specs/custom_matchers.rb +9 -0
- data/lib/longleaf/specs/file_helpers.rb +61 -0
- data/lib/longleaf/specs/metadata_builder.rb +98 -0
- data/lib/longleaf/specs/system_config_builder.rb +27 -0
- data/lib/longleaf/version.rb +1 -1
- data/longleaf.gemspec +20 -7
- data/mkdocs.yml +21 -0
- metadata +308 -24
- data/.travis.yml +0 -4
- data/lib/longleaf/commands/abstract_command.rb +0 -37
- data/lib/longleaf/services/storage_path_validator.rb +0 -16
@@ -1,36 +1,39 @@
|
|
1
1
|
require_relative '../models/app_fields'
|
2
2
|
require_relative '../models/service_definition'
|
3
3
|
|
4
|
-
# Manager which loads and provides access to Longleaf::ServiceDefinition objects
|
5
4
|
module Longleaf
|
5
|
+
# Manager which loads and provides access to Longleaf::ServiceDefinition objects
|
6
6
|
class ServiceDefinitionManager
|
7
7
|
SF ||= Longleaf::ServiceFields
|
8
8
|
AF ||= Longleaf::AppFields
|
9
|
-
|
9
|
+
|
10
|
+
# Hash containing the set of configured services, represented as {ServiceDefinition} objects
|
10
11
|
attr_reader :services
|
11
|
-
|
12
|
+
|
13
|
+
# @param config [Hash] hash representation of the application configuration
|
12
14
|
def initialize(config)
|
13
15
|
raise ArgumentError.new("Configuration must be provided") if config.nil? || config.empty?
|
14
16
|
|
15
17
|
services_config = config[AF::SERVICES]
|
16
18
|
raise ArgumentError.new("Services configuration must be provided") if services_config.nil?
|
17
|
-
|
19
|
+
|
18
20
|
@services = Hash.new
|
19
21
|
config[AF::SERVICES].each do |name, properties|
|
20
22
|
work_script = properties.delete(SF::WORK_SCRIPT)
|
23
|
+
work_class = properties.delete(SF::WORK_CLASS)
|
21
24
|
frequency = properties.delete(SF::FREQUENCY)
|
22
25
|
delay = properties.delete(SF::DELAY)
|
23
26
|
service = Longleaf::ServiceDefinition.new(
|
24
27
|
name: name,
|
25
28
|
work_script: work_script,
|
29
|
+
work_class: work_class,
|
26
30
|
frequency: frequency,
|
27
31
|
delay: delay,
|
28
32
|
properties: properties)
|
29
|
-
|
33
|
+
|
30
34
|
@services[name] = service
|
31
35
|
end
|
32
36
|
@services.freeze
|
33
37
|
end
|
34
|
-
|
35
38
|
end
|
36
|
-
end
|
39
|
+
end
|
@@ -1,32 +1,39 @@
|
|
1
1
|
require 'pathname'
|
2
|
-
|
3
|
-
|
4
|
-
|
2
|
+
require 'longleaf/models/service_fields'
|
3
|
+
require 'longleaf/models/app_fields'
|
4
|
+
require 'longleaf/errors'
|
5
5
|
require_relative 'configuration_validator'
|
6
6
|
|
7
|
-
# Validates application configuration of service definitions
|
8
7
|
module Longleaf
|
8
|
+
# Validates application configuration of service definitions
|
9
9
|
class ServiceDefinitionValidator < ConfigurationValidator
|
10
10
|
SF ||= Longleaf::ServiceFields
|
11
11
|
AF ||= Longleaf::AppFields
|
12
|
-
|
13
|
-
#
|
12
|
+
|
13
|
+
# @param config [Hash] hash containing the application configuration
|
14
|
+
def initialize(config)
|
15
|
+
super(config)
|
16
|
+
end
|
17
|
+
|
18
|
+
protected
|
19
|
+
# Validates configuration to ensure that it is syntactically correct and does not violate
|
14
20
|
# schema requirements.
|
15
21
|
# @param config [Hash] hash containing the application configuration
|
16
|
-
def
|
17
|
-
assert("Configuration must be a hash, but a #{config.class} was provided", config.class == Hash)
|
18
|
-
assert("Configuration must contain a root '#{AF::SERVICES}' key", config.key?(AF::SERVICES))
|
19
|
-
services = config[AF::SERVICES]
|
22
|
+
def validate
|
23
|
+
assert("Configuration must be a hash, but a #{@config.class} was provided", @config.class == Hash)
|
24
|
+
assert("Configuration must contain a root '#{AF::SERVICES}' key", @config.key?(AF::SERVICES))
|
25
|
+
services = @config[AF::SERVICES]
|
20
26
|
assert("'#{AF::SERVICES}' must be a hash of services", services.class == Hash)
|
21
|
-
|
22
|
-
existing_paths = Array.new
|
27
|
+
|
23
28
|
services.each do |name, properties|
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
+
register_on_failure do
|
30
|
+
assert("Name of service definition must be a string, but was of type #{name.class}", name.instance_of?(String))
|
31
|
+
assert("Service definition '#{name}' must be a hash, but a #{properties.class} was provided", properties.is_a?(Hash))
|
32
|
+
|
33
|
+
work_script = properties[SF::WORK_SCRIPT]
|
34
|
+
assert("Service definition '#{name}' must specify a '#{SF::WORK_SCRIPT}' property", !work_script.nil? && !work_script.empty?)
|
35
|
+
end
|
29
36
|
end
|
30
37
|
end
|
31
38
|
end
|
32
|
-
end
|
39
|
+
end
|
@@ -1,21 +1,96 @@
|
|
1
|
-
|
1
|
+
require 'longleaf/helpers/service_date_helper'
|
2
|
+
require 'longleaf/services/service_class_cache'
|
3
|
+
|
2
4
|
module Longleaf
|
5
|
+
# Manager which provides preservation service definitions based on their mappings
|
3
6
|
class ServiceManager
|
4
|
-
|
5
|
-
|
7
|
+
attr_reader :definition_manager
|
8
|
+
attr_reader :mapping_manager
|
9
|
+
|
10
|
+
# @param definition_manager [ServiceDefinitionManager] the service definition manager
|
11
|
+
# @param mapping_manager [ServiceMappingManager] the mapping of services to locations
|
12
|
+
# @param app_manager [ApplicationConfigManager] manager for storage locations
|
13
|
+
def initialize(definition_manager:, mapping_manager:, app_manager:)
|
6
14
|
raise ArgumentError.new('Service definition manager required') if definition_manager.nil?
|
7
15
|
raise ArgumentError.new('Service mappings manager required') if mapping_manager.nil?
|
16
|
+
raise ArgumentError.new('Storage location manager required') if app_manager.nil?
|
8
17
|
@definition_manager = definition_manager
|
9
18
|
@mapping_manager = mapping_manager
|
19
|
+
@app_manager = app_manager
|
20
|
+
@service_class_cache = ServiceClassCache.new(app_manager)
|
10
21
|
end
|
11
|
-
|
12
|
-
#
|
13
|
-
# @param
|
14
|
-
# @return
|
15
|
-
#
|
16
|
-
def
|
22
|
+
|
23
|
+
# Return a service instance instance for provided service name.
|
24
|
+
# @param service_name [String] name of the service
|
25
|
+
# @return Preservation service class for the provided name
|
26
|
+
# @raise ArgumentError if service_name does not reference an existing service
|
27
|
+
def service(service_name)
|
28
|
+
raise ArgumentError.new('Service name is required') if service_name.nil? || service_name.empty?
|
29
|
+
raise ArgumentError.new("No service with name #{service_name}") unless @definition_manager.services.key?(service_name)
|
30
|
+
definition = @definition_manager.services[service_name]
|
31
|
+
@service_class_cache.service_instance(definition)
|
32
|
+
end
|
33
|
+
|
34
|
+
# List the names of services which are applicable to the given criteria
|
35
|
+
# @param location [String] name of the locations to lookup
|
36
|
+
# @param event [String] name of the preservation event taking place
|
37
|
+
# @return [Array] a list of service names which match the provided criteria
|
38
|
+
def list_services(location: nil, event: nil)
|
17
39
|
service_names = @mapping_manager.list_services(location)
|
18
|
-
|
40
|
+
if !event.nil?
|
41
|
+
# Filter service names down by event
|
42
|
+
service_names.select { |name| applicable_for_event?(name, event) }
|
43
|
+
else
|
44
|
+
service_names
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
# List definitions for services which are applicable to the given criteria
|
49
|
+
# @param location [String] name of the locations to lookup
|
50
|
+
# @param event [String] name of the preservation event taking place
|
51
|
+
# @return [Array] List of service definitions which match the provided criteria
|
52
|
+
def list_service_definitions(location: nil, event: nil)
|
53
|
+
names = list_services(location: location, event: event)
|
54
|
+
names.map { |name| @definition_manager.services[name] }
|
55
|
+
end
|
56
|
+
|
57
|
+
# Determines if a service is applicable for a specific preservation event
|
58
|
+
# @param service_name [String] name of the service being evaluated
|
59
|
+
# @param event [String] name of the event to check against
|
60
|
+
# @return [Boolean] true if the service is applicable for the event
|
61
|
+
def applicable_for_event?(service_name, event)
|
62
|
+
service(service_name).is_applicable?(event)
|
63
|
+
end
|
64
|
+
|
65
|
+
# Determine if a service should run for a particular file based on the service's definition and
|
66
|
+
# the file's service related metadata.
|
67
|
+
# @param service_name [String] name of the service being evaluated
|
68
|
+
# @param md_rec [MetadataRecord] metadata record for the file being evaluated
|
69
|
+
# @return [Boolean] true if the service should be run.
|
70
|
+
def service_needed?(service_name, md_rec)
|
71
|
+
service_rec = md_rec.service(service_name)
|
72
|
+
return true if !service_rec.nil? && service_rec.run_needed
|
73
|
+
|
74
|
+
definition = @definition_manager.services[service_name]
|
75
|
+
|
76
|
+
next_run = ServiceDateHelper.next_run_needed(md_rec, definition)
|
77
|
+
|
78
|
+
return false if next_run.nil?
|
79
|
+
|
80
|
+
# If next run timestamp has passed then service is needed
|
81
|
+
now = ServiceDateHelper.formatted_timestamp
|
82
|
+
now >= next_run
|
83
|
+
end
|
84
|
+
|
85
|
+
# Perform the specified service on the file record, in the context of the specified event
|
86
|
+
# @param service_name [String] name of the service
|
87
|
+
# @param file_rec [FileRecord] file record to perform service upon
|
88
|
+
# @param event [String] name of the event service is being performed within.
|
89
|
+
def perform_service(service_name, file_rec, event)
|
90
|
+
definition = @definition_manager.services[service_name]
|
91
|
+
|
92
|
+
service = @service_class_cache.service_instance(definition)
|
93
|
+
service.perform(file_rec, event)
|
19
94
|
end
|
20
95
|
end
|
21
|
-
end
|
96
|
+
end
|
@@ -1,29 +1,30 @@
|
|
1
|
-
|
2
|
-
|
1
|
+
require 'longleaf/models/app_fields'
|
2
|
+
require 'longleaf/models/service_definition'
|
3
3
|
|
4
|
-
# Manager which loads and provides access to location to service mappings
|
5
4
|
module Longleaf
|
5
|
+
# Manager which loads and provides access to location to service mappings
|
6
6
|
class ServiceMappingManager
|
7
7
|
AF ||= Longleaf::AppFields
|
8
|
-
|
8
|
+
|
9
|
+
# @param config [Hash] has representation of the application configuration
|
9
10
|
def initialize(config)
|
10
11
|
raise ArgumentError.new("Configuration must be provided") if config.nil? || config.empty?
|
11
12
|
|
12
13
|
mappings_config = config[AF::SERVICE_MAPPINGS]
|
13
14
|
raise ArgumentError.new("Service mappings configuration must be provided") if mappings_config.nil?
|
14
|
-
|
15
|
+
|
15
16
|
@loc_to_services = Hash.new
|
16
|
-
|
17
|
+
|
17
18
|
mappings_config.each do |mapping|
|
18
19
|
locations = mapping[AF::LOCATIONS]
|
19
20
|
services = mapping[AF::SERVICES]
|
20
|
-
|
21
|
+
|
21
22
|
locations = [locations] if locations.is_a?(String)
|
22
23
|
services = [services] if services.is_a?(String)
|
23
|
-
|
24
|
+
|
24
25
|
locations.each do |loc_name|
|
25
26
|
@loc_to_services[loc_name] = Array.new unless @loc_to_services.key?(loc_name)
|
26
|
-
|
27
|
+
|
27
28
|
service_set = @loc_to_services[loc_name]
|
28
29
|
if services.is_a?(String)
|
29
30
|
service_set.push(services)
|
@@ -32,11 +33,11 @@ module Longleaf
|
|
32
33
|
end
|
33
34
|
end
|
34
35
|
end
|
35
|
-
|
36
|
+
|
36
37
|
@loc_to_services.each { |loc, services| services.uniq! }
|
37
38
|
@loc_to_services.freeze
|
38
39
|
end
|
39
|
-
|
40
|
+
|
40
41
|
# Gets a list of service names associated with the given location
|
41
42
|
# @param loc_name [String] name of the location to lookup
|
42
43
|
# @return [Array] a list of service names associated with the location
|
@@ -44,4 +45,4 @@ module Longleaf
|
|
44
45
|
@loc_to_services[loc_name] || []
|
45
46
|
end
|
46
47
|
end
|
47
|
-
end
|
48
|
+
end
|
@@ -1,49 +1,59 @@
|
|
1
1
|
require 'pathname'
|
2
|
-
|
3
|
-
|
4
|
-
|
2
|
+
require 'longleaf/models/service_fields'
|
3
|
+
require 'longleaf/models/app_fields'
|
4
|
+
require 'longleaf/errors'
|
5
5
|
require_relative 'configuration_validator'
|
6
6
|
|
7
|
-
# Validates application configuration of service to location mappings
|
8
7
|
module Longleaf
|
8
|
+
# Validates application configuration of service to location mappings
|
9
9
|
class ServiceMappingValidator < ConfigurationValidator
|
10
10
|
AF ||= Longleaf::AppFields
|
11
|
-
|
11
|
+
|
12
|
+
# @param config [Hash] hash containing the application configuration
|
13
|
+
def initialize(config)
|
14
|
+
super(config)
|
15
|
+
end
|
16
|
+
|
17
|
+
protected
|
12
18
|
# Validates service mapping configuration to ensure that it is syntactically and referentially correct.
|
13
19
|
# @param config [Hash] hash containing the application configuration
|
14
|
-
def
|
15
|
-
|
16
|
-
assert("Configuration must
|
17
|
-
|
18
|
-
mappings = config[AF::SERVICE_MAPPINGS]
|
20
|
+
def validate
|
21
|
+
assert("Configuration must be a hash, but a #{@config.class} was provided", @config.class == Hash)
|
22
|
+
assert("Configuration must contain a root '#{AF::SERVICE_MAPPINGS}' key", @config.key?(AF::SERVICE_MAPPINGS))
|
23
|
+
mappings = @config[AF::SERVICE_MAPPINGS]
|
19
24
|
return if mappings.nil? || mappings.empty?
|
20
25
|
assert("'#{AF::SERVICE_MAPPINGS}' must be an array of mappings", mappings.is_a?(Array))
|
21
|
-
|
22
|
-
service_names = config[AF::SERVICES].keys
|
23
|
-
location_names = config[AF::LOCATIONS].keys
|
24
|
-
|
25
|
-
existing_paths = Array.new
|
26
|
+
|
27
|
+
service_names = @config[AF::SERVICES].keys
|
28
|
+
location_names = @config[AF::LOCATIONS].keys
|
29
|
+
|
26
30
|
mappings.each do |mapping|
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
+
register_on_failure do
|
32
|
+
assert("Mapping must be a hash, but received #{mapping.inspect} instead", mapping.is_a?(Hash))
|
33
|
+
|
34
|
+
register_on_failure { validate_mapping_field(AF::LOCATIONS, mapping, location_names) }
|
35
|
+
register_on_failure { validate_mapping_field(AF::SERVICES, mapping, service_names) }
|
36
|
+
end
|
31
37
|
end
|
38
|
+
|
39
|
+
@result
|
32
40
|
end
|
33
|
-
|
41
|
+
|
34
42
|
private
|
35
|
-
def
|
43
|
+
def validate_mapping_field(field, mapping, valid_values)
|
36
44
|
assert("Mapping must contain a '#{field}' field", mapping.key?(field))
|
37
45
|
field_values = mapping[field]
|
38
|
-
assert("Mapping '#{field}' field must be either a string or an array, but received '#{field_values.
|
46
|
+
assert("Mapping '#{field}' field must be either a string or an array, but received '#{field_values.class}' instead",
|
39
47
|
field_values.is_a?(Array) || field_values.is_a?(String))
|
40
48
|
assert("Mapping must specify one or more value in the '#{field}' field", !field_values.empty?)
|
41
|
-
|
49
|
+
|
42
50
|
check_values = field_values.is_a?(String) ? [field_values] : field_values
|
43
51
|
check_values.each do |value|
|
44
|
-
|
45
|
-
|
52
|
+
register_on_failure do
|
53
|
+
assert("Mapping specifies value '#{value}', but no #{field} with that name exist",
|
54
|
+
valid_values.include?(value))
|
55
|
+
end
|
46
56
|
end
|
47
57
|
end
|
48
58
|
end
|
49
|
-
end
|
59
|
+
end
|
@@ -1,37 +1,98 @@
|
|
1
|
-
|
2
|
-
|
1
|
+
require 'longleaf/models/app_fields'
|
2
|
+
require 'longleaf/models/storage_types'
|
3
|
+
require 'longleaf/models/filesystem_storage_location'
|
4
|
+
require 'longleaf/models/s3_storage_location'
|
5
|
+
require 'longleaf/models/filesystem_metadata_location'
|
6
|
+
require 'longleaf/errors'
|
3
7
|
|
4
|
-
# Manager which loads and provides access to Longleaf::StorageLocation objects
|
5
8
|
module Longleaf
|
9
|
+
# Manager which loads and provides access to {StorageLocation} objects
|
6
10
|
class StorageLocationManager
|
7
11
|
AF ||= Longleaf::AppFields
|
8
|
-
|
12
|
+
ST ||= Longleaf::StorageTypes
|
13
|
+
|
14
|
+
# Hash mapping storage location names to {StorageLocation} objects
|
9
15
|
attr_reader :locations
|
10
|
-
|
16
|
+
# Mapping of storage types to storage location classes
|
17
|
+
@@storage_type_mappings = {
|
18
|
+
ST::FILESYSTEM_STORAGE_TYPE => Longleaf::FilesystemStorageLocation,
|
19
|
+
ST::S3_STORAGE_TYPE => Longleaf::S3StorageLocation
|
20
|
+
}
|
21
|
+
@@metadata_type_mappings = { ST::FILESYSTEM_STORAGE_TYPE => Longleaf::FilesystemMetadataLocation }
|
22
|
+
|
23
|
+
# @param config [Hash] has representation of the application configuration
|
11
24
|
def initialize(config)
|
12
25
|
raise ArgumentError.new("Configuration must be provided") if config&.empty?
|
13
26
|
|
14
27
|
@locations = Hash.new
|
15
28
|
config[AF::LOCATIONS].each do |name, properties|
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
@locations[name] = location
|
29
|
+
md_loc = instantiate_metadata_location(properties)
|
30
|
+
|
31
|
+
@locations[name] = instantiate_storage_location(name, properties, md_loc)
|
21
32
|
end
|
22
33
|
@locations.freeze
|
23
34
|
end
|
24
|
-
|
25
|
-
# Get the StorageLocation object which should contain the given path
|
35
|
+
|
36
|
+
# Get the {StorageLocation} object which should contain the given path
|
26
37
|
# @return [Longleaf::StorageLocation] location containing the given path
|
27
38
|
# or nil if the path is not contained by a registered location.
|
28
39
|
def get_location_by_path(path)
|
29
40
|
raise ArgumentError.new("Path parameter is required") if path.nil? || path.empty?
|
30
41
|
@locations.each do |name, location|
|
31
|
-
return location if
|
42
|
+
return location if location.contains?(path)
|
43
|
+
end
|
44
|
+
|
45
|
+
nil
|
46
|
+
end
|
47
|
+
|
48
|
+
# Get the {StorageLocation} object which should contain the given metadata path
|
49
|
+
# @return [Longleaf::StorageLocation] location containing the given metadata path
|
50
|
+
# or nil if the path is not contained by a registered location.
|
51
|
+
def get_location_by_metadata_path(md_path)
|
52
|
+
raise ArgumentError.new("Metadata path parameter is required") if md_path.nil? || md_path.empty?
|
53
|
+
@locations.each do |name, location|
|
54
|
+
return location if location.metadata_location.contains?(md_path)
|
32
55
|
end
|
33
|
-
|
56
|
+
|
34
57
|
nil
|
35
58
|
end
|
59
|
+
|
60
|
+
# Raises a {StorageLocationUnavailableError} if the given path is not in a known storage location,
|
61
|
+
# or if it is not within the expected location if provided
|
62
|
+
# @param path [String] file path
|
63
|
+
# @param expected_loc [String] name of the storage location which path should be contained by
|
64
|
+
# @raise [StorageLocationUnavailableError] if the path is not in a known/expected storage location
|
65
|
+
# @return [StorageLocation] the storage location which contains path, if it was within one.
|
66
|
+
def verify_path_in_location(path, expected_loc = nil)
|
67
|
+
location = get_location_by_path(path)
|
68
|
+
if location.nil?
|
69
|
+
raise StorageLocationUnavailableError.new("Path #{path} is not from a known storage location.")
|
70
|
+
elsif !expected_loc.nil? && expected_loc != location.name
|
71
|
+
raise StorageLocationUnavailableError.new("Path #{path} is not contained by storage location #{expected_loc}.")
|
72
|
+
end
|
73
|
+
location
|
74
|
+
end
|
75
|
+
|
76
|
+
private
|
77
|
+
def instantiate_metadata_location(loc_properties)
|
78
|
+
m_config = loc_properties[AF::METADATA_CONFIG]
|
79
|
+
m_type = m_config[AF::STORAGE_TYPE]
|
80
|
+
m_type = ST::FILESYSTEM_STORAGE_TYPE if m_type.nil?
|
81
|
+
|
82
|
+
m_class = @@metadata_type_mappings[m_type]
|
83
|
+
raise ArgumentError.new("Unknown metadata location type #{m_type}") if m_class.nil?
|
84
|
+
|
85
|
+
m_class.new(m_config)
|
86
|
+
end
|
87
|
+
|
88
|
+
def instantiate_storage_location(name, properties, md_loc)
|
89
|
+
s_type = properties[AF::STORAGE_TYPE]
|
90
|
+
s_type = ST::FILESYSTEM_STORAGE_TYPE if s_type.nil?
|
91
|
+
|
92
|
+
s_class = @@storage_type_mappings[s_type]
|
93
|
+
raise ArgumentError.new("Unknown storage location type #{s_type}") if s_class.nil?
|
94
|
+
|
95
|
+
s_class.new(name, properties, md_loc)
|
96
|
+
end
|
36
97
|
end
|
37
|
-
end
|
98
|
+
end
|