longleaf 0.1.0.pre.3 → 1.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.circleci/config.yml +94 -0
- data/.editorconfig +13 -0
- data/.gitignore +4 -1
- data/.rubocop.yml +44 -0
- data/.rubocop_todo.yml +834 -0
- data/.yardopts +1 -0
- data/Gemfile +16 -1
- data/README.md +98 -12
- data/Rakefile +6 -0
- data/bin/setup +16 -1
- data/docs/aboutlongleaf.md +28 -0
- data/docs/extra.css +32 -0
- data/docs/img/change-file.png +0 -0
- data/docs/img/ll-example-preserved.png +0 -0
- data/docs/index.md +19 -0
- data/docs/install.md +66 -0
- data/docs/ll-example/config-example-relative.yml +33 -0
- data/docs/ll-example/files-dir/LLexample-PDF.pdf +0 -0
- data/docs/ll-example/files-dir/LLexample-TOCHANGE.txt +15 -0
- data/docs/ll-example/files-dir/LLexample-tokeep.txt +10 -0
- data/docs/ll-example/metadata-dir/.gitkeep +0 -0
- data/docs/ll-example/replica-files/.gitkeep +0 -0
- data/docs/ll-example/replica-metadata/.gitkeep +0 -0
- data/docs/quickstart.md +270 -0
- data/docs/rdocs/Longleaf.html +135 -0
- data/docs/rdocs/Longleaf/AppFields.html +178 -0
- data/docs/rdocs/Longleaf/ApplicationConfigDeserializer.html +631 -0
- data/docs/rdocs/Longleaf/ApplicationConfigManager.html +610 -0
- data/docs/rdocs/Longleaf/ApplicationConfigValidator.html +238 -0
- data/docs/rdocs/Longleaf/CLI.html +909 -0
- data/docs/rdocs/Longleaf/ChecksumMismatchError.html +151 -0
- data/docs/rdocs/Longleaf/ConfigBuilder.html +1339 -0
- data/docs/rdocs/Longleaf/ConfigurationError.html +143 -0
- data/docs/rdocs/Longleaf/ConfigurationValidator.html +227 -0
- data/docs/rdocs/Longleaf/DeregisterCommand.html +420 -0
- data/docs/rdocs/Longleaf/DeregisterEvent.html +453 -0
- data/docs/rdocs/Longleaf/DeregistrationError.html +151 -0
- data/docs/rdocs/Longleaf/DigestHelper.html +419 -0
- data/docs/rdocs/Longleaf/EventError.html +147 -0
- data/docs/rdocs/Longleaf/EventNames.html +163 -0
- data/docs/rdocs/Longleaf/EventStatusTracking.html +656 -0
- data/docs/rdocs/Longleaf/FileCheckService.html +540 -0
- data/docs/rdocs/Longleaf/FileHelpers.html +520 -0
- data/docs/rdocs/Longleaf/FileRecord.html +716 -0
- data/docs/rdocs/Longleaf/FileSelector.html +901 -0
- data/docs/rdocs/Longleaf/FixityCheckService.html +691 -0
- data/docs/rdocs/Longleaf/IndexManager.html +1155 -0
- data/docs/rdocs/Longleaf/InvalidDigestAlgorithmError.html +143 -0
- data/docs/rdocs/Longleaf/InvalidStoragePathError.html +143 -0
- data/docs/rdocs/Longleaf/Logging.html +405 -0
- data/docs/rdocs/Longleaf/Logging/RedirectingLogger.html +1213 -0
- data/docs/rdocs/Longleaf/LongleafError.html +139 -0
- data/docs/rdocs/Longleaf/MDFields.html +193 -0
- data/docs/rdocs/Longleaf/MetadataBuilder.html +787 -0
- data/docs/rdocs/Longleaf/MetadataDeserializer.html +537 -0
- data/docs/rdocs/Longleaf/MetadataError.html +143 -0
- data/docs/rdocs/Longleaf/MetadataPersistenceManager.html +539 -0
- data/docs/rdocs/Longleaf/MetadataRecord.html +1411 -0
- data/docs/rdocs/Longleaf/MetadataSerializer.html +786 -0
- data/docs/rdocs/Longleaf/PreservationServiceError.html +147 -0
- data/docs/rdocs/Longleaf/PreserveCommand.html +410 -0
- data/docs/rdocs/Longleaf/PreserveEvent.html +491 -0
- data/docs/rdocs/Longleaf/RegisterCommand.html +428 -0
- data/docs/rdocs/Longleaf/RegisterEvent.html +628 -0
- data/docs/rdocs/Longleaf/RegisteredFileSelector.html +446 -0
- data/docs/rdocs/Longleaf/RegistrationError.html +151 -0
- data/docs/rdocs/Longleaf/ReindexCommand.html +576 -0
- data/docs/rdocs/Longleaf/RsyncReplicationService.html +1180 -0
- data/docs/rdocs/Longleaf/SequelIndexDriver.html +1978 -0
- data/docs/rdocs/Longleaf/ServiceCandidateFilesystemIterator.html +572 -0
- data/docs/rdocs/Longleaf/ServiceCandidateIndexIterator.html +532 -0
- data/docs/rdocs/Longleaf/ServiceCandidateLocator.html +333 -0
- data/docs/rdocs/Longleaf/ServiceClassCache.html +725 -0
- data/docs/rdocs/Longleaf/ServiceDateHelper.html +425 -0
- data/docs/rdocs/Longleaf/ServiceDefinition.html +683 -0
- data/docs/rdocs/Longleaf/ServiceDefinitionManager.html +371 -0
- data/docs/rdocs/Longleaf/ServiceDefinitionValidator.html +269 -0
- data/docs/rdocs/Longleaf/ServiceFields.html +173 -0
- data/docs/rdocs/Longleaf/ServiceManager.html +1229 -0
- data/docs/rdocs/Longleaf/ServiceMappingManager.html +410 -0
- data/docs/rdocs/Longleaf/ServiceMappingValidator.html +347 -0
- data/docs/rdocs/Longleaf/ServiceRecord.html +821 -0
- data/docs/rdocs/Longleaf/StorageLocation.html +985 -0
- data/docs/rdocs/Longleaf/StorageLocationManager.html +729 -0
- data/docs/rdocs/Longleaf/StorageLocationUnavailableError.html +143 -0
- data/docs/rdocs/Longleaf/StorageLocationValidator.html +373 -0
- data/docs/rdocs/Longleaf/StoragePathValidator.html +253 -0
- data/docs/rdocs/Longleaf/SystemConfigBuilder.html +441 -0
- data/docs/rdocs/Longleaf/SystemConfigFields.html +163 -0
- data/docs/rdocs/Longleaf/ValidateConfigCommand.html +451 -0
- data/docs/rdocs/Longleaf/ValidateMetadataCommand.html +408 -0
- data/docs/rdocs/_index.html +660 -0
- data/docs/rdocs/class_list.html +51 -0
- data/docs/rdocs/css/common.css +1 -0
- data/docs/rdocs/css/full_list.css +58 -0
- data/docs/rdocs/css/style.css +496 -0
- data/docs/rdocs/file.README.html +165 -0
- data/docs/rdocs/file_list.html +56 -0
- data/docs/rdocs/frames.html +17 -0
- data/docs/rdocs/index.html +165 -0
- data/docs/rdocs/js/app.js +303 -0
- data/docs/rdocs/js/full_list.js +216 -0
- data/docs/rdocs/js/jquery.js +4 -0
- data/docs/rdocs/method_list.html +2051 -0
- data/docs/rdocs/top-level-namespace.html +110 -0
- data/lib/longleaf/candidates/file_selector.rb +150 -0
- data/lib/longleaf/candidates/manifest_digest_provider.rb +17 -0
- data/lib/longleaf/candidates/physical_path_provider.rb +17 -0
- data/lib/longleaf/candidates/registered_file_selector.rb +67 -0
- data/lib/longleaf/candidates/service_candidate_filesystem_iterator.rb +93 -0
- data/lib/longleaf/candidates/service_candidate_index_iterator.rb +84 -0
- data/lib/longleaf/candidates/service_candidate_locator.rb +23 -0
- data/lib/longleaf/candidates/single_digest_provider.rb +13 -0
- data/lib/longleaf/cli.rb +249 -44
- data/lib/longleaf/commands/deregister_command.rb +51 -0
- data/lib/longleaf/commands/preserve_command.rb +50 -0
- data/lib/longleaf/commands/register_command.rb +34 -43
- data/lib/longleaf/commands/reindex_command.rb +92 -0
- data/lib/longleaf/commands/validate_config_command.rb +33 -8
- data/lib/longleaf/commands/validate_metadata_command.rb +51 -0
- data/lib/longleaf/errors.rb +26 -7
- data/lib/longleaf/events/deregister_event.rb +53 -0
- data/lib/longleaf/events/event_names.rb +9 -0
- data/lib/longleaf/events/event_status_tracking.rb +59 -0
- data/lib/longleaf/events/preserve_event.rb +82 -0
- data/lib/longleaf/events/register_event.rb +59 -51
- data/lib/longleaf/helpers/case_insensitive_hash.rb +38 -0
- data/lib/longleaf/helpers/digest_helper.rb +56 -0
- data/lib/longleaf/helpers/s3_uri_helper.rb +86 -0
- data/lib/longleaf/helpers/selection_options_parser.rb +215 -0
- data/lib/longleaf/helpers/service_date_helper.rb +78 -0
- data/lib/longleaf/indexing/index_manager.rb +101 -0
- data/lib/longleaf/indexing/sequel_index_driver.rb +306 -0
- data/lib/longleaf/logging.rb +5 -4
- data/lib/longleaf/logging/redirecting_logger.rb +30 -25
- data/lib/longleaf/models/app_fields.rb +7 -2
- data/lib/longleaf/models/file_record.rb +31 -8
- data/lib/longleaf/models/filesystem_metadata_location.rb +56 -0
- data/lib/longleaf/models/filesystem_storage_location.rb +52 -0
- data/lib/longleaf/models/md_fields.rb +3 -1
- data/lib/longleaf/models/metadata_location.rb +47 -0
- data/lib/longleaf/models/metadata_record.rb +43 -16
- data/lib/longleaf/models/s3_storage_location.rb +138 -0
- data/lib/longleaf/models/service_definition.rb +7 -6
- data/lib/longleaf/models/service_fields.rb +7 -1
- data/lib/longleaf/models/service_record.rb +10 -6
- data/lib/longleaf/models/storage_location.rb +24 -21
- data/lib/longleaf/models/storage_types.rb +9 -0
- data/lib/longleaf/models/system_config_fields.rb +9 -0
- data/lib/longleaf/preservation_services/file_check_service.rb +59 -0
- data/lib/longleaf/preservation_services/fixity_check_service.rb +124 -0
- data/lib/longleaf/preservation_services/rsync_replication_service.rb +198 -0
- data/lib/longleaf/preservation_services/s3_replication_service.rb +131 -0
- data/lib/longleaf/services/application_config_deserializer.rb +80 -21
- data/lib/longleaf/services/application_config_manager.rb +20 -6
- data/lib/longleaf/services/application_config_validator.rb +19 -9
- data/lib/longleaf/services/configuration_validator.rb +67 -4
- data/lib/longleaf/services/filesystem_location_validator.rb +16 -0
- data/lib/longleaf/services/metadata_deserializer.rb +115 -42
- data/lib/longleaf/services/metadata_persistence_manager.rb +47 -0
- data/lib/longleaf/services/metadata_serializer.rb +139 -25
- data/lib/longleaf/services/metadata_validator.rb +76 -0
- data/lib/longleaf/services/s3_location_validator.rb +19 -0
- data/lib/longleaf/services/service_class_cache.rb +112 -0
- data/lib/longleaf/services/service_definition_manager.rb +10 -7
- data/lib/longleaf/services/service_definition_validator.rb +25 -18
- data/lib/longleaf/services/service_manager.rb +86 -11
- data/lib/longleaf/services/service_mapping_manager.rb +13 -12
- data/lib/longleaf/services/service_mapping_validator.rb +36 -26
- data/lib/longleaf/services/storage_location_manager.rb +76 -15
- data/lib/longleaf/services/storage_location_validator.rb +49 -35
- data/lib/longleaf/specs/config_builder.rb +47 -23
- data/lib/longleaf/specs/config_validator_helpers.rb +16 -0
- data/lib/longleaf/specs/custom_matchers.rb +9 -0
- data/lib/longleaf/specs/file_helpers.rb +61 -0
- data/lib/longleaf/specs/metadata_builder.rb +98 -0
- data/lib/longleaf/specs/system_config_builder.rb +27 -0
- data/lib/longleaf/version.rb +1 -1
- data/longleaf.gemspec +20 -7
- data/mkdocs.yml +21 -0
- metadata +310 -26
- data/.travis.yml +0 -4
- data/lib/longleaf/commands/abstract_command.rb +0 -37
- data/lib/longleaf/services/storage_path_validator.rb +0 -16
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html>
|
|
3
|
+
<head>
|
|
4
|
+
<meta charset="utf-8">
|
|
5
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
6
|
+
<title>
|
|
7
|
+
Top Level Namespace
|
|
8
|
+
|
|
9
|
+
— Documentation by YARD 0.9.19
|
|
10
|
+
|
|
11
|
+
</title>
|
|
12
|
+
|
|
13
|
+
<link rel="stylesheet" href="css/style.css" type="text/css" charset="utf-8" />
|
|
14
|
+
|
|
15
|
+
<link rel="stylesheet" href="css/common.css" type="text/css" charset="utf-8" />
|
|
16
|
+
|
|
17
|
+
<script type="text/javascript" charset="utf-8">
|
|
18
|
+
pathId = "";
|
|
19
|
+
relpath = '';
|
|
20
|
+
</script>
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
<script type="text/javascript" charset="utf-8" src="js/jquery.js"></script>
|
|
24
|
+
|
|
25
|
+
<script type="text/javascript" charset="utf-8" src="js/app.js"></script>
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
</head>
|
|
29
|
+
<body>
|
|
30
|
+
<div class="nav_wrap">
|
|
31
|
+
<iframe id="nav" src="class_list.html?1"></iframe>
|
|
32
|
+
<div id="resizer"></div>
|
|
33
|
+
</div>
|
|
34
|
+
|
|
35
|
+
<div id="main" tabindex="-1">
|
|
36
|
+
<div id="header">
|
|
37
|
+
<div id="menu">
|
|
38
|
+
|
|
39
|
+
<a href="_index.html">Index</a> »
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
<span class="title">Top Level Namespace</span>
|
|
43
|
+
|
|
44
|
+
</div>
|
|
45
|
+
|
|
46
|
+
<div id="search">
|
|
47
|
+
|
|
48
|
+
<a class="full_list_link" id="class_list_link"
|
|
49
|
+
href="class_list.html">
|
|
50
|
+
|
|
51
|
+
<svg width="24" height="24">
|
|
52
|
+
<rect x="0" y="4" width="24" height="4" rx="1" ry="1"></rect>
|
|
53
|
+
<rect x="0" y="12" width="24" height="4" rx="1" ry="1"></rect>
|
|
54
|
+
<rect x="0" y="20" width="24" height="4" rx="1" ry="1"></rect>
|
|
55
|
+
</svg>
|
|
56
|
+
</a>
|
|
57
|
+
|
|
58
|
+
</div>
|
|
59
|
+
<div class="clear"></div>
|
|
60
|
+
</div>
|
|
61
|
+
|
|
62
|
+
<div id="content"><h1>Top Level Namespace
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
</h1>
|
|
67
|
+
<div class="box_info">
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
</div>
|
|
80
|
+
|
|
81
|
+
<h2>Defined Under Namespace</h2>
|
|
82
|
+
<p class="children">
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
<strong class="modules">Modules:</strong> <span class='object_link'><a href="Longleaf.html" title="Longleaf (module)">Longleaf</a></span>
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
</p>
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
</div>
|
|
101
|
+
|
|
102
|
+
<div id="footer">
|
|
103
|
+
Generated on Tue May 28 15:48:00 2019 by
|
|
104
|
+
<a href="http://yardoc.org" title="Yay! A Ruby Documentation Tool" target="_parent">yard</a>
|
|
105
|
+
0.9.19 (ruby-2.6.3).
|
|
106
|
+
</div>
|
|
107
|
+
|
|
108
|
+
</div>
|
|
109
|
+
</body>
|
|
110
|
+
</html>
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
require 'longleaf/logging'
|
|
2
|
+
require 'longleaf/candidates/physical_path_provider'
|
|
3
|
+
|
|
4
|
+
module Longleaf
|
|
5
|
+
# Selects and allows for iteration over files which match a provided set of selection criteria
|
|
6
|
+
class FileSelector
|
|
7
|
+
include Longleaf::Logging
|
|
8
|
+
SPECIFICITY_PATH = 'path'
|
|
9
|
+
SPECIFICITY_STORAGE_LOCATION = 'storage_location'
|
|
10
|
+
|
|
11
|
+
attr_reader :specificity
|
|
12
|
+
|
|
13
|
+
# May only provide either file_paths or storage_locations
|
|
14
|
+
def initialize(file_paths: nil, storage_locations: nil, physical_provider: Longleaf::PhysicalPathProvider.new,
|
|
15
|
+
app_config:)
|
|
16
|
+
if nil_or_empty?(file_paths) && nil_or_empty?(storage_locations)
|
|
17
|
+
raise ArgumentError.new("Must provide either file paths or storage locations")
|
|
18
|
+
end
|
|
19
|
+
if !nil_or_empty?(file_paths) && !nil_or_empty?(storage_locations)
|
|
20
|
+
raise ArgumentError.new("Cannot provide both file paths and storage locations")
|
|
21
|
+
end
|
|
22
|
+
@app_config = app_config
|
|
23
|
+
# The top level paths targeted by this selector
|
|
24
|
+
@target_paths = file_paths&.map do |path|
|
|
25
|
+
# Resolve relative paths against pwd
|
|
26
|
+
pathname = Pathname.new(path)
|
|
27
|
+
if !pathname.absolute?
|
|
28
|
+
path = File.join(Dir.pwd, path)
|
|
29
|
+
end
|
|
30
|
+
path = File.expand_path(path)
|
|
31
|
+
|
|
32
|
+
# adding trailing /'s to directories
|
|
33
|
+
if Dir.exist?(path) && !path.end_with?('/')
|
|
34
|
+
path + '/'
|
|
35
|
+
else
|
|
36
|
+
path
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
# The set of storage locations to select file paths from
|
|
40
|
+
@storage_locations = storage_locations
|
|
41
|
+
@physical_provider = physical_provider
|
|
42
|
+
# Validate that the selected storage locations are known
|
|
43
|
+
if @storage_locations.nil?
|
|
44
|
+
@specificity = SPECIFICITY_PATH
|
|
45
|
+
else
|
|
46
|
+
@specificity = SPECIFICITY_STORAGE_LOCATION
|
|
47
|
+
locations = @app_config.location_manager.locations
|
|
48
|
+
@storage_locations.each do |loc_name|
|
|
49
|
+
unless locations.key?(loc_name)
|
|
50
|
+
raise StorageLocationUnavailableError.new("Cannot select unknown storage location #{loc_name}.")
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# @return [Array] a list of top level paths from which files will be selected
|
|
57
|
+
def target_paths
|
|
58
|
+
# If starting from locations, initialize by expanding locations out to their actual paths
|
|
59
|
+
if @target_paths.nil? && !@storage_locations.nil?
|
|
60
|
+
@target_paths = Array.new
|
|
61
|
+
@storage_locations.each do |loc_name|
|
|
62
|
+
@target_paths << @app_config.location_manager.locations[loc_name].path
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
@target_paths
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# Get the next logical file path for this selector.
|
|
70
|
+
# @return [String] an absolute path to the next file targeted by this selector,
|
|
71
|
+
# or nil if no more files selected
|
|
72
|
+
def next_path
|
|
73
|
+
if @paths.nil?
|
|
74
|
+
# Start the paths listing out from the targetted set of paths for this selector
|
|
75
|
+
# In reverse order since using a LIFO structure
|
|
76
|
+
@paths = target_paths.reverse
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# No more paths to return
|
|
80
|
+
return nil if @paths&.empty?
|
|
81
|
+
|
|
82
|
+
# Get the most recently added path for depth first traversal of selected paths
|
|
83
|
+
path = @paths.pop
|
|
84
|
+
until path.nil? do
|
|
85
|
+
@app_config.location_manager.verify_path_in_location(path)
|
|
86
|
+
physical_path = @physical_provider.get_physical_path(path)
|
|
87
|
+
separate_logical = physical_path != path
|
|
88
|
+
if separate_logical
|
|
89
|
+
@app_config.location_manager.verify_path_in_location(physical_path)
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
if File.exist?(physical_path)
|
|
93
|
+
if File.directory?(physical_path)
|
|
94
|
+
if separate_logical
|
|
95
|
+
raise InvalidStoragePathError.new("Cannot specify physical path to a directory: #{physical_path}")
|
|
96
|
+
end
|
|
97
|
+
logger.debug("Expanding directory #{path}")
|
|
98
|
+
# For a directory, add all children to file_paths
|
|
99
|
+
Dir.entries(path).sort.reverse_each do |child|
|
|
100
|
+
@paths << File.join(path, child) unless child == '.' or child == '..'
|
|
101
|
+
end
|
|
102
|
+
else
|
|
103
|
+
logger.debug("Returning file #{path}")
|
|
104
|
+
return path
|
|
105
|
+
end
|
|
106
|
+
else
|
|
107
|
+
raise InvalidStoragePathError.new("File #{physical_path} does not exist.")
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# Returned path was not a suitable file, try the next path
|
|
111
|
+
path = @paths.pop
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# Iterate through the file paths for this selector and execute the provided block with each.
|
|
116
|
+
# A block is required.
|
|
117
|
+
def each
|
|
118
|
+
file_path = next_path
|
|
119
|
+
until file_path.nil?
|
|
120
|
+
yield file_path
|
|
121
|
+
|
|
122
|
+
file_path = next_path
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
# return [Array] a list of all storage locations being targeted by this selector
|
|
127
|
+
def storage_locations
|
|
128
|
+
# Determine what storage_locations are represented by the given file paths
|
|
129
|
+
if @storage_locations.nil? && !@target_paths.nil?
|
|
130
|
+
loc_set = Set.new
|
|
131
|
+
@target_paths.each do |path|
|
|
132
|
+
loc = @app_config.location_manager.get_location_by_path(path)
|
|
133
|
+
loc_set.add(loc.name) unless loc.nil?
|
|
134
|
+
end
|
|
135
|
+
@storage_locations = loc_set.to_a
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
if @storage_locations.nil?
|
|
139
|
+
@storage_locations = Array.new
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
@storage_locations
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
private
|
|
146
|
+
def nil_or_empty?(value)
|
|
147
|
+
value.nil? || value.empty?
|
|
148
|
+
end
|
|
149
|
+
end
|
|
150
|
+
end
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
module Longleaf
|
|
2
|
+
# Provides digests for files from a manifest
|
|
3
|
+
class ManifestDigestProvider
|
|
4
|
+
# @param digests_mapping hash which maps file paths to hashs of digests
|
|
5
|
+
def initialize(digests_mapping)
|
|
6
|
+
@digests_mapping = digests_mapping
|
|
7
|
+
end
|
|
8
|
+
|
|
9
|
+
# @param file_path [String] path of file
|
|
10
|
+
# @return hash containing all the manifested digests for the given path, or nil
|
|
11
|
+
def get_digests(file_path)
|
|
12
|
+
# return nil if key not found, in case the hash has default values
|
|
13
|
+
return nil unless @digests_mapping.key?(file_path)
|
|
14
|
+
@digests_mapping[file_path]
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
module Longleaf
|
|
2
|
+
# Provides physical paths for logical paths from a mapping
|
|
3
|
+
class PhysicalPathProvider
|
|
4
|
+
# @param phys_mapping hash with logical paths as keys, physical paths as values
|
|
5
|
+
def initialize(phys_mapping = Hash.new)
|
|
6
|
+
@phys_mapping = phys_mapping
|
|
7
|
+
end
|
|
8
|
+
|
|
9
|
+
# @param logical_path [String] logical path of file
|
|
10
|
+
# @return physical path of the file
|
|
11
|
+
def get_physical_path(logical_path)
|
|
12
|
+
# return the logical path itself if no physical path is mapped
|
|
13
|
+
return logical_path unless @phys_mapping.key?(logical_path)
|
|
14
|
+
@phys_mapping[logical_path]
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
require 'longleaf/candidates/file_selector'
|
|
2
|
+
require 'longleaf/logging'
|
|
3
|
+
|
|
4
|
+
module Longleaf
|
|
5
|
+
# Selects and allows for iteration over files which are registered and match a provided
|
|
6
|
+
# set of selection criteria
|
|
7
|
+
class RegisteredFileSelector < FileSelector
|
|
8
|
+
include Longleaf::Logging
|
|
9
|
+
|
|
10
|
+
# Get the next file path for this selector.
|
|
11
|
+
# @raise [InvalidStoragePathError] if any of the selected files do not exist
|
|
12
|
+
# @raise [StorageLocationUnavailableError] if any of the selected paths are not
|
|
13
|
+
# in a registered storage location.
|
|
14
|
+
# @return [String] an absolute path to the next file targeted by this selector,
|
|
15
|
+
# or nil if no more files selected
|
|
16
|
+
def next_path
|
|
17
|
+
if @md_paths.nil?
|
|
18
|
+
# Compute the starting paths by looking up the metadata paths for the provided targets,
|
|
19
|
+
# in reverse order since @md_paths is a LIFO stack structure.
|
|
20
|
+
@md_paths = target_paths.reverse_each.map do |file_path|
|
|
21
|
+
storage_loc = @app_config.location_manager.verify_path_in_location(file_path)
|
|
22
|
+
storage_loc.get_metadata_path_for(file_path)
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# No more paths to return
|
|
27
|
+
return nil if @md_paths&.empty?
|
|
28
|
+
|
|
29
|
+
# Get the most recently added path for depth first traversal of selected paths
|
|
30
|
+
md_path = @md_paths.pop
|
|
31
|
+
until md_path.nil? do
|
|
32
|
+
if File.exist?(md_path)
|
|
33
|
+
if File.directory?(md_path)
|
|
34
|
+
logger.debug("Expanding metadata directory #{md_path}")
|
|
35
|
+
# For a directory, add all children to file_paths
|
|
36
|
+
Dir.entries(md_path).sort.reverse_each do |child|
|
|
37
|
+
@md_paths << File.join(md_path, child) unless child == '.' or child == '..'
|
|
38
|
+
end
|
|
39
|
+
elsif md_path.end_with?(MetadataSerializer::metadata_suffix)
|
|
40
|
+
# Convert metadata path to file path before returning
|
|
41
|
+
return file_path_for_metadata(md_path)
|
|
42
|
+
else
|
|
43
|
+
logger.debug("Skipping non-metadata file in metadata directory #{md_path}")
|
|
44
|
+
end
|
|
45
|
+
else
|
|
46
|
+
file_path = file_path_for_metadata(md_path)
|
|
47
|
+
if File.exist?(file_path)
|
|
48
|
+
raise RegistrationError.new("File #{file_path} is not registered.")
|
|
49
|
+
else
|
|
50
|
+
raise InvalidStoragePathError.new("File #{file_path} does not exist.")
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# Returned path was not a suitable file, try the next path
|
|
55
|
+
md_path = @md_paths.pop
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
private
|
|
60
|
+
def file_path_for_metadata(md_path)
|
|
61
|
+
storage_loc = @app_config.location_manager.get_location_by_metadata_path(md_path)
|
|
62
|
+
file_path = storage_loc.get_path_from_metadata_path(md_path)
|
|
63
|
+
logger.debug("Returning next file #{file_path} for metadata path #{md_path}")
|
|
64
|
+
return file_path
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
end
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
require 'longleaf/services/service_manager'
|
|
2
|
+
require 'longleaf/events/event_names'
|
|
3
|
+
require 'longleaf/errors'
|
|
4
|
+
require 'longleaf/logging'
|
|
5
|
+
require 'time'
|
|
6
|
+
|
|
7
|
+
module Longleaf
|
|
8
|
+
# Iterator for getting file candidates which have services which need to be run.
|
|
9
|
+
# Implementation uses metadata files directly from the filesystem for determinations
|
|
10
|
+
# about service status.
|
|
11
|
+
class ServiceCandidateFilesystemIterator
|
|
12
|
+
include Longleaf::Logging
|
|
13
|
+
|
|
14
|
+
def initialize(file_selector, event, app_config, force = false)
|
|
15
|
+
@file_selector = file_selector
|
|
16
|
+
@event = event
|
|
17
|
+
@app_config = app_config
|
|
18
|
+
@force = force
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# Get the file record for the next candidate which needs services run which match the
|
|
22
|
+
# provided file_selector
|
|
23
|
+
# @return [FileRecord] file record of the next candidate with services needing to be run,
|
|
24
|
+
# or nil if there are no more candidates.
|
|
25
|
+
def next_candidate
|
|
26
|
+
loop do
|
|
27
|
+
next_path = @file_selector.next_path
|
|
28
|
+
return nil if next_path.nil?
|
|
29
|
+
|
|
30
|
+
logger.debug("Evaluating candidate #{next_path}")
|
|
31
|
+
storage_loc = @app_config.location_manager.get_location_by_path(next_path)
|
|
32
|
+
file_rec = FileRecord.new(next_path, storage_loc)
|
|
33
|
+
|
|
34
|
+
# Skip over unregistered files
|
|
35
|
+
if !file_rec.metadata_present?
|
|
36
|
+
logger.debug("Ignoring unregistered file #{next_path}")
|
|
37
|
+
next
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
@app_config.md_manager.load(file_rec)
|
|
41
|
+
|
|
42
|
+
# Return the file record if it needs any services run
|
|
43
|
+
return file_rec if needs_run?(file_rec)
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# Iterate through the candidates in this object and execute the provided block with each
|
|
48
|
+
# candidate. A block is required.
|
|
49
|
+
def each
|
|
50
|
+
file_rec = next_candidate
|
|
51
|
+
until file_rec.nil?
|
|
52
|
+
yield file_rec
|
|
53
|
+
|
|
54
|
+
file_rec = next_candidate
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
private
|
|
59
|
+
# Returns true if the file record contains any services which need to be run
|
|
60
|
+
def needs_run?(file_rec)
|
|
61
|
+
md_rec = file_rec.metadata_record
|
|
62
|
+
storage_loc = file_rec.storage_location
|
|
63
|
+
service_manager = @app_config.service_manager
|
|
64
|
+
|
|
65
|
+
# File is not a valid candidate for services if it is deregistered, unless performing cleanup
|
|
66
|
+
if @event != EventNames::CLEANUP && md_rec.deregistered?
|
|
67
|
+
logger.debug("Skipping deregistered file: #{file_rec.path}")
|
|
68
|
+
return false
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
expected_services = service_manager.list_services(
|
|
72
|
+
location: storage_loc.name,
|
|
73
|
+
event: @event)
|
|
74
|
+
|
|
75
|
+
# When in force mode, candidate needs a run as long as there are any services configured for it.
|
|
76
|
+
if @force && !expected_services.empty?
|
|
77
|
+
logger.debug("Forcing run needed for file: #{file_rec.path}")
|
|
78
|
+
return true
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
expected_services.each do |service_name|
|
|
82
|
+
if service_manager.service_needed?(service_name, md_rec)
|
|
83
|
+
logger.debug("Service #{service_name} needed for file: #{file_rec.path}")
|
|
84
|
+
return true
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
logger.debug("Run not needed for file: #{file_rec.path}")
|
|
89
|
+
# No services needed to be run for this file
|
|
90
|
+
false
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
end
|