longleaf 0.1.0 → 1.1.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (185) hide show
  1. checksums.yaml +4 -4
  2. data/.circleci/config.yml +94 -0
  3. data/.editorconfig +13 -0
  4. data/.gitignore +4 -1
  5. data/.rubocop.yml +44 -0
  6. data/.rubocop_todo.yml +834 -0
  7. data/.yardopts +1 -0
  8. data/Gemfile +16 -1
  9. data/README.md +98 -12
  10. data/Rakefile +6 -0
  11. data/bin/setup +16 -1
  12. data/docs/aboutlongleaf.md +28 -0
  13. data/docs/extra.css +32 -0
  14. data/docs/img/change-file.png +0 -0
  15. data/docs/img/ll-example-preserved.png +0 -0
  16. data/docs/index.md +19 -0
  17. data/docs/install.md +66 -0
  18. data/docs/ll-example/config-example-relative.yml +33 -0
  19. data/docs/ll-example/files-dir/LLexample-PDF.pdf +0 -0
  20. data/docs/ll-example/files-dir/LLexample-TOCHANGE.txt +15 -0
  21. data/docs/ll-example/files-dir/LLexample-tokeep.txt +10 -0
  22. data/docs/ll-example/metadata-dir/.gitkeep +0 -0
  23. data/docs/ll-example/replica-files/.gitkeep +0 -0
  24. data/docs/ll-example/replica-metadata/.gitkeep +0 -0
  25. data/docs/quickstart.md +270 -0
  26. data/docs/rdocs/Longleaf.html +135 -0
  27. data/docs/rdocs/Longleaf/AppFields.html +178 -0
  28. data/docs/rdocs/Longleaf/ApplicationConfigDeserializer.html +631 -0
  29. data/docs/rdocs/Longleaf/ApplicationConfigManager.html +610 -0
  30. data/docs/rdocs/Longleaf/ApplicationConfigValidator.html +238 -0
  31. data/docs/rdocs/Longleaf/CLI.html +909 -0
  32. data/docs/rdocs/Longleaf/ChecksumMismatchError.html +151 -0
  33. data/docs/rdocs/Longleaf/ConfigBuilder.html +1339 -0
  34. data/docs/rdocs/Longleaf/ConfigurationError.html +143 -0
  35. data/docs/rdocs/Longleaf/ConfigurationValidator.html +227 -0
  36. data/docs/rdocs/Longleaf/DeregisterCommand.html +420 -0
  37. data/docs/rdocs/Longleaf/DeregisterEvent.html +453 -0
  38. data/docs/rdocs/Longleaf/DeregistrationError.html +151 -0
  39. data/docs/rdocs/Longleaf/DigestHelper.html +419 -0
  40. data/docs/rdocs/Longleaf/EventError.html +147 -0
  41. data/docs/rdocs/Longleaf/EventNames.html +163 -0
  42. data/docs/rdocs/Longleaf/EventStatusTracking.html +656 -0
  43. data/docs/rdocs/Longleaf/FileCheckService.html +540 -0
  44. data/docs/rdocs/Longleaf/FileHelpers.html +520 -0
  45. data/docs/rdocs/Longleaf/FileRecord.html +716 -0
  46. data/docs/rdocs/Longleaf/FileSelector.html +901 -0
  47. data/docs/rdocs/Longleaf/FixityCheckService.html +691 -0
  48. data/docs/rdocs/Longleaf/IndexManager.html +1155 -0
  49. data/docs/rdocs/Longleaf/InvalidDigestAlgorithmError.html +143 -0
  50. data/docs/rdocs/Longleaf/InvalidStoragePathError.html +143 -0
  51. data/docs/rdocs/Longleaf/Logging.html +405 -0
  52. data/docs/rdocs/Longleaf/Logging/RedirectingLogger.html +1213 -0
  53. data/docs/rdocs/Longleaf/LongleafError.html +139 -0
  54. data/docs/rdocs/Longleaf/MDFields.html +193 -0
  55. data/docs/rdocs/Longleaf/MetadataBuilder.html +787 -0
  56. data/docs/rdocs/Longleaf/MetadataDeserializer.html +537 -0
  57. data/docs/rdocs/Longleaf/MetadataError.html +143 -0
  58. data/docs/rdocs/Longleaf/MetadataPersistenceManager.html +539 -0
  59. data/docs/rdocs/Longleaf/MetadataRecord.html +1411 -0
  60. data/docs/rdocs/Longleaf/MetadataSerializer.html +786 -0
  61. data/docs/rdocs/Longleaf/PreservationServiceError.html +147 -0
  62. data/docs/rdocs/Longleaf/PreserveCommand.html +410 -0
  63. data/docs/rdocs/Longleaf/PreserveEvent.html +491 -0
  64. data/docs/rdocs/Longleaf/RegisterCommand.html +428 -0
  65. data/docs/rdocs/Longleaf/RegisterEvent.html +628 -0
  66. data/docs/rdocs/Longleaf/RegisteredFileSelector.html +446 -0
  67. data/docs/rdocs/Longleaf/RegistrationError.html +151 -0
  68. data/docs/rdocs/Longleaf/ReindexCommand.html +576 -0
  69. data/docs/rdocs/Longleaf/RsyncReplicationService.html +1180 -0
  70. data/docs/rdocs/Longleaf/SequelIndexDriver.html +1978 -0
  71. data/docs/rdocs/Longleaf/ServiceCandidateFilesystemIterator.html +572 -0
  72. data/docs/rdocs/Longleaf/ServiceCandidateIndexIterator.html +532 -0
  73. data/docs/rdocs/Longleaf/ServiceCandidateLocator.html +333 -0
  74. data/docs/rdocs/Longleaf/ServiceClassCache.html +725 -0
  75. data/docs/rdocs/Longleaf/ServiceDateHelper.html +425 -0
  76. data/docs/rdocs/Longleaf/ServiceDefinition.html +683 -0
  77. data/docs/rdocs/Longleaf/ServiceDefinitionManager.html +371 -0
  78. data/docs/rdocs/Longleaf/ServiceDefinitionValidator.html +269 -0
  79. data/docs/rdocs/Longleaf/ServiceFields.html +173 -0
  80. data/docs/rdocs/Longleaf/ServiceManager.html +1229 -0
  81. data/docs/rdocs/Longleaf/ServiceMappingManager.html +410 -0
  82. data/docs/rdocs/Longleaf/ServiceMappingValidator.html +347 -0
  83. data/docs/rdocs/Longleaf/ServiceRecord.html +821 -0
  84. data/docs/rdocs/Longleaf/StorageLocation.html +985 -0
  85. data/docs/rdocs/Longleaf/StorageLocationManager.html +729 -0
  86. data/docs/rdocs/Longleaf/StorageLocationUnavailableError.html +143 -0
  87. data/docs/rdocs/Longleaf/StorageLocationValidator.html +373 -0
  88. data/docs/rdocs/Longleaf/StoragePathValidator.html +253 -0
  89. data/docs/rdocs/Longleaf/SystemConfigBuilder.html +441 -0
  90. data/docs/rdocs/Longleaf/SystemConfigFields.html +163 -0
  91. data/docs/rdocs/Longleaf/ValidateConfigCommand.html +451 -0
  92. data/docs/rdocs/Longleaf/ValidateMetadataCommand.html +408 -0
  93. data/docs/rdocs/_index.html +660 -0
  94. data/docs/rdocs/class_list.html +51 -0
  95. data/docs/rdocs/css/common.css +1 -0
  96. data/docs/rdocs/css/full_list.css +58 -0
  97. data/docs/rdocs/css/style.css +496 -0
  98. data/docs/rdocs/file.README.html +165 -0
  99. data/docs/rdocs/file_list.html +56 -0
  100. data/docs/rdocs/frames.html +17 -0
  101. data/docs/rdocs/index.html +165 -0
  102. data/docs/rdocs/js/app.js +303 -0
  103. data/docs/rdocs/js/full_list.js +216 -0
  104. data/docs/rdocs/js/jquery.js +4 -0
  105. data/docs/rdocs/method_list.html +2051 -0
  106. data/docs/rdocs/top-level-namespace.html +110 -0
  107. data/lib/longleaf/candidates/file_selector.rb +150 -0
  108. data/lib/longleaf/candidates/manifest_digest_provider.rb +17 -0
  109. data/lib/longleaf/candidates/physical_path_provider.rb +17 -0
  110. data/lib/longleaf/candidates/registered_file_selector.rb +67 -0
  111. data/lib/longleaf/candidates/service_candidate_filesystem_iterator.rb +93 -0
  112. data/lib/longleaf/candidates/service_candidate_index_iterator.rb +84 -0
  113. data/lib/longleaf/candidates/service_candidate_locator.rb +23 -0
  114. data/lib/longleaf/candidates/single_digest_provider.rb +13 -0
  115. data/lib/longleaf/cli.rb +252 -46
  116. data/lib/longleaf/commands/deregister_command.rb +51 -0
  117. data/lib/longleaf/commands/preserve_command.rb +50 -0
  118. data/lib/longleaf/commands/register_command.rb +34 -43
  119. data/lib/longleaf/commands/reindex_command.rb +92 -0
  120. data/lib/longleaf/commands/validate_config_command.rb +33 -8
  121. data/lib/longleaf/commands/validate_metadata_command.rb +51 -0
  122. data/lib/longleaf/errors.rb +26 -7
  123. data/lib/longleaf/events/deregister_event.rb +53 -0
  124. data/lib/longleaf/events/event_names.rb +9 -0
  125. data/lib/longleaf/events/event_status_tracking.rb +59 -0
  126. data/lib/longleaf/events/preserve_event.rb +82 -0
  127. data/lib/longleaf/events/register_event.rb +59 -51
  128. data/lib/longleaf/helpers/case_insensitive_hash.rb +38 -0
  129. data/lib/longleaf/helpers/digest_helper.rb +56 -0
  130. data/lib/longleaf/helpers/s3_uri_helper.rb +86 -0
  131. data/lib/longleaf/helpers/selection_options_parser.rb +215 -0
  132. data/lib/longleaf/helpers/service_date_helper.rb +78 -0
  133. data/lib/longleaf/indexing/index_manager.rb +101 -0
  134. data/lib/longleaf/indexing/sequel_index_driver.rb +306 -0
  135. data/lib/longleaf/logging.rb +5 -4
  136. data/lib/longleaf/logging/redirecting_logger.rb +30 -25
  137. data/lib/longleaf/models/app_fields.rb +7 -2
  138. data/lib/longleaf/models/file_record.rb +31 -8
  139. data/lib/longleaf/models/filesystem_metadata_location.rb +56 -0
  140. data/lib/longleaf/models/filesystem_storage_location.rb +52 -0
  141. data/lib/longleaf/models/md_fields.rb +3 -1
  142. data/lib/longleaf/models/metadata_location.rb +47 -0
  143. data/lib/longleaf/models/metadata_record.rb +43 -16
  144. data/lib/longleaf/models/s3_storage_location.rb +138 -0
  145. data/lib/longleaf/models/service_definition.rb +7 -6
  146. data/lib/longleaf/models/service_fields.rb +7 -1
  147. data/lib/longleaf/models/service_record.rb +10 -6
  148. data/lib/longleaf/models/storage_location.rb +24 -19
  149. data/lib/longleaf/models/storage_types.rb +9 -0
  150. data/lib/longleaf/models/system_config_fields.rb +9 -0
  151. data/lib/longleaf/preservation_services/file_check_service.rb +59 -0
  152. data/lib/longleaf/preservation_services/fixity_check_service.rb +124 -0
  153. data/lib/longleaf/preservation_services/rsync_replication_service.rb +198 -0
  154. data/lib/longleaf/preservation_services/s3_replication_service.rb +131 -0
  155. data/lib/longleaf/services/application_config_deserializer.rb +81 -24
  156. data/lib/longleaf/services/application_config_manager.rb +20 -6
  157. data/lib/longleaf/services/application_config_validator.rb +19 -9
  158. data/lib/longleaf/services/configuration_validator.rb +67 -4
  159. data/lib/longleaf/services/filesystem_location_validator.rb +16 -0
  160. data/lib/longleaf/services/metadata_deserializer.rb +115 -42
  161. data/lib/longleaf/services/metadata_persistence_manager.rb +47 -0
  162. data/lib/longleaf/services/metadata_serializer.rb +156 -23
  163. data/lib/longleaf/services/metadata_validator.rb +76 -0
  164. data/lib/longleaf/services/s3_location_validator.rb +19 -0
  165. data/lib/longleaf/services/service_class_cache.rb +112 -0
  166. data/lib/longleaf/services/service_definition_manager.rb +10 -7
  167. data/lib/longleaf/services/service_definition_validator.rb +25 -18
  168. data/lib/longleaf/services/service_manager.rb +86 -11
  169. data/lib/longleaf/services/service_mapping_manager.rb +13 -12
  170. data/lib/longleaf/services/service_mapping_validator.rb +36 -26
  171. data/lib/longleaf/services/storage_location_manager.rb +76 -15
  172. data/lib/longleaf/services/storage_location_validator.rb +49 -35
  173. data/lib/longleaf/specs/config_builder.rb +47 -23
  174. data/lib/longleaf/specs/config_validator_helpers.rb +16 -0
  175. data/lib/longleaf/specs/custom_matchers.rb +9 -0
  176. data/lib/longleaf/specs/file_helpers.rb +61 -0
  177. data/lib/longleaf/specs/metadata_builder.rb +98 -0
  178. data/lib/longleaf/specs/system_config_builder.rb +27 -0
  179. data/lib/longleaf/version.rb +1 -1
  180. data/longleaf.gemspec +20 -7
  181. data/mkdocs.yml +21 -0
  182. metadata +308 -24
  183. data/.travis.yml +0 -4
  184. data/lib/longleaf/commands/abstract_command.rb +0 -37
  185. data/lib/longleaf/services/storage_path_validator.rb +0 -16
@@ -0,0 +1,110 @@
1
+ <!DOCTYPE html>
2
+ <html>
3
+ <head>
4
+ <meta charset="utf-8">
5
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
6
+ <title>
7
+ Top Level Namespace
8
+
9
+ &mdash; Documentation by YARD 0.9.19
10
+
11
+ </title>
12
+
13
+ <link rel="stylesheet" href="css/style.css" type="text/css" charset="utf-8" />
14
+
15
+ <link rel="stylesheet" href="css/common.css" type="text/css" charset="utf-8" />
16
+
17
+ <script type="text/javascript" charset="utf-8">
18
+ pathId = "";
19
+ relpath = '';
20
+ </script>
21
+
22
+
23
+ <script type="text/javascript" charset="utf-8" src="js/jquery.js"></script>
24
+
25
+ <script type="text/javascript" charset="utf-8" src="js/app.js"></script>
26
+
27
+
28
+ </head>
29
+ <body>
30
+ <div class="nav_wrap">
31
+ <iframe id="nav" src="class_list.html?1"></iframe>
32
+ <div id="resizer"></div>
33
+ </div>
34
+
35
+ <div id="main" tabindex="-1">
36
+ <div id="header">
37
+ <div id="menu">
38
+
39
+ <a href="_index.html">Index</a> &raquo;
40
+
41
+
42
+ <span class="title">Top Level Namespace</span>
43
+
44
+ </div>
45
+
46
+ <div id="search">
47
+
48
+ <a class="full_list_link" id="class_list_link"
49
+ href="class_list.html">
50
+
51
+ <svg width="24" height="24">
52
+ <rect x="0" y="4" width="24" height="4" rx="1" ry="1"></rect>
53
+ <rect x="0" y="12" width="24" height="4" rx="1" ry="1"></rect>
54
+ <rect x="0" y="20" width="24" height="4" rx="1" ry="1"></rect>
55
+ </svg>
56
+ </a>
57
+
58
+ </div>
59
+ <div class="clear"></div>
60
+ </div>
61
+
62
+ <div id="content"><h1>Top Level Namespace
63
+
64
+
65
+
66
+ </h1>
67
+ <div class="box_info">
68
+
69
+
70
+
71
+
72
+
73
+
74
+
75
+
76
+
77
+
78
+
79
+ </div>
80
+
81
+ <h2>Defined Under Namespace</h2>
82
+ <p class="children">
83
+
84
+
85
+ <strong class="modules">Modules:</strong> <span class='object_link'><a href="Longleaf.html" title="Longleaf (module)">Longleaf</a></span>
86
+
87
+
88
+
89
+
90
+ </p>
91
+
92
+
93
+
94
+
95
+
96
+
97
+
98
+
99
+
100
+ </div>
101
+
102
+ <div id="footer">
103
+ Generated on Tue May 28 15:48:00 2019 by
104
+ <a href="http://yardoc.org" title="Yay! A Ruby Documentation Tool" target="_parent">yard</a>
105
+ 0.9.19 (ruby-2.6.3).
106
+ </div>
107
+
108
+ </div>
109
+ </body>
110
+ </html>
@@ -0,0 +1,150 @@
1
+ require 'longleaf/logging'
2
+ require 'longleaf/candidates/physical_path_provider'
3
+
4
+ module Longleaf
5
+ # Selects and allows for iteration over files which match a provided set of selection criteria
6
+ class FileSelector
7
+ include Longleaf::Logging
8
+ SPECIFICITY_PATH = 'path'
9
+ SPECIFICITY_STORAGE_LOCATION = 'storage_location'
10
+
11
+ attr_reader :specificity
12
+
13
+ # May only provide either file_paths or storage_locations
14
+ def initialize(file_paths: nil, storage_locations: nil, physical_provider: Longleaf::PhysicalPathProvider.new,
15
+ app_config:)
16
+ if nil_or_empty?(file_paths) && nil_or_empty?(storage_locations)
17
+ raise ArgumentError.new("Must provide either file paths or storage locations")
18
+ end
19
+ if !nil_or_empty?(file_paths) && !nil_or_empty?(storage_locations)
20
+ raise ArgumentError.new("Cannot provide both file paths and storage locations")
21
+ end
22
+ @app_config = app_config
23
+ # The top level paths targeted by this selector
24
+ @target_paths = file_paths&.map do |path|
25
+ # Resolve relative paths against pwd
26
+ pathname = Pathname.new(path)
27
+ if !pathname.absolute?
28
+ path = File.join(Dir.pwd, path)
29
+ end
30
+ path = File.expand_path(path)
31
+
32
+ # adding trailing /'s to directories
33
+ if Dir.exist?(path) && !path.end_with?('/')
34
+ path + '/'
35
+ else
36
+ path
37
+ end
38
+ end
39
+ # The set of storage locations to select file paths from
40
+ @storage_locations = storage_locations
41
+ @physical_provider = physical_provider
42
+ # Validate that the selected storage locations are known
43
+ if @storage_locations.nil?
44
+ @specificity = SPECIFICITY_PATH
45
+ else
46
+ @specificity = SPECIFICITY_STORAGE_LOCATION
47
+ locations = @app_config.location_manager.locations
48
+ @storage_locations.each do |loc_name|
49
+ unless locations.key?(loc_name)
50
+ raise StorageLocationUnavailableError.new("Cannot select unknown storage location #{loc_name}.")
51
+ end
52
+ end
53
+ end
54
+ end
55
+
56
+ # @return [Array] a list of top level paths from which files will be selected
57
+ def target_paths
58
+ # If starting from locations, initialize by expanding locations out to their actual paths
59
+ if @target_paths.nil? && !@storage_locations.nil?
60
+ @target_paths = Array.new
61
+ @storage_locations.each do |loc_name|
62
+ @target_paths << @app_config.location_manager.locations[loc_name].path
63
+ end
64
+ end
65
+
66
+ @target_paths
67
+ end
68
+
69
+ # Get the next logical file path for this selector.
70
+ # @return [String] an absolute path to the next file targeted by this selector,
71
+ # or nil if no more files selected
72
+ def next_path
73
+ if @paths.nil?
74
+ # Start the paths listing out from the targetted set of paths for this selector
75
+ # In reverse order since using a LIFO structure
76
+ @paths = target_paths.reverse
77
+ end
78
+
79
+ # No more paths to return
80
+ return nil if @paths&.empty?
81
+
82
+ # Get the most recently added path for depth first traversal of selected paths
83
+ path = @paths.pop
84
+ until path.nil? do
85
+ @app_config.location_manager.verify_path_in_location(path)
86
+ physical_path = @physical_provider.get_physical_path(path)
87
+ separate_logical = physical_path != path
88
+ if separate_logical
89
+ @app_config.location_manager.verify_path_in_location(physical_path)
90
+ end
91
+
92
+ if File.exist?(physical_path)
93
+ if File.directory?(physical_path)
94
+ if separate_logical
95
+ raise InvalidStoragePathError.new("Cannot specify physical path to a directory: #{physical_path}")
96
+ end
97
+ logger.debug("Expanding directory #{path}")
98
+ # For a directory, add all children to file_paths
99
+ Dir.entries(path).sort.reverse_each do |child|
100
+ @paths << File.join(path, child) unless child == '.' or child == '..'
101
+ end
102
+ else
103
+ logger.debug("Returning file #{path}")
104
+ return path
105
+ end
106
+ else
107
+ raise InvalidStoragePathError.new("File #{physical_path} does not exist.")
108
+ end
109
+
110
+ # Returned path was not a suitable file, try the next path
111
+ path = @paths.pop
112
+ end
113
+ end
114
+
115
+ # Iterate through the file paths for this selector and execute the provided block with each.
116
+ # A block is required.
117
+ def each
118
+ file_path = next_path
119
+ until file_path.nil?
120
+ yield file_path
121
+
122
+ file_path = next_path
123
+ end
124
+ end
125
+
126
+ # return [Array] a list of all storage locations being targeted by this selector
127
+ def storage_locations
128
+ # Determine what storage_locations are represented by the given file paths
129
+ if @storage_locations.nil? && !@target_paths.nil?
130
+ loc_set = Set.new
131
+ @target_paths.each do |path|
132
+ loc = @app_config.location_manager.get_location_by_path(path)
133
+ loc_set.add(loc.name) unless loc.nil?
134
+ end
135
+ @storage_locations = loc_set.to_a
136
+ end
137
+
138
+ if @storage_locations.nil?
139
+ @storage_locations = Array.new
140
+ end
141
+
142
+ @storage_locations
143
+ end
144
+
145
+ private
146
+ def nil_or_empty?(value)
147
+ value.nil? || value.empty?
148
+ end
149
+ end
150
+ end
@@ -0,0 +1,17 @@
1
+ module Longleaf
2
+ # Provides digests for files from a manifest
3
+ class ManifestDigestProvider
4
+ # @param digests_mapping hash which maps file paths to hashs of digests
5
+ def initialize(digests_mapping)
6
+ @digests_mapping = digests_mapping
7
+ end
8
+
9
+ # @param file_path [String] path of file
10
+ # @return hash containing all the manifested digests for the given path, or nil
11
+ def get_digests(file_path)
12
+ # return nil if key not found, in case the hash has default values
13
+ return nil unless @digests_mapping.key?(file_path)
14
+ @digests_mapping[file_path]
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,17 @@
1
+ module Longleaf
2
+ # Provides physical paths for logical paths from a mapping
3
+ class PhysicalPathProvider
4
+ # @param phys_mapping hash with logical paths as keys, physical paths as values
5
+ def initialize(phys_mapping = Hash.new)
6
+ @phys_mapping = phys_mapping
7
+ end
8
+
9
+ # @param logical_path [String] logical path of file
10
+ # @return physical path of the file
11
+ def get_physical_path(logical_path)
12
+ # return the logical path itself if no physical path is mapped
13
+ return logical_path unless @phys_mapping.key?(logical_path)
14
+ @phys_mapping[logical_path]
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,67 @@
1
+ require 'longleaf/candidates/file_selector'
2
+ require 'longleaf/logging'
3
+
4
+ module Longleaf
5
+ # Selects and allows for iteration over files which are registered and match a provided
6
+ # set of selection criteria
7
+ class RegisteredFileSelector < FileSelector
8
+ include Longleaf::Logging
9
+
10
+ # Get the next file path for this selector.
11
+ # @raise [InvalidStoragePathError] if any of the selected files do not exist
12
+ # @raise [StorageLocationUnavailableError] if any of the selected paths are not
13
+ # in a registered storage location.
14
+ # @return [String] an absolute path to the next file targeted by this selector,
15
+ # or nil if no more files selected
16
+ def next_path
17
+ if @md_paths.nil?
18
+ # Compute the starting paths by looking up the metadata paths for the provided targets,
19
+ # in reverse order since @md_paths is a LIFO stack structure.
20
+ @md_paths = target_paths.reverse_each.map do |file_path|
21
+ storage_loc = @app_config.location_manager.verify_path_in_location(file_path)
22
+ storage_loc.get_metadata_path_for(file_path)
23
+ end
24
+ end
25
+
26
+ # No more paths to return
27
+ return nil if @md_paths&.empty?
28
+
29
+ # Get the most recently added path for depth first traversal of selected paths
30
+ md_path = @md_paths.pop
31
+ until md_path.nil? do
32
+ if File.exist?(md_path)
33
+ if File.directory?(md_path)
34
+ logger.debug("Expanding metadata directory #{md_path}")
35
+ # For a directory, add all children to file_paths
36
+ Dir.entries(md_path).sort.reverse_each do |child|
37
+ @md_paths << File.join(md_path, child) unless child == '.' or child == '..'
38
+ end
39
+ elsif md_path.end_with?(MetadataSerializer::metadata_suffix)
40
+ # Convert metadata path to file path before returning
41
+ return file_path_for_metadata(md_path)
42
+ else
43
+ logger.debug("Skipping non-metadata file in metadata directory #{md_path}")
44
+ end
45
+ else
46
+ file_path = file_path_for_metadata(md_path)
47
+ if File.exist?(file_path)
48
+ raise RegistrationError.new("File #{file_path} is not registered.")
49
+ else
50
+ raise InvalidStoragePathError.new("File #{file_path} does not exist.")
51
+ end
52
+ end
53
+
54
+ # Returned path was not a suitable file, try the next path
55
+ md_path = @md_paths.pop
56
+ end
57
+ end
58
+
59
+ private
60
+ def file_path_for_metadata(md_path)
61
+ storage_loc = @app_config.location_manager.get_location_by_metadata_path(md_path)
62
+ file_path = storage_loc.get_path_from_metadata_path(md_path)
63
+ logger.debug("Returning next file #{file_path} for metadata path #{md_path}")
64
+ return file_path
65
+ end
66
+ end
67
+ end
@@ -0,0 +1,93 @@
1
+ require 'longleaf/services/service_manager'
2
+ require 'longleaf/events/event_names'
3
+ require 'longleaf/errors'
4
+ require 'longleaf/logging'
5
+ require 'time'
6
+
7
+ module Longleaf
8
+ # Iterator for getting file candidates which have services which need to be run.
9
+ # Implementation uses metadata files directly from the filesystem for determinations
10
+ # about service status.
11
+ class ServiceCandidateFilesystemIterator
12
+ include Longleaf::Logging
13
+
14
+ def initialize(file_selector, event, app_config, force = false)
15
+ @file_selector = file_selector
16
+ @event = event
17
+ @app_config = app_config
18
+ @force = force
19
+ end
20
+
21
+ # Get the file record for the next candidate which needs services run which match the
22
+ # provided file_selector
23
+ # @return [FileRecord] file record of the next candidate with services needing to be run,
24
+ # or nil if there are no more candidates.
25
+ def next_candidate
26
+ loop do
27
+ next_path = @file_selector.next_path
28
+ return nil if next_path.nil?
29
+
30
+ logger.debug("Evaluating candidate #{next_path}")
31
+ storage_loc = @app_config.location_manager.get_location_by_path(next_path)
32
+ file_rec = FileRecord.new(next_path, storage_loc)
33
+
34
+ # Skip over unregistered files
35
+ if !file_rec.metadata_present?
36
+ logger.debug("Ignoring unregistered file #{next_path}")
37
+ next
38
+ end
39
+
40
+ @app_config.md_manager.load(file_rec)
41
+
42
+ # Return the file record if it needs any services run
43
+ return file_rec if needs_run?(file_rec)
44
+ end
45
+ end
46
+
47
+ # Iterate through the candidates in this object and execute the provided block with each
48
+ # candidate. A block is required.
49
+ def each
50
+ file_rec = next_candidate
51
+ until file_rec.nil?
52
+ yield file_rec
53
+
54
+ file_rec = next_candidate
55
+ end
56
+ end
57
+
58
+ private
59
+ # Returns true if the file record contains any services which need to be run
60
+ def needs_run?(file_rec)
61
+ md_rec = file_rec.metadata_record
62
+ storage_loc = file_rec.storage_location
63
+ service_manager = @app_config.service_manager
64
+
65
+ # File is not a valid candidate for services if it is deregistered, unless performing cleanup
66
+ if @event != EventNames::CLEANUP && md_rec.deregistered?
67
+ logger.debug("Skipping deregistered file: #{file_rec.path}")
68
+ return false
69
+ end
70
+
71
+ expected_services = service_manager.list_services(
72
+ location: storage_loc.name,
73
+ event: @event)
74
+
75
+ # When in force mode, candidate needs a run as long as there are any services configured for it.
76
+ if @force && !expected_services.empty?
77
+ logger.debug("Forcing run needed for file: #{file_rec.path}")
78
+ return true
79
+ end
80
+
81
+ expected_services.each do |service_name|
82
+ if service_manager.service_needed?(service_name, md_rec)
83
+ logger.debug("Service #{service_name} needed for file: #{file_rec.path}")
84
+ return true
85
+ end
86
+ end
87
+
88
+ logger.debug("Run not needed for file: #{file_rec.path}")
89
+ # No services needed to be run for this file
90
+ false
91
+ end
92
+ end
93
+ end