RubyGems - longleaf - Versions diffs - 0.2.0.pre.1 → 0.3.0 - Mend

longleaf 0.2.0.pre.1 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (165) hide show

checksums.yaml +4 -4
data/.circleci/config.yml +84 -0
data/.gitignore +4 -2
data/.rubocop.yml +42 -2
data/.rubocop_todo.yml +390 -311
data/.yardopts +1 -0
data/Gemfile +16 -1
data/README.md +67 -13
data/Rakefile +6 -0
data/bin/setup +16 -1
data/docs/aboutlongleaf.md +28 -0
data/docs/extra.css +32 -0
data/docs/img/change-file.png +0 -0
data/docs/img/ll-example-preserved.png +0 -0
data/docs/index.md +19 -0
data/docs/install.md +66 -0
data/docs/ll-example/config-example-relative.yml +33 -0
data/docs/ll-example/files-dir/LLexample-PDF.pdf +0 -0
data/docs/ll-example/files-dir/LLexample-TOCHANGE.txt +15 -0
data/docs/ll-example/files-dir/LLexample-tokeep.txt +10 -0
data/docs/ll-example/metadata-dir/.gitkeep +0 -0
data/docs/ll-example/replica-files/.gitkeep +0 -0
data/docs/ll-example/replica-metadata/.gitkeep +0 -0
data/docs/quickstart.md +270 -0
data/docs/rdocs/Longleaf.html +135 -0
data/docs/rdocs/Longleaf/AppFields.html +178 -0
data/docs/rdocs/Longleaf/ApplicationConfigDeserializer.html +631 -0
data/docs/rdocs/Longleaf/ApplicationConfigManager.html +610 -0
data/docs/rdocs/Longleaf/ApplicationConfigValidator.html +238 -0
data/docs/rdocs/Longleaf/CLI.html +909 -0
data/docs/rdocs/Longleaf/ChecksumMismatchError.html +151 -0
data/docs/rdocs/Longleaf/ConfigBuilder.html +1339 -0
data/docs/rdocs/Longleaf/ConfigurationError.html +143 -0
data/docs/rdocs/Longleaf/ConfigurationValidator.html +227 -0
data/docs/rdocs/Longleaf/DeregisterCommand.html +420 -0
data/docs/rdocs/Longleaf/DeregisterEvent.html +453 -0
data/docs/rdocs/Longleaf/DeregistrationError.html +151 -0
data/docs/rdocs/Longleaf/DigestHelper.html +419 -0
data/docs/rdocs/Longleaf/EventError.html +147 -0
data/docs/rdocs/Longleaf/EventNames.html +163 -0
data/docs/rdocs/Longleaf/EventStatusTracking.html +656 -0
data/docs/rdocs/Longleaf/FileCheckService.html +540 -0
data/docs/rdocs/Longleaf/FileHelpers.html +520 -0
data/docs/rdocs/Longleaf/FileRecord.html +716 -0
data/docs/rdocs/Longleaf/FileSelector.html +901 -0
data/docs/rdocs/Longleaf/FixityCheckService.html +691 -0
data/docs/rdocs/Longleaf/IndexManager.html +1155 -0
data/docs/rdocs/Longleaf/InvalidDigestAlgorithmError.html +143 -0
data/docs/rdocs/Longleaf/InvalidStoragePathError.html +143 -0
data/docs/rdocs/Longleaf/Logging.html +405 -0
data/docs/rdocs/Longleaf/Logging/RedirectingLogger.html +1213 -0
data/docs/rdocs/Longleaf/LongleafError.html +139 -0
data/docs/rdocs/Longleaf/MDFields.html +193 -0
data/docs/rdocs/Longleaf/MetadataBuilder.html +787 -0
data/docs/rdocs/Longleaf/MetadataDeserializer.html +537 -0
data/docs/rdocs/Longleaf/MetadataError.html +143 -0
data/docs/rdocs/Longleaf/MetadataPersistenceManager.html +539 -0
data/docs/rdocs/Longleaf/MetadataRecord.html +1411 -0
data/docs/rdocs/Longleaf/MetadataSerializer.html +786 -0
data/docs/rdocs/Longleaf/PreservationServiceError.html +147 -0
data/docs/rdocs/Longleaf/PreserveCommand.html +410 -0
data/docs/rdocs/Longleaf/PreserveEvent.html +491 -0
data/docs/rdocs/Longleaf/RegisterCommand.html +428 -0
data/docs/rdocs/Longleaf/RegisterEvent.html +628 -0
data/docs/rdocs/Longleaf/RegisteredFileSelector.html +446 -0
data/docs/rdocs/Longleaf/RegistrationError.html +151 -0
data/docs/rdocs/Longleaf/ReindexCommand.html +576 -0
data/docs/rdocs/Longleaf/RsyncReplicationService.html +1180 -0
data/docs/rdocs/Longleaf/SequelIndexDriver.html +1978 -0
data/docs/rdocs/Longleaf/ServiceCandidateFilesystemIterator.html +572 -0
data/docs/rdocs/Longleaf/ServiceCandidateIndexIterator.html +532 -0
data/docs/rdocs/Longleaf/ServiceCandidateLocator.html +333 -0
data/docs/rdocs/Longleaf/ServiceClassCache.html +725 -0
data/docs/rdocs/Longleaf/ServiceDateHelper.html +425 -0
data/docs/rdocs/Longleaf/ServiceDefinition.html +683 -0
data/docs/rdocs/Longleaf/ServiceDefinitionManager.html +371 -0
data/docs/rdocs/Longleaf/ServiceDefinitionValidator.html +269 -0
data/docs/rdocs/Longleaf/ServiceFields.html +173 -0
data/docs/rdocs/Longleaf/ServiceManager.html +1229 -0
data/docs/rdocs/Longleaf/ServiceMappingManager.html +410 -0
data/docs/rdocs/Longleaf/ServiceMappingValidator.html +347 -0
data/docs/rdocs/Longleaf/ServiceRecord.html +821 -0
data/docs/rdocs/Longleaf/StorageLocation.html +985 -0
data/docs/rdocs/Longleaf/StorageLocationManager.html +729 -0
data/docs/rdocs/Longleaf/StorageLocationUnavailableError.html +143 -0
data/docs/rdocs/Longleaf/StorageLocationValidator.html +373 -0
data/docs/rdocs/Longleaf/StoragePathValidator.html +253 -0
data/docs/rdocs/Longleaf/SystemConfigBuilder.html +441 -0
data/docs/rdocs/Longleaf/SystemConfigFields.html +163 -0
data/docs/rdocs/Longleaf/ValidateConfigCommand.html +451 -0
data/docs/rdocs/Longleaf/ValidateMetadataCommand.html +408 -0
data/docs/rdocs/_index.html +660 -0
data/docs/rdocs/class_list.html +51 -0
data/docs/rdocs/css/common.css +1 -0
data/docs/rdocs/css/full_list.css +58 -0
data/docs/rdocs/css/style.css +496 -0
data/docs/rdocs/file.README.html +165 -0
data/docs/rdocs/file_list.html +56 -0
data/docs/rdocs/frames.html +17 -0
data/docs/rdocs/index.html +165 -0
data/docs/rdocs/js/app.js +303 -0
data/docs/rdocs/js/full_list.js +216 -0
data/docs/rdocs/js/jquery.js +4 -0
data/docs/rdocs/method_list.html +2051 -0
data/docs/rdocs/top-level-namespace.html +110 -0
data/lib/longleaf/candidates/file_selector.rb +47 -15
data/lib/longleaf/candidates/registered_file_selector.rb +67 -0
data/lib/longleaf/candidates/service_candidate_filesystem_iterator.rb +29 -35
data/lib/longleaf/candidates/service_candidate_index_iterator.rb +84 -0
data/lib/longleaf/candidates/service_candidate_locator.rb +9 -4
data/lib/longleaf/cli.rb +162 -80
data/lib/longleaf/commands/deregister_command.rb +12 -11
data/lib/longleaf/commands/preserve_command.rb +13 -8
data/lib/longleaf/commands/register_command.rb +9 -6
data/lib/longleaf/commands/reindex_command.rb +92 -0
data/lib/longleaf/commands/validate_config_command.rb +27 -6
data/lib/longleaf/commands/validate_metadata_command.rb +11 -9
data/lib/longleaf/errors.rb +12 -12
data/lib/longleaf/events/deregister_event.rb +13 -15
data/lib/longleaf/events/event_status_tracking.rb +7 -7
data/lib/longleaf/events/preserve_event.rb +24 -14
data/lib/longleaf/events/register_event.rb +21 -35
data/lib/longleaf/helpers/digest_helper.rb +4 -4
data/lib/longleaf/helpers/service_date_helper.rb +5 -6
data/lib/longleaf/indexing/index_manager.rb +101 -0
data/lib/longleaf/indexing/sequel_index_driver.rb +324 -0
data/lib/longleaf/logging.rb +4 -4
data/lib/longleaf/logging/redirecting_logger.rb +20 -20
data/lib/longleaf/models/app_fields.rb +2 -1
data/lib/longleaf/models/file_record.rb +10 -6
data/lib/longleaf/models/md_fields.rb +1 -1
data/lib/longleaf/models/metadata_record.rb +22 -12
data/lib/longleaf/models/service_definition.rb +3 -3
data/lib/longleaf/models/service_fields.rb +1 -1
data/lib/longleaf/models/service_record.rb +6 -5
data/lib/longleaf/models/storage_location.rb +26 -7
data/lib/longleaf/models/system_config_fields.rb +9 -0
data/lib/longleaf/preservation_services/file_check_service.rb +58 -0
data/lib/longleaf/preservation_services/fixity_check_service.rb +16 -14
data/lib/longleaf/preservation_services/rsync_replication_service.rb +32 -31
data/lib/longleaf/services/application_config_deserializer.rb +55 -18
data/lib/longleaf/services/application_config_manager.rb +16 -4
data/lib/longleaf/services/application_config_validator.rb +1 -2
data/lib/longleaf/services/configuration_validator.rb +6 -4
data/lib/longleaf/services/metadata_deserializer.rb +40 -38
data/lib/longleaf/services/metadata_persistence_manager.rb +46 -0
data/lib/longleaf/services/metadata_serializer.rb +23 -22
data/lib/longleaf/services/service_class_cache.rb +15 -15
data/lib/longleaf/services/service_definition_manager.rb +5 -6
data/lib/longleaf/services/service_definition_validator.rb +5 -6
data/lib/longleaf/services/service_manager.rb +37 -17
data/lib/longleaf/services/service_mapping_manager.rb +9 -9
data/lib/longleaf/services/service_mapping_validator.rb +9 -10
data/lib/longleaf/services/storage_location_manager.rb +22 -8
data/lib/longleaf/services/storage_location_validator.rb +11 -8
data/lib/longleaf/services/storage_path_validator.rb +1 -1
data/lib/longleaf/specs/config_builder.rb +30 -17
data/lib/longleaf/specs/custom_matchers.rb +1 -1
data/lib/longleaf/specs/file_helpers.rb +15 -14
data/lib/longleaf/specs/metadata_builder.rb +91 -0
data/lib/longleaf/specs/system_config_builder.rb +27 -0
data/lib/longleaf/version.rb +1 -1
data/longleaf.gemspec +17 -7
data/mkdocs.yml +20 -0
metadata +233 -22

data/docs/rdocs/top-level-namespace.html ADDED Viewed

@@ -0,0 +1,110 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+<title>
+  Top Level Namespace
+    &mdash; Documentation by YARD 0.9.19
+</title>
+  <link rel="stylesheet" href="css/style.css" type="text/css" charset="utf-8" />
+  <link rel="stylesheet" href="css/common.css" type="text/css" charset="utf-8" />
+<script type="text/javascript" charset="utf-8">
+  pathId = "";
+  relpath = '';
+</script>
+  <script type="text/javascript" charset="utf-8" src="js/jquery.js"></script>
+  <script type="text/javascript" charset="utf-8" src="js/app.js"></script>
+  </head>
+  <body>
+    <div class="nav_wrap">
+      <iframe id="nav" src="class_list.html?1"></iframe>
+      <div id="resizer"></div>
+    </div>
+    <div id="main" tabindex="-1">
+      <div id="header">
+        <div id="menu">
+    <a href="_index.html">Index</a> &raquo;
+    <span class="title">Top Level Namespace</span>
+</div>
+        <div id="search">
+    <a class="full_list_link" id="class_list_link"
+        href="class_list.html">
+        <svg width="24" height="24">
+          <rect x="0" y="4" width="24" height="4" rx="1" ry="1"></rect>
+          <rect x="0" y="12" width="24" height="4" rx="1" ry="1"></rect>
+          <rect x="0" y="20" width="24" height="4" rx="1" ry="1"></rect>
+        </svg>
+    </a>
+</div>
+        <div class="clear"></div>
+      </div>
+      <div id="content"><h1>Top Level Namespace
+</h1>
+<div class="box_info">
+</div>
+<h2>Defined Under Namespace</h2>
+<p class="children">
+      <strong class="modules">Modules:</strong> <span class='object_link'><a href="Longleaf.html" title="Longleaf (module)">Longleaf</a></span>
+</p>
+</div>
+      <div id="footer">
+  Generated on Tue May 28 15:48:00 2019 by
+  <a href="http://yardoc.org" title="Yay! A Ruby Documentation Tool" target="_parent">yard</a>
+  0.9.19 (ruby-2.6.3).
+</div>
+    </div>
+  </body>
+</html>

data/lib/longleaf/candidates/file_selector.rb CHANGED Viewed

@@ -4,7 +4,11 @@ module Longleaf
   # Selects and allows for iteration over files which match a provided set of selection criteria
   class FileSelector
     include Longleaf::Logging
+    SPECIFICITY_PATH = 'path'
+    SPECIFICITY_STORAGE_LOCATION = 'storage_location'
+    attr_reader :specificity
     # May only provide either file_paths or storage_locations
     def initialize(file_paths: nil, storage_locations: nil, app_config:)
       if nil_or_empty?(file_paths) && nil_or_empty?(storage_locations)
@@ -15,11 +19,28 @@ module Longleaf
       end
       @app_config = app_config
       # The top level paths targeted by this selector
-      @target_paths = file_paths
+      @target_paths = file_paths&.map do |path|
+        # Resolve relative paths against pwd
+        pathname = Pathname.new(path)
+        if !pathname.absolute?
+          path = File.join(Dir.pwd, path)
+        end
+        path = File.expand_path(path)
+        # adding trailing /'s to directories
+        if Dir.exist?(path) && !path.end_with?('/')
+          path + '/'
+        else
+          path
+        end
+      end
       # The set of storage locations to select file paths from
       @storage_locations = storage_locations
       # Validate that the selected storage locations are known
-      unless @storage_locations.nil?
+      if @storage_locations.nil?
+        @specificity = SPECIFICITY_PATH
+      else
+        @specificity = SPECIFICITY_STORAGE_LOCATION
         locations = @app_config.location_manager.locations
         @storage_locations.each do |loc_name|
           unless locations.key?(loc_name)
@@ -28,7 +49,7 @@ module Longleaf
         end
       end
     end
     # @return [Array] a list of top level paths from which files will be selected
     def target_paths
       # If starting from locations, initialize by expanding locations out to their actual paths
@@ -38,10 +59,10 @@ module Longleaf
           @target_paths << @app_config.location_manager.locations[loc_name].path
         end
       end
       @target_paths
     end
     # Get the next file path for this selector.
     # @return [String] an absolute path to the next file targeted by this selector,
     # or nil if no more files selected
@@ -54,17 +75,17 @@ module Longleaf
       # No more paths to return
       return nil if @paths&.empty?
       # Get the most recently added path for depth first traversal of selected paths
       path = @paths.pop
       until path.nil? do
         @app_config.location_manager.verify_path_in_location(path)
         if File.exist?(path)
           if File.directory?(path)
             logger.debug("Expanding directory #{path}")
             # For a directory, add all children to file_paths
-            Dir.entries(path).sort.reverse.each do |child|
+            Dir.entries(path).sort.reverse_each do |child|
               @paths << File.join(path, child) unless child == '.' or child == '..'
             end
           else
@@ -74,12 +95,23 @@ module Longleaf
         else
           raise InvalidStoragePathError.new("File #{path} does not exist.")
         end
         # Returned path was not a suitable file, try the next path
         path = @paths.pop
       end
     end
+    # Iterate through the file paths for this selector and execute the provided block with each.
+    # A block is required.
+    def each
+      file_path = next_path
+      until file_path.nil?
+        yield file_path
+        file_path = next_path
+      end
+    end
     # return [Array] a list of all storage locations being targeted by this selector
     def storage_locations
       # Determine what storage_locations are represented by the given file paths
@@ -91,17 +123,17 @@ module Longleaf
         end
         @storage_locations = loc_set.to_a
       end
       if @storage_locations.nil?
         @storage_locations = Array.new
       end
       @storage_locations
     end
     private
     def nil_or_empty?(value)
       value.nil? || value.empty?
     end
   end
-end
+end

data/lib/longleaf/candidates/registered_file_selector.rb ADDED Viewed

@@ -0,0 +1,67 @@
+require 'longleaf/candidates/file_selector'
+require 'longleaf/logging'
+module Longleaf
+  # Selects and allows for iteration over files which are registered and match a provided
+  # set of selection criteria
+  class RegisteredFileSelector < FileSelector
+    include Longleaf::Logging
+    # Get the next file path for this selector.
+    # @raise [InvalidStoragePathError] if any of the selected files do not exist
+    # @raise [StorageLocationUnavailableError] if any of the selected paths are not
+    #    in a registered storage location.
+    # @return [String] an absolute path to the next file targeted by this selector,
+    #    or nil if no more files selected
+    def next_path
+      if @md_paths.nil?
+        # Compute the starting paths by looking up the metadata paths for the provided targets,
+        # in reverse order since @md_paths is a LIFO stack structure.
+        @md_paths = target_paths.reverse_each.map do |file_path|
+          storage_loc = @app_config.location_manager.verify_path_in_location(file_path)
+          storage_loc.get_metadata_path_for(file_path)
+        end
+      end
+      # No more paths to return
+      return nil if @md_paths&.empty?
+      # Get the most recently added path for depth first traversal of selected paths
+      md_path = @md_paths.pop
+      until md_path.nil? do
+        if File.exist?(md_path)
+          if File.directory?(md_path)
+            logger.debug("Expanding metadata directory #{md_path}")
+            # For a directory, add all children to file_paths
+            Dir.entries(md_path).sort.reverse_each do |child|
+              @md_paths << File.join(md_path, child) unless child == '.' or child == '..'
+            end
+          elsif md_path.end_with?(MetadataSerializer::metadata_suffix)
+            # Convert metadata path to file path before returning
+            return file_path_for_metadata(md_path)
+          else
+            logger.debug("Skipping non-metadata file in metadata directory #{md_path}")
+          end
+        else
+          file_path = file_path_for_metadata(md_path)
+          if File.exist?(file_path)
+            raise RegistrationError.new("File #{file_path} is not registered.")
+          else
+            raise InvalidStoragePathError.new("File #{file_path} does not exist.")
+          end
+        end
+        # Returned path was not a suitable file, try the next path
+        md_path = @md_paths.pop
+      end
+    end
+    private
+    def file_path_for_metadata(md_path)
+      storage_loc = @app_config.location_manager.get_location_by_metadata_path(md_path)
+      file_path = storage_loc.get_path_from_metadata_path(md_path)
+      logger.debug("Returning next file #{file_path} for metadata path #{md_path}")
+      return file_path
+    end
+  end
+end

data/lib/longleaf/candidates/service_candidate_filesystem_iterator.rb CHANGED Viewed

@@ -1,5 +1,4 @@
 require 'longleaf/services/service_manager'
-require 'longleaf/services/metadata_deserializer'
 require 'longleaf/events/event_names'
 require 'longleaf/errors'
 require 'longleaf/logging'
@@ -11,89 +10,84 @@ module Longleaf
   # about service status.
   class ServiceCandidateFilesystemIterator
     include Longleaf::Logging
     def initialize(file_selector, event, app_config, force = false)
       @file_selector = file_selector
       @event = event
       @app_config = app_config
       @force = force
     end
-    # Get the file record for the next candidate which needs services run which match the
+    # Get the file record for the next candidate which needs services run which match the
     # provided file_selector
     # @return [FileRecord] file record of the next candidate with services needing to be run,
     # or nil if there are no more candidates.
     def next_candidate
       loop do
-        begin
-          next_path = @file_selector.next_path
-          return nil if next_path.nil?
-          logger.debug("Evaluating candidate #{next_path}")
-          storage_loc = @app_config.location_manager.get_location_by_path(next_path)
-          file_rec = FileRecord.new(next_path, storage_loc)
-          # Skip over unregistered files
-          if !file_rec.metadata_present?
-            logger.debug("Ignoring unregistered file #{next_path}")
-            next
-          end
-          file_rec.metadata_record = MetadataDeserializer.deserialize(file_path: file_rec.metadata_path,
-              digest_algs: storage_loc.metadata_digests)
-          # Return the file record if it needs any services run
-          return file_rec if needs_run?(file_rec)
-        rescue InvalidStoragePathError => e
-          logger.warn("Skipping candidate file: #{e.message}")
+        next_path = @file_selector.next_path
+        return nil if next_path.nil?
+        logger.debug("Evaluating candidate #{next_path}")
+        storage_loc = @app_config.location_manager.get_location_by_path(next_path)
+        file_rec = FileRecord.new(next_path, storage_loc)
+        # Skip over unregistered files
+        if !file_rec.metadata_present?
+          logger.debug("Ignoring unregistered file #{next_path}")
+          next
         end
+        @app_config.md_manager.load(file_rec)
+        # Return the file record if it needs any services run
+        return file_rec if needs_run?(file_rec)
       end
     end
     # Iterate through the candidates in this object and execute the provided block with each
     # candidate. A block is required.
     def each
       file_rec = next_candidate
       until file_rec.nil?
         yield file_rec
         file_rec = next_candidate
       end
     end
     private
     # Returns true if the file record contains any services which need to be run
     def needs_run?(file_rec)
       md_rec = file_rec.metadata_record
       storage_loc = file_rec.storage_location
       service_manager = @app_config.service_manager
       # File is not a valid candidate for services if it is deregistered, unless performing cleanup
       if @event != EventNames::CLEANUP && md_rec.deregistered?
         logger.debug("Skipping deregistered file: #{file_rec.path}")
         return false
       end
       expected_services = service_manager.list_services(
           location: storage_loc.name,
           event: @event)
       # When in force mode, candidate needs a run as long as there are any services configured for it.
-      if @force && expected_services.length > 0
+      if @force && !expected_services.empty?
         logger.debug("Forcing run needed for file: #{file_rec.path}")
         return true
       end
       expected_services.each do |service_name|
         if service_manager.service_needed?(service_name, md_rec)
           logger.debug("Service #{service_name} needed for file: #{file_rec.path}")
           return true
         end
       end
       logger.debug("Run not needed for file: #{file_rec.path}")
       # No services needed to be run for this file
       false
     end
   end
-end
+end

data/lib/longleaf/candidates/service_candidate_index_iterator.rb ADDED Viewed

@@ -0,0 +1,84 @@
+require 'longleaf/events/event_names'
+require 'longleaf/errors'
+require 'longleaf/logging'
+require 'time'
+module Longleaf
+  # Iterator for getting file candidates which have services which need to be run.
+  # Implementation uses an index of file metadata to determine if the file needs any
+  # services run.
+  class ServiceCandidateIndexIterator
+    include Longleaf::Logging
+    def initialize(file_selector, event, app_config, force = false)
+      @file_selector = file_selector
+      @event = event
+      @app_config = app_config
+      @force = force
+      @index_manager = @app_config.index_manager
+      @stale_datetime = Time.now.utc
+      @result_set = nil
+    end
+    # Get the file record for the next candidate which needs services run which match the
+    # provided file_selector
+    # @return [FileRecord] file record of the next candidate with services needing to be run,
+    #    or nil if there are no more candidates.
+    def next_candidate
+      file_rec = nil
+      # loop until a candidate with metadata is retrieved
+      loop do
+        # Get the next page of results if the previous page has been processed
+        fetch_next_page if @result_set.nil? || @result_set.empty?
+        # Retrieve the next possible candidate path from the page
+        next_path = @result_set.shift
+        # given that the page was just retrieved, getting a nil path indicates that the retrieved page of
+        # candidates is empty and there are no more candidates to iterate at this time.
+        return nil if next_path.nil?
+        logger.debug("Retrieved candidate #{next_path}")
+        storage_loc = @app_config.location_manager.get_location_by_path(next_path)
+        file_rec = FileRecord.new(next_path, storage_loc)
+        # Keep seeking until a registered candidate is found, according to the file system.
+        if file_rec.metadata_present?
+          break
+        else
+          logger.warn("Encountered #{next_path} in index, but path is not registered. Clearing out of synch entry.")
+          @index_manager.remove(file_rec)
+        end
+      end
+      @app_config.md_manager.load(file_rec)
+      file_rec
+    end
+    # Iterate through the candidates in this object and execute the provided block with each
+    # candidate. A block is required.
+    def each
+      file_rec = next_candidate
+      until file_rec.nil?
+        yield file_rec
+        file_rec = next_candidate
+      end
+    end
+    private
+    def fetch_next_page
+      if @force
+        @result_set = @index_manager.registered_paths(@file_selector)
+      else
+        case @event
+        when EventNames::PRESERVE
+          @result_set = @index_manager.paths_with_stale_services(@file_selector, @stale_datetime)
+        when EventNames::CLEANUP
+          # TODO
+        end
+      end
+      logger.debug("Retrieve result set with #{@result_set&.length} entries")
+    end
+  end
+end