RubyGems - longleaf - Versions diffs - 0.2.0.pre.1 → 0.3.0 - Mend

longleaf 0.2.0.pre.1 → 0.3.0

Files changed (165) hide show

checksums.yaml +4 -4
data/.circleci/config.yml +84 -0
data/.gitignore +4 -2
data/.rubocop.yml +42 -2
data/.rubocop_todo.yml +390 -311
data/.yardopts +1 -0
data/Gemfile +16 -1
data/README.md +67 -13
data/Rakefile +6 -0
data/bin/setup +16 -1
data/docs/aboutlongleaf.md +28 -0
data/docs/extra.css +32 -0
data/docs/img/change-file.png +0 -0
data/docs/img/ll-example-preserved.png +0 -0
data/docs/index.md +19 -0
data/docs/install.md +66 -0
data/docs/ll-example/config-example-relative.yml +33 -0
data/docs/ll-example/files-dir/LLexample-PDF.pdf +0 -0
data/docs/ll-example/files-dir/LLexample-TOCHANGE.txt +15 -0
data/docs/ll-example/files-dir/LLexample-tokeep.txt +10 -0
data/docs/ll-example/metadata-dir/.gitkeep +0 -0
data/docs/ll-example/replica-files/.gitkeep +0 -0
data/docs/ll-example/replica-metadata/.gitkeep +0 -0
data/docs/quickstart.md +270 -0
data/docs/rdocs/Longleaf.html +135 -0
data/docs/rdocs/Longleaf/AppFields.html +178 -0
data/docs/rdocs/Longleaf/ApplicationConfigDeserializer.html +631 -0
data/docs/rdocs/Longleaf/ApplicationConfigManager.html +610 -0
data/docs/rdocs/Longleaf/ApplicationConfigValidator.html +238 -0
data/docs/rdocs/Longleaf/CLI.html +909 -0
data/docs/rdocs/Longleaf/ChecksumMismatchError.html +151 -0
data/docs/rdocs/Longleaf/ConfigBuilder.html +1339 -0
data/docs/rdocs/Longleaf/ConfigurationError.html +143 -0
data/docs/rdocs/Longleaf/ConfigurationValidator.html +227 -0
data/docs/rdocs/Longleaf/DeregisterCommand.html +420 -0
data/docs/rdocs/Longleaf/DeregisterEvent.html +453 -0
data/docs/rdocs/Longleaf/DeregistrationError.html +151 -0
data/docs/rdocs/Longleaf/DigestHelper.html +419 -0
data/docs/rdocs/Longleaf/EventError.html +147 -0
data/docs/rdocs/Longleaf/EventNames.html +163 -0
data/docs/rdocs/Longleaf/EventStatusTracking.html +656 -0
data/docs/rdocs/Longleaf/FileCheckService.html +540 -0
data/docs/rdocs/Longleaf/FileHelpers.html +520 -0
data/docs/rdocs/Longleaf/FileRecord.html +716 -0
data/docs/rdocs/Longleaf/FileSelector.html +901 -0
data/docs/rdocs/Longleaf/FixityCheckService.html +691 -0
data/docs/rdocs/Longleaf/IndexManager.html +1155 -0
data/docs/rdocs/Longleaf/InvalidDigestAlgorithmError.html +143 -0
data/docs/rdocs/Longleaf/InvalidStoragePathError.html +143 -0
data/docs/rdocs/Longleaf/Logging.html +405 -0
data/docs/rdocs/Longleaf/Logging/RedirectingLogger.html +1213 -0
data/docs/rdocs/Longleaf/LongleafError.html +139 -0
data/docs/rdocs/Longleaf/MDFields.html +193 -0
data/docs/rdocs/Longleaf/MetadataBuilder.html +787 -0
data/docs/rdocs/Longleaf/MetadataDeserializer.html +537 -0
data/docs/rdocs/Longleaf/MetadataError.html +143 -0
data/docs/rdocs/Longleaf/MetadataPersistenceManager.html +539 -0
data/docs/rdocs/Longleaf/MetadataRecord.html +1411 -0
data/docs/rdocs/Longleaf/MetadataSerializer.html +786 -0
data/docs/rdocs/Longleaf/PreservationServiceError.html +147 -0
data/docs/rdocs/Longleaf/PreserveCommand.html +410 -0
data/docs/rdocs/Longleaf/PreserveEvent.html +491 -0
data/docs/rdocs/Longleaf/RegisterCommand.html +428 -0
data/docs/rdocs/Longleaf/RegisterEvent.html +628 -0
data/docs/rdocs/Longleaf/RegisteredFileSelector.html +446 -0
data/docs/rdocs/Longleaf/RegistrationError.html +151 -0
data/docs/rdocs/Longleaf/ReindexCommand.html +576 -0
data/docs/rdocs/Longleaf/RsyncReplicationService.html +1180 -0
data/docs/rdocs/Longleaf/SequelIndexDriver.html +1978 -0
data/docs/rdocs/Longleaf/ServiceCandidateFilesystemIterator.html +572 -0
data/docs/rdocs/Longleaf/ServiceCandidateIndexIterator.html +532 -0
data/docs/rdocs/Longleaf/ServiceCandidateLocator.html +333 -0
data/docs/rdocs/Longleaf/ServiceClassCache.html +725 -0
data/docs/rdocs/Longleaf/ServiceDateHelper.html +425 -0
data/docs/rdocs/Longleaf/ServiceDefinition.html +683 -0
data/docs/rdocs/Longleaf/ServiceDefinitionManager.html +371 -0
data/docs/rdocs/Longleaf/ServiceDefinitionValidator.html +269 -0
data/docs/rdocs/Longleaf/ServiceFields.html +173 -0
data/docs/rdocs/Longleaf/ServiceManager.html +1229 -0
data/docs/rdocs/Longleaf/ServiceMappingManager.html +410 -0
data/docs/rdocs/Longleaf/ServiceMappingValidator.html +347 -0
data/docs/rdocs/Longleaf/ServiceRecord.html +821 -0
data/docs/rdocs/Longleaf/StorageLocation.html +985 -0
data/docs/rdocs/Longleaf/StorageLocationManager.html +729 -0
data/docs/rdocs/Longleaf/StorageLocationUnavailableError.html +143 -0
data/docs/rdocs/Longleaf/StorageLocationValidator.html +373 -0
data/docs/rdocs/Longleaf/StoragePathValidator.html +253 -0
data/docs/rdocs/Longleaf/SystemConfigBuilder.html +441 -0
data/docs/rdocs/Longleaf/SystemConfigFields.html +163 -0
data/docs/rdocs/Longleaf/ValidateConfigCommand.html +451 -0
data/docs/rdocs/Longleaf/ValidateMetadataCommand.html +408 -0
data/docs/rdocs/_index.html +660 -0
data/docs/rdocs/class_list.html +51 -0
data/docs/rdocs/css/common.css +1 -0
data/docs/rdocs/css/full_list.css +58 -0
data/docs/rdocs/css/style.css +496 -0
data/docs/rdocs/file.README.html +165 -0
data/docs/rdocs/file_list.html +56 -0
data/docs/rdocs/frames.html +17 -0
data/docs/rdocs/index.html +165 -0
data/docs/rdocs/js/app.js +303 -0
data/docs/rdocs/js/full_list.js +216 -0
data/docs/rdocs/js/jquery.js +4 -0
data/docs/rdocs/method_list.html +2051 -0
data/docs/rdocs/top-level-namespace.html +110 -0
data/lib/longleaf/candidates/file_selector.rb +47 -15
data/lib/longleaf/candidates/registered_file_selector.rb +67 -0
data/lib/longleaf/candidates/service_candidate_filesystem_iterator.rb +29 -35
data/lib/longleaf/candidates/service_candidate_index_iterator.rb +84 -0
data/lib/longleaf/candidates/service_candidate_locator.rb +9 -4
data/lib/longleaf/cli.rb +162 -80
data/lib/longleaf/commands/deregister_command.rb +12 -11
data/lib/longleaf/commands/preserve_command.rb +13 -8
data/lib/longleaf/commands/register_command.rb +9 -6
data/lib/longleaf/commands/reindex_command.rb +92 -0
data/lib/longleaf/commands/validate_config_command.rb +27 -6
data/lib/longleaf/commands/validate_metadata_command.rb +11 -9
data/lib/longleaf/errors.rb +12 -12
data/lib/longleaf/events/deregister_event.rb +13 -15
data/lib/longleaf/events/event_status_tracking.rb +7 -7
data/lib/longleaf/events/preserve_event.rb +24 -14
data/lib/longleaf/events/register_event.rb +21 -35
data/lib/longleaf/helpers/digest_helper.rb +4 -4
data/lib/longleaf/helpers/service_date_helper.rb +5 -6
data/lib/longleaf/indexing/index_manager.rb +101 -0
data/lib/longleaf/indexing/sequel_index_driver.rb +324 -0
data/lib/longleaf/logging.rb +4 -4
data/lib/longleaf/logging/redirecting_logger.rb +20 -20
data/lib/longleaf/models/app_fields.rb +2 -1
data/lib/longleaf/models/file_record.rb +10 -6
data/lib/longleaf/models/md_fields.rb +1 -1
data/lib/longleaf/models/metadata_record.rb +22 -12
data/lib/longleaf/models/service_definition.rb +3 -3
data/lib/longleaf/models/service_fields.rb +1 -1
data/lib/longleaf/models/service_record.rb +6 -5
data/lib/longleaf/models/storage_location.rb +26 -7
data/lib/longleaf/models/system_config_fields.rb +9 -0
data/lib/longleaf/preservation_services/file_check_service.rb +58 -0
data/lib/longleaf/preservation_services/fixity_check_service.rb +16 -14
data/lib/longleaf/preservation_services/rsync_replication_service.rb +32 -31
data/lib/longleaf/services/application_config_deserializer.rb +55 -18
data/lib/longleaf/services/application_config_manager.rb +16 -4
data/lib/longleaf/services/application_config_validator.rb +1 -2
data/lib/longleaf/services/configuration_validator.rb +6 -4
data/lib/longleaf/services/metadata_deserializer.rb +40 -38
data/lib/longleaf/services/metadata_persistence_manager.rb +46 -0
data/lib/longleaf/services/metadata_serializer.rb +23 -22
data/lib/longleaf/services/service_class_cache.rb +15 -15
data/lib/longleaf/services/service_definition_manager.rb +5 -6
data/lib/longleaf/services/service_definition_validator.rb +5 -6
data/lib/longleaf/services/service_manager.rb +37 -17
data/lib/longleaf/services/service_mapping_manager.rb +9 -9
data/lib/longleaf/services/service_mapping_validator.rb +9 -10
data/lib/longleaf/services/storage_location_manager.rb +22 -8
data/lib/longleaf/services/storage_location_validator.rb +11 -8
data/lib/longleaf/services/storage_path_validator.rb +1 -1
data/lib/longleaf/specs/config_builder.rb +30 -17
data/lib/longleaf/specs/custom_matchers.rb +1 -1
data/lib/longleaf/specs/file_helpers.rb +15 -14
data/lib/longleaf/specs/metadata_builder.rb +91 -0
data/lib/longleaf/specs/system_config_builder.rb +27 -0
data/lib/longleaf/version.rb +1 -1
data/longleaf.gemspec +17 -7
data/mkdocs.yml +20 -0
metadata +233 -22

data/docs/rdocs/top-level-namespace.html ADDED Viewed

@@ -0,0 +1,110 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+<title>
+  Top Level Namespace
+    &mdash; Documentation by YARD 0.9.19
+</title>
+  <link rel="stylesheet" href="css/style.css" type="text/css" charset="utf-8" />
+  <link rel="stylesheet" href="css/common.css" type="text/css" charset="utf-8" />
+<script type="text/javascript" charset="utf-8">
+  pathId = "";
+  relpath = '';
+</script>
+  <script type="text/javascript" charset="utf-8" src="js/jquery.js"></script>
+  <script type="text/javascript" charset="utf-8" src="js/app.js"></script>
+  </head>
+  <body>
+    <div class="nav_wrap">
+      <iframe id="nav" src="class_list.html?1"></iframe>
+      <div id="resizer"></div>
+    </div>
+    <div id="main" tabindex="-1">
+      <div id="header">
+        <div id="menu">
+    <a href="_index.html">Index</a> &raquo;
+    <span class="title">Top Level Namespace</span>
+</div>
+        <div id="search">
+    <a class="full_list_link" id="class_list_link"
+        href="class_list.html">
+        <svg width="24" height="24">
+          <rect x="0" y="4" width="24" height="4" rx="1" ry="1"></rect>
+          <rect x="0" y="12" width="24" height="4" rx="1" ry="1"></rect>
+          <rect x="0" y="20" width="24" height="4" rx="1" ry="1"></rect>
+        </svg>
+    </a>
+</div>
+        <div class="clear"></div>
+      </div>
+      <div id="content"><h1>Top Level Namespace
+</h1>
+<div class="box_info">
+</div>
+<h2>Defined Under Namespace</h2>
+<p class="children">
+      <strong class="modules">Modules:</strong> <span class='object_link'><a href="Longleaf.html" title="Longleaf (module)">Longleaf</a></span>
+</p>
+</div>
+      <div id="footer">
+  Generated on Tue May 28 15:48:00 2019 by
+  <a href="http://yardoc.org" title="Yay! A Ruby Documentation Tool" target="_parent">yard</a>
+  0.9.19 (ruby-2.6.3).
+</div>
+    </div>
+  </body>
+</html>

data/lib/longleaf/candidates/file_selector.rb CHANGED Viewed

@@ -4,7 +4,11 @@ module Longleaf
   # Selects and allows for iteration over files which match a provided set of selection criteria
   class FileSelector
     include Longleaf::Logging
+    SPECIFICITY_PATH = 'path'
+    SPECIFICITY_STORAGE_LOCATION = 'storage_location'
+    attr_reader :specificity
     # May only provide either file_paths or storage_locations
     def initialize(file_paths: nil, storage_locations: nil, app_config:)
       if nil_or_empty?(file_paths) && nil_or_empty?(storage_locations)
@@ -15,11 +19,28 @@ module Longleaf
       end
       @app_config = app_config
       # The top level paths targeted by this selector
-      @target_paths = file_paths
+      @target_paths = file_paths&.map do |path|
+        # Resolve relative paths against pwd
+        pathname = Pathname.new(path)
+        if !pathname.absolute?
+          path = File.join(Dir.pwd, path)
+        end
+        path = File.expand_path(path)
+        # adding trailing /'s to directories
+        if Dir.exist?(path) && !path.end_with?('/')
+          path + '/'
+        else
+          path
+        end
+      end
       # The set of storage locations to select file paths from
       @storage_locations = storage_locations
       # Validate that the selected storage locations are known
-      unless @storage_locations.nil?
+      if @storage_locations.nil?
+        @specificity = SPECIFICITY_PATH
+      else
+        @specificity = SPECIFICITY_STORAGE_LOCATION
         locations = @app_config.location_manager.locations
         @storage_locations.each do |loc_name|
           unless locations.key?(loc_name)
@@ -28,7 +49,7 @@ module Longleaf
         end
       end
     end
     # @return [Array] a list of top level paths from which files will be selected
     def target_paths
       # If starting from locations, initialize by expanding locations out to their actual paths
@@ -38,10 +59,10 @@ module Longleaf
           @target_paths << @app_config.location_manager.locations[loc_name].path
         end
       end
       @target_paths
     end
     # Get the next file path for this selector.
     # @return [String] an absolute path to the next file targeted by this selector,
     # or nil if no more files selected
@@ -54,17 +75,17 @@ module Longleaf
       # No more paths to return
       return nil if @paths&.empty?
       # Get the most recently added path for depth first traversal of selected paths
       path = @paths.pop
       until path.nil? do
         @app_config.location_manager.verify_path_in_location(path)
         if File.exist?(path)
           if File.directory?(path)
             logger.debug("Expanding directory #{path}")
             # For a directory, add all children to file_paths
-            Dir.entries(path).sort.reverse.each do |child|
+            Dir.entries(path).sort.reverse_each do |child|
               @paths << File.join(path, child) unless child == '.' or child == '..'
             end
           else
@@ -74,12 +95,23 @@ module Longleaf
         else
           raise InvalidStoragePathError.new("File #{path} does not exist.")
         end
         # Returned path was not a suitable file, try the next path
         path = @paths.pop
       end
     end
+    # Iterate through the file paths for this selector and execute the provided block with each.
+    # A block is required.
+    def each
+      file_path = next_path
+      until file_path.nil?
+        yield file_path
+        file_path = next_path
+      end
+    end
     # return [Array] a list of all storage locations being targeted by this selector
     def storage_locations
       # Determine what storage_locations are represented by the given file paths
@@ -91,17 +123,17 @@ module Longleaf
         end
         @storage_locations = loc_set.to_a
       end
       if @storage_locations.nil?
         @storage_locations = Array.new
       end
       @storage_locations
     end
     private
     def nil_or_empty?(value)
       value.nil? || value.empty?
     end
   end
-end
+end

data/lib/longleaf/candidates/registered_file_selector.rb ADDED Viewed

@@ -0,0 +1,67 @@
+require 'longleaf/candidates/file_selector'
+require 'longleaf/logging'
+module Longleaf
+  # Selects and allows for iteration over files which are registered and match a provided
+  # set of selection criteria
+  class RegisteredFileSelector < FileSelector
+    include Longleaf::Logging
+    # Get the next file path for this selector.
+    # @raise [InvalidStoragePathError] if any of the selected files do not exist
+    # @raise [StorageLocationUnavailableError] if any of the selected paths are not
+    #    in a registered storage location.
+    # @return [String] an absolute path to the next file targeted by this selector,
+    #    or nil if no more files selected
+    def next_path
+      if @md_paths.nil?
+        # Compute the starting paths by looking up the metadata paths for the provided targets,
+        # in reverse order since @md_paths is a LIFO stack structure.
+        @md_paths = target_paths.reverse_each.map do |file_path|
+          storage_loc = @app_config.location_manager.verify_path_in_location(file_path)
+          storage_loc.get_metadata_path_for(file_path)
+        end
+      end
+      # No more paths to return
+      return nil if @md_paths&.empty?
+      # Get the most recently added path for depth first traversal of selected paths
+      md_path = @md_paths.pop
+      until md_path.nil? do
+        if File.exist?(md_path)
+          if File.directory?(md_path)
+            logger.debug("Expanding metadata directory #{md_path}")
+            # For a directory, add all children to file_paths
+            Dir.entries(md_path).sort.reverse_each do |child|
+              @md_paths << File.join(md_path, child) unless child == '.' or child == '..'
+            end
+          elsif md_path.end_with?(MetadataSerializer::metadata_suffix)
+            # Convert metadata path to file path before returning
+            return file_path_for_metadata(md_path)
+          else
+            logger.debug("Skipping non-metadata file in metadata directory #{md_path}")
+          end
+        else
+          file_path = file_path_for_metadata(md_path)
+          if File.exist?(file_path)
+            raise RegistrationError.new("File #{file_path} is not registered.")
+          else
+            raise InvalidStoragePathError.new("File #{file_path} does not exist.")
+          end
+        end
+        # Returned path was not a suitable file, try the next path
+        md_path = @md_paths.pop
+      end
+    end
+    private
+    def file_path_for_metadata(md_path)
+      storage_loc = @app_config.location_manager.get_location_by_metadata_path(md_path)
+      file_path = storage_loc.get_path_from_metadata_path(md_path)
+      logger.debug("Returning next file #{file_path} for metadata path #{md_path}")
+      return file_path
+    end
+  end
+end

data/lib/longleaf/candidates/service_candidate_filesystem_iterator.rb CHANGED Viewed

@@ -1,5 +1,4 @@
 require 'longleaf/services/service_manager'
-require 'longleaf/services/metadata_deserializer'
 require 'longleaf/events/event_names'
 require 'longleaf/errors'
 require 'longleaf/logging'
@@ -11,89 +10,84 @@ module Longleaf
   # about service status.
   class ServiceCandidateFilesystemIterator
     include Longleaf::Logging
     def initialize(file_selector, event, app_config, force = false)
       @file_selector = file_selector
       @event = event
       @app_config = app_config
       @force = force
     end
-    # Get the file record for the next candidate which needs services run which match the
+    # Get the file record for the next candidate which needs services run which match the
     # provided file_selector
     # @return [FileRecord] file record of the next candidate with services needing to be run,
     # or nil if there are no more candidates.
     def next_candidate
       loop do
-        begin
-          next_path = @file_selector.next_path
-          return nil if next_path.nil?
-          logger.debug("Evaluating candidate #{next_path}")
-          storage_loc = @app_config.location_manager.get_location_by_path(next_path)
-          file_rec = FileRecord.new(next_path, storage_loc)
-          # Skip over unregistered files
-          if !file_rec.metadata_present?
-            logger.debug("Ignoring unregistered file #{next_path}")
-            next
-          end
-          file_rec.metadata_record = MetadataDeserializer.deserialize(file_path: file_rec.metadata_path,
-              digest_algs: storage_loc.metadata_digests)
-          # Return the file record if it needs any services run
-          return file_rec if needs_run?(file_rec)
-        rescue InvalidStoragePathError => e
-          logger.warn("Skipping candidate file: #{e.message}")
+        next_path = @file_selector.next_path
+        return nil if next_path.nil?
+        logger.debug("Evaluating candidate #{next_path}")
+        storage_loc = @app_config.location_manager.get_location_by_path(next_path)
+        file_rec = FileRecord.new(next_path, storage_loc)
+        # Skip over unregistered files
+        if !file_rec.metadata_present?
+          logger.debug("Ignoring unregistered file #{next_path}")
+          next
         end
+        @app_config.md_manager.load(file_rec)
+        # Return the file record if it needs any services run
+        return file_rec if needs_run?(file_rec)
       end
     end
     # Iterate through the candidates in this object and execute the provided block with each
     # candidate. A block is required.
     def each
       file_rec = next_candidate
       until file_rec.nil?
         yield file_rec
         file_rec = next_candidate
       end
     end
     private
     # Returns true if the file record contains any services which need to be run
     def needs_run?(file_rec)
       md_rec = file_rec.metadata_record
       storage_loc = file_rec.storage_location
       service_manager = @app_config.service_manager
       # File is not a valid candidate for services if it is deregistered, unless performing cleanup
       if @event != EventNames::CLEANUP && md_rec.deregistered?
         logger.debug("Skipping deregistered file: #{file_rec.path}")
         return false
       end
       expected_services = service_manager.list_services(
           location: storage_loc.name,
           event: @event)
       # When in force mode, candidate needs a run as long as there are any services configured for it.
-      if @force && expected_services.length > 0
+      if @force && !expected_services.empty?
         logger.debug("Forcing run needed for file: #{file_rec.path}")
         return true
       end
       expected_services.each do |service_name|
         if service_manager.service_needed?(service_name, md_rec)
           logger.debug("Service #{service_name} needed for file: #{file_rec.path}")
           return true
         end
       end
       logger.debug("Run not needed for file: #{file_rec.path}")
       # No services needed to be run for this file
       false
     end
   end
-end
+end

data/lib/longleaf/candidates/service_candidate_index_iterator.rb ADDED Viewed

@@ -0,0 +1,84 @@
+require 'longleaf/events/event_names'
+require 'longleaf/errors'
+require 'longleaf/logging'
+require 'time'
+module Longleaf
+  # Iterator for getting file candidates which have services which need to be run.
+  # Implementation uses an index of file metadata to determine if the file needs any
+  # services run.
+  class ServiceCandidateIndexIterator
+    include Longleaf::Logging
+    def initialize(file_selector, event, app_config, force = false)
+      @file_selector = file_selector
+      @event = event
+      @app_config = app_config
+      @force = force
+      @index_manager = @app_config.index_manager
+      @stale_datetime = Time.now.utc
+      @result_set = nil
+    end
+    # Get the file record for the next candidate which needs services run which match the
+    # provided file_selector
+    # @return [FileRecord] file record of the next candidate with services needing to be run,
+    #    or nil if there are no more candidates.
+    def next_candidate
+      file_rec = nil
+      # loop until a candidate with metadata is retrieved
+      loop do
+        # Get the next page of results if the previous page has been processed
+        fetch_next_page if @result_set.nil? || @result_set.empty?
+        # Retrieve the next possible candidate path from the page
+        next_path = @result_set.shift
+        # given that the page was just retrieved, getting a nil path indicates that the retrieved page of
+        # candidates is empty and there are no more candidates to iterate at this time.
+        return nil if next_path.nil?
+        logger.debug("Retrieved candidate #{next_path}")
+        storage_loc = @app_config.location_manager.get_location_by_path(next_path)
+        file_rec = FileRecord.new(next_path, storage_loc)
+        # Keep seeking until a registered candidate is found, according to the file system.
+        if file_rec.metadata_present?
+          break
+        else
+          logger.warn("Encountered #{next_path} in index, but path is not registered. Clearing out of synch entry.")
+          @index_manager.remove(file_rec)
+        end
+      end
+      @app_config.md_manager.load(file_rec)
+      file_rec
+    end
+    # Iterate through the candidates in this object and execute the provided block with each
+    # candidate. A block is required.
+    def each
+      file_rec = next_candidate
+      until file_rec.nil?
+        yield file_rec
+        file_rec = next_candidate
+      end
+    end
+    private
+    def fetch_next_page
+      if @force
+        @result_set = @index_manager.registered_paths(@file_selector)
+      else
+        case @event
+        when EventNames::PRESERVE
+          @result_set = @index_manager.paths_with_stale_services(@file_selector, @stale_datetime)
+        when EventNames::CLEANUP
+          # TODO
+        end
+      end
+      logger.debug("Retrieve result set with #{@result_set&.length} entries")
+    end
+  end
+end