RubyGems - longleaf - Versions diffs - 0.1.0.pre.2 → 1.0.0 - Mend

longleaf 0.1.0.pre.2 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (184) hide show

checksums.yaml +4 -4
data/.circleci/config.yml +94 -0
data/.editorconfig +13 -0
data/.gitignore +4 -1
data/.rubocop.yml +44 -0
data/.rubocop_todo.yml +834 -0
data/.yardopts +1 -0
data/Gemfile +16 -1
data/README.md +98 -12
data/Rakefile +6 -0
data/bin/setup +16 -1
data/docs/aboutlongleaf.md +28 -0
data/docs/extra.css +32 -0
data/docs/img/change-file.png +0 -0
data/docs/img/ll-example-preserved.png +0 -0
data/docs/index.md +19 -0
data/docs/install.md +66 -0
data/docs/ll-example/config-example-relative.yml +33 -0
data/docs/ll-example/files-dir/LLexample-PDF.pdf +0 -0
data/docs/ll-example/files-dir/LLexample-TOCHANGE.txt +15 -0
data/docs/ll-example/files-dir/LLexample-tokeep.txt +10 -0
data/docs/ll-example/metadata-dir/.gitkeep +0 -0
data/docs/ll-example/replica-files/.gitkeep +0 -0
data/docs/ll-example/replica-metadata/.gitkeep +0 -0
data/docs/quickstart.md +270 -0
data/docs/rdocs/Longleaf.html +135 -0
data/docs/rdocs/Longleaf/AppFields.html +178 -0
data/docs/rdocs/Longleaf/ApplicationConfigDeserializer.html +631 -0
data/docs/rdocs/Longleaf/ApplicationConfigManager.html +610 -0
data/docs/rdocs/Longleaf/ApplicationConfigValidator.html +238 -0
data/docs/rdocs/Longleaf/CLI.html +909 -0
data/docs/rdocs/Longleaf/ChecksumMismatchError.html +151 -0
data/docs/rdocs/Longleaf/ConfigBuilder.html +1339 -0
data/docs/rdocs/Longleaf/ConfigurationError.html +143 -0
data/docs/rdocs/Longleaf/ConfigurationValidator.html +227 -0
data/docs/rdocs/Longleaf/DeregisterCommand.html +420 -0
data/docs/rdocs/Longleaf/DeregisterEvent.html +453 -0
data/docs/rdocs/Longleaf/DeregistrationError.html +151 -0
data/docs/rdocs/Longleaf/DigestHelper.html +419 -0
data/docs/rdocs/Longleaf/EventError.html +147 -0
data/docs/rdocs/Longleaf/EventNames.html +163 -0
data/docs/rdocs/Longleaf/EventStatusTracking.html +656 -0
data/docs/rdocs/Longleaf/FileCheckService.html +540 -0
data/docs/rdocs/Longleaf/FileHelpers.html +520 -0
data/docs/rdocs/Longleaf/FileRecord.html +716 -0
data/docs/rdocs/Longleaf/FileSelector.html +901 -0
data/docs/rdocs/Longleaf/FixityCheckService.html +691 -0
data/docs/rdocs/Longleaf/IndexManager.html +1155 -0
data/docs/rdocs/Longleaf/InvalidDigestAlgorithmError.html +143 -0
data/docs/rdocs/Longleaf/InvalidStoragePathError.html +143 -0
data/docs/rdocs/Longleaf/Logging.html +405 -0
data/docs/rdocs/Longleaf/Logging/RedirectingLogger.html +1213 -0
data/docs/rdocs/Longleaf/LongleafError.html +139 -0
data/docs/rdocs/Longleaf/MDFields.html +193 -0
data/docs/rdocs/Longleaf/MetadataBuilder.html +787 -0
data/docs/rdocs/Longleaf/MetadataDeserializer.html +537 -0
data/docs/rdocs/Longleaf/MetadataError.html +143 -0
data/docs/rdocs/Longleaf/MetadataPersistenceManager.html +539 -0
data/docs/rdocs/Longleaf/MetadataRecord.html +1411 -0
data/docs/rdocs/Longleaf/MetadataSerializer.html +786 -0
data/docs/rdocs/Longleaf/PreservationServiceError.html +147 -0
data/docs/rdocs/Longleaf/PreserveCommand.html +410 -0
data/docs/rdocs/Longleaf/PreserveEvent.html +491 -0
data/docs/rdocs/Longleaf/RegisterCommand.html +428 -0
data/docs/rdocs/Longleaf/RegisterEvent.html +628 -0
data/docs/rdocs/Longleaf/RegisteredFileSelector.html +446 -0
data/docs/rdocs/Longleaf/RegistrationError.html +151 -0
data/docs/rdocs/Longleaf/ReindexCommand.html +576 -0
data/docs/rdocs/Longleaf/RsyncReplicationService.html +1180 -0
data/docs/rdocs/Longleaf/SequelIndexDriver.html +1978 -0
data/docs/rdocs/Longleaf/ServiceCandidateFilesystemIterator.html +572 -0
data/docs/rdocs/Longleaf/ServiceCandidateIndexIterator.html +532 -0
data/docs/rdocs/Longleaf/ServiceCandidateLocator.html +333 -0
data/docs/rdocs/Longleaf/ServiceClassCache.html +725 -0
data/docs/rdocs/Longleaf/ServiceDateHelper.html +425 -0
data/docs/rdocs/Longleaf/ServiceDefinition.html +683 -0
data/docs/rdocs/Longleaf/ServiceDefinitionManager.html +371 -0
data/docs/rdocs/Longleaf/ServiceDefinitionValidator.html +269 -0
data/docs/rdocs/Longleaf/ServiceFields.html +173 -0
data/docs/rdocs/Longleaf/ServiceManager.html +1229 -0
data/docs/rdocs/Longleaf/ServiceMappingManager.html +410 -0
data/docs/rdocs/Longleaf/ServiceMappingValidator.html +347 -0
data/docs/rdocs/Longleaf/ServiceRecord.html +821 -0
data/docs/rdocs/Longleaf/StorageLocation.html +985 -0
data/docs/rdocs/Longleaf/StorageLocationManager.html +729 -0
data/docs/rdocs/Longleaf/StorageLocationUnavailableError.html +143 -0
data/docs/rdocs/Longleaf/StorageLocationValidator.html +373 -0
data/docs/rdocs/Longleaf/StoragePathValidator.html +253 -0
data/docs/rdocs/Longleaf/SystemConfigBuilder.html +441 -0
data/docs/rdocs/Longleaf/SystemConfigFields.html +163 -0
data/docs/rdocs/Longleaf/ValidateConfigCommand.html +451 -0
data/docs/rdocs/Longleaf/ValidateMetadataCommand.html +408 -0
data/docs/rdocs/_index.html +660 -0
data/docs/rdocs/class_list.html +51 -0
data/docs/rdocs/css/common.css +1 -0
data/docs/rdocs/css/full_list.css +58 -0
data/docs/rdocs/css/style.css +496 -0
data/docs/rdocs/file.README.html +165 -0
data/docs/rdocs/file_list.html +56 -0
data/docs/rdocs/frames.html +17 -0
data/docs/rdocs/index.html +165 -0
data/docs/rdocs/js/app.js +303 -0
data/docs/rdocs/js/full_list.js +216 -0
data/docs/rdocs/js/jquery.js +4 -0
data/docs/rdocs/method_list.html +2051 -0
data/docs/rdocs/top-level-namespace.html +110 -0
data/lib/longleaf/candidates/file_selector.rb +139 -0
data/lib/longleaf/candidates/manifest_digest_provider.rb +17 -0
data/lib/longleaf/candidates/registered_file_selector.rb +67 -0
data/lib/longleaf/candidates/service_candidate_filesystem_iterator.rb +93 -0
data/lib/longleaf/candidates/service_candidate_index_iterator.rb +84 -0
data/lib/longleaf/candidates/service_candidate_locator.rb +23 -0
data/lib/longleaf/candidates/single_digest_provider.rb +13 -0
data/lib/longleaf/cli.rb +237 -46
data/lib/longleaf/commands/deregister_command.rb +51 -0
data/lib/longleaf/commands/preserve_command.rb +50 -0
data/lib/longleaf/commands/register_command.rb +32 -43
data/lib/longleaf/commands/reindex_command.rb +92 -0
data/lib/longleaf/commands/validate_config_command.rb +33 -8
data/lib/longleaf/commands/validate_metadata_command.rb +51 -0
data/lib/longleaf/errors.rb +26 -7
data/lib/longleaf/events/deregister_event.rb +53 -0
data/lib/longleaf/events/event_names.rb +9 -0
data/lib/longleaf/events/event_status_tracking.rb +59 -0
data/lib/longleaf/events/preserve_event.rb +81 -0
data/lib/longleaf/events/register_event.rb +52 -51
data/lib/longleaf/helpers/case_insensitive_hash.rb +38 -0
data/lib/longleaf/helpers/digest_helper.rb +56 -0
data/lib/longleaf/helpers/s3_uri_helper.rb +86 -0
data/lib/longleaf/helpers/selection_options_parser.rb +189 -0
data/lib/longleaf/helpers/service_date_helper.rb +78 -0
data/lib/longleaf/indexing/index_manager.rb +101 -0
data/lib/longleaf/indexing/sequel_index_driver.rb +306 -0
data/lib/longleaf/logging.rb +5 -4
data/lib/longleaf/logging/redirecting_logger.rb +26 -25
data/lib/longleaf/models/app_fields.rb +7 -2
data/lib/longleaf/models/file_record.rb +17 -8
data/lib/longleaf/models/filesystem_metadata_location.rb +56 -0
data/lib/longleaf/models/filesystem_storage_location.rb +52 -0
data/lib/longleaf/models/md_fields.rb +2 -1
data/lib/longleaf/models/metadata_location.rb +47 -0
data/lib/longleaf/models/metadata_record.rb +39 -15
data/lib/longleaf/models/s3_storage_location.rb +133 -0
data/lib/longleaf/models/service_definition.rb +7 -6
data/lib/longleaf/models/service_fields.rb +7 -1
data/lib/longleaf/models/service_record.rb +10 -6
data/lib/longleaf/models/storage_location.rb +24 -19
data/lib/longleaf/models/storage_types.rb +9 -0
data/lib/longleaf/models/system_config_fields.rb +9 -0
data/lib/longleaf/preservation_services/file_check_service.rb +58 -0
data/lib/longleaf/preservation_services/fixity_check_service.rb +123 -0
data/lib/longleaf/preservation_services/rsync_replication_service.rb +182 -0
data/lib/longleaf/preservation_services/s3_replication_service.rb +143 -0
data/lib/longleaf/services/application_config_deserializer.rb +81 -24
data/lib/longleaf/services/application_config_manager.rb +20 -6
data/lib/longleaf/services/application_config_validator.rb +19 -9
data/lib/longleaf/services/configuration_validator.rb +67 -4
data/lib/longleaf/services/filesystem_location_validator.rb +16 -0
data/lib/longleaf/services/metadata_deserializer.rb +113 -42
data/lib/longleaf/services/metadata_persistence_manager.rb +47 -0
data/lib/longleaf/services/metadata_serializer.rb +138 -25
data/lib/longleaf/services/metadata_validator.rb +76 -0
data/lib/longleaf/services/s3_location_validator.rb +19 -0
data/lib/longleaf/services/service_class_cache.rb +112 -0
data/lib/longleaf/services/service_definition_manager.rb +10 -7
data/lib/longleaf/services/service_definition_validator.rb +25 -18
data/lib/longleaf/services/service_manager.rb +86 -11
data/lib/longleaf/services/service_mapping_manager.rb +13 -12
data/lib/longleaf/services/service_mapping_validator.rb +36 -26
data/lib/longleaf/services/storage_location_manager.rb +76 -15
data/lib/longleaf/services/storage_location_validator.rb +49 -35
data/lib/longleaf/specs/config_builder.rb +47 -23
data/lib/longleaf/specs/config_validator_helpers.rb +16 -0
data/lib/longleaf/specs/custom_matchers.rb +9 -0
data/lib/longleaf/specs/file_helpers.rb +61 -0
data/lib/longleaf/specs/metadata_builder.rb +92 -0
data/lib/longleaf/specs/system_config_builder.rb +27 -0
data/lib/longleaf/version.rb +1 -1
data/longleaf.gemspec +20 -7
data/mkdocs.yml +21 -0
metadata +306 -23
data/.travis.yml +0 -4
data/lib/longleaf/commands/abstract_command.rb +0 -37
data/lib/longleaf/services/storage_path_validator.rb +0 -16

data/docs/rdocs/top-level-namespace.html ADDED

@@ -0,0 +1,110 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+<title>
+  Top Level Namespace
+    &mdash; Documentation by YARD 0.9.19
+</title>
+  <link rel="stylesheet" href="css/style.css" type="text/css" charset="utf-8" />
+  <link rel="stylesheet" href="css/common.css" type="text/css" charset="utf-8" />
+<script type="text/javascript" charset="utf-8">
+  pathId = "";
+  relpath = '';
+</script>
+  <script type="text/javascript" charset="utf-8" src="js/jquery.js"></script>
+  <script type="text/javascript" charset="utf-8" src="js/app.js"></script>
+  </head>
+  <body>
+    <div class="nav_wrap">
+      <iframe id="nav" src="class_list.html?1"></iframe>
+      <div id="resizer"></div>
+    </div>
+    <div id="main" tabindex="-1">
+      <div id="header">
+        <div id="menu">
+    <a href="_index.html">Index</a> &raquo;
+    <span class="title">Top Level Namespace</span>
+</div>
+        <div id="search">
+    <a class="full_list_link" id="class_list_link"
+        href="class_list.html">
+        <svg width="24" height="24">
+          <rect x="0" y="4" width="24" height="4" rx="1" ry="1"></rect>
+          <rect x="0" y="12" width="24" height="4" rx="1" ry="1"></rect>
+          <rect x="0" y="20" width="24" height="4" rx="1" ry="1"></rect>
+        </svg>
+    </a>
+</div>
+        <div class="clear"></div>
+      </div>
+      <div id="content"><h1>Top Level Namespace
+</h1>
+<div class="box_info">
+</div>
+<h2>Defined Under Namespace</h2>
+<p class="children">
+      <strong class="modules">Modules:</strong> <span class='object_link'><a href="Longleaf.html" title="Longleaf (module)">Longleaf</a></span>
+</p>
+</div>
+      <div id="footer">
+  Generated on Tue May 28 15:48:00 2019 by
+  <a href="http://yardoc.org" title="Yay! A Ruby Documentation Tool" target="_parent">yard</a>
+  0.9.19 (ruby-2.6.3).
+</div>
+    </div>
+  </body>
+</html>

data/lib/longleaf/candidates/file_selector.rb ADDED

@@ -0,0 +1,139 @@
+require 'longleaf/logging'
+module Longleaf
+  # Selects and allows for iteration over files which match a provided set of selection criteria
+  class FileSelector
+    include Longleaf::Logging
+    SPECIFICITY_PATH = 'path'
+    SPECIFICITY_STORAGE_LOCATION = 'storage_location'
+    attr_reader :specificity
+    # May only provide either file_paths or storage_locations
+    def initialize(file_paths: nil, storage_locations: nil, app_config:)
+      if nil_or_empty?(file_paths) && nil_or_empty?(storage_locations)
+        raise ArgumentError.new("Must provide either file paths or storage locations")
+      end
+      if !nil_or_empty?(file_paths) && !nil_or_empty?(storage_locations)
+        raise ArgumentError.new("Cannot provide both file paths and storage locations")
+      end
+      @app_config = app_config
+      # The top level paths targeted by this selector
+      @target_paths = file_paths&.map do |path|
+        # Resolve relative paths against pwd
+        pathname = Pathname.new(path)
+        if !pathname.absolute?
+          path = File.join(Dir.pwd, path)
+        end
+        path = File.expand_path(path)
+        # adding trailing /'s to directories
+        if Dir.exist?(path) && !path.end_with?('/')
+          path + '/'
+        else
+          path
+        end
+      end
+      # The set of storage locations to select file paths from
+      @storage_locations = storage_locations
+      # Validate that the selected storage locations are known
+      if @storage_locations.nil?
+        @specificity = SPECIFICITY_PATH
+      else
+        @specificity = SPECIFICITY_STORAGE_LOCATION
+        locations = @app_config.location_manager.locations
+        @storage_locations.each do |loc_name|
+          unless locations.key?(loc_name)
+            raise StorageLocationUnavailableError.new("Cannot select unknown storage location #{loc_name}.")
+          end
+        end
+      end
+    end
+    # @return [Array] a list of top level paths from which files will be selected
+    def target_paths
+      # If starting from locations, initialize by expanding locations out to their actual paths
+      if @target_paths.nil? && !@storage_locations.nil?
+        @target_paths = Array.new
+        @storage_locations.each do |loc_name|
+          @target_paths << @app_config.location_manager.locations[loc_name].path
+        end
+      end
+      @target_paths
+    end
+    # Get the next file path for this selector.
+    # @return [String] an absolute path to the next file targeted by this selector,
+    # or nil if no more files selected
+    def next_path
+      if @paths.nil?
+        # Start the paths listing out from the targetted set of paths for this selector
+        # In reverse order since using a LIFO structure
+        @paths = target_paths.reverse
+      end
+      # No more paths to return
+      return nil if @paths&.empty?
+      # Get the most recently added path for depth first traversal of selected paths
+      path = @paths.pop
+      until path.nil? do
+        @app_config.location_manager.verify_path_in_location(path)
+        if File.exist?(path)
+          if File.directory?(path)
+            logger.debug("Expanding directory #{path}")
+            # For a directory, add all children to file_paths
+            Dir.entries(path).sort.reverse_each do |child|
+              @paths << File.join(path, child) unless child == '.' or child == '..'
+            end
+          else
+            logger.debug("Returning file #{path}")
+            return path
+          end
+        else
+          raise InvalidStoragePathError.new("File #{path} does not exist.")
+        end
+        # Returned path was not a suitable file, try the next path
+        path = @paths.pop
+      end
+    end
+    # Iterate through the file paths for this selector and execute the provided block with each.
+    # A block is required.
+    def each
+      file_path = next_path
+      until file_path.nil?
+        yield file_path
+        file_path = next_path
+      end
+    end
+    # return [Array] a list of all storage locations being targeted by this selector
+    def storage_locations
+      # Determine what storage_locations are represented by the given file paths
+      if @storage_locations.nil? && !@target_paths.nil?
+        loc_set = Set.new
+        @target_paths.each do |path|
+          loc = @app_config.location_manager.get_location_by_path(path)
+          loc_set.add(loc.name) unless loc.nil?
+        end
+        @storage_locations = loc_set.to_a
+      end
+      if @storage_locations.nil?
+        @storage_locations = Array.new
+      end
+      @storage_locations
+    end
+    private
+    def nil_or_empty?(value)
+      value.nil? || value.empty?
+    end
+  end
+end

data/lib/longleaf/candidates/manifest_digest_provider.rb ADDED

@@ -0,0 +1,17 @@
+module Longleaf
+  # Provides digests for files from a manifest
+  class ManifestDigestProvider
+    # @param hash which maps file paths to hashs of digests
+    def initialize(digests_mapping)
+      @digests_mapping = digests_mapping
+    end
+    # @param file_path [String] path of file
+    # @return hash containing all the manifested digests for the given path, or nil
+    def get_digests(file_path)
+      # return nil if key not found, in case the hash has default values
+      return nil unless @digests_mapping.key?(file_path)
+      @digests_mapping[file_path]
+    end
+  end
+end

data/lib/longleaf/candidates/registered_file_selector.rb ADDED

@@ -0,0 +1,67 @@
+require 'longleaf/candidates/file_selector'
+require 'longleaf/logging'
+module Longleaf
+  # Selects and allows for iteration over files which are registered and match a provided
+  # set of selection criteria
+  class RegisteredFileSelector < FileSelector
+    include Longleaf::Logging
+    # Get the next file path for this selector.
+    # @raise [InvalidStoragePathError] if any of the selected files do not exist
+    # @raise [StorageLocationUnavailableError] if any of the selected paths are not
+    #    in a registered storage location.
+    # @return [String] an absolute path to the next file targeted by this selector,
+    #    or nil if no more files selected
+    def next_path
+      if @md_paths.nil?
+        # Compute the starting paths by looking up the metadata paths for the provided targets,
+        # in reverse order since @md_paths is a LIFO stack structure.
+        @md_paths = target_paths.reverse_each.map do |file_path|
+          storage_loc = @app_config.location_manager.verify_path_in_location(file_path)
+          storage_loc.get_metadata_path_for(file_path)
+        end
+      end
+      # No more paths to return
+      return nil if @md_paths&.empty?
+      # Get the most recently added path for depth first traversal of selected paths
+      md_path = @md_paths.pop
+      until md_path.nil? do
+        if File.exist?(md_path)
+          if File.directory?(md_path)
+            logger.debug("Expanding metadata directory #{md_path}")
+            # For a directory, add all children to file_paths
+            Dir.entries(md_path).sort.reverse_each do |child|
+              @md_paths << File.join(md_path, child) unless child == '.' or child == '..'
+            end
+          elsif md_path.end_with?(MetadataSerializer::metadata_suffix)
+            # Convert metadata path to file path before returning
+            return file_path_for_metadata(md_path)
+          else
+            logger.debug("Skipping non-metadata file in metadata directory #{md_path}")
+          end
+        else
+          file_path = file_path_for_metadata(md_path)
+          if File.exist?(file_path)
+            raise RegistrationError.new("File #{file_path} is not registered.")
+          else
+            raise InvalidStoragePathError.new("File #{file_path} does not exist.")
+          end
+        end
+        # Returned path was not a suitable file, try the next path
+        md_path = @md_paths.pop
+      end
+    end
+    private
+    def file_path_for_metadata(md_path)
+      storage_loc = @app_config.location_manager.get_location_by_metadata_path(md_path)
+      file_path = storage_loc.get_path_from_metadata_path(md_path)
+      logger.debug("Returning next file #{file_path} for metadata path #{md_path}")
+      return file_path
+    end
+  end
+end

data/lib/longleaf/candidates/service_candidate_filesystem_iterator.rb ADDED

@@ -0,0 +1,93 @@
+require 'longleaf/services/service_manager'
+require 'longleaf/events/event_names'
+require 'longleaf/errors'
+require 'longleaf/logging'
+require 'time'
+module Longleaf
+  # Iterator for getting file candidates which have services which need to be run.
+  # Implementation uses metadata files directly from the filesystem for determinations
+  # about service status.
+  class ServiceCandidateFilesystemIterator
+    include Longleaf::Logging
+    def initialize(file_selector, event, app_config, force = false)
+      @file_selector = file_selector
+      @event = event
+      @app_config = app_config
+      @force = force
+    end
+    # Get the file record for the next candidate which needs services run which match the
+    # provided file_selector
+    # @return [FileRecord] file record of the next candidate with services needing to be run,
+    # or nil if there are no more candidates.
+    def next_candidate
+      loop do
+        next_path = @file_selector.next_path
+        return nil if next_path.nil?
+        logger.debug("Evaluating candidate #{next_path}")
+        storage_loc = @app_config.location_manager.get_location_by_path(next_path)
+        file_rec = FileRecord.new(next_path, storage_loc)
+        # Skip over unregistered files
+        if !file_rec.metadata_present?
+          logger.debug("Ignoring unregistered file #{next_path}")
+          next
+        end
+        @app_config.md_manager.load(file_rec)
+        # Return the file record if it needs any services run
+        return file_rec if needs_run?(file_rec)
+      end
+    end
+    # Iterate through the candidates in this object and execute the provided block with each
+    # candidate. A block is required.
+    def each
+      file_rec = next_candidate
+      until file_rec.nil?
+        yield file_rec
+        file_rec = next_candidate
+      end
+    end
+    private
+    # Returns true if the file record contains any services which need to be run
+    def needs_run?(file_rec)
+      md_rec = file_rec.metadata_record
+      storage_loc = file_rec.storage_location
+      service_manager = @app_config.service_manager
+      # File is not a valid candidate for services if it is deregistered, unless performing cleanup
+      if @event != EventNames::CLEANUP && md_rec.deregistered?
+        logger.debug("Skipping deregistered file: #{file_rec.path}")
+        return false
+      end
+      expected_services = service_manager.list_services(
+          location: storage_loc.name,
+          event: @event)
+      # When in force mode, candidate needs a run as long as there are any services configured for it.
+      if @force && !expected_services.empty?
+        logger.debug("Forcing run needed for file: #{file_rec.path}")
+        return true
+      end
+      expected_services.each do |service_name|
+        if service_manager.service_needed?(service_name, md_rec)
+          logger.debug("Service #{service_name} needed for file: #{file_rec.path}")
+          return true
+        end
+      end
+      logger.debug("Run not needed for file: #{file_rec.path}")
+      # No services needed to be run for this file
+      false
+    end
+  end
+end

data/lib/longleaf/candidates/service_candidate_index_iterator.rb ADDED

@@ -0,0 +1,84 @@
+require 'longleaf/events/event_names'
+require 'longleaf/errors'
+require 'longleaf/logging'
+require 'time'
+module Longleaf
+  # Iterator for getting file candidates which have services which need to be run.
+  # Implementation uses an index of file metadata to determine if the file needs any
+  # services run.
+  class ServiceCandidateIndexIterator
+    include Longleaf::Logging
+    def initialize(file_selector, event, app_config, force = false)
+      @file_selector = file_selector
+      @event = event
+      @app_config = app_config
+      @force = force
+      @index_manager = @app_config.index_manager
+      @stale_datetime = Time.now.utc
+      @result_set = nil
+    end
+    # Get the file record for the next candidate which needs services run which match the
+    # provided file_selector
+    # @return [FileRecord] file record of the next candidate with services needing to be run,
+    #    or nil if there are no more candidates.
+    def next_candidate
+      file_rec = nil
+      # loop until a candidate with metadata is retrieved
+      loop do
+        # Get the next page of results if the previous page has been processed
+        fetch_next_page if @result_set.nil? || @result_set.empty?
+        # Retrieve the next possible candidate path from the page
+        next_path = @result_set.shift
+        # given that the page was just retrieved, getting a nil path indicates that the retrieved page of
+        # candidates is empty and there are no more candidates to iterate at this time.
+        return nil if next_path.nil?
+        logger.debug("Retrieved candidate #{next_path}")
+        storage_loc = @app_config.location_manager.get_location_by_path(next_path)
+        file_rec = FileRecord.new(next_path, storage_loc)
+        # Keep seeking until a registered candidate is found, according to the file system.
+        if file_rec.metadata_present?
+          break
+        else
+          logger.warn("Encountered #{next_path} in index, but path is not registered. Clearing out of synch entry.")
+          @index_manager.remove(file_rec)
+        end
+      end
+      @app_config.md_manager.load(file_rec)
+      file_rec
+    end
+    # Iterate through the candidates in this object and execute the provided block with each
+    # candidate. A block is required.
+    def each
+      file_rec = next_candidate
+      until file_rec.nil?
+        yield file_rec
+        file_rec = next_candidate
+      end
+    end
+    private
+    def fetch_next_page
+      if @force
+        @result_set = @index_manager.registered_paths(@file_selector)
+      else
+        case @event
+        when EventNames::PRESERVE
+          @result_set = @index_manager.paths_with_stale_services(@file_selector, @stale_datetime)
+        when EventNames::CLEANUP
+          # TODO
+        end
+      end
+      logger.debug("Retrieve result set with #{@result_set&.length} entries")
+    end
+  end
+end