RubyGems - longleaf - Versions diffs - 0.1.0.pre.2 → 1.0.0 - Mend

longleaf 0.1.0.pre.2 → 1.0.0

Files changed (184) hide show

checksums.yaml +4 -4
data/.circleci/config.yml +94 -0
data/.editorconfig +13 -0
data/.gitignore +4 -1
data/.rubocop.yml +44 -0
data/.rubocop_todo.yml +834 -0
data/.yardopts +1 -0
data/Gemfile +16 -1
data/README.md +98 -12
data/Rakefile +6 -0
data/bin/setup +16 -1
data/docs/aboutlongleaf.md +28 -0
data/docs/extra.css +32 -0
data/docs/img/change-file.png +0 -0
data/docs/img/ll-example-preserved.png +0 -0
data/docs/index.md +19 -0
data/docs/install.md +66 -0
data/docs/ll-example/config-example-relative.yml +33 -0
data/docs/ll-example/files-dir/LLexample-PDF.pdf +0 -0
data/docs/ll-example/files-dir/LLexample-TOCHANGE.txt +15 -0
data/docs/ll-example/files-dir/LLexample-tokeep.txt +10 -0
data/docs/ll-example/metadata-dir/.gitkeep +0 -0
data/docs/ll-example/replica-files/.gitkeep +0 -0
data/docs/ll-example/replica-metadata/.gitkeep +0 -0
data/docs/quickstart.md +270 -0
data/docs/rdocs/Longleaf.html +135 -0
data/docs/rdocs/Longleaf/AppFields.html +178 -0
data/docs/rdocs/Longleaf/ApplicationConfigDeserializer.html +631 -0
data/docs/rdocs/Longleaf/ApplicationConfigManager.html +610 -0
data/docs/rdocs/Longleaf/ApplicationConfigValidator.html +238 -0
data/docs/rdocs/Longleaf/CLI.html +909 -0
data/docs/rdocs/Longleaf/ChecksumMismatchError.html +151 -0
data/docs/rdocs/Longleaf/ConfigBuilder.html +1339 -0
data/docs/rdocs/Longleaf/ConfigurationError.html +143 -0
data/docs/rdocs/Longleaf/ConfigurationValidator.html +227 -0
data/docs/rdocs/Longleaf/DeregisterCommand.html +420 -0
data/docs/rdocs/Longleaf/DeregisterEvent.html +453 -0
data/docs/rdocs/Longleaf/DeregistrationError.html +151 -0
data/docs/rdocs/Longleaf/DigestHelper.html +419 -0
data/docs/rdocs/Longleaf/EventError.html +147 -0
data/docs/rdocs/Longleaf/EventNames.html +163 -0
data/docs/rdocs/Longleaf/EventStatusTracking.html +656 -0
data/docs/rdocs/Longleaf/FileCheckService.html +540 -0
data/docs/rdocs/Longleaf/FileHelpers.html +520 -0
data/docs/rdocs/Longleaf/FileRecord.html +716 -0
data/docs/rdocs/Longleaf/FileSelector.html +901 -0
data/docs/rdocs/Longleaf/FixityCheckService.html +691 -0
data/docs/rdocs/Longleaf/IndexManager.html +1155 -0
data/docs/rdocs/Longleaf/InvalidDigestAlgorithmError.html +143 -0
data/docs/rdocs/Longleaf/InvalidStoragePathError.html +143 -0
data/docs/rdocs/Longleaf/Logging.html +405 -0
data/docs/rdocs/Longleaf/Logging/RedirectingLogger.html +1213 -0
data/docs/rdocs/Longleaf/LongleafError.html +139 -0
data/docs/rdocs/Longleaf/MDFields.html +193 -0
data/docs/rdocs/Longleaf/MetadataBuilder.html +787 -0
data/docs/rdocs/Longleaf/MetadataDeserializer.html +537 -0
data/docs/rdocs/Longleaf/MetadataError.html +143 -0
data/docs/rdocs/Longleaf/MetadataPersistenceManager.html +539 -0
data/docs/rdocs/Longleaf/MetadataRecord.html +1411 -0
data/docs/rdocs/Longleaf/MetadataSerializer.html +786 -0
data/docs/rdocs/Longleaf/PreservationServiceError.html +147 -0
data/docs/rdocs/Longleaf/PreserveCommand.html +410 -0
data/docs/rdocs/Longleaf/PreserveEvent.html +491 -0
data/docs/rdocs/Longleaf/RegisterCommand.html +428 -0
data/docs/rdocs/Longleaf/RegisterEvent.html +628 -0
data/docs/rdocs/Longleaf/RegisteredFileSelector.html +446 -0
data/docs/rdocs/Longleaf/RegistrationError.html +151 -0
data/docs/rdocs/Longleaf/ReindexCommand.html +576 -0
data/docs/rdocs/Longleaf/RsyncReplicationService.html +1180 -0
data/docs/rdocs/Longleaf/SequelIndexDriver.html +1978 -0
data/docs/rdocs/Longleaf/ServiceCandidateFilesystemIterator.html +572 -0
data/docs/rdocs/Longleaf/ServiceCandidateIndexIterator.html +532 -0
data/docs/rdocs/Longleaf/ServiceCandidateLocator.html +333 -0
data/docs/rdocs/Longleaf/ServiceClassCache.html +725 -0
data/docs/rdocs/Longleaf/ServiceDateHelper.html +425 -0
data/docs/rdocs/Longleaf/ServiceDefinition.html +683 -0
data/docs/rdocs/Longleaf/ServiceDefinitionManager.html +371 -0
data/docs/rdocs/Longleaf/ServiceDefinitionValidator.html +269 -0
data/docs/rdocs/Longleaf/ServiceFields.html +173 -0
data/docs/rdocs/Longleaf/ServiceManager.html +1229 -0
data/docs/rdocs/Longleaf/ServiceMappingManager.html +410 -0
data/docs/rdocs/Longleaf/ServiceMappingValidator.html +347 -0
data/docs/rdocs/Longleaf/ServiceRecord.html +821 -0
data/docs/rdocs/Longleaf/StorageLocation.html +985 -0
data/docs/rdocs/Longleaf/StorageLocationManager.html +729 -0
data/docs/rdocs/Longleaf/StorageLocationUnavailableError.html +143 -0
data/docs/rdocs/Longleaf/StorageLocationValidator.html +373 -0
data/docs/rdocs/Longleaf/StoragePathValidator.html +253 -0
data/docs/rdocs/Longleaf/SystemConfigBuilder.html +441 -0
data/docs/rdocs/Longleaf/SystemConfigFields.html +163 -0
data/docs/rdocs/Longleaf/ValidateConfigCommand.html +451 -0
data/docs/rdocs/Longleaf/ValidateMetadataCommand.html +408 -0
data/docs/rdocs/_index.html +660 -0
data/docs/rdocs/class_list.html +51 -0
data/docs/rdocs/css/common.css +1 -0
data/docs/rdocs/css/full_list.css +58 -0
data/docs/rdocs/css/style.css +496 -0
data/docs/rdocs/file.README.html +165 -0
data/docs/rdocs/file_list.html +56 -0
data/docs/rdocs/frames.html +17 -0
data/docs/rdocs/index.html +165 -0
data/docs/rdocs/js/app.js +303 -0
data/docs/rdocs/js/full_list.js +216 -0
data/docs/rdocs/js/jquery.js +4 -0
data/docs/rdocs/method_list.html +2051 -0
data/docs/rdocs/top-level-namespace.html +110 -0
data/lib/longleaf/candidates/file_selector.rb +139 -0
data/lib/longleaf/candidates/manifest_digest_provider.rb +17 -0
data/lib/longleaf/candidates/registered_file_selector.rb +67 -0
data/lib/longleaf/candidates/service_candidate_filesystem_iterator.rb +93 -0
data/lib/longleaf/candidates/service_candidate_index_iterator.rb +84 -0
data/lib/longleaf/candidates/service_candidate_locator.rb +23 -0
data/lib/longleaf/candidates/single_digest_provider.rb +13 -0
data/lib/longleaf/cli.rb +237 -46
data/lib/longleaf/commands/deregister_command.rb +51 -0
data/lib/longleaf/commands/preserve_command.rb +50 -0
data/lib/longleaf/commands/register_command.rb +32 -43
data/lib/longleaf/commands/reindex_command.rb +92 -0
data/lib/longleaf/commands/validate_config_command.rb +33 -8
data/lib/longleaf/commands/validate_metadata_command.rb +51 -0
data/lib/longleaf/errors.rb +26 -7
data/lib/longleaf/events/deregister_event.rb +53 -0
data/lib/longleaf/events/event_names.rb +9 -0
data/lib/longleaf/events/event_status_tracking.rb +59 -0
data/lib/longleaf/events/preserve_event.rb +81 -0
data/lib/longleaf/events/register_event.rb +52 -51
data/lib/longleaf/helpers/case_insensitive_hash.rb +38 -0
data/lib/longleaf/helpers/digest_helper.rb +56 -0
data/lib/longleaf/helpers/s3_uri_helper.rb +86 -0
data/lib/longleaf/helpers/selection_options_parser.rb +189 -0
data/lib/longleaf/helpers/service_date_helper.rb +78 -0
data/lib/longleaf/indexing/index_manager.rb +101 -0
data/lib/longleaf/indexing/sequel_index_driver.rb +306 -0
data/lib/longleaf/logging.rb +5 -4
data/lib/longleaf/logging/redirecting_logger.rb +26 -25
data/lib/longleaf/models/app_fields.rb +7 -2
data/lib/longleaf/models/file_record.rb +17 -8
data/lib/longleaf/models/filesystem_metadata_location.rb +56 -0
data/lib/longleaf/models/filesystem_storage_location.rb +52 -0
data/lib/longleaf/models/md_fields.rb +2 -1
data/lib/longleaf/models/metadata_location.rb +47 -0
data/lib/longleaf/models/metadata_record.rb +39 -15
data/lib/longleaf/models/s3_storage_location.rb +133 -0
data/lib/longleaf/models/service_definition.rb +7 -6
data/lib/longleaf/models/service_fields.rb +7 -1
data/lib/longleaf/models/service_record.rb +10 -6
data/lib/longleaf/models/storage_location.rb +24 -19
data/lib/longleaf/models/storage_types.rb +9 -0
data/lib/longleaf/models/system_config_fields.rb +9 -0
data/lib/longleaf/preservation_services/file_check_service.rb +58 -0
data/lib/longleaf/preservation_services/fixity_check_service.rb +123 -0
data/lib/longleaf/preservation_services/rsync_replication_service.rb +182 -0
data/lib/longleaf/preservation_services/s3_replication_service.rb +143 -0
data/lib/longleaf/services/application_config_deserializer.rb +81 -24
data/lib/longleaf/services/application_config_manager.rb +20 -6
data/lib/longleaf/services/application_config_validator.rb +19 -9
data/lib/longleaf/services/configuration_validator.rb +67 -4
data/lib/longleaf/services/filesystem_location_validator.rb +16 -0
data/lib/longleaf/services/metadata_deserializer.rb +113 -42
data/lib/longleaf/services/metadata_persistence_manager.rb +47 -0
data/lib/longleaf/services/metadata_serializer.rb +138 -25
data/lib/longleaf/services/metadata_validator.rb +76 -0
data/lib/longleaf/services/s3_location_validator.rb +19 -0
data/lib/longleaf/services/service_class_cache.rb +112 -0
data/lib/longleaf/services/service_definition_manager.rb +10 -7
data/lib/longleaf/services/service_definition_validator.rb +25 -18
data/lib/longleaf/services/service_manager.rb +86 -11
data/lib/longleaf/services/service_mapping_manager.rb +13 -12
data/lib/longleaf/services/service_mapping_validator.rb +36 -26
data/lib/longleaf/services/storage_location_manager.rb +76 -15
data/lib/longleaf/services/storage_location_validator.rb +49 -35
data/lib/longleaf/specs/config_builder.rb +47 -23
data/lib/longleaf/specs/config_validator_helpers.rb +16 -0
data/lib/longleaf/specs/custom_matchers.rb +9 -0
data/lib/longleaf/specs/file_helpers.rb +61 -0
data/lib/longleaf/specs/metadata_builder.rb +92 -0
data/lib/longleaf/specs/system_config_builder.rb +27 -0
data/lib/longleaf/version.rb +1 -1
data/longleaf.gemspec +20 -7
data/mkdocs.yml +21 -0
metadata +306 -23
data/.travis.yml +0 -4
data/lib/longleaf/commands/abstract_command.rb +0 -37
data/lib/longleaf/services/storage_path_validator.rb +0 -16

data/docs/rdocs/top-level-namespace.html ADDED

@@ -0,0 +1,110 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+<title>
+  Top Level Namespace
+    &mdash; Documentation by YARD 0.9.19
+</title>
+  <link rel="stylesheet" href="css/style.css" type="text/css" charset="utf-8" />
+  <link rel="stylesheet" href="css/common.css" type="text/css" charset="utf-8" />
+<script type="text/javascript" charset="utf-8">
+  pathId = "";
+  relpath = '';
+</script>
+  <script type="text/javascript" charset="utf-8" src="js/jquery.js"></script>
+  <script type="text/javascript" charset="utf-8" src="js/app.js"></script>
+  </head>
+  <body>
+    <div class="nav_wrap">
+      <iframe id="nav" src="class_list.html?1"></iframe>
+      <div id="resizer"></div>
+    </div>
+    <div id="main" tabindex="-1">
+      <div id="header">
+        <div id="menu">
+    <a href="_index.html">Index</a> &raquo;
+    <span class="title">Top Level Namespace</span>
+</div>
+        <div id="search">
+    <a class="full_list_link" id="class_list_link"
+        href="class_list.html">
+        <svg width="24" height="24">
+          <rect x="0" y="4" width="24" height="4" rx="1" ry="1"></rect>
+          <rect x="0" y="12" width="24" height="4" rx="1" ry="1"></rect>
+          <rect x="0" y="20" width="24" height="4" rx="1" ry="1"></rect>
+        </svg>
+    </a>
+</div>
+        <div class="clear"></div>
+      </div>
+      <div id="content"><h1>Top Level Namespace
+</h1>
+<div class="box_info">
+</div>
+<h2>Defined Under Namespace</h2>
+<p class="children">
+      <strong class="modules">Modules:</strong> <span class='object_link'><a href="Longleaf.html" title="Longleaf (module)">Longleaf</a></span>
+</p>
+</div>
+      <div id="footer">
+  Generated on Tue May 28 15:48:00 2019 by
+  <a href="http://yardoc.org" title="Yay! A Ruby Documentation Tool" target="_parent">yard</a>
+  0.9.19 (ruby-2.6.3).
+</div>
+    </div>
+  </body>
+</html>

data/lib/longleaf/candidates/file_selector.rb ADDED

@@ -0,0 +1,139 @@
+require 'longleaf/logging'
+module Longleaf
+  # Selects and allows for iteration over files which match a provided set of selection criteria
+  class FileSelector
+    include Longleaf::Logging
+    SPECIFICITY_PATH = 'path'
+    SPECIFICITY_STORAGE_LOCATION = 'storage_location'
+    attr_reader :specificity
+    # May only provide either file_paths or storage_locations
+    def initialize(file_paths: nil, storage_locations: nil, app_config:)
+      if nil_or_empty?(file_paths) && nil_or_empty?(storage_locations)
+        raise ArgumentError.new("Must provide either file paths or storage locations")
+      end
+      if !nil_or_empty?(file_paths) && !nil_or_empty?(storage_locations)
+        raise ArgumentError.new("Cannot provide both file paths and storage locations")
+      end
+      @app_config = app_config
+      # The top level paths targeted by this selector
+      @target_paths = file_paths&.map do |path|
+        # Resolve relative paths against pwd
+        pathname = Pathname.new(path)
+        if !pathname.absolute?
+          path = File.join(Dir.pwd, path)
+        end
+        path = File.expand_path(path)
+        # adding trailing /'s to directories
+        if Dir.exist?(path) && !path.end_with?('/')
+          path + '/'
+        else
+          path
+        end
+      end
+      # The set of storage locations to select file paths from
+      @storage_locations = storage_locations
+      # Validate that the selected storage locations are known
+      if @storage_locations.nil?
+        @specificity = SPECIFICITY_PATH
+      else
+        @specificity = SPECIFICITY_STORAGE_LOCATION
+        locations = @app_config.location_manager.locations
+        @storage_locations.each do |loc_name|
+          unless locations.key?(loc_name)
+            raise StorageLocationUnavailableError.new("Cannot select unknown storage location #{loc_name}.")
+          end
+        end
+      end
+    end
+    # @return [Array] a list of top level paths from which files will be selected
+    def target_paths
+      # If starting from locations, initialize by expanding locations out to their actual paths
+      if @target_paths.nil? && !@storage_locations.nil?
+        @target_paths = Array.new
+        @storage_locations.each do |loc_name|
+          @target_paths << @app_config.location_manager.locations[loc_name].path
+        end
+      end
+      @target_paths
+    end
+    # Get the next file path for this selector.
+    # @return [String] an absolute path to the next file targeted by this selector,
+    # or nil if no more files selected
+    def next_path
+      if @paths.nil?
+        # Start the paths listing out from the targetted set of paths for this selector
+        # In reverse order since using a LIFO structure
+        @paths = target_paths.reverse
+      end
+      # No more paths to return
+      return nil if @paths&.empty?
+      # Get the most recently added path for depth first traversal of selected paths
+      path = @paths.pop
+      until path.nil? do
+        @app_config.location_manager.verify_path_in_location(path)
+        if File.exist?(path)
+          if File.directory?(path)
+            logger.debug("Expanding directory #{path}")
+            # For a directory, add all children to file_paths
+            Dir.entries(path).sort.reverse_each do |child|
+              @paths << File.join(path, child) unless child == '.' or child == '..'
+            end
+          else
+            logger.debug("Returning file #{path}")
+            return path
+          end
+        else
+          raise InvalidStoragePathError.new("File #{path} does not exist.")
+        end
+        # Returned path was not a suitable file, try the next path
+        path = @paths.pop
+      end
+    end
+    # Iterate through the file paths for this selector and execute the provided block with each.
+    # A block is required.
+    def each
+      file_path = next_path
+      until file_path.nil?
+        yield file_path
+        file_path = next_path
+      end
+    end
+    # return [Array] a list of all storage locations being targeted by this selector
+    def storage_locations
+      # Determine what storage_locations are represented by the given file paths
+      if @storage_locations.nil? && !@target_paths.nil?
+        loc_set = Set.new
+        @target_paths.each do |path|
+          loc = @app_config.location_manager.get_location_by_path(path)
+          loc_set.add(loc.name) unless loc.nil?
+        end
+        @storage_locations = loc_set.to_a
+      end
+      if @storage_locations.nil?
+        @storage_locations = Array.new
+      end
+      @storage_locations
+    end
+    private
+    def nil_or_empty?(value)
+      value.nil? || value.empty?
+    end
+  end
+end

data/lib/longleaf/candidates/manifest_digest_provider.rb ADDED

@@ -0,0 +1,17 @@
+module Longleaf
+  # Provides digests for files from a manifest
+  class ManifestDigestProvider
+    # @param hash which maps file paths to hashs of digests
+    def initialize(digests_mapping)
+      @digests_mapping = digests_mapping
+    end
+    # @param file_path [String] path of file
+    # @return hash containing all the manifested digests for the given path, or nil
+    def get_digests(file_path)
+      # return nil if key not found, in case the hash has default values
+      return nil unless @digests_mapping.key?(file_path)
+      @digests_mapping[file_path]
+    end
+  end
+end

data/lib/longleaf/candidates/registered_file_selector.rb ADDED

@@ -0,0 +1,67 @@
+require 'longleaf/candidates/file_selector'
+require 'longleaf/logging'
+module Longleaf
+  # Selects and allows for iteration over files which are registered and match a provided
+  # set of selection criteria
+  class RegisteredFileSelector < FileSelector
+    include Longleaf::Logging
+    # Get the next file path for this selector.
+    # @raise [InvalidStoragePathError] if any of the selected files do not exist
+    # @raise [StorageLocationUnavailableError] if any of the selected paths are not
+    #    in a registered storage location.
+    # @return [String] an absolute path to the next file targeted by this selector,
+    #    or nil if no more files selected
+    def next_path
+      if @md_paths.nil?
+        # Compute the starting paths by looking up the metadata paths for the provided targets,
+        # in reverse order since @md_paths is a LIFO stack structure.
+        @md_paths = target_paths.reverse_each.map do |file_path|
+          storage_loc = @app_config.location_manager.verify_path_in_location(file_path)
+          storage_loc.get_metadata_path_for(file_path)
+        end
+      end
+      # No more paths to return
+      return nil if @md_paths&.empty?
+      # Get the most recently added path for depth first traversal of selected paths
+      md_path = @md_paths.pop
+      until md_path.nil? do
+        if File.exist?(md_path)
+          if File.directory?(md_path)
+            logger.debug("Expanding metadata directory #{md_path}")
+            # For a directory, add all children to file_paths
+            Dir.entries(md_path).sort.reverse_each do |child|
+              @md_paths << File.join(md_path, child) unless child == '.' or child == '..'
+            end
+          elsif md_path.end_with?(MetadataSerializer::metadata_suffix)
+            # Convert metadata path to file path before returning
+            return file_path_for_metadata(md_path)
+          else
+            logger.debug("Skipping non-metadata file in metadata directory #{md_path}")
+          end
+        else
+          file_path = file_path_for_metadata(md_path)
+          if File.exist?(file_path)
+            raise RegistrationError.new("File #{file_path} is not registered.")
+          else
+            raise InvalidStoragePathError.new("File #{file_path} does not exist.")
+          end
+        end
+        # Returned path was not a suitable file, try the next path
+        md_path = @md_paths.pop
+      end
+    end
+    private
+    def file_path_for_metadata(md_path)
+      storage_loc = @app_config.location_manager.get_location_by_metadata_path(md_path)
+      file_path = storage_loc.get_path_from_metadata_path(md_path)
+      logger.debug("Returning next file #{file_path} for metadata path #{md_path}")
+      return file_path
+    end
+  end
+end

data/lib/longleaf/candidates/service_candidate_filesystem_iterator.rb ADDED

@@ -0,0 +1,93 @@
+require 'longleaf/services/service_manager'
+require 'longleaf/events/event_names'
+require 'longleaf/errors'
+require 'longleaf/logging'
+require 'time'
+module Longleaf
+  # Iterator for getting file candidates which have services which need to be run.
+  # Implementation uses metadata files directly from the filesystem for determinations
+  # about service status.
+  class ServiceCandidateFilesystemIterator
+    include Longleaf::Logging
+    def initialize(file_selector, event, app_config, force = false)
+      @file_selector = file_selector
+      @event = event
+      @app_config = app_config
+      @force = force
+    end
+    # Get the file record for the next candidate which needs services run which match the
+    # provided file_selector
+    # @return [FileRecord] file record of the next candidate with services needing to be run,
+    # or nil if there are no more candidates.
+    def next_candidate
+      loop do
+        next_path = @file_selector.next_path
+        return nil if next_path.nil?
+        logger.debug("Evaluating candidate #{next_path}")
+        storage_loc = @app_config.location_manager.get_location_by_path(next_path)
+        file_rec = FileRecord.new(next_path, storage_loc)
+        # Skip over unregistered files
+        if !file_rec.metadata_present?
+          logger.debug("Ignoring unregistered file #{next_path}")
+          next
+        end
+        @app_config.md_manager.load(file_rec)
+        # Return the file record if it needs any services run
+        return file_rec if needs_run?(file_rec)
+      end
+    end
+    # Iterate through the candidates in this object and execute the provided block with each
+    # candidate. A block is required.
+    def each
+      file_rec = next_candidate
+      until file_rec.nil?
+        yield file_rec
+        file_rec = next_candidate
+      end
+    end
+    private
+    # Returns true if the file record contains any services which need to be run
+    def needs_run?(file_rec)
+      md_rec = file_rec.metadata_record
+      storage_loc = file_rec.storage_location
+      service_manager = @app_config.service_manager
+      # File is not a valid candidate for services if it is deregistered, unless performing cleanup
+      if @event != EventNames::CLEANUP && md_rec.deregistered?
+        logger.debug("Skipping deregistered file: #{file_rec.path}")
+        return false
+      end
+      expected_services = service_manager.list_services(
+          location: storage_loc.name,
+          event: @event)
+      # When in force mode, candidate needs a run as long as there are any services configured for it.
+      if @force && !expected_services.empty?
+        logger.debug("Forcing run needed for file: #{file_rec.path}")
+        return true
+      end
+      expected_services.each do |service_name|
+        if service_manager.service_needed?(service_name, md_rec)
+          logger.debug("Service #{service_name} needed for file: #{file_rec.path}")
+          return true
+        end
+      end
+      logger.debug("Run not needed for file: #{file_rec.path}")
+      # No services needed to be run for this file
+      false
+    end
+  end
+end

data/lib/longleaf/candidates/service_candidate_index_iterator.rb ADDED

@@ -0,0 +1,84 @@
+require 'longleaf/events/event_names'
+require 'longleaf/errors'
+require 'longleaf/logging'
+require 'time'
+module Longleaf
+  # Iterator for getting file candidates which have services which need to be run.
+  # Implementation uses an index of file metadata to determine if the file needs any
+  # services run.
+  class ServiceCandidateIndexIterator
+    include Longleaf::Logging
+    def initialize(file_selector, event, app_config, force = false)
+      @file_selector = file_selector
+      @event = event
+      @app_config = app_config
+      @force = force
+      @index_manager = @app_config.index_manager
+      @stale_datetime = Time.now.utc
+      @result_set = nil
+    end
+    # Get the file record for the next candidate which needs services run which match the
+    # provided file_selector
+    # @return [FileRecord] file record of the next candidate with services needing to be run,
+    #    or nil if there are no more candidates.
+    def next_candidate
+      file_rec = nil
+      # loop until a candidate with metadata is retrieved
+      loop do
+        # Get the next page of results if the previous page has been processed
+        fetch_next_page if @result_set.nil? || @result_set.empty?
+        # Retrieve the next possible candidate path from the page
+        next_path = @result_set.shift
+        # given that the page was just retrieved, getting a nil path indicates that the retrieved page of
+        # candidates is empty and there are no more candidates to iterate at this time.
+        return nil if next_path.nil?
+        logger.debug("Retrieved candidate #{next_path}")
+        storage_loc = @app_config.location_manager.get_location_by_path(next_path)
+        file_rec = FileRecord.new(next_path, storage_loc)
+        # Keep seeking until a registered candidate is found, according to the file system.
+        if file_rec.metadata_present?
+          break
+        else
+          logger.warn("Encountered #{next_path} in index, but path is not registered. Clearing out of synch entry.")
+          @index_manager.remove(file_rec)
+        end
+      end
+      @app_config.md_manager.load(file_rec)
+      file_rec
+    end
+    # Iterate through the candidates in this object and execute the provided block with each
+    # candidate. A block is required.
+    def each
+      file_rec = next_candidate
+      until file_rec.nil?
+        yield file_rec
+        file_rec = next_candidate
+      end
+    end
+    private
+    def fetch_next_page
+      if @force
+        @result_set = @index_manager.registered_paths(@file_selector)
+      else
+        case @event
+        when EventNames::PRESERVE
+          @result_set = @index_manager.paths_with_stale_services(@file_selector, @stale_datetime)
+        when EventNames::CLEANUP
+          # TODO
+        end
+      end
+      logger.debug("Retrieve result set with #{@result_set&.length} entries")
+    end
+  end
+end