longleaf 0.3.0 → 1.0.0

data/lib/longleaf/helpers/selection_options_parser.rb

@@ -0,0 +1,189 @@
+require 'longleaf/candidates/file_selector'
+require 'longleaf/candidates/registered_file_selector'
+require 'longleaf/candidates/manifest_digest_provider'
+require 'longleaf/candidates/single_digest_provider'
+
+module Longleaf
+  # Helper for parsing manifest inputs used for registration
+  class SelectionOptionsParser
+    extend Longleaf::Logging
+
+    # Parses the provided options to construct a file selector and digest provider for
+    # use in registration commands.
+    # @param options [Hash] command options
+    # @param app_config_manager [ApplicationConfigManager] app config manager
+    # @return The file selector and digest provider.
+    def self.parse_registration_selection_options(options, app_config_manager)
+      there_can_be_only_one("Only one of the following selection options may be provided: -m, -f, -s",
+          options, :file, :manifest, :location)
+
+      if !options[:manifest].nil?
+        digests_mapping = self.manifests_to_digest_mapping(options[:manifest])
+        selector = FileSelector.new(file_paths: digests_mapping.keys, app_config: app_config_manager)
+        digest_provider = ManifestDigestProvider.new(digests_mapping)
+      elsif !options[:file].nil?
+        if options[:checksums]
+          checksums = options[:checksums]
+          # validate checksum list format, must a comma delimited list of prefix:checksums
+          if /^[^:,]+:[^:,]+(,[^:,]+:[^:,]+)*$/.match(checksums)
+            # convert checksum list into hash with prefix as key
+            checksums = Hash[*checksums.split(/\s*[:,]\s*/)]
+            digest_provider = SingleDigestProvider.new(checksums)
+          else
+            logger.failure("Invalid checksums parameter format, see `longleaf help <command>` for more information")
+            exit 1
+          end
+        end
+
+        file_paths = options[:file].split(/\s*,\s*/)
+        selector = FileSelector.new(file_paths: file_paths, app_config: app_config_manager)
+      elsif !options[:location].nil?
+        storage_locations = options[:location].split(/\s*,\s*/)
+        selector = FileSelector.new(storage_locations: storage_locations, app_config: app_config_manager)
+        digest_provider = SingleDigestProvider.new(nil)
+      else
+        logger.failure("Must provide one of the following file selection options: -f, l, or -m")
+        exit 1
+      end
+
+      [selector, digest_provider]
+    end
+
+    def self.there_can_be_only_one(failure_msg, options, *names)
+      got_one = false
+      names.each do |name|
+        if !options[name].nil?
+          if got_one
+            logger.failure(failure_msg)
+            exit 1
+          end
+          got_one = true
+        end
+      end
+    end
+
+    # Parses the provided manifest options, reading the contents of the manifests to produce
+    # a mapping from files to one or more algorithms.
+    # @param manifest_vals [Array] List of manifest option values. They may be in one of the following formats:
+    #       <alg_name>:<manifest_path> OR <alg_name>:@-
+    #.      <manifest_path> OR @-
+    # @return a hash containing the aggregated contents of the provided manifests. The keys are
+    #    paths to manifested files. The values are hashes, mapping digest algorithms to digest values.
+    def self.manifests_to_digest_mapping(manifest_vals)
+      alg_manifest_pairs = []
+      # interpret option inputs into a list of algorithms to manifest sources
+      manifest_vals.each do |manifest_val|
+        if manifest_val.include?(':')
+          manifest_parts = manifest_val.split(':', 2)
+          alg_manifest_pairs << manifest_parts
+        else
+          # algorithm not specified in option value
+          alg_manifest_pairs << [nil, manifest_val]
+        end
+      end
+      if alg_manifest_pairs.select { |mpair| mpair[1] == '@-' }.count > 1
+        self.fail("Cannot specify more than one manifest from STDIN")
+      end
+
+      # read the provided manifests to build a mapping from file uri to all supplied digests
+      digests_mapping = Hash.new { |h,k| h[k] = Hash.new }
+      alg_manifest_pairs.each do |mpair|
+        source_stream = nil
+        # Determine if reading from a manifest file or stdin
+        if mpair[1] == '@-'
+          source_stream = $stdin
+        else
+          source_stream = File.new(mpair[1])
+        end
+
+        current_alg = mpair[0]
+        multi_digest_manifest = current_alg.nil?
+        source_stream.each_line do |line|
+          line = line.strip
+          if multi_digest_manifest && /^[a-zA-Z0-9]+:$/ =~ line
+            # Found a digest algorithm header, assuming succeeding entries are of this type
+            current_alg = line.chomp(':')
+            # Verify that the digest algorithm is known to longleaf
+            if !DigestHelper.is_known_algorithm?(current_alg)
+              self.fail("Manifest specifies unknown digest algorithm: #{current_alg}")
+            end
+          else
+            if current_alg.nil?
+              self.fail("Manifest with unknown checksums encountered, an algorithm must be specified")
+            end
+            entry_parts = line.split(' ', 2)
+            if entry_parts.length != 2
+              self.fail("Invalid manifest entry: #{line}")
+            end
+
+            digests_mapping[entry_parts[1]][current_alg] = entry_parts[0]
+          end
+        end
+      end
+
+      digests_mapping
+    end
+
+    # Parses the provided options to create a selector for registered files
+    # @param options [Hash] command options
+    # @param app_config_manager [ApplicationConfigManager] app config manager
+    # @return selector
+    def self.create_registered_selector(options, app_config_manager)
+      there_can_be_only_one("Only one of the following selection options may be provided: -l, -f, -s",
+          options, :file, :location, :from_list)
+          
+      if !options[:from_list].nil?
+        file_paths = read_from_list(options[:from_list])
+        return RegisteredFileSelector.new(file_paths: file_paths, app_config: app_config_manager)
+      elsif !options[:file].nil?
+        file_paths = options[:file].split(/\s*,\s*/)
+        return RegisteredFileSelector.new(file_paths: file_paths, app_config: app_config_manager)
+      elsif !options[:location].nil?
+        storage_locations = options[:location].split(/\s*,\s*/)
+        return RegisteredFileSelector.new(storage_locations: storage_locations, app_config: app_config_manager)
+      else
+        logger.failure("Must provide one of the following file selection options: -l, -f, or -s")
+        exit 1
+      end
+    end
+    
+    # Parses the -l from_list option, reading the list of files specified either from the provided
+    # file path or STDIN
+    # @param from_list option value, either a file path or "@-"
+    # @return list of files from the from_list
+    def self.read_from_list(from_list)
+      from_list = from_list.strip
+      if from_list.empty?
+        logger.failure("List parameter must not be empty")
+        exit 1
+      end
+      
+      if from_list == '@-'
+        source_stream = $stdin
+      else
+        begin
+          source_stream = File.new(from_list)
+        rescue Errno::ENOENT
+          logger.failure("Specified list file does not exist: #{from_list}")
+          exit 1
+        end
+      end
+      
+      lines = []
+      source_stream.each_line do |line|
+        lines << line.strip
+      end
+      
+      if lines.empty?
+        logger.failure("File list is empty, must provide one or more files for this operation")
+        exit 1
+      end
+      lines
+    end
+
+    def self.fail(message)
+      logger.failure(message)
+      exit 1
+    end
+  end
+end

data/lib/longleaf/helpers/service_date_helper.rb

@@ -37,7 +37,7 @@ module Longleaf
       end
 
       modified_time = datetime + (value * unit_modifier)
-      modified_time.iso8601
+      modified_time.iso8601(3)
     end
 
     # Get a timestamp in the format expected for service timestamps.
@@ -46,5 +46,33 @@ module Longleaf
     def self.formatted_timestamp(timestamp = Time.now)
       timestamp.utc.iso8601(3).to_s
     end
+
+    # Get the timestamp for the next time the provided service would need to be run
+    # for the object described by md_rec
+    # @param md_rec [MetadataRecord] metadata record for the file
+    # @param service_def [ServiceDefinition] definition for the service
+    # @return [String] iso8601 timestamp for the next time the service will need to run, or
+    #    nil if the service does not need to run again.
+    def self.next_run_needed(md_rec, service_def)
+      raise ArgumentError.new('Must provide a md_rec parameter') if md_rec.nil?
+      raise ArgumentError.new('Must provide a service_def parameter') if service_def.nil?
+
+      service_name = service_def.name
+      service_rec = md_rec.service(service_name)
+
+      if service_rec.nil? || service_rec.timestamp.nil?
+        if service_def.delay.nil?
+          return md_rec.registered
+        else
+          return ServiceDateHelper.add_to_timestamp(md_rec.registered, service_def.delay)
+        end
+      end
+
+      if service_def.frequency.nil?
+        return nil
+      else
+        return ServiceDateHelper.add_to_timestamp(service_rec.timestamp, service_def.frequency)
+      end
+    end
   end
 end

data/lib/longleaf/indexing/sequel_index_driver.rb

@@ -114,27 +114,9 @@ module Longleaf
 
       expected_services.each do |service_def|
         service_name = service_def.name
-        # Service has never run, set execution time to now
-        if !present_services.include?(service_name)
-          service_times << current_time
-          next
-        end
-
-        service_rec = md_rec.service(service_name)
 
-        # Service either needs a run or has no timestamp, so execution time of now
-        if service_rec.run_needed || service_rec.timestamp.nil?
-          service_times << current_time
-          next
-        end
-
-        # Calculate the next time this service should run based on frequency
-        frequency = service_def.frequency
-        unless frequency.nil?
-          service_timestamp = service_rec.timestamp
-          service_times << ServiceDateHelper.add_to_timestamp(service_timestamp, frequency)
-          next
-        end
+        next_run = ServiceDateHelper.next_run_needed(md_rec, service_def)
+        service_times << next_run unless next_run.nil?
       end
       # Return the lowest service execution time
       service_times.min

data/lib/longleaf/models/app_fields.rb

@@ -7,7 +7,9 @@ module Longleaf
     SYSTEM = 'system'
 
     LOCATION_PATH = 'path'
-    METADATA_PATH = 'metadata_path'
-    METADATA_DIGESTS = 'metadata_digests'
+    METADATA_CONFIG = 'metadata'
+    METADATA_DIGESTS = 'digests'
+
+    STORAGE_TYPE = 'type'
   end
 end

data/lib/longleaf/models/filesystem_metadata_location.rb

@@ -0,0 +1,56 @@
+require 'longleaf/services/metadata_serializer'
+require 'longleaf/models/metadata_location'
+require 'longleaf/models/storage_types'
+
+module Longleaf
+  # A filesystem based location in which metadata associated with registered files is stored.
+  class FilesystemMetadataLocation < MetadataLocation
+    AF ||= Longleaf::AppFields
+
+    def initialize(config)
+      super(config)
+    end
+
+    # @return the storage type for this location
+    def type
+      StorageTypes::FILESYSTEM_STORAGE_TYPE
+    end
+
+    # Get the absolute path for the metadata file for the given file path located in this storage location.
+    # @param file_path [String] path of the file relative its storage location
+    # @return absolute path to the metadata
+    # @raise [ArgumentError] if the file_path is not provided.
+    def metadata_path_for(file_path)
+      raise ArgumentError.new("A file_path parameter is required") if file_path.nil?
+      raise ArgumentError.new("File path must be relative") if Pathname.new(file_path).absolute?
+
+      md_path = File.join(@path, file_path)
+      # If the file_path is to a file, then add metadata suffix.
+      if md_path.end_with?('/')
+        md_path
+      else
+        md_path + MetadataSerializer::metadata_suffix
+      end
+    end
+
+    # Get the metadata path relative to this location
+    # @param md_path [String] metadata file path
+    # @return the metadata path relative to this location
+    # @raise [ArgumentError] if the metadata path is not contained by this location
+    def relativize(md_path)
+      return md_path if Pathname.new(md_path).relative?
+
+      raise ArgumentError.new("Metadata path must be contained by this location") if !md_path.start_with?(@path)
+
+      md_path.sub(@path, "")
+    end
+
+
+    # Checks that the path defined in this metadata location are available
+    # @raise [StorageLocationUnavailableError] if the metadata location is not available
+    def available?
+      raise StorageLocationUnavailableError.new("Metadata path does not exist or is not a directory: #{@path}")\
+          unless Dir.exist?(@path)
+    end
+  end
+end

data/lib/longleaf/models/filesystem_storage_location.rb

@@ -0,0 +1,52 @@
+require 'longleaf/models/storage_location'
+require 'longleaf/models/storage_types'
+
+module Longleaf
+  # A storage location in a local filesystem
+  class FilesystemStorageLocation < StorageLocation
+    # @param name [String] the name of this storage location
+    # @param config [Hash] hash containing the configuration options for this location
+    # @param md_loc [MetadataLocation] metadata location associated with this storage location
+    def initialize(name, config, md_loc)
+      super(name, config, md_loc)
+      @path += File::SEPARATOR unless @path.end_with?(File::SEPARATOR)
+    end
+
+    # @return the storage type for this location
+    def type
+      StorageTypes::FILESYSTEM_STORAGE_TYPE
+    end
+
+    # Get that absolute path to the file associated with the provided metadata path
+    # @param md_path [String] metadata file path
+    # @raise [ArgumentError] if the md_path is not in this storage location
+    # @return [String] the path for the file associated with this metadata
+    def get_path_from_metadata_path(md_path)
+      raise ArgumentError.new("A file_path parameter is required") if md_path.nil? || md_path.empty?
+
+      rel_path = @metadata_location.relative_file_path_for(md_path)
+
+      File.join(@path, rel_path)
+    end
+
+    # Checks that the path and metadata path defined in this location are available
+    # @raise [StorageLocationUnavailableError] if the storage location is not available
+    def available?
+      raise StorageLocationUnavailableError.new("Path does not exist or is not a directory: #{@path}")\
+          unless Dir.exist?(@path)
+      @metadata_location.available?
+    end
+
+    # Get the file path relative to this location
+    # @param file_path [String] file path
+    # @return the file path relative to this location
+    # @raise [ArgumentError] if the file path is not contained by this location
+    def relativize(file_path)
+      return file_path if Pathname.new(file_path).relative?
+
+      raise ArgumentError.new("Metadata path must be contained by this location") if !file_path.start_with?(@path)
+
+      file_path.sub(@path, "")
+    end
+  end
+end

data/lib/longleaf/models/metadata_location.rb

@@ -0,0 +1,47 @@
+require 'longleaf/models/app_fields'
+
+module Longleaf
+  # A location in which metadata associated with registered files is stored.
+  class MetadataLocation
+    AF ||= Longleaf::AppFields
+
+    attr_reader :path
+    attr_reader :digests
+
+    def initialize(config)
+      raise ArgumentError.new("Config parameter is required") unless config
+      @path = config[AF::LOCATION_PATH]
+      raise ArgumentError.new("Parameter path is required") unless @path
+      @path += '/' unless @path.end_with?('/')
+
+      digests = config[AF::METADATA_DIGESTS]
+      if digests.nil?
+        @digests = []
+      elsif digests.is_a?(String)
+        @digests = [digests.downcase]
+      else
+        @digests = digests.map(&:downcase)
+      end
+      DigestHelper::validate_algorithms(@digests)
+    end
+
+    # Transforms the given metadata path into a relative storage location path
+    # @param md_path [String] path of the metadata file or directory to compute file path for.
+    # @return
+    def relative_file_path_for(md_path)
+      rel_md_path = relativize(md_path)
+
+      if rel_md_path.end_with?(MetadataSerializer::metadata_suffix)
+        rel_md_path[0..-MetadataSerializer::metadata_suffix.length - 1]
+      else
+        rel_md_path
+      end
+    end
+
+    # @param [String] metadata path to check
+    # @return true if the metadata path is contained by the path for this location
+    def contains?(md_path)
+      md_path.start_with?(@path)
+    end
+  end
+end

data/lib/longleaf/models/metadata_record.rb

@@ -1,5 +1,6 @@
 require_relative 'md_fields'
 require_relative 'service_record'
+require 'longleaf/helpers/case_insensitive_hash'
 
 module Longleaf
   # Metadata record for a single file
@@ -22,7 +23,8 @@ module Longleaf
       @properties = properties || Hash.new
       @registered = registered
       @deregistered = deregistered
-      @checksums = checksums || Hash.new
+      @checksums = CaseInsensitiveHash.new
+      @checksums.merge!(checksums) unless checksums.nil?
       @services = services || Hash.new
       @file_size = file_size
       @last_modified = last_modified