longleaf 0.1.0.pre.2 → 1.0.0

data/lib/longleaf/events/preserve_event.rb

@@ -0,0 +1,81 @@
+require 'longleaf/services/service_manager'
+require 'longleaf/events/event_names'
+require 'longleaf/events/event_status_tracking'
+require 'longleaf/logging'
+
+module Longleaf
+  # Verify event for a single file
+  class PreserveEvent
+    include Longleaf::Logging
+    include Longleaf::EventStatusTracking
+
+    # @param file_rec [FileRecord] file record
+    # @param app_manager [ApplicationConfigManager] the application configuration
+    # @param force [boolean] if true, then services run regardless of whether they are flagged as needed
+    def initialize(file_rec:, app_manager:, force: false)
+      raise ArgumentError.new('Must provide a file_rec parameter') if file_rec.nil?
+      raise ArgumentError.new('Must provide an ApplicationConfigManager') if app_manager.nil?
+
+      @app_manager = app_manager
+      @file_rec = file_rec
+      @force = force
+    end
+
+    # Perform a preserve event on the given file, updating its metadata record if any services were executed.
+    def perform
+      storage_loc = @file_rec.storage_location
+      service_manager = @app_manager.service_manager
+      md_rec = @file_rec.metadata_record
+      f_path = @file_rec.path
+
+      logger.info("Performing preserve event on #{@file_rec.path}")
+
+      needs_persist = false
+      begin
+        if !File.exist?(f_path)
+          # Need to persist metadata to avoid repeating processing of this file too soon.
+          needs_persist = true
+          record_failure(EventNames::PRESERVE, f_path, "File is registered but missing.")
+          return return_status
+        end
+
+        # get the list of services applicable to this location and event
+        service_manager.list_services(location: storage_loc.name, event: EventNames::PRESERVE).each do |service_name|
+          # Skip over this service if it does not need to be run, unless force flag active
+          unless @force || service_manager.service_needed?(service_name, md_rec)
+            logger.debug("Service #{service_name} not needed for file '#{@file_rec.path}', skipping")
+            next
+          end
+
+          begin
+            logger.info("Performing preserve service #{service_name} for #{@file_rec.path}")
+            needs_persist = true
+            # execute the service
+            service_manager.perform_service(service_name, @file_rec, EventNames::PRESERVE)
+
+            # record the outcome
+            @file_rec.metadata_record.update_service_as_performed(service_name)
+            record_success(EventNames::PRESERVE, f_path, nil, service_name)
+          rescue PreservationServiceError => e
+            @file_rec.metadata_record.update_service_as_failed(service_name)
+            record_failure(EventNames::PRESERVE, f_path, e.message, service_name)
+          rescue StorageLocationUnavailableError => e
+            raise e
+          rescue StandardError => e
+            @file_rec.metadata_record.update_service_as_failed(service_name)
+            record_failure(EventNames::PRESERVE, f_path, nil, service_name, error: e)
+            return return_status
+          end
+        end
+      ensure
+        # persist the metadata out to file if any services were executed
+        if needs_persist
+          # persist the metadata
+          @app_manager.md_manager.persist(@file_rec)
+        end
+      end
+
+      return_status
+    end
+  end
+end

data/lib/longleaf/events/register_event.rb

@@ -1,92 +1,93 @@
 require 'longleaf/errors'
+require 'longleaf/events/event_names'
+require 'longleaf/events/event_status_tracking'
 require 'longleaf/models/metadata_record'
 require 'longleaf/services/metadata_deserializer'
 require 'longleaf/services/metadata_serializer'
 require 'time'
 
-# Event to register a file with longleaf
 module Longleaf
+  # Event to register a file with longleaf
   class RegisterEvent
-    EVENT_NAME = 'register'
-    
+    include Longleaf::EventStatusTracking
+
     # @param file_rec [FileRecord] file record
     # @param app_manager [ApplicationConfigManager] the application configuration
     # @param force [boolean] if true, then already registered files will be re-registered
-    def initialize(file_rec:, app_manager:, force: false, checksums: nil)
+    # @param digest_provider [#get_digests] object which provides digests for files being registered
+    def initialize(file_rec:, app_manager:, force: false, digest_provider: nil)
       raise ArgumentError.new('Must provide a file_rec parameter') if file_rec.nil?
       raise ArgumentError.new('Parameter file_rec must be a FileRecord') \
           unless file_rec.is_a?(FileRecord)
       raise ArgumentError.new('Must provide an ApplicationConfigManager') if app_manager.nil?
       raise ArgumentError.new('Parameter app_manager must be an ApplicationConfigManager') \
           unless app_manager.is_a?(ApplicationConfigManager)
-      
+
       @app_manager = app_manager
       @file_rec = file_rec
       @force = force
-      @checksums = checksums
+      @digest_provider = digest_provider
     end
-    
+
     # Perform a registration event on the given file
-    # @raises RegistrationError if a file cannot be registered 
+    # @raise RegistrationError if a file cannot be registered
     def perform
-      metadata_exists = File.file?(@file_rec.metadata_path)
-      # If the file's metadata exists, only need to register it if the force flag is provided
-      if metadata_exists && !@force
-        raise RegistrationError.new("Unable to register '#{@file_rec.path}', it is already registered.")
-      end
-      
-      # create metadata record
-      md_rec = MetadataRecord.new(registered: Time.now.utc.iso8601)
-      @file_rec.metadata_record = md_rec
-      
-      # retain significant details from former record
-      if metadata_exists
-        retain_existing_properties
+      begin
+        # Only need to re-register file if the force flag is provided
+        if @file_rec.metadata_present? && !@force
+          raise RegistrationError.new("Unable to register '#{@file_rec.path}', it is already registered.")
+        end
+
+        # create metadata record
+        md_rec = MetadataRecord.new(registered: Time.now.utc.iso8601(3))
+        @file_rec.metadata_record = md_rec
+
+        # retain significant details from former record
+        if @file_rec.metadata_present?
+          retain_existing_properties
+        end
+
+        populate_file_properties
+
+        if !@digest_provider.nil?
+          checksums = @digest_provider.get_digests(@file_rec.path)
+          md_rec.checksums.merge!(checksums) unless checksums.nil?
+        end
+
+        # persist the metadata
+        @app_manager.md_manager.persist(@file_rec)
+
+        record_success(EventNames::REGISTER, @file_rec.path)
+      rescue RegistrationError => err
+        record_failure(EventNames::REGISTER, @file_rec.path, err.message)
+      rescue InvalidStoragePathError => err
+        record_failure(EventNames::REGISTER, @file_rec.path, err.message)
       end
-      
-      populate_file_properties
-      
-      md_rec.checksums.merge!(@checksums) unless @checksums.nil?
-      
-      populate_services
-      
-      # persist the metadata out to file
-      MetadataSerializer::write(metadata: md_rec, file_path: @file_rec.metadata_path)
+
+      return_status
     end
-    
+
     private
     def populate_file_properties
       md_rec = @file_rec.metadata_record
-      
+
       # Set file properties
-      md_rec.last_modified = File.mtime(@file_rec.path).utc.iso8601
+      md_rec.last_modified = File.mtime(@file_rec.path).utc.iso8601(3)
       md_rec.file_size = File.size(@file_rec.path)
     end
-    
-    def populate_services
-      md_rec = @file_rec.metadata_record
-      
-      service_manager = @app_manager.service_manager
-      definitions = service_manager.list_service_definitions(location: @file_rec.storage_location.name)
-      
-      # Add service section
-      definitions.each do |serv_def|
-        serv_name = serv_def.name
-        md_rec.add_service(serv_name)
-      end
-    end
-    
+
     # Copy a subset of properties from an existing metadata record to the new record
     def retain_existing_properties
       md_rec = @file_rec.metadata_record
-      
-      old_md = MetadataDeserializer.deserialize(file_path: @file_rec.metadata_path)
+
+      old_md = MetadataDeserializer.deserialize(file_path: @file_rec.metadata_path,
+              digest_algs: @file_rec.storage_location.metadata_location.digests)
       # Copy custom properties
       old_md.properties.each { |name, value| md_rec.properties[name] = value }
       # Copy stale-replicas flag per service
       old_md.list_services.each do |serv_name|
         serv_rec = old_md.service(serv_name)
-        
+
         stale_replicas = serv_rec.stale_replicas
         if stale_replicas
           new_service = md_rec.service(serv_name)
@@ -95,4 +96,4 @@ module Longleaf
       end
     end
   end
-end
+end

data/lib/longleaf/helpers/case_insensitive_hash.rb

@@ -0,0 +1,38 @@
+module Longleaf
+  # Hash subclass which provides case insensitive keys, where keys are always downcased.
+  class CaseInsensitiveHash < Hash
+    def [](key)
+      super _insensitive(key)
+    end
+
+    def []=(key, value)
+      super _insensitive(key), value
+    end
+
+    def delete(key)
+      super _insensitive(key)
+    end
+
+    def has_key?(key)
+      super _insensitive(key)
+    end
+
+    def merge(other_hash)
+      super other_hash.map {|k, v| [_insensitive(k), v] }.to_h
+    end
+
+    def merge!(other_hash)
+      super other_hash.map {|k, v| [_insensitive(k), v] }.to_h
+    end
+
+    # Cause this hash to serialize as a regular hash to avoid deserialization failures
+    def encode_with coder
+      coder.represent_map nil, self
+    end
+
+    protected
+    def _insensitive(key)
+      key.respond_to?(:downcase) ? key.downcase : key
+    end
+  end
+end

data/lib/longleaf/helpers/digest_helper.rb

@@ -0,0 +1,56 @@
+require 'longleaf/errors'
+require 'digest'
+
+module Longleaf
+  # Helper methods for generating digests
+  class DigestHelper
+    KNOWN_DIGESTS ||= ['md5', 'sha1', 'sha2', 'sha256', 'sha384', 'sha512', 'rmd160']
+
+    # @param algs Either a string containing one or an array containing zero or more digest
+    #    algorithm names.
+    # @raise [InvalidDigestAlgorithmError] thrown if any of the digest algorithms listed are not
+    #    known to the system.
+    def self.validate_algorithms(algs)
+      return if algs.nil?
+      if algs.is_a?(String)
+        unless self.is_known_algorithm?(algs)
+          raise InvalidDigestAlgorithmError.new("Unknown digest algorithm #{algs}")
+        end
+      else
+        unknown = algs.select { |alg| !KNOWN_DIGESTS.include?(alg) }
+        unless unknown.empty?
+          raise InvalidDigestAlgorithmError.new("Unknown digest algorithm(s): #{unknown}")
+        end
+      end
+    end
+
+    # @param alg [String] identifier of digest algorithm
+    # @return [Boolean] true if the digest is a valid known algorithm
+    def self.is_known_algorithm?(alg)
+      KNOWN_DIGESTS.include?(alg)
+    end
+
+    # Get a Digest class for the specified algorithm
+    # @param alg [String] name of the digest algorithm
+    # @return [Digest] A digest class for the requested algorithm
+    # @raise [InvalidDigestAlgorithmError] if an unknown digest algorithm is requested
+    def self.start_digest(alg)
+      case alg
+      when 'md5'
+        return Digest::MD5.new
+      when 'sha1'
+        return Digest::SHA1.new
+      when 'sha2', 'sha256'
+        return Digest::SHA2.new
+      when 'sha384'
+        return Digest::SHA2.new(384)
+      when 'sha512'
+        return Digest::SHA2.new(512)
+      when 'rmd160'
+        return Digest::RMD160.new
+      else
+        raise InvalidDigestAlgorithmError.new("Cannot produce digest for unknown algorithm '#{alg}'.")
+      end
+    end
+  end
+end

data/lib/longleaf/helpers/s3_uri_helper.rb

@@ -0,0 +1,86 @@
+require 'uri'
+
+module Longleaf
+  # Helper for interacting with s3 uris
+  class S3UriHelper
+    ENDPOINT_PATTERN = /^(.+\.)?s3[.\-]([a-z0-9\-]+[\-.])?[a-z0-9]+\./
+    ALLOWED_SCHEMES = ['http', 'https', 's3']
+
+    # Extract the name of the s3 bucket from the provided url
+    # @param url s3 url
+    # @return the name of the bucket, or nil if the name could not be identified
+    def self.extract_bucket(url)
+      uri = s3_uri(url)
+
+      matches = ENDPOINT_PATTERN.match(uri.host)
+      if matches.nil?
+        raise ArgumentError.new("Provided URI does match the expected pattern for an S3 URI")
+      end
+
+      prefix = matches[1]
+      if prefix.nil? || prefix.empty?
+        # Is a path style url
+        path = uri.path
+
+        return nil if path == '/'
+
+        path_parts = path.split('/')
+        return nil if path_parts.empty?
+        return path_parts[1]
+      else
+        return prefix[0..-2]
+      end
+    end
+
+    def self.extract_path(url)
+      uri = s3_uri(url)
+
+      matches = ENDPOINT_PATTERN.match(uri.host)
+      if matches.nil?
+        raise ArgumentError.new("Provided URI does match the expected pattern for an S3 URI")
+      end
+
+      path = uri.path
+      return nil if path == '/' || path.empty?
+
+      # trim off the first slash
+      path = path.partition('/').last
+
+      # Determine if the first part of the path is the bucket name
+      prefix = matches[1]
+      if prefix.nil? || prefix.empty?
+        # trim off the bucket name
+        path = path.partition('/').last
+      end
+
+      path
+    end
+
+    def self.extract_region(url)
+      uri = s3_uri(url)
+
+      matches = ENDPOINT_PATTERN.match(uri.host)
+
+      if matches[2].nil?
+        # No region specified
+        nil
+      else
+        matches[2][0..-2]
+      end
+    end
+
+    def self.s3_uri(url)
+      if url.nil?
+        raise ArgumentError.new("url cannot be empty")
+      end
+      uri = URI(url)
+      if !ALLOWED_SCHEMES.include?(uri.scheme&.downcase)
+        raise ArgumentError.new("Invalid scheme for s3 URI #{url}, only http, https and s3 are permitted")
+      end
+      if uri.host.nil?
+        raise ArgumentError.new("Invalid S3 URI, no hostname: #{url}")
+      end
+      uri
+    end
+  end
+end

data/lib/longleaf/helpers/selection_options_parser.rb

@@ -0,0 +1,189 @@
+require 'longleaf/candidates/file_selector'
+require 'longleaf/candidates/registered_file_selector'
+require 'longleaf/candidates/manifest_digest_provider'
+require 'longleaf/candidates/single_digest_provider'
+
+module Longleaf
+  # Helper for parsing manifest inputs used for registration
+  class SelectionOptionsParser
+    extend Longleaf::Logging
+
+    # Parses the provided options to construct a file selector and digest provider for
+    # use in registration commands.
+    # @param options [Hash] command options
+    # @param app_config_manager [ApplicationConfigManager] app config manager
+    # @return The file selector and digest provider.
+    def self.parse_registration_selection_options(options, app_config_manager)
+      there_can_be_only_one("Only one of the following selection options may be provided: -m, -f, -s",
+          options, :file, :manifest, :location)
+
+      if !options[:manifest].nil?
+        digests_mapping = self.manifests_to_digest_mapping(options[:manifest])
+        selector = FileSelector.new(file_paths: digests_mapping.keys, app_config: app_config_manager)
+        digest_provider = ManifestDigestProvider.new(digests_mapping)
+      elsif !options[:file].nil?
+        if options[:checksums]
+          checksums = options[:checksums]
+          # validate checksum list format, must a comma delimited list of prefix:checksums
+          if /^[^:,]+:[^:,]+(,[^:,]+:[^:,]+)*$/.match(checksums)
+            # convert checksum list into hash with prefix as key
+            checksums = Hash[*checksums.split(/\s*[:,]\s*/)]
+            digest_provider = SingleDigestProvider.new(checksums)
+          else
+            logger.failure("Invalid checksums parameter format, see `longleaf help <command>` for more information")
+            exit 1
+          end
+        end
+
+        file_paths = options[:file].split(/\s*,\s*/)
+        selector = FileSelector.new(file_paths: file_paths, app_config: app_config_manager)
+      elsif !options[:location].nil?
+        storage_locations = options[:location].split(/\s*,\s*/)
+        selector = FileSelector.new(storage_locations: storage_locations, app_config: app_config_manager)
+        digest_provider = SingleDigestProvider.new(nil)
+      else
+        logger.failure("Must provide one of the following file selection options: -f, l, or -m")
+        exit 1
+      end
+
+      [selector, digest_provider]
+    end
+
+    def self.there_can_be_only_one(failure_msg, options, *names)
+      got_one = false
+      names.each do |name|
+        if !options[name].nil?
+          if got_one
+            logger.failure(failure_msg)
+            exit 1
+          end
+          got_one = true
+        end
+      end
+    end
+
+    # Parses the provided manifest options, reading the contents of the manifests to produce
+    # a mapping from files to one or more algorithms.
+    # @param manifest_vals [Array] List of manifest option values. They may be in one of the following formats:
+    #       <alg_name>:<manifest_path> OR <alg_name>:@-
+    #.      <manifest_path> OR @-
+    # @return a hash containing the aggregated contents of the provided manifests. The keys are
+    #    paths to manifested files. The values are hashes, mapping digest algorithms to digest values.
+    def self.manifests_to_digest_mapping(manifest_vals)
+      alg_manifest_pairs = []
+      # interpret option inputs into a list of algorithms to manifest sources
+      manifest_vals.each do |manifest_val|
+        if manifest_val.include?(':')
+          manifest_parts = manifest_val.split(':', 2)
+          alg_manifest_pairs << manifest_parts
+        else
+          # algorithm not specified in option value
+          alg_manifest_pairs << [nil, manifest_val]
+        end
+      end
+      if alg_manifest_pairs.select { |mpair| mpair[1] == '@-' }.count > 1
+        self.fail("Cannot specify more than one manifest from STDIN")
+      end
+
+      # read the provided manifests to build a mapping from file uri to all supplied digests
+      digests_mapping = Hash.new { |h,k| h[k] = Hash.new }
+      alg_manifest_pairs.each do |mpair|
+        source_stream = nil
+        # Determine if reading from a manifest file or stdin
+        if mpair[1] == '@-'
+          source_stream = $stdin
+        else
+          source_stream = File.new(mpair[1])
+        end
+
+        current_alg = mpair[0]
+        multi_digest_manifest = current_alg.nil?
+        source_stream.each_line do |line|
+          line = line.strip
+          if multi_digest_manifest && /^[a-zA-Z0-9]+:$/ =~ line
+            # Found a digest algorithm header, assuming succeeding entries are of this type
+            current_alg = line.chomp(':')
+            # Verify that the digest algorithm is known to longleaf
+            if !DigestHelper.is_known_algorithm?(current_alg)
+              self.fail("Manifest specifies unknown digest algorithm: #{current_alg}")
+            end
+          else
+            if current_alg.nil?
+              self.fail("Manifest with unknown checksums encountered, an algorithm must be specified")
+            end
+            entry_parts = line.split(' ', 2)
+            if entry_parts.length != 2
+              self.fail("Invalid manifest entry: #{line}")
+            end
+
+            digests_mapping[entry_parts[1]][current_alg] = entry_parts[0]
+          end
+        end
+      end
+
+      digests_mapping
+    end
+
+    # Parses the provided options to create a selector for registered files
+    # @param options [Hash] command options
+    # @param app_config_manager [ApplicationConfigManager] app config manager
+    # @return selector
+    def self.create_registered_selector(options, app_config_manager)
+      there_can_be_only_one("Only one of the following selection options may be provided: -l, -f, -s",
+          options, :file, :location, :from_list)
+          
+      if !options[:from_list].nil?
+        file_paths = read_from_list(options[:from_list])
+        return RegisteredFileSelector.new(file_paths: file_paths, app_config: app_config_manager)
+      elsif !options[:file].nil?
+        file_paths = options[:file].split(/\s*,\s*/)
+        return RegisteredFileSelector.new(file_paths: file_paths, app_config: app_config_manager)
+      elsif !options[:location].nil?
+        storage_locations = options[:location].split(/\s*,\s*/)
+        return RegisteredFileSelector.new(storage_locations: storage_locations, app_config: app_config_manager)
+      else
+        logger.failure("Must provide one of the following file selection options: -l, -f, or -s")
+        exit 1
+      end
+    end
+    
+    # Parses the -l from_list option, reading the list of files specified either from the provided
+    # file path or STDIN
+    # @param from_list option value, either a file path or "@-"
+    # @return list of files from the from_list
+    def self.read_from_list(from_list)
+      from_list = from_list.strip
+      if from_list.empty?
+        logger.failure("List parameter must not be empty")
+        exit 1
+      end
+      
+      if from_list == '@-'
+        source_stream = $stdin
+      else
+        begin
+          source_stream = File.new(from_list)
+        rescue Errno::ENOENT
+          logger.failure("Specified list file does not exist: #{from_list}")
+          exit 1
+        end
+      end
+      
+      lines = []
+      source_stream.each_line do |line|
+        lines << line.strip
+      end
+      
+      if lines.empty?
+        logger.failure("File list is empty, must provide one or more files for this operation")
+        exit 1
+      end
+      lines
+    end
+
+    def self.fail(message)
+      logger.failure(message)
+      exit 1
+    end
+  end
+end