longleaf 0.1.0.pre.3 → 1.1.0

data/lib/longleaf/models/app_fields.rb

@@ -1,10 +1,15 @@
 module Longleaf
+  # Application configuration field names
   class AppFields
     LOCATIONS = 'locations'
     SERVICES = 'services'
     SERVICE_MAPPINGS = 'service_mappings'
-    
+    SYSTEM = 'system'
+
     LOCATION_PATH = 'path'
-    METADATA_PATH = 'metadata_path'
+    METADATA_CONFIG = 'metadata'
+    METADATA_DIGESTS = 'digests'
+
+    STORAGE_TYPE = 'type'
   end
 end

data/lib/longleaf/models/file_record.rb

@@ -1,25 +1,48 @@
-# Record for an individual file and its associated information 
 module Longleaf
+  # Record for an individual file and its associated information
   class FileRecord
-    
     attr_accessor :metadata_record
     attr_reader :storage_location
     attr_reader :path
-    
+
     # @param file_path [String] path to the file
-    # @param storage_location [Longleaf::StorageLocation] storage location containing the file
-    def initialize(file_path, storage_location)
+    # @param storage_location [StorageLocation] storage location containing the file
+    # @param metadata_record [MetadataRecord] metadata record for this file object. Optional.
+    # @param physical_path [String] physical path where the file is located. Defaults to the file_path.
+    def initialize(file_path, storage_location, metadata_record = nil, physical_path = nil)
       raise ArgumentError.new("FileRecord requires a path") if file_path.nil?
       raise ArgumentError.new("FileRecord requires a storage_location") if storage_location.nil?
-      
+
       @path = file_path
       @storage_location = storage_location
+      @metadata_record = metadata_record
+      @physical_path = physical_path
     end
-    
+
     # @return [String] path for the metadata file for this file
     def metadata_path
       @metadata_path = @storage_location.get_metadata_path_for(path) if @metadata_path.nil?
       @metadata_path
     end
+    
+    def physical_path
+      if @physical_path.nil?
+        if @metadata_record.nil? || @metadata_record.physical_path.nil?
+          @physical_path = @path
+        else
+          @physical_path = @metadata_record.physical_path
+        end
+      end
+      @physical_path
+    end
+
+    def metadata_present?
+      File.exist?(metadata_path)
+    end
+
+    def ==(other_obj)
+      return false unless other_obj.is_a?(FileRecord)
+      path == other_obj.path
+    end
   end
-end
+end

data/lib/longleaf/models/filesystem_metadata_location.rb

@@ -0,0 +1,56 @@
+require 'longleaf/services/metadata_serializer'
+require 'longleaf/models/metadata_location'
+require 'longleaf/models/storage_types'
+
+module Longleaf
+  # A filesystem based location in which metadata associated with registered files is stored.
+  class FilesystemMetadataLocation < MetadataLocation
+    AF ||= Longleaf::AppFields
+
+    def initialize(config)
+      super(config)
+    end
+
+    # @return the storage type for this location
+    def type
+      StorageTypes::FILESYSTEM_STORAGE_TYPE
+    end
+
+    # Get the absolute path for the metadata file for the given file path located in this storage location.
+    # @param file_path [String] path of the file relative its storage location
+    # @return absolute path to the metadata
+    # @raise [ArgumentError] if the file_path is not provided.
+    def metadata_path_for(file_path)
+      raise ArgumentError.new("A file_path parameter is required") if file_path.nil?
+      raise ArgumentError.new("File path must be relative") if Pathname.new(file_path).absolute?
+
+      md_path = File.join(@path, file_path)
+      # If the file_path is to a file, then add metadata suffix.
+      if md_path.end_with?('/')
+        md_path
+      else
+        md_path + MetadataSerializer::metadata_suffix
+      end
+    end
+
+    # Get the metadata path relative to this location
+    # @param md_path [String] metadata file path
+    # @return the metadata path relative to this location
+    # @raise [ArgumentError] if the metadata path is not contained by this location
+    def relativize(md_path)
+      return md_path if Pathname.new(md_path).relative?
+
+      raise ArgumentError.new("Metadata path must be contained by this location") if !md_path.start_with?(@path)
+
+      md_path.sub(@path, "")
+    end
+
+
+    # Checks that the path defined in this metadata location are available
+    # @raise [StorageLocationUnavailableError] if the metadata location is not available
+    def available?
+      raise StorageLocationUnavailableError.new("Metadata path does not exist or is not a directory: #{@path}")\
+          unless Dir.exist?(@path)
+    end
+  end
+end

data/lib/longleaf/models/filesystem_storage_location.rb

@@ -0,0 +1,52 @@
+require 'longleaf/models/storage_location'
+require 'longleaf/models/storage_types'
+
+module Longleaf
+  # A storage location in a local filesystem
+  class FilesystemStorageLocation < StorageLocation
+    # @param name [String] the name of this storage location
+    # @param config [Hash] hash containing the configuration options for this location
+    # @param md_loc [MetadataLocation] metadata location associated with this storage location
+    def initialize(name, config, md_loc)
+      super(name, config, md_loc)
+      @path += File::SEPARATOR unless @path.end_with?(File::SEPARATOR)
+    end
+
+    # @return the storage type for this location
+    def type
+      StorageTypes::FILESYSTEM_STORAGE_TYPE
+    end
+
+    # Get that absolute path to the file associated with the provided metadata path
+    # @param md_path [String] metadata file path
+    # @raise [ArgumentError] if the md_path is not in this storage location
+    # @return [String] the path for the file associated with this metadata
+    def get_path_from_metadata_path(md_path)
+      raise ArgumentError.new("A file_path parameter is required") if md_path.nil? || md_path.empty?
+
+      rel_path = @metadata_location.relative_file_path_for(md_path)
+
+      File.join(@path, rel_path)
+    end
+
+    # Checks that the path and metadata path defined in this location are available
+    # @raise [StorageLocationUnavailableError] if the storage location is not available
+    def available?
+      raise StorageLocationUnavailableError.new("Path does not exist or is not a directory: #{@path}")\
+          unless Dir.exist?(@path)
+      @metadata_location.available?
+    end
+
+    # Get the file path relative to this location
+    # @param file_path [String] file path
+    # @return the file path relative to this location
+    # @raise [ArgumentError] if the file path is not contained by this location
+    def relativize(file_path)
+      return file_path if Pathname.new(file_path).relative?
+
+      raise ArgumentError.new("Metadata path must be contained by this location") if !file_path.start_with?(@path)
+
+      file_path.sub(@path, "")
+    end
+  end
+end

data/lib/longleaf/models/md_fields.rb

@@ -1,13 +1,15 @@
 module Longleaf
+  # File metadata fields
   class MDFields
     DATA = 'data'
     SERVICES = 'services'
-    
+
     REGISTERED_TIMESTAMP = 'registered'
     DEREGISTERED_TIMESTAMP = 'deregistered'
 
     LAST_MODIFIED = 'last-modified'
     FILE_SIZE = 'size'
+    PHYSICAL_PATH = 'physical-path'
 
     CHECKSUMS = 'checksums'
 

data/lib/longleaf/models/metadata_location.rb

@@ -0,0 +1,47 @@
+require 'longleaf/models/app_fields'
+
+module Longleaf
+  # A location in which metadata associated with registered files is stored.
+  class MetadataLocation
+    AF ||= Longleaf::AppFields
+
+    attr_reader :path
+    attr_reader :digests
+
+    def initialize(config)
+      raise ArgumentError.new("Config parameter is required") unless config
+      @path = config[AF::LOCATION_PATH]
+      raise ArgumentError.new("Parameter path is required") unless @path
+      @path += '/' unless @path.end_with?('/')
+
+      digests = config[AF::METADATA_DIGESTS]
+      if digests.nil?
+        @digests = []
+      elsif digests.is_a?(String)
+        @digests = [digests.downcase]
+      else
+        @digests = digests.map(&:downcase)
+      end
+      DigestHelper::validate_algorithms(@digests)
+    end
+
+    # Transforms the given metadata path into a relative storage location path
+    # @param md_path [String] path of the metadata file or directory to compute file path for.
+    # @return
+    def relative_file_path_for(md_path)
+      rel_md_path = relativize(md_path)
+
+      if rel_md_path.end_with?(MetadataSerializer::metadata_suffix)
+        rel_md_path[0..-MetadataSerializer::metadata_suffix.length - 1]
+      else
+        rel_md_path
+      end
+    end
+
+    # @param [String] metadata path to check
+    # @return true if the metadata path is contained by the path for this location
+    def contains?(md_path)
+      md_path.start_with?(@path)
+    end
+  end
+end

data/lib/longleaf/models/metadata_record.rb

@@ -1,14 +1,17 @@
 require_relative 'md_fields'
 require_relative 'service_record'
+require 'longleaf/helpers/case_insensitive_hash'
 
-# Metadata record for a single file
 module Longleaf
+  # Metadata record for a single file
   class MetadataRecord
-    attr_reader :deregistered, :registered
+    attr_reader :registered
+    attr_accessor :deregistered
     attr_reader :checksums
     attr_reader :properties
     attr_accessor :file_size, :last_modified
-    
+    attr_accessor :physical_path
+
     # @param properties [Hash] initial data properties for this record
     # @param services [Hash] initial service property tree
     # @param deregistered [String] deregistered timestamp
@@ -16,42 +19,66 @@ module Longleaf
     # @param checksums [Hash] hash of checksum values
     # @param file_size [Integer] size of file in bytes
     # @param last_modified [String] iso8601 representation of the last modified date of file
-    def initialize(properties: Hash.new, services: Hash.new, deregistered: nil, registered: nil, checksums: Hash.new,
-          file_size: nil, last_modified: nil)
-      @properties = properties
+    # @param physical_path [String] physical path where the file is located
+    def initialize(properties: nil, services: nil, deregistered: nil, registered: nil, checksums: nil,
+          file_size: nil, last_modified: nil, physical_path: nil)
+      @properties = properties || Hash.new
       @registered = registered
       @deregistered = deregistered
-      @checksums = checksums
-      @services = services
+      @checksums = CaseInsensitiveHash.new
+      @checksums.merge!(checksums) unless checksums.nil?
+      @services = services || Hash.new
       @file_size = file_size
       @last_modified = last_modified
+      @physical_path = physical_path
     end
-    
+
     # @return [Boolean] true if the record is deregistered
     def deregistered?
       !@deregistered.nil?
     end
-    
+
     # Adds a service to this record
     #
     # @param name [String] identifier for the service being added
-    # @param service_properties [ServiceRecord] properties for populating the new service
-    def add_service(name, service = Longleaf::ServiceRecord.new)
+    # @param service [ServiceRecord] properties for populating the new service
+    # @return [ServiceRecord] the service added
+    def add_service(name, service = ServiceRecord.new)
       raise ArgumentError.new("Value must be a ServiceRecord object when adding a service") unless service.class == Longleaf::ServiceRecord
       raise IndexError.new("Service with name '#{name}' already exists") if @services.key?(name)
-      
+
       @services[name] = service
     end
-    
+
+    # Updates details of service record as if the service had been executed.
+    # @param service_name [String] name of the service run
+    # @return [ServiceRecord] the service record updated
+    def update_service_as_performed(service_name)
+      service_rec = service(service_name) || add_service(service_name)
+      service_rec.run_needed = false
+      service_rec.timestamp = ServiceDateHelper.formatted_timestamp
+      service_rec
+    end
+
+    # Updates details of service record as if the service had encountered a
+    # failure during execution.
+    # @param service_name [String] name of the service run
+    # @return [ServiceRecord] the service record updated
+    def update_service_as_failed(service_name)
+      service_rec = service(service_name) || add_service(service_name)
+      service_rec.failure_timestamp = ServiceDateHelper.formatted_timestamp
+      service_rec
+    end
+
     # @param name [String] name identifier of the service to retrieve
     # @return [ServiceRecord] the ServiceRecord for the service identified by name, or nil
     def service(name)
       @services[name]
     end
-    
+
     # @return [Array<String>] a list of name identifiers for services registered to this record
     def list_services
       @services.keys
     end
   end
-end
+end

data/lib/longleaf/models/s3_storage_location.rb

@@ -0,0 +1,138 @@
+require 'longleaf/models/storage_location'
+require 'longleaf/models/storage_types'
+require 'longleaf/helpers/s3_uri_helper'
+require 'longleaf/logging'
+require 'uri'
+require 'aws-sdk-s3'
+
+module Longleaf
+  # A storage location in a s3 bucket
+  #
+  # Optionally, the location configuration may include an "options" sub-hash in order to provide
+  # any of the s3 client options specified in Client initializer:
+  # https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html#constructor_details
+
+  class S3StorageLocation < StorageLocation
+    include Longleaf::Logging
+
+    IS_URI_REGEX = /\A#{URI::regexp}\z/
+
+    CLIENT_OPTIONS_FIELD = 'options'
+
+    # @param name [String] the name of this storage location
+    # @param config [Hash] hash containing the configuration options for this location
+    # @param md_loc [MetadataLocation] metadata location associated with this storage location
+    def initialize(name, config, md_loc)
+      super(name, config, md_loc)
+
+      @bucket_name = S3UriHelper.extract_bucket(@path)
+      if @bucket_name.nil?
+        raise ArgumentError.new("Unable to identify bucket for location #{@name} from path #{@path}")
+      end
+
+      # Force path to always end with a slash
+      @path += '/' unless @path.end_with?('/')
+
+      custom_options = config[CLIENT_OPTIONS_FIELD]
+      if custom_options.nil?
+        @client_options = Hash.new
+      else
+        # Clone options and convert keys to symbols
+        @client_options = Hash[custom_options.map { |(k,v)| [k.to_sym,v] } ]
+      end
+      @client_options[:logger] = logger
+      @client_options[:log_level] = :debug if @client_options[:log_level].nil?
+      
+      # If no region directly configured, use region from path
+      if !@client_options.key?(:region)
+        region = S3UriHelper.extract_region(@path)
+        @client_options[:region] = region unless region.nil?
+      end
+      
+      @subpath_prefix = S3UriHelper.extract_path(@path)
+    end
+
+    # @return the storage type for this location
+    def type
+      StorageTypes::S3_STORAGE_TYPE
+    end
+
+    # Get that absolute path to the file associated with the provided metadata path
+    # @param md_path [String] metadata file path
+    # @raise [ArgumentError] if the md_path is not in this storage location
+    # @return [String] the path for the file associated with this metadata
+    def get_path_from_metadata_path(md_path)
+      raise ArgumentError.new("A file_path parameter is required") if md_path.nil? || md_path.empty?
+
+      rel_path = @metadata_location.relative_file_path_for(md_path)
+
+      URI.join(@path, rel_path).to_s
+    end
+
+    # Checks that the path and metadata path defined in this location are available
+    # @raise [StorageLocationUnavailableError] if the storage location is not available
+    def available?
+      begin
+        s3_client().head_bucket({ bucket: @bucket_name, use_accelerate_endpoint: false })
+      rescue StandardError => e
+        raise StorageLocationUnavailableError.new("Destination bucket #{@bucket_name} does not exist " \
+            + "or is not accessible: #{e.message}")
+      end
+      @metadata_location.available?
+    end
+
+    # Get the file path relative to this location
+    # @param file_path [String] file path
+    # @return the file path relative to this location
+    # @raise [ArgumentError] if the file path is not contained by this location
+    def relativize(file_path)
+      raise ArgumentError.new("Must provide a non-nil path to relativize") if file_path.nil?
+
+      if file_path.start_with?(@path)
+        file_path[@path.length..-1]
+      else
+        if file_path =~ IS_URI_REGEX
+          raise ArgumentError.new("Path #{file_path} is not contained by #{@name}")
+        else
+          # path already relative
+          file_path
+        end
+      end
+    end
+    
+    # Prefixes the provided path with the query path portion of the location's path
+    # after the bucket uri, used to place relative paths into the same sub-URL of a bucket.
+    # For example:
+    # Given a location with 'path' http://example.s3-amazonaws.com/env/test/
+    # Where rel_path = 'path/to/text.txt'
+    # The result would be 'env/test/path/to/text.txt'
+    # @param rel_path relative path to work with
+    # @return the given relative path prefixed with the path portion of the storage location path
+    def relative_to_bucket_path(rel_path)
+      raise ArgumentError.new("Must provide a non-nil path") if rel_path.nil?
+      
+      if @subpath_prefix.nil?
+        return rel_path
+      end
+      
+      @subpath_prefix + rel_path
+    end
+
+    # @return the bucket used by this storage location
+    def s3_bucket
+      if @bucket.nil?
+        @s3 = Aws::S3::Resource.new(client: s3_client())
+        @bucket = @s3.bucket(@bucket_name)
+      end
+      @bucket
+    end
+
+    # @return the s3 client used by this storage locatio
+    def s3_client
+      if @client.nil?
+        @client = Aws::S3::Client.new(**@client_options)
+      end
+      @client
+    end
+  end
+end