longleaf 0.1.0.pre.3 → 1.1.0

data/lib/longleaf/helpers/s3_uri_helper.rb

@@ -0,0 +1,86 @@
+require 'uri'
+
+module Longleaf
+  # Helper for interacting with s3 uris
+  class S3UriHelper
+    ENDPOINT_PATTERN = /^(.+\.)?s3[.\-]([a-z0-9\-]+[\-.])?[a-z0-9]+\./
+    ALLOWED_SCHEMES = ['http', 'https', 's3']
+
+    # Extract the name of the s3 bucket from the provided url
+    # @param url s3 url
+    # @return the name of the bucket, or nil if the name could not be identified
+    def self.extract_bucket(url)
+      uri = s3_uri(url)
+
+      matches = ENDPOINT_PATTERN.match(uri.host)
+      if matches.nil?
+        raise ArgumentError.new("Provided URI does match the expected pattern for an S3 URI")
+      end
+
+      prefix = matches[1]
+      if prefix.nil? || prefix.empty?
+        # Is a path style url
+        path = uri.path
+
+        return nil if path == '/'
+
+        path_parts = path.split('/')
+        return nil if path_parts.empty?
+        return path_parts[1]
+      else
+        return prefix[0..-2]
+      end
+    end
+
+    def self.extract_path(url)
+      uri = s3_uri(url)
+
+      matches = ENDPOINT_PATTERN.match(uri.host)
+      if matches.nil?
+        raise ArgumentError.new("Provided URI does match the expected pattern for an S3 URI")
+      end
+
+      path = uri.path
+      return nil if path == '/' || path.empty?
+
+      # trim off the first slash
+      path = path.partition('/').last
+
+      # Determine if the first part of the path is the bucket name
+      prefix = matches[1]
+      if prefix.nil? || prefix.empty?
+        # trim off the bucket name
+        path = path.partition('/').last
+      end
+
+      path
+    end
+
+    def self.extract_region(url)
+      uri = s3_uri(url)
+
+      matches = ENDPOINT_PATTERN.match(uri.host)
+
+      if matches[2].nil?
+        # No region specified
+        nil
+      else
+        matches[2][0..-2]
+      end
+    end
+
+    def self.s3_uri(url)
+      if url.nil?
+        raise ArgumentError.new("url cannot be empty")
+      end
+      uri = URI(url)
+      if !ALLOWED_SCHEMES.include?(uri.scheme&.downcase)
+        raise ArgumentError.new("Invalid scheme for s3 URI #{url}, only http, https and s3 are permitted")
+      end
+      if uri.host.nil?
+        raise ArgumentError.new("Invalid S3 URI, no hostname: #{url}")
+      end
+      uri
+    end
+  end
+end

data/lib/longleaf/helpers/selection_options_parser.rb

@@ -0,0 +1,215 @@
+require 'longleaf/candidates/file_selector'
+require 'longleaf/candidates/registered_file_selector'
+require 'longleaf/candidates/manifest_digest_provider'
+require 'longleaf/candidates/physical_path_provider'
+require 'longleaf/candidates/single_digest_provider'
+
+module Longleaf
+  # Helper for parsing manifest inputs used for registration
+  class SelectionOptionsParser
+    extend Longleaf::Logging
+
+    # Parses the provided options to construct a file selector and digest provider for
+    # use in registration commands.
+    # @param options [Hash] command options
+    # @param app_config_manager [ApplicationConfigManager] app config manager
+    # @return The file selector and digest provider.
+    def self.parse_registration_selection_options(options, app_config_manager)
+      there_can_be_only_one("Only one of the following selection options may be provided: -m, -f, -s",
+          options, :file, :manifest, :location)
+
+      if !options[:manifest].nil?
+        digests_mapping, logical_phys_mapping = self.parse_manifest(options[:manifest])
+        physical_provider = PhysicalPathProvider.new(logical_phys_mapping)
+        selector = FileSelector.new(file_paths: digests_mapping.keys,
+             physical_provider: physical_provider,
+             app_config: app_config_manager)
+        digest_provider = ManifestDigestProvider.new(digests_mapping)
+      elsif !options[:file].nil?
+        if options[:checksums]
+          checksums = options[:checksums]
+          # validate checksum list format, must a comma delimited list of prefix:checksums
+          if /^[^:,]+:[^:,]+(,[^:,]+:[^:,]+)*$/.match(checksums)
+            # convert checksum list into hash with prefix as key
+            checksums = Hash[*checksums.split(/\s*[:,]\s*/)]
+            digest_provider = SingleDigestProvider.new(checksums)
+          else
+            logger.failure("Invalid checksums parameter format, see `longleaf help <command>` for more information")
+            exit 1
+          end
+        end
+
+        file_paths = self.split_quoted(options[:file], "\\s*,\\s*")
+        if !options[:physical_path].nil?
+          physical_paths = self.split_quoted(options[:physical_path], "\\s*,\\s*")
+          if physical_paths.length != file_paths.length
+            logger.failure("Invalid physical paths parameter, number of paths did not match number of logical paths")
+            exit 1
+          end
+          logical_phys_mapping = Hash[file_paths.zip physical_paths]
+          physical_provider = PhysicalPathProvider.new(logical_phys_mapping)
+        else
+          physical_provider = PhysicalPathProvider.new
+        end
+        
+        selector = FileSelector.new(file_paths: file_paths,
+             physical_provider: physical_provider,
+             app_config: app_config_manager)
+      else
+        logger.failure("Must provide one of the following file selection options: -f, l, or -m")
+        exit 1
+      end
+
+      [selector, digest_provider, physical_provider]
+    end
+
+    def self.there_can_be_only_one(failure_msg, options, *names)
+      got_one = false
+      names.each do |name|
+        if !options[name].nil?
+          if got_one
+            logger.failure(failure_msg)
+            exit 1
+          end
+          got_one = true
+        end
+      end
+    end
+
+    # Parses the provided manifest options, reading the contents of the manifests to produce
+    # a mapping from files to one or more algorithms.
+    # @param manifest_vals [Array] List of manifest option values. They may be in one of the following formats:
+    #       <alg_name>:<manifest_path> OR <alg_name>:@-
+    #.      <manifest_path> OR @-
+    # @return a hash containing the aggregated contents of the provided manifests. The keys are
+    #    paths to manifested files. The values are hashes, mapping digest algorithms to digest values.
+    def self.parse_manifest(manifest_vals)
+      alg_manifest_pairs = []
+      # interpret option inputs into a list of algorithms to manifest sources
+      manifest_vals.each do |manifest_val|
+        if manifest_val.include?(':')
+          manifest_parts = manifest_val.split(':', 2)
+          alg_manifest_pairs << manifest_parts
+        else
+          # algorithm not specified in option value
+          alg_manifest_pairs << [nil, manifest_val]
+        end
+      end
+      if alg_manifest_pairs.select { |mpair| mpair[1] == '@-' }.count > 1
+        self.fail("Cannot specify more than one manifest from STDIN")
+      end
+
+      # read the provided manifests to build a mapping from file uri to all supplied digests
+      digests_mapping = Hash.new { |h,k| h[k] = Hash.new }
+      logical_phys_mapping = Hash.new
+      alg_manifest_pairs.each do |mpair|
+        source_stream = nil
+        # Determine if reading from a manifest file or stdin
+        if mpair[1] == '@-'
+          source_stream = $stdin
+        else
+          source_stream = File.new(mpair[1])
+        end
+
+        current_alg = mpair[0]
+        multi_digest_manifest = current_alg.nil?
+        source_stream.each_line do |line|
+          line = line.strip
+          if multi_digest_manifest && /^[a-zA-Z0-9]+:$/ =~ line
+            # Found a digest algorithm header, assuming succeeding entries are of this type
+            current_alg = line.chomp(':')
+            # Verify that the digest algorithm is known to longleaf
+            if !DigestHelper.is_known_algorithm?(current_alg)
+              self.fail("Manifest specifies unknown digest algorithm: #{current_alg}")
+            end
+          else
+            if current_alg.nil?
+              self.fail("Manifest with unknown checksums encountered, an algorithm must be specified")
+            end
+            entry_parts = self.split_quoted(line)
+            if entry_parts.length != 2 && entry_parts.length != 3
+              self.fail("Invalid manifest entry: #{line}")
+            end
+
+            digests_mapping[entry_parts[1]][current_alg] = entry_parts[0]
+            if (entry_parts.length == 3)
+              logical_phys_mapping[entry_parts[1]] = entry_parts[2]
+            end
+          end
+        end
+      end
+
+      [digests_mapping, logical_phys_mapping]
+    end
+    
+    # Splits a string of quoted or unquoted tokens separated by spaces
+    # @param
+    def self.split_quoted(text, delimiter = "\\s+", limit = -1)
+      text.split(/#{delimiter}(?=(?:[^'"]|'[^']*'|"[^"]*")*$)/, limit)
+          .select {|s| not s.empty? }
+          .map {|s| s.gsub(/(^ +)|( +$)|(^["']+)|(["']+$)/, '')}
+    end
+
+    # Parses the provided options to create a selector for registered files
+    # @param options [Hash] command options
+    # @param app_config_manager [ApplicationConfigManager] app config manager
+    # @return selector
+    def self.create_registered_selector(options, app_config_manager)
+      there_can_be_only_one("Only one of the following selection options may be provided: -l, -f, -s",
+          options, :file, :location, :from_list)
+          
+      if !options[:from_list].nil?
+        file_paths = read_from_list(options[:from_list])
+        return RegisteredFileSelector.new(file_paths: file_paths, app_config: app_config_manager)
+      elsif !options[:file].nil?
+        file_paths = options[:file].split(/\s*,\s*/)
+        return RegisteredFileSelector.new(file_paths: file_paths, app_config: app_config_manager)
+      elsif !options[:location].nil?
+        storage_locations = options[:location].split(/\s*,\s*/)
+        return RegisteredFileSelector.new(storage_locations: storage_locations, app_config: app_config_manager)
+      else
+        logger.failure("Must provide one of the following file selection options: -l, -f, or -s")
+        exit 1
+      end
+    end
+    
+    # Parses the -l from_list option, reading the list of files specified either from the provided
+    # file path or STDIN
+    # @param from_list option value, either a file path or "@-"
+    # @return list of files from the from_list
+    def self.read_from_list(from_list)
+      from_list = from_list.strip
+      if from_list.empty?
+        logger.failure("List parameter must not be empty")
+        exit 1
+      end
+      
+      if from_list == '@-'
+        source_stream = $stdin
+      else
+        begin
+          source_stream = File.new(from_list)
+        rescue Errno::ENOENT
+          logger.failure("Specified list file does not exist: #{from_list}")
+          exit 1
+        end
+      end
+      
+      lines = []
+      source_stream.each_line do |line|
+        lines << line.strip
+      end
+      
+      if lines.empty?
+        logger.failure("File list is empty, must provide one or more files for this operation")
+        exit 1
+      end
+      lines
+    end
+
+    def self.fail(message)
+      logger.failure(message)
+      exit 1
+    end
+  end
+end

data/lib/longleaf/helpers/service_date_helper.rb

@@ -0,0 +1,78 @@
+require 'time'
+
+module Longleaf
+  # Helper methods for interacting with dates/timestamps on services
+  class ServiceDateHelper
+    # Adds the amount of time from modifier to the provided timestamp
+    # @param timestamp [String] ISO-8601 timestamp string
+    # @param modifier [String] amount of time to add to the timestamp. It must follow the syntax
+    # "<quantity> <time unit>", where quantity must be a positive whole number and time unit
+    # must be second, minute, hour, day, week, month or year (unit may be plural).
+    # Any info after a comma will be ignored.
+    # @return [String] the original timestamp in ISO-8601 format with the provided amount of time added.
+    def self.add_to_timestamp(timestamp, modifier)
+      if modifier =~ /^(\d+) *(second|minute|hour|day|week|month|year)s?(,.*)?/
+        value = $1.to_i
+        unit = $2
+      else
+        raise ArgumentError.new("Cannot parse time modifier #{modifier}")
+      end
+
+      datetime = Time.iso8601(timestamp)
+      case unit
+      when 'second'
+        unit_modifier = 1
+      when 'minute'
+        unit_modifier = 60
+      when 'hour'
+        unit_modifier = 3600
+      when 'day'
+        unit_modifier = 24 * 3600
+      when 'week'
+        unit_modifier = 7 * 24 * 3600
+      when 'month'
+        unit_modifier = 30 * 24 * 3600
+      when 'year'
+        unit_modifier = 365 * 24 * 3600
+      end
+
+      modified_time = datetime + (value * unit_modifier)
+      modified_time.iso8601(3)
+    end
+
+    # Get a timestamp in the format expected for service timestamps.
+    # @param timestamp [Time] the time to format. Defaults to now.
+    # @return [String] the time formatted as iso8601
+    def self.formatted_timestamp(timestamp = Time.now)
+      timestamp.utc.iso8601(3).to_s
+    end
+
+    # Get the timestamp for the next time the provided service would need to be run
+    # for the object described by md_rec
+    # @param md_rec [MetadataRecord] metadata record for the file
+    # @param service_def [ServiceDefinition] definition for the service
+    # @return [String] iso8601 timestamp for the next time the service will need to run, or
+    #    nil if the service does not need to run again.
+    def self.next_run_needed(md_rec, service_def)
+      raise ArgumentError.new('Must provide a md_rec parameter') if md_rec.nil?
+      raise ArgumentError.new('Must provide a service_def parameter') if service_def.nil?
+
+      service_name = service_def.name
+      service_rec = md_rec.service(service_name)
+
+      if service_rec.nil? || service_rec.timestamp.nil?
+        if service_def.delay.nil?
+          return md_rec.registered
+        else
+          return ServiceDateHelper.add_to_timestamp(md_rec.registered, service_def.delay)
+        end
+      end
+
+      if service_def.frequency.nil?
+        return nil
+      else
+        return ServiceDateHelper.add_to_timestamp(service_rec.timestamp, service_def.frequency)
+      end
+    end
+  end
+end

data/lib/longleaf/indexing/index_manager.rb

@@ -0,0 +1,101 @@
+require 'longleaf/models/system_config_fields'
+require 'longleaf/services/metadata_persistence_manager'
+require 'longleaf/errors'
+
+module Longleaf
+  # Manager configures and provides access to a metadata index if one is specified
+  class IndexManager
+    SYS_FIELDS ||= Longleaf::SystemConfigFields
+
+    # @param config [Hash] The system configuration as a hash
+    # @param app_config_manager [ApplicationConfigManager] the application config
+    def initialize(config, app_config_manager)
+      @config = config
+      @app_config_manager = app_config_manager
+      init_index_driver if @config&.key?(SYS_FIELDS::MD_INDEX)
+    end
+
+    # @return true if the system is configured to use a metadata index
+    def using_index?
+      !@index_driver.nil?
+    end
+
+    # Index the provided file_rec and its metadata
+    #
+    # @param file_rec [FileRecord] file record to index
+    def index(file_rec)
+      @index_driver.index(file_rec)
+    end
+
+    # Remove an entry from the index
+    # @param remove_me The record to remove from the index
+    def remove(remove_me)
+      @index_driver.remove(remove_me)
+    end
+
+    def clear_index(older_than = nil)
+      @index_driver.clear_index(older_than)
+    end
+
+    # @return true if the index should be reindexed
+    def index_stale?
+      @index_driver.is_stale?
+    end
+
+    # Setup initial structure of index implementation
+    def setup_index
+      @index_driver.setup_index
+    end
+
+    def update_index_state
+      @index_driver.update_index_state
+    end
+
+    # Retrieves a set of which have one or more services which need to run.
+    #
+    # @param file_selector [FileSelector] selector for paths to search for files
+    # @param stale_datetime [DateTime] find file_paths with services needing to be run before this value
+    # @return [Array] array of file paths that need one or more services run, in ascending order by
+    #    timestamp.
+    def paths_with_stale_services(file_selector, stale_datetime)
+      @index_driver.paths_with_stale_services(file_selector, stale_datetime)
+    end
+
+    # Retrieves a page of paths for registered files.
+    # @param file_selector [FileSelector] selector for what paths to search for files
+    # @return [Array] array of file paths that are registered
+    def registered_paths(file_selector)
+      @index_driver.registered_paths(file_selector)
+    end
+
+    def each_registered_path(file_selector, older_than: nil, &block)
+      @index_driver.each_registered_path(file_selector, older_than: older_than, &block)
+    end
+
+    private
+    def init_index_driver
+      index_conf = @config[SYS_FIELDS::MD_INDEX]
+      adapter = index_conf[SYS_FIELDS::MD_INDEX_ADAPTER]&.downcase
+
+      raise ConfigurationError.new('Must specify an adapter for the metadata index') if adapter.nil?
+
+      adapter = adapter.to_sym
+
+      case adapter
+      when :postgres, :mysql, :mysql2, :sqlite, :amalgalite
+        page_size = index_conf[SYS_FIELDS::MD_INDEX_PAGE_SIZE]&.to_int
+
+        connection = index_conf[SYS_FIELDS::MD_INDEX_CONNECTION]
+        raise ConfigurationError.new("Must specify connection details for index adapter of type '#{adapter}'") if connection.nil?
+
+        require 'longleaf/indexing/sequel_index_driver'
+        @index_driver = SequelIndexDriver.new(@app_config_manager,
+            adapter,
+            connection,
+            page_size: page_size)
+      else
+        raise ConfigurationError.new("Unknown index adapter '#{adapter}' specified.") if adapter.nil?
+      end
+    end
+  end
+end