longleaf 0.3.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c9420b7edb662a7e6cd5b09d7d1b60aabc135ffcfd517ecdea0eb9237c366887
-  data.tar.gz: 3f91cb150afab07a1943d8a2d29f58e87e6bf74b583746cbbe1a229241ac7e1c
+  metadata.gz: a5f659d06bd9b1094ed6dd0e10d108282bacde2522e4f07f5a2118dcbffb030e
+  data.tar.gz: '0368bb1228372a7b617aa148844f8cc88fbd453b3a704aa0c66d2d1b0abd21ef'
 SHA512:
-  metadata.gz: 43b0469d63ad3390b06e7721d05c539b9415ab447b3e7b9c89b4d9b380bbf062c2c4a5c8ca990c8084914443de11e0ffab3edd4e9c6a4176252c4327bcb02d9b
-  data.tar.gz: 80f3b78cd45ccc1914a16fc2e049e667aebf584f8551870f3a7a843f303521e566cc29a660cb42a07d3e2b0884f5531a36e199b8fafe0e53f7300a959d44f310
+  metadata.gz: 12917813fba8f517585d9e20d22044fce229451b5a22b10edd44f3accaea807a4049d867980fb8b301202f6d0bd4f047b0a06df78d46d054e8e7cc4eeeda9cb6
+  data.tar.gz: fe2f94a2f0bdbe5b2095c63275adca26a5c0f8a6d8dd013bdfa9a1465073085f80a8f2e32b0d5aa7b53c19cce58e48bb1fa9b16479db9b3bcd0af4c649128188

data/.circleci/config.yml

@@ -53,24 +53,34 @@ commands:
 
       - store_artifacts:
           path: coverage
+  install_new_bundler:
+    steps:
+      - run:
+          name: upgrade bundler
+          command: |
+            sudo gem update --system
+            sudo gem uninstall bundler
+            sudo rm /usr/local/bin/bundle
+            sudo gem install bundler
 
 jobs:
   test_with_ruby_23:
     docker:
       - image: circleci/ruby:2.3
         environment:
-          BUNDLER_VERSION: 1.17.3
+          BUNDLER_VERSION: 2.1.4
 
     working_directory: ~/repo
 
     steps:
+      - install_new_bundler
       - build_and_run_tests
 
   test_with_ruby_latest:
     docker:
       - image: circleci/ruby:latest
         environment:
-          BUNDLER_VERSION: 1.17.3
+          BUNDLER_VERSION: 2.1.4
 
     working_directory: ~/repo
 

data/README.md

@@ -1,7 +1,7 @@
 # Longleaf
 Code: [![CircleCI](https://circleci.com/gh/UNC-Libraries/longleaf-preservation.svg?style=svg)](https://circleci.com/gh/UNC-Libraries/longleaf-preservation)
 
-Longleaf is a command-line tool which allows users to configure a set of storage locations and define custom sets of preservation services to run on their contents. These services are executed in response to applicable preservation events issued by clients. Its primary goal is to provide tools to create a simple and customizable preservation environment.  Longleaf:
+Longleaf is a command-line tool which allows users to configure a set of storage locations and define custom sets of preservation services to run on their contents. These services are executed in response to applicable preservation events issued by clients. Its primary goal is to provide tools to create a simple and customizable preservation environment. Longleaf:
 
 * Offers a predictable command-line interface and integrates with standard command-line tools.
 * Offers configurable and customizable criteria based preservation workflows.
@@ -108,6 +108,16 @@ bundle install --with postgres
 
 Options include: postgres, mysql2, mysql, sqlite, amalgalite
 
+To setup an index, you will need to add a `system > index` section to your configuration with the details of the database to use for the index. Then to setup the database, run:
+
+```
+longleaf setup_index -c <config_file>
+```
+And for a one-time indexing:
+```
+longleaf reindex -c <config_file>
+```
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/UNC-Libraries/longleaf-preservation.

data/lib/longleaf/candidates/manifest_digest_provider.rb

@@ -0,0 +1,17 @@
+module Longleaf
+  # Provides digests for files from a manifest
+  class ManifestDigestProvider
+    # @param hash which maps file paths to hashs of digests
+    def initialize(digests_mapping)
+      @digests_mapping = digests_mapping
+    end
+
+    # @param file_path [String] path of file
+    # @return hash containing all the manifested digests for the given path, or nil
+    def get_digests(file_path)
+      # return nil if key not found, in case the hash has default values
+      return nil unless @digests_mapping.key?(file_path)
+      @digests_mapping[file_path]
+    end
+  end
+end

data/lib/longleaf/candidates/single_digest_provider.rb

@@ -0,0 +1,13 @@
+module Longleaf
+  # Provides a single set of digests for files
+  class SingleDigestProvider
+    def initialize(digests)
+      @digests = digests
+    end
+
+    def get_digests(file_path)
+      return nil if @digests.nil?
+      @digests
+    end
+  end
+end

data/lib/longleaf/cli.rb

@@ -9,8 +9,7 @@ require 'longleaf/commands/validate_metadata_command'
 require 'longleaf/commands/register_command'
 require 'longleaf/commands/reindex_command'
 require 'longleaf/commands/preserve_command'
-require 'longleaf/candidates/file_selector'
-require 'longleaf/candidates/registered_file_selector'
+require 'longleaf/helpers/selection_options_parser'
 
 module Longleaf
   # Main commandline interface setup for Longleaf using Thor.
@@ -72,6 +71,16 @@ module Longleaf
               :required => false,
               :desc => 'Name or comma separated names of storage locations to perform this operation over.' })
 
+    add_shared_option(
+        :from_list, :registered_selection, {
+              :aliases => "-l",
+              :required => false,
+              :desc => %q{Provide a list of files to perform this operation on. The list must be new line separated, one file per line.
+                To provide a list from a file:
+                '-l /path/to/file_list.txt'
+                To provide a list from STDIN:
+                '-l @-'}})
+
     # Commands
     map %w[--version] => :__print_version
     desc "--version", "Prints the Longleaf version number."
@@ -81,12 +90,39 @@ module Longleaf
 
     desc "register", "Register files with Longleaf"
     shared_options_group(:file_selection)
+    method_option(:manifest,
+        :aliases => "-m",
+        :type => :array,
+        :desc => %q{Checksum manifests of files to register. Supports the following formats:
+          To submit a md5 manifest from a file
+          '-m md5:/path/to/manifest.txt'
+
+          To provide a sha1 manifest from STDIN
+          '-m sha1:@-'
+          Where the content in STDIN adheres to the format:
+          <digest> <path>
+          <digest> <path>
+          ...
+
+          To submit multiple manifests from files
+          '-m md5:/path/to/manifest1.txt sha1:/path/to/manifest2.txt'
+
+          To provide multiple digests via STDIN
+          '-m @-'
+          Where the content in STDIN adheres to the following format:
+          sha1:
+          <digest> <path>
+          ...
+          md5:
+          <digest> <path>
+          ...})
     method_option(:force,
         :type => :boolean,
         :default => false,
         :desc => 'Force the registration of already registered files.')
     method_option(:checksums,
-        :desc => %q{Checksums for the submitted file. Each checksum must be prefaced with an algorithm prefix. Multiple checksums must be comma separated. If multiple files were submitted, they will be provided with the same checksums. For example:
+        :desc => %q{Checksums for the submitted file. Only applicable with the -f option.
+          Each checksum must be prefaced with an algorithm prefix. Multiple checksums must be comma separated. If multiple files were submitted, they will be provided with the same checksums. For example:
           '--checksums "md5:d8e8fca2dc0f896fd7cb4cb0031ba249,sha1:4e1243bd22c66e76c2ba9eddc1f91394e57f9f83"'})
     shared_options_group(:common)
     # Register event command
@@ -96,25 +132,16 @@ module Longleaf
 
       app_config_manager = load_application_config(options)
 
-      file_selector = create_file_selector(options, app_config_manager)
-      if options[:checksums]
-        checksums = options[:checksums]
-        # validate checksum list format, must a comma delimited list of prefix:checksums
-        if /^[^:,]+:[^:,]+(,[^:,]+:[^:,]+)*$/.match(checksums)
-          # convert checksum list into hash with prefix as key
-          checksums = Hash[*checksums.split(/\s*[:,]\s*/)]
-        else
-          logger.failure("Invalid checksums parameter format, see `longleaf help <command>` for more information")
-          exit 1
-        end
-      end
+      file_selector, digest_provider = SelectionOptionsParser.parse_registration_selection_options(
+          options, app_config_manager)
 
       command = RegisterCommand.new(app_config_manager)
-      exit command.execute(file_selector: file_selector, force: options[:force], checksums: checksums)
+      exit command.execute(file_selector: file_selector, force: options[:force], digest_provider: digest_provider)
     end
 
     desc "deregister", "Deregister files with Longleaf"
     shared_options_group(:file_selection)
+    shared_options_group(:registered_selection)
     method_option(:force,
         :type => :boolean,
         :default => false,
@@ -126,7 +153,7 @@ module Longleaf
       setup_logger(options)
 
       app_config_manager = load_application_config(options)
-      file_selector = create_registered_selector(options, app_config_manager)
+      file_selector = SelectionOptionsParser.create_registered_selector(options, app_config_manager)
 
       command = DeregisterCommand.new(app_config_manager)
       exit command.execute(file_selector: file_selector, force: options[:force])
@@ -134,6 +161,7 @@ module Longleaf
 
     desc "preserve", "Perform preservation services on files with Longleaf"
     shared_options_group(:file_selection)
+    shared_options_group(:registered_selection)
     method_option(:force,
         :type => :boolean,
         :default => false,
@@ -145,7 +173,7 @@ module Longleaf
 
       extend_load_path(options[:load_path])
       app_config_manager = load_application_config(options)
-      file_selector = create_registered_selector(options, app_config_manager)
+      file_selector = SelectionOptionsParser.create_registered_selector(options, app_config_manager)
 
       command = PreserveCommand.new(app_config_manager)
       exit command.execute(file_selector: file_selector, force: options[:force])
@@ -164,6 +192,7 @@ module Longleaf
 
     desc "validate_metadata", "Validate metadata files."
     shared_options_group(:file_selection)
+    shared_options_group(:registered_selection)
     shared_options_group(:common)
     # File metadata validation command
     def validate_metadata
@@ -171,7 +200,7 @@ module Longleaf
       setup_logger(options)
 
       app_config_manager = load_application_config(options)
-      file_selector = create_registered_selector(options, app_config_manager)
+      file_selector = SelectionOptionsParser.create_registered_selector(options, app_config_manager)
 
       exit Longleaf::ValidateMetadataCommand.new(app_config_manager).execute(file_selector: file_selector)
     end
@@ -220,7 +249,7 @@ module Longleaf
         begin
           app_manager = ApplicationConfigDeserializer.deserialize(options[:config])
         rescue ConfigurationError => err
-          logger.failure("Failed to load application configuration due to the following issue:\n#{err.message}")
+          logger.failure("Failed to load application configuration due to the following issue(s):\n#{err.message}")
           exit 1
         end
       end
@@ -231,22 +260,6 @@ module Longleaf
         end
       end
 
-      def create_file_selector(options, app_config_manager, selector_class: FileSelector)
-        file_paths = options[:file]&.split(/\s*,\s*/)
-        storage_locations = options[:location]&.split(/\s*,\s*/)
-
-        begin
-          selector_class.new(file_paths: file_paths, storage_locations: storage_locations, app_config: app_config_manager)
-        rescue ArgumentError => e
-          logger.failure(e.message)
-          exit 1
-        end
-      end
-
-      def create_registered_selector(options, app_config_manager)
-        create_file_selector(options, app_config_manager, selector_class: RegisteredFileSelector)
-      end
-
       def extend_load_path(load_paths)
         load_paths = load_paths&.split(/\s*,\s*/)
         load_paths&.each { |path| $LOAD_PATH.unshift(path) }

data/lib/longleaf/commands/register_command.rb

@@ -16,9 +16,9 @@ module Longleaf
     # Execute the register command on the given parameters
     # @param file_selector [FileSelector] selector for files to register
     # @param force [Boolean] force flag
-    # @param checksums [Array] array of checksums
+    # @param digest_provider [DigestProvider] object which provides digests for files being registered
     # @return [Integer] status code
-    def execute(file_selector:, force: false, checksums: nil)
+    def execute(file_selector:, force: false, digest_provider: nil)
       start_time = Time.now
       logger.info('Performing register command')
       begin
@@ -32,7 +32,7 @@ module Longleaf
           file_rec = FileRecord.new(f_path, storage_location)
 
           register_event = RegisterEvent.new(file_rec: file_rec, force: force, app_manager: @app_manager,
-              checksums: checksums)
+              digest_provider: digest_provider)
           track_status(register_event.perform)
         end
       rescue InvalidStoragePathError, StorageLocationUnavailableError => err

data/lib/longleaf/commands/validate_config_command.rb

@@ -26,7 +26,7 @@ module Longleaf
 
         record_success("Application configuration passed validation: #{@config_path}")
       rescue Longleaf::ConfigurationError, Longleaf::StorageLocationUnavailableError => err
-        record_failure("Application configuration invalid due to the following issue:\n#{err.message}")
+        record_failure("Application configuration invalid due to the following issue(s):\n#{err.message}")
       rescue => err
         record_failure("Failed to validate application configuration", error: err)
       end

data/lib/longleaf/events/register_event.rb

@@ -14,7 +14,8 @@ module Longleaf
     # @param file_rec [FileRecord] file record
     # @param app_manager [ApplicationConfigManager] the application configuration
     # @param force [boolean] if true, then already registered files will be re-registered
-    def initialize(file_rec:, app_manager:, force: false, checksums: nil)
+    # @param digest_provider [#get_digests] object which provides digests for files being registered
+    def initialize(file_rec:, app_manager:, force: false, digest_provider: nil)
       raise ArgumentError.new('Must provide a file_rec parameter') if file_rec.nil?
       raise ArgumentError.new('Parameter file_rec must be a FileRecord') \
           unless file_rec.is_a?(FileRecord)
@@ -25,7 +26,7 @@ module Longleaf
       @app_manager = app_manager
       @file_rec = file_rec
       @force = force
-      @checksums = checksums
+      @digest_provider = digest_provider
     end
 
     # Perform a registration event on the given file
@@ -48,7 +49,10 @@ module Longleaf
 
         populate_file_properties
 
-        md_rec.checksums.merge!(@checksums) unless @checksums.nil?
+        if !@digest_provider.nil?
+          checksums = @digest_provider.get_digests(@file_rec.path)
+          md_rec.checksums.merge!(checksums) unless checksums.nil?
+        end
 
         # persist the metadata
         @app_manager.md_manager.persist(@file_rec)
@@ -77,7 +81,7 @@ module Longleaf
       md_rec = @file_rec.metadata_record
 
       old_md = MetadataDeserializer.deserialize(file_path: @file_rec.metadata_path,
-              digest_algs: @file_rec.storage_location.metadata_digests)
+              digest_algs: @file_rec.storage_location.metadata_location.digests)
       # Copy custom properties
       old_md.properties.each { |name, value| md_rec.properties[name] = value }
       # Copy stale-replicas flag per service

data/lib/longleaf/helpers/case_insensitive_hash.rb

@@ -0,0 +1,38 @@
+module Longleaf
+  # Hash subclass which provides case insensitive keys, where keys are always downcased.
+  class CaseInsensitiveHash < Hash
+    def [](key)
+      super _insensitive(key)
+    end
+
+    def []=(key, value)
+      super _insensitive(key), value
+    end
+
+    def delete(key)
+      super _insensitive(key)
+    end
+
+    def has_key?(key)
+      super _insensitive(key)
+    end
+
+    def merge(other_hash)
+      super other_hash.map {|k, v| [_insensitive(k), v] }.to_h
+    end
+
+    def merge!(other_hash)
+      super other_hash.map {|k, v| [_insensitive(k), v] }.to_h
+    end
+
+    # Cause this hash to serialize as a regular hash to avoid deserialization failures
+    def encode_with coder
+      coder.represent_map nil, self
+    end
+
+    protected
+    def _insensitive(key)
+      key.respond_to?(:downcase) ? key.downcase : key
+    end
+  end
+end

data/lib/longleaf/helpers/digest_helper.rb

@@ -13,7 +13,7 @@ module Longleaf
     def self.validate_algorithms(algs)
       return if algs.nil?
       if algs.is_a?(String)
-        unless KNOWN_DIGESTS.include?(algs)
+        unless self.is_known_algorithm?(algs)
           raise InvalidDigestAlgorithmError.new("Unknown digest algorithm #{algs}")
         end
       else
@@ -24,6 +24,12 @@ module Longleaf
       end
     end
 
+    # @param alg [String] identifier of digest algorithm
+    # @return [Boolean] true if the digest is a valid known algorithm
+    def self.is_known_algorithm?(alg)
+      KNOWN_DIGESTS.include?(alg)
+    end
+
     # Get a Digest class for the specified algorithm
     # @param alg [String] name of the digest algorithm
     # @return [Digest] A digest class for the requested algorithm

data/lib/longleaf/helpers/s3_uri_helper.rb

@@ -0,0 +1,86 @@
+require 'uri'
+
+module Longleaf
+  # Helper for interacting with s3 uris
+  class S3UriHelper
+    ENDPOINT_PATTERN = /^(.+\.)?s3[.\-]([a-z0-9\-]+[\-.])?[a-z0-9]+\./
+    ALLOWED_SCHEMES = ['http', 'https', 's3']
+
+    # Extract the name of the s3 bucket from the provided url
+    # @param url s3 url
+    # @return the name of the bucket, or nil if the name could not be identified
+    def self.extract_bucket(url)
+      uri = s3_uri(url)
+
+      matches = ENDPOINT_PATTERN.match(uri.host)
+      if matches.nil?
+        raise ArgumentError.new("Provided URI does match the expected pattern for an S3 URI")
+      end
+
+      prefix = matches[1]
+      if prefix.nil? || prefix.empty?
+        # Is a path style url
+        path = uri.path
+
+        return nil if path == '/'
+
+        path_parts = path.split('/')
+        return nil if path_parts.empty?
+        return path_parts[1]
+      else
+        return prefix[0..-2]
+      end
+    end
+
+    def self.extract_path(url)
+      uri = s3_uri(url)
+
+      matches = ENDPOINT_PATTERN.match(uri.host)
+      if matches.nil?
+        raise ArgumentError.new("Provided URI does match the expected pattern for an S3 URI")
+      end
+
+      path = uri.path
+      return nil if path == '/' || path.empty?
+
+      # trim off the first slash
+      path = path.partition('/').last
+
+      # Determine if the first part of the path is the bucket name
+      prefix = matches[1]
+      if prefix.nil? || prefix.empty?
+        # trim off the bucket name
+        path = path.partition('/').last
+      end
+
+      path
+    end
+
+    def self.extract_region(url)
+      uri = s3_uri(url)
+
+      matches = ENDPOINT_PATTERN.match(uri.host)
+
+      if matches[2].nil?
+        # No region specified
+        nil
+      else
+        matches[2][0..-2]
+      end
+    end
+
+    def self.s3_uri(url)
+      if url.nil?
+        raise ArgumentError.new("url cannot be empty")
+      end
+      uri = URI(url)
+      if !ALLOWED_SCHEMES.include?(uri.scheme&.downcase)
+        raise ArgumentError.new("Invalid scheme for s3 URI #{url}, only http, https and s3 are permitted")
+      end
+      if uri.host.nil?
+        raise ArgumentError.new("Invalid S3 URI, no hostname: #{url}")
+      end
+      uri
+    end
+  end
+end