longleaf 0.1.0.pre.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 41b149cc264dcc00b79558d1acc06fd42c7810e8b06fe62627baf359fb23a784
+  data.tar.gz: f4634c72c87d9858faace5e8c480d96b85365a37e989addfac8d2a044b957d08
+SHA512:
+  metadata.gz: 07aa3758499e04ff45f1fcbf1bd9e51af22b0d64ef4314ba784dc1eb0fae78bffeb6faf285e0fbe107761cf6c0c89dd024b814eb6a130821718a53d985eebe47
+  data.tar.gz: 4546b08c4e3119f783bd7134b0a49ee7cb0f022509829122b136f872fad95b38c8c31a4db1eed021d51adb639e4cf63c2972cfeeeeaeda8cef194b026fd9051b

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 2.3.0
+before_install: gem install bundler -v 1.11.2

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in longleaf.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,13 @@
+Copyright 2018 UNC Libraries
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,43 @@
+# Longleaf
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/longleaf`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'longleaf'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install longleaf
+
+## Usage
+
+#### Validate configuration files
+Application configuration files can be validated prior to usage with the following command:
+`longleaf validate_config <config.yml>`
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `bundle exec rspec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/longleaf.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "longleaf"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/longleaf

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require 'longleaf/cli'
+Longleaf::CLI.start

data/lib/longleaf/cli.rb

@@ -0,0 +1,78 @@
+require 'thor'
+require 'yaml'
+require 'longleaf/logging'
+require 'longleaf/errors'
+require 'longleaf/commands/validate_config_command'
+require 'longleaf/commands/register_command'
+
+module Longleaf
+  class CLI < Thor
+    include Longleaf::Logging
+    
+    class_option(:config, :aliases => "-c",
+        :default => ENV['LONGLEAF_CFG'],
+        :desc => 'Absolute path to the application configuration used for this command. By default, the value of the environment variable LONGLEAF_CFG is used.')
+    # Logging/output options
+    class_option(:failure_only,
+        :type => :boolean,
+        :default => false,
+        :desc => 'Only display failure messages to STDOUT.')
+    class_option(:log_level,
+        :default => 'WARN',
+        :desc => 'Level of logging to send to STDERR, following standard ruby Logger levels. This includes: DEBUG, INFO, WARN, ERROR, FATAL, UNKNOWN. Default is WARN.')
+    class_option(:log_format,
+        :desc => 'Format to use for log information sent to STDERR. Can contain the following parameters, which must be wrapped in %{}: severity, datetime, progname, msg. Default is "%{severity} [%{datetime}]: %{msg}"')
+    class_option(:log_datetime,
+        :desc => 'Format to use for timestamps used in logging to STDERR, following strftime syntax.')
+    
+    desc "register", "Register files with Longleaf"
+    method_option(:file, :aliases => "-f", 
+        :required => true,
+        :desc => 'File or files to register. Paths must be absolute. If multiple files are provided, they must be comma separated.')
+    method_option(:force,
+        :type => :boolean, 
+        :default => false,
+        :desc => 'Force the registration of already registered files.')
+    method_option(:checksums,
+        :desc => %q{Checksums for the submitted file. Each checksum must be prefaced with an algorithm prefix. Multiple checksums must be comma separated. If multiple files were submitted, they will be provided with the same checksums. For example: 
+          '--checksums "md5:d8e8fca2dc0f896fd7cb4cb0031ba249,sha1:4e1243bd22c66e76c2ba9eddc1f91394e57f9f83"'})
+    # Register event command
+    def register
+      setup_logger(options)
+      
+      config_path = options[:config]
+      file_paths = options[:file]&.split(/\s*,\s*/)
+      if options[:checksums]
+        checksums = options[:checksums]
+        # validate checksum list format, must a comma delimited list of prefix:checksums
+        if /^[^:,]+:[^:,]+(,[^:,]+:[^:,]+)*$/.match(checksums)
+          # convert checksum list into hash with prefix as key
+          checksums = Hash[*checksums.split(/\s*[:,]\s*/)]
+        else
+          logger.failure("Invalid checksums parameter format, see `longleaf help <command>` for more information")
+          exit 1
+        end
+      end
+      
+      command = Longleaf::RegisterCommand.new(config_path)
+      exit command.execute(file_paths: file_paths, force: options[:force], checksums: checksums)
+    end
+    
+    desc "validate_config [CONFIG_PATH]", "Validate an application configuration file"
+    # Application configuration validation command
+    def validate_config(config_path)
+      setup_logger(options)
+      
+      exit Longleaf::ValidateConfigCommand.new(config_path).execute
+    end
+    
+    no_commands do
+      def setup_logger(options)
+        initialize_logger(options[:failure_only],
+            options[:log_level],
+            options[:log_format],
+            options[:log_datetime])
+      end
+    end
+  end
+end

data/lib/longleaf/commands/abstract_command.rb

@@ -0,0 +1,37 @@
+require 'longleaf/logging'
+
+# Parent class for longleaf commands
+module Longleaf
+  class AbstractCommand
+    include Longleaf::Logging
+    
+    # Record a successful operation to the output and the overall status of this command.
+    # @param args [Array] arguments to pass to logger
+    def record_success(*args)
+      logger.success(*args)
+      if @return_status.nil? || @return_status == 0
+        @return_status = 0
+      else
+        @return_status = 2
+      end
+    end
+    
+    # Record a failed operation to the output and the overall status of this command.
+    # @param args [Array] arguments to pass to logger
+    def record_failure(*args)
+      logger.failure(*args)
+      if @return_status.nil? || @return_status == 1
+        @return_status = 1
+      else
+        @return_status = 2
+      end
+    end
+    
+    # @return [Integer] the return status for this command, where 0 indicates success,
+    #    1 indicates failure, and 2 indicates partial failure
+    def return_status
+      @return_status = 0 if @return_status.nil?
+      @return_status
+    end
+  end
+end

data/lib/longleaf/commands/register_command.rb

@@ -0,0 +1,59 @@
+require 'longleaf/services/application_config_deserializer'
+require 'longleaf/events/register_event'
+require 'longleaf/models/file_record'
+require 'longleaf/commands/abstract_command'
+
+# Command for registering files with longleaf
+module Longleaf
+  class RegisterCommand < AbstractCommand
+    
+    def initialize(config_path)
+      @config_path = config_path
+    end
+
+    # Execute the register command on the given parameters
+    def execute(file_paths: nil, force: false, checksums: nil)
+      if file_paths.nil? || file_paths.empty?
+        record_failure("Must provide one or more file paths to register")
+        return return_status
+      end
+      
+      begin
+        # Retrieve the application configuration
+        app_manager = Longleaf::ApplicationConfigDeserializer.deserialize(@config_path)
+        
+        # Perform register events on each of the file paths provided
+        file_paths.each do |f_path|
+          begin
+            storage_location = app_manager.location_manager.get_location_by_path(f_path)
+            if storage_location.nil?
+              raise InvalidStoragePathError.new(
+                  "Unable to register '#{f_path}', it does not belong to any registered storage locations.")
+            end
+          
+            raise InvalidStoragePathError.new("Unable to register '#{f_path}', file does not exist or is unreachable.") \
+                unless File.file?(f_path)
+          
+            file_rec = FileRecord.new(f_path, storage_location)
+            
+            register_event = RegisterEvent.new(file_rec: file_rec, force: force, app_manager: app_manager,
+                checksums: checksums)
+            register_event.perform
+            
+            record_success(RegisterEvent::EVENT_NAME, f_path)
+          rescue RegistrationError => err
+            record_failure(RegisterEvent::EVENT_NAME, f_path, err.message)
+          rescue InvalidStoragePathError => err
+            record_failure(RegisterEvent::EVENT_NAME, f_path, err.message)
+          end
+        end
+      rescue ConfigurationError => err
+        record_failure("Failed to load application configuration due to the following issue:\n#{err.message}")
+      rescue => err
+        record_failure(RegisterEvent::EVENT_NAME, error: err)
+      end
+      
+      return_status
+    end
+  end
+end

data/lib/longleaf/commands/validate_config_command.rb

@@ -0,0 +1,29 @@
+require 'longleaf/services/application_config_deserializer'
+require 'longleaf/commands/abstract_command'
+
+module Longleaf
+  class ValidateConfigCommand < AbstractCommand
+    def initialize(config_path)
+      @config_path = config_path
+    end
+    
+    def execute
+      begin
+        app_config_manager = Longleaf::ApplicationConfigDeserializer.deserialize(@config_path)
+        
+        location_manager = app_config_manager.location_manager
+        location_manager.locations.each do |name, location|
+          location.available?
+        end
+        
+        record_success("Application configuration passed validation: #{@config_path}")
+      rescue Longleaf::ConfigurationError, Longleaf::StorageLocationUnavailableError => err
+        record_failure("Application configuration invalid due to the following issue:\n#{err.message}")
+      rescue => err
+        record_failure("Failed to validate application configuration", error: err)
+      end
+      
+      return_status
+    end
+  end
+end

data/lib/longleaf/errors.rb

@@ -0,0 +1,15 @@
+module Longleaf
+  class LongleafError < StandardError; end
+  
+  class ConfigurationError < LongleafError; end
+  
+  class InvalidStoragePathError < LongleafError; end
+  
+  class MetadataError < LongleafError; end
+  
+  class StorageLocationUnavailableError < LongleafError; end
+  
+  class EventError < LongleafError; end
+  
+  class RegistrationError < EventError; end
+end

data/lib/longleaf/events/register_event.rb

@@ -0,0 +1,98 @@
+require 'longleaf/errors'
+require 'longleaf/models/metadata_record'
+require 'longleaf/services/metadata_deserializer'
+require 'longleaf/services/metadata_serializer'
+require 'time'
+
+# Event to register a file with longleaf
+module Longleaf
+  class RegisterEvent
+    EVENT_NAME = 'register'
+    
+    # @param file_rec [FileRecord] file record
+    # @param app_manager [ApplicationConfigManager] the application configuration
+    # @param force [boolean] if true, then already registered files will be re-registered
+    def initialize(file_rec:, app_manager:, force: false, checksums: nil)
+      raise ArgumentError.new('Must provide a file_rec parameter') if file_rec.nil?
+      raise ArgumentError.new('Parameter file_rec must be a FileRecord') \
+          unless file_rec.is_a?(FileRecord)
+      raise ArgumentError.new('Must provide an ApplicationConfigManager') if app_manager.nil?
+      raise ArgumentError.new('Parameter app_manager must be an ApplicationConfigManager') \
+          unless app_manager.is_a?(ApplicationConfigManager)
+      
+      @app_manager = app_manager
+      @file_rec = file_rec
+      @force = force
+      @checksums = checksums
+    end
+    
+    # Perform a registration event on the given file
+    # @raises RegistrationError if a file cannot be registered 
+    def perform
+      metadata_exists = File.file?(@file_rec.metadata_path)
+      # If the file's metadata exists, only need to register it if the force flag is provided
+      if metadata_exists && !@force
+        raise RegistrationError.new("Unable to register '#{@file_rec.path}', it is already registered.")
+      end
+      
+      # create metadata record
+      md_rec = MetadataRecord.new(registered: Time.now.utc.iso8601)
+      @file_rec.metadata_record = md_rec
+      
+      # retain significant details from former record
+      if metadata_exists
+        retain_existing_properties
+      end
+      
+      populate_file_properties
+      
+      md_rec.checksums.merge!(@checksums) unless @checksums.nil?
+      
+      populate_services
+      
+      # persist the metadata out to file
+      MetadataSerializer::write(metadata: md_rec, file_path: @file_rec.metadata_path)
+    end
+    
+    private
+    def populate_file_properties
+      md_rec = @file_rec.metadata_record
+      
+      # Set file properties
+      md_rec.last_modified = File.mtime(@file_rec.path).utc.iso8601
+      md_rec.file_size = File.size(@file_rec.path)
+    end
+    
+    def populate_services
+      md_rec = @file_rec.metadata_record
+      
+      service_manager = @app_manager.service_manager
+      definitions = service_manager.list_service_definitions(location: @file_rec.storage_location.name)
+      
+      # Add service section
+      definitions.each do |serv_def|
+        serv_name = serv_def.name
+        md_rec.add_service(serv_name)
+      end
+    end
+    
+    # Copy a subset of properties from an existing metadata record to the new record
+    def retain_existing_properties
+      md_rec = @file_rec.metadata_record
+      
+      old_md = MetadataDeserializer.deserialize(file_path: @file_rec.metadata_path)
+      # Copy custom properties
+      old_md.properties.each { |name, value| md_rec.properties[name] = value }
+      # Copy stale-replicas flag per service
+      old_md.list_services.each do |serv_name|
+        serv_rec = old_md.service(serv_name)
+        
+        stale_replicas = serv_rec.stale_replicas
+        if stale_replicas
+          new_service = md_rec.service(serv_name)
+          new_service.stale_replicas = stale_replicas unless new_service.nil?
+        end
+      end
+    end
+  end
+end

data/lib/longleaf/logging/redirecting_logger.rb

@@ -0,0 +1,131 @@
+require 'logger'
+
+# Logger which directs messages to stdout and/or stderr, depending on the nature of the message.
+# Status logging, which includes standard logger methods, goes to STDERR.
+# Operation success and failure messages go to STDOUT, and to STDERR at info level.
+module Longleaf
+  module Logging
+    class RedirectingLogger
+      # @param failure_only [Boolean] If set to true, only failure messages will be output to STDOUT
+      # @param log_level [String] logger level used for output to STDERR
+      # @param log_format [Strfailure_onlying] format string for log entries to STDERR. There are 4 variables available
+      #    for inclusion in the output: severity, datetime, progname, msg. Variables must be wrapped in %{}.
+      # @param datetime_format [String] datetime formatting string used for logger dates appearing in STDERR.
+      def initialize(failure_only: false, log_level: 'WARN', log_format: nil, datetime_format: nil)
+        @stderr_log = Logger.new($stderr)
+        @stderr_log.level = log_level
+        @stderr_log.datetime_format = datetime_format
+        @log_format = log_format
+        if @log_format.nil?
+          @stderr_log.formatter = proc do |severity, datetime, progname, msg|
+            formatted_date = @stderr_log.datetime_format.nil? ? datetime : datetime.strftime(datetime_format)
+            "#{severity} [#{formatted_date}]: #{msg}\n"
+          end
+        elsif @log_format.is_a?(String)
+          @stderr_log.formatter = proc do |severity, datetime, progname, msg|
+            # Make sure the format ends with a newline
+            @log_format = @log_format + "\n" unless @log_format.end_with?("\n")
+        
+            formatted_date = @stderr_log.datetime_format.nil? ? datetime : datetime.strftime(datetime_format)
+            @log_format % { :severity => severity, :datetime => formatted_date, :progname => progname, :msg => msg }
+          end
+        end
+      
+        @stdout_log = Logger.new($stdout)
+        @stdout_log.formatter = proc do |severity, datetime, progname, msg|
+          "#{msg}\n"
+        end
+        if failure_only
+          @stdout_log.level = 'warn'
+        else
+          @stdout_log.level = 'info'
+        end
+      end
+    
+      def debug(progname = nil, &block)
+        @stderr_log.debug(progname, &block)
+      end
+    
+      def info(progname = nil, &block)
+        @stderr_log.info(progname, &block)
+      end
+    
+      def warn(progname = nil, &block)
+        @stderr_log.warn(progname, &block)
+      end
+    
+      def error(progname = nil, &block)
+        @stderr_log.error(progname, &block)
+      end
+    
+      def fatal(progname = nil, &block)
+        @stderr_log.fatal(progname, &block)
+      end
+    
+      def unknown(progname = nil, &block)
+        @stderr_log.unknown(progname, &block)
+      end
+    
+      # Logs a success message to STDOUT, as well as STDERR at info level.
+      # @param eventOrMessage [String] name of the preservation event which succeeded, 
+      #    or the message to output if it is the only parameter. Required.
+      # @param file_name [String] file name which is the subject of this message.
+      # @param message [String] descriptive message to accompany this output
+      # @param service [String] name of the service which executed.
+      def success(eventOrMessage, file_name = nil, message = nil, service = nil)
+        outcome('SUCCESS', eventOrMessage, file_name, message, service)
+      end
+    
+      # Logs a failure message to STDOUT, as well as STDERR at info level.
+      # If an error was provided, it is logged to STDERR at error level.
+      # @param eventOrMessage [String] name of the preservation event which failed, 
+      #    or the message to output if it is the only parameter.
+      # @param file_name [String] file name which is the subject of this message.
+      # @param message [String] descriptive message to accompany this output
+      # @param service [String] name of the service which executed.
+      # @param error [Error] error which occurred
+      def failure(eventOrMessage, file_name = nil, message = nil, service = nil, error: nil)
+        text = outcome_text('FAILURE', eventOrMessage, file_name, message, service, error)
+        @stdout_log.warn(text)
+      
+        @stderr_log.info(text)
+        @stderr_log.error("#{error.message}") unless error.nil?
+      end
+    
+      # Logs an outcome message to STDOUT, as well as STDERR at info level.
+      # If file_name and message are nil, eventOrMessage will be used as the message.
+      #
+      # @param outcome [String] The status of the outcome. Required.
+      # @param eventOrMessage [String] name of the preservation event which was successful, 
+      #    or the message to output if it is the only parameter. Required.
+      # @param file_name [String] file name which is the subject of this message.
+      # @param message [String] descriptive message to accompany this output
+      # @param service [String] name of the service which executed.
+      # @param error [Error] error which occurred
+      def outcome(outcome, eventOrMessage, file_name = nil, message = nil, service = nil, error = nil)
+        text = outcome_text(outcome, eventOrMessage, file_name, message, service, error)
+        @stdout_log.info(text)
+        @stderr_log.info(text)
+      end
+    
+      # FAILURE verify[cdr_fixity_check] /path/to/file: Something terrible
+      private
+      def outcome_text(outcome, eventOrMessage, file_name = nil, message = nil, service = nil, error = nil)
+        message_only = file_name.nil? && message.nil? && error.nil?
+        
+        text = "#{outcome}"
+      
+        if message_only
+          text << ": #{eventOrMessage}"
+        else
+          text << " #{eventOrMessage}"
+          text << "[#{service}]" unless service.nil?
+          text << " #{file_name}" unless file_name.nil?
+          msg = message || error&.message
+          text << ": #{msg}" unless msg.nil?
+        end
+        text
+      end
+    end
+  end
+end

data/lib/longleaf/logging.rb

@@ -0,0 +1,26 @@
+require 'longleaf/logging/redirecting_logger'
+
+module Longleaf
+  module Logging
+    # Get the main logger for longleaf
+    def logger
+      Logging.logger
+    end
+    
+    # Get the main logger for longleaf
+    def self.logger
+      @logger ||= RedirectingLogger.new
+    end
+  
+    def initialize_logger(failure_only, log_level, log_format, datetime_format)
+      Logging.initialize_logger(failure_only, log_level, log_format, datetime_format)
+    end
+  
+    def self.initialize_logger(failure_only, log_level, log_format, datetime_format)
+      @logger = RedirectingLogger.new(failure_only: failure_only,
+          log_level: log_level,
+          log_format: log_format,
+          datetime_format: datetime_format)
+    end
+  end
+end

data/lib/longleaf/models/app_fields.rb

@@ -0,0 +1,10 @@
+module Longleaf
+  class AppFields
+    LOCATIONS = 'locations'
+    SERVICES = 'services'
+    SERVICE_MAPPINGS = 'service_mappings'
+    
+    LOCATION_PATH = 'path'
+    METADATA_PATH = 'metadata_path'
+  end
+end

data/lib/longleaf/models/file_record.rb

@@ -0,0 +1,25 @@
+# Record for an individual file and its associated information 
+module Longleaf
+  class FileRecord
+    
+    attr_accessor :metadata_record
+    attr_reader :storage_location
+    attr_reader :path
+    
+    # @param file_path [String] path to the file
+    # @param storage_location [Longleaf::StorageLocation] storage location containing the file
+    def initialize(file_path, storage_location)
+      raise ArgumentError.new("FileRecord requires a path") if file_path.nil?
+      raise ArgumentError.new("FileRecord requires a storage_location") if storage_location.nil?
+      
+      @path = file_path
+      @storage_location = storage_location
+    end
+    
+    # @return [String] path for the metadata file for this file
+    def metadata_path
+      @metadata_path = @storage_location.get_metadata_path_for(path) if @metadata_path.nil?
+      @metadata_path
+    end
+  end
+end

data/lib/longleaf/models/md_fields.rb

@@ -0,0 +1,18 @@
+module Longleaf
+  class MDFields
+    DATA = 'data'
+    SERVICES = 'services'
+    
+    REGISTERED_TIMESTAMP = 'registered'
+    DEREGISTERED_TIMESTAMP = 'deregistered'
+
+    LAST_MODIFIED = 'last-modified'
+    FILE_SIZE = 'size'
+
+    CHECKSUMS = 'checksums'
+
+    STALE_REPLICAS = 'stale-replicas'
+    SERVICE_TIMESTAMP = 'timestamp'
+    RUN_NEEDED = 'run-needed'
+  end
+end