longleaf 0.3.0 → 1.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (49) hide show
  1. checksums.yaml +4 -4
  2. data/.circleci/config.yml +12 -2
  3. data/README.md +11 -1
  4. data/lib/longleaf/candidates/manifest_digest_provider.rb +17 -0
  5. data/lib/longleaf/candidates/single_digest_provider.rb +13 -0
  6. data/lib/longleaf/cli.rb +49 -36
  7. data/lib/longleaf/commands/register_command.rb +3 -3
  8. data/lib/longleaf/commands/validate_config_command.rb +1 -1
  9. data/lib/longleaf/events/register_event.rb +8 -4
  10. data/lib/longleaf/helpers/case_insensitive_hash.rb +38 -0
  11. data/lib/longleaf/helpers/digest_helper.rb +7 -1
  12. data/lib/longleaf/helpers/s3_uri_helper.rb +86 -0
  13. data/lib/longleaf/helpers/selection_options_parser.rb +189 -0
  14. data/lib/longleaf/helpers/service_date_helper.rb +29 -1
  15. data/lib/longleaf/indexing/sequel_index_driver.rb +2 -20
  16. data/lib/longleaf/models/app_fields.rb +4 -2
  17. data/lib/longleaf/models/filesystem_metadata_location.rb +56 -0
  18. data/lib/longleaf/models/filesystem_storage_location.rb +52 -0
  19. data/lib/longleaf/models/metadata_location.rb +47 -0
  20. data/lib/longleaf/models/metadata_record.rb +3 -1
  21. data/lib/longleaf/models/s3_storage_location.rb +133 -0
  22. data/lib/longleaf/models/service_fields.rb +4 -0
  23. data/lib/longleaf/models/storage_location.rb +17 -48
  24. data/lib/longleaf/models/storage_types.rb +9 -0
  25. data/lib/longleaf/preservation_services/rsync_replication_service.rb +9 -11
  26. data/lib/longleaf/preservation_services/s3_replication_service.rb +143 -0
  27. data/lib/longleaf/services/application_config_deserializer.rb +26 -4
  28. data/lib/longleaf/services/application_config_validator.rb +17 -6
  29. data/lib/longleaf/services/configuration_validator.rb +64 -4
  30. data/lib/longleaf/services/filesystem_location_validator.rb +16 -0
  31. data/lib/longleaf/services/metadata_deserializer.rb +41 -9
  32. data/lib/longleaf/services/metadata_persistence_manager.rb +3 -2
  33. data/lib/longleaf/services/metadata_serializer.rb +94 -13
  34. data/lib/longleaf/services/metadata_validator.rb +76 -0
  35. data/lib/longleaf/services/s3_location_validator.rb +19 -0
  36. data/lib/longleaf/services/service_definition_validator.rb +16 -8
  37. data/lib/longleaf/services/service_manager.rb +7 -15
  38. data/lib/longleaf/services/service_mapping_validator.rb +26 -15
  39. data/lib/longleaf/services/storage_location_manager.rb +38 -12
  40. data/lib/longleaf/services/storage_location_validator.rb +41 -30
  41. data/lib/longleaf/specs/config_builder.rb +10 -3
  42. data/lib/longleaf/specs/config_validator_helpers.rb +16 -0
  43. data/lib/longleaf/specs/metadata_builder.rb +1 -0
  44. data/lib/longleaf/version.rb +1 -1
  45. data/longleaf.gemspec +3 -1
  46. data/mkdocs.yml +2 -1
  47. metadata +48 -8
  48. data/.travis.yml +0 -4
  49. data/lib/longleaf/services/storage_path_validator.rb +0 -16
@@ -0,0 +1,19 @@
1
+ require 'pathname'
2
+ require 'longleaf/errors'
3
+ require 'longleaf/helpers/s3_uri_helper'
4
+
5
+ module Longleaf
6
+ # Validates the configuration of a s3 based location
7
+ class S3LocationValidator
8
+ def self.validate(p_validator, name, path_prop, section_name, path)
9
+ base_msg = "Storage location '#{name}' specifies invalid #{section_name} '#{path_prop}' property: "
10
+ p_validator.assert(base_msg + 'Path must not be empty', !path.nil? && !path.to_s.strip.empty?)
11
+ begin
12
+ bucket_name = S3UriHelper.extract_bucket(path)
13
+ p_validator.assert(base_msg + 'Path must specify a bucket', !bucket_name.nil?)
14
+ rescue ArgumentError => e
15
+ p_validator.fail(base_msg + e.message)
16
+ end
17
+ end
18
+ end
19
+ end
@@ -10,21 +10,29 @@ module Longleaf
10
10
  SF ||= Longleaf::ServiceFields
11
11
  AF ||= Longleaf::AppFields
12
12
 
13
+ # @param config [Hash] hash containing the application configuration
14
+ def initialize(config)
15
+ super(config)
16
+ end
17
+
18
+ protected
13
19
  # Validates configuration to ensure that it is syntactically correct and does not violate
14
20
  # schema requirements.
15
21
  # @param config [Hash] hash containing the application configuration
16
- def self.validate_config(config)
17
- assert("Configuration must be a hash, but a #{config.class} was provided", config.class == Hash)
18
- assert("Configuration must contain a root '#{AF::SERVICES}' key", config.key?(AF::SERVICES))
19
- services = config[AF::SERVICES]
22
+ def validate
23
+ assert("Configuration must be a hash, but a #{@config.class} was provided", @config.class == Hash)
24
+ assert("Configuration must contain a root '#{AF::SERVICES}' key", @config.key?(AF::SERVICES))
25
+ services = @config[AF::SERVICES]
20
26
  assert("'#{AF::SERVICES}' must be a hash of services", services.class == Hash)
21
27
 
22
28
  services.each do |name, properties|
23
- assert("Name of service definition must be a string, but was of type #{name.class}", name.instance_of?(String))
24
- assert("Service definition '#{name}' must be a hash, but a #{properties.class} was provided", properties.is_a?(Hash))
29
+ register_on_failure do
30
+ assert("Name of service definition must be a string, but was of type #{name.class}", name.instance_of?(String))
31
+ assert("Service definition '#{name}' must be a hash, but a #{properties.class} was provided", properties.is_a?(Hash))
25
32
 
26
- work_script = properties[SF::WORK_SCRIPT]
27
- assert("Service definition '#{name}' must specify a '#{SF::WORK_SCRIPT}' property", !work_script.nil? && !work_script.empty?)
33
+ work_script = properties[SF::WORK_SCRIPT]
34
+ assert("Service definition '#{name}' must specify a '#{SF::WORK_SCRIPT}' property", !work_script.nil? && !work_script.empty?)
35
+ end
28
36
  end
29
37
  end
30
38
  end
@@ -68,26 +68,18 @@ module Longleaf
68
68
  # @param md_rec [MetadataRecord] metadata record for the file being evaluated
69
69
  # @return [Boolean] true if the service should be run.
70
70
  def service_needed?(service_name, md_rec)
71
- # If service not recorded for file, then it is needed
72
- present_services = md_rec.list_services
73
- return true unless present_services.include?(service_name)
74
-
75
71
  service_rec = md_rec.service(service_name)
76
-
77
- return true if service_rec.run_needed
78
- return true if service_rec.timestamp.nil?
72
+ return true if !service_rec.nil? && service_rec.run_needed
79
73
 
80
74
  definition = @definition_manager.services[service_name]
81
75
 
82
- # Check if the amount of time defined in frequency has passed since the service timestamp
83
- frequency = definition.frequency
84
- unless frequency.nil?
85
- service_timestamp = service_rec.timestamp
86
- now = ServiceDateHelper.formatted_timestamp
76
+ next_run = ServiceDateHelper.next_run_needed(md_rec, definition)
87
77
 
88
- return true if now > ServiceDateHelper.add_to_timestamp(service_timestamp, frequency)
89
- end
90
- false
78
+ return false if next_run.nil?
79
+
80
+ # If next run timestamp has passed then service is needed
81
+ now = ServiceDateHelper.formatted_timestamp
82
+ now >= next_run
91
83
  end
92
84
 
93
85
  # Perform the specified service on the file record, in the context of the specified event
@@ -9,40 +9,51 @@ module Longleaf
9
9
  class ServiceMappingValidator < ConfigurationValidator
10
10
  AF ||= Longleaf::AppFields
11
11
 
12
+ # @param config [Hash] hash containing the application configuration
13
+ def initialize(config)
14
+ super(config)
15
+ end
16
+
17
+ protected
12
18
  # Validates service mapping configuration to ensure that it is syntactically and referentially correct.
13
19
  # @param config [Hash] hash containing the application configuration
14
- def self.validate_config(config)
15
- assert("Configuration must be a hash, but a #{config.class} was provided", config.class == Hash)
16
- assert("Configuration must contain a root '#{AF::SERVICE_MAPPINGS}' key", config.key?(AF::SERVICE_MAPPINGS))
17
- mappings = config[AF::SERVICE_MAPPINGS]
20
+ def validate
21
+ assert("Configuration must be a hash, but a #{@config.class} was provided", @config.class == Hash)
22
+ assert("Configuration must contain a root '#{AF::SERVICE_MAPPINGS}' key", @config.key?(AF::SERVICE_MAPPINGS))
23
+ mappings = @config[AF::SERVICE_MAPPINGS]
18
24
  return if mappings.nil? || mappings.empty?
19
25
  assert("'#{AF::SERVICE_MAPPINGS}' must be an array of mappings", mappings.is_a?(Array))
20
26
 
21
- service_names = config[AF::SERVICES].keys
22
- location_names = config[AF::LOCATIONS].keys
27
+ service_names = @config[AF::SERVICES].keys
28
+ location_names = @config[AF::LOCATIONS].keys
23
29
 
24
30
  mappings.each do |mapping|
25
- assert("Mapping must be a hash, but received #{mapping.inspect} instead", mapping.is_a?(Hash))
31
+ register_on_failure do
32
+ assert("Mapping must be a hash, but received #{mapping.inspect} instead", mapping.is_a?(Hash))
26
33
 
27
- validate_mapping_field(AF::LOCATIONS, mapping, location_names)
28
- validate_mapping_field(AF::SERVICES, mapping, service_names)
34
+ register_on_failure { validate_mapping_field(AF::LOCATIONS, mapping, location_names) }
35
+ register_on_failure { validate_mapping_field(AF::SERVICES, mapping, service_names) }
36
+ end
29
37
  end
38
+
39
+ @result
30
40
  end
31
41
 
32
- def self.validate_mapping_field(field, mapping, valid_values)
42
+ private
43
+ def validate_mapping_field(field, mapping, valid_values)
33
44
  assert("Mapping must contain a '#{field}' field", mapping.key?(field))
34
45
  field_values = mapping[field]
35
- assert("Mapping '#{field}' field must be either a string or an array, but received '#{field_values.inspect}' instead",
46
+ assert("Mapping '#{field}' field must be either a string or an array, but received '#{field_values.class}' instead",
36
47
  field_values.is_a?(Array) || field_values.is_a?(String))
37
48
  assert("Mapping must specify one or more value in the '#{field}' field", !field_values.empty?)
38
49
 
39
50
  check_values = field_values.is_a?(String) ? [field_values] : field_values
40
51
  check_values.each do |value|
41
- assert("Mapping '#{field}' specifies value '#{value}', but no #{field} with that name exist",
42
- valid_values.include?(value))
52
+ register_on_failure do
53
+ assert("Mapping specifies value '#{value}', but no #{field} with that name exist",
54
+ valid_values.include?(value))
55
+ end
43
56
  end
44
57
  end
45
-
46
- private_class_method :validate_mapping_field
47
58
  end
48
59
  end
@@ -1,14 +1,24 @@
1
1
  require 'longleaf/models/app_fields'
2
- require 'longleaf/models/storage_location'
2
+ require 'longleaf/models/storage_types'
3
+ require 'longleaf/models/filesystem_storage_location'
4
+ require 'longleaf/models/s3_storage_location'
5
+ require 'longleaf/models/filesystem_metadata_location'
3
6
  require 'longleaf/errors'
4
7
 
5
8
  module Longleaf
6
9
  # Manager which loads and provides access to {StorageLocation} objects
7
10
  class StorageLocationManager
8
11
  AF ||= Longleaf::AppFields
12
+ ST ||= Longleaf::StorageTypes
9
13
 
10
14
  # Hash mapping storage location names to {StorageLocation} objects
11
15
  attr_reader :locations
16
+ # Mapping of storage types to storage location classes
17
+ @@storage_type_mappings = {
18
+ ST::FILESYSTEM_STORAGE_TYPE => Longleaf::FilesystemStorageLocation,
19
+ ST::S3_STORAGE_TYPE => Longleaf::S3StorageLocation
20
+ }
21
+ @@metadata_type_mappings = { ST::FILESYSTEM_STORAGE_TYPE => Longleaf::FilesystemMetadataLocation }
12
22
 
13
23
  # @param config [Hash] has representation of the application configuration
14
24
  def initialize(config)
@@ -16,15 +26,9 @@ module Longleaf
16
26
 
17
27
  @locations = Hash.new
18
28
  config[AF::LOCATIONS].each do |name, properties|
19
- path = properties[AF::LOCATION_PATH]
20
- md_path = properties[AF::METADATA_PATH]
21
- md_digests = properties[AF::METADATA_DIGESTS]
22
- location = Longleaf::StorageLocation.new(name: name,
23
- path: path,
24
- metadata_path: md_path,
25
- metadata_digests: md_digests)
26
-
27
- @locations[name] = location
29
+ md_loc = instantiate_metadata_location(properties)
30
+
31
+ @locations[name] = instantiate_storage_location(name, properties, md_loc)
28
32
  end
29
33
  @locations.freeze
30
34
  end
@@ -35,7 +39,7 @@ module Longleaf
35
39
  def get_location_by_path(path)
36
40
  raise ArgumentError.new("Path parameter is required") if path.nil? || path.empty?
37
41
  @locations.each do |name, location|
38
- return location if path.start_with?(location.path)
42
+ return location if location.contains?(path)
39
43
  end
40
44
 
41
45
  nil
@@ -47,7 +51,7 @@ module Longleaf
47
51
  def get_location_by_metadata_path(md_path)
48
52
  raise ArgumentError.new("Metadata path parameter is required") if md_path.nil? || md_path.empty?
49
53
  @locations.each do |name, location|
50
- return location if md_path.start_with?(location.metadata_path)
54
+ return location if location.metadata_location.contains?(md_path)
51
55
  end
52
56
 
53
57
  nil
@@ -68,5 +72,27 @@ module Longleaf
68
72
  end
69
73
  location
70
74
  end
75
+
76
+ private
77
+ def instantiate_metadata_location(loc_properties)
78
+ m_config = loc_properties[AF::METADATA_CONFIG]
79
+ m_type = m_config[AF::STORAGE_TYPE]
80
+ m_type = ST::FILESYSTEM_STORAGE_TYPE if m_type.nil?
81
+
82
+ m_class = @@metadata_type_mappings[m_type]
83
+ raise ArgumentError.new("Unknown metadata location type #{m_type}") if m_class.nil?
84
+
85
+ m_class.new(m_config)
86
+ end
87
+
88
+ def instantiate_storage_location(name, properties, md_loc)
89
+ s_type = properties[AF::STORAGE_TYPE]
90
+ s_type = ST::FILESYSTEM_STORAGE_TYPE if s_type.nil?
91
+
92
+ s_class = @@storage_type_mappings[s_type]
93
+ raise ArgumentError.new("Unknown storage location type #{s_type}") if s_class.nil?
94
+
95
+ s_class.new(name, properties, md_loc)
96
+ end
71
97
  end
72
98
  end
@@ -1,63 +1,74 @@
1
1
  require 'pathname'
2
- require 'longleaf/models/storage_location'
3
2
  require 'longleaf/models/app_fields'
3
+ require 'longleaf/models/storage_types'
4
4
  require 'longleaf/errors'
5
5
  require_relative 'configuration_validator'
6
- require 'longleaf/services/storage_path_validator'
6
+ require 'longleaf/services/filesystem_location_validator'
7
+ require 'longleaf/services/s3_location_validator'
7
8
 
8
9
  module Longleaf
9
10
  # Validates application configuration of storage locations
10
11
  class StorageLocationValidator < ConfigurationValidator
11
12
  AF ||= Longleaf::AppFields
13
+ ST ||= Longleaf::StorageTypes
12
14
 
15
+ @@storage_type_mappings = {
16
+ ST::FILESYSTEM_STORAGE_TYPE => Longleaf::FilesystemLocationValidator,
17
+ ST::S3_STORAGE_TYPE => Longleaf::S3LocationValidator
18
+ }
19
+
20
+ # @param config [Hash] hash containing the application configuration
21
+ def initialize(config)
22
+ super(config)
23
+ @existing_paths = Array.new
24
+ end
25
+
26
+ protected
13
27
  # Validates configuration to ensure that it is syntactically correct and does not violate
14
28
  # schema and uniqueness requirements.
15
- # @param config [Hash] hash containing the application configuration
16
- def self.validate_config(config)
17
- assert("Configuration must be a hash, but a #{config.class} was provided", config.class == Hash)
18
- assert("Configuration must contain a root '#{AF::LOCATIONS}' key", config.key?(AF::LOCATIONS))
19
- locations = config[AF::LOCATIONS]
29
+ def validate
30
+ assert("Configuration must be a hash, but a #{@config.class} was provided", @config.class == Hash)
31
+ assert("Configuration must contain a root '#{AF::LOCATIONS}' key", @config.key?(AF::LOCATIONS))
32
+ locations = @config[AF::LOCATIONS]
20
33
  assert("'#{AF::LOCATIONS}' must be a hash of locations", locations.class == Hash)
21
34
 
22
- existing_paths = Array.new
23
35
  locations.each do |name, properties|
24
- assert("Name of storage location must be a string, but was of type #{name.class}", name.instance_of?(String))
25
- assert("Storage location '#{name}' must be a hash, but a #{properties.class} was provided", properties.is_a?(Hash))
36
+ register_on_failure do
37
+ assert("Name of storage location must be a string, but was of type #{name.class}", name.instance_of?(String))
38
+ assert("Storage location '#{name}' must be a hash, but a #{properties.class} was provided", properties.is_a?(Hash))
39
+
40
+ register_on_failure { assert_path_property_valid(name, AF::LOCATION_PATH, properties, 'location') }
26
41
 
27
- assert_path_property_valid(name, AF::LOCATION_PATH, properties, existing_paths)
28
- assert_path_property_valid(name, AF::METADATA_PATH, properties, existing_paths)
42
+ assert("Metadata location must be present for location '#{name}'", properties.key?(AF::METADATA_CONFIG))
43
+ assert_path_property_valid(name, AF::LOCATION_PATH, properties[AF::METADATA_CONFIG], 'metadata')
44
+ end
29
45
  end
46
+
47
+ @result
30
48
  end
31
49
 
32
- def self.assert_path_property_valid(name, path_prop, properties, existing_paths)
50
+ private
51
+ def assert_path_property_valid(name, path_prop, properties, section_name)
33
52
  path = properties[path_prop]
34
- begin
35
- StoragePathValidator::validate(path)
36
- rescue InvalidStoragePathError => err
37
- raise ConfigurationError.new(
38
- "Storage location '#{name}' specifies invalid '#{path_prop}' property: #{err.message}")
39
- end
40
- assert("Storage location '#{name}' must specify a '#{path_prop}' property", !path.nil? && !path.empty?)
41
- assert("Storage location '#{name}' must specify an absolute path for property '#{path_prop}'",
42
- Pathname.new(path).absolute? && !path.include?('/..'))
43
- assert("Storage location '#{name}' specifies a '#{path_prop}' directory which does not exist", Dir.exist?(path))
53
+
54
+ storage_type = properties[AF::STORAGE_TYPE] || ST::DEFAULT_STORAGE_TYPE
55
+ type_validator = @@storage_type_mappings[storage_type]
56
+ type_validator.validate(self, name, path_prop, section_name, path)
44
57
 
45
58
  # Ensure paths have trailing slash to avoid matching on partial directory names
46
59
  path += '/' unless path.end_with?('/')
47
60
  # Verify that the (metadata_)path property's value is not inside of another storage location or vice versa
48
- existing_paths.each do |existing|
61
+ @existing_paths.each do |existing|
49
62
  if existing.start_with?(path) || path.start_with?(existing)
50
- msg = "Location '#{name}' defines property #{path_prop} with value '#{path}'" \
63
+ msg = "Location '#{name}' defines property #{section_name} #{path_prop} with value '#{path}'" \
51
64
  " which overlaps with another configured path '#{existing}'." \
52
- " Storage locations must not define #{AF::LOCATION_PATH} or #{AF::METADATA_PATH}" \
65
+ " Storage locations must not define #{AF::LOCATION_PATH}" \
53
66
  " properties which are contained by another location property"
54
- raise ConfigurationError.new(msg)
67
+ fail(msg)
55
68
  end
56
69
  end
57
70
 
58
- existing_paths << path
71
+ @existing_paths << path
59
72
  end
60
-
61
- private_class_method :assert_path_property_valid
62
73
  end
63
74
  end
@@ -27,14 +27,21 @@ module Longleaf
27
27
  # @param path [String] value for the 'path' field
28
28
  # @param md_path [String] value for the 'metadata_path' field
29
29
  # @return this builder
30
- def with_location(name:, path: '/file/path/', md_path: '/metadata/path/', md_digests: nil)
30
+ def with_location(name:, path: '/file/path/', s_type: nil, md_path: '/metadata/path/', md_type: nil, md_digests: nil)
31
31
  @config[AF::LOCATIONS] = Hash.new unless @config.key?(AF::LOCATIONS)
32
32
 
33
33
  location = {}
34
34
  @config[AF::LOCATIONS][name] = location
35
35
  location[AF::LOCATION_PATH] = path unless path.nil?
36
- location[AF::METADATA_PATH] = md_path unless md_path.nil?
37
- location[AF::METADATA_DIGESTS] = md_digests unless md_digests.nil?
36
+ location[AF::STORAGE_TYPE] = s_type unless s_type.nil?
37
+
38
+ if !md_path.nil?
39
+ md_loc = { AF::LOCATION_PATH => md_path }
40
+ location[AF::METADATA_CONFIG] = md_loc
41
+
42
+ md_loc[AF::METADATA_DIGESTS] = md_digests unless md_digests.nil?
43
+ md_loc[AF::STORAGE_TYPE] = md_type unless md_type.nil?
44
+ end
38
45
  self
39
46
  end
40
47
 
@@ -0,0 +1,16 @@
1
+ module Longleaf
2
+ module ConfigValidatorHelpers
3
+ def fails_validation_with_error(validator, *error_messages)
4
+ result = validator.validate_config
5
+ expect(result.valid?).to be false
6
+ error_messages.each do |error_message|
7
+ expect(result.errors).to include(error_message)
8
+ end
9
+ end
10
+
11
+ def passes_validation(validator)
12
+ result = validator.validate_config
13
+ expect(result.valid?).to eq(true), "expected validation to pass, but received errors:\n#{result.errors&.join("\n")}"
14
+ end
15
+ end
16
+ end
@@ -45,6 +45,7 @@ module Longleaf
45
45
 
46
46
  def with_properties(properties)
47
47
  @properties = properties
48
+ self
48
49
  end
49
50
 
50
51
  # @return the constructed metadata record
@@ -1,3 +1,3 @@
1
1
  module Longleaf
2
- VERSION = "0.3.0"
2
+ VERSION = "1.0.0"
3
3
  end
@@ -29,10 +29,12 @@ Gem::Specification.new do |spec|
29
29
  spec.add_dependency "thor", "~> 0.20.0"
30
30
  spec.add_dependency "yard", "~> 0.9.16"
31
31
  spec.add_dependency "sequel", "~> 5.20"
32
+ spec.add_dependency "aws-sdk-s3", "~> 1.56"
32
33
 
33
- spec.add_development_dependency "bundler", "~> 1.16"
34
+ spec.add_development_dependency "bundler", "~> 2.1"
34
35
  spec.add_development_dependency "rake", "~> 12.0"
35
36
  spec.add_development_dependency "rspec", "~> 3.8"
37
+ spec.add_development_dependency "rspec-core", "~> 3.8"
36
38
  spec.add_development_dependency "rspec_junit_formatter", "~> 0.4"
37
39
  spec.add_development_dependency "factory_bot", "~> 5.0"
38
40
  spec.add_development_dependency "aruba", "~> 0.14.9"
data/mkdocs.yml CHANGED
@@ -1,9 +1,10 @@
1
1
  site_name: Longleaf Documentation
2
2
  site_url: https://unc-libraries.github.io/longleaf-preservation
3
3
  theme: readthedocs
4
- #repo_name: 'UNC-Libraries/Longleaf-preservation'
4
+ # repo_name: 'UNC-Libraries/Longleaf-preservation'
5
5
  #setting edit_uri to empty string removes the "Edit in" link in upper right hand corner of pages, but keeps the Github link to repo in the bottom of sidepane
6
6
  repo_url: https://github.com/UNC-Libraries/longleaf-preservation/
7
+ site_description: Longleaf is a command-line tool which allows users to configure a set of storage locations and define custom sets of preservation services to run on their contents. These services are executed in response to applicable preservation events issued by clients. Its primary goal is to provide tools to create a simple and customizable preservation environment.
7
8
  edit_uri: ''
8
9
 
9
10
  # include extra css file, in docs/
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: longleaf
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.0
4
+ version: 1.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ben Pennell
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2019-05-30 00:00:00.000000000 Z
11
+ date: 2020-10-14 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: thor
@@ -52,20 +52,34 @@ dependencies:
52
52
  - - "~>"
53
53
  - !ruby/object:Gem::Version
54
54
  version: '5.20'
55
+ - !ruby/object:Gem::Dependency
56
+ name: aws-sdk-s3
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '1.56'
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '1.56'
55
69
  - !ruby/object:Gem::Dependency
56
70
  name: bundler
57
71
  requirement: !ruby/object:Gem::Requirement
58
72
  requirements:
59
73
  - - "~>"
60
74
  - !ruby/object:Gem::Version
61
- version: '1.16'
75
+ version: '2.1'
62
76
  type: :development
63
77
  prerelease: false
64
78
  version_requirements: !ruby/object:Gem::Requirement
65
79
  requirements:
66
80
  - - "~>"
67
81
  - !ruby/object:Gem::Version
68
- version: '1.16'
82
+ version: '2.1'
69
83
  - !ruby/object:Gem::Dependency
70
84
  name: rake
71
85
  requirement: !ruby/object:Gem::Requirement
@@ -94,6 +108,20 @@ dependencies:
94
108
  - - "~>"
95
109
  - !ruby/object:Gem::Version
96
110
  version: '3.8'
111
+ - !ruby/object:Gem::Dependency
112
+ name: rspec-core
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - "~>"
116
+ - !ruby/object:Gem::Version
117
+ version: '3.8'
118
+ type: :development
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - "~>"
123
+ - !ruby/object:Gem::Version
124
+ version: '3.8'
97
125
  - !ruby/object:Gem::Dependency
98
126
  name: rspec_junit_formatter
99
127
  requirement: !ruby/object:Gem::Requirement
@@ -238,7 +266,6 @@ files:
238
266
  - ".rspec"
239
267
  - ".rubocop.yml"
240
268
  - ".rubocop_todo.yml"
241
- - ".travis.yml"
242
269
  - ".yardopts"
243
270
  - Gemfile
244
271
  - LICENSE.txt
@@ -344,10 +371,12 @@ files:
344
371
  - exe/longleaf
345
372
  - lib/longleaf.rb
346
373
  - lib/longleaf/candidates/file_selector.rb
374
+ - lib/longleaf/candidates/manifest_digest_provider.rb
347
375
  - lib/longleaf/candidates/registered_file_selector.rb
348
376
  - lib/longleaf/candidates/service_candidate_filesystem_iterator.rb
349
377
  - lib/longleaf/candidates/service_candidate_index_iterator.rb
350
378
  - lib/longleaf/candidates/service_candidate_locator.rb
379
+ - lib/longleaf/candidates/single_digest_provider.rb
351
380
  - lib/longleaf/cli.rb
352
381
  - lib/longleaf/commands/deregister_command.rb
353
382
  - lib/longleaf/commands/preserve_command.rb
@@ -361,7 +390,10 @@ files:
361
390
  - lib/longleaf/events/event_status_tracking.rb
362
391
  - lib/longleaf/events/preserve_event.rb
363
392
  - lib/longleaf/events/register_event.rb
393
+ - lib/longleaf/helpers/case_insensitive_hash.rb
364
394
  - lib/longleaf/helpers/digest_helper.rb
395
+ - lib/longleaf/helpers/s3_uri_helper.rb
396
+ - lib/longleaf/helpers/selection_options_parser.rb
365
397
  - lib/longleaf/helpers/service_date_helper.rb
366
398
  - lib/longleaf/indexing/index_manager.rb
367
399
  - lib/longleaf/indexing/sequel_index_driver.rb
@@ -369,23 +401,32 @@ files:
369
401
  - lib/longleaf/logging/redirecting_logger.rb
370
402
  - lib/longleaf/models/app_fields.rb
371
403
  - lib/longleaf/models/file_record.rb
404
+ - lib/longleaf/models/filesystem_metadata_location.rb
405
+ - lib/longleaf/models/filesystem_storage_location.rb
372
406
  - lib/longleaf/models/md_fields.rb
407
+ - lib/longleaf/models/metadata_location.rb
373
408
  - lib/longleaf/models/metadata_record.rb
409
+ - lib/longleaf/models/s3_storage_location.rb
374
410
  - lib/longleaf/models/service_definition.rb
375
411
  - lib/longleaf/models/service_fields.rb
376
412
  - lib/longleaf/models/service_record.rb
377
413
  - lib/longleaf/models/storage_location.rb
414
+ - lib/longleaf/models/storage_types.rb
378
415
  - lib/longleaf/models/system_config_fields.rb
379
416
  - lib/longleaf/preservation_services/file_check_service.rb
380
417
  - lib/longleaf/preservation_services/fixity_check_service.rb
381
418
  - lib/longleaf/preservation_services/rsync_replication_service.rb
419
+ - lib/longleaf/preservation_services/s3_replication_service.rb
382
420
  - lib/longleaf/services/application_config_deserializer.rb
383
421
  - lib/longleaf/services/application_config_manager.rb
384
422
  - lib/longleaf/services/application_config_validator.rb
385
423
  - lib/longleaf/services/configuration_validator.rb
424
+ - lib/longleaf/services/filesystem_location_validator.rb
386
425
  - lib/longleaf/services/metadata_deserializer.rb
387
426
  - lib/longleaf/services/metadata_persistence_manager.rb
388
427
  - lib/longleaf/services/metadata_serializer.rb
428
+ - lib/longleaf/services/metadata_validator.rb
429
+ - lib/longleaf/services/s3_location_validator.rb
389
430
  - lib/longleaf/services/service_class_cache.rb
390
431
  - lib/longleaf/services/service_definition_manager.rb
391
432
  - lib/longleaf/services/service_definition_validator.rb
@@ -394,8 +435,8 @@ files:
394
435
  - lib/longleaf/services/service_mapping_validator.rb
395
436
  - lib/longleaf/services/storage_location_manager.rb
396
437
  - lib/longleaf/services/storage_location_validator.rb
397
- - lib/longleaf/services/storage_path_validator.rb
398
438
  - lib/longleaf/specs/config_builder.rb
439
+ - lib/longleaf/specs/config_validator_helpers.rb
399
440
  - lib/longleaf/specs/custom_matchers.rb
400
441
  - lib/longleaf/specs/file_helpers.rb
401
442
  - lib/longleaf/specs/metadata_builder.rb
@@ -423,8 +464,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
423
464
  - !ruby/object:Gem::Version
424
465
  version: '0'
425
466
  requirements: []
426
- rubyforge_project:
427
- rubygems_version: 2.7.7
467
+ rubygems_version: 3.1.2
428
468
  signing_key:
429
469
  specification_version: 4
430
470
  summary: Longleaf preservation services tool