longleaf 0.3.0 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.circleci/config.yml +12 -2
- data/README.md +11 -1
- data/lib/longleaf/candidates/manifest_digest_provider.rb +17 -0
- data/lib/longleaf/candidates/single_digest_provider.rb +13 -0
- data/lib/longleaf/cli.rb +49 -36
- data/lib/longleaf/commands/register_command.rb +3 -3
- data/lib/longleaf/commands/validate_config_command.rb +1 -1
- data/lib/longleaf/events/register_event.rb +8 -4
- data/lib/longleaf/helpers/case_insensitive_hash.rb +38 -0
- data/lib/longleaf/helpers/digest_helper.rb +7 -1
- data/lib/longleaf/helpers/s3_uri_helper.rb +86 -0
- data/lib/longleaf/helpers/selection_options_parser.rb +189 -0
- data/lib/longleaf/helpers/service_date_helper.rb +29 -1
- data/lib/longleaf/indexing/sequel_index_driver.rb +2 -20
- data/lib/longleaf/models/app_fields.rb +4 -2
- data/lib/longleaf/models/filesystem_metadata_location.rb +56 -0
- data/lib/longleaf/models/filesystem_storage_location.rb +52 -0
- data/lib/longleaf/models/metadata_location.rb +47 -0
- data/lib/longleaf/models/metadata_record.rb +3 -1
- data/lib/longleaf/models/s3_storage_location.rb +133 -0
- data/lib/longleaf/models/service_fields.rb +4 -0
- data/lib/longleaf/models/storage_location.rb +17 -48
- data/lib/longleaf/models/storage_types.rb +9 -0
- data/lib/longleaf/preservation_services/rsync_replication_service.rb +9 -11
- data/lib/longleaf/preservation_services/s3_replication_service.rb +143 -0
- data/lib/longleaf/services/application_config_deserializer.rb +26 -4
- data/lib/longleaf/services/application_config_validator.rb +17 -6
- data/lib/longleaf/services/configuration_validator.rb +64 -4
- data/lib/longleaf/services/filesystem_location_validator.rb +16 -0
- data/lib/longleaf/services/metadata_deserializer.rb +41 -9
- data/lib/longleaf/services/metadata_persistence_manager.rb +3 -2
- data/lib/longleaf/services/metadata_serializer.rb +94 -13
- data/lib/longleaf/services/metadata_validator.rb +76 -0
- data/lib/longleaf/services/s3_location_validator.rb +19 -0
- data/lib/longleaf/services/service_definition_validator.rb +16 -8
- data/lib/longleaf/services/service_manager.rb +7 -15
- data/lib/longleaf/services/service_mapping_validator.rb +26 -15
- data/lib/longleaf/services/storage_location_manager.rb +38 -12
- data/lib/longleaf/services/storage_location_validator.rb +41 -30
- data/lib/longleaf/specs/config_builder.rb +10 -3
- data/lib/longleaf/specs/config_validator_helpers.rb +16 -0
- data/lib/longleaf/specs/metadata_builder.rb +1 -0
- data/lib/longleaf/version.rb +1 -1
- data/longleaf.gemspec +3 -1
- data/mkdocs.yml +2 -1
- metadata +48 -8
- data/.travis.yml +0 -4
- data/lib/longleaf/services/storage_path_validator.rb +0 -16
@@ -0,0 +1,19 @@
|
|
1
|
+
require 'pathname'
|
2
|
+
require 'longleaf/errors'
|
3
|
+
require 'longleaf/helpers/s3_uri_helper'
|
4
|
+
|
5
|
+
module Longleaf
|
6
|
+
# Validates the configuration of a s3 based location
|
7
|
+
class S3LocationValidator
|
8
|
+
def self.validate(p_validator, name, path_prop, section_name, path)
|
9
|
+
base_msg = "Storage location '#{name}' specifies invalid #{section_name} '#{path_prop}' property: "
|
10
|
+
p_validator.assert(base_msg + 'Path must not be empty', !path.nil? && !path.to_s.strip.empty?)
|
11
|
+
begin
|
12
|
+
bucket_name = S3UriHelper.extract_bucket(path)
|
13
|
+
p_validator.assert(base_msg + 'Path must specify a bucket', !bucket_name.nil?)
|
14
|
+
rescue ArgumentError => e
|
15
|
+
p_validator.fail(base_msg + e.message)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -10,21 +10,29 @@ module Longleaf
|
|
10
10
|
SF ||= Longleaf::ServiceFields
|
11
11
|
AF ||= Longleaf::AppFields
|
12
12
|
|
13
|
+
# @param config [Hash] hash containing the application configuration
|
14
|
+
def initialize(config)
|
15
|
+
super(config)
|
16
|
+
end
|
17
|
+
|
18
|
+
protected
|
13
19
|
# Validates configuration to ensure that it is syntactically correct and does not violate
|
14
20
|
# schema requirements.
|
15
21
|
# @param config [Hash] hash containing the application configuration
|
16
|
-
def
|
17
|
-
assert("Configuration must be a hash, but a #{config.class} was provided", config.class == Hash)
|
18
|
-
assert("Configuration must contain a root '#{AF::SERVICES}' key", config.key?(AF::SERVICES))
|
19
|
-
services = config[AF::SERVICES]
|
22
|
+
def validate
|
23
|
+
assert("Configuration must be a hash, but a #{@config.class} was provided", @config.class == Hash)
|
24
|
+
assert("Configuration must contain a root '#{AF::SERVICES}' key", @config.key?(AF::SERVICES))
|
25
|
+
services = @config[AF::SERVICES]
|
20
26
|
assert("'#{AF::SERVICES}' must be a hash of services", services.class == Hash)
|
21
27
|
|
22
28
|
services.each do |name, properties|
|
23
|
-
|
24
|
-
|
29
|
+
register_on_failure do
|
30
|
+
assert("Name of service definition must be a string, but was of type #{name.class}", name.instance_of?(String))
|
31
|
+
assert("Service definition '#{name}' must be a hash, but a #{properties.class} was provided", properties.is_a?(Hash))
|
25
32
|
|
26
|
-
|
27
|
-
|
33
|
+
work_script = properties[SF::WORK_SCRIPT]
|
34
|
+
assert("Service definition '#{name}' must specify a '#{SF::WORK_SCRIPT}' property", !work_script.nil? && !work_script.empty?)
|
35
|
+
end
|
28
36
|
end
|
29
37
|
end
|
30
38
|
end
|
@@ -68,26 +68,18 @@ module Longleaf
|
|
68
68
|
# @param md_rec [MetadataRecord] metadata record for the file being evaluated
|
69
69
|
# @return [Boolean] true if the service should be run.
|
70
70
|
def service_needed?(service_name, md_rec)
|
71
|
-
# If service not recorded for file, then it is needed
|
72
|
-
present_services = md_rec.list_services
|
73
|
-
return true unless present_services.include?(service_name)
|
74
|
-
|
75
71
|
service_rec = md_rec.service(service_name)
|
76
|
-
|
77
|
-
return true if service_rec.run_needed
|
78
|
-
return true if service_rec.timestamp.nil?
|
72
|
+
return true if !service_rec.nil? && service_rec.run_needed
|
79
73
|
|
80
74
|
definition = @definition_manager.services[service_name]
|
81
75
|
|
82
|
-
|
83
|
-
frequency = definition.frequency
|
84
|
-
unless frequency.nil?
|
85
|
-
service_timestamp = service_rec.timestamp
|
86
|
-
now = ServiceDateHelper.formatted_timestamp
|
76
|
+
next_run = ServiceDateHelper.next_run_needed(md_rec, definition)
|
87
77
|
|
88
|
-
|
89
|
-
|
90
|
-
|
78
|
+
return false if next_run.nil?
|
79
|
+
|
80
|
+
# If next run timestamp has passed then service is needed
|
81
|
+
now = ServiceDateHelper.formatted_timestamp
|
82
|
+
now >= next_run
|
91
83
|
end
|
92
84
|
|
93
85
|
# Perform the specified service on the file record, in the context of the specified event
|
@@ -9,40 +9,51 @@ module Longleaf
|
|
9
9
|
class ServiceMappingValidator < ConfigurationValidator
|
10
10
|
AF ||= Longleaf::AppFields
|
11
11
|
|
12
|
+
# @param config [Hash] hash containing the application configuration
|
13
|
+
def initialize(config)
|
14
|
+
super(config)
|
15
|
+
end
|
16
|
+
|
17
|
+
protected
|
12
18
|
# Validates service mapping configuration to ensure that it is syntactically and referentially correct.
|
13
19
|
# @param config [Hash] hash containing the application configuration
|
14
|
-
def
|
15
|
-
assert("Configuration must be a hash, but a #{config.class} was provided", config.class == Hash)
|
16
|
-
assert("Configuration must contain a root '#{AF::SERVICE_MAPPINGS}' key", config.key?(AF::SERVICE_MAPPINGS))
|
17
|
-
mappings = config[AF::SERVICE_MAPPINGS]
|
20
|
+
def validate
|
21
|
+
assert("Configuration must be a hash, but a #{@config.class} was provided", @config.class == Hash)
|
22
|
+
assert("Configuration must contain a root '#{AF::SERVICE_MAPPINGS}' key", @config.key?(AF::SERVICE_MAPPINGS))
|
23
|
+
mappings = @config[AF::SERVICE_MAPPINGS]
|
18
24
|
return if mappings.nil? || mappings.empty?
|
19
25
|
assert("'#{AF::SERVICE_MAPPINGS}' must be an array of mappings", mappings.is_a?(Array))
|
20
26
|
|
21
|
-
service_names = config[AF::SERVICES].keys
|
22
|
-
location_names = config[AF::LOCATIONS].keys
|
27
|
+
service_names = @config[AF::SERVICES].keys
|
28
|
+
location_names = @config[AF::LOCATIONS].keys
|
23
29
|
|
24
30
|
mappings.each do |mapping|
|
25
|
-
|
31
|
+
register_on_failure do
|
32
|
+
assert("Mapping must be a hash, but received #{mapping.inspect} instead", mapping.is_a?(Hash))
|
26
33
|
|
27
|
-
|
28
|
-
|
34
|
+
register_on_failure { validate_mapping_field(AF::LOCATIONS, mapping, location_names) }
|
35
|
+
register_on_failure { validate_mapping_field(AF::SERVICES, mapping, service_names) }
|
36
|
+
end
|
29
37
|
end
|
38
|
+
|
39
|
+
@result
|
30
40
|
end
|
31
41
|
|
32
|
-
|
42
|
+
private
|
43
|
+
def validate_mapping_field(field, mapping, valid_values)
|
33
44
|
assert("Mapping must contain a '#{field}' field", mapping.key?(field))
|
34
45
|
field_values = mapping[field]
|
35
|
-
assert("Mapping '#{field}' field must be either a string or an array, but received '#{field_values.
|
46
|
+
assert("Mapping '#{field}' field must be either a string or an array, but received '#{field_values.class}' instead",
|
36
47
|
field_values.is_a?(Array) || field_values.is_a?(String))
|
37
48
|
assert("Mapping must specify one or more value in the '#{field}' field", !field_values.empty?)
|
38
49
|
|
39
50
|
check_values = field_values.is_a?(String) ? [field_values] : field_values
|
40
51
|
check_values.each do |value|
|
41
|
-
|
42
|
-
|
52
|
+
register_on_failure do
|
53
|
+
assert("Mapping specifies value '#{value}', but no #{field} with that name exist",
|
54
|
+
valid_values.include?(value))
|
55
|
+
end
|
43
56
|
end
|
44
57
|
end
|
45
|
-
|
46
|
-
private_class_method :validate_mapping_field
|
47
58
|
end
|
48
59
|
end
|
@@ -1,14 +1,24 @@
|
|
1
1
|
require 'longleaf/models/app_fields'
|
2
|
-
require 'longleaf/models/
|
2
|
+
require 'longleaf/models/storage_types'
|
3
|
+
require 'longleaf/models/filesystem_storage_location'
|
4
|
+
require 'longleaf/models/s3_storage_location'
|
5
|
+
require 'longleaf/models/filesystem_metadata_location'
|
3
6
|
require 'longleaf/errors'
|
4
7
|
|
5
8
|
module Longleaf
|
6
9
|
# Manager which loads and provides access to {StorageLocation} objects
|
7
10
|
class StorageLocationManager
|
8
11
|
AF ||= Longleaf::AppFields
|
12
|
+
ST ||= Longleaf::StorageTypes
|
9
13
|
|
10
14
|
# Hash mapping storage location names to {StorageLocation} objects
|
11
15
|
attr_reader :locations
|
16
|
+
# Mapping of storage types to storage location classes
|
17
|
+
@@storage_type_mappings = {
|
18
|
+
ST::FILESYSTEM_STORAGE_TYPE => Longleaf::FilesystemStorageLocation,
|
19
|
+
ST::S3_STORAGE_TYPE => Longleaf::S3StorageLocation
|
20
|
+
}
|
21
|
+
@@metadata_type_mappings = { ST::FILESYSTEM_STORAGE_TYPE => Longleaf::FilesystemMetadataLocation }
|
12
22
|
|
13
23
|
# @param config [Hash] has representation of the application configuration
|
14
24
|
def initialize(config)
|
@@ -16,15 +26,9 @@ module Longleaf
|
|
16
26
|
|
17
27
|
@locations = Hash.new
|
18
28
|
config[AF::LOCATIONS].each do |name, properties|
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
location = Longleaf::StorageLocation.new(name: name,
|
23
|
-
path: path,
|
24
|
-
metadata_path: md_path,
|
25
|
-
metadata_digests: md_digests)
|
26
|
-
|
27
|
-
@locations[name] = location
|
29
|
+
md_loc = instantiate_metadata_location(properties)
|
30
|
+
|
31
|
+
@locations[name] = instantiate_storage_location(name, properties, md_loc)
|
28
32
|
end
|
29
33
|
@locations.freeze
|
30
34
|
end
|
@@ -35,7 +39,7 @@ module Longleaf
|
|
35
39
|
def get_location_by_path(path)
|
36
40
|
raise ArgumentError.new("Path parameter is required") if path.nil? || path.empty?
|
37
41
|
@locations.each do |name, location|
|
38
|
-
return location if
|
42
|
+
return location if location.contains?(path)
|
39
43
|
end
|
40
44
|
|
41
45
|
nil
|
@@ -47,7 +51,7 @@ module Longleaf
|
|
47
51
|
def get_location_by_metadata_path(md_path)
|
48
52
|
raise ArgumentError.new("Metadata path parameter is required") if md_path.nil? || md_path.empty?
|
49
53
|
@locations.each do |name, location|
|
50
|
-
return location if
|
54
|
+
return location if location.metadata_location.contains?(md_path)
|
51
55
|
end
|
52
56
|
|
53
57
|
nil
|
@@ -68,5 +72,27 @@ module Longleaf
|
|
68
72
|
end
|
69
73
|
location
|
70
74
|
end
|
75
|
+
|
76
|
+
private
|
77
|
+
def instantiate_metadata_location(loc_properties)
|
78
|
+
m_config = loc_properties[AF::METADATA_CONFIG]
|
79
|
+
m_type = m_config[AF::STORAGE_TYPE]
|
80
|
+
m_type = ST::FILESYSTEM_STORAGE_TYPE if m_type.nil?
|
81
|
+
|
82
|
+
m_class = @@metadata_type_mappings[m_type]
|
83
|
+
raise ArgumentError.new("Unknown metadata location type #{m_type}") if m_class.nil?
|
84
|
+
|
85
|
+
m_class.new(m_config)
|
86
|
+
end
|
87
|
+
|
88
|
+
def instantiate_storage_location(name, properties, md_loc)
|
89
|
+
s_type = properties[AF::STORAGE_TYPE]
|
90
|
+
s_type = ST::FILESYSTEM_STORAGE_TYPE if s_type.nil?
|
91
|
+
|
92
|
+
s_class = @@storage_type_mappings[s_type]
|
93
|
+
raise ArgumentError.new("Unknown storage location type #{s_type}") if s_class.nil?
|
94
|
+
|
95
|
+
s_class.new(name, properties, md_loc)
|
96
|
+
end
|
71
97
|
end
|
72
98
|
end
|
@@ -1,63 +1,74 @@
|
|
1
1
|
require 'pathname'
|
2
|
-
require 'longleaf/models/storage_location'
|
3
2
|
require 'longleaf/models/app_fields'
|
3
|
+
require 'longleaf/models/storage_types'
|
4
4
|
require 'longleaf/errors'
|
5
5
|
require_relative 'configuration_validator'
|
6
|
-
require 'longleaf/services/
|
6
|
+
require 'longleaf/services/filesystem_location_validator'
|
7
|
+
require 'longleaf/services/s3_location_validator'
|
7
8
|
|
8
9
|
module Longleaf
|
9
10
|
# Validates application configuration of storage locations
|
10
11
|
class StorageLocationValidator < ConfigurationValidator
|
11
12
|
AF ||= Longleaf::AppFields
|
13
|
+
ST ||= Longleaf::StorageTypes
|
12
14
|
|
15
|
+
@@storage_type_mappings = {
|
16
|
+
ST::FILESYSTEM_STORAGE_TYPE => Longleaf::FilesystemLocationValidator,
|
17
|
+
ST::S3_STORAGE_TYPE => Longleaf::S3LocationValidator
|
18
|
+
}
|
19
|
+
|
20
|
+
# @param config [Hash] hash containing the application configuration
|
21
|
+
def initialize(config)
|
22
|
+
super(config)
|
23
|
+
@existing_paths = Array.new
|
24
|
+
end
|
25
|
+
|
26
|
+
protected
|
13
27
|
# Validates configuration to ensure that it is syntactically correct and does not violate
|
14
28
|
# schema and uniqueness requirements.
|
15
|
-
|
16
|
-
|
17
|
-
assert("Configuration must
|
18
|
-
|
19
|
-
locations = config[AF::LOCATIONS]
|
29
|
+
def validate
|
30
|
+
assert("Configuration must be a hash, but a #{@config.class} was provided", @config.class == Hash)
|
31
|
+
assert("Configuration must contain a root '#{AF::LOCATIONS}' key", @config.key?(AF::LOCATIONS))
|
32
|
+
locations = @config[AF::LOCATIONS]
|
20
33
|
assert("'#{AF::LOCATIONS}' must be a hash of locations", locations.class == Hash)
|
21
34
|
|
22
|
-
existing_paths = Array.new
|
23
35
|
locations.each do |name, properties|
|
24
|
-
|
25
|
-
|
36
|
+
register_on_failure do
|
37
|
+
assert("Name of storage location must be a string, but was of type #{name.class}", name.instance_of?(String))
|
38
|
+
assert("Storage location '#{name}' must be a hash, but a #{properties.class} was provided", properties.is_a?(Hash))
|
39
|
+
|
40
|
+
register_on_failure { assert_path_property_valid(name, AF::LOCATION_PATH, properties, 'location') }
|
26
41
|
|
27
|
-
|
28
|
-
|
42
|
+
assert("Metadata location must be present for location '#{name}'", properties.key?(AF::METADATA_CONFIG))
|
43
|
+
assert_path_property_valid(name, AF::LOCATION_PATH, properties[AF::METADATA_CONFIG], 'metadata')
|
44
|
+
end
|
29
45
|
end
|
46
|
+
|
47
|
+
@result
|
30
48
|
end
|
31
49
|
|
32
|
-
|
50
|
+
private
|
51
|
+
def assert_path_property_valid(name, path_prop, properties, section_name)
|
33
52
|
path = properties[path_prop]
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
"Storage location '#{name}' specifies invalid '#{path_prop}' property: #{err.message}")
|
39
|
-
end
|
40
|
-
assert("Storage location '#{name}' must specify a '#{path_prop}' property", !path.nil? && !path.empty?)
|
41
|
-
assert("Storage location '#{name}' must specify an absolute path for property '#{path_prop}'",
|
42
|
-
Pathname.new(path).absolute? && !path.include?('/..'))
|
43
|
-
assert("Storage location '#{name}' specifies a '#{path_prop}' directory which does not exist", Dir.exist?(path))
|
53
|
+
|
54
|
+
storage_type = properties[AF::STORAGE_TYPE] || ST::DEFAULT_STORAGE_TYPE
|
55
|
+
type_validator = @@storage_type_mappings[storage_type]
|
56
|
+
type_validator.validate(self, name, path_prop, section_name, path)
|
44
57
|
|
45
58
|
# Ensure paths have trailing slash to avoid matching on partial directory names
|
46
59
|
path += '/' unless path.end_with?('/')
|
47
60
|
# Verify that the (metadata_)path property's value is not inside of another storage location or vice versa
|
48
|
-
existing_paths.each do |existing|
|
61
|
+
@existing_paths.each do |existing|
|
49
62
|
if existing.start_with?(path) || path.start_with?(existing)
|
50
|
-
msg = "Location '#{name}' defines property #{path_prop} with value '#{path}'" \
|
63
|
+
msg = "Location '#{name}' defines property #{section_name} #{path_prop} with value '#{path}'" \
|
51
64
|
" which overlaps with another configured path '#{existing}'." \
|
52
|
-
" Storage locations must not define #{AF::LOCATION_PATH}
|
65
|
+
" Storage locations must not define #{AF::LOCATION_PATH}" \
|
53
66
|
" properties which are contained by another location property"
|
54
|
-
|
67
|
+
fail(msg)
|
55
68
|
end
|
56
69
|
end
|
57
70
|
|
58
|
-
existing_paths << path
|
71
|
+
@existing_paths << path
|
59
72
|
end
|
60
|
-
|
61
|
-
private_class_method :assert_path_property_valid
|
62
73
|
end
|
63
74
|
end
|
@@ -27,14 +27,21 @@ module Longleaf
|
|
27
27
|
# @param path [String] value for the 'path' field
|
28
28
|
# @param md_path [String] value for the 'metadata_path' field
|
29
29
|
# @return this builder
|
30
|
-
def with_location(name:, path: '/file/path/', md_path: '/metadata/path/', md_digests: nil)
|
30
|
+
def with_location(name:, path: '/file/path/', s_type: nil, md_path: '/metadata/path/', md_type: nil, md_digests: nil)
|
31
31
|
@config[AF::LOCATIONS] = Hash.new unless @config.key?(AF::LOCATIONS)
|
32
32
|
|
33
33
|
location = {}
|
34
34
|
@config[AF::LOCATIONS][name] = location
|
35
35
|
location[AF::LOCATION_PATH] = path unless path.nil?
|
36
|
-
location[AF::
|
37
|
-
|
36
|
+
location[AF::STORAGE_TYPE] = s_type unless s_type.nil?
|
37
|
+
|
38
|
+
if !md_path.nil?
|
39
|
+
md_loc = { AF::LOCATION_PATH => md_path }
|
40
|
+
location[AF::METADATA_CONFIG] = md_loc
|
41
|
+
|
42
|
+
md_loc[AF::METADATA_DIGESTS] = md_digests unless md_digests.nil?
|
43
|
+
md_loc[AF::STORAGE_TYPE] = md_type unless md_type.nil?
|
44
|
+
end
|
38
45
|
self
|
39
46
|
end
|
40
47
|
|
@@ -0,0 +1,16 @@
|
|
1
|
+
module Longleaf
|
2
|
+
module ConfigValidatorHelpers
|
3
|
+
def fails_validation_with_error(validator, *error_messages)
|
4
|
+
result = validator.validate_config
|
5
|
+
expect(result.valid?).to be false
|
6
|
+
error_messages.each do |error_message|
|
7
|
+
expect(result.errors).to include(error_message)
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
def passes_validation(validator)
|
12
|
+
result = validator.validate_config
|
13
|
+
expect(result.valid?).to eq(true), "expected validation to pass, but received errors:\n#{result.errors&.join("\n")}"
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
data/lib/longleaf/version.rb
CHANGED
data/longleaf.gemspec
CHANGED
@@ -29,10 +29,12 @@ Gem::Specification.new do |spec|
|
|
29
29
|
spec.add_dependency "thor", "~> 0.20.0"
|
30
30
|
spec.add_dependency "yard", "~> 0.9.16"
|
31
31
|
spec.add_dependency "sequel", "~> 5.20"
|
32
|
+
spec.add_dependency "aws-sdk-s3", "~> 1.56"
|
32
33
|
|
33
|
-
spec.add_development_dependency "bundler", "~> 1
|
34
|
+
spec.add_development_dependency "bundler", "~> 2.1"
|
34
35
|
spec.add_development_dependency "rake", "~> 12.0"
|
35
36
|
spec.add_development_dependency "rspec", "~> 3.8"
|
37
|
+
spec.add_development_dependency "rspec-core", "~> 3.8"
|
36
38
|
spec.add_development_dependency "rspec_junit_formatter", "~> 0.4"
|
37
39
|
spec.add_development_dependency "factory_bot", "~> 5.0"
|
38
40
|
spec.add_development_dependency "aruba", "~> 0.14.9"
|
data/mkdocs.yml
CHANGED
@@ -1,9 +1,10 @@
|
|
1
1
|
site_name: Longleaf Documentation
|
2
2
|
site_url: https://unc-libraries.github.io/longleaf-preservation
|
3
3
|
theme: readthedocs
|
4
|
-
#repo_name: 'UNC-Libraries/Longleaf-preservation'
|
4
|
+
# repo_name: 'UNC-Libraries/Longleaf-preservation'
|
5
5
|
#setting edit_uri to empty string removes the "Edit in" link in upper right hand corner of pages, but keeps the Github link to repo in the bottom of sidepane
|
6
6
|
repo_url: https://github.com/UNC-Libraries/longleaf-preservation/
|
7
|
+
site_description: Longleaf is a command-line tool which allows users to configure a set of storage locations and define custom sets of preservation services to run on their contents. These services are executed in response to applicable preservation events issued by clients. Its primary goal is to provide tools to create a simple and customizable preservation environment.
|
7
8
|
edit_uri: ''
|
8
9
|
|
9
10
|
# include extra css file, in docs/
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: longleaf
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 1.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ben Pennell
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-10-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: thor
|
@@ -52,20 +52,34 @@ dependencies:
|
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '5.20'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: aws-sdk-s3
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '1.56'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '1.56'
|
55
69
|
- !ruby/object:Gem::Dependency
|
56
70
|
name: bundler
|
57
71
|
requirement: !ruby/object:Gem::Requirement
|
58
72
|
requirements:
|
59
73
|
- - "~>"
|
60
74
|
- !ruby/object:Gem::Version
|
61
|
-
version: '1
|
75
|
+
version: '2.1'
|
62
76
|
type: :development
|
63
77
|
prerelease: false
|
64
78
|
version_requirements: !ruby/object:Gem::Requirement
|
65
79
|
requirements:
|
66
80
|
- - "~>"
|
67
81
|
- !ruby/object:Gem::Version
|
68
|
-
version: '1
|
82
|
+
version: '2.1'
|
69
83
|
- !ruby/object:Gem::Dependency
|
70
84
|
name: rake
|
71
85
|
requirement: !ruby/object:Gem::Requirement
|
@@ -94,6 +108,20 @@ dependencies:
|
|
94
108
|
- - "~>"
|
95
109
|
- !ruby/object:Gem::Version
|
96
110
|
version: '3.8'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: rspec-core
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - "~>"
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '3.8'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - "~>"
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '3.8'
|
97
125
|
- !ruby/object:Gem::Dependency
|
98
126
|
name: rspec_junit_formatter
|
99
127
|
requirement: !ruby/object:Gem::Requirement
|
@@ -238,7 +266,6 @@ files:
|
|
238
266
|
- ".rspec"
|
239
267
|
- ".rubocop.yml"
|
240
268
|
- ".rubocop_todo.yml"
|
241
|
-
- ".travis.yml"
|
242
269
|
- ".yardopts"
|
243
270
|
- Gemfile
|
244
271
|
- LICENSE.txt
|
@@ -344,10 +371,12 @@ files:
|
|
344
371
|
- exe/longleaf
|
345
372
|
- lib/longleaf.rb
|
346
373
|
- lib/longleaf/candidates/file_selector.rb
|
374
|
+
- lib/longleaf/candidates/manifest_digest_provider.rb
|
347
375
|
- lib/longleaf/candidates/registered_file_selector.rb
|
348
376
|
- lib/longleaf/candidates/service_candidate_filesystem_iterator.rb
|
349
377
|
- lib/longleaf/candidates/service_candidate_index_iterator.rb
|
350
378
|
- lib/longleaf/candidates/service_candidate_locator.rb
|
379
|
+
- lib/longleaf/candidates/single_digest_provider.rb
|
351
380
|
- lib/longleaf/cli.rb
|
352
381
|
- lib/longleaf/commands/deregister_command.rb
|
353
382
|
- lib/longleaf/commands/preserve_command.rb
|
@@ -361,7 +390,10 @@ files:
|
|
361
390
|
- lib/longleaf/events/event_status_tracking.rb
|
362
391
|
- lib/longleaf/events/preserve_event.rb
|
363
392
|
- lib/longleaf/events/register_event.rb
|
393
|
+
- lib/longleaf/helpers/case_insensitive_hash.rb
|
364
394
|
- lib/longleaf/helpers/digest_helper.rb
|
395
|
+
- lib/longleaf/helpers/s3_uri_helper.rb
|
396
|
+
- lib/longleaf/helpers/selection_options_parser.rb
|
365
397
|
- lib/longleaf/helpers/service_date_helper.rb
|
366
398
|
- lib/longleaf/indexing/index_manager.rb
|
367
399
|
- lib/longleaf/indexing/sequel_index_driver.rb
|
@@ -369,23 +401,32 @@ files:
|
|
369
401
|
- lib/longleaf/logging/redirecting_logger.rb
|
370
402
|
- lib/longleaf/models/app_fields.rb
|
371
403
|
- lib/longleaf/models/file_record.rb
|
404
|
+
- lib/longleaf/models/filesystem_metadata_location.rb
|
405
|
+
- lib/longleaf/models/filesystem_storage_location.rb
|
372
406
|
- lib/longleaf/models/md_fields.rb
|
407
|
+
- lib/longleaf/models/metadata_location.rb
|
373
408
|
- lib/longleaf/models/metadata_record.rb
|
409
|
+
- lib/longleaf/models/s3_storage_location.rb
|
374
410
|
- lib/longleaf/models/service_definition.rb
|
375
411
|
- lib/longleaf/models/service_fields.rb
|
376
412
|
- lib/longleaf/models/service_record.rb
|
377
413
|
- lib/longleaf/models/storage_location.rb
|
414
|
+
- lib/longleaf/models/storage_types.rb
|
378
415
|
- lib/longleaf/models/system_config_fields.rb
|
379
416
|
- lib/longleaf/preservation_services/file_check_service.rb
|
380
417
|
- lib/longleaf/preservation_services/fixity_check_service.rb
|
381
418
|
- lib/longleaf/preservation_services/rsync_replication_service.rb
|
419
|
+
- lib/longleaf/preservation_services/s3_replication_service.rb
|
382
420
|
- lib/longleaf/services/application_config_deserializer.rb
|
383
421
|
- lib/longleaf/services/application_config_manager.rb
|
384
422
|
- lib/longleaf/services/application_config_validator.rb
|
385
423
|
- lib/longleaf/services/configuration_validator.rb
|
424
|
+
- lib/longleaf/services/filesystem_location_validator.rb
|
386
425
|
- lib/longleaf/services/metadata_deserializer.rb
|
387
426
|
- lib/longleaf/services/metadata_persistence_manager.rb
|
388
427
|
- lib/longleaf/services/metadata_serializer.rb
|
428
|
+
- lib/longleaf/services/metadata_validator.rb
|
429
|
+
- lib/longleaf/services/s3_location_validator.rb
|
389
430
|
- lib/longleaf/services/service_class_cache.rb
|
390
431
|
- lib/longleaf/services/service_definition_manager.rb
|
391
432
|
- lib/longleaf/services/service_definition_validator.rb
|
@@ -394,8 +435,8 @@ files:
|
|
394
435
|
- lib/longleaf/services/service_mapping_validator.rb
|
395
436
|
- lib/longleaf/services/storage_location_manager.rb
|
396
437
|
- lib/longleaf/services/storage_location_validator.rb
|
397
|
-
- lib/longleaf/services/storage_path_validator.rb
|
398
438
|
- lib/longleaf/specs/config_builder.rb
|
439
|
+
- lib/longleaf/specs/config_validator_helpers.rb
|
399
440
|
- lib/longleaf/specs/custom_matchers.rb
|
400
441
|
- lib/longleaf/specs/file_helpers.rb
|
401
442
|
- lib/longleaf/specs/metadata_builder.rb
|
@@ -423,8 +464,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
423
464
|
- !ruby/object:Gem::Version
|
424
465
|
version: '0'
|
425
466
|
requirements: []
|
426
|
-
|
427
|
-
rubygems_version: 2.7.7
|
467
|
+
rubygems_version: 3.1.2
|
428
468
|
signing_key:
|
429
469
|
specification_version: 4
|
430
470
|
summary: Longleaf preservation services tool
|