longleaf 0.1.0.pre.2 → 1.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (184) hide show
  1. checksums.yaml +4 -4
  2. data/.circleci/config.yml +94 -0
  3. data/.editorconfig +13 -0
  4. data/.gitignore +4 -1
  5. data/.rubocop.yml +44 -0
  6. data/.rubocop_todo.yml +834 -0
  7. data/.yardopts +1 -0
  8. data/Gemfile +16 -1
  9. data/README.md +98 -12
  10. data/Rakefile +6 -0
  11. data/bin/setup +16 -1
  12. data/docs/aboutlongleaf.md +28 -0
  13. data/docs/extra.css +32 -0
  14. data/docs/img/change-file.png +0 -0
  15. data/docs/img/ll-example-preserved.png +0 -0
  16. data/docs/index.md +19 -0
  17. data/docs/install.md +66 -0
  18. data/docs/ll-example/config-example-relative.yml +33 -0
  19. data/docs/ll-example/files-dir/LLexample-PDF.pdf +0 -0
  20. data/docs/ll-example/files-dir/LLexample-TOCHANGE.txt +15 -0
  21. data/docs/ll-example/files-dir/LLexample-tokeep.txt +10 -0
  22. data/docs/ll-example/metadata-dir/.gitkeep +0 -0
  23. data/docs/ll-example/replica-files/.gitkeep +0 -0
  24. data/docs/ll-example/replica-metadata/.gitkeep +0 -0
  25. data/docs/quickstart.md +270 -0
  26. data/docs/rdocs/Longleaf.html +135 -0
  27. data/docs/rdocs/Longleaf/AppFields.html +178 -0
  28. data/docs/rdocs/Longleaf/ApplicationConfigDeserializer.html +631 -0
  29. data/docs/rdocs/Longleaf/ApplicationConfigManager.html +610 -0
  30. data/docs/rdocs/Longleaf/ApplicationConfigValidator.html +238 -0
  31. data/docs/rdocs/Longleaf/CLI.html +909 -0
  32. data/docs/rdocs/Longleaf/ChecksumMismatchError.html +151 -0
  33. data/docs/rdocs/Longleaf/ConfigBuilder.html +1339 -0
  34. data/docs/rdocs/Longleaf/ConfigurationError.html +143 -0
  35. data/docs/rdocs/Longleaf/ConfigurationValidator.html +227 -0
  36. data/docs/rdocs/Longleaf/DeregisterCommand.html +420 -0
  37. data/docs/rdocs/Longleaf/DeregisterEvent.html +453 -0
  38. data/docs/rdocs/Longleaf/DeregistrationError.html +151 -0
  39. data/docs/rdocs/Longleaf/DigestHelper.html +419 -0
  40. data/docs/rdocs/Longleaf/EventError.html +147 -0
  41. data/docs/rdocs/Longleaf/EventNames.html +163 -0
  42. data/docs/rdocs/Longleaf/EventStatusTracking.html +656 -0
  43. data/docs/rdocs/Longleaf/FileCheckService.html +540 -0
  44. data/docs/rdocs/Longleaf/FileHelpers.html +520 -0
  45. data/docs/rdocs/Longleaf/FileRecord.html +716 -0
  46. data/docs/rdocs/Longleaf/FileSelector.html +901 -0
  47. data/docs/rdocs/Longleaf/FixityCheckService.html +691 -0
  48. data/docs/rdocs/Longleaf/IndexManager.html +1155 -0
  49. data/docs/rdocs/Longleaf/InvalidDigestAlgorithmError.html +143 -0
  50. data/docs/rdocs/Longleaf/InvalidStoragePathError.html +143 -0
  51. data/docs/rdocs/Longleaf/Logging.html +405 -0
  52. data/docs/rdocs/Longleaf/Logging/RedirectingLogger.html +1213 -0
  53. data/docs/rdocs/Longleaf/LongleafError.html +139 -0
  54. data/docs/rdocs/Longleaf/MDFields.html +193 -0
  55. data/docs/rdocs/Longleaf/MetadataBuilder.html +787 -0
  56. data/docs/rdocs/Longleaf/MetadataDeserializer.html +537 -0
  57. data/docs/rdocs/Longleaf/MetadataError.html +143 -0
  58. data/docs/rdocs/Longleaf/MetadataPersistenceManager.html +539 -0
  59. data/docs/rdocs/Longleaf/MetadataRecord.html +1411 -0
  60. data/docs/rdocs/Longleaf/MetadataSerializer.html +786 -0
  61. data/docs/rdocs/Longleaf/PreservationServiceError.html +147 -0
  62. data/docs/rdocs/Longleaf/PreserveCommand.html +410 -0
  63. data/docs/rdocs/Longleaf/PreserveEvent.html +491 -0
  64. data/docs/rdocs/Longleaf/RegisterCommand.html +428 -0
  65. data/docs/rdocs/Longleaf/RegisterEvent.html +628 -0
  66. data/docs/rdocs/Longleaf/RegisteredFileSelector.html +446 -0
  67. data/docs/rdocs/Longleaf/RegistrationError.html +151 -0
  68. data/docs/rdocs/Longleaf/ReindexCommand.html +576 -0
  69. data/docs/rdocs/Longleaf/RsyncReplicationService.html +1180 -0
  70. data/docs/rdocs/Longleaf/SequelIndexDriver.html +1978 -0
  71. data/docs/rdocs/Longleaf/ServiceCandidateFilesystemIterator.html +572 -0
  72. data/docs/rdocs/Longleaf/ServiceCandidateIndexIterator.html +532 -0
  73. data/docs/rdocs/Longleaf/ServiceCandidateLocator.html +333 -0
  74. data/docs/rdocs/Longleaf/ServiceClassCache.html +725 -0
  75. data/docs/rdocs/Longleaf/ServiceDateHelper.html +425 -0
  76. data/docs/rdocs/Longleaf/ServiceDefinition.html +683 -0
  77. data/docs/rdocs/Longleaf/ServiceDefinitionManager.html +371 -0
  78. data/docs/rdocs/Longleaf/ServiceDefinitionValidator.html +269 -0
  79. data/docs/rdocs/Longleaf/ServiceFields.html +173 -0
  80. data/docs/rdocs/Longleaf/ServiceManager.html +1229 -0
  81. data/docs/rdocs/Longleaf/ServiceMappingManager.html +410 -0
  82. data/docs/rdocs/Longleaf/ServiceMappingValidator.html +347 -0
  83. data/docs/rdocs/Longleaf/ServiceRecord.html +821 -0
  84. data/docs/rdocs/Longleaf/StorageLocation.html +985 -0
  85. data/docs/rdocs/Longleaf/StorageLocationManager.html +729 -0
  86. data/docs/rdocs/Longleaf/StorageLocationUnavailableError.html +143 -0
  87. data/docs/rdocs/Longleaf/StorageLocationValidator.html +373 -0
  88. data/docs/rdocs/Longleaf/StoragePathValidator.html +253 -0
  89. data/docs/rdocs/Longleaf/SystemConfigBuilder.html +441 -0
  90. data/docs/rdocs/Longleaf/SystemConfigFields.html +163 -0
  91. data/docs/rdocs/Longleaf/ValidateConfigCommand.html +451 -0
  92. data/docs/rdocs/Longleaf/ValidateMetadataCommand.html +408 -0
  93. data/docs/rdocs/_index.html +660 -0
  94. data/docs/rdocs/class_list.html +51 -0
  95. data/docs/rdocs/css/common.css +1 -0
  96. data/docs/rdocs/css/full_list.css +58 -0
  97. data/docs/rdocs/css/style.css +496 -0
  98. data/docs/rdocs/file.README.html +165 -0
  99. data/docs/rdocs/file_list.html +56 -0
  100. data/docs/rdocs/frames.html +17 -0
  101. data/docs/rdocs/index.html +165 -0
  102. data/docs/rdocs/js/app.js +303 -0
  103. data/docs/rdocs/js/full_list.js +216 -0
  104. data/docs/rdocs/js/jquery.js +4 -0
  105. data/docs/rdocs/method_list.html +2051 -0
  106. data/docs/rdocs/top-level-namespace.html +110 -0
  107. data/lib/longleaf/candidates/file_selector.rb +139 -0
  108. data/lib/longleaf/candidates/manifest_digest_provider.rb +17 -0
  109. data/lib/longleaf/candidates/registered_file_selector.rb +67 -0
  110. data/lib/longleaf/candidates/service_candidate_filesystem_iterator.rb +93 -0
  111. data/lib/longleaf/candidates/service_candidate_index_iterator.rb +84 -0
  112. data/lib/longleaf/candidates/service_candidate_locator.rb +23 -0
  113. data/lib/longleaf/candidates/single_digest_provider.rb +13 -0
  114. data/lib/longleaf/cli.rb +237 -46
  115. data/lib/longleaf/commands/deregister_command.rb +51 -0
  116. data/lib/longleaf/commands/preserve_command.rb +50 -0
  117. data/lib/longleaf/commands/register_command.rb +32 -43
  118. data/lib/longleaf/commands/reindex_command.rb +92 -0
  119. data/lib/longleaf/commands/validate_config_command.rb +33 -8
  120. data/lib/longleaf/commands/validate_metadata_command.rb +51 -0
  121. data/lib/longleaf/errors.rb +26 -7
  122. data/lib/longleaf/events/deregister_event.rb +53 -0
  123. data/lib/longleaf/events/event_names.rb +9 -0
  124. data/lib/longleaf/events/event_status_tracking.rb +59 -0
  125. data/lib/longleaf/events/preserve_event.rb +81 -0
  126. data/lib/longleaf/events/register_event.rb +52 -51
  127. data/lib/longleaf/helpers/case_insensitive_hash.rb +38 -0
  128. data/lib/longleaf/helpers/digest_helper.rb +56 -0
  129. data/lib/longleaf/helpers/s3_uri_helper.rb +86 -0
  130. data/lib/longleaf/helpers/selection_options_parser.rb +189 -0
  131. data/lib/longleaf/helpers/service_date_helper.rb +78 -0
  132. data/lib/longleaf/indexing/index_manager.rb +101 -0
  133. data/lib/longleaf/indexing/sequel_index_driver.rb +306 -0
  134. data/lib/longleaf/logging.rb +5 -4
  135. data/lib/longleaf/logging/redirecting_logger.rb +26 -25
  136. data/lib/longleaf/models/app_fields.rb +7 -2
  137. data/lib/longleaf/models/file_record.rb +17 -8
  138. data/lib/longleaf/models/filesystem_metadata_location.rb +56 -0
  139. data/lib/longleaf/models/filesystem_storage_location.rb +52 -0
  140. data/lib/longleaf/models/md_fields.rb +2 -1
  141. data/lib/longleaf/models/metadata_location.rb +47 -0
  142. data/lib/longleaf/models/metadata_record.rb +39 -15
  143. data/lib/longleaf/models/s3_storage_location.rb +133 -0
  144. data/lib/longleaf/models/service_definition.rb +7 -6
  145. data/lib/longleaf/models/service_fields.rb +7 -1
  146. data/lib/longleaf/models/service_record.rb +10 -6
  147. data/lib/longleaf/models/storage_location.rb +24 -19
  148. data/lib/longleaf/models/storage_types.rb +9 -0
  149. data/lib/longleaf/models/system_config_fields.rb +9 -0
  150. data/lib/longleaf/preservation_services/file_check_service.rb +58 -0
  151. data/lib/longleaf/preservation_services/fixity_check_service.rb +123 -0
  152. data/lib/longleaf/preservation_services/rsync_replication_service.rb +182 -0
  153. data/lib/longleaf/preservation_services/s3_replication_service.rb +143 -0
  154. data/lib/longleaf/services/application_config_deserializer.rb +81 -24
  155. data/lib/longleaf/services/application_config_manager.rb +20 -6
  156. data/lib/longleaf/services/application_config_validator.rb +19 -9
  157. data/lib/longleaf/services/configuration_validator.rb +67 -4
  158. data/lib/longleaf/services/filesystem_location_validator.rb +16 -0
  159. data/lib/longleaf/services/metadata_deserializer.rb +113 -42
  160. data/lib/longleaf/services/metadata_persistence_manager.rb +47 -0
  161. data/lib/longleaf/services/metadata_serializer.rb +138 -25
  162. data/lib/longleaf/services/metadata_validator.rb +76 -0
  163. data/lib/longleaf/services/s3_location_validator.rb +19 -0
  164. data/lib/longleaf/services/service_class_cache.rb +112 -0
  165. data/lib/longleaf/services/service_definition_manager.rb +10 -7
  166. data/lib/longleaf/services/service_definition_validator.rb +25 -18
  167. data/lib/longleaf/services/service_manager.rb +86 -11
  168. data/lib/longleaf/services/service_mapping_manager.rb +13 -12
  169. data/lib/longleaf/services/service_mapping_validator.rb +36 -26
  170. data/lib/longleaf/services/storage_location_manager.rb +76 -15
  171. data/lib/longleaf/services/storage_location_validator.rb +49 -35
  172. data/lib/longleaf/specs/config_builder.rb +47 -23
  173. data/lib/longleaf/specs/config_validator_helpers.rb +16 -0
  174. data/lib/longleaf/specs/custom_matchers.rb +9 -0
  175. data/lib/longleaf/specs/file_helpers.rb +61 -0
  176. data/lib/longleaf/specs/metadata_builder.rb +92 -0
  177. data/lib/longleaf/specs/system_config_builder.rb +27 -0
  178. data/lib/longleaf/version.rb +1 -1
  179. data/longleaf.gemspec +20 -7
  180. data/mkdocs.yml +21 -0
  181. metadata +306 -23
  182. data/.travis.yml +0 -4
  183. data/lib/longleaf/commands/abstract_command.rb +0 -37
  184. data/lib/longleaf/services/storage_path_validator.rb +0 -16
@@ -0,0 +1,16 @@
1
+ module Longleaf
2
+ module ConfigValidatorHelpers
3
+ def fails_validation_with_error(validator, *error_messages)
4
+ result = validator.validate_config
5
+ expect(result.valid?).to be false
6
+ error_messages.each do |error_message|
7
+ expect(result.errors).to include(error_message)
8
+ end
9
+ end
10
+
11
+ def passes_validation(validator)
12
+ result = validator.validate_config
13
+ expect(result.valid?).to eq(true), "expected validation to pass, but received errors:\n#{result.errors&.join("\n")}"
14
+ end
15
+ end
16
+ end
@@ -0,0 +1,9 @@
1
+ module Longleaf
2
+ # Match if the parameter is a {FileRecord} with the expected path
3
+ RSpec::Matchers.define :be_file_record_for do |expected|
4
+ match do |actual|
5
+ return false if actual.nil? || !actual.is_a?(Longleaf::FileRecord)
6
+ actual.path == expected
7
+ end
8
+ end
9
+ end
@@ -0,0 +1,61 @@
1
+ require 'tempfile'
2
+ require 'tmpdir'
3
+
4
+ module Longleaf
5
+ # Test helper methods for creating test files
6
+ module FileHelpers
7
+ def make_test_dir(parent: nil, name: 'dir')
8
+ FileHelpers.make_test_dir(parent: parent, name: name)
9
+ end
10
+
11
+ def self.make_test_dir(parent: nil, name: 'dir')
12
+ if parent.nil?
13
+ Dir.mktmpdir(name)
14
+ else
15
+ path = File.join(parent, name)
16
+ Dir.mkdir(path)
17
+ path
18
+ end
19
+ end
20
+
21
+ def create_test_file(dir: nil, name: nil, content: 'content')
22
+ FileHelpers.create_test_file(dir: dir, name: name, content: content)
23
+ end
24
+
25
+ def self.create_test_file(dir: nil, name: nil, content: 'content')
26
+ if dir.nil? || name.nil?
27
+ name = 'test_file' if name.nil?
28
+ file = Tempfile.create(name, dir)
29
+ file << content
30
+ file.close
31
+ return file.path
32
+ else
33
+ path = File.join(dir, name)
34
+ File.open(path, 'w') { |f| f.write(content) }
35
+ path
36
+ end
37
+ end
38
+
39
+ def create_work_class(lib_dir, class_name, file_name, module_name = nil, is_applicable: true, init_body: "", perform: "")
40
+ FileHelpers.create_work_class(lib_dir, class_name, file_name, module_name, is_applicable, init_body, perform)
41
+ end
42
+
43
+ def self.create_work_class(lib_dir, class_name, file_name, module_name, is_applicable, init_body, perform)
44
+ class_contents = %Q(
45
+ class #{class_name}
46
+ def initialize(service_def, app_manager)
47
+ #{init_body}
48
+ end
49
+ def perform(file_rec, event)
50
+ #{perform}
51
+ end
52
+ def is_applicable?(event)
53
+ #{is_applicable}
54
+ end
55
+ end
56
+ )
57
+ class_contents = "module #{module_name}\n#{class_contents}\nend" unless module_name.nil?
58
+ create_test_file(dir: lib_dir, name: file_name, content: class_contents)
59
+ end
60
+ end
61
+ end
@@ -0,0 +1,92 @@
1
+ require 'longleaf/models/md_fields'
2
+ require 'longleaf/helpers/service_date_helper'
3
+ require 'yaml'
4
+
5
+ module Longleaf
6
+ # Test helper for constructing file metadata records
7
+ class MetadataBuilder
8
+ MF ||= Longleaf::MDFields
9
+
10
+ def initialize(file_path: nil, registered: ServiceDateHelper::formatted_timestamp)
11
+ @data = Hash.new
12
+ @services = Hash.new
13
+
14
+ unless file_path.nil?
15
+ @last_modified = File.mtime(file_path).utc.iso8601(3)
16
+ @file_size = File.size(file_path)
17
+ end
18
+
19
+ @registered = registered
20
+ end
21
+
22
+ def deregistered(timestamp = ServiceDateHelper::formatted_timestamp)
23
+ @deregistered = timestamp
24
+ self
25
+ end
26
+
27
+ def with_checksum(alg, value)
28
+ @checksums = Hash.new unless @data.key?(MF::CHECKSUMS)
29
+ @checksums[alg] = value
30
+ self
31
+ end
32
+
33
+ def with_service(name, timestamp: ServiceDateHelper::formatted_timestamp, run_needed: false, properties: nil,
34
+ failure_timestamp: nil)
35
+ timestamp = format_timestamp(timestamp)
36
+ failure_timestamp = format_timestamp(failure_timestamp) unless failure_timestamp.nil?
37
+
38
+ @services[name] = ServiceRecord.new(
39
+ properties: properties.nil? ? Hash.new : nil,
40
+ timestamp: timestamp,
41
+ run_needed: run_needed)
42
+ @services[name].failure_timestamp = failure_timestamp
43
+ self
44
+ end
45
+
46
+ def with_properties(properties)
47
+ @properties = properties
48
+ self
49
+ end
50
+
51
+ # @return the constructed metadata record
52
+ def get_metadata_record
53
+ MetadataRecord.new(properties: @properties,
54
+ services: @services,
55
+ deregistered: @deregistered,
56
+ registered: @registered,
57
+ checksums: @checksums,
58
+ file_size: @file_size,
59
+ last_modified: @last_modified)
60
+ end
61
+
62
+ # Add the generated metadata record to the given file record
63
+ def register_to(file_rec)
64
+ file_rec.metadata_record = get_metadata_record
65
+ self
66
+ end
67
+
68
+ # Writes the metadata record from this builder into a temporary file, or if a file
69
+ # record is provided, then to the expected metadata path for the record, and assigns
70
+ # the result as the metadata record for the file record.
71
+ # @return the file path of the config file
72
+ def write_to_yaml_file(file_rec: nil)
73
+ md_path = nil
74
+ if file_rec.nil?
75
+ md_path = TempFile.new(['metadata', 'yml']).path
76
+ else
77
+ md_path = file_rec.metadata_path
78
+ end
79
+
80
+ md_rec = get_metadata_record
81
+ MetadataSerializer::write(metadata: md_rec, file_path: md_path)
82
+ file_rec.metadata_record = md_rec
83
+
84
+ md_path
85
+ end
86
+
87
+ private
88
+ def format_timestamp(timestamp)
89
+ timestamp.kind_of?(Time) ? ServiceDateHelper::formatted_timestamp(timestamp) : timestamp
90
+ end
91
+ end
92
+ end
@@ -0,0 +1,27 @@
1
+ require 'longleaf/models/system_config_fields'
2
+ require 'yaml'
3
+
4
+ module Longleaf
5
+ # Test helper for constructing system configuration hashes
6
+ class SystemConfigBuilder
7
+ SCF ||= Longleaf::SystemConfigFields
8
+
9
+ attr_accessor :config
10
+
11
+ def initialize
12
+ @config = Hash.new
13
+ end
14
+
15
+ def with_index(adapter, connection)
16
+ @config[SCF::MD_INDEX] = Hash.new
17
+ @config[SCF::MD_INDEX][SCF::MD_INDEX_ADAPTER] = adapter
18
+ @config[SCF::MD_INDEX][SCF::MD_INDEX_CONNECTION] = connection
19
+ self
20
+ end
21
+
22
+ # @return the constructed configuration
23
+ def get
24
+ @config
25
+ end
26
+ end
27
+ end
@@ -1,3 +1,3 @@
1
1
  module Longleaf
2
- VERSION = "0.1.0.pre.2"
2
+ VERSION = "1.0.0"
3
3
  end
@@ -6,14 +6,17 @@ require 'longleaf/version'
6
6
  Gem::Specification.new do |spec|
7
7
  spec.name = "longleaf"
8
8
  spec.version = Longleaf::VERSION
9
- spec.authors = ["bbpennel"]
9
+ spec.authors = ["Ben Pennell"]
10
10
  spec.email = ["bbpennel@email.unc.edu"]
11
11
 
12
12
  spec.summary = %q{Longleaf preservation services tool}
13
- spec.description = %q{Provides a framework for performing preservation services over sets of files.}
14
- spec.homepage = "https://github.com/UNC-Libraries/"
13
+ spec.description = %q{Longleaf is a command-line tool which allows users to configure a set of storage locations and define custom sets of preservation services to run on their contents. These services are executed in response to applicable preservation events issued by clients. Its primary goal is to provide tools to create a simple and customizable preservation environment.}
14
+ spec.homepage = "https://unc-libraries.github.io/longleaf-preservation"
15
+ spec.metadata = { "source_code_uri" => "https://github.com/UNC-Libraries/longleaf-preservation" }
15
16
  spec.license = "Apache-2.0"
16
17
 
18
+ spec.required_ruby_version = '>= 2.3'
19
+
17
20
  # Specify which files should be added to the gem when it is released.
18
21
  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
19
22
  spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do
@@ -25,10 +28,20 @@ Gem::Specification.new do |spec|
25
28
 
26
29
  spec.add_dependency "thor", "~> 0.20.0"
27
30
  spec.add_dependency "yard", "~> 0.9.16"
31
+ spec.add_dependency "sequel", "~> 5.20"
32
+ spec.add_dependency "aws-sdk-s3", "~> 1.56"
28
33
 
29
- spec.add_development_dependency "bundler", "~> 1.16"
34
+ spec.add_development_dependency "bundler", "~> 2.1"
30
35
  spec.add_development_dependency "rake", "~> 12.0"
31
- spec.add_development_dependency "rspec", "~> 3.6.0"
32
- spec.add_development_dependency "factory_bot", "~> 4.0"
33
- spec.add_development_dependency "aruba", "~> 0.14.0"
36
+ spec.add_development_dependency "rspec", "~> 3.8"
37
+ spec.add_development_dependency "rspec-core", "~> 3.8"
38
+ spec.add_development_dependency "rspec_junit_formatter", "~> 0.4"
39
+ spec.add_development_dependency "factory_bot", "~> 5.0"
40
+ spec.add_development_dependency "aruba", "~> 0.14.9"
41
+ spec.add_development_dependency "rubocop", '~> 0.70.0'
42
+ spec.add_development_dependency "rubocop-rspec", '~> 1.33'
43
+ spec.add_development_dependency "rubocop-performance", '~> 1.3'
44
+ spec.add_development_dependency "rubocop-sequel", '~> 0.0.5'
45
+ spec.add_development_dependency "amalgalite", "~> 1.6"
46
+ spec.add_development_dependency "simplecov", "~> 0.16"
34
47
  end
@@ -0,0 +1,21 @@
1
+ site_name: Longleaf Documentation
2
+ site_url: https://unc-libraries.github.io/longleaf-preservation
3
+ theme: readthedocs
4
+ # repo_name: 'UNC-Libraries/Longleaf-preservation'
5
+ #setting edit_uri to empty string removes the "Edit in" link in upper right hand corner of pages, but keeps the Github link to repo in the bottom of sidepane
6
+ repo_url: https://github.com/UNC-Libraries/longleaf-preservation/
7
+ site_description: Longleaf is a command-line tool which allows users to configure a set of storage locations and define custom sets of preservation services to run on their contents. These services are executed in response to applicable preservation events issued by clients. Its primary goal is to provide tools to create a simple and customizable preservation environment.
8
+ edit_uri: ''
9
+
10
+ # include extra css file, in docs/
11
+ extra_css: [extra.css]
12
+
13
+ # docs markdown pages to include in sidepane navigation
14
+ nav:
15
+ - Welcome: index.md
16
+ - About Longleaf: aboutlongleaf.md
17
+ - Installation: install.md
18
+ - Basic Usage: quickstart.md
19
+ # pages to add and include later
20
+ # - Troubleshooting: trouble.md
21
+ # - FAQs: faq.md
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: longleaf
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0.pre.2
4
+ version: 1.0.0
5
5
  platform: ruby
6
6
  authors:
7
- - bbpennel
7
+ - Ben Pennell
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2018-10-18 00:00:00.000000000 Z
11
+ date: 2020-10-14 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: thor
@@ -38,20 +38,48 @@ dependencies:
38
38
  - - "~>"
39
39
  - !ruby/object:Gem::Version
40
40
  version: 0.9.16
41
+ - !ruby/object:Gem::Dependency
42
+ name: sequel
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '5.20'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '5.20'
55
+ - !ruby/object:Gem::Dependency
56
+ name: aws-sdk-s3
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '1.56'
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '1.56'
41
69
  - !ruby/object:Gem::Dependency
42
70
  name: bundler
43
71
  requirement: !ruby/object:Gem::Requirement
44
72
  requirements:
45
73
  - - "~>"
46
74
  - !ruby/object:Gem::Version
47
- version: '1.16'
75
+ version: '2.1'
48
76
  type: :development
49
77
  prerelease: false
50
78
  version_requirements: !ruby/object:Gem::Requirement
51
79
  requirements:
52
80
  - - "~>"
53
81
  - !ruby/object:Gem::Version
54
- version: '1.16'
82
+ version: '2.1'
55
83
  - !ruby/object:Gem::Dependency
56
84
  name: rake
57
85
  requirement: !ruby/object:Gem::Requirement
@@ -72,44 +100,159 @@ dependencies:
72
100
  requirements:
73
101
  - - "~>"
74
102
  - !ruby/object:Gem::Version
75
- version: 3.6.0
103
+ version: '3.8'
76
104
  type: :development
77
105
  prerelease: false
78
106
  version_requirements: !ruby/object:Gem::Requirement
79
107
  requirements:
80
108
  - - "~>"
81
109
  - !ruby/object:Gem::Version
82
- version: 3.6.0
110
+ version: '3.8'
111
+ - !ruby/object:Gem::Dependency
112
+ name: rspec-core
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - "~>"
116
+ - !ruby/object:Gem::Version
117
+ version: '3.8'
118
+ type: :development
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - "~>"
123
+ - !ruby/object:Gem::Version
124
+ version: '3.8'
125
+ - !ruby/object:Gem::Dependency
126
+ name: rspec_junit_formatter
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - "~>"
130
+ - !ruby/object:Gem::Version
131
+ version: '0.4'
132
+ type: :development
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - "~>"
137
+ - !ruby/object:Gem::Version
138
+ version: '0.4'
83
139
  - !ruby/object:Gem::Dependency
84
140
  name: factory_bot
85
141
  requirement: !ruby/object:Gem::Requirement
86
142
  requirements:
87
143
  - - "~>"
88
144
  - !ruby/object:Gem::Version
89
- version: '4.0'
145
+ version: '5.0'
90
146
  type: :development
91
147
  prerelease: false
92
148
  version_requirements: !ruby/object:Gem::Requirement
93
149
  requirements:
94
150
  - - "~>"
95
151
  - !ruby/object:Gem::Version
96
- version: '4.0'
152
+ version: '5.0'
97
153
  - !ruby/object:Gem::Dependency
98
154
  name: aruba
99
155
  requirement: !ruby/object:Gem::Requirement
100
156
  requirements:
101
157
  - - "~>"
102
158
  - !ruby/object:Gem::Version
103
- version: 0.14.0
159
+ version: 0.14.9
160
+ type: :development
161
+ prerelease: false
162
+ version_requirements: !ruby/object:Gem::Requirement
163
+ requirements:
164
+ - - "~>"
165
+ - !ruby/object:Gem::Version
166
+ version: 0.14.9
167
+ - !ruby/object:Gem::Dependency
168
+ name: rubocop
169
+ requirement: !ruby/object:Gem::Requirement
170
+ requirements:
171
+ - - "~>"
172
+ - !ruby/object:Gem::Version
173
+ version: 0.70.0
104
174
  type: :development
105
175
  prerelease: false
106
176
  version_requirements: !ruby/object:Gem::Requirement
107
177
  requirements:
108
178
  - - "~>"
109
179
  - !ruby/object:Gem::Version
110
- version: 0.14.0
111
- description: Provides a framework for performing preservation services over sets of
112
- files.
180
+ version: 0.70.0
181
+ - !ruby/object:Gem::Dependency
182
+ name: rubocop-rspec
183
+ requirement: !ruby/object:Gem::Requirement
184
+ requirements:
185
+ - - "~>"
186
+ - !ruby/object:Gem::Version
187
+ version: '1.33'
188
+ type: :development
189
+ prerelease: false
190
+ version_requirements: !ruby/object:Gem::Requirement
191
+ requirements:
192
+ - - "~>"
193
+ - !ruby/object:Gem::Version
194
+ version: '1.33'
195
+ - !ruby/object:Gem::Dependency
196
+ name: rubocop-performance
197
+ requirement: !ruby/object:Gem::Requirement
198
+ requirements:
199
+ - - "~>"
200
+ - !ruby/object:Gem::Version
201
+ version: '1.3'
202
+ type: :development
203
+ prerelease: false
204
+ version_requirements: !ruby/object:Gem::Requirement
205
+ requirements:
206
+ - - "~>"
207
+ - !ruby/object:Gem::Version
208
+ version: '1.3'
209
+ - !ruby/object:Gem::Dependency
210
+ name: rubocop-sequel
211
+ requirement: !ruby/object:Gem::Requirement
212
+ requirements:
213
+ - - "~>"
214
+ - !ruby/object:Gem::Version
215
+ version: 0.0.5
216
+ type: :development
217
+ prerelease: false
218
+ version_requirements: !ruby/object:Gem::Requirement
219
+ requirements:
220
+ - - "~>"
221
+ - !ruby/object:Gem::Version
222
+ version: 0.0.5
223
+ - !ruby/object:Gem::Dependency
224
+ name: amalgalite
225
+ requirement: !ruby/object:Gem::Requirement
226
+ requirements:
227
+ - - "~>"
228
+ - !ruby/object:Gem::Version
229
+ version: '1.6'
230
+ type: :development
231
+ prerelease: false
232
+ version_requirements: !ruby/object:Gem::Requirement
233
+ requirements:
234
+ - - "~>"
235
+ - !ruby/object:Gem::Version
236
+ version: '1.6'
237
+ - !ruby/object:Gem::Dependency
238
+ name: simplecov
239
+ requirement: !ruby/object:Gem::Requirement
240
+ requirements:
241
+ - - "~>"
242
+ - !ruby/object:Gem::Version
243
+ version: '0.16'
244
+ type: :development
245
+ prerelease: false
246
+ version_requirements: !ruby/object:Gem::Requirement
247
+ requirements:
248
+ - - "~>"
249
+ - !ruby/object:Gem::Version
250
+ version: '0.16'
251
+ description: Longleaf is a command-line tool which allows users to configure a set
252
+ of storage locations and define custom sets of preservation services to run on their
253
+ contents. These services are executed in response to applicable preservation events
254
+ issued by clients. Its primary goal is to provide tools to create a simple and customizable
255
+ preservation environment.
113
256
  email:
114
257
  - bbpennel@email.unc.edu
115
258
  executables:
@@ -117,39 +260,174 @@ executables:
117
260
  extensions: []
118
261
  extra_rdoc_files: []
119
262
  files:
263
+ - ".circleci/config.yml"
264
+ - ".editorconfig"
120
265
  - ".gitignore"
121
266
  - ".rspec"
122
- - ".travis.yml"
267
+ - ".rubocop.yml"
268
+ - ".rubocop_todo.yml"
269
+ - ".yardopts"
123
270
  - Gemfile
124
271
  - LICENSE.txt
125
272
  - README.md
126
273
  - Rakefile
127
274
  - bin/console
128
275
  - bin/setup
276
+ - docs/aboutlongleaf.md
277
+ - docs/extra.css
278
+ - docs/img/change-file.png
279
+ - docs/img/ll-example-preserved.png
280
+ - docs/index.md
281
+ - docs/install.md
282
+ - docs/ll-example/config-example-relative.yml
283
+ - docs/ll-example/files-dir/LLexample-PDF.pdf
284
+ - docs/ll-example/files-dir/LLexample-TOCHANGE.txt
285
+ - docs/ll-example/files-dir/LLexample-tokeep.txt
286
+ - docs/ll-example/metadata-dir/.gitkeep
287
+ - docs/ll-example/replica-files/.gitkeep
288
+ - docs/ll-example/replica-metadata/.gitkeep
289
+ - docs/quickstart.md
290
+ - docs/rdocs/Longleaf.html
291
+ - docs/rdocs/Longleaf/AppFields.html
292
+ - docs/rdocs/Longleaf/ApplicationConfigDeserializer.html
293
+ - docs/rdocs/Longleaf/ApplicationConfigManager.html
294
+ - docs/rdocs/Longleaf/ApplicationConfigValidator.html
295
+ - docs/rdocs/Longleaf/CLI.html
296
+ - docs/rdocs/Longleaf/ChecksumMismatchError.html
297
+ - docs/rdocs/Longleaf/ConfigBuilder.html
298
+ - docs/rdocs/Longleaf/ConfigurationError.html
299
+ - docs/rdocs/Longleaf/ConfigurationValidator.html
300
+ - docs/rdocs/Longleaf/DeregisterCommand.html
301
+ - docs/rdocs/Longleaf/DeregisterEvent.html
302
+ - docs/rdocs/Longleaf/DeregistrationError.html
303
+ - docs/rdocs/Longleaf/DigestHelper.html
304
+ - docs/rdocs/Longleaf/EventError.html
305
+ - docs/rdocs/Longleaf/EventNames.html
306
+ - docs/rdocs/Longleaf/EventStatusTracking.html
307
+ - docs/rdocs/Longleaf/FileCheckService.html
308
+ - docs/rdocs/Longleaf/FileHelpers.html
309
+ - docs/rdocs/Longleaf/FileRecord.html
310
+ - docs/rdocs/Longleaf/FileSelector.html
311
+ - docs/rdocs/Longleaf/FixityCheckService.html
312
+ - docs/rdocs/Longleaf/IndexManager.html
313
+ - docs/rdocs/Longleaf/InvalidDigestAlgorithmError.html
314
+ - docs/rdocs/Longleaf/InvalidStoragePathError.html
315
+ - docs/rdocs/Longleaf/Logging.html
316
+ - docs/rdocs/Longleaf/Logging/RedirectingLogger.html
317
+ - docs/rdocs/Longleaf/LongleafError.html
318
+ - docs/rdocs/Longleaf/MDFields.html
319
+ - docs/rdocs/Longleaf/MetadataBuilder.html
320
+ - docs/rdocs/Longleaf/MetadataDeserializer.html
321
+ - docs/rdocs/Longleaf/MetadataError.html
322
+ - docs/rdocs/Longleaf/MetadataPersistenceManager.html
323
+ - docs/rdocs/Longleaf/MetadataRecord.html
324
+ - docs/rdocs/Longleaf/MetadataSerializer.html
325
+ - docs/rdocs/Longleaf/PreservationServiceError.html
326
+ - docs/rdocs/Longleaf/PreserveCommand.html
327
+ - docs/rdocs/Longleaf/PreserveEvent.html
328
+ - docs/rdocs/Longleaf/RegisterCommand.html
329
+ - docs/rdocs/Longleaf/RegisterEvent.html
330
+ - docs/rdocs/Longleaf/RegisteredFileSelector.html
331
+ - docs/rdocs/Longleaf/RegistrationError.html
332
+ - docs/rdocs/Longleaf/ReindexCommand.html
333
+ - docs/rdocs/Longleaf/RsyncReplicationService.html
334
+ - docs/rdocs/Longleaf/SequelIndexDriver.html
335
+ - docs/rdocs/Longleaf/ServiceCandidateFilesystemIterator.html
336
+ - docs/rdocs/Longleaf/ServiceCandidateIndexIterator.html
337
+ - docs/rdocs/Longleaf/ServiceCandidateLocator.html
338
+ - docs/rdocs/Longleaf/ServiceClassCache.html
339
+ - docs/rdocs/Longleaf/ServiceDateHelper.html
340
+ - docs/rdocs/Longleaf/ServiceDefinition.html
341
+ - docs/rdocs/Longleaf/ServiceDefinitionManager.html
342
+ - docs/rdocs/Longleaf/ServiceDefinitionValidator.html
343
+ - docs/rdocs/Longleaf/ServiceFields.html
344
+ - docs/rdocs/Longleaf/ServiceManager.html
345
+ - docs/rdocs/Longleaf/ServiceMappingManager.html
346
+ - docs/rdocs/Longleaf/ServiceMappingValidator.html
347
+ - docs/rdocs/Longleaf/ServiceRecord.html
348
+ - docs/rdocs/Longleaf/StorageLocation.html
349
+ - docs/rdocs/Longleaf/StorageLocationManager.html
350
+ - docs/rdocs/Longleaf/StorageLocationUnavailableError.html
351
+ - docs/rdocs/Longleaf/StorageLocationValidator.html
352
+ - docs/rdocs/Longleaf/StoragePathValidator.html
353
+ - docs/rdocs/Longleaf/SystemConfigBuilder.html
354
+ - docs/rdocs/Longleaf/SystemConfigFields.html
355
+ - docs/rdocs/Longleaf/ValidateConfigCommand.html
356
+ - docs/rdocs/Longleaf/ValidateMetadataCommand.html
357
+ - docs/rdocs/_index.html
358
+ - docs/rdocs/class_list.html
359
+ - docs/rdocs/css/common.css
360
+ - docs/rdocs/css/full_list.css
361
+ - docs/rdocs/css/style.css
362
+ - docs/rdocs/file.README.html
363
+ - docs/rdocs/file_list.html
364
+ - docs/rdocs/frames.html
365
+ - docs/rdocs/index.html
366
+ - docs/rdocs/js/app.js
367
+ - docs/rdocs/js/full_list.js
368
+ - docs/rdocs/js/jquery.js
369
+ - docs/rdocs/method_list.html
370
+ - docs/rdocs/top-level-namespace.html
129
371
  - exe/longleaf
130
372
  - lib/longleaf.rb
373
+ - lib/longleaf/candidates/file_selector.rb
374
+ - lib/longleaf/candidates/manifest_digest_provider.rb
375
+ - lib/longleaf/candidates/registered_file_selector.rb
376
+ - lib/longleaf/candidates/service_candidate_filesystem_iterator.rb
377
+ - lib/longleaf/candidates/service_candidate_index_iterator.rb
378
+ - lib/longleaf/candidates/service_candidate_locator.rb
379
+ - lib/longleaf/candidates/single_digest_provider.rb
131
380
  - lib/longleaf/cli.rb
132
- - lib/longleaf/commands/abstract_command.rb
381
+ - lib/longleaf/commands/deregister_command.rb
382
+ - lib/longleaf/commands/preserve_command.rb
133
383
  - lib/longleaf/commands/register_command.rb
384
+ - lib/longleaf/commands/reindex_command.rb
134
385
  - lib/longleaf/commands/validate_config_command.rb
386
+ - lib/longleaf/commands/validate_metadata_command.rb
135
387
  - lib/longleaf/errors.rb
388
+ - lib/longleaf/events/deregister_event.rb
389
+ - lib/longleaf/events/event_names.rb
390
+ - lib/longleaf/events/event_status_tracking.rb
391
+ - lib/longleaf/events/preserve_event.rb
136
392
  - lib/longleaf/events/register_event.rb
393
+ - lib/longleaf/helpers/case_insensitive_hash.rb
394
+ - lib/longleaf/helpers/digest_helper.rb
395
+ - lib/longleaf/helpers/s3_uri_helper.rb
396
+ - lib/longleaf/helpers/selection_options_parser.rb
397
+ - lib/longleaf/helpers/service_date_helper.rb
398
+ - lib/longleaf/indexing/index_manager.rb
399
+ - lib/longleaf/indexing/sequel_index_driver.rb
137
400
  - lib/longleaf/logging.rb
138
401
  - lib/longleaf/logging/redirecting_logger.rb
139
402
  - lib/longleaf/models/app_fields.rb
140
403
  - lib/longleaf/models/file_record.rb
404
+ - lib/longleaf/models/filesystem_metadata_location.rb
405
+ - lib/longleaf/models/filesystem_storage_location.rb
141
406
  - lib/longleaf/models/md_fields.rb
407
+ - lib/longleaf/models/metadata_location.rb
142
408
  - lib/longleaf/models/metadata_record.rb
409
+ - lib/longleaf/models/s3_storage_location.rb
143
410
  - lib/longleaf/models/service_definition.rb
144
411
  - lib/longleaf/models/service_fields.rb
145
412
  - lib/longleaf/models/service_record.rb
146
413
  - lib/longleaf/models/storage_location.rb
414
+ - lib/longleaf/models/storage_types.rb
415
+ - lib/longleaf/models/system_config_fields.rb
416
+ - lib/longleaf/preservation_services/file_check_service.rb
417
+ - lib/longleaf/preservation_services/fixity_check_service.rb
418
+ - lib/longleaf/preservation_services/rsync_replication_service.rb
419
+ - lib/longleaf/preservation_services/s3_replication_service.rb
147
420
  - lib/longleaf/services/application_config_deserializer.rb
148
421
  - lib/longleaf/services/application_config_manager.rb
149
422
  - lib/longleaf/services/application_config_validator.rb
150
423
  - lib/longleaf/services/configuration_validator.rb
424
+ - lib/longleaf/services/filesystem_location_validator.rb
151
425
  - lib/longleaf/services/metadata_deserializer.rb
426
+ - lib/longleaf/services/metadata_persistence_manager.rb
152
427
  - lib/longleaf/services/metadata_serializer.rb
428
+ - lib/longleaf/services/metadata_validator.rb
429
+ - lib/longleaf/services/s3_location_validator.rb
430
+ - lib/longleaf/services/service_class_cache.rb
153
431
  - lib/longleaf/services/service_definition_manager.rb
154
432
  - lib/longleaf/services/service_definition_validator.rb
155
433
  - lib/longleaf/services/service_manager.rb
@@ -157,14 +435,20 @@ files:
157
435
  - lib/longleaf/services/service_mapping_validator.rb
158
436
  - lib/longleaf/services/storage_location_manager.rb
159
437
  - lib/longleaf/services/storage_location_validator.rb
160
- - lib/longleaf/services/storage_path_validator.rb
161
438
  - lib/longleaf/specs/config_builder.rb
439
+ - lib/longleaf/specs/config_validator_helpers.rb
440
+ - lib/longleaf/specs/custom_matchers.rb
441
+ - lib/longleaf/specs/file_helpers.rb
442
+ - lib/longleaf/specs/metadata_builder.rb
443
+ - lib/longleaf/specs/system_config_builder.rb
162
444
  - lib/longleaf/version.rb
163
445
  - longleaf.gemspec
164
- homepage: https://github.com/UNC-Libraries/
446
+ - mkdocs.yml
447
+ homepage: https://unc-libraries.github.io/longleaf-preservation
165
448
  licenses:
166
449
  - Apache-2.0
167
- metadata: {}
450
+ metadata:
451
+ source_code_uri: https://github.com/UNC-Libraries/longleaf-preservation
168
452
  post_install_message:
169
453
  rdoc_options: []
170
454
  require_paths:
@@ -173,15 +457,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
173
457
  requirements:
174
458
  - - ">="
175
459
  - !ruby/object:Gem::Version
176
- version: '0'
460
+ version: '2.3'
177
461
  required_rubygems_version: !ruby/object:Gem::Requirement
178
462
  requirements:
179
- - - ">"
463
+ - - ">="
180
464
  - !ruby/object:Gem::Version
181
- version: 1.3.1
465
+ version: '0'
182
466
  requirements: []
183
- rubyforge_project:
184
- rubygems_version: 2.7.6
467
+ rubygems_version: 3.1.2
185
468
  signing_key:
186
469
  specification_version: 4
187
470
  summary: Longleaf preservation services tool