longleaf 0.1.0.pre.3 → 1.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (185) hide show
  1. checksums.yaml +4 -4
  2. data/.circleci/config.yml +94 -0
  3. data/.editorconfig +13 -0
  4. data/.gitignore +4 -1
  5. data/.rubocop.yml +44 -0
  6. data/.rubocop_todo.yml +834 -0
  7. data/.yardopts +1 -0
  8. data/Gemfile +16 -1
  9. data/README.md +98 -12
  10. data/Rakefile +6 -0
  11. data/bin/setup +16 -1
  12. data/docs/aboutlongleaf.md +28 -0
  13. data/docs/extra.css +32 -0
  14. data/docs/img/change-file.png +0 -0
  15. data/docs/img/ll-example-preserved.png +0 -0
  16. data/docs/index.md +19 -0
  17. data/docs/install.md +66 -0
  18. data/docs/ll-example/config-example-relative.yml +33 -0
  19. data/docs/ll-example/files-dir/LLexample-PDF.pdf +0 -0
  20. data/docs/ll-example/files-dir/LLexample-TOCHANGE.txt +15 -0
  21. data/docs/ll-example/files-dir/LLexample-tokeep.txt +10 -0
  22. data/docs/ll-example/metadata-dir/.gitkeep +0 -0
  23. data/docs/ll-example/replica-files/.gitkeep +0 -0
  24. data/docs/ll-example/replica-metadata/.gitkeep +0 -0
  25. data/docs/quickstart.md +270 -0
  26. data/docs/rdocs/Longleaf.html +135 -0
  27. data/docs/rdocs/Longleaf/AppFields.html +178 -0
  28. data/docs/rdocs/Longleaf/ApplicationConfigDeserializer.html +631 -0
  29. data/docs/rdocs/Longleaf/ApplicationConfigManager.html +610 -0
  30. data/docs/rdocs/Longleaf/ApplicationConfigValidator.html +238 -0
  31. data/docs/rdocs/Longleaf/CLI.html +909 -0
  32. data/docs/rdocs/Longleaf/ChecksumMismatchError.html +151 -0
  33. data/docs/rdocs/Longleaf/ConfigBuilder.html +1339 -0
  34. data/docs/rdocs/Longleaf/ConfigurationError.html +143 -0
  35. data/docs/rdocs/Longleaf/ConfigurationValidator.html +227 -0
  36. data/docs/rdocs/Longleaf/DeregisterCommand.html +420 -0
  37. data/docs/rdocs/Longleaf/DeregisterEvent.html +453 -0
  38. data/docs/rdocs/Longleaf/DeregistrationError.html +151 -0
  39. data/docs/rdocs/Longleaf/DigestHelper.html +419 -0
  40. data/docs/rdocs/Longleaf/EventError.html +147 -0
  41. data/docs/rdocs/Longleaf/EventNames.html +163 -0
  42. data/docs/rdocs/Longleaf/EventStatusTracking.html +656 -0
  43. data/docs/rdocs/Longleaf/FileCheckService.html +540 -0
  44. data/docs/rdocs/Longleaf/FileHelpers.html +520 -0
  45. data/docs/rdocs/Longleaf/FileRecord.html +716 -0
  46. data/docs/rdocs/Longleaf/FileSelector.html +901 -0
  47. data/docs/rdocs/Longleaf/FixityCheckService.html +691 -0
  48. data/docs/rdocs/Longleaf/IndexManager.html +1155 -0
  49. data/docs/rdocs/Longleaf/InvalidDigestAlgorithmError.html +143 -0
  50. data/docs/rdocs/Longleaf/InvalidStoragePathError.html +143 -0
  51. data/docs/rdocs/Longleaf/Logging.html +405 -0
  52. data/docs/rdocs/Longleaf/Logging/RedirectingLogger.html +1213 -0
  53. data/docs/rdocs/Longleaf/LongleafError.html +139 -0
  54. data/docs/rdocs/Longleaf/MDFields.html +193 -0
  55. data/docs/rdocs/Longleaf/MetadataBuilder.html +787 -0
  56. data/docs/rdocs/Longleaf/MetadataDeserializer.html +537 -0
  57. data/docs/rdocs/Longleaf/MetadataError.html +143 -0
  58. data/docs/rdocs/Longleaf/MetadataPersistenceManager.html +539 -0
  59. data/docs/rdocs/Longleaf/MetadataRecord.html +1411 -0
  60. data/docs/rdocs/Longleaf/MetadataSerializer.html +786 -0
  61. data/docs/rdocs/Longleaf/PreservationServiceError.html +147 -0
  62. data/docs/rdocs/Longleaf/PreserveCommand.html +410 -0
  63. data/docs/rdocs/Longleaf/PreserveEvent.html +491 -0
  64. data/docs/rdocs/Longleaf/RegisterCommand.html +428 -0
  65. data/docs/rdocs/Longleaf/RegisterEvent.html +628 -0
  66. data/docs/rdocs/Longleaf/RegisteredFileSelector.html +446 -0
  67. data/docs/rdocs/Longleaf/RegistrationError.html +151 -0
  68. data/docs/rdocs/Longleaf/ReindexCommand.html +576 -0
  69. data/docs/rdocs/Longleaf/RsyncReplicationService.html +1180 -0
  70. data/docs/rdocs/Longleaf/SequelIndexDriver.html +1978 -0
  71. data/docs/rdocs/Longleaf/ServiceCandidateFilesystemIterator.html +572 -0
  72. data/docs/rdocs/Longleaf/ServiceCandidateIndexIterator.html +532 -0
  73. data/docs/rdocs/Longleaf/ServiceCandidateLocator.html +333 -0
  74. data/docs/rdocs/Longleaf/ServiceClassCache.html +725 -0
  75. data/docs/rdocs/Longleaf/ServiceDateHelper.html +425 -0
  76. data/docs/rdocs/Longleaf/ServiceDefinition.html +683 -0
  77. data/docs/rdocs/Longleaf/ServiceDefinitionManager.html +371 -0
  78. data/docs/rdocs/Longleaf/ServiceDefinitionValidator.html +269 -0
  79. data/docs/rdocs/Longleaf/ServiceFields.html +173 -0
  80. data/docs/rdocs/Longleaf/ServiceManager.html +1229 -0
  81. data/docs/rdocs/Longleaf/ServiceMappingManager.html +410 -0
  82. data/docs/rdocs/Longleaf/ServiceMappingValidator.html +347 -0
  83. data/docs/rdocs/Longleaf/ServiceRecord.html +821 -0
  84. data/docs/rdocs/Longleaf/StorageLocation.html +985 -0
  85. data/docs/rdocs/Longleaf/StorageLocationManager.html +729 -0
  86. data/docs/rdocs/Longleaf/StorageLocationUnavailableError.html +143 -0
  87. data/docs/rdocs/Longleaf/StorageLocationValidator.html +373 -0
  88. data/docs/rdocs/Longleaf/StoragePathValidator.html +253 -0
  89. data/docs/rdocs/Longleaf/SystemConfigBuilder.html +441 -0
  90. data/docs/rdocs/Longleaf/SystemConfigFields.html +163 -0
  91. data/docs/rdocs/Longleaf/ValidateConfigCommand.html +451 -0
  92. data/docs/rdocs/Longleaf/ValidateMetadataCommand.html +408 -0
  93. data/docs/rdocs/_index.html +660 -0
  94. data/docs/rdocs/class_list.html +51 -0
  95. data/docs/rdocs/css/common.css +1 -0
  96. data/docs/rdocs/css/full_list.css +58 -0
  97. data/docs/rdocs/css/style.css +496 -0
  98. data/docs/rdocs/file.README.html +165 -0
  99. data/docs/rdocs/file_list.html +56 -0
  100. data/docs/rdocs/frames.html +17 -0
  101. data/docs/rdocs/index.html +165 -0
  102. data/docs/rdocs/js/app.js +303 -0
  103. data/docs/rdocs/js/full_list.js +216 -0
  104. data/docs/rdocs/js/jquery.js +4 -0
  105. data/docs/rdocs/method_list.html +2051 -0
  106. data/docs/rdocs/top-level-namespace.html +110 -0
  107. data/lib/longleaf/candidates/file_selector.rb +150 -0
  108. data/lib/longleaf/candidates/manifest_digest_provider.rb +17 -0
  109. data/lib/longleaf/candidates/physical_path_provider.rb +17 -0
  110. data/lib/longleaf/candidates/registered_file_selector.rb +67 -0
  111. data/lib/longleaf/candidates/service_candidate_filesystem_iterator.rb +93 -0
  112. data/lib/longleaf/candidates/service_candidate_index_iterator.rb +84 -0
  113. data/lib/longleaf/candidates/service_candidate_locator.rb +23 -0
  114. data/lib/longleaf/candidates/single_digest_provider.rb +13 -0
  115. data/lib/longleaf/cli.rb +249 -44
  116. data/lib/longleaf/commands/deregister_command.rb +51 -0
  117. data/lib/longleaf/commands/preserve_command.rb +50 -0
  118. data/lib/longleaf/commands/register_command.rb +34 -43
  119. data/lib/longleaf/commands/reindex_command.rb +92 -0
  120. data/lib/longleaf/commands/validate_config_command.rb +33 -8
  121. data/lib/longleaf/commands/validate_metadata_command.rb +51 -0
  122. data/lib/longleaf/errors.rb +26 -7
  123. data/lib/longleaf/events/deregister_event.rb +53 -0
  124. data/lib/longleaf/events/event_names.rb +9 -0
  125. data/lib/longleaf/events/event_status_tracking.rb +59 -0
  126. data/lib/longleaf/events/preserve_event.rb +82 -0
  127. data/lib/longleaf/events/register_event.rb +59 -51
  128. data/lib/longleaf/helpers/case_insensitive_hash.rb +38 -0
  129. data/lib/longleaf/helpers/digest_helper.rb +56 -0
  130. data/lib/longleaf/helpers/s3_uri_helper.rb +86 -0
  131. data/lib/longleaf/helpers/selection_options_parser.rb +215 -0
  132. data/lib/longleaf/helpers/service_date_helper.rb +78 -0
  133. data/lib/longleaf/indexing/index_manager.rb +101 -0
  134. data/lib/longleaf/indexing/sequel_index_driver.rb +306 -0
  135. data/lib/longleaf/logging.rb +5 -4
  136. data/lib/longleaf/logging/redirecting_logger.rb +30 -25
  137. data/lib/longleaf/models/app_fields.rb +7 -2
  138. data/lib/longleaf/models/file_record.rb +31 -8
  139. data/lib/longleaf/models/filesystem_metadata_location.rb +56 -0
  140. data/lib/longleaf/models/filesystem_storage_location.rb +52 -0
  141. data/lib/longleaf/models/md_fields.rb +3 -1
  142. data/lib/longleaf/models/metadata_location.rb +47 -0
  143. data/lib/longleaf/models/metadata_record.rb +43 -16
  144. data/lib/longleaf/models/s3_storage_location.rb +138 -0
  145. data/lib/longleaf/models/service_definition.rb +7 -6
  146. data/lib/longleaf/models/service_fields.rb +7 -1
  147. data/lib/longleaf/models/service_record.rb +10 -6
  148. data/lib/longleaf/models/storage_location.rb +24 -21
  149. data/lib/longleaf/models/storage_types.rb +9 -0
  150. data/lib/longleaf/models/system_config_fields.rb +9 -0
  151. data/lib/longleaf/preservation_services/file_check_service.rb +59 -0
  152. data/lib/longleaf/preservation_services/fixity_check_service.rb +124 -0
  153. data/lib/longleaf/preservation_services/rsync_replication_service.rb +198 -0
  154. data/lib/longleaf/preservation_services/s3_replication_service.rb +131 -0
  155. data/lib/longleaf/services/application_config_deserializer.rb +80 -21
  156. data/lib/longleaf/services/application_config_manager.rb +20 -6
  157. data/lib/longleaf/services/application_config_validator.rb +19 -9
  158. data/lib/longleaf/services/configuration_validator.rb +67 -4
  159. data/lib/longleaf/services/filesystem_location_validator.rb +16 -0
  160. data/lib/longleaf/services/metadata_deserializer.rb +115 -42
  161. data/lib/longleaf/services/metadata_persistence_manager.rb +47 -0
  162. data/lib/longleaf/services/metadata_serializer.rb +139 -25
  163. data/lib/longleaf/services/metadata_validator.rb +76 -0
  164. data/lib/longleaf/services/s3_location_validator.rb +19 -0
  165. data/lib/longleaf/services/service_class_cache.rb +112 -0
  166. data/lib/longleaf/services/service_definition_manager.rb +10 -7
  167. data/lib/longleaf/services/service_definition_validator.rb +25 -18
  168. data/lib/longleaf/services/service_manager.rb +86 -11
  169. data/lib/longleaf/services/service_mapping_manager.rb +13 -12
  170. data/lib/longleaf/services/service_mapping_validator.rb +36 -26
  171. data/lib/longleaf/services/storage_location_manager.rb +76 -15
  172. data/lib/longleaf/services/storage_location_validator.rb +49 -35
  173. data/lib/longleaf/specs/config_builder.rb +47 -23
  174. data/lib/longleaf/specs/config_validator_helpers.rb +16 -0
  175. data/lib/longleaf/specs/custom_matchers.rb +9 -0
  176. data/lib/longleaf/specs/file_helpers.rb +61 -0
  177. data/lib/longleaf/specs/metadata_builder.rb +98 -0
  178. data/lib/longleaf/specs/system_config_builder.rb +27 -0
  179. data/lib/longleaf/version.rb +1 -1
  180. data/longleaf.gemspec +20 -7
  181. data/mkdocs.yml +21 -0
  182. metadata +310 -26
  183. data/.travis.yml +0 -4
  184. data/lib/longleaf/commands/abstract_command.rb +0 -37
  185. data/lib/longleaf/services/storage_path_validator.rb +0 -16
@@ -0,0 +1,84 @@
1
+ require 'longleaf/events/event_names'
2
+ require 'longleaf/errors'
3
+ require 'longleaf/logging'
4
+ require 'time'
5
+
6
+ module Longleaf
7
+ # Iterator for getting file candidates which have services which need to be run.
8
+ # Implementation uses an index of file metadata to determine if the file needs any
9
+ # services run.
10
+ class ServiceCandidateIndexIterator
11
+ include Longleaf::Logging
12
+
13
+ def initialize(file_selector, event, app_config, force = false)
14
+ @file_selector = file_selector
15
+ @event = event
16
+ @app_config = app_config
17
+ @force = force
18
+ @index_manager = @app_config.index_manager
19
+ @stale_datetime = Time.now.utc
20
+ @result_set = nil
21
+ end
22
+
23
+ # Get the file record for the next candidate which needs services run which match the
24
+ # provided file_selector
25
+ # @return [FileRecord] file record of the next candidate with services needing to be run,
26
+ # or nil if there are no more candidates.
27
+ def next_candidate
28
+ file_rec = nil
29
+ # loop until a candidate with metadata is retrieved
30
+ loop do
31
+ # Get the next page of results if the previous page has been processed
32
+ fetch_next_page if @result_set.nil? || @result_set.empty?
33
+
34
+ # Retrieve the next possible candidate path from the page
35
+ next_path = @result_set.shift
36
+ # given that the page was just retrieved, getting a nil path indicates that the retrieved page of
37
+ # candidates is empty and there are no more candidates to iterate at this time.
38
+ return nil if next_path.nil?
39
+
40
+ logger.debug("Retrieved candidate #{next_path}")
41
+ storage_loc = @app_config.location_manager.get_location_by_path(next_path)
42
+ file_rec = FileRecord.new(next_path, storage_loc)
43
+
44
+ # Keep seeking until a registered candidate is found, according to the file system.
45
+ if file_rec.metadata_present?
46
+ break
47
+ else
48
+ logger.warn("Encountered #{next_path} in index, but path is not registered. Clearing out of synch entry.")
49
+ @index_manager.remove(file_rec)
50
+ end
51
+ end
52
+
53
+ @app_config.md_manager.load(file_rec)
54
+
55
+ file_rec
56
+ end
57
+
58
+ # Iterate through the candidates in this object and execute the provided block with each
59
+ # candidate. A block is required.
60
+ def each
61
+ file_rec = next_candidate
62
+ until file_rec.nil?
63
+ yield file_rec
64
+
65
+ file_rec = next_candidate
66
+ end
67
+ end
68
+
69
+ private
70
+ def fetch_next_page
71
+ if @force
72
+ @result_set = @index_manager.registered_paths(@file_selector)
73
+ else
74
+ case @event
75
+ when EventNames::PRESERVE
76
+ @result_set = @index_manager.paths_with_stale_services(@file_selector, @stale_datetime)
77
+ when EventNames::CLEANUP
78
+ # TODO
79
+ end
80
+ end
81
+ logger.debug("Retrieve result set with #{@result_set&.length} entries")
82
+ end
83
+ end
84
+ end
@@ -0,0 +1,23 @@
1
+ require 'longleaf/candidates/service_candidate_filesystem_iterator'
2
+ require 'longleaf/candidates/service_candidate_index_iterator'
3
+
4
+ module Longleaf
5
+ # Service which locates files that have services which need to be performed on them.
6
+ class ServiceCandidateLocator
7
+ def initialize(app_config)
8
+ @app_config = app_config
9
+ end
10
+
11
+ # Get a iterator of the candidates matching the given FileSelector which need services run.
12
+ # @param file_selector [FileSelector] selector identifying the files to pull candidates from.
13
+ # @return an iterator
14
+ def candidate_iterator(file_selector, event, force = false)
15
+ if @app_config.index_manager.using_index?
16
+ ServiceCandidateIndexIterator.new(file_selector, event, @app_config, force)
17
+ else
18
+ # Get filesystem based implementation
19
+ ServiceCandidateFilesystemIterator.new(file_selector, event, @app_config, force)
20
+ end
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,13 @@
1
+ module Longleaf
2
+ # Provides a single set of digests for files
3
+ class SingleDigestProvider
4
+ def initialize(digests)
5
+ @digests = digests
6
+ end
7
+
8
+ def get_digests(file_path)
9
+ return nil if @digests.nil?
10
+ @digests
11
+ end
12
+ end
13
+ end
data/lib/longleaf/cli.rb CHANGED
@@ -2,71 +2,256 @@ require 'thor'
2
2
  require 'yaml'
3
3
  require 'longleaf/logging'
4
4
  require 'longleaf/errors'
5
+ require 'longleaf/version'
6
+ require 'longleaf/commands/deregister_command'
5
7
  require 'longleaf/commands/validate_config_command'
8
+ require 'longleaf/commands/validate_metadata_command'
6
9
  require 'longleaf/commands/register_command'
10
+ require 'longleaf/commands/reindex_command'
11
+ require 'longleaf/commands/preserve_command'
12
+ require 'longleaf/helpers/selection_options_parser'
7
13
 
8
14
  module Longleaf
15
+ # Main commandline interface setup for Longleaf using Thor.
9
16
  class CLI < Thor
10
17
  include Longleaf::Logging
11
-
12
- class_option(:config, :aliases => "-c",
13
- :default => ENV['LONGLEAF_CFG'],
14
- :required => true,
15
- :desc => 'Absolute path to the application configuration used for this command. By default, the value of the environment variable LONGLEAF_CFG is used.')
16
- # Logging/output options
17
- class_option(:failure_only,
18
- :type => :boolean,
19
- :default => false,
20
- :desc => 'Only display failure messages to STDOUT.')
21
- class_option(:log_level,
22
- :default => 'WARN',
23
- :desc => 'Level of logging to send to STDERR, following standard ruby Logger levels. This includes: DEBUG, INFO, WARN, ERROR, FATAL, UNKNOWN. Default is WARN.')
24
- class_option(:log_format,
25
- :desc => 'Format to use for log information sent to STDERR. Can contain the following parameters, which must be wrapped in %{}: severity, datetime, progname, msg. Default is "%{severity} [%{datetime}]: %{msg}"')
26
- class_option(:log_datetime,
27
- :desc => 'Format to use for timestamps used in logging to STDERR, following strftime syntax.')
28
-
18
+
19
+ # Register a shared method option in a shared option group
20
+ def self.add_shared_option(name, group, options = {})
21
+ @shared_groups = {} if @shared_groups.nil?
22
+ @shared_groups[group] = {} if @shared_groups[group].nil?
23
+ @shared_groups[group][name] = options
24
+ end
25
+
26
+ # Add all of the shared options in the specified group as method options
27
+ def self.shared_options_group(group_name)
28
+ @shared_groups[group_name].each do |opt_name, opt|
29
+ option opt_name, opt
30
+ end
31
+ end
32
+
33
+ # Config options
34
+ add_shared_option(
35
+ :config, :common, {
36
+ :aliases => "-c",
37
+ :default => ENV['LONGLEAF_CFG'],
38
+ :required => false,
39
+ :desc => 'Path to the application configuration used for this command. By default, the value of the environment variable LONGLEAF_CFG is used. A config path is required for most commands.' })
40
+ add_shared_option(
41
+ :load_path, :common, {
42
+ :aliases => "-I",
43
+ :desc => 'Specify comma seperated directories to add to the $LOAD_PATH, which can be used to specify additional paths from which to load preservation services.' })
44
+
45
+ # Logging options
46
+ add_shared_option(
47
+ :failure_only, :common, {
48
+ :type => :boolean,
49
+ :default => false,
50
+ :desc => 'Only display failure messages to STDOUT.' })
51
+ add_shared_option(
52
+ :log_level, :common, {
53
+ :default => 'WARN',
54
+ :desc => 'Level of logging to send to STDERR, following standard ruby Logger levels. This includes: DEBUG, INFO, WARN, ERROR, FATAL, UNKNOWN.' })
55
+ add_shared_option(
56
+ :log_format, :common, {
57
+ :desc => 'Format to use for log information sent to STDERR. Can contain the following parameters, which must be wrapped in %{}: severity, datetime, progname, msg. Default is "%{severity} [%{datetime}]: %{msg}"' })
58
+ add_shared_option(
59
+ :log_datetime, :common, {
60
+ :desc => 'Format to use for timestamps used in logging to STDERR, following strftime syntax.' })
61
+
62
+ # File selection options
63
+ add_shared_option(
64
+ :file, :file_selection, {
65
+ :aliases => "-f",
66
+ :required => false,
67
+ :desc => 'File or files to perform this operation on. If multiple files are provided, they must be comma separated.' })
68
+
69
+ add_shared_option(
70
+ :location, :registered_selection, {
71
+ :aliases => "-s",
72
+ :required => false,
73
+ :desc => 'Name or comma separated names of storage locations to perform this operation over.' })
74
+ add_shared_option(
75
+ :from_list, :registered_selection, {
76
+ :aliases => "-l",
77
+ :required => false,
78
+ :desc => %q{Provide a list of files to perform this operation on. The list must be new line separated, one file per line.
79
+ To provide a list from a file:
80
+ '-l /path/to/file_list.txt'
81
+ To provide a list from STDIN:
82
+ '-l @-'}})
83
+
84
+ # Commands
85
+ map %w[--version] => :__print_version
86
+ desc "--version", "Prints the Longleaf version number."
87
+ def __print_version
88
+ puts "longleaf version #{Longleaf::VERSION}"
89
+ end
90
+
29
91
  desc "register", "Register files with Longleaf"
30
- method_option(:file, :aliases => "-f",
31
- :required => true,
32
- :desc => 'File or files to register. Paths must be absolute. If multiple files are provided, they must be comma separated.')
92
+ shared_options_group(:file_selection)
93
+ method_option(:manifest,
94
+ :aliases => "-m",
95
+ :type => :array,
96
+ :desc => %q{Checksum manifests of files to register. Supports the following formats:
97
+ To submit a md5 manifest from a file
98
+ '-m md5:/path/to/manifest.txt'
99
+
100
+ To provide a sha1 manifest from STDIN
101
+ '-m sha1:@-'
102
+ Where the content in STDIN adheres to the format:
103
+ <digest> <path>
104
+ <digest> <path>
105
+ ...
106
+
107
+ To submit multiple manifests from files
108
+ '-m md5:/path/to/manifest1.txt sha1:/path/to/manifest2.txt'
109
+
110
+ To provide multiple digests via STDIN
111
+ '-m @-'
112
+ Where the content in STDIN adheres to the following format:
113
+ sha1:
114
+ <digest> <path>
115
+ ...
116
+ md5:
117
+ <digest> <path>
118
+ ...
119
+
120
+ To provide separate logical and physical paths, add a physical path column:
121
+ '-m sha1:@-'
122
+ Where the content in STDIN adheres to the format:
123
+ <digest> <logical path> <physical path>
124
+ ...
125
+ })
126
+ method_option(:physical_path,
127
+ :aliases => "-p",
128
+ :required => false,
129
+ :desc => %q{Comma separated list of physical paths of files to register. Only needed
130
+ if the physical and logical paths of the files differ, otherwise they will be assumed to be the same.
131
+ Only applicable when used with the -f option, and only for individual files, not directories.
132
+ Must be provided in the same order as the logical paths.})
33
133
  method_option(:force,
34
- :type => :boolean,
134
+ :type => :boolean,
35
135
  :default => false,
36
136
  :desc => 'Force the registration of already registered files.')
37
137
  method_option(:checksums,
38
- :desc => %q{Checksums for the submitted file. Each checksum must be prefaced with an algorithm prefix. Multiple checksums must be comma separated. If multiple files were submitted, they will be provided with the same checksums. For example:
138
+ :desc => %q{Checksums for the submitted file. Only applicable with the -f option.
139
+ Each checksum must be prefaced with an algorithm prefix. Multiple checksums must be comma separated. If multiple files were submitted, they will be provided with the same checksums. For example:
39
140
  '--checksums "md5:d8e8fca2dc0f896fd7cb4cb0031ba249,sha1:4e1243bd22c66e76c2ba9eddc1f91394e57f9f83"'})
141
+ shared_options_group(:common)
40
142
  # Register event command
41
143
  def register
144
+ verify_config_provided(options)
42
145
  setup_logger(options)
43
-
44
- config_path = options[:config]
45
- file_paths = options[:file]&.split(/\s*,\s*/)
46
- if options[:checksums]
47
- checksums = options[:checksums]
48
- # validate checksum list format, must a comma delimited list of prefix:checksums
49
- if /^[^:,]+:[^:,]+(,[^:,]+:[^:,]+)*$/.match(checksums)
50
- # convert checksum list into hash with prefix as key
51
- checksums = Hash[*checksums.split(/\s*[:,]\s*/)]
52
- else
53
- logger.failure("Invalid checksums parameter format, see `longleaf help <command>` for more information")
54
- exit 1
55
- end
56
- end
57
-
58
- command = Longleaf::RegisterCommand.new(config_path)
59
- exit command.execute(file_paths: file_paths, force: options[:force], checksums: checksums)
146
+
147
+ app_config_manager = load_application_config(options)
148
+
149
+ file_selector, digest_provider, physical_provider = SelectionOptionsParser
150
+ .parse_registration_selection_options(options, app_config_manager)
151
+
152
+ command = RegisterCommand.new(app_config_manager)
153
+ exit command.execute(file_selector: file_selector, force: options[:force], digest_provider: digest_provider,
154
+ physical_provider: physical_provider)
60
155
  end
61
-
156
+
157
+ desc "deregister", "Deregister files with Longleaf"
158
+ shared_options_group(:file_selection)
159
+ shared_options_group(:registered_selection)
160
+ method_option(:force,
161
+ :type => :boolean,
162
+ :default => false,
163
+ :desc => 'Force the deregistration of already deregistered files.')
164
+ shared_options_group(:common)
165
+ # Deregister event command
166
+ def deregister
167
+ verify_config_provided(options)
168
+ setup_logger(options)
169
+
170
+ app_config_manager = load_application_config(options)
171
+ file_selector = SelectionOptionsParser.create_registered_selector(options, app_config_manager)
172
+
173
+ command = DeregisterCommand.new(app_config_manager)
174
+ exit command.execute(file_selector: file_selector, force: options[:force])
175
+ end
176
+
177
+ desc "preserve", "Perform preservation services on files with Longleaf"
178
+ shared_options_group(:file_selection)
179
+ shared_options_group(:registered_selection)
180
+ method_option(:force,
181
+ :type => :boolean,
182
+ :default => false,
183
+ :desc => 'Force the execution of preservation services, disregarding scheduling information.')
184
+ shared_options_group(:common)
185
+ def preserve
186
+ verify_config_provided(options)
187
+ setup_logger(options)
188
+
189
+ extend_load_path(options[:load_path])
190
+ app_config_manager = load_application_config(options)
191
+ file_selector = SelectionOptionsParser.create_registered_selector(options, app_config_manager)
192
+
193
+ command = PreserveCommand.new(app_config_manager)
194
+ exit command.execute(file_selector: file_selector, force: options[:force])
195
+ end
196
+
62
197
  desc "validate_config", "Validate an application configuration file, provided using --config."
198
+ shared_options_group(:common)
63
199
  # Application configuration validation command
64
200
  def validate_config
201
+ verify_config_provided(options)
65
202
  setup_logger(options)
66
-
203
+ extend_load_path(options[:load_path])
204
+
67
205
  exit Longleaf::ValidateConfigCommand.new(options[:config]).execute
68
206
  end
69
-
207
+
208
+ desc "validate_metadata", "Validate metadata files."
209
+ shared_options_group(:file_selection)
210
+ shared_options_group(:registered_selection)
211
+ shared_options_group(:common)
212
+ # File metadata validation command
213
+ def validate_metadata
214
+ verify_config_provided(options)
215
+ setup_logger(options)
216
+
217
+ app_config_manager = load_application_config(options)
218
+ file_selector = SelectionOptionsParser.create_registered_selector(options, app_config_manager)
219
+
220
+ exit Longleaf::ValidateMetadataCommand.new(app_config_manager).execute(file_selector: file_selector)
221
+ end
222
+
223
+ desc "setup_index", "Sets up the structure of the metadata index, if one is configured using the system configuration file provided using the --system_config option. Some index types may require additional steps to be taken by an administrator before hand, such as creating users and databases."
224
+ shared_options_group(:common)
225
+ def setup_index
226
+ verify_config_provided(options)
227
+ setup_logger(options)
228
+
229
+ app_config_manager = load_application_config(options)
230
+
231
+ if app_config_manager.index_manager.using_index?
232
+ app_config_manager.index_manager.setup_index
233
+ logger.success("Setup of index complete")
234
+ exit 0
235
+ else
236
+ logger.failure("No index configured, unable to perform setup.")
237
+ exit 1
238
+ end
239
+ end
240
+
241
+ desc "reindex", "Perform a full reindex of file metadata stored within the configured storage locations."
242
+ method_option(:if_stale,
243
+ :type => :boolean,
244
+ :default => false,
245
+ :desc => 'Only perform the reindex if the index is known to be stale, generally after an config file change.')
246
+ shared_options_group(:common)
247
+ def reindex
248
+ verify_config_provided(options)
249
+ setup_logger(options)
250
+ app_config_manager = load_application_config(options)
251
+
252
+ exit Longleaf::ReindexCommand.new(app_config_manager).execute(only_if_stale: options[:if_stale])
253
+ end
254
+
70
255
  no_commands do
71
256
  def setup_logger(options)
72
257
  initialize_logger(options[:failure_only],
@@ -74,6 +259,26 @@ module Longleaf
74
259
  options[:log_format],
75
260
  options[:log_datetime])
76
261
  end
262
+
263
+ def load_application_config(options)
264
+ begin
265
+ app_manager = ApplicationConfigDeserializer.deserialize(options[:config])
266
+ rescue ConfigurationError => err
267
+ logger.failure("Failed to load application configuration due to the following issue(s):\n#{err.message}")
268
+ exit 1
269
+ end
270
+ end
271
+
272
+ def verify_config_provided(options)
273
+ if options[:config].nil? || options[:config].empty?
274
+ raise "No value provided for required options '--config'"
275
+ end
276
+ end
277
+
278
+ def extend_load_path(load_paths)
279
+ load_paths = load_paths&.split(/\s*,\s*/)
280
+ load_paths&.each { |path| $LOAD_PATH.unshift(path) }
281
+ end
77
282
  end
78
283
  end
79
- end
284
+ end