longleaf 0.1.0 → 0.2.0.pre.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.editorconfig +13 -0
- data/.gitignore +1 -0
- data/.rubocop.yml +4 -0
- data/.rubocop_todo.yml +755 -0
- data/README.md +29 -7
- data/lib/longleaf/candidates/file_selector.rb +107 -0
- data/lib/longleaf/candidates/service_candidate_filesystem_iterator.rb +99 -0
- data/lib/longleaf/candidates/service_candidate_locator.rb +18 -0
- data/lib/longleaf/cli.rb +102 -6
- data/lib/longleaf/commands/deregister_command.rb +50 -0
- data/lib/longleaf/commands/preserve_command.rb +45 -0
- data/lib/longleaf/commands/register_command.rb +24 -38
- data/lib/longleaf/commands/validate_config_command.rb +6 -2
- data/lib/longleaf/commands/validate_metadata_command.rb +49 -0
- data/lib/longleaf/errors.rb +19 -0
- data/lib/longleaf/events/deregister_event.rb +55 -0
- data/lib/longleaf/events/event_names.rb +9 -0
- data/lib/longleaf/events/event_status_tracking.rb +59 -0
- data/lib/longleaf/events/preserve_event.rb +71 -0
- data/lib/longleaf/events/register_event.rb +37 -26
- data/lib/longleaf/helpers/digest_helper.rb +50 -0
- data/lib/longleaf/helpers/service_date_helper.rb +51 -0
- data/lib/longleaf/logging.rb +1 -0
- data/lib/longleaf/logging/redirecting_logger.rb +9 -8
- data/lib/longleaf/models/app_fields.rb +2 -0
- data/lib/longleaf/models/file_record.rb +8 -3
- data/lib/longleaf/models/md_fields.rb +1 -0
- data/lib/longleaf/models/metadata_record.rb +16 -4
- data/lib/longleaf/models/service_definition.rb +4 -3
- data/lib/longleaf/models/service_fields.rb +2 -0
- data/lib/longleaf/models/service_record.rb +4 -1
- data/lib/longleaf/models/storage_location.rb +18 -1
- data/lib/longleaf/preservation_services/fixity_check_service.rb +121 -0
- data/lib/longleaf/preservation_services/rsync_replication_service.rb +183 -0
- data/lib/longleaf/services/application_config_deserializer.rb +4 -6
- data/lib/longleaf/services/application_config_manager.rb +4 -2
- data/lib/longleaf/services/application_config_validator.rb +1 -1
- data/lib/longleaf/services/configuration_validator.rb +1 -0
- data/lib/longleaf/services/metadata_deserializer.rb +47 -10
- data/lib/longleaf/services/metadata_serializer.rb +42 -6
- data/lib/longleaf/services/service_class_cache.rb +112 -0
- data/lib/longleaf/services/service_definition_manager.rb +5 -1
- data/lib/longleaf/services/service_definition_validator.rb +4 -4
- data/lib/longleaf/services/service_manager.rb +72 -9
- data/lib/longleaf/services/service_mapping_manager.rb +4 -3
- data/lib/longleaf/services/service_mapping_validator.rb +4 -4
- data/lib/longleaf/services/storage_location_manager.rb +26 -5
- data/lib/longleaf/services/storage_location_validator.rb +1 -1
- data/lib/longleaf/services/storage_path_validator.rb +2 -2
- data/lib/longleaf/specs/config_builder.rb +9 -5
- data/lib/longleaf/specs/custom_matchers.rb +9 -0
- data/lib/longleaf/specs/file_helpers.rb +60 -0
- data/lib/longleaf/version.rb +1 -1
- data/longleaf.gemspec +1 -0
- metadata +39 -7
- data/lib/longleaf/commands/abstract_command.rb +0 -37
@@ -0,0 +1,183 @@
|
|
1
|
+
require 'longleaf/events/event_names'
|
2
|
+
require 'longleaf/logging'
|
3
|
+
require 'longleaf/errors'
|
4
|
+
require 'longleaf/models/file_record'
|
5
|
+
require 'longleaf/models/service_fields'
|
6
|
+
require 'longleaf/events/register_event'
|
7
|
+
require 'open3'
|
8
|
+
|
9
|
+
module Longleaf
|
10
|
+
# Preservation service which performs replication of a file to one or more destinations using rsync.
|
11
|
+
#
|
12
|
+
# The service definition must contain one or more destinations, specified with the "to" property.
|
13
|
+
# These destinations must be either a known storage location name, a remote path, or absolute path.
|
14
|
+
#
|
15
|
+
# Optional service configuration properties:
|
16
|
+
# * replica_collision_policy = specifies the desired outcome if the service attempts to replicate
|
17
|
+
# a file which already exists at a destination. Default: "replace".
|
18
|
+
# * rsync_command = the command to invoke in order to execute rsync. Default: "rsync"
|
19
|
+
# * rsync_options = additional parameters that will be passed along to rsync. Cannot include options
|
20
|
+
# which change the target of the command or prevent its execution, such as "files-from", "dry-run",
|
21
|
+
# "help", etc. Command will always include "-R". Default "-a".
|
22
|
+
class RsyncReplicationService
|
23
|
+
include Longleaf::Logging
|
24
|
+
|
25
|
+
COLLISION_PROPERTY = "replica_collision_policy"
|
26
|
+
DEFAULT_COLLISION_POLICY = "replace"
|
27
|
+
VALID_COLLISION_POLICIES = ["replace"]
|
28
|
+
|
29
|
+
RSYNC_COMMAND_PROPERTY = "rsync_command"
|
30
|
+
DEFAULT_COMMAND = "rsync"
|
31
|
+
|
32
|
+
RSYNC_OPTIONS_PROPERTY = "rsync_options"
|
33
|
+
DEFAULT_OPTIONS = "-a"
|
34
|
+
DISALLOWED_OPTIONS = ["files-from", "n", "dry-run", "exclude", "exclude-from", "cvs-exclude",
|
35
|
+
"h", "help", "f", "F", "filter"]
|
36
|
+
|
37
|
+
attr_reader :command, :options, :collision_policy
|
38
|
+
|
39
|
+
# Initialize a RsyncReplicationService from the given service definition
|
40
|
+
#
|
41
|
+
# @param service_def [ServiceDefinition] the configuration for this service
|
42
|
+
# @param location_manager [StorageLocationManager] manager for configured storage locations
|
43
|
+
def initialize(service_def, app_manager)
|
44
|
+
@service_def = service_def
|
45
|
+
@app_manager = app_manager
|
46
|
+
|
47
|
+
@command = @service_def.properties[RSYNC_COMMAND_PROPERTY] || DEFAULT_COMMAND
|
48
|
+
|
49
|
+
# Validate rsync parameters
|
50
|
+
@options = @service_def.properties[RSYNC_OPTIONS_PROPERTY] || DEFAULT_OPTIONS
|
51
|
+
if contains_disallowed_option?(@options)
|
52
|
+
raise ArgumentError.new("Service #{service_def.name} specifies a disallowed rsync paramter," \
|
53
|
+
+ " rsync_options may not include the following: #{DISALLOWED_OPTIONS.join(" ")}")
|
54
|
+
end
|
55
|
+
|
56
|
+
# Add -R (--relative) in to command options to ensure full path gets replicated
|
57
|
+
@options = @options + " -R"
|
58
|
+
|
59
|
+
# Set and validate the replica collision policy
|
60
|
+
@collision_policy = @service_def.properties[COLLISION_PROPERTY] || DEFAULT_COLLISION_POLICY
|
61
|
+
if !VALID_COLLISION_POLICIES.include?(@collision_policy)
|
62
|
+
raise ArgumentError.new("Service #{service_def.name} received invalid #{COLLISION_PROPERTY}" \
|
63
|
+
+ " value #{collision_policy}")
|
64
|
+
end
|
65
|
+
|
66
|
+
# Store and validate destinations
|
67
|
+
replicate_to = @service_def.properties[ServiceFields::REPLICATE_TO]
|
68
|
+
if replicate_to.nil? || replicate_to.empty?
|
69
|
+
raise ArgumentError.new("Service #{service_def.name} must provide one or more replication destinations.")
|
70
|
+
end
|
71
|
+
|
72
|
+
loc_manager = app_manager.location_manager
|
73
|
+
# Build list of destinations, translating to storage locations when relevant
|
74
|
+
@destinations = Array.new
|
75
|
+
replicate_to.each do |dest|
|
76
|
+
# Assume that if destination contains a : or / it is a path rather than storage location
|
77
|
+
if dest =~ /[:\/]/
|
78
|
+
@destinations << dest
|
79
|
+
else
|
80
|
+
if loc_manager.locations.key?(dest)
|
81
|
+
@destinations << loc_manager.locations[dest]
|
82
|
+
else
|
83
|
+
raise ArgumentError.new("Service #{service_def.name} specifies unknown storage location '#{dest}'" \
|
84
|
+
+ " as a replication destination")
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
# During a replication event, perform replication of the specified file to all configured destinations
|
91
|
+
# as necessary.
|
92
|
+
#
|
93
|
+
# @param file_rec [FileRecord] record representing the file to perform the service on.
|
94
|
+
# @param event [String] name of the event this service is being invoked by.
|
95
|
+
# @raise [PreservationServiceError] if the rsync replication fails
|
96
|
+
def perform(file_rec, event)
|
97
|
+
@destinations.each do |destination|
|
98
|
+
dest_is_storage_loc = destination.is_a?(Longleaf::StorageLocation)
|
99
|
+
|
100
|
+
if dest_is_storage_loc
|
101
|
+
dest_path = destination.path
|
102
|
+
else
|
103
|
+
dest_path = destination
|
104
|
+
end
|
105
|
+
|
106
|
+
# Determine the path to the file being replicated relative to its storage location
|
107
|
+
rel_path = file_rec.path.sub(/\A#{file_rec.storage_location.path}/, "")
|
108
|
+
# source path with . so that rsync will only create destination directories starting from that point
|
109
|
+
source_path = File.join(file_rec.storage_location.path, "./#{rel_path}")
|
110
|
+
|
111
|
+
# Check that the destination is available because attempting to write
|
112
|
+
verify_destination_available(destination, file_rec)
|
113
|
+
|
114
|
+
logger.debug("Invoking rsync with command: #{@command} \"#{source_path}\" \"#{dest_path}\" #{@options}")
|
115
|
+
stdout, stderr, status = Open3.capture3("#{@command} \"#{source_path}\" \"#{dest_path}\" #{@options}")
|
116
|
+
raise PreservationServiceError.new("Failed to replicate #{file_rec.path} to #{dest_path}: #{stderr}") \
|
117
|
+
unless status.success?
|
118
|
+
|
119
|
+
logger.info("Replicated #{file_rec.path} to destination #{dest_path}")
|
120
|
+
|
121
|
+
# For destinations which are storage locations, register the replica with longleaf
|
122
|
+
if dest_is_storage_loc
|
123
|
+
register_replica(destination, rel_path, file_rec)
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
# Determine if this service is applicable for the provided event, given the configured service definition
|
129
|
+
#
|
130
|
+
# @param event [String] name of the event
|
131
|
+
# @return [Boolean] returns true if this service is applicable for the provided event
|
132
|
+
def is_applicable?(event)
|
133
|
+
case event
|
134
|
+
when EventNames::PRESERVE
|
135
|
+
true
|
136
|
+
else
|
137
|
+
false
|
138
|
+
end
|
139
|
+
end
|
140
|
+
|
141
|
+
private
|
142
|
+
def contains_disallowed_option?(options)
|
143
|
+
DISALLOWED_OPTIONS.each do |disallowed|
|
144
|
+
if disallowed.length == 1
|
145
|
+
if options =~ /(\A| )-[a-zA-Z0-9]*#{disallowed}[a-zA-Z0-9]*( |=|\z)/
|
146
|
+
return true
|
147
|
+
end
|
148
|
+
else
|
149
|
+
if options =~ /(\A| )--#{disallowed}( |=|\z)/
|
150
|
+
return true
|
151
|
+
end
|
152
|
+
end
|
153
|
+
end
|
154
|
+
|
155
|
+
false
|
156
|
+
end
|
157
|
+
|
158
|
+
def verify_destination_available(destination, file_rec)
|
159
|
+
if destination.is_a?(Longleaf::StorageLocation)
|
160
|
+
begin
|
161
|
+
destination.available?
|
162
|
+
rescue StorageLocationUnavailableError => e
|
163
|
+
raise StorageLocationUnavailableError.new("Cannot replicate #{file_rec.path} to destination" \
|
164
|
+
+ " storage location #{destination.name}, it is unavailable.") unless destination.available?
|
165
|
+
end
|
166
|
+
elsif destination.start_with?("/")
|
167
|
+
raise StorageLocationUnavailableError.new("Cannot replicate #{file_rec.path} to destination" \
|
168
|
+
+ " #{destination}, path does not exist.") unless Dir.exist?(destination)
|
169
|
+
end
|
170
|
+
end
|
171
|
+
|
172
|
+
def register_replica(destination, rel_path, file_rec)
|
173
|
+
dest_file_path = File.join(destination.path, rel_path)
|
174
|
+
dest_file_rec = FileRecord.new(dest_file_path, destination)
|
175
|
+
|
176
|
+
register_event = RegisterEvent.new(file_rec: dest_file_rec,
|
177
|
+
app_manager: @app_manager,
|
178
|
+
force: true,
|
179
|
+
checksums: file_rec.metadata_record.checksums)
|
180
|
+
register_event.perform
|
181
|
+
end
|
182
|
+
end
|
183
|
+
end
|
@@ -1,14 +1,14 @@
|
|
1
1
|
require 'longleaf/services/application_config_validator'
|
2
2
|
require 'longleaf/services/application_config_manager'
|
3
3
|
|
4
|
-
# Deserializer for application configuration files
|
5
4
|
module Longleaf
|
5
|
+
# Deserializer for application configuration files
|
6
6
|
class ApplicationConfigDeserializer
|
7
7
|
|
8
8
|
# Deserializes a valid application configuration file as a ApplicationConfigManager option
|
9
9
|
# @param config_path [String] file path to the application configuration file
|
10
10
|
# @param format [String] encoding format of the config file
|
11
|
-
# return [
|
11
|
+
# return [ApplicationConfigManager] manager for the loaded configuration
|
12
12
|
def self.deserialize(config_path, format: 'yaml')
|
13
13
|
config = load(config_path, format: format)
|
14
14
|
|
@@ -35,11 +35,9 @@ module Longleaf
|
|
35
35
|
YAML.load_file(config_path)
|
36
36
|
rescue Errno::ENOENT => err
|
37
37
|
raise Longleaf::ConfigurationError.new(
|
38
|
-
"
|
38
|
+
"Configuration file #{config_path} does not exist.")
|
39
39
|
rescue => err
|
40
|
-
raise Longleaf::ConfigurationError.new(
|
41
|
-
%Q(Failed to load application configuration due to the following reason:
|
42
|
-
#{err.message}))
|
40
|
+
raise Longleaf::ConfigurationError.new(err)
|
43
41
|
end
|
44
42
|
end
|
45
43
|
end
|
@@ -6,8 +6,8 @@ require_relative 'service_mapping_validator'
|
|
6
6
|
require_relative 'service_mapping_manager'
|
7
7
|
require_relative 'service_manager'
|
8
8
|
|
9
|
-
# Manager which loads and provides access to the configuration of the application
|
10
9
|
module Longleaf
|
10
|
+
# Manager which loads and provides access to the configuration of the application
|
11
11
|
class ApplicationConfigManager
|
12
12
|
attr_reader :service_manager
|
13
13
|
attr_reader :location_manager
|
@@ -18,7 +18,9 @@ module Longleaf
|
|
18
18
|
definition_manager = Longleaf::ServiceDefinitionManager.new(config)
|
19
19
|
mapping_manager = Longleaf::ServiceMappingManager.new(config)
|
20
20
|
@service_manager = Longleaf::ServiceManager.new(
|
21
|
-
definition_manager: definition_manager,
|
21
|
+
definition_manager: definition_manager,
|
22
|
+
mapping_manager: mapping_manager,
|
23
|
+
app_manager: self)
|
22
24
|
end
|
23
25
|
end
|
24
26
|
end
|
@@ -2,8 +2,8 @@ require_relative 'storage_location_validator'
|
|
2
2
|
require_relative 'service_definition_validator'
|
3
3
|
require_relative 'service_mapping_validator'
|
4
4
|
|
5
|
-
# Validator for Longleaf application configuration
|
6
5
|
module Longleaf
|
6
|
+
# Validator for Longleaf application configuration
|
7
7
|
class ApplicationConfigValidator
|
8
8
|
|
9
9
|
# Validates the application configuration provided. Will raise ConfigurationError
|
@@ -1,26 +1,28 @@
|
|
1
1
|
require 'yaml'
|
2
|
-
|
3
|
-
|
4
|
-
|
2
|
+
require 'longleaf/models/metadata_record'
|
3
|
+
require 'longleaf/models/md_fields'
|
4
|
+
require 'longleaf/errors'
|
5
|
+
require 'longleaf/logging'
|
5
6
|
|
6
|
-
# Service which deserializes metadata files into MetadataRecord objects
|
7
7
|
module Longleaf
|
8
|
+
# Service which deserializes metadata files into MetadataRecord objects
|
8
9
|
class MetadataDeserializer
|
9
|
-
|
10
|
+
extend Longleaf::Logging
|
11
|
+
MDF ||= MDFields
|
10
12
|
|
11
13
|
# Deserialize a file into a MetadataRecord object
|
12
14
|
#
|
13
15
|
# @param file_path [String] path of the file to read. Required.
|
14
16
|
# @param format [String] format the file is stored in. Default is 'yaml'.
|
15
|
-
def self.deserialize(file_path:, format: 'yaml')
|
17
|
+
def self.deserialize(file_path:, format: 'yaml', digest_algs: [])
|
16
18
|
case format
|
17
19
|
when 'yaml'
|
18
|
-
md = from_yaml(file_path)
|
20
|
+
md = from_yaml(file_path, digest_algs)
|
19
21
|
else
|
20
22
|
raise ArgumentError.new('Invalid deserialization format #{format} specified')
|
21
23
|
end
|
22
24
|
|
23
|
-
if !md || !md.key?(MDF::DATA) || !md.key?(MDF::SERVICES)
|
25
|
+
if !md || !md.is_a?(Hash) || !md.key?(MDF::DATA) || !md.key?(MDF::SERVICES)
|
24
26
|
raise Longleaf::MetadataError.new("Invalid metadata file, did not contain data or services fields: #{file_path}")
|
25
27
|
end
|
26
28
|
|
@@ -61,8 +63,43 @@ module Longleaf
|
|
61
63
|
last_modified: last_modified)
|
62
64
|
end
|
63
65
|
|
64
|
-
|
65
|
-
|
66
|
+
# Load configuration a yaml encoded configuration file
|
67
|
+
def self.from_yaml(file_path, digest_algs)
|
68
|
+
File.open(file_path, 'r:bom|utf-8') do |f|
|
69
|
+
contents = f.read
|
70
|
+
|
71
|
+
verify_digests(file_path, contents, digest_algs)
|
72
|
+
|
73
|
+
YAML.load(contents)
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
def self.verify_digests(file_path, contents, digest_algs)
|
78
|
+
return if digest_algs.nil? || digest_algs.empty?
|
79
|
+
|
80
|
+
digest_algs.each do |alg|
|
81
|
+
if file_path.respond_to?(:path)
|
82
|
+
path = file_path.path
|
83
|
+
else
|
84
|
+
path = file_path
|
85
|
+
end
|
86
|
+
digest_path = "#{path}.#{alg}"
|
87
|
+
unless File.exist?(digest_path)
|
88
|
+
logger.warn("Missing expected #{alg} digest for #{path}")
|
89
|
+
next
|
90
|
+
end
|
91
|
+
|
92
|
+
digest = DigestHelper::start_digest(alg)
|
93
|
+
result = digest.hexdigest(contents)
|
94
|
+
existing_digest = IO.read(digest_path)
|
95
|
+
|
96
|
+
if result == existing_digest
|
97
|
+
logger.info("Metadata fixity check using algorithm '#{alg}' succeeded for file #{path}")
|
98
|
+
else
|
99
|
+
raise ChecksumMismatchError.new("Metadata digest of type #{alg} did not match the contents of #{path}:" \
|
100
|
+
+ " expected #{existing_digest}, calculated #{result}")
|
101
|
+
end
|
102
|
+
end
|
66
103
|
end
|
67
104
|
end
|
68
105
|
end
|
@@ -1,20 +1,27 @@
|
|
1
1
|
require 'yaml'
|
2
|
-
|
3
|
-
|
2
|
+
require 'longleaf/models/metadata_record'
|
3
|
+
require 'longleaf/models/md_fields'
|
4
|
+
require 'longleaf/helpers/digest_helper'
|
5
|
+
require 'longleaf/errors'
|
6
|
+
require 'longleaf/logging'
|
7
|
+
require 'pathname'
|
4
8
|
|
5
|
-
# Service which serializes MetadataRecord objects
|
6
9
|
module Longleaf
|
10
|
+
# Service which serializes MetadataRecord objects
|
7
11
|
class MetadataSerializer
|
8
|
-
|
12
|
+
extend Longleaf::Logging
|
13
|
+
MDF ||= MDFields
|
9
14
|
|
10
15
|
# Serialize the contents of the provided metadata record to the specified path
|
11
16
|
#
|
12
17
|
# @param metadata [MetadataRecord] metadata record to serialize. Required.
|
13
18
|
# @param file_path [String] path to write the file to. Required.
|
14
19
|
# @param format [String] format to serialize the metadata in. Default is 'yaml'.
|
15
|
-
|
20
|
+
# @param digest_algs [Array] if provided, sidecar digest files for the metadata file
|
21
|
+
# will be generated for each algorithm.
|
22
|
+
def self.write(metadata:, file_path:, format: 'yaml', digest_algs: [])
|
16
23
|
raise ArgumentError.new('metadata parameter must be a MetadataRecord') \
|
17
|
-
unless metadata.class ==
|
24
|
+
unless metadata.class == MetadataRecord
|
18
25
|
|
19
26
|
case format
|
20
27
|
when 'yaml'
|
@@ -23,7 +30,12 @@ module Longleaf
|
|
23
30
|
raise ArgumentError.new('Invalid serialization format #{format} specified')
|
24
31
|
end
|
25
32
|
|
33
|
+
# Fill in parent directories if they do not exist
|
34
|
+
parent_dir = Pathname(file_path).parent
|
35
|
+
parent_dir.mkpath unless parent_dir.exist?
|
36
|
+
|
26
37
|
File.write(file_path, content)
|
38
|
+
write_digests(file_path, content, digest_algs)
|
27
39
|
end
|
28
40
|
|
29
41
|
# @param metadata [MetadataRecord] metadata record to transform
|
@@ -33,6 +45,8 @@ module Longleaf
|
|
33
45
|
props.to_yaml
|
34
46
|
end
|
35
47
|
|
48
|
+
# Create a hash representation of the given MetadataRecord file
|
49
|
+
# @param metadata [MetadataRecord] metadata record to transform into a hash
|
36
50
|
def self.to_hash(metadata)
|
37
51
|
props = Hash.new
|
38
52
|
|
@@ -59,6 +73,9 @@ module Longleaf
|
|
59
73
|
props
|
60
74
|
end
|
61
75
|
|
76
|
+
# @param format [String] encoding format used for metadata file
|
77
|
+
# @return [String] the suffix used to indicate that a file is a metadata file in the provided encoding
|
78
|
+
# @raise [ArgumentError] raised if the provided format is not a supported metadata encoding format
|
62
79
|
def self.metadata_suffix(format: 'yaml')
|
63
80
|
case format
|
64
81
|
when 'yaml'
|
@@ -67,5 +84,24 @@ module Longleaf
|
|
67
84
|
raise ArgumentError.new('Invalid serialization format #{format} specified')
|
68
85
|
end
|
69
86
|
end
|
87
|
+
|
88
|
+
private
|
89
|
+
def self.write_digests(file_path, content, digests)
|
90
|
+
return if digests.nil? || digests.empty?
|
91
|
+
|
92
|
+
digests.each do |alg|
|
93
|
+
digest_class = DigestHelper::start_digest(alg)
|
94
|
+
result = digest_class.hexdigest(content)
|
95
|
+
if file_path.respond_to?(:path)
|
96
|
+
digest_path = "#{file_path.path}.#{alg}"
|
97
|
+
else
|
98
|
+
digest_path = "#{file_path}.#{alg}"
|
99
|
+
end
|
100
|
+
|
101
|
+
File.write(digest_path, result)
|
102
|
+
|
103
|
+
self.logger.debug("Generated #{alg} digest for metadata file #{file_path}: #{result}")
|
104
|
+
end
|
105
|
+
end
|
70
106
|
end
|
71
107
|
end
|
@@ -0,0 +1,112 @@
|
|
1
|
+
require 'pathname'
|
2
|
+
|
3
|
+
module Longleaf
|
4
|
+
# Cache for loading and retrieving preservation service classes
|
5
|
+
class ServiceClassCache
|
6
|
+
STD_PRESERVATION_SERVICE_PATH = 'longleaf/preservation_services/'
|
7
|
+
|
8
|
+
def initialize(app_manager)
|
9
|
+
@app_manager = app_manager
|
10
|
+
# Cache storing per service definition instances of service classes
|
11
|
+
@service_instance_cache = Hash.new
|
12
|
+
# Cache storing per script path class of service
|
13
|
+
@class_cache = Hash.new
|
14
|
+
end
|
15
|
+
|
16
|
+
# Returns an instance of the preversation service defined for the provided service definition,
|
17
|
+
# based on the work_script and work_class properties provided.
|
18
|
+
#
|
19
|
+
# @param service_def [ServiceDefinition] definition of service to instantiate
|
20
|
+
# @return [PreservationService] Instance of the preservation service class for the definition.
|
21
|
+
def service_instance(service_def)
|
22
|
+
service_name = service_def.name
|
23
|
+
# Return the cached instance of the service
|
24
|
+
if @service_instance_cache.key?(service_name)
|
25
|
+
return @service_instance_cache[service_name]
|
26
|
+
end
|
27
|
+
|
28
|
+
clazz = service_class(service_def)
|
29
|
+
# Cache and return the class instance
|
30
|
+
@service_instance_cache[service_name] = clazz.new(service_def, @app_manager)
|
31
|
+
end
|
32
|
+
|
33
|
+
# Load and return the PreservationService class assigned to the provided service definition,
|
34
|
+
# based on the work_script and work_class properties provided.
|
35
|
+
#
|
36
|
+
# @param service_def [ServiceDefinition] definition of service to retrieve class for
|
37
|
+
# @return [Class] class of work_script
|
38
|
+
def service_class(service_def)
|
39
|
+
service_name = service_def.name
|
40
|
+
work_script = service_def.work_script
|
41
|
+
|
42
|
+
if work_script.include?('/')
|
43
|
+
expanded_path = Pathname.new(work_script).expand_path.to_s
|
44
|
+
if !from_permitted_path?(expanded_path)
|
45
|
+
raise ConfigurationError.new("Unable to load work_script for service #{service_name}, #{work_script} is not in a known library path.")
|
46
|
+
end
|
47
|
+
|
48
|
+
last_slash_index = work_script.rindex('/')
|
49
|
+
script_path = work_script[0..last_slash_index]
|
50
|
+
script_name = work_script[(last_slash_index + 1)..-1]
|
51
|
+
else
|
52
|
+
script_path = STD_PRESERVATION_SERVICE_PATH
|
53
|
+
script_name = work_script
|
54
|
+
end
|
55
|
+
|
56
|
+
# Strip off the extension
|
57
|
+
script_name.sub!('.rb', '')
|
58
|
+
|
59
|
+
require_path = File.join(script_path, script_name)
|
60
|
+
# Return the cached Class if this path has been encountered before
|
61
|
+
if @class_cache.key?(require_path)
|
62
|
+
return @class_cache[require_path]
|
63
|
+
end
|
64
|
+
|
65
|
+
# Load the script
|
66
|
+
begin
|
67
|
+
require require_path
|
68
|
+
rescue LoadError => e
|
69
|
+
raise ConfigurationError.new("Failed to load work_script '#{script_name}' for service #{service_name}")
|
70
|
+
end
|
71
|
+
|
72
|
+
# Generate the class name, either configured or from file naming convention if possible
|
73
|
+
if service_def.work_class
|
74
|
+
class_name = service_def.work_class
|
75
|
+
else
|
76
|
+
class_name = script_name.split('_').map(&:capitalize).join
|
77
|
+
# Assume the longleaf module for classes in the standard path
|
78
|
+
class_name = 'Longleaf::' + class_name if script_path == STD_PRESERVATION_SERVICE_PATH
|
79
|
+
end
|
80
|
+
|
81
|
+
begin
|
82
|
+
class_constant = constantize(class_name)
|
83
|
+
# cache the class for this work_script and return it
|
84
|
+
@class_cache[require_path] = class_constant
|
85
|
+
rescue NameError
|
86
|
+
raise ConfigurationError.new("Failed to load work_script '#{script_name}' for service #{service_name}, class name #{class_name} was not found.")
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
private
|
91
|
+
# Borrowed from sidekiq implementation
|
92
|
+
def constantize(str)
|
93
|
+
names = str.split('::')
|
94
|
+
names.shift if names.empty? || names.first.empty?
|
95
|
+
|
96
|
+
names.inject(Object) do |constant, name|
|
97
|
+
# the false flag limits search for name to under the constant namespace
|
98
|
+
# which mimics Rails' behaviour
|
99
|
+
constant.const_defined?(name, false) ? constant.const_get(name, false) : constant.const_missing(name)
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
def from_permitted_path?(script_path)
|
104
|
+
$LOAD_PATH.each do |lib_path|
|
105
|
+
if script_path.start_with?(lib_path)
|
106
|
+
return true
|
107
|
+
end
|
108
|
+
end
|
109
|
+
false
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|