moab-versioning 4.3.0 → 5.0.0.beta1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/moab/bagger.rb +7 -2
- data/lib/moab/config.rb +40 -7
- data/lib/moab/exceptions.rb +6 -0
- data/lib/moab/file_group.rb +12 -9
- data/lib/moab/file_group_difference.rb +26 -23
- data/lib/moab/file_group_difference_subset.rb +5 -3
- data/lib/moab/file_instance.rb +4 -1
- data/lib/moab/file_instance_difference.rb +5 -3
- data/lib/moab/file_inventory.rb +13 -9
- data/lib/moab/file_inventory_difference.rb +8 -6
- data/lib/moab/file_manifestation.rb +5 -2
- data/lib/moab/file_signature.rb +12 -7
- data/lib/moab/signature_catalog.rb +13 -13
- data/lib/moab/signature_catalog_entry.rb +6 -4
- data/lib/moab/stanford.rb +2 -10
- data/lib/moab/storage_object.rb +11 -5
- data/lib/moab/storage_object_validator.rb +24 -10
- data/lib/moab/storage_object_version.rb +19 -12
- data/lib/moab/storage_repository.rb +49 -7
- data/lib/moab/storage_services.rb +12 -9
- data/lib/moab/utc_time.rb +2 -0
- data/lib/moab/verification_result.rb +4 -3
- data/lib/moab/version_metadata_entry.rb +6 -4
- data/lib/moab.rb +2 -9
- data/lib/serializer/manifest.rb +4 -2
- data/lib/serializer/serializable.rb +6 -1
- data/lib/serializer.rb +2 -0
- data/lib/stanford/content_inventory.rb +23 -19
- data/lib/stanford/storage_object_validator.rb +2 -0
- data/lib/stanford/storage_repository.rb +6 -2
- data/lib/stanford/storage_services.rb +2 -0
- metadata +22 -42
- data/lib/moab/deposit_bag_validator.rb +0 -323
- data/lib/moab/version_metadata.rb +0 -32
- data/lib/moab/version_metadata_event.rb +0 -40
- data/lib/stanford/active_fedora_object.rb +0 -28
- data/lib/stanford/dor_metadata.rb +0 -41
- data/lib/stanford/moab_storage_directory.rb +0 -36
data/lib/moab/file_signature.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Moab
|
2
4
|
# The fixity properties of a file, used to determine file content equivalence regardless of filename.
|
3
5
|
# Placing this data in a class by itself facilitates using file size together with the MD5 and SHA1 checksums
|
@@ -44,19 +46,19 @@ module Moab
|
|
44
46
|
|
45
47
|
# @attribute
|
46
48
|
# @return [Integer] The size of the file in bytes
|
47
|
-
attribute :size, Integer, :
|
49
|
+
attribute :size, Integer, on_save: proc { |n| n.to_s }
|
48
50
|
|
49
51
|
# @attribute
|
50
52
|
# @return [String] The MD5 checksum value of the file
|
51
|
-
attribute :md5, String, :
|
53
|
+
attribute :md5, String, on_save: proc { |n| n.nil? ? "" : n.to_s }
|
52
54
|
|
53
55
|
# @attribute
|
54
56
|
# @return [String] The SHA1 checksum value of the file
|
55
|
-
attribute :sha1, String, :
|
57
|
+
attribute :sha1, String, on_save: proc { |n| n.nil? ? "" : n.to_s }
|
56
58
|
|
57
59
|
# @attribute
|
58
60
|
# @return [String] The SHA256 checksum value of the file
|
59
|
-
attribute :sha256, String, :
|
61
|
+
attribute :sha256, String, on_save: proc { |n| n.nil? ? "" : n.to_s }
|
60
62
|
|
61
63
|
KNOWN_ALGOS = {
|
62
64
|
md5: proc { Digest::MD5.new },
|
@@ -83,7 +85,7 @@ module Moab
|
|
83
85
|
end
|
84
86
|
end
|
85
87
|
|
86
|
-
new(signatures.
|
88
|
+
new(signatures.transform_values(&:hexdigest).merge(size: pathname.size))
|
87
89
|
end
|
88
90
|
|
89
91
|
# @param type [Symbol,String] The type of checksum
|
@@ -128,10 +130,12 @@ module Moab
|
|
128
130
|
def eql?(other)
|
129
131
|
return false unless other.respond_to?(:size) && other.respond_to?(:checksums)
|
130
132
|
return false if size.to_i != other.size.to_i
|
133
|
+
|
131
134
|
self_checksums = checksums
|
132
135
|
other_checksums = other.checksums
|
133
136
|
matching_keys = self_checksums.keys & other_checksums.keys
|
134
137
|
return false if matching_keys.empty?
|
138
|
+
|
135
139
|
matching_keys.each do |key|
|
136
140
|
return false if self_checksums[key] != other_checksums[key]
|
137
141
|
end
|
@@ -176,6 +180,7 @@ module Moab
|
|
176
180
|
def normalized_signature(pathname)
|
177
181
|
sig_from_file = FileSignature.new.signature_from_file(pathname)
|
178
182
|
return sig_from_file if eql?(sig_from_file)
|
183
|
+
|
179
184
|
# The full signature from file is consistent with current values, or...
|
180
185
|
# One or more of the fixity values is inconsistent, so raise an exception
|
181
186
|
raise(MoabRuntimeError, "Signature inconsistent between inventory and file for #{pathname}: #{diff(sig_from_file).inspect}")
|
@@ -185,8 +190,8 @@ module Moab
|
|
185
190
|
def self.checksum_names_for_type
|
186
191
|
{
|
187
192
|
md5: ['MD5'],
|
188
|
-
sha1: [
|
189
|
-
sha256: [
|
193
|
+
sha1: %w[SHA-1 SHA1],
|
194
|
+
sha256: %w[SHA-256 SHA256]
|
190
195
|
}
|
191
196
|
end
|
192
197
|
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Moab
|
2
4
|
# A digital object's Signature Catalog is derived from an filtered aggregation of the file inventories
|
3
5
|
# of a digital object's set of versions. (see {#update})
|
@@ -37,20 +39,20 @@ module Moab
|
|
37
39
|
|
38
40
|
# @attribute
|
39
41
|
# @return [String] The object ID (druid)
|
40
|
-
attribute :digital_object_id, String, :
|
42
|
+
attribute :digital_object_id, String, tag: 'objectId'
|
41
43
|
|
42
44
|
# @attribute
|
43
45
|
# @return [Integer] The ordinal version number
|
44
|
-
attribute :version_id, Integer, :
|
46
|
+
attribute :version_id, Integer, tag: 'versionId', key: true, on_save: proc { |n| n.to_s }
|
45
47
|
|
46
48
|
# @return [String] The unique identifier concatenating digital object id with version id
|
47
49
|
def composite_key
|
48
|
-
@digital_object_id
|
50
|
+
"#{@digital_object_id}-#{StorageObject.version_dirname(@version_id)}"
|
49
51
|
end
|
50
52
|
|
51
53
|
# @attribute
|
52
54
|
# @return [String] The datetime at which the catalog was updated
|
53
|
-
attribute :catalog_datetime, Time, :
|
55
|
+
attribute :catalog_datetime, Time, tag: 'catalogDatetime'
|
54
56
|
|
55
57
|
def catalog_datetime=(datetime)
|
56
58
|
@catalog_datetime = Moab::UtcTime.input(datetime)
|
@@ -62,7 +64,7 @@ module Moab
|
|
62
64
|
|
63
65
|
# @attribute
|
64
66
|
# @return [Integer] The total number of data files (dynamically calculated)
|
65
|
-
attribute :file_count, Integer, :
|
67
|
+
attribute :file_count, Integer, tag: 'fileCount', on_save: proc { |t| t.to_s }
|
66
68
|
|
67
69
|
def file_count
|
68
70
|
entries.size
|
@@ -70,7 +72,7 @@ module Moab
|
|
70
72
|
|
71
73
|
# @attribute
|
72
74
|
# @return [Integer] The total size (in bytes) of all data files (dynamically calculated)
|
73
|
-
attribute :byte_count, Integer, :
|
75
|
+
attribute :byte_count, Integer, tag: 'byteCount', on_save: proc { |t| t.to_s }
|
74
76
|
|
75
77
|
def byte_count
|
76
78
|
entries.inject(0) { |sum, entry| sum + entry.signature.size.to_i }
|
@@ -78,7 +80,7 @@ module Moab
|
|
78
80
|
|
79
81
|
# @attribute
|
80
82
|
# @return [Integer] The total disk usage (in 1 kB blocks) of all data files (estimating du -k result) (dynamically calculated)
|
81
|
-
attribute :block_count, Integer, :
|
83
|
+
attribute :block_count, Integer, tag: 'blockCount', on_save: proc { |t| t.to_s }
|
82
84
|
|
83
85
|
def block_count
|
84
86
|
block_size = 1024
|
@@ -92,7 +94,7 @@ module Moab
|
|
92
94
|
|
93
95
|
# @attribute
|
94
96
|
# @return [Array<SignatureCatalogEntry>] The set of data groups comprising the version
|
95
|
-
has_many :entries, SignatureCatalogEntry, :
|
97
|
+
has_many :entries, SignatureCatalogEntry, tag: 'entry'
|
96
98
|
|
97
99
|
def entries=(entry_array)
|
98
100
|
entry_array.each do |entry|
|
@@ -176,14 +178,12 @@ module Moab
|
|
176
178
|
# containing only those files that were added in this version
|
177
179
|
# @example {include:file:spec/features/catalog/version_additions_spec.rb}
|
178
180
|
def version_additions(version_inventory)
|
179
|
-
version_additions = FileInventory.new(:
|
181
|
+
version_additions = FileInventory.new(type: 'additions')
|
180
182
|
version_additions.copy_ids(version_inventory)
|
181
183
|
version_inventory.groups.each do |group|
|
182
|
-
group_addtions = FileGroup.new(:
|
184
|
+
group_addtions = FileGroup.new(group_id: group.group_id)
|
183
185
|
group.files.each do |file|
|
184
|
-
unless @signature_hash.key?(file.signature)
|
185
|
-
group_addtions.add_file_instance(file.signature, file.instances[0])
|
186
|
-
end
|
186
|
+
group_addtions.add_file_instance(file.signature, file.instances[0]) unless @signature_hash.key?(file.signature)
|
187
187
|
end
|
188
188
|
version_additions.groups << group_addtions unless group_addtions.files.empty?
|
189
189
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Moab
|
2
4
|
# A file-level entry in a digital object's {SignatureCatalog}.
|
3
5
|
# It has a child {FileSignature} element that identifies the file's contents (the bytestream)
|
@@ -23,19 +25,19 @@ module Moab
|
|
23
25
|
|
24
26
|
# @attribute
|
25
27
|
# @return [Integer] The ordinal version number
|
26
|
-
attribute :version_id, Integer, :
|
28
|
+
attribute :version_id, Integer, tag: 'originalVersion', key: true, on_save: proc { |n| n.to_s }
|
27
29
|
|
28
30
|
# @attribute
|
29
31
|
# @return [String] The name of the file group
|
30
|
-
attribute :group_id, String, :
|
32
|
+
attribute :group_id, String, tag: 'groupId', key: true
|
31
33
|
|
32
34
|
# @attribute
|
33
35
|
# @return [String] The id is the filename path, relative to the file group's base directory
|
34
|
-
attribute :path, String, :
|
36
|
+
attribute :path, String, key: true, tag: 'storagePath'
|
35
37
|
|
36
38
|
# @attribute
|
37
39
|
# @return [FileSignature] The fixity data of the file instance
|
38
|
-
element :signature, FileSignature, :
|
40
|
+
element :signature, FileSignature, tag: 'fileSignature'
|
39
41
|
|
40
42
|
def signature
|
41
43
|
# HappyMapper's parser tries to put an array of signatures in the signature field
|
data/lib/moab/stanford.rb
CHANGED
@@ -1,19 +1,11 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'moab'
|
2
4
|
require 'stanford/content_inventory'
|
3
|
-
require 'stanford/dor_metadata'
|
4
5
|
require 'stanford/storage_repository'
|
5
6
|
require 'stanford/storage_services'
|
6
|
-
require 'stanford/active_fedora_object'
|
7
|
-
require 'stanford/moab_storage_directory'
|
8
7
|
require 'stanford/storage_object_validator'
|
9
8
|
|
10
9
|
# Stanford is a module that isolates classes specific to the Stanford Digital Repository
|
11
|
-
#
|
12
|
-
# ====Data Model
|
13
|
-
# * <b>{DorMetadata} = utility methods for interfacing with Stanford metadata files (esp contentMetadata)</b>
|
14
|
-
# * {ActiveFedoraObject} [1..*] = utility for extracting content or other information from a Fedora Instance
|
15
|
-
#
|
16
|
-
# @note Copyright (c) 2012 by The Board of Trustees of the Leland Stanford Junior University.
|
17
|
-
# All rights reserved. See {file:LICENSE.rdoc} for details.
|
18
10
|
module Stanford
|
19
11
|
end
|
data/lib/moab/storage_object.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Moab
|
2
4
|
# A class to represent a digital object's repository storage location
|
3
5
|
# and methods for
|
@@ -82,10 +84,10 @@ module Moab
|
|
82
84
|
# @return [FileInventory] The file inventory of the specified type for this version
|
83
85
|
def versionize_bag(bag_dir, current_version, new_version)
|
84
86
|
new_inventory = FileInventory.new(
|
85
|
-
:
|
86
|
-
:
|
87
|
-
:
|
88
|
-
:
|
87
|
+
type: 'version',
|
88
|
+
digital_object_id: @digital_object_id,
|
89
|
+
version_id: new_version.version_id,
|
90
|
+
inventory_datetime: Time.now
|
89
91
|
)
|
90
92
|
new_inventory.inventory_from_bagit_bag(bag_dir)
|
91
93
|
new_inventory.write_xml_file(bag_dir)
|
@@ -113,6 +115,7 @@ module Moab
|
|
113
115
|
storage_filepath = @object_pathname.join(catalog_filepath)
|
114
116
|
errmsg = "#{catalog_filepath} missing from storage location #{storage_filepath}"
|
115
117
|
raise FileNotFoundException, errmsg unless storage_filepath.exist?
|
118
|
+
|
116
119
|
storage_filepath
|
117
120
|
end
|
118
121
|
|
@@ -127,9 +130,10 @@ module Moab
|
|
127
130
|
def version_id_list
|
128
131
|
list = []
|
129
132
|
return list unless @object_pathname.exist?
|
133
|
+
|
130
134
|
@object_pathname.children.each do |dirname|
|
131
135
|
vnum = dirname.basename.to_s
|
132
|
-
list << vnum[1
|
136
|
+
list << vnum[1..].to_i if vnum =~ /^v(\d+)$/
|
133
137
|
end
|
134
138
|
list.sort
|
135
139
|
end
|
@@ -163,6 +167,7 @@ module Moab
|
|
163
167
|
if version_inventory.version_id != (current_version_id + 1)
|
164
168
|
raise(MoabRuntimeError, "version mismatch - current: #{current_version_id} new: #{version_inventory.version_id}")
|
165
169
|
end
|
170
|
+
|
166
171
|
true
|
167
172
|
end
|
168
173
|
|
@@ -188,6 +193,7 @@ module Moab
|
|
188
193
|
# * Current version + 1 is used for creation of a new version
|
189
194
|
def storage_object_version(version_id)
|
190
195
|
raise(MoabRuntimeError, "Version ID not specified") unless version_id
|
196
|
+
|
191
197
|
StorageObjectVersion.new(self, version_id)
|
192
198
|
end
|
193
199
|
|
@@ -1,15 +1,17 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'set'
|
2
4
|
|
3
5
|
module Moab
|
4
6
|
# Given a druid path, are the contents actually a well-formed Moab?
|
5
7
|
# Shameless green: repetitious code included.
|
6
8
|
class StorageObjectValidator
|
7
|
-
METADATA_DIR = "metadata"
|
8
|
-
CONTENT_DIR = "content"
|
9
|
+
METADATA_DIR = "metadata"
|
10
|
+
CONTENT_DIR = "content"
|
9
11
|
EXPECTED_DATA_SUB_DIRS = [CONTENT_DIR, METADATA_DIR].freeze
|
10
12
|
IMPLICIT_DIRS = ['.', '..'].freeze # unlike Find.find, Dir.entries returns the current/parent dirs
|
11
|
-
DATA_DIR = "data"
|
12
|
-
MANIFESTS_DIR = 'manifests'
|
13
|
+
DATA_DIR = "data"
|
14
|
+
MANIFESTS_DIR = 'manifests'
|
13
15
|
EXPECTED_VERSION_SUB_DIRS = [DATA_DIR, MANIFESTS_DIR].freeze
|
14
16
|
MANIFEST_INVENTORY_PATH = File.join(MANIFESTS_DIR, "manifestInventory.xml").freeze
|
15
17
|
SIGNATURE_CATALOG_PATH = File.join(MANIFESTS_DIR, "signatureCatalog.xml").freeze
|
@@ -83,13 +85,14 @@ module Moab
|
|
83
85
|
end
|
84
86
|
|
85
87
|
def version_dir_format?(dirname)
|
86
|
-
dirname =~ /^
|
88
|
+
dirname =~ /^v\d{4}$/
|
87
89
|
end
|
88
90
|
|
89
91
|
# call only if the version directories are "correctly named" vdddd
|
90
92
|
def check_sequential_version_dirs
|
91
93
|
version_directories.each_with_index do |dir_name, index|
|
92
|
-
next if dir_name[1
|
94
|
+
next if dir_name[1..].to_i == index + 1 # version numbering starts at 1, array indexing at 0
|
95
|
+
|
93
96
|
return [result_hash(VERSIONS_NOT_IN_ORDER, version_directories)]
|
94
97
|
end
|
95
98
|
[]
|
@@ -113,6 +116,7 @@ module Moab
|
|
113
116
|
return expected_version_sub_dirs(version_path, version) if count == EXPECTED_VERSION_SUB_DIRS.size
|
114
117
|
return found_unexpected(version_sub_dirs, version, EXPECTED_VERSION_SUB_DIRS) if count > EXPECTED_VERSION_SUB_DIRS.size
|
115
118
|
return missing_dir(version_sub_dirs, version, EXPECTED_VERSION_SUB_DIRS) if count < EXPECTED_VERSION_SUB_DIRS.size
|
119
|
+
|
116
120
|
[]
|
117
121
|
end
|
118
122
|
|
@@ -122,15 +126,20 @@ module Moab
|
|
122
126
|
data_sub_dirs = directory_entries(data_dir_path)
|
123
127
|
errors.concat check_data_sub_dirs(version, data_sub_dirs)
|
124
128
|
errors.concat check_metadata_dir_files_only(version_path) if errors.empty?
|
125
|
-
|
129
|
+
if data_sub_dirs.include?('content') && errors.empty?
|
130
|
+
errors.concat check_optional_content_dir(version_path, allow_content_subdirs)
|
131
|
+
end
|
126
132
|
errors
|
127
133
|
end
|
128
134
|
|
129
135
|
def check_data_sub_dirs(version, data_sub_dirs)
|
130
136
|
return found_unexpected(data_sub_dirs, version, EXPECTED_DATA_SUB_DIRS) if data_sub_dirs.size > EXPECTED_DATA_SUB_DIRS.size
|
137
|
+
|
131
138
|
errors = []
|
132
139
|
errors.concat missing_dir(data_sub_dirs, version, [METADATA_DIR]) unless data_sub_dirs.include?(METADATA_DIR)
|
133
|
-
|
140
|
+
unless data_sub_dirs.to_set.subset?(EXPECTED_DATA_SUB_DIRS.to_set)
|
141
|
+
errors.concat found_unexpected(data_sub_dirs, version, EXPECTED_DATA_SUB_DIRS)
|
142
|
+
end
|
134
143
|
errors
|
135
144
|
end
|
136
145
|
|
@@ -139,7 +148,9 @@ module Moab
|
|
139
148
|
content_dir_path = File.join(version_path, DATA_DIR, CONTENT_DIR)
|
140
149
|
errors << result_hash(NO_FILES_IN_CONTENT_DIR, basename(version_path)) if directory_entries(content_dir_path).empty?
|
141
150
|
content_sub_dir = contains_sub_dir?(content_dir_path)
|
142
|
-
|
151
|
+
if content_sub_dir && !allow_content_subdirs
|
152
|
+
errors << result_hash(CONTENT_SUB_DIRS_DETECTED, version: basename(version_path), dir: content_sub_dir)
|
153
|
+
end
|
143
154
|
if allow_content_subdirs && contains_sub_dir?(content_dir_path) && contains_forbidden_content_sub_dir?(content_dir_path)
|
144
155
|
errors << result_hash(BAD_SUB_DIR_IN_CONTENT_DIR, basename(version_path))
|
145
156
|
end
|
@@ -159,7 +170,9 @@ module Moab
|
|
159
170
|
metadata_dir_path = File.join(version_path, DATA_DIR, METADATA_DIR)
|
160
171
|
errors << result_hash(NO_FILES_IN_METADATA_DIR, basename(version_path)) if directory_entries(metadata_dir_path).empty?
|
161
172
|
metadata_sub_dir = contains_sub_dir?(metadata_dir_path)
|
162
|
-
|
173
|
+
if metadata_sub_dir
|
174
|
+
errors << result_hash(METADATA_SUB_DIRS_DETECTED, version: basename(version_path), dir: metadata_sub_dir)
|
175
|
+
end
|
163
176
|
errors
|
164
177
|
end
|
165
178
|
|
@@ -228,6 +241,7 @@ module Moab
|
|
228
241
|
|
229
242
|
def check_required_manifest_files(dir, version)
|
230
243
|
return [result_hash(NO_FILES_IN_MANIFEST_DIR, version)] unless contains_file?(File.join(dir, MANIFESTS_DIR))
|
244
|
+
|
231
245
|
errors = []
|
232
246
|
errors << result_hash(NO_MANIFEST_INVENTORY, version) unless File.exist?(File.join(dir, MANIFEST_INVENTORY_PATH))
|
233
247
|
errors << result_hash(NO_SIGNATURE_CATALOG, version) unless File.exist?(File.join(dir, SIGNATURE_CATALOG_PATH))
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Moab
|
2
4
|
# A class to represent a version subdirectory within an object's home directory in preservation storage
|
3
5
|
# ====Data Model
|
@@ -43,7 +45,7 @@ module Moab
|
|
43
45
|
|
44
46
|
# @return [String] The unique identifier concatenating digital object id with version id
|
45
47
|
def composite_key
|
46
|
-
@storage_object.digital_object_id
|
48
|
+
"#{@storage_object.digital_object_id}-#{StorageObject.version_dirname(@version_id)}"
|
47
49
|
end
|
48
50
|
|
49
51
|
# @return [Boolean] true if the object version directory exists
|
@@ -68,6 +70,7 @@ module Moab
|
|
68
70
|
def find_filepath(file_category, file_id)
|
69
71
|
this_version_filepath = file_pathname(file_category, file_id)
|
70
72
|
return this_version_filepath if this_version_filepath.exist?
|
73
|
+
|
71
74
|
if file_category == 'manifest'
|
72
75
|
msg = "manifest file #{file_id} not found for #{@storage_object.digital_object_id} - #{@version_id}"
|
73
76
|
raise FileNotFoundException, msg
|
@@ -109,14 +112,15 @@ module Moab
|
|
109
112
|
def file_inventory(type)
|
110
113
|
if version_id > 0
|
111
114
|
return @inventory_cache[type] if @inventory_cache.key?(type)
|
115
|
+
|
112
116
|
@inventory_cache[type] = FileInventory.read_xml_file(@version_pathname.join('manifests'), type)
|
113
117
|
else
|
114
|
-
groups = %w[content metadata].collect { |id| FileGroup.new(:
|
118
|
+
groups = %w[content metadata].collect { |id| FileGroup.new(group_id: id) }
|
115
119
|
FileInventory.new(
|
116
|
-
:
|
117
|
-
:
|
118
|
-
:
|
119
|
-
:
|
120
|
+
type: 'version',
|
121
|
+
digital_object_id: @storage_object.digital_object_id,
|
122
|
+
version_id: @version_id,
|
123
|
+
groups: groups
|
120
124
|
)
|
121
125
|
end
|
122
126
|
end
|
@@ -127,7 +131,7 @@ module Moab
|
|
127
131
|
if version_id > 0
|
128
132
|
SignatureCatalog.read_xml_file(@version_pathname.join('manifests'))
|
129
133
|
else
|
130
|
-
SignatureCatalog.new(:
|
134
|
+
SignatureCatalog.new(digital_object_id: @storage_object.digital_object_id)
|
131
135
|
end
|
132
136
|
end
|
133
137
|
|
@@ -136,6 +140,7 @@ module Moab
|
|
136
140
|
# @return [void] Create the version subdirectory and move files into it
|
137
141
|
def ingest_bag_data(bag_dir)
|
138
142
|
raise(MoabRuntimeError, "Version already exists: #{@version_pathname}") if @version_pathname.exist?
|
143
|
+
|
139
144
|
@version_pathname.join('manifests').mkpath
|
140
145
|
bag_dir = Pathname(bag_dir)
|
141
146
|
ingest_dir(bag_dir.join('data'), @version_pathname.join('data'))
|
@@ -150,6 +155,7 @@ module Moab
|
|
150
155
|
# @return [void] recursively link or copy the source directory contents to the target directory
|
151
156
|
def ingest_dir(source_dir, target_dir, use_links = true)
|
152
157
|
raise(MoabRuntimeError, "cannot copy - target already exists: #{target_dir.expand_path}") if target_dir.exist?
|
158
|
+
|
153
159
|
target_dir.mkpath
|
154
160
|
source_dir.children.each do |child|
|
155
161
|
if child.directory?
|
@@ -196,12 +202,12 @@ module Moab
|
|
196
202
|
# @return [void] examine the version's directory and create/serialize a {FileInventory} containing the manifest files
|
197
203
|
def generate_manifest_inventory
|
198
204
|
manifest_inventory = FileInventory.new(
|
199
|
-
:
|
200
|
-
:
|
201
|
-
:
|
205
|
+
type: 'manifests',
|
206
|
+
digital_object_id: @storage_object.digital_object_id,
|
207
|
+
version_id: @version_id
|
202
208
|
)
|
203
209
|
pathname = @version_pathname.join('manifests')
|
204
|
-
manifest_inventory.groups << FileGroup.new(:
|
210
|
+
manifest_inventory.groups << FileGroup.new(group_id: 'manifests').group_from_directory(pathname, false)
|
205
211
|
manifest_inventory.write_xml_file(pathname)
|
206
212
|
end
|
207
213
|
|
@@ -306,7 +312,7 @@ module Moab
|
|
306
312
|
version_additions = file_inventory('additions')
|
307
313
|
result.subentities << VerificationResult.verify_value('composite_key', composite_key, version_additions.composite_key)
|
308
314
|
data_directory = @version_pathname.join('data')
|
309
|
-
directory_inventory = FileInventory.new(:
|
315
|
+
directory_inventory = FileInventory.new(type: 'directory').inventory_from_directory(data_directory)
|
310
316
|
diff = FileInventoryDifference.new
|
311
317
|
diff.compare(version_additions, directory_inventory)
|
312
318
|
compare_result = VerificationResult.new('file_differences')
|
@@ -321,6 +327,7 @@ module Moab
|
|
321
327
|
# @return [null] Deactivate this object version by moving it to another directory. (Used by restore operation)
|
322
328
|
def deactivate(timestamp)
|
323
329
|
return unless @version_pathname.exist?
|
330
|
+
|
324
331
|
timestamp_pathname = @version_pathname.parent.join(timestamp.utc.iso8601.gsub(/[-:]/, ''))
|
325
332
|
timestamp_pathname.mkpath
|
326
333
|
demote_pathame = timestamp_pathname.join(@version_pathname.basename)
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Moab
|
2
4
|
# A class to represent the SDR repository store
|
3
5
|
#
|
@@ -19,8 +21,10 @@ module Moab
|
|
19
21
|
def storage_roots
|
20
22
|
unless defined?(@storage_roots)
|
21
23
|
raise(MoabRuntimeError, "Moab::Config.storage_roots not found in config file") if Moab::Config.storage_roots.nil?
|
24
|
+
|
22
25
|
@storage_roots = [Moab::Config.storage_roots].flatten.collect { |filesystem| Pathname(filesystem) }
|
23
26
|
raise(MoabRuntimeError, "Moab::Config.storage_roots empty") if @storage_roots.empty?
|
27
|
+
|
24
28
|
@storage_roots.each { |root| raise(MoabRuntimeError, "Storage root #{root} not found on system") unless root.exist? }
|
25
29
|
end
|
26
30
|
@storage_roots
|
@@ -30,7 +34,8 @@ module Moab
|
|
30
34
|
def storage_trunk
|
31
35
|
unless defined?(@storage_trunk)
|
32
36
|
raise(MoabRuntimeError, "Moab::Config.storage_trunk not found in config file") if Moab::Config.storage_trunk.nil?
|
33
|
-
|
37
|
+
|
38
|
+
@storage_trunk = Moab::Config.storage_trunk
|
34
39
|
end
|
35
40
|
@storage_trunk
|
36
41
|
end
|
@@ -50,7 +55,7 @@ module Moab
|
|
50
55
|
unless defined?(@deposit_trunk)
|
51
56
|
# do not raise error. this parameter will be ignored if missing
|
52
57
|
# raise "Moab::Config.deposit_trunk not found in config file" if Moab::Config.deposit_trunk.nil?
|
53
|
-
@deposit_trunk
|
58
|
+
@deposit_trunk = Moab::Config.deposit_trunk
|
54
59
|
end
|
55
60
|
@deposit_trunk
|
56
61
|
end
|
@@ -71,6 +76,7 @@ module Moab
|
|
71
76
|
storage_roots.each do |root|
|
72
77
|
root_trunk = root.join(storage_trunk)
|
73
78
|
raise(MoabRuntimeError, "Storage area not found at #{root_trunk}") unless root_trunk.exist?
|
79
|
+
|
74
80
|
root_trunk_branch = root_trunk.join(branch)
|
75
81
|
return root if root_trunk_branch.exist?
|
76
82
|
end
|
@@ -80,6 +86,7 @@ module Moab
|
|
80
86
|
storage_roots.each do |root|
|
81
87
|
root_trunk = root.join(deposit_trunk)
|
82
88
|
raise(MoabRuntimeError, "Deposit area not found at #{root_trunk}") unless root_trunk.exist?
|
89
|
+
|
83
90
|
root_trunk_branch = root_trunk.join(branch)
|
84
91
|
return root if root_trunk_branch.exist?
|
85
92
|
end
|
@@ -93,10 +100,35 @@ module Moab
|
|
93
100
|
# @return [StorageObject] The representation of a digitial object's storage directory, which might not exist yet.
|
94
101
|
def find_storage_object(object_id, include_deposit = false)
|
95
102
|
root = find_storage_root(object_id, include_deposit)
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
103
|
+
create_storage_object(object_id, root)
|
104
|
+
end
|
105
|
+
|
106
|
+
# @param object_id [String] The identifier of the digital object
|
107
|
+
# @param include_deposit [Boolean] specifies whether to look in deposit areas for objects in process of initial ingest
|
108
|
+
# @return [Array<StorageObject>] Representations of a digitial object's storage directories, or an empty array if none found.
|
109
|
+
def search_storage_objects(object_id, include_deposit = false)
|
110
|
+
storage_objects = []
|
111
|
+
# Search for the object's home directory in the storage areas
|
112
|
+
branch = storage_branch(object_id)
|
113
|
+
storage_roots.each do |root|
|
114
|
+
root_trunk = root.join(storage_trunk)
|
115
|
+
raise(MoabRuntimeError, "Storage area not found at #{root_trunk}") unless root_trunk.exist?
|
116
|
+
|
117
|
+
root_trunk_branch = root_trunk.join(branch)
|
118
|
+
storage_objects << create_storage_object(object_id, root) if root_trunk_branch.exist?
|
119
|
+
end
|
120
|
+
# Search for the object's directory in the deposit areas
|
121
|
+
if include_deposit && deposit_trunk
|
122
|
+
branch = deposit_branch(object_id)
|
123
|
+
storage_roots.each do |root|
|
124
|
+
root_trunk = root.join(deposit_trunk)
|
125
|
+
raise(MoabRuntimeError, "Deposit area not found at #{root_trunk}") unless root_trunk.exist?
|
126
|
+
|
127
|
+
root_trunk_branch = root_trunk.join(branch)
|
128
|
+
storage_objects << create_storage_object(object_id, root) if root_trunk_branch.exist?
|
129
|
+
end
|
130
|
+
end
|
131
|
+
storage_objects
|
100
132
|
end
|
101
133
|
|
102
134
|
# @param object_id [String] The identifier of the digital object whose size is desired
|
@@ -118,6 +150,7 @@ module Moab
|
|
118
150
|
storage_object = find_storage_object(object_id)
|
119
151
|
unless storage_object.object_pathname.exist?
|
120
152
|
raise ObjectNotFoundException, "No storage object found for #{object_id}" unless create
|
153
|
+
|
121
154
|
storage_object.object_pathname.mkpath
|
122
155
|
end
|
123
156
|
storage_object
|
@@ -127,7 +160,16 @@ module Moab
|
|
127
160
|
# @param druid [String] The object identifier
|
128
161
|
# @return [void] transfer the object to the preservation repository
|
129
162
|
def store_new_object_version(druid, bag_pathname)
|
130
|
-
storage_object(druid,
|
163
|
+
storage_object(druid, true).ingest_bag(bag_pathname)
|
164
|
+
end
|
165
|
+
|
166
|
+
private
|
167
|
+
|
168
|
+
def create_storage_object(object_id, root)
|
169
|
+
storage_pathname = root.join(storage_trunk, storage_branch(object_id))
|
170
|
+
storage_object = StorageObject.new(object_id, storage_pathname)
|
171
|
+
storage_object.storage_root = root
|
172
|
+
storage_object
|
131
173
|
end
|
132
174
|
end
|
133
175
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Moab
|
2
4
|
# An interface class to support access to SDR storage via a RESTful server
|
3
5
|
#
|
@@ -41,6 +43,13 @@ module Moab
|
|
41
43
|
@@repository.find_storage_object(object_id, include_deposit)
|
42
44
|
end
|
43
45
|
|
46
|
+
# @param object_id [String] The identifier of the digital object
|
47
|
+
# @param [Object] include_deposit
|
48
|
+
# @return [Array<StorageObject>] Representations of a digitial object's storage directories, or an empty array if none found.
|
49
|
+
def self.search_storage_objects(object_id, include_deposit = false)
|
50
|
+
@@repository.search_storage_objects(object_id, include_deposit)
|
51
|
+
end
|
52
|
+
|
44
53
|
# @param object_id [String] The identifier of the digital object whose size is desired
|
45
54
|
# @param include_deposit [Boolean] specifies whether to look in deposit areas for objects in process of initial ingest
|
46
55
|
# @return [Integer] the size occupied on disk by the storage object, in bytes. this is the entire moab (all versions).
|
@@ -74,12 +83,6 @@ module Moab
|
|
74
83
|
@@repository.storage_object(object_id).current_version_id
|
75
84
|
end
|
76
85
|
|
77
|
-
# @param [String] object_id The digital object identifier of the object
|
78
|
-
# @return [Pathname] Pathname object containing the full path for the specified file
|
79
|
-
def self.version_metadata(object_id)
|
80
|
-
retrieve_file('metadata', 'versionMetadata.xml', object_id)
|
81
|
-
end
|
82
|
-
|
83
86
|
# @param [String] object_id The digital object identifier of the object
|
84
87
|
# @param [Integer] version_id The ID of the version, if nil use latest version
|
85
88
|
# @return [FileInventory] the file inventory for the specified object version
|
@@ -101,7 +104,7 @@ module Moab
|
|
101
104
|
# @return [Pathname] Pathname object containing the full path for the specified file
|
102
105
|
def self.retrieve_file(file_category, file_id, object_id, version_id = nil)
|
103
106
|
storage_object_version = @@repository.storage_object(object_id).find_object_version(version_id)
|
104
|
-
|
107
|
+
storage_object_version.find_filepath(file_category, file_id)
|
105
108
|
end
|
106
109
|
|
107
110
|
# @param [String] file_category The category of file ('content', 'metdata', or 'manifest')
|
@@ -111,7 +114,7 @@ module Moab
|
|
111
114
|
# @return [Pathname] Pathname object containing the full path for the specified file
|
112
115
|
def self.retrieve_file_using_signature(file_category, file_signature, object_id, version_id = nil)
|
113
116
|
storage_object_version = @@repository.storage_object(object_id).find_object_version(version_id)
|
114
|
-
|
117
|
+
storage_object_version.find_filepath_using_signature(file_category, file_signature)
|
115
118
|
end
|
116
119
|
|
117
120
|
# @param [String] file_category The category of file ('content', 'metdata', or 'manifest')
|
@@ -121,7 +124,7 @@ module Moab
|
|
121
124
|
# @return [FileSignature] The signature of the file
|
122
125
|
def self.retrieve_file_signature(file_category, file_id, object_id, version_id = nil)
|
123
126
|
storage_object_version = @@repository.storage_object(object_id).find_object_version(version_id)
|
124
|
-
|
127
|
+
storage_object_version.find_signature(file_category, file_id)
|
125
128
|
end
|
126
129
|
|
127
130
|
# @param [String] object_id The digital object identifier of the object
|
data/lib/moab/utc_time.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Moab
|
2
4
|
# A recursive "Tree" type object for verifications
|
3
5
|
class VerificationResult
|
@@ -52,10 +54,9 @@ module Moab
|
|
52
54
|
# @return [Hash] The verification result serialized to a hash
|
53
55
|
def to_hash(verbose = false, level = 0)
|
54
56
|
hash = { 'verified' => verified }
|
55
|
-
if verbose || !verified
|
56
|
-
hash['details'] = details || subentities_to_hash(verbose, level)
|
57
|
-
end
|
57
|
+
hash['details'] = details || subentities_to_hash(verbose, level) if verbose || !verified
|
58
58
|
return hash if level > 0
|
59
|
+
|
59
60
|
{ entity => hash }
|
60
61
|
end
|
61
62
|
|