moab-versioning 4.3.0 → 5.0.0.beta1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/moab/bagger.rb +7 -2
- data/lib/moab/config.rb +40 -7
- data/lib/moab/exceptions.rb +6 -0
- data/lib/moab/file_group.rb +12 -9
- data/lib/moab/file_group_difference.rb +26 -23
- data/lib/moab/file_group_difference_subset.rb +5 -3
- data/lib/moab/file_instance.rb +4 -1
- data/lib/moab/file_instance_difference.rb +5 -3
- data/lib/moab/file_inventory.rb +13 -9
- data/lib/moab/file_inventory_difference.rb +8 -6
- data/lib/moab/file_manifestation.rb +5 -2
- data/lib/moab/file_signature.rb +12 -7
- data/lib/moab/signature_catalog.rb +13 -13
- data/lib/moab/signature_catalog_entry.rb +6 -4
- data/lib/moab/stanford.rb +2 -10
- data/lib/moab/storage_object.rb +11 -5
- data/lib/moab/storage_object_validator.rb +24 -10
- data/lib/moab/storage_object_version.rb +19 -12
- data/lib/moab/storage_repository.rb +49 -7
- data/lib/moab/storage_services.rb +12 -9
- data/lib/moab/utc_time.rb +2 -0
- data/lib/moab/verification_result.rb +4 -3
- data/lib/moab/version_metadata_entry.rb +6 -4
- data/lib/moab.rb +2 -9
- data/lib/serializer/manifest.rb +4 -2
- data/lib/serializer/serializable.rb +6 -1
- data/lib/serializer.rb +2 -0
- data/lib/stanford/content_inventory.rb +23 -19
- data/lib/stanford/storage_object_validator.rb +2 -0
- data/lib/stanford/storage_repository.rb +6 -2
- data/lib/stanford/storage_services.rb +2 -0
- metadata +22 -42
- data/lib/moab/deposit_bag_validator.rb +0 -323
- data/lib/moab/version_metadata.rb +0 -32
- data/lib/moab/version_metadata_event.rb +0 -40
- data/lib/stanford/active_fedora_object.rb +0 -28
- data/lib/stanford/dor_metadata.rb +0 -41
- data/lib/stanford/moab_storage_directory.rb +0 -36
data/lib/moab/file_signature.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Moab
|
2
4
|
# The fixity properties of a file, used to determine file content equivalence regardless of filename.
|
3
5
|
# Placing this data in a class by itself facilitates using file size together with the MD5 and SHA1 checksums
|
@@ -44,19 +46,19 @@ module Moab
|
|
44
46
|
|
45
47
|
# @attribute
|
46
48
|
# @return [Integer] The size of the file in bytes
|
47
|
-
attribute :size, Integer, :
|
49
|
+
attribute :size, Integer, on_save: proc { |n| n.to_s }
|
48
50
|
|
49
51
|
# @attribute
|
50
52
|
# @return [String] The MD5 checksum value of the file
|
51
|
-
attribute :md5, String, :
|
53
|
+
attribute :md5, String, on_save: proc { |n| n.nil? ? "" : n.to_s }
|
52
54
|
|
53
55
|
# @attribute
|
54
56
|
# @return [String] The SHA1 checksum value of the file
|
55
|
-
attribute :sha1, String, :
|
57
|
+
attribute :sha1, String, on_save: proc { |n| n.nil? ? "" : n.to_s }
|
56
58
|
|
57
59
|
# @attribute
|
58
60
|
# @return [String] The SHA256 checksum value of the file
|
59
|
-
attribute :sha256, String, :
|
61
|
+
attribute :sha256, String, on_save: proc { |n| n.nil? ? "" : n.to_s }
|
60
62
|
|
61
63
|
KNOWN_ALGOS = {
|
62
64
|
md5: proc { Digest::MD5.new },
|
@@ -83,7 +85,7 @@ module Moab
|
|
83
85
|
end
|
84
86
|
end
|
85
87
|
|
86
|
-
new(signatures.
|
88
|
+
new(signatures.transform_values(&:hexdigest).merge(size: pathname.size))
|
87
89
|
end
|
88
90
|
|
89
91
|
# @param type [Symbol,String] The type of checksum
|
@@ -128,10 +130,12 @@ module Moab
|
|
128
130
|
def eql?(other)
|
129
131
|
return false unless other.respond_to?(:size) && other.respond_to?(:checksums)
|
130
132
|
return false if size.to_i != other.size.to_i
|
133
|
+
|
131
134
|
self_checksums = checksums
|
132
135
|
other_checksums = other.checksums
|
133
136
|
matching_keys = self_checksums.keys & other_checksums.keys
|
134
137
|
return false if matching_keys.empty?
|
138
|
+
|
135
139
|
matching_keys.each do |key|
|
136
140
|
return false if self_checksums[key] != other_checksums[key]
|
137
141
|
end
|
@@ -176,6 +180,7 @@ module Moab
|
|
176
180
|
def normalized_signature(pathname)
|
177
181
|
sig_from_file = FileSignature.new.signature_from_file(pathname)
|
178
182
|
return sig_from_file if eql?(sig_from_file)
|
183
|
+
|
179
184
|
# The full signature from file is consistent with current values, or...
|
180
185
|
# One or more of the fixity values is inconsistent, so raise an exception
|
181
186
|
raise(MoabRuntimeError, "Signature inconsistent between inventory and file for #{pathname}: #{diff(sig_from_file).inspect}")
|
@@ -185,8 +190,8 @@ module Moab
|
|
185
190
|
def self.checksum_names_for_type
|
186
191
|
{
|
187
192
|
md5: ['MD5'],
|
188
|
-
sha1: [
|
189
|
-
sha256: [
|
193
|
+
sha1: %w[SHA-1 SHA1],
|
194
|
+
sha256: %w[SHA-256 SHA256]
|
190
195
|
}
|
191
196
|
end
|
192
197
|
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Moab
|
2
4
|
# A digital object's Signature Catalog is derived from an filtered aggregation of the file inventories
|
3
5
|
# of a digital object's set of versions. (see {#update})
|
@@ -37,20 +39,20 @@ module Moab
|
|
37
39
|
|
38
40
|
# @attribute
|
39
41
|
# @return [String] The object ID (druid)
|
40
|
-
attribute :digital_object_id, String, :
|
42
|
+
attribute :digital_object_id, String, tag: 'objectId'
|
41
43
|
|
42
44
|
# @attribute
|
43
45
|
# @return [Integer] The ordinal version number
|
44
|
-
attribute :version_id, Integer, :
|
46
|
+
attribute :version_id, Integer, tag: 'versionId', key: true, on_save: proc { |n| n.to_s }
|
45
47
|
|
46
48
|
# @return [String] The unique identifier concatenating digital object id with version id
|
47
49
|
def composite_key
|
48
|
-
@digital_object_id
|
50
|
+
"#{@digital_object_id}-#{StorageObject.version_dirname(@version_id)}"
|
49
51
|
end
|
50
52
|
|
51
53
|
# @attribute
|
52
54
|
# @return [String] The datetime at which the catalog was updated
|
53
|
-
attribute :catalog_datetime, Time, :
|
55
|
+
attribute :catalog_datetime, Time, tag: 'catalogDatetime'
|
54
56
|
|
55
57
|
def catalog_datetime=(datetime)
|
56
58
|
@catalog_datetime = Moab::UtcTime.input(datetime)
|
@@ -62,7 +64,7 @@ module Moab
|
|
62
64
|
|
63
65
|
# @attribute
|
64
66
|
# @return [Integer] The total number of data files (dynamically calculated)
|
65
|
-
attribute :file_count, Integer, :
|
67
|
+
attribute :file_count, Integer, tag: 'fileCount', on_save: proc { |t| t.to_s }
|
66
68
|
|
67
69
|
def file_count
|
68
70
|
entries.size
|
@@ -70,7 +72,7 @@ module Moab
|
|
70
72
|
|
71
73
|
# @attribute
|
72
74
|
# @return [Integer] The total size (in bytes) of all data files (dynamically calculated)
|
73
|
-
attribute :byte_count, Integer, :
|
75
|
+
attribute :byte_count, Integer, tag: 'byteCount', on_save: proc { |t| t.to_s }
|
74
76
|
|
75
77
|
def byte_count
|
76
78
|
entries.inject(0) { |sum, entry| sum + entry.signature.size.to_i }
|
@@ -78,7 +80,7 @@ module Moab
|
|
78
80
|
|
79
81
|
# @attribute
|
80
82
|
# @return [Integer] The total disk usage (in 1 kB blocks) of all data files (estimating du -k result) (dynamically calculated)
|
81
|
-
attribute :block_count, Integer, :
|
83
|
+
attribute :block_count, Integer, tag: 'blockCount', on_save: proc { |t| t.to_s }
|
82
84
|
|
83
85
|
def block_count
|
84
86
|
block_size = 1024
|
@@ -92,7 +94,7 @@ module Moab
|
|
92
94
|
|
93
95
|
# @attribute
|
94
96
|
# @return [Array<SignatureCatalogEntry>] The set of data groups comprising the version
|
95
|
-
has_many :entries, SignatureCatalogEntry, :
|
97
|
+
has_many :entries, SignatureCatalogEntry, tag: 'entry'
|
96
98
|
|
97
99
|
def entries=(entry_array)
|
98
100
|
entry_array.each do |entry|
|
@@ -176,14 +178,12 @@ module Moab
|
|
176
178
|
# containing only those files that were added in this version
|
177
179
|
# @example {include:file:spec/features/catalog/version_additions_spec.rb}
|
178
180
|
def version_additions(version_inventory)
|
179
|
-
version_additions = FileInventory.new(:
|
181
|
+
version_additions = FileInventory.new(type: 'additions')
|
180
182
|
version_additions.copy_ids(version_inventory)
|
181
183
|
version_inventory.groups.each do |group|
|
182
|
-
group_addtions = FileGroup.new(:
|
184
|
+
group_addtions = FileGroup.new(group_id: group.group_id)
|
183
185
|
group.files.each do |file|
|
184
|
-
unless @signature_hash.key?(file.signature)
|
185
|
-
group_addtions.add_file_instance(file.signature, file.instances[0])
|
186
|
-
end
|
186
|
+
group_addtions.add_file_instance(file.signature, file.instances[0]) unless @signature_hash.key?(file.signature)
|
187
187
|
end
|
188
188
|
version_additions.groups << group_addtions unless group_addtions.files.empty?
|
189
189
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Moab
|
2
4
|
# A file-level entry in a digital object's {SignatureCatalog}.
|
3
5
|
# It has a child {FileSignature} element that identifies the file's contents (the bytestream)
|
@@ -23,19 +25,19 @@ module Moab
|
|
23
25
|
|
24
26
|
# @attribute
|
25
27
|
# @return [Integer] The ordinal version number
|
26
|
-
attribute :version_id, Integer, :
|
28
|
+
attribute :version_id, Integer, tag: 'originalVersion', key: true, on_save: proc { |n| n.to_s }
|
27
29
|
|
28
30
|
# @attribute
|
29
31
|
# @return [String] The name of the file group
|
30
|
-
attribute :group_id, String, :
|
32
|
+
attribute :group_id, String, tag: 'groupId', key: true
|
31
33
|
|
32
34
|
# @attribute
|
33
35
|
# @return [String] The id is the filename path, relative to the file group's base directory
|
34
|
-
attribute :path, String, :
|
36
|
+
attribute :path, String, key: true, tag: 'storagePath'
|
35
37
|
|
36
38
|
# @attribute
|
37
39
|
# @return [FileSignature] The fixity data of the file instance
|
38
|
-
element :signature, FileSignature, :
|
40
|
+
element :signature, FileSignature, tag: 'fileSignature'
|
39
41
|
|
40
42
|
def signature
|
41
43
|
# HappyMapper's parser tries to put an array of signatures in the signature field
|
data/lib/moab/stanford.rb
CHANGED
@@ -1,19 +1,11 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'moab'
|
2
4
|
require 'stanford/content_inventory'
|
3
|
-
require 'stanford/dor_metadata'
|
4
5
|
require 'stanford/storage_repository'
|
5
6
|
require 'stanford/storage_services'
|
6
|
-
require 'stanford/active_fedora_object'
|
7
|
-
require 'stanford/moab_storage_directory'
|
8
7
|
require 'stanford/storage_object_validator'
|
9
8
|
|
10
9
|
# Stanford is a module that isolates classes specific to the Stanford Digital Repository
|
11
|
-
#
|
12
|
-
# ====Data Model
|
13
|
-
# * <b>{DorMetadata} = utility methods for interfacing with Stanford metadata files (esp contentMetadata)</b>
|
14
|
-
# * {ActiveFedoraObject} [1..*] = utility for extracting content or other information from a Fedora Instance
|
15
|
-
#
|
16
|
-
# @note Copyright (c) 2012 by The Board of Trustees of the Leland Stanford Junior University.
|
17
|
-
# All rights reserved. See {file:LICENSE.rdoc} for details.
|
18
10
|
module Stanford
|
19
11
|
end
|
data/lib/moab/storage_object.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Moab
|
2
4
|
# A class to represent a digital object's repository storage location
|
3
5
|
# and methods for
|
@@ -82,10 +84,10 @@ module Moab
|
|
82
84
|
# @return [FileInventory] The file inventory of the specified type for this version
|
83
85
|
def versionize_bag(bag_dir, current_version, new_version)
|
84
86
|
new_inventory = FileInventory.new(
|
85
|
-
:
|
86
|
-
:
|
87
|
-
:
|
88
|
-
:
|
87
|
+
type: 'version',
|
88
|
+
digital_object_id: @digital_object_id,
|
89
|
+
version_id: new_version.version_id,
|
90
|
+
inventory_datetime: Time.now
|
89
91
|
)
|
90
92
|
new_inventory.inventory_from_bagit_bag(bag_dir)
|
91
93
|
new_inventory.write_xml_file(bag_dir)
|
@@ -113,6 +115,7 @@ module Moab
|
|
113
115
|
storage_filepath = @object_pathname.join(catalog_filepath)
|
114
116
|
errmsg = "#{catalog_filepath} missing from storage location #{storage_filepath}"
|
115
117
|
raise FileNotFoundException, errmsg unless storage_filepath.exist?
|
118
|
+
|
116
119
|
storage_filepath
|
117
120
|
end
|
118
121
|
|
@@ -127,9 +130,10 @@ module Moab
|
|
127
130
|
def version_id_list
|
128
131
|
list = []
|
129
132
|
return list unless @object_pathname.exist?
|
133
|
+
|
130
134
|
@object_pathname.children.each do |dirname|
|
131
135
|
vnum = dirname.basename.to_s
|
132
|
-
list << vnum[1
|
136
|
+
list << vnum[1..].to_i if vnum =~ /^v(\d+)$/
|
133
137
|
end
|
134
138
|
list.sort
|
135
139
|
end
|
@@ -163,6 +167,7 @@ module Moab
|
|
163
167
|
if version_inventory.version_id != (current_version_id + 1)
|
164
168
|
raise(MoabRuntimeError, "version mismatch - current: #{current_version_id} new: #{version_inventory.version_id}")
|
165
169
|
end
|
170
|
+
|
166
171
|
true
|
167
172
|
end
|
168
173
|
|
@@ -188,6 +193,7 @@ module Moab
|
|
188
193
|
# * Current version + 1 is used for creation of a new version
|
189
194
|
def storage_object_version(version_id)
|
190
195
|
raise(MoabRuntimeError, "Version ID not specified") unless version_id
|
196
|
+
|
191
197
|
StorageObjectVersion.new(self, version_id)
|
192
198
|
end
|
193
199
|
|
@@ -1,15 +1,17 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'set'
|
2
4
|
|
3
5
|
module Moab
|
4
6
|
# Given a druid path, are the contents actually a well-formed Moab?
|
5
7
|
# Shameless green: repetitious code included.
|
6
8
|
class StorageObjectValidator
|
7
|
-
METADATA_DIR = "metadata"
|
8
|
-
CONTENT_DIR = "content"
|
9
|
+
METADATA_DIR = "metadata"
|
10
|
+
CONTENT_DIR = "content"
|
9
11
|
EXPECTED_DATA_SUB_DIRS = [CONTENT_DIR, METADATA_DIR].freeze
|
10
12
|
IMPLICIT_DIRS = ['.', '..'].freeze # unlike Find.find, Dir.entries returns the current/parent dirs
|
11
|
-
DATA_DIR = "data"
|
12
|
-
MANIFESTS_DIR = 'manifests'
|
13
|
+
DATA_DIR = "data"
|
14
|
+
MANIFESTS_DIR = 'manifests'
|
13
15
|
EXPECTED_VERSION_SUB_DIRS = [DATA_DIR, MANIFESTS_DIR].freeze
|
14
16
|
MANIFEST_INVENTORY_PATH = File.join(MANIFESTS_DIR, "manifestInventory.xml").freeze
|
15
17
|
SIGNATURE_CATALOG_PATH = File.join(MANIFESTS_DIR, "signatureCatalog.xml").freeze
|
@@ -83,13 +85,14 @@ module Moab
|
|
83
85
|
end
|
84
86
|
|
85
87
|
def version_dir_format?(dirname)
|
86
|
-
dirname =~ /^
|
88
|
+
dirname =~ /^v\d{4}$/
|
87
89
|
end
|
88
90
|
|
89
91
|
# call only if the version directories are "correctly named" vdddd
|
90
92
|
def check_sequential_version_dirs
|
91
93
|
version_directories.each_with_index do |dir_name, index|
|
92
|
-
next if dir_name[1
|
94
|
+
next if dir_name[1..].to_i == index + 1 # version numbering starts at 1, array indexing at 0
|
95
|
+
|
93
96
|
return [result_hash(VERSIONS_NOT_IN_ORDER, version_directories)]
|
94
97
|
end
|
95
98
|
[]
|
@@ -113,6 +116,7 @@ module Moab
|
|
113
116
|
return expected_version_sub_dirs(version_path, version) if count == EXPECTED_VERSION_SUB_DIRS.size
|
114
117
|
return found_unexpected(version_sub_dirs, version, EXPECTED_VERSION_SUB_DIRS) if count > EXPECTED_VERSION_SUB_DIRS.size
|
115
118
|
return missing_dir(version_sub_dirs, version, EXPECTED_VERSION_SUB_DIRS) if count < EXPECTED_VERSION_SUB_DIRS.size
|
119
|
+
|
116
120
|
[]
|
117
121
|
end
|
118
122
|
|
@@ -122,15 +126,20 @@ module Moab
|
|
122
126
|
data_sub_dirs = directory_entries(data_dir_path)
|
123
127
|
errors.concat check_data_sub_dirs(version, data_sub_dirs)
|
124
128
|
errors.concat check_metadata_dir_files_only(version_path) if errors.empty?
|
125
|
-
|
129
|
+
if data_sub_dirs.include?('content') && errors.empty?
|
130
|
+
errors.concat check_optional_content_dir(version_path, allow_content_subdirs)
|
131
|
+
end
|
126
132
|
errors
|
127
133
|
end
|
128
134
|
|
129
135
|
def check_data_sub_dirs(version, data_sub_dirs)
|
130
136
|
return found_unexpected(data_sub_dirs, version, EXPECTED_DATA_SUB_DIRS) if data_sub_dirs.size > EXPECTED_DATA_SUB_DIRS.size
|
137
|
+
|
131
138
|
errors = []
|
132
139
|
errors.concat missing_dir(data_sub_dirs, version, [METADATA_DIR]) unless data_sub_dirs.include?(METADATA_DIR)
|
133
|
-
|
140
|
+
unless data_sub_dirs.to_set.subset?(EXPECTED_DATA_SUB_DIRS.to_set)
|
141
|
+
errors.concat found_unexpected(data_sub_dirs, version, EXPECTED_DATA_SUB_DIRS)
|
142
|
+
end
|
134
143
|
errors
|
135
144
|
end
|
136
145
|
|
@@ -139,7 +148,9 @@ module Moab
|
|
139
148
|
content_dir_path = File.join(version_path, DATA_DIR, CONTENT_DIR)
|
140
149
|
errors << result_hash(NO_FILES_IN_CONTENT_DIR, basename(version_path)) if directory_entries(content_dir_path).empty?
|
141
150
|
content_sub_dir = contains_sub_dir?(content_dir_path)
|
142
|
-
|
151
|
+
if content_sub_dir && !allow_content_subdirs
|
152
|
+
errors << result_hash(CONTENT_SUB_DIRS_DETECTED, version: basename(version_path), dir: content_sub_dir)
|
153
|
+
end
|
143
154
|
if allow_content_subdirs && contains_sub_dir?(content_dir_path) && contains_forbidden_content_sub_dir?(content_dir_path)
|
144
155
|
errors << result_hash(BAD_SUB_DIR_IN_CONTENT_DIR, basename(version_path))
|
145
156
|
end
|
@@ -159,7 +170,9 @@ module Moab
|
|
159
170
|
metadata_dir_path = File.join(version_path, DATA_DIR, METADATA_DIR)
|
160
171
|
errors << result_hash(NO_FILES_IN_METADATA_DIR, basename(version_path)) if directory_entries(metadata_dir_path).empty?
|
161
172
|
metadata_sub_dir = contains_sub_dir?(metadata_dir_path)
|
162
|
-
|
173
|
+
if metadata_sub_dir
|
174
|
+
errors << result_hash(METADATA_SUB_DIRS_DETECTED, version: basename(version_path), dir: metadata_sub_dir)
|
175
|
+
end
|
163
176
|
errors
|
164
177
|
end
|
165
178
|
|
@@ -228,6 +241,7 @@ module Moab
|
|
228
241
|
|
229
242
|
def check_required_manifest_files(dir, version)
|
230
243
|
return [result_hash(NO_FILES_IN_MANIFEST_DIR, version)] unless contains_file?(File.join(dir, MANIFESTS_DIR))
|
244
|
+
|
231
245
|
errors = []
|
232
246
|
errors << result_hash(NO_MANIFEST_INVENTORY, version) unless File.exist?(File.join(dir, MANIFEST_INVENTORY_PATH))
|
233
247
|
errors << result_hash(NO_SIGNATURE_CATALOG, version) unless File.exist?(File.join(dir, SIGNATURE_CATALOG_PATH))
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Moab
|
2
4
|
# A class to represent a version subdirectory within an object's home directory in preservation storage
|
3
5
|
# ====Data Model
|
@@ -43,7 +45,7 @@ module Moab
|
|
43
45
|
|
44
46
|
# @return [String] The unique identifier concatenating digital object id with version id
|
45
47
|
def composite_key
|
46
|
-
@storage_object.digital_object_id
|
48
|
+
"#{@storage_object.digital_object_id}-#{StorageObject.version_dirname(@version_id)}"
|
47
49
|
end
|
48
50
|
|
49
51
|
# @return [Boolean] true if the object version directory exists
|
@@ -68,6 +70,7 @@ module Moab
|
|
68
70
|
def find_filepath(file_category, file_id)
|
69
71
|
this_version_filepath = file_pathname(file_category, file_id)
|
70
72
|
return this_version_filepath if this_version_filepath.exist?
|
73
|
+
|
71
74
|
if file_category == 'manifest'
|
72
75
|
msg = "manifest file #{file_id} not found for #{@storage_object.digital_object_id} - #{@version_id}"
|
73
76
|
raise FileNotFoundException, msg
|
@@ -109,14 +112,15 @@ module Moab
|
|
109
112
|
def file_inventory(type)
|
110
113
|
if version_id > 0
|
111
114
|
return @inventory_cache[type] if @inventory_cache.key?(type)
|
115
|
+
|
112
116
|
@inventory_cache[type] = FileInventory.read_xml_file(@version_pathname.join('manifests'), type)
|
113
117
|
else
|
114
|
-
groups = %w[content metadata].collect { |id| FileGroup.new(:
|
118
|
+
groups = %w[content metadata].collect { |id| FileGroup.new(group_id: id) }
|
115
119
|
FileInventory.new(
|
116
|
-
:
|
117
|
-
:
|
118
|
-
:
|
119
|
-
:
|
120
|
+
type: 'version',
|
121
|
+
digital_object_id: @storage_object.digital_object_id,
|
122
|
+
version_id: @version_id,
|
123
|
+
groups: groups
|
120
124
|
)
|
121
125
|
end
|
122
126
|
end
|
@@ -127,7 +131,7 @@ module Moab
|
|
127
131
|
if version_id > 0
|
128
132
|
SignatureCatalog.read_xml_file(@version_pathname.join('manifests'))
|
129
133
|
else
|
130
|
-
SignatureCatalog.new(:
|
134
|
+
SignatureCatalog.new(digital_object_id: @storage_object.digital_object_id)
|
131
135
|
end
|
132
136
|
end
|
133
137
|
|
@@ -136,6 +140,7 @@ module Moab
|
|
136
140
|
# @return [void] Create the version subdirectory and move files into it
|
137
141
|
def ingest_bag_data(bag_dir)
|
138
142
|
raise(MoabRuntimeError, "Version already exists: #{@version_pathname}") if @version_pathname.exist?
|
143
|
+
|
139
144
|
@version_pathname.join('manifests').mkpath
|
140
145
|
bag_dir = Pathname(bag_dir)
|
141
146
|
ingest_dir(bag_dir.join('data'), @version_pathname.join('data'))
|
@@ -150,6 +155,7 @@ module Moab
|
|
150
155
|
# @return [void] recursively link or copy the source directory contents to the target directory
|
151
156
|
def ingest_dir(source_dir, target_dir, use_links = true)
|
152
157
|
raise(MoabRuntimeError, "cannot copy - target already exists: #{target_dir.expand_path}") if target_dir.exist?
|
158
|
+
|
153
159
|
target_dir.mkpath
|
154
160
|
source_dir.children.each do |child|
|
155
161
|
if child.directory?
|
@@ -196,12 +202,12 @@ module Moab
|
|
196
202
|
# @return [void] examine the version's directory and create/serialize a {FileInventory} containing the manifest files
|
197
203
|
def generate_manifest_inventory
|
198
204
|
manifest_inventory = FileInventory.new(
|
199
|
-
:
|
200
|
-
:
|
201
|
-
:
|
205
|
+
type: 'manifests',
|
206
|
+
digital_object_id: @storage_object.digital_object_id,
|
207
|
+
version_id: @version_id
|
202
208
|
)
|
203
209
|
pathname = @version_pathname.join('manifests')
|
204
|
-
manifest_inventory.groups << FileGroup.new(:
|
210
|
+
manifest_inventory.groups << FileGroup.new(group_id: 'manifests').group_from_directory(pathname, false)
|
205
211
|
manifest_inventory.write_xml_file(pathname)
|
206
212
|
end
|
207
213
|
|
@@ -306,7 +312,7 @@ module Moab
|
|
306
312
|
version_additions = file_inventory('additions')
|
307
313
|
result.subentities << VerificationResult.verify_value('composite_key', composite_key, version_additions.composite_key)
|
308
314
|
data_directory = @version_pathname.join('data')
|
309
|
-
directory_inventory = FileInventory.new(:
|
315
|
+
directory_inventory = FileInventory.new(type: 'directory').inventory_from_directory(data_directory)
|
310
316
|
diff = FileInventoryDifference.new
|
311
317
|
diff.compare(version_additions, directory_inventory)
|
312
318
|
compare_result = VerificationResult.new('file_differences')
|
@@ -321,6 +327,7 @@ module Moab
|
|
321
327
|
# @return [null] Deactivate this object version by moving it to another directory. (Used by restore operation)
|
322
328
|
def deactivate(timestamp)
|
323
329
|
return unless @version_pathname.exist?
|
330
|
+
|
324
331
|
timestamp_pathname = @version_pathname.parent.join(timestamp.utc.iso8601.gsub(/[-:]/, ''))
|
325
332
|
timestamp_pathname.mkpath
|
326
333
|
demote_pathame = timestamp_pathname.join(@version_pathname.basename)
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Moab
|
2
4
|
# A class to represent the SDR repository store
|
3
5
|
#
|
@@ -19,8 +21,10 @@ module Moab
|
|
19
21
|
def storage_roots
|
20
22
|
unless defined?(@storage_roots)
|
21
23
|
raise(MoabRuntimeError, "Moab::Config.storage_roots not found in config file") if Moab::Config.storage_roots.nil?
|
24
|
+
|
22
25
|
@storage_roots = [Moab::Config.storage_roots].flatten.collect { |filesystem| Pathname(filesystem) }
|
23
26
|
raise(MoabRuntimeError, "Moab::Config.storage_roots empty") if @storage_roots.empty?
|
27
|
+
|
24
28
|
@storage_roots.each { |root| raise(MoabRuntimeError, "Storage root #{root} not found on system") unless root.exist? }
|
25
29
|
end
|
26
30
|
@storage_roots
|
@@ -30,7 +34,8 @@ module Moab
|
|
30
34
|
def storage_trunk
|
31
35
|
unless defined?(@storage_trunk)
|
32
36
|
raise(MoabRuntimeError, "Moab::Config.storage_trunk not found in config file") if Moab::Config.storage_trunk.nil?
|
33
|
-
|
37
|
+
|
38
|
+
@storage_trunk = Moab::Config.storage_trunk
|
34
39
|
end
|
35
40
|
@storage_trunk
|
36
41
|
end
|
@@ -50,7 +55,7 @@ module Moab
|
|
50
55
|
unless defined?(@deposit_trunk)
|
51
56
|
# do not raise error. this parameter will be ignored if missing
|
52
57
|
# raise "Moab::Config.deposit_trunk not found in config file" if Moab::Config.deposit_trunk.nil?
|
53
|
-
@deposit_trunk
|
58
|
+
@deposit_trunk = Moab::Config.deposit_trunk
|
54
59
|
end
|
55
60
|
@deposit_trunk
|
56
61
|
end
|
@@ -71,6 +76,7 @@ module Moab
|
|
71
76
|
storage_roots.each do |root|
|
72
77
|
root_trunk = root.join(storage_trunk)
|
73
78
|
raise(MoabRuntimeError, "Storage area not found at #{root_trunk}") unless root_trunk.exist?
|
79
|
+
|
74
80
|
root_trunk_branch = root_trunk.join(branch)
|
75
81
|
return root if root_trunk_branch.exist?
|
76
82
|
end
|
@@ -80,6 +86,7 @@ module Moab
|
|
80
86
|
storage_roots.each do |root|
|
81
87
|
root_trunk = root.join(deposit_trunk)
|
82
88
|
raise(MoabRuntimeError, "Deposit area not found at #{root_trunk}") unless root_trunk.exist?
|
89
|
+
|
83
90
|
root_trunk_branch = root_trunk.join(branch)
|
84
91
|
return root if root_trunk_branch.exist?
|
85
92
|
end
|
@@ -93,10 +100,35 @@ module Moab
|
|
93
100
|
# @return [StorageObject] The representation of a digitial object's storage directory, which might not exist yet.
|
94
101
|
def find_storage_object(object_id, include_deposit = false)
|
95
102
|
root = find_storage_root(object_id, include_deposit)
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
103
|
+
create_storage_object(object_id, root)
|
104
|
+
end
|
105
|
+
|
106
|
+
# @param object_id [String] The identifier of the digital object
|
107
|
+
# @param include_deposit [Boolean] specifies whether to look in deposit areas for objects in process of initial ingest
|
108
|
+
# @return [Array<StorageObject>] Representations of a digitial object's storage directories, or an empty array if none found.
|
109
|
+
def search_storage_objects(object_id, include_deposit = false)
|
110
|
+
storage_objects = []
|
111
|
+
# Search for the object's home directory in the storage areas
|
112
|
+
branch = storage_branch(object_id)
|
113
|
+
storage_roots.each do |root|
|
114
|
+
root_trunk = root.join(storage_trunk)
|
115
|
+
raise(MoabRuntimeError, "Storage area not found at #{root_trunk}") unless root_trunk.exist?
|
116
|
+
|
117
|
+
root_trunk_branch = root_trunk.join(branch)
|
118
|
+
storage_objects << create_storage_object(object_id, root) if root_trunk_branch.exist?
|
119
|
+
end
|
120
|
+
# Search for the object's directory in the deposit areas
|
121
|
+
if include_deposit && deposit_trunk
|
122
|
+
branch = deposit_branch(object_id)
|
123
|
+
storage_roots.each do |root|
|
124
|
+
root_trunk = root.join(deposit_trunk)
|
125
|
+
raise(MoabRuntimeError, "Deposit area not found at #{root_trunk}") unless root_trunk.exist?
|
126
|
+
|
127
|
+
root_trunk_branch = root_trunk.join(branch)
|
128
|
+
storage_objects << create_storage_object(object_id, root) if root_trunk_branch.exist?
|
129
|
+
end
|
130
|
+
end
|
131
|
+
storage_objects
|
100
132
|
end
|
101
133
|
|
102
134
|
# @param object_id [String] The identifier of the digital object whose size is desired
|
@@ -118,6 +150,7 @@ module Moab
|
|
118
150
|
storage_object = find_storage_object(object_id)
|
119
151
|
unless storage_object.object_pathname.exist?
|
120
152
|
raise ObjectNotFoundException, "No storage object found for #{object_id}" unless create
|
153
|
+
|
121
154
|
storage_object.object_pathname.mkpath
|
122
155
|
end
|
123
156
|
storage_object
|
@@ -127,7 +160,16 @@ module Moab
|
|
127
160
|
# @param druid [String] The object identifier
|
128
161
|
# @return [void] transfer the object to the preservation repository
|
129
162
|
def store_new_object_version(druid, bag_pathname)
|
130
|
-
storage_object(druid,
|
163
|
+
storage_object(druid, true).ingest_bag(bag_pathname)
|
164
|
+
end
|
165
|
+
|
166
|
+
private
|
167
|
+
|
168
|
+
def create_storage_object(object_id, root)
|
169
|
+
storage_pathname = root.join(storage_trunk, storage_branch(object_id))
|
170
|
+
storage_object = StorageObject.new(object_id, storage_pathname)
|
171
|
+
storage_object.storage_root = root
|
172
|
+
storage_object
|
131
173
|
end
|
132
174
|
end
|
133
175
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Moab
|
2
4
|
# An interface class to support access to SDR storage via a RESTful server
|
3
5
|
#
|
@@ -41,6 +43,13 @@ module Moab
|
|
41
43
|
@@repository.find_storage_object(object_id, include_deposit)
|
42
44
|
end
|
43
45
|
|
46
|
+
# @param object_id [String] The identifier of the digital object
|
47
|
+
# @param [Object] include_deposit
|
48
|
+
# @return [Array<StorageObject>] Representations of a digitial object's storage directories, or an empty array if none found.
|
49
|
+
def self.search_storage_objects(object_id, include_deposit = false)
|
50
|
+
@@repository.search_storage_objects(object_id, include_deposit)
|
51
|
+
end
|
52
|
+
|
44
53
|
# @param object_id [String] The identifier of the digital object whose size is desired
|
45
54
|
# @param include_deposit [Boolean] specifies whether to look in deposit areas for objects in process of initial ingest
|
46
55
|
# @return [Integer] the size occupied on disk by the storage object, in bytes. this is the entire moab (all versions).
|
@@ -74,12 +83,6 @@ module Moab
|
|
74
83
|
@@repository.storage_object(object_id).current_version_id
|
75
84
|
end
|
76
85
|
|
77
|
-
# @param [String] object_id The digital object identifier of the object
|
78
|
-
# @return [Pathname] Pathname object containing the full path for the specified file
|
79
|
-
def self.version_metadata(object_id)
|
80
|
-
retrieve_file('metadata', 'versionMetadata.xml', object_id)
|
81
|
-
end
|
82
|
-
|
83
86
|
# @param [String] object_id The digital object identifier of the object
|
84
87
|
# @param [Integer] version_id The ID of the version, if nil use latest version
|
85
88
|
# @return [FileInventory] the file inventory for the specified object version
|
@@ -101,7 +104,7 @@ module Moab
|
|
101
104
|
# @return [Pathname] Pathname object containing the full path for the specified file
|
102
105
|
def self.retrieve_file(file_category, file_id, object_id, version_id = nil)
|
103
106
|
storage_object_version = @@repository.storage_object(object_id).find_object_version(version_id)
|
104
|
-
|
107
|
+
storage_object_version.find_filepath(file_category, file_id)
|
105
108
|
end
|
106
109
|
|
107
110
|
# @param [String] file_category The category of file ('content', 'metdata', or 'manifest')
|
@@ -111,7 +114,7 @@ module Moab
|
|
111
114
|
# @return [Pathname] Pathname object containing the full path for the specified file
|
112
115
|
def self.retrieve_file_using_signature(file_category, file_signature, object_id, version_id = nil)
|
113
116
|
storage_object_version = @@repository.storage_object(object_id).find_object_version(version_id)
|
114
|
-
|
117
|
+
storage_object_version.find_filepath_using_signature(file_category, file_signature)
|
115
118
|
end
|
116
119
|
|
117
120
|
# @param [String] file_category The category of file ('content', 'metdata', or 'manifest')
|
@@ -121,7 +124,7 @@ module Moab
|
|
121
124
|
# @return [FileSignature] The signature of the file
|
122
125
|
def self.retrieve_file_signature(file_category, file_id, object_id, version_id = nil)
|
123
126
|
storage_object_version = @@repository.storage_object(object_id).find_object_version(version_id)
|
124
|
-
|
127
|
+
storage_object_version.find_signature(file_category, file_id)
|
125
128
|
end
|
126
129
|
|
127
130
|
# @param [String] object_id The digital object identifier of the object
|
data/lib/moab/utc_time.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Moab
|
2
4
|
# A recursive "Tree" type object for verifications
|
3
5
|
class VerificationResult
|
@@ -52,10 +54,9 @@ module Moab
|
|
52
54
|
# @return [Hash] The verification result serialized to a hash
|
53
55
|
def to_hash(verbose = false, level = 0)
|
54
56
|
hash = { 'verified' => verified }
|
55
|
-
if verbose || !verified
|
56
|
-
hash['details'] = details || subentities_to_hash(verbose, level)
|
57
|
-
end
|
57
|
+
hash['details'] = details || subentities_to_hash(verbose, level) if verbose || !verified
|
58
58
|
return hash if level > 0
|
59
|
+
|
59
60
|
{ entity => hash }
|
60
61
|
end
|
61
62
|
|