sdr-client 0.4.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +6 -0
- data/.rubocop_todo.yml +0 -5
- data/exe/sdr +15 -1
- data/lib/sdr-client.rb +3 -0
- data/lib/sdr_client/cli.rb +3 -0
- data/lib/sdr_client/credentials.rb +4 -4
- data/lib/sdr_client/deposit.rb +15 -3
- data/lib/sdr_client/deposit/file.rb +29 -1
- data/lib/sdr_client/deposit/file_set.rb +12 -3
- data/lib/sdr_client/deposit/files/direct_upload_request.rb +5 -5
- data/lib/sdr_client/deposit/matching_file_grouping_strategy.rb +17 -0
- data/lib/sdr_client/deposit/metadata_builder.rb +42 -0
- data/lib/sdr_client/deposit/process.rb +33 -55
- data/lib/sdr_client/deposit/request.rb +3 -1
- data/lib/sdr_client/deposit/single_file_grouping_strategy.rb +14 -0
- data/lib/sdr_client/deposit/upload_files.rb +87 -0
- data/lib/sdr_client/version.rb +1 -1
- metadata +8 -4
- data/lib/sdr_client/deposit/default_file_set_builder.rb +0 -18
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: d4ac0ef485cd0ede90e6bcb2b99dcf001f92f3f296720520e6f1d4f5dcccef47
|
|
4
|
+
data.tar.gz: 52d7af6780779c41dcb67d87b9f87a139568132e612c2118ae5170617b023d06
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 48f9cb820ed8e23aada68d2bd65fe676b94941bb47e618832449df5bbcfe53ffb632fc961e1f5d3279d5fc9a8d6458dc32f28fd5b6e3ba240911cc50a9f137c8
|
|
7
|
+
data.tar.gz: 6ad9e1dd66c890a2ae771c95d53c3204872fc8c8fc3eb851863f021c1150748134a426b6aedd511b378a9bba29566d828c4eefa9b027462fedeeea0a170c9df7
|
data/.rubocop.yml
CHANGED
data/.rubocop_todo.yml
CHANGED
data/exe/sdr
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
|
|
4
4
|
$LOAD_PATH.unshift 'lib'
|
|
5
5
|
require 'optparse'
|
|
6
|
+
require 'sdr_client'
|
|
6
7
|
|
|
7
8
|
options = {}
|
|
8
9
|
global = OptionParser.new do |opts|
|
|
@@ -60,6 +61,20 @@ subcommands = {
|
|
|
60
61
|
options[:source_id] = source_id
|
|
61
62
|
end
|
|
62
63
|
|
|
64
|
+
opts.on('--strategy STRATEGY',
|
|
65
|
+
'The strategy to use for distributing files into filesets. Either "default" or "filename"') do |strategy|
|
|
66
|
+
strategy_class = case strategy
|
|
67
|
+
when 'filename'
|
|
68
|
+
SdrClient::Deposit::MatchingFileGroupingStrategy
|
|
69
|
+
when 'default'
|
|
70
|
+
SdrClient::Deposit::SingleFileGroupingStrategy
|
|
71
|
+
else
|
|
72
|
+
warn "Unknown strategy #{strategy}"
|
|
73
|
+
exit(1)
|
|
74
|
+
end
|
|
75
|
+
options[:grouping_strategy] = strategy_class
|
|
76
|
+
end
|
|
77
|
+
|
|
63
78
|
opts.on('-h', '--help', 'Display this screen') do
|
|
64
79
|
puts opts
|
|
65
80
|
exit
|
|
@@ -75,6 +90,5 @@ end
|
|
|
75
90
|
|
|
76
91
|
subcommands[command].order!
|
|
77
92
|
|
|
78
|
-
require 'sdr_client'
|
|
79
93
|
options[:files] = ARGV unless ARGV.empty?
|
|
80
94
|
SdrClient::CLI.start(command, options)
|
data/lib/sdr-client.rb
ADDED
data/lib/sdr_client/cli.rb
CHANGED
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
module SdrClient
|
|
4
4
|
# The stored credentials
|
|
5
5
|
class Credentials
|
|
6
|
+
class NoCredentialsError < StandardError; end
|
|
7
|
+
|
|
6
8
|
# @param [String] a json string that contains a field 'token'
|
|
7
9
|
def self.write(body)
|
|
8
10
|
json = JSON.parse(body)
|
|
@@ -10,14 +12,12 @@ module SdrClient
|
|
|
10
12
|
File.open(credentials_file, 'w', 0o600) do |file|
|
|
11
13
|
file.write(json.fetch('token'))
|
|
12
14
|
end
|
|
13
|
-
puts 'Signed in.'
|
|
14
15
|
end
|
|
15
16
|
|
|
16
17
|
def self.read
|
|
17
|
-
|
|
18
|
+
raise NoCredentialsError unless ::File.exist?(credentials_file)
|
|
18
19
|
|
|
19
|
-
|
|
20
|
-
exit(1)
|
|
20
|
+
IO.readlines(credentials_file, chomp: true).first
|
|
21
21
|
end
|
|
22
22
|
|
|
23
23
|
def self.credentials_path
|
data/lib/sdr_client/deposit.rb
CHANGED
|
@@ -1,15 +1,22 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require 'logger'
|
|
4
|
+
|
|
3
5
|
module SdrClient
|
|
4
6
|
# The namespace for the "deposit" command
|
|
5
7
|
module Deposit
|
|
8
|
+
# rubocop:disable Metrics/ParameterLists
|
|
6
9
|
def self.run(label: nil,
|
|
7
10
|
type: 'http://cocina.sul.stanford.edu/models/book.jsonld',
|
|
8
11
|
apo:,
|
|
9
12
|
collection:,
|
|
10
13
|
catkey: nil,
|
|
11
14
|
source_id:,
|
|
12
|
-
url:,
|
|
15
|
+
url:,
|
|
16
|
+
files: [],
|
|
17
|
+
files_metadata: {},
|
|
18
|
+
grouping_strategy: SingleFileGroupingStrategy,
|
|
19
|
+
logger: Logger.new(STDOUT))
|
|
13
20
|
token = Credentials.read
|
|
14
21
|
|
|
15
22
|
metadata = Request.new(label: label,
|
|
@@ -18,15 +25,20 @@ module SdrClient
|
|
|
18
25
|
collection: collection,
|
|
19
26
|
source_id: source_id,
|
|
20
27
|
catkey: catkey)
|
|
21
|
-
Process.new(metadata: metadata, url: url, token: token, files: files
|
|
28
|
+
Process.new(metadata: metadata, url: url, token: token, files: files,
|
|
29
|
+
files_metadata: files_metadata, grouping_strategy: grouping_strategy, logger: logger).run
|
|
22
30
|
end
|
|
31
|
+
# rubocop:enable Metrics/ParameterLists
|
|
23
32
|
end
|
|
24
33
|
end
|
|
25
34
|
require 'json'
|
|
26
|
-
require 'sdr_client/deposit/
|
|
35
|
+
require 'sdr_client/deposit/single_file_grouping_strategy'
|
|
36
|
+
require 'sdr_client/deposit/matching_file_grouping_strategy'
|
|
27
37
|
require 'sdr_client/deposit/files/direct_upload_request'
|
|
28
38
|
require 'sdr_client/deposit/files/direct_upload_response'
|
|
29
39
|
require 'sdr_client/deposit/file'
|
|
30
40
|
require 'sdr_client/deposit/file_set'
|
|
31
41
|
require 'sdr_client/deposit/request'
|
|
42
|
+
require 'sdr_client/deposit/upload_files'
|
|
43
|
+
require 'sdr_client/deposit/metadata_builder'
|
|
32
44
|
require 'sdr_client/deposit/process'
|
|
@@ -4,15 +4,23 @@ module SdrClient
|
|
|
4
4
|
module Deposit
|
|
5
5
|
# This represents the File metadata that we send to the server for doing a deposit
|
|
6
6
|
class File
|
|
7
|
-
|
|
7
|
+
# rubocop:disable Metrics/ParameterLists
|
|
8
|
+
def initialize(external_identifier:, label:, filename:,
|
|
9
|
+
access: 'dark', preserve: false, shelve: false,
|
|
10
|
+
mime_type: nil, md5: nil, sha1: nil)
|
|
8
11
|
@external_identifier = external_identifier
|
|
9
12
|
@label = label
|
|
10
13
|
@filename = filename
|
|
11
14
|
@access = access
|
|
12
15
|
@preserve = preserve
|
|
13
16
|
@shelve = shelve
|
|
17
|
+
@mime_type = mime_type
|
|
18
|
+
@md5 = md5
|
|
19
|
+
@sha1 = sha1
|
|
14
20
|
end
|
|
21
|
+
# rubocop:enable Metrics/ParameterLists
|
|
15
22
|
|
|
23
|
+
# rubocop:disable Metrics/MethodLength
|
|
16
24
|
def as_json
|
|
17
25
|
{
|
|
18
26
|
"type": 'http://cocina.sul.stanford.edu/models/file.jsonld',
|
|
@@ -26,6 +34,26 @@ module SdrClient
|
|
|
26
34
|
sdrPreserve: @preserve,
|
|
27
35
|
shelve: @shelve
|
|
28
36
|
}
|
|
37
|
+
}.tap do |json|
|
|
38
|
+
json['hasMessageDigests'] = message_digests unless message_digests.empty?
|
|
39
|
+
json['hasMimeType'] = @mime_type if @mime_type
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
# rubocop:enable Metrics/MethodLength
|
|
43
|
+
|
|
44
|
+
private
|
|
45
|
+
|
|
46
|
+
def message_digests
|
|
47
|
+
@message_digests ||= [].tap do |message_digests|
|
|
48
|
+
message_digests << create_message_digest('md5', @md5) unless @md5.nil?
|
|
49
|
+
message_digests << create_message_digest('sha1', @sha1) unless @sha1.nil?
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def create_message_digest(algorithm, digest)
|
|
54
|
+
{
|
|
55
|
+
"type": algorithm,
|
|
56
|
+
digest: digest
|
|
29
57
|
}
|
|
30
58
|
end
|
|
31
59
|
end
|
|
@@ -4,11 +4,11 @@ module SdrClient
|
|
|
4
4
|
module Deposit
|
|
5
5
|
# This represents the FileSet metadata that we send to the server for doing a deposit
|
|
6
6
|
class FileSet
|
|
7
|
-
def initialize(uploads: [], files: [], label:)
|
|
7
|
+
def initialize(uploads: [], uploads_metadata: {}, files: [], label:)
|
|
8
8
|
@label = label
|
|
9
9
|
@files = if !uploads.empty?
|
|
10
10
|
uploads.map do |upload|
|
|
11
|
-
File.new(
|
|
11
|
+
File.new(file_args(upload, uploads_metadata.fetch(upload.filename, {})))
|
|
12
12
|
end
|
|
13
13
|
else
|
|
14
14
|
files
|
|
@@ -20,7 +20,7 @@ module SdrClient
|
|
|
20
20
|
"type": 'http://cocina.sul.stanford.edu/models/fileset.jsonld',
|
|
21
21
|
"label": label,
|
|
22
22
|
structural: {
|
|
23
|
-
|
|
23
|
+
contains: files.map(&:as_json)
|
|
24
24
|
}
|
|
25
25
|
}
|
|
26
26
|
end
|
|
@@ -28,6 +28,15 @@ module SdrClient
|
|
|
28
28
|
private
|
|
29
29
|
|
|
30
30
|
attr_reader :files, :label
|
|
31
|
+
|
|
32
|
+
def file_args(upload, upload_metadata)
|
|
33
|
+
args = {
|
|
34
|
+
external_identifier: upload.signed_id,
|
|
35
|
+
label: upload.filename,
|
|
36
|
+
filename: upload.filename
|
|
37
|
+
}
|
|
38
|
+
args.merge(upload_metadata)
|
|
39
|
+
end
|
|
31
40
|
end
|
|
32
41
|
end
|
|
33
42
|
end
|
|
@@ -6,12 +6,12 @@ module SdrClient
|
|
|
6
6
|
module Deposit
|
|
7
7
|
module Files
|
|
8
8
|
DirectUploadRequest = Struct.new(:checksum, :byte_size, :content_type, :filename, keyword_init: true) do
|
|
9
|
-
def self.from_file(
|
|
10
|
-
checksum = Digest::MD5.file(
|
|
9
|
+
def self.from_file(path, file_name:, content_type:)
|
|
10
|
+
checksum = Digest::MD5.file(path).base64digest
|
|
11
11
|
new(checksum: checksum,
|
|
12
|
-
byte_size: ::File.size(
|
|
13
|
-
content_type: '
|
|
14
|
-
filename:
|
|
12
|
+
byte_size: ::File.size(path),
|
|
13
|
+
content_type: content_type || 'application/octet-stream',
|
|
14
|
+
filename: file_name)
|
|
15
15
|
end
|
|
16
16
|
|
|
17
17
|
def as_json
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SdrClient
|
|
4
|
+
module Deposit
|
|
5
|
+
# This strategy is for building one file set per set of similarly prefixed uploaded files
|
|
6
|
+
class MatchingFileGroupingStrategy
|
|
7
|
+
# @param [Array<SdrClient::Deposit::Files::DirectUploadResponse>] uploads the uploaded files to attach.
|
|
8
|
+
# @return [Array<Array<SdrClient::Deposit::Files::DirectUploadResponse>>] uploads the grouped uploaded files.
|
|
9
|
+
def self.run(uploads: [])
|
|
10
|
+
# Call `#values` on the result of the grouping operation because 1)
|
|
11
|
+
# `Process#build_filesets` expects an array of arrays, not an array of
|
|
12
|
+
# hashes, and 2) the keys aren't used anywhere
|
|
13
|
+
uploads.group_by { |ul| ::File.basename(ul.filename, '.*') }.values
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'logger'
|
|
4
|
+
|
|
5
|
+
module SdrClient
|
|
6
|
+
module Deposit
|
|
7
|
+
# Constructs the deposit metadata for the DRO
|
|
8
|
+
class MetadataBuilder
|
|
9
|
+
# @param [Request] metadata information about the object
|
|
10
|
+
# @param [Class] grouping_strategy class whose run method groups an array of uploads
|
|
11
|
+
# @param [Hash<String, Hash<String, String>>] files_metadata file name, hash of additional file metadata
|
|
12
|
+
# Additional metadata includes access, preserve, shelve, md5, sha1
|
|
13
|
+
# @param [Logger] logger the logger to use
|
|
14
|
+
def initialize(metadata:, grouping_strategy:, files_metadata:, logger:)
|
|
15
|
+
@metadata = metadata
|
|
16
|
+
@logger = logger
|
|
17
|
+
@grouping_strategy = grouping_strategy
|
|
18
|
+
@files_metadata = files_metadata
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def with_uploads(upload_responses)
|
|
22
|
+
file_sets = build_filesets(uploads: upload_responses)
|
|
23
|
+
metadata.with_file_sets(file_sets)
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
private
|
|
27
|
+
|
|
28
|
+
attr_reader :metadata, :files, :logger, :grouping_strategy, :files_metadata
|
|
29
|
+
|
|
30
|
+
# @param [Array<SdrClient::Deposit::Files::DirectUploadResponse>] uploads the uploaded files to attach.
|
|
31
|
+
# @return [Array<SdrClient::Deposit::FileSet>] the uploads transformed to filesets
|
|
32
|
+
def build_filesets(uploads:)
|
|
33
|
+
grouped_uploads = grouping_strategy.run(uploads: uploads)
|
|
34
|
+
grouped_uploads.map.with_index(1) do |upload_group, i|
|
|
35
|
+
metadata_group = {}
|
|
36
|
+
upload_group.each { |upload| metadata_group[upload.filename] = files_metadata.fetch(upload.filename, {}) }
|
|
37
|
+
FileSet.new(uploads: upload_group, uploads_metadata: metadata_group, label: "Object #{i}")
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
@@ -6,36 +6,43 @@ module SdrClient
|
|
|
6
6
|
module Deposit
|
|
7
7
|
# The process for doing a deposit
|
|
8
8
|
class Process
|
|
9
|
-
BLOB_PATH = '/v1/direct_uploads'
|
|
10
9
|
DRO_PATH = '/v1/resources'
|
|
11
10
|
# @param [Request] metadata information about the object
|
|
12
|
-
# @param [
|
|
11
|
+
# @param [Class] grouping_strategy class whose run method groups an array of uploads
|
|
13
12
|
# @param [String] url the server to send to
|
|
14
13
|
# @param [String] token the bearer auth token for the server
|
|
15
14
|
# @param [Array<String>] files a list of file names to upload
|
|
15
|
+
# @param [Hash<String, Hash<String, String>>] files_metadata file name, hash of additional file metadata
|
|
16
|
+
# Additional metadata includes access, preserve, shelve, md5, sha1
|
|
16
17
|
# @param [Logger] logger the logger to use
|
|
17
|
-
|
|
18
|
-
|
|
18
|
+
# rubocop:disable Metrics/ParameterLists
|
|
19
|
+
def initialize(metadata:, grouping_strategy: SingleFileGroupingStrategy, url:,
|
|
20
|
+
token:, files: [], files_metadata: {}, logger: Logger.new(STDOUT))
|
|
19
21
|
@files = files
|
|
20
22
|
@url = url
|
|
21
23
|
@token = token
|
|
22
24
|
@metadata = metadata
|
|
23
25
|
@logger = logger
|
|
24
|
-
@
|
|
26
|
+
@grouping_strategy = grouping_strategy
|
|
27
|
+
@files_metadata = files_metadata
|
|
25
28
|
end
|
|
29
|
+
# rubocop:enable Metrics/ParameterLists
|
|
26
30
|
|
|
27
31
|
def run
|
|
28
32
|
check_files_exist
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
+
upload_responses = UploadFiles.new(files: files, logger: logger,
|
|
34
|
+
connection: connection, files_metadata: files_metadata).run
|
|
35
|
+
metadata_builder = MetadataBuilder.new(metadata: metadata,
|
|
36
|
+
grouping_strategy: grouping_strategy,
|
|
37
|
+
files_metadata: files_metadata,
|
|
38
|
+
logger: logger)
|
|
39
|
+
request = metadata_builder.with_uploads(upload_responses)
|
|
33
40
|
upload_metadata(request.as_json)
|
|
34
41
|
end
|
|
35
42
|
|
|
36
43
|
private
|
|
37
44
|
|
|
38
|
-
attr_reader :metadata, :files, :url, :token, :logger, :
|
|
45
|
+
attr_reader :metadata, :files, :url, :token, :logger, :grouping_strategy, :files_metadata
|
|
39
46
|
|
|
40
47
|
def check_files_exist
|
|
41
48
|
logger.info('checking to see if files exist')
|
|
@@ -44,51 +51,6 @@ module SdrClient
|
|
|
44
51
|
end
|
|
45
52
|
end
|
|
46
53
|
|
|
47
|
-
def collect_file_metadata
|
|
48
|
-
files.each_with_object({}) do |filename, obj|
|
|
49
|
-
obj[filename] = Files::DirectUploadRequest.from_file(filename)
|
|
50
|
-
end
|
|
51
|
-
end
|
|
52
|
-
|
|
53
|
-
# @param [Hash<String,Files::DirectUploadRequest>] file_metadata the filenames and their upload request
|
|
54
|
-
def upload_file_metadata(file_metadata)
|
|
55
|
-
Hash[file_metadata.map { |filename, metadata| [filename, direct_upload(metadata.to_json)] }]
|
|
56
|
-
end
|
|
57
|
-
|
|
58
|
-
def direct_upload(metadata_json)
|
|
59
|
-
logger.info("Starting an upload request: #{metadata_json}")
|
|
60
|
-
response = connection.post(BLOB_PATH, metadata_json, 'Content-Type' => 'application/json')
|
|
61
|
-
raise "unexpected response: #{response.inspect}" unless response.status == 200
|
|
62
|
-
|
|
63
|
-
logger.info("Response from server: #{response.body}")
|
|
64
|
-
|
|
65
|
-
Files::DirectUploadResponse.new(JSON.parse(response.body))
|
|
66
|
-
end
|
|
67
|
-
|
|
68
|
-
# @param [Hash<String,Files::DirectUploadResponse>] upload_responses the filenames and their upload response
|
|
69
|
-
def upload_files(upload_responses)
|
|
70
|
-
upload_responses.each do |filename, response|
|
|
71
|
-
upload_file(filename: filename,
|
|
72
|
-
url: response.direct_upload.fetch('url'),
|
|
73
|
-
content_type: response.content_type,
|
|
74
|
-
content_length: response.byte_size)
|
|
75
|
-
|
|
76
|
-
logger.info('Upload complete')
|
|
77
|
-
end
|
|
78
|
-
end
|
|
79
|
-
|
|
80
|
-
def upload_file(filename:, url:, content_type:, content_length:)
|
|
81
|
-
logger.info("Uploading `#{filename}' to #{url}")
|
|
82
|
-
|
|
83
|
-
upload_response = connection.put(url) do |req|
|
|
84
|
-
req.body = ::File.open(filename)
|
|
85
|
-
req.headers['Content-Type'] = content_type
|
|
86
|
-
req.headers['Content-Length'] = content_length.to_s
|
|
87
|
-
end
|
|
88
|
-
|
|
89
|
-
raise "unexpected response: #{upload_response.inspect}" unless upload_response.status == 204
|
|
90
|
-
end
|
|
91
|
-
|
|
92
54
|
# @return [Hash<Symbol,String>] the result of the metadata call
|
|
93
55
|
def upload_metadata(metadata)
|
|
94
56
|
logger.info("Starting upload metadata: #{metadata}")
|
|
@@ -114,6 +76,22 @@ module SdrClient
|
|
|
114
76
|
conn.adapter :net_http
|
|
115
77
|
end
|
|
116
78
|
end
|
|
79
|
+
|
|
80
|
+
# @param [Array<SdrClient::Deposit::Files::DirectUploadResponse>] uploads the uploaded files to attach.
|
|
81
|
+
# @return [Array<SdrClient::Deposit::FileSet>] the uploads transformed to filesets
|
|
82
|
+
def build_filesets(uploads:)
|
|
83
|
+
grouped_uploads = grouping_strategy.run(uploads: uploads)
|
|
84
|
+
grouped_uploads.map.with_index(1) do |upload_group, i|
|
|
85
|
+
metadata_group = {}
|
|
86
|
+
upload_group.each { |upload| metadata_group[upload.filename] = file_metadata_for(upload.filename) }
|
|
87
|
+
FileSet.new(uploads: upload_group, uploads_metadata: metadata_group, label: "Object #{i}")
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# @return [Hash<Symbol,String>] the file metadata for this file
|
|
92
|
+
def file_metadata_for(filename)
|
|
93
|
+
files_metadata.fetch(filename, {})
|
|
94
|
+
end
|
|
117
95
|
end
|
|
118
96
|
end
|
|
119
97
|
end
|
|
@@ -7,6 +7,7 @@ module SdrClient
|
|
|
7
7
|
# @param [String] label the required object label
|
|
8
8
|
# @param [String] type (http://cocina.sul.stanford.edu/models/object.jsonld) the required object type.
|
|
9
9
|
# @param [Array<FileSet>] file_sets the file sets to attach.
|
|
10
|
+
# rubocop:disable Metrics/ParameterLists
|
|
10
11
|
def initialize(label: nil,
|
|
11
12
|
apo:,
|
|
12
13
|
collection:,
|
|
@@ -22,6 +23,7 @@ module SdrClient
|
|
|
22
23
|
@apo = apo
|
|
23
24
|
@file_sets = file_sets
|
|
24
25
|
end
|
|
26
|
+
# rubocop:enable Metrics/ParameterLists
|
|
25
27
|
|
|
26
28
|
def as_json
|
|
27
29
|
{
|
|
@@ -65,7 +67,7 @@ module SdrClient
|
|
|
65
67
|
def structural
|
|
66
68
|
{
|
|
67
69
|
isMemberOf: collection,
|
|
68
|
-
|
|
70
|
+
contains: file_sets.map(&:as_json)
|
|
69
71
|
}
|
|
70
72
|
end
|
|
71
73
|
end
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SdrClient
|
|
4
|
+
module Deposit
|
|
5
|
+
# This strategy is for building one file set per uploaded file
|
|
6
|
+
class SingleFileGroupingStrategy
|
|
7
|
+
# @param [Array<SdrClient::Deposit::Files::DirectUploadResponse>] uploads the uploaded files to attach.
|
|
8
|
+
# @return [Array<Array<SdrClient::Deposit::Files::DirectUploadResponse>>] uploads the grouped uploaded files.
|
|
9
|
+
def self.run(uploads: [])
|
|
10
|
+
uploads.map { |upload| [upload] }
|
|
11
|
+
end
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
end
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'logger'
|
|
4
|
+
|
|
5
|
+
module SdrClient
|
|
6
|
+
module Deposit
|
|
7
|
+
# The file uploading part of a deposit
|
|
8
|
+
class UploadFiles
|
|
9
|
+
BLOB_PATH = '/v1/direct_uploads'
|
|
10
|
+
# @param [Array<String>] files a list of file names to upload
|
|
11
|
+
# @param [Logger] logger the logger to use
|
|
12
|
+
# @param [Faraday::Connection] connection
|
|
13
|
+
# @param [Hash<String, Hash<String, String>>] files_metadata file name, hash of additional file metadata
|
|
14
|
+
def initialize(files:, files_metadata:, logger:, connection:)
|
|
15
|
+
@files = files
|
|
16
|
+
@files_metadata = files_metadata
|
|
17
|
+
@logger = logger
|
|
18
|
+
@connection = connection
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# @return [Array<SdrClient::Deposit::Files::DirectUploadResponse>] the responses from the server for the uploads
|
|
22
|
+
def run
|
|
23
|
+
file_metadata = collect_file_metadata
|
|
24
|
+
upload_responses = upload_file_metadata(file_metadata)
|
|
25
|
+
upload_files(upload_responses)
|
|
26
|
+
upload_responses.values
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
private
|
|
30
|
+
|
|
31
|
+
attr_reader :files, :files_metadata, :logger, :connection
|
|
32
|
+
|
|
33
|
+
def collect_file_metadata
|
|
34
|
+
files.each_with_object({}) do |path, obj|
|
|
35
|
+
file_name = ::File.basename(path)
|
|
36
|
+
obj[path] = Files::DirectUploadRequest.from_file(path,
|
|
37
|
+
file_name: file_name,
|
|
38
|
+
content_type: file_metadata_for(file_name)[:mime_type])
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# @param [Hash<String,Files::DirectUploadRequest>] file_metadata the filenames and their upload request
|
|
43
|
+
def upload_file_metadata(file_metadata)
|
|
44
|
+
Hash[file_metadata.map { |filename, metadata| [filename, direct_upload(metadata.to_json)] }]
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def direct_upload(metadata_json)
|
|
48
|
+
logger.info("Starting an upload request: #{metadata_json}")
|
|
49
|
+
response = connection.post(BLOB_PATH, metadata_json, 'Content-Type' => 'application/json')
|
|
50
|
+
raise "unexpected response: #{response.inspect}" unless response.status == 200
|
|
51
|
+
|
|
52
|
+
logger.info("Response from server: #{response.body}")
|
|
53
|
+
|
|
54
|
+
Files::DirectUploadResponse.new(JSON.parse(response.body))
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# @param [Hash<String,Files::DirectUploadResponse>] upload_responses the filenames and their upload response
|
|
58
|
+
def upload_files(upload_responses)
|
|
59
|
+
upload_responses.each do |filename, response|
|
|
60
|
+
upload_file(filename: filename,
|
|
61
|
+
url: response.direct_upload.fetch('url'),
|
|
62
|
+
content_type: response.content_type,
|
|
63
|
+
content_length: response.byte_size)
|
|
64
|
+
|
|
65
|
+
logger.info('Upload complete')
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def upload_file(filename:, url:, content_type:, content_length:)
|
|
70
|
+
logger.info("Uploading `#{filename}' to #{url}")
|
|
71
|
+
|
|
72
|
+
upload_response = connection.put(url) do |req|
|
|
73
|
+
req.body = ::File.open(filename)
|
|
74
|
+
req.headers['Content-Type'] = content_type
|
|
75
|
+
req.headers['Content-Length'] = content_length.to_s
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
raise "unexpected response: #{upload_response.inspect}" unless upload_response.status == 204
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# @return [Hash<Symbol,String>] the file metadata for this file
|
|
82
|
+
def file_metadata_for(filename)
|
|
83
|
+
files_metadata.fetch(filename, {})
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
end
|
data/lib/sdr_client/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: sdr-client
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.8.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Justin Coyne
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2020-01-
|
|
11
|
+
date: 2020-01-31 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: dry-monads
|
|
@@ -158,17 +158,21 @@ files:
|
|
|
158
158
|
- bin/console
|
|
159
159
|
- bin/setup
|
|
160
160
|
- exe/sdr
|
|
161
|
+
- lib/sdr-client.rb
|
|
161
162
|
- lib/sdr_client.rb
|
|
162
163
|
- lib/sdr_client/cli.rb
|
|
163
164
|
- lib/sdr_client/credentials.rb
|
|
164
165
|
- lib/sdr_client/deposit.rb
|
|
165
|
-
- lib/sdr_client/deposit/default_file_set_builder.rb
|
|
166
166
|
- lib/sdr_client/deposit/file.rb
|
|
167
167
|
- lib/sdr_client/deposit/file_set.rb
|
|
168
168
|
- lib/sdr_client/deposit/files/direct_upload_request.rb
|
|
169
169
|
- lib/sdr_client/deposit/files/direct_upload_response.rb
|
|
170
|
+
- lib/sdr_client/deposit/matching_file_grouping_strategy.rb
|
|
171
|
+
- lib/sdr_client/deposit/metadata_builder.rb
|
|
170
172
|
- lib/sdr_client/deposit/process.rb
|
|
171
173
|
- lib/sdr_client/deposit/request.rb
|
|
174
|
+
- lib/sdr_client/deposit/single_file_grouping_strategy.rb
|
|
175
|
+
- lib/sdr_client/deposit/upload_files.rb
|
|
172
176
|
- lib/sdr_client/login.rb
|
|
173
177
|
- lib/sdr_client/login_prompt.rb
|
|
174
178
|
- lib/sdr_client/version.rb
|
|
@@ -194,7 +198,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
194
198
|
- !ruby/object:Gem::Version
|
|
195
199
|
version: '0'
|
|
196
200
|
requirements: []
|
|
197
|
-
rubygems_version: 3.0.
|
|
201
|
+
rubygems_version: 3.0.6
|
|
198
202
|
signing_key:
|
|
199
203
|
specification_version: 4
|
|
200
204
|
summary: The CLI for https://github.com/sul-dlss/sdr-api
|
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module SdrClient
|
|
4
|
-
module Deposit
|
|
5
|
-
# This strategy is for building one file set per uploaded file
|
|
6
|
-
class DefaultFileSetBuilder
|
|
7
|
-
# @return [Request] request The initial request
|
|
8
|
-
# @param [Array<SdrClient::Deposit::Files::DirectUploadResponse>] uploads the uploaded files to attach.
|
|
9
|
-
# @return [Request] a clone of this request with the uploads added
|
|
10
|
-
def self.run(request:, uploads: [])
|
|
11
|
-
file_sets = uploads.each_with_index.map do |upload, i|
|
|
12
|
-
FileSet.new(uploads: [upload], label: "Object #{i + 1}")
|
|
13
|
-
end
|
|
14
|
-
request.with_file_sets(file_sets)
|
|
15
|
-
end
|
|
16
|
-
end
|
|
17
|
-
end
|
|
18
|
-
end
|