envoi-mam-agent 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +8 -0
- data/.rbenv-gemsets +3 -0
- data/.ruby-version +1 -0
- data/.travis.yml +5 -0
- data/CODE_OF_CONDUCT.md +74 -0
- data/Gemfile +7 -0
- data/LICENSE.txt +21 -0
- data/README.md +43 -0
- data/Rakefile +10 -0
- data/bin/console +17 -0
- data/bin/setup +8 -0
- data/envoi-mam-agent.gemspec +41 -0
- data/exe/envoi-mam-agent +15 -0
- data/lib/envoi/mam/agent.rb +119 -0
- data/lib/envoi/mam/agent/cli.rb +37 -0
- data/lib/envoi/mam/agent/cli/commands.rb +89 -0
- data/lib/envoi/mam/agent/cli/commands/iconik.rb +71 -0
- data/lib/envoi/mam/agent/cli/commands/mediasilo.rb +296 -0
- data/lib/envoi/mam/agent/cli/commands/vidispine.rb +60 -0
- data/lib/envoi/mam/agent/cli/commands/wiredrive.rb +223 -0
- data/lib/envoi/mam/agent/config_service_client.rb +60 -0
- data/lib/envoi/mam/agent/transfer_client.rb +37 -0
- data/lib/envoi/mam/agent/transfer_client/aspera.rb +91 -0
- data/lib/envoi/mam/agent/transfer_client/s3.rb +70 -0
- data/lib/envoi/mam/agent/version.rb +7 -0
- data/lib/envoi/mam/iconik/agent.rb +195 -0
- data/lib/envoi/mam/mediasilo/agent.rb +286 -0
- data/lib/envoi/mam/vidispine/agent.rb +214 -0
- data/lib/envoi/mam/wiredrive/agent.rb +117 -0
- metadata +256 -0
@@ -0,0 +1,71 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
lib_path = __FILE__ == '(irb)' ? File.join(Dir.cwd, 'lib') : File.expand_path('../../../lib', __FILE__)
|
3
|
+
$:.unshift(lib_path) unless $:.include?(lib_path) or !File.exists?(lib_path)
|
4
|
+
require 'rubygems'
|
5
|
+
require 'optparse'
|
6
|
+
|
7
|
+
require 'envoi/mam/agent/cli'
|
8
|
+
require 'envoi/mam/iconik/agent'
|
9
|
+
|
10
|
+
current_command = ARGV.shift
|
11
|
+
ARGV << '--help' if ARGV.empty?
|
12
|
+
|
13
|
+
default_config_file_paths = Envoi::Mam::Agent::Cli::CONFIG_FILE_PATHS
|
14
|
+
|
15
|
+
aspera_ascp_paths = Envoi::Mam::Agent::TransferClient::Aspera::ASCP_PATHS
|
16
|
+
default_aspera_ascp_path = aspera_ascp_paths.find { |v| File.exist? v } || aspera_ascp_paths.first
|
17
|
+
|
18
|
+
# test_args = %w(--download --send-to-envoi --destination-path /tmp)
|
19
|
+
# ARGV.concat test_args
|
20
|
+
args = {
|
21
|
+
:config_file_path => default_config_file_paths,
|
22
|
+
:default_ascp_path => default_aspera_ascp_path,
|
23
|
+
:dry_run => false,
|
24
|
+
:ingest_after_upload => true,
|
25
|
+
:preserve_path => true,
|
26
|
+
:destination_path => File.expand_path('.'),
|
27
|
+
:should_download_presentation => false,
|
28
|
+
:should_add_presentation_assets_to_envoi => false
|
29
|
+
}
|
30
|
+
|
31
|
+
args[:config_service_app_id] = ENV['ENVOI_MAM_AGENT_APP_ID']
|
32
|
+
args[:config_service_app_token] = ENV['ENVOI_MAM_AGENT_APP_TOKEN']
|
33
|
+
args[:config_service_app_url] = ENV['ENVOI_MAM_AGENT_APP_URL']
|
34
|
+
|
35
|
+
op = OptionParser.new
|
36
|
+
op.on('-c', '--config-file-path FILEPATH', 'The path to the configuration file.') { |v| args[:config_file_path] = v }
|
37
|
+
op.on('--agent-app-id ID', '') { |v| args[:config_service_app_id] = v }
|
38
|
+
op.on('--agent-token TOKEN', '') { |v| args[:config_service_app_token] = v }
|
39
|
+
op.on('--agent-service-api URL', '') { |v| args[:config_service_api_url] = v }
|
40
|
+
|
41
|
+
op.on('--asset-id ASSETID', 'The Iconik item ID.') { |v| args[:asset_id] = v }
|
42
|
+
op.on('-f', '--format-name FORMATNAME', 'The Iconik format name used to find the file to download.') { |v| args[:format_name] = v }
|
43
|
+
op.on('--storage-id STORAGEID', 'The Iconik storage id to use for the transfer.') { |v| args[:storage_id] = v}
|
44
|
+
|
45
|
+
|
46
|
+
op.on('--file-path PATH', '') { |v| args[:file_path] = v }
|
47
|
+
op.on('-d', '--destination-path DIRPATH', 'The destination path to download or upload to.',
|
48
|
+
"default: '#{args[:destination_path]}'") { |v| args[:destination_path] = v }
|
49
|
+
|
50
|
+
op.on('-p','--[no-]preserve-path', "default: #{args[:preserve_path]}") { |v| args[:preserve_path] = v }
|
51
|
+
|
52
|
+
op.on('-o', '--operation OP', [ :download, :upload ], 'Transfer Direction.',
|
53
|
+
"default: #{args[:operation]}" ) { |v| args[:operation] = v }
|
54
|
+
op.on('--transfer-type TYPE', [ :aspera, :s3 ],'Force the type of transfer to use. Ex: aspera || s3') { |v| args[:transfer_type] = v }
|
55
|
+
|
56
|
+
|
57
|
+
op.on('-i', '--[no-]import', 'Determines if an asset is created after a upload completes.',
|
58
|
+
"default: #{args[:ingest_after_upload]}") { |v| args[:ingest_after_upload] = v}
|
59
|
+
op.on('--[no-]analyze-asset', 'Determines if an analyze job is started on an asset after it is imported',
|
60
|
+
"default: #{args[:analyze_asset]}") { |v| args[:analyze_asset] = v }
|
61
|
+
|
62
|
+
# op.on('--dry-run', 'Run without executing the transfer.') { |v| args[:dry_run] = v }
|
63
|
+
op.on('--help', 'Show this message.') { puts op; exit }
|
64
|
+
op.load
|
65
|
+
op.parse!
|
66
|
+
|
67
|
+
args[:file_path] ||= ARGV.shift
|
68
|
+
args[:destination_path] ||= ARGV.shift unless ARGV.empty?
|
69
|
+
|
70
|
+
agent = Envoi::Mam::Iconik::Agent.load_config_and_init(args)
|
71
|
+
agent.run_operation
|
@@ -0,0 +1,296 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# lib_path = __FILE__ == '(irb)' ? File.join(Dir.cwd, 'lib') : File.expand_path('../../../lib', __FILE__)
|
3
|
+
# $:.unshift(lib_path) unless $:.include?(lib_path) or !File.exists?(lib_path)
|
4
|
+
# require 'rubygems'
|
5
|
+
require 'optparse'
|
6
|
+
require 'pp'
|
7
|
+
|
8
|
+
require 'envoi/mam/agent/cli'
|
9
|
+
require 'envoi/mam/mediasilo/agent'
|
10
|
+
|
11
|
+
current_command = ARGV.shift
|
12
|
+
ARGV << '--help' if ARGV.empty?
|
13
|
+
|
14
|
+
default_config_file_paths = Envoi::Mam::Agent::Cli::CONFIG_FILE_PATHS
|
15
|
+
|
16
|
+
aspera_ascp_paths = Envoi::Mam::Agent::TransferClient::Aspera::ASCP_PATHS
|
17
|
+
default_aspera_ascp_path = aspera_ascp_paths.find { |v| File.exist? v } || aspera_ascp_paths.first
|
18
|
+
|
19
|
+
args = {
|
20
|
+
:config_file_path => default_config_file_paths,
|
21
|
+
:default_aspera_ascp_path => default_aspera_ascp_path,
|
22
|
+
:dry_run => false,
|
23
|
+
:operation => :download,
|
24
|
+
:preserve_path => true,
|
25
|
+
:transfer_type => '',
|
26
|
+
|
27
|
+
:asset_derivative_type => 'source',
|
28
|
+
}
|
29
|
+
|
30
|
+
args[:config_service_app_id] = ENV['ENVOI_MAM_AGENT_APP_ID']
|
31
|
+
args[:config_service_app_token] = ENV['ENVOI_MAM_AGENT_APP_TOKEN']
|
32
|
+
args[:config_service_app_url] = ENV['ENVOI_MAM_AGENT_APP_URL']
|
33
|
+
|
34
|
+
op = OptionParser.new
|
35
|
+
op.on('--mediasilo-path PATH', 'The path on MediaSilo to download from or upload to.', "PROJECTNAME/[FOLDERNAME]/.../[ASSETTITLE]") { |v| args[:mediasilo_path] = v }
|
36
|
+
op.on('--mediasilo-project-name PROJECTNAME') { |v| args[:mediasilo_project_name] = v }
|
37
|
+
op.on('--mediasilo-asset-derivative-type DERIVATIVE', 'The derivative to use when downloading and asset.', "default: #{args[:asset_derivative_type]}") { |v| args[:asset_derivative_type] = v }
|
38
|
+
op.on('--mediasilo-asset-id ASSETID', '') { |v| args[:asset_id] = v }
|
39
|
+
|
40
|
+
op.on('-c', '--config-file-path FILEPATH', 'The path to the configuration file.') { |v| args[:config_file_path] = v }
|
41
|
+
op.on('--agent-app-id ID', '') { |v| args[:config_service_app_id] = v }
|
42
|
+
op.on('--agent-token TOKEN', '') { |v| args[:config_service_app_token] = v }
|
43
|
+
op.on('--agent-service-api URL', '') { |v| args[:config_service_api_url] = v }
|
44
|
+
|
45
|
+
op.on('--file-path PATH', '') { |v| args[:file_path] = v }
|
46
|
+
op.on('-d', '--destination-path DIRPATH', 'The destination path to download or upload to.', "default: '.'") { |v| args[:destination_path] = v }
|
47
|
+
op.on('-p','--[no-]preserve-path', "default: true") { |v| args[:preserve_path] = v }
|
48
|
+
op.on('-o', '--operation OP', [ :download, :upload ], 'Transfer Direction.', "default: #{args[:operation]}" ) { |v| args[:operation] = v }
|
49
|
+
op.on('--transfer-type TYPE', [ :aspera, :s3 ],'Force the type of transfer to use. Ex: aspera || s3') { |v| args[:transfer_type] = v }
|
50
|
+
op.on('--transfer-token TOKEN', '') { |v| args[:transfer_token] = v }
|
51
|
+
op.on('--dry-run', 'Run without executing the transfer.') { |v| args[:dry_run] = v }
|
52
|
+
op.on('--help', 'Show this message.') { puts op; exit }
|
53
|
+
op.load
|
54
|
+
op.parse!
|
55
|
+
|
56
|
+
args[:file_path] ||= ARGV.shift
|
57
|
+
args[:destination_path] ||= ARGV.shift unless ARGV.empty?
|
58
|
+
|
59
|
+
# module Envoi
|
60
|
+
#
|
61
|
+
# module Mam
|
62
|
+
#
|
63
|
+
# class MediaSilo
|
64
|
+
#
|
65
|
+
# attr_accessor :logger, :config, :api_client
|
66
|
+
#
|
67
|
+
# class Agent < Envoi::Mam::Agent
|
68
|
+
#
|
69
|
+
# DEFAULT_DOWNLOAD_DESTINATION_PATH = '.'
|
70
|
+
#
|
71
|
+
# class CaseSensitiveHeaderKey < String
|
72
|
+
# # def downcase; self end
|
73
|
+
# def capitalize; self end
|
74
|
+
# end
|
75
|
+
#
|
76
|
+
# class FileToSend < File
|
77
|
+
# def bytesize; size end
|
78
|
+
# end
|
79
|
+
#
|
80
|
+
# def initialize_api_client(args = { })
|
81
|
+
# @api_client = args[:api_client] || begin
|
82
|
+
# ms_config = config['mediasilo']
|
83
|
+
# mediasilo_hostname = ms_config['hostname']
|
84
|
+
# mediasilo_username = ms_config['username']
|
85
|
+
# mediasilo_password = ms_config['password']
|
86
|
+
# mediasilo_api_key = ms_config['api_key']
|
87
|
+
#
|
88
|
+
# client_args = { }
|
89
|
+
# client_args[:hostname] = mediasilo_hostname if mediasilo_hostname
|
90
|
+
# client_args[:username] = mediasilo_username if mediasilo_username
|
91
|
+
# client_args[:password] = mediasilo_password if mediasilo_password
|
92
|
+
# client_args[:api_key] = mediasilo_api_key if mediasilo_api_key
|
93
|
+
# Ubiquity::MediaSilo::API::V3::Utilities.new(client_args)
|
94
|
+
# end
|
95
|
+
#
|
96
|
+
# end
|
97
|
+
#
|
98
|
+
# def download(args = { })
|
99
|
+
# transfer_type = args[:transfer_type]
|
100
|
+
#
|
101
|
+
# file_path = args[:file_path]
|
102
|
+
# project_id = args[:project_id]
|
103
|
+
# folder_id = args[:folder_id]
|
104
|
+
# asset_id = args[:asset_id]
|
105
|
+
#
|
106
|
+
# mediasilo_path = args[:mediasilo_path]
|
107
|
+
# project_name = args[:project_name]
|
108
|
+
# folder_name = args[:folder_name]
|
109
|
+
#
|
110
|
+
# if mediasilo_path || project_name || folder_name
|
111
|
+
# check_path_result = api_client.check_path(mediasilo_path)
|
112
|
+
# found = check_path_result[:existing]
|
113
|
+
#
|
114
|
+
# project = found[:project] || { }
|
115
|
+
# project_id = project['id']
|
116
|
+
#
|
117
|
+
# folders = found[:folders]
|
118
|
+
# folder_id = (folders.last || { })['id']
|
119
|
+
#
|
120
|
+
# asset = found[:asset] || { }
|
121
|
+
# asset_id = asset['id']
|
122
|
+
# end
|
123
|
+
#
|
124
|
+
# asset ||= asset_id
|
125
|
+
#
|
126
|
+
# destination_file_path = args[:destination_path] || DEFAULT_DOWNLOAD_DESTINATION_PATH
|
127
|
+
#
|
128
|
+
# case transfer_type
|
129
|
+
# when :aspera; download_using_aspera(asset_id, destination_file_path, args)
|
130
|
+
# else
|
131
|
+
# do_upload_response = download_using_http(asset, destination_file_path, args)
|
132
|
+
# asset_url = do_upload_response[:asset_url]
|
133
|
+
# end
|
134
|
+
#
|
135
|
+
# end
|
136
|
+
#
|
137
|
+
# def download_using_aspera(asset_id, destination_file_path, args = { })
|
138
|
+
# derivative_type = args[:asset_derivative_type]
|
139
|
+
# derivative_type = derivative_type.downcase == 'source' ? 'source' : 'proxy'
|
140
|
+
# ticket = api.aspera_file_download_ticket_create(:asset_id => asset_id, :target => derivative_type)
|
141
|
+
#
|
142
|
+
# host = ticket['server']
|
143
|
+
# username = ticket['username']
|
144
|
+
# password = ticket['password']
|
145
|
+
# token = ticket['token']
|
146
|
+
#
|
147
|
+
# original_file_name = ticket['fileName']
|
148
|
+
#
|
149
|
+
# file_path = ticket['path']
|
150
|
+
# target_path = destination_file_path
|
151
|
+
#
|
152
|
+
# FileUtils.mkdir_p(target_path) if target_path.end_with?('/') && !File.directory?(target_path)
|
153
|
+
# if target_path.end_with?('/') || File.directory?(target_path)
|
154
|
+
# target_path = File.join(target_path, original_file_name)
|
155
|
+
# end
|
156
|
+
#
|
157
|
+
# aspera_config = { }
|
158
|
+
# aspera_config['host'] = host
|
159
|
+
# aspera_config['username'] = username
|
160
|
+
# aspera_config['password'] = password
|
161
|
+
# aspera_config['token'] = token
|
162
|
+
#
|
163
|
+
# client = Envoi::Mam::Agent::TransferClient::Aspera.new(agent: self)
|
164
|
+
# client.download(aspera_config, file_path, target_path)
|
165
|
+
# end
|
166
|
+
#
|
167
|
+
# def download_using_http(asset, destination_file_path, args)
|
168
|
+
# overwrite = args.fetch(:overwrite, false)
|
169
|
+
# derivative_type = args[:asset_derivative_type]
|
170
|
+
# api_client.asset_download_derivative(derivative_type, asset, destination_file_path, overwrite)
|
171
|
+
# end
|
172
|
+
#
|
173
|
+
# def upload(args = { })
|
174
|
+
# transfer_type = args[:transfer_type]
|
175
|
+
#
|
176
|
+
# file_path = args[:file_path]
|
177
|
+
# project_id = args[:project_id]
|
178
|
+
# folder_id = args[:folder_id]
|
179
|
+
# asset_id = args[:asset_id]
|
180
|
+
#
|
181
|
+
# mediasilo_path = args[:mediasilo_path]
|
182
|
+
# project_name = args[:project_name]
|
183
|
+
# folder_name = args[:folder_name]
|
184
|
+
#
|
185
|
+
# if mediasilo_path || project_name || folder_name
|
186
|
+
# check_path_result = api_client.check_path(mediasilo_path, true)
|
187
|
+
# found = check_path_result[:existing]
|
188
|
+
#
|
189
|
+
# project = found[:project] || { }
|
190
|
+
# project_id = project['id']
|
191
|
+
#
|
192
|
+
# folders = found[:folders]
|
193
|
+
# folder_id = (folders.last || { })['id']
|
194
|
+
# end
|
195
|
+
#
|
196
|
+
# # # preserve_path = args.fetch(:preserve_path, vidispine_storage_true)
|
197
|
+
# # preserve_path = args.fetch(:preserve_path, true)
|
198
|
+
# #
|
199
|
+
# # destination_path = args[:destination_path] || DEFAULT_DESTINATION_PATH
|
200
|
+
# # relative_path = preserve_path ? File.dirname(file_path) : nil
|
201
|
+
# # relative_path = nil if relative_path == '.'
|
202
|
+
# #
|
203
|
+
# # target_path = relative_path ? File.join(destination_path, relative_path) : destination_path
|
204
|
+
# # target_path = target_path[0..-1] if target_path.start_with?('/') && !destination_path.start_with?('/')
|
205
|
+
#
|
206
|
+
# # upload file
|
207
|
+
# case transfer_type
|
208
|
+
# when :aspera; upload_using_aspera(file_path, args)
|
209
|
+
# else
|
210
|
+
# do_upload_response = upload_using_http(file_path)
|
211
|
+
# asset_url = do_upload_response[:asset_url]
|
212
|
+
# end
|
213
|
+
#
|
214
|
+
# asset_create_args = {
|
215
|
+
# 'projectId' => project_id,
|
216
|
+
# 'folderId' => folder_id,
|
217
|
+
# :source_url => asset_url
|
218
|
+
# }
|
219
|
+
# response = api_client.asset_create(asset_create_args)
|
220
|
+
# end
|
221
|
+
#
|
222
|
+
# def upload_using_aspera(file_path, args = { })
|
223
|
+
# raise 'Upload using Aspera not yet Implemented.'
|
224
|
+
#
|
225
|
+
# file_name = File.basename(file_path)
|
226
|
+
#
|
227
|
+
# ticket = api.aspera_file_upload_ticket_create(:file_name => file_name)
|
228
|
+
#
|
229
|
+
# host = ticket['server']
|
230
|
+
# username = ticket['username']
|
231
|
+
# password = ticket['password']
|
232
|
+
# token = ticket['token']
|
233
|
+
# destination = ticket['destination']
|
234
|
+
#
|
235
|
+
# target_path = destination
|
236
|
+
#
|
237
|
+
# aspera_config = { }
|
238
|
+
# aspera_config['host'] = host
|
239
|
+
# aspera_config['username'] = username
|
240
|
+
# aspera_config['password'] = password
|
241
|
+
# aspera_config['token'] = token
|
242
|
+
#
|
243
|
+
# client = Envoi::Mam::Agent::TransferClient::Aspera.new(agent: self)
|
244
|
+
# client.upload(aspera_config, file_path, target_path)
|
245
|
+
# end
|
246
|
+
#
|
247
|
+
# def upload_using_http(file_path)
|
248
|
+
# file_name = File.basename(file_path)
|
249
|
+
#
|
250
|
+
# res = api_client.asset_upload_ticket_create(:file_name => file_name)
|
251
|
+
# uri_str = res['assetUrl']
|
252
|
+
#
|
253
|
+
# uri = URI.parse(uri_str)
|
254
|
+
#
|
255
|
+
# amz_date = res['amzDate']
|
256
|
+
# amz_acl = res['amzAcl']
|
257
|
+
# content_type = res['contentType']
|
258
|
+
# authorization = res['authorization']
|
259
|
+
#
|
260
|
+
# file = File.open(file_path)
|
261
|
+
# # file = FileToSend.open(file_path)
|
262
|
+
#
|
263
|
+
# req = Net::HTTP::Put.new(uri.request_uri)
|
264
|
+
# req['x-amz-date'] = amz_date
|
265
|
+
# req['x-amz-acl'] = amz_acl
|
266
|
+
# req[CaseSensitiveHeaderKey.new('Content-Type')] = content_type
|
267
|
+
# res[CaseSensitiveHeaderKey.new('Authorization')] = authorization
|
268
|
+
# req[CaseSensitiveHeaderKey.new('Transfer-Encoding')] = 'chunked'
|
269
|
+
# req[CaseSensitiveHeaderKey.new('Content-Length')] = file.size
|
270
|
+
#
|
271
|
+
# req.body_stream = file
|
272
|
+
#
|
273
|
+
# # req.body = FileToSend.open(file_path)
|
274
|
+
#
|
275
|
+
# request = req
|
276
|
+
# http = Net::HTTP.new(uri.host, uri.port)
|
277
|
+
# http.use_ssl = true
|
278
|
+
# logger.debug { %(REQUEST: #{request.method} https://#{http.address}:#{http.port}#{request.path} HEADERS: #{request.to_hash.inspect} #{api_client.http_client.log_request_body and request.request_body_permitted? ? "BODY: #{api_client.http_client.format_body_for_log_output(request)}" : ''}) }
|
279
|
+
#
|
280
|
+
# res = http.request(req)
|
281
|
+
# { :asset_url => uri_str }
|
282
|
+
# end
|
283
|
+
#
|
284
|
+
#
|
285
|
+
# end
|
286
|
+
#
|
287
|
+
# end
|
288
|
+
#
|
289
|
+
# end
|
290
|
+
#
|
291
|
+
# end
|
292
|
+
|
293
|
+
agent = Envoi::Mam::MediaSilo::Agent.load_config_and_init(args)
|
294
|
+
agent.run_operation
|
295
|
+
|
296
|
+
|
@@ -0,0 +1,60 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
lib_path = __FILE__ == '(irb)' ? File.join(Dir.cwd, 'lib') : File.expand_path('../../../lib', __FILE__)
|
3
|
+
$:.unshift(lib_path) unless $:.include?(lib_path) or !File.exists?(lib_path)
|
4
|
+
require 'rubygems'
|
5
|
+
require 'optparse'
|
6
|
+
require 'open3'
|
7
|
+
|
8
|
+
require 'envoi/mam/agent/cli'
|
9
|
+
require 'envoi/mam/vidispine/agent'
|
10
|
+
|
11
|
+
default_config_file_paths = Envoi::Mam::Agent::Cli::CONFIG_FILE_PATHS
|
12
|
+
|
13
|
+
aspera_ascp_paths = Envoi::Mam::Agent::TransferClient::Aspera::ASCP_PATHS
|
14
|
+
default_aspera_ascp_path = aspera_ascp_paths.find { |v| File.exist? v } || aspera_ascp_paths.first
|
15
|
+
|
16
|
+
current_command = ARGV.shift
|
17
|
+
ARGV << '--help' if ARGV.empty?
|
18
|
+
|
19
|
+
args = {
|
20
|
+
:config_file_path => default_config_file_paths,
|
21
|
+
:default_ascp_path => default_aspera_ascp_path,
|
22
|
+
:dry_run => false,
|
23
|
+
:operation => :download,
|
24
|
+
:preserve_path => true,
|
25
|
+
:transfer_type => '',
|
26
|
+
}
|
27
|
+
|
28
|
+
args[:config_service_app_id] = ENV['ENVOI_MAM_AGENT_APP_ID']
|
29
|
+
args[:config_service_app_token] = ENV['ENVOI_MAM_AGENT_APP_TOKEN']
|
30
|
+
args[:config_service_app_url] = ENV['ENVOI_MAM_AGENT_APP_URL']
|
31
|
+
|
32
|
+
op = OptionParser.new
|
33
|
+
op.on('-c', '--config-file-path FILEPATH', 'The path to the configuration file.',
|
34
|
+
"Default Path(s): #{args[:config_file_path]}") { |v| args[:config_file_path] = v }
|
35
|
+
op.on('--agent-app-id ID', '') { |v| args[:config_service_app_id] = v }
|
36
|
+
op.on('--agent-token TOKEN', '') { |v| args[:config_service_app_token] = v }
|
37
|
+
op.on('--agent-service-api URL', '') { |v| args[:config_service_api_url] = v }
|
38
|
+
|
39
|
+
op.on('-i', '--item-id ITEMID', 'The Vidispine item id.') { |v| args[:item_id] = v }
|
40
|
+
op.on('-s', '--shape-id SHAPEID', 'The Vidispine shape id.') { |v| args[:shape_id] = v }
|
41
|
+
op.on('-t', '--shape-tag SHAPETAG', 'The Vidispine shape tag used to find the shape when a shape id is not supplied.') { |v| args[:shape_tag] = v }
|
42
|
+
op.on('--storage-id STORAGEID', 'The Vidispine storage id to use for the transfer.') { |v| args[:storage_id] = v}
|
43
|
+
|
44
|
+
op.on('-d', '--destination-path DIRPATH', 'The destination path to download or upload to.',
|
45
|
+
"default: '.'") { |v| args[:destination_path] = v }
|
46
|
+
op.on('-p','--[no-]preserve-path', "default: true") { |v| args[:preserve_path] = v }
|
47
|
+
op.on('-p', '--operation OP', [ :download, :upload ], 'Transfer Direction.',
|
48
|
+
"default: #{args[:operation]}" ) { |v| args[:operation] = v }
|
49
|
+
op.on('--transfer-type TYPE', [ :aspera, :s3 ],'Force the type of transfer to use. Ex: aspera || s3') { |v| args[:transfer_type] = v }
|
50
|
+
op.on('--transfer-token TOKEN', '') { |v| args[:transfer_token] = v }
|
51
|
+
op.on('--dry-run', 'Run without executing the transfer.') { |v| args[:dry_run] = v }
|
52
|
+
op.on('--help', 'Show this message.') { puts op; exit }
|
53
|
+
op.load
|
54
|
+
op.parse!
|
55
|
+
|
56
|
+
args[:file_path] ||= ARGV.shift
|
57
|
+
args[:destination_path] ||= ARGV.shift unless ARGV.empty?
|
58
|
+
|
59
|
+
agent = Envoi::Mam::Vidispine::Agent.load_config_and_init(args)
|
60
|
+
agent.run_operation
|
@@ -0,0 +1,223 @@
|
|
1
|
+
require 'cgi'
|
2
|
+
require 'open-uri'
|
3
|
+
|
4
|
+
require 'ubiquity/wiredrive/presentation_utility'
|
5
|
+
require 'ubiquity/envoi/api/utilities'
|
6
|
+
|
7
|
+
require 'envoi/mam/agent/cli'
|
8
|
+
require 'envoi/mam/wiredrive/agent'
|
9
|
+
require 'envoi/mam/iconik/agent'
|
10
|
+
|
11
|
+
current_command = ARGV.shift
|
12
|
+
ARGV << '--help' if ARGV.empty?
|
13
|
+
|
14
|
+
default_config_file_paths = Envoi::Mam::Agent::Cli::CONFIG_FILE_PATHS
|
15
|
+
# test_args = %w(--download --send-to-envoi --destination-path /tmp)
|
16
|
+
# ARGV.concat test_args
|
17
|
+
args = {
|
18
|
+
:config_file_path => default_config_file_paths,
|
19
|
+
:dry_run => false,
|
20
|
+
:preserve_path => true,
|
21
|
+
:destination_path => File.expand_path('.'),
|
22
|
+
:should_download_presentation => false,
|
23
|
+
:should_add_presentation_assets_to_envoi => false
|
24
|
+
}
|
25
|
+
|
26
|
+
args[:config_service_app_id] = ENV['ENVOI_MAM_AGENT_APP_ID']
|
27
|
+
args[:config_service_app_token] = ENV['ENVOI_MAM_AGENT_APP_TOKEN']
|
28
|
+
args[:config_service_app_url] = ENV['ENVOI_MAM_AGENT_APP_URL']
|
29
|
+
|
30
|
+
op = OptionParser.new
|
31
|
+
op.on('-c', '--config-file-path FILEPATH', 'The path to the configuration file.') { |v| args[:config_file_path] = v }
|
32
|
+
op.on('--agent-app-id ID', '') { |v| args[:config_service_app_id] = v }
|
33
|
+
op.on('--agent-token TOKEN', '') { |v| args[:config_service_app_token] = v }
|
34
|
+
op.on('--agent-service-api URL', '') { |v| args[:config_service_api_url] = v }
|
35
|
+
|
36
|
+
op.on('--presentation-invitation-token TOKEN', 'Presentation invitation token.') { |v| args[:presentation_invitation_token] = v }
|
37
|
+
op.on('--presentation-password PASSWORD', 'Presentation password') { |v| args[:presentation_password] = v }
|
38
|
+
op.on('--presentation-url URL', 'A shared presentation URL.') { |v| args[:presentation_url] = v }
|
39
|
+
# op.on('--presentation-id ID', '') { |v| args[:presentation_id] = v }
|
40
|
+
#
|
41
|
+
op.on('--[no-]download-presentation-assets', 'Determines if presentation asset files should be downloaded.') { |v| args[:should_download_presentation] = v }
|
42
|
+
op.on('--[no-]add-assets-to-envoi', 'Determines if presentation assets should be added to Envoi.') { |v| args[:should_add_presentation_assets_to_envoi] = v }
|
43
|
+
op.on('--[no-]add-assets-to-iconik', '') { |v| args[:should_add_presentation_assets_to_iconik] = v }
|
44
|
+
op.on('--iconik-storage-id STORAGEID', '') { |v| args[:iconik_storage_id] = v }
|
45
|
+
|
46
|
+
op.on('-d', '--destination-path DIRPATH', 'The destination path to download or upload to.', "default: '#{args[:destination_path]}'") { |v| args[:destination_path] = v }
|
47
|
+
op.on('-p','--[no-]preserve-path', "default: #{args[:preserve_path]}") { |v| args[:preserve_path] = v }
|
48
|
+
|
49
|
+
# op.on('--dry-run', 'Run without executing the transfer.') { |v| args[:dry_run] = v }
|
50
|
+
op.on('--help', 'Show this message.') { puts op; exit }
|
51
|
+
op.load
|
52
|
+
op.parse!
|
53
|
+
|
54
|
+
args[:destination_path] ||= ARGV.shift unless ARGV.empty?
|
55
|
+
|
56
|
+
presentation_url ||= args[:presentation_url] || ARGV.shift
|
57
|
+
if presentation_url
|
58
|
+
uri = URI(presentation_url)
|
59
|
+
presentation_invitation_token = uri.path
|
60
|
+
presentation_url_token = presentation_invitation_token[1..-1] if presentation_invitation_token.start_with?('/')
|
61
|
+
|
62
|
+
query_data = uri.query ? Hash[uri.query.split('&').map { |v| v.split('=') }] : { }
|
63
|
+
query_password = query_data['password']
|
64
|
+
presentation_url_password ||= query_password
|
65
|
+
|
66
|
+
args[:presentation_invitation_token] ||= presentation_url_token
|
67
|
+
args[:presentation_password] ||= presentation_url_password
|
68
|
+
end
|
69
|
+
|
70
|
+
class EnvoiImportUtility
|
71
|
+
|
72
|
+
attr_accessor :initial_args, :api_client
|
73
|
+
|
74
|
+
def initialize(args = { })
|
75
|
+
@initial_args = args
|
76
|
+
initialize_api_client(args)
|
77
|
+
end
|
78
|
+
|
79
|
+
def after_initialize
|
80
|
+
@default_original_storage_id = args[:default_original_storage_id]
|
81
|
+
@default_thumbnail_storage_id = args[:default_thumbnail_storage_id]
|
82
|
+
end
|
83
|
+
|
84
|
+
def initialize_api_client(args = { })
|
85
|
+
@api_client = Ubiquity::Envoi::API::Utilities.new(args)
|
86
|
+
end
|
87
|
+
|
88
|
+
def import_wiredrive_presentation_asset(asset, options = { })
|
89
|
+
title = asset['title'] || asset['label']
|
90
|
+
description = asset['description']
|
91
|
+
keywords = asset['keywords']
|
92
|
+
|
93
|
+
media_elements = asset['media']
|
94
|
+
|
95
|
+
media = media_elements.find { |me| me['type'] == 'original' }
|
96
|
+
raise 'No original media found.' unless media
|
97
|
+
url = media['url']
|
98
|
+
uri = URI(url)
|
99
|
+
mime_type = media['mimeType']
|
100
|
+
size = media['fileSize']
|
101
|
+
|
102
|
+
|
103
|
+
url_path = uri.path.split('?').first
|
104
|
+
filename = File.basename(url_path)
|
105
|
+
|
106
|
+
original_storage_id = options[:original_storage_id] || options[:storage_id] || @default_original_storage_id
|
107
|
+
create_args = {
|
108
|
+
:name => title,
|
109
|
+
:description => description,
|
110
|
+
:file => {
|
111
|
+
:mime => 'video/mp4', # mime_type,
|
112
|
+
:path => url,
|
113
|
+
:name => filename,
|
114
|
+
:storage_key => url_path,
|
115
|
+
:setting_id => original_storage_id,
|
116
|
+
'size' => size || 0
|
117
|
+
}
|
118
|
+
}
|
119
|
+
r = api_client.media_file_create(create_args)
|
120
|
+
|
121
|
+
media_file_id = r['id']
|
122
|
+
raise 'No id found in response' unless media_file_id
|
123
|
+
|
124
|
+
video_thumb = asset['thumbnails'].find { |e| e['category'] == 'max' }
|
125
|
+
if video_thumb
|
126
|
+
thumbnail_storage_id = options[:thumbnail_storage_id] || options[:storage_id] || @default_thumbnail_storage_id
|
127
|
+
video_thumb_url = video_thumb['url']
|
128
|
+
video_thumb_uri = URI(video_thumb_url)
|
129
|
+
video_thumb_path = video_thumb_uri.path.split('?').first
|
130
|
+
media_file_file_add_args = {
|
131
|
+
'media_file_id' => media_file_id,
|
132
|
+
'shape_type' => 'thumb',
|
133
|
+
'shape_label' => 'Thumbnail',
|
134
|
+
'name' => File.basename(video_thumb_path),
|
135
|
+
'mime' => 'image/jpeg',
|
136
|
+
'path' => video_thumb_url,
|
137
|
+
'size' => 0,
|
138
|
+
'setting_id' => thumbnail_storage_id,
|
139
|
+
'storage_key' => video_thumb_path
|
140
|
+
}
|
141
|
+
r = api_client.media_file_file_add(media_file_file_add_args)
|
142
|
+
|
143
|
+
end
|
144
|
+
end
|
145
|
+
|
146
|
+
end
|
147
|
+
|
148
|
+
@args = args
|
149
|
+
def wdpu(args = @args); @wdpu ||= Ubiquity::Wiredrive::PresentationUtility.new(args) end
|
150
|
+
|
151
|
+
def presentation_assets_get(args = @args)
|
152
|
+
presentation_invitation_token = args[:presentation_invitation_token]
|
153
|
+
presentation_password = args[:presentation_password]
|
154
|
+
wdpu.presentation_assets_get_using_token(presentation_invitation_token, presentation_password)
|
155
|
+
end
|
156
|
+
def presentation_assets; @presentation_assets ||= presentation_assets_get end
|
157
|
+
|
158
|
+
if args[:should_download_presentation]
|
159
|
+
presentation_assets_download_results = wdpu.presentation_assets_download(assets: presentation_assets, destination_path: args[:destination_path], overwrite: args[:overwrite_on_download])
|
160
|
+
end
|
161
|
+
|
162
|
+
if args[:should_add_presentation_assets_to_envoi]
|
163
|
+
|
164
|
+
wda = Envoi::Mam::Wiredrive::Agent.load_config_and_init(args)
|
165
|
+
envoi_config = wda.config['envoi']
|
166
|
+
envoi_http_host_address = envoi_config['http_host_address']
|
167
|
+
envoi_username = envoi_config['username']
|
168
|
+
envoi_password = envoi_config['password']
|
169
|
+
envoi_original_storage_id = envoi_config['original_storage_id'] || envoi_config['default_storage_id']
|
170
|
+
envoi_thumbnail_storage_id = envoi_config['thumbnail_storage_id'] || envoi_config['default_storage_id']
|
171
|
+
envoi_args = { http_host_address: envoi_http_host_address, username: envoi_username, password: envoi_password }
|
172
|
+
envoi = EnvoiImportUtility.new(envoi_args)
|
173
|
+
presentation_assets.each do |a|
|
174
|
+
begin
|
175
|
+
envoi.import_wiredrive_presentation_asset(a, { original_storage_id: envoi_original_storage_id, thumbnail_storage_id: envoi_thumbnail_storage_id })
|
176
|
+
rescue => e
|
177
|
+
warn "#{e.message} #{$@.first}"
|
178
|
+
end
|
179
|
+
|
180
|
+
end
|
181
|
+
end
|
182
|
+
|
183
|
+
if args[:should_add_presentation_assets_to_iconik]
|
184
|
+
presentation_assets_download_results ||= wdpu.presentation_assets_download(assets: presentation_assets, destination_path: '/tmp')
|
185
|
+
|
186
|
+
# wda = Envoi::Mam::Wiredrive::Agent.load_config_and_init(args)
|
187
|
+
# iconik_config = wda.config['iconik']
|
188
|
+
|
189
|
+
iconik = Envoi::Mam::Iconik::Agent.load_config_and_init(args)
|
190
|
+
iconik_storage_id = args[:iconik_storage_id]
|
191
|
+
presentation_assets_download_results.each do |r|
|
192
|
+
# asset = r[:asset]
|
193
|
+
media_elements_downloaded = r[:media_element_download_responses]
|
194
|
+
media_element_downloaded = media_elements_downloaded.first
|
195
|
+
file_path = media_element_downloaded[:destination_file_path]
|
196
|
+
begin
|
197
|
+
# puts "WD FP: #{media_element_downloaded}"
|
198
|
+
iconik.upload(file_path: file_path, storage_id: iconik_storage_id, destination_path: '/', preserve_path: false)
|
199
|
+
rescue => e
|
200
|
+
warn "#{e.message} #{$@.first}"
|
201
|
+
end
|
202
|
+
end
|
203
|
+
|
204
|
+
end
|
205
|
+
|
206
|
+
if args[:should_add_presentation_assets_to_vidispine]
|
207
|
+
presentation_assets_download_results ||= wdpu.presentation_assets_download(assets: presentation_assets, destination_path: '/tmp')
|
208
|
+
|
209
|
+
vidispine = Envoi::Mam::Vidispine::Agent.load_config_and_init(args)
|
210
|
+
vidispine_storage_id = args[:vidispine_storage_id]
|
211
|
+
presentation_assets_download_results.each do |r|
|
212
|
+
# asset = r[:asset]
|
213
|
+
media_elements_downloaded = r[:media_element_download_responses]
|
214
|
+
media_element_downloaded = media_elements_downloaded.first
|
215
|
+
file_path = media_element_downloaded[:destination_file_path]
|
216
|
+
begin
|
217
|
+
vidispine.upload(file_path: file_path, storage_id: vidispine_storage_id, destination_path: '/', preserve_path: false)
|
218
|
+
rescue => e
|
219
|
+
warn "#{e.message} #{$@.first}"
|
220
|
+
end
|
221
|
+
end
|
222
|
+
|
223
|
+
end
|