cantemo-portal-agent 1.0.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1 @@
1
+ require 'faraday'
@@ -0,0 +1,70 @@
1
+ require 'aws-sdk-s3'
2
+
3
+ require 'envoi/mam/agent/transfer_client'
4
+
5
+ module Envoi
6
+ module Mam
7
+ class Agent
8
+ class TransferClient
9
+
10
+ class S3 < TransferClient
11
+
12
+ def after_initialize(args = initial_args)
13
+
14
+ end
15
+
16
+ def initialize_s3_client(config)
17
+ region = config['region']
18
+ access_key_id = config['access_key_id']
19
+ secret_access_key = config['secret_access_key']
20
+
21
+ resource_args = { }
22
+ resource_args[:region] = region if region && !region.empty?
23
+ resource_args[:access_key_id] = access_key_id if access_key_id && !access_key_id.empty?
24
+ resource_args[:secret_access_key] = secret_access_key if secret_access_key && !secret_access_key.empty?
25
+ @s3 = Aws::S3::Resource.new(resource_args)
26
+ end
27
+
28
+ def download(config, path, destination_path = DEFAULT_DESTINATION_PATH)
29
+ s3 = initialize_s3_client(config)
30
+ bucket_name = config['bucket_name']
31
+ bucket = s3.bucket(bucket_name)
32
+ object_prefix = config['object_prefix']
33
+ object_path = object_prefix && !object_prefix.empty? ? File.join(object_prefix, path) : path
34
+ object_path = object_path[1..-1] if object_path.start_with?('/')
35
+ object = bucket.object(object_path)
36
+ # puts object.data
37
+ # puts "Path: '#{path}' DPath: '#{destination_path}'"
38
+ object.get(response_target: destination_path) if agent && !agent.dry_run?
39
+ end
40
+
41
+ def upload(config, path, destination_path)
42
+ s3 = initialize_s3_client(config)
43
+ bucket_name = config['bucket_name']
44
+ bucket = s3.bucket(bucket_name)
45
+ object_prefix = config['object_prefix']
46
+ object_path = object_prefix && !object_prefix.empty? ? File.join(object_prefix, destination_path) : path
47
+ object_path = object_path[1..-1] if object_path.start_with?('/')
48
+ object_path = object_path[0..-2] if object_path.end_with?('/')
49
+ object = bucket.object(object_path)
50
+ logger.debug { "Uploading File '#{path}' to '#{destination_path}' Object Path: '#{object_path}'" }
51
+
52
+ object.upload_file(path) if agent && !agent.dry_run?
53
+ object
54
+
55
+ # puts object.data
56
+ # puts "Path: '#{path}' DPath: '#{destination_path}'"
57
+ # abort
58
+ end
59
+
60
+ # S3TransferClient
61
+ end
62
+
63
+ # TransferClient
64
+ end
65
+ # Agent
66
+ end
67
+ # Mam
68
+ end
69
+ # Envoi
70
+ end
@@ -0,0 +1,7 @@
1
+ module Envoi
2
+ module Mam
3
+ class Agent
4
+ VERSION = '1.0.2'.freeze
5
+ end
6
+ end
7
+ end
@@ -0,0 +1,183 @@
1
+ require 'envoi/mam/agent/transfer_client/aspera'
2
+ require 'envoi/mam/agent/transfer_client/s3'
3
+
4
+ require 'ubiquity/iconik/api/utilities'
5
+
6
+ module Envoi
7
+
8
+ module Mam
9
+
10
+ class Iconik
11
+
12
+ class Agent < Envoi::Mam::Agent
13
+
14
+ DEFAULT_FORMAT_NAME = 'ORIGINAL'
15
+ DEFAULT_DESTINATION_PATH = '.'
16
+
17
+ def after_initialize
18
+ @default_format_name = initial_args[:default_format_name] || DEFAULT_FORMAT_NAME
19
+ end
20
+
21
+ def iconik_config
22
+ config['iconik']
23
+ end
24
+
25
+ def initialize_api_client(args = { })
26
+ @api_client = args[:api_client] || begin
27
+ http_host_address = iconik_config['http_host_address']
28
+ application_id = iconik_config['application_id']
29
+ token = iconik_config['token']
30
+ client_args = { }
31
+ client_args[:http_host_address] = http_host_address if http_host_address && !http_host_address.empty?
32
+ client_args[:application_id] = application_id if application_id && !application_id.empty?
33
+ client_args[:token] = token if token && !token.empty?
34
+ Ubiquity::Iconik::API::Utilities.new(client_args)
35
+ end
36
+ end
37
+
38
+ def download(args = { })
39
+
40
+ asset_id = args[:asset_id]
41
+ file_id = args[:file_id]
42
+ if file_id && !file_id.empty?
43
+ asset_file = api_client.asset_file_get(asset_id: asset_id, file_id: file_id)
44
+ else
45
+ format_name = args[:format_name] || @default_format_name
46
+ format = api_client.asset_format_get_by_name(asset_id: asset_id, format_name: format_name.upcase)
47
+ format_id = format['id']
48
+
49
+ asset_files_get_response = api_client.asset_files_get(asset_id: asset_id, generate_signed_url: true)
50
+ asset_files = asset_files_get_response['objects']
51
+
52
+ asset_files_for_format = asset_files.find_all { |f| f['format_id'] == format_id }
53
+
54
+ asset_file = asset_files_for_format.first
55
+ end
56
+
57
+ # file = files.first
58
+ files = [ asset_file ] # just do the first file for now
59
+ files.each do |file|
60
+ begin
61
+ download_file(args, file)
62
+ rescue => e
63
+ logger.warn { "Exception: #{$!}. #{$@.first}" }
64
+ end
65
+ end
66
+ logger.info { 'DONE' }
67
+ end
68
+
69
+ def download_file(args, file)
70
+ logger.debug { "File: #{file}"}
71
+ transfer_type = args[:transfer_type] || ''
72
+
73
+ file_storage_id = file['storage_id']
74
+ file_directory_path = file['directory_path']
75
+ file_path = !file_directory_path || file_directory_path.empty? ? file['original_name'] : File.join(file_directory_path, file['original_name'])
76
+
77
+ file_storage_config = iconik_config['storages'][file_storage_id]
78
+
79
+ unless file_storage_config && !file_storage_config.empty?
80
+ raise Exception, "No configuration found for storage '#{file_storage_id}'"
81
+ end
82
+
83
+ logger.info { "Transferring File Path: '#{file_path}'" }
84
+ preserve_path = args.fetch(:preserve_path, file_storage_config.fetch('preserve_path', true))
85
+
86
+ destination_path = args[:destination_path] || file_storage_config['destination_path'] || DEFAULT_DESTINATION_PATH
87
+ relative_path = preserve_path ? file_directory_path : nil
88
+ relative_path = nil if relative_path == '.'
89
+
90
+ target_path = relative_path ? File.join(destination_path, relative_path) : destination_path
91
+ target_path = target_path[0..-1] if target_path.start_with?('/') && !destination_path.start_with?('/')
92
+
93
+ aspera_config = file_storage_config['aspera']
94
+ if (transfer_type.empty? || transfer_type == :aspera) && (aspera_config && !aspera_config.empty?)
95
+ client = Envoi::Mam::Agent::TransferClient::Aspera.new(agent: self)
96
+ return client.download(aspera_config, file_path, target_path)
97
+ end
98
+
99
+ s3_config = file_storage_config['s3']
100
+ if (transfer_type.empty? || transfer_type == :s3) && (s3_config && !s3_config.empty?)
101
+ target_path = File.expand_path(target_path) if target_path == '.'
102
+ target_path = File.join(target_path, File.basename(file_path))
103
+ client = Envoi::Mam::Agent::TransferClient::S3.new(agent: self)
104
+ return client.download(s3_config, file_path, target_path)
105
+ end
106
+
107
+ logger.warn { "No Supported TransferClient Configuration#{transfer_type && !transfer_type.empty? ? " for transfer type '#{transfer_type}' " : ''}Found in Storage Configuration." }
108
+ end
109
+
110
+ def upload(args = { })
111
+ file_path = args[:file_path]
112
+ if File.directory?(file_path)
113
+ file_paths = Dir.glob(File.join(file_path, '*.*'))
114
+ logger.debug { "File Paths: #{file_paths}"}
115
+ file_paths.map { |fp| upload(args.merge(file_path: fp))}
116
+ return file_paths
117
+ end
118
+ logger.debug { "Preparing to upload '#{file_path}'" }
119
+
120
+ transfer_type = args[:transfer_type] || ''
121
+ storage_id = args[:storage_id]
122
+ path_on_storage = nil
123
+ iconik_storage_config = iconik_config['storages'][storage_id]
124
+
125
+ unless iconik_storage_config && !iconik_storage_config.empty?
126
+ raise Exception, "No configuration found for storage '#{storage_id}'"
127
+ end
128
+
129
+ preserve_path = args.fetch(:preserve_path, iconik_storage_config.fetch('preserve_path', true))
130
+
131
+ destination_path = args[:destination_path] || iconik_storage_config['destination_path'] || '/'
132
+ relative_path = preserve_path ? File.dirname(file_path) : nil
133
+ relative_path = File.expand_path(relative_path) if relative_path == '.'
134
+
135
+ target_path = relative_path ? File.join(destination_path, relative_path) : destination_path
136
+ target_path = target_path[1..-1] if target_path.start_with?('/') && !destination_path.start_with?('/')
137
+
138
+ # abort("FP: #{file_path} TP: #{target_path}")
139
+
140
+ transfer_response = begin
141
+ aspera_config = iconik_storage_config['aspera']
142
+ if (transfer_type.empty? || transfer_type == :aspera) && (aspera_config && !aspera_config.empty?)
143
+ client = Envoi::Mam::Agent::TransferClient::Aspera.new(agent: self)
144
+ resp = client.upload(aspera_config, file_path, target_path)
145
+ end
146
+
147
+ s3_config = iconik_storage_config['s3']
148
+ if (!resp) && (transfer_type.empty? || transfer_type == :s3) && (s3_config && !s3_config.empty?)
149
+ _target_path = target_path
150
+ _target_path = '' if target_path == '.'
151
+ _target_path = File.join(_target_path, File.basename(file_path))
152
+ client = Envoi::Mam::Agent::TransferClient::S3.new(agent: self)
153
+ resp = client.upload(s3_config, file_path, _target_path)
154
+ path_on_storage = _target_path
155
+ end
156
+
157
+ resp
158
+ end
159
+
160
+ if transfer_response.nil?
161
+ logger.warn { "No supported TransferClient configuration#{transfer_type && !transfer_type.empty? ? " for transfer type '#{transfer_type}' " : ''}found in storage configuration." }
162
+ return false
163
+ end
164
+
165
+ unless transfer_response
166
+ logger.warn { ""}
167
+ return false
168
+ end
169
+
170
+ file_size = File.exist?(file_path) ? File.size?(file_path) : 1024
171
+ path_on_storage ||= File.join(target_path, File.basename(file_path))
172
+
173
+ api_client.asset_add_using_file_path(file_path: path_on_storage, storage_id: storage_id, file_size: file_size)
174
+ end
175
+
176
+
177
+ end
178
+
179
+ end
180
+
181
+ end
182
+
183
+ end
@@ -0,0 +1,286 @@
1
+ require 'net/https'
2
+ require 'faraday'
3
+
4
+ require 'ubiquity/mediasilo/api/v3/utilities'
5
+ require 'envoi/mam/agent/transfer_client/aspera'
6
+
7
+ module Envoi
8
+
9
+ module Mam
10
+
11
+ class MediaSilo
12
+
13
+ attr_accessor :logger, :config, :api_client
14
+
15
+ class Agent < Envoi::Mam::Agent
16
+
17
+ DEFAULT_DOWNLOAD_DESTINATION_PATH = '.'
18
+
19
+ class CaseSensitiveHeaderKey < String
20
+ # def downcase; self end
21
+ def capitalize; self end
22
+ end
23
+
24
+ class File < ::File
25
+ def bytesize; size end
26
+ end
27
+
28
+ def initialize_api_client(args = { })
29
+ @api_client = args[:api_client] || begin
30
+ ms_config = config['mediasilo']
31
+ mediasilo_hostname = ms_config['hostname']
32
+ mediasilo_username = ms_config['username']
33
+ mediasilo_password = ms_config['password']
34
+ mediasilo_api_key = ms_config['api_key']
35
+
36
+ client_args = { }
37
+ client_args[:hostname] = mediasilo_hostname if mediasilo_hostname
38
+ client_args[:username] = mediasilo_username if mediasilo_username
39
+ client_args[:password] = mediasilo_password if mediasilo_password
40
+ client_args[:api_key] = mediasilo_api_key if mediasilo_api_key
41
+ Ubiquity::MediaSilo::API::V3::Utilities.new(client_args)
42
+ end
43
+
44
+ end
45
+
46
+ def download(args = { })
47
+ transfer_type = args[:transfer_type]
48
+
49
+ file_path = args[:file_path]
50
+ project_id = args[:project_id]
51
+ folder_id = args[:folder_id]
52
+ asset_id = args[:asset_id]
53
+
54
+ mediasilo_path = args[:mediasilo_path]
55
+ project_name = args[:project_name]
56
+ folder_name = args[:folder_name]
57
+
58
+ if mediasilo_path || project_name || folder_name
59
+ check_path_result = api_client.check_path(mediasilo_path)
60
+ found = check_path_result[:existing]
61
+
62
+ project = found[:project] || { }
63
+ project_id = project['id']
64
+
65
+ folders = found[:folders]
66
+ folder_id = (folders.last || { })['id']
67
+
68
+ asset = found[:asset] || { }
69
+ asset_id = asset['id']
70
+ end
71
+
72
+ asset ||= asset_id
73
+
74
+ destination_file_path = args[:destination_path] || DEFAULT_DOWNLOAD_DESTINATION_PATH
75
+
76
+ case transfer_type
77
+ when :aspera; download_using_aspera(asset_id, destination_file_path, args)
78
+ else
79
+ do_upload_response = download_using_http(asset, destination_file_path, args)
80
+ asset_url = do_upload_response[:asset_url]
81
+ end
82
+
83
+ end
84
+
85
+ def download_using_aspera(asset_id, destination_file_path, args = { })
86
+ derivative_type = args[:asset_derivative_type]
87
+ derivative_type = derivative_type.downcase == 'source' ? 'source' : 'proxy'
88
+ ticket = api_client.aspera_file_download_ticket_create(:asset_id => asset_id, :target => derivative_type)
89
+ return false unless ticket.is_a?(Hash)
90
+
91
+ host = ticket['server']
92
+ username = ticket['username']
93
+ password = ticket['password']
94
+ token = ticket['token']
95
+
96
+ original_file_name = ticket['fileName']
97
+
98
+ file_path = ticket['path']
99
+ target_path = destination_file_path
100
+
101
+ FileUtils.mkdir_p(target_path) if target_path.end_with?('/') && !File.directory?(target_path)
102
+ if target_path.end_with?('/') || File.directory?(target_path)
103
+ target_path = File.join(target_path, original_file_name)
104
+ end
105
+
106
+ aspera_config = { }
107
+ aspera_config['host'] = host
108
+ aspera_config['username'] = username
109
+ aspera_config['password'] = password
110
+ aspera_config['token'] = token
111
+
112
+ client = Envoi::Mam::Agent::TransferClient::Aspera.new(agent: self)
113
+ client.download(aspera_config, file_path, target_path)
114
+ end
115
+
116
+ def download_using_http(asset, destination_file_path, args)
117
+ overwrite = args.fetch(:overwrite, false)
118
+ derivative_type = args[:asset_derivative_type]
119
+ api_client.asset_download_derivative(derivative_type, asset, destination_file_path, overwrite)
120
+ end
121
+
122
+
123
+ def upload(args = { })
124
+ file_path = args[:file_path]
125
+ raise "File not found: '#{file_path}'" unless File.exists?(file_path)
126
+
127
+ if File.directory?(file_path)
128
+ file_paths = Dir.glob(File.join(file_path, '*.*'))
129
+ logger.debug { "File Paths: #{file_paths}"}
130
+ file_paths.map { |fp| upload(args.merge(file_path: fp))}
131
+ return file_paths
132
+ end
133
+ logger.debug { "Preparing to upload '#{file_path}'" }
134
+
135
+ transfer_type = args[:transfer_type]
136
+ project_id = args[:project_id]
137
+ folder_id = args[:folder_id]
138
+ # asset_id = args[:asset_id]
139
+
140
+ mediasilo_path = args[:mediasilo_path]
141
+ project_name = args[:project_name]
142
+ folder_name = args[:folder_name]
143
+
144
+ if mediasilo_path || project_name || folder_name
145
+ mediasilo_path ||= File.join(project_name, folder_name)
146
+
147
+ check_path_result = api_client.check_path(mediasilo_path, false)
148
+ found = check_path_result[:existing]
149
+
150
+ project = found[:project] || { }
151
+ project_id = project['id']
152
+
153
+ folders = found[:folders]
154
+ folder_id = (folders.last || { })['id']
155
+ end
156
+
157
+ # # preserve_path = args.fetch(:preserve_path, vidispine_storage_true)
158
+ # preserve_path = args.fetch(:preserve_path, true)
159
+ #
160
+ # destination_path = args[:destination_path] || DEFAULT_DESTINATION_PATH
161
+ # relative_path = preserve_path ? File.dirname(file_path) : nil
162
+ # relative_path = nil if relative_path == '.'
163
+ #
164
+ # target_path = relative_path ? File.join(destination_path, relative_path) : destination_path
165
+ # target_path = target_path[0..-1] if target_path.start_with?('/') && !destination_path.start_with?('/')
166
+
167
+ # upload file
168
+ case transfer_type
169
+ when :aspera; do_upload_response = upload_using_aspera(file_path, args)
170
+ else
171
+ do_upload_response = upload_using_http(file_path)
172
+ asset_url = do_upload_response[:asset_url]
173
+ end
174
+
175
+ return false unless do_upload_response[:success]
176
+
177
+ asset_create_args = {
178
+ 'projectId' => project_id,
179
+ 'folderId' => folder_id,
180
+ :source_url => asset_url
181
+ }
182
+ response = api_client.asset_create(asset_create_args)
183
+ end
184
+
185
+ def upload_using_aspera(file_path, args = { })
186
+ raise 'Upload using Aspera not yet Implemented.'
187
+
188
+ file_name = File.basename(file_path)
189
+
190
+ ticket = api_client.aspera_file_upload_ticket_create(:file_name => file_name)
191
+
192
+ host = ticket['server']
193
+ username = ticket['username']
194
+ password = ticket['password']
195
+ token = ticket['token']
196
+ destination = ticket['destination']
197
+
198
+ target_path = destination
199
+
200
+ aspera_config = { }
201
+ aspera_config['host'] = host
202
+ aspera_config['username'] = username
203
+ aspera_config['password'] = password
204
+ aspera_config['token'] = token
205
+
206
+ client = AsperaTransferClient.new(agent: self)
207
+ client.upload(aspera_config, file_path, target_path)
208
+ { :success => true }
209
+ end
210
+
211
+ def upload_using_faraday(url, file_path, options = { })
212
+ headers = options[:headers]
213
+
214
+ uri = URI(url)
215
+ connection_url = "#{uri.scheme}://#{uri.host}:#{uri.port}"
216
+ conn = Faraday.new(connection_url, { headers: headers, ssl: { } }) do |f|
217
+ f.adapter :net_http
218
+ end
219
+ payload = Faraday::UploadIO.new(file_path, headers['Content-Type'])
220
+ response = conn.put do |req|
221
+ req.url uri.path
222
+ req.body = payload
223
+ req.headers = headers
224
+ end
225
+ end
226
+
227
+ def upload_using_http(file_path)
228
+ file_name = File.basename(file_path)
229
+
230
+ res = api_client.asset_upload_ticket_create(:file_name => file_name)
231
+ uri_str = res['assetUrl']
232
+
233
+ uri = URI.parse(uri_str)
234
+
235
+ amz_date = res['amzDate']
236
+ amz_acl = res['amzAcl']
237
+ content_type = res['contentType']
238
+ authorization = res['authorization']
239
+
240
+ # file = FileToSend.open(file_path)
241
+ #
242
+ file = File.open(file_path)
243
+
244
+ headers = {
245
+ 'x-amz-date' => amz_date,
246
+ 'x-amz-acl' => amz_acl,
247
+ 'Content-Type' => content_type,
248
+ 'Authorization' => authorization,
249
+ 'Content-Length' => file.size.to_s
250
+ }
251
+
252
+
253
+ response = upload_using_faraday(uri_str, file_path, { headers: headers })
254
+
255
+ unless response
256
+ req = Net::HTTP::Put.new(uri.request_uri)
257
+ req['x-amz-date'] = amz_date
258
+ req['x-amz-acl'] = amz_acl
259
+ req[CaseSensitiveHeaderKey.new('Content-Type')] = content_type
260
+ res[CaseSensitiveHeaderKey.new('Authorization')] = authorization
261
+ # req[CaseSensitiveHeaderKey.new('Transfer-Encoding')] = 'chunked'
262
+ req[CaseSensitiveHeaderKey.new('Content-Length')] = file.size
263
+
264
+ req.body_stream = file
265
+ # req.body = file
266
+
267
+ request = req
268
+ http = Net::HTTP.new(uri.host, uri.port)
269
+ http.use_ssl = true
270
+ logger.debug { %(REQUEST: #{request.method} https://#{http.address}:#{http.port}#{request.path} HEADERS: #{request.to_hash.inspect} #{api_client.http_client.log_request_body and request.request_body_permitted? ? "BODY: #{api_client.http_client.format_body_for_log_output(request)}" : ''}) }
271
+
272
+ response = res = http.request(req)
273
+ end
274
+ logger.debug { %(RESPONSE: #{response.inspect})} #HEADERS: #{response.to_hash.inspect} #{api_client.http_client.log_response_body and response.respond_to?(:body) ? "BODY: #{api_client.http_client.format_body_for_log_output(response)}" : ''}) }
275
+ response_code = response.respond_to?(:status) ? response.status.to_s : response.code
276
+ { :success => response_code.start_with?('2'), asset_url: uri_str }
277
+ end
278
+
279
+
280
+ end
281
+
282
+ end
283
+
284
+ end
285
+
286
+ end