envoi-mam-agent 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +8 -0
- data/.rbenv-gemsets +3 -0
- data/.ruby-version +1 -0
- data/.travis.yml +5 -0
- data/CODE_OF_CONDUCT.md +74 -0
- data/Gemfile +7 -0
- data/LICENSE.txt +21 -0
- data/README.md +43 -0
- data/Rakefile +10 -0
- data/bin/console +17 -0
- data/bin/setup +8 -0
- data/envoi-mam-agent.gemspec +41 -0
- data/exe/envoi-mam-agent +15 -0
- data/lib/envoi/mam/agent.rb +119 -0
- data/lib/envoi/mam/agent/cli.rb +37 -0
- data/lib/envoi/mam/agent/cli/commands.rb +89 -0
- data/lib/envoi/mam/agent/cli/commands/iconik.rb +71 -0
- data/lib/envoi/mam/agent/cli/commands/mediasilo.rb +296 -0
- data/lib/envoi/mam/agent/cli/commands/vidispine.rb +60 -0
- data/lib/envoi/mam/agent/cli/commands/wiredrive.rb +223 -0
- data/lib/envoi/mam/agent/config_service_client.rb +60 -0
- data/lib/envoi/mam/agent/transfer_client.rb +37 -0
- data/lib/envoi/mam/agent/transfer_client/aspera.rb +91 -0
- data/lib/envoi/mam/agent/transfer_client/s3.rb +70 -0
- data/lib/envoi/mam/agent/version.rb +7 -0
- data/lib/envoi/mam/iconik/agent.rb +195 -0
- data/lib/envoi/mam/mediasilo/agent.rb +286 -0
- data/lib/envoi/mam/vidispine/agent.rb +214 -0
- data/lib/envoi/mam/wiredrive/agent.rb +117 -0
- metadata +256 -0
@@ -0,0 +1,60 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'net/http'
|
3
|
+
require 'json'
|
4
|
+
|
5
|
+
module Envoi
|
6
|
+
module Mam
|
7
|
+
class Agent
|
8
|
+
class ConfigServiceClient
|
9
|
+
|
10
|
+
DEFAULT_API_URL = 'https://9nyhwwwjw7.execute-api.us-east-1.amazonaws.com/latest/'
|
11
|
+
|
12
|
+
|
13
|
+
def self.config_get(args = { })
|
14
|
+
app_id = args[:app_id]
|
15
|
+
token = args[:token]
|
16
|
+
api_url = args[:api_url] || DEFAULT_API_URL
|
17
|
+
request_url = "#{api_url}getuserrecord?token=#{token}&application_id=#{app_id}"
|
18
|
+
uri = URI(request_url)
|
19
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
20
|
+
http.use_ssl = true
|
21
|
+
req = Net::HTTP::Get.new(uri.request_uri)
|
22
|
+
response = http.request(req)
|
23
|
+
|
24
|
+
code = response.code
|
25
|
+
body_raw = response.body
|
26
|
+
|
27
|
+
unless code == '200'
|
28
|
+
puts uri.to_s
|
29
|
+
puts uri.request_uri
|
30
|
+
puts code
|
31
|
+
puts body_raw
|
32
|
+
return false
|
33
|
+
end
|
34
|
+
|
35
|
+
body_parsed = JSON.parse(body_raw)
|
36
|
+
result = body_parsed['result']
|
37
|
+
raise "Invalid response from configuration service. No result found. '#{body_raw}'" unless result
|
38
|
+
|
39
|
+
app_data = result.first
|
40
|
+
raise "Invalid response from configuration service. No app data found. '#{body_raw}'" unless app_data
|
41
|
+
|
42
|
+
agent_data_raw = app_data['agentdata']
|
43
|
+
raise "Invalid response from configuration service. No agentdata found. '#{body_raw}'" unless agent_data_raw
|
44
|
+
|
45
|
+
begin
|
46
|
+
agent_data = JSON.parse(agent_data_raw)
|
47
|
+
rescue => e
|
48
|
+
raise "Error parsing agentdata. #{e.message} '#{body_raw}'"
|
49
|
+
end
|
50
|
+
|
51
|
+
agent_data
|
52
|
+
end
|
53
|
+
|
54
|
+
end
|
55
|
+
|
56
|
+
end
|
57
|
+
|
58
|
+
end
|
59
|
+
|
60
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
|
2
|
+
require 'envoi/mam/agent'
|
3
|
+
|
4
|
+
module Envoi
|
5
|
+
module Mam
|
6
|
+
class Agent
|
7
|
+
class TransferClient
|
8
|
+
|
9
|
+
attr_accessor :agent, :logger, :initial_args
|
10
|
+
|
11
|
+
def initialize(args = { })
|
12
|
+
@initial_args = args
|
13
|
+
@agent = args[:agent]
|
14
|
+
initialize_logger(args)
|
15
|
+
|
16
|
+
after_initialize
|
17
|
+
end
|
18
|
+
|
19
|
+
def after_initialize
|
20
|
+
# To be overridden by child class
|
21
|
+
end
|
22
|
+
|
23
|
+
def initialize_logger(args = { })
|
24
|
+
@logger = agent.logger if agent && agent.respond_to?(:logger) && agent.logger
|
25
|
+
end
|
26
|
+
|
27
|
+
# TransferClient
|
28
|
+
end
|
29
|
+
|
30
|
+
# Agent
|
31
|
+
end
|
32
|
+
|
33
|
+
# Mam
|
34
|
+
end
|
35
|
+
|
36
|
+
# Envoi
|
37
|
+
end
|
@@ -0,0 +1,91 @@
|
|
1
|
+
require 'envoi/mam/agent/transfer_client'
|
2
|
+
|
3
|
+
module Envoi
|
4
|
+
module Mam
|
5
|
+
class Agent
|
6
|
+
class TransferClient
|
7
|
+
|
8
|
+
class Aspera < TransferClient
|
9
|
+
|
10
|
+
DEFAULT_DESTINATION_PATH = '.'
|
11
|
+
DEFAULT_ASCP_ARGS = '-v -k3 --overwrite=diff -P 33001'
|
12
|
+
|
13
|
+
ASCP_PATHS = [
|
14
|
+
'/usr/local/bin/ascp',
|
15
|
+
'/usr/bin/ascp',
|
16
|
+
'/Library/Aspera/bin/ascp',
|
17
|
+
'~/Applications/Aspera CLI/bin/ascp',
|
18
|
+
'~/Applications/Aspera Connect.app/Contents/Resources/ascp',
|
19
|
+
'/Applications/Aspera Connect.app/Contents/Resources/ascp'
|
20
|
+
]
|
21
|
+
ASCP_PATHS.map { |v| File.expand_path(v) }
|
22
|
+
|
23
|
+
attr_accessor :default_ascp_path, :ascp_path, :default_ascp_args
|
24
|
+
|
25
|
+
def after_initialize(args = initial_args)
|
26
|
+
@default_ascp_path = args[:default_aspera_ascp_path]
|
27
|
+
@default_ascp_path ||= ASCP_PATHS.find { |v| File.exists? v } || ASCP_PATHS.first
|
28
|
+
@default_ascp_args = args[:default_ascp_args] || args[:default_aspera_ascp_args] || DEFAULT_ASCP_ARGS
|
29
|
+
end
|
30
|
+
|
31
|
+
def ascp_path_exists?
|
32
|
+
File.executable?(@ascp_path)
|
33
|
+
end
|
34
|
+
|
35
|
+
def build_ascp_command(config, mode, source_path, target_path)
|
36
|
+
aspera_host_address = config['host_address'] || config['host']
|
37
|
+
aspera_username = config['username']
|
38
|
+
aspera_password = config['password']
|
39
|
+
aspera_token = config['aspera_transfer_token'] || config['aspera_token'] || config['token'] || config['transfer_token']
|
40
|
+
|
41
|
+
@ascp_path = config['ascp_path'] || default_ascp_path
|
42
|
+
@ascp_path = File.expand_path(@ascp_path)
|
43
|
+
|
44
|
+
aspera_ascp_path = @ascp_path
|
45
|
+
|
46
|
+
ascp_args = config['ascp_args'] || default_ascp_args || agent.default_ascp_args
|
47
|
+
# command = %(export ASPERA_SCP_PASS="#{aspera_password}"; "#{aspera_ascp_path}" #{ascp_args} --host="#{aspera_host_address}" --user=#{aspera_username} --mode=#{mode} "#{source_path}" "#{target_path}" )
|
48
|
+
command = ''
|
49
|
+
command << %(export ASPERA_SCP_PASS="#{aspera_password}"; ) if aspera_password
|
50
|
+
command << %("#{aspera_ascp_path}" --mode=#{mode} --host="#{aspera_host_address}" --user="#{aspera_username}")
|
51
|
+
command << %( #{ascp_args}) if ascp_args && !ascp_args.empty?
|
52
|
+
command << %( -W "#{aspera_token}") if aspera_token && !aspera_token.empty?
|
53
|
+
command << %( "#{source_path.gsub('"', '\"')}" "#{target_path.gsub('"', '\"')}")
|
54
|
+
end
|
55
|
+
|
56
|
+
def download(config, path, destination_path = DEFAULT_DESTINATION_PATH)
|
57
|
+
aspera_base_path = config['base_path'] || ''
|
58
|
+
source_path = File.join(aspera_base_path, path)
|
59
|
+
command = build_ascp_command(config, 'recv', source_path, destination_path)
|
60
|
+
|
61
|
+
unless ascp_path_exists?
|
62
|
+
warn "ASCP not found. '#{ascp_path}'"
|
63
|
+
return false
|
64
|
+
end
|
65
|
+
agent.shell_execute(command)
|
66
|
+
end
|
67
|
+
|
68
|
+
def upload(config, path, destination_path = DEFAULT_DESTINATION_PATH)
|
69
|
+
aspera_base_path = config['base_path'] || ''
|
70
|
+
source_path = path
|
71
|
+
destination_path = aspera_base_path.empty? ? destination_path : File.join(aspera_base_path, destination_path)
|
72
|
+
command = build_ascp_command(config, 'send', source_path, destination_path)
|
73
|
+
|
74
|
+
unless ascp_path_exists?
|
75
|
+
warn "ASCP not found. '#{ascp_path}'"
|
76
|
+
return false
|
77
|
+
end
|
78
|
+
agent.shell_execute(command)
|
79
|
+
end
|
80
|
+
|
81
|
+
# AsperaTransferClient
|
82
|
+
end
|
83
|
+
|
84
|
+
# TransferClient
|
85
|
+
end
|
86
|
+
# Agent
|
87
|
+
end
|
88
|
+
# Mam
|
89
|
+
end
|
90
|
+
# Envoi
|
91
|
+
end
|
@@ -0,0 +1,70 @@
|
|
1
|
+
require 'aws-sdk-s3'
|
2
|
+
|
3
|
+
require 'envoi/mam/agent/transfer_client'
|
4
|
+
|
5
|
+
module Envoi
|
6
|
+
module Mam
|
7
|
+
class Agent
|
8
|
+
class TransferClient
|
9
|
+
|
10
|
+
class S3 < TransferClient
|
11
|
+
|
12
|
+
def after_initialize(args = initial_args)
|
13
|
+
|
14
|
+
end
|
15
|
+
|
16
|
+
def initialize_s3_client(config)
|
17
|
+
region = config['region']
|
18
|
+
access_key_id = config['access_key_id']
|
19
|
+
secret_access_key = config['secret_access_key']
|
20
|
+
|
21
|
+
resource_args = { }
|
22
|
+
resource_args[:region] = region if region && !region.empty?
|
23
|
+
resource_args[:access_key_id] = access_key_id if access_key_id && !access_key_id.empty?
|
24
|
+
resource_args[:secret_access_key] = secret_access_key if secret_access_key && !secret_access_key.empty?
|
25
|
+
@s3 = Aws::S3::Resource.new(resource_args)
|
26
|
+
end
|
27
|
+
|
28
|
+
def download(config, path, destination_path = DEFAULT_DESTINATION_PATH)
|
29
|
+
s3 = initialize_s3_client(config)
|
30
|
+
bucket_name = config['bucket_name']
|
31
|
+
bucket = s3.bucket(bucket_name)
|
32
|
+
object_prefix = config['object_prefix']
|
33
|
+
object_path = object_prefix && !object_prefix.empty? ? File.join(object_prefix, path) : path
|
34
|
+
object_path = object_path[1..-1] if object_path.start_with?('/')
|
35
|
+
object = bucket.object(object_path)
|
36
|
+
# puts object.data
|
37
|
+
# puts "Path: '#{path}' DPath: '#{destination_path}'"
|
38
|
+
object.get(response_target: destination_path) if agent && !agent.dry_run?
|
39
|
+
end
|
40
|
+
|
41
|
+
def upload(config, path, destination_path)
|
42
|
+
s3 = initialize_s3_client(config)
|
43
|
+
bucket_name = config['bucket_name']
|
44
|
+
bucket = s3.bucket(bucket_name)
|
45
|
+
object_prefix = config['object_prefix']
|
46
|
+
object_path = object_prefix && !object_prefix.empty? ? File.join(object_prefix, destination_path) : path
|
47
|
+
object_path = object_path[1..-1] if object_path.start_with?('/')
|
48
|
+
object_path = object_path[0..-2] if object_path.end_with?('/')
|
49
|
+
object = bucket.object(object_path)
|
50
|
+
logger.debug { "Uploading File '#{path}' to '#{destination_path}' Object Path: '#{object_path}'" }
|
51
|
+
|
52
|
+
object.upload_file(path) if agent && !agent.dry_run?
|
53
|
+
object
|
54
|
+
|
55
|
+
# puts object.data
|
56
|
+
# puts "Path: '#{path}' DPath: '#{destination_path}'"
|
57
|
+
# abort
|
58
|
+
end
|
59
|
+
|
60
|
+
# S3TransferClient
|
61
|
+
end
|
62
|
+
|
63
|
+
# TransferClient
|
64
|
+
end
|
65
|
+
# Agent
|
66
|
+
end
|
67
|
+
# Mam
|
68
|
+
end
|
69
|
+
# Envoi
|
70
|
+
end
|
@@ -0,0 +1,195 @@
|
|
1
|
+
require 'envoi/mam/agent/transfer_client/aspera'
|
2
|
+
require 'envoi/mam/agent/transfer_client/s3'
|
3
|
+
|
4
|
+
require 'ubiquity/iconik/api/utilities'
|
5
|
+
|
6
|
+
module Envoi
|
7
|
+
|
8
|
+
module Mam
|
9
|
+
|
10
|
+
class Iconik
|
11
|
+
|
12
|
+
class Agent < Envoi::Mam::Agent
|
13
|
+
|
14
|
+
DEFAULT_FORMAT_NAME = 'ORIGINAL'
|
15
|
+
DEFAULT_DESTINATION_PATH = '.'
|
16
|
+
|
17
|
+
def after_initialize
|
18
|
+
@default_format_name = initial_args[:default_format_name] || DEFAULT_FORMAT_NAME
|
19
|
+
end
|
20
|
+
|
21
|
+
def iconik_config
|
22
|
+
config['iconik']
|
23
|
+
end
|
24
|
+
|
25
|
+
def initialize_api_client(args = { })
|
26
|
+
@api_client = args[:api_client] || begin
|
27
|
+
http_host_address = iconik_config['http_host_address']
|
28
|
+
application_id = iconik_config['application_id']
|
29
|
+
token = iconik_config['token']
|
30
|
+
client_args = { }
|
31
|
+
client_args[:http_host_address] = http_host_address if http_host_address && !http_host_address.empty?
|
32
|
+
client_args[:application_id] = application_id if application_id && !application_id.empty?
|
33
|
+
client_args[:token] = token if token && !token.empty?
|
34
|
+
Ubiquity::Iconik::API::Utilities.new(client_args)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def download(args = { })
|
39
|
+
|
40
|
+
asset_id = args[:asset_id]
|
41
|
+
file_id = args[:file_id]
|
42
|
+
if file_id && !file_id.empty?
|
43
|
+
asset_file = api_client.asset_file_get(asset_id: asset_id, file_id: file_id)
|
44
|
+
else
|
45
|
+
format_name = args[:format_name] || @default_format_name
|
46
|
+
format = api_client.asset_format_get_by_name(asset_id: asset_id, format_name: format_name.upcase)
|
47
|
+
format_id = format['id']
|
48
|
+
|
49
|
+
asset_files_get_response = api_client.asset_files_get(asset_id: asset_id, generate_signed_url: true)
|
50
|
+
asset_files = asset_files_get_response['objects']
|
51
|
+
|
52
|
+
asset_files_for_format = asset_files.find_all { |f| f['format_id'] == format_id }
|
53
|
+
|
54
|
+
asset_file = asset_files_for_format.first
|
55
|
+
end
|
56
|
+
|
57
|
+
# file = files.first
|
58
|
+
files = [ asset_file ] # just do the first file for now
|
59
|
+
files.each do |file|
|
60
|
+
begin
|
61
|
+
download_file(args, file)
|
62
|
+
rescue => e
|
63
|
+
logger.warn { "Exception: #{$!}. #{$@.first}" }
|
64
|
+
end
|
65
|
+
end
|
66
|
+
logger.info { 'DONE' }
|
67
|
+
end
|
68
|
+
|
69
|
+
def download_file(args, file)
|
70
|
+
logger.debug { "File: #{file}"}
|
71
|
+
transfer_type = args[:transfer_type] || ''
|
72
|
+
|
73
|
+
file_storage_id = file['storage_id']
|
74
|
+
file_directory_path = file['directory_path']
|
75
|
+
file_path = !file_directory_path || file_directory_path.empty? ? file['original_name'] : File.join(file_directory_path, file['original_name'])
|
76
|
+
|
77
|
+
file_storage_config = iconik_config['storages'][file_storage_id]
|
78
|
+
|
79
|
+
unless file_storage_config && !file_storage_config.empty?
|
80
|
+
raise Exception, "No configuration found for storage '#{file_storage_id}'"
|
81
|
+
end
|
82
|
+
|
83
|
+
logger.info { "Transferring File Path: '#{file_path}'" }
|
84
|
+
preserve_path = args.fetch(:preserve_path, file_storage_config.fetch('preserve_path', true))
|
85
|
+
|
86
|
+
destination_path = args[:destination_path] || file_storage_config['destination_path'] || DEFAULT_DESTINATION_PATH
|
87
|
+
relative_path = preserve_path ? file_directory_path : nil
|
88
|
+
relative_path = nil if relative_path == '.'
|
89
|
+
|
90
|
+
target_path = relative_path ? File.join(destination_path, relative_path) : destination_path
|
91
|
+
target_path = target_path[0..-1] if target_path.start_with?('/') && !destination_path.start_with?('/')
|
92
|
+
|
93
|
+
aspera_config = file_storage_config['aspera']
|
94
|
+
if (transfer_type.empty? || transfer_type == :aspera) && (aspera_config && !aspera_config.empty?)
|
95
|
+
client = Envoi::Mam::Agent::TransferClient::Aspera.new(agent: self)
|
96
|
+
return client.download(aspera_config, file_path, target_path)
|
97
|
+
end
|
98
|
+
|
99
|
+
s3_config = file_storage_config['s3']
|
100
|
+
if (transfer_type.empty? || transfer_type == :s3) && (s3_config && !s3_config.empty?)
|
101
|
+
target_path = File.expand_path(target_path) if target_path == '.'
|
102
|
+
target_path = File.join(target_path, File.basename(file_path))
|
103
|
+
client = Envoi::Mam::Agent::TransferClient::S3.new(agent: self)
|
104
|
+
return client.download(s3_config, file_path, target_path)
|
105
|
+
end
|
106
|
+
|
107
|
+
logger.warn { "No Supported TransferClient Configuration#{transfer_type && !transfer_type.empty? ? " for transfer type '#{transfer_type}' " : ''}Found in Storage Configuration." }
|
108
|
+
end
|
109
|
+
|
110
|
+
def upload(args = { })
|
111
|
+
file_path = args[:file_path]
|
112
|
+
if File.directory?(file_path)
|
113
|
+
file_paths = Dir.glob(File.join(file_path, '*.*'))
|
114
|
+
logger.debug { "File Paths: #{file_paths}"}
|
115
|
+
return file_paths.map { |fp| upload(args.merge(file_path: fp))}
|
116
|
+
end
|
117
|
+
logger.debug { "Preparing to upload '#{file_path}'" }
|
118
|
+
|
119
|
+
ingest_after_upload = args[:ingest_after_upload]
|
120
|
+
analyze_asset = args[:analyze_asset]
|
121
|
+
|
122
|
+
transfer_type = args[:transfer_type] || ''
|
123
|
+
storage_id = args[:storage_id]
|
124
|
+
path_on_storage = nil
|
125
|
+
iconik_storage_config = iconik_config['storages'][storage_id]
|
126
|
+
|
127
|
+
unless iconik_storage_config && !iconik_storage_config.empty?
|
128
|
+
raise Exception, "No configuration found for storage '#{storage_id}'"
|
129
|
+
end
|
130
|
+
|
131
|
+
preserve_path = args.fetch(:preserve_path, iconik_storage_config.fetch('preserve_path', true))
|
132
|
+
|
133
|
+
destination_path = args[:destination_path] || iconik_storage_config['destination_path'] || '/'
|
134
|
+
relative_path = preserve_path ? File.dirname(file_path) : nil
|
135
|
+
relative_path = File.expand_path(relative_path) if relative_path == '.'
|
136
|
+
|
137
|
+
target_path = relative_path ? File.join(destination_path, relative_path) : destination_path
|
138
|
+
target_path = target_path[1..-1] if target_path.start_with?('/') && !destination_path.start_with?('/')
|
139
|
+
|
140
|
+
# abort("FP: #{file_path} TP: #{target_path}")
|
141
|
+
|
142
|
+
transfer_response = begin
|
143
|
+
aspera_config = iconik_storage_config['aspera']
|
144
|
+
if (transfer_type.empty? || transfer_type == :aspera) && (aspera_config && !aspera_config.empty?)
|
145
|
+
client = Envoi::Mam::Agent::TransferClient::Aspera.new(agent: self)
|
146
|
+
resp = client.upload(aspera_config, file_path, target_path)
|
147
|
+
end
|
148
|
+
|
149
|
+
s3_config = iconik_storage_config['s3']
|
150
|
+
if (!resp) && (transfer_type.empty? || transfer_type == :s3) && (s3_config && !s3_config.empty?)
|
151
|
+
_target_path = target_path
|
152
|
+
_target_path = '' if target_path == '.'
|
153
|
+
_target_path = File.join(_target_path, File.basename(file_path))
|
154
|
+
client = Envoi::Mam::Agent::TransferClient::S3.new(agent: self)
|
155
|
+
resp = client.upload(s3_config, file_path, _target_path)
|
156
|
+
path_on_storage = _target_path
|
157
|
+
end
|
158
|
+
|
159
|
+
resp
|
160
|
+
end
|
161
|
+
|
162
|
+
if transfer_response.nil?
|
163
|
+
logger.warn { "No supported TransferClient configuration#{transfer_type && !transfer_type.empty? ? " for transfer type '#{transfer_type}' " : ''}found in storage configuration." }
|
164
|
+
return false
|
165
|
+
end
|
166
|
+
|
167
|
+
unless transfer_response
|
168
|
+
logger.warn { ""}
|
169
|
+
return false
|
170
|
+
end
|
171
|
+
|
172
|
+
file_size = File.exist?(file_path) ? File.size?(file_path) : 1024
|
173
|
+
path_on_storage ||= File.join(target_path, File.basename(file_path))
|
174
|
+
|
175
|
+
if ingest_after_upload
|
176
|
+
asset_create_response = api_client.asset_add_using_file_path(file_path: path_on_storage,
|
177
|
+
storage_id: storage_id,
|
178
|
+
file_size: file_size)
|
179
|
+
if analyze_asset
|
180
|
+
asset_id = asset_create_response[:asset]['id']
|
181
|
+
api_client.asset_analyze(:asset_id => asset_id)
|
182
|
+
end
|
183
|
+
|
184
|
+
end
|
185
|
+
|
186
|
+
end
|
187
|
+
|
188
|
+
|
189
|
+
end
|
190
|
+
|
191
|
+
end
|
192
|
+
|
193
|
+
end
|
194
|
+
|
195
|
+
end
|