envoi-mam-agent 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,60 @@
1
+ require 'rubygems'
2
+ require 'net/http'
3
+ require 'json'
4
+
5
+ module Envoi
6
+ module Mam
7
+ class Agent
8
+ class ConfigServiceClient
9
+
10
+ DEFAULT_API_URL = 'https://9nyhwwwjw7.execute-api.us-east-1.amazonaws.com/latest/'
11
+
12
+
13
+ def self.config_get(args = { })
14
+ app_id = args[:app_id]
15
+ token = args[:token]
16
+ api_url = args[:api_url] || DEFAULT_API_URL
17
+ request_url = "#{api_url}getuserrecord?token=#{token}&application_id=#{app_id}"
18
+ uri = URI(request_url)
19
+ http = Net::HTTP.new(uri.host, uri.port)
20
+ http.use_ssl = true
21
+ req = Net::HTTP::Get.new(uri.request_uri)
22
+ response = http.request(req)
23
+
24
+ code = response.code
25
+ body_raw = response.body
26
+
27
+ unless code == '200'
28
+ puts uri.to_s
29
+ puts uri.request_uri
30
+ puts code
31
+ puts body_raw
32
+ return false
33
+ end
34
+
35
+ body_parsed = JSON.parse(body_raw)
36
+ result = body_parsed['result']
37
+ raise "Invalid response from configuration service. No result found. '#{body_raw}'" unless result
38
+
39
+ app_data = result.first
40
+ raise "Invalid response from configuration service. No app data found. '#{body_raw}'" unless app_data
41
+
42
+ agent_data_raw = app_data['agentdata']
43
+ raise "Invalid response from configuration service. No agentdata found. '#{body_raw}'" unless agent_data_raw
44
+
45
+ begin
46
+ agent_data = JSON.parse(agent_data_raw)
47
+ rescue => e
48
+ raise "Error parsing agentdata. #{e.message} '#{body_raw}'"
49
+ end
50
+
51
+ agent_data
52
+ end
53
+
54
+ end
55
+
56
+ end
57
+
58
+ end
59
+
60
+ end
@@ -0,0 +1,37 @@
1
+
2
+ require 'envoi/mam/agent'
3
+
4
+ module Envoi
5
+ module Mam
6
+ class Agent
7
+ class TransferClient
8
+
9
+ attr_accessor :agent, :logger, :initial_args
10
+
11
+ def initialize(args = { })
12
+ @initial_args = args
13
+ @agent = args[:agent]
14
+ initialize_logger(args)
15
+
16
+ after_initialize
17
+ end
18
+
19
+ def after_initialize
20
+ # To be overridden by child class
21
+ end
22
+
23
+ def initialize_logger(args = { })
24
+ @logger = agent.logger if agent && agent.respond_to?(:logger) && agent.logger
25
+ end
26
+
27
+ # TransferClient
28
+ end
29
+
30
+ # Agent
31
+ end
32
+
33
+ # Mam
34
+ end
35
+
36
+ # Envoi
37
+ end
@@ -0,0 +1,91 @@
1
+ require 'envoi/mam/agent/transfer_client'
2
+
3
+ module Envoi
4
+ module Mam
5
+ class Agent
6
+ class TransferClient
7
+
8
+ class Aspera < TransferClient
9
+
10
+ DEFAULT_DESTINATION_PATH = '.'
11
+ DEFAULT_ASCP_ARGS = '-v -k3 --overwrite=diff -P 33001'
12
+
13
+ ASCP_PATHS = [
14
+ '/usr/local/bin/ascp',
15
+ '/usr/bin/ascp',
16
+ '/Library/Aspera/bin/ascp',
17
+ '~/Applications/Aspera CLI/bin/ascp',
18
+ '~/Applications/Aspera Connect.app/Contents/Resources/ascp',
19
+ '/Applications/Aspera Connect.app/Contents/Resources/ascp'
20
+ ]
21
+ ASCP_PATHS.map { |v| File.expand_path(v) }
22
+
23
+ attr_accessor :default_ascp_path, :ascp_path, :default_ascp_args
24
+
25
+ def after_initialize(args = initial_args)
26
+ @default_ascp_path = args[:default_aspera_ascp_path]
27
+ @default_ascp_path ||= ASCP_PATHS.find { |v| File.exists? v } || ASCP_PATHS.first
28
+ @default_ascp_args = args[:default_ascp_args] || args[:default_aspera_ascp_args] || DEFAULT_ASCP_ARGS
29
+ end
30
+
31
+ def ascp_path_exists?
32
+ File.executable?(@ascp_path)
33
+ end
34
+
35
+ def build_ascp_command(config, mode, source_path, target_path)
36
+ aspera_host_address = config['host_address'] || config['host']
37
+ aspera_username = config['username']
38
+ aspera_password = config['password']
39
+ aspera_token = config['aspera_transfer_token'] || config['aspera_token'] || config['token'] || config['transfer_token']
40
+
41
+ @ascp_path = config['ascp_path'] || default_ascp_path
42
+ @ascp_path = File.expand_path(@ascp_path)
43
+
44
+ aspera_ascp_path = @ascp_path
45
+
46
+ ascp_args = config['ascp_args'] || default_ascp_args || agent.default_ascp_args
47
+ # command = %(export ASPERA_SCP_PASS="#{aspera_password}"; "#{aspera_ascp_path}" #{ascp_args} --host="#{aspera_host_address}" --user=#{aspera_username} --mode=#{mode} "#{source_path}" "#{target_path}" )
48
+ command = ''
49
+ command << %(export ASPERA_SCP_PASS="#{aspera_password}"; ) if aspera_password
50
+ command << %("#{aspera_ascp_path}" --mode=#{mode} --host="#{aspera_host_address}" --user="#{aspera_username}")
51
+ command << %( #{ascp_args}) if ascp_args && !ascp_args.empty?
52
+ command << %( -W "#{aspera_token}") if aspera_token && !aspera_token.empty?
53
+ command << %( "#{source_path.gsub('"', '\"')}" "#{target_path.gsub('"', '\"')}")
54
+ end
55
+
56
+ def download(config, path, destination_path = DEFAULT_DESTINATION_PATH)
57
+ aspera_base_path = config['base_path'] || ''
58
+ source_path = File.join(aspera_base_path, path)
59
+ command = build_ascp_command(config, 'recv', source_path, destination_path)
60
+
61
+ unless ascp_path_exists?
62
+ warn "ASCP not found. '#{ascp_path}'"
63
+ return false
64
+ end
65
+ agent.shell_execute(command)
66
+ end
67
+
68
+ def upload(config, path, destination_path = DEFAULT_DESTINATION_PATH)
69
+ aspera_base_path = config['base_path'] || ''
70
+ source_path = path
71
+ destination_path = aspera_base_path.empty? ? destination_path : File.join(aspera_base_path, destination_path)
72
+ command = build_ascp_command(config, 'send', source_path, destination_path)
73
+
74
+ unless ascp_path_exists?
75
+ warn "ASCP not found. '#{ascp_path}'"
76
+ return false
77
+ end
78
+ agent.shell_execute(command)
79
+ end
80
+
81
+ # AsperaTransferClient
82
+ end
83
+
84
+ # TransferClient
85
+ end
86
+ # Agent
87
+ end
88
+ # Mam
89
+ end
90
+ # Envoi
91
+ end
@@ -0,0 +1,70 @@
1
+ require 'aws-sdk-s3'
2
+
3
+ require 'envoi/mam/agent/transfer_client'
4
+
5
+ module Envoi
6
+ module Mam
7
+ class Agent
8
+ class TransferClient
9
+
10
+ class S3 < TransferClient
11
+
12
+ def after_initialize(args = initial_args)
13
+
14
+ end
15
+
16
+ def initialize_s3_client(config)
17
+ region = config['region']
18
+ access_key_id = config['access_key_id']
19
+ secret_access_key = config['secret_access_key']
20
+
21
+ resource_args = { }
22
+ resource_args[:region] = region if region && !region.empty?
23
+ resource_args[:access_key_id] = access_key_id if access_key_id && !access_key_id.empty?
24
+ resource_args[:secret_access_key] = secret_access_key if secret_access_key && !secret_access_key.empty?
25
+ @s3 = Aws::S3::Resource.new(resource_args)
26
+ end
27
+
28
+ def download(config, path, destination_path = DEFAULT_DESTINATION_PATH)
29
+ s3 = initialize_s3_client(config)
30
+ bucket_name = config['bucket_name']
31
+ bucket = s3.bucket(bucket_name)
32
+ object_prefix = config['object_prefix']
33
+ object_path = object_prefix && !object_prefix.empty? ? File.join(object_prefix, path) : path
34
+ object_path = object_path[1..-1] if object_path.start_with?('/')
35
+ object = bucket.object(object_path)
36
+ # puts object.data
37
+ # puts "Path: '#{path}' DPath: '#{destination_path}'"
38
+ object.get(response_target: destination_path) if agent && !agent.dry_run?
39
+ end
40
+
41
+ def upload(config, path, destination_path)
42
+ s3 = initialize_s3_client(config)
43
+ bucket_name = config['bucket_name']
44
+ bucket = s3.bucket(bucket_name)
45
+ object_prefix = config['object_prefix']
46
+ object_path = object_prefix && !object_prefix.empty? ? File.join(object_prefix, destination_path) : path
47
+ object_path = object_path[1..-1] if object_path.start_with?('/')
48
+ object_path = object_path[0..-2] if object_path.end_with?('/')
49
+ object = bucket.object(object_path)
50
+ logger.debug { "Uploading File '#{path}' to '#{destination_path}' Object Path: '#{object_path}'" }
51
+
52
+ object.upload_file(path) if agent && !agent.dry_run?
53
+ object
54
+
55
+ # puts object.data
56
+ # puts "Path: '#{path}' DPath: '#{destination_path}'"
57
+ # abort
58
+ end
59
+
60
+ # S3TransferClient
61
+ end
62
+
63
+ # TransferClient
64
+ end
65
+ # Agent
66
+ end
67
+ # Mam
68
+ end
69
+ # Envoi
70
+ end
@@ -0,0 +1,7 @@
1
+ module Envoi
2
+ module Mam
3
+ class Agent
4
+ VERSION = '1.1.0'.freeze
5
+ end
6
+ end
7
+ end
@@ -0,0 +1,195 @@
1
+ require 'envoi/mam/agent/transfer_client/aspera'
2
+ require 'envoi/mam/agent/transfer_client/s3'
3
+
4
+ require 'ubiquity/iconik/api/utilities'
5
+
6
+ module Envoi
7
+
8
+ module Mam
9
+
10
+ class Iconik
11
+
12
+ class Agent < Envoi::Mam::Agent
13
+
14
+ DEFAULT_FORMAT_NAME = 'ORIGINAL'
15
+ DEFAULT_DESTINATION_PATH = '.'
16
+
17
+ def after_initialize
18
+ @default_format_name = initial_args[:default_format_name] || DEFAULT_FORMAT_NAME
19
+ end
20
+
21
+ def iconik_config
22
+ config['iconik']
23
+ end
24
+
25
+ def initialize_api_client(args = { })
26
+ @api_client = args[:api_client] || begin
27
+ http_host_address = iconik_config['http_host_address']
28
+ application_id = iconik_config['application_id']
29
+ token = iconik_config['token']
30
+ client_args = { }
31
+ client_args[:http_host_address] = http_host_address if http_host_address && !http_host_address.empty?
32
+ client_args[:application_id] = application_id if application_id && !application_id.empty?
33
+ client_args[:token] = token if token && !token.empty?
34
+ Ubiquity::Iconik::API::Utilities.new(client_args)
35
+ end
36
+ end
37
+
38
+ def download(args = { })
39
+
40
+ asset_id = args[:asset_id]
41
+ file_id = args[:file_id]
42
+ if file_id && !file_id.empty?
43
+ asset_file = api_client.asset_file_get(asset_id: asset_id, file_id: file_id)
44
+ else
45
+ format_name = args[:format_name] || @default_format_name
46
+ format = api_client.asset_format_get_by_name(asset_id: asset_id, format_name: format_name.upcase)
47
+ format_id = format['id']
48
+
49
+ asset_files_get_response = api_client.asset_files_get(asset_id: asset_id, generate_signed_url: true)
50
+ asset_files = asset_files_get_response['objects']
51
+
52
+ asset_files_for_format = asset_files.find_all { |f| f['format_id'] == format_id }
53
+
54
+ asset_file = asset_files_for_format.first
55
+ end
56
+
57
+ # file = files.first
58
+ files = [ asset_file ] # just do the first file for now
59
+ files.each do |file|
60
+ begin
61
+ download_file(args, file)
62
+ rescue => e
63
+ logger.warn { "Exception: #{$!}. #{$@.first}" }
64
+ end
65
+ end
66
+ logger.info { 'DONE' }
67
+ end
68
+
69
+ def download_file(args, file)
70
+ logger.debug { "File: #{file}"}
71
+ transfer_type = args[:transfer_type] || ''
72
+
73
+ file_storage_id = file['storage_id']
74
+ file_directory_path = file['directory_path']
75
+ file_path = !file_directory_path || file_directory_path.empty? ? file['original_name'] : File.join(file_directory_path, file['original_name'])
76
+
77
+ file_storage_config = iconik_config['storages'][file_storage_id]
78
+
79
+ unless file_storage_config && !file_storage_config.empty?
80
+ raise Exception, "No configuration found for storage '#{file_storage_id}'"
81
+ end
82
+
83
+ logger.info { "Transferring File Path: '#{file_path}'" }
84
+ preserve_path = args.fetch(:preserve_path, file_storage_config.fetch('preserve_path', true))
85
+
86
+ destination_path = args[:destination_path] || file_storage_config['destination_path'] || DEFAULT_DESTINATION_PATH
87
+ relative_path = preserve_path ? file_directory_path : nil
88
+ relative_path = nil if relative_path == '.'
89
+
90
+ target_path = relative_path ? File.join(destination_path, relative_path) : destination_path
91
+ target_path = target_path[0..-1] if target_path.start_with?('/') && !destination_path.start_with?('/')
92
+
93
+ aspera_config = file_storage_config['aspera']
94
+ if (transfer_type.empty? || transfer_type == :aspera) && (aspera_config && !aspera_config.empty?)
95
+ client = Envoi::Mam::Agent::TransferClient::Aspera.new(agent: self)
96
+ return client.download(aspera_config, file_path, target_path)
97
+ end
98
+
99
+ s3_config = file_storage_config['s3']
100
+ if (transfer_type.empty? || transfer_type == :s3) && (s3_config && !s3_config.empty?)
101
+ target_path = File.expand_path(target_path) if target_path == '.'
102
+ target_path = File.join(target_path, File.basename(file_path))
103
+ client = Envoi::Mam::Agent::TransferClient::S3.new(agent: self)
104
+ return client.download(s3_config, file_path, target_path)
105
+ end
106
+
107
+ logger.warn { "No Supported TransferClient Configuration#{transfer_type && !transfer_type.empty? ? " for transfer type '#{transfer_type}' " : ''}Found in Storage Configuration." }
108
+ end
109
+
110
+ def upload(args = { })
111
+ file_path = args[:file_path]
112
+ if File.directory?(file_path)
113
+ file_paths = Dir.glob(File.join(file_path, '*.*'))
114
+ logger.debug { "File Paths: #{file_paths}"}
115
+ return file_paths.map { |fp| upload(args.merge(file_path: fp))}
116
+ end
117
+ logger.debug { "Preparing to upload '#{file_path}'" }
118
+
119
+ ingest_after_upload = args[:ingest_after_upload]
120
+ analyze_asset = args[:analyze_asset]
121
+
122
+ transfer_type = args[:transfer_type] || ''
123
+ storage_id = args[:storage_id]
124
+ path_on_storage = nil
125
+ iconik_storage_config = iconik_config['storages'][storage_id]
126
+
127
+ unless iconik_storage_config && !iconik_storage_config.empty?
128
+ raise Exception, "No configuration found for storage '#{storage_id}'"
129
+ end
130
+
131
+ preserve_path = args.fetch(:preserve_path, iconik_storage_config.fetch('preserve_path', true))
132
+
133
+ destination_path = args[:destination_path] || iconik_storage_config['destination_path'] || '/'
134
+ relative_path = preserve_path ? File.dirname(file_path) : nil
135
+ relative_path = File.expand_path(relative_path) if relative_path == '.'
136
+
137
+ target_path = relative_path ? File.join(destination_path, relative_path) : destination_path
138
+ target_path = target_path[1..-1] if target_path.start_with?('/') && !destination_path.start_with?('/')
139
+
140
+ # abort("FP: #{file_path} TP: #{target_path}")
141
+
142
+ transfer_response = begin
143
+ aspera_config = iconik_storage_config['aspera']
144
+ if (transfer_type.empty? || transfer_type == :aspera) && (aspera_config && !aspera_config.empty?)
145
+ client = Envoi::Mam::Agent::TransferClient::Aspera.new(agent: self)
146
+ resp = client.upload(aspera_config, file_path, target_path)
147
+ end
148
+
149
+ s3_config = iconik_storage_config['s3']
150
+ if (!resp) && (transfer_type.empty? || transfer_type == :s3) && (s3_config && !s3_config.empty?)
151
+ _target_path = target_path
152
+ _target_path = '' if target_path == '.'
153
+ _target_path = File.join(_target_path, File.basename(file_path))
154
+ client = Envoi::Mam::Agent::TransferClient::S3.new(agent: self)
155
+ resp = client.upload(s3_config, file_path, _target_path)
156
+ path_on_storage = _target_path
157
+ end
158
+
159
+ resp
160
+ end
161
+
162
+ if transfer_response.nil?
163
+ logger.warn { "No supported TransferClient configuration#{transfer_type && !transfer_type.empty? ? " for transfer type '#{transfer_type}' " : ''}found in storage configuration." }
164
+ return false
165
+ end
166
+
167
+ unless transfer_response
168
+ logger.warn { ""}
169
+ return false
170
+ end
171
+
172
+ file_size = File.exist?(file_path) ? File.size?(file_path) : 1024
173
+ path_on_storage ||= File.join(target_path, File.basename(file_path))
174
+
175
+ if ingest_after_upload
176
+ asset_create_response = api_client.asset_add_using_file_path(file_path: path_on_storage,
177
+ storage_id: storage_id,
178
+ file_size: file_size)
179
+ if analyze_asset
180
+ asset_id = asset_create_response[:asset]['id']
181
+ api_client.asset_analyze(:asset_id => asset_id)
182
+ end
183
+
184
+ end
185
+
186
+ end
187
+
188
+
189
+ end
190
+
191
+ end
192
+
193
+ end
194
+
195
+ end