cantemo-portal-agent 1.0.9 → 1.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/exe/cantemo-portal-agent +8 -6
- data/lib/cantemo/portal/agent/WatchFolderUtility/foreman.rb +59 -0
- data/lib/cantemo/portal/agent/cli/commands/watch_folders-working.rb +237 -0
- data/lib/cantemo/portal/agent/cli/commands/watch_folders.rb +63 -0
- data/lib/cantemo/portal/agent/version.rb +1 -1
- data/lib/envoi/aspera/watch_service/client.rb +397 -0
- data/lib/envoi/aspera/watch_service/snapshot.rb +11 -0
- data/lib/envoi/aspera/watch_service/subscription.rb +0 -0
- data/lib/envoi/aspera/watch_service/watch_folder.rb +322 -0
- data/lib/envoi/mam/agent/cli/commands/cantemo-agent.rb +62 -0
- data/lib/envoi/mam/agent/cli/commands/cantemo-watch_folders.rb +41 -0
- data/lib/envoi/mam/agent/cli/commands/cantemo.rb +5 -0
- data/lib/envoi/mam/agent/cli/commands/iconik.rb +21 -11
- data/lib/envoi/mam/agent/cli/commands/mediasilo.rb +1 -1
- data/lib/envoi/mam/agent/cli/commands/vidispine.rb +7 -4
- data/lib/envoi/mam/agent/cli/commands/wiredrive.rb +15 -2
- data/lib/envoi/mam/agent/cli/commands.rb +4 -4
- data/lib/envoi/mam/agent/cli.rb +3 -3
- data/lib/envoi/mam/agent/transfer_client/aspera.rb +145 -7
- data/lib/envoi/mam/agent/version.rb +1 -1
- data/lib/envoi/mam/agent/watch_folder_utility/foreman.rb +76 -0
- data/lib/envoi/mam/agent.rb +6 -1
- data/lib/envoi/mam/cantemo/agent/watch_folder_handler-working.rb +111 -0
- data/lib/envoi/mam/cantemo/agent/watch_folder_handler.rb +176 -0
- data/lib/envoi/mam/cantemo/agent/watch_folder_handler_aspera.rb +112 -0
- data/lib/envoi/mam/cantemo/agent.rb +288 -0
- data/lib/envoi/mam/iconik/agent.rb +15 -3
- data/lib/envoi/mam/vidispine/agent.rb +6 -1
- data/lib/envoi/watch_folder_utility/watch_folder/handler/listen.rb +189 -0
- metadata +48 -4
@@ -0,0 +1,322 @@
|
|
1
|
+
require 'json'
|
2
|
+
|
3
|
+
require 'envoi/aspera/watch_service/client'
|
4
|
+
|
5
|
+
module Envoi
|
6
|
+
module Aspera
|
7
|
+
module WatchService
|
8
|
+
|
9
|
+
class WatchFolder
|
10
|
+
class State
|
11
|
+
|
12
|
+
attr_accessor :subscription
|
13
|
+
attr_accessor :known_path_map
|
14
|
+
attr_accessor :details
|
15
|
+
|
16
|
+
|
17
|
+
def initialize
|
18
|
+
@known_path_map = {}
|
19
|
+
@details = {}
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
23
|
+
|
24
|
+
class Subscription
|
25
|
+
|
26
|
+
class Snapshot
|
27
|
+
|
28
|
+
class Entry < Hash
|
29
|
+
|
30
|
+
def initialize(data)
|
31
|
+
merge! data
|
32
|
+
end
|
33
|
+
|
34
|
+
def path;
|
35
|
+
self[:path]
|
36
|
+
end
|
37
|
+
|
38
|
+
def stat;
|
39
|
+
self[:stat] ||= ((json = self[:stat_as_json]) ? JSON.parse(json) : nil)
|
40
|
+
end
|
41
|
+
|
42
|
+
def stat_as_json;
|
43
|
+
self[:stat_as_json]
|
44
|
+
end
|
45
|
+
|
46
|
+
if MatchData.method_defined?(:named_captures)
|
47
|
+
def self.new_from_match_data(match_data)
|
48
|
+
new(match_data.named_captures)
|
49
|
+
end
|
50
|
+
else
|
51
|
+
def self.new_from_match_data(match_data)
|
52
|
+
new(Hash[match_data.names.map(&:to_sym).zip(match_data.captures)])
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
# Entry
|
57
|
+
end
|
58
|
+
|
59
|
+
attr_accessor :subscription, :version, :entries
|
60
|
+
|
61
|
+
def initialize(subscription, version)
|
62
|
+
@subscription = subscription
|
63
|
+
@version = version
|
64
|
+
@entries = nil
|
65
|
+
end
|
66
|
+
|
67
|
+
def client
|
68
|
+
subscription.client
|
69
|
+
end
|
70
|
+
|
71
|
+
def logger
|
72
|
+
client.logger
|
73
|
+
end
|
74
|
+
|
75
|
+
def entries
|
76
|
+
@entries ||= client.subscription_snapshot_entries_get(subscription_id: subscription['identifier'],
|
77
|
+
snapshot_version: version)
|
78
|
+
end
|
79
|
+
|
80
|
+
def differential(from = nil)
|
81
|
+
(@differentials ||= {})[from] ||= differential_no_cache(from: from)
|
82
|
+
end
|
83
|
+
|
84
|
+
def differential_no_cache(args = {})
|
85
|
+
args_out = args.dup
|
86
|
+
args_out[:subscription_id] ||= subscription['identifier']
|
87
|
+
args_out[:to] ||= version
|
88
|
+
client.subscription_snapshot_differential(args_out)
|
89
|
+
end
|
90
|
+
|
91
|
+
def entries_by_path
|
92
|
+
@entries_by_hash ||= Hash[entries.map { |e| [e[:path], e] }]
|
93
|
+
end
|
94
|
+
|
95
|
+
# Snapshot
|
96
|
+
end
|
97
|
+
|
98
|
+
attr_accessor :attributes, :client, :definition, :snapshots
|
99
|
+
|
100
|
+
def initialize(data, client, definition = nil, snapshots = { })
|
101
|
+
@attributes = data
|
102
|
+
@client = client
|
103
|
+
@definition = definition
|
104
|
+
@snapshots = {}
|
105
|
+
end
|
106
|
+
|
107
|
+
def logger
|
108
|
+
client.logger
|
109
|
+
end
|
110
|
+
|
111
|
+
def [](key)
|
112
|
+
attributes[key]
|
113
|
+
end
|
114
|
+
|
115
|
+
def identifier
|
116
|
+
attributes['identifier']
|
117
|
+
end
|
118
|
+
|
119
|
+
alias id identifier
|
120
|
+
|
121
|
+
def snapshot_create(args = {})
|
122
|
+
args_out = args.dup
|
123
|
+
args_out[:subscription_id] = attributes['identifier']
|
124
|
+
snapshot_id = client.subscription_snapshot_create(args_out)
|
125
|
+
snapshots[snapshot_id] ||= Snapshot.new(self, snapshot_id)
|
126
|
+
end
|
127
|
+
|
128
|
+
def resubscribe
|
129
|
+
client.subscription_resubscribe(subscription_id: id)
|
130
|
+
end
|
131
|
+
|
132
|
+
|
133
|
+
def self.get_or_create(client, definition, snapshots = { })
|
134
|
+
watch_folder_path = definition['path']
|
135
|
+
scan_period = definition['scan_period']
|
136
|
+
expire_in = definition['expire_in']
|
137
|
+
|
138
|
+
subscriptions = nil
|
139
|
+
subscriptions_for_path = client.subscription_find_for_path(path: watch_folder_path, subscriptions: subscriptions)
|
140
|
+
subscriptions_for_path.delete_if { |s| s['scan_period']['sec'] != scan_period } if scan_period
|
141
|
+
|
142
|
+
if subscriptions_for_path.empty?
|
143
|
+
args_out = { watch_folder_path: watch_folder_path }
|
144
|
+
args_out[:scan_period] = scan_period if scan_period
|
145
|
+
args_out[:expire_in] = expire_in if expire_in
|
146
|
+
# Create Subscription
|
147
|
+
subscription = client.subscription_create(args_out)
|
148
|
+
else
|
149
|
+
subscription = subscriptions_for_path.first
|
150
|
+
end
|
151
|
+
|
152
|
+
new(subscription, client, definition, snapshots)
|
153
|
+
end
|
154
|
+
|
155
|
+
# Subscription
|
156
|
+
end
|
157
|
+
|
158
|
+
attr_accessor :definition, :state, :client, :watcher,
|
159
|
+
:poll_interval, :previous_poll_time, :last_poll_time
|
160
|
+
|
161
|
+
def initialize(definition, state = nil, client = nil)
|
162
|
+
@definition = definition
|
163
|
+
@state = state || State.new
|
164
|
+
@client = @watcher = client || Client.new
|
165
|
+
@poll_interval = definition.fetch('poll_interval', 15)
|
166
|
+
process_definition(definition)
|
167
|
+
end
|
168
|
+
|
169
|
+
def logger
|
170
|
+
client.logger
|
171
|
+
end
|
172
|
+
|
173
|
+
def path
|
174
|
+
definition['path']
|
175
|
+
end
|
176
|
+
|
177
|
+
def process_definition(watch_folder_def)
|
178
|
+
state.subscription = subscription_get_or_create(watch_folder_def)
|
179
|
+
end
|
180
|
+
|
181
|
+
def subscription_get_or_create(watch_folder_def)
|
182
|
+
Subscription.get_or_create(client, watch_folder_def)
|
183
|
+
end
|
184
|
+
|
185
|
+
def process_stable_entry(entry)
|
186
|
+
logger.debug { "Stable Entry Detected: #{entry.stat}" }
|
187
|
+
end
|
188
|
+
|
189
|
+
def events(from, to)
|
190
|
+
state.subscription.client.subscription_snapshot_differential(subscription_id: state.subscription['identifier'],
|
191
|
+
from: from,
|
192
|
+
to: to)
|
193
|
+
end
|
194
|
+
|
195
|
+
def poll
|
196
|
+
subscription = state.subscription
|
197
|
+
subscription.resubscribe
|
198
|
+
previous_snapshot_version = subscription.snapshots.keys.last
|
199
|
+
|
200
|
+
# Delete other snapshots
|
201
|
+
subscription.snapshots.delete_if { |p, e| p != previous_snapshot_version } if previous_snapshot_version
|
202
|
+
|
203
|
+
snapshot = subscription.snapshot_create
|
204
|
+
@previous_poll_time = @last_poll_time
|
205
|
+
@last_poll_time = Time.now
|
206
|
+
|
207
|
+
no_change = snapshot.version == previous_snapshot_version
|
208
|
+
if !no_change
|
209
|
+
events = snapshot.differential(previous_snapshot_version)
|
210
|
+
logger.debug { "Events: #{events}" }
|
211
|
+
end
|
212
|
+
|
213
|
+
_known_path_map = state.known_path_map.dup
|
214
|
+
|
215
|
+
logger.debug { "Subscription ID: #{subscription['identifier']}" }
|
216
|
+
logger.debug { "Previous Snapshot: #{previous_snapshot_version}" }
|
217
|
+
logger.debug { "Current Snapshot: #{snapshot.version}" }
|
218
|
+
logger.debug { "Known Paths: #{_known_path_map.keys}" }
|
219
|
+
|
220
|
+
current_path_map = snapshot.entries_by_path
|
221
|
+
current_paths = current_path_map.keys
|
222
|
+
|
223
|
+
new_path_map = current_path_map.dup
|
224
|
+
|
225
|
+
deleted_path_map = {}
|
226
|
+
stable_path_map = {}
|
227
|
+
unstable_path_map = {}
|
228
|
+
_known_path_map.delete_if do |p, e|
|
229
|
+
deleted = !no_change && !current_paths.include?(p)
|
230
|
+
if deleted
|
231
|
+
logger.debug { "DELETED '#{p}'" }
|
232
|
+
deleted_path_map[p] = e
|
233
|
+
else
|
234
|
+
new_path_map.delete(p) # The file is not new, so remove it from the new list
|
235
|
+
|
236
|
+
new_entry = current_path_map[p]
|
237
|
+
previous_stat = e.stat.values_at('mtime', 'size')
|
238
|
+
new_stat = new_entry.stat.values_at('mtime', 'size')
|
239
|
+
|
240
|
+
if no_change || new_stat === previous_stat
|
241
|
+
logger.debug { "UNCHANGED '#{p}' '#{new_stat}' == '#{previous_stat}'" }
|
242
|
+
stable_poll_count = e[:stable_poll_count] || 0
|
243
|
+
stable_poll_count += 1
|
244
|
+
e[:stable_poll_count] = stable_poll_count
|
245
|
+
stable_path_map[p] = e
|
246
|
+
else
|
247
|
+
logger.debug { "MODIFIED '#{p}' '#{new_stat}' != '#{previous_stat}'" }
|
248
|
+
e[:stable_poll_count] = 0
|
249
|
+
unstable_path_map[p] = new_entry
|
250
|
+
end
|
251
|
+
end
|
252
|
+
deleted
|
253
|
+
end
|
254
|
+
|
255
|
+
new_path_map.each do |p, e|
|
256
|
+
logger.debug { "CREATED '#{p}' #{e.stat.values_at('mtime', 'size')}" }
|
257
|
+
e[:stable_poll_count] = 0
|
258
|
+
unstable_path_map[p] = e.dup
|
259
|
+
end
|
260
|
+
|
261
|
+
_known_path_map.merge! stable_path_map
|
262
|
+
_known_path_map.merge! unstable_path_map unless no_change
|
263
|
+
state.known_path_map = _known_path_map
|
264
|
+
state.details = {
|
265
|
+
previous_snapshot_version: previous_snapshot_version,
|
266
|
+
maps: {
|
267
|
+
new: new_path_map,
|
268
|
+
deleted: deleted_path_map,
|
269
|
+
stable: stable_path_map,
|
270
|
+
unstable: unstable_path_map
|
271
|
+
}
|
272
|
+
}
|
273
|
+
|
274
|
+
logger.debug { "Known Paths: #{_known_path_map.keys}" }
|
275
|
+
logger.debug { "New Paths: #{new_path_map.map { |p, e| "#{p} #{e.stat}" }}" }
|
276
|
+
|
277
|
+
logger.debug { "Deleted Paths: #{deleted_path_map.keys}" }
|
278
|
+
logger.debug { "Unstable Paths: #{unstable_path_map.keys}" }
|
279
|
+
logger.debug { "Stable Paths: #{stable_path_map.map { |p, e| [p, e[:stable_poll_count]] }}" }
|
280
|
+
|
281
|
+
yield self if block_given?
|
282
|
+
|
283
|
+
end
|
284
|
+
|
285
|
+
def self.process_watch_folder_def(watch_folder_def)
|
286
|
+
new(watch_folder_def)
|
287
|
+
end
|
288
|
+
|
289
|
+
def self.process_watch_folder_defs(watch_folder_defs)
|
290
|
+
|
291
|
+
|
292
|
+
end
|
293
|
+
|
294
|
+
def self.process_watch_folder(watch_folder, &block)
|
295
|
+
watch_folder.poll(&block) if (Time.now - watch_folder.last_poll_time) >= watch_folder.poll_interval
|
296
|
+
end
|
297
|
+
|
298
|
+
def self.process_watch_folders(watch_folders, &block)
|
299
|
+
watch_folders.each { |watch_folder| process_watch_folder(watch_folder, &block) }
|
300
|
+
end
|
301
|
+
|
302
|
+
def self.run_once(watch_folders, &block)
|
303
|
+
process_watch_folders(watch_folders, &block)
|
304
|
+
end
|
305
|
+
|
306
|
+
def self.run(watch_folders, poll_interval = 15, &block)
|
307
|
+
puts 'Starting...'
|
308
|
+
loop do
|
309
|
+
begin
|
310
|
+
run_once(watch_folders, &block)
|
311
|
+
sleep 1
|
312
|
+
rescue SystemExit, Interrupt
|
313
|
+
break
|
314
|
+
end
|
315
|
+
end
|
316
|
+
puts 'Exiting...'
|
317
|
+
end
|
318
|
+
|
319
|
+
end
|
320
|
+
end
|
321
|
+
end
|
322
|
+
end
|
@@ -0,0 +1,62 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
lib_path = __FILE__ == '(irb)' ? File.join(Dir.cwd, 'lib') : File.expand_path('../../../lib', __FILE__)
|
3
|
+
$:.unshift(lib_path) unless $:.include?(lib_path) or !File.exists?(lib_path)
|
4
|
+
require 'rubygems'
|
5
|
+
require 'optparse'
|
6
|
+
require 'open3'
|
7
|
+
|
8
|
+
require 'envoi/mam/agent/cli'
|
9
|
+
require 'envoi/mam/cantemo/agent'
|
10
|
+
|
11
|
+
default_config_file_paths = Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS
|
12
|
+
|
13
|
+
aspera_ascp_paths = Envoi::Mam::Agent::TransferClient::Aspera::ASCP_PATHS
|
14
|
+
default_aspera_ascp_path = aspera_ascp_paths.find { |v| File.exist? v } || aspera_ascp_paths.first
|
15
|
+
|
16
|
+
current_command = ARGV.shift
|
17
|
+
ARGV << '--help' if ARGV.empty?
|
18
|
+
|
19
|
+
args = {
|
20
|
+
:config_file_path => default_config_file_paths,
|
21
|
+
:default_ascp_path => default_aspera_ascp_path,
|
22
|
+
:dry_run => false,
|
23
|
+
:operation => :download,
|
24
|
+
:preserve_path => true,
|
25
|
+
:transfer_type => '',
|
26
|
+
}
|
27
|
+
|
28
|
+
args[:config_service_app_id] = ENV['ENVOI_MAM_AGENT_APP_ID']
|
29
|
+
args[:config_service_app_token] = ENV['ENVOI_MAM_AGENT_APP_TOKEN']
|
30
|
+
args[:config_service_app_url] = ENV['ENVOI_MAM_AGENT_APP_URL']
|
31
|
+
|
32
|
+
op = OptionParser.new
|
33
|
+
op.on('-c', '--config-file-path FILEPATH', 'The path to the configuration file.',
|
34
|
+
"Default Path(s): #{args[:config_file_path]}") { |v| args[:config_file_path] = v }
|
35
|
+
op.on('--agent-app-id ID', '') { |v| args[:config_service_app_id] = v }
|
36
|
+
op.on('--agent-token TOKEN', '') { |v| args[:config_service_app_token] = v }
|
37
|
+
op.on('--agent-service-api URL', '') { |v| args[:config_service_api_url] = v }
|
38
|
+
|
39
|
+
op.on('-i', '--item-id ITEMID', 'The Vidispine item id.') { |v| args[:item_id] = v }
|
40
|
+
op.on('-s', '--shape-id SHAPEID', 'The Vidispine shape id.') { |v| args[:shape_id] = v }
|
41
|
+
op.on('-t', '--shape-tag SHAPETAG', 'The Vidispine shape tag used to find the shape when a shape id is not supplied.') { |v| args[:shape_tag] = v }
|
42
|
+
op.on('--storage-id STORAGEID', 'The Vidispine storage id to use for the transfer.') { |v| args[:storage_id] = v}
|
43
|
+
|
44
|
+
op.on('-d', '--destination-path DIRPATH', 'The destination path to download or upload to.',
|
45
|
+
"default: '.'") { |v| args[:destination_path] = v }
|
46
|
+
op.on('-p','--[no-]preserve-path', "default: true") { |v| args[:preserve_path] = v }
|
47
|
+
op.on('-p', '--operation OP', [ :download, :upload ], 'Transfer Direction.',
|
48
|
+
"default: #{args[:operation]}" ) { |v| args[:operation] = v }
|
49
|
+
op.on('--transfer-type TYPE', [ :aspera, :s3 ],'Force the type of transfer to use. Ex: aspera || s3') { |v| args[:transfer_type] = v }
|
50
|
+
op.on('--transfer-token TOKEN', '') { |v| args[:transfer_token] = v }
|
51
|
+
op.on('--dry-run', 'Run without executing the transfer.') { |v| args[:dry_run] = v }
|
52
|
+
op.on('--help', 'Show this message.') { puts op; exit }
|
53
|
+
op.load
|
54
|
+
op.parse!
|
55
|
+
|
56
|
+
args[:file_path] ||= ARGV.shift
|
57
|
+
args[:destination_path] ||= ARGV.shift unless ARGV.empty?
|
58
|
+
|
59
|
+
agent = Envoi::Mam::Cantemo::Agent.load_config_and_init(args)
|
60
|
+
|
61
|
+
exit
|
62
|
+
agent.run_operation
|
@@ -0,0 +1,41 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
lib_path = __FILE__ == '(irb)' ? File.join(Dir.cwd, 'lib') : File.expand_path('../../../lib', __FILE__)
|
3
|
+
$:.unshift(lib_path) unless $:.include?(lib_path) or !File.exists?(lib_path)
|
4
|
+
require 'rubygems'
|
5
|
+
require 'optparse'
|
6
|
+
require 'open3'
|
7
|
+
|
8
|
+
require 'envoi/mam/agent/cli'
|
9
|
+
require 'envoi/mam/cantemo/agent'
|
10
|
+
require 'aspeeralm/fasp/installation'
|
11
|
+
|
12
|
+
aspera = Asperalm::Fasp::Installation
|
13
|
+
|
14
|
+
default_config_file_paths = Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS
|
15
|
+
|
16
|
+
aspera_ascp_paths = Envoi::Mam::Agent::TransferClient::Aspera::ASCP_PATHS
|
17
|
+
default_aspera_ascp_path = aspera_ascp_paths.find { |v| File.exist? v } || aspera_ascp_paths.first
|
18
|
+
|
19
|
+
current_command = ARGV.shift
|
20
|
+
ARGV << '--help' if ARGV.empty?
|
21
|
+
|
22
|
+
args = {
|
23
|
+
:config_file_path => default_config_file_paths,
|
24
|
+
:default_ascp_path => default_aspera_ascp_path,
|
25
|
+
:dry_run => false,
|
26
|
+
:operation => :download,
|
27
|
+
:preserve_path => true,
|
28
|
+
:transfer_type => '',
|
29
|
+
}
|
30
|
+
|
31
|
+
args[:config_service_app_id] = ENV['ENVOI_MAM_AGENT_APP_ID']
|
32
|
+
args[:config_service_app_token] = ENV['ENVOI_MAM_AGENT_APP_TOKEN']
|
33
|
+
args[:config_service_app_url] = ENV['ENVOI_MAM_AGENT_APP_URL']
|
34
|
+
|
35
|
+
op = OptionParser.new
|
36
|
+
op.on('-c', '--config-file-path FILEPATH', 'The path to the configuration file.',
|
37
|
+
"Default Path(s): #{args[:config_file_path]}") { |v| args[:config_file_path] = v }
|
38
|
+
op.on('--help', 'Show this message.') { puts op; exit }
|
39
|
+
op.load
|
40
|
+
op.parse!
|
41
|
+
|
@@ -10,7 +10,7 @@ require 'envoi/mam/iconik/agent'
|
|
10
10
|
current_command = ARGV.shift
|
11
11
|
ARGV << '--help' if ARGV.empty?
|
12
12
|
|
13
|
-
default_config_file_paths = Envoi::Mam::Agent::
|
13
|
+
default_config_file_paths = Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS
|
14
14
|
|
15
15
|
aspera_ascp_paths = Envoi::Mam::Agent::TransferClient::Aspera::ASCP_PATHS
|
16
16
|
default_aspera_ascp_path = aspera_ascp_paths.find { |v| File.exist? v } || aspera_ascp_paths.first
|
@@ -18,13 +18,14 @@ default_aspera_ascp_path = aspera_ascp_paths.find { |v| File.exist? v } || asper
|
|
18
18
|
# test_args = %w(--download --send-to-envoi --destination-path /tmp)
|
19
19
|
# ARGV.concat test_args
|
20
20
|
args = {
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
21
|
+
:config_file_path => default_config_file_paths,
|
22
|
+
:default_ascp_path => default_aspera_ascp_path,
|
23
|
+
:dry_run => false,
|
24
|
+
:ingest_after_upload => true,
|
25
|
+
:preserve_path => true,
|
26
|
+
:destination_path => File.expand_path('.'),
|
27
|
+
:should_download_presentation => false,
|
28
|
+
:should_add_presentation_assets_to_envoi => false
|
28
29
|
}
|
29
30
|
|
30
31
|
args[:config_service_app_id] = ENV['ENVOI_MAM_AGENT_APP_ID']
|
@@ -40,15 +41,24 @@ op.on('--agent-service-api URL', '') { |v| args[:config_service_api_url] = v }
|
|
40
41
|
op.on('--asset-id ASSETID', 'The Iconik item ID.') { |v| args[:asset_id] = v }
|
41
42
|
op.on('-f', '--format-name FORMATNAME', 'The Iconik format name used to find the file to download.') { |v| args[:format_name] = v }
|
42
43
|
op.on('--storage-id STORAGEID', 'The Iconik storage id to use for the transfer.') { |v| args[:storage_id] = v}
|
43
|
-
|
44
|
+
|
44
45
|
|
45
46
|
op.on('--file-path PATH', '') { |v| args[:file_path] = v }
|
46
|
-
op.on('-d', '--destination-path DIRPATH', 'The destination path to download or upload to.',
|
47
|
+
op.on('-d', '--destination-path DIRPATH', 'The destination path to download or upload to.',
|
48
|
+
"default: '#{args[:destination_path]}'") { |v| args[:destination_path] = v }
|
49
|
+
|
47
50
|
op.on('-p','--[no-]preserve-path', "default: #{args[:preserve_path]}") { |v| args[:preserve_path] = v }
|
48
51
|
|
49
|
-
op.on('-o', '--operation OP', [ :download, :upload ], 'Transfer Direction.',
|
52
|
+
op.on('-o', '--operation OP', [ :download, :upload ], 'Transfer Direction.',
|
53
|
+
"default: #{args[:operation]}" ) { |v| args[:operation] = v }
|
50
54
|
op.on('--transfer-type TYPE', [ :aspera, :s3 ],'Force the type of transfer to use. Ex: aspera || s3') { |v| args[:transfer_type] = v }
|
51
55
|
|
56
|
+
|
57
|
+
op.on('-i', '--[no-]import', 'Determines if an asset is created after a upload completes.',
|
58
|
+
"default: #{args[:ingest_after_upload]}") { |v| args[:ingest_after_upload] = v}
|
59
|
+
op.on('--[no-]analyze-asset', 'Determines if an analyze job is started on an asset after it is imported',
|
60
|
+
"default: #{args[:analyze_asset]}") { |v| args[:analyze_asset] = v }
|
61
|
+
|
52
62
|
# op.on('--dry-run', 'Run without executing the transfer.') { |v| args[:dry_run] = v }
|
53
63
|
op.on('--help', 'Show this message.') { puts op; exit }
|
54
64
|
op.load
|
@@ -11,7 +11,7 @@ require 'envoi/mam/mediasilo/agent'
|
|
11
11
|
current_command = ARGV.shift
|
12
12
|
ARGV << '--help' if ARGV.empty?
|
13
13
|
|
14
|
-
default_config_file_paths = Envoi::Mam::Agent::
|
14
|
+
default_config_file_paths = Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS
|
15
15
|
|
16
16
|
aspera_ascp_paths = Envoi::Mam::Agent::TransferClient::Aspera::ASCP_PATHS
|
17
17
|
default_aspera_ascp_path = aspera_ascp_paths.find { |v| File.exist? v } || aspera_ascp_paths.first
|
@@ -8,7 +8,7 @@ require 'open3'
|
|
8
8
|
require 'envoi/mam/agent/cli'
|
9
9
|
require 'envoi/mam/vidispine/agent'
|
10
10
|
|
11
|
-
default_config_file_paths = Envoi::Mam::Agent::
|
11
|
+
default_config_file_paths = Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS
|
12
12
|
|
13
13
|
aspera_ascp_paths = Envoi::Mam::Agent::TransferClient::Aspera::ASCP_PATHS
|
14
14
|
default_aspera_ascp_path = aspera_ascp_paths.find { |v| File.exist? v } || aspera_ascp_paths.first
|
@@ -30,7 +30,8 @@ args[:config_service_app_token] = ENV['ENVOI_MAM_AGENT_APP_TOKEN']
|
|
30
30
|
args[:config_service_app_url] = ENV['ENVOI_MAM_AGENT_APP_URL']
|
31
31
|
|
32
32
|
op = OptionParser.new
|
33
|
-
op.on('-c', '--config-file-path FILEPATH', 'The path to the configuration file.',
|
33
|
+
op.on('-c', '--config-file-path FILEPATH', 'The path to the configuration file.',
|
34
|
+
"Default Path(s): #{args[:config_file_path]}") { |v| args[:config_file_path] = v }
|
34
35
|
op.on('--agent-app-id ID', '') { |v| args[:config_service_app_id] = v }
|
35
36
|
op.on('--agent-token TOKEN', '') { |v| args[:config_service_app_token] = v }
|
36
37
|
op.on('--agent-service-api URL', '') { |v| args[:config_service_api_url] = v }
|
@@ -40,9 +41,11 @@ op.on('-s', '--shape-id SHAPEID', 'The Vidispine shape id.') { |v| args[:shape_i
|
|
40
41
|
op.on('-t', '--shape-tag SHAPETAG', 'The Vidispine shape tag used to find the shape when a shape id is not supplied.') { |v| args[:shape_tag] = v }
|
41
42
|
op.on('--storage-id STORAGEID', 'The Vidispine storage id to use for the transfer.') { |v| args[:storage_id] = v}
|
42
43
|
|
43
|
-
op.on('-d', '--destination-path DIRPATH', 'The destination path to download or upload to.',
|
44
|
+
op.on('-d', '--destination-path DIRPATH', 'The destination path to download or upload to.',
|
45
|
+
"default: '.'") { |v| args[:destination_path] = v }
|
44
46
|
op.on('-p','--[no-]preserve-path', "default: true") { |v| args[:preserve_path] = v }
|
45
|
-
op.on('-p', '--operation OP', [ :download, :upload ], 'Transfer Direction.',
|
47
|
+
op.on('-p', '--operation OP', [ :download, :upload ], 'Transfer Direction.',
|
48
|
+
"default: #{args[:operation]}" ) { |v| args[:operation] = v }
|
46
49
|
op.on('--transfer-type TYPE', [ :aspera, :s3 ],'Force the type of transfer to use. Ex: aspera || s3') { |v| args[:transfer_type] = v }
|
47
50
|
op.on('--transfer-token TOKEN', '') { |v| args[:transfer_token] = v }
|
48
51
|
op.on('--dry-run', 'Run without executing the transfer.') { |v| args[:dry_run] = v }
|
@@ -11,7 +11,7 @@ require 'envoi/mam/iconik/agent'
|
|
11
11
|
current_command = ARGV.shift
|
12
12
|
ARGV << '--help' if ARGV.empty?
|
13
13
|
|
14
|
-
default_config_file_paths = Envoi::Mam::Agent::
|
14
|
+
default_config_file_paths = Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS
|
15
15
|
# test_args = %w(--download --send-to-envoi --destination-path /tmp)
|
16
16
|
# ARGV.concat test_args
|
17
17
|
args = {
|
@@ -121,6 +121,19 @@ class EnvoiImportUtility
|
|
121
121
|
media_file_id = r['id']
|
122
122
|
raise 'No id found in response' unless media_file_id
|
123
123
|
|
124
|
+
create_args = {
|
125
|
+
'media_file_id' => media_file_id,
|
126
|
+
'shape_type' => 'web',
|
127
|
+
'shape_label' => 'web',
|
128
|
+
:mime => 'video/mp4', # mime_type,
|
129
|
+
:path => url,
|
130
|
+
:name => filename,
|
131
|
+
:storage_key => url_path,
|
132
|
+
:setting_id => original_storage_id,
|
133
|
+
'size' => size || 0
|
134
|
+
}
|
135
|
+
r = api_client.media_file_file_add(create_args)
|
136
|
+
|
124
137
|
video_thumb = asset['thumbnails'].find { |e| e['category'] == 'max' }
|
125
138
|
if video_thumb
|
126
139
|
thumbnail_storage_id = options[:thumbnail_storage_id] || options[:storage_id] || @default_thumbnail_storage_id
|
@@ -176,7 +189,7 @@ if args[:should_add_presentation_assets_to_envoi]
|
|
176
189
|
rescue => e
|
177
190
|
warn "#{e.message} #{$@.first}"
|
178
191
|
end
|
179
|
-
|
192
|
+
break
|
180
193
|
end
|
181
194
|
end
|
182
195
|
|
@@ -70,14 +70,14 @@ end
|
|
70
70
|
|
71
71
|
|
72
72
|
# Execute the command if it is in object form, otherwise it's just a script and executed when we required it
|
73
|
-
if Envoi::Mam::Agent::
|
73
|
+
if Envoi::Mam::Agent::CLI.const_defined?('Commands')
|
74
74
|
command_class_name = command_name.to_s.capitalize
|
75
|
-
if Envoi::Mam::Agent::
|
76
|
-
command_object = Envoi::Mam::Agent::
|
75
|
+
if Envoi::Mam::Agent::CLI::Commands.const_defined?(command_class_name)
|
76
|
+
command_object = Envoi::Mam::Agent::CLI::Commands.const_get(command_class_name.capitalize ) rescue nil
|
77
77
|
end
|
78
78
|
command_object ||= begin
|
79
79
|
command_class_name = command_class_name.upcase
|
80
|
-
Envoi::Mam::Agent::
|
80
|
+
Envoi::Mam::Agent::CLI::Commands.const_defined?(command_class_name) ? Envoi::Mam::Agent::CLI::Commands.const_get(command_class_name ) : false
|
81
81
|
end
|
82
82
|
if command_object
|
83
83
|
command_instance = command_object.new if command_object.respond_to?(:new)
|
data/lib/envoi/mam/agent/cli.rb
CHANGED
@@ -6,11 +6,11 @@ module Envoi
|
|
6
6
|
module Mam
|
7
7
|
class Agent
|
8
8
|
|
9
|
-
class
|
9
|
+
class CLI
|
10
10
|
|
11
11
|
CONFIG_FILE_PATHS = [
|
12
|
-
|
13
|
-
|
12
|
+
'./envoi-mam-agent-config.json',
|
13
|
+
'~/envoi-mam-agent-config.json',
|
14
14
|
]
|
15
15
|
CONFIG_FILE_PATHS.map! { |v| File.expand_path(v) }
|
16
16
|
|