cantemo-portal-agent 1.0.9 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/exe/cantemo-portal-agent +8 -6
- data/lib/cantemo/portal/agent/WatchFolderUtility/foreman.rb +59 -0
- data/lib/cantemo/portal/agent/cli/commands/watch_folders-working.rb +237 -0
- data/lib/cantemo/portal/agent/cli/commands/watch_folders.rb +63 -0
- data/lib/cantemo/portal/agent/version.rb +1 -1
- data/lib/envoi/aspera/watch_service/client.rb +397 -0
- data/lib/envoi/aspera/watch_service/snapshot.rb +11 -0
- data/lib/envoi/aspera/watch_service/subscription.rb +0 -0
- data/lib/envoi/aspera/watch_service/watch_folder.rb +322 -0
- data/lib/envoi/mam/agent/cli/commands/cantemo-agent.rb +62 -0
- data/lib/envoi/mam/agent/cli/commands/cantemo-watch_folders.rb +41 -0
- data/lib/envoi/mam/agent/cli/commands/cantemo.rb +5 -0
- data/lib/envoi/mam/agent/cli/commands/iconik.rb +21 -11
- data/lib/envoi/mam/agent/cli/commands/mediasilo.rb +1 -1
- data/lib/envoi/mam/agent/cli/commands/vidispine.rb +7 -4
- data/lib/envoi/mam/agent/cli/commands/wiredrive.rb +15 -2
- data/lib/envoi/mam/agent/cli/commands.rb +4 -4
- data/lib/envoi/mam/agent/cli.rb +3 -3
- data/lib/envoi/mam/agent/transfer_client/aspera.rb +145 -7
- data/lib/envoi/mam/agent/version.rb +1 -1
- data/lib/envoi/mam/agent/watch_folder_utility/foreman.rb +76 -0
- data/lib/envoi/mam/agent.rb +6 -1
- data/lib/envoi/mam/cantemo/agent/watch_folder_handler-working.rb +111 -0
- data/lib/envoi/mam/cantemo/agent/watch_folder_handler.rb +176 -0
- data/lib/envoi/mam/cantemo/agent/watch_folder_handler_aspera.rb +112 -0
- data/lib/envoi/mam/cantemo/agent.rb +288 -0
- data/lib/envoi/mam/iconik/agent.rb +15 -3
- data/lib/envoi/mam/vidispine/agent.rb +6 -1
- data/lib/envoi/watch_folder_utility/watch_folder/handler/listen.rb +189 -0
- metadata +48 -4
@@ -0,0 +1,322 @@
|
|
1
|
+
require 'json'
|
2
|
+
|
3
|
+
require 'envoi/aspera/watch_service/client'
|
4
|
+
|
5
|
+
module Envoi
|
6
|
+
module Aspera
|
7
|
+
module WatchService
|
8
|
+
|
9
|
+
class WatchFolder
|
10
|
+
class State
|
11
|
+
|
12
|
+
attr_accessor :subscription
|
13
|
+
attr_accessor :known_path_map
|
14
|
+
attr_accessor :details
|
15
|
+
|
16
|
+
|
17
|
+
def initialize
|
18
|
+
@known_path_map = {}
|
19
|
+
@details = {}
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
23
|
+
|
24
|
+
class Subscription
|
25
|
+
|
26
|
+
class Snapshot
|
27
|
+
|
28
|
+
class Entry < Hash
|
29
|
+
|
30
|
+
def initialize(data)
|
31
|
+
merge! data
|
32
|
+
end
|
33
|
+
|
34
|
+
def path;
|
35
|
+
self[:path]
|
36
|
+
end
|
37
|
+
|
38
|
+
def stat;
|
39
|
+
self[:stat] ||= ((json = self[:stat_as_json]) ? JSON.parse(json) : nil)
|
40
|
+
end
|
41
|
+
|
42
|
+
def stat_as_json;
|
43
|
+
self[:stat_as_json]
|
44
|
+
end
|
45
|
+
|
46
|
+
if MatchData.method_defined?(:named_captures)
|
47
|
+
def self.new_from_match_data(match_data)
|
48
|
+
new(match_data.named_captures)
|
49
|
+
end
|
50
|
+
else
|
51
|
+
def self.new_from_match_data(match_data)
|
52
|
+
new(Hash[match_data.names.map(&:to_sym).zip(match_data.captures)])
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
# Entry
|
57
|
+
end
|
58
|
+
|
59
|
+
attr_accessor :subscription, :version, :entries
|
60
|
+
|
61
|
+
def initialize(subscription, version)
|
62
|
+
@subscription = subscription
|
63
|
+
@version = version
|
64
|
+
@entries = nil
|
65
|
+
end
|
66
|
+
|
67
|
+
def client
|
68
|
+
subscription.client
|
69
|
+
end
|
70
|
+
|
71
|
+
def logger
|
72
|
+
client.logger
|
73
|
+
end
|
74
|
+
|
75
|
+
def entries
|
76
|
+
@entries ||= client.subscription_snapshot_entries_get(subscription_id: subscription['identifier'],
|
77
|
+
snapshot_version: version)
|
78
|
+
end
|
79
|
+
|
80
|
+
def differential(from = nil)
|
81
|
+
(@differentials ||= {})[from] ||= differential_no_cache(from: from)
|
82
|
+
end
|
83
|
+
|
84
|
+
def differential_no_cache(args = {})
|
85
|
+
args_out = args.dup
|
86
|
+
args_out[:subscription_id] ||= subscription['identifier']
|
87
|
+
args_out[:to] ||= version
|
88
|
+
client.subscription_snapshot_differential(args_out)
|
89
|
+
end
|
90
|
+
|
91
|
+
def entries_by_path
|
92
|
+
@entries_by_hash ||= Hash[entries.map { |e| [e[:path], e] }]
|
93
|
+
end
|
94
|
+
|
95
|
+
# Snapshot
|
96
|
+
end
|
97
|
+
|
98
|
+
attr_accessor :attributes, :client, :definition, :snapshots
|
99
|
+
|
100
|
+
def initialize(data, client, definition = nil, snapshots = { })
|
101
|
+
@attributes = data
|
102
|
+
@client = client
|
103
|
+
@definition = definition
|
104
|
+
@snapshots = {}
|
105
|
+
end
|
106
|
+
|
107
|
+
def logger
|
108
|
+
client.logger
|
109
|
+
end
|
110
|
+
|
111
|
+
def [](key)
|
112
|
+
attributes[key]
|
113
|
+
end
|
114
|
+
|
115
|
+
def identifier
|
116
|
+
attributes['identifier']
|
117
|
+
end
|
118
|
+
|
119
|
+
alias id identifier
|
120
|
+
|
121
|
+
def snapshot_create(args = {})
|
122
|
+
args_out = args.dup
|
123
|
+
args_out[:subscription_id] = attributes['identifier']
|
124
|
+
snapshot_id = client.subscription_snapshot_create(args_out)
|
125
|
+
snapshots[snapshot_id] ||= Snapshot.new(self, snapshot_id)
|
126
|
+
end
|
127
|
+
|
128
|
+
def resubscribe
|
129
|
+
client.subscription_resubscribe(subscription_id: id)
|
130
|
+
end
|
131
|
+
|
132
|
+
|
133
|
+
def self.get_or_create(client, definition, snapshots = { })
|
134
|
+
watch_folder_path = definition['path']
|
135
|
+
scan_period = definition['scan_period']
|
136
|
+
expire_in = definition['expire_in']
|
137
|
+
|
138
|
+
subscriptions = nil
|
139
|
+
subscriptions_for_path = client.subscription_find_for_path(path: watch_folder_path, subscriptions: subscriptions)
|
140
|
+
subscriptions_for_path.delete_if { |s| s['scan_period']['sec'] != scan_period } if scan_period
|
141
|
+
|
142
|
+
if subscriptions_for_path.empty?
|
143
|
+
args_out = { watch_folder_path: watch_folder_path }
|
144
|
+
args_out[:scan_period] = scan_period if scan_period
|
145
|
+
args_out[:expire_in] = expire_in if expire_in
|
146
|
+
# Create Subscription
|
147
|
+
subscription = client.subscription_create(args_out)
|
148
|
+
else
|
149
|
+
subscription = subscriptions_for_path.first
|
150
|
+
end
|
151
|
+
|
152
|
+
new(subscription, client, definition, snapshots)
|
153
|
+
end
|
154
|
+
|
155
|
+
# Subscription
|
156
|
+
end
|
157
|
+
|
158
|
+
attr_accessor :definition, :state, :client, :watcher,
|
159
|
+
:poll_interval, :previous_poll_time, :last_poll_time
|
160
|
+
|
161
|
+
def initialize(definition, state = nil, client = nil)
|
162
|
+
@definition = definition
|
163
|
+
@state = state || State.new
|
164
|
+
@client = @watcher = client || Client.new
|
165
|
+
@poll_interval = definition.fetch('poll_interval', 15)
|
166
|
+
process_definition(definition)
|
167
|
+
end
|
168
|
+
|
169
|
+
def logger
|
170
|
+
client.logger
|
171
|
+
end
|
172
|
+
|
173
|
+
def path
|
174
|
+
definition['path']
|
175
|
+
end
|
176
|
+
|
177
|
+
def process_definition(watch_folder_def)
|
178
|
+
state.subscription = subscription_get_or_create(watch_folder_def)
|
179
|
+
end
|
180
|
+
|
181
|
+
def subscription_get_or_create(watch_folder_def)
|
182
|
+
Subscription.get_or_create(client, watch_folder_def)
|
183
|
+
end
|
184
|
+
|
185
|
+
def process_stable_entry(entry)
|
186
|
+
logger.debug { "Stable Entry Detected: #{entry.stat}" }
|
187
|
+
end
|
188
|
+
|
189
|
+
def events(from, to)
|
190
|
+
state.subscription.client.subscription_snapshot_differential(subscription_id: state.subscription['identifier'],
|
191
|
+
from: from,
|
192
|
+
to: to)
|
193
|
+
end
|
194
|
+
|
195
|
+
def poll
|
196
|
+
subscription = state.subscription
|
197
|
+
subscription.resubscribe
|
198
|
+
previous_snapshot_version = subscription.snapshots.keys.last
|
199
|
+
|
200
|
+
# Delete other snapshots
|
201
|
+
subscription.snapshots.delete_if { |p, e| p != previous_snapshot_version } if previous_snapshot_version
|
202
|
+
|
203
|
+
snapshot = subscription.snapshot_create
|
204
|
+
@previous_poll_time = @last_poll_time
|
205
|
+
@last_poll_time = Time.now
|
206
|
+
|
207
|
+
no_change = snapshot.version == previous_snapshot_version
|
208
|
+
if !no_change
|
209
|
+
events = snapshot.differential(previous_snapshot_version)
|
210
|
+
logger.debug { "Events: #{events}" }
|
211
|
+
end
|
212
|
+
|
213
|
+
_known_path_map = state.known_path_map.dup
|
214
|
+
|
215
|
+
logger.debug { "Subscription ID: #{subscription['identifier']}" }
|
216
|
+
logger.debug { "Previous Snapshot: #{previous_snapshot_version}" }
|
217
|
+
logger.debug { "Current Snapshot: #{snapshot.version}" }
|
218
|
+
logger.debug { "Known Paths: #{_known_path_map.keys}" }
|
219
|
+
|
220
|
+
current_path_map = snapshot.entries_by_path
|
221
|
+
current_paths = current_path_map.keys
|
222
|
+
|
223
|
+
new_path_map = current_path_map.dup
|
224
|
+
|
225
|
+
deleted_path_map = {}
|
226
|
+
stable_path_map = {}
|
227
|
+
unstable_path_map = {}
|
228
|
+
_known_path_map.delete_if do |p, e|
|
229
|
+
deleted = !no_change && !current_paths.include?(p)
|
230
|
+
if deleted
|
231
|
+
logger.debug { "DELETED '#{p}'" }
|
232
|
+
deleted_path_map[p] = e
|
233
|
+
else
|
234
|
+
new_path_map.delete(p) # The file is not new, so remove it from the new list
|
235
|
+
|
236
|
+
new_entry = current_path_map[p]
|
237
|
+
previous_stat = e.stat.values_at('mtime', 'size')
|
238
|
+
new_stat = new_entry.stat.values_at('mtime', 'size')
|
239
|
+
|
240
|
+
if no_change || new_stat === previous_stat
|
241
|
+
logger.debug { "UNCHANGED '#{p}' '#{new_stat}' == '#{previous_stat}'" }
|
242
|
+
stable_poll_count = e[:stable_poll_count] || 0
|
243
|
+
stable_poll_count += 1
|
244
|
+
e[:stable_poll_count] = stable_poll_count
|
245
|
+
stable_path_map[p] = e
|
246
|
+
else
|
247
|
+
logger.debug { "MODIFIED '#{p}' '#{new_stat}' != '#{previous_stat}'" }
|
248
|
+
e[:stable_poll_count] = 0
|
249
|
+
unstable_path_map[p] = new_entry
|
250
|
+
end
|
251
|
+
end
|
252
|
+
deleted
|
253
|
+
end
|
254
|
+
|
255
|
+
new_path_map.each do |p, e|
|
256
|
+
logger.debug { "CREATED '#{p}' #{e.stat.values_at('mtime', 'size')}" }
|
257
|
+
e[:stable_poll_count] = 0
|
258
|
+
unstable_path_map[p] = e.dup
|
259
|
+
end
|
260
|
+
|
261
|
+
_known_path_map.merge! stable_path_map
|
262
|
+
_known_path_map.merge! unstable_path_map unless no_change
|
263
|
+
state.known_path_map = _known_path_map
|
264
|
+
state.details = {
|
265
|
+
previous_snapshot_version: previous_snapshot_version,
|
266
|
+
maps: {
|
267
|
+
new: new_path_map,
|
268
|
+
deleted: deleted_path_map,
|
269
|
+
stable: stable_path_map,
|
270
|
+
unstable: unstable_path_map
|
271
|
+
}
|
272
|
+
}
|
273
|
+
|
274
|
+
logger.debug { "Known Paths: #{_known_path_map.keys}" }
|
275
|
+
logger.debug { "New Paths: #{new_path_map.map { |p, e| "#{p} #{e.stat}" }}" }
|
276
|
+
|
277
|
+
logger.debug { "Deleted Paths: #{deleted_path_map.keys}" }
|
278
|
+
logger.debug { "Unstable Paths: #{unstable_path_map.keys}" }
|
279
|
+
logger.debug { "Stable Paths: #{stable_path_map.map { |p, e| [p, e[:stable_poll_count]] }}" }
|
280
|
+
|
281
|
+
yield self if block_given?
|
282
|
+
|
283
|
+
end
|
284
|
+
|
285
|
+
def self.process_watch_folder_def(watch_folder_def)
|
286
|
+
new(watch_folder_def)
|
287
|
+
end
|
288
|
+
|
289
|
+
def self.process_watch_folder_defs(watch_folder_defs)
|
290
|
+
|
291
|
+
|
292
|
+
end
|
293
|
+
|
294
|
+
def self.process_watch_folder(watch_folder, &block)
|
295
|
+
watch_folder.poll(&block) if (Time.now - watch_folder.last_poll_time) >= watch_folder.poll_interval
|
296
|
+
end
|
297
|
+
|
298
|
+
def self.process_watch_folders(watch_folders, &block)
|
299
|
+
watch_folders.each { |watch_folder| process_watch_folder(watch_folder, &block) }
|
300
|
+
end
|
301
|
+
|
302
|
+
def self.run_once(watch_folders, &block)
|
303
|
+
process_watch_folders(watch_folders, &block)
|
304
|
+
end
|
305
|
+
|
306
|
+
def self.run(watch_folders, poll_interval = 15, &block)
|
307
|
+
puts 'Starting...'
|
308
|
+
loop do
|
309
|
+
begin
|
310
|
+
run_once(watch_folders, &block)
|
311
|
+
sleep 1
|
312
|
+
rescue SystemExit, Interrupt
|
313
|
+
break
|
314
|
+
end
|
315
|
+
end
|
316
|
+
puts 'Exiting...'
|
317
|
+
end
|
318
|
+
|
319
|
+
end
|
320
|
+
end
|
321
|
+
end
|
322
|
+
end
|
@@ -0,0 +1,62 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
lib_path = __FILE__ == '(irb)' ? File.join(Dir.cwd, 'lib') : File.expand_path('../../../lib', __FILE__)
|
3
|
+
$:.unshift(lib_path) unless $:.include?(lib_path) or !File.exists?(lib_path)
|
4
|
+
require 'rubygems'
|
5
|
+
require 'optparse'
|
6
|
+
require 'open3'
|
7
|
+
|
8
|
+
require 'envoi/mam/agent/cli'
|
9
|
+
require 'envoi/mam/cantemo/agent'
|
10
|
+
|
11
|
+
default_config_file_paths = Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS
|
12
|
+
|
13
|
+
aspera_ascp_paths = Envoi::Mam::Agent::TransferClient::Aspera::ASCP_PATHS
|
14
|
+
default_aspera_ascp_path = aspera_ascp_paths.find { |v| File.exist? v } || aspera_ascp_paths.first
|
15
|
+
|
16
|
+
current_command = ARGV.shift
|
17
|
+
ARGV << '--help' if ARGV.empty?
|
18
|
+
|
19
|
+
args = {
|
20
|
+
:config_file_path => default_config_file_paths,
|
21
|
+
:default_ascp_path => default_aspera_ascp_path,
|
22
|
+
:dry_run => false,
|
23
|
+
:operation => :download,
|
24
|
+
:preserve_path => true,
|
25
|
+
:transfer_type => '',
|
26
|
+
}
|
27
|
+
|
28
|
+
args[:config_service_app_id] = ENV['ENVOI_MAM_AGENT_APP_ID']
|
29
|
+
args[:config_service_app_token] = ENV['ENVOI_MAM_AGENT_APP_TOKEN']
|
30
|
+
args[:config_service_app_url] = ENV['ENVOI_MAM_AGENT_APP_URL']
|
31
|
+
|
32
|
+
op = OptionParser.new
|
33
|
+
op.on('-c', '--config-file-path FILEPATH', 'The path to the configuration file.',
|
34
|
+
"Default Path(s): #{args[:config_file_path]}") { |v| args[:config_file_path] = v }
|
35
|
+
op.on('--agent-app-id ID', '') { |v| args[:config_service_app_id] = v }
|
36
|
+
op.on('--agent-token TOKEN', '') { |v| args[:config_service_app_token] = v }
|
37
|
+
op.on('--agent-service-api URL', '') { |v| args[:config_service_api_url] = v }
|
38
|
+
|
39
|
+
op.on('-i', '--item-id ITEMID', 'The Vidispine item id.') { |v| args[:item_id] = v }
|
40
|
+
op.on('-s', '--shape-id SHAPEID', 'The Vidispine shape id.') { |v| args[:shape_id] = v }
|
41
|
+
op.on('-t', '--shape-tag SHAPETAG', 'The Vidispine shape tag used to find the shape when a shape id is not supplied.') { |v| args[:shape_tag] = v }
|
42
|
+
op.on('--storage-id STORAGEID', 'The Vidispine storage id to use for the transfer.') { |v| args[:storage_id] = v}
|
43
|
+
|
44
|
+
op.on('-d', '--destination-path DIRPATH', 'The destination path to download or upload to.',
|
45
|
+
"default: '.'") { |v| args[:destination_path] = v }
|
46
|
+
op.on('-p','--[no-]preserve-path', "default: true") { |v| args[:preserve_path] = v }
|
47
|
+
op.on('-p', '--operation OP', [ :download, :upload ], 'Transfer Direction.',
|
48
|
+
"default: #{args[:operation]}" ) { |v| args[:operation] = v }
|
49
|
+
op.on('--transfer-type TYPE', [ :aspera, :s3 ],'Force the type of transfer to use. Ex: aspera || s3') { |v| args[:transfer_type] = v }
|
50
|
+
op.on('--transfer-token TOKEN', '') { |v| args[:transfer_token] = v }
|
51
|
+
op.on('--dry-run', 'Run without executing the transfer.') { |v| args[:dry_run] = v }
|
52
|
+
op.on('--help', 'Show this message.') { puts op; exit }
|
53
|
+
op.load
|
54
|
+
op.parse!
|
55
|
+
|
56
|
+
args[:file_path] ||= ARGV.shift
|
57
|
+
args[:destination_path] ||= ARGV.shift unless ARGV.empty?
|
58
|
+
|
59
|
+
agent = Envoi::Mam::Cantemo::Agent.load_config_and_init(args)
|
60
|
+
|
61
|
+
exit
|
62
|
+
agent.run_operation
|
@@ -0,0 +1,41 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
lib_path = __FILE__ == '(irb)' ? File.join(Dir.cwd, 'lib') : File.expand_path('../../../lib', __FILE__)
|
3
|
+
$:.unshift(lib_path) unless $:.include?(lib_path) or !File.exists?(lib_path)
|
4
|
+
require 'rubygems'
|
5
|
+
require 'optparse'
|
6
|
+
require 'open3'
|
7
|
+
|
8
|
+
require 'envoi/mam/agent/cli'
|
9
|
+
require 'envoi/mam/cantemo/agent'
|
10
|
+
require 'aspeeralm/fasp/installation'
|
11
|
+
|
12
|
+
aspera = Asperalm::Fasp::Installation
|
13
|
+
|
14
|
+
default_config_file_paths = Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS
|
15
|
+
|
16
|
+
aspera_ascp_paths = Envoi::Mam::Agent::TransferClient::Aspera::ASCP_PATHS
|
17
|
+
default_aspera_ascp_path = aspera_ascp_paths.find { |v| File.exist? v } || aspera_ascp_paths.first
|
18
|
+
|
19
|
+
current_command = ARGV.shift
|
20
|
+
ARGV << '--help' if ARGV.empty?
|
21
|
+
|
22
|
+
args = {
|
23
|
+
:config_file_path => default_config_file_paths,
|
24
|
+
:default_ascp_path => default_aspera_ascp_path,
|
25
|
+
:dry_run => false,
|
26
|
+
:operation => :download,
|
27
|
+
:preserve_path => true,
|
28
|
+
:transfer_type => '',
|
29
|
+
}
|
30
|
+
|
31
|
+
args[:config_service_app_id] = ENV['ENVOI_MAM_AGENT_APP_ID']
|
32
|
+
args[:config_service_app_token] = ENV['ENVOI_MAM_AGENT_APP_TOKEN']
|
33
|
+
args[:config_service_app_url] = ENV['ENVOI_MAM_AGENT_APP_URL']
|
34
|
+
|
35
|
+
op = OptionParser.new
|
36
|
+
op.on('-c', '--config-file-path FILEPATH', 'The path to the configuration file.',
|
37
|
+
"Default Path(s): #{args[:config_file_path]}") { |v| args[:config_file_path] = v }
|
38
|
+
op.on('--help', 'Show this message.') { puts op; exit }
|
39
|
+
op.load
|
40
|
+
op.parse!
|
41
|
+
|
@@ -10,7 +10,7 @@ require 'envoi/mam/iconik/agent'
|
|
10
10
|
current_command = ARGV.shift
|
11
11
|
ARGV << '--help' if ARGV.empty?
|
12
12
|
|
13
|
-
default_config_file_paths = Envoi::Mam::Agent::
|
13
|
+
default_config_file_paths = Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS
|
14
14
|
|
15
15
|
aspera_ascp_paths = Envoi::Mam::Agent::TransferClient::Aspera::ASCP_PATHS
|
16
16
|
default_aspera_ascp_path = aspera_ascp_paths.find { |v| File.exist? v } || aspera_ascp_paths.first
|
@@ -18,13 +18,14 @@ default_aspera_ascp_path = aspera_ascp_paths.find { |v| File.exist? v } || asper
|
|
18
18
|
# test_args = %w(--download --send-to-envoi --destination-path /tmp)
|
19
19
|
# ARGV.concat test_args
|
20
20
|
args = {
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
21
|
+
:config_file_path => default_config_file_paths,
|
22
|
+
:default_ascp_path => default_aspera_ascp_path,
|
23
|
+
:dry_run => false,
|
24
|
+
:ingest_after_upload => true,
|
25
|
+
:preserve_path => true,
|
26
|
+
:destination_path => File.expand_path('.'),
|
27
|
+
:should_download_presentation => false,
|
28
|
+
:should_add_presentation_assets_to_envoi => false
|
28
29
|
}
|
29
30
|
|
30
31
|
args[:config_service_app_id] = ENV['ENVOI_MAM_AGENT_APP_ID']
|
@@ -40,15 +41,24 @@ op.on('--agent-service-api URL', '') { |v| args[:config_service_api_url] = v }
|
|
40
41
|
op.on('--asset-id ASSETID', 'The Iconik item ID.') { |v| args[:asset_id] = v }
|
41
42
|
op.on('-f', '--format-name FORMATNAME', 'The Iconik format name used to find the file to download.') { |v| args[:format_name] = v }
|
42
43
|
op.on('--storage-id STORAGEID', 'The Iconik storage id to use for the transfer.') { |v| args[:storage_id] = v}
|
43
|
-
|
44
|
+
|
44
45
|
|
45
46
|
op.on('--file-path PATH', '') { |v| args[:file_path] = v }
|
46
|
-
op.on('-d', '--destination-path DIRPATH', 'The destination path to download or upload to.',
|
47
|
+
op.on('-d', '--destination-path DIRPATH', 'The destination path to download or upload to.',
|
48
|
+
"default: '#{args[:destination_path]}'") { |v| args[:destination_path] = v }
|
49
|
+
|
47
50
|
op.on('-p','--[no-]preserve-path', "default: #{args[:preserve_path]}") { |v| args[:preserve_path] = v }
|
48
51
|
|
49
|
-
op.on('-o', '--operation OP', [ :download, :upload ], 'Transfer Direction.',
|
52
|
+
op.on('-o', '--operation OP', [ :download, :upload ], 'Transfer Direction.',
|
53
|
+
"default: #{args[:operation]}" ) { |v| args[:operation] = v }
|
50
54
|
op.on('--transfer-type TYPE', [ :aspera, :s3 ],'Force the type of transfer to use. Ex: aspera || s3') { |v| args[:transfer_type] = v }
|
51
55
|
|
56
|
+
|
57
|
+
op.on('-i', '--[no-]import', 'Determines if an asset is created after a upload completes.',
|
58
|
+
"default: #{args[:ingest_after_upload]}") { |v| args[:ingest_after_upload] = v}
|
59
|
+
op.on('--[no-]analyze-asset', 'Determines if an analyze job is started on an asset after it is imported',
|
60
|
+
"default: #{args[:analyze_asset]}") { |v| args[:analyze_asset] = v }
|
61
|
+
|
52
62
|
# op.on('--dry-run', 'Run without executing the transfer.') { |v| args[:dry_run] = v }
|
53
63
|
op.on('--help', 'Show this message.') { puts op; exit }
|
54
64
|
op.load
|
@@ -11,7 +11,7 @@ require 'envoi/mam/mediasilo/agent'
|
|
11
11
|
current_command = ARGV.shift
|
12
12
|
ARGV << '--help' if ARGV.empty?
|
13
13
|
|
14
|
-
default_config_file_paths = Envoi::Mam::Agent::
|
14
|
+
default_config_file_paths = Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS
|
15
15
|
|
16
16
|
aspera_ascp_paths = Envoi::Mam::Agent::TransferClient::Aspera::ASCP_PATHS
|
17
17
|
default_aspera_ascp_path = aspera_ascp_paths.find { |v| File.exist? v } || aspera_ascp_paths.first
|
@@ -8,7 +8,7 @@ require 'open3'
|
|
8
8
|
require 'envoi/mam/agent/cli'
|
9
9
|
require 'envoi/mam/vidispine/agent'
|
10
10
|
|
11
|
-
default_config_file_paths = Envoi::Mam::Agent::
|
11
|
+
default_config_file_paths = Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS
|
12
12
|
|
13
13
|
aspera_ascp_paths = Envoi::Mam::Agent::TransferClient::Aspera::ASCP_PATHS
|
14
14
|
default_aspera_ascp_path = aspera_ascp_paths.find { |v| File.exist? v } || aspera_ascp_paths.first
|
@@ -30,7 +30,8 @@ args[:config_service_app_token] = ENV['ENVOI_MAM_AGENT_APP_TOKEN']
|
|
30
30
|
args[:config_service_app_url] = ENV['ENVOI_MAM_AGENT_APP_URL']
|
31
31
|
|
32
32
|
op = OptionParser.new
|
33
|
-
op.on('-c', '--config-file-path FILEPATH', 'The path to the configuration file.',
|
33
|
+
op.on('-c', '--config-file-path FILEPATH', 'The path to the configuration file.',
|
34
|
+
"Default Path(s): #{args[:config_file_path]}") { |v| args[:config_file_path] = v }
|
34
35
|
op.on('--agent-app-id ID', '') { |v| args[:config_service_app_id] = v }
|
35
36
|
op.on('--agent-token TOKEN', '') { |v| args[:config_service_app_token] = v }
|
36
37
|
op.on('--agent-service-api URL', '') { |v| args[:config_service_api_url] = v }
|
@@ -40,9 +41,11 @@ op.on('-s', '--shape-id SHAPEID', 'The Vidispine shape id.') { |v| args[:shape_i
|
|
40
41
|
op.on('-t', '--shape-tag SHAPETAG', 'The Vidispine shape tag used to find the shape when a shape id is not supplied.') { |v| args[:shape_tag] = v }
|
41
42
|
op.on('--storage-id STORAGEID', 'The Vidispine storage id to use for the transfer.') { |v| args[:storage_id] = v}
|
42
43
|
|
43
|
-
op.on('-d', '--destination-path DIRPATH', 'The destination path to download or upload to.',
|
44
|
+
op.on('-d', '--destination-path DIRPATH', 'The destination path to download or upload to.',
|
45
|
+
"default: '.'") { |v| args[:destination_path] = v }
|
44
46
|
op.on('-p','--[no-]preserve-path', "default: true") { |v| args[:preserve_path] = v }
|
45
|
-
op.on('-p', '--operation OP', [ :download, :upload ], 'Transfer Direction.',
|
47
|
+
op.on('-p', '--operation OP', [ :download, :upload ], 'Transfer Direction.',
|
48
|
+
"default: #{args[:operation]}" ) { |v| args[:operation] = v }
|
46
49
|
op.on('--transfer-type TYPE', [ :aspera, :s3 ],'Force the type of transfer to use. Ex: aspera || s3') { |v| args[:transfer_type] = v }
|
47
50
|
op.on('--transfer-token TOKEN', '') { |v| args[:transfer_token] = v }
|
48
51
|
op.on('--dry-run', 'Run without executing the transfer.') { |v| args[:dry_run] = v }
|
@@ -11,7 +11,7 @@ require 'envoi/mam/iconik/agent'
|
|
11
11
|
current_command = ARGV.shift
|
12
12
|
ARGV << '--help' if ARGV.empty?
|
13
13
|
|
14
|
-
default_config_file_paths = Envoi::Mam::Agent::
|
14
|
+
default_config_file_paths = Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS
|
15
15
|
# test_args = %w(--download --send-to-envoi --destination-path /tmp)
|
16
16
|
# ARGV.concat test_args
|
17
17
|
args = {
|
@@ -121,6 +121,19 @@ class EnvoiImportUtility
|
|
121
121
|
media_file_id = r['id']
|
122
122
|
raise 'No id found in response' unless media_file_id
|
123
123
|
|
124
|
+
create_args = {
|
125
|
+
'media_file_id' => media_file_id,
|
126
|
+
'shape_type' => 'web',
|
127
|
+
'shape_label' => 'web',
|
128
|
+
:mime => 'video/mp4', # mime_type,
|
129
|
+
:path => url,
|
130
|
+
:name => filename,
|
131
|
+
:storage_key => url_path,
|
132
|
+
:setting_id => original_storage_id,
|
133
|
+
'size' => size || 0
|
134
|
+
}
|
135
|
+
r = api_client.media_file_file_add(create_args)
|
136
|
+
|
124
137
|
video_thumb = asset['thumbnails'].find { |e| e['category'] == 'max' }
|
125
138
|
if video_thumb
|
126
139
|
thumbnail_storage_id = options[:thumbnail_storage_id] || options[:storage_id] || @default_thumbnail_storage_id
|
@@ -176,7 +189,7 @@ if args[:should_add_presentation_assets_to_envoi]
|
|
176
189
|
rescue => e
|
177
190
|
warn "#{e.message} #{$@.first}"
|
178
191
|
end
|
179
|
-
|
192
|
+
break
|
180
193
|
end
|
181
194
|
end
|
182
195
|
|
@@ -70,14 +70,14 @@ end
|
|
70
70
|
|
71
71
|
|
72
72
|
# Execute the command if it is in object form, otherwise it's just a script and executed when we required it
|
73
|
-
if Envoi::Mam::Agent::
|
73
|
+
if Envoi::Mam::Agent::CLI.const_defined?('Commands')
|
74
74
|
command_class_name = command_name.to_s.capitalize
|
75
|
-
if Envoi::Mam::Agent::
|
76
|
-
command_object = Envoi::Mam::Agent::
|
75
|
+
if Envoi::Mam::Agent::CLI::Commands.const_defined?(command_class_name)
|
76
|
+
command_object = Envoi::Mam::Agent::CLI::Commands.const_get(command_class_name.capitalize ) rescue nil
|
77
77
|
end
|
78
78
|
command_object ||= begin
|
79
79
|
command_class_name = command_class_name.upcase
|
80
|
-
Envoi::Mam::Agent::
|
80
|
+
Envoi::Mam::Agent::CLI::Commands.const_defined?(command_class_name) ? Envoi::Mam::Agent::CLI::Commands.const_get(command_class_name ) : false
|
81
81
|
end
|
82
82
|
if command_object
|
83
83
|
command_instance = command_object.new if command_object.respond_to?(:new)
|
data/lib/envoi/mam/agent/cli.rb
CHANGED
@@ -6,11 +6,11 @@ module Envoi
|
|
6
6
|
module Mam
|
7
7
|
class Agent
|
8
8
|
|
9
|
-
class
|
9
|
+
class CLI
|
10
10
|
|
11
11
|
CONFIG_FILE_PATHS = [
|
12
|
-
|
13
|
-
|
12
|
+
'./envoi-mam-agent-config.json',
|
13
|
+
'~/envoi-mam-agent-config.json',
|
14
14
|
]
|
15
15
|
CONFIG_FILE_PATHS.map! { |v| File.expand_path(v) }
|
16
16
|
|