cantemo-portal-agent 1.1.0 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/cantemo/portal/agent/cli/commands/watch_folders-working.rb +2 -2
- data/lib/cantemo/portal/agent/cli/commands/watch_folders.rb +23 -7
- data/lib/cantemo/portal/agent/version.rb +1 -1
- data/lib/envoi/mam/agent.rb +5 -2
- data/lib/envoi/mam/agent/transfer_client/aspera.rb +39 -12
- data/lib/envoi/mam/cantemo/agent.rb +106 -22
- data/lib/envoi/mam/cantemo/agent/watch_folder_handler-working.rb +5 -2
- data/lib/envoi/mam/cantemo/agent/watch_folder_manager.rb +426 -0
- data/lib/envoi/mam/vidispine/agent.rb +1 -1
- data/lib/envoi/watch_folder_utility/watch_folder/handler/listen.rb +152 -23
- metadata +3 -3
- data/lib/envoi/mam/cantemo/agent/watch_folder_handler.rb +0 -176
@@ -50,7 +50,7 @@ module Envoi
|
|
50
50
|
::Vidispine::API::Utilities.new(client_args)
|
51
51
|
end
|
52
52
|
|
53
|
-
@
|
53
|
+
@default_vidispine_original_shape_tag = args[:default_shape_tag] || _vidispine_config['default_shape_tag'] || _vidispine_config['shape_tag'] || DEFAULT_SHAPE_TAG
|
54
54
|
|
55
55
|
end
|
56
56
|
|
@@ -1,48 +1,113 @@
|
|
1
1
|
require 'listen' # https://github.com/guard/listen
|
2
2
|
require 'pp'
|
3
|
+
Pathname
|
3
4
|
module Envoi
|
4
5
|
module WatchFolderUtility
|
5
6
|
class WatchFolder
|
6
7
|
class Handler
|
7
8
|
class Listen
|
8
9
|
|
9
|
-
attr_accessor :logger,
|
10
|
+
attr_accessor :logger,
|
11
|
+
:definition,
|
12
|
+
:paths,
|
10
13
|
:known_path_map,
|
14
|
+
:ignored_files_map,
|
11
15
|
:last_poll_time,
|
12
16
|
:min_stable_poll_count,
|
13
|
-
:
|
17
|
+
:min_stable_time,
|
14
18
|
:poll_interval,
|
15
19
|
|
16
20
|
:listener, :lock,
|
17
21
|
|
18
|
-
|
19
22
|
:quarantine_directory_path
|
20
23
|
|
24
|
+
class DiscoveredPath < OpenStruct
|
25
|
+
|
26
|
+
def min_stable_time
|
27
|
+
self[:monitor].min_stable_time
|
28
|
+
end
|
29
|
+
|
30
|
+
def min_stable_poll_count
|
31
|
+
self[:monitor].min_stable_poll_count
|
32
|
+
end
|
33
|
+
|
34
|
+
def stable?
|
35
|
+
return false if deleted?
|
36
|
+
return false if !exist?
|
37
|
+
|
38
|
+
_min_stable_time = min_stable_time
|
39
|
+
if _min_stable_time
|
40
|
+
return ((Time.now - self[:event_timestamp]) >= _min_stable_time)
|
41
|
+
else
|
42
|
+
return (self[:stable_poll_count] >= min_stable_poll_count)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def exist?
|
47
|
+
File.exist?(self[:path])
|
48
|
+
end
|
49
|
+
|
50
|
+
def deleted?
|
51
|
+
self[:event_type] == :deleted
|
52
|
+
end
|
53
|
+
|
54
|
+
def summary
|
55
|
+
_table = @table.dup
|
56
|
+
_table.delete(:monitor)
|
57
|
+
_table
|
58
|
+
end
|
59
|
+
|
60
|
+
def watch_folder
|
61
|
+
self[:monitor]
|
62
|
+
end
|
63
|
+
|
64
|
+
def watch_folder=(wf)
|
65
|
+
self[:monitor] = wf
|
66
|
+
end
|
67
|
+
|
68
|
+
def ignore
|
69
|
+
self[:monitor].add_to_ignore(self)
|
70
|
+
end
|
71
|
+
|
72
|
+
def ignore?
|
73
|
+
self[:monitor].ignored_files_map[self[:path]] == self
|
74
|
+
end
|
75
|
+
|
76
|
+
end
|
77
|
+
|
21
78
|
def initialize(args = { })
|
22
79
|
initialize_logger(args)
|
23
|
-
logger.debug { "Initializing
|
80
|
+
logger.debug { "Initializing Watch Folder. Args: #{args}" }
|
24
81
|
|
25
82
|
@definition = args[:definition]
|
26
83
|
raise ArgumentError, "Definition is a required argument." unless definition
|
27
84
|
|
85
|
+
@known_path_map = { }
|
86
|
+
@ignored_files_map = { }
|
87
|
+
|
88
|
+
@lock = Mutex.new
|
89
|
+
|
90
|
+
@processing = { }
|
28
91
|
|
92
|
+
process_definition(@definition)
|
93
|
+
initialize_listener
|
94
|
+
end
|
29
95
|
|
96
|
+
def process_definition(work_folder_def)
|
30
97
|
# @paths = [ File.expand_path('~/watch_folders/aspera') ]
|
31
98
|
@paths = definition['paths'] || []
|
32
99
|
path = definition['path']
|
33
|
-
@paths << path if path
|
100
|
+
@paths << path if path && !@paths.include?(path)
|
34
101
|
|
35
|
-
@known_path_map = { }
|
36
|
-
@ignored_files_map = { }
|
37
|
-
|
38
|
-
@lock = Mutex.new
|
39
102
|
# @quarantine_directory_path = definition['quarantine_directory_path'] || definition['quarantine_path']
|
40
103
|
@poll_interval = definition['poll_interval'] || 15
|
41
|
-
@min_stable_poll_count = 3
|
42
|
-
@
|
43
|
-
@
|
104
|
+
@min_stable_poll_count = definition['minimum_stable_poll_count'] || 3
|
105
|
+
@min_stable_time = definition['minimum_stable_time']
|
106
|
+
if @min_stable_time.nil? && @min_stable_poll_count && @min_stable_time
|
107
|
+
@min_stable_time = @poll_interval * @min_stable_poll_count
|
108
|
+
end
|
109
|
+
@processing_limit = definition['processing_limit'] || 10
|
44
110
|
|
45
|
-
initialize_listener
|
46
111
|
end
|
47
112
|
|
48
113
|
def initialize_logger(args = { })
|
@@ -55,10 +120,27 @@ module Envoi
|
|
55
120
|
@logger
|
56
121
|
end
|
57
122
|
|
123
|
+
def process_deleted_path(path)
|
124
|
+
return path.each { |path| process_deleted_path(path, event_type) } if path.is_a?(Array)
|
125
|
+
logger.debug { "PATH #{:deleted} '#{path}'" }
|
126
|
+
file = known_path_map[path]
|
127
|
+
known_path_map.delete(path)
|
128
|
+
return unless file
|
129
|
+
file.event_type = :deleted
|
130
|
+
file.event_timestamp = Time.now
|
131
|
+
file.stable_poll_count = 0
|
132
|
+
file.deleted = true
|
133
|
+
end
|
134
|
+
|
58
135
|
def process_add_or_modified_path(path, event_type)
|
59
136
|
return path.each { |path| process_add_or_modified_path(path, event_type) } if path.is_a?(Array)
|
60
137
|
logger.debug { "PATH #{event_type} '#{path}'" }
|
61
|
-
lock.synchronize { known_path_map[path] = OpenStruct.new({ type: event_type, path: path, stable_poll_count: 0, timestamp: Time.now}) }
|
138
|
+
# lock.synchronize { known_path_map[path] = OpenStruct.new({ type: event_type, path: path, stable_poll_count: 0, timestamp: Time.now}) }
|
139
|
+
file = known_path_map[path] ||= DiscoveredPath.new({ path: path, processed: false, monitor: self })
|
140
|
+
file.event_type = event_type
|
141
|
+
file.event_timestamp = Time.now
|
142
|
+
file.stable_poll_count = 0
|
143
|
+
file
|
62
144
|
end
|
63
145
|
|
64
146
|
def initialize_listener(args = { })
|
@@ -70,7 +152,7 @@ module Envoi
|
|
70
152
|
@listener = ::Listen.to(*paths, {}) do |m, a, r|
|
71
153
|
if !r.empty?
|
72
154
|
log "#{r.length} file(s) removed."
|
73
|
-
lock.synchronize { r.each { |f|
|
155
|
+
lock.synchronize { r.each { |f| process_deleted_path(f) } }
|
74
156
|
end
|
75
157
|
|
76
158
|
if !m.empty?
|
@@ -123,28 +205,75 @@ module Envoi
|
|
123
205
|
# end
|
124
206
|
# end
|
125
207
|
|
126
|
-
def poll
|
208
|
+
def poll(options = { })
|
209
|
+
should_increment_counter = options.fetch(:should_increment_counter, true)
|
210
|
+
|
127
211
|
@previous_poll_time = @last_poll_time
|
128
212
|
@last_poll_time = Time.now
|
129
|
-
stable_files = []
|
130
213
|
lock.synchronize do
|
131
214
|
@known_path_map.each do |fp, f|
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
215
|
+
next if f.ignore?
|
216
|
+
next unless f.exist?
|
217
|
+
|
218
|
+
logger.debug { "Incrementing Stable Stats: #{f.summary}" }
|
219
|
+
|
220
|
+
f.last_poll_time ||= Time.now
|
221
|
+
f[:stable_poll_count] += 1 if should_increment_counter
|
222
|
+
|
223
|
+
#
|
224
|
+
# if @min_stable_time
|
225
|
+
# if (Time.now - f.event_timestamp) >= @min_stable_time
|
226
|
+
# is_stable = true
|
227
|
+
# end
|
228
|
+
# else
|
229
|
+
# if f[:stable_poll_count] >= @min_stable_poll_count
|
230
|
+
# is_stable = true
|
231
|
+
# # handle_stable_file(f)
|
232
|
+
# end
|
233
|
+
# end
|
138
234
|
end
|
139
235
|
end
|
140
236
|
stable_files
|
141
237
|
end
|
142
238
|
|
239
|
+
def stable_files
|
240
|
+
@known_path_map.select { |p,f| f.stable? }.values
|
241
|
+
end
|
242
|
+
|
143
243
|
def run(options = { })
|
144
244
|
logger.info { "Starting handler for #{paths}" }
|
145
245
|
listener.start
|
146
246
|
end
|
147
247
|
|
248
|
+
def stop
|
249
|
+
listener.stop if listener && listener.respond_to?(:stop)
|
250
|
+
end
|
251
|
+
|
252
|
+
InspectKey = :__inspect_key__ # :nodoc:
|
253
|
+
|
254
|
+
def inspect
|
255
|
+
str = "#<#{self.class}"
|
256
|
+
|
257
|
+
ids = (Thread.current[InspectKey] ||= [])
|
258
|
+
if ids.include?(object_id)
|
259
|
+
return str << ' ...>'
|
260
|
+
end
|
261
|
+
|
262
|
+
ids << object_id
|
263
|
+
# begin
|
264
|
+
# first = true
|
265
|
+
# for k,v in @table
|
266
|
+
# str << "," unless first
|
267
|
+
# first = false
|
268
|
+
# str << " #{k}=#{v.inspect}"
|
269
|
+
# end
|
270
|
+
# return str << '>'
|
271
|
+
# ensure
|
272
|
+
# ids.pop
|
273
|
+
# end
|
274
|
+
end
|
275
|
+
alias :to_s :inspect
|
276
|
+
|
148
277
|
end
|
149
278
|
end
|
150
279
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cantemo-portal-agent
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.1.
|
4
|
+
version: 1.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- John Whitson
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-10-
|
11
|
+
date: 2018-10-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: asperalm
|
@@ -181,8 +181,8 @@ files:
|
|
181
181
|
- lib/envoi/mam/agent/watch_folder_utility/foreman.rb
|
182
182
|
- lib/envoi/mam/cantemo/agent.rb
|
183
183
|
- lib/envoi/mam/cantemo/agent/watch_folder_handler-working.rb
|
184
|
-
- lib/envoi/mam/cantemo/agent/watch_folder_handler.rb
|
185
184
|
- lib/envoi/mam/cantemo/agent/watch_folder_handler_aspera.rb
|
185
|
+
- lib/envoi/mam/cantemo/agent/watch_folder_manager.rb
|
186
186
|
- lib/envoi/mam/iconik/agent.rb
|
187
187
|
- lib/envoi/mam/mediasilo/agent.rb
|
188
188
|
- lib/envoi/mam/vdms/agent.rb
|
@@ -1,176 +0,0 @@
|
|
1
|
-
require 'envoi/mam/cantemo/agent'
|
2
|
-
require 'envoi/aspera/watch_service/watch_folder'
|
3
|
-
require 'envoi/watch_folder_utility/watch_folder/handler/listen'
|
4
|
-
|
5
|
-
module Envoi::Mam::Cantemo
|
6
|
-
|
7
|
-
class Agent
|
8
|
-
|
9
|
-
class WatchFolderHandler
|
10
|
-
|
11
|
-
AWF = Envoi::Aspera::WatchService::WatchFolder
|
12
|
-
LWF = Envoi::WatchFolderUtility::WatchFolder::Handler::Listen
|
13
|
-
|
14
|
-
attr_accessor :logger, :agent, :config, :watch_folder_defs, :watch_folders
|
15
|
-
|
16
|
-
def initialize(args = { })
|
17
|
-
initialize_logger(args)
|
18
|
-
|
19
|
-
@agent = Envoi::Mam::Cantemo::Agent.load_from_config_file(args)
|
20
|
-
@config = agent.config
|
21
|
-
cantemo_config = config[:cantemo] || config['cantemo']
|
22
|
-
@watch_folder_defs = cantemo_config[:watch_folders] || cantemo_config['watch_folders']
|
23
|
-
|
24
|
-
@ignored_file_paths_by_watch_folder = Hash.new { |h, k| h[k] = [] }
|
25
|
-
process_watch_folder_defs
|
26
|
-
end
|
27
|
-
|
28
|
-
def process_watch_folder_def(watch_folder_def)
|
29
|
-
name = watch_folder_def['name']
|
30
|
-
|
31
|
-
path = watch_folder_def['path']
|
32
|
-
path ||= name
|
33
|
-
(watch_folder_def['paths'] ||= []).concat [*path]
|
34
|
-
watch_folder_def['paths'].uniq!
|
35
|
-
paths = watch_folder_def['paths']
|
36
|
-
watch_folder_def['path'] ||= paths.first if path.length == 1
|
37
|
-
|
38
|
-
exclude = watch_folder_def['exclude']
|
39
|
-
exclude ||= '**/.*'
|
40
|
-
(watch_folder_def['excludes'] ||= []).concat [*exclude]
|
41
|
-
watch_folder_def['excludes'].uniq!
|
42
|
-
|
43
|
-
args_out = { }
|
44
|
-
logging = watch_folder_def['logging'] || watch_folder_def
|
45
|
-
log_to = logging['log_to']
|
46
|
-
log_level = logging['log_level']
|
47
|
-
args_out[:log_to] ||= log_to if log_to && !log_to.empty?
|
48
|
-
args_out[:log_level] ||= log_level if log_level && !log_level.empty?
|
49
|
-
args_out[:logger] ||= logger.dup unless log_to
|
50
|
-
args_out[:definition] = watch_folder_def
|
51
|
-
|
52
|
-
LWF.new(args_out)
|
53
|
-
end
|
54
|
-
|
55
|
-
def process_watch_folder_defs
|
56
|
-
if watch_folder_defs.is_a?(Array)
|
57
|
-
@watch_folders = watch_folder_defs.map { |watch_folder_def| process_watch_folder_def(watch_folder_def) }
|
58
|
-
elsif watch_folder_defs.is_a?(Hash)
|
59
|
-
@watch_folders = watch_folder_defs.map do |name, watch_folder_def|
|
60
|
-
watch_folder_def['name'] ||= name
|
61
|
-
|
62
|
-
process_watch_folder_def(watch_folder_def)
|
63
|
-
end
|
64
|
-
else
|
65
|
-
raise "Unhandled format: #{watch_folder_defs.class.name}"
|
66
|
-
end
|
67
|
-
end
|
68
|
-
|
69
|
-
def initialize_logger(args = { })
|
70
|
-
@logger = args[:logger] ||= Logger.new(args[:log_to] || STDOUT)
|
71
|
-
log_level = args[:log_level]
|
72
|
-
if log_level
|
73
|
-
@logger.level = log_level
|
74
|
-
args[:logger] = @logger
|
75
|
-
end
|
76
|
-
@logger
|
77
|
-
end
|
78
|
-
|
79
|
-
def add_to_ignore(watch_folder, file)
|
80
|
-
logger.debug { "Adding File to Ignore Cache: '#{file.path}'"}
|
81
|
-
@ignored_file_paths_by_watch_folder[watch_folder] << file.path
|
82
|
-
end
|
83
|
-
|
84
|
-
def process_file(watch_folder, file, storage_id = nil, quarantine_directory_path = nil)
|
85
|
-
# full_file_path = File.join(watch_folder.path, file.path)
|
86
|
-
full_file_path = file.path
|
87
|
-
if storage_id && agent.upload(file_path: full_file_path, storage_id: storage_id)
|
88
|
-
FileUtils.rm full_file_path
|
89
|
-
else
|
90
|
-
if Dir.exist?(quarantine_directory_path)
|
91
|
-
FileUtils.mv full_file_path, quarantine_directory_path
|
92
|
-
else
|
93
|
-
add_to_ignore(watch_folder, file)
|
94
|
-
end
|
95
|
-
end
|
96
|
-
end
|
97
|
-
|
98
|
-
def process_watch_folder_stable_files(wf, stable_files)
|
99
|
-
storage_id = wf.definition['upload_to_storage_id'] || wf.definition['storage_id']
|
100
|
-
quarantine_directory_path = wf.definition['quarantine_path']
|
101
|
-
exclude = wf.definition['excludes']
|
102
|
-
|
103
|
-
ignored_file_paths = @ignored_file_paths_by_watch_folder[wf]
|
104
|
-
stable_files.each do |file|
|
105
|
-
next if ignored_file_paths.include?(file.path)
|
106
|
-
|
107
|
-
if exclude && [*exclude].find do |ep|
|
108
|
-
excluded = (r = Regexp.try_convert(ep) ? r.match(ep) : File.fnmatch(ep, file.path))
|
109
|
-
logger.debug { "#{ep} #{excluded ? 'matched' : "didn't match"} #{file.path}" }
|
110
|
-
excluded
|
111
|
-
end
|
112
|
-
add_to_ignore(wf, file)
|
113
|
-
next
|
114
|
-
end
|
115
|
-
|
116
|
-
process_file(wf, file, storage_id, quarantine_directory_path)
|
117
|
-
end
|
118
|
-
|
119
|
-
end
|
120
|
-
|
121
|
-
def poll
|
122
|
-
stable_files_by_watch_folder = {} # Hash.new { |h, k| h[k] = [] }
|
123
|
-
watch_folders.each do |watch_folder|
|
124
|
-
if !watch_folder.last_poll_time || (Time.now - watch_folder.last_poll_time) >= watch_folder.poll_interval
|
125
|
-
stable_files = watch_folder.poll
|
126
|
-
stable_files_by_watch_folder[watch_folder] = stable_files
|
127
|
-
end
|
128
|
-
end
|
129
|
-
stable_files_by_watch_folder.each do |watch_folder, stable_files|
|
130
|
-
process_watch_folder_stable_files(watch_folder, stable_files)
|
131
|
-
end
|
132
|
-
end
|
133
|
-
|
134
|
-
def run
|
135
|
-
# AWF.run_once(watch_folders) { |wf| pp wf }
|
136
|
-
# AWF.run(watch_folders) { |wf| process_watch_folder(wf) }
|
137
|
-
|
138
|
-
logger.info { 'Running...' }
|
139
|
-
watch_folders.map { |wf| wf.run }
|
140
|
-
loop do
|
141
|
-
begin
|
142
|
-
poll
|
143
|
-
sleep 1
|
144
|
-
rescue SystemExit, Interrupt
|
145
|
-
break
|
146
|
-
end
|
147
|
-
end
|
148
|
-
logger.info { 'Exiting...' }
|
149
|
-
rescue => e
|
150
|
-
logger.error { "An error occurred.\n#{e.message}\n#{e.backtrace.join("\n")}\n#{e.message}" }
|
151
|
-
raise e
|
152
|
-
end
|
153
|
-
|
154
|
-
def run_once
|
155
|
-
# AWF.run_once(watch_folders) { |wf| process_watch_folder(wf) }
|
156
|
-
end
|
157
|
-
|
158
|
-
def self.run(args)
|
159
|
-
w = self.new(args)
|
160
|
-
w.run
|
161
|
-
rescue => e
|
162
|
-
abort("An error occurred.\n#{e.message}\n#{e.message}")
|
163
|
-
end
|
164
|
-
|
165
|
-
def self.run_as_daemon(args)
|
166
|
-
# ARGV.unshift 'run' unless %w(start stop restart run zap killall status).include? ARGV.first
|
167
|
-
require 'daemons'
|
168
|
-
# Daemons.run_proc('cantemo-portal-watch-folders', { force: true }) { self.run(args) }
|
169
|
-
Daemons.run_proc('cantemo-portal-watch-folders') { self.run(args) }
|
170
|
-
end
|
171
|
-
|
172
|
-
end
|
173
|
-
|
174
|
-
end
|
175
|
-
|
176
|
-
end
|