cantemo-portal-agent 1.1.0 → 1.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -50,7 +50,7 @@ module Envoi
50
50
  ::Vidispine::API::Utilities.new(client_args)
51
51
  end
52
52
 
53
- @default_vidispine_shape_tag = args[:default_shape_tag] || _vidispine_config['default_shape_tag'] || _vidispine_config['shape_tag'] || DEFAULT_SHAPE_TAG
53
+ @default_vidispine_original_shape_tag = args[:default_shape_tag] || _vidispine_config['default_shape_tag'] || _vidispine_config['shape_tag'] || DEFAULT_SHAPE_TAG
54
54
 
55
55
  end
56
56
 
@@ -1,48 +1,113 @@
1
1
  require 'listen' # https://github.com/guard/listen
2
2
  require 'pp'
3
+ Pathname
3
4
  module Envoi
4
5
  module WatchFolderUtility
5
6
  class WatchFolder
6
7
  class Handler
7
8
  class Listen
8
9
 
9
- attr_accessor :logger, :definition,
10
+ attr_accessor :logger,
11
+ :definition,
12
+ :paths,
10
13
  :known_path_map,
14
+ :ignored_files_map,
11
15
  :last_poll_time,
12
16
  :min_stable_poll_count,
13
- :paths,
17
+ :min_stable_time,
14
18
  :poll_interval,
15
19
 
16
20
  :listener, :lock,
17
21
 
18
-
19
22
  :quarantine_directory_path
20
23
 
24
+ class DiscoveredPath < OpenStruct
25
+
26
+ def min_stable_time
27
+ self[:monitor].min_stable_time
28
+ end
29
+
30
+ def min_stable_poll_count
31
+ self[:monitor].min_stable_poll_count
32
+ end
33
+
34
+ def stable?
35
+ return false if deleted?
36
+ return false if !exist?
37
+
38
+ _min_stable_time = min_stable_time
39
+ if _min_stable_time
40
+ return ((Time.now - self[:event_timestamp]) >= _min_stable_time)
41
+ else
42
+ return (self[:stable_poll_count] >= min_stable_poll_count)
43
+ end
44
+ end
45
+
46
+ def exist?
47
+ File.exist?(self[:path])
48
+ end
49
+
50
+ def deleted?
51
+ self[:event_type] == :deleted
52
+ end
53
+
54
+ def summary
55
+ _table = @table.dup
56
+ _table.delete(:monitor)
57
+ _table
58
+ end
59
+
60
+ def watch_folder
61
+ self[:monitor]
62
+ end
63
+
64
+ def watch_folder=(wf)
65
+ self[:monitor] = wf
66
+ end
67
+
68
+ def ignore
69
+ self[:monitor].add_to_ignore(self)
70
+ end
71
+
72
+ def ignore?
73
+ self[:monitor].ignored_files_map[self[:path]] == self
74
+ end
75
+
76
+ end
77
+
21
78
  def initialize(args = { })
22
79
  initialize_logger(args)
23
- logger.debug { "Initializing Work Folder. Args: #{args}" }
80
+ logger.debug { "Initializing Watch Folder. Args: #{args}" }
24
81
 
25
82
  @definition = args[:definition]
26
83
  raise ArgumentError, "Definition is a required argument." unless definition
27
84
 
85
+ @known_path_map = { }
86
+ @ignored_files_map = { }
87
+
88
+ @lock = Mutex.new
89
+
90
+ @processing = { }
28
91
 
92
+ process_definition(@definition)
93
+ initialize_listener
94
+ end
29
95
 
96
+ def process_definition(work_folder_def)
30
97
  # @paths = [ File.expand_path('~/watch_folders/aspera') ]
31
98
  @paths = definition['paths'] || []
32
99
  path = definition['path']
33
- @paths << path if path
100
+ @paths << path if path && !@paths.include?(path)
34
101
 
35
- @known_path_map = { }
36
- @ignored_files_map = { }
37
-
38
- @lock = Mutex.new
39
102
  # @quarantine_directory_path = definition['quarantine_directory_path'] || definition['quarantine_path']
40
103
  @poll_interval = definition['poll_interval'] || 15
41
- @min_stable_poll_count = 3
42
- @processing_limit = 10
43
- @processing = { }
104
+ @min_stable_poll_count = definition['minimum_stable_poll_count'] || 3
105
+ @min_stable_time = definition['minimum_stable_time']
106
+ if @min_stable_time.nil? && @min_stable_poll_count && @min_stable_time
107
+ @min_stable_time = @poll_interval * @min_stable_poll_count
108
+ end
109
+ @processing_limit = definition['processing_limit'] || 10
44
110
 
45
- initialize_listener
46
111
  end
47
112
 
48
113
  def initialize_logger(args = { })
@@ -55,10 +120,27 @@ module Envoi
55
120
  @logger
56
121
  end
57
122
 
123
+ def process_deleted_path(path)
124
+ return path.each { |path| process_deleted_path(path, event_type) } if path.is_a?(Array)
125
+ logger.debug { "PATH #{:deleted} '#{path}'" }
126
+ file = known_path_map[path]
127
+ known_path_map.delete(path)
128
+ return unless file
129
+ file.event_type = :deleted
130
+ file.event_timestamp = Time.now
131
+ file.stable_poll_count = 0
132
+ file.deleted = true
133
+ end
134
+
58
135
  def process_add_or_modified_path(path, event_type)
59
136
  return path.each { |path| process_add_or_modified_path(path, event_type) } if path.is_a?(Array)
60
137
  logger.debug { "PATH #{event_type} '#{path}'" }
61
- lock.synchronize { known_path_map[path] = OpenStruct.new({ type: event_type, path: path, stable_poll_count: 0, timestamp: Time.now}) }
138
+ # lock.synchronize { known_path_map[path] = OpenStruct.new({ type: event_type, path: path, stable_poll_count: 0, timestamp: Time.now}) }
139
+ file = known_path_map[path] ||= DiscoveredPath.new({ path: path, processed: false, monitor: self })
140
+ file.event_type = event_type
141
+ file.event_timestamp = Time.now
142
+ file.stable_poll_count = 0
143
+ file
62
144
  end
63
145
 
64
146
  def initialize_listener(args = { })
@@ -70,7 +152,7 @@ module Envoi
70
152
  @listener = ::Listen.to(*paths, {}) do |m, a, r|
71
153
  if !r.empty?
72
154
  log "#{r.length} file(s) removed."
73
- lock.synchronize { r.each { |f| known_path_map.delete(f) } }
155
+ lock.synchronize { r.each { |f| process_deleted_path(f) } }
74
156
  end
75
157
 
76
158
  if !m.empty?
@@ -123,28 +205,75 @@ module Envoi
123
205
  # end
124
206
  # end
125
207
 
126
- def poll
208
+ def poll(options = { })
209
+ should_increment_counter = options.fetch(:should_increment_counter, true)
210
+
127
211
  @previous_poll_time = @last_poll_time
128
212
  @last_poll_time = Time.now
129
- stable_files = []
130
213
  lock.synchronize do
131
214
  @known_path_map.each do |fp, f|
132
- logger.debug { "Incrementing Stable Count: #{f.inspect}" }
133
- f[:stable_poll_count] += 1
134
- if f[:stable_poll_count] >= @min_stable_poll_count
135
- stable_files << f
136
- # handle_stable_file(f)
137
- end
215
+ next if f.ignore?
216
+ next unless f.exist?
217
+
218
+ logger.debug { "Incrementing Stable Stats: #{f.summary}" }
219
+
220
+ f.last_poll_time ||= Time.now
221
+ f[:stable_poll_count] += 1 if should_increment_counter
222
+
223
+ #
224
+ # if @min_stable_time
225
+ # if (Time.now - f.event_timestamp) >= @min_stable_time
226
+ # is_stable = true
227
+ # end
228
+ # else
229
+ # if f[:stable_poll_count] >= @min_stable_poll_count
230
+ # is_stable = true
231
+ # # handle_stable_file(f)
232
+ # end
233
+ # end
138
234
  end
139
235
  end
140
236
  stable_files
141
237
  end
142
238
 
239
+ def stable_files
240
+ @known_path_map.select { |p,f| f.stable? }.values
241
+ end
242
+
143
243
  def run(options = { })
144
244
  logger.info { "Starting handler for #{paths}" }
145
245
  listener.start
146
246
  end
147
247
 
248
+ def stop
249
+ listener.stop if listener && listener.respond_to?(:stop)
250
+ end
251
+
252
+ InspectKey = :__inspect_key__ # :nodoc:
253
+
254
+ def inspect
255
+ str = "#<#{self.class}"
256
+
257
+ ids = (Thread.current[InspectKey] ||= [])
258
+ if ids.include?(object_id)
259
+ return str << ' ...>'
260
+ end
261
+
262
+ ids << object_id
263
+ # begin
264
+ # first = true
265
+ # for k,v in @table
266
+ # str << "," unless first
267
+ # first = false
268
+ # str << " #{k}=#{v.inspect}"
269
+ # end
270
+ # return str << '>'
271
+ # ensure
272
+ # ids.pop
273
+ # end
274
+ end
275
+ alias :to_s :inspect
276
+
148
277
  end
149
278
  end
150
279
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: cantemo-portal-agent
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.1.0
4
+ version: 1.1.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - John Whitson
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2018-10-02 00:00:00.000000000 Z
11
+ date: 2018-10-17 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: asperalm
@@ -181,8 +181,8 @@ files:
181
181
  - lib/envoi/mam/agent/watch_folder_utility/foreman.rb
182
182
  - lib/envoi/mam/cantemo/agent.rb
183
183
  - lib/envoi/mam/cantemo/agent/watch_folder_handler-working.rb
184
- - lib/envoi/mam/cantemo/agent/watch_folder_handler.rb
185
184
  - lib/envoi/mam/cantemo/agent/watch_folder_handler_aspera.rb
185
+ - lib/envoi/mam/cantemo/agent/watch_folder_manager.rb
186
186
  - lib/envoi/mam/iconik/agent.rb
187
187
  - lib/envoi/mam/mediasilo/agent.rb
188
188
  - lib/envoi/mam/vdms/agent.rb
@@ -1,176 +0,0 @@
1
- require 'envoi/mam/cantemo/agent'
2
- require 'envoi/aspera/watch_service/watch_folder'
3
- require 'envoi/watch_folder_utility/watch_folder/handler/listen'
4
-
5
- module Envoi::Mam::Cantemo
6
-
7
- class Agent
8
-
9
- class WatchFolderHandler
10
-
11
- AWF = Envoi::Aspera::WatchService::WatchFolder
12
- LWF = Envoi::WatchFolderUtility::WatchFolder::Handler::Listen
13
-
14
- attr_accessor :logger, :agent, :config, :watch_folder_defs, :watch_folders
15
-
16
- def initialize(args = { })
17
- initialize_logger(args)
18
-
19
- @agent = Envoi::Mam::Cantemo::Agent.load_from_config_file(args)
20
- @config = agent.config
21
- cantemo_config = config[:cantemo] || config['cantemo']
22
- @watch_folder_defs = cantemo_config[:watch_folders] || cantemo_config['watch_folders']
23
-
24
- @ignored_file_paths_by_watch_folder = Hash.new { |h, k| h[k] = [] }
25
- process_watch_folder_defs
26
- end
27
-
28
- def process_watch_folder_def(watch_folder_def)
29
- name = watch_folder_def['name']
30
-
31
- path = watch_folder_def['path']
32
- path ||= name
33
- (watch_folder_def['paths'] ||= []).concat [*path]
34
- watch_folder_def['paths'].uniq!
35
- paths = watch_folder_def['paths']
36
- watch_folder_def['path'] ||= paths.first if path.length == 1
37
-
38
- exclude = watch_folder_def['exclude']
39
- exclude ||= '**/.*'
40
- (watch_folder_def['excludes'] ||= []).concat [*exclude]
41
- watch_folder_def['excludes'].uniq!
42
-
43
- args_out = { }
44
- logging = watch_folder_def['logging'] || watch_folder_def
45
- log_to = logging['log_to']
46
- log_level = logging['log_level']
47
- args_out[:log_to] ||= log_to if log_to && !log_to.empty?
48
- args_out[:log_level] ||= log_level if log_level && !log_level.empty?
49
- args_out[:logger] ||= logger.dup unless log_to
50
- args_out[:definition] = watch_folder_def
51
-
52
- LWF.new(args_out)
53
- end
54
-
55
- def process_watch_folder_defs
56
- if watch_folder_defs.is_a?(Array)
57
- @watch_folders = watch_folder_defs.map { |watch_folder_def| process_watch_folder_def(watch_folder_def) }
58
- elsif watch_folder_defs.is_a?(Hash)
59
- @watch_folders = watch_folder_defs.map do |name, watch_folder_def|
60
- watch_folder_def['name'] ||= name
61
-
62
- process_watch_folder_def(watch_folder_def)
63
- end
64
- else
65
- raise "Unhandled format: #{watch_folder_defs.class.name}"
66
- end
67
- end
68
-
69
- def initialize_logger(args = { })
70
- @logger = args[:logger] ||= Logger.new(args[:log_to] || STDOUT)
71
- log_level = args[:log_level]
72
- if log_level
73
- @logger.level = log_level
74
- args[:logger] = @logger
75
- end
76
- @logger
77
- end
78
-
79
- def add_to_ignore(watch_folder, file)
80
- logger.debug { "Adding File to Ignore Cache: '#{file.path}'"}
81
- @ignored_file_paths_by_watch_folder[watch_folder] << file.path
82
- end
83
-
84
- def process_file(watch_folder, file, storage_id = nil, quarantine_directory_path = nil)
85
- # full_file_path = File.join(watch_folder.path, file.path)
86
- full_file_path = file.path
87
- if storage_id && agent.upload(file_path: full_file_path, storage_id: storage_id)
88
- FileUtils.rm full_file_path
89
- else
90
- if Dir.exist?(quarantine_directory_path)
91
- FileUtils.mv full_file_path, quarantine_directory_path
92
- else
93
- add_to_ignore(watch_folder, file)
94
- end
95
- end
96
- end
97
-
98
- def process_watch_folder_stable_files(wf, stable_files)
99
- storage_id = wf.definition['upload_to_storage_id'] || wf.definition['storage_id']
100
- quarantine_directory_path = wf.definition['quarantine_path']
101
- exclude = wf.definition['excludes']
102
-
103
- ignored_file_paths = @ignored_file_paths_by_watch_folder[wf]
104
- stable_files.each do |file|
105
- next if ignored_file_paths.include?(file.path)
106
-
107
- if exclude && [*exclude].find do |ep|
108
- excluded = (r = Regexp.try_convert(ep) ? r.match(ep) : File.fnmatch(ep, file.path))
109
- logger.debug { "#{ep} #{excluded ? 'matched' : "didn't match"} #{file.path}" }
110
- excluded
111
- end
112
- add_to_ignore(wf, file)
113
- next
114
- end
115
-
116
- process_file(wf, file, storage_id, quarantine_directory_path)
117
- end
118
-
119
- end
120
-
121
- def poll
122
- stable_files_by_watch_folder = {} # Hash.new { |h, k| h[k] = [] }
123
- watch_folders.each do |watch_folder|
124
- if !watch_folder.last_poll_time || (Time.now - watch_folder.last_poll_time) >= watch_folder.poll_interval
125
- stable_files = watch_folder.poll
126
- stable_files_by_watch_folder[watch_folder] = stable_files
127
- end
128
- end
129
- stable_files_by_watch_folder.each do |watch_folder, stable_files|
130
- process_watch_folder_stable_files(watch_folder, stable_files)
131
- end
132
- end
133
-
134
- def run
135
- # AWF.run_once(watch_folders) { |wf| pp wf }
136
- # AWF.run(watch_folders) { |wf| process_watch_folder(wf) }
137
-
138
- logger.info { 'Running...' }
139
- watch_folders.map { |wf| wf.run }
140
- loop do
141
- begin
142
- poll
143
- sleep 1
144
- rescue SystemExit, Interrupt
145
- break
146
- end
147
- end
148
- logger.info { 'Exiting...' }
149
- rescue => e
150
- logger.error { "An error occurred.\n#{e.message}\n#{e.backtrace.join("\n")}\n#{e.message}" }
151
- raise e
152
- end
153
-
154
- def run_once
155
- # AWF.run_once(watch_folders) { |wf| process_watch_folder(wf) }
156
- end
157
-
158
- def self.run(args)
159
- w = self.new(args)
160
- w.run
161
- rescue => e
162
- abort("An error occurred.\n#{e.message}\n#{e.message}")
163
- end
164
-
165
- def self.run_as_daemon(args)
166
- # ARGV.unshift 'run' unless %w(start stop restart run zap killall status).include? ARGV.first
167
- require 'daemons'
168
- # Daemons.run_proc('cantemo-portal-watch-folders', { force: true }) { self.run(args) }
169
- Daemons.run_proc('cantemo-portal-watch-folders') { self.run(args) }
170
- end
171
-
172
- end
173
-
174
- end
175
-
176
- end