cantemo-portal-agent 1.1.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/cantemo/portal/agent/cli/commands/watch_folder.rb +1 -0
- data/lib/cantemo/portal/agent/cli/commands/watch_folders.rb +4 -3
- data/lib/cantemo/portal/agent/version.rb +1 -1
- data/lib/envoi/mam/agent.rb +16 -9
- data/lib/envoi/mam/agent/cli/commands/cantemo-agent.rb +0 -2
- data/lib/envoi/mam/agent/cli/commands/cantemo.rb +1 -1
- data/lib/envoi/mam/cantemo/agent.rb +123 -99
- data/lib/envoi/mam/cantemo/agent/watch_folder_manager.rb +75 -127
- data/lib/envoi/watch_folder_utility/watch_folder.rb +307 -0
- data/lib/envoi/watch_folder_utility/watch_folder/handler/listen.rb +28 -55
- metadata +6 -4
@@ -1,5 +1,6 @@
|
|
1
1
|
require 'envoi/mam/cantemo/agent'
|
2
2
|
require 'envoi/aspera/watch_service/watch_folder'
|
3
|
+
require 'envoi/watch_folder_utility/watch_folder'
|
3
4
|
require 'envoi/watch_folder_utility/watch_folder/handler/listen'
|
4
5
|
|
5
6
|
module Envoi::Mam::Cantemo
|
@@ -13,21 +14,33 @@ module Envoi::Mam::Cantemo
|
|
13
14
|
|
14
15
|
DEFAULT_WATCH_FOLDER_PROCESSOR_LIMIT = 10
|
15
16
|
|
16
|
-
attr_accessor :logger, :
|
17
|
+
attr_accessor :logger, :default_agent_class, :default_agent, :config, :watch_folder_defs, :watch_folders
|
17
18
|
|
18
19
|
def initialize(args = {})
|
19
20
|
initialize_logger(args)
|
20
21
|
|
21
22
|
logger.debug { 'Initializing Agent Watch Folder Manager.' }
|
22
23
|
|
23
|
-
logger.debug { 'Initializing Cantemo Portal Agent.' }
|
24
24
|
args[:default_preserve_file_path] = args.fetch(:default_preserve_file_path, false)
|
25
|
-
@agent = Envoi::Mam::Cantemo::Agent.load_from_config_file(args)
|
26
|
-
logger.debug { 'Cantemo Portal Agent Initialized.' }
|
27
25
|
|
28
|
-
@config
|
29
|
-
cantemo_config
|
30
|
-
|
26
|
+
@config = Envoi::Mam::Cantemo::Agent.load_config_from_file(args)
|
27
|
+
cantemo_config = config[:cantemo] || config['cantemo']
|
28
|
+
|
29
|
+
@default_agent_class = Envoi::Mam::Cantemo::Agent
|
30
|
+
|
31
|
+
if cantemo_config
|
32
|
+
logger.debug { 'Initializing Default Cantemo Portal Agent.' }
|
33
|
+
@default_agent = @default_agent_class.new(args.merge({ config: cantemo_config }))
|
34
|
+
logger.debug { 'Default Cantemo Portal Agent Initialized.' }
|
35
|
+
@default_storages = @default_agent.agent_config_storages
|
36
|
+
else
|
37
|
+
@default_agent = nil
|
38
|
+
end
|
39
|
+
|
40
|
+
@watch_folder_defs = config[:watch_folders] || config['watch_folders']
|
41
|
+
if !@watch_folder_defs && cantemo_config
|
42
|
+
@watch_folder_defs = cantemo_config[:watch_folders] || cantemo_config['watch_folders']
|
43
|
+
end
|
31
44
|
|
32
45
|
@ignored_file_paths_by_watch_folder = Hash.new { |h, k| h[k] = [] }
|
33
46
|
@ignored_file_paths_lock = Mutex.new
|
@@ -50,8 +63,6 @@ module Envoi::Mam::Cantemo
|
|
50
63
|
@logger
|
51
64
|
end
|
52
65
|
|
53
|
-
# @TODO Move most of this code into a watch folder class
|
54
|
-
#
|
55
66
|
# @param [Hash] watch_folder_def
|
56
67
|
# @option watch_folder_def [String] path
|
57
68
|
# @option watch_folder_def [String] upload_to_storage_id
|
@@ -66,96 +77,6 @@ module Envoi::Mam::Cantemo
|
|
66
77
|
# @option watch_folder_def [Integer|False] maximum_active_processors (@default_maximum_active_processors)
|
67
78
|
# @option watch_folder_def [Hash] logging
|
68
79
|
def process_watch_folder_def(watch_folder_def)
|
69
|
-
logger.debug { "Initializing Watch Folder #{watch_folder_def.inspect}" }
|
70
|
-
|
71
|
-
logger.debug { "Initializing parameter 'paths'." }
|
72
|
-
name = watch_folder_def['name']
|
73
|
-
|
74
|
-
path = watch_folder_def['path']
|
75
|
-
|
76
|
-
paths = watch_folder_def['paths'] ||= []
|
77
|
-
paths = [ paths ] if paths.is_a?(String)
|
78
|
-
paths.concat [*path] if path
|
79
|
-
paths.map! { |p| File.expand_path(p) }
|
80
|
-
if paths.empty?
|
81
|
-
name_as_path = File.expand_path(name)
|
82
|
-
paths.concat name_as_path if Dir.exist?(name_as_path)
|
83
|
-
end
|
84
|
-
paths.uniq!
|
85
|
-
watch_folder_def['paths'] = paths
|
86
|
-
# watch_folder_def['path'] ||= paths.first if paths.length == 1
|
87
|
-
watch_folder_def.delete('path')
|
88
|
-
|
89
|
-
if paths.empty?
|
90
|
-
logger.error { "Failed to initialize watch folder. No path found in watch folder definition." }
|
91
|
-
return false
|
92
|
-
end
|
93
|
-
logger.debug { "Parameter 'paths' initialized." }
|
94
|
-
|
95
|
-
logger.debug { "Initializing parameter 'includes'." }
|
96
|
-
include = watch_folder_def['include']
|
97
|
-
includes = (watch_folder_def['includes'] ||= [])
|
98
|
-
includes.concat [*include] if include
|
99
|
-
includes.uniq!
|
100
|
-
includes.map! { |e| Regexp.try_convert(e) || e }
|
101
|
-
watch_folder_def['includes'] = includes
|
102
|
-
watch_folder_def.delete('include')
|
103
|
-
logger.debug { "Parameter `includes` initialized." }
|
104
|
-
|
105
|
-
logger.debug { "Initializing parameter 'excludes'." }
|
106
|
-
exclude = watch_folder_def['exclude']
|
107
|
-
exclude ||= '**/.*'
|
108
|
-
excludes = (watch_folder_def['excludes'] ||= [])
|
109
|
-
excludes.concat [*exclude] if exclude
|
110
|
-
excludes.uniq!
|
111
|
-
excludes.map! { |e| Regexp.try_convert(e) || e }
|
112
|
-
watch_folder_def['excludes'] = excludes
|
113
|
-
watch_folder_def.delete('exclude')
|
114
|
-
logger.debug { "Parameter `excludes` initialized." }
|
115
|
-
|
116
|
-
|
117
|
-
logger.debug { "Initializing parameter `quarantine directory path`." }
|
118
|
-
quarantine_directory_path = watch_folder_def['quarantine_directory_path'] || watch_folder_def['quarantine_path']
|
119
|
-
if quarantine_directory_path
|
120
|
-
quarantine_directory_path = File.expand_path(quarantine_directory_path)
|
121
|
-
watch_folder_def['quarantine_directory_path'] = quarantine_directory_path
|
122
|
-
|
123
|
-
unless Dir.exist?(quarantine_directory_path)
|
124
|
-
logger.warn { "Quarantine directory path '#{quarantine_directory_path}' does not exist. Files will be ignored instead." }
|
125
|
-
end
|
126
|
-
end
|
127
|
-
watch_folder_def.delete('quarantine_path')
|
128
|
-
logger.debug { "Parameter `quarantine directory path` initialized." }
|
129
|
-
|
130
|
-
logger.debug { "Initializing parameter 'completed directory path'." }
|
131
|
-
completed_directory_path = watch_folder_def['completed_directory_path'] || watch_folder_def['completed_path']
|
132
|
-
if completed_directory_path
|
133
|
-
completed_directory_path = File.expand_path(completed_directory_path)
|
134
|
-
watch_folder_def['completed_directory_path'] = completed_directory_path
|
135
|
-
|
136
|
-
unless Dir.exist?(completed_directory_path)
|
137
|
-
logger.warn { "Completed directory path '#{completed_directory_path}' does not exist. File will be ignored instead." }
|
138
|
-
end
|
139
|
-
end
|
140
|
-
watch_folder_def.delete('completed_path')
|
141
|
-
logger.debug { "Parameter 'completed directory path' initialized." }
|
142
|
-
|
143
|
-
|
144
|
-
logger.debug { "Initializing parameter `upload to storage id`." }
|
145
|
-
storage_id = watch_folder_def['upload_to_storage_id'] || watch_folder_def['storage_id']
|
146
|
-
watch_folder_def['upload_to_storage_id'] ||= storage_id
|
147
|
-
watch_folder_def.delete('storage_id')
|
148
|
-
unless storage_id
|
149
|
-
logger.warn { "No `upload to storage id` specified. Uploading will be skipped for this watch folder." }
|
150
|
-
end
|
151
|
-
logger.debug { "Parameter 'upload to storage id' initialized." }
|
152
|
-
|
153
|
-
maximum_active_processors = watch_folder_def['maximum_active_processors']
|
154
|
-
if maximum_active_processors.nil?
|
155
|
-
maximum_active_processors = @default_maximum_active_processors
|
156
|
-
watch_folder_def['maximum_active_processors'] = maximum_active_processors
|
157
|
-
end
|
158
|
-
|
159
80
|
args_out = {}
|
160
81
|
logging = watch_folder_def['logging'] || watch_folder_def
|
161
82
|
log_to = logging['log_to']
|
@@ -163,12 +84,11 @@ module Envoi::Mam::Cantemo
|
|
163
84
|
args_out[:log_to] ||= log_to if log_to && !log_to.empty?
|
164
85
|
args_out[:log_level] ||= log_level if log_level && !log_level.empty?
|
165
86
|
args_out[:logger] ||= logger.dup unless log_to
|
87
|
+
args_out[:default_agent] = default_agent
|
88
|
+
args_out[:default_agent_class] = default_agent_class
|
166
89
|
args_out[:definition] = watch_folder_def
|
167
90
|
|
168
|
-
|
169
|
-
watch_folder = LWF.new(args_out)
|
170
|
-
logger.debug { "Watch Folder Instance Created." }
|
171
|
-
watch_folder
|
91
|
+
Envoi::WatchFolderUtility::WatchFolder.new(args_out)
|
172
92
|
end
|
173
93
|
|
174
94
|
# Iterates through watch_folder_defs and populates @watch_folders with watch folders initialized from the watch
|
@@ -189,27 +109,30 @@ module Envoi::Mam::Cantemo
|
|
189
109
|
logger.debug { 'Processing of watch folder definitions completed.' }
|
190
110
|
end
|
191
111
|
|
192
|
-
# @param [Object] watch_folder
|
193
112
|
# @param [Object] file
|
194
|
-
def add_to_ignore(
|
113
|
+
def add_to_ignore(file)
|
195
114
|
logger.debug { "Adding File to Ignore Cache: '#{file.path}'" }
|
196
115
|
@ignored_file_paths_lock.synchronize do
|
197
|
-
@ignored_file_paths_by_watch_folder[watch_folder] << file.path
|
116
|
+
@ignored_file_paths_by_watch_folder[file.watch_folder] << file.path
|
198
117
|
file.ignore if file.respond_to?(:ignore)
|
199
118
|
end
|
200
119
|
end
|
201
120
|
|
202
|
-
# @param [Object] watch_folder
|
203
121
|
# @param [Object] file
|
204
122
|
# @return [FalseClass]
|
205
|
-
def process_file(
|
123
|
+
def process_file(file)
|
206
124
|
file.processing = true
|
207
125
|
file_name = file.name || file.path
|
208
126
|
logger.debug { "Processing File '#{file_name}'" }
|
127
|
+
watch_folder = file.watch_folder
|
128
|
+
agent = watch_folder.respond_to?(:agent) ? watch_folder.agent : default_agent
|
129
|
+
agent ||= default_agent
|
209
130
|
|
210
131
|
storage_id = watch_folder.definition['upload_to_storage_id']
|
211
132
|
quarantine_directory_path = watch_folder.definition['quarantine_directory_path']
|
212
133
|
completed_directory_path = watch_folder.definition['completed_directory_path']
|
134
|
+
import_args = watch_folder.definition['import_args']
|
135
|
+
import_options = watch_folder.definition['import_options']
|
213
136
|
|
214
137
|
# full_file_path = File.join(watch_folder.path, file.path)
|
215
138
|
full_file_path = file.path
|
@@ -218,7 +141,8 @@ module Envoi::Mam::Cantemo
|
|
218
141
|
return { success: false, message: 'Missing storage ID.' }
|
219
142
|
end
|
220
143
|
|
221
|
-
_response = agent.upload(file_path: full_file_path,
|
144
|
+
_response = agent.upload(file_path: full_file_path,
|
145
|
+
storage_id: storage_id, import_args: import_args, import_options: import_options)
|
222
146
|
_response = { success: _response } if _response == true || _response == false
|
223
147
|
|
224
148
|
if _response[:success]
|
@@ -228,7 +152,7 @@ module Envoi::Mam::Cantemo
|
|
228
152
|
FileUtils.mv full_file_path, completed_directory_path
|
229
153
|
else
|
230
154
|
logger.warn { "Completed directory path not found: '#{completed_directory_path}'" }
|
231
|
-
add_to_ignore(
|
155
|
+
add_to_ignore(file)
|
232
156
|
end
|
233
157
|
else
|
234
158
|
FileUtils.rm full_file_path
|
@@ -239,7 +163,7 @@ module Envoi::Mam::Cantemo
|
|
239
163
|
FileUtils.mv full_file_path, quarantine_directory_path
|
240
164
|
else
|
241
165
|
logger.warn { "Adding '#{full_file_path}' to the temporary ignore list." }
|
242
|
-
add_to_ignore(
|
166
|
+
add_to_ignore(file)
|
243
167
|
end
|
244
168
|
end
|
245
169
|
|
@@ -269,9 +193,14 @@ module Envoi::Mam::Cantemo
|
|
269
193
|
includes = wf.definition['includes']
|
270
194
|
excludes = wf.definition['excludes']
|
271
195
|
|
272
|
-
ignored_file_paths =
|
196
|
+
ignored_file_paths = wf.ignored_file_paths if wf.respond_to?(:ingored_file_paths)
|
197
|
+
unless ignored_file_paths
|
198
|
+
ignored_file_paths = @ignored_file_paths_by_watch_folder[wf]
|
199
|
+
end
|
200
|
+
|
201
|
+
|
273
202
|
stable_files.each do |file|
|
274
|
-
next if ignored_file_paths.include?(file.path)
|
203
|
+
next if file.respond_to?(:ignore?) ? file.ignore? : ignored_file_paths.include?(file.path)
|
275
204
|
next if file.processing
|
276
205
|
next if file.processed
|
277
206
|
|
@@ -319,11 +248,15 @@ module Envoi::Mam::Cantemo
|
|
319
248
|
def poll
|
320
249
|
stable_files_by_watch_folder = {} # Hash.new { |h, k| h[k] = [] }
|
321
250
|
watch_folders.each do |watch_folder|
|
322
|
-
|
323
|
-
|
324
|
-
|
325
|
-
|
251
|
+
next unless watch_folder.poll_interval_elapsed?
|
252
|
+
|
253
|
+
logger.debug { "Polling Watch Folder: #{watch_folder.name}" }
|
254
|
+
watch_folder.poll
|
255
|
+
|
256
|
+
stable_files = watch_folder.stable_files
|
257
|
+
stable_files_by_watch_folder[watch_folder] = stable_files
|
326
258
|
end
|
259
|
+
|
327
260
|
stable_files_by_watch_folder.each do |watch_folder, stable_files|
|
328
261
|
process_watch_folder_stable_files(watch_folder, stable_files)
|
329
262
|
end
|
@@ -357,14 +290,33 @@ module Envoi::Mam::Cantemo
|
|
357
290
|
# AWF.run_once(watch_folders) { |wf| process_watch_folder(wf) }
|
358
291
|
end
|
359
292
|
|
293
|
+
# Converts hash keys to symbols
|
294
|
+
#
|
295
|
+
# @param [Hash] value hash
|
296
|
+
# @param [Boolean] recursive Will recurse into any values that are hashes or arrays
|
297
|
+
def symbolize_keys (value, recursive = true)
|
298
|
+
case value
|
299
|
+
when Hash
|
300
|
+
Hash[value.map { |k,v| [ k.respond_to?(:to_sym) ? k.to_sym : k, recursive ? symbolize_keys(v, true) : v ] }]
|
301
|
+
when Array
|
302
|
+
value.map { |v| symbolize_keys(v, recursive) }
|
303
|
+
else
|
304
|
+
value
|
305
|
+
end
|
306
|
+
# symbolize_keys
|
307
|
+
end
|
308
|
+
|
360
309
|
def self.run(args)
|
361
310
|
w = self.new(args)
|
362
311
|
w.run
|
363
312
|
rescue => e
|
364
|
-
abort("An error occurred
|
313
|
+
abort("An error occurred. #{e.message}\n#{e.backtrace.join("\n")}")
|
365
314
|
end
|
366
315
|
|
367
316
|
def self.daemons_run_proc_with_cleanup(options, &block)
|
317
|
+
options[:dir_mode] = :normal
|
318
|
+
options[:dir] = File.split(__FILE__)[0]
|
319
|
+
|
368
320
|
if block_given?
|
369
321
|
options[:mode] = :proc
|
370
322
|
options[:proc] = block
|
@@ -404,19 +356,15 @@ module Envoi::Mam::Cantemo
|
|
404
356
|
def self.run_as_daemon(args, options = {})
|
405
357
|
# ARGV.unshift 'run' unless %w(start stop restart run zap killall status).include? ARGV.first
|
406
358
|
require 'daemons'
|
407
|
-
# Daemons.run_proc('cantemo-portal-watch-folders', { force: true }) { self.run(args) }
|
408
|
-
app_name = 'cantemo-portal-watch-folders'
|
409
359
|
proc = Proc.new { self.run(args) }
|
360
|
+
app_name = 'cantemo-portal-watch-folders'
|
410
361
|
|
411
|
-
options[:app_name] = app_name
|
412
|
-
options
|
413
|
-
options[:dir_mode] = :normal
|
414
|
-
options[:dir] = File.split(__FILE__)[0]
|
362
|
+
# options[:app_name] = app_name
|
363
|
+
# daemons_run_proc_with_cleanup(options, &proc)
|
415
364
|
|
416
|
-
|
365
|
+
# options[:force] = true
|
366
|
+
Daemons.run_proc(app_name, options, &proc)
|
417
367
|
|
418
|
-
# app_controller = Daemons::Controller.new(options, ARGV)
|
419
|
-
# Daemons.run_proc(app_name, options, &proc)
|
420
368
|
end
|
421
369
|
|
422
370
|
end
|
@@ -0,0 +1,307 @@
|
|
1
|
+
require 'envoi/mam/cantemo/agent'
|
2
|
+
require 'envoi/aspera/watch_service/watch_folder'
|
3
|
+
require 'envoi/watch_folder_utility/watch_folder/handler/listen'
|
4
|
+
|
5
|
+
module Envoi
|
6
|
+
|
7
|
+
module WatchFolderUtility
|
8
|
+
|
9
|
+
class WatchFolder
|
10
|
+
|
11
|
+
DEFAULT_POLL_INTERVAL = 15
|
12
|
+
DEFAULT_PROCESSOR_COUNT_LIMIT = 10
|
13
|
+
|
14
|
+
attr_accessor :logger, :definition, :agent, :handler
|
15
|
+
|
16
|
+
attr_accessor :processors
|
17
|
+
|
18
|
+
attr_accessor :last_poll_time,
|
19
|
+
:min_stable_poll_count,
|
20
|
+
:min_stable_time,
|
21
|
+
:poll_interval
|
22
|
+
|
23
|
+
|
24
|
+
def initialize(args = { })
|
25
|
+
initialize_logger(args)
|
26
|
+
@definition = args[:definition].dup
|
27
|
+
|
28
|
+
logger.debug { "Initializing Watch Folder. #{Object.__id__}" }
|
29
|
+
|
30
|
+
@ignored_file_paths_lock = Mutex.new
|
31
|
+
|
32
|
+
@threaded = args.fetch(:threaded, true)
|
33
|
+
|
34
|
+
@default_maximum_active_processors = DEFAULT_PROCESSOR_COUNT_LIMIT
|
35
|
+
@processors = Hash.new { |h, k| h[k] = {} }
|
36
|
+
|
37
|
+
@default_handler_class = Envoi::WatchFolderUtility::WatchFolder::Handler::Listen
|
38
|
+
process_watch_folder_def
|
39
|
+
|
40
|
+
initialize_handler
|
41
|
+
|
42
|
+
@default_agent = args[:default_agent]
|
43
|
+
@default_agent_class = args[:default_agent_class] || @default_agent.class
|
44
|
+
|
45
|
+
process_agent_defs
|
46
|
+
logger.debug { "Watch Folder Initialized. #{Object.__id__}" }
|
47
|
+
end
|
48
|
+
|
49
|
+
def initialize_logger(args = {})
|
50
|
+
@logger = args[:logger] ||= Logger.new(args[:log_to] || STDOUT)
|
51
|
+
log_level = args[:log_level] = Logger::DEBUG
|
52
|
+
if log_level
|
53
|
+
@logger.level = log_level
|
54
|
+
args[:logger] = @logger
|
55
|
+
end
|
56
|
+
@logger
|
57
|
+
end
|
58
|
+
|
59
|
+
def initialize_agent(args = {})
|
60
|
+
|
61
|
+
@agent ||= begin
|
62
|
+
logger.debug { "Initializing Agent. #{@default_agent_class} #{args}" }
|
63
|
+
_agent = @default_agent_class.new(config: args)
|
64
|
+
logger.debug { "Agent Instance created." }
|
65
|
+
_agent
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def ignored_file_paths
|
70
|
+
handler.ignored_files_map.keys
|
71
|
+
end
|
72
|
+
|
73
|
+
def process_agent_def(agent_def)
|
74
|
+
if @default_agent
|
75
|
+
_agent_def_storages = agent_def['storages'] || {}
|
76
|
+
if _agent_def_storages
|
77
|
+
agent_def['storages'] = default_agent.agent_config_storages.merge(_agent_def_storages)
|
78
|
+
end
|
79
|
+
end
|
80
|
+
initialize_agent(agent_def)
|
81
|
+
end
|
82
|
+
|
83
|
+
def name
|
84
|
+
@name ||= definition['name'] || definition['paths'] || definition['path'] || Object.__id__
|
85
|
+
end
|
86
|
+
|
87
|
+
def process_agent_defs
|
88
|
+
agent_defs = definition['agents']
|
89
|
+
if agent_defs
|
90
|
+
if agent_defs.is_a?(Hash)
|
91
|
+
_agents = agent_defs.map do |name, agent_def|
|
92
|
+
agent_def['name'] ||= name
|
93
|
+
# agent_def['type'] ||= name
|
94
|
+
process_agent_def(agent_def)
|
95
|
+
end
|
96
|
+
elsif agent_defs.is_a?(Array)
|
97
|
+
_agents = agent_defs.map { |agent_def| process_agent_def(agent_def) }
|
98
|
+
else
|
99
|
+
|
100
|
+
end
|
101
|
+
end
|
102
|
+
@agent ||= @default_agent
|
103
|
+
end
|
104
|
+
|
105
|
+
def initialize_handler(watch_folder_def = @definition)
|
106
|
+
args_out = {}
|
107
|
+
logging = watch_folder_def['logging'] || watch_folder_def
|
108
|
+
log_to = logging['log_to']
|
109
|
+
log_level = logging['log_level']
|
110
|
+
args_out[:log_to] ||= log_to if log_to && !log_to.empty?
|
111
|
+
args_out[:log_level] ||= log_level if log_level && !log_level.empty?
|
112
|
+
args_out[:logger] ||= logger.dup unless log_to
|
113
|
+
args_out[:definition] = watch_folder_def
|
114
|
+
|
115
|
+
handler_class = @default_handler_class
|
116
|
+
logger.debug { "Creating Watch Folder Handler Instance. #{handler_class.name}" }
|
117
|
+
@handler = handler_class.new(args_out)
|
118
|
+
logger.debug { "Watch Folder Handler Instance Created. #{handler_class.name}" }
|
119
|
+
end
|
120
|
+
|
121
|
+
#
|
122
|
+
# @param [Hash] watch_folder_def
|
123
|
+
# @option watch_folder_def [String] path
|
124
|
+
# @option watch_folder_def [String] upload_to_storage_id
|
125
|
+
# @option watch_folder_def [String] name (path)
|
126
|
+
# @option watch_folder_def [Array<String>] paths ([path])
|
127
|
+
# @option watch_folder_def [String] exclude ('**/.*')
|
128
|
+
# @option watch_folder_def [Array<string>] excludes ([exclude])
|
129
|
+
# @option watch_folder_def [String] include
|
130
|
+
# @option watch_folder_def [Array<String>] includes ([include])
|
131
|
+
# @option watch_folder_def [String] quarantine_directory_path
|
132
|
+
# @option watch_folder_def [String] completed_directory_path
|
133
|
+
# @option watch_folder_def [Hash] import_args ({})
|
134
|
+
# @option watch_folder_def [Hash] import_options ({})
|
135
|
+
# @option watch_folder_def [Integer|False] maximum_active_processors (@default_maximum_active_processors)
|
136
|
+
# @option watch_folder_def [Hash] logging
|
137
|
+
def process_watch_folder_def(watch_folder_def = @definition)
|
138
|
+
logger.debug { "Initializing Watch Folder #{watch_folder_def.inspect}" }
|
139
|
+
|
140
|
+
logger.debug { "Initializing parameter 'paths'." }
|
141
|
+
name = watch_folder_def['name']
|
142
|
+
|
143
|
+
path = watch_folder_def['path']
|
144
|
+
|
145
|
+
paths = watch_folder_def['paths'] ||= []
|
146
|
+
paths = [ paths ] if paths.is_a?(String)
|
147
|
+
paths.concat [*path] if path
|
148
|
+
paths.map! { |p| File.expand_path(p) }
|
149
|
+
if paths.empty?
|
150
|
+
name_as_path = File.expand_path(name)
|
151
|
+
paths.concat name_as_path if Dir.exist?(name_as_path)
|
152
|
+
end
|
153
|
+
paths.uniq!
|
154
|
+
watch_folder_def['paths'] = paths
|
155
|
+
# watch_folder_def['path'] ||= paths.first if paths.length == 1
|
156
|
+
watch_folder_def.delete('path')
|
157
|
+
|
158
|
+
if paths.empty?
|
159
|
+
logger.error { "Failed to initialize watch folder. No path found in watch folder definition." }
|
160
|
+
return false
|
161
|
+
end
|
162
|
+
logger.debug { "Parameter 'paths' initialized." }
|
163
|
+
|
164
|
+
logger.debug { "Initializing parameter 'includes'." }
|
165
|
+
include = watch_folder_def['include']
|
166
|
+
includes = (watch_folder_def['includes'] ||= [])
|
167
|
+
includes.concat [*include] if include
|
168
|
+
includes.uniq!
|
169
|
+
includes.map! { |e| Regexp.try_convert(e) || e }
|
170
|
+
watch_folder_def['includes'] = includes
|
171
|
+
watch_folder_def.delete('include')
|
172
|
+
logger.debug { "Parameter `includes` initialized." }
|
173
|
+
|
174
|
+
logger.debug { "Initializing parameter 'excludes'." }
|
175
|
+
exclude = watch_folder_def['exclude']
|
176
|
+
exclude ||= '**/.*'
|
177
|
+
excludes = (watch_folder_def['excludes'] ||= [])
|
178
|
+
excludes.concat [*exclude] if exclude
|
179
|
+
excludes.uniq!
|
180
|
+
excludes.map! { |e| Regexp.try_convert(e) || e }
|
181
|
+
watch_folder_def['excludes'] = excludes
|
182
|
+
watch_folder_def.delete('exclude')
|
183
|
+
logger.debug { "Parameter `excludes` initialized." }
|
184
|
+
|
185
|
+
|
186
|
+
logger.debug { "Initializing parameter `quarantine directory path`." }
|
187
|
+
quarantine_directory_path = watch_folder_def['quarantine_directory_path'] || watch_folder_def['quarantine_path']
|
188
|
+
if quarantine_directory_path
|
189
|
+
quarantine_directory_path = File.expand_path(quarantine_directory_path)
|
190
|
+
watch_folder_def['quarantine_directory_path'] = quarantine_directory_path
|
191
|
+
|
192
|
+
unless Dir.exist?(quarantine_directory_path)
|
193
|
+
logger.warn { "Quarantine directory path '#{quarantine_directory_path}' does not exist. Files will be ignored instead." }
|
194
|
+
end
|
195
|
+
end
|
196
|
+
watch_folder_def.delete('quarantine_path')
|
197
|
+
logger.debug { "Parameter `quarantine directory path` initialized." }
|
198
|
+
|
199
|
+
logger.debug { "Initializing parameter 'completed directory path'." }
|
200
|
+
completed_directory_path = watch_folder_def['completed_directory_path'] || watch_folder_def['completed_path']
|
201
|
+
if completed_directory_path
|
202
|
+
completed_directory_path = File.expand_path(completed_directory_path)
|
203
|
+
watch_folder_def['completed_directory_path'] = completed_directory_path
|
204
|
+
|
205
|
+
unless Dir.exist?(completed_directory_path)
|
206
|
+
logger.warn { "Completed directory path '#{completed_directory_path}' does not exist. File will be ignored instead." }
|
207
|
+
end
|
208
|
+
end
|
209
|
+
watch_folder_def.delete('completed_path')
|
210
|
+
logger.debug { "Parameter 'completed directory path' initialized." }
|
211
|
+
|
212
|
+
|
213
|
+
logger.debug { "Initializing parameter `upload to storage id`." }
|
214
|
+
storage_id = watch_folder_def['upload_to_storage_id'] || watch_folder_def['storage_id']
|
215
|
+
watch_folder_def['upload_to_storage_id'] ||= storage_id
|
216
|
+
watch_folder_def.delete('storage_id')
|
217
|
+
unless storage_id
|
218
|
+
logger.warn { "No `upload to storage id` specified. Uploading will be skipped for this watch folder." }
|
219
|
+
end
|
220
|
+
logger.debug { "Parameter 'upload to storage id' initialized." }
|
221
|
+
|
222
|
+
logger.debug { "Initializing parameter 'import args'." }
|
223
|
+
import_args = watch_folder_def['import_args'] || { }
|
224
|
+
import_options = watch_folder_def['import_options'] || { }
|
225
|
+
|
226
|
+
add_item_to_collection = import_options.fetch(:add_item_to_collection, import_args[:add_item_to_collection])
|
227
|
+
if add_item_to_collection.nil? || add_item_to_collection
|
228
|
+
_add_item_to_collection = false
|
229
|
+
collection_id = watch_folder_def['collection_id']
|
230
|
+
if collection_id
|
231
|
+
import_args[:collection_id] ||= collection_id
|
232
|
+
_add_item_to_collection = true
|
233
|
+
watch_folder_def.delete('collection_id')
|
234
|
+
else
|
235
|
+
collection_name = watch_folder_def['collection_name']
|
236
|
+
if collection_name
|
237
|
+
import_args[:collection_name] ||= collection_name
|
238
|
+
_add_item_to_collection = true
|
239
|
+
watch_folder_def.delete('collection_name')
|
240
|
+
else
|
241
|
+
file_path_collection_name_position = import_args[:file_path_collection_name_position]
|
242
|
+
if file_path_collection_name_position
|
243
|
+
import_args[:file_path_collection_name_position] = file_path_collection_name_position
|
244
|
+
_add_item_to_collection = true
|
245
|
+
watch_folder_def.delete('file_path_collection_name_position')
|
246
|
+
end
|
247
|
+
end
|
248
|
+
end
|
249
|
+
import_options[:add_item_to_collection] ||= _add_item_to_collection
|
250
|
+
end
|
251
|
+
|
252
|
+
metadata = watch_folder_def['metadata']
|
253
|
+
if metadata
|
254
|
+
import_args[:metadata] = metadata
|
255
|
+
watch_folder_def.delete('metadata')
|
256
|
+
metadata_map = watch_folder_def['metadata_map']
|
257
|
+
import_options[:metadata_map] ||= metadata_map if metadata_map
|
258
|
+
end
|
259
|
+
logger.debug { "Parameter 'import args' initialized." }
|
260
|
+
|
261
|
+
|
262
|
+
maximum_active_processors = watch_folder_def['maximum_active_processors']
|
263
|
+
if maximum_active_processors.nil?
|
264
|
+
maximum_active_processors = @default_maximum_active_processors
|
265
|
+
watch_folder_def['maximum_active_processors'] = maximum_active_processors
|
266
|
+
end
|
267
|
+
|
268
|
+
logger.debug { "Initializing parameter 'agents'." }
|
269
|
+
agent = watch_folder_def['agent']
|
270
|
+
agents = watch_folder_def['agents'] ||= [ ]
|
271
|
+
if agents.is_a?(Hash)
|
272
|
+
agents = agents.map { |k,v| v['name'] ||= k; v }
|
273
|
+
end
|
274
|
+
if agent
|
275
|
+
if agent.is_a?(Hash)
|
276
|
+
if agent.keys.length == 1
|
277
|
+
agent = agent.map { |k,v| v['name'] ||= k; v }
|
278
|
+
end
|
279
|
+
end
|
280
|
+
agents.concat [*agent]
|
281
|
+
watch_folder_def.delete('agent')
|
282
|
+
end
|
283
|
+
logger.debug { "Parameter 'agent' initialized." }
|
284
|
+
|
285
|
+
@poll_interval = watch_folder_def['poll_interval'] ||= DEFAULT_POLL_INTERVAL
|
286
|
+
end
|
287
|
+
|
288
|
+
def poll
|
289
|
+
@previous_poll_time = @last_poll_time
|
290
|
+
@last_poll_time = Time.now
|
291
|
+
|
292
|
+
handler.poll if handler.respond_to?(:poll)
|
293
|
+
end
|
294
|
+
|
295
|
+
def poll_interval_elapsed?
|
296
|
+
!last_poll_time || (Time.now - last_poll_time) >= poll_interval
|
297
|
+
end
|
298
|
+
|
299
|
+
def stable_files
|
300
|
+
handler.stable_files
|
301
|
+
end
|
302
|
+
|
303
|
+
end
|
304
|
+
|
305
|
+
end
|
306
|
+
|
307
|
+
end
|