cantemo-portal-agent 1.1.1 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,6 @@
1
1
  require 'envoi/mam/cantemo/agent'
2
2
  require 'envoi/aspera/watch_service/watch_folder'
3
+ require 'envoi/watch_folder_utility/watch_folder'
3
4
  require 'envoi/watch_folder_utility/watch_folder/handler/listen'
4
5
 
5
6
  module Envoi::Mam::Cantemo
@@ -13,21 +14,33 @@ module Envoi::Mam::Cantemo
13
14
 
14
15
  DEFAULT_WATCH_FOLDER_PROCESSOR_LIMIT = 10
15
16
 
16
- attr_accessor :logger, :agent, :config, :watch_folder_defs, :watch_folders
17
+ attr_accessor :logger, :default_agent_class, :default_agent, :config, :watch_folder_defs, :watch_folders
17
18
 
18
19
  def initialize(args = {})
19
20
  initialize_logger(args)
20
21
 
21
22
  logger.debug { 'Initializing Agent Watch Folder Manager.' }
22
23
 
23
- logger.debug { 'Initializing Cantemo Portal Agent.' }
24
24
  args[:default_preserve_file_path] = args.fetch(:default_preserve_file_path, false)
25
- @agent = Envoi::Mam::Cantemo::Agent.load_from_config_file(args)
26
- logger.debug { 'Cantemo Portal Agent Initialized.' }
27
25
 
28
- @config = agent.config
29
- cantemo_config = config[:cantemo] || config['cantemo']
30
- @watch_folder_defs = cantemo_config[:watch_folders] || cantemo_config['watch_folders']
26
+ @config = Envoi::Mam::Cantemo::Agent.load_config_from_file(args)
27
+ cantemo_config = config[:cantemo] || config['cantemo']
28
+
29
+ @default_agent_class = Envoi::Mam::Cantemo::Agent
30
+
31
+ if cantemo_config
32
+ logger.debug { 'Initializing Default Cantemo Portal Agent.' }
33
+ @default_agent = @default_agent_class.new(args.merge({ config: cantemo_config }))
34
+ logger.debug { 'Default Cantemo Portal Agent Initialized.' }
35
+ @default_storages = @default_agent.agent_config_storages
36
+ else
37
+ @default_agent = nil
38
+ end
39
+
40
+ @watch_folder_defs = config[:watch_folders] || config['watch_folders']
41
+ if !@watch_folder_defs && cantemo_config
42
+ @watch_folder_defs = cantemo_config[:watch_folders] || cantemo_config['watch_folders']
43
+ end
31
44
 
32
45
  @ignored_file_paths_by_watch_folder = Hash.new { |h, k| h[k] = [] }
33
46
  @ignored_file_paths_lock = Mutex.new
@@ -50,8 +63,6 @@ module Envoi::Mam::Cantemo
50
63
  @logger
51
64
  end
52
65
 
53
- # @TODO Move most of this code into a watch folder class
54
- #
55
66
  # @param [Hash] watch_folder_def
56
67
  # @option watch_folder_def [String] path
57
68
  # @option watch_folder_def [String] upload_to_storage_id
@@ -66,96 +77,6 @@ module Envoi::Mam::Cantemo
66
77
  # @option watch_folder_def [Integer|False] maximum_active_processors (@default_maximum_active_processors)
67
78
  # @option watch_folder_def [Hash] logging
68
79
  def process_watch_folder_def(watch_folder_def)
69
- logger.debug { "Initializing Watch Folder #{watch_folder_def.inspect}" }
70
-
71
- logger.debug { "Initializing parameter 'paths'." }
72
- name = watch_folder_def['name']
73
-
74
- path = watch_folder_def['path']
75
-
76
- paths = watch_folder_def['paths'] ||= []
77
- paths = [ paths ] if paths.is_a?(String)
78
- paths.concat [*path] if path
79
- paths.map! { |p| File.expand_path(p) }
80
- if paths.empty?
81
- name_as_path = File.expand_path(name)
82
- paths.concat name_as_path if Dir.exist?(name_as_path)
83
- end
84
- paths.uniq!
85
- watch_folder_def['paths'] = paths
86
- # watch_folder_def['path'] ||= paths.first if paths.length == 1
87
- watch_folder_def.delete('path')
88
-
89
- if paths.empty?
90
- logger.error { "Failed to initialize watch folder. No path found in watch folder definition." }
91
- return false
92
- end
93
- logger.debug { "Parameter 'paths' initialized." }
94
-
95
- logger.debug { "Initializing parameter 'includes'." }
96
- include = watch_folder_def['include']
97
- includes = (watch_folder_def['includes'] ||= [])
98
- includes.concat [*include] if include
99
- includes.uniq!
100
- includes.map! { |e| Regexp.try_convert(e) || e }
101
- watch_folder_def['includes'] = includes
102
- watch_folder_def.delete('include')
103
- logger.debug { "Parameter `includes` initialized." }
104
-
105
- logger.debug { "Initializing parameter 'excludes'." }
106
- exclude = watch_folder_def['exclude']
107
- exclude ||= '**/.*'
108
- excludes = (watch_folder_def['excludes'] ||= [])
109
- excludes.concat [*exclude] if exclude
110
- excludes.uniq!
111
- excludes.map! { |e| Regexp.try_convert(e) || e }
112
- watch_folder_def['excludes'] = excludes
113
- watch_folder_def.delete('exclude')
114
- logger.debug { "Parameter `excludes` initialized." }
115
-
116
-
117
- logger.debug { "Initializing parameter `quarantine directory path`." }
118
- quarantine_directory_path = watch_folder_def['quarantine_directory_path'] || watch_folder_def['quarantine_path']
119
- if quarantine_directory_path
120
- quarantine_directory_path = File.expand_path(quarantine_directory_path)
121
- watch_folder_def['quarantine_directory_path'] = quarantine_directory_path
122
-
123
- unless Dir.exist?(quarantine_directory_path)
124
- logger.warn { "Quarantine directory path '#{quarantine_directory_path}' does not exist. Files will be ignored instead." }
125
- end
126
- end
127
- watch_folder_def.delete('quarantine_path')
128
- logger.debug { "Parameter `quarantine directory path` initialized." }
129
-
130
- logger.debug { "Initializing parameter 'completed directory path'." }
131
- completed_directory_path = watch_folder_def['completed_directory_path'] || watch_folder_def['completed_path']
132
- if completed_directory_path
133
- completed_directory_path = File.expand_path(completed_directory_path)
134
- watch_folder_def['completed_directory_path'] = completed_directory_path
135
-
136
- unless Dir.exist?(completed_directory_path)
137
- logger.warn { "Completed directory path '#{completed_directory_path}' does not exist. File will be ignored instead." }
138
- end
139
- end
140
- watch_folder_def.delete('completed_path')
141
- logger.debug { "Parameter 'completed directory path' initialized." }
142
-
143
-
144
- logger.debug { "Initializing parameter `upload to storage id`." }
145
- storage_id = watch_folder_def['upload_to_storage_id'] || watch_folder_def['storage_id']
146
- watch_folder_def['upload_to_storage_id'] ||= storage_id
147
- watch_folder_def.delete('storage_id')
148
- unless storage_id
149
- logger.warn { "No `upload to storage id` specified. Uploading will be skipped for this watch folder." }
150
- end
151
- logger.debug { "Parameter 'upload to storage id' initialized." }
152
-
153
- maximum_active_processors = watch_folder_def['maximum_active_processors']
154
- if maximum_active_processors.nil?
155
- maximum_active_processors = @default_maximum_active_processors
156
- watch_folder_def['maximum_active_processors'] = maximum_active_processors
157
- end
158
-
159
80
  args_out = {}
160
81
  logging = watch_folder_def['logging'] || watch_folder_def
161
82
  log_to = logging['log_to']
@@ -163,12 +84,11 @@ module Envoi::Mam::Cantemo
163
84
  args_out[:log_to] ||= log_to if log_to && !log_to.empty?
164
85
  args_out[:log_level] ||= log_level if log_level && !log_level.empty?
165
86
  args_out[:logger] ||= logger.dup unless log_to
87
+ args_out[:default_agent] = default_agent
88
+ args_out[:default_agent_class] = default_agent_class
166
89
  args_out[:definition] = watch_folder_def
167
90
 
168
- logger.debug { "Initializing Watch Folder Instance from Def. #{watch_folder_def}" }
169
- watch_folder = LWF.new(args_out)
170
- logger.debug { "Watch Folder Instance Created." }
171
- watch_folder
91
+ Envoi::WatchFolderUtility::WatchFolder.new(args_out)
172
92
  end
173
93
 
174
94
  # Iterates through watch_folder_defs and populates @watch_folders with watch folders initialized from the watch
@@ -189,27 +109,30 @@ module Envoi::Mam::Cantemo
189
109
  logger.debug { 'Processing of watch folder definitions completed.' }
190
110
  end
191
111
 
192
- # @param [Object] watch_folder
193
112
  # @param [Object] file
194
- def add_to_ignore(watch_folder, file)
113
+ def add_to_ignore(file)
195
114
  logger.debug { "Adding File to Ignore Cache: '#{file.path}'" }
196
115
  @ignored_file_paths_lock.synchronize do
197
- @ignored_file_paths_by_watch_folder[watch_folder] << file.path
116
+ @ignored_file_paths_by_watch_folder[file.watch_folder] << file.path
198
117
  file.ignore if file.respond_to?(:ignore)
199
118
  end
200
119
  end
201
120
 
202
- # @param [Object] watch_folder
203
121
  # @param [Object] file
204
122
  # @return [FalseClass]
205
- def process_file(watch_folder, file)
123
+ def process_file(file)
206
124
  file.processing = true
207
125
  file_name = file.name || file.path
208
126
  logger.debug { "Processing File '#{file_name}'" }
127
+ watch_folder = file.watch_folder
128
+ agent = watch_folder.respond_to?(:agent) ? watch_folder.agent : default_agent
129
+ agent ||= default_agent
209
130
 
210
131
  storage_id = watch_folder.definition['upload_to_storage_id']
211
132
  quarantine_directory_path = watch_folder.definition['quarantine_directory_path']
212
133
  completed_directory_path = watch_folder.definition['completed_directory_path']
134
+ import_args = watch_folder.definition['import_args']
135
+ import_options = watch_folder.definition['import_options']
213
136
 
214
137
  # full_file_path = File.join(watch_folder.path, file.path)
215
138
  full_file_path = file.path
@@ -218,7 +141,8 @@ module Envoi::Mam::Cantemo
218
141
  return { success: false, message: 'Missing storage ID.' }
219
142
  end
220
143
 
221
- _response = agent.upload(file_path: full_file_path, storage_id: storage_id)
144
+ _response = agent.upload(file_path: full_file_path,
145
+ storage_id: storage_id, import_args: import_args, import_options: import_options)
222
146
  _response = { success: _response } if _response == true || _response == false
223
147
 
224
148
  if _response[:success]
@@ -228,7 +152,7 @@ module Envoi::Mam::Cantemo
228
152
  FileUtils.mv full_file_path, completed_directory_path
229
153
  else
230
154
  logger.warn { "Completed directory path not found: '#{completed_directory_path}'" }
231
- add_to_ignore(watch_folder, file)
155
+ add_to_ignore(file)
232
156
  end
233
157
  else
234
158
  FileUtils.rm full_file_path
@@ -239,7 +163,7 @@ module Envoi::Mam::Cantemo
239
163
  FileUtils.mv full_file_path, quarantine_directory_path
240
164
  else
241
165
  logger.warn { "Adding '#{full_file_path}' to the temporary ignore list." }
242
- add_to_ignore(watch_folder, file)
166
+ add_to_ignore(file)
243
167
  end
244
168
  end
245
169
 
@@ -269,9 +193,14 @@ module Envoi::Mam::Cantemo
269
193
  includes = wf.definition['includes']
270
194
  excludes = wf.definition['excludes']
271
195
 
272
- ignored_file_paths = @ignored_file_paths_by_watch_folder[wf]
196
+ ignored_file_paths = wf.ignored_file_paths if wf.respond_to?(:ingored_file_paths)
197
+ unless ignored_file_paths
198
+ ignored_file_paths = @ignored_file_paths_by_watch_folder[wf]
199
+ end
200
+
201
+
273
202
  stable_files.each do |file|
274
- next if ignored_file_paths.include?(file.path)
203
+ next if file.respond_to?(:ignore?) ? file.ignore? : ignored_file_paths.include?(file.path)
275
204
  next if file.processing
276
205
  next if file.processed
277
206
 
@@ -319,11 +248,15 @@ module Envoi::Mam::Cantemo
319
248
  def poll
320
249
  stable_files_by_watch_folder = {} # Hash.new { |h, k| h[k] = [] }
321
250
  watch_folders.each do |watch_folder|
322
- if !watch_folder.last_poll_time || (Time.now - watch_folder.last_poll_time) >= watch_folder.poll_interval
323
- stable_files = watch_folder.poll
324
- stable_files_by_watch_folder[watch_folder] = stable_files
325
- end
251
+ next unless watch_folder.poll_interval_elapsed?
252
+
253
+ logger.debug { "Polling Watch Folder: #{watch_folder.name}" }
254
+ watch_folder.poll
255
+
256
+ stable_files = watch_folder.stable_files
257
+ stable_files_by_watch_folder[watch_folder] = stable_files
326
258
  end
259
+
327
260
  stable_files_by_watch_folder.each do |watch_folder, stable_files|
328
261
  process_watch_folder_stable_files(watch_folder, stable_files)
329
262
  end
@@ -357,14 +290,33 @@ module Envoi::Mam::Cantemo
357
290
  # AWF.run_once(watch_folders) { |wf| process_watch_folder(wf) }
358
291
  end
359
292
 
293
+ # Converts hash keys to symbols
294
+ #
295
+ # @param [Hash] value hash
296
+ # @param [Boolean] recursive Will recurse into any values that are hashes or arrays
297
+ def symbolize_keys (value, recursive = true)
298
+ case value
299
+ when Hash
300
+ Hash[value.map { |k,v| [ k.respond_to?(:to_sym) ? k.to_sym : k, recursive ? symbolize_keys(v, true) : v ] }]
301
+ when Array
302
+ value.map { |v| symbolize_keys(v, recursive) }
303
+ else
304
+ value
305
+ end
306
+ # symbolize_keys
307
+ end
308
+
360
309
  def self.run(args)
361
310
  w = self.new(args)
362
311
  w.run
363
312
  rescue => e
364
- abort("An error occurred.\n#{e.message}\n#{e.message}")
313
+ abort("An error occurred. #{e.message}\n#{e.backtrace.join("\n")}")
365
314
  end
366
315
 
367
316
  def self.daemons_run_proc_with_cleanup(options, &block)
317
+ options[:dir_mode] = :normal
318
+ options[:dir] = File.split(__FILE__)[0]
319
+
368
320
  if block_given?
369
321
  options[:mode] = :proc
370
322
  options[:proc] = block
@@ -404,19 +356,15 @@ module Envoi::Mam::Cantemo
404
356
  def self.run_as_daemon(args, options = {})
405
357
  # ARGV.unshift 'run' unless %w(start stop restart run zap killall status).include? ARGV.first
406
358
  require 'daemons'
407
- # Daemons.run_proc('cantemo-portal-watch-folders', { force: true }) { self.run(args) }
408
- app_name = 'cantemo-portal-watch-folders'
409
359
  proc = Proc.new { self.run(args) }
360
+ app_name = 'cantemo-portal-watch-folders'
410
361
 
411
- options[:app_name] = app_name
412
- options[:mode] = :proc
413
- options[:dir_mode] = :normal
414
- options[:dir] = File.split(__FILE__)[0]
362
+ # options[:app_name] = app_name
363
+ # daemons_run_proc_with_cleanup(options, &proc)
415
364
 
416
- daemons_run_proc_with_cleanup(options, &proc)
365
+ # options[:force] = true
366
+ Daemons.run_proc(app_name, options, &proc)
417
367
 
418
- # app_controller = Daemons::Controller.new(options, ARGV)
419
- # Daemons.run_proc(app_name, options, &proc)
420
368
  end
421
369
 
422
370
  end
@@ -0,0 +1,307 @@
1
+ require 'envoi/mam/cantemo/agent'
2
+ require 'envoi/aspera/watch_service/watch_folder'
3
+ require 'envoi/watch_folder_utility/watch_folder/handler/listen'
4
+
5
+ module Envoi
6
+
7
+ module WatchFolderUtility
8
+
9
+ class WatchFolder
10
+
11
+ DEFAULT_POLL_INTERVAL = 15
12
+ DEFAULT_PROCESSOR_COUNT_LIMIT = 10
13
+
14
+ attr_accessor :logger, :definition, :agent, :handler
15
+
16
+ attr_accessor :processors
17
+
18
+ attr_accessor :last_poll_time,
19
+ :min_stable_poll_count,
20
+ :min_stable_time,
21
+ :poll_interval
22
+
23
+
24
+ def initialize(args = { })
25
+ initialize_logger(args)
26
+ @definition = args[:definition].dup
27
+
28
+ logger.debug { "Initializing Watch Folder. #{Object.__id__}" }
29
+
30
+ @ignored_file_paths_lock = Mutex.new
31
+
32
+ @threaded = args.fetch(:threaded, true)
33
+
34
+ @default_maximum_active_processors = DEFAULT_PROCESSOR_COUNT_LIMIT
35
+ @processors = Hash.new { |h, k| h[k] = {} }
36
+
37
+ @default_handler_class = Envoi::WatchFolderUtility::WatchFolder::Handler::Listen
38
+ process_watch_folder_def
39
+
40
+ initialize_handler
41
+
42
+ @default_agent = args[:default_agent]
43
+ @default_agent_class = args[:default_agent_class] || @default_agent.class
44
+
45
+ process_agent_defs
46
+ logger.debug { "Watch Folder Initialized. #{Object.__id__}" }
47
+ end
48
+
49
+ def initialize_logger(args = {})
50
+ @logger = args[:logger] ||= Logger.new(args[:log_to] || STDOUT)
51
+ log_level = args[:log_level] = Logger::DEBUG
52
+ if log_level
53
+ @logger.level = log_level
54
+ args[:logger] = @logger
55
+ end
56
+ @logger
57
+ end
58
+
59
+ def initialize_agent(args = {})
60
+
61
+ @agent ||= begin
62
+ logger.debug { "Initializing Agent. #{@default_agent_class} #{args}" }
63
+ _agent = @default_agent_class.new(config: args)
64
+ logger.debug { "Agent Instance created." }
65
+ _agent
66
+ end
67
+ end
68
+
69
+ def ignored_file_paths
70
+ handler.ignored_files_map.keys
71
+ end
72
+
73
+ def process_agent_def(agent_def)
74
+ if @default_agent
75
+ _agent_def_storages = agent_def['storages'] || {}
76
+ if _agent_def_storages
77
+ agent_def['storages'] = default_agent.agent_config_storages.merge(_agent_def_storages)
78
+ end
79
+ end
80
+ initialize_agent(agent_def)
81
+ end
82
+
83
+ def name
84
+ @name ||= definition['name'] || definition['paths'] || definition['path'] || Object.__id__
85
+ end
86
+
87
+ def process_agent_defs
88
+ agent_defs = definition['agents']
89
+ if agent_defs
90
+ if agent_defs.is_a?(Hash)
91
+ _agents = agent_defs.map do |name, agent_def|
92
+ agent_def['name'] ||= name
93
+ # agent_def['type'] ||= name
94
+ process_agent_def(agent_def)
95
+ end
96
+ elsif agent_defs.is_a?(Array)
97
+ _agents = agent_defs.map { |agent_def| process_agent_def(agent_def) }
98
+ else
99
+
100
+ end
101
+ end
102
+ @agent ||= @default_agent
103
+ end
104
+
105
+ def initialize_handler(watch_folder_def = @definition)
106
+ args_out = {}
107
+ logging = watch_folder_def['logging'] || watch_folder_def
108
+ log_to = logging['log_to']
109
+ log_level = logging['log_level']
110
+ args_out[:log_to] ||= log_to if log_to && !log_to.empty?
111
+ args_out[:log_level] ||= log_level if log_level && !log_level.empty?
112
+ args_out[:logger] ||= logger.dup unless log_to
113
+ args_out[:definition] = watch_folder_def
114
+
115
+ handler_class = @default_handler_class
116
+ logger.debug { "Creating Watch Folder Handler Instance. #{handler_class.name}" }
117
+ @handler = handler_class.new(args_out)
118
+ logger.debug { "Watch Folder Handler Instance Created. #{handler_class.name}" }
119
+ end
120
+
121
+ #
122
+ # @param [Hash] watch_folder_def
123
+ # @option watch_folder_def [String] path
124
+ # @option watch_folder_def [String] upload_to_storage_id
125
+ # @option watch_folder_def [String] name (path)
126
+ # @option watch_folder_def [Array<String>] paths ([path])
127
+ # @option watch_folder_def [String] exclude ('**/.*')
128
+ # @option watch_folder_def [Array<string>] excludes ([exclude])
129
+ # @option watch_folder_def [String] include
130
+ # @option watch_folder_def [Array<String>] includes ([include])
131
+ # @option watch_folder_def [String] quarantine_directory_path
132
+ # @option watch_folder_def [String] completed_directory_path
133
+ # @option watch_folder_def [Hash] import_args ({})
134
+ # @option watch_folder_def [Hash] import_options ({})
135
+ # @option watch_folder_def [Integer|False] maximum_active_processors (@default_maximum_active_processors)
136
+ # @option watch_folder_def [Hash] logging
137
+ def process_watch_folder_def(watch_folder_def = @definition)
138
+ logger.debug { "Initializing Watch Folder #{watch_folder_def.inspect}" }
139
+
140
+ logger.debug { "Initializing parameter 'paths'." }
141
+ name = watch_folder_def['name']
142
+
143
+ path = watch_folder_def['path']
144
+
145
+ paths = watch_folder_def['paths'] ||= []
146
+ paths = [ paths ] if paths.is_a?(String)
147
+ paths.concat [*path] if path
148
+ paths.map! { |p| File.expand_path(p) }
149
+ if paths.empty?
150
+ name_as_path = File.expand_path(name)
151
+ paths.concat name_as_path if Dir.exist?(name_as_path)
152
+ end
153
+ paths.uniq!
154
+ watch_folder_def['paths'] = paths
155
+ # watch_folder_def['path'] ||= paths.first if paths.length == 1
156
+ watch_folder_def.delete('path')
157
+
158
+ if paths.empty?
159
+ logger.error { "Failed to initialize watch folder. No path found in watch folder definition." }
160
+ return false
161
+ end
162
+ logger.debug { "Parameter 'paths' initialized." }
163
+
164
+ logger.debug { "Initializing parameter 'includes'." }
165
+ include = watch_folder_def['include']
166
+ includes = (watch_folder_def['includes'] ||= [])
167
+ includes.concat [*include] if include
168
+ includes.uniq!
169
+ includes.map! { |e| Regexp.try_convert(e) || e }
170
+ watch_folder_def['includes'] = includes
171
+ watch_folder_def.delete('include')
172
+ logger.debug { "Parameter `includes` initialized." }
173
+
174
+ logger.debug { "Initializing parameter 'excludes'." }
175
+ exclude = watch_folder_def['exclude']
176
+ exclude ||= '**/.*'
177
+ excludes = (watch_folder_def['excludes'] ||= [])
178
+ excludes.concat [*exclude] if exclude
179
+ excludes.uniq!
180
+ excludes.map! { |e| Regexp.try_convert(e) || e }
181
+ watch_folder_def['excludes'] = excludes
182
+ watch_folder_def.delete('exclude')
183
+ logger.debug { "Parameter `excludes` initialized." }
184
+
185
+
186
+ logger.debug { "Initializing parameter `quarantine directory path`." }
187
+ quarantine_directory_path = watch_folder_def['quarantine_directory_path'] || watch_folder_def['quarantine_path']
188
+ if quarantine_directory_path
189
+ quarantine_directory_path = File.expand_path(quarantine_directory_path)
190
+ watch_folder_def['quarantine_directory_path'] = quarantine_directory_path
191
+
192
+ unless Dir.exist?(quarantine_directory_path)
193
+ logger.warn { "Quarantine directory path '#{quarantine_directory_path}' does not exist. Files will be ignored instead." }
194
+ end
195
+ end
196
+ watch_folder_def.delete('quarantine_path')
197
+ logger.debug { "Parameter `quarantine directory path` initialized." }
198
+
199
+ logger.debug { "Initializing parameter 'completed directory path'." }
200
+ completed_directory_path = watch_folder_def['completed_directory_path'] || watch_folder_def['completed_path']
201
+ if completed_directory_path
202
+ completed_directory_path = File.expand_path(completed_directory_path)
203
+ watch_folder_def['completed_directory_path'] = completed_directory_path
204
+
205
+ unless Dir.exist?(completed_directory_path)
206
+ logger.warn { "Completed directory path '#{completed_directory_path}' does not exist. File will be ignored instead." }
207
+ end
208
+ end
209
+ watch_folder_def.delete('completed_path')
210
+ logger.debug { "Parameter 'completed directory path' initialized." }
211
+
212
+
213
+ logger.debug { "Initializing parameter `upload to storage id`." }
214
+ storage_id = watch_folder_def['upload_to_storage_id'] || watch_folder_def['storage_id']
215
+ watch_folder_def['upload_to_storage_id'] ||= storage_id
216
+ watch_folder_def.delete('storage_id')
217
+ unless storage_id
218
+ logger.warn { "No `upload to storage id` specified. Uploading will be skipped for this watch folder." }
219
+ end
220
+ logger.debug { "Parameter 'upload to storage id' initialized." }
221
+
222
+ logger.debug { "Initializing parameter 'import args'." }
223
+ import_args = watch_folder_def['import_args'] || { }
224
+ import_options = watch_folder_def['import_options'] || { }
225
+
226
+ add_item_to_collection = import_options.fetch(:add_item_to_collection, import_args[:add_item_to_collection])
227
+ if add_item_to_collection.nil? || add_item_to_collection
228
+ _add_item_to_collection = false
229
+ collection_id = watch_folder_def['collection_id']
230
+ if collection_id
231
+ import_args[:collection_id] ||= collection_id
232
+ _add_item_to_collection = true
233
+ watch_folder_def.delete('collection_id')
234
+ else
235
+ collection_name = watch_folder_def['collection_name']
236
+ if collection_name
237
+ import_args[:collection_name] ||= collection_name
238
+ _add_item_to_collection = true
239
+ watch_folder_def.delete('collection_name')
240
+ else
241
+ file_path_collection_name_position = import_args[:file_path_collection_name_position]
242
+ if file_path_collection_name_position
243
+ import_args[:file_path_collection_name_position] = file_path_collection_name_position
244
+ _add_item_to_collection = true
245
+ watch_folder_def.delete('file_path_collection_name_position')
246
+ end
247
+ end
248
+ end
249
+ import_options[:add_item_to_collection] ||= _add_item_to_collection
250
+ end
251
+
252
+ metadata = watch_folder_def['metadata']
253
+ if metadata
254
+ import_args[:metadata] = metadata
255
+ watch_folder_def.delete('metadata')
256
+ metadata_map = watch_folder_def['metadata_map']
257
+ import_options[:metadata_map] ||= metadata_map if metadata_map
258
+ end
259
+ logger.debug { "Parameter 'import args' initialized." }
260
+
261
+
262
+ maximum_active_processors = watch_folder_def['maximum_active_processors']
263
+ if maximum_active_processors.nil?
264
+ maximum_active_processors = @default_maximum_active_processors
265
+ watch_folder_def['maximum_active_processors'] = maximum_active_processors
266
+ end
267
+
268
+ logger.debug { "Initializing parameter 'agents'." }
269
+ agent = watch_folder_def['agent']
270
+ agents = watch_folder_def['agents'] ||= [ ]
271
+ if agents.is_a?(Hash)
272
+ agents = agents.map { |k,v| v['name'] ||= k; v }
273
+ end
274
+ if agent
275
+ if agent.is_a?(Hash)
276
+ if agent.keys.length == 1
277
+ agent = agent.map { |k,v| v['name'] ||= k; v }
278
+ end
279
+ end
280
+ agents.concat [*agent]
281
+ watch_folder_def.delete('agent')
282
+ end
283
+ logger.debug { "Parameter 'agent' initialized." }
284
+
285
+ @poll_interval = watch_folder_def['poll_interval'] ||= DEFAULT_POLL_INTERVAL
286
+ end
287
+
288
+ def poll
289
+ @previous_poll_time = @last_poll_time
290
+ @last_poll_time = Time.now
291
+
292
+ handler.poll if handler.respond_to?(:poll)
293
+ end
294
+
295
+ def poll_interval_elapsed?
296
+ !last_poll_time || (Time.now - last_poll_time) >= poll_interval
297
+ end
298
+
299
+ def stable_files
300
+ handler.stable_files
301
+ end
302
+
303
+ end
304
+
305
+ end
306
+
307
+ end