cantemo-portal-agent 1.3.2 → 1.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/cantemo/portal/agent/cli/commands/watch_folders.rb +63 -15
- data/lib/cantemo/portal/agent/version.rb +1 -1
- data/lib/envoi/mam/cantemo/agent/watch_folder_manager.rb +24 -152
- data/lib/envoi/watch_folder_utility/watch_folder.rb +152 -22
- data/lib/envoi/watch_folder_utility/watch_folder/handler/listen.rb +8 -4
- metadata +8 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7c2c43f547f0f607e087665ce93f3a5c63dd4d5a
|
4
|
+
data.tar.gz: 96c2cc23d5b30fa48abbc3d329e01287782b343e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 16d67112efc3e3748edbb090c97e86c5874945e18c17bedd34184ac0b9709f5771bccd7fc40b1f6c2ec382e17dc551c041c3ab33d901e6d809632ef183c9a60f
|
7
|
+
data.tar.gz: 8d93bcc5a774aba3a369e3dbe77c09a9c4dfe2b05d71572bfa0c5e387be9f4220c26f3beac924771e5e7541be06fb85123530fb2d9e1254cf22376f320542c28
|
@@ -11,7 +11,6 @@ require 'envoi/mam/agent/cli'
|
|
11
11
|
require 'envoi/mam/cantemo/agent'
|
12
12
|
require 'envoi/mam/cantemo/agent/watch_folder_manager'
|
13
13
|
|
14
|
-
|
15
14
|
Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS.clear
|
16
15
|
Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS.concat [
|
17
16
|
'./cantemo-portal-agent-config.json',
|
@@ -21,7 +20,6 @@ Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS.concat [
|
|
21
20
|
]
|
22
21
|
default_config_file_paths = Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS
|
23
22
|
current_command = ARGV.shift
|
24
|
-
ARGV << 'run' if ARGV.empty?
|
25
23
|
# ARGV << '--help' if ARGV.empty?
|
26
24
|
|
27
25
|
@args = {
|
@@ -47,12 +45,13 @@ op.load
|
|
47
45
|
op.parse!
|
48
46
|
|
49
47
|
sub_command = (ARGV.first || 'run').downcase
|
48
|
+
sub_command.tr('-_', '')
|
49
|
+
|
50
50
|
if sub_command == 'help'
|
51
51
|
puts op
|
52
52
|
exit
|
53
53
|
end
|
54
54
|
|
55
|
-
|
56
55
|
config_file_path = args[:config_file_path]
|
57
56
|
if config_file_path.is_a?(Array)
|
58
57
|
args[:config_file_path].map! { |v| File.expand_path(v) }
|
@@ -95,25 +94,72 @@ module Application
|
|
95
94
|
require 'win32/daemon'
|
96
95
|
class Watcher < Win32::Daemon
|
97
96
|
|
98
|
-
|
97
|
+
LOG_FILE_PATH = 'C:\\tmp\\win32_daemon_test.log'
|
98
|
+
log_file_dir_path = File.dirname(LOG_FILE_PATH)
|
99
|
+
Dir.mkdir_p(log_file_dir_path) unless Dir.exist?(log_file_dir_path)
|
99
100
|
|
101
|
+
def write_log(message = nil, &block)
|
102
|
+
if block_given?
|
103
|
+
begin
|
104
|
+
f = File.open(LOG_FILE_PATH, 'a')
|
105
|
+
yield f
|
106
|
+
ensure
|
107
|
+
f.close if f && f.open?
|
108
|
+
end
|
109
|
+
else
|
110
|
+
File.open(LOG_FILE_PATH, 'a') { |f| f.puts(message) }
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
def service_init
|
115
|
+
@process_thread = nil
|
100
116
|
end
|
101
117
|
|
102
118
|
def service_main(*args)
|
103
|
-
|
104
|
-
|
119
|
+
msg = 'service_main entered at: ' + Time.now.to_s
|
120
|
+
|
121
|
+
write_log{ |f|
|
122
|
+
f.puts msg
|
123
|
+
f.puts "Args: " + args.join(',')
|
124
|
+
}
|
125
|
+
|
126
|
+
@watch_folder_manager = nil
|
127
|
+
@process_thread = Thread.new(@watch_folder_manager, args) do |watch_folder_manager, args|
|
128
|
+
watch_folder_manager = WATCH_FOLDER_MANAGER_CLASS.new(args)
|
129
|
+
watch_folder_manager.run
|
130
|
+
end
|
131
|
+
|
132
|
+
while running?
|
133
|
+
if state == RUNNING
|
134
|
+
sleep 20
|
135
|
+
msg = 'Service is running as of: ' + Time.now.to_s
|
136
|
+
write_log { |f| f.puts msg }
|
137
|
+
else # PAUSED or IDLE
|
138
|
+
sleep 0.5
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
|
143
|
+
# We've left the loop, the daemon is about to exit.
|
144
|
+
|
145
|
+
write_log{ |f| f.puts "STATE: #{state}" }
|
146
|
+
|
147
|
+
msg = 'service_main left at: ' + Time.now.to_s
|
148
|
+
|
149
|
+
write_log{ |f| f.puts msg }
|
150
|
+
|
105
151
|
end
|
106
152
|
|
107
153
|
def service_stop
|
108
|
-
@
|
154
|
+
@watch_folder_manager.stop
|
109
155
|
end
|
110
156
|
|
111
157
|
def service_pause
|
112
|
-
@
|
158
|
+
@watch_folder_manager.pause
|
113
159
|
end
|
114
160
|
|
115
161
|
def service_resume
|
116
|
-
@
|
162
|
+
@watch_folder_manager.resume
|
117
163
|
end
|
118
164
|
|
119
165
|
# Watcher - Windows
|
@@ -230,6 +276,8 @@ module Application
|
|
230
276
|
end
|
231
277
|
|
232
278
|
end
|
279
|
+
|
280
|
+
|
233
281
|
# Options:
|
234
282
|
# install - Installs the service.
|
235
283
|
# start - Starts the service.
|
@@ -245,8 +293,6 @@ include RbConfig
|
|
245
293
|
|
246
294
|
SERVICE_NAME = 'cantemo_portal_watch_folder_agent'
|
247
295
|
SERVICE_DISPLAYNAME = 'Cantemo Portal Watch Folder Agent'
|
248
|
-
EXECUTABLE_NAME = __FILE__
|
249
|
-
|
250
296
|
|
251
297
|
# You must provide at least one argument.
|
252
298
|
abort('No argument provided.') unless sub_command
|
@@ -255,10 +301,11 @@ abort('Service not installed.') unless Service.exists?(SERVICE_NAME) || sub_comm
|
|
255
301
|
|
256
302
|
case sub_command
|
257
303
|
when 'runasservice'
|
258
|
-
Application::Watcher.mainloop
|
304
|
+
Application::Watcher.mainloop
|
259
305
|
when 'start'
|
260
306
|
if Service.status(SERVICE_NAME).current_state != 'running'
|
261
|
-
Service.start(SERVICE_NAME, nil)
|
307
|
+
# Service.start(SERVICE_NAME, nil, args)
|
308
|
+
Service.start(SERVICE_NAME)
|
262
309
|
while Service.status(SERVICE_NAME).current_state != 'running'
|
263
310
|
puts 'One moment...' + Service.status(SERVICE_NAME).current_state
|
264
311
|
sleep 1
|
@@ -305,8 +352,9 @@ when 'status'
|
|
305
352
|
when 'install'
|
306
353
|
# Quote the full path to deal with possible spaces in the path name.
|
307
354
|
ruby = File.join(CONFIG['bindir'], CONFIG['ruby_install_name']).tr('/', '\\')
|
308
|
-
path =
|
309
|
-
cmd = %Q("#{ruby}" "#{path}" runasservice --config-file-path "#{config_file_path}")
|
355
|
+
path = __FILE__.tr('/', '\\')
|
356
|
+
# cmd = %Q("#{ruby}" "#{path}" runasservice --config-file-path "#{config_file_path}")
|
357
|
+
cmd = %Q("#{ruby}" "#{path}" run-as-service)
|
310
358
|
|
311
359
|
Service.new(
|
312
360
|
:service_name => SERVICE_NAME,
|
@@ -18,7 +18,13 @@ module Envoi::Mam::Cantemo
|
|
18
18
|
|
19
19
|
%w(log debug info warn error fatal unknown level=).each do |m|
|
20
20
|
define_method(m) do |*args, &block|
|
21
|
-
@
|
21
|
+
@current_message = nil
|
22
|
+
if block
|
23
|
+
@targets.map { |t| t.send(m, *args) { @current_message ||= block.call }}
|
24
|
+
else
|
25
|
+
@targets.map { |t| t.send(m, *args) }
|
26
|
+
end
|
27
|
+
|
22
28
|
end
|
23
29
|
end
|
24
30
|
end
|
@@ -140,166 +146,32 @@ module Envoi::Mam::Cantemo
|
|
140
146
|
logger.debug { 'Processing of watch folder definitions completed.' }
|
141
147
|
end
|
142
148
|
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
file.ignore if file.respond_to?(:ignore)
|
149
|
-
end
|
150
|
-
end
|
151
|
-
|
152
|
-
# @param [Object] file
|
153
|
-
# @return [Hash]
|
154
|
-
def process_file(file)
|
155
|
-
file.processing = true
|
156
|
-
file_name = file.name || file.path
|
157
|
-
logger.debug { "Processing File '#{file_name}'" }
|
158
|
-
watch_folder = file.watch_folder
|
159
|
-
agent = watch_folder.respond_to?(:agent) ? watch_folder.agent : default_agent
|
160
|
-
agent ||= default_agent
|
161
|
-
|
162
|
-
storage_id = watch_folder.definition['upload_to_storage_id']
|
163
|
-
|
164
|
-
unless storage_id
|
165
|
-
logger.warn { "Skipping processing of file because of missing storage ID." }
|
166
|
-
return { success: false, message: 'Missing storage ID.' }
|
167
|
-
end
|
168
|
-
|
169
|
-
quarantine_directory_path = watch_folder.definition['quarantine_directory_path']
|
170
|
-
completed_directory_path = watch_folder.definition['completed_directory_path']
|
171
|
-
watch_folder_upload_args = watch_folder.definition['upload_args']
|
172
|
-
|
173
|
-
# full_file_path = File.join(watch_folder.path, file.path)
|
174
|
-
full_file_path = file.path
|
175
|
-
|
176
|
-
upload_args = {
|
177
|
-
file_path: full_file_path,
|
178
|
-
storage_id: storage_id
|
179
|
-
}
|
180
|
-
upload_args.merge!(watch_folder_upload_args) if watch_folder_upload_args.is_a?(Hash)
|
181
|
-
|
182
|
-
logger.debug { "Executing Upload. #{upload_args}" }
|
183
|
-
_response = agent.upload(upload_args)
|
184
|
-
_response = { success: _response } if _response == true || _response == false
|
185
|
-
|
186
|
-
if _response[:success]
|
187
|
-
if completed_directory_path
|
188
|
-
if Dir.exist?(completed_directory_path)
|
189
|
-
logger.debug { "Moving '#{full_file_path}' to completed directory path '#{completed_directory_path}'" }
|
190
|
-
FileUtils.mv full_file_path, completed_directory_path
|
191
|
-
else
|
192
|
-
logger.warn { "Completed directory path not found: '#{completed_directory_path}'" }
|
193
|
-
add_to_ignore(file)
|
194
|
-
end
|
195
|
-
else
|
196
|
-
FileUtils.rm full_file_path
|
197
|
-
end
|
198
|
-
else
|
199
|
-
if quarantine_directory_path && Dir.exist?(quarantine_directory_path)
|
200
|
-
logger.warn { "Moving '#{full_file_path}' to quarantine directory path '#{quarantine_directory_path}'" }
|
201
|
-
FileUtils.mv full_file_path, quarantine_directory_path
|
202
|
-
else
|
203
|
-
logger.warn { "Adding '#{full_file_path}' to the temporary ignore list." }
|
204
|
-
add_to_ignore(file)
|
205
|
-
end
|
206
|
-
end
|
207
|
-
|
208
|
-
file.processed = true
|
209
|
-
|
210
|
-
_response
|
211
|
-
rescue => e
|
212
|
-
file.exception = e
|
213
|
-
raise e
|
214
|
-
ensure
|
215
|
-
file.processing = false
|
216
|
-
end
|
217
|
-
|
218
|
-
# Used to compare file to patterns
|
219
|
-
def find_in_patterns(patterns, file)
|
220
|
-
patterns.find do |pattern|
|
221
|
-
matched = pattern.is_a?(Regexp) ? pattern.match(file.path) : File.fnmatch(pattern, file.path)
|
222
|
-
logger.debug { "#{pattern} #{matched ? 'matched' : "didn't match"} #{file.path}" }
|
223
|
-
matched
|
224
|
-
end
|
225
|
-
end
|
226
|
-
|
227
|
-
# This should be part of the watch folder but it is here to track active processors globally
|
228
|
-
def process_watch_folder_stable_files(wf)
|
229
|
-
stable_files = wf.stable_files
|
230
|
-
active_processors = @processors_by_watch_folder[wf]
|
231
|
-
maximum_active_processors = wf.definition['maximum_active_processors']
|
232
|
-
|
233
|
-
includes = wf.definition['includes']
|
234
|
-
excludes = wf.definition['excludes']
|
235
|
-
|
236
|
-
ignored_file_paths = wf.ignored_file_paths if wf.respond_to?(:ingored_file_paths)
|
237
|
-
unless ignored_file_paths
|
238
|
-
ignored_file_paths = @ignored_file_paths_by_watch_folder[wf]
|
239
|
-
end
|
240
|
-
|
241
|
-
stable_files.each do |file|
|
242
|
-
file.watch_folder ||= wf
|
243
|
-
next if file.respond_to?(:ignore?) ? file.ignore? : ignored_file_paths.include?(file.path)
|
244
|
-
next if file.processing || file.processed
|
245
|
-
|
246
|
-
if includes && !includes.empty?
|
247
|
-
should_include = find_in_patterns(includes, file)
|
248
|
-
unless should_include
|
249
|
-
add_to_ignore(file)
|
250
|
-
next
|
251
|
-
end
|
252
|
-
end
|
253
|
-
|
254
|
-
should_exclude = find_in_patterns(excludes, file)
|
255
|
-
if should_exclude
|
256
|
-
add_to_ignore(file)
|
257
|
-
next
|
149
|
+
def poll
|
150
|
+
watch_folders_with_stable_files = [ ]
|
151
|
+
watch_folders.each do |watch_folder|
|
152
|
+
if watch_folder.respond_to?(:poll_interval_elapsed?)
|
153
|
+
next unless watch_folder.poll_interval_elapsed?
|
258
154
|
end
|
259
155
|
|
260
|
-
if
|
261
|
-
|
262
|
-
|
263
|
-
logger.debug { "Maximum number of active processors reached for watch folder. #{wf.name || wf.paths}" }
|
264
|
-
break
|
265
|
-
end
|
266
|
-
t = Thread.new(file) do |file|
|
267
|
-
wf = file.watch_folder
|
268
|
-
begin
|
269
|
-
process_file(file)
|
270
|
-
rescue => e
|
271
|
-
logger.error { "Exception '#{e.message}' in thread for `#{wf.name || wf.paths}` `#{file.path}`. " }
|
272
|
-
raise e
|
273
|
-
ensure
|
274
|
-
file.processing = false rescue nil
|
275
|
-
end
|
276
|
-
end
|
277
|
-
t.join
|
278
|
-
active_processors[file] = t if file.processing
|
279
|
-
else
|
280
|
-
process_file(file)
|
156
|
+
if watch_folder.respond_to?(:poll)
|
157
|
+
logger.debug { "Polling Watch Folder: #{watch_folder.name}" }
|
158
|
+
watch_folder.poll
|
281
159
|
end
|
282
160
|
|
161
|
+
watch_folders_with_stable_files << watch_folder unless watch_folder.stable_files.empty?
|
283
162
|
end
|
284
163
|
|
285
|
-
|
164
|
+
# @TODO create processor/worker pool to limit total file processing across all watch folders
|
165
|
+
watch_folders_with_stable_files.each do |watch_folder|
|
166
|
+
#process_watch_folder_stable_files(watch_folder)
|
286
167
|
|
287
|
-
|
288
|
-
|
289
|
-
|
290
|
-
watch_folders.each do |watch_folder|
|
291
|
-
next unless watch_folder.poll_interval_elapsed?
|
292
|
-
|
293
|
-
logger.debug { "Polling Watch Folder: #{watch_folder.name}" }
|
294
|
-
watch_folder.poll
|
295
|
-
|
296
|
-
stable_files = watch_folder.stable_files
|
297
|
-
stable_files_by_watch_folder[watch_folder] = stable_files
|
168
|
+
total_active_processors = watch_folders.inject(0) { |sum, wf| sum + wf.processors.length }
|
169
|
+
watch_folder.process_stable_files
|
170
|
+
logger.debug { "Total active processors #{total_active_processors}" }
|
298
171
|
end
|
299
172
|
|
300
|
-
|
301
|
-
|
302
|
-
end
|
173
|
+
# Thread.list.each {|t| t.join unless t == Thread.current }
|
174
|
+
# @processors_by_watch_folder.each { |wf, ap| ap.each { |f, t| t.join } } if @threaded
|
303
175
|
end
|
304
176
|
|
305
177
|
def run_once
|
@@ -13,7 +13,7 @@ module Envoi
|
|
13
13
|
|
14
14
|
attr_accessor :logger, :definition, :agent, :handler
|
15
15
|
|
16
|
-
attr_accessor :processors
|
16
|
+
attr_accessor :processors
|
17
17
|
|
18
18
|
attr_accessor :last_poll_time,
|
19
19
|
:min_stable_poll_count,
|
@@ -26,12 +26,15 @@ module Envoi
|
|
26
26
|
|
27
27
|
logger.debug { "Initializing Watch Folder. #{Object.__id__}" }
|
28
28
|
|
29
|
+
@ignored_file_paths = [ ]
|
29
30
|
@ignored_file_paths_lock = Mutex.new
|
30
31
|
|
31
32
|
@threaded = args.fetch(:threaded, true)
|
32
33
|
|
33
34
|
@default_maximum_active_processors = DEFAULT_PROCESSOR_COUNT_LIMIT
|
34
|
-
@processors =
|
35
|
+
@processors = { }
|
36
|
+
# @processors = Hash.new { |h, k| h[k] = {} }
|
37
|
+
|
35
38
|
|
36
39
|
@default_handler_class = Envoi::WatchFolderUtility::WatchFolder::Handler::Listen
|
37
40
|
process_watch_folder_def
|
@@ -70,7 +73,6 @@ module Envoi
|
|
70
73
|
end
|
71
74
|
|
72
75
|
def initialize_agent(args = {})
|
73
|
-
|
74
76
|
@agent ||= begin
|
75
77
|
logger.debug { "Initializing Agent. #{@default_agent_class} #{args}" }
|
76
78
|
_agent = @default_agent_class.new(config: args, logger: logger, default_preserve_file_path: false)
|
@@ -79,10 +81,50 @@ module Envoi
|
|
79
81
|
end
|
80
82
|
end
|
81
83
|
|
84
|
+
def initialize_handler(watch_folder_def = @definition)
|
85
|
+
args_out = {}
|
86
|
+
args_out[:logger] ||= logger.dup
|
87
|
+
args_out[:definition] = watch_folder_def
|
88
|
+
|
89
|
+
handler_class = @default_handler_class
|
90
|
+
logger.debug { "Creating Watch Folder Handler Instance. #{handler_class.name}" }
|
91
|
+
@handler = handler_class.new(args_out)
|
92
|
+
logger.debug { "Watch Folder Handler Instance Created. #{handler_class.name}" }
|
93
|
+
end
|
94
|
+
|
95
|
+
# @param [Object] file
|
96
|
+
def add_file_to_ignored_file_paths(file)
|
97
|
+
logger.debug { "Adding File to Ignore Cache: '#{file.path}'" }
|
98
|
+
@ignored_file_paths_lock.synchronize do
|
99
|
+
@ignored_file_paths << file.path
|
100
|
+
file.ignore if file.respond_to?(:ignore)
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
# Used to compare file to patterns
|
105
|
+
def find_in_patterns(patterns, file)
|
106
|
+
patterns.find do |pattern|
|
107
|
+
matched = pattern.is_a?(Regexp) ? pattern.match(file.path) : File.fnmatch(pattern, file.path)
|
108
|
+
logger.debug { "#{pattern} #{matched ? 'matched' : "didn't match"} #{file.path}" }
|
109
|
+
matched
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
82
113
|
def ignored_file_paths
|
83
114
|
handler.ignored_files_map.keys
|
84
115
|
end
|
85
116
|
|
117
|
+
def poll
|
118
|
+
@previous_poll_time = @last_poll_time
|
119
|
+
@last_poll_time = Time.now
|
120
|
+
|
121
|
+
handler.poll if handler.respond_to?(:poll)
|
122
|
+
end
|
123
|
+
|
124
|
+
def poll_interval_elapsed?
|
125
|
+
!last_poll_time || (Time.now - last_poll_time) >= poll_interval
|
126
|
+
end
|
127
|
+
|
86
128
|
def process_agent_def(agent_def)
|
87
129
|
if @default_agent
|
88
130
|
_agent_def_storages = agent_def['storages'] || {}
|
@@ -115,17 +157,6 @@ module Envoi
|
|
115
157
|
@agent ||= @default_agent
|
116
158
|
end
|
117
159
|
|
118
|
-
def initialize_handler(watch_folder_def = @definition)
|
119
|
-
args_out = {}
|
120
|
-
args_out[:logger] ||= logger.dup
|
121
|
-
args_out[:definition] = watch_folder_def
|
122
|
-
|
123
|
-
handler_class = @default_handler_class
|
124
|
-
logger.debug { "Creating Watch Folder Handler Instance. #{handler_class.name}" }
|
125
|
-
@handler = handler_class.new(args_out)
|
126
|
-
logger.debug { "Watch Folder Handler Instance Created. #{handler_class.name}" }
|
127
|
-
end
|
128
|
-
|
129
160
|
#
|
130
161
|
# @param [Hash] watch_folder_def
|
131
162
|
# @option watch_folder_def [String] path
|
@@ -190,7 +221,6 @@ module Envoi
|
|
190
221
|
watch_folder_def.delete('exclude')
|
191
222
|
logger.debug { "Parameter `excludes` initialized." }
|
192
223
|
|
193
|
-
|
194
224
|
logger.debug { "Initializing parameter `quarantine directory path`." }
|
195
225
|
quarantine_directory_path = watch_folder_def['quarantine_directory_path'] || watch_folder_def['quarantine_path']
|
196
226
|
if quarantine_directory_path
|
@@ -217,7 +247,6 @@ module Envoi
|
|
217
247
|
watch_folder_def.delete('completed_path')
|
218
248
|
logger.debug { "Parameter 'completed directory path' initialized." }
|
219
249
|
|
220
|
-
|
221
250
|
logger.debug { "Initializing parameter `upload to storage id`." }
|
222
251
|
storage_id = watch_folder_def['upload_to_storage_id'] || watch_folder_def['storage_id']
|
223
252
|
watch_folder_def['upload_to_storage_id'] ||= storage_id
|
@@ -325,15 +354,116 @@ module Envoi
|
|
325
354
|
@poll_interval = watch_folder_def['poll_interval'] ||= DEFAULT_POLL_INTERVAL
|
326
355
|
end
|
327
356
|
|
328
|
-
def
|
329
|
-
|
330
|
-
|
357
|
+
def process_stable_file(file)
|
358
|
+
file.processing = true
|
359
|
+
file_name = file.name || file.path
|
360
|
+
logger.debug { "Processing File '#{file_name}'" }
|
331
361
|
|
332
|
-
|
362
|
+
storage_id = definition['upload_to_storage_id']
|
363
|
+
|
364
|
+
unless storage_id
|
365
|
+
logger.warn { "Skipping processing of file because of missing storage ID." }
|
366
|
+
return { success: false, message: 'Missing storage ID.' }
|
367
|
+
end
|
368
|
+
|
369
|
+
quarantine_directory_path = definition['quarantine_directory_path']
|
370
|
+
completed_directory_path = definition['completed_directory_path']
|
371
|
+
watch_folder_upload_args = definition['upload_args']
|
372
|
+
|
373
|
+
# full_file_path = File.join(watch_folder.path, file.path)
|
374
|
+
full_file_path = file.path
|
375
|
+
|
376
|
+
upload_args = {
|
377
|
+
file_path: full_file_path,
|
378
|
+
storage_id: storage_id
|
379
|
+
}
|
380
|
+
upload_args.merge!(watch_folder_upload_args) if watch_folder_upload_args.is_a?(Hash)
|
381
|
+
|
382
|
+
logger.debug { "Executing Upload. #{upload_args}" }
|
383
|
+
_response = agent.upload(upload_args)
|
384
|
+
_response = { success: _response } if _response == true || _response == false
|
385
|
+
|
386
|
+
if _response[:success]
|
387
|
+
if completed_directory_path
|
388
|
+
if Dir.exist?(completed_directory_path)
|
389
|
+
logger.debug { "Moving '#{full_file_path}' to completed directory path '#{completed_directory_path}'" }
|
390
|
+
FileUtils.mv full_file_path, completed_directory_path
|
391
|
+
else
|
392
|
+
logger.warn { "Completed directory path not found: '#{completed_directory_path}'" }
|
393
|
+
add_file_to_ignored_file_paths(file)
|
394
|
+
end
|
395
|
+
else
|
396
|
+
FileUtils.rm full_file_path
|
397
|
+
end
|
398
|
+
else
|
399
|
+
if quarantine_directory_path && Dir.exist?(quarantine_directory_path)
|
400
|
+
logger.warn { "Moving '#{full_file_path}' to quarantine directory path '#{quarantine_directory_path}'" }
|
401
|
+
FileUtils.mv full_file_path, quarantine_directory_path
|
402
|
+
else
|
403
|
+
logger.warn { "Adding '#{full_file_path}' to the temporary ignore list." }
|
404
|
+
add_file_to_ignored_file_paths(file)
|
405
|
+
end
|
406
|
+
end
|
407
|
+
|
408
|
+
file.processed = true
|
409
|
+
|
410
|
+
_response
|
411
|
+
rescue => e
|
412
|
+
file.exception = e
|
413
|
+
raise e
|
414
|
+
ensure
|
415
|
+
file.processing = false
|
333
416
|
end
|
334
417
|
|
335
|
-
|
336
|
-
|
418
|
+
|
419
|
+
def process_stable_files
|
420
|
+
maximum_active_processors = definition['maximum_active_processors']
|
421
|
+
|
422
|
+
includes = definition['includes']
|
423
|
+
excludes = definition['excludes']
|
424
|
+
|
425
|
+
stable_files.each do |file|
|
426
|
+
file.watch_folder ||= self
|
427
|
+
next if file.respond_to?(:ignore?) ? file.ignore? : ignored_file_paths.include?(file.path)
|
428
|
+
next if file.processing || file.processed
|
429
|
+
|
430
|
+
if includes && !includes.empty?
|
431
|
+
should_include = find_in_patterns(includes, file)
|
432
|
+
unless should_include
|
433
|
+
add_file_to_ignored_file_paths(file)
|
434
|
+
next
|
435
|
+
end
|
436
|
+
end
|
437
|
+
|
438
|
+
should_exclude = find_in_patterns(excludes, file)
|
439
|
+
if should_exclude
|
440
|
+
add_file_to_ignored_file_paths(file)
|
441
|
+
next
|
442
|
+
end
|
443
|
+
|
444
|
+
if @threaded
|
445
|
+
processors.keep_if { |k, v| k.processing }
|
446
|
+
if processors.length >= maximum_active_processors
|
447
|
+
logger.debug { "Maximum number of active processors reached for watch folder. #{wf.name || wf.paths}" }
|
448
|
+
break
|
449
|
+
end
|
450
|
+
t = Thread.new(file) do |file|
|
451
|
+
begin
|
452
|
+
process_stable_file(file)
|
453
|
+
rescue => e
|
454
|
+
logger.error { "Exception '#{e.message}' in thread for `#{name}` `#{file.path}`. " }
|
455
|
+
raise e
|
456
|
+
ensure
|
457
|
+
file.processing = false rescue nil
|
458
|
+
end
|
459
|
+
end
|
460
|
+
# t.join
|
461
|
+
processors[file] = t if file.processing
|
462
|
+
else
|
463
|
+
process_stable_file(file)
|
464
|
+
end
|
465
|
+
|
466
|
+
end
|
337
467
|
end
|
338
468
|
|
339
469
|
def run
|
@@ -33,13 +33,13 @@ module Envoi
|
|
33
33
|
|
34
34
|
def stable?
|
35
35
|
return false if deleted?
|
36
|
-
return false
|
36
|
+
return false unless exist?
|
37
37
|
|
38
38
|
_min_stable_time = min_stable_time
|
39
39
|
if _min_stable_time
|
40
|
-
|
40
|
+
((Time.now - self[:event_timestamp]) >= _min_stable_time)
|
41
41
|
else
|
42
|
-
|
42
|
+
(self[:stable_poll_count] >= min_stable_poll_count)
|
43
43
|
end
|
44
44
|
end
|
45
45
|
|
@@ -200,6 +200,7 @@ module Envoi
|
|
200
200
|
|
201
201
|
# Really this is just for debugging as polling isn't used to determine file stability
|
202
202
|
def poll(options = { })
|
203
|
+
# return if @min_stable_time
|
203
204
|
should_increment_counter = options.fetch(:should_increment_counter, true)
|
204
205
|
|
205
206
|
@previous_poll_time = @last_poll_time
|
@@ -208,7 +209,10 @@ module Envoi
|
|
208
209
|
lock.synchronize do
|
209
210
|
@known_path_map.each do |fp, f|
|
210
211
|
next if f.ignore?
|
211
|
-
|
212
|
+
unless f.exist?
|
213
|
+
logger.debug { "Skipping Missing File. #{f.summary}" }
|
214
|
+
next
|
215
|
+
end
|
212
216
|
|
213
217
|
logger.debug { "Incrementing Stable Stats: #{f.summary}" }
|
214
218
|
f.last_poll_time = Time.now
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cantemo-portal-agent
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.3.
|
4
|
+
version: 1.3.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- John Whitson
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-01-
|
11
|
+
date: 2019-01-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: asperalm
|
@@ -71,6 +71,9 @@ dependencies:
|
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
73
|
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '1.6'
|
76
|
+
- - ">="
|
74
77
|
- !ruby/object:Gem::Version
|
75
78
|
version: 1.6.5
|
76
79
|
type: :runtime
|
@@ -78,6 +81,9 @@ dependencies:
|
|
78
81
|
version_requirements: !ruby/object:Gem::Requirement
|
79
82
|
requirements:
|
80
83
|
- - "~>"
|
84
|
+
- !ruby/object:Gem::Version
|
85
|
+
version: '1.6'
|
86
|
+
- - ">="
|
81
87
|
- !ruby/object:Gem::Version
|
82
88
|
version: 1.6.5
|
83
89
|
- !ruby/object:Gem::Dependency
|