cantemo-portal-agent 1.3.2 → 1.3.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/cantemo/portal/agent/cli/commands/watch_folders.rb +63 -15
- data/lib/cantemo/portal/agent/version.rb +1 -1
- data/lib/envoi/mam/cantemo/agent/watch_folder_manager.rb +24 -152
- data/lib/envoi/watch_folder_utility/watch_folder.rb +152 -22
- data/lib/envoi/watch_folder_utility/watch_folder/handler/listen.rb +8 -4
- metadata +8 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7c2c43f547f0f607e087665ce93f3a5c63dd4d5a
|
4
|
+
data.tar.gz: 96c2cc23d5b30fa48abbc3d329e01287782b343e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 16d67112efc3e3748edbb090c97e86c5874945e18c17bedd34184ac0b9709f5771bccd7fc40b1f6c2ec382e17dc551c041c3ab33d901e6d809632ef183c9a60f
|
7
|
+
data.tar.gz: 8d93bcc5a774aba3a369e3dbe77c09a9c4dfe2b05d71572bfa0c5e387be9f4220c26f3beac924771e5e7541be06fb85123530fb2d9e1254cf22376f320542c28
|
@@ -11,7 +11,6 @@ require 'envoi/mam/agent/cli'
|
|
11
11
|
require 'envoi/mam/cantemo/agent'
|
12
12
|
require 'envoi/mam/cantemo/agent/watch_folder_manager'
|
13
13
|
|
14
|
-
|
15
14
|
Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS.clear
|
16
15
|
Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS.concat [
|
17
16
|
'./cantemo-portal-agent-config.json',
|
@@ -21,7 +20,6 @@ Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS.concat [
|
|
21
20
|
]
|
22
21
|
default_config_file_paths = Envoi::Mam::Agent::CLI::CONFIG_FILE_PATHS
|
23
22
|
current_command = ARGV.shift
|
24
|
-
ARGV << 'run' if ARGV.empty?
|
25
23
|
# ARGV << '--help' if ARGV.empty?
|
26
24
|
|
27
25
|
@args = {
|
@@ -47,12 +45,13 @@ op.load
|
|
47
45
|
op.parse!
|
48
46
|
|
49
47
|
sub_command = (ARGV.first || 'run').downcase
|
48
|
+
sub_command.tr('-_', '')
|
49
|
+
|
50
50
|
if sub_command == 'help'
|
51
51
|
puts op
|
52
52
|
exit
|
53
53
|
end
|
54
54
|
|
55
|
-
|
56
55
|
config_file_path = args[:config_file_path]
|
57
56
|
if config_file_path.is_a?(Array)
|
58
57
|
args[:config_file_path].map! { |v| File.expand_path(v) }
|
@@ -95,25 +94,72 @@ module Application
|
|
95
94
|
require 'win32/daemon'
|
96
95
|
class Watcher < Win32::Daemon
|
97
96
|
|
98
|
-
|
97
|
+
LOG_FILE_PATH = 'C:\\tmp\\win32_daemon_test.log'
|
98
|
+
log_file_dir_path = File.dirname(LOG_FILE_PATH)
|
99
|
+
Dir.mkdir_p(log_file_dir_path) unless Dir.exist?(log_file_dir_path)
|
99
100
|
|
101
|
+
def write_log(message = nil, &block)
|
102
|
+
if block_given?
|
103
|
+
begin
|
104
|
+
f = File.open(LOG_FILE_PATH, 'a')
|
105
|
+
yield f
|
106
|
+
ensure
|
107
|
+
f.close if f && f.open?
|
108
|
+
end
|
109
|
+
else
|
110
|
+
File.open(LOG_FILE_PATH, 'a') { |f| f.puts(message) }
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
def service_init
|
115
|
+
@process_thread = nil
|
100
116
|
end
|
101
117
|
|
102
118
|
def service_main(*args)
|
103
|
-
|
104
|
-
|
119
|
+
msg = 'service_main entered at: ' + Time.now.to_s
|
120
|
+
|
121
|
+
write_log{ |f|
|
122
|
+
f.puts msg
|
123
|
+
f.puts "Args: " + args.join(',')
|
124
|
+
}
|
125
|
+
|
126
|
+
@watch_folder_manager = nil
|
127
|
+
@process_thread = Thread.new(@watch_folder_manager, args) do |watch_folder_manager, args|
|
128
|
+
watch_folder_manager = WATCH_FOLDER_MANAGER_CLASS.new(args)
|
129
|
+
watch_folder_manager.run
|
130
|
+
end
|
131
|
+
|
132
|
+
while running?
|
133
|
+
if state == RUNNING
|
134
|
+
sleep 20
|
135
|
+
msg = 'Service is running as of: ' + Time.now.to_s
|
136
|
+
write_log { |f| f.puts msg }
|
137
|
+
else # PAUSED or IDLE
|
138
|
+
sleep 0.5
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
|
143
|
+
# We've left the loop, the daemon is about to exit.
|
144
|
+
|
145
|
+
write_log{ |f| f.puts "STATE: #{state}" }
|
146
|
+
|
147
|
+
msg = 'service_main left at: ' + Time.now.to_s
|
148
|
+
|
149
|
+
write_log{ |f| f.puts msg }
|
150
|
+
|
105
151
|
end
|
106
152
|
|
107
153
|
def service_stop
|
108
|
-
@
|
154
|
+
@watch_folder_manager.stop
|
109
155
|
end
|
110
156
|
|
111
157
|
def service_pause
|
112
|
-
@
|
158
|
+
@watch_folder_manager.pause
|
113
159
|
end
|
114
160
|
|
115
161
|
def service_resume
|
116
|
-
@
|
162
|
+
@watch_folder_manager.resume
|
117
163
|
end
|
118
164
|
|
119
165
|
# Watcher - Windows
|
@@ -230,6 +276,8 @@ module Application
|
|
230
276
|
end
|
231
277
|
|
232
278
|
end
|
279
|
+
|
280
|
+
|
233
281
|
# Options:
|
234
282
|
# install - Installs the service.
|
235
283
|
# start - Starts the service.
|
@@ -245,8 +293,6 @@ include RbConfig
|
|
245
293
|
|
246
294
|
SERVICE_NAME = 'cantemo_portal_watch_folder_agent'
|
247
295
|
SERVICE_DISPLAYNAME = 'Cantemo Portal Watch Folder Agent'
|
248
|
-
EXECUTABLE_NAME = __FILE__
|
249
|
-
|
250
296
|
|
251
297
|
# You must provide at least one argument.
|
252
298
|
abort('No argument provided.') unless sub_command
|
@@ -255,10 +301,11 @@ abort('Service not installed.') unless Service.exists?(SERVICE_NAME) || sub_comm
|
|
255
301
|
|
256
302
|
case sub_command
|
257
303
|
when 'runasservice'
|
258
|
-
Application::Watcher.mainloop
|
304
|
+
Application::Watcher.mainloop
|
259
305
|
when 'start'
|
260
306
|
if Service.status(SERVICE_NAME).current_state != 'running'
|
261
|
-
Service.start(SERVICE_NAME, nil)
|
307
|
+
# Service.start(SERVICE_NAME, nil, args)
|
308
|
+
Service.start(SERVICE_NAME)
|
262
309
|
while Service.status(SERVICE_NAME).current_state != 'running'
|
263
310
|
puts 'One moment...' + Service.status(SERVICE_NAME).current_state
|
264
311
|
sleep 1
|
@@ -305,8 +352,9 @@ when 'status'
|
|
305
352
|
when 'install'
|
306
353
|
# Quote the full path to deal with possible spaces in the path name.
|
307
354
|
ruby = File.join(CONFIG['bindir'], CONFIG['ruby_install_name']).tr('/', '\\')
|
308
|
-
path =
|
309
|
-
cmd = %Q("#{ruby}" "#{path}" runasservice --config-file-path "#{config_file_path}")
|
355
|
+
path = __FILE__.tr('/', '\\')
|
356
|
+
# cmd = %Q("#{ruby}" "#{path}" runasservice --config-file-path "#{config_file_path}")
|
357
|
+
cmd = %Q("#{ruby}" "#{path}" run-as-service)
|
310
358
|
|
311
359
|
Service.new(
|
312
360
|
:service_name => SERVICE_NAME,
|
@@ -18,7 +18,13 @@ module Envoi::Mam::Cantemo
|
|
18
18
|
|
19
19
|
%w(log debug info warn error fatal unknown level=).each do |m|
|
20
20
|
define_method(m) do |*args, &block|
|
21
|
-
@
|
21
|
+
@current_message = nil
|
22
|
+
if block
|
23
|
+
@targets.map { |t| t.send(m, *args) { @current_message ||= block.call }}
|
24
|
+
else
|
25
|
+
@targets.map { |t| t.send(m, *args) }
|
26
|
+
end
|
27
|
+
|
22
28
|
end
|
23
29
|
end
|
24
30
|
end
|
@@ -140,166 +146,32 @@ module Envoi::Mam::Cantemo
|
|
140
146
|
logger.debug { 'Processing of watch folder definitions completed.' }
|
141
147
|
end
|
142
148
|
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
file.ignore if file.respond_to?(:ignore)
|
149
|
-
end
|
150
|
-
end
|
151
|
-
|
152
|
-
# @param [Object] file
|
153
|
-
# @return [Hash]
|
154
|
-
def process_file(file)
|
155
|
-
file.processing = true
|
156
|
-
file_name = file.name || file.path
|
157
|
-
logger.debug { "Processing File '#{file_name}'" }
|
158
|
-
watch_folder = file.watch_folder
|
159
|
-
agent = watch_folder.respond_to?(:agent) ? watch_folder.agent : default_agent
|
160
|
-
agent ||= default_agent
|
161
|
-
|
162
|
-
storage_id = watch_folder.definition['upload_to_storage_id']
|
163
|
-
|
164
|
-
unless storage_id
|
165
|
-
logger.warn { "Skipping processing of file because of missing storage ID." }
|
166
|
-
return { success: false, message: 'Missing storage ID.' }
|
167
|
-
end
|
168
|
-
|
169
|
-
quarantine_directory_path = watch_folder.definition['quarantine_directory_path']
|
170
|
-
completed_directory_path = watch_folder.definition['completed_directory_path']
|
171
|
-
watch_folder_upload_args = watch_folder.definition['upload_args']
|
172
|
-
|
173
|
-
# full_file_path = File.join(watch_folder.path, file.path)
|
174
|
-
full_file_path = file.path
|
175
|
-
|
176
|
-
upload_args = {
|
177
|
-
file_path: full_file_path,
|
178
|
-
storage_id: storage_id
|
179
|
-
}
|
180
|
-
upload_args.merge!(watch_folder_upload_args) if watch_folder_upload_args.is_a?(Hash)
|
181
|
-
|
182
|
-
logger.debug { "Executing Upload. #{upload_args}" }
|
183
|
-
_response = agent.upload(upload_args)
|
184
|
-
_response = { success: _response } if _response == true || _response == false
|
185
|
-
|
186
|
-
if _response[:success]
|
187
|
-
if completed_directory_path
|
188
|
-
if Dir.exist?(completed_directory_path)
|
189
|
-
logger.debug { "Moving '#{full_file_path}' to completed directory path '#{completed_directory_path}'" }
|
190
|
-
FileUtils.mv full_file_path, completed_directory_path
|
191
|
-
else
|
192
|
-
logger.warn { "Completed directory path not found: '#{completed_directory_path}'" }
|
193
|
-
add_to_ignore(file)
|
194
|
-
end
|
195
|
-
else
|
196
|
-
FileUtils.rm full_file_path
|
197
|
-
end
|
198
|
-
else
|
199
|
-
if quarantine_directory_path && Dir.exist?(quarantine_directory_path)
|
200
|
-
logger.warn { "Moving '#{full_file_path}' to quarantine directory path '#{quarantine_directory_path}'" }
|
201
|
-
FileUtils.mv full_file_path, quarantine_directory_path
|
202
|
-
else
|
203
|
-
logger.warn { "Adding '#{full_file_path}' to the temporary ignore list." }
|
204
|
-
add_to_ignore(file)
|
205
|
-
end
|
206
|
-
end
|
207
|
-
|
208
|
-
file.processed = true
|
209
|
-
|
210
|
-
_response
|
211
|
-
rescue => e
|
212
|
-
file.exception = e
|
213
|
-
raise e
|
214
|
-
ensure
|
215
|
-
file.processing = false
|
216
|
-
end
|
217
|
-
|
218
|
-
# Used to compare file to patterns
|
219
|
-
def find_in_patterns(patterns, file)
|
220
|
-
patterns.find do |pattern|
|
221
|
-
matched = pattern.is_a?(Regexp) ? pattern.match(file.path) : File.fnmatch(pattern, file.path)
|
222
|
-
logger.debug { "#{pattern} #{matched ? 'matched' : "didn't match"} #{file.path}" }
|
223
|
-
matched
|
224
|
-
end
|
225
|
-
end
|
226
|
-
|
227
|
-
# This should be part of the watch folder but it is here to track active processors globally
|
228
|
-
def process_watch_folder_stable_files(wf)
|
229
|
-
stable_files = wf.stable_files
|
230
|
-
active_processors = @processors_by_watch_folder[wf]
|
231
|
-
maximum_active_processors = wf.definition['maximum_active_processors']
|
232
|
-
|
233
|
-
includes = wf.definition['includes']
|
234
|
-
excludes = wf.definition['excludes']
|
235
|
-
|
236
|
-
ignored_file_paths = wf.ignored_file_paths if wf.respond_to?(:ingored_file_paths)
|
237
|
-
unless ignored_file_paths
|
238
|
-
ignored_file_paths = @ignored_file_paths_by_watch_folder[wf]
|
239
|
-
end
|
240
|
-
|
241
|
-
stable_files.each do |file|
|
242
|
-
file.watch_folder ||= wf
|
243
|
-
next if file.respond_to?(:ignore?) ? file.ignore? : ignored_file_paths.include?(file.path)
|
244
|
-
next if file.processing || file.processed
|
245
|
-
|
246
|
-
if includes && !includes.empty?
|
247
|
-
should_include = find_in_patterns(includes, file)
|
248
|
-
unless should_include
|
249
|
-
add_to_ignore(file)
|
250
|
-
next
|
251
|
-
end
|
252
|
-
end
|
253
|
-
|
254
|
-
should_exclude = find_in_patterns(excludes, file)
|
255
|
-
if should_exclude
|
256
|
-
add_to_ignore(file)
|
257
|
-
next
|
149
|
+
def poll
|
150
|
+
watch_folders_with_stable_files = [ ]
|
151
|
+
watch_folders.each do |watch_folder|
|
152
|
+
if watch_folder.respond_to?(:poll_interval_elapsed?)
|
153
|
+
next unless watch_folder.poll_interval_elapsed?
|
258
154
|
end
|
259
155
|
|
260
|
-
if
|
261
|
-
|
262
|
-
|
263
|
-
logger.debug { "Maximum number of active processors reached for watch folder. #{wf.name || wf.paths}" }
|
264
|
-
break
|
265
|
-
end
|
266
|
-
t = Thread.new(file) do |file|
|
267
|
-
wf = file.watch_folder
|
268
|
-
begin
|
269
|
-
process_file(file)
|
270
|
-
rescue => e
|
271
|
-
logger.error { "Exception '#{e.message}' in thread for `#{wf.name || wf.paths}` `#{file.path}`. " }
|
272
|
-
raise e
|
273
|
-
ensure
|
274
|
-
file.processing = false rescue nil
|
275
|
-
end
|
276
|
-
end
|
277
|
-
t.join
|
278
|
-
active_processors[file] = t if file.processing
|
279
|
-
else
|
280
|
-
process_file(file)
|
156
|
+
if watch_folder.respond_to?(:poll)
|
157
|
+
logger.debug { "Polling Watch Folder: #{watch_folder.name}" }
|
158
|
+
watch_folder.poll
|
281
159
|
end
|
282
160
|
|
161
|
+
watch_folders_with_stable_files << watch_folder unless watch_folder.stable_files.empty?
|
283
162
|
end
|
284
163
|
|
285
|
-
|
164
|
+
# @TODO create processor/worker pool to limit total file processing across all watch folders
|
165
|
+
watch_folders_with_stable_files.each do |watch_folder|
|
166
|
+
#process_watch_folder_stable_files(watch_folder)
|
286
167
|
|
287
|
-
|
288
|
-
|
289
|
-
|
290
|
-
watch_folders.each do |watch_folder|
|
291
|
-
next unless watch_folder.poll_interval_elapsed?
|
292
|
-
|
293
|
-
logger.debug { "Polling Watch Folder: #{watch_folder.name}" }
|
294
|
-
watch_folder.poll
|
295
|
-
|
296
|
-
stable_files = watch_folder.stable_files
|
297
|
-
stable_files_by_watch_folder[watch_folder] = stable_files
|
168
|
+
total_active_processors = watch_folders.inject(0) { |sum, wf| sum + wf.processors.length }
|
169
|
+
watch_folder.process_stable_files
|
170
|
+
logger.debug { "Total active processors #{total_active_processors}" }
|
298
171
|
end
|
299
172
|
|
300
|
-
|
301
|
-
|
302
|
-
end
|
173
|
+
# Thread.list.each {|t| t.join unless t == Thread.current }
|
174
|
+
# @processors_by_watch_folder.each { |wf, ap| ap.each { |f, t| t.join } } if @threaded
|
303
175
|
end
|
304
176
|
|
305
177
|
def run_once
|
@@ -13,7 +13,7 @@ module Envoi
|
|
13
13
|
|
14
14
|
attr_accessor :logger, :definition, :agent, :handler
|
15
15
|
|
16
|
-
attr_accessor :processors
|
16
|
+
attr_accessor :processors
|
17
17
|
|
18
18
|
attr_accessor :last_poll_time,
|
19
19
|
:min_stable_poll_count,
|
@@ -26,12 +26,15 @@ module Envoi
|
|
26
26
|
|
27
27
|
logger.debug { "Initializing Watch Folder. #{Object.__id__}" }
|
28
28
|
|
29
|
+
@ignored_file_paths = [ ]
|
29
30
|
@ignored_file_paths_lock = Mutex.new
|
30
31
|
|
31
32
|
@threaded = args.fetch(:threaded, true)
|
32
33
|
|
33
34
|
@default_maximum_active_processors = DEFAULT_PROCESSOR_COUNT_LIMIT
|
34
|
-
@processors =
|
35
|
+
@processors = { }
|
36
|
+
# @processors = Hash.new { |h, k| h[k] = {} }
|
37
|
+
|
35
38
|
|
36
39
|
@default_handler_class = Envoi::WatchFolderUtility::WatchFolder::Handler::Listen
|
37
40
|
process_watch_folder_def
|
@@ -70,7 +73,6 @@ module Envoi
|
|
70
73
|
end
|
71
74
|
|
72
75
|
def initialize_agent(args = {})
|
73
|
-
|
74
76
|
@agent ||= begin
|
75
77
|
logger.debug { "Initializing Agent. #{@default_agent_class} #{args}" }
|
76
78
|
_agent = @default_agent_class.new(config: args, logger: logger, default_preserve_file_path: false)
|
@@ -79,10 +81,50 @@ module Envoi
|
|
79
81
|
end
|
80
82
|
end
|
81
83
|
|
84
|
+
def initialize_handler(watch_folder_def = @definition)
|
85
|
+
args_out = {}
|
86
|
+
args_out[:logger] ||= logger.dup
|
87
|
+
args_out[:definition] = watch_folder_def
|
88
|
+
|
89
|
+
handler_class = @default_handler_class
|
90
|
+
logger.debug { "Creating Watch Folder Handler Instance. #{handler_class.name}" }
|
91
|
+
@handler = handler_class.new(args_out)
|
92
|
+
logger.debug { "Watch Folder Handler Instance Created. #{handler_class.name}" }
|
93
|
+
end
|
94
|
+
|
95
|
+
# @param [Object] file
|
96
|
+
def add_file_to_ignored_file_paths(file)
|
97
|
+
logger.debug { "Adding File to Ignore Cache: '#{file.path}'" }
|
98
|
+
@ignored_file_paths_lock.synchronize do
|
99
|
+
@ignored_file_paths << file.path
|
100
|
+
file.ignore if file.respond_to?(:ignore)
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
# Used to compare file to patterns
|
105
|
+
def find_in_patterns(patterns, file)
|
106
|
+
patterns.find do |pattern|
|
107
|
+
matched = pattern.is_a?(Regexp) ? pattern.match(file.path) : File.fnmatch(pattern, file.path)
|
108
|
+
logger.debug { "#{pattern} #{matched ? 'matched' : "didn't match"} #{file.path}" }
|
109
|
+
matched
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
82
113
|
def ignored_file_paths
|
83
114
|
handler.ignored_files_map.keys
|
84
115
|
end
|
85
116
|
|
117
|
+
def poll
|
118
|
+
@previous_poll_time = @last_poll_time
|
119
|
+
@last_poll_time = Time.now
|
120
|
+
|
121
|
+
handler.poll if handler.respond_to?(:poll)
|
122
|
+
end
|
123
|
+
|
124
|
+
def poll_interval_elapsed?
|
125
|
+
!last_poll_time || (Time.now - last_poll_time) >= poll_interval
|
126
|
+
end
|
127
|
+
|
86
128
|
def process_agent_def(agent_def)
|
87
129
|
if @default_agent
|
88
130
|
_agent_def_storages = agent_def['storages'] || {}
|
@@ -115,17 +157,6 @@ module Envoi
|
|
115
157
|
@agent ||= @default_agent
|
116
158
|
end
|
117
159
|
|
118
|
-
def initialize_handler(watch_folder_def = @definition)
|
119
|
-
args_out = {}
|
120
|
-
args_out[:logger] ||= logger.dup
|
121
|
-
args_out[:definition] = watch_folder_def
|
122
|
-
|
123
|
-
handler_class = @default_handler_class
|
124
|
-
logger.debug { "Creating Watch Folder Handler Instance. #{handler_class.name}" }
|
125
|
-
@handler = handler_class.new(args_out)
|
126
|
-
logger.debug { "Watch Folder Handler Instance Created. #{handler_class.name}" }
|
127
|
-
end
|
128
|
-
|
129
160
|
#
|
130
161
|
# @param [Hash] watch_folder_def
|
131
162
|
# @option watch_folder_def [String] path
|
@@ -190,7 +221,6 @@ module Envoi
|
|
190
221
|
watch_folder_def.delete('exclude')
|
191
222
|
logger.debug { "Parameter `excludes` initialized." }
|
192
223
|
|
193
|
-
|
194
224
|
logger.debug { "Initializing parameter `quarantine directory path`." }
|
195
225
|
quarantine_directory_path = watch_folder_def['quarantine_directory_path'] || watch_folder_def['quarantine_path']
|
196
226
|
if quarantine_directory_path
|
@@ -217,7 +247,6 @@ module Envoi
|
|
217
247
|
watch_folder_def.delete('completed_path')
|
218
248
|
logger.debug { "Parameter 'completed directory path' initialized." }
|
219
249
|
|
220
|
-
|
221
250
|
logger.debug { "Initializing parameter `upload to storage id`." }
|
222
251
|
storage_id = watch_folder_def['upload_to_storage_id'] || watch_folder_def['storage_id']
|
223
252
|
watch_folder_def['upload_to_storage_id'] ||= storage_id
|
@@ -325,15 +354,116 @@ module Envoi
|
|
325
354
|
@poll_interval = watch_folder_def['poll_interval'] ||= DEFAULT_POLL_INTERVAL
|
326
355
|
end
|
327
356
|
|
328
|
-
def
|
329
|
-
|
330
|
-
|
357
|
+
def process_stable_file(file)
|
358
|
+
file.processing = true
|
359
|
+
file_name = file.name || file.path
|
360
|
+
logger.debug { "Processing File '#{file_name}'" }
|
331
361
|
|
332
|
-
|
362
|
+
storage_id = definition['upload_to_storage_id']
|
363
|
+
|
364
|
+
unless storage_id
|
365
|
+
logger.warn { "Skipping processing of file because of missing storage ID." }
|
366
|
+
return { success: false, message: 'Missing storage ID.' }
|
367
|
+
end
|
368
|
+
|
369
|
+
quarantine_directory_path = definition['quarantine_directory_path']
|
370
|
+
completed_directory_path = definition['completed_directory_path']
|
371
|
+
watch_folder_upload_args = definition['upload_args']
|
372
|
+
|
373
|
+
# full_file_path = File.join(watch_folder.path, file.path)
|
374
|
+
full_file_path = file.path
|
375
|
+
|
376
|
+
upload_args = {
|
377
|
+
file_path: full_file_path,
|
378
|
+
storage_id: storage_id
|
379
|
+
}
|
380
|
+
upload_args.merge!(watch_folder_upload_args) if watch_folder_upload_args.is_a?(Hash)
|
381
|
+
|
382
|
+
logger.debug { "Executing Upload. #{upload_args}" }
|
383
|
+
_response = agent.upload(upload_args)
|
384
|
+
_response = { success: _response } if _response == true || _response == false
|
385
|
+
|
386
|
+
if _response[:success]
|
387
|
+
if completed_directory_path
|
388
|
+
if Dir.exist?(completed_directory_path)
|
389
|
+
logger.debug { "Moving '#{full_file_path}' to completed directory path '#{completed_directory_path}'" }
|
390
|
+
FileUtils.mv full_file_path, completed_directory_path
|
391
|
+
else
|
392
|
+
logger.warn { "Completed directory path not found: '#{completed_directory_path}'" }
|
393
|
+
add_file_to_ignored_file_paths(file)
|
394
|
+
end
|
395
|
+
else
|
396
|
+
FileUtils.rm full_file_path
|
397
|
+
end
|
398
|
+
else
|
399
|
+
if quarantine_directory_path && Dir.exist?(quarantine_directory_path)
|
400
|
+
logger.warn { "Moving '#{full_file_path}' to quarantine directory path '#{quarantine_directory_path}'" }
|
401
|
+
FileUtils.mv full_file_path, quarantine_directory_path
|
402
|
+
else
|
403
|
+
logger.warn { "Adding '#{full_file_path}' to the temporary ignore list." }
|
404
|
+
add_file_to_ignored_file_paths(file)
|
405
|
+
end
|
406
|
+
end
|
407
|
+
|
408
|
+
file.processed = true
|
409
|
+
|
410
|
+
_response
|
411
|
+
rescue => e
|
412
|
+
file.exception = e
|
413
|
+
raise e
|
414
|
+
ensure
|
415
|
+
file.processing = false
|
333
416
|
end
|
334
417
|
|
335
|
-
|
336
|
-
|
418
|
+
|
419
|
+
def process_stable_files
|
420
|
+
maximum_active_processors = definition['maximum_active_processors']
|
421
|
+
|
422
|
+
includes = definition['includes']
|
423
|
+
excludes = definition['excludes']
|
424
|
+
|
425
|
+
stable_files.each do |file|
|
426
|
+
file.watch_folder ||= self
|
427
|
+
next if file.respond_to?(:ignore?) ? file.ignore? : ignored_file_paths.include?(file.path)
|
428
|
+
next if file.processing || file.processed
|
429
|
+
|
430
|
+
if includes && !includes.empty?
|
431
|
+
should_include = find_in_patterns(includes, file)
|
432
|
+
unless should_include
|
433
|
+
add_file_to_ignored_file_paths(file)
|
434
|
+
next
|
435
|
+
end
|
436
|
+
end
|
437
|
+
|
438
|
+
should_exclude = find_in_patterns(excludes, file)
|
439
|
+
if should_exclude
|
440
|
+
add_file_to_ignored_file_paths(file)
|
441
|
+
next
|
442
|
+
end
|
443
|
+
|
444
|
+
if @threaded
|
445
|
+
processors.keep_if { |k, v| k.processing }
|
446
|
+
if processors.length >= maximum_active_processors
|
447
|
+
logger.debug { "Maximum number of active processors reached for watch folder. #{wf.name || wf.paths}" }
|
448
|
+
break
|
449
|
+
end
|
450
|
+
t = Thread.new(file) do |file|
|
451
|
+
begin
|
452
|
+
process_stable_file(file)
|
453
|
+
rescue => e
|
454
|
+
logger.error { "Exception '#{e.message}' in thread for `#{name}` `#{file.path}`. " }
|
455
|
+
raise e
|
456
|
+
ensure
|
457
|
+
file.processing = false rescue nil
|
458
|
+
end
|
459
|
+
end
|
460
|
+
# t.join
|
461
|
+
processors[file] = t if file.processing
|
462
|
+
else
|
463
|
+
process_stable_file(file)
|
464
|
+
end
|
465
|
+
|
466
|
+
end
|
337
467
|
end
|
338
468
|
|
339
469
|
def run
|
@@ -33,13 +33,13 @@ module Envoi
|
|
33
33
|
|
34
34
|
def stable?
|
35
35
|
return false if deleted?
|
36
|
-
return false
|
36
|
+
return false unless exist?
|
37
37
|
|
38
38
|
_min_stable_time = min_stable_time
|
39
39
|
if _min_stable_time
|
40
|
-
|
40
|
+
((Time.now - self[:event_timestamp]) >= _min_stable_time)
|
41
41
|
else
|
42
|
-
|
42
|
+
(self[:stable_poll_count] >= min_stable_poll_count)
|
43
43
|
end
|
44
44
|
end
|
45
45
|
|
@@ -200,6 +200,7 @@ module Envoi
|
|
200
200
|
|
201
201
|
# Really this is just for debugging as polling isn't used to determine file stability
|
202
202
|
def poll(options = { })
|
203
|
+
# return if @min_stable_time
|
203
204
|
should_increment_counter = options.fetch(:should_increment_counter, true)
|
204
205
|
|
205
206
|
@previous_poll_time = @last_poll_time
|
@@ -208,7 +209,10 @@ module Envoi
|
|
208
209
|
lock.synchronize do
|
209
210
|
@known_path_map.each do |fp, f|
|
210
211
|
next if f.ignore?
|
211
|
-
|
212
|
+
unless f.exist?
|
213
|
+
logger.debug { "Skipping Missing File. #{f.summary}" }
|
214
|
+
next
|
215
|
+
end
|
212
216
|
|
213
217
|
logger.debug { "Incrementing Stable Stats: #{f.summary}" }
|
214
218
|
f.last_poll_time = Time.now
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cantemo-portal-agent
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.3.
|
4
|
+
version: 1.3.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- John Whitson
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-01-
|
11
|
+
date: 2019-01-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: asperalm
|
@@ -71,6 +71,9 @@ dependencies:
|
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
73
|
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '1.6'
|
76
|
+
- - ">="
|
74
77
|
- !ruby/object:Gem::Version
|
75
78
|
version: 1.6.5
|
76
79
|
type: :runtime
|
@@ -78,6 +81,9 @@ dependencies:
|
|
78
81
|
version_requirements: !ruby/object:Gem::Requirement
|
79
82
|
requirements:
|
80
83
|
- - "~>"
|
84
|
+
- !ruby/object:Gem::Version
|
85
|
+
version: '1.6'
|
86
|
+
- - ">="
|
81
87
|
- !ruby/object:Gem::Version
|
82
88
|
version: 1.6.5
|
83
89
|
- !ruby/object:Gem::Dependency
|