at_email 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,3 @@
1
+ module At_email::Core ; end
2
+
3
+ require "at_email/core/default"
@@ -0,0 +1,62 @@
1
+
2
+ class At_email::Core::Default
3
+
4
+ CLASS_DISPLAY_NAME = 'atEmail - Default Class'
5
+
6
+ attr_accessor :properties
7
+
8
+ def initialize()
9
+ @properties = {}
10
+ @properties['State'] = 'New'
11
+ @properties['ID'] = At_email::Formatting.new.get_random_id(5)
12
+ @properties['Display Name'] = CLASS_DISPLAY_NAME
13
+ @properties['Log Prefix'] = @properties['Display Name'] + ': ' + @properties['ID']
14
+ @properties['State'] = 'Initialized'
15
+ end
16
+
17
+ def get(property)
18
+ if !@properties[property]
19
+ log_event event_tag='ERROR', 'Cannot get property: ' + property + "\n" + __LINE__ + "\n" + (pp caller())
20
+ return false
21
+ end
22
+ return @properties[property]
23
+ end
24
+
25
+ def get_formatted(property)
26
+ value = get(property)
27
+ output = property + ': ' + value.to_s
28
+ return output
29
+ end
30
+
31
+ def get_state()
32
+ return get('State')
33
+ end
34
+
35
+ def set_state(value)
36
+ set('State', value)
37
+ end
38
+
39
+ ### start of private methods
40
+ private
41
+
42
+ def log_event(event_tag='', event_data)
43
+ log_prefix = get['Log Prefix']
44
+ log_string = log_prefix + ' - ' + event_data
45
+ $logger.event(event_tag, event_data)
46
+ end
47
+
48
+ def set(property, value)
49
+ if @properties[property]
50
+ @properties[property] = value
51
+ end
52
+ end
53
+
54
+ def create_new_property(property, value)
55
+ if @properties[property]
56
+ return false
57
+ else
58
+ @properties[property] = value
59
+ end
60
+ end
61
+
62
+ end
@@ -0,0 +1,3 @@
1
+ module At_email::Tasks ; end
2
+
3
+ require "at_email/tasks/imap_to_fs"
@@ -0,0 +1,756 @@
1
+
2
+ require 'net/imap'
3
+ require 'zlib'
4
+ require 'json'
5
+ require 'digest'
6
+ require 'thread'
7
+ require 'yaml'
8
+ require 'fileutils'
9
+
10
+ class At_email::Tasks::Imap_To_Fs
11
+
12
+ METADATA_ATTRIBUTES = [ 'ENVELOPE', 'FLAGS', 'RFC822.SIZE' ].freeze
13
+ REQUESTED_ATTRIBUTES = [ 'INTERNALDATE', 'RFC822', 'RFC822.SIZE' ].freeze
14
+
15
+ def initialize
16
+ @dir_base_path = $config[:output_base_dir] + '/' + $config[:account_id]
17
+ end
18
+
19
+ def get_imap
20
+ @imap = @connection.imap
21
+ end
22
+
23
+ def get_server_message_folder_list()
24
+ output = []
25
+ root_path = @imap.list("", "")[0].name
26
+ folder_data = @imap.list(root_path, "*")
27
+ output = []
28
+ folder_data.map do |this_folder_data|
29
+ output << this_folder_data.name
30
+ end
31
+ return output
32
+ end
33
+
34
+ def get_imap_metadata(uid_list)
35
+ output = {}
36
+ if uid_list.count > 0
37
+ output = @imap.uid_fetch(uid_list, METADATA_ATTRIBUTES)
38
+ end
39
+ return output
40
+ end
41
+
42
+ def get_message_data(uid)
43
+ data_raw = @imap.uid_fetch(uid.to_i, REQUESTED_ATTRIBUTES)
44
+ if data_raw
45
+ if data_raw[0]
46
+ if data_raw[0].attr
47
+ output = data_raw[0].attr
48
+ return output
49
+ end
50
+ end
51
+ end
52
+ end
53
+
54
+ def get_file_path_prefix(folder_name, message_config)
55
+ if $config[:storage_file_name_uid_padding]
56
+ if $config[:storage_file_name_uid_padding_length] <= 1
57
+ storage_file_name_uid_padding = 2
58
+ else
59
+ storage_file_name_uid_padding = $config[:storage_file_name_uid_padding_length]
60
+ end
61
+ uid_based_prefix = message_config['uid'] + (10 ** (storage_file_name_uid_padding - 1))
62
+ else
63
+ uid_based_prefix = message_config['uid']
64
+ end
65
+ hash_length = $config[:storage_file_name_hash_length]
66
+ dir_path = @dir_base_path + '/' + folder_name
67
+ file_name_hash = Digest::MD5.hexdigest(Digest::MD5.hexdigest(YAML.dump(message_config['envelope']))+'-'+Digest::MD5.hexdigest(YAML.dump(message_config['flags'])))[0...hash_length]
68
+ output = dir_path+'/'+uid_based_prefix.to_s+'-'+file_name_hash
69
+ return output
70
+ end
71
+
72
+ def get_file_path_for_message(folder_name, message_config)
73
+ output = get_file_path_prefix(folder_name, message_config)+'.'+$config[:storage_file_extension]
74
+ return output
75
+ end
76
+
77
+ def get_uids()
78
+ uid_list = @imap.uid_search(["ALL"]).sort
79
+ return uid_list
80
+ end
81
+
82
+ def generate_metadata_for_account()
83
+ @account_metadata = {}
84
+ @account_metadata[:account] = {}
85
+ @account_metadata[:account]['directories'] = {}
86
+ @account_metadata[:account]['directories']['server'] = get_server_dir_list()
87
+ @account_metadata[:account]['directories']['local'] = get_local_dir_list()
88
+ @account_metadata[:account]['messages'] = {}
89
+ @account_metadata[:account]['messages']['server'] = {}
90
+ @account_metadata[:account]['messages']['server']['folders'] = get_server_message_folder_list()
91
+ @account_metadata[:task] = {}
92
+ @account_metadata[:task] = {}
93
+ @account_metadata[:folders] = {}
94
+ end
95
+
96
+ def generate_metadata_for_folder(folder_name)
97
+ @account_metadata[:folders][folder_name] = {}
98
+ @account_metadata[:folders][folder_name]['messages'] = {}
99
+ @account_metadata[:folders][folder_name]['messages']['server'] = {}
100
+ @account_metadata[:folders][folder_name]['messages']['server']['uids'] = get_uids()
101
+ @account_metadata[:folders][folder_name]['messages']['server']['metadata'] = get_imap_metadata(@account_metadata[:folders][folder_name]['messages']['server']['uids'])
102
+ @account_metadata[:folders][folder_name]['messages']['server']['config'] = update_message_config(folder_name, @account_metadata[:folders][folder_name]['messages']['server']['metadata'])
103
+ @account_metadata[:folders][folder_name]['files'] = {}
104
+ @account_metadata[:folders][folder_name]['files']['server'] = get_server_file_list(folder_name, @account_metadata[:folders][folder_name]['messages']['server']['config'])
105
+ @account_metadata[:folders][folder_name]['files']['local'] = get_local_file_list(folder_name)
106
+ @account_metadata[:folders][folder_name]['files']['deletes'] = get_delete_list(folder_name)
107
+ @account_metadata[:folders][folder_name]['process'] = {}
108
+ end
109
+
110
+ def update_message_config(folder_name, metadata)
111
+ output = {}
112
+ metadata.each do |message_metadata|
113
+ uid = message_metadata.attr['UID']
114
+ output[uid] = {}
115
+ output[uid]['uid'] = uid
116
+ output[uid]['envelope'] = message_metadata.attr['ENVELOPE'].to_h
117
+ output[uid]['flags'] = message_metadata.attr['FLAGS']
118
+ output[uid]['size'] = message_metadata.attr['RFC822.SIZE']
119
+ output[uid]['file_path'] = get_file_path_for_message(folder_name, output[uid])
120
+ end
121
+ return output
122
+ end
123
+
124
+ def get_local_file_list(folder_name)
125
+ output = []
126
+ dir_path = @dir_base_path + '/' + folder_name
127
+ if Dir.exists?(dir_path)
128
+ Dir.chdir(dir_path)
129
+ file_list = Dir.glob("*."+$config[:storage_file_extension]).sort_by{|f| f.split(" ")[0].to_i }
130
+ file_list.each do |file_name|
131
+ if File.file?(file_name)
132
+ output << dir_path+'/'+file_name
133
+ end
134
+ end
135
+ end
136
+ return output
137
+ end
138
+
139
+ def get_server_file_list(folder_name, server_message_config)
140
+ output = {}
141
+ server_message_config.each do |uid, message_config|
142
+ file_path = get_file_path_for_message(folder_name, message_config)
143
+ output[file_path] = uid
144
+ end
145
+ return output
146
+ end
147
+
148
+ def get_local_dir_list()
149
+ output = []
150
+ if Dir.exists?(@dir_base_path)
151
+ Dir.chdir(@dir_base_path)
152
+ file_list = Dir.glob("**/*/")
153
+ file_list.each do |object_name|
154
+ if File.directory?(object_name)
155
+ output << @dir_base_path + '/' + object_name
156
+ end
157
+ end
158
+ end
159
+ return output
160
+ end
161
+
162
+ def get_server_dir_list()
163
+ output = []
164
+ message_folder_list = get_server_message_folder_list()
165
+ message_folder_list.each do |message_folder|
166
+ output << @dir_base_path + '/' + message_folder
167
+ end
168
+ return output
169
+ end
170
+
171
+ def delete_directories(folder_name)
172
+ server_dirs = @account_metadata[:folders][folder_name]['directories']['server']
173
+ @account_metadata[:folders][folder_name]['directories']['local'].each do |local_dir|
174
+ if !server_dirs[local_dir]
175
+ $logger.info( folder_name + ' - Deleting Directory: ' + local_dir )
176
+ end
177
+ end
178
+ end
179
+
180
+ def get_download_list(folder_name)
181
+ output = []
182
+ local_file_list = @account_metadata[:folders][folder_name]['files']['local']
183
+ @account_metadata[:folders][folder_name]['messages']['server']['config'].each do |uid, message_config|
184
+ file_path = message_config['file_path']
185
+ if !local_file_list.include?(file_path)
186
+ output << uid
187
+ end
188
+ end
189
+ return output
190
+ end
191
+
192
+ def get_delete_list(folder_name)
193
+ output = []
194
+ server_files = @account_metadata[:folders][folder_name]['files']['server']
195
+ @account_metadata[:folders][folder_name]['files']['local'].each do |file_path|
196
+ if !server_files[file_path]
197
+ if File.exists?(file_path)
198
+ output << file_path
199
+ end
200
+ end
201
+ end
202
+ return output
203
+ end
204
+
205
+ def delete_files(folder_name)
206
+ @account_metadata[:folders][folder_name]['files']['deletes'].each do |file_path|
207
+ if File.exists?(file_path)
208
+ $logger.info( folder_name + ' - Deleting File: ' + file_path )
209
+ File.delete(file_path)
210
+ end
211
+ end
212
+ end
213
+
214
+ def get_ratio_formatted(a, b, delimiter = '/')
215
+ output = a.to_s + delimiter + b.to_s
216
+ return output
217
+ end
218
+
219
+ def get_uid_string_formatted(uid)
220
+ output = 'UID:' + uid.to_s
221
+ return output
222
+ end
223
+
224
+ def set_thread_state(folder_name, uid, msg_ratio, state)
225
+ uid_string = get_uid_string_formatted(uid)
226
+ set_thread_property(folder_name, uid, 'State', state.to_s)
227
+ log_thread_property(folder_name, uid, msg_ratio, 'State')
228
+ end
229
+
230
+ def set_thread_property(folder_name, uid, property, value)
231
+ @account_metadata[:folders][folder_name]['process']['threads']['uids'][uid][property] = value
232
+ end
233
+
234
+ def get_thread_property(folder_name, uid, property)
235
+ if @account_metadata[:folders][folder_name]['process']['threads']['uids'][uid][property]
236
+ output = @account_metadata[:folders][folder_name]['process']['threads']['uids'][uid][property]
237
+ return output
238
+ else
239
+ return false
240
+ end
241
+ end
242
+
243
+ def log_thread_property(folder_name, uid, msg_ratio, property_name)
244
+ log_string = get_formatted_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
245
+ log_thread_event(folder_name, uid, msg_ratio, 'INFO', '', log_string)
246
+ end
247
+
248
+ def get_formatted_thread_property(folder_name, uid, property_name)
249
+ ### getting property
250
+ name = property_name
251
+ value = get_thread_property(folder_name, uid, name) || 'Unable to get property: ' + name
252
+ #####
253
+ output = name + ': ' + value.to_s
254
+ return output
255
+ end
256
+
257
+ def increase_thread_property(folder_name, uid, property)
258
+ @account_metadata[:folders][folder_name]['process']['threads']['uids'][uid][property] += 1
259
+ end
260
+
261
+ def increase_thread_property(folder_name, uid, property)
262
+ @account_metadata[:folders][folder_name]['process']['threads']['uids'][uid][property] -= 1
263
+ end
264
+
265
+ def increase_thread_count(folder_name)
266
+ @account_metadata[:folders][folder_name]['process']['threads']['count'] += 1
267
+ end
268
+
269
+ def decrease_thread_count(folder_name)
270
+ @account_metadata[:folders][folder_name]['process']['threads']['count'] -= 1
271
+ end
272
+
273
+ def get_thread_id_string(uid, msg_ratio)
274
+ uid_string = get_uid_string_formatted(uid)
275
+ output = msg_ratio + ' - ' + uid_string
276
+ end
277
+
278
+ def log_thread_on_success(folder_name, uid, msg_ratio)
279
+ ### getting property
280
+ property_name = 'Queue Time'
281
+ queue_time = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
282
+ #####
283
+ ### getting property
284
+ property_name = 'End Time'
285
+ end_time = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
286
+ #####
287
+ ### getting property
288
+ property_name = 'Server Size'
289
+ server_size = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
290
+ #####
291
+ ### getting property
292
+ property_name = 'Local Size'
293
+ local_size = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
294
+ #####
295
+ ### getting data
296
+ duration_formatted = get_duration_formatted(queue_time, end_time) || 'Unable to get data'
297
+ #####
298
+ ### getting data
299
+ size_ratio_formatted = get_ratio_formatted(local_size, server_size, ' / ') || 'Unable to get data'
300
+ size_formatted = size_ratio_formatted + ' Bytes'
301
+ #####
302
+ log_string = 'Duration: ' + duration_formatted + ' - Size: ' + size_formatted
303
+ log_thread_event(folder_name, uid, msg_ratio, 'INFO', '', log_string)
304
+ end
305
+
306
+ def log_thread_on_failure(folder_name, uid, msg_ratio, event_data)
307
+ ### getting property
308
+ property_name = 'Queue Time'
309
+ queue_time = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
310
+ #####
311
+ ### getting property
312
+ property_name = 'End Time'
313
+ end_time = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
314
+ #####
315
+ ### getting property
316
+ property_name = 'Server Size'
317
+ server_size = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
318
+ #####
319
+ ### getting property
320
+ property_name = 'Local Size'
321
+ local_size = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
322
+ #####
323
+ ### getting data
324
+ duration_formatted = get_duration_formatted(queue_time, end_time) || 'Unable to get data'
325
+ #####
326
+ ### getting data
327
+ size_formatted = server_size + ' Bytes'
328
+ #####
329
+ log_string = 'Duration: ' + duration_formatted + ' - Size: ' + size_formatted
330
+ log_thread_event(folder_name, uid, msg_ratio, 'ERROR', 'E', log_string)
331
+ log_thread_event(folder_name, uid, msg_ratio, 'ERROR', 'ERROR', event_data)
332
+ end
333
+
334
+ def log_thread_on_retry(folder_name, uid, msg_ratio, event_data)
335
+ log_thread_event(folder_name, uid, msg_ratio, 'WARN', 'W', event_data)
336
+ end
337
+
338
+ def log_thread_event(folder_name, uid, msg_ratio, event_level, event_tag='', event_data)
339
+ uid_string = get_uid_string_formatted(uid) || 'Unable to get UID string'
340
+ thread_id_string = get_thread_id_string(uid, msg_ratio) || 'Unable to get thread ID string'
341
+ log_string = thread_id_string + ' - ' + event_data
342
+ log_folder_event(folder_name, event_level, event_tag, log_string)
343
+ end
344
+
345
+ def log_folder_event(folder_name, event_level, event_tag, event_data)
346
+ log_string = folder_name + ' - ' + event_data
347
+ $logger.event(event_level, event_tag, log_string)
348
+ end
349
+
350
+ def get_message_config(folder_name, uid)
351
+ if @account_metadata[:folders][folder_name]['messages']['server']['config'][uid]
352
+ output = @account_metadata[:folders][folder_name]['messages']['server']['config'][uid]
353
+ return output
354
+ else
355
+ raise '***** ERROR ***** - Message Config: Cannot get message config for folder: ' + folder_name + ' - UID:' + uid
356
+ end
357
+ end
358
+
359
+ def bootstrap_thread(folder_name, uid)
360
+ message_config = get_message_config(folder_name, uid)
361
+ set_thread_property(folder_name, uid, 'File Name', File.basename(message_config['file_path']))
362
+ set_thread_property(folder_name, uid, 'File Path', message_config['file_path'])
363
+ set_thread_property(folder_name, uid, 'Server Size', message_config['size'])
364
+ set_thread_property(folder_name, uid, 'Retry Count', 0)
365
+ end
366
+
367
+ def retry_check(folder_name, uid, msg_ratio)
368
+ if get_thread_property(folder_name, uid, 'Retry Count') <= 3
369
+ log_thread_on_retry(folder_name, uid, msg_ratio, 'Queuing for a retry')
370
+ increase_thread_property(folder_name, uid, 'Retry Count')
371
+ set_thread_state(folder_name, uid, msg_ratio, 'Retrying')
372
+ queue_thread(folder_name, uid, msg_ratio)
373
+ return true
374
+ else
375
+ set_thread_property(folder_name, uid, 'End Time', Time.now)
376
+ set_thread_state(folder_name, uid, msg_ratio, 'Failed')
377
+ return false
378
+ end
379
+ end
380
+
381
+ def queue_thread(folder_name, uid, msg_ratio)
382
+ set_thread_state(folder_name, uid, msg_ratio, 'Queued')
383
+ bootstrap_thread(folder_name, uid)
384
+ set_thread_property(folder_name, uid, 'Queue Time', Time.now)
385
+ increase_thread_count(folder_name)
386
+ Thread.abort_on_exception = true
387
+ @account_metadata[:folders][folder_name]['process']['threads']['uids'][uid]['Thread'] = Thread.new do
388
+ sleep 0.1
389
+ write_data_for_uid(folder_name, uid, msg_ratio)
390
+ end
391
+ end
392
+
393
+ def write_data_for_uid(folder_name, uid, msg_ratio)
394
+ uid_string = get_uid_string_formatted(uid)
395
+ if get_thread_property(folder_name, uid, 'State') === 'Queued'
396
+ set_thread_state(folder_name, uid, msg_ratio, 'Active')
397
+ log_thread_property(folder_name, uid, msg_ratio, 'File Name')
398
+ file_path = get_thread_property(folder_name, uid, 'File Path')
399
+ file_path_tmp = file_path + '.tmp'
400
+ if message_data = get_message_data(uid)
401
+ File.write(file_path_tmp, Zlib::Deflate.deflate(YAML.dump(message_data)))
402
+ FileUtils.mv(file_path_tmp, file_path)
403
+ set_thread_property(folder_name, uid, 'Local Size', File.size(get_thread_property(folder_name, uid, 'File Path')))
404
+ if get_thread_property(folder_name, uid, 'Local Size') > (get_thread_property(folder_name, uid, 'Server Size') * 3)
405
+ if !retry_check(folder_name, uid, msg_ratio)
406
+ log_thread_on_failure(folder_name, uid, msg_ratio, 'File size mismatch after download')
407
+ end
408
+ if File.exists?(file_path)
409
+ ### getting property
410
+ property_name = 'File Name'
411
+ file_name = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
412
+ #####
413
+ log_thread_event(folder_name, uid, msg_ratio, 'WARN', 'W', 'File size mismatch detected, cleaning up')
414
+ log_thread_event(folder_name, uid, msg_ratio, 'WARN', 'W', 'Deleting File: ' + file_name)
415
+ File.delete(file_path)
416
+ end
417
+ else
418
+ set_thread_property(folder_name, uid, 'End Time', Time.now)
419
+ set_thread_state(folder_name, uid, msg_ratio, 'Successful')
420
+ log_thread_on_success(folder_name, uid, msg_ratio)
421
+ end
422
+ else
423
+ if !retry_check(folder_name, uid, msg_ratio)
424
+ log_thread_on_failure(folder_name, uid, msg_ratio, 'Unable to get message data')
425
+ end
426
+ end
427
+ decrease_thread_count(folder_name)
428
+ else
429
+ ### getting property
430
+ property_name = 'State'
431
+ formatted_property = get_formatted_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
432
+ #####
433
+ log_string = 'Thread state incorrect - ' + formatted_property
434
+ log_folder_event(folder_name, 'ERROR', 'E', log_string)
435
+ raise log_string
436
+ end
437
+ end
438
+
439
+
440
+
441
+ def process_thread_data_before_finish(folder_name)
442
+ has_failed = false
443
+ @account_metadata[:folders][folder_name]['process']['threads']['uids'].each do |uid, thread_data|
444
+ if thread_data['State'] != 'Successful'
445
+ thread_data['State'] = 'Failed'
446
+ end
447
+ if thread_data['State'] === 'Failed'
448
+ kill_thread(folder_name, uid)
449
+ has_failed = true
450
+ end
451
+ end
452
+ if has_failed
453
+ sleep 5
454
+ end
455
+ end
456
+
457
+ def process_shutdown(folder_name)
458
+ time_started_waiting = Time.now.to_i
459
+ while @account_metadata[:folders][folder_name]['process']['threads']['count'] > 0
460
+ time_since_started_waiting = Time.now.to_i - time_started_waiting
461
+ if time_since_started_waiting >= $config[:thread_timeout]
462
+ $logger.warn folder_name + ' - *E* - Thread pool shutdown - Timeout: ' + $config[:thread_timeout].to_s + ' Sec(s)'
463
+ @account_metadata[:folders][folder_name]['process']['threads']['count'] = 0
464
+ else
465
+ $logger.info folder_name + ' - Waiting for threads - Remaining: ' + @account_metadata[:folders][folder_name]['process']['threads']['count'].to_s + '/' + $config[:threads].to_s + ' - Timeout: ' + time_since_started_waiting.to_s + '/' + $config[:thread_timeout].to_s + ' Sec(s)'
466
+ sleep 10
467
+ end
468
+ end
469
+ end
470
+
471
+ def log_folder_info(folder_name)
472
+ $logger.debug folder_name + ' - Server Files: ' + @account_metadata[:folders][folder_name]['files']['server'].count.to_s
473
+ @account_metadata[:folders][folder_name]['files']['server'].each do |file_path, uid|
474
+ $logger.debug folder_name + ' - ' + File.basename(file_path)
475
+ end
476
+ $logger.debug folder_name + ' - Local Files: ' + @account_metadata[:folders][folder_name]['files']['local'].count.to_s
477
+ @account_metadata[:folders][folder_name]['files']['local'].each do |file_path|
478
+ $logger.debug folder_name + ' - ' + File.basename(file_path)
479
+ end
480
+ end
481
+
482
+ def create_output_directory(folder_name)
483
+ dir_path = @dir_base_path + '/' + folder_name
484
+ if !Dir.exists?(dir_path)
485
+ $logger.debug folder_name + ' - Creating Directory: ' + dir_path
486
+ FileUtils.mkdir_p dir_path
487
+ end
488
+ end
489
+
490
+ def process_deletes(folder_name)
491
+ if $config[:delete_on_sync]
492
+ if @account_metadata[:folders][folder_name]['files']['deletes'].count > 0
493
+ $logger.info folder_name + ' - Delete Count: ' + @account_metadata[:folders][folder_name]['files']['deletes'].count.to_s
494
+ $logger.info folder_name + ' - Executing Deletes'
495
+ delete_files(folder_name)
496
+ end
497
+ end
498
+ end
499
+
500
+ def kill_thread(folder_name, uid)
501
+ log_folder_event(folder_name, 'WARN', 'ERROR', 'Thread Failure - ' + get_uid_string_formatted(uid))
502
+ log_folder_event(folder_name, 'WARN', 'ERROR', ' Killing Thread')
503
+ Thread.kill(get_thread_property(folder_name, uid, 'Thread'))
504
+ ### getting formatted thread property
505
+ property_name = 'Queue Time'
506
+ log_string = get_formatted_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
507
+ #####
508
+ log_folder_event(folder_name, 'WARN', 'ERROR', ' ' + log_string)
509
+ ### getting formatted thread property
510
+ property_name = 'End Time'
511
+ log_string = get_formatted_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
512
+ #####
513
+ log_folder_event(folder_name, 'WARN', 'ERROR', ' ' + log_string)
514
+ $logger.error folder_name + ' - Duration: ' + get_duration_formatted(get_thread_property(folder_name, uid, 'Queue Time'), get_thread_property(folder_name, uid, 'End Time'))
515
+ ### getting formatted thread property
516
+ property_name = 'File Name'
517
+ log_string = get_formatted_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
518
+ #####
519
+ log_folder_event(folder_name, 'WARN', 'ERROR', ' ' + log_string)
520
+ end
521
+
522
+ def process_downloads(folder_name, download_list)
523
+ process_healthy = true
524
+ if download_list.count > 0
525
+ $logger.info( folder_name + ' - ' + download_list.count.to_s + ' Message(s) to download - Thread Limit: ' + $config[:threads].to_s )
526
+ message_count = 0
527
+ ### Creating / resetting active thread data structure
528
+ @account_metadata[:folders][folder_name]['process']['threads'] = {}
529
+ @account_metadata[:folders][folder_name]['process']['threads']['count'] = 0
530
+ @account_metadata[:folders][folder_name]['process']['threads']['uids'] = {}
531
+ message_total = download_list.count
532
+ download_list.each do |uid|
533
+ if process_healthy
534
+ message_count += 1
535
+ uid_string = get_uid_string_formatted(uid)
536
+ msg_ratio = get_ratio_formatted(message_count, message_total)
537
+ last_queue_time = Time.now
538
+ if @account_metadata[:folders][folder_name]['process']['threads']['count'] >= $config[:threads]
539
+ $logger.debug folder_name + ' - Thread Count: ' + @account_metadata[:folders][folder_name]['process']['threads']['count'].to_s + ' - Waiting...'
540
+ end
541
+ while @account_metadata[:folders][folder_name]['process']['threads']['count'] >= $config[:threads] && process_healthy
542
+ if (Time.now.to_i - last_queue_time.to_i) >= ($config[:thread_timeout] * $config[:timeout_warning_ratio])
543
+ if (Time.now.to_i - last_queue_time.to_i) >= $config[:thread_timeout]
544
+ $logger.error folder_name + ' - *E* Thread queuing has stopped - Timeout: ' + $config[:thread_timeout].to_s + ' Sec(s)'
545
+ process_healthy = false
546
+ else
547
+ $logger.warn folder_name + ' - *W* Thread queuing has stopped - Timeout: ' + (Time.now.to_i - last_queue_time.to_i).to_s + '/' + $config[:thread_timeout].to_s + ' Sec(s)'
548
+ sleep rand(10..20)
549
+ end
550
+ else
551
+ sleep 0.2
552
+ end
553
+ end
554
+ if process_healthy
555
+ @account_metadata[:folders][folder_name]['process']['threads']['uids'][uid] = {}
556
+ queue_thread(folder_name, uid, msg_ratio)
557
+ sleep 0.1
558
+ while get_thread_property(folder_name, uid, 'State') === 'Queued'
559
+ if (Time.now.to_i - get_thread_property(folder_name, uid, 'Queue Time').to_i) >= ($config[:thread_timeout] * $config[:timeout_warning_ratio])
560
+ if (Time.now.to_i - get_thread_property(folder_name, uid, 'Queue Time').to_i) >= $config[:thread_timeout]
561
+ $logger.error folder_name + ' - ' + msg_ratio + ' - ' + uid_string + ' - *E* Thread state - Moving on... - Timeout: ' + $config[:thread_timeout].to_s + ' Sec(s)'
562
+ set_thread_state(folder_name, uid, msg_ratio, 'Failed')
563
+ kill_thread(folder_name, uid)
564
+ else
565
+ $logger.warn folder_name + ' - ' + msg_ratio + ' - ' + uid_string + ' - *W* Thread state - Timeout: ' + (Time.now.to_i - get_thread_property(folder_name, uid, 'Queue Time').to_i).to_s + '/' + $config[:thread_timeout].to_s + ' Sec(s)'
566
+ sleep rand(10..20)
567
+ end
568
+ else
569
+ sleep 0.2
570
+ end
571
+ end
572
+ end
573
+ end
574
+ end
575
+ if process_healthy
576
+ sleep 2
577
+ process_shutdown(folder_name)
578
+ end
579
+ process_thread_data_before_finish(folder_name)
580
+ end
581
+ end
582
+
583
+ def write_data_for_folder(folder_name)
584
+ $logger.debug ''
585
+ $logger.info 'Checking ' + folder_name
586
+ @imap.examine(folder_name)
587
+ generate_metadata_for_folder(folder_name)
588
+ log_folder_info(folder_name)
589
+ download_list = get_download_list(folder_name)
590
+ create_output_directory(folder_name)
591
+ process_deletes(folder_name)
592
+ process_downloads(folder_name, download_list)
593
+ end
594
+
595
+ def get_data_for_all_folders
596
+ folder_list = @account_metadata[:account]['messages']['server']['folders']
597
+ $logger.debug ''
598
+ $logger.debug 'Folder Count: ' + folder_list.count.to_s
599
+ folder_list.each do |folder_name|
600
+ if check_folder_included(folder_name)
601
+ $logger.debug ' ' + folder_name
602
+ else
603
+ $logger.debug ' ' + folder_name + ' - SKIPPED'
604
+ end
605
+ end
606
+ folder_list.each do |folder_name|
607
+ if check_folder_included(folder_name)
608
+ write_data_for_folder(folder_name)
609
+ end
610
+ end
611
+ $logger.debug ''
612
+ end
613
+
614
+ def check_folder_included(folder_name)
615
+ ignore_list = []
616
+ ignore_list << "[Gmail]"
617
+ ignore_list << "[Gmail]/All Mail"
618
+ ignore_list << "[Gmail]/Important"
619
+ ignore_list << "[Gmail]/Starred"
620
+ if ignore_list.include?(folder_name)
621
+ return false
622
+ else
623
+ return true
624
+ end
625
+ end
626
+
627
+ def execute
628
+ @connection = At_email::Account::ImapConnection.new()
629
+ @connection.login
630
+ get_imap
631
+ generate_metadata_for_account
632
+ @account_metadata[:task][:start_time] = Time.now
633
+ get_data_for_all_folders
634
+ @connection.disconnect
635
+ @account_metadata[:task][:end_time] = Time.now
636
+ task_report
637
+ end
638
+
639
+ def task_report
640
+ stats = get_task_stats()
641
+ $logger.info ''
642
+ $logger.info '##### Task Report #####'
643
+ $logger.info ''
644
+ $logger.info ' Task: ' + $config[:task]
645
+ $logger.info ''
646
+ $logger.info ' Server: ' + $config[:server]
647
+ $logger.info ' Username: ' + $config[:username]
648
+ $logger.info ' Output Directory: ' + @dir_base_path
649
+ $logger.info ''
650
+ $logger.info ' Start Time: ' + @account_metadata[:task][:start_time].to_s
651
+ $logger.info ' End Time: ' + @account_metadata[:task][:end_time].to_s
652
+ $logger.info ' Duration: ' + get_duration_formatted(@account_metadata[:task][:start_time], @account_metadata[:task][:end_time])
653
+ $logger.info ''
654
+ $logger.info ' Folders Processed: ' + @account_metadata[:folders].count.to_s
655
+ $logger.info ''
656
+ @account_metadata[:folders].each do |folder_name, folder_data|
657
+ $logger.info ' ' + folder_name
658
+ $logger.info ''
659
+ $logger.info ' Message Stats'
660
+ $logger.info ' Total: ' + folder_data['messages']['server']['config'].count.to_s
661
+ $logger.info ' Total Size: ' + stats[:folders][folder_name]['messages']['size'].to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse + ' bytes'
662
+ $logger.info ''
663
+ $logger.info ' Thread Stats'
664
+ $logger.info ' Total: ' + stats[:folders][folder_name]['threads']['Total'].to_s
665
+ $logger.info ' Success: ' + stats[:folders][folder_name]['threads']['Success'].to_s
666
+ $logger.info ' Failed: ' + stats[:folders][folder_name]['threads']['Failed'].to_s
667
+ if folder_data['process']
668
+ if folder_data['process']['threads']
669
+ if folder_data['process']['threads']['uids']
670
+ $logger.info ' Threads'
671
+ folder_data['process']['threads']['uids'].each do |uid, thread_data|
672
+ if thread_data['State'] === 'Successful'
673
+ $logger.debug ' UID: ' + uid.to_s
674
+ $logger.debug ' Path: ' + folder_data['messages']['server']['config'][uid]['file_path']
675
+ $logger.debug ' Thread State: ' + thread_data['State']
676
+ else
677
+ $logger.info ' UID: ' + uid.to_s
678
+ $logger.info ' Path: ' + message_config['file_path']
679
+ $logger.info ' Thread State: ' + thread_data['State']
680
+ end
681
+ end
682
+ end
683
+ end
684
+ end
685
+ $logger.info ''
686
+ end
687
+ $logger.info ''
688
+ end
689
+
690
+ def get_task_stats()
691
+ output = {}
692
+ output[:account] = {}
693
+ output[:folders] = {}
694
+ @account_metadata[:folders].each do |folder_name, folder_data|
695
+ output[:folders][folder_name] = {}
696
+ output[:folders][folder_name]['messages'] = {}
697
+ output[:folders][folder_name]['messages']['size'] = 0
698
+ folder_data['messages']['server']['config'].each do |uid, message_config|
699
+ output[:folders][folder_name]['messages']['size'] += message_config['size']
700
+ end
701
+ output[:folders][folder_name]['threads'] = {}
702
+ output[:folders][folder_name]['threads']['Total'] = 0
703
+ output[:folders][folder_name]['threads']['Success'] = 0
704
+ output[:folders][folder_name]['threads']['Failed'] = 0
705
+ if folder_data['process']
706
+ if folder_data['process']['threads']
707
+ if folder_data['process']['threads']['uids']
708
+ folder_data['process']['threads']['uids'].each do |uid, thread_data|
709
+ output[:folders][folder_name]['threads']['Total'] += 1
710
+ if thread_data['State'] === 'Successful'
711
+ output[:folders][folder_name]['threads']['Success'] += 1
712
+ else
713
+ output[:folders][folder_name]['threads']['Failed'] += 1
714
+ end
715
+ end
716
+ end
717
+ end
718
+ end
719
+ end
720
+ return output
721
+ end
722
+
723
+ def get_duration(start_time, end_time)
724
+ output = end_time.to_f - start_time.to_f
725
+ return output
726
+ end
727
+
728
+ def get_duration_formatted(start_time, end_time)
729
+ duration_seconds = get_duration(start_time, end_time)
730
+ if duration_seconds
731
+ if duration_seconds <= 60
732
+ output = sprintf("%.6f", duration_seconds) + ' Sec(s)'
733
+ else
734
+ nanoseconds = (sprintf("%.6f", duration_seconds).to_f * 1000000) % 1000000
735
+ seconds = duration_seconds % 60
736
+ minutes = (duration_seconds / 60) % 60
737
+ hours = duration_seconds / (60 * 60)
738
+ output = format("%02d:%02d:%02d.%6d", hours, minutes, seconds, nanoseconds)
739
+ end
740
+ else
741
+ return false
742
+ end
743
+ return output
744
+ end
745
+
746
+ def format_file_size(file_size)
747
+ output = format_number(file_size) + ' bytes'
748
+ return output
749
+ end
750
+
751
+ def format_number(number)
752
+ output = number.to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse
753
+ return output
754
+ end
755
+
756
+ end