at_email 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/README.md +28 -0
- data/bin/at_email +24 -0
- data/lib/at_email.rb +210 -0
- data/lib/at_email/account.rb +3 -0
- data/lib/at_email/account/connection.rb +32 -0
- data/lib/at_email/config.rb +4 -0
- data/lib/at_email/config/cmd_opt_parser.rb +58 -0
- data/lib/at_email/config/config_file.rb +100 -0
- data/lib/at_email/core.rb +3 -0
- data/lib/at_email/core/default.rb +62 -0
- data/lib/at_email/tasks.rb +3 -0
- data/lib/at_email/tasks/imap_to_fs.rb +756 -0
- data/lib/at_email/threads.rb +3 -0
- data/lib/at_email/threads/thread_queue.rb +83 -0
- data/lib/at_email/version.rb +10 -0
- metadata +59 -0
@@ -0,0 +1,62 @@
|
|
1
|
+
|
2
|
+
class At_email::Core::Default
|
3
|
+
|
4
|
+
CLASS_DISPLAY_NAME = 'atEmail - Default Class'
|
5
|
+
|
6
|
+
attr_accessor :properties
|
7
|
+
|
8
|
+
def initialize()
|
9
|
+
@properties = {}
|
10
|
+
@properties['State'] = 'New'
|
11
|
+
@properties['ID'] = At_email::Formatting.new.get_random_id(5)
|
12
|
+
@properties['Display Name'] = CLASS_DISPLAY_NAME
|
13
|
+
@properties['Log Prefix'] = @properties['Display Name'] + ': ' + @properties['ID']
|
14
|
+
@properties['State'] = 'Initialized'
|
15
|
+
end
|
16
|
+
|
17
|
+
def get(property)
|
18
|
+
if !@properties[property]
|
19
|
+
log_event event_tag='ERROR', 'Cannot get property: ' + property + "\n" + __LINE__ + "\n" + (pp caller())
|
20
|
+
return false
|
21
|
+
end
|
22
|
+
return @properties[property]
|
23
|
+
end
|
24
|
+
|
25
|
+
def get_formatted(property)
|
26
|
+
value = get(property)
|
27
|
+
output = property + ': ' + value.to_s
|
28
|
+
return output
|
29
|
+
end
|
30
|
+
|
31
|
+
def get_state()
|
32
|
+
return get('State')
|
33
|
+
end
|
34
|
+
|
35
|
+
def set_state(value)
|
36
|
+
set('State', value)
|
37
|
+
end
|
38
|
+
|
39
|
+
### start of private methods
|
40
|
+
private
|
41
|
+
|
42
|
+
def log_event(event_tag='', event_data)
|
43
|
+
log_prefix = get['Log Prefix']
|
44
|
+
log_string = log_prefix + ' - ' + event_data
|
45
|
+
$logger.event(event_tag, event_data)
|
46
|
+
end
|
47
|
+
|
48
|
+
def set(property, value)
|
49
|
+
if @properties[property]
|
50
|
+
@properties[property] = value
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def create_new_property(property, value)
|
55
|
+
if @properties[property]
|
56
|
+
return false
|
57
|
+
else
|
58
|
+
@properties[property] = value
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
end
|
@@ -0,0 +1,756 @@
|
|
1
|
+
|
2
|
+
require 'net/imap'
|
3
|
+
require 'zlib'
|
4
|
+
require 'json'
|
5
|
+
require 'digest'
|
6
|
+
require 'thread'
|
7
|
+
require 'yaml'
|
8
|
+
require 'fileutils'
|
9
|
+
|
10
|
+
class At_email::Tasks::Imap_To_Fs
|
11
|
+
|
12
|
+
METADATA_ATTRIBUTES = [ 'ENVELOPE', 'FLAGS', 'RFC822.SIZE' ].freeze
|
13
|
+
REQUESTED_ATTRIBUTES = [ 'INTERNALDATE', 'RFC822', 'RFC822.SIZE' ].freeze
|
14
|
+
|
15
|
+
def initialize
|
16
|
+
@dir_base_path = $config[:output_base_dir] + '/' + $config[:account_id]
|
17
|
+
end
|
18
|
+
|
19
|
+
def get_imap
|
20
|
+
@imap = @connection.imap
|
21
|
+
end
|
22
|
+
|
23
|
+
def get_server_message_folder_list()
|
24
|
+
output = []
|
25
|
+
root_path = @imap.list("", "")[0].name
|
26
|
+
folder_data = @imap.list(root_path, "*")
|
27
|
+
output = []
|
28
|
+
folder_data.map do |this_folder_data|
|
29
|
+
output << this_folder_data.name
|
30
|
+
end
|
31
|
+
return output
|
32
|
+
end
|
33
|
+
|
34
|
+
def get_imap_metadata(uid_list)
|
35
|
+
output = {}
|
36
|
+
if uid_list.count > 0
|
37
|
+
output = @imap.uid_fetch(uid_list, METADATA_ATTRIBUTES)
|
38
|
+
end
|
39
|
+
return output
|
40
|
+
end
|
41
|
+
|
42
|
+
def get_message_data(uid)
|
43
|
+
data_raw = @imap.uid_fetch(uid.to_i, REQUESTED_ATTRIBUTES)
|
44
|
+
if data_raw
|
45
|
+
if data_raw[0]
|
46
|
+
if data_raw[0].attr
|
47
|
+
output = data_raw[0].attr
|
48
|
+
return output
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def get_file_path_prefix(folder_name, message_config)
|
55
|
+
if $config[:storage_file_name_uid_padding]
|
56
|
+
if $config[:storage_file_name_uid_padding_length] <= 1
|
57
|
+
storage_file_name_uid_padding = 2
|
58
|
+
else
|
59
|
+
storage_file_name_uid_padding = $config[:storage_file_name_uid_padding_length]
|
60
|
+
end
|
61
|
+
uid_based_prefix = message_config['uid'] + (10 ** (storage_file_name_uid_padding - 1))
|
62
|
+
else
|
63
|
+
uid_based_prefix = message_config['uid']
|
64
|
+
end
|
65
|
+
hash_length = $config[:storage_file_name_hash_length]
|
66
|
+
dir_path = @dir_base_path + '/' + folder_name
|
67
|
+
file_name_hash = Digest::MD5.hexdigest(Digest::MD5.hexdigest(YAML.dump(message_config['envelope']))+'-'+Digest::MD5.hexdigest(YAML.dump(message_config['flags'])))[0...hash_length]
|
68
|
+
output = dir_path+'/'+uid_based_prefix.to_s+'-'+file_name_hash
|
69
|
+
return output
|
70
|
+
end
|
71
|
+
|
72
|
+
def get_file_path_for_message(folder_name, message_config)
|
73
|
+
output = get_file_path_prefix(folder_name, message_config)+'.'+$config[:storage_file_extension]
|
74
|
+
return output
|
75
|
+
end
|
76
|
+
|
77
|
+
def get_uids()
|
78
|
+
uid_list = @imap.uid_search(["ALL"]).sort
|
79
|
+
return uid_list
|
80
|
+
end
|
81
|
+
|
82
|
+
def generate_metadata_for_account()
|
83
|
+
@account_metadata = {}
|
84
|
+
@account_metadata[:account] = {}
|
85
|
+
@account_metadata[:account]['directories'] = {}
|
86
|
+
@account_metadata[:account]['directories']['server'] = get_server_dir_list()
|
87
|
+
@account_metadata[:account]['directories']['local'] = get_local_dir_list()
|
88
|
+
@account_metadata[:account]['messages'] = {}
|
89
|
+
@account_metadata[:account]['messages']['server'] = {}
|
90
|
+
@account_metadata[:account]['messages']['server']['folders'] = get_server_message_folder_list()
|
91
|
+
@account_metadata[:task] = {}
|
92
|
+
@account_metadata[:task] = {}
|
93
|
+
@account_metadata[:folders] = {}
|
94
|
+
end
|
95
|
+
|
96
|
+
def generate_metadata_for_folder(folder_name)
|
97
|
+
@account_metadata[:folders][folder_name] = {}
|
98
|
+
@account_metadata[:folders][folder_name]['messages'] = {}
|
99
|
+
@account_metadata[:folders][folder_name]['messages']['server'] = {}
|
100
|
+
@account_metadata[:folders][folder_name]['messages']['server']['uids'] = get_uids()
|
101
|
+
@account_metadata[:folders][folder_name]['messages']['server']['metadata'] = get_imap_metadata(@account_metadata[:folders][folder_name]['messages']['server']['uids'])
|
102
|
+
@account_metadata[:folders][folder_name]['messages']['server']['config'] = update_message_config(folder_name, @account_metadata[:folders][folder_name]['messages']['server']['metadata'])
|
103
|
+
@account_metadata[:folders][folder_name]['files'] = {}
|
104
|
+
@account_metadata[:folders][folder_name]['files']['server'] = get_server_file_list(folder_name, @account_metadata[:folders][folder_name]['messages']['server']['config'])
|
105
|
+
@account_metadata[:folders][folder_name]['files']['local'] = get_local_file_list(folder_name)
|
106
|
+
@account_metadata[:folders][folder_name]['files']['deletes'] = get_delete_list(folder_name)
|
107
|
+
@account_metadata[:folders][folder_name]['process'] = {}
|
108
|
+
end
|
109
|
+
|
110
|
+
def update_message_config(folder_name, metadata)
|
111
|
+
output = {}
|
112
|
+
metadata.each do |message_metadata|
|
113
|
+
uid = message_metadata.attr['UID']
|
114
|
+
output[uid] = {}
|
115
|
+
output[uid]['uid'] = uid
|
116
|
+
output[uid]['envelope'] = message_metadata.attr['ENVELOPE'].to_h
|
117
|
+
output[uid]['flags'] = message_metadata.attr['FLAGS']
|
118
|
+
output[uid]['size'] = message_metadata.attr['RFC822.SIZE']
|
119
|
+
output[uid]['file_path'] = get_file_path_for_message(folder_name, output[uid])
|
120
|
+
end
|
121
|
+
return output
|
122
|
+
end
|
123
|
+
|
124
|
+
def get_local_file_list(folder_name)
|
125
|
+
output = []
|
126
|
+
dir_path = @dir_base_path + '/' + folder_name
|
127
|
+
if Dir.exists?(dir_path)
|
128
|
+
Dir.chdir(dir_path)
|
129
|
+
file_list = Dir.glob("*."+$config[:storage_file_extension]).sort_by{|f| f.split(" ")[0].to_i }
|
130
|
+
file_list.each do |file_name|
|
131
|
+
if File.file?(file_name)
|
132
|
+
output << dir_path+'/'+file_name
|
133
|
+
end
|
134
|
+
end
|
135
|
+
end
|
136
|
+
return output
|
137
|
+
end
|
138
|
+
|
139
|
+
def get_server_file_list(folder_name, server_message_config)
|
140
|
+
output = {}
|
141
|
+
server_message_config.each do |uid, message_config|
|
142
|
+
file_path = get_file_path_for_message(folder_name, message_config)
|
143
|
+
output[file_path] = uid
|
144
|
+
end
|
145
|
+
return output
|
146
|
+
end
|
147
|
+
|
148
|
+
def get_local_dir_list()
|
149
|
+
output = []
|
150
|
+
if Dir.exists?(@dir_base_path)
|
151
|
+
Dir.chdir(@dir_base_path)
|
152
|
+
file_list = Dir.glob("**/*/")
|
153
|
+
file_list.each do |object_name|
|
154
|
+
if File.directory?(object_name)
|
155
|
+
output << @dir_base_path + '/' + object_name
|
156
|
+
end
|
157
|
+
end
|
158
|
+
end
|
159
|
+
return output
|
160
|
+
end
|
161
|
+
|
162
|
+
def get_server_dir_list()
|
163
|
+
output = []
|
164
|
+
message_folder_list = get_server_message_folder_list()
|
165
|
+
message_folder_list.each do |message_folder|
|
166
|
+
output << @dir_base_path + '/' + message_folder
|
167
|
+
end
|
168
|
+
return output
|
169
|
+
end
|
170
|
+
|
171
|
+
def delete_directories(folder_name)
|
172
|
+
server_dirs = @account_metadata[:folders][folder_name]['directories']['server']
|
173
|
+
@account_metadata[:folders][folder_name]['directories']['local'].each do |local_dir|
|
174
|
+
if !server_dirs[local_dir]
|
175
|
+
$logger.info( folder_name + ' - Deleting Directory: ' + local_dir )
|
176
|
+
end
|
177
|
+
end
|
178
|
+
end
|
179
|
+
|
180
|
+
def get_download_list(folder_name)
|
181
|
+
output = []
|
182
|
+
local_file_list = @account_metadata[:folders][folder_name]['files']['local']
|
183
|
+
@account_metadata[:folders][folder_name]['messages']['server']['config'].each do |uid, message_config|
|
184
|
+
file_path = message_config['file_path']
|
185
|
+
if !local_file_list.include?(file_path)
|
186
|
+
output << uid
|
187
|
+
end
|
188
|
+
end
|
189
|
+
return output
|
190
|
+
end
|
191
|
+
|
192
|
+
def get_delete_list(folder_name)
|
193
|
+
output = []
|
194
|
+
server_files = @account_metadata[:folders][folder_name]['files']['server']
|
195
|
+
@account_metadata[:folders][folder_name]['files']['local'].each do |file_path|
|
196
|
+
if !server_files[file_path]
|
197
|
+
if File.exists?(file_path)
|
198
|
+
output << file_path
|
199
|
+
end
|
200
|
+
end
|
201
|
+
end
|
202
|
+
return output
|
203
|
+
end
|
204
|
+
|
205
|
+
def delete_files(folder_name)
|
206
|
+
@account_metadata[:folders][folder_name]['files']['deletes'].each do |file_path|
|
207
|
+
if File.exists?(file_path)
|
208
|
+
$logger.info( folder_name + ' - Deleting File: ' + file_path )
|
209
|
+
File.delete(file_path)
|
210
|
+
end
|
211
|
+
end
|
212
|
+
end
|
213
|
+
|
214
|
+
def get_ratio_formatted(a, b, delimiter = '/')
|
215
|
+
output = a.to_s + delimiter + b.to_s
|
216
|
+
return output
|
217
|
+
end
|
218
|
+
|
219
|
+
def get_uid_string_formatted(uid)
|
220
|
+
output = 'UID:' + uid.to_s
|
221
|
+
return output
|
222
|
+
end
|
223
|
+
|
224
|
+
def set_thread_state(folder_name, uid, msg_ratio, state)
|
225
|
+
uid_string = get_uid_string_formatted(uid)
|
226
|
+
set_thread_property(folder_name, uid, 'State', state.to_s)
|
227
|
+
log_thread_property(folder_name, uid, msg_ratio, 'State')
|
228
|
+
end
|
229
|
+
|
230
|
+
def set_thread_property(folder_name, uid, property, value)
|
231
|
+
@account_metadata[:folders][folder_name]['process']['threads']['uids'][uid][property] = value
|
232
|
+
end
|
233
|
+
|
234
|
+
def get_thread_property(folder_name, uid, property)
|
235
|
+
if @account_metadata[:folders][folder_name]['process']['threads']['uids'][uid][property]
|
236
|
+
output = @account_metadata[:folders][folder_name]['process']['threads']['uids'][uid][property]
|
237
|
+
return output
|
238
|
+
else
|
239
|
+
return false
|
240
|
+
end
|
241
|
+
end
|
242
|
+
|
243
|
+
def log_thread_property(folder_name, uid, msg_ratio, property_name)
|
244
|
+
log_string = get_formatted_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
|
245
|
+
log_thread_event(folder_name, uid, msg_ratio, 'INFO', '', log_string)
|
246
|
+
end
|
247
|
+
|
248
|
+
def get_formatted_thread_property(folder_name, uid, property_name)
|
249
|
+
### getting property
|
250
|
+
name = property_name
|
251
|
+
value = get_thread_property(folder_name, uid, name) || 'Unable to get property: ' + name
|
252
|
+
#####
|
253
|
+
output = name + ': ' + value.to_s
|
254
|
+
return output
|
255
|
+
end
|
256
|
+
|
257
|
+
def increase_thread_property(folder_name, uid, property)
|
258
|
+
@account_metadata[:folders][folder_name]['process']['threads']['uids'][uid][property] += 1
|
259
|
+
end
|
260
|
+
|
261
|
+
def increase_thread_property(folder_name, uid, property)
|
262
|
+
@account_metadata[:folders][folder_name]['process']['threads']['uids'][uid][property] -= 1
|
263
|
+
end
|
264
|
+
|
265
|
+
def increase_thread_count(folder_name)
|
266
|
+
@account_metadata[:folders][folder_name]['process']['threads']['count'] += 1
|
267
|
+
end
|
268
|
+
|
269
|
+
def decrease_thread_count(folder_name)
|
270
|
+
@account_metadata[:folders][folder_name]['process']['threads']['count'] -= 1
|
271
|
+
end
|
272
|
+
|
273
|
+
def get_thread_id_string(uid, msg_ratio)
|
274
|
+
uid_string = get_uid_string_formatted(uid)
|
275
|
+
output = msg_ratio + ' - ' + uid_string
|
276
|
+
end
|
277
|
+
|
278
|
+
def log_thread_on_success(folder_name, uid, msg_ratio)
|
279
|
+
### getting property
|
280
|
+
property_name = 'Queue Time'
|
281
|
+
queue_time = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
|
282
|
+
#####
|
283
|
+
### getting property
|
284
|
+
property_name = 'End Time'
|
285
|
+
end_time = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
|
286
|
+
#####
|
287
|
+
### getting property
|
288
|
+
property_name = 'Server Size'
|
289
|
+
server_size = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
|
290
|
+
#####
|
291
|
+
### getting property
|
292
|
+
property_name = 'Local Size'
|
293
|
+
local_size = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
|
294
|
+
#####
|
295
|
+
### getting data
|
296
|
+
duration_formatted = get_duration_formatted(queue_time, end_time) || 'Unable to get data'
|
297
|
+
#####
|
298
|
+
### getting data
|
299
|
+
size_ratio_formatted = get_ratio_formatted(local_size, server_size, ' / ') || 'Unable to get data'
|
300
|
+
size_formatted = size_ratio_formatted + ' Bytes'
|
301
|
+
#####
|
302
|
+
log_string = 'Duration: ' + duration_formatted + ' - Size: ' + size_formatted
|
303
|
+
log_thread_event(folder_name, uid, msg_ratio, 'INFO', '', log_string)
|
304
|
+
end
|
305
|
+
|
306
|
+
def log_thread_on_failure(folder_name, uid, msg_ratio, event_data)
|
307
|
+
### getting property
|
308
|
+
property_name = 'Queue Time'
|
309
|
+
queue_time = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
|
310
|
+
#####
|
311
|
+
### getting property
|
312
|
+
property_name = 'End Time'
|
313
|
+
end_time = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
|
314
|
+
#####
|
315
|
+
### getting property
|
316
|
+
property_name = 'Server Size'
|
317
|
+
server_size = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
|
318
|
+
#####
|
319
|
+
### getting property
|
320
|
+
property_name = 'Local Size'
|
321
|
+
local_size = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
|
322
|
+
#####
|
323
|
+
### getting data
|
324
|
+
duration_formatted = get_duration_formatted(queue_time, end_time) || 'Unable to get data'
|
325
|
+
#####
|
326
|
+
### getting data
|
327
|
+
size_formatted = server_size + ' Bytes'
|
328
|
+
#####
|
329
|
+
log_string = 'Duration: ' + duration_formatted + ' - Size: ' + size_formatted
|
330
|
+
log_thread_event(folder_name, uid, msg_ratio, 'ERROR', 'E', log_string)
|
331
|
+
log_thread_event(folder_name, uid, msg_ratio, 'ERROR', 'ERROR', event_data)
|
332
|
+
end
|
333
|
+
|
334
|
+
def log_thread_on_retry(folder_name, uid, msg_ratio, event_data)
|
335
|
+
log_thread_event(folder_name, uid, msg_ratio, 'WARN', 'W', event_data)
|
336
|
+
end
|
337
|
+
|
338
|
+
def log_thread_event(folder_name, uid, msg_ratio, event_level, event_tag='', event_data)
|
339
|
+
uid_string = get_uid_string_formatted(uid) || 'Unable to get UID string'
|
340
|
+
thread_id_string = get_thread_id_string(uid, msg_ratio) || 'Unable to get thread ID string'
|
341
|
+
log_string = thread_id_string + ' - ' + event_data
|
342
|
+
log_folder_event(folder_name, event_level, event_tag, log_string)
|
343
|
+
end
|
344
|
+
|
345
|
+
def log_folder_event(folder_name, event_level, event_tag, event_data)
|
346
|
+
log_string = folder_name + ' - ' + event_data
|
347
|
+
$logger.event(event_level, event_tag, log_string)
|
348
|
+
end
|
349
|
+
|
350
|
+
def get_message_config(folder_name, uid)
|
351
|
+
if @account_metadata[:folders][folder_name]['messages']['server']['config'][uid]
|
352
|
+
output = @account_metadata[:folders][folder_name]['messages']['server']['config'][uid]
|
353
|
+
return output
|
354
|
+
else
|
355
|
+
raise '***** ERROR ***** - Message Config: Cannot get message config for folder: ' + folder_name + ' - UID:' + uid
|
356
|
+
end
|
357
|
+
end
|
358
|
+
|
359
|
+
def bootstrap_thread(folder_name, uid)
|
360
|
+
message_config = get_message_config(folder_name, uid)
|
361
|
+
set_thread_property(folder_name, uid, 'File Name', File.basename(message_config['file_path']))
|
362
|
+
set_thread_property(folder_name, uid, 'File Path', message_config['file_path'])
|
363
|
+
set_thread_property(folder_name, uid, 'Server Size', message_config['size'])
|
364
|
+
set_thread_property(folder_name, uid, 'Retry Count', 0)
|
365
|
+
end
|
366
|
+
|
367
|
+
def retry_check(folder_name, uid, msg_ratio)
|
368
|
+
if get_thread_property(folder_name, uid, 'Retry Count') <= 3
|
369
|
+
log_thread_on_retry(folder_name, uid, msg_ratio, 'Queuing for a retry')
|
370
|
+
increase_thread_property(folder_name, uid, 'Retry Count')
|
371
|
+
set_thread_state(folder_name, uid, msg_ratio, 'Retrying')
|
372
|
+
queue_thread(folder_name, uid, msg_ratio)
|
373
|
+
return true
|
374
|
+
else
|
375
|
+
set_thread_property(folder_name, uid, 'End Time', Time.now)
|
376
|
+
set_thread_state(folder_name, uid, msg_ratio, 'Failed')
|
377
|
+
return false
|
378
|
+
end
|
379
|
+
end
|
380
|
+
|
381
|
+
def queue_thread(folder_name, uid, msg_ratio)
|
382
|
+
set_thread_state(folder_name, uid, msg_ratio, 'Queued')
|
383
|
+
bootstrap_thread(folder_name, uid)
|
384
|
+
set_thread_property(folder_name, uid, 'Queue Time', Time.now)
|
385
|
+
increase_thread_count(folder_name)
|
386
|
+
Thread.abort_on_exception = true
|
387
|
+
@account_metadata[:folders][folder_name]['process']['threads']['uids'][uid]['Thread'] = Thread.new do
|
388
|
+
sleep 0.1
|
389
|
+
write_data_for_uid(folder_name, uid, msg_ratio)
|
390
|
+
end
|
391
|
+
end
|
392
|
+
|
393
|
+
def write_data_for_uid(folder_name, uid, msg_ratio)
|
394
|
+
uid_string = get_uid_string_formatted(uid)
|
395
|
+
if get_thread_property(folder_name, uid, 'State') === 'Queued'
|
396
|
+
set_thread_state(folder_name, uid, msg_ratio, 'Active')
|
397
|
+
log_thread_property(folder_name, uid, msg_ratio, 'File Name')
|
398
|
+
file_path = get_thread_property(folder_name, uid, 'File Path')
|
399
|
+
file_path_tmp = file_path + '.tmp'
|
400
|
+
if message_data = get_message_data(uid)
|
401
|
+
File.write(file_path_tmp, Zlib::Deflate.deflate(YAML.dump(message_data)))
|
402
|
+
FileUtils.mv(file_path_tmp, file_path)
|
403
|
+
set_thread_property(folder_name, uid, 'Local Size', File.size(get_thread_property(folder_name, uid, 'File Path')))
|
404
|
+
if get_thread_property(folder_name, uid, 'Local Size') > (get_thread_property(folder_name, uid, 'Server Size') * 3)
|
405
|
+
if !retry_check(folder_name, uid, msg_ratio)
|
406
|
+
log_thread_on_failure(folder_name, uid, msg_ratio, 'File size mismatch after download')
|
407
|
+
end
|
408
|
+
if File.exists?(file_path)
|
409
|
+
### getting property
|
410
|
+
property_name = 'File Name'
|
411
|
+
file_name = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
|
412
|
+
#####
|
413
|
+
log_thread_event(folder_name, uid, msg_ratio, 'WARN', 'W', 'File size mismatch detected, cleaning up')
|
414
|
+
log_thread_event(folder_name, uid, msg_ratio, 'WARN', 'W', 'Deleting File: ' + file_name)
|
415
|
+
File.delete(file_path)
|
416
|
+
end
|
417
|
+
else
|
418
|
+
set_thread_property(folder_name, uid, 'End Time', Time.now)
|
419
|
+
set_thread_state(folder_name, uid, msg_ratio, 'Successful')
|
420
|
+
log_thread_on_success(folder_name, uid, msg_ratio)
|
421
|
+
end
|
422
|
+
else
|
423
|
+
if !retry_check(folder_name, uid, msg_ratio)
|
424
|
+
log_thread_on_failure(folder_name, uid, msg_ratio, 'Unable to get message data')
|
425
|
+
end
|
426
|
+
end
|
427
|
+
decrease_thread_count(folder_name)
|
428
|
+
else
|
429
|
+
### getting property
|
430
|
+
property_name = 'State'
|
431
|
+
formatted_property = get_formatted_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
|
432
|
+
#####
|
433
|
+
log_string = 'Thread state incorrect - ' + formatted_property
|
434
|
+
log_folder_event(folder_name, 'ERROR', 'E', log_string)
|
435
|
+
raise log_string
|
436
|
+
end
|
437
|
+
end
|
438
|
+
|
439
|
+
|
440
|
+
|
441
|
+
def process_thread_data_before_finish(folder_name)
|
442
|
+
has_failed = false
|
443
|
+
@account_metadata[:folders][folder_name]['process']['threads']['uids'].each do |uid, thread_data|
|
444
|
+
if thread_data['State'] != 'Successful'
|
445
|
+
thread_data['State'] = 'Failed'
|
446
|
+
end
|
447
|
+
if thread_data['State'] === 'Failed'
|
448
|
+
kill_thread(folder_name, uid)
|
449
|
+
has_failed = true
|
450
|
+
end
|
451
|
+
end
|
452
|
+
if has_failed
|
453
|
+
sleep 5
|
454
|
+
end
|
455
|
+
end
|
456
|
+
|
457
|
+
def process_shutdown(folder_name)
|
458
|
+
time_started_waiting = Time.now.to_i
|
459
|
+
while @account_metadata[:folders][folder_name]['process']['threads']['count'] > 0
|
460
|
+
time_since_started_waiting = Time.now.to_i - time_started_waiting
|
461
|
+
if time_since_started_waiting >= $config[:thread_timeout]
|
462
|
+
$logger.warn folder_name + ' - *E* - Thread pool shutdown - Timeout: ' + $config[:thread_timeout].to_s + ' Sec(s)'
|
463
|
+
@account_metadata[:folders][folder_name]['process']['threads']['count'] = 0
|
464
|
+
else
|
465
|
+
$logger.info folder_name + ' - Waiting for threads - Remaining: ' + @account_metadata[:folders][folder_name]['process']['threads']['count'].to_s + '/' + $config[:threads].to_s + ' - Timeout: ' + time_since_started_waiting.to_s + '/' + $config[:thread_timeout].to_s + ' Sec(s)'
|
466
|
+
sleep 10
|
467
|
+
end
|
468
|
+
end
|
469
|
+
end
|
470
|
+
|
471
|
+
def log_folder_info(folder_name)
|
472
|
+
$logger.debug folder_name + ' - Server Files: ' + @account_metadata[:folders][folder_name]['files']['server'].count.to_s
|
473
|
+
@account_metadata[:folders][folder_name]['files']['server'].each do |file_path, uid|
|
474
|
+
$logger.debug folder_name + ' - ' + File.basename(file_path)
|
475
|
+
end
|
476
|
+
$logger.debug folder_name + ' - Local Files: ' + @account_metadata[:folders][folder_name]['files']['local'].count.to_s
|
477
|
+
@account_metadata[:folders][folder_name]['files']['local'].each do |file_path|
|
478
|
+
$logger.debug folder_name + ' - ' + File.basename(file_path)
|
479
|
+
end
|
480
|
+
end
|
481
|
+
|
482
|
+
def create_output_directory(folder_name)
|
483
|
+
dir_path = @dir_base_path + '/' + folder_name
|
484
|
+
if !Dir.exists?(dir_path)
|
485
|
+
$logger.debug folder_name + ' - Creating Directory: ' + dir_path
|
486
|
+
FileUtils.mkdir_p dir_path
|
487
|
+
end
|
488
|
+
end
|
489
|
+
|
490
|
+
def process_deletes(folder_name)
|
491
|
+
if $config[:delete_on_sync]
|
492
|
+
if @account_metadata[:folders][folder_name]['files']['deletes'].count > 0
|
493
|
+
$logger.info folder_name + ' - Delete Count: ' + @account_metadata[:folders][folder_name]['files']['deletes'].count.to_s
|
494
|
+
$logger.info folder_name + ' - Executing Deletes'
|
495
|
+
delete_files(folder_name)
|
496
|
+
end
|
497
|
+
end
|
498
|
+
end
|
499
|
+
|
500
|
+
def kill_thread(folder_name, uid)
|
501
|
+
log_folder_event(folder_name, 'WARN', 'ERROR', 'Thread Failure - ' + get_uid_string_formatted(uid))
|
502
|
+
log_folder_event(folder_name, 'WARN', 'ERROR', ' Killing Thread')
|
503
|
+
Thread.kill(get_thread_property(folder_name, uid, 'Thread'))
|
504
|
+
### getting formatted thread property
|
505
|
+
property_name = 'Queue Time'
|
506
|
+
log_string = get_formatted_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
|
507
|
+
#####
|
508
|
+
log_folder_event(folder_name, 'WARN', 'ERROR', ' ' + log_string)
|
509
|
+
### getting formatted thread property
|
510
|
+
property_name = 'End Time'
|
511
|
+
log_string = get_formatted_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
|
512
|
+
#####
|
513
|
+
log_folder_event(folder_name, 'WARN', 'ERROR', ' ' + log_string)
|
514
|
+
$logger.error folder_name + ' - Duration: ' + get_duration_formatted(get_thread_property(folder_name, uid, 'Queue Time'), get_thread_property(folder_name, uid, 'End Time'))
|
515
|
+
### getting formatted thread property
|
516
|
+
property_name = 'File Name'
|
517
|
+
log_string = get_formatted_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
|
518
|
+
#####
|
519
|
+
log_folder_event(folder_name, 'WARN', 'ERROR', ' ' + log_string)
|
520
|
+
end
|
521
|
+
|
522
|
+
def process_downloads(folder_name, download_list)
|
523
|
+
process_healthy = true
|
524
|
+
if download_list.count > 0
|
525
|
+
$logger.info( folder_name + ' - ' + download_list.count.to_s + ' Message(s) to download - Thread Limit: ' + $config[:threads].to_s )
|
526
|
+
message_count = 0
|
527
|
+
### Creating / resetting active thread data structure
|
528
|
+
@account_metadata[:folders][folder_name]['process']['threads'] = {}
|
529
|
+
@account_metadata[:folders][folder_name]['process']['threads']['count'] = 0
|
530
|
+
@account_metadata[:folders][folder_name]['process']['threads']['uids'] = {}
|
531
|
+
message_total = download_list.count
|
532
|
+
download_list.each do |uid|
|
533
|
+
if process_healthy
|
534
|
+
message_count += 1
|
535
|
+
uid_string = get_uid_string_formatted(uid)
|
536
|
+
msg_ratio = get_ratio_formatted(message_count, message_total)
|
537
|
+
last_queue_time = Time.now
|
538
|
+
if @account_metadata[:folders][folder_name]['process']['threads']['count'] >= $config[:threads]
|
539
|
+
$logger.debug folder_name + ' - Thread Count: ' + @account_metadata[:folders][folder_name]['process']['threads']['count'].to_s + ' - Waiting...'
|
540
|
+
end
|
541
|
+
while @account_metadata[:folders][folder_name]['process']['threads']['count'] >= $config[:threads] && process_healthy
|
542
|
+
if (Time.now.to_i - last_queue_time.to_i) >= ($config[:thread_timeout] * $config[:timeout_warning_ratio])
|
543
|
+
if (Time.now.to_i - last_queue_time.to_i) >= $config[:thread_timeout]
|
544
|
+
$logger.error folder_name + ' - *E* Thread queuing has stopped - Timeout: ' + $config[:thread_timeout].to_s + ' Sec(s)'
|
545
|
+
process_healthy = false
|
546
|
+
else
|
547
|
+
$logger.warn folder_name + ' - *W* Thread queuing has stopped - Timeout: ' + (Time.now.to_i - last_queue_time.to_i).to_s + '/' + $config[:thread_timeout].to_s + ' Sec(s)'
|
548
|
+
sleep rand(10..20)
|
549
|
+
end
|
550
|
+
else
|
551
|
+
sleep 0.2
|
552
|
+
end
|
553
|
+
end
|
554
|
+
if process_healthy
|
555
|
+
@account_metadata[:folders][folder_name]['process']['threads']['uids'][uid] = {}
|
556
|
+
queue_thread(folder_name, uid, msg_ratio)
|
557
|
+
sleep 0.1
|
558
|
+
while get_thread_property(folder_name, uid, 'State') === 'Queued'
|
559
|
+
if (Time.now.to_i - get_thread_property(folder_name, uid, 'Queue Time').to_i) >= ($config[:thread_timeout] * $config[:timeout_warning_ratio])
|
560
|
+
if (Time.now.to_i - get_thread_property(folder_name, uid, 'Queue Time').to_i) >= $config[:thread_timeout]
|
561
|
+
$logger.error folder_name + ' - ' + msg_ratio + ' - ' + uid_string + ' - *E* Thread state - Moving on... - Timeout: ' + $config[:thread_timeout].to_s + ' Sec(s)'
|
562
|
+
set_thread_state(folder_name, uid, msg_ratio, 'Failed')
|
563
|
+
kill_thread(folder_name, uid)
|
564
|
+
else
|
565
|
+
$logger.warn folder_name + ' - ' + msg_ratio + ' - ' + uid_string + ' - *W* Thread state - Timeout: ' + (Time.now.to_i - get_thread_property(folder_name, uid, 'Queue Time').to_i).to_s + '/' + $config[:thread_timeout].to_s + ' Sec(s)'
|
566
|
+
sleep rand(10..20)
|
567
|
+
end
|
568
|
+
else
|
569
|
+
sleep 0.2
|
570
|
+
end
|
571
|
+
end
|
572
|
+
end
|
573
|
+
end
|
574
|
+
end
|
575
|
+
if process_healthy
|
576
|
+
sleep 2
|
577
|
+
process_shutdown(folder_name)
|
578
|
+
end
|
579
|
+
process_thread_data_before_finish(folder_name)
|
580
|
+
end
|
581
|
+
end
|
582
|
+
|
583
|
+
def write_data_for_folder(folder_name)
|
584
|
+
$logger.debug ''
|
585
|
+
$logger.info 'Checking ' + folder_name
|
586
|
+
@imap.examine(folder_name)
|
587
|
+
generate_metadata_for_folder(folder_name)
|
588
|
+
log_folder_info(folder_name)
|
589
|
+
download_list = get_download_list(folder_name)
|
590
|
+
create_output_directory(folder_name)
|
591
|
+
process_deletes(folder_name)
|
592
|
+
process_downloads(folder_name, download_list)
|
593
|
+
end
|
594
|
+
|
595
|
+
def get_data_for_all_folders
|
596
|
+
folder_list = @account_metadata[:account]['messages']['server']['folders']
|
597
|
+
$logger.debug ''
|
598
|
+
$logger.debug 'Folder Count: ' + folder_list.count.to_s
|
599
|
+
folder_list.each do |folder_name|
|
600
|
+
if check_folder_included(folder_name)
|
601
|
+
$logger.debug ' ' + folder_name
|
602
|
+
else
|
603
|
+
$logger.debug ' ' + folder_name + ' - SKIPPED'
|
604
|
+
end
|
605
|
+
end
|
606
|
+
folder_list.each do |folder_name|
|
607
|
+
if check_folder_included(folder_name)
|
608
|
+
write_data_for_folder(folder_name)
|
609
|
+
end
|
610
|
+
end
|
611
|
+
$logger.debug ''
|
612
|
+
end
|
613
|
+
|
614
|
+
def check_folder_included(folder_name)
|
615
|
+
ignore_list = []
|
616
|
+
ignore_list << "[Gmail]"
|
617
|
+
ignore_list << "[Gmail]/All Mail"
|
618
|
+
ignore_list << "[Gmail]/Important"
|
619
|
+
ignore_list << "[Gmail]/Starred"
|
620
|
+
if ignore_list.include?(folder_name)
|
621
|
+
return false
|
622
|
+
else
|
623
|
+
return true
|
624
|
+
end
|
625
|
+
end
|
626
|
+
|
627
|
+
def execute
|
628
|
+
@connection = At_email::Account::ImapConnection.new()
|
629
|
+
@connection.login
|
630
|
+
get_imap
|
631
|
+
generate_metadata_for_account
|
632
|
+
@account_metadata[:task][:start_time] = Time.now
|
633
|
+
get_data_for_all_folders
|
634
|
+
@connection.disconnect
|
635
|
+
@account_metadata[:task][:end_time] = Time.now
|
636
|
+
task_report
|
637
|
+
end
|
638
|
+
|
639
|
+
def task_report
|
640
|
+
stats = get_task_stats()
|
641
|
+
$logger.info ''
|
642
|
+
$logger.info '##### Task Report #####'
|
643
|
+
$logger.info ''
|
644
|
+
$logger.info ' Task: ' + $config[:task]
|
645
|
+
$logger.info ''
|
646
|
+
$logger.info ' Server: ' + $config[:server]
|
647
|
+
$logger.info ' Username: ' + $config[:username]
|
648
|
+
$logger.info ' Output Directory: ' + @dir_base_path
|
649
|
+
$logger.info ''
|
650
|
+
$logger.info ' Start Time: ' + @account_metadata[:task][:start_time].to_s
|
651
|
+
$logger.info ' End Time: ' + @account_metadata[:task][:end_time].to_s
|
652
|
+
$logger.info ' Duration: ' + get_duration_formatted(@account_metadata[:task][:start_time], @account_metadata[:task][:end_time])
|
653
|
+
$logger.info ''
|
654
|
+
$logger.info ' Folders Processed: ' + @account_metadata[:folders].count.to_s
|
655
|
+
$logger.info ''
|
656
|
+
@account_metadata[:folders].each do |folder_name, folder_data|
|
657
|
+
$logger.info ' ' + folder_name
|
658
|
+
$logger.info ''
|
659
|
+
$logger.info ' Message Stats'
|
660
|
+
$logger.info ' Total: ' + folder_data['messages']['server']['config'].count.to_s
|
661
|
+
$logger.info ' Total Size: ' + stats[:folders][folder_name]['messages']['size'].to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse + ' bytes'
|
662
|
+
$logger.info ''
|
663
|
+
$logger.info ' Thread Stats'
|
664
|
+
$logger.info ' Total: ' + stats[:folders][folder_name]['threads']['Total'].to_s
|
665
|
+
$logger.info ' Success: ' + stats[:folders][folder_name]['threads']['Success'].to_s
|
666
|
+
$logger.info ' Failed: ' + stats[:folders][folder_name]['threads']['Failed'].to_s
|
667
|
+
if folder_data['process']
|
668
|
+
if folder_data['process']['threads']
|
669
|
+
if folder_data['process']['threads']['uids']
|
670
|
+
$logger.info ' Threads'
|
671
|
+
folder_data['process']['threads']['uids'].each do |uid, thread_data|
|
672
|
+
if thread_data['State'] === 'Successful'
|
673
|
+
$logger.debug ' UID: ' + uid.to_s
|
674
|
+
$logger.debug ' Path: ' + folder_data['messages']['server']['config'][uid]['file_path']
|
675
|
+
$logger.debug ' Thread State: ' + thread_data['State']
|
676
|
+
else
|
677
|
+
$logger.info ' UID: ' + uid.to_s
|
678
|
+
$logger.info ' Path: ' + message_config['file_path']
|
679
|
+
$logger.info ' Thread State: ' + thread_data['State']
|
680
|
+
end
|
681
|
+
end
|
682
|
+
end
|
683
|
+
end
|
684
|
+
end
|
685
|
+
$logger.info ''
|
686
|
+
end
|
687
|
+
$logger.info ''
|
688
|
+
end
|
689
|
+
|
690
|
+
def get_task_stats()
|
691
|
+
output = {}
|
692
|
+
output[:account] = {}
|
693
|
+
output[:folders] = {}
|
694
|
+
@account_metadata[:folders].each do |folder_name, folder_data|
|
695
|
+
output[:folders][folder_name] = {}
|
696
|
+
output[:folders][folder_name]['messages'] = {}
|
697
|
+
output[:folders][folder_name]['messages']['size'] = 0
|
698
|
+
folder_data['messages']['server']['config'].each do |uid, message_config|
|
699
|
+
output[:folders][folder_name]['messages']['size'] += message_config['size']
|
700
|
+
end
|
701
|
+
output[:folders][folder_name]['threads'] = {}
|
702
|
+
output[:folders][folder_name]['threads']['Total'] = 0
|
703
|
+
output[:folders][folder_name]['threads']['Success'] = 0
|
704
|
+
output[:folders][folder_name]['threads']['Failed'] = 0
|
705
|
+
if folder_data['process']
|
706
|
+
if folder_data['process']['threads']
|
707
|
+
if folder_data['process']['threads']['uids']
|
708
|
+
folder_data['process']['threads']['uids'].each do |uid, thread_data|
|
709
|
+
output[:folders][folder_name]['threads']['Total'] += 1
|
710
|
+
if thread_data['State'] === 'Successful'
|
711
|
+
output[:folders][folder_name]['threads']['Success'] += 1
|
712
|
+
else
|
713
|
+
output[:folders][folder_name]['threads']['Failed'] += 1
|
714
|
+
end
|
715
|
+
end
|
716
|
+
end
|
717
|
+
end
|
718
|
+
end
|
719
|
+
end
|
720
|
+
return output
|
721
|
+
end
|
722
|
+
|
723
|
+
def get_duration(start_time, end_time)
|
724
|
+
output = end_time.to_f - start_time.to_f
|
725
|
+
return output
|
726
|
+
end
|
727
|
+
|
728
|
+
def get_duration_formatted(start_time, end_time)
|
729
|
+
duration_seconds = get_duration(start_time, end_time)
|
730
|
+
if duration_seconds
|
731
|
+
if duration_seconds <= 60
|
732
|
+
output = sprintf("%.6f", duration_seconds) + ' Sec(s)'
|
733
|
+
else
|
734
|
+
nanoseconds = (sprintf("%.6f", duration_seconds).to_f * 1000000) % 1000000
|
735
|
+
seconds = duration_seconds % 60
|
736
|
+
minutes = (duration_seconds / 60) % 60
|
737
|
+
hours = duration_seconds / (60 * 60)
|
738
|
+
output = format("%02d:%02d:%02d.%6d", hours, minutes, seconds, nanoseconds)
|
739
|
+
end
|
740
|
+
else
|
741
|
+
return false
|
742
|
+
end
|
743
|
+
return output
|
744
|
+
end
|
745
|
+
|
746
|
+
def format_file_size(file_size)
|
747
|
+
output = format_number(file_size) + ' bytes'
|
748
|
+
return output
|
749
|
+
end
|
750
|
+
|
751
|
+
def format_number(number)
|
752
|
+
output = number.to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse
|
753
|
+
return output
|
754
|
+
end
|
755
|
+
|
756
|
+
end
|