at_email 0.0.1

data/lib/at_email/core.rb

@@ -0,0 +1,3 @@
+module At_email::Core ; end
+
+require "at_email/core/default"

data/lib/at_email/core/default.rb

@@ -0,0 +1,62 @@
+
+class At_email::Core::Default
+
+  CLASS_DISPLAY_NAME = 'atEmail - Default Class'
+
+  attr_accessor :properties
+
+  def initialize()
+    @properties = {}
+    @properties['State'] = 'New'
+    @properties['ID'] = At_email::Formatting.new.get_random_id(5)
+    @properties['Display Name'] = CLASS_DISPLAY_NAME
+    @properties['Log Prefix'] = @properties['Display Name'] + ': ' + @properties['ID']
+    @properties['State'] = 'Initialized'
+  end
+
+  def get(property)
+    if !@properties[property]
+      log_event event_tag='ERROR', 'Cannot get property: ' + property + "\n" + __LINE__ + "\n" + (pp caller())
+      return false
+    end
+    return @properties[property]
+  end
+
+  def get_formatted(property)
+    value = get(property)
+    output = property + ': ' + value.to_s
+    return output
+  end
+
+  def get_state()
+    return get('State')
+  end
+
+  def set_state(value)
+    set('State', value)
+  end
+
+  ### start of private methods
+  private
+
+  def log_event(event_tag='', event_data)
+    log_prefix = get['Log Prefix']
+    log_string = log_prefix + ' - ' + event_data
+    $logger.event(event_tag, event_data)
+  end
+
+  def set(property, value)
+    if @properties[property]
+      @properties[property] = value
+    end
+  end
+
+  def create_new_property(property, value)
+    if @properties[property]
+      return false
+    else
+      @properties[property] = value
+    end
+  end
+
+end

data/lib/at_email/tasks.rb

@@ -0,0 +1,3 @@
+module At_email::Tasks ; end
+
+require "at_email/tasks/imap_to_fs"

data/lib/at_email/tasks/imap_to_fs.rb

@@ -0,0 +1,756 @@
+
+require 'net/imap'
+require 'zlib'
+require 'json'
+require 'digest'
+require 'thread'
+require 'yaml'
+require 'fileutils'
+
+class At_email::Tasks::Imap_To_Fs
+
+  METADATA_ATTRIBUTES = [ 'ENVELOPE', 'FLAGS', 'RFC822.SIZE' ].freeze
+  REQUESTED_ATTRIBUTES = [ 'INTERNALDATE', 'RFC822', 'RFC822.SIZE' ].freeze
+
+  def initialize
+    @dir_base_path = $config[:output_base_dir] + '/' + $config[:account_id]
+  end
+
+  def get_imap
+    @imap = @connection.imap
+  end
+
+  def get_server_message_folder_list()
+    output = []
+    root_path = @imap.list("", "")[0].name
+    folder_data = @imap.list(root_path, "*")
+    output = []
+    folder_data.map do |this_folder_data|
+      output << this_folder_data.name
+    end
+    return output
+  end
+
+  def get_imap_metadata(uid_list)
+    output = {}
+    if uid_list.count > 0
+      output = @imap.uid_fetch(uid_list, METADATA_ATTRIBUTES)
+    end
+    return output
+  end
+
+  def get_message_data(uid)
+    data_raw = @imap.uid_fetch(uid.to_i, REQUESTED_ATTRIBUTES)
+    if data_raw
+      if data_raw[0]
+        if data_raw[0].attr
+          output = data_raw[0].attr
+          return output
+        end
+      end
+    end
+  end
+
+  def get_file_path_prefix(folder_name, message_config)
+    if $config[:storage_file_name_uid_padding]
+      if $config[:storage_file_name_uid_padding_length] <= 1
+        storage_file_name_uid_padding = 2
+      else
+        storage_file_name_uid_padding = $config[:storage_file_name_uid_padding_length]
+      end
+      uid_based_prefix = message_config['uid'] + (10 ** (storage_file_name_uid_padding - 1))
+    else
+      uid_based_prefix = message_config['uid']
+    end
+    hash_length = $config[:storage_file_name_hash_length]
+    dir_path = @dir_base_path + '/' + folder_name
+    file_name_hash = Digest::MD5.hexdigest(Digest::MD5.hexdigest(YAML.dump(message_config['envelope']))+'-'+Digest::MD5.hexdigest(YAML.dump(message_config['flags'])))[0...hash_length]
+    output = dir_path+'/'+uid_based_prefix.to_s+'-'+file_name_hash
+    return output
+  end
+
+  def get_file_path_for_message(folder_name, message_config)
+    output = get_file_path_prefix(folder_name, message_config)+'.'+$config[:storage_file_extension]
+    return output
+  end
+
+  def get_uids()
+    uid_list = @imap.uid_search(["ALL"]).sort
+    return uid_list
+  end
+
+  def generate_metadata_for_account()
+    @account_metadata = {}
+    @account_metadata[:account] = {}
+    @account_metadata[:account]['directories'] = {}
+    @account_metadata[:account]['directories']['server'] = get_server_dir_list()
+    @account_metadata[:account]['directories']['local'] = get_local_dir_list()
+    @account_metadata[:account]['messages'] = {}
+    @account_metadata[:account]['messages']['server'] = {}
+    @account_metadata[:account]['messages']['server']['folders'] = get_server_message_folder_list()
+    @account_metadata[:task] = {}
+    @account_metadata[:task] = {}
+    @account_metadata[:folders] = {}
+  end
+
+  def generate_metadata_for_folder(folder_name)
+    @account_metadata[:folders][folder_name] = {}
+    @account_metadata[:folders][folder_name]['messages'] = {}
+    @account_metadata[:folders][folder_name]['messages']['server'] = {}
+    @account_metadata[:folders][folder_name]['messages']['server']['uids'] = get_uids()
+    @account_metadata[:folders][folder_name]['messages']['server']['metadata'] = get_imap_metadata(@account_metadata[:folders][folder_name]['messages']['server']['uids'])
+    @account_metadata[:folders][folder_name]['messages']['server']['config'] = update_message_config(folder_name, @account_metadata[:folders][folder_name]['messages']['server']['metadata'])
+    @account_metadata[:folders][folder_name]['files'] = {}
+    @account_metadata[:folders][folder_name]['files']['server'] = get_server_file_list(folder_name, @account_metadata[:folders][folder_name]['messages']['server']['config'])
+    @account_metadata[:folders][folder_name]['files']['local'] = get_local_file_list(folder_name)
+    @account_metadata[:folders][folder_name]['files']['deletes'] = get_delete_list(folder_name)
+    @account_metadata[:folders][folder_name]['process'] = {}
+  end
+
+  def update_message_config(folder_name, metadata)
+    output = {}
+    metadata.each do |message_metadata|
+      uid = message_metadata.attr['UID']
+      output[uid] = {}
+      output[uid]['uid'] = uid
+      output[uid]['envelope'] = message_metadata.attr['ENVELOPE'].to_h
+      output[uid]['flags'] = message_metadata.attr['FLAGS']
+      output[uid]['size'] = message_metadata.attr['RFC822.SIZE']
+      output[uid]['file_path'] = get_file_path_for_message(folder_name, output[uid])
+    end
+    return output
+  end
+
+  def get_local_file_list(folder_name)
+    output = []
+    dir_path = @dir_base_path + '/' + folder_name
+    if Dir.exists?(dir_path)
+      Dir.chdir(dir_path)
+      file_list = Dir.glob("*."+$config[:storage_file_extension]).sort_by{|f| f.split(" ")[0].to_i }
+      file_list.each do |file_name|
+        if File.file?(file_name)
+          output << dir_path+'/'+file_name
+        end
+      end
+    end
+    return output
+  end
+
+  def get_server_file_list(folder_name, server_message_config)
+    output = {}
+    server_message_config.each do |uid, message_config|
+      file_path = get_file_path_for_message(folder_name, message_config)
+      output[file_path] = uid
+    end
+    return output
+  end
+
+  def get_local_dir_list()
+    output = []
+    if Dir.exists?(@dir_base_path)
+      Dir.chdir(@dir_base_path)
+      file_list = Dir.glob("**/*/")
+      file_list.each do |object_name|
+        if File.directory?(object_name)
+          output << @dir_base_path + '/' + object_name
+        end
+      end
+    end
+    return output
+  end
+
+  def get_server_dir_list()
+    output = []
+    message_folder_list = get_server_message_folder_list()
+    message_folder_list.each do |message_folder|
+      output << @dir_base_path + '/' + message_folder
+    end
+    return output
+  end
+
+  def delete_directories(folder_name)
+    server_dirs = @account_metadata[:folders][folder_name]['directories']['server']
+    @account_metadata[:folders][folder_name]['directories']['local'].each do |local_dir|
+      if !server_dirs[local_dir]
+        $logger.info( folder_name + ' - Deleting Directory: ' + local_dir )
+      end
+    end
+  end
+
+  def get_download_list(folder_name)
+    output = []
+    local_file_list = @account_metadata[:folders][folder_name]['files']['local']
+    @account_metadata[:folders][folder_name]['messages']['server']['config'].each do |uid, message_config|
+      file_path = message_config['file_path']
+      if !local_file_list.include?(file_path)
+        output << uid
+      end
+    end
+    return output
+  end
+
+  def get_delete_list(folder_name)
+    output = []
+    server_files = @account_metadata[:folders][folder_name]['files']['server']
+    @account_metadata[:folders][folder_name]['files']['local'].each do |file_path|
+      if !server_files[file_path]
+        if File.exists?(file_path)
+          output << file_path
+        end
+      end
+    end
+    return output
+  end
+
+  def delete_files(folder_name)
+    @account_metadata[:folders][folder_name]['files']['deletes'].each do |file_path|
+      if File.exists?(file_path)
+        $logger.info( folder_name + ' -  Deleting File: ' + file_path )
+        File.delete(file_path)
+      end
+    end
+  end
+
+  def get_ratio_formatted(a, b, delimiter = '/')
+    output = a.to_s + delimiter + b.to_s
+    return output
+  end
+
+  def get_uid_string_formatted(uid)
+    output = 'UID:' + uid.to_s
+    return output
+  end
+
+  def set_thread_state(folder_name, uid, msg_ratio, state)
+    uid_string = get_uid_string_formatted(uid)
+    set_thread_property(folder_name, uid, 'State', state.to_s)
+    log_thread_property(folder_name, uid, msg_ratio, 'State')
+  end
+
+  def set_thread_property(folder_name, uid, property, value)
+    @account_metadata[:folders][folder_name]['process']['threads']['uids'][uid][property] = value
+  end
+
+  def get_thread_property(folder_name, uid, property)
+    if @account_metadata[:folders][folder_name]['process']['threads']['uids'][uid][property]
+      output = @account_metadata[:folders][folder_name]['process']['threads']['uids'][uid][property]
+      return output
+    else
+      return false
+    end
+  end
+
+  def log_thread_property(folder_name, uid, msg_ratio, property_name)
+    log_string = get_formatted_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
+    log_thread_event(folder_name, uid, msg_ratio, 'INFO', '', log_string)
+  end
+
+  def get_formatted_thread_property(folder_name, uid, property_name)
+    ### getting property
+    name = property_name
+    value = get_thread_property(folder_name, uid, name) || 'Unable to get property: ' + name
+    #####
+    output = name + ': ' + value.to_s
+    return output
+  end
+
+  def increase_thread_property(folder_name, uid, property)
+    @account_metadata[:folders][folder_name]['process']['threads']['uids'][uid][property] += 1
+  end
+
+  def increase_thread_property(folder_name, uid, property)
+    @account_metadata[:folders][folder_name]['process']['threads']['uids'][uid][property] -= 1
+  end
+
+  def increase_thread_count(folder_name)
+    @account_metadata[:folders][folder_name]['process']['threads']['count'] += 1
+  end
+
+  def decrease_thread_count(folder_name)
+    @account_metadata[:folders][folder_name]['process']['threads']['count'] -= 1
+  end
+
+  def get_thread_id_string(uid, msg_ratio)
+    uid_string = get_uid_string_formatted(uid)
+    output = msg_ratio + ' - ' + uid_string
+  end
+
+  def log_thread_on_success(folder_name, uid, msg_ratio)
+    ### getting property
+    property_name = 'Queue Time'
+    queue_time = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
+    #####
+    ### getting property
+    property_name = 'End Time'
+    end_time = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
+    #####
+    ### getting property
+    property_name = 'Server Size'
+    server_size = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
+    #####
+    ### getting property
+    property_name = 'Local Size'
+    local_size = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
+    #####
+    ### getting data
+    duration_formatted = get_duration_formatted(queue_time, end_time) || 'Unable to get data'
+    #####
+    ### getting data
+    size_ratio_formatted = get_ratio_formatted(local_size, server_size, ' / ') || 'Unable to get data'
+    size_formatted = size_ratio_formatted + ' Bytes'
+    #####
+    log_string = 'Duration: ' + duration_formatted + ' -  Size: ' + size_formatted
+    log_thread_event(folder_name, uid, msg_ratio, 'INFO', '', log_string)
+  end
+
+  def log_thread_on_failure(folder_name, uid, msg_ratio, event_data)
+    ### getting property
+    property_name = 'Queue Time'
+    queue_time = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
+    #####
+    ### getting property
+    property_name = 'End Time'
+    end_time = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
+    #####
+    ### getting property
+    property_name = 'Server Size'
+    server_size = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
+    #####
+    ### getting property
+    property_name = 'Local Size'
+    local_size = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
+    #####
+    ### getting data
+    duration_formatted = get_duration_formatted(queue_time, end_time) || 'Unable to get data'
+    #####
+    ### getting data
+    size_formatted = server_size + ' Bytes'
+    #####
+    log_string = 'Duration: ' + duration_formatted + ' -  Size: ' + size_formatted
+    log_thread_event(folder_name, uid, msg_ratio, 'ERROR', 'E', log_string)
+    log_thread_event(folder_name, uid, msg_ratio, 'ERROR', 'ERROR', event_data)
+  end
+
+  def log_thread_on_retry(folder_name, uid, msg_ratio, event_data)
+    log_thread_event(folder_name, uid, msg_ratio, 'WARN', 'W', event_data)
+  end
+
+  def log_thread_event(folder_name, uid, msg_ratio, event_level, event_tag='', event_data)
+    uid_string = get_uid_string_formatted(uid) || 'Unable to get UID string'
+    thread_id_string = get_thread_id_string(uid, msg_ratio) || 'Unable to get thread ID string'
+    log_string = thread_id_string + ' - ' + event_data
+    log_folder_event(folder_name, event_level, event_tag, log_string)
+  end
+
+  def log_folder_event(folder_name, event_level, event_tag, event_data)
+    log_string = folder_name + ' - ' + event_data
+    $logger.event(event_level, event_tag, log_string)
+  end
+
+  def get_message_config(folder_name, uid)
+    if @account_metadata[:folders][folder_name]['messages']['server']['config'][uid]
+      output = @account_metadata[:folders][folder_name]['messages']['server']['config'][uid]
+      return output
+    else
+      raise '***** ERROR ***** - Message Config: Cannot get message config for folder: ' + folder_name + ' - UID:' + uid
+    end
+  end
+
+  def bootstrap_thread(folder_name, uid)
+    message_config = get_message_config(folder_name, uid)
+    set_thread_property(folder_name, uid, 'File Name', File.basename(message_config['file_path']))
+    set_thread_property(folder_name, uid, 'File Path', message_config['file_path'])
+    set_thread_property(folder_name, uid, 'Server Size', message_config['size'])
+    set_thread_property(folder_name, uid, 'Retry Count', 0)
+  end
+
+  def retry_check(folder_name, uid, msg_ratio)
+    if get_thread_property(folder_name, uid, 'Retry Count') <= 3
+      log_thread_on_retry(folder_name, uid, msg_ratio, 'Queuing for a retry')
+      increase_thread_property(folder_name, uid, 'Retry Count')
+      set_thread_state(folder_name, uid, msg_ratio, 'Retrying')
+      queue_thread(folder_name, uid, msg_ratio)
+      return true
+    else
+      set_thread_property(folder_name, uid, 'End Time', Time.now)
+      set_thread_state(folder_name, uid, msg_ratio, 'Failed')
+      return false
+    end
+  end
+
+  def queue_thread(folder_name, uid, msg_ratio)
+    set_thread_state(folder_name, uid, msg_ratio, 'Queued')
+    bootstrap_thread(folder_name, uid)
+    set_thread_property(folder_name, uid, 'Queue Time', Time.now)
+    increase_thread_count(folder_name)
+    Thread.abort_on_exception = true
+    @account_metadata[:folders][folder_name]['process']['threads']['uids'][uid]['Thread'] = Thread.new do
+      sleep 0.1
+      write_data_for_uid(folder_name, uid, msg_ratio)
+    end
+  end
+
+  def write_data_for_uid(folder_name, uid, msg_ratio)
+    uid_string = get_uid_string_formatted(uid)
+    if get_thread_property(folder_name, uid, 'State') === 'Queued'
+      set_thread_state(folder_name, uid, msg_ratio, 'Active')
+      log_thread_property(folder_name, uid, msg_ratio, 'File Name')
+      file_path = get_thread_property(folder_name, uid, 'File Path')
+      file_path_tmp = file_path + '.tmp'
+      if message_data = get_message_data(uid)
+        File.write(file_path_tmp, Zlib::Deflate.deflate(YAML.dump(message_data)))
+        FileUtils.mv(file_path_tmp, file_path)
+        set_thread_property(folder_name, uid, 'Local Size', File.size(get_thread_property(folder_name, uid, 'File Path')))
+        if get_thread_property(folder_name, uid, 'Local Size') > (get_thread_property(folder_name, uid, 'Server Size') * 3)
+          if !retry_check(folder_name, uid, msg_ratio)
+            log_thread_on_failure(folder_name, uid, msg_ratio, 'File size mismatch after download')
+          end
+          if File.exists?(file_path)
+            ### getting property
+            property_name = 'File Name'
+            file_name = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
+            #####
+            log_thread_event(folder_name, uid, msg_ratio, 'WARN', 'W', 'File size mismatch detected, cleaning up')
+            log_thread_event(folder_name, uid, msg_ratio, 'WARN', 'W', 'Deleting File: ' + file_name)
+            File.delete(file_path)
+          end
+        else
+          set_thread_property(folder_name, uid, 'End Time', Time.now)
+          set_thread_state(folder_name, uid, msg_ratio, 'Successful')
+          log_thread_on_success(folder_name, uid, msg_ratio)
+        end
+      else
+        if !retry_check(folder_name, uid, msg_ratio)
+          log_thread_on_failure(folder_name, uid, msg_ratio, 'Unable to get message data')
+        end
+      end
+      decrease_thread_count(folder_name)
+    else
+      ### getting property
+      property_name = 'State'
+      formatted_property = get_formatted_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
+      #####
+      log_string = 'Thread state incorrect - ' + formatted_property
+      log_folder_event(folder_name, 'ERROR', 'E', log_string)
+      raise log_string
+    end
+  end
+
+
+
+  def process_thread_data_before_finish(folder_name)
+    has_failed = false
+    @account_metadata[:folders][folder_name]['process']['threads']['uids'].each do |uid, thread_data|
+      if thread_data['State'] != 'Successful'
+        thread_data['State'] = 'Failed'
+      end
+      if thread_data['State'] === 'Failed'
+        kill_thread(folder_name, uid)
+        has_failed = true
+      end
+    end
+    if has_failed
+      sleep 5
+    end
+  end
+
+  def process_shutdown(folder_name)
+    time_started_waiting = Time.now.to_i
+    while @account_metadata[:folders][folder_name]['process']['threads']['count'] > 0
+      time_since_started_waiting = Time.now.to_i - time_started_waiting
+      if time_since_started_waiting >= $config[:thread_timeout]
+        $logger.warn folder_name + ' - *E* - Thread pool shutdown - Timeout: ' + $config[:thread_timeout].to_s + ' Sec(s)'
+        @account_metadata[:folders][folder_name]['process']['threads']['count'] = 0
+      else
+        $logger.info folder_name + ' - Waiting for threads - Remaining: ' + @account_metadata[:folders][folder_name]['process']['threads']['count'].to_s + '/' + $config[:threads].to_s + ' - Timeout: ' + time_since_started_waiting.to_s + '/' + $config[:thread_timeout].to_s + ' Sec(s)'
+        sleep 10
+      end
+    end
+  end
+
+  def log_folder_info(folder_name)
+    $logger.debug folder_name + ' - Server Files: ' + @account_metadata[:folders][folder_name]['files']['server'].count.to_s
+    @account_metadata[:folders][folder_name]['files']['server'].each do |file_path, uid|
+      $logger.debug folder_name + ' -  ' + File.basename(file_path)
+    end
+    $logger.debug folder_name + ' - Local Files: ' + @account_metadata[:folders][folder_name]['files']['local'].count.to_s
+    @account_metadata[:folders][folder_name]['files']['local'].each do |file_path|
+      $logger.debug folder_name + ' -  ' + File.basename(file_path)
+    end
+  end
+
+  def create_output_directory(folder_name)
+    dir_path = @dir_base_path + '/' + folder_name
+    if !Dir.exists?(dir_path)
+      $logger.debug folder_name + ' - Creating Directory: ' + dir_path
+      FileUtils.mkdir_p dir_path
+    end
+  end
+
+  def process_deletes(folder_name)
+    if $config[:delete_on_sync]
+      if @account_metadata[:folders][folder_name]['files']['deletes'].count > 0
+        $logger.info folder_name + ' - Delete Count: ' + @account_metadata[:folders][folder_name]['files']['deletes'].count.to_s
+        $logger.info folder_name + ' - Executing Deletes'
+        delete_files(folder_name)
+      end
+    end
+  end
+
+  def kill_thread(folder_name, uid)
+    log_folder_event(folder_name, 'WARN', 'ERROR', 'Thread Failure - ' + get_uid_string_formatted(uid))
+    log_folder_event(folder_name, 'WARN', 'ERROR', ' Killing Thread')
+    Thread.kill(get_thread_property(folder_name, uid, 'Thread'))
+    ### getting formatted thread property
+    property_name = 'Queue Time'
+    log_string = get_formatted_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
+    #####
+    log_folder_event(folder_name, 'WARN', 'ERROR', ' ' + log_string)
+    ### getting formatted thread property
+    property_name = 'End Time'
+    log_string = get_formatted_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
+    #####
+    log_folder_event(folder_name, 'WARN', 'ERROR', ' ' + log_string)
+    $logger.error folder_name + ' -  Duration: ' + get_duration_formatted(get_thread_property(folder_name, uid, 'Queue Time'), get_thread_property(folder_name, uid, 'End Time'))
+    ### getting formatted thread property
+    property_name = 'File Name'
+    log_string = get_formatted_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
+    #####
+    log_folder_event(folder_name, 'WARN', 'ERROR', ' ' + log_string)
+  end
+
+  def process_downloads(folder_name, download_list)
+    process_healthy = true
+    if download_list.count > 0
+      $logger.info( folder_name + ' - ' + download_list.count.to_s + ' Message(s) to download - Thread Limit: ' + $config[:threads].to_s )
+      message_count = 0
+      ### Creating / resetting active thread data structure
+      @account_metadata[:folders][folder_name]['process']['threads'] = {}
+      @account_metadata[:folders][folder_name]['process']['threads']['count'] = 0
+      @account_metadata[:folders][folder_name]['process']['threads']['uids'] = {}
+      message_total = download_list.count
+      download_list.each do |uid|
+        if process_healthy
+          message_count += 1
+          uid_string = get_uid_string_formatted(uid)
+          msg_ratio = get_ratio_formatted(message_count, message_total)
+          last_queue_time = Time.now
+          if @account_metadata[:folders][folder_name]['process']['threads']['count'] >= $config[:threads]
+            $logger.debug folder_name + ' - Thread Count: ' + @account_metadata[:folders][folder_name]['process']['threads']['count'].to_s + ' - Waiting...'
+          end
+          while @account_metadata[:folders][folder_name]['process']['threads']['count'] >= $config[:threads] && process_healthy
+            if (Time.now.to_i - last_queue_time.to_i) >= ($config[:thread_timeout] * $config[:timeout_warning_ratio])
+              if (Time.now.to_i - last_queue_time.to_i) >= $config[:thread_timeout]
+                $logger.error folder_name + ' - *E* Thread queuing has stopped - Timeout: ' + $config[:thread_timeout].to_s + ' Sec(s)'
+                process_healthy = false
+              else
+                $logger.warn folder_name + ' - *W* Thread queuing has stopped - Timeout: ' + (Time.now.to_i - last_queue_time.to_i).to_s + '/' + $config[:thread_timeout].to_s + ' Sec(s)'
+                sleep rand(10..20)
+              end
+            else
+              sleep 0.2
+            end
+          end
+          if process_healthy
+            @account_metadata[:folders][folder_name]['process']['threads']['uids'][uid] = {}
+            queue_thread(folder_name, uid, msg_ratio)
+            sleep 0.1
+            while get_thread_property(folder_name, uid, 'State') === 'Queued'
+              if (Time.now.to_i - get_thread_property(folder_name, uid, 'Queue Time').to_i) >= ($config[:thread_timeout] * $config[:timeout_warning_ratio])
+                if (Time.now.to_i - get_thread_property(folder_name, uid, 'Queue Time').to_i) >= $config[:thread_timeout]
+                  $logger.error folder_name + ' - ' + msg_ratio + ' - ' + uid_string + ' - *E* Thread state - Moving on... - Timeout: ' + $config[:thread_timeout].to_s + ' Sec(s)'
+                  set_thread_state(folder_name, uid, msg_ratio, 'Failed')
+                  kill_thread(folder_name, uid)
+                else
+                  $logger.warn folder_name + ' - ' + msg_ratio + ' - ' + uid_string + ' - *W* Thread state - Timeout: ' + (Time.now.to_i - get_thread_property(folder_name, uid, 'Queue Time').to_i).to_s + '/' + $config[:thread_timeout].to_s + ' Sec(s)'
+                  sleep rand(10..20)
+                end
+              else
+                sleep 0.2
+              end
+            end
+          end
+        end
+      end
+      if process_healthy
+        sleep 2
+        process_shutdown(folder_name)
+      end
+      process_thread_data_before_finish(folder_name)
+    end
+  end
+
+  def write_data_for_folder(folder_name)
+    $logger.debug ''
+    $logger.info 'Checking ' + folder_name
+    @imap.examine(folder_name)
+    generate_metadata_for_folder(folder_name)
+    log_folder_info(folder_name)
+    download_list = get_download_list(folder_name)
+    create_output_directory(folder_name)
+    process_deletes(folder_name)
+    process_downloads(folder_name, download_list)
+  end
+
+  def get_data_for_all_folders
+    folder_list = @account_metadata[:account]['messages']['server']['folders']
+    $logger.debug ''
+    $logger.debug 'Folder Count: ' + folder_list.count.to_s
+    folder_list.each do |folder_name|
+      if check_folder_included(folder_name)
+        $logger.debug ' ' + folder_name
+      else
+        $logger.debug ' ' + folder_name + ' - SKIPPED'
+      end
+    end
+    folder_list.each do |folder_name|
+      if check_folder_included(folder_name)
+        write_data_for_folder(folder_name)
+      end
+    end
+    $logger.debug ''
+  end
+
+  def check_folder_included(folder_name)
+    ignore_list = []
+    ignore_list << "[Gmail]"
+    ignore_list << "[Gmail]/All Mail"
+    ignore_list << "[Gmail]/Important"
+    ignore_list << "[Gmail]/Starred"
+    if ignore_list.include?(folder_name)
+      return false
+    else
+      return true
+    end
+  end
+
+  def execute
+    @connection = At_email::Account::ImapConnection.new()
+    @connection.login
+    get_imap
+    generate_metadata_for_account
+    @account_metadata[:task][:start_time] = Time.now
+    get_data_for_all_folders
+    @connection.disconnect
+    @account_metadata[:task][:end_time] = Time.now
+    task_report
+  end
+
+  def task_report
+    stats = get_task_stats()
+    $logger.info ''
+    $logger.info '##### Task Report #####'
+    $logger.info ''
+    $logger.info ' Task: ' + $config[:task]
+    $logger.info ''
+    $logger.info ' Server: ' + $config[:server]
+    $logger.info ' Username: ' + $config[:username]
+    $logger.info ' Output Directory: ' + @dir_base_path
+    $logger.info ''
+    $logger.info ' Start Time: ' + @account_metadata[:task][:start_time].to_s
+    $logger.info ' End Time: ' + @account_metadata[:task][:end_time].to_s
+    $logger.info ' Duration: ' + get_duration_formatted(@account_metadata[:task][:start_time], @account_metadata[:task][:end_time])
+    $logger.info ''
+    $logger.info ' Folders Processed: ' + @account_metadata[:folders].count.to_s
+    $logger.info ''
+    @account_metadata[:folders].each do |folder_name, folder_data|
+      $logger.info '  ' + folder_name
+      $logger.info ''
+      $logger.info '   Message Stats'
+      $logger.info '    Total: ' + folder_data['messages']['server']['config'].count.to_s
+      $logger.info '    Total Size: ' + stats[:folders][folder_name]['messages']['size'].to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse + ' bytes'
+      $logger.info ''
+      $logger.info '   Thread Stats'
+      $logger.info '    Total: ' + stats[:folders][folder_name]['threads']['Total'].to_s
+      $logger.info '    Success: ' + stats[:folders][folder_name]['threads']['Success'].to_s
+      $logger.info '    Failed: ' + stats[:folders][folder_name]['threads']['Failed'].to_s
+      if folder_data['process']
+        if folder_data['process']['threads']
+          if folder_data['process']['threads']['uids']
+            $logger.info '   Threads'
+            folder_data['process']['threads']['uids'].each do |uid, thread_data|
+              if thread_data['State'] === 'Successful'
+                $logger.debug '    UID: ' + uid.to_s
+                $logger.debug '     Path: ' + folder_data['messages']['server']['config'][uid]['file_path']
+                $logger.debug '     Thread State: ' + thread_data['State']
+              else
+                $logger.info '    UID: ' + uid.to_s
+                $logger.info '     Path: ' + message_config['file_path']
+                $logger.info '     Thread State: ' + thread_data['State']
+              end
+            end
+          end
+        end
+      end
+      $logger.info ''
+    end
+    $logger.info ''
+  end
+
+  def get_task_stats()
+    output = {}
+    output[:account] = {}
+    output[:folders] = {}
+    @account_metadata[:folders].each do |folder_name, folder_data|
+      output[:folders][folder_name] = {}
+      output[:folders][folder_name]['messages'] = {}
+      output[:folders][folder_name]['messages']['size'] = 0
+      folder_data['messages']['server']['config'].each do |uid, message_config|
+        output[:folders][folder_name]['messages']['size'] += message_config['size']
+      end
+      output[:folders][folder_name]['threads'] = {}
+      output[:folders][folder_name]['threads']['Total'] = 0
+      output[:folders][folder_name]['threads']['Success'] = 0
+      output[:folders][folder_name]['threads']['Failed'] = 0
+      if folder_data['process']
+        if folder_data['process']['threads']
+          if folder_data['process']['threads']['uids']
+            folder_data['process']['threads']['uids'].each do |uid, thread_data|
+              output[:folders][folder_name]['threads']['Total'] += 1
+              if thread_data['State'] === 'Successful'
+                output[:folders][folder_name]['threads']['Success'] += 1
+              else
+                output[:folders][folder_name]['threads']['Failed'] += 1
+              end
+            end
+          end
+        end
+      end
+    end
+    return output
+  end
+
+  def get_duration(start_time, end_time)
+    output = end_time.to_f - start_time.to_f
+    return output
+  end
+
+  def get_duration_formatted(start_time, end_time)
+    duration_seconds = get_duration(start_time, end_time)
+    if duration_seconds
+      if duration_seconds <= 60
+        output = sprintf("%.6f", duration_seconds) + ' Sec(s)'
+      else
+        nanoseconds = (sprintf("%.6f", duration_seconds).to_f * 1000000) % 1000000
+        seconds = duration_seconds % 60
+        minutes = (duration_seconds / 60) % 60
+        hours = duration_seconds / (60 * 60)
+        output = format("%02d:%02d:%02d.%6d", hours, minutes, seconds, nanoseconds)
+      end
+    else
+      return false
+    end
+    return output
+  end
+
+  def format_file_size(file_size)
+    output = format_number(file_size) + ' bytes'
+    return output
+  end
+
+  def format_number(number)
+    output = number.to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse
+    return output
+  end
+
+end