imap-backup 6.0.0.rc2 → 6.1.0

data/lib/imap/backup/cli/local.rb

@@ -5,6 +5,8 @@ module Imap::Backup
     include Thor::Actions
     include CLI::Helpers
 
+    MAX_SUBJECT = 60
+
     desc "accounts", "List locally backed-up accounts"
     def accounts
       accounts = CLI::Accounts.new
@@ -29,26 +31,16 @@ module Imap::Backup
       end
       raise "Folder '#{folder_name}' not found" if !folder_serializer
 
-      max_subject = 60
       Kernel.puts format(
-        "%-10<uid>s  %-#{max_subject}<subject>s - %<date>s",
+        "%-10<uid>s  %-#{MAX_SUBJECT}<subject>s - %<date>s",
         {uid: "UID", subject: "Subject", date: "Date"}
       )
-      Kernel.puts "-" * (12 + max_subject + 28)
+      Kernel.puts "-" * (12 + MAX_SUBJECT + 28)
 
       uids = folder_serializer.uids
 
       folder_serializer.each_message(uids).map do |uid, message|
-        m = {
-          uid: uid,
-          date: message.date.to_s,
-          subject: message.subject || ""
-        }
-        if m[:subject].length > max_subject
-          Kernel.puts format("% 10<uid>u: %.#{max_subject - 3}<subject>s... - %<date>s", m)
-        else
-          Kernel.puts format("% 10<uid>u: %-#{max_subject}<subject>s - %<date>s", m)
-        end
+        list_message uid, message
       end
     end
 
@@ -78,5 +70,20 @@ module Imap::Backup
         Kernel.puts message.supplied_body
       end
     end
+
+    no_commands do
+      def list_message(uid, message)
+        m = {
+          uid: uid,
+          date: message.date.to_s,
+          subject: message.subject || ""
+        }
+        if m[:subject].length > MAX_SUBJECT
+          Kernel.puts format("% 10<uid>u: %.#{MAX_SUBJECT - 3}<subject>s... - %<date>s", m)
+        else
+          Kernel.puts format("% 10<uid>u: %-#{MAX_SUBJECT}<subject>s - %<date>s", m)
+        end
+      end
+    end
   end
 end

data/lib/imap/backup/cli/migrate.rb

@@ -38,13 +38,9 @@ module Imap::Backup
           raise "Source and destination accounts cannot be the same!"
         end
 
-        if !destination_account
-          raise "Account '#{destination_email}' does not exist"
-        end
+        raise "Account '#{destination_email}' does not exist" if !destination_account
 
-        if !source_account
-          raise "Account '#{source_email}' does not exist"
-        end
+        raise "Account '#{source_email}' does not exist" if !source_account
       end
 
       def config
@@ -69,16 +65,14 @@ module Imap::Backup
 
       def folder_for(source_folder)
         no_source_prefix =
-          if source_prefix != "" &&
-              source_folder.start_with?(source_prefix)
+          if source_prefix != "" && source_folder.start_with?(source_prefix)
             source_folder.delete_prefix(source_prefix)
           else
             source_folder.to_s
           end
 
         with_destination_prefix =
-          if destination_prefix &&
-              destination_prefix != ""
+          if destination_prefix && destination_prefix != ""
             destination_prefix + no_source_prefix
           else
             no_source_prefix

data/lib/imap/backup/cli/restore.rb

@@ -7,11 +7,9 @@ module Imap::Backup
     attr_reader :account_names
 
     def initialize(email = nil, options)
+      super([])
       @email = email
-      @account_names =
-        if options.key?(:accounts)
-          options[:accounts].split(",")
-        end
+      @account_names = options[:accounts].split(",") if options.key?(:accounts)
     end
 
     no_commands do
@@ -22,12 +20,15 @@ module Imap::Backup
           connection.restore
         when !email && !account_names
           Logger.logger.info "Calling restore without an EMAIL parameter is deprecated"
-          each_connection([]) { |connection| connection.restore }
+          each_connection([], &:restore)
         when email && account_names.any?
           raise "Pass either an email or the --accounts option, not both"
         when account_names.any?
-          Logger.logger.info "Calling restore with the --account option is deprected, please pass a single EMAIL argument"
-          each_connection(account_names) { |connection| connection.restore }
+          Logger.logger.info(
+            "Calling restore with the --account option is deprected, " \
+            "please pass a single EMAIL argument"
+          )
+          each_connection(account_names, &:restore)
         end
       end
     end

data/lib/imap/backup/cli/setup.rb

@@ -1,13 +1,15 @@
-class Imap::Backup::CLI::Setup < Thor
-  include Thor::Actions
+module Imap::Backup
+  class CLI::Setup < Thor
+    include Thor::Actions
 
-  def initialize
-    super([])
-  end
+    def initialize
+      super([])
+    end
 
-  no_commands do
-    def run
-      Imap::Backup::Setup.new.run
+    no_commands do
+      def run
+        Setup.new.run
+      end
     end
   end
 end

data/lib/imap/backup/cli/stats.rb

@@ -0,0 +1,78 @@
+module Imap::Backup
+  class CLI::Stats < Thor
+    include Thor::Actions
+    include CLI::Helpers
+
+    TEXT_COLUMNS = [
+      {name: :folder, width: 20, alignment: :left},
+      {name: :remote, width: 8, alignment: :right},
+      {name: :both, width: 8, alignment: :right},
+      {name: :local, width: 8, alignment: :right}
+    ].freeze
+    ALIGNMENT_FORMAT_SYMBOL = {left: "-", right: " "}.freeze
+
+    attr_reader :email
+    attr_reader :options
+
+    def initialize(email, options = {})
+      super([])
+      @email = email
+      @options = options
+    end
+
+    no_commands do
+      def run
+        case options[:format]
+        when "json"
+          Kernel.puts stats.to_json
+        else
+          format_text stats
+        end
+      end
+
+      def stats
+        connection = connection(email)
+
+        connection.backup_folders.map do |folder|
+          next if !folder.exist?
+
+          serializer = Serializer.new(connection.account.local_path, folder.name)
+          local_uids = serializer.uids
+          remote_uids = folder.uids
+          {
+            folder: folder.name,
+            remote: (remote_uids - local_uids).count,
+            both: (serializer.uids & folder.uids).count,
+            local: (local_uids - remote_uids).count
+          }
+        end.compact
+      end
+
+      def format_text(stats)
+        Kernel.puts text_header
+
+        stats.each do |stat|
+          columns = TEXT_COLUMNS.map do |column|
+            symbol = ALIGNMENT_FORMAT_SYMBOL[column[:alignment]]
+            count = stat[column[:name]]
+            format("%#{symbol}#{column[:width]}s", count)
+          end.join("|")
+
+          Kernel.puts columns
+        end
+      end
+
+      def text_header
+        titles = TEXT_COLUMNS.map do |column|
+          format("%-#{column[:width]}s", column[:name])
+        end.join("|")
+
+        underline = TEXT_COLUMNS.map do |column|
+          "-" * column[:width]
+        end.join("|")
+
+        "#{titles}\n#{underline}"
+      end
+    end
+  end
+end

data/lib/imap/backup/cli/status.rb

@@ -13,11 +13,11 @@ module Imap::Backup
     no_commands do
       def run
         each_connection(account_names) do |connection|
-          puts connection.account.username
+          Kernel.puts connection.account.username
           folders = connection.status
           folders.each do |f|
             missing_locally = f[:remote] - f[:local]
-            puts "#{f[:name]}: #{missing_locally.size}"
+            Kernel.puts "#{f[:name]}: #{missing_locally.size}"
           end
         end
       end

data/lib/imap/backup/cli/utils.rb

@@ -47,11 +47,9 @@ module Imap::Backup
       profile = thunderbird_profile(profile_name)
 
       if !profile
-        if profile_name
-          raise "Thunderbird profile '#{profile_name}' not found"
-        else
-          raise "Default Thunderbird profile not found"
-        end
+        raise "Thunderbird profile '#{profile_name}' not found" if profile_name
+
+        raise "Default Thunderbird profile not found"
       end
 
       connection.local_folders.each do |serializer, _folder|
@@ -64,7 +62,7 @@ module Imap::Backup
     no_commands do
       def do_ignore_folder_history(folder, serializer)
         uids = folder.uids - serializer.uids
-        Imap::Backup::Logger.logger.info "Folder '#{folder.name}' - #{uids.length} messages"
+        Logger.logger.info "Folder '#{folder.name}' - #{uids.length} messages"
 
         serializer.apply_uid_validity(folder.uid_validity)
 

data/lib/imap/backup/cli.rb

@@ -13,6 +13,7 @@ module Imap::Backup
     autoload :Remote, "imap/backup/cli/remote"
     autoload :Restore, "imap/backup/cli/restore"
     autoload :Setup, "imap/backup/cli/setup"
+    autoload :Stats, "imap/backup/cli/stats"
     autoload :Status, "imap/backup/cli/status"
     autoload :Utils, "imap/backup/cli/utils"
 
@@ -61,8 +62,11 @@ module Imap::Backup
     desc "local SUBCOMMAND [OPTIONS]", "View local info"
     subcommand "local", Local
 
-    desc "migrate SOURCE_EMAIL DESTINATION_EMAIL [OPTIONS]",
-      "[Experimental] Uploads backed-up emails from account SOURCE_EMAIL to account DESTINATION_EMAIL"
+    desc(
+      "migrate SOURCE_EMAIL DESTINATION_EMAIL [OPTIONS]",
+      "[Experimental] " \
+      "Uploads backed-up emails from account SOURCE_EMAIL to account DESTINATION_EMAIL"
+    )
     long_desc <<~DESC
       All emails which have been backed up for the "source account" (SOURCE_EMAIL) are
       uploaded to the "destination account" (DESTINATION_EMAIL).
@@ -125,9 +129,26 @@ module Imap::Backup
       Setup.new.run
     end
 
-    desc "status", "Show backup status"
+    desc "stats EMAIL [OPTIONS]", "Print stats for each account folder"
+    long_desc <<~DESC
+      For each account folder, lists emails that are yet to be downloaded "server",
+      are downloaded (exist on server and locally) "both" and those which
+      are only present in the backup (as they have been deleted on the server) "local".
+    DESC
+    method_option(
+      "format",
+      type: :string,
+      desc: "the output type, text (plain text) or json",
+      aliases: ["-f"]
+    )
+    def stats(email)
+      Stats.new(email, symbolized(options)).run
+    end
+
+    desc "status", "This command is deprecated, use `imap-backup stats ACCOUNT`"
     long_desc <<~DESC
       For each configured account and folder, lists the number of emails yet to be downloaded.
+      This command is deprecated.
     DESC
     accounts_option
     def status

data/lib/imap/backup/configuration.rb

@@ -6,7 +6,7 @@ require "imap/backup/account"
 module Imap::Backup
   class Configuration
     CONFIGURATION_DIRECTORY = File.expand_path("~/.imap-backup")
-    VERSION = "2.0"
+    VERSION = "2.0".freeze
 
     attr_reader :pathname
 
@@ -81,23 +81,21 @@ module Imap::Backup
 
     def data
       @data ||=
-        begin
-          if File.exist?(pathname)
-            Utils.check_permissions(pathname, 0o600) if !windows?
-            contents = File.read(pathname)
-            JSON.parse(contents, symbolize_names: true)
-          else
-            {accounts: []}
-          end
+        if File.exist?(pathname)
+          Utils.check_permissions(pathname, 0o600) if !windows?
+          contents = File.read(pathname)
+          JSON.parse(contents, symbolize_names: true)
+        else
+          {accounts: []}
         end
     end
 
     def remove_modified_flags
-      accounts.each { |a| a.clear_changes }
+      accounts.each(&:clear_changes)
     end
 
     def remove_deleted_accounts
-      accounts.reject! { |a| a.marked_for_deletion? }
+      accounts.reject!(&:marked_for_deletion?)
     end
 
     def make_private(path)

data/lib/imap/backup/downloader.rb

@@ -5,57 +5,101 @@ module Imap::Backup
     attr_reader :folder
     attr_reader :serializer
     attr_reader :multi_fetch_size
+    # Some IMAP providers, notably Apple Mail, set the '\Seen' flag
+    # on emails when they are fetched. By setting `:reset_seen_flags_after_fetch`,
+    # a workaround is activated which checks which emails are 'unseen' before
+    # and after the fetch, and removes the '\Seen' flag from those which have changed.
+    # As this check is susceptible to 'race conditions', i.e. when a different
+    # client sets the '\Seen' flag while imap-backup is fetching, it is best
+    # to only use it when required (i.e. for IMAP providers which always
+    # mark messages as '\Seen' when accessed).
+    attr_reader :reset_seen_flags_after_fetch
 
-    def initialize(folder, serializer, multi_fetch_size: 1)
+    def initialize(folder, serializer, multi_fetch_size: 1, reset_seen_flags_after_fetch: false)
       @folder = folder
       @serializer = serializer
       @multi_fetch_size = multi_fetch_size
+      @reset_seen_flags_after_fetch = reset_seen_flags_after_fetch
+      @uids = nil
     end
 
     def run
-      uids = folder.uids - serializer.uids
-      count = uids.count
-      debug "#{count} new messages"
-      uids.each_slice(multi_fetch_size).with_index do |block, i|
-        uids_and_bodies = folder.fetch_multi(block)
-        if uids_and_bodies.nil?
-          if multi_fetch_size > 1
-            debug("Multi fetch failed for UIDs #{block.join(", ")}, switching to single fetches")
-            raise MultiFetchFailedError
-          else
-            debug("Fetch failed for UID #{block[0]} - skipping")
-            next
-          end
-        end
+      debug "#{uids.count} new messages"
 
-        offset = i * multi_fetch_size + 1
-        uids_and_bodies.each.with_index do |uid_and_body, j|
-          uid = uid_and_body[:uid]
-          body = uid_and_body[:body]
-          case
-          when !body
-            info("Fetch returned empty body - skipping")
-          when !uid
-            info("Fetch returned empty UID - skipping")
-          else
-            debug("uid: #{uid} (#{offset + j}/#{count}) - #{body.size} bytes")
-            serializer.append uid, body
-          end
-        end
+      uids.each_slice(multi_fetch_size).with_index do |block, i|
+        multifetch_failed = download_block(block, i)
+        raise MultiFetchFailedError if multifetch_failed
       end
     rescue MultiFetchFailedError
+      @count = nil
       @multi_fetch_size = 1
+      @uids = nil
       retry
     end
 
     private
 
+    def download_block(block, index)
+      uids_and_bodies =
+        if reset_seen_flags_after_fetch
+          before_unseen = folder.unseen(block)
+          debug "Pre-fetch unseen messages: #{before_unseen.join(', ')}"
+          uids_and_bodies = folder.fetch_multi(block)
+          after_unseen = folder.unseen(block)
+          debug "Post-fetch unseen messages: #{after_unseen.join(', ')}"
+          changed = before_unseen - after_unseen
+          if changed.any?
+            ids = changed.join(", ")
+            debug "Removing '\Seen' flag for the following messages: #{ids}"
+            folder.unset_flags(changed, [:Seen])
+          end
+          uids_and_bodies
+        else
+          folder.fetch_multi(block)
+        end
+      if uids_and_bodies.nil?
+        if multi_fetch_size > 1
+          uids = block.join(", ")
+          debug("Multi fetch failed for UIDs #{uids}, switching to single fetches")
+          return true
+        else
+          debug("Fetch failed for UID #{block[0]} - skipping")
+          return false
+        end
+      end
+
+      offset = (index * multi_fetch_size) + 1
+      uids_and_bodies.each.with_index do |uid_and_body, j|
+        handle_uid_and_body uid_and_body, offset + j
+      end
+
+      false
+    end
+
+    def handle_uid_and_body(uid_and_body, index)
+      uid = uid_and_body[:uid]
+      body = uid_and_body[:body]
+      case
+      when !body
+        info("Fetch returned empty body - skipping")
+      when !uid
+        info("Fetch returned empty UID - skipping")
+      else
+        debug("uid: #{uid} (#{index}/#{uids.count}) - #{body.size} bytes")
+        serializer.append uid, body
+      end
+    end
+
+    def uids
+      @uids ||= folder.uids - serializer.uids
+    end
+
     def info(message)
-      Imap::Backup::Logger.logger.info("[#{folder.name}] #{message}")
+      Logger.logger.info("[#{folder.name}] #{message}")
     end
 
     def debug(message)
-      Imap::Backup::Logger.logger.debug("[#{folder.name}] #{message}")
+      Logger.logger.debug("[#{folder.name}] #{message}")
     end
   end
 end

data/lib/imap/backup/migrator.rb

@@ -15,12 +15,12 @@ module Imap::Backup
       folder.create
       ensure_destination_empty!
 
-      Imap::Backup::Logger.logger.debug "[#{folder.name}] #{count} to migrate"
+      Logger.logger.debug "[#{folder.name}] #{count} to migrate"
       serializer.each_message(serializer.uids).with_index do |(uid, message), i|
         next if message.nil?
 
         log_prefix = "[#{folder.name}] uid: #{uid} (#{i + 1}/#{count}) -"
-        Imap::Backup::Logger.logger.debug(
+        Logger.logger.debug(
           "#{log_prefix} #{message.supplied_body.size} bytes"
         )
 
@@ -42,9 +42,9 @@ module Imap::Backup
       return if folder.uids.empty?
 
       raise <<~ERROR
-      The destination folder '#{folder.name}' is not empty.
-      Pass the --reset flag if you want to clear existing emails from destination
-      folders before uploading.
+        The destination folder '#{folder.name}' is not empty.
+        Pass the --reset flag if you want to clear existing emails from destination
+        folders before uploading.
       ERROR
     end
   end

data/lib/imap/backup/sanitizer.rb

@@ -19,6 +19,7 @@ module Imap::Backup
       loop do
         line, newline, rest = @current.partition("\n")
         break if newline != "\n"
+
         clean = sanitize(line)
         output.puts clean
         @current = rest
@@ -34,9 +35,9 @@ module Imap::Backup
 
     private
 
-    def sanitize(t)
+    def sanitize(text)
       # Hide password in Net::IMAP debug output
-      t.gsub(
+      text.gsub(
         /\A(C: RUBY\d+ LOGIN \S+) \S+/,
         "\\1 [PASSWORD REDACTED]"
       )

data/lib/imap/backup/serializer/appender.rb

@@ -0,0 +1,49 @@
+module Imap::Backup
+  class Serializer::Appender
+    attr_reader :imap
+    attr_reader :folder
+    attr_reader :mbox
+
+    def initialize(folder:, imap:, mbox:)
+      @folder = folder
+      @imap = imap
+      @mbox = mbox
+    end
+
+    def run(uid:, message:)
+      raise "Can't add messages without uid_validity" if !imap.uid_validity
+
+      uid = uid.to_i
+      if imap.include?(uid)
+        Logger.logger.debug(
+          "[#{folder}] message #{uid} already downloaded - skipping"
+        )
+        return
+      end
+
+      do_append uid, message
+    end
+
+    private
+
+    def do_append(uid, message)
+      mboxrd_message = Email::Mboxrd::Message.new(message)
+      initial = mbox.length || 0
+      mbox_appended = false
+      begin
+        mbox.append mboxrd_message.to_serialized
+        mbox_appended = true
+        imap.append uid
+      rescue StandardError => e
+        mbox.rewind(initial) if mbox_appended
+
+        message = <<-ERROR.gsub(/^\s*/m, "")
+          [#{folder}] failed to append message #{uid}:
+          #{message}. #{e}:
+          #{e.backtrace.join("\n")}"
+        ERROR
+        Logger.logger.warn message
+      end
+    end
+  end
+end

data/lib/imap/backup/serializer/imap.rb

@@ -12,6 +12,15 @@ module Imap::Backup
       @loaded = false
       @uid_validity = nil
       @uids = nil
+      @version = nil
+    end
+
+    def valid?
+      return false if !exist?
+      return false if version != CURRENT_VERSION
+      return false if !uid_validity
+
+      true
     end
 
     def append(uid)
@@ -19,8 +28,10 @@ module Imap::Backup
       save
     end
 
-    def exist?
-      File.exist?(pathname)
+    def delete
+      return if !exist?
+
+      File.unlink(pathname)
     end
 
     def include?(uid)
@@ -66,12 +77,21 @@ module Imap::Backup
       save
     end
 
+    def version
+      ensure_loaded
+      @version
+    end
+
     private
 
     def pathname
       "#{folder_path}.imap"
     end
 
+    def exist?
+      File.exist?(pathname)
+    end
+
     def ensure_loaded
       return if loaded
 
@@ -79,9 +99,11 @@ module Imap::Backup
       if data
         @uids = data[:uids].map(&:to_i)
         @uid_validity = data[:uid_validity]
+        @version = data[:version]
       else
         @uids = []
         @uid_validity = nil
+        @version = CURRENT_VERSION
       end
       @loaded = true
     end
@@ -97,6 +119,8 @@ module Imap::Backup
         return nil
       end
 
+      return nil if !data.key?(:version)
+      return nil if !data.key?(:uid_validity)
       return nil if !data.key?(:uids)
       return nil if !data[:uids].is_a?(Array)
 
@@ -109,7 +133,7 @@ module Imap::Backup
       raise "Cannot save metadata without a uid_validity" if !uid_validity
 
       data = {
-        version: CURRENT_VERSION,
+        version: @version,
         uid_validity: @uid_validity,
         uids: @uids
       }