imap-backup 6.0.0.rc2 → 6.0.0

data/lib/imap/backup/cli/stats.rb

@@ -0,0 +1,78 @@
+module Imap::Backup
+  class CLI::Stats < Thor
+    include Thor::Actions
+    include CLI::Helpers
+
+    TEXT_COLUMNS = [
+      {name: :folder, width: 20, alignment: :left},
+      {name: :remote, width: 8, alignment: :right},
+      {name: :both, width: 8, alignment: :right},
+      {name: :local, width: 8, alignment: :right}
+    ].freeze
+    ALIGNMENT_FORMAT_SYMBOL = {left: "-", right: " "}.freeze
+
+    attr_reader :email
+    attr_reader :options
+
+    def initialize(email, options = {})
+      super([])
+      @email = email
+      @options = options
+    end
+
+    no_commands do
+      def run
+        case options[:format]
+        when "json"
+          Kernel.puts stats.to_json
+        else
+          format_text stats
+        end
+      end
+
+      def stats
+        connection = connection(email)
+
+        connection.backup_folders.map do |folder|
+          next if !folder.exist?
+
+          serializer = Serializer.new(connection.account.local_path, folder.name)
+          local_uids = serializer.uids
+          remote_uids = folder.uids
+          {
+            folder: folder.name,
+            remote: (remote_uids - local_uids).count,
+            both: (serializer.uids & folder.uids).count,
+            local: (local_uids - remote_uids).count
+          }
+        end.compact
+      end
+
+      def format_text(stats)
+        Kernel.puts text_header
+
+        stats.each do |stat|
+          columns = TEXT_COLUMNS.map do |column|
+            symbol = ALIGNMENT_FORMAT_SYMBOL[column[:alignment]]
+            count = stat[column[:name]]
+            format("%#{symbol}#{column[:width]}s", count)
+          end.join("|")
+
+          Kernel.puts columns
+        end
+      end
+
+      def text_header
+        titles = TEXT_COLUMNS.map do |column|
+          format("%-#{column[:width]}s", column[:name])
+        end.join("|")
+
+        underline = TEXT_COLUMNS.map do |column|
+          "-" * column[:width]
+        end.join("|")
+
+        "#{titles}\n#{underline}"
+      end
+    end
+  end
+end

data/lib/imap/backup/cli/status.rb

@@ -13,11 +13,11 @@ module Imap::Backup
     no_commands do
       def run
         each_connection(account_names) do |connection|
-          puts connection.account.username
+          Kernel.puts connection.account.username
           folders = connection.status
           folders.each do |f|
             missing_locally = f[:remote] - f[:local]
-            puts "#{f[:name]}: #{missing_locally.size}"
+            Kernel.puts "#{f[:name]}: #{missing_locally.size}"
           end
         end
       end

data/lib/imap/backup/cli/utils.rb

@@ -47,11 +47,9 @@ module Imap::Backup
       profile = thunderbird_profile(profile_name)
 
       if !profile
-        if profile_name
-          raise "Thunderbird profile '#{profile_name}' not found"
-        else
-          raise "Default Thunderbird profile not found"
-        end
+        raise "Thunderbird profile '#{profile_name}' not found" if profile_name
+
+        raise "Default Thunderbird profile not found"
       end
 
       connection.local_folders.each do |serializer, _folder|
@@ -64,7 +62,7 @@ module Imap::Backup
     no_commands do
       def do_ignore_folder_history(folder, serializer)
         uids = folder.uids - serializer.uids
-        Imap::Backup::Logger.logger.info "Folder '#{folder.name}' - #{uids.length} messages"
+        Logger.logger.info "Folder '#{folder.name}' - #{uids.length} messages"
 
         serializer.apply_uid_validity(folder.uid_validity)
 

data/lib/imap/backup/cli.rb

@@ -13,6 +13,7 @@ module Imap::Backup
     autoload :Remote, "imap/backup/cli/remote"
     autoload :Restore, "imap/backup/cli/restore"
     autoload :Setup, "imap/backup/cli/setup"
+    autoload :Stats, "imap/backup/cli/stats"
     autoload :Status, "imap/backup/cli/status"
     autoload :Utils, "imap/backup/cli/utils"
 
@@ -61,8 +62,11 @@ module Imap::Backup
     desc "local SUBCOMMAND [OPTIONS]", "View local info"
     subcommand "local", Local
 
-    desc "migrate SOURCE_EMAIL DESTINATION_EMAIL [OPTIONS]",
-      "[Experimental] Uploads backed-up emails from account SOURCE_EMAIL to account DESTINATION_EMAIL"
+    desc(
+      "migrate SOURCE_EMAIL DESTINATION_EMAIL [OPTIONS]",
+      "[Experimental] " \
+      "Uploads backed-up emails from account SOURCE_EMAIL to account DESTINATION_EMAIL"
+    )
     long_desc <<~DESC
       All emails which have been backed up for the "source account" (SOURCE_EMAIL) are
       uploaded to the "destination account" (DESTINATION_EMAIL).
@@ -125,9 +129,26 @@ module Imap::Backup
       Setup.new.run
     end
 
-    desc "status", "Show backup status"
+    desc "stats EMAIL [OPTIONS]", "Print stats for each account folder"
+    long_desc <<~DESC
+      For each account folder, lists emails that are yet to be downloaded "server",
+      are downloaded (exist on server and locally) "both" and those which
+      are only present in the backup (as they have been deleted on the server) "local".
+    DESC
+    method_option(
+      "format",
+      type: :string,
+      desc: "the output type, text (plain text) or json",
+      aliases: ["-f"]
+    )
+    def stats(email)
+      Stats.new(email, symbolized(options)).run
+    end
+
+    desc "status", "This command is deprecated, use `imap-backup stats ACCOUNT`"
     long_desc <<~DESC
       For each configured account and folder, lists the number of emails yet to be downloaded.
+      This command is deprecated.
     DESC
     accounts_option
     def status

data/lib/imap/backup/configuration.rb

@@ -6,7 +6,7 @@ require "imap/backup/account"
 module Imap::Backup
   class Configuration
     CONFIGURATION_DIRECTORY = File.expand_path("~/.imap-backup")
-    VERSION = "2.0"
+    VERSION = "2.0".freeze
 
     attr_reader :pathname
 
@@ -81,23 +81,21 @@ module Imap::Backup
 
     def data
       @data ||=
-        begin
-          if File.exist?(pathname)
-            Utils.check_permissions(pathname, 0o600) if !windows?
-            contents = File.read(pathname)
-            JSON.parse(contents, symbolize_names: true)
-          else
-            {accounts: []}
-          end
+        if File.exist?(pathname)
+          Utils.check_permissions(pathname, 0o600) if !windows?
+          contents = File.read(pathname)
+          JSON.parse(contents, symbolize_names: true)
+        else
+          {accounts: []}
         end
     end
 
     def remove_modified_flags
-      accounts.each { |a| a.clear_changes }
+      accounts.each(&:clear_changes)
     end
 
     def remove_deleted_accounts
-      accounts.reject! { |a| a.marked_for_deletion? }
+      accounts.reject!(&:marked_for_deletion?)
     end
 
     def make_private(path)

data/lib/imap/backup/downloader.rb

@@ -10,52 +10,70 @@ module Imap::Backup
       @folder = folder
       @serializer = serializer
       @multi_fetch_size = multi_fetch_size
+      @uids = nil
     end
 
     def run
-      uids = folder.uids - serializer.uids
-      count = uids.count
-      debug "#{count} new messages"
-      uids.each_slice(multi_fetch_size).with_index do |block, i|
-        uids_and_bodies = folder.fetch_multi(block)
-        if uids_and_bodies.nil?
-          if multi_fetch_size > 1
-            debug("Multi fetch failed for UIDs #{block.join(", ")}, switching to single fetches")
-            raise MultiFetchFailedError
-          else
-            debug("Fetch failed for UID #{block[0]} - skipping")
-            next
-          end
-        end
+      debug "#{uids.count} new messages"
 
-        offset = i * multi_fetch_size + 1
-        uids_and_bodies.each.with_index do |uid_and_body, j|
-          uid = uid_and_body[:uid]
-          body = uid_and_body[:body]
-          case
-          when !body
-            info("Fetch returned empty body - skipping")
-          when !uid
-            info("Fetch returned empty UID - skipping")
-          else
-            debug("uid: #{uid} (#{offset + j}/#{count}) - #{body.size} bytes")
-            serializer.append uid, body
-          end
-        end
+      uids.each_slice(multi_fetch_size).with_index do |block, i|
+        multifetch_failed = download_block(block, i)
+        raise MultiFetchFailedError if multifetch_failed
       end
     rescue MultiFetchFailedError
+      @count = nil
       @multi_fetch_size = 1
+      @uids = nil
       retry
     end
 
     private
 
+    def download_block(block, index)
+      uids_and_bodies = folder.fetch_multi(block)
+      if uids_and_bodies.nil?
+        if multi_fetch_size > 1
+          uids = block.join(", ")
+          debug("Multi fetch failed for UIDs #{uids}, switching to single fetches")
+          return true
+        else
+          debug("Fetch failed for UID #{block[0]} - skipping")
+          return false
+        end
+      end
+
+      offset = (index * multi_fetch_size) + 1
+      uids_and_bodies.each.with_index do |uid_and_body, j|
+        handle_uid_and_body uid_and_body, offset + j
+      end
+
+      false
+    end
+
+    def handle_uid_and_body(uid_and_body, index)
+      uid = uid_and_body[:uid]
+      body = uid_and_body[:body]
+      case
+      when !body
+        info("Fetch returned empty body - skipping")
+      when !uid
+        info("Fetch returned empty UID - skipping")
+      else
+        debug("uid: #{uid} (#{index}/#{uids.count}) - #{body.size} bytes")
+        serializer.append uid, body
+      end
+    end
+
+    def uids
+      @uids ||= folder.uids - serializer.uids
+    end
+
     def info(message)
-      Imap::Backup::Logger.logger.info("[#{folder.name}] #{message}")
+      Logger.logger.info("[#{folder.name}] #{message}")
     end
 
     def debug(message)
-      Imap::Backup::Logger.logger.debug("[#{folder.name}] #{message}")
+      Logger.logger.debug("[#{folder.name}] #{message}")
     end
   end
 end

data/lib/imap/backup/migrator.rb

@@ -15,12 +15,12 @@ module Imap::Backup
       folder.create
       ensure_destination_empty!
 
-      Imap::Backup::Logger.logger.debug "[#{folder.name}] #{count} to migrate"
+      Logger.logger.debug "[#{folder.name}] #{count} to migrate"
       serializer.each_message(serializer.uids).with_index do |(uid, message), i|
         next if message.nil?
 
         log_prefix = "[#{folder.name}] uid: #{uid} (#{i + 1}/#{count}) -"
-        Imap::Backup::Logger.logger.debug(
+        Logger.logger.debug(
           "#{log_prefix} #{message.supplied_body.size} bytes"
         )
 
@@ -42,9 +42,9 @@ module Imap::Backup
       return if folder.uids.empty?
 
       raise <<~ERROR
-      The destination folder '#{folder.name}' is not empty.
-      Pass the --reset flag if you want to clear existing emails from destination
-      folders before uploading.
+        The destination folder '#{folder.name}' is not empty.
+        Pass the --reset flag if you want to clear existing emails from destination
+        folders before uploading.
       ERROR
     end
   end

data/lib/imap/backup/sanitizer.rb

@@ -19,6 +19,7 @@ module Imap::Backup
       loop do
         line, newline, rest = @current.partition("\n")
         break if newline != "\n"
+
         clean = sanitize(line)
         output.puts clean
         @current = rest
@@ -34,9 +35,9 @@ module Imap::Backup
 
     private
 
-    def sanitize(t)
+    def sanitize(text)
       # Hide password in Net::IMAP debug output
-      t.gsub(
+      text.gsub(
         /\A(C: RUBY\d+ LOGIN \S+) \S+/,
         "\\1 [PASSWORD REDACTED]"
       )

data/lib/imap/backup/serializer/appender.rb

@@ -0,0 +1,49 @@
+module Imap::Backup
+  class Serializer::Appender
+    attr_reader :imap
+    attr_reader :folder
+    attr_reader :mbox
+
+    def initialize(folder:, imap:, mbox:)
+      @folder = folder
+      @imap = imap
+      @mbox = mbox
+    end
+
+    def run(uid:, message:)
+      raise "Can't add messages without uid_validity" if !imap.uid_validity
+
+      uid = uid.to_i
+      if imap.include?(uid)
+        Logger.logger.debug(
+          "[#{folder}] message #{uid} already downloaded - skipping"
+        )
+        return
+      end
+
+      do_append uid, message
+    end
+
+    private
+
+    def do_append(uid, message)
+      mboxrd_message = Email::Mboxrd::Message.new(message)
+      initial = mbox.length || 0
+      mbox_appended = false
+      begin
+        mbox.append mboxrd_message.to_serialized
+        mbox_appended = true
+        imap.append uid
+      rescue StandardError => e
+        mbox.rewind(initial) if mbox_appended
+
+        message = <<-ERROR.gsub(/^\s*/m, "")
+          [#{folder}] failed to append message #{uid}:
+          #{message}. #{e}:
+          #{e.backtrace.join("\n")}"
+        ERROR
+        Logger.logger.warn message
+      end
+    end
+  end
+end

data/lib/imap/backup/serializer/message_enumerator.rb

@@ -0,0 +1,29 @@
+require "email/mboxrd/message"
+require "imap/backup/serializer/mbox_enumerator"
+
+module Imap::Backup
+  class Serializer::MessageEnumerator
+    attr_reader :imap
+    attr_reader :mbox
+
+    def initialize(imap:, mbox:)
+      @imap = imap
+      @mbox = mbox
+    end
+
+    def run(uids:)
+      indexes = uids.each.with_object({}) do |uid_maybe_string, acc|
+        uid = uid_maybe_string.to_i
+        index = imap.index(uid)
+        acc[index] = uid if index
+      end
+      enumerator = Serializer::MboxEnumerator.new(mbox.pathname)
+      enumerator.each.with_index do |raw, i|
+        uid = indexes[i]
+        next if !uid
+
+        yield uid, Email::Mboxrd::Message.from_serialized(raw)
+      end
+    end
+  end
+end

data/lib/imap/backup/serializer/unused_name_finder.rb

@@ -0,0 +1,27 @@
+module Imap::Backup
+  class Serializer::UnusedNameFinder
+    attr_reader :serializer
+
+    def initialize(serializer:)
+      @serializer = serializer
+    end
+
+    def run
+      digit = 0
+      name = nil
+
+      loop do
+        extra = digit.zero? ? "" : "-#{digit}"
+        name = "#{serializer.folder}-#{serializer.uid_validity}#{extra}"
+        new_folder_path = Serializer.folder_path_for(path: serializer.path, folder: name)
+        test_mbox = Serializer::Mbox.new(new_folder_path)
+        test_imap = Serializer::Imap.new(new_folder_path)
+        break if !test_mbox.exist? && !test_imap.exist?
+
+        digit += 1
+      end
+
+      name
+    end
+  end
+end

data/lib/imap/backup/serializer.rb

@@ -1,12 +1,20 @@
 require "forwardable"
 
 require "email/mboxrd/message"
+require "imap/backup/serializer/appender"
 require "imap/backup/serializer/imap"
 require "imap/backup/serializer/mbox"
 require "imap/backup/serializer/mbox_enumerator"
+require "imap/backup/serializer/message_enumerator"
+require "imap/backup/serializer/unused_name_finder"
 
 module Imap::Backup
   class Serializer
+    def self.folder_path_for(path:, folder:)
+      relative = File.join(path, folder)
+      File.expand_path(relative)
+    end
+
     extend Forwardable
 
     def_delegator :mbox, :pathname, :mbox_pathname
@@ -38,17 +46,8 @@ module Imap::Backup
     end
 
     def append(uid, message)
-      raise "Can't add messages without uid_validity" if !imap.uid_validity
-
-      uid = uid.to_i
-      if imap.include?(uid)
-        Logger.logger.debug(
-          "[#{folder}] message #{uid} already downloaded - skipping"
-        )
-        return
-      end
-
-      do_append uid, message
+      appender = Serializer::Appender.new(folder: folder, imap: imap, mbox: mbox)
+      appender.run(uid: uid, message: message)
     end
 
     def load(uid_maybe_string)
@@ -69,59 +68,26 @@ module Imap::Backup
       nil
     end
 
-    def each_message(required_uids)
-      return enum_for(:each_message, required_uids) if !block_given?
+    def each_message(required_uids, &block)
+      return enum_for(:each_message, required_uids) if !block
 
-      indexes = required_uids.each.with_object({}) do |uid_maybe_string, acc|
-        uid = uid_maybe_string.to_i
-        index = imap.index(uid)
-        acc[index] = uid if index
-      end
-      enumerator = Serializer::MboxEnumerator.new(mbox.pathname)
-      enumerator.each.with_index do |raw, i|
-        uid = indexes[i]
-        next if !uid
-
-        yield uid, Email::Mboxrd::Message.from_serialized(raw)
-      end
+      enumerator = Serializer::MessageEnumerator.new(imap: imap, mbox: mbox)
+      enumerator.run(uids: required_uids, &block)
     end
 
     def rename(new_name)
-      # Initialize so we get memoized instances with the correct folder_path
-      mbox
-      imap
-      @folder = new_name
-      ensure_containing_directory
-      mbox.rename folder_path
-      imap.rename folder_path
+      destination = self.class.folder_path_for(path: path, folder: new_name)
+      ensure_containing_directory(new_name)
+      mbox.rename destination
+      imap.rename destination
     end
 
     private
 
-    def do_append(uid, message)
-      mboxrd_message = Email::Mboxrd::Message.new(message)
-      initial = mbox.length || 0
-      mbox_appended = false
-      begin
-        mbox.append mboxrd_message.to_serialized
-        mbox_appended = true
-        imap.append uid
-      rescue StandardError => e
-        mbox.rewind(initial) if mbox_appended
-
-        message = <<-ERROR.gsub(/^\s*/m, "")
-          [#{folder}] failed to append message #{uid}:
-          #{message}. #{e}:
-          #{e.backtrace.join("\n")}"
-        ERROR
-        Logger.logger.warn message
-      end
-    end
-
     def mbox
       @mbox ||=
         begin
-          ensure_containing_directory
+          ensure_containing_directory(folder)
           Serializer::Mbox.new(folder_path)
         end
     end
@@ -129,21 +95,16 @@ module Imap::Backup
     def imap
       @imap ||=
         begin
-          ensure_containing_directory
+          ensure_containing_directory(folder)
           Serializer::Imap.new(folder_path)
         end
     end
 
     def folder_path
-      folder_path_for(path, folder)
-    end
-
-    def folder_path_for(path, folder)
-      relative = File.join(path, folder)
-      File.expand_path(relative)
+      self.class.folder_path_for(path: path, folder: folder)
     end
 
-    def ensure_containing_directory
+    def ensure_containing_directory(folder)
       relative = File.dirname(folder)
       directory = Serializer::Directory.new(path, relative)
       directory.ensure_exists
@@ -160,23 +121,8 @@ module Imap::Backup
     end
 
     def rename_existing_folder
-      digit = 0
-      new_name = nil
-      loop do
-        extra = digit.zero? ? "" : "-#{digit}"
-        new_name = "#{folder}-#{imap.uid_validity}#{extra}"
-        new_folder_path = folder_path_for(path, new_name)
-        test_mbox = Serializer::Mbox.new(new_folder_path)
-        test_imap = Serializer::Imap.new(new_folder_path)
-        break if !test_mbox.exist? && !test_imap.exist?
-
-        digit += 1
-      end
-
-      previous = folder
-      rename(new_name)
-      @folder = previous
-
+      new_name = Serializer::UnusedNameFinder.new(serializer: self).run
+      rename new_name
       new_name
     end
   end