imap-backup 9.3.2 → 10.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 35c15f0cf7b9909868b55bb5c9ec928cb5c22e6943a9856fbf52b1bd9fb0212b
-  data.tar.gz: d4cfaf196b031f91c0e745a55def95cdc480906c7b387a888949c80bc607867f
+  metadata.gz: a244374a14bae1b8d9bee48293eb744266d4d9fa67be560a3ca52a25b49a0cfa
+  data.tar.gz: 03fc3fe13037bcc3c9927117a5fb9edd4a85e053ea3d8e8f2380d8daee514c9d
 SHA512:
-  metadata.gz: bca6edbdf041a3795d9f97ba1df9f63feaeee7d024df73da389990cc2a235b73a0379096261b1c515344746cb256380f79c7b2a99abdd78b2a279049b6184bbf
-  data.tar.gz: 966e04781677d2f281810a2160a5f56dfea17dd69c7694948cf8a36e96616685708d5c599e8bb661f7441fbdf7a0accc66fbcbb9d278bbe375c685af62e8a0b7
+  metadata.gz: fc751d9d7f9af7f483536251234b5b40ffa9813d947d0739229ab14bf3d35d8878af6656d2180a490ad21b06d5b99c746547a95b75b6925c968fca51b91ba917
+  data.tar.gz: 8f4944c21e5458dda098554b3338c5baaecfe2e111fc5acc23ca0bcf1a20794202a80d0139a9932bba28960702ed8531881ea5bc7030af1b04a2e98afdc6dd22

data/lib/imap/backup/account/backup.rb

@@ -0,0 +1,65 @@
+require "imap/backup/account/folder_ensurer"
+require "imap/backup/account/local_only_folder_deleter"
+require "imap/backup/account/serialized_folders"
+require "imap/backup/downloader"
+require "imap/backup/flag_refresher"
+require "imap/backup/local_only_message_deleter"
+
+module Imap::Backup
+  class Account; end
+
+  class Account::Backup
+    attr_reader :account
+    attr_reader :refresh
+
+    def initialize(account:, refresh: false)
+      @account = account
+      @refresh = refresh
+    end
+
+    def run
+      Logger.logger.info "Running backup of account: #{account.username}"
+      # start the connection so we get logging messages in the right order
+      account.client
+
+      Account::FolderEnsurer.new(account: account).run
+      Account::LocalOnlyFolderDeleter.new(account: account).run if account.mirror_mode
+      each_folder do |folder, serializer|
+        begin
+          next if !folder.exist?
+        rescue Encoding::UndefinedConversionError
+          message = "Skipping backup for '#{folder.name}' " \
+                    "as it is not UTF-7 encoded correctly"
+          Logger.logger.info message
+          next
+        end
+
+        Logger.logger.debug "[#{folder.name}] running backup"
+        serializer.apply_uid_validity(folder.uid_validity)
+        Downloader.new(
+          folder,
+          serializer,
+          multi_fetch_size: account.multi_fetch_size,
+          reset_seen_flags_after_fetch: account.reset_seen_flags_after_fetch
+        ).run
+        if account.mirror_mode
+          Logger.logger.info "Mirror mode - Deleting messages only present locally"
+          LocalOnlyMessageDeleter.new(folder, serializer).run
+        end
+        FlagRefresher.new(folder, serializer).run if account.mirror_mode || refresh
+      end
+    end
+
+    private
+
+    def each_folder
+      backup_folders = Account::BackupFolders.new(
+        client: account.client, account: account
+      )
+      backup_folders.each do |folder|
+        serializer = Serializer.new(account.local_path, folder.name)
+        yield folder, serializer
+      end
+    end
+  end
+end

data/lib/imap/backup/account/{connection/backup_folders.rb → backup_folders.rb}

@@ -1,8 +1,7 @@
 module Imap::Backup
   class Account; end
-  class Account::Connection; end
 
-  class Account::Connection::BackupFolders
+  class Account::BackupFolders
     attr_reader :account
     attr_reader :client
 
@@ -11,8 +10,10 @@ module Imap::Backup
       @account = account
     end
 
-    def run
-      all_names = Account::Connection::FolderNames.new(client: client, account: account).run
+    def each(&block)
+      return enum_for(:each) if !block
+
+      all_names = client.list
 
       configured =
         case
@@ -31,7 +32,13 @@ module Imap::Backup
           all_names & configured
         end
 
-      names.map { |name| Account::Folder.new(account.connection, name) }
+      names.each { |name| block.call(Account::Folder.new(client, name)) }
+    end
+
+    def map(&block)
+      each.map do |folder|
+        block.call(folder)
+      end
     end
   end
 end

data/lib/imap/backup/account/{connection/client_factory.rb → client_factory.rb}

@@ -1,11 +1,17 @@
+require "socket"
+
 require "email/provider"
+require "imap/backup/client/apple_mail"
+require "imap/backup/client/default"
 require "retry_on_error"
 
 module Imap::Backup
-  class Account::Connection::ClientFactory
+  class Account; end
+
+  class Account::ClientFactory
     include RetryOnError
 
-    LOGIN_RETRY_CLASSES = [EOFError, Errno::ECONNRESET, SocketError].freeze
+    LOGIN_RETRY_CLASSES = [::EOFError, ::Errno::ECONNRESET, ::SocketError].freeze
 
     attr_reader :account
 
@@ -23,23 +29,17 @@ module Imap::Backup
         )
         client =
           if provider.is_a?(Email::Provider::AppleMail)
-            Client::AppleMail.new(server, options)
+            Client::AppleMail.new(server, account, options)
           else
-            Client::Default.new(server, options)
+            Client::Default.new(server, account, options)
           end
-        Logger.logger.debug "Logging in: #{account.username}/#{masked_password}"
-        client.login(account.username, account.password)
-        Logger.logger.debug "Login complete"
+        client.login
         client
       end
     end
 
     private
 
-    def masked_password
-      account.password.gsub(/./, "x")
-    end
-
     def provider
       @provider ||= Email::Provider.for_address(account.username)
     end

data/lib/imap/backup/account/folder.rb

@@ -1,4 +1,5 @@
 require "forwardable"
+require "net/imap"
 
 require "retry_on_error"
 
@@ -12,28 +13,21 @@ module Imap::Backup
     include RetryOnError
 
     BODY_ATTRIBUTE = "BODY[]".freeze
-    UID_FETCH_RETRY_CLASSES = [EOFError, Errno::ECONNRESET, IOError].freeze
-    APPEND_RETRY_CLASSES = [Net::IMAP::BadResponseError].freeze
-    CREATE_RETRY_CLASSES = [Net::IMAP::BadResponseError].freeze
-    EXAMINE_RETRY_CLASSES = [Net::IMAP::BadResponseError].freeze
+    UID_FETCH_RETRY_CLASSES = [::EOFError, ::Errno::ECONNRESET, ::IOError].freeze
+    APPEND_RETRY_CLASSES = [::Net::IMAP::BadResponseError].freeze
+    CREATE_RETRY_CLASSES = [::Net::IMAP::BadResponseError].freeze
+    EXAMINE_RETRY_CLASSES = [::Net::IMAP::BadResponseError].freeze
     PERMITTED_FLAGS = %i(Answered Draft Flagged Seen).freeze
 
-    attr_reader :connection
+    attr_reader :client
     attr_reader :name
 
-    delegate client: :connection
-
-    def initialize(connection, name)
-      @connection = connection
+    def initialize(client, name)
+      @client = client
       @name = name
       @uid_validity = nil
     end
 
-    # Deprecated: use #name
-    def folder
-      name
-    end
-
     def exist?
       retry_on_error(errors: EXAMINE_RETRY_CLASSES) do
         examine

data/lib/imap/backup/account/folder_ensurer.rb

@@ -0,0 +1,24 @@
+require "imap/backup/serializer/directory"
+require "imap/backup/serializer/folder_maker"
+
+module Imap::Backup
+  class Account; end
+
+  class Account::FolderEnsurer
+    attr_reader :account
+
+    def initialize(account:)
+      @account = account
+    end
+
+    def run
+      raise "The backup path for #{account.username} is not set" if !account.local_path
+
+      Serializer::FolderMaker.new(
+        base: File.dirname(account.local_path),
+        path: File.basename(account.local_path),
+        permissions: Serializer::Directory::DIRECTORY_PERMISSIONS
+      ).run
+    end
+  end
+end

data/lib/imap/backup/account/local_only_folder_deleter.rb

@@ -0,0 +1,26 @@
+require "imap/backup/account/backup_folders"
+require "imap/backup/account/serialized_folders"
+
+module Imap::Backup
+  class Account; end
+
+  # Deletes serialized folders that are not configured to be backed up
+  class Account::LocalOnlyFolderDeleter
+    attr_reader :account
+
+    def initialize(account:)
+      @account = account
+    end
+
+    def run
+      backup_folders = Account::BackupFolders.new(
+        client: account.client, account: account
+      )
+      wanted = backup_folders.map(&:name)
+      serialized_folders = Account::SerializedFolders.new(account: account)
+      serialized_folders.each do |serializer, _folder|
+        serializer.delete if !wanted.include?(serializer.folder)
+      end
+    end
+  end
+end

data/lib/imap/backup/account/restore.rb

@@ -0,0 +1,20 @@
+require "imap/backup/account/serialized_folders"
+
+module Imap::Backup
+  class Account; end
+
+  class Account::Restore
+    attr_reader :account
+
+    def initialize(account:)
+      @account = account
+    end
+
+    def run
+      serialized_folders = Account::SerializedFolders.new(account: account)
+      serialized_folders.each do |serializer, folder|
+        Uploader.new(folder, serializer).run
+      end
+    end
+  end
+end

data/lib/imap/backup/account/serialized_folders.rb

@@ -0,0 +1,41 @@
+require "imap/backup/account/folder_ensurer"
+
+module Imap::Backup
+  class Account; end
+
+  class Account::SerializedFolders
+    include Enumerable
+
+    attr_reader :account
+
+    def initialize(account:)
+      @account = account
+    end
+
+    def each(&block)
+      return enum_for(:each) if !block
+
+      glob.each do |path|
+        name = path.relative_path_from(base).to_s[0..-6]
+        serializer = Serializer.new(account.local_path, name)
+        folder = Account::Folder.new(account.client, name)
+        block.call(serializer, folder)
+      end
+    end
+
+    private
+
+    def base
+      @base ||= Pathname.new(account.local_path)
+    end
+
+    def glob
+      @glob ||= begin
+        Account::FolderEnsurer.new(account: account).run
+
+        pattern = File.join(account.local_path, "**", "*.imap")
+        Pathname.glob(pattern)
+      end
+    end
+  end
+end

data/lib/imap/backup/account.rb

@@ -1,3 +1,6 @@
+require "imap/backup/account/client_factory"
+require "imap/backup/account/restore"
+
 module Imap; end
 
 module Imap::Backup
@@ -26,13 +29,22 @@ module Imap::Backup
       @connection_options = options[:connection_options]
       @multi_fetch_size = options[:multi_fetch_size]
       @reset_seen_flags_after_fetch = options[:reset_seen_flags_after_fetch]
-      @connection = nil
+      @client = nil
       @changes = {}
       @marked_for_deletion = false
     end
 
-    def connection
-      @connection ||= Account::Connection.new(self)
+    def client
+      @client ||= Account::ClientFactory.new(account: self).run
+    end
+
+    def namespaces
+      client.namespace
+    end
+
+    def restore
+      restore = Account::Restore.new(account: self)
+      restore.run
     end
 
     def valid?
@@ -145,7 +157,7 @@ module Imap::Backup
         changes[field] = {from: current, to: value} if value != current
       end
 
-      @connection = nil
+      @client = nil
       instance_variable_set(key, value)
     end
   end

data/lib/imap/backup/cli/backup.rb

@@ -1,3 +1,5 @@
+require "imap/backup/account/backup"
+
 module Imap::Backup
   class CLI::Backup < Thor
     include Thor::Actions
@@ -13,21 +15,18 @@ module Imap::Backup
     no_commands do
       def run
         config = load_config(**options)
-        each_connection(config, emails) do |connection|
-          connection.run_backup(refresh: refresh)
+        requested_accounts(config).each do |account|
+          backup = Account::Backup.new(account: account, refresh: refresh)
+          backup.run
         rescue StandardError => e
           message =
-            "Backup for account '#{connection.account.username}' " \
+            "Backup for account '#{account.username}' " \
             "failed with error #{e}"
           Logger.logger.warn message
           next
         end
       end
 
-      def emails
-        (options[:accounts] || "").split(",")
-      end
-
       def refresh
         options.key?(:refresh) ? !!options[:refresh] : false
       end

data/lib/imap/backup/cli/folder_enumerator.rb

@@ -72,7 +72,7 @@ module Imap::Backup
       destination_name = with_destination_prefix.join(destination_delimiter)
 
       Account::Folder.new(
-        destination.connection,
+        destination.client,
         destination_name
       )
     end

data/lib/imap/backup/cli/helpers.rb

@@ -4,6 +4,15 @@ module Imap::Backup
   module CLI::Helpers
     def self.included(base)
       base.class_eval do
+        def self.accounts_option
+          method_option(
+            "accounts",
+            type: :string,
+            desc: "a comma-separated list of accounts (defaults to all configured accounts)",
+            aliases: ["-a"]
+          )
+        end
+
         def self.config_option
           method_option(
             "config",
@@ -78,22 +87,13 @@ module Imap::Backup
       account
     end
 
-    def connection(config, email)
-      account = account(config, email)
-
-      Account::Connection.new(account)
-    end
-
-    def each_connection(config, names)
-      return enum_for(:each_connection, config, names) if !block_given?
-
-      config.accounts.each do |account|
-        next if names.any? && !names.include?(account.username)
-
-        yield account.connection
+    def requested_accounts(config)
+      emails = (options[:accounts] || "").split(",")
+      if emails.any?
+        config.accounts.filter { |a| emails.include?(a.username) }
+      else
+        config.accounts
       end
-    rescue ConfigurationNotFound
-      raise "imap-backup is not configured. Run `imap-backup setup`"
     end
   end
 end

data/lib/imap/backup/cli/local.rb

@@ -8,8 +8,9 @@ module Imap::Backup
     desc "accounts", "List locally backed-up accounts"
     config_option
     format_option
+    quiet_option
+    verbose_option
     def accounts
-      config = load_config(**options)
       names = config.accounts.map(&:username)
       case options[:format]
       when "json"
@@ -29,13 +30,16 @@ module Imap::Backup
       type: :boolean,
       desc: "deletes any corrupted folders - USE WITH CAUTION!"
     )
+    accounts_option
     config_option
     format_option
+    quiet_option
+    verbose_option
     def check
-      config = load_config(**options)
-      results = each_connection(config, emails).map do |connection|
-        folders = connection.local_folders
-        folder_results = folders.map do |serializer|
+      results = requested_accounts(config).map do |account|
+        serialized_folders = Account::SerializedFolders.new(account: account)
+        folder_results = serialized_folders.map do |serializer, _folder|
+          puts "serializer: #{serializer.inspect}"
           serializer.check_integrity!
           {name: serializer.folder, result: "OK"}
         rescue Serializer::FolderIntegrityError => e
@@ -50,7 +54,7 @@ module Imap::Backup
             result: message
           }
         end
-        {account: connection.account.username, folders: folder_results}
+        {account: account.username, folders: folder_results}
       end
 
       case options[:format]
@@ -64,17 +68,18 @@ module Imap::Backup
     desc "folders EMAIL", "List backed up folders"
     config_option
     format_option
+    quiet_option
+    verbose_option
     def folders(email)
-      config = load_config(**options)
-      connection = connection(config, email)
+      account = account(config, email)
 
-      folders = connection.local_folders
+      serialized_folders = Account::SerializedFolders.new(account: account)
       case options[:format]
       when "json"
-        list = folders.map { |_s, f| {name: f.name} }
+        list = serialized_folders.map { |_s, f| {name: f.name} }
         Kernel.puts list.to_json
       else
-        folders.each do |_s, f|
+        serialized_folders.each do |_s, f|
           Kernel.puts %("#{f.name}")
         end
       end
@@ -83,11 +88,13 @@ module Imap::Backup
     desc "list EMAIL FOLDER", "List emails in a folder"
     config_option
     format_option
+    quiet_option
+    verbose_option
     def list(email, folder_name)
-      config = load_config(**options)
-      connection = connection(config, email)
+      account = account(config, email)
 
-      serializer, _folder = connection.local_folders.find do |(_s, f)|
+      serialized_folders = Account::SerializedFolders.new(account: account)
+      serializer, _folder = serialized_folders.find do |_s, f|
         f.name == folder_name
       end
       raise "Folder '#{folder_name}' not found" if !serializer
@@ -108,11 +115,13 @@ module Imap::Backup
     DESC
     config_option
     format_option
+    quiet_option
+    verbose_option
     def show(email, folder_name, uids)
-      config = load_config(**options)
-      connection = connection(config, email)
+      account = account(config, email)
 
-      serializer, _folder = connection.local_folders.find do |(_s, f)|
+      serialized_folders = Account::SerializedFolders.new(account: account)
+      serializer, _folder = serialized_folders.find do |_s, f|
         f.name == folder_name
       end
       raise "Folder '#{folder_name}' not found" if !serializer
@@ -197,8 +206,11 @@ module Imap::Backup
         end
       end
 
-      def emails
-        (options[:accounts] || "").split(",")
+      def config
+        @config ||= begin
+          non_logging_options = Logger.setup_logging(options)
+          load_config(**non_logging_options)
+        end
       end
     end
   end

data/lib/imap/backup/cli/mirror.rb

@@ -61,7 +61,7 @@ module Imap::Backup
         message =
           "The account '#{source_account.username}' " \
           "is not set up to make mirror backups"
-        Logger.logger.info message
+        Logger.logger.warn message
       end
 
       def config

data/lib/imap/backup/cli/remote.rb

@@ -12,12 +12,12 @@ module Imap::Backup
     verbose_option
     def folders(email)
       Imap::Backup::Logger.setup_logging options
-      names = names(email)
+      folder_names = folder_names(email)
       case options[:format]
       when "json"
-        json_format_names names
+        json_format_names folder_names
       else
-        list_names names
+        list_names folder_names
       end
     end
 
@@ -35,8 +35,8 @@ module Imap::Backup
     def namespaces(email)
       Imap::Backup::Logger.setup_logging options
       config = load_config(**options)
-      connection = connection(config, email)
-      namespaces = connection.namespaces
+      account = account(config, email)
+      namespaces = account.namespaces
       case options[:format]
       when "json"
         json_format_namespaces namespaces
@@ -46,11 +46,11 @@ module Imap::Backup
     end
 
     no_commands do
-      def names(email)
+      def folder_names(email)
         config = load_config(**options)
-        connection = connection(config, email)
+        account = account(config, email)
 
-        connection.folder_names
+        account.client.list
       end
 
       def json_format_names(names)

data/lib/imap/backup/cli/restore.rb

@@ -16,26 +16,22 @@ module Imap::Backup
       def run
         config = load_config(**options)
         case
-        when email && !emails
-          connection = connection(config, email)
-          connection.restore
-        when !email && !emails
+        when email && !options.key?(:accounts)
+          account = account(config, email)
+          account.restore
+        when !email && !options.key?(:accounts)
           Logger.logger.info "Calling restore without an EMAIL parameter is deprecated"
-          each_connection(config, [], &:restore)
-        when email && emails.any?
-          raise "Pass either an email or the --accounts option, not both"
-        when emails.any?
+          config.accounts.map(&:restore)
+        when email && options.key?(:accounts)
+          raise "Missing EMAIL parameter"
+        when !email && options.key?(:accounts)
           Logger.logger.info(
             "Calling restore with the --account option is deprected, " \
-            "please pass a single EMAIL argument"
+            "please pass a single EMAIL parameter"
           )
-          each_connection(config, emails, &:restore)
+          requested_accounts(config).each(&:restore)
         end
       end
-
-      def emails
-        @emails ||= options[:accounts].split(",") if options.key?(:accounts)
-      end
     end
   end
 end

data/lib/imap/backup/cli/stats.rb

@@ -1,3 +1,5 @@
+require "imap/backup/account/backup_folders"
+
 module Imap::Backup
   class CLI::Stats < Thor
     include Thor::Actions
@@ -32,12 +34,15 @@ module Imap::Backup
 
       def stats
         config = load_config(**options)
-        connection = connection(config, email)
+        account = account(config, email)
 
-        connection.backup_folders.map do |folder|
+        backup_folders = Account::BackupFolders.new(
+          client: account.client, account: account
+        )
+        backup_folders.map do |folder|
           next if !folder.exist?
 
-          serializer = Serializer.new(connection.account.local_path, folder.name)
+          serializer = Serializer.new(account.local_path, folder.name)
           local_uids = serializer.uids
           remote_uids = folder.uids
           {