imap-backup 9.4.0.pre1 → 10.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bd7a94d72f09a4364d3fd424b2e7fabbf4885726b07f0ffbb3e28a6a08e1507f
-  data.tar.gz: ddc3624e1e7441422b93be0d5b651b80b4e7f78268e267573338cec557de2ae2
+  metadata.gz: a244374a14bae1b8d9bee48293eb744266d4d9fa67be560a3ca52a25b49a0cfa
+  data.tar.gz: 03fc3fe13037bcc3c9927117a5fb9edd4a85e053ea3d8e8f2380d8daee514c9d
 SHA512:
-  metadata.gz: '09734ca2ee62e60830358a0d5404d2a5d4f8b8f47b1002cf5373c41a6f4f42cbd3487c66fd7c964c193b41b5b99fa81a173ae3061d43ebcfd8ca56eaee66010d'
-  data.tar.gz: 1ebccf8170e1830222ba9f720be48668b067288bfe89754a98f0bc0146ed96086e8df2fb943c4ecd4f7ed7c4d75297b456a61f96225701e74d79c7aaba39529d
+  metadata.gz: fc751d9d7f9af7f483536251234b5b40ffa9813d947d0739229ab14bf3d35d8878af6656d2180a490ad21b06d5b99c746547a95b75b6925c968fca51b91ba917
+  data.tar.gz: 8f4944c21e5458dda098554b3338c5baaecfe2e111fc5acc23ca0bcf1a20794202a80d0139a9932bba28960702ed8531881ea5bc7030af1b04a2e98afdc6dd22

data/docs/development.md

@@ -21,6 +21,12 @@ Before running the test suite, it needs to be started:
 $ docker-compose up -d
 ```
 
+or, with Podman
+
+```sh
+$ podman-compose -f docker-compose.yml up -d
+```
+
 ```sh
 $ rake
 ```

data/lib/email/mboxrd/message.rb

@@ -38,7 +38,7 @@ module Email::Mboxrd
     end
 
     def subject
-      parsed.subject&.force_encoding(Encoding::ASCII_8BIT)
+      parsed.subject
     end
 
     def imap_body
@@ -62,8 +62,9 @@ module Email::Mboxrd
 
     def best_from
       return first_from if first_from
-      return sender if sender
-      return return_path if return_path
+      return parsed.sender if parsed.sender
+      return parsed.envelope_from if parsed.envelope_from
+      return parsed.return_path if parsed.return_path
 
       ""
     end
@@ -71,16 +72,7 @@ module Email::Mboxrd
     def first_from
       return nil if !parsed.from.is_a?(Enumerable)
 
-      first = parsed.from.compact.first
-      first&.force_encoding(Encoding::ASCII_8BIT)
-    end
-
-    def return_path
-      parsed.return_path&.force_encoding(Encoding::ASCII_8BIT)
-    end
-
-    def sender
-      parsed.sender&.force_encoding(Encoding::ASCII_8BIT)
+      parsed.from.find { |from| from }
     end
 
     def mboxrd_body

data/lib/imap/backup/account/backup.rb

@@ -0,0 +1,65 @@
+require "imap/backup/account/folder_ensurer"
+require "imap/backup/account/local_only_folder_deleter"
+require "imap/backup/account/serialized_folders"
+require "imap/backup/downloader"
+require "imap/backup/flag_refresher"
+require "imap/backup/local_only_message_deleter"
+
+module Imap::Backup
+  class Account; end
+
+  class Account::Backup
+    attr_reader :account
+    attr_reader :refresh
+
+    def initialize(account:, refresh: false)
+      @account = account
+      @refresh = refresh
+    end
+
+    def run
+      Logger.logger.info "Running backup of account: #{account.username}"
+      # start the connection so we get logging messages in the right order
+      account.client
+
+      Account::FolderEnsurer.new(account: account).run
+      Account::LocalOnlyFolderDeleter.new(account: account).run if account.mirror_mode
+      each_folder do |folder, serializer|
+        begin
+          next if !folder.exist?
+        rescue Encoding::UndefinedConversionError
+          message = "Skipping backup for '#{folder.name}' " \
+                    "as it is not UTF-7 encoded correctly"
+          Logger.logger.info message
+          next
+        end
+
+        Logger.logger.debug "[#{folder.name}] running backup"
+        serializer.apply_uid_validity(folder.uid_validity)
+        Downloader.new(
+          folder,
+          serializer,
+          multi_fetch_size: account.multi_fetch_size,
+          reset_seen_flags_after_fetch: account.reset_seen_flags_after_fetch
+        ).run
+        if account.mirror_mode
+          Logger.logger.info "Mirror mode - Deleting messages only present locally"
+          LocalOnlyMessageDeleter.new(folder, serializer).run
+        end
+        FlagRefresher.new(folder, serializer).run if account.mirror_mode || refresh
+      end
+    end
+
+    private
+
+    def each_folder
+      backup_folders = Account::BackupFolders.new(
+        client: account.client, account: account
+      )
+      backup_folders.each do |folder|
+        serializer = Serializer.new(account.local_path, folder.name)
+        yield folder, serializer
+      end
+    end
+  end
+end

data/lib/imap/backup/account/{connection/backup_folders.rb → backup_folders.rb}

@@ -1,8 +1,7 @@
 module Imap::Backup
   class Account; end
-  class Account::Connection; end
 
-  class Account::Connection::BackupFolders
+  class Account::BackupFolders
     attr_reader :account
     attr_reader :client
 
@@ -11,8 +10,10 @@ module Imap::Backup
       @account = account
     end
 
-    def run
-      all_names = Account::Connection::FolderNames.new(client: client, account: account).run
+    def each(&block)
+      return enum_for(:each) if !block
+
+      all_names = client.list
 
       configured =
         case
@@ -31,7 +32,13 @@ module Imap::Backup
           all_names & configured
         end
 
-      names.map { |name| Account::Folder.new(account.connection, name) }
+      names.each { |name| block.call(Account::Folder.new(client, name)) }
+    end
+
+    def map(&block)
+      each.map do |folder|
+        block.call(folder)
+      end
     end
   end
 end

data/lib/imap/backup/account/{connection/client_factory.rb → client_factory.rb}

@@ -1,11 +1,17 @@
+require "socket"
+
 require "email/provider"
+require "imap/backup/client/apple_mail"
+require "imap/backup/client/default"
 require "retry_on_error"
 
 module Imap::Backup
-  class Account::Connection::ClientFactory
+  class Account; end
+
+  class Account::ClientFactory
     include RetryOnError
 
-    LOGIN_RETRY_CLASSES = [EOFError, Errno::ECONNRESET, SocketError].freeze
+    LOGIN_RETRY_CLASSES = [::EOFError, ::Errno::ECONNRESET, ::SocketError].freeze
 
     attr_reader :account
 
@@ -23,23 +29,17 @@ module Imap::Backup
         )
         client =
           if provider.is_a?(Email::Provider::AppleMail)
-            Client::AppleMail.new(server, options)
+            Client::AppleMail.new(server, account, options)
           else
-            Client::Default.new(server, options)
+            Client::Default.new(server, account, options)
           end
-        Logger.logger.debug "Logging in: #{account.username}/#{masked_password}"
-        client.login(account.username, account.password)
-        Logger.logger.debug "Login complete"
+        client.login
         client
       end
     end
 
     private
 
-    def masked_password
-      account.password.gsub(/./, "x")
-    end
-
     def provider
       @provider ||= Email::Provider.for_address(account.username)
     end

data/lib/imap/backup/account/folder.rb

@@ -1,4 +1,5 @@
 require "forwardable"
+require "net/imap"
 
 require "retry_on_error"
 
@@ -12,28 +13,21 @@ module Imap::Backup
     include RetryOnError
 
     BODY_ATTRIBUTE = "BODY[]".freeze
-    UID_FETCH_RETRY_CLASSES = [EOFError, Errno::ECONNRESET, IOError].freeze
-    APPEND_RETRY_CLASSES = [Net::IMAP::BadResponseError].freeze
-    CREATE_RETRY_CLASSES = [Net::IMAP::BadResponseError].freeze
-    EXAMINE_RETRY_CLASSES = [Net::IMAP::BadResponseError].freeze
+    UID_FETCH_RETRY_CLASSES = [::EOFError, ::Errno::ECONNRESET, ::IOError].freeze
+    APPEND_RETRY_CLASSES = [::Net::IMAP::BadResponseError].freeze
+    CREATE_RETRY_CLASSES = [::Net::IMAP::BadResponseError].freeze
+    EXAMINE_RETRY_CLASSES = [::Net::IMAP::BadResponseError].freeze
     PERMITTED_FLAGS = %i(Answered Draft Flagged Seen).freeze
 
-    attr_reader :connection
+    attr_reader :client
     attr_reader :name
 
-    delegate client: :connection
-
-    def initialize(connection, name)
-      @connection = connection
+    def initialize(client, name)
+      @client = client
       @name = name
       @uid_validity = nil
     end
 
-    # Deprecated: use #name
-    def folder
-      name
-    end
-
     def exist?
       retry_on_error(errors: EXAMINE_RETRY_CLASSES) do
         examine

data/lib/imap/backup/account/folder_ensurer.rb

@@ -0,0 +1,24 @@
+require "imap/backup/serializer/directory"
+require "imap/backup/serializer/folder_maker"
+
+module Imap::Backup
+  class Account; end
+
+  class Account::FolderEnsurer
+    attr_reader :account
+
+    def initialize(account:)
+      @account = account
+    end
+
+    def run
+      raise "The backup path for #{account.username} is not set" if !account.local_path
+
+      Serializer::FolderMaker.new(
+        base: File.dirname(account.local_path),
+        path: File.basename(account.local_path),
+        permissions: Serializer::Directory::DIRECTORY_PERMISSIONS
+      ).run
+    end
+  end
+end

data/lib/imap/backup/account/local_only_folder_deleter.rb

@@ -0,0 +1,26 @@
+require "imap/backup/account/backup_folders"
+require "imap/backup/account/serialized_folders"
+
+module Imap::Backup
+  class Account; end
+
+  # Deletes serialized folders that are not configured to be backed up
+  class Account::LocalOnlyFolderDeleter
+    attr_reader :account
+
+    def initialize(account:)
+      @account = account
+    end
+
+    def run
+      backup_folders = Account::BackupFolders.new(
+        client: account.client, account: account
+      )
+      wanted = backup_folders.map(&:name)
+      serialized_folders = Account::SerializedFolders.new(account: account)
+      serialized_folders.each do |serializer, _folder|
+        serializer.delete if !wanted.include?(serializer.folder)
+      end
+    end
+  end
+end

data/lib/imap/backup/account/restore.rb

@@ -0,0 +1,20 @@
+require "imap/backup/account/serialized_folders"
+
+module Imap::Backup
+  class Account; end
+
+  class Account::Restore
+    attr_reader :account
+
+    def initialize(account:)
+      @account = account
+    end
+
+    def run
+      serialized_folders = Account::SerializedFolders.new(account: account)
+      serialized_folders.each do |serializer, folder|
+        Uploader.new(folder, serializer).run
+      end
+    end
+  end
+end

data/lib/imap/backup/account/serialized_folders.rb

@@ -0,0 +1,41 @@
+require "imap/backup/account/folder_ensurer"
+
+module Imap::Backup
+  class Account; end
+
+  class Account::SerializedFolders
+    include Enumerable
+
+    attr_reader :account
+
+    def initialize(account:)
+      @account = account
+    end
+
+    def each(&block)
+      return enum_for(:each) if !block
+
+      glob.each do |path|
+        name = path.relative_path_from(base).to_s[0..-6]
+        serializer = Serializer.new(account.local_path, name)
+        folder = Account::Folder.new(account.client, name)
+        block.call(serializer, folder)
+      end
+    end
+
+    private
+
+    def base
+      @base ||= Pathname.new(account.local_path)
+    end
+
+    def glob
+      @glob ||= begin
+        Account::FolderEnsurer.new(account: account).run
+
+        pattern = File.join(account.local_path, "**", "*.imap")
+        Pathname.glob(pattern)
+      end
+    end
+  end
+end

data/lib/imap/backup/account.rb

@@ -1,3 +1,6 @@
+require "imap/backup/account/client_factory"
+require "imap/backup/account/restore"
+
 module Imap; end
 
 module Imap::Backup
@@ -26,13 +29,22 @@ module Imap::Backup
       @connection_options = options[:connection_options]
       @multi_fetch_size = options[:multi_fetch_size]
       @reset_seen_flags_after_fetch = options[:reset_seen_flags_after_fetch]
-      @connection = nil
+      @client = nil
       @changes = {}
       @marked_for_deletion = false
     end
 
-    def connection
-      @connection ||= Account::Connection.new(self)
+    def client
+      @client ||= Account::ClientFactory.new(account: self).run
+    end
+
+    def namespaces
+      client.namespace
+    end
+
+    def restore
+      restore = Account::Restore.new(account: self)
+      restore.run
     end
 
     def valid?
@@ -145,7 +157,7 @@ module Imap::Backup
         changes[field] = {from: current, to: value} if value != current
       end
 
-      @connection = nil
+      @client = nil
       instance_variable_set(key, value)
     end
   end

data/lib/imap/backup/cli/backup.rb

@@ -1,3 +1,5 @@
+require "imap/backup/account/backup"
+
 module Imap::Backup
   class CLI::Backup < Thor
     include Thor::Actions
@@ -13,21 +15,18 @@ module Imap::Backup
     no_commands do
       def run
         config = load_config(**options)
-        each_connection(config, emails) do |connection|
-          connection.run_backup(refresh: refresh)
+        requested_accounts(config).each do |account|
+          backup = Account::Backup.new(account: account, refresh: refresh)
+          backup.run
         rescue StandardError => e
           message =
-            "Backup for account '#{connection.account.username}' " \
+            "Backup for account '#{account.username}' " \
             "failed with error #{e}"
           Logger.logger.warn message
           next
         end
       end
 
-      def emails
-        (options[:accounts] || "").split(",")
-      end
-
       def refresh
         options.key?(:refresh) ? !!options[:refresh] : false
       end

data/lib/imap/backup/cli/folder_enumerator.rb

@@ -72,7 +72,7 @@ module Imap::Backup
       destination_name = with_destination_prefix.join(destination_delimiter)
 
       Account::Folder.new(
-        destination.connection,
+        destination.client,
         destination_name
       )
     end

data/lib/imap/backup/cli/helpers.rb

@@ -4,6 +4,15 @@ module Imap::Backup
   module CLI::Helpers
     def self.included(base)
       base.class_eval do
+        def self.accounts_option
+          method_option(
+            "accounts",
+            type: :string,
+            desc: "a comma-separated list of accounts (defaults to all configured accounts)",
+            aliases: ["-a"]
+          )
+        end
+
         def self.config_option
           method_option(
             "config",
@@ -78,22 +87,13 @@ module Imap::Backup
       account
     end
 
-    def connection(config, email)
-      account = account(config, email)
-
-      Account::Connection.new(account)
-    end
-
-    def each_connection(config, names)
-      return enum_for(:each_connection, config, names) if !block_given?
-
-      config.accounts.each do |account|
-        next if names.any? && !names.include?(account.username)
-
-        yield account.connection
+    def requested_accounts(config)
+      emails = (options[:accounts] || "").split(",")
+      if emails.any?
+        config.accounts.filter { |a| emails.include?(a.username) }
+      else
+        config.accounts
       end
-    rescue ConfigurationNotFound
-      raise "imap-backup is not configured. Run `imap-backup setup`"
     end
   end
 end

data/lib/imap/backup/cli/local.rb

@@ -8,8 +8,9 @@ module Imap::Backup
     desc "accounts", "List locally backed-up accounts"
     config_option
     format_option
+    quiet_option
+    verbose_option
     def accounts
-      config = load_config(**options)
       names = config.accounts.map(&:username)
       case options[:format]
       when "json"
@@ -29,13 +30,16 @@ module Imap::Backup
       type: :boolean,
       desc: "deletes any corrupted folders - USE WITH CAUTION!"
     )
+    accounts_option
     config_option
     format_option
+    quiet_option
+    verbose_option
     def check
-      config = load_config(**options)
-      results = each_connection(config, emails).map do |connection|
-        folders = connection.local_folders
-        folder_results = folders.map do |serializer|
+      results = requested_accounts(config).map do |account|
+        serialized_folders = Account::SerializedFolders.new(account: account)
+        folder_results = serialized_folders.map do |serializer, _folder|
+          puts "serializer: #{serializer.inspect}"
           serializer.check_integrity!
           {name: serializer.folder, result: "OK"}
         rescue Serializer::FolderIntegrityError => e
@@ -50,7 +54,7 @@ module Imap::Backup
             result: message
           }
         end
-        {account: connection.account.username, folders: folder_results}
+        {account: account.username, folders: folder_results}
       end
 
       case options[:format]
@@ -64,17 +68,18 @@ module Imap::Backup
     desc "folders EMAIL", "List backed up folders"
     config_option
     format_option
+    quiet_option
+    verbose_option
     def folders(email)
-      config = load_config(**options)
-      connection = connection(config, email)
+      account = account(config, email)
 
-      folders = connection.local_folders
+      serialized_folders = Account::SerializedFolders.new(account: account)
       case options[:format]
       when "json"
-        list = folders.map { |_s, f| {name: f.name} }
+        list = serialized_folders.map { |_s, f| {name: f.name} }
         Kernel.puts list.to_json
       else
-        folders.each do |_s, f|
+        serialized_folders.each do |_s, f|
           Kernel.puts %("#{f.name}")
         end
       end
@@ -83,11 +88,13 @@ module Imap::Backup
     desc "list EMAIL FOLDER", "List emails in a folder"
     config_option
     format_option
+    quiet_option
+    verbose_option
     def list(email, folder_name)
-      config = load_config(**options)
-      connection = connection(config, email)
+      account = account(config, email)
 
-      serializer, _folder = connection.local_folders.find do |(_s, f)|
+      serialized_folders = Account::SerializedFolders.new(account: account)
+      serializer, _folder = serialized_folders.find do |_s, f|
         f.name == folder_name
       end
       raise "Folder '#{folder_name}' not found" if !serializer
@@ -108,11 +115,13 @@ module Imap::Backup
     DESC
     config_option
     format_option
+    quiet_option
+    verbose_option
     def show(email, folder_name, uids)
-      config = load_config(**options)
-      connection = connection(config, email)
+      account = account(config, email)
 
-      serializer, _folder = connection.local_folders.find do |(_s, f)|
+      serialized_folders = Account::SerializedFolders.new(account: account)
+      serializer, _folder = serialized_folders.find do |_s, f|
         f.name == folder_name
       end
       raise "Folder '#{folder_name}' not found" if !serializer
@@ -197,8 +206,11 @@ module Imap::Backup
         end
       end
 
-      def emails
-        (options[:accounts] || "").split(",")
+      def config
+        @config ||= begin
+          non_logging_options = Logger.setup_logging(options)
+          load_config(**non_logging_options)
+        end
       end
     end
   end

data/lib/imap/backup/cli/mirror.rb

@@ -61,7 +61,7 @@ module Imap::Backup
         message =
           "The account '#{source_account.username}' " \
           "is not set up to make mirror backups"
-        Logger.logger.info message
+        Logger.logger.warn message
       end
 
       def config

data/lib/imap/backup/cli/remote.rb

@@ -12,12 +12,12 @@ module Imap::Backup
     verbose_option
     def folders(email)
       Imap::Backup::Logger.setup_logging options
-      names = names(email)
+      folder_names = folder_names(email)
       case options[:format]
       when "json"
-        json_format_names names
+        json_format_names folder_names
       else
-        list_names names
+        list_names folder_names
       end
     end
 
@@ -35,8 +35,8 @@ module Imap::Backup
     def namespaces(email)
       Imap::Backup::Logger.setup_logging options
       config = load_config(**options)
-      connection = connection(config, email)
-      namespaces = connection.namespaces
+      account = account(config, email)
+      namespaces = account.namespaces
       case options[:format]
       when "json"
         json_format_namespaces namespaces
@@ -46,11 +46,11 @@ module Imap::Backup
     end
 
     no_commands do
-      def names(email)
+      def folder_names(email)
         config = load_config(**options)
-        connection = connection(config, email)
+        account = account(config, email)
 
-        connection.folder_names
+        account.client.list
       end
 
       def json_format_names(names)