imap-backup 7.0.0.rc1 → 7.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d858a14c5bbb9300fd99fe6338cc2ff83fd596071dc2db48ca792f45eee430a3
-  data.tar.gz: c28f18cd459cd9c2c9a62fb5a55234615218bcff2c3c46e0422af9c6893e6043
+  metadata.gz: cb709e6431c47afc24c55d39ef7d9213fc0c27b42929c0692843a254cfd83c1d
+  data.tar.gz: 702d93af210afa826e64c0b5b1a4d1e16780c6078201e4a8db13dd4703ac62d3
 SHA512:
-  metadata.gz: c669d9a170ec1d500e898324bbdc2072cc42597e9c2d30b6b284b7095151afecd0451c9fa7cb0adb93bbf46f1f2b1ae188554507141161acb6ba125349c1cee9
-  data.tar.gz: c53ecb54ed6d33ab574caa18e62ae57e214e76975af65c72b11b8b2612d45487422be234c025966c792260d86d20e5431bf27845772e40d82cc8fea0132f9963
+  metadata.gz: 04a0c173fea0dd80e87443db040383402eb9eacbc7f557b4db8339cfd16c20153b068da8f4ade84d82643e3e9d3b9a76f04d654cbbc931620ef197546b24492a
+  data.tar.gz: e3cb1c12f73c421a66e41016b826ecdf3d199fd5920f9e5cbb04e0ef41619984791c8ce436283a91e6264707b04db7ecef94703f5df9af4c64cbcbb75c0fd7d7

data/README.md

@@ -11,21 +11,15 @@ The backups can then be restored, used to migrate to another service,
 inspected or exported.
 
   * [Source Code]
-  * [Documentation]
+  * [Code Documentation]
   * [Rubygem]
   * [CI Status]
 
 [Source Code]: https://github.com/joeyates/imap-backup "Source code at GitHub"
-[Documentation]: https://rubydoc.info/gems/imap-backup/frames "RDoc API Documentation at Rubydoc.info"
+[Code Documentation]: https://rubydoc.info/gems/imap-backup/frames "Code Documentation at Rubydoc.info"
 [Rubygem]: https://rubygems.org/gems/imap-backup "Ruby gem at rubygems.org"
 [CI Status]: https://github.com/joeyates/imap-backup/actions/workflows/main.yml
 
-# Backup Emails
-
-imap-backup downloads emails and stores them on disk.
-
-The backup is incremental and interruptable, so backups won't get messed if your connection goes down during an operation.
-
 # Installation
 
 ## Homebrew (macOS)
@@ -63,7 +57,7 @@ imap-backup setup
 
 To use imap-backup with GMail, you will need to enable 'App passwords' on your account.
 
-# Run Backup
+# Backup
 
 Manually, from the command line:
 
@@ -73,14 +67,22 @@ imap-backup
 
 Alternatively, add it to your crontab.
 
-Emails are stored on disk in [Mbox files](./docs/files/mboxrd.md) for each folder, with metadata
-stored in [Imap files](./docs/files/imap.md).
-
 # Commands
 
-* [folders](./commands/folders.md)
-* [restore](./commands/restore.md)
-* [status](./commands/status.md)
+* [backup](docs/commands/backup.md)
+* [folders](docs/commands/folders.md)
+* [local accounts](docs/commands/local-accounts.md)
+* [local folders](docs/commands/local-folders.md)
+* [local list](docs/commands/local-list.md)
+* [local show](docs/commands/local-show.md)
+* [migrate](docs/commands/migrate.md)
+* [mirror](docs/commands/mirror.md)
+* [remote folders](docs/commands/remote-folders.md)
+* [restore](docs/commands/restore.md)
+* [setup](docs/commands/setup.md)
+* [status](docs/commands/status.md)
+* [utils export-to-thunderbird](docs/commands/utils-export-to-thunderbird.md)
+* [utils ignore-history](docs/commands/utils-ignore-history.md)
 
 For a full list of available commands, run
 
@@ -94,12 +96,6 @@ For more information about a command, run
 imap-backup help COMMAND
 ```
 
-## Configuration
-
-`imap-backup setup` creates the file `~/.imap-backup/config.json`.
-
-[More information about configuration is available in the specific documentation](./docs/configuration.md).
-
 # Troubleshooting
 
 If you have problems:

data/bin/imap-backup

@@ -8,8 +8,6 @@ CliCoverage.conditionally_activate
 require "imap/backup/cli"
 require "imap/backup/logger"
 
-Imap::Backup::Logger.setup_logging
-
 Imap::Backup::Logger.sanitize_stderr do
   Imap::Backup::CLI.start(ARGV)
 end

data/docs/development.md

@@ -44,7 +44,7 @@ use `last_command_started.output`.
 require "net/imap"
 
 imap = Net::IMAP.new("localhost", {port: 8993, ssl: {verify_mode: 0}})
-username = "address@example.org"
+username = "address@example.com"
 imap.login(username, "pass")
 
 message = "From: #{username}\nSubject: Some Subject\n\nHello!\n"

data/lib/email/mboxrd/message.rb

@@ -5,9 +5,6 @@ module Email; end
 
 module Email::Mboxrd
   class Message
-    extend Forwardable
-    def_delegators :@parsed, :subject
-
     attr_reader :supplied_body
 
     def self.clean_serialized(serialized)
@@ -40,6 +37,10 @@ module Email::Mboxrd
       nil
     end
 
+    def subject
+      parsed.subject
+    end
+
     def imap_body
       supplied_body.gsub(/(?<!\r)\n/, "\r\n")
     end

data/lib/imap/backup/account/connection.rb

@@ -3,6 +3,8 @@ require "imap/backup/client/default"
 require "imap/backup/account/connection/backup_folders"
 require "imap/backup/account/connection/client_factory"
 require "imap/backup/account/connection/folder_names"
+require "imap/backup/flag_refresher"
+require "imap/backup/local_only_message_deleter"
 require "imap/backup/serializer/directory"
 
 module Imap::Backup
@@ -33,11 +35,18 @@ module Imap::Backup
       end
     end
 
-    def run_backup
-      Logger.logger.debug "Running backup of account: #{account.username}"
+    def run_backup(refresh: false)
+      Logger.logger.info "Running backup of account: #{account.username}"
       # start the connection so we get logging messages in the right order
       client
       ensure_account_folder
+      if account.mirror_mode
+        # Delete serialized folders that are not to be backed up
+        wanted = backup_folders.map(&:name)
+        local_folders do |serializer, _folder|
+          serializer.delete if !wanted.include?(serializer.folder)
+        end
+      end
       each_folder do |folder, serializer|
         begin
           next if !folder.exist?
@@ -57,6 +66,11 @@ module Imap::Backup
             multi_fetch_size: account.multi_fetch_size,
             reset_seen_flags_after_fetch: account.reset_seen_flags_after_fetch
           ).run
+          if account.mirror_mode
+            Logger.logger.info "Mirror mode - Deleting messages only present locally"
+            LocalOnlyMessageDeleter.new(folder, serializer).run
+          end
+          FlagRefresher.new(folder, serializer).run if account.mirror_mode || refresh
         rescue Net::IMAP::ByeResponseError
           reconnect
           retry

data/lib/imap/backup/account/folder.rb

@@ -58,22 +58,22 @@ module Imap::Backup
     rescue FolderNotFound
       []
     rescue NoMethodError
-      message = <<~MESSAGE
-        Folder '#{name}' caused NoMethodError
-        probably
-        `undefined method `[]' for nil:NilClass (NoMethodError)`
-        in `search_internal` in stdlib net/imap.rb.
-        This is caused by `@responses["SEARCH"] being unset/undefined
-      MESSAGE
+      message =
+        "Folder '#{name}' caused a NoMethodError. " \
+        "Probably this was `undefined method `[]' for nil:NilClass (NoMethodError)` " \
+        "in `search_internal` in stdlib net/imap.rb. " \
+        'This is caused by `@responses["SEARCH"] being unset/undefined. ' \
+        "Among others, Apple Mail servers send empty responses when " \
+        "folders are empty, causing this error."
       Logger.logger.warn message
       []
     end
 
-    def fetch_multi(uids)
+    def fetch_multi(uids, attr = [BODY_ATTRIBUTE, "FLAGS"])
       examine
       fetch_data_items =
         retry_on_error(errors: UID_FETCH_RETRY_CLASSES) do
-          client.uid_fetch(uids, [BODY_ATTRIBUTE, "FLAGS"])
+          client.uid_fetch(uids, attr)
         end
       return nil if fetch_data_items.nil?
 
@@ -90,18 +90,30 @@ module Imap::Backup
       nil
     end
 
-    def append(message, flags: nil)
+    def append(message)
       body = message.imap_body
       date = message.date&.to_time
       retry_on_error(errors: APPEND_RETRY_CLASSES, limit: 3) do
-        response = client.append(utf7_encoded_name, body, flags, date)
+        response = client.append(utf7_encoded_name, body, message.flags, date)
         extract_uid(response)
       end
     end
 
+    def delete_multi(uids)
+      set_flags(uids, [:Deleted])
+      client.expunge
+    end
+
+    def apply_flags(uids, flags)
+      client.select(utf7_encoded_name)
+      flags.reject! { |f| f == :Recent }
+      client.uid_store(uids, "FLAGS", flags)
+    end
+
     def set_flags(uids, flags)
       # Use read-write access, via `select`
       client.select(utf7_encoded_name)
+      flags.reject! { |f| f == :Recent }
       client.uid_store(uids, "+FLAGS", flags)
     end
 
@@ -111,7 +123,10 @@ module Imap::Backup
     end
 
     def clear
-      set_flags(uids, [:Deleted])
+      existing = uids
+      return if existing.empty?
+
+      set_flags(existing, [:Deleted])
       client.expunge
     end
 

data/lib/imap/backup/account.rb

@@ -1,3 +1,5 @@
+module Imap; end
+
 module Imap::Backup
   class Account
     DEFAULT_MULTI_FETCH_SIZE = 1
@@ -6,6 +8,7 @@ module Imap::Backup
     attr_reader :password
     attr_reader :local_path
     attr_reader :folders
+    attr_reader :mirror_mode
     attr_reader :server
     attr_reader :connection_options
     attr_reader :reset_seen_flags_after_fetch
@@ -16,6 +19,7 @@ module Imap::Backup
       @password = options[:password]
       @local_path = options[:local_path]
       @folders = options[:folders]
+      @mirror_mode = options[:mirror_mode]
       @server = options[:server]
       @connection_options = options[:connection_options]
       @multi_fetch_size = options[:multi_fetch_size]
@@ -53,6 +57,7 @@ module Imap::Backup
       h = {username: @username, password: @password}
       h[:local_path] = @local_path if @local_path
       h[:folders] = @folders if @folders
+      h[:mirror_mode] = true if @mirror_mode
       h[:server] = @server if @server
       h[:connection_options] = @connection_options if @connection_options
       h[:multi_fetch_size] = multi_fetch_size if @multi_fetch_size
@@ -80,12 +85,16 @@ module Imap::Backup
       update(:folders, value)
     end
 
+    def mirror_mode=(value)
+      update(:mirror_mode, value)
+    end
+
     def server=(value)
       update(:server, value)
     end
 
     def connection_options=(value)
-      parsed = JSON.parse(value)
+      parsed = JSON.parse(value, symbolize_names: true)
       update(:connection_options, parsed)
     end
 

data/lib/imap/backup/cli/backup.rb

@@ -4,15 +4,19 @@ module Imap::Backup
     include CLI::Helpers
 
     attr_reader :account_names
+    attr_reader :refresh
 
     def initialize(options)
       super([])
       @account_names = (options[:accounts] || "").split(",")
+      @refresh = options.key?(:refresh) ? !!options[:refresh] : false
     end
 
     no_commands do
       def run
-        each_connection(account_names, &:run_backup)
+        each_connection(account_names) do |connection|
+          connection.run_backup(refresh: refresh)
+        end
       end
     end
   end

data/lib/imap/backup/cli/helpers.rb

@@ -3,6 +3,28 @@ require "imap/backup/cli/accounts"
 
 module Imap::Backup
   module CLI::Helpers
+    def self.included(base)
+      base.class_eval do
+        def self.verbose_option
+          method_option(
+            "verbose",
+            type: :boolean,
+            desc: "increase the amount of logging",
+            aliases: ["-v"]
+          )
+        end
+
+        def self.quiet_option
+          method_option(
+            "quiet",
+            type: :boolean,
+            desc: "silence all output",
+            aliases: ["-q"]
+          )
+        end
+      end
+    end
+
     def symbolized(options)
       options.each.with_object({}) do |(k, v), acc|
         key = k.gsub("-", "_").intern

data/lib/imap/backup/cli/local.rb

@@ -39,8 +39,8 @@ module Imap::Backup
 
       uids = folder_serializer.uids
 
-      folder_serializer.each_message(uids).map do |uid, message|
-        list_message uid, message
+      folder_serializer.each_message(uids).map do |message|
+        list_message message
       end
     end
 
@@ -59,22 +59,22 @@ module Imap::Backup
       raise "Folder '#{folder_name}' not found" if !folder_serializer
 
       uid_list = uids.split(",")
-      folder_serializer.each_message(uid_list).each do |uid, message|
+      folder_serializer.each_message(uid_list).each do |message|
         if uid_list.count > 1
           Kernel.puts <<~HEADER
             #{'-' * 80}
-            #{format('| UID: %-71s |', uid)}
+            #{format('| UID: %-71s |', message.uid)}
             #{'-' * 80}
           HEADER
         end
-        Kernel.puts message.supplied_body
+        Kernel.puts message.body
       end
     end
 
     no_commands do
-      def list_message(uid, message)
+      def list_message(message)
         m = {
-          uid: uid,
+          uid: message.uid,
           date: message.date.to_s,
           subject: message.subject || ""
         }

data/lib/imap/backup/cli/mirror.rb

@@ -0,0 +1,118 @@
+require "imap/backup/mirror"
+
+module Imap::Backup
+  class CLI::Mirror < Thor
+    include Thor::Actions
+
+    attr_reader :destination_email
+    attr_reader :destination_prefix
+    attr_reader :source_email
+    attr_reader :source_prefix
+
+    def initialize(
+      source_email,
+      destination_email,
+      destination_prefix: "",
+      source_prefix: ""
+    )
+      super([])
+      @destination_email = destination_email
+      @destination_prefix = destination_prefix
+      @source_email = source_email
+      @source_prefix = source_prefix
+    end
+
+    no_commands do
+      def run
+        check_accounts!
+        warn_if_source_account_is_not_in_mirror_mode
+
+        CLI::Backup.new(accounts: source_email).run
+
+        folders.each do |serializer, folder|
+          Mirror.new(serializer, folder).run
+        end
+      end
+
+      def check_accounts!
+        if destination_email == source_email
+          raise "Source and destination accounts cannot be the same!"
+        end
+
+        raise "Account '#{destination_email}' does not exist" if !destination_account
+
+        raise "Account '#{source_email}' does not exist" if !source_account
+      end
+
+      def warn_if_source_account_is_not_in_mirror_mode
+        return if source_account.mirror_mode
+
+        message =
+          "The account '#{source_account.username}' " \
+          "is not set up to make mirror backups"
+        Logger.logger.info message
+      end
+
+      def config
+        Configuration.new
+      end
+
+      def destination_account
+        config.accounts.find { |a| a.username == destination_email }
+      end
+
+      def folders
+        return enum_for(:folders) if !block_given?
+
+        glob = File.join(source_local_path, "**", "*.imap")
+        Pathname.glob(glob) do |path|
+          name = source_folder_name(path)
+          serializer = Serializer.new(source_local_path, name)
+          folder = folder_for(name)
+          yield serializer, folder
+        end
+      end
+
+      def folder_for(source_folder)
+        no_source_prefix =
+          if source_prefix != "" && source_folder.start_with?(source_prefix)
+            source_folder.delete_prefix(source_prefix)
+          else
+            source_folder.to_s
+          end
+
+        with_destination_prefix =
+          if destination_prefix && destination_prefix != ""
+            destination_prefix + no_source_prefix
+          else
+            no_source_prefix
+          end
+
+        Account::Folder.new(
+          destination_account.connection,
+          with_destination_prefix
+        )
+      end
+
+      def source_local_path
+        source_account.local_path
+      end
+
+      def source_account
+        config.accounts.find { |a| a.username == source_email }
+      end
+
+      def source_folder_name(imap_pathname)
+        base = Pathname.new(source_local_path)
+        imap_name = imap_pathname.relative_path_from(base).to_s
+        dir = File.dirname(imap_name)
+        stripped = File.basename(imap_name, ".imap")
+        if dir == "."
+          stripped
+        else
+          File.join(dir, stripped)
+        end
+      end
+    end
+  end
+end

data/lib/imap/backup/cli/utils.rb

@@ -8,7 +8,10 @@ module Imap::Backup
     FAKE_EMAIL = "fake@email.com".freeze
 
     desc "ignore-history EMAIL", "Skip downloading emails up to today for all configured folders"
+    verbose_option
+    quiet_option
     def ignore_history(email)
+      Imap::Backup::Logger.setup_logging symbolized(options)
       connection = connection(email)
 
       connection.backup_folders.each do |folder|
@@ -26,6 +29,8 @@ module Imap::Backup
         A folder called 'imap-backup/EMAIL' is created under 'Local Folders'.
       DOC
     )
+    verbose_option
+    quiet_option
     method_option(
       "force",
       type: :boolean,
@@ -40,6 +45,7 @@ module Imap::Backup
     )
     def export_to_thunderbird(email)
       opts = symbolized(options)
+      Imap::Backup::Logger.setup_logging opts
       force = opts.key?(:force) ? opts[:force] : false
       profile_name = opts[:profile]
 

data/lib/imap/backup/cli.rb

@@ -1,4 +1,5 @@
 require "thor"
+require "imap/backup/logger"
 
 module Imap; end
 
@@ -10,6 +11,7 @@ module Imap::Backup
     autoload :Folders, "imap/backup/cli/folders"
     autoload :Local, "imap/backup/cli/local"
     autoload :Migrate, "imap/backup/cli/migrate"
+    autoload :Mirror, "imap/backup/cli/mirror"
     autoload :Remote, "imap/backup/cli/remote"
     autoload :Restore, "imap/backup/cli/restore"
     autoload :Setup, "imap/backup/cli/setup"
@@ -43,7 +45,16 @@ module Imap::Backup
       The setup tool can be used to choose a specific list of folders to back up.
     DESC
     accounts_option
+    verbose_option
+    quiet_option
+    method_option(
+      "refresh",
+      type: :boolean,
+      desc: "in 'keep all emails' mode, update flags for messages that are already downloaded",
+      aliases: ["-r"]
+    )
     def backup
+      Imap::Backup::Logger.setup_logging symbolized(options)
       Backup.new(symbolized(options)).run
     end
 
@@ -85,6 +96,8 @@ module Imap::Backup
       use the `--reset` option. In this case, all existing emails are
       deleted before uploading the migrated emails.
     DESC
+    verbose_option
+    quiet_option
     method_option(
       "destination-prefix",
       type: :string,
@@ -104,9 +117,49 @@ module Imap::Backup
       aliases: ["-s"]
     )
     def migrate(source_email, destination_email)
+      Imap::Backup::Logger.setup_logging symbolized(options)
       Migrate.new(source_email, destination_email, **symbolized(options)).run
     end
 
+    desc(
+      "mirror SOURCE_EMAIL DESTINATION_EMAIL [OPTIONS]",
+      "Keeps the DESTINATION_EMAIL account aligned with the SOURCE_EMAIL account"
+    )
+    long_desc <<~DESC
+      This command updates the DESTINATION_EMAIL account's folders to have the same contents
+      as those on the SOURCE_EMAIL account.
+
+      If a folder list is configured for the SOURCE_EMAIL account,
+      only the folders indicated by the setting are copied.
+
+      First, runs the download of the SOURCE_EMAIL account.
+      If the SOURCE_EMAIL account is **not** configured to be in 'mirror' mode,
+      a warning is printed.
+
+      When the mirror command is used, for each folder that is processed,
+      a new file is created alongside the normal backup files (.imap and .mbox)
+      This file has a '.mirror' extension. This file contains a mapping of
+      the known UIDs on the source account to those on the destination account.
+    DESC
+    verbose_option
+    quiet_option
+    method_option(
+      "destination-prefix",
+      type: :string,
+      desc: "the prefix (namespace) to add to destination folder names",
+      aliases: ["-d"]
+    )
+    method_option(
+      "source-prefix",
+      type: :string,
+      desc: "the prefix (namespace) to strip from source folder names",
+      aliases: ["-s"]
+    )
+    def mirror(source_email, destination_email)
+      Imap::Backup::Logger.setup_logging symbolized(options)
+      Mirror.new(source_email, destination_email, **symbolized(options)).run
+    end
+
     desc "remote SUBCOMMAND [OPTIONS]", "View info about online accounts"
     subcommand "remote", Remote
 
@@ -116,7 +169,10 @@ module Imap::Backup
       their original server.
     DESC
     accounts_option
+    verbose_option
+    quiet_option
     def restore(email = nil)
+      Imap::Backup::Logger.setup_logging symbolized(options)
       Restore.new(email, symbolized(options)).run
     end
 
@@ -125,7 +181,10 @@ module Imap::Backup
       A menu-driven command-line application used to configure imap-backup.
       Configure email accounts to back up.
     DESC
+    verbose_option
+    quiet_option
     def setup
+      Imap::Backup::Logger.setup_logging symbolized(options)
       Setup.new.run
     end
 

data/lib/imap/backup/configuration.rb

@@ -20,8 +20,6 @@ module Imap::Backup
 
     def initialize(pathname = self.class.default_pathname)
       @pathname = pathname
-      @saved_debug = nil
-      @debug = nil
     end
 
     def path
@@ -36,8 +34,7 @@ module Imap::Backup
       remove_deleted_accounts
       save_data = {
         version: VERSION,
-        accounts: accounts.map(&:to_h),
-        debug: debug?
+        accounts: accounts.map(&:to_h)
       }
       File.open(pathname, "w") { |f| f.write(JSON.pretty_generate(save_data)) }
       FileUtils.chmod(0o600, pathname) if !windows?
@@ -53,29 +50,16 @@ module Imap::Backup
 
     def modified?
       ensure_loaded!
-      return true if @saved_debug != @debug
 
       accounts.any? { |a| a.modified? || a.marked_for_deletion? }
     end
 
-    def debug?
-      ensure_loaded!
-      @debug
-    end
-
-    def debug=(value)
-      ensure_loaded!
-      @debug = [true, false].include?(value) ? value : false
-    end
-
     private
 
     def ensure_loaded!
       return true if @data
 
       data
-      @debug = data.key?(:debug) ? data[:debug] == true : false
-      @saved_debug = @debug
       true
     end