imap-backup 10.0.0 → 11.0.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a244374a14bae1b8d9bee48293eb744266d4d9fa67be560a3ca52a25b49a0cfa
-  data.tar.gz: 03fc3fe13037bcc3c9927117a5fb9edd4a85e053ea3d8e8f2380d8daee514c9d
+  metadata.gz: e9cff069fefde8147d1995b254f3f0613575ff4f52232c2fc64c91dc00c49bde
+  data.tar.gz: a1b8cb3d48f41b2773fd20b842b8d3e3b71e673b6797c83a345e341d087060d8
 SHA512:
-  metadata.gz: fc751d9d7f9af7f483536251234b5b40ffa9813d947d0739229ab14bf3d35d8878af6656d2180a490ad21b06d5b99c746547a95b75b6925c968fca51b91ba917
-  data.tar.gz: 8f4944c21e5458dda098554b3338c5baaecfe2e111fc5acc23ca0bcf1a20794202a80d0139a9932bba28960702ed8531881ea5bc7030af1b04a2e98afdc6dd22
+  metadata.gz: 4e7b83bac1004b853287689e98ed4d03d7c3c87bd05c4a22831ffe592782ff8916ab576778e41de572f872505e3beab5b923ff5d1d78ccf3e1a99d701ff95737
+  data.tar.gz: a0cc1cc27b225c315a68743dff999fc2d8b0d0c9fd5d68f03641b59d53ddc491da21f318b9c075927263367b9db0a50e80af727de0445c86a049bc55d2a32fdf

data/README.md

@@ -100,6 +100,54 @@ For more information about a command, run
 imap-backup help COMMAND
 ```
 
+# Performace
+
+There are a couple of performance tweaks that you can use
+to improve backup speed.
+
+These are activated via two settings:
+
+* Global setting "Delay download writes"
+* Account setting "Multi-fetch size"
+
+As with all performance tweaks, there are trade-offs.
+If you are using a small virtual server or Raspberry Pi
+to run your backups, you will probably want to leave
+the deafult settings.
+If, on the other hand, you are using a computer with a
+fair bit of RAM, and you are dealing with a *lot* of email,
+then changing these settings may be worthwhile.
+
+## Delay download writes
+
+This setting affects all account backups.
+
+When not set, each message is written to disk, one at a time.
+Doing so means the message itself is appended to the MBox file,
+but more importantly, the JSON metadata is rewritten to disk
+from scratch.
+
+When in use, all of a mailboxes unbackupped messages are
+downloaded first, and then written to disk just once.
+
+This speeds up backup as the metadata file is not rewritten
+after each message is added, but it potentially uses much more memory.
+
+## Multi-fetch Size
+
+By default, during backup, each message is downloaded one-by-one.
+
+Using this setting, you can download chunks of emails at a time,
+potentially speeding up the process.
+
+If you're not using "Delayed downlaod writes",
+using multi-fetch *will* mean that the backup process will use
+more memory - equivalent to the size of the greater number
+of messages downloaded at a time.
+
+This behaviour may also exceed limits on your email provider,
+so it's best to check before cranking it up!
+
 # Troubleshooting
 
 If you have problems:

data/docs/development.md

@@ -43,6 +43,15 @@ or
 $ rspec --tag ~docker
 ```
 
+# Performance Specs
+
+```sh
+PERFORMANCE=1 rspec --order=defined
+```
+
+Beware: the performance spec (just backup for now) takes a very
+long time to run, approximately 24 hours!
+
 ### Debugging
 
 The feature specs are run 'out of process' via the Aruba gem.

data/lib/imap/backup/account/backup.rb

@@ -5,6 +5,8 @@ require "imap/backup/downloader"
 require "imap/backup/flag_refresher"
 require "imap/backup/local_only_message_deleter"
 
+module Imap; end
+
 module Imap::Backup
   class Account; end
 
@@ -36,12 +38,19 @@ module Imap::Backup
 
         Logger.logger.debug "[#{folder.name}] running backup"
         serializer.apply_uid_validity(folder.uid_validity)
-        Downloader.new(
+        downloader = Downloader.new(
           folder,
           serializer,
           multi_fetch_size: account.multi_fetch_size,
           reset_seen_flags_after_fetch: account.reset_seen_flags_after_fetch
-        ).run
+        )
+        if account.delay_download_writes
+          serializer.transaction do
+            downloader.run
+          end
+        else
+          downloader.run
+        end
         if account.mirror_mode
           Logger.logger.info "Mirror mode - Deleting messages only present locally"
           LocalOnlyMessageDeleter.new(folder, serializer).run

data/lib/imap/backup/account/backup_folders.rb

@@ -1,3 +1,5 @@
+module Imap; end
+
 module Imap::Backup
   class Account; end
 

data/lib/imap/backup/account/client_factory.rb

@@ -5,6 +5,8 @@ require "imap/backup/client/apple_mail"
 require "imap/backup/client/default"
 require "retry_on_error"
 
+module Imap; end
+
 module Imap::Backup
   class Account; end
 

data/lib/imap/backup/account/folder.rb

@@ -3,6 +3,8 @@ require "net/imap"
 
 require "retry_on_error"
 
+module Imap; end
+
 module Imap::Backup
   class Account; end
 

data/lib/imap/backup/account/folder_ensurer.rb

@@ -1,6 +1,8 @@
 require "imap/backup/serializer/directory"
 require "imap/backup/serializer/folder_maker"
 
+module Imap; end
+
 module Imap::Backup
   class Account; end
 

data/lib/imap/backup/account/local_only_folder_deleter.rb

@@ -1,6 +1,8 @@
 require "imap/backup/account/backup_folders"
 require "imap/backup/account/serialized_folders"
 
+module Imap; end
+
 module Imap::Backup
   class Account; end
 

data/lib/imap/backup/account/restore.rb

@@ -1,5 +1,7 @@
 require "imap/backup/account/serialized_folders"
 
+module Imap; end
+
 module Imap::Backup
   class Account; end
 

data/lib/imap/backup/account/serialized_folders.rb

@@ -1,5 +1,7 @@
 require "imap/backup/account/folder_ensurer"
 
+module Imap; end
+
 module Imap::Backup
   class Account; end
 

data/lib/imap/backup/account.rb

@@ -15,6 +15,7 @@ module Imap::Backup
     attr_reader :mirror_mode
     attr_reader :server
     attr_reader :connection_options
+    attr_accessor :delay_download_writes
     attr_reader :reset_seen_flags_after_fetch
     attr_reader :changes
 
@@ -27,7 +28,8 @@ module Imap::Backup
       @mirror_mode = options[:mirror_mode]
       @server = options[:server]
       @connection_options = options[:connection_options]
-      @multi_fetch_size = options[:multi_fetch_size]
+      @delay_download_writes = true
+      @multi_fetch_size_orignal = options[:multi_fetch_size]
       @reset_seen_flags_after_fetch = options[:reset_seen_flags_after_fetch]
       @client = nil
       @changes = {}
@@ -42,6 +44,10 @@ module Imap::Backup
       client.namespace
     end
 
+    def capabilities
+      client.capability
+    end
+
     def restore
       restore = Account::Restore.new(account: self)
       restore.run
@@ -75,7 +81,8 @@ module Imap::Backup
       h[:mirror_mode] = true if @mirror_mode
       h[:server] = @server if @server
       h[:connection_options] = @connection_options if @connection_options
-      h[:multi_fetch_size] = multi_fetch_size if @multi_fetch_size
+      h[:delay_download_writes] = delay_download_writes
+      h[:multi_fetch_size] = multi_fetch_size
       if @reset_seen_flags_after_fetch
         h[:reset_seen_flags_after_fetch] = @reset_seen_flags_after_fetch
       end
@@ -123,11 +130,13 @@ module Imap::Backup
     end
 
     def multi_fetch_size
-      int = @multi_fetch_size.to_i
-      if int.positive?
-        int
-      else
-        DEFAULT_MULTI_FETCH_SIZE
+      @multi_fetch_size ||= begin
+        int = @multi_fetch_size_orignal.to_i
+        if int.positive?
+          int
+        else
+          DEFAULT_MULTI_FETCH_SIZE
+        end
       end
     end
 

data/lib/imap/backup/cli/backup.rb

@@ -1,5 +1,7 @@
 require "imap/backup/account/backup"
 
+module Imap; end
+
 module Imap::Backup
   class CLI::Backup < Thor
     include Thor::Actions

data/lib/imap/backup/cli/folder_enumerator.rb

@@ -1,3 +1,5 @@
+module Imap; end
+
 module Imap::Backup
   class CLI::FolderEnumerator
     attr_reader :destination

data/lib/imap/backup/cli/helpers.rb

@@ -1,5 +1,7 @@
 require "imap/backup"
 
+module Imap; end
+
 module Imap::Backup
   module CLI::Helpers
     def self.included(base)

data/lib/imap/backup/cli/local.rb

@@ -1,3 +1,5 @@
+module Imap; end
+
 module Imap::Backup
   class CLI::Local < Thor
     include Thor::Actions
@@ -39,7 +41,6 @@ module Imap::Backup
       results = requested_accounts(config).map do |account|
         serialized_folders = Account::SerializedFolders.new(account: account)
         folder_results = serialized_folders.map do |serializer, _folder|
-          puts "serializer: #{serializer.inspect}"
           serializer.check_integrity!
           {name: serializer.folder, result: "OK"}
         rescue Serializer::FolderIntegrityError => e

data/lib/imap/backup/cli/migrate.rb

@@ -1,6 +1,8 @@
 require "imap/backup/cli/folder_enumerator"
 require "imap/backup/migrator"
 
+module Imap; end
+
 module Imap::Backup
   class CLI::Migrate < Thor
     include Thor::Actions

data/lib/imap/backup/cli/mirror.rb

@@ -1,6 +1,8 @@
 require "imap/backup/cli/folder_enumerator"
 require "imap/backup/mirror"
 
+module Imap; end
+
 module Imap::Backup
   class CLI::Mirror < Thor
     include Thor::Actions

data/lib/imap/backup/cli/remote.rb

@@ -1,5 +1,7 @@
 require "imap/backup/logger"
 
+module Imap; end
+
 module Imap::Backup
   class CLI::Remote < Thor
     include Thor::Actions
@@ -21,6 +23,22 @@ module Imap::Backup
       end
     end
 
+    desc "capabilities EMAIL", "List server capabilities"
+    long_desc <<~DESC
+      Lists the IMAP capabilities supported by the IMAP server.
+    DESC
+    config_option
+    format_option
+    quiet_option
+    verbose_option
+    def capabilities(email)
+      Imap::Backup::Logger.setup_logging options
+      config = load_config(**options)
+      account = account(config, email)
+      capabilities = account.capabilities
+      Kernel.puts capabilities.join(", ")
+    end
+
     desc "namespaces EMAIL", "List account namespaces"
     long_desc <<~DESC
       Lists namespaces defined for an email account.

data/lib/imap/backup/cli/restore.rb

@@ -1,3 +1,5 @@
+module Imap; end
+
 module Imap::Backup
   class CLI::Restore < Thor
     include Thor::Actions

data/lib/imap/backup/cli/setup.rb

@@ -1,3 +1,5 @@
+module Imap; end
+
 module Imap::Backup
   class CLI::Setup < Thor
     include Thor::Actions

data/lib/imap/backup/cli/stats.rb

@@ -1,5 +1,7 @@
 require "imap/backup/account/backup_folders"
 
+module Imap; end
+
 module Imap::Backup
   class CLI::Stats < Thor
     include Thor::Actions

data/lib/imap/backup/cli/utils.rb

@@ -2,6 +2,8 @@ require "imap/backup/account/backup_folders"
 require "imap/backup/account/serialized_folders"
 require "imap/backup/thunderbird/mailbox_exporter"
 
+module Imap; end
+
 module Imap::Backup
   class CLI::Utils < Thor
     include Thor::Actions

data/lib/imap/backup/client/apple_mail.rb

@@ -1,5 +1,7 @@
 require "imap/backup/client/default"
 
+module Imap; end
+
 module Imap::Backup
   class Client::AppleMail < Client::Default
     # With Apple Mails's IMAP, passing "/" to list

data/lib/imap/backup/client/default.rb

@@ -1,13 +1,15 @@
 require "forwardable"
 require "net/imap"
 
+module Imap; end
+
 module Imap::Backup
   module Client; end
 
   class Client::Default
     extend Forwardable
     def_delegators :imap, *%i(
-      append authenticate create expunge namespace
+      append authenticate capability create expunge namespace
       responses uid_fetch uid_search uid_store
     )
 

data/lib/imap/backup/configuration.rb

@@ -6,12 +6,16 @@ require "imap/backup/account"
 require "imap/backup/file_mode"
 require "imap/backup/serializer/permission_checker"
 
+module Imap; end
+
 module Imap::Backup
   class Configuration
     CONFIGURATION_DIRECTORY = File.expand_path("~/.imap-backup")
     VERSION = "2.0".freeze
 
     attr_reader :pathname
+    attr_reader :delay_download_writes
+    attr_reader :delay_download_writes_modified
 
     def self.default_pathname
       File.join(CONFIGURATION_DIRECTORY, "config.json")
@@ -23,6 +27,8 @@ module Imap::Backup
 
     def initialize(path: nil)
       @pathname = path || self.class.default_pathname
+      @delay_download_writes = false
+      @delay_download_writes_modified = false
     end
 
     def path
@@ -37,7 +43,8 @@ module Imap::Backup
       remove_deleted_accounts
       save_data = {
         version: VERSION,
-        accounts: accounts.map(&:to_h)
+        accounts: accounts.map(&:to_h),
+        delay_download_writes: delay_download_writes
       }
       File.open(pathname, "w") { |f| f.write(JSON.pretty_generate(save_data)) }
       FileUtils.chmod(0o600, pathname) if !windows?
@@ -47,13 +54,26 @@ module Imap::Backup
     def accounts
       @accounts ||= begin
         ensure_loaded!
-        data[:accounts].map { |data| Account.new(data) }
+        accounts = data[:accounts].map do |attr|
+          Account.new(attr)
+        end
+        inject_global_attributes(accounts)
       end
     end
 
+    def delay_download_writes=(value)
+      ensure_loaded!
+
+      @delay_download_writes = value
+      @delay_download_writes_modified = true
+      inject_global_attributes(accounts)
+    end
+
     def modified?
       ensure_loaded!
 
+      return true if delay_download_writes_modified
+
       accounts.any? { |a| a.modified? || a.marked_for_deletion? }
     end
 
@@ -63,6 +83,7 @@ module Imap::Backup
       return true if @data
 
       data
+      @delay_download_writes = data[:delay_download_writes]
       true
     end
 
@@ -81,6 +102,7 @@ module Imap::Backup
     end
 
     def remove_modified_flags
+      @delay_download_writes_modified = false
       accounts.each(&:clear_changes)
     end
 
@@ -88,6 +110,13 @@ module Imap::Backup
       accounts.reject!(&:marked_for_deletion?)
     end
 
+    def inject_global_attributes(accounts)
+      accounts.map do |a|
+        a.delay_download_writes = delay_download_writes
+        a
+      end
+    end
+
     def make_private(path)
       FileUtils.chmod(0o700, path) if FileMode.new(filename: path).mode != 0o700
     end

data/lib/imap/backup/downloader.rb

@@ -1,3 +1,5 @@
+module Imap; end
+
 module Imap::Backup
   class MultiFetchFailedError < StandardError; end
 

data/lib/imap/backup/file_mode.rb

@@ -1,3 +1,5 @@
+module Imap; end
+
 module Imap::Backup
   class FileMode
     attr_reader :filename

data/lib/imap/backup/flag_refresher.rb

@@ -1,3 +1,5 @@
+module Imap; end
+
 module Imap::Backup
   class FlagRefresher
     attr_reader :folder

data/lib/imap/backup/local_only_message_deleter.rb

@@ -1,3 +1,5 @@
+module Imap; end
+
 module Imap::Backup
   class LocalOnlyMessageDeleter
     attr_reader :folder

data/lib/imap/backup/logger.rb

@@ -4,6 +4,8 @@ require "singleton"
 require "imap/backup/configuration"
 require "text/sanitizer"
 
+module Imap; end
+
 module Imap::Backup
   class Logger
     include Singleton

data/lib/imap/backup/migrator.rb

@@ -1,3 +1,5 @@
+module Imap; end
+
 module Imap::Backup
   class Migrator
     attr_reader :folder

data/lib/imap/backup/mirror/map.rb

@@ -1,3 +1,5 @@
+module Imap; end
+
 module Imap::Backup
   class Mirror; end
 

data/lib/imap/backup/mirror.rb

@@ -1,5 +1,7 @@
 require "imap/backup/mirror/map"
 
+module Imap; end
+
 module Imap::Backup
   class Mirror
     attr_reader :serializer

data/lib/imap/backup/naming.rb

@@ -1,3 +1,5 @@
+module Imap; end
+
 module Imap::Backup
   class Naming
     INVALID_FILENAME_CHARACTERS = ":%;".freeze

data/lib/imap/backup/serializer/appender.rb

@@ -1,3 +1,5 @@
+module Imap; end
+
 module Imap::Backup
   class Serializer; end
 
@@ -12,7 +14,7 @@ module Imap::Backup
       @mbox = mbox
     end
 
-    def run(uid:, message:, flags:)
+    def single(uid:, message:, flags:)
       raise "Can't add messages without uid_validity" if !imap.uid_validity
 
       uid = uid.to_i
@@ -24,29 +26,43 @@ module Imap::Backup
         return
       end
 
-      do_append uid, message, flags
+      rollback_on_error do
+        do_append uid, message, flags
+      rescue StandardError => e
+        raise <<-ERROR.gsub(/^\s*/m, "")
+          [#{folder}] failed to append message #{uid}:
+          #{message}. #{e}:
+          #{e.backtrace.join("\n")}"
+        ERROR
+      end
+    end
+
+    def multi(appends)
+      rollback_on_error do
+        appends.each do |a|
+          do_append a[:uid], a[:message], a[:flags]
+        end
+      end
     end
 
     private
 
     def do_append(uid, message, flags)
       mboxrd_message = Email::Mboxrd::Message.new(message)
-      initial = mbox.length || 0
-      mbox_appended = false
-      begin
-        serialized = mboxrd_message.to_serialized
-        mbox.append serialized
-        mbox_appended = true
-        imap.append uid, serialized.length, flags
-      rescue StandardError => e
-        mbox.rewind(initial) if mbox_appended
+      serialized = mboxrd_message.to_serialized
+      mbox.append serialized
+      imap.append uid, serialized.length, flags: flags
+    end
 
-        error = <<-ERROR.gsub(/^\s*/m, "")
-          [#{folder}] failed to append message #{uid}:
-          #{message}. #{e}:
-          #{e.backtrace.join("\n")}"
-        ERROR
-        Logger.logger.warn error
+    def rollback_on_error(&block)
+      imap.transaction do
+        mbox.transaction do
+          block.call
+        rescue StandardError => e
+          Logger.logger.error e
+          imap.rollback
+          mbox.rollback
+        end
       end
     end
   end

data/lib/imap/backup/serializer/directory.rb

@@ -2,6 +2,8 @@ require "os"
 
 require "imap/backup/serializer/folder_maker"
 
+module Imap; end
+
 module Imap::Backup
   class Serializer; end
 

data/lib/imap/backup/serializer/folder_maker.rb

@@ -1,5 +1,7 @@
 require "fileutils"
 
+module Imap; end
+
 module Imap::Backup
   class Serializer; end
 

data/lib/imap/backup/serializer/imap.rb

@@ -1,5 +1,7 @@
 require "json"
 
+module Imap; end
+
 module Imap::Backup
   class Serializer::Imap
     CURRENT_VERSION = 3
@@ -13,6 +15,26 @@ module Imap::Backup
       @uid_validity = nil
       @messages = nil
       @version = nil
+      @savepoint = nil
+    end
+
+    def transaction(&block)
+      fail_in_transaction!(message: "Serializer::Imap: nested transactions are not supported")
+
+      ensure_loaded
+      @savepoint = {messages: messages.dup, uid_validity: uid_validity}
+
+      block.call
+
+      @savepoint = nil
+      save
+    end
+
+    def rollback
+      fail_outside_transaction!
+
+      @messages = @savepoint[:messages]
+      @uid_validity = @savepoint[:uid_validity]
     end
 
     def pathname
@@ -31,7 +53,7 @@ module Imap::Backup
       true
     end
 
-    def append(uid, length, flags = [])
+    def append(uid, length, flags: [])
       offset =
         if messages.empty?
           0
@@ -107,14 +129,16 @@ module Imap::Backup
     end
 
     def save
+      return if @savepoint
+
       ensure_loaded
 
       raise "Cannot save metadata without a uid_validity" if !uid_validity
 
       data = {
-        version: @version,
-        uid_validity: @uid_validity,
-        messages: @messages.map(&:to_h)
+        version: version,
+        uid_validity: uid_validity,
+        messages: messages.map(&:to_h)
       }
       content = data.to_json
       File.open(pathname, "w") { |f| f.write content }
@@ -160,5 +184,13 @@ module Imap::Backup
     def mbox
       @mbox ||= Serializer::Mbox.new(folder_path)
     end
+
+    def fail_in_transaction!(message: "Serializer::Imap: method not supported inside trasactions")
+      raise message if @savepoint
+    end
+
+    def fail_outside_transaction!
+      raise "This method can only be called inside a transaction" if !@savepoint
+    end
   end
 end

data/lib/imap/backup/serializer/mbox.rb

@@ -1,9 +1,27 @@
+module Imap; end
+
 module Imap::Backup
   class Serializer::Mbox
     attr_reader :folder_path
+    attr_reader :savepoint
 
     def initialize(folder_path)
       @folder_path = folder_path
+      @savepoint = nil
+    end
+
+    def transaction(&block)
+      fail_in_transaction!(message: "Nested transactions are not supported")
+
+      @savepoint = {length: length}
+      block.call
+      @savepoint = nil
+    end
+
+    def rollback
+      fail_outside_transaction!
+
+      rewind(@savepoint[:length])
     end
 
     def valid?
@@ -29,6 +47,10 @@ module Imap::Backup
       File.unlink(pathname)
     end
 
+    def exist?
+      File.exist?(pathname)
+    end
+
     def length
       return nil if !exist?
 
@@ -49,18 +71,24 @@ module Imap::Backup
       end
     end
 
+    def touch
+      File.open(pathname, "a") {}
+    end
+
+    private
+
     def rewind(length)
       File.open(pathname, File::RDWR | File::CREAT, 0o644) do |f|
         f.truncate(length)
       end
     end
 
-    def touch
-      File.open(pathname, "a") {}
+    def fail_in_transaction!(message: "Method not supported inside trasactions")
+      raise message if savepoint
     end
 
-    def exist?
-      File.exist?(pathname)
+    def fail_outside_transaction!
+      raise "This method can only be called inside a transaction" if !savepoint
     end
   end
 end

data/lib/imap/backup/serializer/message.rb

@@ -1,5 +1,7 @@
 require "email/mboxrd/message"
 
+module Imap; end
+
 module Imap::Backup
   class Serializer::Message
     attr_accessor :uid

data/lib/imap/backup/serializer/message_enumerator.rb

@@ -1,3 +1,5 @@
+module Imap; end
+
 module Imap::Backup
   class Serializer::MessageEnumerator
     attr_reader :imap

data/lib/imap/backup/serializer/permission_checker.rb

@@ -1,3 +1,5 @@
+module Imap; end
+
 module Imap::Backup
   class Serializer::PermissionChecker
     attr_reader :filename

data/lib/imap/backup/serializer/unused_name_finder.rb

@@ -1,3 +1,5 @@
+module Imap; end
+
 module Imap::Backup
   class Serializer::UnusedNameFinder
     attr_reader :serializer

data/lib/imap/backup/serializer/version2_migrator.rb

@@ -1,5 +1,7 @@
 require "json"
 
+module Imap; end
+
 module Imap::Backup
   class Serializer::Version2Migrator
     attr_reader :folder_path

data/lib/imap/backup/serializer.rb

@@ -10,6 +10,8 @@ require "imap/backup/serializer/message_enumerator"
 require "imap/backup/serializer/version2_migrator"
 require "imap/backup/serializer/unused_name_finder"
 
+module Imap; end
+
 module Imap::Backup
   class Serializer
     def self.folder_path_for(path:, folder:)
@@ -26,16 +28,36 @@ module Imap::Backup
 
     attr_reader :folder
     attr_reader :path
+    attr_reader :dirty
 
     def initialize(path, folder)
       @path = path
       @folder = folder
       @validated = nil
+      @dirty = nil
+    end
+
+    def transaction(&block)
+      fail_in_transaction!(:transaction, message: "nested transactions are not supported")
+
+      validate!
+      @dirty = {append: []}
+
+      block.call
+
+      if dirty[:append].any?
+        appender = Serializer::Appender.new(folder: sanitized, imap: imap, mbox: mbox)
+        appender.multi(dirty[:append])
+      end
+
+      @dirty = nil
     end
 
     # Returns true if there are existing, valid files
     # false otherwise (in which case any existing files are deleted)
     def validate!
+      fail_in_transaction!(:validate!)
+
       return true if @validated
 
       optionally_migrate2to3
@@ -51,6 +73,8 @@ module Imap::Backup
     end
 
     def check_integrity!
+      fail_in_transaction!(:check_integrity!)
+
       if !imap.valid?
         message = ".imap file '#{imap.pathname}' is corrupt"
         raise FolderIntegrityError, message
@@ -102,6 +126,8 @@ module Imap::Backup
     end
 
     def delete
+      fail_in_transaction!(:delete)
+
       imap.delete
       @imap = nil
       mbox.delete
@@ -109,6 +135,7 @@ module Imap::Backup
     end
 
     def apply_uid_validity(value)
+      fail_in_transaction!(:apply_uid_validity)
       validate!
 
       case
@@ -124,19 +151,26 @@ module Imap::Backup
     end
 
     def force_uid_validity(value)
+      fail_in_transaction!(:force_uid_validity)
       validate!
 
       internal_force_uid_validity(value)
     end
 
     def append(uid, message, flags)
-      validate!
+      if dirty
+        dirty[:append] << {uid: uid, message: message, flags: flags}
+      else
+        validate!
 
-      appender = Serializer::Appender.new(folder: sanitized, imap: imap, mbox: mbox)
-      appender.run(uid: uid, message: message, flags: flags)
+        appender = Serializer::Appender.new(folder: sanitized, imap: imap, mbox: mbox)
+        appender.single(uid: uid, message: message, flags: flags)
+      end
     end
 
     def update(uid, flags: nil)
+      fail_in_transaction!(:update)
+
       message = imap.get(uid)
       return if !message
 
@@ -145,17 +179,21 @@ module Imap::Backup
     end
 
     def each_message(required_uids = nil, &block)
+      fail_in_transaction!(:each_message)
+
+      return enum_for(:each_message, required_uids) if !block
+
       required_uids ||= uids
 
       validate!
 
-      return enum_for(:each_message, required_uids) if !block
-
       enumerator = Serializer::MessageEnumerator.new(imap: imap)
       enumerator.run(uids: required_uids, &block)
     end
 
     def filter(&block)
+      fail_in_transaction!(:filter)
+
       temp_name = Serializer::UnusedNameFinder.new(serializer: self).run
       temp_folder_path = self.class.folder_path_for(path: path, folder: temp_name)
       new_mbox = Serializer::Mbox.new(temp_folder_path)
@@ -165,7 +203,7 @@ module Imap::Backup
       enumerator = Serializer::MessageEnumerator.new(imap: imap)
       enumerator.run(uids: uids) do |message|
         keep = block.call(message)
-        appender.run(uid: message.uid, message: message.body, flags: message.flags) if keep
+        appender.single(uid: message.uid, message: message.body, flags: message.flags) if keep
       end
       imap.delete
       new_imap.rename imap.folder_path
@@ -249,5 +287,13 @@ module Imap::Backup
       rename new_name
       new_name
     end
+
+    def fail_in_transaction!(method, message: "not supported inside trasactions")
+      raise "Serializer##{method} #{message}" if dirty
+    end
+
+    def fail_outside_transaction!(method)
+      raise "Serializer##{method} can only be called inside a transaction" if !dirty
+    end
   end
 end

data/lib/imap/backup/setup/account/header.rb

@@ -1,5 +1,7 @@
 require "imap/backup/setup/helpers"
 
+module Imap; end
+
 module Imap::Backup
   class Setup; end
   class Setup::Account; end

data/lib/imap/backup/setup/account.rb

@@ -2,6 +2,8 @@ require "imap/backup/setup/account/header"
 require "imap/backup/setup/backup_path"
 require "imap/backup/setup/email"
 
+module Imap; end
+
 module Imap::Backup
   class Setup; end
 

data/lib/imap/backup/setup/asker.rb

@@ -1,3 +1,5 @@
+module Imap; end
+
 module Imap::Backup
   class Setup; end
 

data/lib/imap/backup/setup/backup_path.rb

@@ -1,3 +1,5 @@
+module Imap; end
+
 module Imap::Backup
   class Setup; end
 

data/lib/imap/backup/setup/connection_tester.rb

@@ -1,3 +1,5 @@
+module Imap; end
+
 module Imap::Backup
   class Setup; end
 

data/lib/imap/backup/setup/email.rb

@@ -1,5 +1,7 @@
 require "email/provider"
 
+module Imap; end
+
 module Imap::Backup
   class Setup; end
 

data/lib/imap/backup/setup/folder_chooser.rb

@@ -1,5 +1,7 @@
 require "imap/backup/setup/helpers"
 
+module Imap; end
+
 module Imap::Backup
   class Setup; end
 

data/lib/imap/backup/setup/helpers.rb

@@ -1,5 +1,7 @@
 require "imap/backup/version"
 
+module Imap; end
+
 module Imap::Backup
   class Setup; end
 

data/lib/imap/backup/setup.rb

@@ -4,6 +4,8 @@ require "email/provider"
 require "imap/backup/account"
 require "imap/backup/setup/helpers"
 
+module Imap; end
+
 module Imap::Backup
   class Setup
     class << self
@@ -37,6 +39,7 @@ module Imap::Backup
         MENU
         account_items menu
         add_account_item menu
+        toggle_delay_download_writes menu
         if config.modified?
           menu.choice("save and exit") do
             config.save
@@ -68,6 +71,16 @@ module Imap::Backup
       end
     end
 
+    def toggle_delay_download_writes(menu)
+      new_value = config.delay_download_writes ? false : true
+      modified = config.delay_download_writes_modified ? " *" : ""
+      change = config.delay_download_writes ? "don't delay" : "delay"
+      menu_item = "#{change} download writes#{modified}"
+      menu.choice(menu_item) do
+        config.delay_download_writes = new_value
+      end
+    end
+
     def default_account_config(username)
       Imap::Backup::Account.new(
         username: username,

data/lib/imap/backup/thunderbird/mailbox_exporter.rb

@@ -1,6 +1,8 @@
 require "thunderbird/local_folder"
 require "thunderbird/profiles"
 
+module Imap; end
+
 module Imap::Backup
   class Thunderbird::MailboxExporter
     EXPORT_PREFIX = "imap-backup".freeze

data/lib/imap/backup/uploader.rb

@@ -1,3 +1,5 @@
+module Imap; end
+
 module Imap::Backup
   class Uploader
     attr_reader :folder

data/lib/imap/backup/version.rb

@@ -1,9 +1,9 @@
 module Imap; end
 
 module Imap::Backup
-  MAJOR    = 10
+  MAJOR    = 11
   MINOR    = 0
   REVISION = 0
-  PRE      = nil
+  PRE      = "rc1".freeze
   VERSION  = [MAJOR, MINOR, REVISION, PRE].compact.map(&:to_s).join(".")
 end

data/lib/imap/backup.rb

@@ -1,5 +1,3 @@
-module Imap; end
-
 require "imap/backup/account/folder"
 require "imap/backup/configuration"
 require "imap/backup/downloader"
@@ -13,6 +11,8 @@ require "imap/backup/setup/connection_tester"
 require "imap/backup/setup/folder_chooser"
 require "imap/backup/version"
 
+module Imap; end
+
 module Imap::Backup
   class ConfigurationNotFound < StandardError; end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: imap-backup
 version: !ruby/object:Gem::Version
-  version: 10.0.0
+  version: 11.0.0.rc1
 platform: ruby
 authors:
 - Joe Yates
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-07-16 00:00:00.000000000 Z
+date: 2023-07-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: highline
@@ -305,9 +305,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.6'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.3.7
 signing_key: