imap-backup 11.0.0.rc1 → 11.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e9cff069fefde8147d1995b254f3f0613575ff4f52232c2fc64c91dc00c49bde
-  data.tar.gz: a1b8cb3d48f41b2773fd20b842b8d3e3b71e673b6797c83a345e341d087060d8
+  metadata.gz: 1392301e243a9affe750a9361f14622228879596b8cb0a29e4944dd91af52d44
+  data.tar.gz: d85f59f5387d5f3c5ac1f6ce97a4a2e5a685ca675b02be3982c914774485d5fc
 SHA512:
-  metadata.gz: 4e7b83bac1004b853287689e98ed4d03d7c3c87bd05c4a22831ffe592782ff8916ab576778e41de572f872505e3beab5b923ff5d1d78ccf3e1a99d701ff95737
-  data.tar.gz: a0cc1cc27b225c315a68743dff999fc2d8b0d0c9fd5d68f03641b59d53ddc491da21f318b9c075927263367b9db0a50e80af727de0445c86a049bc55d2a32fdf
+  metadata.gz: ef636f637cf6ab48a173439e6d4c908fa7bf0b8a171a7ba60a80b83062e95d30aafba0e52a4647b5c74b23a40b7504132803ad4ddc57907e3b08298e11b9a26c
+  data.tar.gz: 1274d7462c5e302bd1ea84529b12d74e6cc0ebe47ba6d755e2f23cdb5829dfde6fdde858c2640abea5f6f13eab8e5867334e82f5e4b7649ae41385a49f5706ca

data/lib/imap/backup/account/backup.rb

@@ -1,6 +1,7 @@
 require "imap/backup/account/folder_ensurer"
 require "imap/backup/account/local_only_folder_deleter"
 require "imap/backup/account/serialized_folders"
+require "imap/backup/serializer/delayed_metadata_serializer"
 require "imap/backup/downloader"
 require "imap/backup/flag_refresher"
 require "imap/backup/local_only_message_deleter"
@@ -37,20 +38,38 @@ module Imap::Backup
         end
 
         Logger.logger.debug "[#{folder.name}] running backup"
+
         serializer.apply_uid_validity(folder.uid_validity)
+
+        download_serializer =
+          case account.download_strategy
+          when "direct"
+            serializer
+          when "delay_metadata"
+            Serializer::DelayedMetadataSerializer.new(serializer: serializer)
+          else
+            raise "Unknown download strategy '#{account.download_strategy}'"
+          end
+
         downloader = Downloader.new(
           folder,
-          serializer,
+          download_serializer,
           multi_fetch_size: account.multi_fetch_size,
           reset_seen_flags_after_fetch: account.reset_seen_flags_after_fetch
         )
-        if account.delay_download_writes
-          serializer.transaction do
-            downloader.run
-          end
-        else
+        # rubocop:disable Lint/RescueException
+        download_serializer.transaction do
           downloader.run
+        rescue Exception => e
+          message = <<~ERROR
+            #{self.class} error #{e}
+            #{e.backtrace.join("\n")}
+          ERROR
+          Logger.logger.error message
+          download_serializer.rollback
+          raise e
         end
+        # rubocop:enable Lint/RescueException
         if account.mirror_mode
           Logger.logger.info "Mirror mode - Deleting messages only present locally"
           LocalOnlyMessageDeleter.new(folder, serializer).run

data/lib/imap/backup/account.rb

@@ -15,7 +15,7 @@ module Imap::Backup
     attr_reader :mirror_mode
     attr_reader :server
     attr_reader :connection_options
-    attr_accessor :delay_download_writes
+    attr_accessor :download_strategy
     attr_reader :reset_seen_flags_after_fetch
     attr_reader :changes
 
@@ -28,7 +28,7 @@ module Imap::Backup
       @mirror_mode = options[:mirror_mode]
       @server = options[:server]
       @connection_options = options[:connection_options]
-      @delay_download_writes = true
+      @download_strategy = options[:download_strategy]
       @multi_fetch_size_orignal = options[:multi_fetch_size]
       @reset_seen_flags_after_fetch = options[:reset_seen_flags_after_fetch]
       @client = nil
@@ -81,7 +81,6 @@ module Imap::Backup
       h[:mirror_mode] = true if @mirror_mode
       h[:server] = @server if @server
       h[:connection_options] = @connection_options if @connection_options
-      h[:delay_download_writes] = delay_download_writes
       h[:multi_fetch_size] = multi_fetch_size
       if @reset_seen_flags_after_fetch
         h[:reset_seen_flags_after_fetch] = @reset_seen_flags_after_fetch

data/lib/imap/backup/cli/backup.rb

@@ -1,3 +1,4 @@
+require "net/imap"
 require "imap/backup/account/backup"
 
 module Imap; end
@@ -17,21 +18,34 @@ module Imap::Backup
     no_commands do
       def run
         config = load_config(**options)
+        exit_code = nil
         requested_accounts(config).each do |account|
           backup = Account::Backup.new(account: account, refresh: refresh)
           backup.run
         rescue StandardError => e
-          message =
-            "Backup for account '#{account.username}' " \
-            "failed with error #{e}"
-          Logger.logger.warn message
+          exit_code ||= choose_exit_code(e)
+          message = <<~ERROR
+            Backup for account '#{account.username}' failed with error #{e}
+            #{e.backtrace.join("\n")}
+          ERROR
+          Logger.logger.error message
           next
         end
+        exit(exit_code) if exit_code
       end
 
       def refresh
         options.key?(:refresh) ? !!options[:refresh] : false
       end
+
+      def choose_exit_code(exception)
+        case exception
+        when Net::IMAP::NoResponseError, Errno::ECONNREFUSED
+          111
+        else
+          1
+        end
+      end
     end
   end
 end

data/lib/imap/backup/configuration.rb

@@ -11,11 +11,17 @@ module Imap; end
 module Imap::Backup
   class Configuration
     CONFIGURATION_DIRECTORY = File.expand_path("~/.imap-backup")
-    VERSION = "2.0".freeze
+    VERSION = "2.1".freeze
+    DEFAULT_STRATEGY = "delay_metadata".freeze
+    DOWNLOAD_STRATEGIES = [
+      {key: "direct", description: "write straight to disk"},
+      {key: DEFAULT_STRATEGY, description: "delay writing metadata"}
+    ].freeze
 
     attr_reader :pathname
-    attr_reader :delay_download_writes
-    attr_reader :delay_download_writes_modified
+    attr_reader :download_strategy
+    attr_reader :download_strategy_original
+    attr_reader :download_strategy_modified
 
     def self.default_pathname
       File.join(CONFIGURATION_DIRECTORY, "config.json")
@@ -27,8 +33,9 @@ module Imap::Backup
 
     def initialize(path: nil)
       @pathname = path || self.class.default_pathname
-      @delay_download_writes = false
-      @delay_download_writes_modified = false
+      @download_strategy = nil
+      @download_strategy_original = nil
+      @download_strategy_modified = false
     end
 
     def path
@@ -44,7 +51,7 @@ module Imap::Backup
       save_data = {
         version: VERSION,
         accounts: accounts.map(&:to_h),
-        delay_download_writes: delay_download_writes
+        download_strategy: download_strategy
       }
       File.open(pathname, "w") { |f| f.write(JSON.pretty_generate(save_data)) }
       FileUtils.chmod(0o600, pathname) if !windows?
@@ -61,18 +68,20 @@ module Imap::Backup
       end
     end
 
-    def delay_download_writes=(value)
+    def download_strategy=(value)
+      raise "Unknown strategy '#{value}'" if !DOWNLOAD_STRATEGIES.find { |s| s[:key] == value }
+
       ensure_loaded!
 
-      @delay_download_writes = value
-      @delay_download_writes_modified = true
+      @download_strategy = value
+      @download_strategy_modified = value != download_strategy_original
       inject_global_attributes(accounts)
     end
 
     def modified?
       ensure_loaded!
 
-      return true if delay_download_writes_modified
+      return true if download_strategy_modified
 
       accounts.any? { |a| a.modified? || a.marked_for_deletion? }
     end
@@ -83,7 +92,8 @@ module Imap::Backup
       return true if @data
 
       data
-      @delay_download_writes = data[:delay_download_writes]
+      @download_strategy = data[:download_strategy]
+      @download_strategy_original = data[:download_strategy]
       true
     end
 
@@ -95,14 +105,21 @@ module Imap::Backup
           )
           permission_checker.run if !windows?
           contents = File.read(pathname)
-          JSON.parse(contents, symbolize_names: true)
+          data = JSON.parse(contents, symbolize_names: true)
+          data[:download_strategy] =
+            if DOWNLOAD_STRATEGIES.find { |s| s[:key] == data[:download_strategy] }
+              data[:download_strategy]
+            else
+              DEFAULT_STRATEGY
+            end
+          data
         else
           {accounts: []}
         end
     end
 
     def remove_modified_flags
-      @delay_download_writes_modified = false
+      @download_strategy_modified = false
       accounts.each(&:clear_changes)
     end
 
@@ -112,7 +129,7 @@ module Imap::Backup
 
     def inject_global_attributes(accounts)
       accounts.map do |a|
-        a.delay_download_writes = delay_download_writes
+        a.download_strategy = download_strategy
         a
       end
     end

data/lib/imap/backup/serializer/appender.rb

@@ -14,7 +14,7 @@ module Imap::Backup
       @mbox = mbox
     end
 
-    def single(uid:, message:, flags:)
+    def append(uid:, message:, flags:)
       raise "Can't add messages without uid_validity" if !imap.uid_validity
 
       uid = uid.to_i
@@ -27,31 +27,23 @@ module Imap::Backup
       end
 
       rollback_on_error do
-        do_append uid, message, flags
+        serialized = to_serialized(message)
+        mbox.append serialized
+        imap.append uid, serialized.length, flags: flags
       rescue StandardError => e
         raise <<-ERROR.gsub(/^\s*/m, "")
-          [#{folder}] failed to append message #{uid}:
-          #{message}. #{e}:
+          [#{folder}] failed to append message #{uid}: #{message}.
+          #{e}:
           #{e.backtrace.join("\n")}"
         ERROR
       end
     end
 
-    def multi(appends)
-      rollback_on_error do
-        appends.each do |a|
-          do_append a[:uid], a[:message], a[:flags]
-        end
-      end
-    end
-
     private
 
-    def do_append(uid, message, flags)
+    def to_serialized(message)
       mboxrd_message = Email::Mboxrd::Message.new(message)
-      serialized = mboxrd_message.to_serialized
-      mbox.append serialized
-      imap.append uid, serialized.length, flags: flags
+      mboxrd_message.to_serialized
     end
 
     def rollback_on_error(&block)
@@ -62,6 +54,11 @@ module Imap::Backup
           Logger.logger.error e
           imap.rollback
           mbox.rollback
+        rescue SignalException => e
+          Logger.logger.error e
+          imap.rollback
+          mbox.rollback
+          raise
         end
       end
     end

data/lib/imap/backup/serializer/delayed_metadata_serializer.rb

@@ -0,0 +1,77 @@
+require "imap/backup/serializer/imap"
+require "imap/backup/serializer/mbox"
+require "imap/backup/serializer/transaction"
+
+module Imap; end
+
+module Imap::Backup
+  class Serializer::DelayedMetadataSerializer
+    extend Forwardable
+
+    attr_reader :serializer
+
+    def_delegator :serializer, :uids
+
+    def initialize(serializer:)
+      @serializer = serializer
+      @tsx = nil
+    end
+
+    def transaction(&block)
+      tsx.fail_in_transaction!(:transaction, message: "nested transactions are not supported")
+
+      # rubocop:disable Lint/RescueException
+      tsx.begin({metadata: []}) do
+        mbox.transaction do
+          block.call
+
+          commit
+        rescue Exception => e
+          message = <<~ERROR
+            #{self.class} error #{e}
+            #{e.backtrace.join("\n")}
+          ERROR
+          Logger.logger.error message
+          mbox.rollback
+          raise e
+        end
+      end
+      # rubocop:enable Lint/RescueException
+    end
+
+    def append(uid, message, flags)
+      tsx.fail_outside_transaction!(:append)
+      mboxrd_message = Email::Mboxrd::Message.new(message)
+      serialized = mboxrd_message.to_serialized
+      tsx.data[:metadata] << {uid: uid, length: serialized.length, flags: flags}
+      mbox.append(serialized)
+    end
+
+    private
+
+    def commit
+      # rubocop:disable Lint/RescueException
+      imap.transaction do
+        tsx.data[:metadata].each do |m|
+          imap.append m[:uid], m[:length], flags: m[:flags]
+        end
+      rescue Exception => e
+        imap.rollback
+        raise e
+      end
+      # rubocop:enable Lint/RescueException
+    end
+
+    def mbox
+      @mbox ||= Serializer::Mbox.new(serializer.folder_path)
+    end
+
+    def imap
+      @imap ||= Serializer::Imap.new(serializer.folder_path)
+    end
+
+    def tsx
+      @tsx ||= Serializer::Transaction.new(owner: self)
+    end
+  end
+end

data/lib/imap/backup/serializer/imap.rb

@@ -1,5 +1,8 @@
 require "json"
 
+require "imap/backup/serializer/message"
+require "imap/backup/serializer/transaction"
+
 module Imap; end
 
 module Imap::Backup
@@ -15,26 +18,32 @@ module Imap::Backup
       @uid_validity = nil
       @messages = nil
       @version = nil
-      @savepoint = nil
+      @tsx = nil
     end
 
     def transaction(&block)
-      fail_in_transaction!(message: "Serializer::Imap: nested transactions are not supported")
+      tsx.fail_in_transaction!(:transaction, message: "nested transactions are not supported")
 
       ensure_loaded
-      @savepoint = {messages: messages.dup, uid_validity: uid_validity}
-
-      block.call
-
-      @savepoint = nil
-      save
+      # rubocop:disable Lint/RescueException
+      tsx.begin({savepoint: {messages: messages.dup, uid_validity: uid_validity}}) do
+        block.call
+
+        save_internal(version: version, uid_validity: uid_validity, messages: messages) if tsx.data
+      rescue Exception => e
+        rollback
+        raise e
+      end
+      # rubocop:enable Lint/RescueException
     end
 
     def rollback
-      fail_outside_transaction!
+      tsx.fail_outside_transaction!(:rollback)
+
+      @messages = tsx.data[:savepoint][:messages]
+      @uid_validity = tsx.data[:savepoint][:uid_validity]
 
-      @messages = @savepoint[:messages]
-      @uid_validity = @savepoint[:uid_validity]
+      tsx.clear
     end
 
     def pathname
@@ -129,10 +138,16 @@ module Imap::Backup
     end
 
     def save
-      return if @savepoint
+      return if tsx.in_transaction?
 
       ensure_loaded
 
+      save_internal(version: version, uid_validity: uid_validity, messages: messages)
+    end
+
+    private
+
+    def save_internal(version:, uid_validity:, messages:)
       raise "Cannot save metadata without a uid_validity" if !uid_validity
 
       data = {
@@ -144,8 +159,6 @@ module Imap::Backup
       File.open(pathname, "w") { |f| f.write content }
     end
 
-    private
-
     def ensure_loaded
       return if loaded
 
@@ -185,12 +198,8 @@ module Imap::Backup
       @mbox ||= Serializer::Mbox.new(folder_path)
     end
 
-    def fail_in_transaction!(message: "Serializer::Imap: method not supported inside trasactions")
-      raise message if @savepoint
-    end
-
-    def fail_outside_transaction!
-      raise "This method can only be called inside a transaction" if !@savepoint
+    def tsx
+      @tsx ||= Serializer::Transaction.new(owner: self)
     end
   end
 end

data/lib/imap/backup/serializer/integrity_checker.rb

@@ -0,0 +1,87 @@
+module Imap; end
+
+module Imap::Backup
+  class Serializer; end
+
+  class Serializer::FolderIntegrityError < StandardError; end
+
+  class Serializer::IntegrityChecker
+    attr_reader :imap
+    attr_reader :mbox
+
+    def initialize(imap:, mbox:)
+      @imap = imap
+      @mbox = mbox
+    end
+
+    def run
+      if !imap.valid?
+        message = ".imap file '#{imap.pathname}' is corrupt"
+        raise Serializer::FolderIntegrityError, message
+      end
+
+      if !mbox.exist?
+        message = ".mbox file '#{mbox.pathname}' is missing"
+        raise Serializer::FolderIntegrityError, message
+      end
+
+      if imap.messages.empty?
+        if mbox.length.positive?
+          message =
+            ".imap file '#{imap.pathname}' lists no messages, " \
+            "but .mbox file '#{mbox.pathname}' is not empty"
+          raise Serializer::FolderIntegrityError, message
+        end
+        return
+      end
+
+      check_offset_ordering!
+      check_mbox_length!
+      check_message_starts!
+
+      nil
+    end
+
+    private
+
+    def check_offset_ordering!
+      offsets = imap.messages.map(&:offset)
+
+      if offsets != offsets.sort
+        message = ".imap file '#{imap.pathname}' has offset data which is out of order"
+        raise Serializer::FolderIntegrityError, message
+      end
+    end
+
+    def check_mbox_length!
+      last = imap.messages[-1]
+
+      if mbox.length < last.offset + last.length
+        message =
+          ".mbox file '#{mbox.pathname}' is shorter than indicated by " \
+          ".imap file '#{imap.pathname}'"
+        raise Serializer::FolderIntegrityError, message
+      end
+
+      if mbox.length > last.offset + last.length
+        message =
+          ".mbox file '#{mbox.pathname}' is longer than indicated by " \
+          ".imap file '#{imap.pathname}'"
+        raise Serializer::FolderIntegrityError, message
+      end
+    end
+
+    def check_message_starts!
+      imap.messages.each do |m|
+        text = mbox.read(m.offset, m.length)
+
+        next if text.start_with?("From ")
+
+        message =
+          "Message #{m.uid} not found at expected offset #{m.offset} " \
+          "in file '#{mbox.pathname}'"
+        raise Serializer::FolderIntegrityError, message
+      end
+    end
+  end
+end

data/lib/imap/backup/serializer/mbox.rb

@@ -1,3 +1,5 @@
+require "imap/backup/serializer/transaction"
+
 module Imap; end
 
 module Imap::Backup
@@ -7,21 +9,32 @@ module Imap::Backup
 
     def initialize(folder_path)
       @folder_path = folder_path
-      @savepoint = nil
+      @tsx = nil
     end
 
     def transaction(&block)
-      fail_in_transaction!(message: "Nested transactions are not supported")
-
-      @savepoint = {length: length}
-      block.call
-      @savepoint = nil
+      tsx.fail_in_transaction!(:transaction, message: "nested transactions are not supported")
+
+      # rubocop:disable Lint/RescueException
+      tsx.begin({savepoint: {length: length}}) do
+        block.call
+      rescue Exception => e
+        message = <<~ERROR
+          #{self.class} error #{e}
+          #{e.backtrace.join("\n")}
+        ERROR
+        Logger.logger.error message
+        rollback
+        raise e
+      end
+      # rubocop:enable Lint/RescueException
     end
 
     def rollback
-      fail_outside_transaction!
+      tsx.fail_outside_transaction!(:rollback)
 
-      rewind(@savepoint[:length])
+      rewind(tsx.data[:savepoint][:length])
+      tsx.clear
     end
 
     def valid?
@@ -83,12 +96,8 @@ module Imap::Backup
       end
     end
 
-    def fail_in_transaction!(message: "Method not supported inside trasactions")
-      raise message if savepoint
-    end
-
-    def fail_outside_transaction!
-      raise "This method can only be called inside a transaction" if !savepoint
+    def tsx
+      @tsx ||= Serializer::Transaction.new(owner: self)
     end
   end
 end

data/lib/imap/backup/serializer/transaction.rb

@@ -0,0 +1,37 @@
+module Imap; end
+
+module Imap::Backup
+  class Serializer::Transaction
+    attr_reader :owner
+    attr_reader :data
+
+    def initialize(owner:)
+      @data = nil
+      @owner = owner
+      @in_transaction = false
+    end
+
+    def begin(data, &block)
+      @data = data
+      @in_transaction = true
+      block.call
+      @in_transaction = false
+    end
+
+    def clear
+      @data = nil
+    end
+
+    def in_transaction?
+      @in_transaction
+    end
+
+    def fail_in_transaction!(method, message: "not supported inside trasactions")
+      raise "#{owner.class}##{method} #{message}" if in_transaction?
+    end
+
+    def fail_outside_transaction!(method)
+      raise "#{owner.class}##{method} can only be called inside a transaction" if !in_transaction?
+    end
+  end
+end

data/lib/imap/backup/serializer.rb

@@ -3,8 +3,8 @@ require "forwardable"
 require "email/mboxrd/message"
 require "imap/backup/naming"
 require "imap/backup/serializer/appender"
+require "imap/backup/serializer/integrity_checker"
 require "imap/backup/serializer/imap"
-require "imap/backup/serializer/message"
 require "imap/backup/serializer/mbox"
 require "imap/backup/serializer/message_enumerator"
 require "imap/backup/serializer/version2_migrator"
@@ -21,43 +21,28 @@ module Imap::Backup
 
     extend Forwardable
 
-    class FolderIntegrityError < StandardError; end
-
     def_delegator :mbox, :pathname, :mbox_pathname
     def_delegators :imap, :get, :messages, :uid_validity, :uids, :update_uid
 
     attr_reader :folder
     attr_reader :path
-    attr_reader :dirty
 
     def initialize(path, folder)
       @path = path
       @folder = folder
       @validated = nil
-      @dirty = nil
     end
 
     def transaction(&block)
-      fail_in_transaction!(:transaction, message: "nested transactions are not supported")
-
-      validate!
-      @dirty = {append: []}
-
       block.call
+    end
 
-      if dirty[:append].any?
-        appender = Serializer::Appender.new(folder: sanitized, imap: imap, mbox: mbox)
-        appender.multi(dirty[:append])
-      end
-
-      @dirty = nil
+    def rollback
     end
 
     # Returns true if there are existing, valid files
     # false otherwise (in which case any existing files are deleted)
     def validate!
-      fail_in_transaction!(:validate!)
-
       return true if @validated
 
       optionally_migrate2to3
@@ -73,61 +58,10 @@ module Imap::Backup
     end
 
     def check_integrity!
-      fail_in_transaction!(:check_integrity!)
-
-      if !imap.valid?
-        message = ".imap file '#{imap.pathname}' is corrupt"
-        raise FolderIntegrityError, message
-      end
-
-      if !mbox.exist?
-        message = ".mbox file '#{mbox.pathname}' is missing"
-        raise FolderIntegrityError, message
-      end
-
-      return if imap.messages.empty?
-
-      offsets = imap.messages.map(&:offset)
-
-      if offsets != offsets.sort
-        message = ".imap file '#{imap.pathname}' has offset data which is out of order"
-        raise FolderIntegrityError, message
-      end
-
-      if mbox.length < offsets[-1]
-        message =
-          ".imap file '#{imap.pathname}' has offsets past the end " \
-          "of .mbox file '#{mbox.pathname}'"
-        raise FolderIntegrityError, message
-      end
-
-      imap.messages.each do |m|
-        text = mbox.read(m.offset, m.length)
-        if text.length < m.length
-          message = "Message #{m.uid} is incomplete in file '#{mbox.pathname}'"
-          raise FolderIntegrityError, message
-        end
-
-        next if text.start_with?("From ")
-
-        message =
-          "Message #{m.uid} not found at expected offset #{m.offset} " \
-          "in file '#{mbox.pathname}'"
-        raise FolderIntegrityError, message
-      end
-
-      last = imap.messages.last
-      expected_length = last.offset + last.length
-      actual_length = mbox.length
-      return if actual_length == expected_length
-
-      message = "Mbox file '#{mbox.pathname}' contains unexpected trailing data"
-      raise FolderIntegrityError, message
+      IntegrityChecker.new(imap: imap, mbox: mbox).run
     end
 
     def delete
-      fail_in_transaction!(:delete)
-
       imap.delete
       @imap = nil
       mbox.delete
@@ -135,7 +69,6 @@ module Imap::Backup
     end
 
     def apply_uid_validity(value)
-      fail_in_transaction!(:apply_uid_validity)
       validate!
 
       case
@@ -151,26 +84,19 @@ module Imap::Backup
     end
 
     def force_uid_validity(value)
-      fail_in_transaction!(:force_uid_validity)
       validate!
 
       internal_force_uid_validity(value)
     end
 
     def append(uid, message, flags)
-      if dirty
-        dirty[:append] << {uid: uid, message: message, flags: flags}
-      else
-        validate!
+      validate!
 
-        appender = Serializer::Appender.new(folder: sanitized, imap: imap, mbox: mbox)
-        appender.single(uid: uid, message: message, flags: flags)
-      end
+      appender = Serializer::Appender.new(folder: sanitized, imap: imap, mbox: mbox)
+      appender.append(uid: uid, message: message, flags: flags)
     end
 
     def update(uid, flags: nil)
-      fail_in_transaction!(:update)
-
       message = imap.get(uid)
       return if !message
 
@@ -179,8 +105,6 @@ module Imap::Backup
     end
 
     def each_message(required_uids = nil, &block)
-      fail_in_transaction!(:each_message)
-
       return enum_for(:each_message, required_uids) if !block
 
       required_uids ||= uids
@@ -192,8 +116,6 @@ module Imap::Backup
     end
 
     def filter(&block)
-      fail_in_transaction!(:filter)
-
       temp_name = Serializer::UnusedNameFinder.new(serializer: self).run
       temp_folder_path = self.class.folder_path_for(path: path, folder: temp_name)
       new_mbox = Serializer::Mbox.new(temp_folder_path)
@@ -203,7 +125,7 @@ module Imap::Backup
       enumerator = Serializer::MessageEnumerator.new(imap: imap)
       enumerator.run(uids: uids) do |message|
         keep = block.call(message)
-        appender.single(uid: message.uid, message: message.body, flags: message.flags) if keep
+        appender.append(uid: message.uid, message: message.body, flags: message.flags) if keep
       end
       imap.delete
       new_imap.rename imap.folder_path
@@ -217,6 +139,10 @@ module Imap::Backup
       self.class.folder_path_for(path: path, folder: sanitized)
     end
 
+    def sanitized
+      @sanitized ||= Naming.to_local_path(folder)
+    end
+
     private
 
     def rename(new_name)
@@ -249,10 +175,6 @@ module Imap::Backup
         end
     end
 
-    def sanitized
-      @sanitized ||= Naming.to_local_path(folder)
-    end
-
     def optionally_migrate2to3
       migrator = Version2Migrator.new(folder_path)
       return if !migrator.required?
@@ -287,13 +209,5 @@ module Imap::Backup
       rename new_name
       new_name
     end
-
-    def fail_in_transaction!(method, message: "not supported inside trasactions")
-      raise "Serializer##{method} #{message}" if dirty
-    end
-
-    def fail_outside_transaction!(method)
-      raise "Serializer##{method} can only be called inside a transaction" if !dirty
-    end
   end
 end

data/lib/imap/backup/setup/global_options/download_strategy_chooser.rb

@@ -0,0 +1,85 @@
+require "imap/backup/configuration"
+
+module Imap; end
+module Imap::Backup; end
+class Imap::Backup::Setup; end
+
+class Imap::Backup::Setup::GlobalOptions
+  class DownloadStrategyChooser
+    attr_reader :config
+    attr_reader :highline
+
+    def initialize(config:, highline:)
+      @config = config
+      @highline = highline
+    end
+
+    def run
+      catch :done do
+        loop do
+          Kernel.system("clear")
+          create_menu
+        end
+      end
+    end
+
+    private
+
+    def create_menu
+      strategies = Imap::Backup::Configuration::DOWNLOAD_STRATEGIES
+      highline.choose do |menu|
+        menu.header = "Choose a Download Strategy"
+
+        strategies.each do |s|
+          current = s[:key] == config.download_strategy ? " <- current" : ""
+          topic = "#{s[:description]}#{current}"
+          menu.choice(topic) do
+            config.download_strategy = s[:key]
+          end
+        end
+        show_help menu
+        menu.choice("(q) return to main menu") { throw :done }
+        menu.hidden("quit") { throw :done }
+      end
+    end
+
+    def show_help(menu)
+      menu.choice("help") do
+        Kernel.puts <<~HELP
+          This setting changes how often data is written to disk during backups.
+
+          imap-backup uses two files per folder, a .mbox file with the actual
+          messages and a .imap file with metadata like message lengths and their
+          offsets within the .mbox file.
+
+          # write straight to disk
+
+          With this setting, each message and its metadata are written to disk
+          as they are downloaded.
+
+          This choice uses least memory and so is suitable for backing up onto
+          devices with limited memory, like Raspberry Pis.
+
+          # delay writing metadata
+
+          This is the default setting.
+
+          Here, messages (which are potentially very large) are appended to the
+          .mbox file as they are received, but the metadata is only written to
+          the .imap file once all the folder's messages have been downloaded.
+
+          This choice uses a little more memory than the previous setting, but
+          is **much** faster for large folders (potentially >30 times for
+          folders with >100k messages) and is less wearing on the disk.
+
+          # Other Performance Settings
+
+          Another configuration which affects backup performance is the
+          `multi_fetch_size` account-level setting.
+
+        HELP
+        highline.ask "Press a key "
+      end
+    end
+  end
+end

data/lib/imap/backup/setup/global_options.rb

@@ -0,0 +1,53 @@
+require "imap/backup/configuration"
+require "imap/backup/setup/global_options/download_strategy_chooser"
+
+module Imap; end
+
+module Imap::Backup
+  class Setup; end
+
+  class Setup::GlobalOptions
+    attr_reader :config
+    attr_reader :highline
+
+    def initialize(config:, highline:)
+      @config = config
+      @highline = highline
+    end
+
+    def run
+      catch :done do
+        loop do
+          Kernel.system("clear")
+          show_menu
+        end
+      end
+    end
+
+    private
+
+    def show_menu
+      highline.choose do |menu|
+        menu.header = <<~MENU.chomp
+          Global Options
+
+          These settings affect all accounts.
+
+          Choose an action
+        MENU
+        change_download_strategy menu
+        menu.choice("(q) return to main menu") { throw :done }
+        menu.hidden("quit") { throw :done }
+      end
+    end
+
+    def change_download_strategy(menu)
+      strategies = Imap::Backup::Configuration::DOWNLOAD_STRATEGIES
+      current = strategies.find { |s| s[:key] == config.download_strategy }
+      changed = config.download_strategy_modified ? " *" : ""
+      menu.choice("change download strategy (currently: '#{current[:description]}')#{changed}") do
+        DownloadStrategyChooser.new(config: config, highline: Setup.highline).run
+      end
+    end
+  end
+end

data/lib/imap/backup/setup.rb

@@ -2,6 +2,7 @@ require "highline"
 
 require "email/provider"
 require "imap/backup/account"
+require "imap/backup/setup/global_options"
 require "imap/backup/setup/helpers"
 
 module Imap; end
@@ -39,7 +40,7 @@ module Imap::Backup
         MENU
         account_items menu
         add_account_item menu
-        toggle_delay_download_writes menu
+        modify_global_options menu
         if config.modified?
           menu.choice("save and exit") do
             config.save
@@ -47,7 +48,8 @@ module Imap::Backup
           end
           menu.choice("exit without saving changes") { throw :done }
         else
-          menu.choice("quit") { throw :done }
+          menu.choice("(q) quit") { throw :done }
+          menu.hidden("quit") { throw :done }
         end
       end
     end
@@ -71,13 +73,10 @@ module Imap::Backup
       end
     end
 
-    def toggle_delay_download_writes(menu)
-      new_value = config.delay_download_writes ? false : true
-      modified = config.delay_download_writes_modified ? " *" : ""
-      change = config.delay_download_writes ? "don't delay" : "delay"
-      menu_item = "#{change} download writes#{modified}"
-      menu.choice(menu_item) do
-        config.delay_download_writes = new_value
+    def modify_global_options(menu)
+      changed = config.modified? ? " *" : ""
+      menu.choice("modify global options#{changed}") do
+        GlobalOptions.new(config: config, highline: Setup.highline).run
       end
     end
 

data/lib/imap/backup/version.rb

@@ -4,6 +4,6 @@ module Imap::Backup
   MAJOR    = 11
   MINOR    = 0
   REVISION = 0
-  PRE      = "rc1".freeze
+  PRE      = nil
   VERSION  = [MAJOR, MINOR, REVISION, PRE].compact.map(&:to_s).join(".")
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: imap-backup
 version: !ruby/object:Gem::Version
-  version: 11.0.0.rc1
+  version: 11.0.0
 platform: ruby
 authors:
 - Joe Yates
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-07-18 00:00:00.000000000 Z
+date: 2023-07-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: highline
@@ -266,13 +266,16 @@ files:
 - lib/imap/backup/naming.rb
 - lib/imap/backup/serializer.rb
 - lib/imap/backup/serializer/appender.rb
+- lib/imap/backup/serializer/delayed_metadata_serializer.rb
 - lib/imap/backup/serializer/directory.rb
 - lib/imap/backup/serializer/folder_maker.rb
 - lib/imap/backup/serializer/imap.rb
+- lib/imap/backup/serializer/integrity_checker.rb
 - lib/imap/backup/serializer/mbox.rb
 - lib/imap/backup/serializer/message.rb
 - lib/imap/backup/serializer/message_enumerator.rb
 - lib/imap/backup/serializer/permission_checker.rb
+- lib/imap/backup/serializer/transaction.rb
 - lib/imap/backup/serializer/unused_name_finder.rb
 - lib/imap/backup/serializer/version2_migrator.rb
 - lib/imap/backup/setup.rb
@@ -283,6 +286,8 @@ files:
 - lib/imap/backup/setup/connection_tester.rb
 - lib/imap/backup/setup/email.rb
 - lib/imap/backup/setup/folder_chooser.rb
+- lib/imap/backup/setup/global_options.rb
+- lib/imap/backup/setup/global_options/download_strategy_chooser.rb
 - lib/imap/backup/setup/helpers.rb
 - lib/imap/backup/thunderbird/mailbox_exporter.rb
 - lib/imap/backup/uploader.rb
@@ -305,9 +310,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.6'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubygems_version: 3.3.7
 signing_key: