imap-backup 10.0.1 → 11.0.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7675c421b9705eb8e4543fc9df035c6a9750b291688805e25cc7f6c2537936e8
-  data.tar.gz: e3027362856d02201bf5d2633804b6744238684bb30979c07d8cc53ef4afaf36
+  metadata.gz: e9cff069fefde8147d1995b254f3f0613575ff4f52232c2fc64c91dc00c49bde
+  data.tar.gz: a1b8cb3d48f41b2773fd20b842b8d3e3b71e673b6797c83a345e341d087060d8
 SHA512:
-  metadata.gz: 34771e1aa4dd8d8444e37b149d3a491b506363d6dfff00cd6ed6a8a7e501f6ac04e5d7ec24774fd36488bda96c6dfa6454b388d6afbe0f86e0b11317877100f0
-  data.tar.gz: f721a34ed6bf6af08d7a87335241e684a3d878538534300821c9466bdff5215b539f1f9cffebe484567370481460d114922f62fc7d73daba56801594ac3171b0
+  metadata.gz: 4e7b83bac1004b853287689e98ed4d03d7c3c87bd05c4a22831ffe592782ff8916ab576778e41de572f872505e3beab5b923ff5d1d78ccf3e1a99d701ff95737
+  data.tar.gz: a0cc1cc27b225c315a68743dff999fc2d8b0d0c9fd5d68f03641b59d53ddc491da21f318b9c075927263367b9db0a50e80af727de0445c86a049bc55d2a32fdf

data/README.md

@@ -100,6 +100,54 @@ For more information about a command, run
 imap-backup help COMMAND
 ```
 
+# Performace
+
+There are a couple of performance tweaks that you can use
+to improve backup speed.
+
+These are activated via two settings:
+
+* Global setting "Delay download writes"
+* Account setting "Multi-fetch size"
+
+As with all performance tweaks, there are trade-offs.
+If you are using a small virtual server or Raspberry Pi
+to run your backups, you will probably want to leave
+the deafult settings.
+If, on the other hand, you are using a computer with a
+fair bit of RAM, and you are dealing with a *lot* of email,
+then changing these settings may be worthwhile.
+
+## Delay download writes
+
+This setting affects all account backups.
+
+When not set, each message is written to disk, one at a time.
+Doing so means the message itself is appended to the MBox file,
+but more importantly, the JSON metadata is rewritten to disk
+from scratch.
+
+When in use, all of a mailboxes unbackupped messages are
+downloaded first, and then written to disk just once.
+
+This speeds up backup as the metadata file is not rewritten
+after each message is added, but it potentially uses much more memory.
+
+## Multi-fetch Size
+
+By default, during backup, each message is downloaded one-by-one.
+
+Using this setting, you can download chunks of emails at a time,
+potentially speeding up the process.
+
+If you're not using "Delayed downlaod writes",
+using multi-fetch *will* mean that the backup process will use
+more memory - equivalent to the size of the greater number
+of messages downloaded at a time.
+
+This behaviour may also exceed limits on your email provider,
+so it's best to check before cranking it up!
+
 # Troubleshooting
 
 If you have problems:

data/docs/development.md

@@ -43,6 +43,15 @@ or
 $ rspec --tag ~docker
 ```
 
+# Performance Specs
+
+```sh
+PERFORMANCE=1 rspec --order=defined
+```
+
+Beware: the performance spec (just backup for now) takes a very
+long time to run, approximately 24 hours!
+
 ### Debugging
 
 The feature specs are run 'out of process' via the Aruba gem.

data/lib/imap/backup/account/backup.rb

@@ -38,12 +38,19 @@ module Imap::Backup
 
         Logger.logger.debug "[#{folder.name}] running backup"
         serializer.apply_uid_validity(folder.uid_validity)
-        Downloader.new(
+        downloader = Downloader.new(
           folder,
           serializer,
           multi_fetch_size: account.multi_fetch_size,
           reset_seen_flags_after_fetch: account.reset_seen_flags_after_fetch
-        ).run
+        )
+        if account.delay_download_writes
+          serializer.transaction do
+            downloader.run
+          end
+        else
+          downloader.run
+        end
         if account.mirror_mode
           Logger.logger.info "Mirror mode - Deleting messages only present locally"
           LocalOnlyMessageDeleter.new(folder, serializer).run

data/lib/imap/backup/account.rb

@@ -15,6 +15,7 @@ module Imap::Backup
     attr_reader :mirror_mode
     attr_reader :server
     attr_reader :connection_options
+    attr_accessor :delay_download_writes
     attr_reader :reset_seen_flags_after_fetch
     attr_reader :changes
 
@@ -27,7 +28,8 @@ module Imap::Backup
       @mirror_mode = options[:mirror_mode]
       @server = options[:server]
       @connection_options = options[:connection_options]
-      @multi_fetch_size = options[:multi_fetch_size]
+      @delay_download_writes = true
+      @multi_fetch_size_orignal = options[:multi_fetch_size]
       @reset_seen_flags_after_fetch = options[:reset_seen_flags_after_fetch]
       @client = nil
       @changes = {}
@@ -42,6 +44,10 @@ module Imap::Backup
       client.namespace
     end
 
+    def capabilities
+      client.capability
+    end
+
     def restore
       restore = Account::Restore.new(account: self)
       restore.run
@@ -75,7 +81,8 @@ module Imap::Backup
       h[:mirror_mode] = true if @mirror_mode
       h[:server] = @server if @server
       h[:connection_options] = @connection_options if @connection_options
-      h[:multi_fetch_size] = multi_fetch_size if @multi_fetch_size
+      h[:delay_download_writes] = delay_download_writes
+      h[:multi_fetch_size] = multi_fetch_size
       if @reset_seen_flags_after_fetch
         h[:reset_seen_flags_after_fetch] = @reset_seen_flags_after_fetch
       end
@@ -123,11 +130,13 @@ module Imap::Backup
     end
 
     def multi_fetch_size
-      int = @multi_fetch_size.to_i
-      if int.positive?
-        int
-      else
-        DEFAULT_MULTI_FETCH_SIZE
+      @multi_fetch_size ||= begin
+        int = @multi_fetch_size_orignal.to_i
+        if int.positive?
+          int
+        else
+          DEFAULT_MULTI_FETCH_SIZE
+        end
       end
     end
 

data/lib/imap/backup/cli/local.rb

@@ -41,7 +41,6 @@ module Imap::Backup
       results = requested_accounts(config).map do |account|
         serialized_folders = Account::SerializedFolders.new(account: account)
         folder_results = serialized_folders.map do |serializer, _folder|
-          puts "serializer: #{serializer.inspect}"
           serializer.check_integrity!
           {name: serializer.folder, result: "OK"}
         rescue Serializer::FolderIntegrityError => e

data/lib/imap/backup/cli/remote.rb

@@ -23,6 +23,22 @@ module Imap::Backup
       end
     end
 
+    desc "capabilities EMAIL", "List server capabilities"
+    long_desc <<~DESC
+      Lists the IMAP capabilities supported by the IMAP server.
+    DESC
+    config_option
+    format_option
+    quiet_option
+    verbose_option
+    def capabilities(email)
+      Imap::Backup::Logger.setup_logging options
+      config = load_config(**options)
+      account = account(config, email)
+      capabilities = account.capabilities
+      Kernel.puts capabilities.join(", ")
+    end
+
     desc "namespaces EMAIL", "List account namespaces"
     long_desc <<~DESC
       Lists namespaces defined for an email account.

data/lib/imap/backup/client/default.rb

@@ -9,7 +9,7 @@ module Imap::Backup
   class Client::Default
     extend Forwardable
     def_delegators :imap, *%i(
-      append authenticate create expunge namespace
+      append authenticate capability create expunge namespace
       responses uid_fetch uid_search uid_store
     )
 

data/lib/imap/backup/configuration.rb

@@ -14,6 +14,8 @@ module Imap::Backup
     VERSION = "2.0".freeze
 
     attr_reader :pathname
+    attr_reader :delay_download_writes
+    attr_reader :delay_download_writes_modified
 
     def self.default_pathname
       File.join(CONFIGURATION_DIRECTORY, "config.json")
@@ -25,6 +27,8 @@ module Imap::Backup
 
     def initialize(path: nil)
       @pathname = path || self.class.default_pathname
+      @delay_download_writes = false
+      @delay_download_writes_modified = false
     end
 
     def path
@@ -39,7 +43,8 @@ module Imap::Backup
       remove_deleted_accounts
       save_data = {
         version: VERSION,
-        accounts: accounts.map(&:to_h)
+        accounts: accounts.map(&:to_h),
+        delay_download_writes: delay_download_writes
       }
       File.open(pathname, "w") { |f| f.write(JSON.pretty_generate(save_data)) }
       FileUtils.chmod(0o600, pathname) if !windows?
@@ -49,13 +54,26 @@ module Imap::Backup
     def accounts
       @accounts ||= begin
         ensure_loaded!
-        data[:accounts].map { |data| Account.new(data) }
+        accounts = data[:accounts].map do |attr|
+          Account.new(attr)
+        end
+        inject_global_attributes(accounts)
       end
     end
 
+    def delay_download_writes=(value)
+      ensure_loaded!
+
+      @delay_download_writes = value
+      @delay_download_writes_modified = true
+      inject_global_attributes(accounts)
+    end
+
     def modified?
       ensure_loaded!
 
+      return true if delay_download_writes_modified
+
       accounts.any? { |a| a.modified? || a.marked_for_deletion? }
     end
 
@@ -65,6 +83,7 @@ module Imap::Backup
       return true if @data
 
       data
+      @delay_download_writes = data[:delay_download_writes]
       true
     end
 
@@ -83,6 +102,7 @@ module Imap::Backup
     end
 
     def remove_modified_flags
+      @delay_download_writes_modified = false
       accounts.each(&:clear_changes)
     end
 
@@ -90,6 +110,13 @@ module Imap::Backup
       accounts.reject!(&:marked_for_deletion?)
     end
 
+    def inject_global_attributes(accounts)
+      accounts.map do |a|
+        a.delay_download_writes = delay_download_writes
+        a
+      end
+    end
+
     def make_private(path)
       FileUtils.chmod(0o700, path) if FileMode.new(filename: path).mode != 0o700
     end

data/lib/imap/backup/serializer/appender.rb

@@ -14,7 +14,7 @@ module Imap::Backup
       @mbox = mbox
     end
 
-    def run(uid:, message:, flags:)
+    def single(uid:, message:, flags:)
       raise "Can't add messages without uid_validity" if !imap.uid_validity
 
       uid = uid.to_i
@@ -26,29 +26,43 @@ module Imap::Backup
         return
       end
 
-      do_append uid, message, flags
+      rollback_on_error do
+        do_append uid, message, flags
+      rescue StandardError => e
+        raise <<-ERROR.gsub(/^\s*/m, "")
+          [#{folder}] failed to append message #{uid}:
+          #{message}. #{e}:
+          #{e.backtrace.join("\n")}"
+        ERROR
+      end
+    end
+
+    def multi(appends)
+      rollback_on_error do
+        appends.each do |a|
+          do_append a[:uid], a[:message], a[:flags]
+        end
+      end
     end
 
     private
 
     def do_append(uid, message, flags)
       mboxrd_message = Email::Mboxrd::Message.new(message)
-      initial = mbox.length || 0
-      mbox_appended = false
-      begin
-        serialized = mboxrd_message.to_serialized
-        mbox.append serialized
-        mbox_appended = true
-        imap.append uid, serialized.length, flags
-      rescue StandardError => e
-        mbox.rewind(initial) if mbox_appended
+      serialized = mboxrd_message.to_serialized
+      mbox.append serialized
+      imap.append uid, serialized.length, flags: flags
+    end
 
-        error = <<-ERROR.gsub(/^\s*/m, "")
-          [#{folder}] failed to append message #{uid}:
-          #{message}. #{e}:
-          #{e.backtrace.join("\n")}"
-        ERROR
-        Logger.logger.warn error
+    def rollback_on_error(&block)
+      imap.transaction do
+        mbox.transaction do
+          block.call
+        rescue StandardError => e
+          Logger.logger.error e
+          imap.rollback
+          mbox.rollback
+        end
       end
     end
   end

data/lib/imap/backup/serializer/imap.rb

@@ -15,6 +15,26 @@ module Imap::Backup
       @uid_validity = nil
       @messages = nil
       @version = nil
+      @savepoint = nil
+    end
+
+    def transaction(&block)
+      fail_in_transaction!(message: "Serializer::Imap: nested transactions are not supported")
+
+      ensure_loaded
+      @savepoint = {messages: messages.dup, uid_validity: uid_validity}
+
+      block.call
+
+      @savepoint = nil
+      save
+    end
+
+    def rollback
+      fail_outside_transaction!
+
+      @messages = @savepoint[:messages]
+      @uid_validity = @savepoint[:uid_validity]
     end
 
     def pathname
@@ -33,7 +53,7 @@ module Imap::Backup
       true
     end
 
-    def append(uid, length, flags = [])
+    def append(uid, length, flags: [])
       offset =
         if messages.empty?
           0
@@ -109,14 +129,16 @@ module Imap::Backup
     end
 
     def save
+      return if @savepoint
+
       ensure_loaded
 
       raise "Cannot save metadata without a uid_validity" if !uid_validity
 
       data = {
-        version: @version,
-        uid_validity: @uid_validity,
-        messages: @messages.map(&:to_h)
+        version: version,
+        uid_validity: uid_validity,
+        messages: messages.map(&:to_h)
       }
       content = data.to_json
       File.open(pathname, "w") { |f| f.write content }
@@ -162,5 +184,13 @@ module Imap::Backup
     def mbox
       @mbox ||= Serializer::Mbox.new(folder_path)
     end
+
+    def fail_in_transaction!(message: "Serializer::Imap: method not supported inside trasactions")
+      raise message if @savepoint
+    end
+
+    def fail_outside_transaction!
+      raise "This method can only be called inside a transaction" if !@savepoint
+    end
   end
 end

data/lib/imap/backup/serializer/mbox.rb

@@ -3,9 +3,25 @@ module Imap; end
 module Imap::Backup
   class Serializer::Mbox
     attr_reader :folder_path
+    attr_reader :savepoint
 
     def initialize(folder_path)
       @folder_path = folder_path
+      @savepoint = nil
+    end
+
+    def transaction(&block)
+      fail_in_transaction!(message: "Nested transactions are not supported")
+
+      @savepoint = {length: length}
+      block.call
+      @savepoint = nil
+    end
+
+    def rollback
+      fail_outside_transaction!
+
+      rewind(@savepoint[:length])
     end
 
     def valid?
@@ -31,6 +47,10 @@ module Imap::Backup
       File.unlink(pathname)
     end
 
+    def exist?
+      File.exist?(pathname)
+    end
+
     def length
       return nil if !exist?
 
@@ -51,18 +71,24 @@ module Imap::Backup
       end
     end
 
+    def touch
+      File.open(pathname, "a") {}
+    end
+
+    private
+
     def rewind(length)
       File.open(pathname, File::RDWR | File::CREAT, 0o644) do |f|
         f.truncate(length)
       end
     end
 
-    def touch
-      File.open(pathname, "a") {}
+    def fail_in_transaction!(message: "Method not supported inside trasactions")
+      raise message if savepoint
     end
 
-    def exist?
-      File.exist?(pathname)
+    def fail_outside_transaction!
+      raise "This method can only be called inside a transaction" if !savepoint
     end
   end
 end

data/lib/imap/backup/serializer.rb

@@ -28,16 +28,36 @@ module Imap::Backup
 
     attr_reader :folder
     attr_reader :path
+    attr_reader :dirty
 
     def initialize(path, folder)
       @path = path
       @folder = folder
       @validated = nil
+      @dirty = nil
+    end
+
+    def transaction(&block)
+      fail_in_transaction!(:transaction, message: "nested transactions are not supported")
+
+      validate!
+      @dirty = {append: []}
+
+      block.call
+
+      if dirty[:append].any?
+        appender = Serializer::Appender.new(folder: sanitized, imap: imap, mbox: mbox)
+        appender.multi(dirty[:append])
+      end
+
+      @dirty = nil
     end
 
     # Returns true if there are existing, valid files
     # false otherwise (in which case any existing files are deleted)
     def validate!
+      fail_in_transaction!(:validate!)
+
       return true if @validated
 
       optionally_migrate2to3
@@ -53,6 +73,8 @@ module Imap::Backup
     end
 
     def check_integrity!
+      fail_in_transaction!(:check_integrity!)
+
       if !imap.valid?
         message = ".imap file '#{imap.pathname}' is corrupt"
         raise FolderIntegrityError, message
@@ -104,6 +126,8 @@ module Imap::Backup
     end
 
     def delete
+      fail_in_transaction!(:delete)
+
       imap.delete
       @imap = nil
       mbox.delete
@@ -111,6 +135,7 @@ module Imap::Backup
     end
 
     def apply_uid_validity(value)
+      fail_in_transaction!(:apply_uid_validity)
       validate!
 
       case
@@ -126,19 +151,26 @@ module Imap::Backup
     end
 
     def force_uid_validity(value)
+      fail_in_transaction!(:force_uid_validity)
       validate!
 
       internal_force_uid_validity(value)
     end
 
     def append(uid, message, flags)
-      validate!
+      if dirty
+        dirty[:append] << {uid: uid, message: message, flags: flags}
+      else
+        validate!
 
-      appender = Serializer::Appender.new(folder: sanitized, imap: imap, mbox: mbox)
-      appender.run(uid: uid, message: message, flags: flags)
+        appender = Serializer::Appender.new(folder: sanitized, imap: imap, mbox: mbox)
+        appender.single(uid: uid, message: message, flags: flags)
+      end
     end
 
     def update(uid, flags: nil)
+      fail_in_transaction!(:update)
+
       message = imap.get(uid)
       return if !message
 
@@ -147,17 +179,21 @@ module Imap::Backup
     end
 
     def each_message(required_uids = nil, &block)
+      fail_in_transaction!(:each_message)
+
+      return enum_for(:each_message, required_uids) if !block
+
       required_uids ||= uids
 
       validate!
 
-      return enum_for(:each_message, required_uids) if !block
-
       enumerator = Serializer::MessageEnumerator.new(imap: imap)
       enumerator.run(uids: required_uids, &block)
     end
 
     def filter(&block)
+      fail_in_transaction!(:filter)
+
       temp_name = Serializer::UnusedNameFinder.new(serializer: self).run
       temp_folder_path = self.class.folder_path_for(path: path, folder: temp_name)
       new_mbox = Serializer::Mbox.new(temp_folder_path)
@@ -167,7 +203,7 @@ module Imap::Backup
       enumerator = Serializer::MessageEnumerator.new(imap: imap)
       enumerator.run(uids: uids) do |message|
         keep = block.call(message)
-        appender.run(uid: message.uid, message: message.body, flags: message.flags) if keep
+        appender.single(uid: message.uid, message: message.body, flags: message.flags) if keep
       end
       imap.delete
       new_imap.rename imap.folder_path
@@ -251,5 +287,13 @@ module Imap::Backup
       rename new_name
       new_name
     end
+
+    def fail_in_transaction!(method, message: "not supported inside trasactions")
+      raise "Serializer##{method} #{message}" if dirty
+    end
+
+    def fail_outside_transaction!(method)
+      raise "Serializer##{method} can only be called inside a transaction" if !dirty
+    end
   end
 end

data/lib/imap/backup/setup.rb

@@ -39,6 +39,7 @@ module Imap::Backup
         MENU
         account_items menu
         add_account_item menu
+        toggle_delay_download_writes menu
         if config.modified?
           menu.choice("save and exit") do
             config.save
@@ -70,6 +71,16 @@ module Imap::Backup
       end
     end
 
+    def toggle_delay_download_writes(menu)
+      new_value = config.delay_download_writes ? false : true
+      modified = config.delay_download_writes_modified ? " *" : ""
+      change = config.delay_download_writes ? "don't delay" : "delay"
+      menu_item = "#{change} download writes#{modified}"
+      menu.choice(menu_item) do
+        config.delay_download_writes = new_value
+      end
+    end
+
     def default_account_config(username)
       Imap::Backup::Account.new(
         username: username,

data/lib/imap/backup/version.rb

@@ -1,9 +1,9 @@
 module Imap; end
 
 module Imap::Backup
-  MAJOR    = 10
+  MAJOR    = 11
   MINOR    = 0
-  REVISION = 1
-  PRE      = nil
+  REVISION = 0
+  PRE      = "rc1".freeze
   VERSION  = [MAJOR, MINOR, REVISION, PRE].compact.map(&:to_s).join(".")
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: imap-backup
 version: !ruby/object:Gem::Version
-  version: 10.0.1
+  version: 11.0.0.rc1
 platform: ruby
 authors:
 - Joe Yates
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-07-16 00:00:00.000000000 Z
+date: 2023-07-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: highline
@@ -305,9 +305,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.6'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.3.7
 signing_key: