imap-backup 4.0.5 → 4.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6eebcf60acbb2e97a007e5da03c63fb7c32e24413397f28d3ec9f94154b8def5
-  data.tar.gz: 5f81961b55811c8c9fadabf2def11e1e00ab8572fd844f44b8fda371aa190bbb
+  metadata.gz: 00a744249ff80a26be2c57655709be59d9678c9bb998dc1c9f3ea4bf3539d747
+  data.tar.gz: 4f44c050c66fe2ef2e136b46dec581a91f77ab4b650e9b3466a893540e481a24
 SHA512:
-  metadata.gz: 6ebb0d93413c0911487235584837f76f6409d66cc06a64ad9e7db7ec0a4ccca2694b9be363e02922bbf1c981d944a9772e6705190ad92b461252d7b57bf1e4b0
-  data.tar.gz: 6f29640122bf11c098050476e60caa073ebb868a0859624d6e7b74df435e85a1d6ebbcf964840058c6978a42d16034d86341741caead66d99722554ffc2ca4a3
+  metadata.gz: 66067924fda33810aff4b02aa047c20cd26ce3fc31d8e752fc43a8f78af3e1c952871fa0ba372c670a5ee399e7fc686f9df9601fc5cbb7caa6f2065b4ac95b90
+  data.tar.gz: 443a86c3bda9cc1652fc9d1c5619347c1d82498befed16851a6f8a227eaf3299813dd6a4084e1bb0d180170adb1b63ccf8e5d2a8bb19f0457611e7916d796c4d

data/bin/imap-backup

@@ -2,7 +2,10 @@
 
 $LOAD_PATH.unshift(File.expand_path("../lib/", __dir__))
 require "imap/backup/cli"
+require "imap/backup/logger"
 
-Imap::Backup::Configuration::List.new.setup_logging
+Imap::Backup::Logger.setup_logging
 
-Imap::Backup::CLI.start(ARGV)
+Imap::Backup::Logger.sanitize_stderr do
+  Imap::Backup::CLI.start(ARGV)
+end

data/lib/imap/backup/account/connection.rb

@@ -4,63 +4,74 @@ require "imap/backup/client/default"
 require "retry_on_error"
 
 module Imap::Backup
-  module Account; end
+  class Account; end
 
   class Account::Connection
     include RetryOnError
 
     LOGIN_RETRY_CLASSES = [EOFError, Errno::ECONNRESET, SocketError].freeze
 
-    attr_reader :connection_options
-    attr_reader :local_path
-    attr_reader :password
-    attr_reader :username
-
-    def initialize(options)
-      @username = options[:username]
-      @password = options[:password]
-      @local_path = options[:local_path]
-      @config_folders = options[:folders]
-      @server = options[:server]
-      @connection_options = options[:connection_options] || {}
-      @folders = nil
+    attr_reader :account
+
+    def initialize(account)
+      @account = account
+      reset
       create_account_folder
     end
 
-    def folders
-      @folders ||=
+    # TODO: Make this private once the 'folders' command
+    # has been removed.
+    def folder_names
+      @folder_names ||=
         begin
-          folders = client.list
+          folder_names = client.list
 
-          if folders.empty?
-            message = "Unable to get folder list for account #{username}"
-            Imap::Backup.logger.info message
+          if folder_names.empty?
+            message = "Unable to get folder list for account #{account.username}"
+            Imap::Backup::Logger.logger.info message
             raise message
           end
 
-          folders
+          folder_names
+        end
+    end
+
+    def backup_folders
+      @backup_folders ||=
+        begin
+          names =
+            if account.folders&.any?
+              account.folders.map { |af| af[:name] }
+            else
+              folder_names
+            end
+
+          names.map do |name|
+            Account::Folder.new(self, name)
+          end
         end
     end
 
     def status
-      backup_folders.map do |backup_folder|
-        f = Account::Folder.new(self, backup_folder[:name])
-        s = Serializer::Mbox.new(local_path, backup_folder[:name])
-        {name: backup_folder[:name], local: s.uids, remote: f.uids}
+      backup_folders.map do |folder|
+        s = Serializer::Mbox.new(account.local_path, folder.name)
+        {name: folder.name, local: s.uids, remote: folder.uids}
       end
     end
 
     def run_backup
-      Imap::Backup.logger.debug "Running backup of account: #{username}"
+      Imap::Backup::Logger.logger.debug "Running backup of account: #{account.username}"
       # start the connection so we get logging messages in the right order
       client
       each_folder do |folder, serializer|
         next if !folder.exist?
 
-        Imap::Backup.logger.debug "[#{folder.name}] running backup"
+        Imap::Backup::Logger.logger.debug "[#{folder.name}] running backup"
         serializer.apply_uid_validity(folder.uid_validity)
         begin
-          Downloader.new(folder, serializer).run
+          Downloader.new(
+            folder, serializer, block_size: config.download_block_size
+          ).run
         rescue Net::IMAP::ByeResponseError
           reconnect
           retry
@@ -71,11 +82,11 @@ module Imap::Backup
     def local_folders
       return enum_for(:local_folders) if !block_given?
 
-      glob = File.join(local_path, "**", "*.imap")
-      base = Pathname.new(local_path)
+      glob = File.join(account.local_path, "**", "*.imap")
+      base = Pathname.new(account.local_path)
       Pathname.glob(glob) do |path|
         name = path.relative_path_from(base).to_s[0..-6]
-        serializer = Serializer::Mbox.new(local_path, name)
+        serializer = Serializer::Mbox.new(account.local_path, name)
         folder = Account::Folder.new(self, name)
         yield serializer, folder
       end
@@ -89,18 +100,27 @@ module Imap::Backup
 
     def disconnect
       client.disconnect if @client
+      reset
     end
 
     def reconnect
       disconnect
+    end
+
+    def reset
+      @backup_folders = nil
       @client = nil
+      @config = nil
+      @folder_names = nil
+      @provider = nil
+      @server = nil
     end
 
     def client
       @client ||=
         retry_on_error(errors: LOGIN_RETRY_CLASSES) do
           options = provider_options
-          Imap::Backup.logger.debug(
+          Imap::Backup::Logger.logger.debug(
             "Creating IMAP instance: #{server}, options: #{options.inspect}"
           )
           client =
@@ -109,23 +129,22 @@ module Imap::Backup
             else
               Client::Default.new(server, options)
             end
-          Imap::Backup.logger.debug "Logging in: #{username}/#{masked_password}"
-          client.login(username, password)
-          Imap::Backup.logger.debug "Login complete"
+          Imap::Backup::Logger.logger.debug "Logging in: #{account.username}/#{masked_password}"
+          client.login(account.username, account.password)
+          Imap::Backup::Logger.logger.debug "Login complete"
           client
         end
     end
 
     def server
-      @server ||= provider.host
+      @server ||= account.server || provider.host
     end
 
     private
 
     def each_folder
-      backup_folders.each do |backup_folder|
-        folder = Account::Folder.new(self, backup_folder[:name])
-        serializer = Serializer::Mbox.new(local_path, backup_folder[:name])
+      backup_folders.each do |folder|
+        serializer = Serializer::Mbox.new(account.local_path, folder.name)
         yield folder, serializer
       end
     end
@@ -133,16 +152,16 @@ module Imap::Backup
     def restore_folder(serializer, folder)
       existing_uids = folder.uids
       if existing_uids.any?
-        Imap::Backup.logger.debug(
+        Imap::Backup::Logger.logger.debug(
           "There's already a '#{folder.name}' folder with emails"
         )
         new_name = serializer.apply_uid_validity(folder.uid_validity)
         old_name = serializer.folder
         if new_name
-          Imap::Backup.logger.debug(
+          Imap::Backup::Logger.logger.debug(
             "Backup '#{old_name}' renamed and restored to '#{new_name}'"
           )
-          new_serializer = Serializer::Mbox.new(local_path, new_name)
+          new_serializer = Serializer::Mbox.new(account.local_path, new_name)
           new_folder = Account::Folder.new(self, new_name)
           new_folder.create
           new_serializer.force_uid_validity(new_folder.uid_validity)
@@ -159,33 +178,26 @@ module Imap::Backup
 
     def create_account_folder
       Utils.make_folder(
-        File.dirname(local_path),
-        File.basename(local_path),
+        File.dirname(account.local_path),
+        File.basename(account.local_path),
         Serializer::DIRECTORY_PERMISSIONS
       )
     end
 
     def masked_password
-      password.gsub(/./, "x")
-    end
-
-    def backup_folders
-      @backup_folders ||=
-        begin
-          if @config_folders&.any?
-            @config_folders
-          else
-            folders.map { |name| {name: name} }
-          end
-        end
+      account.password.gsub(/./, "x")
     end
 
     def provider
-      @provider ||= Email::Provider.for_address(username)
+      @provider ||= Email::Provider.for_address(account.username)
     end
 
     def provider_options
-      provider.options.merge(connection_options)
+      provider.options.merge(account.connection_options || {})
+    end
+
+    def config
+      @config ||= Configuration.new
     end
   end
 end

data/lib/imap/backup/account/folder.rb

@@ -3,7 +3,7 @@ require "forwardable"
 require "retry_on_error"
 
 module Imap::Backup
-  module Account; end
+  class Account; end
 
   class FolderNotFound < StandardError; end
 
@@ -64,7 +64,7 @@ module Imap::Backup
         in `search_internal` in stdlib net/imap.rb.
         This is caused by `@responses["SEARCH"] being unset/undefined
       MESSAGE
-      Imap::Backup.logger.warn message
+      Imap::Backup::Logger.logger.warn message
       []
     end
 
@@ -84,6 +84,26 @@ module Imap::Backup
       nil
     end
 
+    def fetch_multi(uids)
+      examine
+      fetch_data_items =
+        retry_on_error(errors: UID_FETCH_RETRY_CLASSES) do
+          client.uid_fetch(uids, [BODY_ATTRIBUTE])
+        end
+      return nil if fetch_data_items.nil?
+
+      fetch_data_items.map do |item|
+        attributes = item.attr
+
+        {
+          uid: attributes["UID"],
+          body: attributes[BODY_ATTRIBUTE]
+        }
+      end
+    rescue FolderNotFound
+      nil
+    end
+
     def append(message)
       body = message.imap_body
       date = message.date&.to_time
@@ -96,7 +116,7 @@ module Imap::Backup
     def examine
       client.examine(utf7_encoded_name)
     rescue Net::IMAP::NoResponseError
-      Imap::Backup.logger.warn "Folder '#{name}' does not exist on server"
+      Imap::Backup::Logger.logger.warn "Folder '#{name}' does not exist on server"
       raise FolderNotFound, "Folder '#{name}' does not exist on server"
     end
 

data/lib/imap/backup/account.rb

@@ -20,6 +20,10 @@ module Imap::Backup
       @marked_for_deletion = false
     end
 
+    def connection
+      Account::Connection.new(self)
+    end
+
     def valid?
       username && password
     end
@@ -83,14 +87,9 @@ module Imap::Backup
     def update(field, value)
       if changes[field]
         change = changes[field]
-        if change[:from] == value
-          changes.delete(field)
-        else
-          set_field!(field, value)
-        end
-      else
-        set_field!(field, value)
+        changes.delete(field) if change[:from] == value
       end
+      set_field!(field, value)
     end
 
     def set_field!(field, value)

data/lib/imap/backup/cli/accounts.rb

@@ -0,0 +1,43 @@
+module Imap::Backup
+  class CLI; end
+
+  class CLI::Accounts
+    include Enumerable
+
+    attr_reader :required_accounts
+
+    def initialize(required_accounts = [])
+      @required_accounts = required_accounts
+    end
+
+    def each(&block)
+      return enum_for(:each) if !block
+
+      accounts.each(&block)
+    end
+
+    private
+
+    def accounts
+      @accounts ||=
+        if required_accounts.empty?
+          config.accounts
+        else
+          config.accounts.select do |account|
+            required_accounts.include?(account.username)
+          end
+        end
+    end
+
+    def config
+      @config ||= begin
+        exists = Configuration.exist?
+        if !exists
+          path = Configuration.default_pathname
+          raise ConfigurationNotFound, "Configuration file '#{path}' not found"
+        end
+        Configuration.new
+      end
+    end
+  end
+end

data/lib/imap/backup/cli/folders.rb

@@ -14,7 +14,9 @@ module Imap::Backup
       def run
         each_connection(account_names) do |connection|
           puts connection.username
-          folders = connection.folders
+          # TODO: Make folder_names private once this command
+          # has been removed.
+          folders = connection.folder_names
           if folders.nil?
             warn "Unable to list account folders"
             return false

data/lib/imap/backup/cli/helpers.rb

@@ -1,4 +1,5 @@
 require "imap/backup"
+require "imap/backup/cli/accounts"
 
 module Imap::Backup::CLI::Helpers
   def symbolized(options)
@@ -6,8 +7,8 @@ module Imap::Backup::CLI::Helpers
   end
 
   def account(email)
-    connections = Imap::Backup::Configuration::List.new
-    account = connections.accounts.find { |a| a[:username] == email }
+    accounts = Imap::Backup::CLI::Accounts.new
+    account = accounts.find { |a| a.username == email }
     raise "#{email} is not a configured account" if !account
 
     account
@@ -20,14 +21,12 @@ module Imap::Backup::CLI::Helpers
   end
 
   def each_connection(names)
-    begin
-      connections = Imap::Backup::Configuration::List.new(names)
-    rescue Imap::Backup::ConfigurationNotFound
-      raise "imap-backup is not configured. Run `imap-backup setup`"
-    end
+    accounts = Imap::Backup::CLI::Accounts.new(names)
 
-    connections.each_connection do |connection|
-      yield connection
+    accounts.each do |account|
+      yield account.connection
     end
+  rescue Imap::Backup::ConfigurationNotFound
+    raise "imap-backup is not configured. Run `imap-backup setup`"
   end
 end

data/lib/imap/backup/cli/local.rb

@@ -1,3 +1,5 @@
+require "imap/backup/cli/accounts"
+
 module Imap::Backup
   class CLI::Local < Thor
     include Thor::Actions
@@ -5,8 +7,8 @@ module Imap::Backup
 
     desc "accounts", "List locally backed-up accounts"
     def accounts
-      connections = Imap::Backup::Configuration::List.new
-      connections.accounts.each { |a| Kernel.puts a[:username] }
+      accounts = CLI::Accounts.new
+      accounts.each { |a| Kernel.puts a.username }
     end
 
     desc "folders EMAIL", "List account folders"

data/lib/imap/backup/cli/setup.rb

@@ -7,7 +7,7 @@ class Imap::Backup::CLI::Setup < Thor
 
   no_commands do
     def run
-      Imap::Backup::Configuration::Setup.new.run
+      Imap::Backup::Setup.new.run
     end
   end
 end

data/lib/imap/backup/cli/status.rb

@@ -13,7 +13,7 @@ module Imap::Backup
     no_commands do
       def run
         each_connection(account_names) do |connection|
-          puts connection.username
+          puts connection.account.username
           folders = connection.status
           folders.each do |f|
             missing_locally = f[:remote] - f[:local]

data/lib/imap/backup/cli/utils.rb

@@ -11,9 +11,10 @@ module Imap::Backup
     def ignore_history(email)
       connection = connection(email)
 
-      connection.local_folders.each do |serializer, folder|
+      connection.backup_folders.each do |folder|
         next if !folder.exist?
 
+        serializer = Serializer::Mbox.new(connection.account.local_path, folder.name)
         do_ignore_folder_history(folder, serializer)
       end
     end
@@ -63,7 +64,7 @@ module Imap::Backup
     no_commands do
       def do_ignore_folder_history(folder, serializer)
         uids = folder.uids - serializer.uids
-        Imap::Backup.logger.info "Folder '#{folder.name}' - #{uids.length} messages"
+        Imap::Backup::Logger.logger.info "Folder '#{folder.name}' - #{uids.length} messages"
 
         serializer.apply_uid_validity(folder.uid_validity)
 

data/lib/imap/backup/{configuration/store.rb → configuration.rb}

@@ -1,11 +1,13 @@
 require "json"
 require "os"
 
-module Imap::Backup
-  module Configuration; end
+require "imap/backup/account"
 
-  class Configuration::Store
+module Imap::Backup
+  class Configuration
     CONFIGURATION_DIRECTORY = File.expand_path("~/.imap-backup")
+    DEFAULT_DOWNLOAD_BLOCK_SIZE = 1
+    VERSION = "2.0"
 
     attr_reader :pathname
 
@@ -19,6 +21,8 @@ module Imap::Backup
 
     def initialize(pathname = self.class.default_pathname)
       @pathname = pathname
+      @saved_debug = nil
+      @debug = nil
     end
 
     def path
@@ -26,53 +30,84 @@ module Imap::Backup
     end
 
     def save
+      ensure_loaded!
       FileUtils.mkdir(path) if !File.directory?(path)
       make_private(path) if !windows?
       remove_modified_flags
       remove_deleted_accounts
-      File.open(pathname, "w") { |f| f.write(JSON.pretty_generate(data)) }
+      save_data = {
+        version: VERSION,
+        accounts: accounts.map(&:to_h),
+        debug: debug?
+      }
+      File.open(pathname, "w") { |f| f.write(JSON.pretty_generate(save_data)) }
       FileUtils.chmod(0o600, pathname) if !windows?
+      @data = nil
     end
 
     def accounts
-      data[:accounts]
+      @accounts ||= begin
+        ensure_loaded!
+        data[:accounts].map { |data| Account.new(data) }
+      end
+    end
+
+    def download_block_size
+      size = ENV["DOWNLOAD_BLOCK_SIZE"].to_i
+      if size > 0
+        size
+      else
+        DEFAULT_DOWNLOAD_BLOCK_SIZE
+      end
     end
 
     def modified?
-      accounts.any? { |a| a[:modified] || a[:delete] }
+      ensure_loaded!
+      return true if @saved_debug != @debug
+
+      accounts.any? { |a| a.modified? || a.marked_for_deletion? }
     end
 
     def debug?
-      data[:debug]
+      ensure_loaded!
+      @debug
     end
 
     def debug=(value)
-      data[:debug] = [true, false].include?(value) ? value : false
+      ensure_loaded!
+      @debug = [true, false].include?(value) ? value : false
     end
 
     private
 
+    def ensure_loaded!
+      return true if @data
+
+      data
+      @debug = data.key?(:debug) ? data[:debug] == true : false
+      @saved_debug = @debug
+      true
+    end
+
     def data
       @data ||=
         begin
           if File.exist?(pathname)
             Utils.check_permissions(pathname, 0o600) if !windows?
             contents = File.read(pathname)
-            data = JSON.parse(contents, symbolize_names: true)
+            JSON.parse(contents, symbolize_names: true)
           else
-            data = {accounts: []}
+            {accounts: []}
           end
-          data[:debug] = data.key?(:debug) ? data[:debug] == true : false
-          data
         end
     end
 
     def remove_modified_flags
-      accounts.each { |a| a.delete(:modified) }
+      accounts.each { |a| a.clear_changes! }
     end
 
     def remove_deleted_accounts
-      accounts.reject! { |a| a[:delete] }
+      accounts.reject! { |a| a.marked_for_deletion? }
     end
 
     def make_private(path)

data/lib/imap/backup/downloader.rb

@@ -2,27 +2,41 @@ module Imap::Backup
   class Downloader
     attr_reader :folder
     attr_reader :serializer
+    attr_reader :block_size
 
-    def initialize(folder, serializer)
+    def initialize(folder, serializer, block_size: 1)
       @folder = folder
       @serializer = serializer
+      @block_size = block_size
     end
 
     def run
       uids = folder.uids - serializer.uids
       count = uids.count
-      Imap::Backup.logger.debug "[#{folder.name}] #{count} new messages"
-      uids.each.with_index do |uid, i|
-        body = folder.fetch(uid)
-        log_prefix = "[#{folder.name}] uid: #{uid} (#{i + 1}/#{count}) -"
-        if body.nil?
-          Imap::Backup.logger.debug("#{log_prefix} not available - skipped")
-          next
+      Imap::Backup::Logger.logger.debug "[#{folder.name}] #{count} new messages"
+      uids.each_slice(block_size).with_index do |block, i|
+        offset = i * block_size + 1
+        uids_and_bodies = folder.fetch_multi(block)
+        if uids_and_bodies.nil?
+          if block_size > 1
+            Imap::Backup::Logger.logger.debug("[#{folder.name}] Multi fetch failed for UIDs #{block.join(", ")}, switching to single fetches")
+            @block_size = 1
+            redo
+          else
+            Imap::Backup::Logger.logger.debug("[#{folder.name}] Fetch failed for UID #{block[0]} - skipping")
+            next
+          end
+        end
+
+        uids_and_bodies.each.with_index do |uid_and_body, j|
+          uid = uid_and_body[:uid]
+          body = uid_and_body[:body]
+          Imap::Backup::Logger.logger.debug(
+            "[#{folder.name}] uid: #{uid} (#{offset +j}/#{count}) - " \
+              "#{body.size} bytes"
+          )
+          serializer.save(uid, body)
         end
-        Imap::Backup.logger.debug(
-          "#{log_prefix} #{body.size} bytes"
-        )
-        serializer.save(uid, body)
       end
     end
   end