imap-backup 4.0.6 → 4.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7b543b8f8b913b95b68c9ad9532143c375d90b468b0977982faa3522c0d3932b
-  data.tar.gz: '0939b42c8dad41d14212a18444d94c209ce025646683faded89b9808e6181416'
+  metadata.gz: e0cd29188ed972abfb2b2ce21d237ce3b7aed5b434717ba64fa2e3475da38ded
+  data.tar.gz: 597b58cdee1ab1b5881ea29111ac85e7fdedfbaa370824d0d24ba1b19c869dcb
 SHA512:
-  metadata.gz: 52f9058554faa9a3d31bef3cb7254408032cdddee98293ea5f7c8ef108d3187b564b7fd71a18bdb1b31829b46ed22bdd51626704f82e98d88076bd4cec887d44
-  data.tar.gz: 186036e2f11ed8e0ae578ab44d73b2b4c1f89bf9c65eedc66a143b4d1b861adacba1d954190d566c399cc79184ba3645ebb53a3341ee82a03ad9b5d5d7fa777d
+  metadata.gz: a3adc1984eea76a5ef993a40e89520a01997e6d01357e0a1e41cd4b58a4efc56801fc3d4f8801b95e828ec3a6a1e7884be974aaabf40f921a3faccdc99cf9b98
+  data.tar.gz: 55c0c91fdfb4829ba77be5292b4e475c6ea31e9e6e9ea5abf9702121f8a12c20b9b0105da5e1b97bcb2482105807c614530dea9e058d64af2cde96f7fd19348b

data/bin/imap-backup

@@ -2,7 +2,10 @@
 
 $LOAD_PATH.unshift(File.expand_path("../lib/", __dir__))
 require "imap/backup/cli"
+require "imap/backup/logger"
 
-Imap::Backup::Configuration::List.new.setup_logging
+Imap::Backup::Logger.setup_logging
 
-Imap::Backup::CLI.start(ARGV)
+Imap::Backup::Logger.sanitize_stderr do
+  Imap::Backup::CLI.start(ARGV)
+end

data/lib/email/mboxrd/message.rb

@@ -10,7 +10,7 @@ module Email::Mboxrd
 
     attr_reader :supplied_body
 
-    def self.from_serialized(serialized)
+    def self.clean_serialized(serialized)
       cleaned = serialized.gsub(/^>(>*From)/, "\\1")
       # Serialized messages in this format *should* start with a line
       #   From xxx yy zz
@@ -19,7 +19,11 @@ module Email::Mboxrd
         cleaned = cleaned.sub(/^From .*[\r\n]*/, "")
       end
       # rubocop:enable Style/IfUnlessModifier
-      new(cleaned)
+      cleaned
+    end
+
+    def self.from_serialized(serialized)
+      new(clean_serialized(serialized))
     end
 
     def initialize(supplied_body)

data/lib/imap/backup/account/connection.rb

@@ -15,44 +15,63 @@ module Imap::Backup
 
     def initialize(account)
       @account = account
-      @folders = nil
+      reset
       create_account_folder
     end
 
-    def folders
-      @folders ||=
+    # TODO: Make this private once the 'folders' command
+    # has been removed.
+    def folder_names
+      @folder_names ||=
         begin
-          folders = client.list
+          folder_names = client.list
 
-          if folders.empty?
+          if folder_names.empty?
             message = "Unable to get folder list for account #{account.username}"
-            Imap::Backup.logger.info message
+            Imap::Backup::Logger.logger.info message
             raise message
           end
 
-          folders
+          folder_names
+        end
+    end
+
+    def backup_folders
+      @backup_folders ||=
+        begin
+          names =
+            if account.folders&.any?
+              account.folders.map { |af| af[:name] }
+            else
+              folder_names
+            end
+
+          names.map do |name|
+            Account::Folder.new(self, name)
+          end
         end
     end
 
     def status
-      backup_folders.map do |backup_folder|
-        f = Account::Folder.new(self, backup_folder[:name])
-        s = Serializer::Mbox.new(account.local_path, backup_folder[:name])
-        {name: backup_folder[:name], local: s.uids, remote: f.uids}
+      backup_folders.map do |folder|
+        s = Serializer::Mbox.new(account.local_path, folder.name)
+        {name: folder.name, local: s.uids, remote: folder.uids}
       end
     end
 
     def run_backup
-      Imap::Backup.logger.debug "Running backup of account: #{account.username}"
+      Imap::Backup::Logger.logger.debug "Running backup of account: #{account.username}"
       # start the connection so we get logging messages in the right order
       client
       each_folder do |folder, serializer|
         next if !folder.exist?
 
-        Imap::Backup.logger.debug "[#{folder.name}] running backup"
+        Imap::Backup::Logger.logger.debug "[#{folder.name}] running backup"
         serializer.apply_uid_validity(folder.uid_validity)
         begin
-          Downloader.new(folder, serializer).run
+          Downloader.new(
+            folder, serializer, block_size: config.download_block_size
+          ).run
         rescue Net::IMAP::ByeResponseError
           reconnect
           retry
@@ -81,18 +100,27 @@ module Imap::Backup
 
     def disconnect
       client.disconnect if @client
+      reset
     end
 
     def reconnect
       disconnect
+    end
+
+    def reset
+      @backup_folders = nil
       @client = nil
+      @config = nil
+      @folder_names = nil
+      @provider = nil
+      @server = nil
     end
 
     def client
       @client ||=
         retry_on_error(errors: LOGIN_RETRY_CLASSES) do
           options = provider_options
-          Imap::Backup.logger.debug(
+          Imap::Backup::Logger.logger.debug(
             "Creating IMAP instance: #{server}, options: #{options.inspect}"
           )
           client =
@@ -101,9 +129,9 @@ module Imap::Backup
             else
               Client::Default.new(server, options)
             end
-          Imap::Backup.logger.debug "Logging in: #{account.username}/#{masked_password}"
+          Imap::Backup::Logger.logger.debug "Logging in: #{account.username}/#{masked_password}"
           client.login(account.username, account.password)
-          Imap::Backup.logger.debug "Login complete"
+          Imap::Backup::Logger.logger.debug "Login complete"
           client
         end
     end
@@ -115,9 +143,8 @@ module Imap::Backup
     private
 
     def each_folder
-      backup_folders.each do |backup_folder|
-        folder = Account::Folder.new(self, backup_folder[:name])
-        serializer = Serializer::Mbox.new(account.local_path, backup_folder[:name])
+      backup_folders.each do |folder|
+        serializer = Serializer::Mbox.new(account.local_path, folder.name)
         yield folder, serializer
       end
     end
@@ -125,13 +152,13 @@ module Imap::Backup
     def restore_folder(serializer, folder)
       existing_uids = folder.uids
       if existing_uids.any?
-        Imap::Backup.logger.debug(
+        Imap::Backup::Logger.logger.debug(
           "There's already a '#{folder.name}' folder with emails"
         )
         new_name = serializer.apply_uid_validity(folder.uid_validity)
         old_name = serializer.folder
         if new_name
-          Imap::Backup.logger.debug(
+          Imap::Backup::Logger.logger.debug(
             "Backup '#{old_name}' renamed and restored to '#{new_name}'"
           )
           new_serializer = Serializer::Mbox.new(account.local_path, new_name)
@@ -161,17 +188,6 @@ module Imap::Backup
       account.password.gsub(/./, "x")
     end
 
-    def backup_folders
-      @backup_folders ||=
-        begin
-          if account.folders&.any?
-            account.folders
-          else
-            folders.map { |name| {name: name} }
-          end
-        end
-    end
-
     def provider
       @provider ||= Email::Provider.for_address(account.username)
     end
@@ -179,5 +195,9 @@ module Imap::Backup
     def provider_options
       provider.options.merge(account.connection_options || {})
     end
+
+    def config
+      @config ||= Configuration.new
+    end
   end
 end

data/lib/imap/backup/account/folder.rb

@@ -64,7 +64,7 @@ module Imap::Backup
         in `search_internal` in stdlib net/imap.rb.
         This is caused by `@responses["SEARCH"] being unset/undefined
       MESSAGE
-      Imap::Backup.logger.warn message
+      Imap::Backup::Logger.logger.warn message
       []
     end
 
@@ -84,6 +84,26 @@ module Imap::Backup
       nil
     end
 
+    def fetch_multi(uids)
+      examine
+      fetch_data_items =
+        retry_on_error(errors: UID_FETCH_RETRY_CLASSES) do
+          client.uid_fetch(uids, [BODY_ATTRIBUTE])
+        end
+      return nil if fetch_data_items.nil?
+
+      fetch_data_items.map do |item|
+        attributes = item.attr
+
+        {
+          uid: attributes["UID"],
+          body: attributes[BODY_ATTRIBUTE]
+        }
+      end
+    rescue FolderNotFound
+      nil
+    end
+
     def append(message)
       body = message.imap_body
       date = message.date&.to_time
@@ -96,7 +116,7 @@ module Imap::Backup
     def examine
       client.examine(utf7_encoded_name)
     rescue Net::IMAP::NoResponseError
-      Imap::Backup.logger.warn "Folder '#{name}' does not exist on server"
+      Imap::Backup::Logger.logger.warn "Folder '#{name}' does not exist on server"
       raise FolderNotFound, "Folder '#{name}' does not exist on server"
     end
 

data/lib/imap/backup/account.rb

@@ -20,6 +20,10 @@ module Imap::Backup
       @marked_for_deletion = false
     end
 
+    def connection
+      Account::Connection.new(self)
+    end
+
     def valid?
       username && password
     end

data/lib/imap/backup/cli/accounts.rb

@@ -0,0 +1,43 @@
+module Imap::Backup
+  class CLI; end
+
+  class CLI::Accounts
+    include Enumerable
+
+    attr_reader :required_accounts
+
+    def initialize(required_accounts = [])
+      @required_accounts = required_accounts
+    end
+
+    def each(&block)
+      return enum_for(:each) if !block
+
+      accounts.each(&block)
+    end
+
+    private
+
+    def accounts
+      @accounts ||=
+        if required_accounts.empty?
+          config.accounts
+        else
+          config.accounts.select do |account|
+            required_accounts.include?(account.username)
+          end
+        end
+    end
+
+    def config
+      @config ||= begin
+        exists = Configuration.exist?
+        if !exists
+          path = Configuration.default_pathname
+          raise ConfigurationNotFound, "Configuration file '#{path}' not found"
+        end
+        Configuration.new
+      end
+    end
+  end
+end

data/lib/imap/backup/cli/folders.rb

@@ -14,7 +14,9 @@ module Imap::Backup
       def run
         each_connection(account_names) do |connection|
           puts connection.username
-          folders = connection.folders
+          # TODO: Make folder_names private once this command
+          # has been removed.
+          folders = connection.folder_names
           if folders.nil?
             warn "Unable to list account folders"
             return false

data/lib/imap/backup/cli/helpers.rb

@@ -1,4 +1,5 @@
 require "imap/backup"
+require "imap/backup/cli/accounts"
 
 module Imap::Backup::CLI::Helpers
   def symbolized(options)
@@ -6,8 +7,8 @@ module Imap::Backup::CLI::Helpers
   end
 
   def account(email)
-    connections = Imap::Backup::Configuration::List.new
-    account = connections.accounts.find { |a| a.username == email }
+    accounts = Imap::Backup::CLI::Accounts.new
+    account = accounts.find { |a| a.username == email }
     raise "#{email} is not a configured account" if !account
 
     account
@@ -20,14 +21,12 @@ module Imap::Backup::CLI::Helpers
   end
 
   def each_connection(names)
-    begin
-      connections = Imap::Backup::Configuration::List.new(names)
-    rescue Imap::Backup::ConfigurationNotFound
-      raise "imap-backup is not configured. Run `imap-backup setup`"
-    end
+    accounts = Imap::Backup::CLI::Accounts.new(names)
 
-    connections.each_connection do |connection|
-      yield connection
+    accounts.each do |account|
+      yield account.connection
     end
+  rescue Imap::Backup::ConfigurationNotFound
+    raise "imap-backup is not configured. Run `imap-backup setup`"
   end
 end

data/lib/imap/backup/cli/local.rb

@@ -1,3 +1,5 @@
+require "imap/backup/cli/accounts"
+
 module Imap::Backup
   class CLI::Local < Thor
     include Thor::Actions
@@ -5,8 +7,8 @@ module Imap::Backup
 
     desc "accounts", "List locally backed-up accounts"
     def accounts
-      connections = Imap::Backup::Configuration::List.new
-      connections.accounts.each { |a| Kernel.puts a.username }
+      accounts = CLI::Accounts.new
+      accounts.each { |a| Kernel.puts a.username }
     end
 
     desc "folders EMAIL", "List account folders"

data/lib/imap/backup/cli/setup.rb

@@ -7,7 +7,7 @@ class Imap::Backup::CLI::Setup < Thor
 
   no_commands do
     def run
-      Imap::Backup::Configuration::Setup.new.run
+      Imap::Backup::Setup.new.run
     end
   end
 end

data/lib/imap/backup/cli/status.rb

@@ -13,7 +13,7 @@ module Imap::Backup
     no_commands do
       def run
         each_connection(account_names) do |connection|
-          puts connection.username
+          puts connection.account.username
           folders = connection.status
           folders.each do |f|
             missing_locally = f[:remote] - f[:local]

data/lib/imap/backup/cli/utils.rb

@@ -11,9 +11,10 @@ module Imap::Backup
     def ignore_history(email)
       connection = connection(email)
 
-      connection.local_folders.each do |serializer, folder|
+      connection.backup_folders.each do |folder|
         next if !folder.exist?
 
+        serializer = Serializer::Mbox.new(connection.account.local_path, folder.name)
         do_ignore_folder_history(folder, serializer)
       end
     end
@@ -63,7 +64,7 @@ module Imap::Backup
     no_commands do
       def do_ignore_folder_history(folder, serializer)
         uids = folder.uids - serializer.uids
-        Imap::Backup.logger.info "Folder '#{folder.name}' - #{uids.length} messages"
+        Imap::Backup::Logger.logger.info "Folder '#{folder.name}' - #{uids.length} messages"
 
         serializer.apply_uid_validity(folder.uid_validity)
 

data/lib/imap/backup/{configuration/store.rb → configuration.rb}

@@ -4,10 +4,9 @@ require "os"
 require "imap/backup/account"
 
 module Imap::Backup
-  module Configuration; end
-
-  class Configuration::Store
+  class Configuration
     CONFIGURATION_DIRECTORY = File.expand_path("~/.imap-backup")
+    DEFAULT_DOWNLOAD_BLOCK_SIZE = 1
     VERSION = "2.0"
 
     attr_reader :pathname
@@ -22,6 +21,7 @@ module Imap::Backup
 
     def initialize(pathname = self.class.default_pathname)
       @pathname = pathname
+      @saved_debug = nil
       @debug = nil
     end
 
@@ -42,6 +42,7 @@ module Imap::Backup
       }
       File.open(pathname, "w") { |f| f.write(JSON.pretty_generate(save_data)) }
       FileUtils.chmod(0o600, pathname) if !windows?
+      @data = nil
     end
 
     def accounts
@@ -51,8 +52,19 @@ module Imap::Backup
       end
     end
 
+    def download_block_size
+      size = ENV["DOWNLOAD_BLOCK_SIZE"].to_i
+      if size > 0
+        size
+      else
+        DEFAULT_DOWNLOAD_BLOCK_SIZE
+      end
+    end
+
     def modified?
       ensure_loaded!
+      return true if @saved_debug != @debug
+
       accounts.any? { |a| a.modified? || a.marked_for_deletion? }
     end
 
@@ -73,6 +85,7 @@ module Imap::Backup
 
       data
       @debug = data.key?(:debug) ? data[:debug] == true : false
+      @saved_debug = @debug
       true
     end
 

data/lib/imap/backup/downloader.rb

@@ -2,27 +2,41 @@ module Imap::Backup
   class Downloader
     attr_reader :folder
     attr_reader :serializer
+    attr_reader :block_size
 
-    def initialize(folder, serializer)
+    def initialize(folder, serializer, block_size: 1)
       @folder = folder
       @serializer = serializer
+      @block_size = block_size
     end
 
     def run
       uids = folder.uids - serializer.uids
       count = uids.count
-      Imap::Backup.logger.debug "[#{folder.name}] #{count} new messages"
-      uids.each.with_index do |uid, i|
-        body = folder.fetch(uid)
-        log_prefix = "[#{folder.name}] uid: #{uid} (#{i + 1}/#{count}) -"
-        if body.nil?
-          Imap::Backup.logger.debug("#{log_prefix} not available - skipped")
-          next
+      Imap::Backup::Logger.logger.debug "[#{folder.name}] #{count} new messages"
+      uids.each_slice(block_size).with_index do |block, i|
+        offset = i * block_size + 1
+        uids_and_bodies = folder.fetch_multi(block)
+        if uids_and_bodies.nil?
+          if block_size > 1
+            Imap::Backup::Logger.logger.debug("[#{folder.name}] Multi fetch failed for UIDs #{block.join(", ")}, switching to single fetches")
+            @block_size = 1
+            redo
+          else
+            Imap::Backup::Logger.logger.debug("[#{folder.name}] Fetch failed for UID #{block[0]} - skipping")
+            next
+          end
+        end
+
+        uids_and_bodies.each.with_index do |uid_and_body, j|
+          uid = uid_and_body[:uid]
+          body = uid_and_body[:body]
+          Imap::Backup::Logger.logger.debug(
+            "[#{folder.name}] uid: #{uid} (#{offset +j}/#{count}) - " \
+              "#{body.size} bytes"
+          )
+          serializer.save(uid, body)
         end
-        Imap::Backup.logger.debug(
-          "#{log_prefix} #{body.size} bytes"
-        )
-        serializer.save(uid, body)
       end
     end
   end

data/lib/imap/backup/logger.rb

@@ -0,0 +1,42 @@
+require "logger"
+require "singleton"
+
+require "imap/backup/configuration"
+require "imap/backup/sanitizer"
+
+module Imap::Backup
+  class Logger
+    include Singleton
+
+    def self.logger
+      Logger.instance.logger
+    end
+
+    def self.setup_logging(config = Configuration.new)
+      logger.level =
+        if config.debug?
+          ::Logger::Severity::DEBUG
+        else
+          ::Logger::Severity::ERROR
+        end
+      Net::IMAP.debug = config.debug?
+    end
+
+    def self.sanitize_stderr
+      sanitizer = Sanitizer.new($stdout)
+      previous_stderr = $stderr
+      $stderr = sanitizer
+      yield
+    ensure
+      sanitizer.flush
+      $stderr = previous_stderr
+    end
+
+    attr_reader :logger
+
+    def initialize
+      @logger = ::Logger.new($stdout)
+      $stdout.sync = true
+    end
+  end
+end

data/lib/imap/backup/sanitizer.rb

@@ -0,0 +1,42 @@
+module Imap::Backup
+  class Sanitizer
+    attr_reader :output
+
+    def initialize(output)
+      @output = output
+      @current = ""
+    end
+
+    def write(*args)
+      output.write(*args)
+    end
+
+    def print(*args)
+      @current << args.join
+      loop do
+        line, newline, rest = @current.partition("\n")
+        break if newline != "\n"
+        clean = sanitize(line)
+        output.puts clean
+        @current = rest
+      end
+    end
+
+    def flush
+      return if @current == ""
+
+      clean = sanitize(@current)
+      output.puts clean
+    end
+
+    private
+
+    def sanitize(t)
+      # Hide password in Net::IMAP debug output
+      t.gsub(
+        /\A(C: RUBY\d+ LOGIN \S+) \S+/,
+        "\\1 [PASSWORD REDACTED]"
+      )
+    end
+  end
+end

data/lib/imap/backup/serializer/mbox_store.rb

@@ -46,7 +46,7 @@ module Imap::Backup
 
       uid = uid.to_i
       if uids.include?(uid)
-        Imap::Backup.logger.debug(
+        Imap::Backup::Logger.logger.debug(
           "[#{folder}] message #{uid} already downloaded - skipping"
         )
         return
@@ -65,7 +65,7 @@ module Imap::Backup
           #{body}. #{e}:
           #{e.backtrace.join("\n")}"
         ERROR
-        Imap::Backup.logger.warn message
+        Imap::Backup::Logger.logger.warn message
       ensure
         mbox&.close
       end