imap-backup 4.0.4 → 4.1.1

data/lib/imap/backup/{configuration/store.rb → configuration.rb}

@@ -1,11 +1,13 @@
 require "json"
 require "os"
 
-module Imap::Backup
-  module Configuration; end
+require "imap/backup/account"
 
-  class Configuration::Store
+module Imap::Backup
+  class Configuration
     CONFIGURATION_DIRECTORY = File.expand_path("~/.imap-backup")
+    DEFAULT_DOWNLOAD_BLOCK_SIZE = 1
+    VERSION = "2.0"
 
     attr_reader :pathname
 
@@ -19,6 +21,8 @@ module Imap::Backup
 
     def initialize(pathname = self.class.default_pathname)
       @pathname = pathname
+      @saved_debug = nil
+      @debug = nil
     end
 
     def path
@@ -26,53 +30,84 @@ module Imap::Backup
     end
 
     def save
+      ensure_loaded!
       FileUtils.mkdir(path) if !File.directory?(path)
       make_private(path) if !windows?
       remove_modified_flags
       remove_deleted_accounts
-      File.open(pathname, "w") { |f| f.write(JSON.pretty_generate(data)) }
+      save_data = {
+        version: VERSION,
+        accounts: accounts.map(&:to_h),
+        debug: debug?
+      }
+      File.open(pathname, "w") { |f| f.write(JSON.pretty_generate(save_data)) }
       FileUtils.chmod(0o600, pathname) if !windows?
+      @data = nil
     end
 
     def accounts
-      data[:accounts]
+      @accounts ||= begin
+        ensure_loaded!
+        data[:accounts].map { |data| Account.new(data) }
+      end
+    end
+
+    def download_block_size
+      size = ENV["DOWNLOAD_BLOCK_SIZE"].to_i
+      if size > 0
+        size
+      else
+        DEFAULT_DOWNLOAD_BLOCK_SIZE
+      end
     end
 
     def modified?
-      accounts.any? { |a| a[:modified] || a[:delete] }
+      ensure_loaded!
+      return true if @saved_debug != @debug
+
+      accounts.any? { |a| a.modified? || a.marked_for_deletion? }
     end
 
     def debug?
-      data[:debug]
+      ensure_loaded!
+      @debug
     end
 
     def debug=(value)
-      data[:debug] = [true, false].include?(value) ? value : false
+      ensure_loaded!
+      @debug = [true, false].include?(value) ? value : false
     end
 
     private
 
+    def ensure_loaded!
+      return true if @data
+
+      data
+      @debug = data.key?(:debug) ? data[:debug] == true : false
+      @saved_debug = @debug
+      true
+    end
+
     def data
       @data ||=
         begin
           if File.exist?(pathname)
             Utils.check_permissions(pathname, 0o600) if !windows?
             contents = File.read(pathname)
-            data = JSON.parse(contents, symbolize_names: true)
+            JSON.parse(contents, symbolize_names: true)
           else
-            data = {accounts: []}
+            {accounts: []}
           end
-          data[:debug] = data.key?(:debug) ? data[:debug] == true : false
-          data
         end
     end
 
     def remove_modified_flags
-      accounts.each { |a| a.delete(:modified) }
+      accounts.each { |a| a.clear_changes! }
     end
 
     def remove_deleted_accounts
-      accounts.reject! { |a| a[:delete] }
+      accounts.reject! { |a| a.marked_for_deletion? }
     end
 
     def make_private(path)

data/lib/imap/backup/downloader.rb

@@ -2,27 +2,41 @@ module Imap::Backup
   class Downloader
     attr_reader :folder
     attr_reader :serializer
+    attr_reader :block_size
 
-    def initialize(folder, serializer)
+    def initialize(folder, serializer, block_size: 1)
       @folder = folder
       @serializer = serializer
+      @block_size = block_size
     end
 
     def run
       uids = folder.uids - serializer.uids
       count = uids.count
-      Imap::Backup.logger.debug "[#{folder.name}] #{count} new messages"
-      uids.each.with_index do |uid, i|
-        body = folder.fetch(uid)
-        log_prefix = "[#{folder.name}] uid: #{uid} (#{i + 1}/#{count}) -"
-        if body.nil?
-          Imap::Backup.logger.debug("#{log_prefix} not available - skipped")
-          next
+      Imap::Backup::Logger.logger.debug "[#{folder.name}] #{count} new messages"
+      uids.each_slice(block_size).with_index do |block, i|
+        offset = i * block_size + 1
+        uids_and_bodies = folder.fetch_multi(block)
+        if uids_and_bodies.nil?
+          if block_size > 1
+            Imap::Backup::Logger.logger.debug("[#{folder.name}] Multi fetch failed for UIDs #{block.join(", ")}, switching to single fetches")
+            @block_size = 1
+            redo
+          else
+            Imap::Backup::Logger.logger.debug("[#{folder.name}] Fetch failed for UID #{block[0]} - skipping")
+            next
+          end
+        end
+
+        uids_and_bodies.each.with_index do |uid_and_body, j|
+          uid = uid_and_body[:uid]
+          body = uid_and_body[:body]
+          Imap::Backup::Logger.logger.debug(
+            "[#{folder.name}] uid: #{uid} (#{offset +j}/#{count}) - " \
+              "#{body.size} bytes"
+          )
+          serializer.save(uid, body)
         end
-        Imap::Backup.logger.debug(
-          "#{log_prefix} #{body.size} bytes"
-        )
-        serializer.save(uid, body)
       end
     end
   end

data/lib/imap/backup/logger.rb

@@ -0,0 +1,42 @@
+require "logger"
+require "singleton"
+
+require "imap/backup/configuration"
+require "imap/backup/sanitizer"
+
+module Imap::Backup
+  class Logger
+    include Singleton
+
+    def self.logger
+      Logger.instance.logger
+    end
+
+    def self.setup_logging(config = Configuration.new)
+      logger.level =
+        if config.debug?
+          ::Logger::Severity::DEBUG
+        else
+          ::Logger::Severity::ERROR
+        end
+      Net::IMAP.debug = config.debug?
+    end
+
+    def self.sanitize_stderr
+      sanitizer = Sanitizer.new($stdout)
+      previous_stderr = $stderr
+      $stderr = sanitizer
+      yield
+    ensure
+      sanitizer.flush
+      $stderr = previous_stderr
+    end
+
+    attr_reader :logger
+
+    def initialize
+      @logger = ::Logger.new($stdout)
+      $stdout.sync = true
+    end
+  end
+end

data/lib/imap/backup/sanitizer.rb

@@ -0,0 +1,42 @@
+module Imap::Backup
+  class Sanitizer
+    attr_reader :output
+
+    def initialize(output)
+      @output = output
+      @current = ""
+    end
+
+    def write(*args)
+      output.write(*args)
+    end
+
+    def print(*args)
+      @current << args.join
+      loop do
+        line, newline, rest = @current.partition("\n")
+        break if newline != "\n"
+        clean = sanitize(line)
+        output.puts clean
+        @current = rest
+      end
+    end
+
+    def flush
+      return if @current == ""
+
+      clean = sanitize(@current)
+      output.puts clean
+    end
+
+    private
+
+    def sanitize(t)
+      # Hide password in Net::IMAP debug output
+      t.gsub(
+        /\A(C: RUBY\d+ LOGIN \S+) \S+/,
+        "\\1 [PASSWORD REDACTED]"
+      )
+    end
+  end
+end

data/lib/imap/backup/serializer/mbox_store.rb

@@ -46,7 +46,7 @@ module Imap::Backup
 
       uid = uid.to_i
       if uids.include?(uid)
-        Imap::Backup.logger.debug(
+        Imap::Backup::Logger.logger.debug(
           "[#{folder}] message #{uid} already downloaded - skipping"
         )
         return
@@ -65,7 +65,7 @@ module Imap::Backup
           #{body}. #{e}:
           #{e.backtrace.join("\n")}"
         ERROR
-        Imap::Backup.logger.warn message
+        Imap::Backup::Logger.logger.warn message
       ensure
         mbox&.close
       end

data/lib/imap/backup/setup/account.rb

@@ -0,0 +1,177 @@
+require "imap/backup/setup/helpers"
+
+module Imap::Backup
+  class Setup; end
+
+  Setup::Account = Struct.new(:config, :account, :highline) do
+    def initialize(config, account, highline)
+      super
+    end
+
+    def run
+      catch :done do
+        loop do
+          Kernel.system("clear")
+          create_menu
+        end
+      end
+    end
+
+    private
+
+    def create_menu
+      highline.choose do |menu|
+        header menu
+        modify_email menu
+        modify_password menu
+        modify_backup_path menu
+        choose_folders menu
+        modify_server menu
+        modify_connection_options menu
+        test_connection menu
+        delete_account menu
+        menu.choice("(q) return to main menu") { throw :done }
+        menu.hidden("quit") { throw :done }
+      end
+    end
+
+    def header(menu)
+      modified = account.modified? ? "*" : ""
+      connection_options =
+        if account.connection_options
+          escaped =
+            JSON.generate(account.connection_options).
+            gsub('"', '\"')
+          "\n  connection options #{escaped}"
+        end
+      menu.header = <<~HEADER.chomp
+        #{helpers.title_prefix} Account#{modified}
+
+        email      #{account.username}
+        password   #{masked_password}
+        path       #{account.local_path}
+        folders    #{folders.map { |f| f[:name] }.join(', ')}
+        server     #{account.server}#{connection_options}
+
+        Choose an action
+      HEADER
+    end
+
+    def modify_email(menu)
+      menu.choice("modify email") do
+        username = Setup::Asker.email(username)
+        Kernel.puts "username: #{username}"
+        other_accounts = config.accounts.reject { |a| a == account }
+        others = other_accounts.map { |a| a.username }
+        Kernel.puts "others: #{others.inspect}"
+        if others.include?(username)
+          Kernel.puts(
+            "There is already an account set up with that email address"
+          )
+        else
+          account.username = username
+          # rubocop:disable Style/IfUnlessModifier
+          default = default_server(username)
+          if default && (account.server.nil? || (account.server == ""))
+            account.server = default
+          end
+          # rubocop:enable Style/IfUnlessModifier
+        end
+      end
+    end
+
+    def modify_password(menu)
+      menu.choice("modify password") do
+        password = Setup::Asker.password
+
+        account.password = password if !password.nil?
+      end
+    end
+
+    def modify_server(menu)
+      menu.choice("modify server") do
+        server = highline.ask("server: ")
+        account.server = server if !server.nil?
+      end
+    end
+
+    def modify_connection_options(menu)
+      menu.choice("modify connection options") do
+        connection_options = highline.ask("connections options (as JSON): ")
+        account.connection_options = connection_options if !connection_options.nil?
+      end
+    end
+
+    def path_modification_validator(path)
+      same = config.accounts.find do |a|
+        a.username != account.username && a.local_path == path
+      end
+      if same
+        Kernel.puts "The path '#{path}' is used to backup " \
+          "the account '#{same.username}'"
+        false
+      else
+        true
+      end
+    end
+
+    def modify_backup_path(menu)
+      menu.choice("modify backup path") do
+        existing = account.local_path.clone
+        account.local_path = Setup::Asker.backup_path(
+          account.local_path, ->(path) { path_modification_validator(path) }
+        )
+      end
+    end
+
+    def choose_folders(menu)
+      menu.choice("choose backup folders") do
+        Setup::FolderChooser.new(account).run
+      end
+    end
+
+    def test_connection(menu)
+      menu.choice("test connection") do
+        result = Setup::ConnectionTester.new(account).test
+        Kernel.puts result
+        highline.ask "Press a key "
+      end
+    end
+
+    def delete_account(menu)
+      menu.choice("delete") do
+        if highline.agree("Are you sure? (y/n) ")
+          account.mark_for_deletion!
+          throw :done
+        end
+      end
+    end
+
+    def folders
+      account.folders || []
+    end
+
+    def masked_password
+      if (account.password == "") || account.password.nil?
+        "(unset)"
+      else
+        account.password.gsub(/./, "x")
+      end
+    end
+
+    def default_server(username)
+      provider = Email::Provider.for_address(username)
+
+      if provider.is_a?(Email::Provider::Unknown)
+        Kernel.puts "Can't decide provider for email address '#{username}'"
+        return nil
+      end
+
+      provider.host
+    end
+
+    def helpers
+      Setup::Helpers.new
+    end
+  end
+end

data/lib/imap/backup/{configuration → setup}/asker.rb

@@ -1,7 +1,7 @@
 module Imap::Backup
-  module Configuration; end
+  class Setup; end
 
-  Configuration::Asker = Struct.new(:highline) do
+  Setup::Asker = Struct.new(:highline) do
     EMAIL_MATCHER = /^[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]+$/i.freeze
 
     def initialize(highline)
@@ -40,15 +40,15 @@ module Imap::Backup
     end
 
     def self.email(default = "")
-      new(Configuration::Setup.highline).email(default)
+      new(Setup.highline).email(default)
     end
 
     def self.password
-      new(Configuration::Setup.highline).password
+      new(Setup.highline).password
     end
 
     def self.backup_path(default, validator)
-      new(Configuration::Setup.highline).backup_path(default, validator)
+      new(Setup.highline).backup_path(default, validator)
     end
   end
 end

data/lib/imap/backup/setup/connection_tester.rb

@@ -0,0 +1,26 @@
+module Imap::Backup
+  class Setup; end
+
+  class Setup::ConnectionTester
+    attr_reader :account
+
+    def initialize(account)
+      @account = account
+    end
+
+    def test
+      connection.client
+      "Connection successful"
+    rescue Net::IMAP::NoResponseError
+      "No response"
+    rescue StandardError => e
+      "Unexpected error: #{e}"
+    end
+
+    private
+
+    def connection
+      Account::Connection.new(account)
+    end
+  end
+end

data/lib/imap/backup/{configuration → setup}/folder_chooser.rb

@@ -1,7 +1,9 @@
+require "imap/backup/setup/helpers"
+
 module Imap::Backup
-  module Configuration; end
+  class Setup; end
 
-  class Configuration::FolderChooser
+  class Setup::FolderChooser
     attr_reader :account
 
     def initialize(account)
@@ -10,13 +12,13 @@ module Imap::Backup
 
     def run
       if connection.nil?
-        Imap::Backup.logger.warn "Connection failed"
+        Imap::Backup::Logger.logger.warn "Connection failed"
         highline.ask "Press a key "
         return
       end
 
       if imap_folders.nil?
-        Imap::Backup.logger.warn "Unable to get folder list"
+        Imap::Backup::Logger.logger.warn "Unable to get folder list"
         highline.ask "Press a key "
         return
       end
@@ -35,10 +37,14 @@ module Imap::Backup
 
     def show_menu
       highline.choose do |menu|
-        menu.header = "Add/remove folders"
+        menu.header = <<~MENU.chomp
+          #{helpers.title_prefix} Add/remove folders
+
+          Select a folder (toggles)
+        MENU
         menu.index = :number
         add_folders menu
-        menu.choice("return to the account menu") { throw :done }
+        menu.choice("(q) return to the account menu") { throw :done }
         menu.hidden("quit") { throw :done }
       end
     end
@@ -53,7 +59,7 @@ module Imap::Backup
     end
 
     def selected?(folder_name)
-      config_folders = account[:folders]
+      config_folders = account.folders
       return false if config_folders.nil?
 
       config_folders.find { |f| f[:name] == folder_name }
@@ -62,7 +68,7 @@ module Imap::Backup
     def remove_missing
       removed = []
       config_folders = []
-      account[:folders].each do |f|
+      account.folders.each do |f|
         found = imap_folders.find { |folder| folder == f[:name] }
         if found
           config_folders << f
@@ -73,8 +79,7 @@ module Imap::Backup
 
       return if removed.empty?
 
-      account[:folders] = config_folders
-      account[:modified] = true
+      account.folders = config_folders
 
       Kernel.puts <<~MESSAGE
         The following folders have been removed: #{removed.join(', ')}
@@ -85,12 +90,11 @@ module Imap::Backup
 
     def toggle_selection(folder_name)
       if selected?(folder_name)
-        changed = account[:folders].reject! { |f| f[:name] == folder_name }
-        account[:modified] = true if changed
+        new_list = account.folders.select { |f| f[:name] != folder_name }
+        account.folders = new_list
       else
-        account[:folders] ||= []
-        account[:folders] << {name: folder_name}
-        account[:modified] = true
+        existing = account.folders || []
+        account.folders = existing + [{name: folder_name}]
       end
     end
 
@@ -101,11 +105,15 @@ module Imap::Backup
     end
 
     def imap_folders
-      @imap_folders ||= connection.folders
+      @imap_folders ||= connection.folder_names
     end
 
     def highline
-      Configuration::Setup.highline
+      Setup.highline
+    end
+
+    def helpers
+      Setup::Helpers.new
     end
   end
 end

data/lib/imap/backup/setup/helpers.rb

@@ -0,0 +1,15 @@
+require "imap/backup/version"
+
+module Imap::Backup
+  class Setup; end
+
+  class Setup::Helpers
+    def title_prefix
+      "imap-backup –"
+    end
+
+    def version
+      Version::VERSION
+    end
+  end
+end