imap-backup 4.2.0 → 5.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e0cd29188ed972abfb2b2ce21d237ce3b7aed5b434717ba64fa2e3475da38ded
-  data.tar.gz: 597b58cdee1ab1b5881ea29111ac85e7fdedfbaa370824d0d24ba1b19c869dcb
+  metadata.gz: 59babb556b2f47a07590debff15dcd5456f1e61454c3aba2e504610ce4322647
+  data.tar.gz: 89909077444746e44fe3955070df6b590942da0d44da58c9b08b282e694000d5
 SHA512:
-  metadata.gz: a3adc1984eea76a5ef993a40e89520a01997e6d01357e0a1e41cd4b58a4efc56801fc3d4f8801b95e828ec3a6a1e7884be974aaabf40f921a3faccdc99cf9b98
-  data.tar.gz: 55c0c91fdfb4829ba77be5292b4e475c6ea31e9e6e9ea5abf9702121f8a12c20b9b0105da5e1b97bcb2482105807c614530dea9e058d64af2cde96f7fd19348b
+  metadata.gz: ea00bb55bb853cef73ec12171a4bdde9b0b5ef88a914065bbe307f48712db3329e8c8f1712cfa48b4a49102d20fb38cc465dc463c2d3ea9e0caba000a184766b
+  data.tar.gz: 24d368c209c11db4ae7f05a93b9cf4e0435b65eef0cffd7e58f8c669e634ca735aa3187b6a6506b7b10ba3bec961d3aa9d7387df40d4880f1bed73df7f24bbb9

data/imap-backup.gemspec

@@ -27,7 +27,9 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "os"
   gem.add_runtime_dependency "rake"
   gem.add_runtime_dependency "thor", "~> 1.1"
+  gem.add_runtime_dependency "thunderbird", ">= 0.0.0"
 
+  gem.add_development_dependency "aruba", ">= 0.0.0"
   gem.add_development_dependency "codeclimate-test-reporter", "~> 0.4.8"
   if RUBY_ENGINE == "jruby"
     gem.add_development_dependency "pry-debugger-jruby"

data/lib/imap/backup/account/connection.rb

@@ -16,11 +16,8 @@ module Imap::Backup
     def initialize(account)
       @account = account
       reset
-      create_account_folder
     end
 
-    # TODO: Make this private once the 'folders' command
-    # has been removed.
     def folder_names
       @folder_names ||=
         begin
@@ -53,6 +50,7 @@ module Imap::Backup
     end
 
     def status
+      ensure_account_folder
       backup_folders.map do |folder|
         s = Serializer::Mbox.new(account.local_path, folder.name)
         {name: folder.name, local: s.uids, remote: folder.uids}
@@ -63,6 +61,7 @@ module Imap::Backup
       Imap::Backup::Logger.logger.debug "Running backup of account: #{account.username}"
       # start the connection so we get logging messages in the right order
       client
+      ensure_account_folder
       each_folder do |folder, serializer|
         next if !folder.exist?
 
@@ -82,6 +81,7 @@ module Imap::Backup
     def local_folders
       return enum_for(:local_folders) if !block_given?
 
+      ensure_account_folder
       glob = File.join(account.local_path, "**", "*.imap")
       base = Pathname.new(account.local_path)
       Pathname.glob(glob) do |path|
@@ -176,7 +176,7 @@ module Imap::Backup
       end
     end
 
-    def create_account_folder
+    def ensure_account_folder
       Utils.make_folder(
         File.dirname(account.local_path),
         File.basename(account.local_path),

data/lib/imap/backup/account/folder.rb

@@ -111,6 +111,14 @@ module Imap::Backup
       extract_uid(response)
     end
 
+    def clear
+      existing = uids
+      # Use read-write access, via `select`
+      client.select(utf7_encoded_name)
+      client.uid_store(existing, "+FLAGS", [:Deleted])
+      client.expunge
+    end
+
     private
 
     def examine

data/lib/imap/backup/cli/folders.rb

@@ -13,7 +13,7 @@ module Imap::Backup
     no_commands do
       def run
         each_connection(account_names) do |connection|
-          puts connection.username
+          puts connection.account.username
           # TODO: Make folder_names private once this command
           # has been removed.
           folders = connection.folder_names

data/lib/imap/backup/cli/helpers.rb

@@ -3,7 +3,10 @@ require "imap/backup/cli/accounts"
 
 module Imap::Backup::CLI::Helpers
   def symbolized(options)
-    options.each.with_object({}) { |(k, v), acc| acc[k.intern] = v }
+    options.each.with_object({}) do |(k, v), acc|
+      key = k.gsub("-", "_").intern
+      acc[key] = v
+    end
   end
 
   def account(email)

data/lib/imap/backup/cli/local.rb

@@ -11,7 +11,7 @@ module Imap::Backup
       accounts.each { |a| Kernel.puts a.username }
     end
 
-    desc "folders EMAIL", "List account folders"
+    desc "folders EMAIL", "List backed up folders"
     def folders(email)
       connection = connection(email)
 

data/lib/imap/backup/cli/migrate.rb

@@ -0,0 +1,114 @@
+require "imap/backup/migrator"
+
+module Imap::Backup
+  class CLI::Migrate < Thor
+    include Thor::Actions
+
+    attr_reader :destination_email
+    attr_reader :destination_prefix
+    attr_reader :reset
+    attr_reader :source_email
+    attr_reader :source_prefix
+
+    def initialize(
+      source_email,
+      destination_email,
+      destination_prefix: "",
+      reset: false,
+      source_prefix: ""
+    )
+      super([])
+      @destination_email = destination_email
+      @destination_prefix = destination_prefix
+      @reset = reset
+      @source_email = source_email
+      @source_prefix = source_prefix
+    end
+
+    no_commands do
+      def run
+        check_accounts!
+        folders.each do |serializer, folder|
+          Migrator.new(serializer, folder, reset: reset).run
+        end
+      end
+
+      def check_accounts!
+        if destination_email == source_email
+          raise "Source and destination accounts cannot be the same!"
+        end
+
+        if !destination_account
+          raise "Account #{destination_email} does not exist"
+        end
+
+        if !source_account
+          raise "Account #{source_email} does not exist"
+        end
+      end
+
+      def config
+        Configuration.new
+      end
+
+      def destination_account
+        config.accounts.find { |a| a.username == destination_email }
+      end
+
+      def folders
+        return enum_for(:folders) if !block_given?
+
+        glob = File.join(source_local_path, "**", "*.imap")
+        Pathname.glob(glob) do |path|
+          name = source_folder_name(path)
+          serializer = Serializer::Mbox.new(source_local_path, name)
+          folder = folder_for(name)
+          yield serializer, folder
+        end
+      end
+
+      def folder_for(source_folder)
+        no_source_prefix =
+          if source_prefix != "" &&
+              source_folder.start_with?(source_prefix)
+            source_folder.delete_prefix(source_prefix)
+          else
+            source_folder.to_s
+          end
+
+        with_destination_prefix =
+          if destination_prefix &&
+              destination_prefix != ""
+            destination_prefix + no_source_prefix
+          else
+            no_source_prefix
+          end
+
+        Account::Folder.new(
+          destination_account.connection,
+          with_destination_prefix
+        )
+      end
+
+      def source_local_path
+        source_account.local_path
+      end
+
+      def source_account
+        config.accounts.find { |a| a.username == source_email }
+      end
+
+      def source_folder_name(imap_pathname)
+        base = Pathname.new(source_local_path)
+        imap_name = imap_pathname.relative_path_from(base).to_s
+        dir = File.dirname(imap_name)
+        stripped = File.basename(imap_name, ".imap")
+        if dir == "."
+          stripped
+        else
+          File.join(dir, stripped)
+        end
+      end
+    end
+  end
+end

data/lib/imap/backup/cli/remote.rb

@@ -0,0 +1,15 @@
+module Imap::Backup
+  class CLI::Remote < Thor
+    include Thor::Actions
+    include CLI::Helpers
+
+    desc "folders EMAIL", "List account folders"
+    def folders(email)
+      connection = connection(email)
+
+      connection.folder_names.each do |name|
+        Kernel.puts %("#{name}")
+      end
+    end
+  end
+end

data/lib/imap/backup/cli/restore.rb

@@ -3,16 +3,32 @@ module Imap::Backup
     include Thor::Actions
     include CLI::Helpers
 
+    attr_reader :email
     attr_reader :account_names
 
-    def initialize(options)
-      super([])
-      @account_names = (options[:accounts] || "").split(",")
+    def initialize(email = nil, options)
+      @email = email
+      @account_names =
+        if options.key?(:accounts)
+          options[:accounts].split(",")
+        end
     end
 
     no_commands do
       def run
-        each_connection(account_names) { |connection| connection.restore }
+        case
+        when email && !account_names
+          connection = connection(email)
+          connection.restore
+        when !email && !account_names
+          Logger.logger.info "Calling restore without an EMAIL parameter is deprecated"
+          each_connection([]) { |connection| connection.restore }
+        when email && account_names.any?
+          raise "Pass either an email or the --accounts option, not both"
+        when account_names.any?
+          Logger.logger.info "Calling restore with the --account option is deprected, please pass a single EMAIL argument"
+          each_connection(account_names) { |connection| connection.restore }
+        end
       end
     end
   end

data/lib/imap/backup/cli.rb

@@ -9,6 +9,7 @@ module Imap::Backup
     autoload :Backup, "imap/backup/cli/backup"
     autoload :Folders, "imap/backup/cli/folders"
     autoload :Local, "imap/backup/cli/local"
+    autoload :Migrate, "imap/backup/cli/migrate"
     autoload :Remote, "imap/backup/cli/remote"
     autoload :Restore, "imap/backup/cli/restore"
     autoload :Setup, "imap/backup/cli/setup"
@@ -27,7 +28,7 @@ module Imap::Backup
       method_option(
         "accounts",
         type: :string,
-        banner: "a comma-separated list of accounts (defaults to all configured accounts)",
+        desc: "a comma-separated list of accounts (defaults to all configured accounts)",
         aliases: ["-a"]
       )
     end
@@ -57,15 +58,62 @@ module Imap::Backup
       Folders.new(symbolized(options)).run
     end
 
-    desc "restore [OPTIONS]", "This command is deprecated, use `imap-backup restore ACCOUNT`"
+    desc "local SUBCOMMAND [OPTIONS]", "View local info"
+    subcommand "local", Local
+
+    desc "migrate SOURCE_EMAIL DESTINATION_EMAIL [OPTIONS]",
+      "[Experimental] Uploads backed-up emails from account SOURCE_EMAIL to account DESTINATION_EMAIL"
     long_desc <<~DESC
-      By default, restores all local emails to their respective servers.
-      This command is deprecated.
-      Instead, use `imap-backup restore ACCOUNT` to restore a single account.
+      All emails which have been backed up for the "source account" (SOURCE_EMAIL) are
+      uploaded to the "destination account" (DESTINATION_EMAIL).
+
+      When one or other account has namespaces (i.e. prefixes like "INBOX."),
+      use the `--source-prefix=` and/or `--destination-prefix=` options.
+
+      Usually, you should migrate to an account with empty folders.
+
+      Before migrating each folder, `imap-backup` checks if the destination
+      folder is empty.
+
+      If it finds a non-empty destination folder, it halts with an error.
+
+      If you are sure that these destination emails can be deleted,
+      use the `--reset` option. In this case, all existing emails are
+      deleted before uploading the migrated emails.
+    DESC
+    method_option(
+      "destination-prefix",
+      type: :string,
+      desc: "the prefix (namespace) to add to destination folder names",
+      aliases: ["-d"]
+    )
+    method_option(
+      "reset",
+      type: :boolean,
+      desc: "DANGER! This option deletes all messages from destination folders before uploading",
+      aliases: ["-r"]
+    )
+    method_option(
+      "source-prefix",
+      type: :string,
+      desc: "the prefix (namespace) to strip from source folder names",
+      aliases: ["-s"]
+    )
+    def migrate(source_email, destination_email)
+      Migrate.new(source_email, destination_email, symbolized(options)).run
+    end
+
+    desc "remote SUBCOMMAND [OPTIONS]", "View info about online accounts"
+    subcommand "remote", Remote
+
+    desc "restore EMAIL", "Restores a single account"
+    long_desc <<~DESC
+      Restores all backed-up emails for the supplied account to
+      their original server.
     DESC
     accounts_option
-    def restore
-      Restore.new(symbolized(options)).run
+    def restore(email = nil)
+      Restore.new(email, symbolized(options)).run
     end
 
     desc "setup", "Configure imap-backup"
@@ -86,9 +134,6 @@ module Imap::Backup
       Status.new(symbolized(options)).run
     end
 
-    desc "local SUBCOMMAND [OPTIONS]", "View local info"
-    subcommand "local", Local
-
     desc "utils SUBCOMMAND [OPTIONS]", "Various utilities"
     subcommand "utils", Utils
   end

data/lib/imap/backup/client/default.rb

@@ -7,8 +7,8 @@ module Imap::Backup
   class Client::Default
     extend Forwardable
     def_delegators :imap, *%i(
-      append authenticate create disconnect examine
-      login responses uid_fetch uid_search
+      append authenticate create disconnect examine expunge
+      login responses select uid_fetch uid_search uid_store
     )
 
     attr_reader :args

data/lib/imap/backup/downloader.rb

@@ -13,31 +13,45 @@ module Imap::Backup
     def run
       uids = folder.uids - serializer.uids
       count = uids.count
-      Imap::Backup::Logger.logger.debug "[#{folder.name}] #{count} new messages"
+      debug "#{count} new messages"
       uids.each_slice(block_size).with_index do |block, i|
-        offset = i * block_size + 1
         uids_and_bodies = folder.fetch_multi(block)
         if uids_and_bodies.nil?
           if block_size > 1
-            Imap::Backup::Logger.logger.debug("[#{folder.name}] Multi fetch failed for UIDs #{block.join(", ")}, switching to single fetches")
+            debug("Multi fetch failed for UIDs #{block.join(", ")}, switching to single fetches")
             @block_size = 1
             redo
           else
-            Imap::Backup::Logger.logger.debug("[#{folder.name}] Fetch failed for UID #{block[0]} - skipping")
+            debug("Fetch failed for UID #{block[0]} - skipping")
             next
           end
         end
 
+        offset = i * block_size + 1
         uids_and_bodies.each.with_index do |uid_and_body, j|
           uid = uid_and_body[:uid]
           body = uid_and_body[:body]
-          Imap::Backup::Logger.logger.debug(
-            "[#{folder.name}] uid: #{uid} (#{offset +j}/#{count}) - " \
-              "#{body.size} bytes"
-          )
-          serializer.save(uid, body)
+          case
+          when !body
+            info("Fetch returned empty body - skipping")
+          when !uid
+            info("Fetch returned empty UID - skipping")
+          else
+            debug("uid: #{uid} (#{offset + j}/#{count}) - #{body.size} bytes")
+            serializer.save(uid, body)
+          end
         end
       end
     end
+
+    private
+
+    def info(message)
+      Imap::Backup::Logger.logger.info("[#{folder.name}] #{message}")
+    end
+
+    def debug(message)
+      Imap::Backup::Logger.logger.debug("[#{folder.name}] #{message}")
+    end
   end
 end

data/lib/imap/backup/migrator.rb

@@ -0,0 +1,51 @@
+module Imap::Backup
+  class Migrator
+    attr_reader :folder
+    attr_reader :reset
+    attr_reader :serializer
+
+    def initialize(serializer, folder, reset: false)
+      @folder = folder
+      @reset = reset
+      @serializer = serializer
+    end
+
+    def run
+      count = serializer.uids.count
+      folder.create
+      ensure_destination_empty!
+
+      Imap::Backup::Logger.logger.debug "[#{folder.name}] #{count} to migrate"
+      serializer.each_message(serializer.uids).with_index do |(uid, message), i|
+        next if message.nil?
+
+        log_prefix = "[#{folder.name}] uid: #{uid} (#{i + 1}/#{count}) -"
+        Imap::Backup::Logger.logger.debug(
+          "#{log_prefix} #{message.supplied_body.size} bytes"
+        )
+
+        folder.append(message)
+      end
+    end
+
+    private
+
+    def ensure_destination_empty!
+      if reset
+        folder.clear
+      else
+        fail_if_destination_not_empty!
+      end
+    end
+
+    def fail_if_destination_not_empty!
+      return if folder.uids.empty?
+
+      raise <<~ERROR
+      The destination folder '#{folder.name}' is not empty.
+      Pass the --reset flag if you want to clear existing emails from destination
+      folders before uploading.
+      ERROR
+    end
+  end
+end

data/lib/imap/backup/sanitizer.rb

@@ -1,16 +1,19 @@
+require "forwardable"
+
 module Imap::Backup
   class Sanitizer
+    extend Forwardable
+
     attr_reader :output
 
+    delegate puts: :output
+    delegate write: :output
+
     def initialize(output)
       @output = output
       @current = ""
     end
 
-    def write(*args)
-      output.write(*args)
-    end
-
     def print(*args)
       @current << args.join
       loop do

data/lib/imap/backup/version.rb

@@ -1,8 +1,8 @@
 module Imap; end
 
 module Imap::Backup
-  MAJOR    = 4
-  MINOR    = 2
+  MAJOR    = 5
+  MINOR    = 1
   REVISION = 0
   PRE      = nil
   VERSION  = [MAJOR, MINOR, REVISION, PRE].compact.map(&:to_s).join(".")

data/spec/features/backup_spec.rb

@@ -1,28 +1,28 @@
 require "features/helper"
 
-RSpec.describe "backup", type: :feature, docker: true do
+RSpec.describe "backup", type: :aruba, docker: true do
   include_context "imap-backup connection"
   include_context "message-fixtures"
 
+  let(:local_backup_path) { File.expand_path("~/backup") }
+  let(:backup_folders) { [{name: folder}] }
+  let(:folder) { "my-stuff" }
   let(:messages_as_mbox) do
     message_as_mbox_entry(msg1) + message_as_mbox_entry(msg2)
   end
-  let(:folder) { "my-stuff" }
-  let(:email1) { send_email folder, msg1 }
-  let(:email2) { send_email folder, msg2 }
+
   let!(:pre) {}
   let!(:setup) do
     server_create_folder folder
-    email1
-    email2
-    connection.run_backup
+    send_email folder, msg1
+    send_email folder, msg2
+    create_config(accounts: [account.to_h])
+
+    run_command_and_stop("imap-backup backup")
   end
 
   after do
-    FileUtils.rm_rf local_backup_path
-    delete_emails folder
     server_delete_folder folder
-    connection.disconnect
   end
 
   it "downloads messages" do
@@ -51,11 +51,10 @@ RSpec.describe "backup", type: :feature, docker: true do
 
     context "when uid_validity does not match" do
       let(:new_name) { "NEWNAME" }
-      let(:email3) { send_email folder, msg3 }
       let(:original_folder_uid_validity) { server_uid_validity(folder) }
       let!(:pre) do
         server_create_folder folder
-        email3
+        send_email folder, msg3
         original_folder_uid_validity
         connection.run_backup
         connection.disconnect
@@ -78,6 +77,7 @@ RSpec.describe "backup", type: :feature, docker: true do
       context "when a renamed local backup exists" do
         let!(:pre) do
           super()
+          create_directory local_backup_path
           File.write(imap_path(renamed_folder), "existing imap")
           File.write(mbox_path(renamed_folder), "existing mbox")
         end
@@ -91,6 +91,7 @@ RSpec.describe "backup", type: :feature, docker: true do
 
     context "when an unversioned .imap file is found" do
       let!(:pre) do
+        create_directory local_backup_path
         File.open(imap_path(folder), "w") { |f| f.write "old format imap" }
         File.open(mbox_path(folder), "w") { |f| f.write "old format emails" }
       end

data/spec/features/configuration/minimal_configuration.rb

@@ -0,0 +1,15 @@
+require "features/helper"
+
+RSpec.describe "Running commands", type: :aruba do
+  before do
+    create_config(accounts: [])
+    run_command("imap-backup local accounts")
+    last_command_started.stop
+  end
+
+  context "when no accounts are configured" do
+    it "succeeds" do
+      expect(last_command_started).to have_exit_status(0)
+    end
+  end
+end

data/spec/features/configuration/missing_configuration.rb

@@ -0,0 +1,14 @@
+require "features/helper"
+
+RSpec.describe "Running commands", type: :aruba do
+  before do
+    run_command("imap-backup")
+    last_command_started.stop
+  end
+
+  context "when the configuration file is missing" do
+    it "fails" do
+      expect(last_command_started).not_to have_exit_status(0)
+    end
+  end
+end

data/spec/features/folders_spec.rb

@@ -0,0 +1,36 @@
+require "features/helper"
+require "imap/backup/cli/folders"
+
+RSpec.describe "folders", type: :feature, docker: true do
+  include_context "imap-backup connection"
+
+  let(:options) do
+    {accounts: "address@example.org"}
+  end
+  let(:folder) { "my-stuff" }
+  let(:output) { StringIO.new }
+
+  before do
+    allow(Imap::Backup::CLI::Accounts).to receive(:new) { [account] }
+    server_create_folder folder
+  end
+
+  around do |example|
+    stdout = $stdout
+    $stdout = output
+    example.run
+    $stdout = stdout
+  end
+
+  after do
+    FileUtils.rm_rf local_backup_path
+    server_delete_folder folder
+    connection.disconnect
+  end
+
+  it "lists account folders" do
+    Imap::Backup::CLI::Folders.new(options).run
+
+    expect(output.string).to match(/^\tmy-stuff\n/)
+  end
+end