imap-backup 4.1.2 → 5.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 00a744249ff80a26be2c57655709be59d9678c9bb998dc1c9f3ea4bf3539d747
-  data.tar.gz: 4f44c050c66fe2ef2e136b46dec581a91f77ab4b650e9b3466a893540e481a24
+  metadata.gz: 16911b1cea1983b844c3bbf7f74c03ff7436990fb5aa4d5b49fa18f03ae2e229
+  data.tar.gz: 265f63995e70ba3f68b98ec72d3a12b3e95100f7e6b320026b18bcc156888a92
 SHA512:
-  metadata.gz: 66067924fda33810aff4b02aa047c20cd26ce3fc31d8e752fc43a8f78af3e1c952871fa0ba372c670a5ee399e7fc686f9df9601fc5cbb7caa6f2065b4ac95b90
-  data.tar.gz: 443a86c3bda9cc1652fc9d1c5619347c1d82498befed16851a6f8a227eaf3299813dd6a4084e1bb0d180170adb1b63ccf8e5d2a8bb19f0457611e7916d796c4d
+  metadata.gz: 4a541a7830ddab458f2366bdad4924f3e6cbc3ba858651839e6dcd1a530acbbeb01e2c59a44331af73a2d77f606f2d6b39eacd2b5a705d78ec20e93d2c2f2527
+  data.tar.gz: 7b60170e3d29e9882f3bcbf13b04daa162434944d59311a5a03a4b1c1022a7a9501b292c3eb929d6e07039c47cc433316a338e062ced4e8a6b7ea424e6c0678b

data/imap-backup.gemspec

@@ -28,6 +28,7 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "rake"
   gem.add_runtime_dependency "thor", "~> 1.1"
 
+  gem.add_development_dependency "aruba", ">= 0.0.0"
   gem.add_development_dependency "codeclimate-test-reporter", "~> 0.4.8"
   if RUBY_ENGINE == "jruby"
     gem.add_development_dependency "pry-debugger-jruby"

data/lib/email/mboxrd/message.rb

@@ -10,7 +10,7 @@ module Email::Mboxrd
 
     attr_reader :supplied_body
 
-    def self.from_serialized(serialized)
+    def self.clean_serialized(serialized)
       cleaned = serialized.gsub(/^>(>*From)/, "\\1")
       # Serialized messages in this format *should* start with a line
       #   From xxx yy zz
@@ -19,7 +19,11 @@ module Email::Mboxrd
         cleaned = cleaned.sub(/^From .*[\r\n]*/, "")
       end
       # rubocop:enable Style/IfUnlessModifier
-      new(cleaned)
+      cleaned
+    end
+
+    def self.from_serialized(serialized)
+      new(clean_serialized(serialized))
     end
 
     def initialize(supplied_body)

data/lib/imap/backup/account/connection.rb

@@ -16,11 +16,8 @@ module Imap::Backup
     def initialize(account)
       @account = account
       reset
-      create_account_folder
     end
 
-    # TODO: Make this private once the 'folders' command
-    # has been removed.
     def folder_names
       @folder_names ||=
         begin
@@ -53,6 +50,7 @@ module Imap::Backup
     end
 
     def status
+      ensure_account_folder
       backup_folders.map do |folder|
         s = Serializer::Mbox.new(account.local_path, folder.name)
         {name: folder.name, local: s.uids, remote: folder.uids}
@@ -63,6 +61,7 @@ module Imap::Backup
       Imap::Backup::Logger.logger.debug "Running backup of account: #{account.username}"
       # start the connection so we get logging messages in the right order
       client
+      ensure_account_folder
       each_folder do |folder, serializer|
         next if !folder.exist?
 
@@ -82,6 +81,7 @@ module Imap::Backup
     def local_folders
       return enum_for(:local_folders) if !block_given?
 
+      ensure_account_folder
       glob = File.join(account.local_path, "**", "*.imap")
       base = Pathname.new(account.local_path)
       Pathname.glob(glob) do |path|
@@ -176,7 +176,7 @@ module Imap::Backup
       end
     end
 
-    def create_account_folder
+    def ensure_account_folder
       Utils.make_folder(
         File.dirname(account.local_path),
         File.basename(account.local_path),

data/lib/imap/backup/account/folder.rb

@@ -111,6 +111,14 @@ module Imap::Backup
       extract_uid(response)
     end
 
+    def clear
+      existing = uids
+      # Use read-write access, via `select`
+      client.select(utf7_encoded_name)
+      client.uid_store(existing, "+FLAGS", [:Deleted])
+      client.expunge
+    end
+
     private
 
     def examine

data/lib/imap/backup/cli/folders.rb

@@ -13,7 +13,7 @@ module Imap::Backup
     no_commands do
       def run
         each_connection(account_names) do |connection|
-          puts connection.username
+          puts connection.account.username
           # TODO: Make folder_names private once this command
           # has been removed.
           folders = connection.folder_names

data/lib/imap/backup/cli/helpers.rb

@@ -3,7 +3,10 @@ require "imap/backup/cli/accounts"
 
 module Imap::Backup::CLI::Helpers
   def symbolized(options)
-    options.each.with_object({}) { |(k, v), acc| acc[k.intern] = v }
+    options.each.with_object({}) do |(k, v), acc|
+      key = k.gsub("-", "_").intern
+      acc[key] = v
+    end
   end
 
   def account(email)

data/lib/imap/backup/cli/local.rb

@@ -11,7 +11,7 @@ module Imap::Backup
       accounts.each { |a| Kernel.puts a.username }
     end
 
-    desc "folders EMAIL", "List account folders"
+    desc "folders EMAIL", "List backed up folders"
     def folders(email)
       connection = connection(email)
 

data/lib/imap/backup/cli/migrate.rb

@@ -0,0 +1,114 @@
+require "imap/backup/migrator"
+
+module Imap::Backup
+  class CLI::Migrate < Thor
+    include Thor::Actions
+
+    attr_reader :destination_email
+    attr_reader :destination_prefix
+    attr_reader :reset
+    attr_reader :source_email
+    attr_reader :source_prefix
+
+    def initialize(
+      source_email,
+      destination_email,
+      destination_prefix: "",
+      reset: false,
+      source_prefix: ""
+    )
+      super([])
+      @destination_email = destination_email
+      @destination_prefix = destination_prefix
+      @reset = reset
+      @source_email = source_email
+      @source_prefix = source_prefix
+    end
+
+    no_commands do
+      def run
+        check_accounts!
+        folders.each do |serializer, folder|
+          Migrator.new(serializer, folder, reset: reset).run
+        end
+      end
+
+      def check_accounts!
+        if destination_email == source_email
+          raise "Source and destination accounts cannot be the same!"
+        end
+
+        if !destination_account
+          raise "Account #{destination_email} does not exist"
+        end
+
+        if !source_account
+          raise "Account #{source_email} does not exist"
+        end
+      end
+
+      def config
+        Configuration.new
+      end
+
+      def destination_account
+        config.accounts.find { |a| a.username == destination_email }
+      end
+
+      def folders
+        return enum_for(:folders) if !block_given?
+
+        glob = File.join(source_local_path, "**", "*.imap")
+        Pathname.glob(glob) do |path|
+          name = source_folder_name(path)
+          serializer = Serializer::Mbox.new(source_local_path, name)
+          folder = folder_for(name)
+          yield serializer, folder
+        end
+      end
+
+      def folder_for(source_folder)
+        no_source_prefix =
+          if source_prefix != "" &&
+              source_folder.start_with?(source_prefix)
+            source_folder.delete_prefix(source_prefix)
+          else
+            source_folder.to_s
+          end
+
+        with_destination_prefix =
+          if destination_prefix &&
+              destination_prefix != ""
+            destination_prefix + no_source_prefix
+          else
+            no_source_prefix
+          end
+
+        Account::Folder.new(
+          destination_account.connection,
+          with_destination_prefix
+        )
+      end
+
+      def source_local_path
+        source_account.local_path
+      end
+
+      def source_account
+        config.accounts.find { |a| a.username == source_email }
+      end
+
+      def source_folder_name(imap_pathname)
+        base = Pathname.new(source_local_path)
+        imap_name = imap_pathname.relative_path_from(base).to_s
+        dir = File.dirname(imap_name)
+        stripped = File.basename(imap_name, ".imap")
+        if dir == "."
+          stripped
+        else
+          File.join(dir, stripped)
+        end
+      end
+    end
+  end
+end

data/lib/imap/backup/cli/remote.rb

@@ -0,0 +1,15 @@
+module Imap::Backup
+  class CLI::Remote < Thor
+    include Thor::Actions
+    include CLI::Helpers
+
+    desc "folders EMAIL", "List account folders"
+    def folders(email)
+      connection = connection(email)
+
+      connection.folder_names.each do |name|
+        Kernel.puts %("#{name}")
+      end
+    end
+  end
+end

data/lib/imap/backup/cli.rb

@@ -9,6 +9,7 @@ module Imap::Backup
     autoload :Backup, "imap/backup/cli/backup"
     autoload :Folders, "imap/backup/cli/folders"
     autoload :Local, "imap/backup/cli/local"
+    autoload :Migrate, "imap/backup/cli/migrate"
     autoload :Remote, "imap/backup/cli/remote"
     autoload :Restore, "imap/backup/cli/restore"
     autoload :Setup, "imap/backup/cli/setup"
@@ -27,7 +28,7 @@ module Imap::Backup
       method_option(
         "accounts",
         type: :string,
-        banner: "a comma-separated list of accounts (defaults to all configured accounts)",
+        desc: "a comma-separated list of accounts (defaults to all configured accounts)",
         aliases: ["-a"]
       )
     end
@@ -57,6 +58,54 @@ module Imap::Backup
       Folders.new(symbolized(options)).run
     end
 
+    desc "local SUBCOMMAND [OPTIONS]", "View local info"
+    subcommand "local", Local
+
+    desc "migrate SOURCE_EMAIL DESTINATION_EMAIL [OPTIONS]",
+      "[Experimental] Uploads backed-up emails from account SOURCE_EMAIL to account DESTINATION_EMAIL"
+    long_desc <<~DESC
+      All emails which have been backed up for the "source account" (SOURCE_EMAIL) are
+      uploaded to the "destination account" (DESTINATION_EMAIL).
+
+      When one or other account has namespaces (i.e. prefixes like "INBOX."),
+      use the `--source-prefix=` and/or `--destination-prefix=` options.
+
+      Usually, you should migrate to an account with empty folders.
+
+      Before migrating each folder, `imap-backup` checks if the destination
+      folder is empty.
+
+      If it finds a non-empty destination folder, it halts with an error.
+
+      If you are sure that these destination emails can be deleted,
+      use the `--reset` option. In this case, all existing emails are
+      deleted before uploading the migrated emails.
+    DESC
+    method_option(
+      "destination-prefix",
+      type: :string,
+      desc: "the prefix (namespace) to add to destination folder names",
+      aliases: ["-d"]
+    )
+    method_option(
+      "reset",
+      type: :boolean,
+      desc: "DANGER! This option deletes all messages from destination folders before uploading",
+      aliases: ["-r"]
+    )
+    method_option(
+      "source-prefix",
+      type: :string,
+      desc: "the prefix (namespace) to strip from source folder names",
+      aliases: ["-s"]
+    )
+    def migrate(source_email, destination_email)
+      Migrate.new(source_email, destination_email, symbolized(options)).run
+    end
+
+    desc "remote SUBCOMMAND [OPTIONS]", "View info about online accounts"
+    subcommand "remote", Remote
+
     desc "restore [OPTIONS]", "This command is deprecated, use `imap-backup restore ACCOUNT`"
     long_desc <<~DESC
       By default, restores all local emails to their respective servers.
@@ -86,9 +135,6 @@ module Imap::Backup
       Status.new(symbolized(options)).run
     end
 
-    desc "local SUBCOMMAND [OPTIONS]", "View local info"
-    subcommand "local", Local
-
     desc "utils SUBCOMMAND [OPTIONS]", "Various utilities"
     subcommand "utils", Utils
   end

data/lib/imap/backup/client/default.rb

@@ -7,8 +7,8 @@ module Imap::Backup
   class Client::Default
     extend Forwardable
     def_delegators :imap, *%i(
-      append authenticate create disconnect examine
-      login responses uid_fetch uid_search
+      append authenticate create disconnect examine expunge
+      login responses select uid_fetch uid_search uid_store
     )
 
     attr_reader :args

data/lib/imap/backup/downloader.rb

@@ -13,31 +13,45 @@ module Imap::Backup
     def run
       uids = folder.uids - serializer.uids
       count = uids.count
-      Imap::Backup::Logger.logger.debug "[#{folder.name}] #{count} new messages"
+      debug "#{count} new messages"
       uids.each_slice(block_size).with_index do |block, i|
-        offset = i * block_size + 1
         uids_and_bodies = folder.fetch_multi(block)
         if uids_and_bodies.nil?
           if block_size > 1
-            Imap::Backup::Logger.logger.debug("[#{folder.name}] Multi fetch failed for UIDs #{block.join(", ")}, switching to single fetches")
+            debug("Multi fetch failed for UIDs #{block.join(", ")}, switching to single fetches")
             @block_size = 1
             redo
           else
-            Imap::Backup::Logger.logger.debug("[#{folder.name}] Fetch failed for UID #{block[0]} - skipping")
+            debug("Fetch failed for UID #{block[0]} - skipping")
             next
           end
         end
 
+        offset = i * block_size + 1
         uids_and_bodies.each.with_index do |uid_and_body, j|
           uid = uid_and_body[:uid]
           body = uid_and_body[:body]
-          Imap::Backup::Logger.logger.debug(
-            "[#{folder.name}] uid: #{uid} (#{offset +j}/#{count}) - " \
-              "#{body.size} bytes"
-          )
-          serializer.save(uid, body)
+          case
+          when !body
+            info("Fetch returned empty body - skipping")
+          when !uid
+            info("Fetch returned empty UID - skipping")
+          else
+            debug("uid: #{uid} (#{offset + j}/#{count}) - #{body.size} bytes")
+            serializer.save(uid, body)
+          end
         end
       end
     end
+
+    private
+
+    def info(message)
+      Imap::Backup::Logger.logger.info("[#{folder.name}] #{message}")
+    end
+
+    def debug(message)
+      Imap::Backup::Logger.logger.debug("[#{folder.name}] #{message}")
+    end
   end
 end

data/lib/imap/backup/migrator.rb

@@ -0,0 +1,51 @@
+module Imap::Backup
+  class Migrator
+    attr_reader :folder
+    attr_reader :reset
+    attr_reader :serializer
+
+    def initialize(serializer, folder, reset: false)
+      @folder = folder
+      @reset = reset
+      @serializer = serializer
+    end
+
+    def run
+      count = serializer.uids.count
+      folder.create
+      ensure_destination_empty!
+
+      Imap::Backup::Logger.logger.debug "[#{folder.name}] #{count} to migrate"
+      serializer.each_message(serializer.uids).with_index do |(uid, message), i|
+        next if message.nil?
+
+        log_prefix = "[#{folder.name}] uid: #{uid} (#{i + 1}/#{count}) -"
+        Imap::Backup::Logger.logger.debug(
+          "#{log_prefix} #{message.supplied_body.size} bytes"
+        )
+
+        folder.append(message)
+      end
+    end
+
+    private
+
+    def ensure_destination_empty!
+      if reset
+        folder.clear
+      else
+        fail_if_destination_not_empty!
+      end
+    end
+
+    def fail_if_destination_not_empty!
+      return if folder.uids.empty?
+
+      raise <<~ERROR
+      The destination folder '#{folder.name}' is not empty.
+      Pass the --reset flag if you want to clear existing emails from destination
+      folders before uploading.
+      ERROR
+    end
+  end
+end

data/lib/imap/backup/sanitizer.rb

@@ -1,16 +1,19 @@
+require "forwardable"
+
 module Imap::Backup
   class Sanitizer
+    extend Forwardable
+
     attr_reader :output
 
+    delegate puts: :output
+    delegate write: :output
+
     def initialize(output)
       @output = output
       @current = ""
     end
 
-    def write(*args)
-      output.write(*args)
-    end
-
     def print(*args)
       @current << args.join
       loop do

data/lib/imap/backup/thunderbird/mailbox_exporter.rb

@@ -40,7 +40,16 @@ module Imap::Backup
         end
       end
 
-      FileUtils.cp serializer.mbox_pathname, local_folder.full_path
+      File.open(local_folder.full_path, "w") do |f|
+        enumerator = Serializer::MboxEnumerator.new(serializer.mbox_pathname)
+        enumerator.each.with_index do |raw, i|
+          clean = Email::Mboxrd::Message.clean_serialized(raw)
+          timestamp = Time.now.strftime("%a %b %d %H:%M:%S %Y")
+          thunderbird_fom_line = "From - #{timestamp}"
+          output = "#{thunderbird_fom_line}\n#{clean}\n"
+          f.write output
+        end
+      end
 
       true
     end

data/lib/imap/backup/version.rb

@@ -1,9 +1,9 @@
 module Imap; end
 
 module Imap::Backup
-  MAJOR    = 4
-  MINOR    = 1
-  REVISION = 2
+  MAJOR    = 5
+  MINOR    = 0
+  REVISION = 0
   PRE      = nil
   VERSION  = [MAJOR, MINOR, REVISION, PRE].compact.map(&:to_s).join(".")
 end

data/lib/thunderbird/subdirectory.rb

@@ -77,17 +77,14 @@ class Thunderbird::Subdirectory
 
   def check
     case
-    when placeholder.exists? && !exists?
-      Kernel.puts "Can't set up folder '#{folder_path}': '#{placeholder.path}' exists, but '#{full_path}' is missing"
-      false
     when exists? && !placeholder.exists?
-      Kernel.puts "Can't set up folder '#{folder_path}': '#{full_path}' exists, but '#{placeholder.path}' is missing"
+      Kernel.puts "Can't set up folder '#{full_path}': '#{full_path}' exists, but '#{placeholder.path}' is missing"
       false
     when placeholder.exists? && !placeholder.regular?
-      Kernel.puts "Can't set up folder '#{folder_path}': '#{placeholder.path}' exists, but it is not a regular file"
+      Kernel.puts "Can't set up folder '#{full_path}': '#{placeholder.path}' exists, but it is not a regular file"
       false
     when exists? && !directory?
-      Kernel.puts "Can't set up folder '#{folder_path}': '#{full_path}' exists, but it is not a directory"
+      Kernel.puts "Can't set up folder '#{full_path}': '#{full_path}' exists, but it is not a directory"
       false
     else
       true

data/spec/features/configuration/minimal_configuration.rb

@@ -0,0 +1,15 @@
+require "features/helper"
+
+RSpec.describe "Running commands", type: :aruba do
+  before do
+    create_config(accounts: [])
+    run_command("imap-backup local accounts")
+    last_command_started.stop
+  end
+
+  context "when no accounts are configured" do
+    it "succeeds" do
+      expect(last_command_started).to have_exit_status(0)
+    end
+  end
+end

data/spec/features/configuration/missing_configuration.rb

@@ -0,0 +1,14 @@
+require "features/helper"
+
+RSpec.describe "Running commands", type: :aruba do
+  before do
+    run_command("imap-backup")
+    last_command_started.stop
+  end
+
+  context "when the configuration file is missing" do
+    it "fails" do
+      expect(last_command_started).not_to have_exit_status(0)
+    end
+  end
+end

data/spec/features/folders_spec.rb

@@ -0,0 +1,36 @@
+require "features/helper"
+require "imap/backup/cli/folders"
+
+RSpec.describe "folders", type: :feature, docker: true do
+  include_context "imap-backup connection"
+
+  let(:options) do
+    {accounts: "address@example.org"}
+  end
+  let(:folder) { "my-stuff" }
+  let(:output) { StringIO.new }
+
+  before do
+    allow(Imap::Backup::CLI::Accounts).to receive(:new) { [account] }
+    server_create_folder folder
+  end
+
+  around do |example|
+    stdout = $stdout
+    $stdout = output
+    example.run
+    $stdout = stdout
+  end
+
+  after do
+    FileUtils.rm_rf local_backup_path
+    server_delete_folder folder
+    connection.disconnect
+  end
+
+  it "lists account folders" do
+    Imap::Backup::CLI::Folders.new(options).run
+
+    expect(output.string).to match(/^\tmy-stuff\n/)
+  end
+end

data/spec/features/local/list_accounts_spec.rb

@@ -0,0 +1,12 @@
+require "features/helper"
+
+RSpec.describe "Listing accounts", type: :aruba do
+  before do
+    create_config(accounts: [{"username": "me@example.com"}])
+    run_command_and_stop("imap-backup local accounts")
+  end
+
+  it "lists accounts" do
+    expect(last_command_started).to have_output("me@example.com")
+  end
+end

data/spec/features/local/list_emails_spec.rb

@@ -0,0 +1,21 @@
+require "features/helper"
+
+RSpec.describe "Listing emails", type: :aruba do
+  let(:email) { "me@example.com" }
+  let(:account) do
+    {
+      username: email,
+      local_path: File.join(config_path, email.gsub("@", "_"))
+    }
+  end
+
+  before do
+    create_config(accounts: [account])
+    store_email(email: email, folder: "my_folder", subject: "Ciao")
+    run_command_and_stop("imap-backup local list #{email} my_folder")
+  end
+
+  it "lists emails" do
+    expect(last_command_started).to have_output(/1: Ciao/)
+  end
+end

data/spec/features/local/list_folders_spec.rb

@@ -0,0 +1,21 @@
+require "features/helper"
+
+RSpec.describe "Listing account folders", type: :aruba do
+  let(:email) { "me@example.com" }
+  let(:account) do
+    {
+      username: email,
+      local_path: File.join(config_path, email.gsub("@", "_"))
+    }
+  end
+
+  before do
+    create_config(accounts: [account])
+    store_email(email: email, folder: "my_folder", body: "Hi")
+    run_command_and_stop("imap-backup local folders #{email}")
+  end
+
+  it "lists folders that have been backed up" do
+    expect(last_command_started).to have_output(%q("my_folder"))
+  end
+end

data/spec/features/local/show_an_email_spec.rb

@@ -0,0 +1,34 @@
+require "features/helper"
+
+RSpec.describe "Show an email", type: :aruba do
+  let(:email) { "me@example.com" }
+  let(:account) do
+    {
+      username: email,
+      local_path: File.join(config_path, email.gsub("@", "_"))
+    }
+  end
+
+  before do
+    create_config(accounts: [account])
+    store_email(
+      email: email,
+      folder: "my_folder",
+      uid: 99,
+      from: "me@example.com",
+      subject: "Hello",
+      body: "How're things?"
+    )
+    run_command_and_stop("imap-backup local show #{email} my_folder 99")
+  end
+
+  it "shows the email" do
+    expected = <<~BODY
+      From: me@example.com
+      Subject: Hello
+      
+      How're things?
+    BODY
+    expect(last_command_started).to have_output(expected)
+  end
+end

data/spec/features/migrate_spec.rb

@@ -0,0 +1,35 @@
+require "features/helper"
+
+RSpec.describe "Migration", type: :aruba, docker: true do
+  let(:email) { "me@example.com" }
+  let(:folder) { "my_folder" }
+  let(:source_account) do
+    {
+      username: email,
+      local_path: File.join(config_path, email.gsub("@", "_"))
+    }
+  end
+  let(:destination_account) { fixture("connection") }
+
+  before do
+    create_config(accounts: [source_account, destination_account])
+    store_email(email: email, folder: folder, subject: "Ciao")
+    run_command_and_stop("imap-backup migrate #{email} #{destination_account[:username]}")
+  end
+
+  after do
+    delete_emails(folder)
+  end
+
+  it "copies email to the destination account" do
+    messages = server_messages(folder)
+    expected = <<~MESSAGE.gsub("\n", "\r\n")
+      From: sender@example.com
+      Subject: Ciao
+
+      body
+
+    MESSAGE
+    expect(messages[0]["BODY[]"]).to eq(expected)
+  end
+end

data/spec/features/remote/list_account_folders_spec.rb

@@ -0,0 +1,16 @@
+require "features/helper"
+
+RSpec.describe "List account folders", type: :aruba do
+  let(:account) do
+    fixture("connection")
+  end
+
+  before do
+    create_config(accounts: [account])
+    run_command_and_stop("imap-backup remote folders #{account[:username]}")
+  end
+
+  it "lists folders" do
+    expect(last_command_started).to have_output(/"INBOX"/)
+  end
+end

data/spec/features/support/aruba.rb

@@ -0,0 +1,69 @@
+require "aruba/rspec"
+
+Aruba.configure do |config|
+  config.home_directory = File.expand_path("./tmp/home")
+end
+
+module ConfigurationHelpers
+  def config_path
+    File.expand_path("~/.imap-backup")
+  end
+
+  def create_config(accounts:, debug: false)
+    pathname = File.join(config_path, "config.json")
+    save_data = {
+      version: Imap::Backup::Configuration::VERSION,
+      accounts: accounts,
+      debug: debug
+    }
+    FileUtils.mkdir_p config_path
+    File.open(pathname, "w") { |f| f.write(JSON.pretty_generate(save_data)) }
+    FileUtils.chmod(0o600, pathname)
+  end
+end
+
+module StoreHelpers
+  def store_email(
+    email:, folder:,
+    uid: 1,
+    from: "sender@example.com",
+    subject: "The Subject",
+    body: "body"
+  )
+    account = config.accounts.find { |a| a.username == email }
+    raise "Account not found" if !account
+    FileUtils.mkdir_p account.local_path
+    store = Imap::Backup::Serializer::MboxStore.new(account.local_path, folder)
+    store.uid_validity = "42" if !store.uid_validity
+    serialized = to_serialized(from: from, subject: subject, body: body)
+    store.add(uid, serialized)
+  end
+
+  def to_serialized(from:, subject:, body:)
+    body_and_headers = <<~BODY
+      From: #{from}
+      Subject: #{subject}
+
+      #{body}
+    BODY
+  end
+
+  def config
+    Imap::Backup::Configuration.new(
+      File.expand_path("~/.imap-backup/config.json")
+    )
+  end
+end
+
+RSpec.configure do |config|
+  config.include ConfigurationHelpers, type: :aruba
+  config.include StoreHelpers, type: :aruba
+
+  config.before(:suite) do
+    FileUtils.rm_rf "./tmp/home"
+  end
+
+  config.after do
+    FileUtils.rm_rf "./tmp/home"
+  end
+end

data/spec/features/support/email_server.rb

@@ -106,5 +106,6 @@ module EmailServerHelpers
 end
 
 RSpec.configure do |config|
+  config.include EmailServerHelpers, type: :aruba
   config.include EmailServerHelpers, type: :feature
 end

data/spec/unit/imap/backup/account/connection_spec.rb

@@ -61,14 +61,6 @@ describe Imap::Backup::Account::Connection do
     end
   end
 
-  describe "#initialize" do
-    it "creates the path" do
-      expect(Imap::Backup::Utils).to receive(:make_folder)
-
-      subject
-    end
-  end
-
   describe "#client" do
     let(:result) { subject.client }
 
@@ -127,6 +119,12 @@ describe Imap::Backup::Account::Connection do
       allow(Imap::Backup::Serializer::Mbox).to receive(:new) { serializer }
     end
 
+    it "creates the path" do
+      expect(Imap::Backup::Utils).to receive(:make_folder)
+
+      subject.status
+    end
+
     it "returns the names of folders" do
       expect(subject.status[0][:name]).to eq(IMAP_FOLDER)
     end

data/spec/unit/imap/backup/account/folder_spec.rb

@@ -12,7 +12,10 @@ describe Imap::Backup::Account::Folder do
       append: append_response,
       create: nil,
       examine: nil,
-      responses: responses
+      expunge: nil,
+      responses: responses,
+      select: nil,
+      uid_store: nil
     )
   end
   let(:connection) do
@@ -27,12 +30,11 @@ describe Imap::Backup::Account::Folder do
   end
   let(:responses) { [] }
   let(:append_response) { nil }
+  let(:uids) { [5678, 123] }
 
-  describe "#uids" do
-    let(:uids) { [5678, 123] }
-
-    before { allow(client).to receive(:uid_search) { uids } }
+  before { allow(client).to receive(:uid_search) { uids } }
 
+  describe "#uids" do
     it "lists available messages" do
       expect(subject.uids).to eq(uids.reverse)
     end
@@ -233,6 +235,25 @@ describe Imap::Backup::Account::Folder do
       expect(subject.uid_validity).to eq(1)
     end
   end
+
+  describe "#clear" do
+    before do
+      subject.clear
+    end
+
+    it "uses select to have read-write access" do
+      expect(client).to have_received(:select)
+    end
+
+    it "marks all emails as deleted" do
+      expect(client).
+        to have_received(:uid_store).with(uids.sort, "+FLAGS", [:Deleted])
+    end
+
+    it "deletes marked emails" do
+      expect(client).to have_received(:expunge)
+    end
+  end
 end
 
 # rubocop:enable RSpec/PredicateMatcher

data/spec/unit/imap/backup/cli/helpers_spec.rb

@@ -0,0 +1,87 @@
+require "imap/backup/cli/helpers"
+
+module Imap::Backup
+  class WithHelpers
+    include CLI::Helpers
+  end
+
+  RSpec.describe CLI::Helpers do
+    subject { WithHelpers.new }
+
+    let(:accounts) { instance_double(CLI::Accounts) }
+    let(:email) { "email@example.com" }
+    let(:account1) { instance_double(Account, username: email, connection: "c1") }
+    let(:account2) { instance_double(Account, username: "foo", connection: "c2") }
+    let(:items) { [account1, account2] }
+
+    before do
+      allow(CLI::Accounts).to receive(:new) { accounts }
+      allow(accounts).to receive(:each).
+        and_yield(account1).
+        and_yield(account2)
+      allow(accounts).to receive(:find) do |&block|
+        items.find { |a| block.call(a) }
+      end
+    end
+
+    describe ".symbolized" do
+      let(:arguments) { {"foo" => 1, "bar" => 2} }
+      let(:result) { subject.symbolized(arguments) }
+
+      it "converts string keys to symbols" do
+        expect(result.keys).to eq([:foo, :bar])
+      end
+
+      context "when keys have hyphens" do
+        let(:arguments) { {"some-option"=> 3} }
+
+        it "replaces them with underscores" do
+          expect(result.keys).to eq([:some_option])
+        end
+      end
+    end
+
+    describe ".account" do
+      it "returns any account with a matching username" do
+        expect(subject.account(email)).to eq(account1)
+      end
+
+      context "when no match is found" do
+        let(:items) { [account2] }
+        it "fails" do
+          expect do
+            subject.account(email)
+          end.to raise_error(RuntimeError, /not a configured account/)
+        end
+      end
+    end
+
+    describe ".connection" do
+      it "returns the connection for any account with a matching username" do
+        result = subject.connection(email)
+        expect(result).to be_a(Account::Connection)
+        expect(result.account).to eq(account1)
+      end
+    end
+
+    describe ".each_connection" do
+      it "yields each connection" do
+        expect { |b| subject.each_connection([email, "foo"], &b) }.
+          to yield_successive_args("c1", "c2")
+      end
+
+      context "when there is no configuration" do
+        before do
+          allow(accounts).to receive(:each).
+            and_raise(Imap::Backup::ConfigurationNotFound)
+        end
+
+        it "fails" do
+          expect do
+            subject.each_connection([email])
+          end.to raise_error(RuntimeError, /not configured/)
+        end
+      end
+    end
+  end
+end

data/spec/unit/imap/backup/migrator_spec.rb

@@ -0,0 +1,58 @@
+require "imap/backup/migrator"
+
+module Imap::Backup
+  RSpec.describe Migrator do
+    subject { described_class.new(serializer, folder, reset: reset) }
+
+    let(:serializer) { instance_double(Serializer::MboxStore, uids: [1]) }
+    let(:folder) do
+      instance_double(
+        Account::Folder,
+        append: nil, clear: nil, create: nil, name: "name", uids: folder_uids
+      )
+    end
+    let(:folder_uids) { [] }
+    let(:reset) { false }
+    let(:messages) { [[1, message]] }
+    let(:message) { instance_double(Email::Mboxrd::Message, supplied_body: body) }
+    let(:body) { "body" }
+
+    before do
+      allow(serializer).to receive(:each_message) do
+        messages.enum_for(:each)
+      end
+    end
+
+    it "creates the folder" do
+      subject.run
+
+      expect(folder).to have_received(:create)
+    end
+
+    it "uploads messages" do
+      subject.run
+
+      expect(folder).to have_received(:append).with(message)
+    end
+
+    context "when the folder is not empty" do
+      let(:folder_uids) { [99] }
+
+      it "fails" do
+        expect do
+          subject.run
+        end.to raise_error(RuntimeError, /The destination folder 'name' is not empty/)
+      end
+
+      context "when `reset` is true" do
+        let(:reset) { true }
+
+        it "clears the folder" do
+          subject.run
+
+          expect(folder).to have_received(:clear)
+        end
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: imap-backup
 version: !ruby/object:Gem::Version
-  version: 4.1.2
+  version: 5.0.0
 platform: ruby
 authors:
 - Joe Yates
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-01-07 00:00:00.000000000 Z
+date: 2022-02-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: highline
@@ -80,6 +80,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: aruba
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.0
 - !ruby/object:Gem::Dependency
   name: codeclimate-test-reporter
   requirement: !ruby/object:Gem::Requirement
@@ -196,6 +210,8 @@ files:
 - lib/imap/backup/cli/folders.rb
 - lib/imap/backup/cli/helpers.rb
 - lib/imap/backup/cli/local.rb
+- lib/imap/backup/cli/migrate.rb
+- lib/imap/backup/cli/remote.rb
 - lib/imap/backup/cli/restore.rb
 - lib/imap/backup/cli/setup.rb
 - lib/imap/backup/cli/status.rb
@@ -205,6 +221,7 @@ files:
 - lib/imap/backup/configuration.rb
 - lib/imap/backup/downloader.rb
 - lib/imap/backup/logger.rb
+- lib/imap/backup/migrator.rb
 - lib/imap/backup/sanitizer.rb
 - lib/imap/backup/serializer.rb
 - lib/imap/backup/serializer/mbox.rb
@@ -229,9 +246,19 @@ files:
 - lib/thunderbird/subdirectory.rb
 - lib/thunderbird/subdirectory_placeholder.rb
 - spec/features/backup_spec.rb
+- spec/features/configuration/minimal_configuration.rb
+- spec/features/configuration/missing_configuration.rb
+- spec/features/folders_spec.rb
 - spec/features/helper.rb
+- spec/features/local/list_accounts_spec.rb
+- spec/features/local/list_emails_spec.rb
+- spec/features/local/list_folders_spec.rb
+- spec/features/local/show_an_email_spec.rb
+- spec/features/migrate_spec.rb
+- spec/features/remote/list_account_folders_spec.rb
 - spec/features/restore_spec.rb
 - spec/features/status_spec.rb
+- spec/features/support/aruba.rb
 - spec/features/support/backup_directory.rb
 - spec/features/support/email_server.rb
 - spec/features/support/shared/connection_context.rb
@@ -251,12 +278,14 @@ files:
 - spec/unit/imap/backup/account/connection_spec.rb
 - spec/unit/imap/backup/account/folder_spec.rb
 - spec/unit/imap/backup/cli/accounts_spec.rb
+- spec/unit/imap/backup/cli/helpers_spec.rb
 - spec/unit/imap/backup/cli/local_spec.rb
 - spec/unit/imap/backup/cli/utils_spec.rb
 - spec/unit/imap/backup/client/default_spec.rb
 - spec/unit/imap/backup/configuration_spec.rb
 - spec/unit/imap/backup/downloader_spec.rb
 - spec/unit/imap/backup/logger_spec.rb
+- spec/unit/imap/backup/migrator_spec.rb
 - spec/unit/imap/backup/serializer/mbox_enumerator_spec.rb
 - spec/unit/imap/backup/serializer/mbox_spec.rb
 - spec/unit/imap/backup/serializer/mbox_store_spec.rb
@@ -271,7 +300,7 @@ homepage: https://github.com/joeyates/imap-backup
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -287,46 +316,58 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.1.6
-signing_key: 
+signing_key:
 specification_version: 4
 summary: Backup GMail (or other IMAP) accounts to disk
 test_files:
-- spec/features/helper.rb
-- spec/features/support/backup_directory.rb
-- spec/features/support/shared/message_fixtures.rb
-- spec/features/support/shared/connection_context.rb
-- spec/features/support/email_server.rb
-- spec/features/restore_spec.rb
-- spec/features/status_spec.rb
-- spec/features/backup_spec.rb
-- spec/support/higline_test_helpers.rb
-- spec/support/fixtures.rb
-- spec/support/silence_logging.rb
-- spec/gather_rspec_coverage.rb
-- spec/unit/email/provider/gmail_spec.rb
-- spec/unit/email/provider/fastmail_spec.rb
-- spec/unit/email/provider/base_spec.rb
-- spec/unit/email/provider/apple_mail_spec.rb
-- spec/unit/email/mboxrd/message_spec.rb
-- spec/unit/email/provider_spec.rb
-- spec/unit/imap/backup/cli/utils_spec.rb
-- spec/unit/imap/backup/cli/accounts_spec.rb
-- spec/unit/imap/backup/cli/local_spec.rb
+- spec/spec_helper.rb
+- spec/unit/imap/backup/serializer/mbox_spec.rb
+- spec/unit/imap/backup/serializer/mbox_store_spec.rb
+- spec/unit/imap/backup/serializer/mbox_enumerator_spec.rb
+- spec/unit/imap/backup/setup_spec.rb
+- spec/unit/imap/backup/logger_spec.rb
+- spec/unit/imap/backup/configuration_spec.rb
 - spec/unit/imap/backup/setup/account_spec.rb
 - spec/unit/imap/backup/setup/connection_tester_spec.rb
-- spec/unit/imap/backup/setup/folder_chooser_spec.rb
 - spec/unit/imap/backup/setup/asker_spec.rb
-- spec/unit/imap/backup/uploader_spec.rb
-- spec/unit/imap/backup/utils_spec.rb
-- spec/unit/imap/backup/configuration_spec.rb
-- spec/unit/imap/backup/setup_spec.rb
-- spec/unit/imap/backup/logger_spec.rb
-- spec/unit/imap/backup/serializer/mbox_store_spec.rb
-- spec/unit/imap/backup/serializer/mbox_enumerator_spec.rb
-- spec/unit/imap/backup/serializer/mbox_spec.rb
+- spec/unit/imap/backup/setup/folder_chooser_spec.rb
+- spec/unit/imap/backup/cli/helpers_spec.rb
+- spec/unit/imap/backup/cli/accounts_spec.rb
+- spec/unit/imap/backup/cli/utils_spec.rb
+- spec/unit/imap/backup/cli/local_spec.rb
 - spec/unit/imap/backup/downloader_spec.rb
+- spec/unit/imap/backup/migrator_spec.rb
+- spec/unit/imap/backup/uploader_spec.rb
 - spec/unit/imap/backup/account/folder_spec.rb
 - spec/unit/imap/backup/account/connection_spec.rb
+- spec/unit/imap/backup/utils_spec.rb
 - spec/unit/imap/backup/client/default_spec.rb
-- spec/spec_helper.rb
+- spec/unit/email/mboxrd/message_spec.rb
+- spec/unit/email/provider/gmail_spec.rb
+- spec/unit/email/provider/apple_mail_spec.rb
+- spec/unit/email/provider/fastmail_spec.rb
+- spec/unit/email/provider/base_spec.rb
+- spec/unit/email/provider_spec.rb
+- spec/features/folders_spec.rb
+- spec/features/configuration/missing_configuration.rb
+- spec/features/configuration/minimal_configuration.rb
+- spec/features/backup_spec.rb
+- spec/features/helper.rb
+- spec/features/restore_spec.rb
+- spec/features/local/list_emails_spec.rb
+- spec/features/local/list_accounts_spec.rb
+- spec/features/local/show_an_email_spec.rb
+- spec/features/local/list_folders_spec.rb
+- spec/features/status_spec.rb
+- spec/features/support/email_server.rb
+- spec/features/support/backup_directory.rb
+- spec/features/support/aruba.rb
+- spec/features/support/shared/connection_context.rb
+- spec/features/support/shared/message_fixtures.rb
+- spec/features/migrate_spec.rb
+- spec/features/remote/list_account_folders_spec.rb
+- spec/support/fixtures.rb
+- spec/support/silence_logging.rb
+- spec/support/higline_test_helpers.rb
 - spec/fixtures/connection.yml
+- spec/gather_rspec_coverage.rb