imap-backup 6.0.0.rc2 → 6.1.0

data/lib/imap/backup/setup/folder_chooser.rb

@@ -12,13 +12,13 @@ module Imap::Backup
 
     def run
       if connection.nil?
-        Imap::Backup::Logger.logger.warn "Connection failed"
+        Logger.logger.warn "Connection failed"
         highline.ask "Press a key "
         return
       end
 
       if imap_folders.nil?
-        Imap::Backup::Logger.logger.warn "Unable to get folder list"
+        Logger.logger.warn "Unable to get folder list"
         highline.ask "Press a key "
         return
       end
@@ -90,7 +90,7 @@ module Imap::Backup
 
     def toggle_selection(folder_name)
       if selected?(folder_name)
-        new_list = account.folders.select { |f| f[:name] != folder_name }
+        new_list = account.folders.reject { |f| f[:name] == folder_name }
         account.folders = new_list
       else
         existing = account.folders || []

data/lib/imap/backup/setup/helpers.rb

@@ -5,7 +5,7 @@ module Imap::Backup
 
   class Setup::Helpers
     def title_prefix
-      "imap-backup –"
+      "imap-backup -"
     end
 
     def version

data/lib/imap/backup/setup.rb

@@ -1,5 +1,6 @@
 require "highline"
 
+require "email/provider"
 require "imap/backup/account"
 require "imap/backup/setup/helpers"
 
@@ -36,7 +37,7 @@ module Imap::Backup
             config.save
             throw :done
           end
-          menu.choice("exit without saving changes")  { throw :done }
+          menu.choice("exit without saving changes") { throw :done }
         else
           menu.choice("quit") { throw :done }
         end
@@ -67,7 +68,7 @@ module Imap::Backup
       new_setting = !config.debug?
       menu.choice(menu_item) do
         config.debug = new_setting
-        Imap::Backup::Logger.setup_logging(config)
+        Logger.setup_logging(config)
       end
     end
 
@@ -76,14 +77,14 @@ module Imap::Backup
     end
 
     def default_account_config(username)
-      ::Imap::Backup::Account.new(
+      Imap::Backup::Account.new(
         username: username,
         password: "",
-        local_path: File.join(config.path, username.tr("@", "_")),
         folders: []
       ).tap do |a|
-        server = Email::Provider.for_address(username)
-        a.server = server.host if server.host
+        provider = ::Email::Provider.for_address(username)
+        a.server = provider.host if provider.host
+        a.reset_seen_flags_after_fetch = true if provider.sets_seen_flags_on_fetch?
       end
     end
 

data/lib/imap/backup/thunderbird/mailbox_exporter.rb

@@ -21,28 +21,51 @@ module Imap::Backup
       local_folder_ok = local_folder.set_up
       return false if !local_folder_ok
 
-      if local_folder.msf_exists?
-        if force
-          Kernel.puts "Deleting '#{local_folder.msf_path}' as --force option was supplied"
-          File.unlink local_folder.msf_path
-        else
-          Kernel.puts "Skipping export of '#{serializer.folder}' as '#{local_folder.msf_path}' exists"
-          return false
-        end
+      skip_for_msf = check_msf
+      return false if skip_for_msf
+
+      skip_for_local_folder = check_local_folder
+      return false if skip_for_local_folder
+
+      copy_messages
+
+      true
+    end
+
+    private
+
+    def check_local_folder
+      return false if !local_folder.exists?
+
+      if force
+        Kernel.puts "Overwriting '#{local_folder.path}' as --force option was supplied"
+        return false
       end
 
-      if local_folder.exists?
-        if force
-          Kernel.puts "Overwriting '#{local_folder.path}' as --force option was supplied"
-        else
-          Kernel.puts "Skipping export of '#{serializer.folder}' as '#{local_folder.path}' exists"
-          return false
-        end
+      Kernel.puts "Skipping export of '#{serializer.folder}' as '#{local_folder.path}' exists"
+      true
+    end
+
+    def check_msf
+      return false if !local_folder.msf_exists?
+
+      if force
+        Kernel.puts "Deleting '#{local_folder.msf_path}' as --force option was supplied"
+        File.unlink local_folder.msf_path
+        return false
       end
 
+      Kernel.puts(
+        "Skipping export of '#{serializer.folder}' " \
+        "as '#{local_folder.msf_path}' exists"
+      )
+      true
+    end
+
+    def copy_messages
       File.open(local_folder.full_path, "w") do |f|
         enumerator = Serializer::MboxEnumerator.new(serializer.mbox_pathname)
-        enumerator.each.with_index do |raw, i|
+        enumerator.each do |raw|
           clean = Email::Mboxrd::Message.clean_serialized(raw)
           timestamp = Time.now.strftime("%a %b %d %H:%M:%S %Y")
           thunderbird_fom_line = "From - #{timestamp}"
@@ -50,12 +73,8 @@ module Imap::Backup
           f.write output
         end
       end
-
-      true
     end
 
-    private
-
     def local_folder
       @local_folder ||= begin
         top_level_folders = [EXPORT_PREFIX, email]

data/lib/imap/backup/uploader.rb

@@ -9,27 +9,65 @@ module Imap::Backup
     end
 
     def run
-      count = missing_uids.count
+      if folder.uids.any?
+        rename_serialized_folder
+      else
+        folder.create
+        serializer.force_uid_validity(folder.uid_validity)
+      end
+
       return if count.zero?
 
-      Imap::Backup::Logger.logger.debug "[#{folder.name}] #{count} to restore"
+      Logger.logger.debug "[#{folder.name}] #{count} to restore"
       serializer.each_message(missing_uids).with_index do |(uid, message), i|
-        next if message.nil?
+        upload_message uid, message, i + 1
+      end
+    end
+
+    private
+
+    def upload_message(uid, message, index)
+      return if message.nil?
 
-        log_prefix = "[#{folder.name}] uid: #{uid} (#{i + 1}/#{count}) -"
-        Imap::Backup::Logger.logger.debug(
-          "#{log_prefix} #{message.supplied_body.size} bytes"
-        )
+      log_prefix = "[#{folder.name}] uid: #{uid} (#{index}/#{count}) -"
+      Logger.logger.debug(
+        "#{log_prefix} #{message.supplied_body.size} bytes"
+      )
 
+      begin
         new_uid = folder.append(message)
         serializer.update_uid(uid, new_uid)
+      rescue StandardError => e
+        Logger.logger.warn "#{log_prefix} append error: #{e}"
       end
     end
 
-    private
+    def count
+      @count ||= missing_uids.count
+    end
 
     def missing_uids
       serializer.uids - folder.uids
     end
+
+    def rename_serialized_folder
+      Logger.logger.debug(
+        "There's already a '#{folder.name}' folder with emails"
+      )
+
+      # Rename the local folder to a unique name
+      new_name = serializer.apply_uid_validity(folder.uid_validity)
+
+      return if !new_name
+
+      # Restore the renamed folder
+      Logger.logger.debug(
+        "Backup '#{serializer.folder}' renamed and restored to '#{new_name}'"
+      )
+      @folder = Account::Folder.new(folder.connection, new_name)
+      folder.create
+      @serializer = Serializer.new(serializer.path, new_name)
+      serializer.force_uid_validity(@folder.uid_validity)
+    end
   end
 end

data/lib/imap/backup/utils.rb

@@ -11,7 +11,7 @@ module Imap::Backup
 
       message = format(
         "Permissions on '%<filename>s' " \
-          "should be 0%<limit>o, not 0%<actual>o",
+        "should be 0%<limit>o, not 0%<actual>o",
         filename: filename, limit: limit, actual: actual
       )
       raise message

data/lib/imap/backup/version.rb

@@ -2,8 +2,8 @@ module Imap; end
 
 module Imap::Backup
   MAJOR    = 6
-  MINOR    = 0
+  MINOR    = 1
   REVISION = 0
-  PRE      = "rc2".freeze
+  PRE      = nil
   VERSION  = [MAJOR, MINOR, REVISION, PRE].compact.map(&:to_s).join(".")
 end

data/lib/imap/backup.rb

@@ -14,7 +14,6 @@ require "imap/backup/setup/asker"
 require "imap/backup/setup/connection_tester"
 require "imap/backup/setup/folder_chooser"
 require "imap/backup/version"
-require "email/provider"
 
 module Imap::Backup
   class ConfigurationNotFound < StandardError; end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: imap-backup
 version: !ruby/object:Gem::Version
-  version: 6.0.0.rc2
+  version: 6.1.0
 platform: ruby
 authors:
 - Joe Yates
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-03-04 00:00:00.000000000 Z
+date: 2022-07-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: highline
@@ -164,6 +164,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Backup GMail, or any other IMAP email service, to disk.
 email:
 - joe.g.yates@gmail.com
@@ -186,10 +200,14 @@ files:
 - lib/email/provider/base.rb
 - lib/email/provider/fastmail.rb
 - lib/email/provider/gmail.rb
+- lib/email/provider/purelymail.rb
 - lib/email/provider/unknown.rb
 - lib/imap/backup.rb
 - lib/imap/backup/account.rb
 - lib/imap/backup/account/connection.rb
+- lib/imap/backup/account/connection/backup_folders.rb
+- lib/imap/backup/account/connection/client_factory.rb
+- lib/imap/backup/account/connection/folder_names.rb
 - lib/imap/backup/account/folder.rb
 - lib/imap/backup/cli.rb
 - lib/imap/backup/cli/accounts.rb
@@ -201,6 +219,7 @@ files:
 - lib/imap/backup/cli/remote.rb
 - lib/imap/backup/cli/restore.rb
 - lib/imap/backup/cli/setup.rb
+- lib/imap/backup/cli/stats.rb
 - lib/imap/backup/cli/status.rb
 - lib/imap/backup/cli/utils.rb
 - lib/imap/backup/client/apple_mail.rb
@@ -211,14 +230,20 @@ files:
 - lib/imap/backup/migrator.rb
 - lib/imap/backup/sanitizer.rb
 - lib/imap/backup/serializer.rb
+- lib/imap/backup/serializer/appender.rb
 - lib/imap/backup/serializer/directory.rb
 - lib/imap/backup/serializer/imap.rb
 - lib/imap/backup/serializer/mbox.rb
 - lib/imap/backup/serializer/mbox_enumerator.rb
+- lib/imap/backup/serializer/message_enumerator.rb
+- lib/imap/backup/serializer/unused_name_finder.rb
 - lib/imap/backup/setup.rb
 - lib/imap/backup/setup/account.rb
+- lib/imap/backup/setup/account/header.rb
 - lib/imap/backup/setup/asker.rb
+- lib/imap/backup/setup/backup_path.rb
 - lib/imap/backup/setup/connection_tester.rb
+- lib/imap/backup/setup/email.rb
 - lib/imap/backup/setup/folder_chooser.rb
 - lib/imap/backup/setup/helpers.rb
 - lib/imap/backup/thunderbird/mailbox_exporter.rb
@@ -226,74 +251,11 @@ files:
 - lib/imap/backup/utils.rb
 - lib/imap/backup/version.rb
 - lib/retry_on_error.rb
-- spec/features/backup_spec.rb
-- spec/features/configuration/minimal_configuration.rb
-- spec/features/configuration/missing_configuration.rb
-- spec/features/folders_spec.rb
-- spec/features/helper.rb
-- spec/features/local/list_accounts_spec.rb
-- spec/features/local/list_emails_spec.rb
-- spec/features/local/list_folders_spec.rb
-- spec/features/local/show_an_email_spec.rb
-- spec/features/migrate_spec.rb
-- spec/features/remote/list_account_folders_spec.rb
-- spec/features/restore_spec.rb
-- spec/features/status_spec.rb
-- spec/features/support/aruba.rb
-- spec/features/support/backup_directory.rb
-- spec/features/support/email_server.rb
-- spec/features/support/shared/connection_context.rb
-- spec/features/support/shared/message_fixtures.rb
-- spec/fixtures/connection.yml
-- spec/spec_helper.rb
-- spec/support/fixtures.rb
-- spec/support/higline_test_helpers.rb
-- spec/support/silence_logging.rb
-- spec/unit/email/mboxrd/message_spec.rb
-- spec/unit/email/provider/apple_mail_spec.rb
-- spec/unit/email/provider/base_spec.rb
-- spec/unit/email/provider/fastmail_spec.rb
-- spec/unit/email/provider/gmail_spec.rb
-- spec/unit/email/provider_spec.rb
-- spec/unit/imap/backup/account/connection_spec.rb
-- spec/unit/imap/backup/account/folder_spec.rb
-- spec/unit/imap/backup/account_spec.rb
-- spec/unit/imap/backup/cli/accounts_spec.rb
-- spec/unit/imap/backup/cli/backup_spec.rb
-- spec/unit/imap/backup/cli/folders_spec.rb
-- spec/unit/imap/backup/cli/helpers_spec.rb
-- spec/unit/imap/backup/cli/local_spec.rb
-- spec/unit/imap/backup/cli/migrate_spec.rb
-- spec/unit/imap/backup/cli/restore_spec.rb
-- spec/unit/imap/backup/cli/setup_spec.rb
-- spec/unit/imap/backup/cli/utils_spec.rb
-- spec/unit/imap/backup/cli_spec.rb
-- spec/unit/imap/backup/client/apple_mail_spec.rb
-- spec/unit/imap/backup/client/default_spec.rb
-- spec/unit/imap/backup/configuration_spec.rb
-- spec/unit/imap/backup/downloader_spec.rb
-- spec/unit/imap/backup/logger_spec.rb
-- spec/unit/imap/backup/migrator_spec.rb
-- spec/unit/imap/backup/sanitizer_spec.rb
-- spec/unit/imap/backup/serializer/directory_spec.rb
-- spec/unit/imap/backup/serializer/imap_spec.rb
-- spec/unit/imap/backup/serializer/mbox_enumerator_spec.rb
-- spec/unit/imap/backup/serializer/mbox_spec.rb
-- spec/unit/imap/backup/serializer_spec.rb
-- spec/unit/imap/backup/setup/account_spec.rb
-- spec/unit/imap/backup/setup/asker_spec.rb
-- spec/unit/imap/backup/setup/connection_tester_spec.rb
-- spec/unit/imap/backup/setup/folder_chooser_spec.rb
-- spec/unit/imap/backup/setup/helpers_spec.rb
-- spec/unit/imap/backup/setup_spec.rb
-- spec/unit/imap/backup/thunderbird/mailbox_exporter_spec.rb
-- spec/unit/imap/backup/uploader_spec.rb
-- spec/unit/imap/backup/utils_spec.rb
-- spec/unit/retry_on_error_spec.rb
 homepage: https://github.com/joeyates/imap-backup
 licenses:
 - MIT
-metadata: {}
+metadata:
+  rubygems_mfa_required: 'true'
 post_install_message:
 rdoc_options: []
 require_paths:
@@ -305,76 +267,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.5'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Backup GMail (or other IMAP) accounts to disk
-test_files:
-- spec/spec_helper.rb
-- spec/unit/imap/backup/account_spec.rb
-- spec/unit/imap/backup/serializer/imap_spec.rb
-- spec/unit/imap/backup/serializer/directory_spec.rb
-- spec/unit/imap/backup/serializer/mbox_spec.rb
-- spec/unit/imap/backup/serializer/mbox_enumerator_spec.rb
-- spec/unit/imap/backup/cli_spec.rb
-- spec/unit/imap/backup/setup_spec.rb
-- spec/unit/imap/backup/logger_spec.rb
-- spec/unit/imap/backup/configuration_spec.rb
-- spec/unit/imap/backup/setup/account_spec.rb
-- spec/unit/imap/backup/setup/connection_tester_spec.rb
-- spec/unit/imap/backup/setup/asker_spec.rb
-- spec/unit/imap/backup/setup/helpers_spec.rb
-- spec/unit/imap/backup/setup/folder_chooser_spec.rb
-- spec/unit/imap/backup/cli/folders_spec.rb
-- spec/unit/imap/backup/cli/setup_spec.rb
-- spec/unit/imap/backup/cli/backup_spec.rb
-- spec/unit/imap/backup/cli/restore_spec.rb
-- spec/unit/imap/backup/cli/helpers_spec.rb
-- spec/unit/imap/backup/cli/accounts_spec.rb
-- spec/unit/imap/backup/cli/migrate_spec.rb
-- spec/unit/imap/backup/cli/utils_spec.rb
-- spec/unit/imap/backup/cli/local_spec.rb
-- spec/unit/imap/backup/downloader_spec.rb
-- spec/unit/imap/backup/migrator_spec.rb
-- spec/unit/imap/backup/uploader_spec.rb
-- spec/unit/imap/backup/account/folder_spec.rb
-- spec/unit/imap/backup/account/connection_spec.rb
-- spec/unit/imap/backup/utils_spec.rb
-- spec/unit/imap/backup/serializer_spec.rb
-- spec/unit/imap/backup/client/default_spec.rb
-- spec/unit/imap/backup/client/apple_mail_spec.rb
-- spec/unit/imap/backup/thunderbird/mailbox_exporter_spec.rb
-- spec/unit/imap/backup/sanitizer_spec.rb
-- spec/unit/retry_on_error_spec.rb
-- spec/unit/email/mboxrd/message_spec.rb
-- spec/unit/email/provider/gmail_spec.rb
-- spec/unit/email/provider/apple_mail_spec.rb
-- spec/unit/email/provider/fastmail_spec.rb
-- spec/unit/email/provider/base_spec.rb
-- spec/unit/email/provider_spec.rb
-- spec/features/folders_spec.rb
-- spec/features/configuration/missing_configuration.rb
-- spec/features/configuration/minimal_configuration.rb
-- spec/features/backup_spec.rb
-- spec/features/helper.rb
-- spec/features/restore_spec.rb
-- spec/features/local/list_emails_spec.rb
-- spec/features/local/list_accounts_spec.rb
-- spec/features/local/show_an_email_spec.rb
-- spec/features/local/list_folders_spec.rb
-- spec/features/status_spec.rb
-- spec/features/support/email_server.rb
-- spec/features/support/backup_directory.rb
-- spec/features/support/aruba.rb
-- spec/features/support/shared/connection_context.rb
-- spec/features/support/shared/message_fixtures.rb
-- spec/features/migrate_spec.rb
-- spec/features/remote/list_account_folders_spec.rb
-- spec/support/fixtures.rb
-- spec/support/silence_logging.rb
-- spec/support/higline_test_helpers.rb
-- spec/fixtures/connection.yml
+test_files: []

data/spec/features/backup_spec.rb

@@ -1,100 +0,0 @@
-require "features/helper"
-
-RSpec.describe "backup", type: :aruba, docker: true do
-  include_context "imap-backup connection"
-  include_context "message-fixtures"
-
-  let(:local_backup_path) { File.expand_path("~/backup") }
-  let(:backup_folders) { [{name: folder}] }
-  let(:folder) { "my-stuff" }
-  let(:messages_as_mbox) do
-    message_as_mbox_entry(msg1) + message_as_mbox_entry(msg2)
-  end
-
-  let!(:pre) {}
-  let!(:setup) do
-    server_create_folder folder
-    send_email folder, msg1
-    send_email folder, msg2
-    create_config(accounts: [account.to_h])
-
-    run_command_and_stop("imap-backup backup")
-  end
-
-  after do
-    server_delete_folder folder
-  end
-
-  it "downloads messages" do
-    expect(mbox_content(folder)).to eq(messages_as_mbox)
-  end
-
-  describe "IMAP metadata" do
-    let(:imap_metadata) { imap_parsed(folder) }
-    let(:folder_uids) { server_uids(folder) }
-
-    it "saves IMAP metadata in a JSON file" do
-      expect { imap_metadata }.to_not raise_error
-    end
-
-    it "saves a file version" do
-      expect(imap_metadata[:version].to_s).to match(/^[0-9.]$/)
-    end
-
-    it "records IMAP ids" do
-      expect(imap_metadata[:uids]).to eq(folder_uids)
-    end
-
-    it "records uid_validity" do
-      expect(imap_metadata[:uid_validity]).to eq(server_uid_validity(folder))
-    end
-
-    context "when uid_validity does not match" do
-      let(:new_name) { "NEWNAME" }
-      let(:original_folder_uid_validity) { server_uid_validity(folder) }
-      let!(:pre) do
-        server_create_folder folder
-        send_email folder, msg3
-        original_folder_uid_validity
-        connection.run_backup
-        connection.disconnect
-        server_rename_folder folder, new_name
-      end
-      let(:renamed_folder) { "#{folder}-#{original_folder_uid_validity}" }
-
-      after do
-        server_delete_folder new_name
-      end
-
-      it "renames the old backup" do
-        expect(mbox_content(renamed_folder)).to eq(message_as_mbox_entry(msg3))
-      end
-
-      it "renames the old metadata file" do
-        expect(imap_parsed(renamed_folder)).to be_a Hash
-      end
-
-      it "downloads messages" do
-        expect(mbox_content(folder)).to eq(messages_as_mbox)
-      end
-
-      it "creates a metadata file" do
-        expect(imap_parsed(folder)).to be_a Hash
-      end
-
-      context "when a renamed local backup exists" do
-        let!(:pre) do
-          super()
-          create_directory local_backup_path
-          File.write(imap_path(renamed_folder), "existing imap")
-          File.write(mbox_path(renamed_folder), "existing mbox")
-        end
-
-        it "moves the old backup to a uniquely named directory" do
-          renamed = "#{folder}-#{original_folder_uid_validity}-1"
-          expect(mbox_content(renamed)).to eq(message_as_mbox_entry(msg3))
-        end
-      end
-    end
-  end
-end

data/spec/features/configuration/minimal_configuration.rb

@@ -1,15 +0,0 @@
-require "features/helper"
-
-RSpec.describe "Running commands", type: :aruba do
-  before do
-    create_config(accounts: [])
-    run_command("imap-backup local accounts")
-    last_command_started.stop
-  end
-
-  context "when no accounts are configured" do
-    it "succeeds" do
-      expect(last_command_started).to have_exit_status(0)
-    end
-  end
-end

data/spec/features/configuration/missing_configuration.rb

@@ -1,14 +0,0 @@
-require "features/helper"
-
-RSpec.describe "Running commands", type: :aruba do
-  before do
-    run_command("imap-backup")
-    last_command_started.stop
-  end
-
-  context "when the configuration file is missing" do
-    it "fails" do
-      expect(last_command_started).not_to have_exit_status(0)
-    end
-  end
-end

data/spec/features/folders_spec.rb

@@ -1,36 +0,0 @@
-require "features/helper"
-require "imap/backup/cli/folders"
-
-RSpec.describe "folders", type: :feature, docker: true do
-  include_context "imap-backup connection"
-
-  let(:options) do
-    {accounts: "address@example.org"}
-  end
-  let(:folder) { "my-stuff" }
-  let(:output) { StringIO.new }
-
-  before do
-    allow(Imap::Backup::CLI::Accounts).to receive(:new) { [account] }
-    server_create_folder folder
-  end
-
-  around do |example|
-    stdout = $stdout
-    $stdout = output
-    example.run
-    $stdout = stdout
-  end
-
-  after do
-    FileUtils.rm_rf local_backup_path
-    server_delete_folder folder
-    connection.disconnect
-  end
-
-  it "lists account folders" do
-    Imap::Backup::CLI::Folders.new(options).run
-
-    expect(output.string).to match(/^\tmy-stuff\n/)
-  end
-end

data/spec/features/helper.rb

@@ -1,2 +0,0 @@
-support_glob = File.expand_path("support/**/*.rb", __dir__)
-Dir[support_glob].sort.each { |f| require f }