imap-backup 5.2.0 → 6.0.1

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: imap-backup
 version: !ruby/object:Gem::Version
-  version: 5.2.0
+  version: 6.0.1
 platform: ruby
 authors:
 - Joe Yates
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-02-24 00:00:00.000000000 Z
+date: 2022-07-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: highline
@@ -164,6 +164,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Backup GMail, or any other IMAP email service, to disk.
 email:
 - joe.g.yates@gmail.com
@@ -190,6 +204,9 @@ files:
 - lib/imap/backup.rb
 - lib/imap/backup/account.rb
 - lib/imap/backup/account/connection.rb
+- lib/imap/backup/account/connection/backup_folders.rb
+- lib/imap/backup/account/connection/client_factory.rb
+- lib/imap/backup/account/connection/folder_names.rb
 - lib/imap/backup/account/folder.rb
 - lib/imap/backup/cli.rb
 - lib/imap/backup/cli/accounts.rb
@@ -201,6 +218,7 @@ files:
 - lib/imap/backup/cli/remote.rb
 - lib/imap/backup/cli/restore.rb
 - lib/imap/backup/cli/setup.rb
+- lib/imap/backup/cli/stats.rb
 - lib/imap/backup/cli/status.rb
 - lib/imap/backup/cli/utils.rb
 - lib/imap/backup/client/apple_mail.rb
@@ -211,13 +229,20 @@ files:
 - lib/imap/backup/migrator.rb
 - lib/imap/backup/sanitizer.rb
 - lib/imap/backup/serializer.rb
+- lib/imap/backup/serializer/appender.rb
+- lib/imap/backup/serializer/directory.rb
+- lib/imap/backup/serializer/imap.rb
 - lib/imap/backup/serializer/mbox.rb
 - lib/imap/backup/serializer/mbox_enumerator.rb
-- lib/imap/backup/serializer/mbox_store.rb
+- lib/imap/backup/serializer/message_enumerator.rb
+- lib/imap/backup/serializer/unused_name_finder.rb
 - lib/imap/backup/setup.rb
 - lib/imap/backup/setup/account.rb
+- lib/imap/backup/setup/account/header.rb
 - lib/imap/backup/setup/asker.rb
+- lib/imap/backup/setup/backup_path.rb
 - lib/imap/backup/setup/connection_tester.rb
+- lib/imap/backup/setup/email.rb
 - lib/imap/backup/setup/folder_chooser.rb
 - lib/imap/backup/setup/helpers.rb
 - lib/imap/backup/thunderbird/mailbox_exporter.rb
@@ -225,60 +250,11 @@ files:
 - lib/imap/backup/utils.rb
 - lib/imap/backup/version.rb
 - lib/retry_on_error.rb
-- spec/features/backup_spec.rb
-- spec/features/configuration/minimal_configuration.rb
-- spec/features/configuration/missing_configuration.rb
-- spec/features/folders_spec.rb
-- spec/features/helper.rb
-- spec/features/local/list_accounts_spec.rb
-- spec/features/local/list_emails_spec.rb
-- spec/features/local/list_folders_spec.rb
-- spec/features/local/show_an_email_spec.rb
-- spec/features/migrate_spec.rb
-- spec/features/remote/list_account_folders_spec.rb
-- spec/features/restore_spec.rb
-- spec/features/status_spec.rb
-- spec/features/support/aruba.rb
-- spec/features/support/backup_directory.rb
-- spec/features/support/email_server.rb
-- spec/features/support/shared/connection_context.rb
-- spec/features/support/shared/message_fixtures.rb
-- spec/fixtures/connection.yml
-- spec/spec_helper.rb
-- spec/support/fixtures.rb
-- spec/support/higline_test_helpers.rb
-- spec/support/silence_logging.rb
-- spec/unit/email/mboxrd/message_spec.rb
-- spec/unit/email/provider/apple_mail_spec.rb
-- spec/unit/email/provider/base_spec.rb
-- spec/unit/email/provider/fastmail_spec.rb
-- spec/unit/email/provider/gmail_spec.rb
-- spec/unit/email/provider_spec.rb
-- spec/unit/imap/backup/account/connection_spec.rb
-- spec/unit/imap/backup/account/folder_spec.rb
-- spec/unit/imap/backup/cli/accounts_spec.rb
-- spec/unit/imap/backup/cli/helpers_spec.rb
-- spec/unit/imap/backup/cli/local_spec.rb
-- spec/unit/imap/backup/cli/utils_spec.rb
-- spec/unit/imap/backup/client/default_spec.rb
-- spec/unit/imap/backup/configuration_spec.rb
-- spec/unit/imap/backup/downloader_spec.rb
-- spec/unit/imap/backup/logger_spec.rb
-- spec/unit/imap/backup/migrator_spec.rb
-- spec/unit/imap/backup/serializer/mbox_enumerator_spec.rb
-- spec/unit/imap/backup/serializer/mbox_spec.rb
-- spec/unit/imap/backup/serializer/mbox_store_spec.rb
-- spec/unit/imap/backup/setup/account_spec.rb
-- spec/unit/imap/backup/setup/asker_spec.rb
-- spec/unit/imap/backup/setup/connection_tester_spec.rb
-- spec/unit/imap/backup/setup/folder_chooser_spec.rb
-- spec/unit/imap/backup/setup_spec.rb
-- spec/unit/imap/backup/uploader_spec.rb
-- spec/unit/imap/backup/utils_spec.rb
 homepage: https://github.com/joeyates/imap-backup
 licenses:
 - MIT
-metadata: {}
+metadata:
+  rubygems_mfa_required: 'true'
 post_install_message:
 rdoc_options: []
 require_paths:
@@ -298,54 +274,4 @@ rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Backup GMail (or other IMAP) accounts to disk
-test_files:
-- spec/spec_helper.rb
-- spec/unit/imap/backup/serializer/mbox_spec.rb
-- spec/unit/imap/backup/serializer/mbox_store_spec.rb
-- spec/unit/imap/backup/serializer/mbox_enumerator_spec.rb
-- spec/unit/imap/backup/setup_spec.rb
-- spec/unit/imap/backup/logger_spec.rb
-- spec/unit/imap/backup/configuration_spec.rb
-- spec/unit/imap/backup/setup/account_spec.rb
-- spec/unit/imap/backup/setup/connection_tester_spec.rb
-- spec/unit/imap/backup/setup/asker_spec.rb
-- spec/unit/imap/backup/setup/folder_chooser_spec.rb
-- spec/unit/imap/backup/cli/helpers_spec.rb
-- spec/unit/imap/backup/cli/accounts_spec.rb
-- spec/unit/imap/backup/cli/utils_spec.rb
-- spec/unit/imap/backup/cli/local_spec.rb
-- spec/unit/imap/backup/downloader_spec.rb
-- spec/unit/imap/backup/migrator_spec.rb
-- spec/unit/imap/backup/uploader_spec.rb
-- spec/unit/imap/backup/account/folder_spec.rb
-- spec/unit/imap/backup/account/connection_spec.rb
-- spec/unit/imap/backup/utils_spec.rb
-- spec/unit/imap/backup/client/default_spec.rb
-- spec/unit/email/mboxrd/message_spec.rb
-- spec/unit/email/provider/gmail_spec.rb
-- spec/unit/email/provider/apple_mail_spec.rb
-- spec/unit/email/provider/fastmail_spec.rb
-- spec/unit/email/provider/base_spec.rb
-- spec/unit/email/provider_spec.rb
-- spec/features/folders_spec.rb
-- spec/features/configuration/missing_configuration.rb
-- spec/features/configuration/minimal_configuration.rb
-- spec/features/backup_spec.rb
-- spec/features/helper.rb
-- spec/features/restore_spec.rb
-- spec/features/local/list_emails_spec.rb
-- spec/features/local/list_accounts_spec.rb
-- spec/features/local/show_an_email_spec.rb
-- spec/features/local/list_folders_spec.rb
-- spec/features/status_spec.rb
-- spec/features/support/email_server.rb
-- spec/features/support/backup_directory.rb
-- spec/features/support/aruba.rb
-- spec/features/support/shared/connection_context.rb
-- spec/features/support/shared/message_fixtures.rb
-- spec/features/migrate_spec.rb
-- spec/features/remote/list_account_folders_spec.rb
-- spec/support/fixtures.rb
-- spec/support/silence_logging.rb
-- spec/support/higline_test_helpers.rb
-- spec/fixtures/connection.yml
+test_files: []

data/lib/imap/backup/serializer/mbox_store.rb

@@ -1,217 +0,0 @@
-require "json"
-
-require "email/mboxrd/message"
-require "imap/backup/serializer/mbox_enumerator"
-
-module Imap::Backup
-  class Serializer::MboxStore
-    CURRENT_VERSION = 2
-
-    attr_reader :folder
-    attr_reader :path
-    attr_reader :loaded
-
-    def initialize(path, folder)
-      @path = path
-      @folder = folder
-      @loaded = false
-      @uids = nil
-      @uid_validity = nil
-    end
-
-    def exist?
-      mbox_exist? && imap_exist?
-    end
-
-    def uid_validity
-      do_load if !loaded
-      @uid_validity
-    end
-
-    def uid_validity=(value)
-      do_load if !loaded
-      @uid_validity = value
-      @uids ||= []
-      write_imap_file
-    end
-
-    def uids
-      do_load if !loaded
-      @uids || []
-    end
-
-    def add(uid, body)
-      do_load if !loaded
-      raise "Can't add messages without uid_validity" if !uid_validity
-
-      uid = uid.to_i
-      if uids.include?(uid)
-        Imap::Backup::Logger.logger.debug(
-          "[#{folder}] message #{uid} already downloaded - skipping"
-        )
-        return
-      end
-
-      mboxrd_message = Email::Mboxrd::Message.new(body)
-      mbox = nil
-      begin
-        mbox = File.open(mbox_pathname, "ab")
-        mbox.write mboxrd_message.to_serialized
-        @uids << uid
-        write_imap_file
-      rescue StandardError => e
-        message = <<-ERROR.gsub(/^\s*/m, "")
-          [#{folder}] failed to save message #{uid}:
-          #{body}. #{e}:
-          #{e.backtrace.join("\n")}"
-        ERROR
-        Imap::Backup::Logger.logger.warn message
-      ensure
-        mbox&.close
-      end
-    end
-
-    def load(uid_maybe_string)
-      do_load if !loaded
-      uid = uid_maybe_string.to_i
-      message_index = uids.find_index(uid)
-      return nil if message_index.nil?
-
-      load_nth(message_index)
-    end
-
-    def each_message(required_uids)
-      return enum_for(:each_message, required_uids) if !block_given?
-
-      indexes = required_uids.each.with_object({}) do |uid_maybe_string, acc|
-        uid = uid_maybe_string.to_i
-        index = uids.find_index(uid)
-        acc[index] = uid if index
-      end
-      enumerator = Serializer::MboxEnumerator.new(mbox_pathname)
-      enumerator.each.with_index do |raw, i|
-        uid = indexes[i]
-        next if !uid
-
-        yield uid, Email::Mboxrd::Message.from_serialized(raw)
-      end
-    end
-
-    def update_uid(old, new)
-      index = uids.find_index(old.to_i)
-      return if index.nil?
-
-      uids[index] = new.to_i
-      write_imap_file
-    end
-
-    def reset
-      @uids = nil
-      @uid_validity = nil
-      @loaded = false
-      delete_files
-      write_blank_mbox_file
-    end
-
-    def rename(new_name)
-      new_mbox_pathname = absolute_path("#{new_name}.mbox")
-      new_imap_pathname = absolute_path("#{new_name}.imap")
-      File.rename(mbox_pathname, new_mbox_pathname)
-      File.rename(imap_pathname, new_imap_pathname)
-      @folder = new_name
-    end
-
-    def mbox_pathname
-      absolute_path("#{folder}.mbox")
-    end
-
-    def imap_pathname
-      absolute_path("#{folder}.imap")
-    end
-
-    private
-
-    def do_load
-      data = imap_data
-      if data
-        @uids = data[:uids].map(&:to_i)
-        @uid_validity = data[:uid_validity]
-        @loaded = true
-      else
-        reset
-      end
-    end
-
-    def imap_data
-      return nil if !imap_ok?
-
-      imap_data = nil
-
-      begin
-        imap_data = JSON.parse(File.read(imap_pathname), symbolize_names: true)
-      rescue JSON::ParserError
-        return nil
-      end
-
-      return nil if !imap_data.key?(:uids)
-      return nil if !imap_data[:uids].is_a?(Array)
-
-      imap_data
-    end
-
-    def imap_ok?
-      return false if !exist?
-      return false if !imap_looks_like_json?
-
-      true
-    end
-
-    def load_nth(index)
-      enumerator = Serializer::MboxEnumerator.new(mbox_pathname)
-      enumerator.each.with_index do |raw, i|
-        next if i != index
-
-        return Email::Mboxrd::Message.from_serialized(raw)
-      end
-      nil
-    end
-
-    def imap_looks_like_json?
-      return false if !imap_exist?
-
-      content = File.read(imap_pathname)
-      content.start_with?("{")
-    end
-
-    def write_imap_file
-      imap_data = {
-        version: CURRENT_VERSION,
-        uid_validity: @uid_validity,
-        uids: @uids
-      }
-      content = imap_data.to_json
-      File.open(imap_pathname, "w") { |f| f.write content }
-    end
-
-    def write_blank_mbox_file
-      File.open(mbox_pathname, "w") { |f| f.write "" }
-    end
-
-    def delete_files
-      File.unlink(imap_pathname) if imap_exist?
-      File.unlink(mbox_pathname) if mbox_exist?
-    end
-
-    def mbox_exist?
-      File.exist?(mbox_pathname)
-    end
-
-    def imap_exist?
-      File.exist?(imap_pathname)
-    end
-
-    def absolute_path(relative_path)
-      File.join(path, relative_path)
-    end
-  end
-end

data/spec/features/backup_spec.rb

@@ -1,108 +0,0 @@
-require "features/helper"
-
-RSpec.describe "backup", type: :aruba, docker: true do
-  include_context "imap-backup connection"
-  include_context "message-fixtures"
-
-  let(:local_backup_path) { File.expand_path("~/backup") }
-  let(:backup_folders) { [{name: folder}] }
-  let(:folder) { "my-stuff" }
-  let(:messages_as_mbox) do
-    message_as_mbox_entry(msg1) + message_as_mbox_entry(msg2)
-  end
-
-  let!(:pre) {}
-  let!(:setup) do
-    server_create_folder folder
-    send_email folder, msg1
-    send_email folder, msg2
-    create_config(accounts: [account.to_h])
-
-    run_command_and_stop("imap-backup backup")
-  end
-
-  after do
-    server_delete_folder folder
-  end
-
-  it "downloads messages" do
-    expect(mbox_content(folder)).to eq(messages_as_mbox)
-  end
-
-  describe "IMAP metadata" do
-    let(:imap_metadata) { imap_parsed(folder) }
-    let(:folder_uids) { server_uids(folder) }
-
-    it "saves IMAP metadata in a JSON file" do
-      expect { imap_metadata }.to_not raise_error
-    end
-
-    it "saves a file version" do
-      expect(imap_metadata[:version].to_s).to match(/^[0-9.]$/)
-    end
-
-    it "records IMAP ids" do
-      expect(imap_metadata[:uids]).to eq(folder_uids)
-    end
-
-    it "records uid_validity" do
-      expect(imap_metadata[:uid_validity]).to eq(server_uid_validity(folder))
-    end
-
-    context "when uid_validity does not match" do
-      let(:new_name) { "NEWNAME" }
-      let(:original_folder_uid_validity) { server_uid_validity(folder) }
-      let!(:pre) do
-        server_create_folder folder
-        send_email folder, msg3
-        original_folder_uid_validity
-        connection.run_backup
-        connection.disconnect
-        server_rename_folder folder, new_name
-      end
-      let(:renamed_folder) { "#{folder}-#{original_folder_uid_validity}" }
-
-      after do
-        server_delete_folder new_name
-      end
-
-      it "renames the old backup" do
-        expect(mbox_content(renamed_folder)).to eq(message_as_mbox_entry(msg3))
-      end
-
-      it "downloads messages" do
-        expect(mbox_content(folder)).to eq(messages_as_mbox)
-      end
-
-      context "when a renamed local backup exists" do
-        let!(:pre) do
-          super()
-          create_directory local_backup_path
-          File.write(imap_path(renamed_folder), "existing imap")
-          File.write(mbox_path(renamed_folder), "existing mbox")
-        end
-
-        it "moves the old backup to a uniquely named directory" do
-          renamed = "#{folder}-#{original_folder_uid_validity}-1"
-          expect(mbox_content(renamed)).to eq(message_as_mbox_entry(msg3))
-        end
-      end
-    end
-
-    context "when an unversioned .imap file is found" do
-      let!(:pre) do
-        create_directory local_backup_path
-        File.open(imap_path(folder), "w") { |f| f.write "old format imap" }
-        File.open(mbox_path(folder), "w") { |f| f.write "old format emails" }
-      end
-
-      it "replaces the .imap file with a versioned JSON file" do
-        expect(imap_metadata[:uids]).to eq(folder_uids)
-      end
-
-      it "does the download" do
-        expect(mbox_content(folder)).to eq(messages_as_mbox)
-      end
-    end
-  end
-end

data/spec/features/configuration/minimal_configuration.rb

@@ -1,15 +0,0 @@
-require "features/helper"
-
-RSpec.describe "Running commands", type: :aruba do
-  before do
-    create_config(accounts: [])
-    run_command("imap-backup local accounts")
-    last_command_started.stop
-  end
-
-  context "when no accounts are configured" do
-    it "succeeds" do
-      expect(last_command_started).to have_exit_status(0)
-    end
-  end
-end

data/spec/features/configuration/missing_configuration.rb

@@ -1,14 +0,0 @@
-require "features/helper"
-
-RSpec.describe "Running commands", type: :aruba do
-  before do
-    run_command("imap-backup")
-    last_command_started.stop
-  end
-
-  context "when the configuration file is missing" do
-    it "fails" do
-      expect(last_command_started).not_to have_exit_status(0)
-    end
-  end
-end

data/spec/features/folders_spec.rb

@@ -1,36 +0,0 @@
-require "features/helper"
-require "imap/backup/cli/folders"
-
-RSpec.describe "folders", type: :feature, docker: true do
-  include_context "imap-backup connection"
-
-  let(:options) do
-    {accounts: "address@example.org"}
-  end
-  let(:folder) { "my-stuff" }
-  let(:output) { StringIO.new }
-
-  before do
-    allow(Imap::Backup::CLI::Accounts).to receive(:new) { [account] }
-    server_create_folder folder
-  end
-
-  around do |example|
-    stdout = $stdout
-    $stdout = output
-    example.run
-    $stdout = stdout
-  end
-
-  after do
-    FileUtils.rm_rf local_backup_path
-    server_delete_folder folder
-    connection.disconnect
-  end
-
-  it "lists account folders" do
-    Imap::Backup::CLI::Folders.new(options).run
-
-    expect(output.string).to match(/^\tmy-stuff\n/)
-  end
-end

data/spec/features/helper.rb

@@ -1,2 +0,0 @@
-support_glob = File.expand_path("support/**/*.rb", __dir__)
-Dir[support_glob].sort.each { |f| require f }

data/spec/features/local/list_accounts_spec.rb

@@ -1,12 +0,0 @@
-require "features/helper"
-
-RSpec.describe "Listing accounts", type: :aruba do
-  before do
-    create_config(accounts: [{"username": "me@example.com"}])
-    run_command_and_stop("imap-backup local accounts")
-  end
-
-  it "lists accounts" do
-    expect(last_command_started).to have_output("me@example.com")
-  end
-end

data/spec/features/local/list_emails_spec.rb

@@ -1,21 +0,0 @@
-require "features/helper"
-
-RSpec.describe "Listing emails", type: :aruba do
-  let(:email) { "me@example.com" }
-  let(:account) do
-    {
-      username: email,
-      local_path: File.join(config_path, email.gsub("@", "_"))
-    }
-  end
-
-  before do
-    create_config(accounts: [account])
-    store_email(email: email, folder: "my_folder", subject: "Ciao")
-    run_command_and_stop("imap-backup local list #{email} my_folder")
-  end
-
-  it "lists emails" do
-    expect(last_command_started).to have_output(/1: Ciao/)
-  end
-end

data/spec/features/local/list_folders_spec.rb

@@ -1,21 +0,0 @@
-require "features/helper"
-
-RSpec.describe "Listing account folders", type: :aruba do
-  let(:email) { "me@example.com" }
-  let(:account) do
-    {
-      username: email,
-      local_path: File.join(config_path, email.gsub("@", "_"))
-    }
-  end
-
-  before do
-    create_config(accounts: [account])
-    store_email(email: email, folder: "my_folder", body: "Hi")
-    run_command_and_stop("imap-backup local folders #{email}")
-  end
-
-  it "lists folders that have been backed up" do
-    expect(last_command_started).to have_output(%q("my_folder"))
-  end
-end

data/spec/features/local/show_an_email_spec.rb

@@ -1,34 +0,0 @@
-require "features/helper"
-
-RSpec.describe "Show an email", type: :aruba do
-  let(:email) { "me@example.com" }
-  let(:account) do
-    {
-      username: email,
-      local_path: File.join(config_path, email.gsub("@", "_"))
-    }
-  end
-
-  before do
-    create_config(accounts: [account])
-    store_email(
-      email: email,
-      folder: "my_folder",
-      uid: 99,
-      from: "me@example.com",
-      subject: "Hello",
-      body: "How're things?"
-    )
-    run_command_and_stop("imap-backup local show #{email} my_folder 99")
-  end
-
-  it "shows the email" do
-    expected = <<~BODY
-      From: me@example.com
-      Subject: Hello
-      
-      How're things?
-    BODY
-    expect(last_command_started).to have_output(expected)
-  end
-end