imap-backup 5.2.0 → 6.0.0.rc2

data/lib/imap/backup/serializer/mbox.rb

@@ -1,115 +1,44 @@
-require "forwardable"
-
-require "imap/backup/serializer/mbox_store"
-
 module Imap::Backup
   class Serializer::Mbox
-    extend Forwardable
-    def_delegators :store, :mbox_pathname
+    attr_reader :folder_path
 
-    attr_reader :path
-    attr_reader :folder
-
-    def initialize(path, folder)
-      @path = path
-      @folder = folder
+    def initialize(folder_path)
+      @folder_path = folder_path
     end
 
-    def apply_uid_validity(value)
-      case
-      when store.uid_validity.nil?
-        store.uid_validity = value
-        nil
-      when store.uid_validity == value
-        # NOOP
-        nil
-      else
-        apply_new_uid_validity value
+    def append(message)
+      File.open(pathname, "ab") do |file|
+        file.write message
       end
     end
 
-    def force_uid_validity(value)
-      store.uid_validity = value
+    def exist?
+      File.exist?(pathname)
     end
 
-    def uids
-      store.uids
-    end
-
-    def load(uid)
-      store.load(uid)
-    end
-
-    def each_message(uids)
-      store.each_message(uids)
-    end
-
-    def save(uid, message)
-      store.add(uid, message)
-    end
-
-    def rename(new_name)
-      @folder = new_name
-      store.rename new_name
-    end
+    def length
+      return nil if !exist?
 
-    def update_uid(old, new)
-      store.update_uid old, new
+      File.stat(pathname).size
     end
 
-    private
-
-    def store
-      @store ||=
-        begin
-          create_containing_directory
-          Serializer::MboxStore.new(path, folder)
-        end
+    def pathname
+      "#{folder_path}.mbox"
     end
 
-    def apply_new_uid_validity(value)
-      digit = 0
-      new_name = nil
-      loop do
-        extra = digit.zero? ? "" : "-#{digit}"
-        new_name = "#{folder}-#{store.uid_validity}#{extra}"
-        test_store = Serializer::MboxStore.new(path, new_name)
-        break if !test_store.exist?
-
-        digit += 1
+    def rename(new_path)
+      if exist?
+        old_pathname = pathname
+        @folder_path = new_path
+        File.rename(old_pathname, pathname)
+      else
+        @folder_path = new_path
       end
-      rename_store new_name, value
-    end
-
-    def rename_store(new_name, value)
-      store.rename new_name
-      @store = nil
-      store.uid_validity = value
-      new_name
-    end
-
-    def relative_path
-      File.dirname(folder)
     end
 
-    def containing_directory
-      File.join(path, relative_path)
-    end
-
-    def full_path
-      File.expand_path(containing_directory)
-    end
-
-    def create_containing_directory
-      if !File.directory?(full_path)
-        Utils.make_folder(
-          path, relative_path, Serializer::DIRECTORY_PERMISSIONS
-        )
-      end
-
-      if Utils.mode(full_path) !=
-         Serializer::DIRECTORY_PERMISSIONS
-        FileUtils.chmod Serializer::DIRECTORY_PERMISSIONS, full_path
+    def rewind(length)
+      File.open(pathname, File::RDWR | File::CREAT, 0o644) do |f|
+        f.truncate(length)
       end
     end
   end

data/lib/imap/backup/serializer/mbox_enumerator.rb

@@ -1,4 +1,6 @@
 module Imap::Backup
+  class Serializer; end
+
   class Serializer::MboxEnumerator
     attr_reader :mbox_pathname
 

data/lib/imap/backup/serializer.rb

@@ -1,6 +1,183 @@
+require "forwardable"
+
+require "email/mboxrd/message"
+require "imap/backup/serializer/imap"
+require "imap/backup/serializer/mbox"
+require "imap/backup/serializer/mbox_enumerator"
+
 module Imap::Backup
-  module Serializer
-    DIRECTORY_PERMISSIONS = 0o700
-    FILE_PERMISSIONS      = 0o600
+  class Serializer
+    extend Forwardable
+
+    def_delegator :mbox, :pathname, :mbox_pathname
+    def_delegators :imap, :uid_validity, :uids, :update_uid
+
+    attr_reader :folder
+    attr_reader :path
+
+    def initialize(path, folder)
+      @path = path
+      @folder = folder
+    end
+
+    def apply_uid_validity(value)
+      case
+      when uid_validity.nil?
+        imap.uid_validity = value
+        nil
+      when uid_validity == value
+        # NOOP
+        nil
+      else
+        apply_new_uid_validity value
+      end
+    end
+
+    def force_uid_validity(value)
+      imap.uid_validity = value
+    end
+
+    def append(uid, message)
+      raise "Can't add messages without uid_validity" if !imap.uid_validity
+
+      uid = uid.to_i
+      if imap.include?(uid)
+        Logger.logger.debug(
+          "[#{folder}] message #{uid} already downloaded - skipping"
+        )
+        return
+      end
+
+      do_append uid, message
+    end
+
+    def load(uid_maybe_string)
+      uid = uid_maybe_string.to_i
+      message_index = imap.index(uid)
+      return nil if message_index.nil?
+
+      load_nth(message_index)
+    end
+
+    def load_nth(index)
+      enumerator = Serializer::MboxEnumerator.new(mbox.pathname)
+      enumerator.each.with_index do |raw, i|
+        next if i != index
+
+        return Email::Mboxrd::Message.from_serialized(raw)
+      end
+      nil
+    end
+
+    def each_message(required_uids)
+      return enum_for(:each_message, required_uids) if !block_given?
+
+      indexes = required_uids.each.with_object({}) do |uid_maybe_string, acc|
+        uid = uid_maybe_string.to_i
+        index = imap.index(uid)
+        acc[index] = uid if index
+      end
+      enumerator = Serializer::MboxEnumerator.new(mbox.pathname)
+      enumerator.each.with_index do |raw, i|
+        uid = indexes[i]
+        next if !uid
+
+        yield uid, Email::Mboxrd::Message.from_serialized(raw)
+      end
+    end
+
+    def rename(new_name)
+      # Initialize so we get memoized instances with the correct folder_path
+      mbox
+      imap
+      @folder = new_name
+      ensure_containing_directory
+      mbox.rename folder_path
+      imap.rename folder_path
+    end
+
+    private
+
+    def do_append(uid, message)
+      mboxrd_message = Email::Mboxrd::Message.new(message)
+      initial = mbox.length || 0
+      mbox_appended = false
+      begin
+        mbox.append mboxrd_message.to_serialized
+        mbox_appended = true
+        imap.append uid
+      rescue StandardError => e
+        mbox.rewind(initial) if mbox_appended
+
+        message = <<-ERROR.gsub(/^\s*/m, "")
+          [#{folder}] failed to append message #{uid}:
+          #{message}. #{e}:
+          #{e.backtrace.join("\n")}"
+        ERROR
+        Logger.logger.warn message
+      end
+    end
+
+    def mbox
+      @mbox ||=
+        begin
+          ensure_containing_directory
+          Serializer::Mbox.new(folder_path)
+        end
+    end
+
+    def imap
+      @imap ||=
+        begin
+          ensure_containing_directory
+          Serializer::Imap.new(folder_path)
+        end
+    end
+
+    def folder_path
+      folder_path_for(path, folder)
+    end
+
+    def folder_path_for(path, folder)
+      relative = File.join(path, folder)
+      File.expand_path(relative)
+    end
+
+    def ensure_containing_directory
+      relative = File.dirname(folder)
+      directory = Serializer::Directory.new(path, relative)
+      directory.ensure_exists
+    end
+
+    def apply_new_uid_validity(value)
+      new_name = rename_existing_folder
+      # Clear memoization so we get empty data
+      @mbox = nil
+      @imap = nil
+      imap.uid_validity = value
+
+      new_name
+    end
+
+    def rename_existing_folder
+      digit = 0
+      new_name = nil
+      loop do
+        extra = digit.zero? ? "" : "-#{digit}"
+        new_name = "#{folder}-#{imap.uid_validity}#{extra}"
+        new_folder_path = folder_path_for(path, new_name)
+        test_mbox = Serializer::Mbox.new(new_folder_path)
+        test_imap = Serializer::Imap.new(new_folder_path)
+        break if !test_mbox.exist? && !test_imap.exist?
+
+        digit += 1
+      end
+
+      previous = folder
+      rename(new_name)
+      @folder = previous
+
+      new_name
+    end
   end
 end

data/lib/imap/backup/setup/account.rb

@@ -26,6 +26,7 @@ module Imap::Backup
         modify_password menu
         modify_backup_path menu
         choose_folders menu
+        modify_multi_fetch_size menu
         modify_server menu
         modify_connection_options menu
         test_connection menu
@@ -37,21 +38,30 @@ module Imap::Backup
 
     def header(menu)
       modified = account.modified? ? "*" : ""
-      connection_options =
-        if account.connection_options
-          escaped =
-            JSON.generate(account.connection_options).
-            gsub('"', '\"')
-          "\n  connection options #{escaped}"
-        end
+
+      if account.multi_fetch_size > 1
+        multi_fetch_size = "\nmulti-fetch #{account.multi_fetch_size}"
+      end
+
+      if account.connection_options
+        escaped =
+          JSON.generate(account.connection_options)
+        connection_options =
+          "\nconnection options  '#{escaped}'"
+        space = " " * 12
+      else
+        connection_options = nil
+        space = " " * 4
+      end
+
       menu.header = <<~HEADER.chomp
         #{helpers.title_prefix} Account#{modified}
 
-        email      #{account.username}
-        password   #{masked_password}
-        path       #{account.local_path}
-        folders    #{folders.map { |f| f[:name] }.join(', ')}
-        server     #{account.server}#{connection_options}
+        email   #{space}#{account.username}
+        password#{space}#{masked_password}
+        path    #{space}#{account.local_path}
+        folders #{space}#{folders.map { |f| f[:name] }.join(', ')}#{multi_fetch_size}
+        server  #{space}#{account.server}#{connection_options}
 
         Choose an action
       HEADER
@@ -70,12 +80,10 @@ module Imap::Backup
           )
         else
           account.username = username
-          # rubocop:disable Style/IfUnlessModifier
           default = default_server(username)
           if default && (account.server.nil? || (account.server == ""))
             account.server = default
           end
-          # rubocop:enable Style/IfUnlessModifier
         end
       end
     end
@@ -88,20 +96,6 @@ module Imap::Backup
       end
     end
 
-    def modify_server(menu)
-      menu.choice("modify server") do
-        server = highline.ask("server: ")
-        account.server = server if !server.nil?
-      end
-    end
-
-    def modify_connection_options(menu)
-      menu.choice("modify connection options") do
-        connection_options = highline.ask("connections options (as JSON): ")
-        account.connection_options = connection_options if !connection_options.nil?
-      end
-    end
-
     def path_modification_validator(path)
       same = config.accounts.find do |a|
         a.username != account.username && a.local_path == path
@@ -130,6 +124,35 @@ module Imap::Backup
       end
     end
 
+    def modify_multi_fetch_size(menu)
+      menu.choice("modify multi-fetch size (number of emails to fetch at a time)") do
+        size = highline.ask("size: ")
+        int = size.to_i
+        account.multi_fetch_size = int if int.positive?
+      end
+    end
+
+    def modify_server(menu)
+      menu.choice("modify server") do
+        server = highline.ask("server: ")
+        account.server = server if !server.nil?
+      end
+    end
+
+    def modify_connection_options(menu)
+      menu.choice("modify connection options") do
+        connection_options = highline.ask("connections options (as JSON): ")
+        if !connection_options.nil?
+          begin
+            account.connection_options = connection_options
+          rescue JSON::ParserError
+            Kernel.puts "Malformed JSON, please try again"
+            highline.ask "Press a key "
+          end
+        end
+      end
+    end
+
     def test_connection(menu)
       menu.choice("test connection") do
         result = Setup::ConnectionTester.new(account).test
@@ -141,7 +164,7 @@ module Imap::Backup
     def delete_account(menu)
       menu.choice("delete") do
         if highline.agree("Are you sure? (y/n) ")
-          account.mark_for_deletion!
+          account.mark_for_deletion
           throw :done
         end
       end

data/lib/imap/backup/setup/helpers.rb

@@ -9,7 +9,7 @@ module Imap::Backup
     end
 
     def version
-      Version::VERSION
+      VERSION
     end
   end
 end

data/lib/imap/backup/thunderbird/mailbox_exporter.rb

@@ -19,7 +19,7 @@ module Imap::Backup
 
     def run
       local_folder_ok = local_folder.set_up
-      return if !local_folder_ok
+      return false if !local_folder_ok
 
       if local_folder.msf_exists?
         if force

data/lib/imap/backup/version.rb

@@ -1,9 +1,9 @@
 module Imap; end
 
 module Imap::Backup
-  MAJOR    = 5
-  MINOR    = 2
+  MAJOR    = 6
+  MINOR    = 0
   REVISION = 0
-  PRE      = nil
+  PRE      = "rc2".freeze
   VERSION  = [MAJOR, MINOR, REVISION, PRE].compact.map(&:to_s).join(".")
 end

data/lib/imap/backup.rb

@@ -8,7 +8,6 @@ require "imap/backup/downloader"
 require "imap/backup/logger"
 require "imap/backup/uploader"
 require "imap/backup/serializer"
-require "imap/backup/serializer/mbox"
 require "imap/backup/setup"
 require "imap/backup/setup/account"
 require "imap/backup/setup/asker"

data/spec/features/backup_spec.rb

@@ -70,10 +70,18 @@ RSpec.describe "backup", type: :aruba, docker: true do
         expect(mbox_content(renamed_folder)).to eq(message_as_mbox_entry(msg3))
       end
 
+      it "renames the old metadata file" do
+        expect(imap_parsed(renamed_folder)).to be_a Hash
+      end
+
       it "downloads messages" do
         expect(mbox_content(folder)).to eq(messages_as_mbox)
       end
 
+      it "creates a metadata file" do
+        expect(imap_parsed(folder)).to be_a Hash
+      end
+
       context "when a renamed local backup exists" do
         let!(:pre) do
           super()
@@ -88,21 +96,5 @@ RSpec.describe "backup", type: :aruba, docker: true do
         end
       end
     end
-
-    context "when an unversioned .imap file is found" do
-      let!(:pre) do
-        create_directory local_backup_path
-        File.open(imap_path(folder), "w") { |f| f.write "old format imap" }
-        File.open(mbox_path(folder), "w") { |f| f.write "old format emails" }
-      end
-
-      it "replaces the .imap file with a versioned JSON file" do
-        expect(imap_metadata[:uids]).to eq(folder_uids)
-      end
-
-      it "does the download" do
-        expect(mbox_content(folder)).to eq(messages_as_mbox)
-      end
-    end
   end
 end

data/spec/features/support/aruba.rb

@@ -1,6 +1,7 @@
 require "aruba/rspec"
 
 require_relative "backup_directory"
+require "imap/backup/serializer/mbox"
 
 Aruba.configure do |config|
   config.home_directory = File.expand_path("./tmp/home")
@@ -36,10 +37,10 @@ module StoreHelpers
     account = config.accounts.find { |a| a.username == email }
     raise "Account not found" if !account
     FileUtils.mkdir_p account.local_path
-    store = Imap::Backup::Serializer::MboxStore.new(account.local_path, folder)
-    store.uid_validity = "42" if !store.uid_validity
+    serializer = Imap::Backup::Serializer.new(account.local_path, folder)
+    serializer.force_uid_validity("42") if !serializer.uid_validity
     serialized = to_serialized(from: from, subject: subject, body: body)
-    store.add(uid, serialized)
+    serializer.append uid, serialized
   end
 
   def to_serialized(from:, subject:, body:)

data/spec/unit/imap/backup/account/connection_spec.rb

@@ -24,21 +24,24 @@ describe Imap::Backup::Account::Connection do
   let(:account) do
     instance_double(
       Imap::Backup::Account,
-      username: USERNAME,
+      username: username,
       password: PASSWORD,
       local_path: LOCAL_PATH,
       folders: config_folders,
+      multi_fetch_size: multi_fetch_size,
       server: server,
       connection_options: nil
     )
   end
+  let(:username) { USERNAME }
   let(:config_folders) { [FOLDER_CONFIG] }
+  let(:multi_fetch_size) { 1 }
   let(:root_info) do
     instance_double(Net::IMAP::MailboxList, name: ROOT_NAME)
   end
   let(:serializer) do
     instance_double(
-      Imap::Backup::Serializer::Mbox,
+      Imap::Backup::Serializer,
       folder: serialized_folder,
       force_uid_validity: nil,
       apply_uid_validity: new_uid_validity,
@@ -87,6 +90,23 @@ describe Imap::Backup::Account::Connection do
       end
     end
 
+    context "when the provider is Apple" do
+      let(:username) { "user@mac.com" }
+      let(:apple_client) do
+        instance_double(
+          Imap::Backup::Client::AppleMail, login: nil
+        )
+      end
+
+      before do
+        allow(Imap::Backup::Client::AppleMail).to receive(:new) { apple_client }
+      end
+
+      it "returns the Apple client" do
+        expect(result).to eq(apple_client)
+      end
+    end
+
     context "when run" do
       before { subject.client }
 
@@ -116,7 +136,7 @@ describe Imap::Backup::Account::Connection do
 
     before do
       allow(Imap::Backup::Account::Folder).to receive(:new) { folder }
-      allow(Imap::Backup::Serializer::Mbox).to receive(:new) { serializer }
+      allow(Imap::Backup::Serializer).to receive(:new) { serializer }
     end
 
     it "creates the path" do
@@ -150,16 +170,24 @@ describe Imap::Backup::Account::Connection do
     let(:exists) { true }
     let(:uid_validity) { 123 }
     let(:downloader) { instance_double(Imap::Backup::Downloader, run: nil) }
+    let(:multi_fetch_size) { 10 }
 
     before do
       allow(Imap::Backup::Downloader).
         to receive(:new).with(folder, serializer, anything) { downloader }
       allow(Imap::Backup::Account::Folder).to receive(:new).
         with(subject, BACKUP_FOLDER) { folder }
-      allow(Imap::Backup::Serializer::Mbox).to receive(:new).
+      allow(Imap::Backup::Serializer).to receive(:new).
         with(LOCAL_PATH, IMAP_FOLDER) { serializer }
     end
 
+    it "passes the multi_fetch_size" do
+      subject.run_backup
+
+      expect(Imap::Backup::Downloader).to have_received(:new).
+        with(anything, anything, {multi_fetch_size: 10})
+    end
+
     context "with supplied config_folders" do
       it "runs the downloader" do
         expect(downloader).to receive(:run)
@@ -184,7 +212,7 @@ describe Imap::Backup::Account::Connection do
       before do
         allow(Imap::Backup::Account::Folder).to receive(:new).
           with(subject, ROOT_NAME) { folder }
-        allow(Imap::Backup::Serializer::Mbox).to receive(:new).
+        allow(Imap::Backup::Serializer).to receive(:new).
           with(LOCAL_PATH, ROOT_NAME) { serializer }
       end
 
@@ -273,18 +301,18 @@ describe Imap::Backup::Account::Connection do
     end
     let(:updated_serializer) do
       instance_double(
-        Imap::Backup::Serializer::Mbox, force_uid_validity: nil
+        Imap::Backup::Serializer, force_uid_validity: nil
       )
     end
 
     before do
       allow(Imap::Backup::Account::Folder).to receive(:new).
         with(subject, FOLDER_NAME) { folder }
-      allow(Imap::Backup::Serializer::Mbox).to receive(:new).
+      allow(Imap::Backup::Serializer).to receive(:new).
         with(anything, FOLDER_NAME) { serializer }
       allow(Imap::Backup::Account::Folder).to receive(:new).
         with(subject, "new name") { updated_folder }
-      allow(Imap::Backup::Serializer::Mbox).to receive(:new).
+      allow(Imap::Backup::Serializer).to receive(:new).
         with(anything, "new name") { updated_serializer }
       allow(Imap::Backup::Uploader).to receive(:new).
         with(folder, serializer) { uploader }

data/spec/unit/imap/backup/account/folder_spec.rb

@@ -64,6 +64,16 @@ describe Imap::Backup::Account::Folder do
         expect(subject.uids).to eq([])
       end
     end
+
+    context "when the UID search fails" do
+      before do
+        allow(client).to receive(:uid_search).and_raise(NoMethodError)
+      end
+
+      it "returns an empty array" do
+        expect(subject.uids).to eq([])
+      end
+    end
   end
 
   describe "#fetch_multi" do