imap-backup 1.4.2 → 2.0.0.rc1

data/lib/imap/backup/serializer/mbox_store.rb

@@ -0,0 +1,214 @@
+require "json"
+
+require "email/mboxrd/message"
+
+module Imap::Backup
+  class Serializer::MboxStore
+    CURRENT_VERSION = 2
+
+    attr_reader :folder
+    attr_reader :path
+    attr_reader :loaded
+
+    def initialize(path, folder)
+      @path = path
+      @folder = folder
+      @loaded = false
+      @uids = nil
+      @uid_validity = nil
+    end
+
+    def exist?
+      mbox_exist? && imap_exist?
+    end
+
+    def uid_validity
+      do_load if !loaded
+      @uid_validity
+    end
+
+    def uid_validity=(value)
+      do_load if !loaded
+      @uid_validity = value
+      @uids ||= []
+      write_imap_file
+    end
+
+    def uids
+      do_load if !loaded
+      @uids || []
+    end
+
+    def add(uid, message)
+      do_load if !loaded
+      raise "Can't add messages without uid_validity" if !uid_validity
+      uid = uid.to_i
+      if uids.include?(uid)
+        Imap::Backup.logger.debug(
+          "[#{folder}] message #{uid} already downloaded - skipping"
+        )
+        return
+      end
+
+      body = message["RFC822"]
+      mboxrd_message = Email::Mboxrd::Message.new(body)
+      mbox = nil
+      begin
+        mbox = File.open(mbox_pathname, "ab")
+        mbox.write mboxrd_message.to_serialized
+        @uids << uid
+        write_imap_file
+      rescue => e
+        message = <<-ERROR.gsub(/^\s*/m, "")
+          [#{folder}] failed to save message #{uid}:
+          #{body}. #{e}:
+          #{e.backtrace.join("\n")}"
+        ERROR
+        Imap::Backup.logger.warn message
+      ensure
+        mbox.close if mbox
+      end
+    end
+
+    def load(uid)
+      do_load if !loaded
+      message_index = uids.find_index(uid)
+      return nil if message_index.nil?
+      load_nth(message_index)
+    end
+
+    def update_uid(old, new)
+      index = uids.find_index(old.to_i)
+      return if index.nil?
+      uids[index] = new.to_i
+      write_imap_file
+    end
+
+    def reset
+      @uids = nil
+      @uid_validity = nil
+      @loaded = false
+      delete_files
+      write_blank_mbox_file
+    end
+
+    def rename(new_name)
+      new_mbox_pathname = absolute_path(new_name + ".mbox")
+      new_imap_pathname = absolute_path(new_name + ".imap")
+      File.rename(mbox_pathname, new_mbox_pathname)
+      File.rename(imap_pathname, new_imap_pathname)
+      @folder = new_name
+    end
+
+    def relative_path
+      File.dirname(folder)
+    end
+
+    private
+
+    def do_load
+      data = imap_data
+      if data
+        @uids = data[:uids].map(&:to_i).sort
+        @uid_validity = data[:uid_validity]
+        @loaded = true
+      else
+        reset
+      end
+    end
+
+    def imap_data
+      return nil if !imap_ok?
+
+      imap_data = nil
+
+      begin
+        imap_data = JSON.parse(File.read(imap_pathname), symbolize_names: true)
+      rescue JSON::ParserError
+        return nil
+      end
+
+      return nil if !imap_data.has_key?(:uids)
+      return nil if !imap_data[:uids].is_a?(Array)
+
+      imap_data
+    end
+
+    def imap_ok?
+      return false if !exist?
+      return false if !imap_looks_like_json?
+      true
+    end
+
+    def load_nth(index)
+      each_mbox_message.with_index do |raw, i|
+        next unless i == index
+        return Email::Mboxrd::Message.from_serialized(raw)
+      end
+      nil
+    end
+
+    def each_mbox_message
+      Enumerator.new do |e|
+        File.open(mbox_pathname) do |f|
+          lines = []
+
+          while line = f.gets
+            if line.start_with?("From ")
+              e.yield lines.join if lines.count > 0
+              lines = [line]
+            else
+              lines << line
+            end
+          end
+          e.yield lines.join if lines.count > 0
+        end
+      end
+    end
+
+    def imap_looks_like_json?
+      return false unless imap_exist?
+      content = File.read(imap_pathname)
+      content.start_with?("{")
+    end
+
+    def write_imap_file
+      imap_data = {
+        version: CURRENT_VERSION,
+        uid_validity: @uid_validity,
+        uids: @uids
+      }
+      content = imap_data.to_json
+      File.open(imap_pathname, "w") { |f| f.write content }
+    end
+
+    def write_blank_mbox_file
+      File.open(mbox_pathname, "w") { |f| f.write "" }
+    end
+
+    def delete_files
+      File.unlink(imap_pathname) if imap_exist?
+      File.unlink(mbox_pathname) if mbox_exist?
+    end
+
+    def mbox_exist?
+      File.exist?(mbox_pathname)
+    end
+
+    def imap_exist?
+      File.exist?(imap_pathname)
+    end
+
+    def absolute_path(relative_path)
+      File.join(path, relative_path)
+    end
+
+    def mbox_pathname
+      absolute_path(folder + ".mbox")
+    end
+
+    def imap_pathname
+      absolute_path(folder + ".imap")
+    end
+  end
+end

data/lib/imap/backup/uploader.rb

@@ -0,0 +1,26 @@
+module Imap::Backup
+  class Uploader
+    attr_reader :folder
+    attr_reader :serializer
+
+    def initialize(folder, serializer)
+      @folder = folder
+      @serializer = serializer
+    end
+
+    def run
+      missing_uids.each do |uid|
+        message = serializer.load(uid)
+        next if message.nil?
+        new_uid = folder.append(message)
+        serializer.update_uid(uid, new_uid)
+      end
+    end
+
+    private
+
+    def missing_uids
+      serializer.uids - folder.uids
+    end
+  end
+end

data/lib/imap/backup/utils.rb

@@ -3,7 +3,7 @@ require "fileutils"
 module Imap::Backup
   module Utils
     def self.check_permissions(filename, limit)
-      actual = stat(filename)
+      actual = mode(filename)
       return nil if actual.nil?
       mask = ~limit & 0o777
       if actual & mask != 0
@@ -16,8 +16,8 @@ module Imap::Backup
       end
     end
 
-    def self.stat(filename)
-      return nil unless File.exist?(filename)
+    def self.mode(filename)
+      return nil if !File.exist?(filename)
 
       stat = File.stat(filename)
       stat.mode & 0o777

data/lib/imap/backup/version.rb

@@ -1,8 +1,9 @@
 module Imap; end
 
 module Imap::Backup
-  MAJOR    = 1
-  MINOR    = 4
-  REVISION = 2
-  VERSION  = [MAJOR, MINOR, REVISION].map(&:to_s).join(".")
+  MAJOR    = 2
+  MINOR    = 0
+  REVISION = 0
+  PRE      = "rc1"
+  VERSION  = [MAJOR, MINOR, REVISION, PRE].compact.map(&:to_s).join(".")
 end

data/spec/features/backup_spec.rb

@@ -7,21 +7,99 @@ RSpec.describe "backup", type: :feature, docker: true do
   let(:messages_as_mbox) do
     message_as_mbox_entry(msg1) + message_as_mbox_entry(msg2)
   end
-  let(:folder) { "INBOX" }
-
-  before do
-    send_email folder, msg1
-    send_email folder, msg2
+  let(:folder) { "my-stuff" }
+  let(:email1) { send_email folder, msg1 }
+  let(:email2) { send_email folder, msg2 }
+  let!(:pre) { }
+  let!(:setup) do
+    server_create_folder folder
+    email1
+    email2
+    connection.run_backup
   end
 
   after do
     FileUtils.rm_rf local_backup_path
     delete_emails folder
+    server_delete_folder folder
+    connection.disconnect
   end
 
   it "downloads messages" do
-    connection.run_backup
-
     expect(mbox_content(folder)).to eq(messages_as_mbox)
   end
+
+  context "IMAP metadata" do
+    let(:imap_metadata) { imap_parsed(folder) }
+    let(:folder_uids) { server_uids(folder) }
+
+    it "saves IMAP metadata in a JSON file" do
+      expect { imap_metadata }.to_not raise_error
+    end
+
+    it "saves a file version" do
+      expect(imap_metadata[:version].to_s).to match(/^[0-9\.]$/)
+    end
+
+    it "records IMAP ids" do
+      expect(imap_metadata[:uids]).to eq(folder_uids)
+    end
+
+    it "records uid_validity" do
+      expect(imap_metadata[:uid_validity]).to eq(server_uid_validity(folder))
+    end
+
+    context "when uid_validity does not match" do
+      let(:new_name) { "NEWNAME" }
+      let(:email3) { send_email folder, msg3 }
+      let!(:pre) do
+        server_create_folder folder
+        email3
+        @original_folder_uid_validity = server_uid_validity(folder)
+        connection.run_backup
+        server_rename_folder folder, new_name
+      end
+      let(:renamed_folder) { folder + "." + @original_folder_uid_validity.to_s }
+
+      after do
+        server_delete_folder new_name
+      end
+
+      it "renames the old backup" do
+        expect(mbox_content(renamed_folder)).to eq(message_as_mbox_entry(msg3))
+      end
+
+      it "downloads messages" do
+        expect(mbox_content(folder)).to eq(messages_as_mbox)
+      end
+
+      context "when a renamed local backup exists" do
+        let!(:pre) do
+          super()
+          File.write(imap_path(renamed_folder), "existing imap")
+          File.write(mbox_path(renamed_folder), "existing mbox")
+        end
+
+        it "moves the old backup to a uniquely named directory" do
+          renamed = folder + "." + @original_folder_uid_validity.to_s + ".1"
+          expect(mbox_content(renamed)).to eq(message_as_mbox_entry(msg3))
+        end
+      end
+    end
+
+    context "when an unversioned .imap file is found" do
+      let!(:pre) do
+        File.open(imap_path(folder), "w") { |f| f.write "old format imap" }
+        File.open(mbox_path(folder), "w") { |f| f.write "old format emails" }
+      end
+
+      it "replaces the .imap file with a versioned JSON file" do
+        expect(imap_metadata[:uids]).to eq(folder_uids)
+      end
+
+      it "does the download" do
+        expect(mbox_content(folder)).to eq(messages_as_mbox)
+      end
+    end
+  end
 end

data/spec/features/restore_spec.rb

@@ -0,0 +1,112 @@
+require "features/helper"
+
+RSpec.describe "restore", type: :feature, docker: true do
+  include_context "imap-backup connection"
+  include_context "message-fixtures"
+
+  let(:messages_as_mbox) do
+    message_as_mbox_entry(msg1) + message_as_mbox_entry(msg2)
+  end
+  let(:messages_as_server_messages) do
+    [message_as_server_message(msg1), message_as_server_message(msg2)]
+  end
+  let(:message_uids) { [msg1[:uid], msg2[:uid]] }
+  let(:existing_imap_content) { imap_data(uid_validity, message_uids).to_json }
+  let(:folder) { "my-stuff" }
+  let(:uid_validity) { 1234 }
+
+  let!(:pre) {}
+  let!(:setup) do
+    File.write(imap_path(folder), existing_imap_content)
+    File.write(mbox_path(folder), messages_as_mbox)
+    connection.restore
+  end
+  let(:cleanup) do
+    FileUtils.rm_rf local_backup_path
+    server_delete_folder folder
+    connection.disconnect
+  end
+
+  after { cleanup }
+
+  context "when the folder doesn't exist" do
+    it "restores messages" do
+      messages = server_messages(folder).map { |m| server_message_to_body(m) }
+      expect(messages).to eq(messages_as_server_messages)
+    end
+
+    it "updates local uids to match the new server ones" do
+      updated_imap_content = imap_parsed(folder)
+      expect(server_uids(folder)).to eq(updated_imap_content[:uids])
+    end
+
+    it "sets the backup uid_validity to match the new folder" do
+      updated_imap_content = imap_parsed(folder)
+      expect(updated_imap_content[:uid_validity]).
+        to eq(server_uid_validity(folder))
+    end
+  end
+
+  context "when the folder exists" do
+    let(:email3) { send_email folder, msg3 }
+
+    context "when the uid_validity matches" do
+      let(:pre) do
+        server_create_folder folder
+        email3
+        uid_validity
+      end
+      let(:messages_as_server_messages) do
+        [
+          message_as_server_message(msg3),
+          message_as_server_message(msg1),
+          message_as_server_message(msg2)
+        ]
+      end
+      let(:uid_validity) { server_uid_validity(folder) }
+
+      it "appends to the existing folder" do
+        messages = server_messages(folder).map { |m| server_message_to_body(m) }
+        expect(messages).to eq(messages_as_server_messages)
+      end
+    end
+
+    context "when the uid_validity doesn't match" do
+      let(:pre) do
+        server_create_folder folder
+        email3
+      end
+      let(:new_folder) { "#{folder}.#{uid_validity}" }
+      let(:cleanup) do
+        server_delete_folder new_folder
+        super()
+      end
+
+      it "sets the backup uid_validity to match the new folder" do
+        updated_imap_content = imap_parsed(new_folder)
+        expect(updated_imap_content[:uid_validity]).
+          to eq(server_uid_validity(new_folder))
+      end
+
+      it "renames the backup" do
+        expect(mbox_content(new_folder)).to eq(messages_as_mbox)
+      end
+
+      it "leaves the existing folder as is" do
+        messages = server_messages(folder).map { |m| server_message_to_body(m) }
+        expect(messages).to eq([message_as_server_message(msg3)])
+      end
+
+      it "creates the new folder" do
+        expect(server_folders.map(&:name)).to include(new_folder)
+      end
+
+      it "uploads to the new folder" do
+        messages = server_messages(new_folder).map do |m|
+          server_message_to_body(m)
+        end
+        expect(messages).to eq(messages_as_server_messages)
+      end
+    end
+  end
+end