imap-backup 2.0.0 → 2.2.2

data/lib/imap/backup/serializer/mbox_enumerator.rb

@@ -0,0 +1,31 @@
+module Imap::Backup
+  class Serializer::MboxEnumerator
+    attr_reader :mbox_pathname
+
+    def initialize(mbox_pathname)
+      @mbox_pathname = mbox_pathname
+    end
+
+    def each
+      return enum_for(:each) if !block_given?
+
+      File.open(mbox_pathname, "rb") do |f|
+        lines = []
+
+        loop do
+          line = f.gets
+          break if !line
+
+          if line.start_with?("From ")
+            yield lines.join if lines.count.positive?
+            lines = [line]
+          else
+            lines << line
+          end
+        end
+
+        yield lines.join if lines.count.positive?
+      end
+    end
+  end
+end

data/lib/imap/backup/serializer/mbox_store.rb

@@ -1,6 +1,7 @@
 require "json"
 
 require "email/mboxrd/message"
+require "imap/backup/serializer/mbox_enumerator"
 
 module Imap::Backup
   class Serializer::MboxStore
@@ -42,6 +43,7 @@ module Imap::Backup
     def add(uid, message)
       do_load if !loaded
       raise "Can't add messages without uid_validity" if !uid_validity
+
       uid = uid.to_i
       if uids.include?(uid)
         Imap::Backup.logger.debug(
@@ -58,7 +60,7 @@ module Imap::Backup
         mbox.write mboxrd_message.to_serialized
         @uids << uid
         write_imap_file
-      rescue => e
+      rescue StandardError => e
         message = <<-ERROR.gsub(/^\s*/m, "")
           [#{folder}] failed to save message #{uid}:
           #{body}. #{e}:
@@ -66,20 +68,39 @@ module Imap::Backup
         ERROR
         Imap::Backup.logger.warn message
       ensure
-        mbox.close if mbox
+        mbox&.close
       end
     end
 
-    def load(uid)
+    def load(uid_maybe_string)
       do_load if !loaded
+      uid = uid_maybe_string.to_i
       message_index = uids.find_index(uid)
       return nil if message_index.nil?
+
       load_nth(message_index)
     end
 
+    def each_message(required_uids)
+      return enum_for(:each_message, required_uids) if !block_given?
+
+      indexes = required_uids.each.with_object({}) do |uid, acc|
+        index = uids.find_index(uid)
+        acc[index] = uid if index
+      end
+      enumerator = Serializer::MboxEnumerator.new(mbox_pathname)
+      enumerator.each.with_index do |raw, i|
+        uid = indexes[i]
+        next if !uid
+
+        yield uid, Email::Mboxrd::Message.from_serialized(raw)
+      end
+    end
+
     def update_uid(old, new)
       index = uids.find_index(old.to_i)
       return if index.nil?
+
       uids[index] = new.to_i
       write_imap_file
     end
@@ -93,23 +114,19 @@ module Imap::Backup
     end
 
     def rename(new_name)
-      new_mbox_pathname = absolute_path(new_name + ".mbox")
-      new_imap_pathname = absolute_path(new_name + ".imap")
+      new_mbox_pathname = absolute_path("#{new_name}.mbox")
+      new_imap_pathname = absolute_path("#{new_name}.imap")
       File.rename(mbox_pathname, new_mbox_pathname)
       File.rename(imap_pathname, new_imap_pathname)
       @folder = new_name
     end
 
-    def relative_path
-      File.dirname(folder)
-    end
-
     private
 
     def do_load
       data = imap_data
       if data
-        @uids = data[:uids].map(&:to_i).sort
+        @uids = data[:uids].map(&:to_i)
         @uid_validity = data[:uid_validity]
         @loaded = true
       else
@@ -128,7 +145,7 @@ module Imap::Backup
         return nil
       end
 
-      return nil if !imap_data.has_key?(:uids)
+      return nil if !imap_data.key?(:uids)
       return nil if !imap_data[:uids].is_a?(Array)
 
       imap_data
@@ -137,37 +154,23 @@ module Imap::Backup
     def imap_ok?
       return false if !exist?
       return false if !imap_looks_like_json?
+
       true
     end
 
     def load_nth(index)
-      each_mbox_message.with_index do |raw, i|
+      enumerator = Serializer::MboxEnumerator.new(mbox_pathname)
+      enumerator.each.with_index do |raw, i|
         next unless i == index
+
         return Email::Mboxrd::Message.from_serialized(raw)
       end
       nil
     end
 
-    def each_mbox_message
-      Enumerator.new do |e|
-        File.open(mbox_pathname) do |f|
-          lines = []
-
-          while line = f.gets
-            if line.start_with?("From ")
-              e.yield lines.join if lines.count > 0
-              lines = [line]
-            else
-              lines << line
-            end
-          end
-          e.yield lines.join if lines.count > 0
-        end
-      end
-    end
-
     def imap_looks_like_json?
       return false unless imap_exist?
+
       content = File.read(imap_pathname)
       content.start_with?("{")
     end
@@ -204,11 +207,11 @@ module Imap::Backup
     end
 
     def mbox_pathname
-      absolute_path(folder + ".mbox")
+      absolute_path("#{folder}.mbox")
     end
 
     def imap_pathname
-      absolute_path(folder + ".imap")
+      absolute_path("#{folder}.imap")
     end
   end
 end

data/lib/imap/backup/uploader.rb

@@ -9,9 +9,18 @@ module Imap::Backup
     end
 
     def run
-      missing_uids.each do |uid|
-        message = serializer.load(uid)
+      count = missing_uids.count
+      return if count.zero?
+
+      Imap::Backup.logger.debug "[#{folder.name}] #{count} to restore"
+      serializer.each_message(missing_uids).with_index do |(uid, message), i|
         next if message.nil?
+
+        log_prefix = "[#{folder.name}] uid: #{uid} (#{i + 1}/#{count}) -"
+        Imap::Backup.logger.debug(
+          "#{log_prefix} #{message.supplied_body.size} bytes"
+        )
+
         new_uid = folder.append(message)
         serializer.update_uid(uid, new_uid)
       end

data/lib/imap/backup/utils.rb

@@ -5,15 +5,16 @@ module Imap::Backup
     def self.check_permissions(filename, limit)
       actual = mode(filename)
       return nil if actual.nil?
+
       mask = ~limit & 0o777
-      if actual & mask != 0
-        message = format(
-          "Permissions on '%<filename>s' " \
-            "should be 0%<limit>o, not 0%<actual>o",
-          filename: filename, limit: limit, actual: actual
-        )
-        raise message
-      end
+      return if (actual & mask).zero?
+
+      message = format(
+        "Permissions on '%<filename>s' " \
+          "should be 0%<limit>o, not 0%<actual>o",
+        filename: filename, limit: limit, actual: actual
+      )
+      raise message
     end
 
     def self.mode(filename)
@@ -25,7 +26,8 @@ module Imap::Backup
 
     def self.make_folder(base_path, path, permissions)
       parts = path.split("/")
-      return if parts.size == 0
+      return if parts.empty?
+
       full_path = File.join(base_path, path)
       FileUtils.mkdir_p full_path
       path = base_path

data/lib/imap/backup/version.rb

@@ -2,7 +2,7 @@ module Imap; end
 
 module Imap::Backup
   MAJOR    = 2
-  MINOR    = 0
-  REVISION = 0
+  MINOR    = 2
+  REVISION = 2
   VERSION  = [MAJOR, MINOR, REVISION].compact.map(&:to_s).join(".")
 end

data/spec/features/backup_spec.rb

@@ -10,7 +10,7 @@ RSpec.describe "backup", type: :feature, docker: true do
   let(:folder) { "my-stuff" }
   let(:email1) { send_email folder, msg1 }
   let(:email2) { send_email folder, msg2 }
-  let!(:pre) { }
+  let!(:pre) {}
   let!(:setup) do
     server_create_folder folder
     email1
@@ -29,7 +29,7 @@ RSpec.describe "backup", type: :feature, docker: true do
     expect(mbox_content(folder)).to eq(messages_as_mbox)
   end
 
-  context "IMAP metadata" do
+  describe "IMAP metadata" do
     let(:imap_metadata) { imap_parsed(folder) }
     let(:folder_uids) { server_uids(folder) }
 
@@ -52,14 +52,15 @@ RSpec.describe "backup", type: :feature, docker: true do
     context "when uid_validity does not match" do
       let(:new_name) { "NEWNAME" }
       let(:email3) { send_email folder, msg3 }
+      let(:original_folder_uid_validity) { server_uid_validity(folder) }
       let!(:pre) do
         server_create_folder folder
         email3
-        @original_folder_uid_validity = server_uid_validity(folder)
+        original_folder_uid_validity
         connection.run_backup
         server_rename_folder folder, new_name
       end
-      let(:renamed_folder) { folder + "." + @original_folder_uid_validity.to_s }
+      let(:renamed_folder) { "#{folder}.#{original_folder_uid_validity}" }
 
       after do
         server_delete_folder new_name
@@ -81,7 +82,7 @@ RSpec.describe "backup", type: :feature, docker: true do
         end
 
         it "moves the old backup to a uniquely named directory" do
-          renamed = folder + "." + @original_folder_uid_validity.to_s + ".1"
+          renamed = "#{folder}.#{original_folder_uid_validity}.1"
           expect(mbox_content(renamed)).to eq(message_as_mbox_entry(msg3))
         end
       end

data/spec/features/helper.rb

@@ -1,2 +1,2 @@
 support_glob = File.expand_path("support/**/*.rb", __dir__)
-Dir[support_glob].each { |f| require f }
+Dir[support_glob].sort.each { |f| require f }

data/spec/features/restore_spec.rb

@@ -72,41 +72,89 @@ RSpec.describe "restore", type: :feature, docker: true do
     end
 
     context "when the uid_validity doesn't match" do
-      let(:pre) do
-        server_create_folder folder
-        email3
-      end
-      let(:new_folder) { "#{folder}.#{uid_validity}" }
-      let(:cleanup) do
-        server_delete_folder new_folder
-        super()
-      end
+      context "when the folder is empty" do
+        let(:pre) do
+          server_create_folder folder
+        end
 
-      it "sets the backup uid_validity to match the new folder" do
-        updated_imap_content = imap_parsed(new_folder)
-        expect(updated_imap_content[:uid_validity]).
-          to eq(server_uid_validity(new_folder))
-      end
+        it "sets the backup uid_validity to match the folder" do
+          updated_imap_content = imap_parsed(folder)
+          expect(updated_imap_content[:uid_validity]).
+            to eq(server_uid_validity(folder))
+        end
 
-      it "renames the backup" do
-        expect(mbox_content(new_folder)).to eq(messages_as_mbox)
+        it "uploads to the new folder" do
+          messages = server_messages(folder).map do |m|
+            server_message_to_body(m)
+          end
+          expect(messages).to eq(messages_as_server_messages)
+        end
       end
 
-      it "leaves the existing folder as is" do
-        messages = server_messages(folder).map { |m| server_message_to_body(m) }
-        expect(messages).to eq([message_as_server_message(msg3)])
-      end
+      context "when the folder has content" do
+        let(:new_folder) { "#{folder}.#{uid_validity}" }
+        let(:cleanup) do
+          server_delete_folder new_folder
+          super()
+        end
 
-      it "creates the new folder" do
-        expect(server_folders.map(&:name)).to include(new_folder)
-      end
+        let(:pre) do
+          server_create_folder folder
+          email3
+        end
 
-      it "uploads to the new folder" do
-        messages = server_messages(new_folder).map do |m|
-          server_message_to_body(m)
+        it "renames the backup" do
+          expect(mbox_content(new_folder)).to eq(messages_as_mbox)
+        end
+
+        it "leaves the existing folder as is" do
+          messages = server_messages(folder).map do |m|
+            server_message_to_body(m)
+          end
+          expect(messages).to eq([message_as_server_message(msg3)])
+        end
+
+        it "creates the new folder" do
+          expect(server_folders.map(&:name)).to include(new_folder)
+        end
+
+        it "sets the backup uid_validity to match the new folder" do
+          updated_imap_content = imap_parsed(new_folder)
+          expect(updated_imap_content[:uid_validity]).
+            to eq(server_uid_validity(new_folder))
+        end
+
+        it "uploads to the new folder" do
+          messages = server_messages(new_folder).map do |m|
+            server_message_to_body(m)
+          end
+          expect(messages).to eq(messages_as_server_messages)
         end
-        expect(messages).to eq(messages_as_server_messages)
       end
     end
   end
+
+  context "when non-Unicode encodings are used" do
+    let(:server_message) do
+      message_as_server_message(msg_iso8859)
+    end
+    let(:messages_as_mbox) do
+      message_as_mbox_entry(msg_iso8859)
+    end
+    let(:message_uids) { [uid_iso8859] }
+    let(:uid_validity) { server_uid_validity(folder) }
+
+    let(:pre) do
+      server_create_folder folder
+      uid_validity
+    end
+
+    it "maintains encodings" do
+      message =
+        server_messages(folder).
+        first["RFC822"]
+
+      expect(message).to eq(server_message)
+    end
+  end
 end

data/spec/features/support/backup_directory.rb

@@ -3,12 +3,12 @@ module BackupDirectoryHelpers
     from = fixture("connection")[:username]
     subject = options[:subject]
     body = options[:body]
-    body_and_headers = <<-EOT
-From: #{from}
-Subject: #{subject}
+    body_and_headers = <<~BODY
+      From: #{from}
+      Subject: #{subject}
 
-#{body}
-    EOT
+      #{body}
+    BODY
 
     "From #{from}\n#{body_and_headers}\n"
   end
@@ -26,11 +26,11 @@ Subject: #{subject}
   end
 
   def mbox_path(name)
-    File.join(local_backup_path, name + ".mbox")
+    File.join(local_backup_path, "#{name}.mbox")
   end
 
   def imap_path(name)
-    File.join(local_backup_path, name + ".imap")
+    File.join(local_backup_path, "#{name}.imap")
   end
 
   def imap_content(name)