imap-backup 2.1.1 → 3.0.0

data/lib/imap/backup/configuration/setup.rb

@@ -75,7 +75,10 @@ module Imap::Backup
         password: "",
         local_path: File.join(config.path, username.tr("@", "_")),
         folders: []
-      }
+      }.tap do |c|
+        server = Email::Provider.for_address(username)
+        c[:server] = server.host if server.host
+      end
     end
 
     def edit_account(username)

data/lib/imap/backup/serializer/mbox.rb

@@ -35,6 +35,10 @@ module Imap::Backup
       store.load(uid)
     end
 
+    def each_message(uids)
+      store.each_message(uids)
+    end
+
     def save(uid, message)
       store.add(uid, message)
     end

data/lib/imap/backup/serializer/mbox_enumerator.rb

@@ -9,7 +9,7 @@ module Imap::Backup
     def each
       return enum_for(:each) if !block_given?
 
-      File.open(mbox_pathname) do |f|
+      File.open(mbox_pathname, "rb") do |f|
         lines = []
 
         loop do

data/lib/imap/backup/serializer/mbox_store.rb

@@ -81,6 +81,22 @@ module Imap::Backup
       load_nth(message_index)
     end
 
+    def each_message(required_uids)
+      return enum_for(:each_message, required_uids) if !block_given?
+
+      indexes = required_uids.each.with_object({}) do |uid, acc|
+        index = uids.find_index(uid)
+        acc[index] = uid if index
+      end
+      enumerator = Serializer::MboxEnumerator.new(mbox_pathname)
+      enumerator.each.with_index do |raw, i|
+        uid = indexes[i]
+        next if !uid
+
+        yield uid, Email::Mboxrd::Message.from_serialized(raw)
+      end
+    end
+
     def update_uid(old, new)
       index = uids.find_index(old.to_i)
       return if index.nil?
@@ -98,8 +114,8 @@ module Imap::Backup
     end
 
     def rename(new_name)
-      new_mbox_pathname = absolute_path(new_name + ".mbox")
-      new_imap_pathname = absolute_path(new_name + ".imap")
+      new_mbox_pathname = absolute_path("#{new_name}.mbox")
+      new_imap_pathname = absolute_path("#{new_name}.imap")
       File.rename(mbox_pathname, new_mbox_pathname)
       File.rename(imap_pathname, new_imap_pathname)
       @folder = new_name
@@ -191,11 +207,11 @@ module Imap::Backup
     end
 
     def mbox_pathname
-      absolute_path(folder + ".mbox")
+      absolute_path("#{folder}.mbox")
     end
 
     def imap_pathname
-      absolute_path(folder + ".imap")
+      absolute_path("#{folder}.imap")
     end
   end
 end

data/lib/imap/backup/uploader.rb

@@ -9,10 +9,18 @@ module Imap::Backup
     end
 
     def run
-      missing_uids.each do |uid|
-        message = serializer.load(uid)
+      count = missing_uids.count
+      return if count.zero?
+
+      Imap::Backup.logger.debug "[#{folder.name}] #{count} to restore"
+      serializer.each_message(missing_uids).with_index do |(uid, message), i|
         next if message.nil?
 
+        log_prefix = "[#{folder.name}] uid: #{uid} (#{i + 1}/#{count}) -"
+        Imap::Backup.logger.debug(
+          "#{log_prefix} #{message.supplied_body.size} bytes"
+        )
+
         new_uid = folder.append(message)
         serializer.update_uid(uid, new_uid)
       end

data/lib/imap/backup/version.rb

@@ -1,8 +1,9 @@
 module Imap; end
 
 module Imap::Backup
-  MAJOR    = 2
-  MINOR    = 1
-  REVISION = 1
-  VERSION  = [MAJOR, MINOR, REVISION].compact.map(&:to_s).join(".")
+  MAJOR    = 3
+  MINOR    = 0
+  REVISION = 0
+  PRE      = nil
+  VERSION  = [MAJOR, MINOR, REVISION, PRE].compact.map(&:to_s).join(".")
 end

data/spec/features/backup_spec.rb

@@ -38,7 +38,7 @@ RSpec.describe "backup", type: :feature, docker: true do
     end
 
     it "saves a file version" do
-      expect(imap_metadata[:version].to_s).to match(/^[0-9\.]$/)
+      expect(imap_metadata[:version].to_s).to match(/^[0-9.]$/)
     end
 
     it "records IMAP ids" do
@@ -60,7 +60,7 @@ RSpec.describe "backup", type: :feature, docker: true do
         connection.run_backup
         server_rename_folder folder, new_name
       end
-      let(:renamed_folder) { folder + "." + original_folder_uid_validity.to_s }
+      let(:renamed_folder) { "#{folder}.#{original_folder_uid_validity}" }
 
       after do
         server_delete_folder new_name
@@ -82,7 +82,7 @@ RSpec.describe "backup", type: :feature, docker: true do
         end
 
         it "moves the old backup to a uniquely named directory" do
-          renamed = folder + "." + original_folder_uid_validity.to_s + ".1"
+          renamed = "#{folder}.#{original_folder_uid_validity}.1"
           expect(mbox_content(renamed)).to eq(message_as_mbox_entry(msg3))
         end
       end

data/spec/features/helper.rb

@@ -1,2 +1,2 @@
 support_glob = File.expand_path("support/**/*.rb", __dir__)
-Dir[support_glob].each { |f| require f }
+Dir[support_glob].sort.each { |f| require f }

data/spec/features/restore_spec.rb

@@ -72,41 +72,89 @@ RSpec.describe "restore", type: :feature, docker: true do
     end
 
     context "when the uid_validity doesn't match" do
-      let(:pre) do
-        server_create_folder folder
-        email3
-      end
-      let(:new_folder) { "#{folder}.#{uid_validity}" }
-      let(:cleanup) do
-        server_delete_folder new_folder
-        super()
-      end
+      context "when the folder is empty" do
+        let(:pre) do
+          server_create_folder folder
+        end
 
-      it "sets the backup uid_validity to match the new folder" do
-        updated_imap_content = imap_parsed(new_folder)
-        expect(updated_imap_content[:uid_validity]).
-          to eq(server_uid_validity(new_folder))
-      end
+        it "sets the backup uid_validity to match the folder" do
+          updated_imap_content = imap_parsed(folder)
+          expect(updated_imap_content[:uid_validity]).
+            to eq(server_uid_validity(folder))
+        end
 
-      it "renames the backup" do
-        expect(mbox_content(new_folder)).to eq(messages_as_mbox)
+        it "uploads to the new folder" do
+          messages = server_messages(folder).map do |m|
+            server_message_to_body(m)
+          end
+          expect(messages).to eq(messages_as_server_messages)
+        end
       end
 
-      it "leaves the existing folder as is" do
-        messages = server_messages(folder).map { |m| server_message_to_body(m) }
-        expect(messages).to eq([message_as_server_message(msg3)])
-      end
+      context "when the folder has content" do
+        let(:new_folder) { "#{folder}.#{uid_validity}" }
+        let(:cleanup) do
+          server_delete_folder new_folder
+          super()
+        end
 
-      it "creates the new folder" do
-        expect(server_folders.map(&:name)).to include(new_folder)
-      end
+        let(:pre) do
+          server_create_folder folder
+          email3
+        end
 
-      it "uploads to the new folder" do
-        messages = server_messages(new_folder).map do |m|
-          server_message_to_body(m)
+        it "renames the backup" do
+          expect(mbox_content(new_folder)).to eq(messages_as_mbox)
+        end
+
+        it "leaves the existing folder as is" do
+          messages = server_messages(folder).map do |m|
+            server_message_to_body(m)
+          end
+          expect(messages).to eq([message_as_server_message(msg3)])
+        end
+
+        it "creates the new folder" do
+          expect(server_folders.map(&:name)).to include(new_folder)
+        end
+
+        it "sets the backup uid_validity to match the new folder" do
+          updated_imap_content = imap_parsed(new_folder)
+          expect(updated_imap_content[:uid_validity]).
+            to eq(server_uid_validity(new_folder))
+        end
+
+        it "uploads to the new folder" do
+          messages = server_messages(new_folder).map do |m|
+            server_message_to_body(m)
+          end
+          expect(messages).to eq(messages_as_server_messages)
         end
-        expect(messages).to eq(messages_as_server_messages)
       end
     end
   end
+
+  context "when non-Unicode encodings are used" do
+    let(:server_message) do
+      message_as_server_message(msg_iso8859)
+    end
+    let(:messages_as_mbox) do
+      message_as_mbox_entry(msg_iso8859)
+    end
+    let(:message_uids) { [uid_iso8859] }
+    let(:uid_validity) { server_uid_validity(folder) }
+
+    let(:pre) do
+      server_create_folder folder
+      uid_validity
+    end
+
+    it "maintains encodings" do
+      message =
+        server_messages(folder).
+        first["RFC822"]
+
+      expect(message).to eq(server_message)
+    end
+  end
 end

data/spec/features/support/backup_directory.rb

@@ -26,11 +26,11 @@ module BackupDirectoryHelpers
   end
 
   def mbox_path(name)
-    File.join(local_backup_path, name + ".mbox")
+    File.join(local_backup_path, "#{name}.mbox")
   end
 
   def imap_path(name)
-    File.join(local_backup_path, name + ".imap")
+    File.join(local_backup_path, "#{name}.imap")
   end
 
   def imap_content(name)

data/spec/features/support/email_server.rb

@@ -38,9 +38,7 @@ module EmailServerHelpers
     return nil if fetch_data_items.nil?
 
     fetch_data_item = fetch_data_items[0]
-    attributes = fetch_data_item.attr
-    attributes["RFC822"].force_encoding("utf-8")
-    attributes
+    fetch_data_item.attr
   end
 
   def delete_emails(folder)

data/spec/features/support/shared/message_fixtures.rb

@@ -2,7 +2,15 @@ shared_context "message-fixtures" do
   let(:uid1) { 123 }
   let(:uid2) { 345 }
   let(:uid3) { 567 }
+  let(:uid_iso8859) { 890 }
   let(:msg1) { {uid: uid1, subject: "Test 1", body: "body 1\nHi"} }
   let(:msg2) { {uid: uid2, subject: "Test 2", body: "body 2"} }
   let(:msg3) { {uid: uid3, subject: "Test 3", body: "body 3"} }
+  let(:msg_iso8859) do
+    {
+      uid: uid_iso8859,
+      subject: "iso8859 Body",
+      body: "Ma, perchè?".encode(Encoding::ISO_8859_1).force_encoding("binary")
+    }
+  end
 end

data/spec/spec_helper.rb

@@ -7,7 +7,7 @@ spec_path = File.dirname(__FILE__)
 $LOAD_PATH << File.expand_path("../lib", spec_path)
 
 support_glob = File.join(spec_path, "support", "**", "*.rb")
-Dir[support_glob].each { |f| require f }
+Dir[support_glob].sort.each { |f| require f }
 
 require "simplecov"
 SimpleCov.start do

data/spec/support/fixtures.rb

@@ -1,6 +1,6 @@
 def fixture(name)
   spec_root = File.expand_path("..", File.dirname(__FILE__))
-  fixture_path = File.join(spec_root, "fixtures", name + ".yml")
+  fixture_path = File.join(spec_root, "fixtures", "#{name}.yml")
   fixture = File.read(fixture_path)
   YAML.safe_load(fixture, [Symbol])
 end

data/spec/unit/email/mboxrd/message_spec.rb

@@ -79,12 +79,6 @@ describe Email::Mboxrd::Message do
       allow(Mail).to receive(:new).with(cloned_message_body) { mail }
     end
 
-    it "does not modify the message" do
-      subject.to_serialized
-
-      expect(message_body).to_not have_received(:force_encoding).with("binary")
-    end
-
     it "adds a 'From ' line at the start" do
       expected = "From #{from} #{date.asctime}\n"
       expect(subject.to_serialized).to start_with(expected)
@@ -135,7 +129,7 @@ describe Email::Mboxrd::Message do
       let(:message_body) { msg_no_from_but_return_path }
 
       it "'return path' is used as 'from'" do
-        expect(subject.to_serialized).to start_with("From " + from + "\n")
+        expect(subject.to_serialized).to start_with("From #{from}\n")
       end
     end
 
@@ -143,7 +137,7 @@ describe Email::Mboxrd::Message do
       let(:message_body) { msg_no_from_but_sender }
 
       it "Sender is used as 'from'" do
-        expect(subject.to_serialized).to start_with("From " + from + "\n")
+        expect(subject.to_serialized).to start_with("From #{from}\n")
       end
     end
   end

data/spec/unit/email/provider_spec.rb

@@ -1,4 +1,6 @@
 describe Email::Provider do
+  subject { described_class.new(:gmail) }
+
   describe ".for_address" do
     context "with known providers" do
       [
@@ -20,8 +22,6 @@ describe Email::Provider do
     end
   end
 
-  subject { described_class.new(:gmail) }
-
   describe "#options" do
     it "returns options" do
       expect(subject.options).to be_a(Hash)

data/spec/unit/gmail/authenticator_spec.rb

@@ -0,0 +1,138 @@
+require "gmail/authenticator"
+require "googleauth"
+
+describe Gmail::Authenticator do
+  ACCESS_TOKEN = "access_token".freeze
+  AUTHORIZATION_URL = "authorization_url".freeze
+  CLIENT_ID = "client_id".freeze
+  CLIENT_SECRET = "client_secret".freeze
+  CODE = "code".freeze
+  CREDENTIALS = "credentials".freeze
+  EMAIL = "email".freeze
+  EXPIRATION_TIME_MILLIS = "expiration_time_millis".freeze
+  GMAIL_READ_SCOPE = "https://mail.google.com/".freeze
+  IMAP_BACKUP_TOKEN = "imap_backup_token".freeze
+  OOB_URI = "urn:ietf:wg:oauth:2.0:oob".freeze
+  REFRESH_TOKEN = "refresh_token".freeze
+
+  subject { described_class.new(**params) }
+
+  let(:params) do
+    {
+      email: EMAIL,
+      token: IMAP_BACKUP_TOKEN
+    }
+  end
+
+  let(:authorizer) do
+    instance_double(Google::Auth::UserAuthorizer)
+  end
+
+  let(:imap_backup_token) do
+    instance_double(
+      Gmail::Authenticator::ImapBackupToken,
+      access_token: ACCESS_TOKEN,
+      client_id: CLIENT_ID,
+      client_secret: CLIENT_SECRET,
+      expiration_time_millis: EXPIRATION_TIME_MILLIS,
+      refresh_token: REFRESH_TOKEN,
+      valid?: true
+    )
+  end
+
+  let(:token_store) do
+    instance_double(Google::Auth::Stores::InMemoryTokenStore)
+  end
+
+  let(:credentials) do
+    instance_double(Google::Auth::UserRefreshCredentials, refresh!: true)
+  end
+
+  let(:expired) { false }
+
+  before do
+    allow(Google::Auth::UserAuthorizer).
+      to receive(:new).
+        with(
+          instance_of(Google::Auth::ClientId),
+          GMAIL_READ_SCOPE,
+          token_store
+        ) { authorizer }
+    allow(authorizer).to receive(:get_authorization_url).
+      with(base_url: OOB_URI) { AUTHORIZATION_URL }
+    allow(authorizer).to receive(:get_credentials).
+      with(EMAIL) { credentials }
+    allow(authorizer).to receive(:get_credentials_from_code).
+      with(user_id: EMAIL, code: CODE, base_url: OOB_URI) { CREDENTIALS }
+
+    allow(Google::Auth::UserRefreshCredentials).
+      to receive(:new) { credentials }
+    allow(credentials).to receive(:expired?) { expired }
+
+    allow(Google::Auth::Stores::InMemoryTokenStore).
+      to receive(:new) { token_store }
+    allow(token_store).to receive(:store).
+      with(EMAIL, anything) # TODO: use a JSON matcher
+    allow(Gmail::Authenticator::ImapBackupToken).
+      to receive(:new).
+        with(IMAP_BACKUP_TOKEN) { imap_backup_token }
+  end
+
+  describe "#initialize" do
+    [:email, :token].each do |param|
+      context "parameter #{param}" do
+        let(:params) { super().dup.reject { |k| k == param } }
+
+        it "is expected" do
+          expect { subject }.to raise_error(
+            ArgumentError, /missing keyword: :#{param}/
+          )
+        end
+      end
+    end
+  end
+
+  describe "#credentials" do
+    let!(:result) { subject.credentials }
+
+    it "attempts to get credentials" do
+      expect(authorizer).to have_received(:get_credentials)
+    end
+
+    it "returns the result" do
+      expect(result).to eq(credentials)
+    end
+
+    context "when the access_token has expired" do
+      let(:expired) { true }
+
+      it "refreshes it" do
+        expect(credentials).to have_received(:refresh!)
+      end
+    end
+  end
+
+  describe "#authorization_url" do
+    let!(:result) { subject.authorization_url }
+
+    it "requests an authorization URL" do
+      expect(authorizer).to have_received(:get_authorization_url)
+    end
+
+    it "returns the result" do
+      expect(result).to eq(AUTHORIZATION_URL)
+    end
+  end
+
+  describe "#credentials_from_code" do
+    let!(:result) { subject.credentials_from_code(CODE) }
+
+    it "requests credentials" do
+      expect(authorizer).to have_received(:get_credentials_from_code)
+    end
+
+    it "returns credentials" do
+      expect(result).to eq(CREDENTIALS)
+    end
+  end
+end