imap-backup 1.4.2 → 2.0.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 37b1f605db79f10d3520f0f718e457d38c839003380f53fc7c1259a16293952e
-  data.tar.gz: e13a7ab06d0c3a9e93ef52709de8f23a6fe4eaba720f51a3535aa2b984468df5
+  metadata.gz: 3f897dc0bac78683c9cae100718a03cf934adc0348866a4688ce5d2196bbdcfa
+  data.tar.gz: 164bc47c5acb9e448b3c61e6d58d18c3f48cd03fd6a2cf236682e5f5abd11232
 SHA512:
-  metadata.gz: 9961d6f0ddde8b511f77d1cf0a1ebec932d2a98a0b6762c18ea36496365125b451818c1d9565527b146ccb8a7c495e17d6f590600caee8aa7a02d128edf8fe19
-  data.tar.gz: 511c7025476ff56508d6538bc74ba382d2af741ba1c2ee8eabf36ceb6ce002b9d18ef51fc731d1aed4e8f20cfbeb998c3ccd39000528cccaa45c365403ffea3f
+  metadata.gz: 1ef7c3cea9a72c2fec7f87491c80eebc1aeeeca50c207fec4c713259a2de34c986a7a2457aed7cc20b36e4173f52fd9d30ac0340bca64600d23792d00433ac3f
+  data.tar.gz: 787c00650b5dd72258e895f51457599e0c20bd4d5897c5ec8c812fd4a54ce96796538b8c88cc689a02365f8fed5122237724a566aae9849dd5b807804a3a644f

data/.rspec-all

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/README.md

@@ -16,6 +16,12 @@
 [Rubygem]: http://rubygems.org/gems/imap-backup "Ruby gem at rubygems.org"
 [Continuous Integration]: http://travis-ci.org/joeyates/imap-backup "Build status by Travis-CI"
 
+## Version 2
+
+With versions above 2.x, this gems stores IMAP metadata in a
+backwardly-incompatible way. When upgrading, all old backups will be gradually
+deleted to allow for the new file format to be introduced.
+
 # Installation
 
 ```shell
@@ -158,6 +164,13 @@ If you have problems:
   "debug": true
 }
 ```
+# Restore
+
+All missing messages are pushed to the IMAP server.
+Existing messages are left unchanged.
+
+This functionality requires that the IMAP server supports the UIDPLUS
+extension to IMAP4.
 
 # Other Usage
 
@@ -190,13 +203,33 @@ $ imap-backup status
 
 ## Integration Tests
 
-Integration tests are run against a Docker image
+Integration tests (feature specs) are run against a Docker image
 (antespi/docker-imap-devel:latest).
 
+In one shell, run the Docker image:
+
+```
+docker run \
+  --env MAIL_ADDRESS=address@example.org \
+  --env MAIL_PASS=pass \
+  --env MAILNAME=example.org \
+  --publish 8993:993 \
+  antespi/docker-imap-devel:latest
+```
+
 Currently, the integration tests with Docker are excluded from the CI run.
 
-The image has a pre-existing user:
-`address@example.org` with password `pass`
+To run **just** the Docker-based tests:
+
+```
+rspec -t docker
+```
+
+To run **all** specs, including the integration tests, do the following:
+
+```
+rspec -O .rspec-all
+```
 
 ## Contributing
 

data/bin/imap-backup

@@ -8,6 +8,7 @@ KNOWN_COMMANDS = [
   {name: "setup",   help: "Create/edit the configuration file"},
   {name: "backup",  help: "Do the backup (default)"},
   {name: "folders", help: "List folders for all (or selected) accounts"},
+  {name: "restore", help: "Restore emails to server"},
   {name: "status",  help: "List count of non backed-up emails per folder"},
   {name: "help",    help: "Show usage"}
 ].freeze
@@ -84,6 +85,10 @@ when "folders"
     end
     folders.each { |f| puts "\t" + f.name }
   end
+when "restore"
+  configuration.each_connection do |connection|
+    connection.restore
+  end
 when "status"
   configuration.each_connection do |connection|
     puts connection.username

data/imap-backup.gemspec

@@ -15,6 +15,13 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = Imap::Backup::VERSION
 
+  gem.post_install_message = <<-MESSAGE.gsub(/^\s{4}/m, "")
+    When upgrading #{gem.name} from version 1.x to 2.x not that the
+    metadata storage method has changed (from flat file to JSON).
+    As a result, on the first run after an upgrade, old backup folders will be
+    **deleted** and a full new backup created.
+  MESSAGE
+
   gem.add_runtime_dependency "rake"
   gem.add_runtime_dependency "highline"
   gem.add_runtime_dependency "mail"

data/lib/email/mboxrd/message.rb

@@ -22,7 +22,15 @@ module Email::Mboxrd
     end
 
     def to_serialized
-      "From " + from + "\n" + mboxrd_body + "\n"
+      "From " + from + "\n" + mboxrd_body
+    end
+
+    def date
+      parsed.date
+    end
+
+    def imap_body
+      supplied_body.gsub(/(?<!\r)\n/, "\r\n")
     end
 
     private
@@ -57,9 +65,10 @@ module Email::Mboxrd
     def mboxrd_body
       @mboxrd_body ||=
         begin
-          @mboxrd_body = add_extra_quote(supplied_body)
-          @mboxrd_body += "\n" if !@mboxrd_body.end_with?("\n")
-          @mboxrd_body
+          mboxrd_body = add_extra_quote(supplied_body.gsub("\r\n", "\n"))
+          mboxrd_body += "\n" if !mboxrd_body.end_with?("\n")
+          mboxrd_body += "\n" if !mboxrd_body.end_with?("\n\n")
+          mboxrd_body
         end
     end
 
@@ -75,11 +84,5 @@ module Email::Mboxrd
     def asctime
       @asctime ||= date ? date.asctime : ""
     end
-
-    def date
-      parsed.date
-    rescue
-      nil
-    end
   end
 end

data/lib/email/provider.rb

@@ -3,6 +3,8 @@ module Email; end
 class Email::Provider
   def self.for_address(address)
     case
+    when address.end_with?("@fastmail.com")
+      new(:fastmail)
     when address.end_with?("@gmail.com")
       new(:gmail)
     when address.end_with?("@fastmail.fm")
@@ -19,7 +21,14 @@ class Email::Provider
   end
 
   def options
-    {port: 993, ssl: {ssl_version: :TLSv1_2}}
+    case provider
+    when :gmail
+      {port: 993, ssl: true}
+    when :fastmail
+      {port: 993, ssl: true}
+    else
+      {port: 993, ssl: true}
+    end
   end
 
   def host
@@ -27,7 +36,7 @@ class Email::Provider
     when :gmail
       "imap.gmail.com"
     when :fastmail
-      "mail.messagingengine.com"
+      "imap.fastmail.com"
     end
   end
 end

data/lib/imap/backup.rb

@@ -11,7 +11,8 @@ require "imap/backup/configuration/list"
 require "imap/backup/configuration/setup"
 require "imap/backup/configuration/store"
 require "imap/backup/downloader"
-require "imap/backup/serializer/base"
+require "imap/backup/uploader"
+require "imap/backup/serializer"
 require "imap/backup/serializer/mbox"
 require "imap/backup/version"
 require "email/provider"

data/lib/imap/backup/account/connection.rb

@@ -47,14 +47,44 @@ module Imap::Backup
       imap
       each_folder do |folder, serializer|
         Imap::Backup.logger.debug "[#{folder.name}] running backup"
+        serializer.set_uid_validity(folder.uid_validity)
         Downloader.new(folder, serializer).run
       end
     end
 
+    def restore
+      local_folders do |serializer, folder|
+        exists = folder.exist?
+        if exists
+          new_name = serializer.set_uid_validity(folder.uid_validity)
+          old_name = serializer.folder
+          if new_name
+            Imap::Backup.logger.debug "Backup '#{old_name}' renamed and restored to '#{new_name}'"
+            new_serializer = Serializer::Mbox.new(local_path, new_name)
+            new_folder = Account::Folder.new(self, new_name)
+            new_folder.create
+            new_serializer.force_uid_validity(new_folder.uid_validity)
+            Uploader.new(new_folder, new_serializer).run
+          else
+            Uploader.new(folder, serializer).run
+          end
+        else
+          folder.create
+          serializer.force_uid_validity(folder.uid_validity)
+          Uploader.new(folder, serializer).run
+        end
+      end
+    end
+
     def disconnect
       imap.disconnect
     end
 
+    def reconnect
+      disconnect
+      @imap = nil
+    end
+
     def imap
       return @imap unless @imap.nil?
       options = provider_options
@@ -95,6 +125,17 @@ module Imap::Backup
       (folders || []).map { |f| {name: f.name} }
     end
 
+    def local_folders
+      glob = File.join(local_path, "**", "*.imap")
+      base = Pathname.new(local_path)
+      Pathname.glob(glob) do |path|
+        name = path.relative_path_from(base).to_s[0..-6]
+        serializer = Serializer::Mbox.new(local_path, name)
+        folder = Account::Folder.new(self, name)
+        yield serializer, folder
+      end
+    end
+
     def provider
       @provider ||= Email::Provider.for_address(username)
     end

data/lib/imap/backup/account/folder.rb

@@ -16,14 +16,36 @@ module Imap::Backup
     def initialize(connection, name)
       @connection = connection
       @name = name
+      @uid_validity = nil
     end
 
+    # Deprecated: use #name
     def folder
       name
     end
 
+    def exist?
+      examine
+      true
+    rescue Net::IMAP::NoResponseError => e
+      false
+    end
+
+    def create
+      return if exist?
+      imap.create(name)
+    end
+
+    def uid_validity
+      @uid_validity ||=
+        begin
+          examine
+          imap.responses["UIDVALIDITY"][-1]
+        end
+    end
+
     def uids
-      imap.examine(name)
+      examine
       imap.uid_search(["ALL"]).sort
     rescue Net::IMAP::NoResponseError
       Imap::Backup.logger.warn "Folder '#{name}' does not exist"
@@ -31,7 +53,7 @@ module Imap::Backup
     end
 
     def fetch(uid)
-      imap.examine(name)
+      examine
       fetch_data_items = imap.uid_fetch([uid.to_i], REQUESTED_ATTRIBUTES)
       return nil if fetch_data_items.nil?
       fetch_data_item = fetch_data_items[0]
@@ -42,5 +64,23 @@ module Imap::Backup
       Imap::Backup.logger.warn "Folder '#{name}' does not exist"
       nil
     end
+
+    def append(message)
+      body = message.imap_body
+      date = message.date.to_time
+      response = imap.append(name, body, nil, date)
+      extract_uid(response)
+    end
+
+    private
+
+    def examine
+      imap.examine(name)
+    end
+
+    def extract_uid(response)
+      @uid_validity, uid = response.data.code.data.split(" ").map(&:to_i)
+      uid
+    end
   end
 end

data/lib/imap/backup/serializer.rb

@@ -0,0 +1,6 @@
+module Imap::Backup
+  module Serializer
+    DIRECTORY_PERMISSIONS = 0o700
+    FILE_PERMISSIONS      = 0o600
+  end
+end

data/lib/imap/backup/serializer/mbox.rb

@@ -1,112 +1,92 @@
-require "csv"
-require "email/mboxrd/message"
+require "imap/backup/serializer/mbox_store"
 
 module Imap::Backup
-  module Serializer; end
+  class Serializer::Mbox
+    attr_reader :path
+    attr_reader :folder
 
-  class Serializer::Mbox < Serializer::Base
     def initialize(path, folder)
-      super
-      create_containing_directory
-      assert_files
+      @path = path
+      @folder = folder
     end
 
-    # TODO: cleanup locks, close file handles
-
-    def uids
-      return @uids if @uids
-
-      @uids = []
-      return @uids if !exist?
-
-      CSV.foreach(imap_pathname) do |row|
-        @uids << row[0]
+    def set_uid_validity(value)
+      existing_uid_validity = store.uid_validity
+      case
+      when existing_uid_validity.nil?
+        store.uid_validity = value
+        nil
+      when existing_uid_validity == value
+        # NOOP
+        nil
+      else
+        digit = nil
+        new_name = nil
+        loop do
+          extra = digit ? ".#{digit}" : ""
+          new_name = "#{folder}.#{existing_uid_validity}#{extra}"
+          test_store = Serializer::MboxStore.new(path, new_name)
+          break if !test_store.exist?
+          digit ||= 0
+          digit += 1
+        end
+        store.rename new_name
+        @store = nil
+        store.uid_validity = value
+        new_name
       end
-      @uids = @uids.map(&:to_i).sort
-      @uids
     end
 
-    def save(uid, message)
-      uid = uid.to_s
-      if uids.include?(uid)
-        Imap::Backup.logger.debug(
-          "[#{folder}] message #{uid} already downloaded - skipping"
-        )
-        return
-      end
-
-      # invalidate cache
-      @uids = nil
-
-      body = message["RFC822"]
-      mboxrd_message = Email::Mboxrd::Message.new(body)
-      mbox = imap = nil
-      begin
-        mbox = File.open(mbox_pathname, "ab")
-        imap = File.open(imap_pathname, "ab")
-        mbox.write mboxrd_message.to_serialized
-        imap.write uid + "\n"
-      rescue => e
-        message = <<-ERROR.gsub(/^\s*/m, "")
-          [#{folder}] failed to save message #{uid}:
-          #{body}. #{e}:
-          #{e.backtrace.join("\n")}"
-        ERROR
-        Imap::Backup.logger.warn message
-      ensure
-        mbox.close if mbox
-        imap.close if imap
-      end
+    def force_uid_validity(value)
+      store.uid_validity = value
     end
 
-    private
-
-    def assert_files
-      mbox = mbox_exist?
-      imap = imap_exist?
-      raise ".imap file missing" if mbox && (!imap)
-      raise ".mbox file missing" if imap && (!mbox)
+    def uids
+      store.uids
     end
 
-    def create_containing_directory
-      mbox_relative_path = File.dirname(mbox_relative_pathname)
-      return if mbox_relative_path == "."
-      Utils.make_folder(
-        path, mbox_relative_path, Serializer::DIRECTORY_PERMISSIONS
-      )
+    def load(uid)
+      store.load(uid)
     end
 
-    def exist?
-      mbox_exist? && imap_exist?
+    def save(uid, message)
+      store.add(uid, message)
     end
 
-    def mbox_exist?
-      File.exist?(mbox_pathname)
+    def rename(new_name)
+      @folder = new_name
+      store.rename new_name
     end
 
-    def imap_exist?
-      File.exist?(imap_pathname)
+    def update_uid(old, new)
+      store.update_uid old, new
     end
 
-    def mbox_relative_pathname
-      folder + ".mbox"
-    end
+    private
 
-    def mbox_pathname
-      File.join(path, mbox_relative_pathname)
+    def store
+      @store ||=
+        begin
+          create_containing_directory
+          Serializer::MboxStore.new(path, folder)
+        end
     end
 
-    def imap_pathname
-      filename = folder + ".imap"
-      File.join(path, filename)
-    end
+    def create_containing_directory
+      relative_path = File.dirname(folder)
+      containing_directory = File.join(path, relative_path)
+      full_path = File.expand_path(containing_directory)
 
-    def lock
-      # lock mbox and imap files
-      # create both empty if missing
-    end
+      if !File.directory?(full_path)
+        Imap::Backup::Utils.make_folder(
+          path, relative_path, Serializer::DIRECTORY_PERMISSIONS
+        )
+      end
 
-    def unlock
+      if Imap::Backup::Utils.mode(full_path) !=
+          Serializer::DIRECTORY_PERMISSIONS
+        FileUtils.chmod Serializer::DIRECTORY_PERMISSIONS, full_path
+      end
     end
   end
 end