imap-backup 6.0.0.rc2 → 6.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4f9cac7966fac9858fdc380d8e9638c8d479035ee4053444a104b8c3eb5a304c
-  data.tar.gz: 5d9785b5e34d90535c8052d216d9b3dbf46ea743672f05129604729662f42f34
+  metadata.gz: 2a05fedc57275cadd2a1bd22efbe88ef5ec6ee72843400499cac8b8df8e80c35
+  data.tar.gz: f0e8e859b5fb1bb8480730b3ea62abc7470f5d0ddb469113bbdac0e4f2a88d1a
 SHA512:
-  metadata.gz: a4595e4316dcdc20123a8ed2bec524d37c1de95da49281a30484349c317a996cf3c6946617a489880a0c41db41698b2ee862d9aee4f9c9b8fa34d67707ba924c
-  data.tar.gz: ccee268de9cb6225c72fe0cb11a2552fc793f664429d8dfa918e5b4697961c8e5e15e47ed653dac241f377385806443f17683614b8ff76075a8ff2f12e07f98c
+  metadata.gz: bfb4de1a3048a9f29ee30fb56c7493524a7d688a6eab2ec3307005ba3c8d341a99c961acd1d8f588384dc91571b4528482931e52724df42dff19a26553568fb0
+  data.tar.gz: a411ad26b4ce3eb631c97d94c9242c1eb48fd57d23cffcec84f5c829d9354b506115b6c8278ee38430e27f0d6ca1400d55f74adce5eb99f94db2963f235c6d7a

data/imap-backup.gemspec

@@ -18,7 +18,6 @@ Gem::Specification.new do |gem|
   gem.files         += %w[LICENSE README.md]
 
   gem.executables   = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) }
-  gem.test_files    = Dir.glob("spec/**/*{.rb,.yml}")
   gem.require_paths = ["lib"]
   gem.required_ruby_version = ">= 2.5"
 
@@ -34,4 +33,9 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency "rspec", ">= 3.0.0"
   gem.add_development_dependency "rubocop-rspec"
   gem.add_development_dependency "simplecov"
+  gem.add_development_dependency "yard"
+
+  gem.metadata = {
+    "rubygems_mfa_required" => "true"
+  }
 end

data/lib/cli_coverage.rb

@@ -1,18 +1,18 @@
 class CliCoverage
   def self.conditionally_activate
-    if ENV["COVERAGE"]
-      require "simplecov"
+    return if !ENV["COVERAGE"]
 
-      # Collect coverage separately
-      SimpleCov.command_name "#{ENV['COVERAGE']} #{ARGV.join(' ')} coverage"
+    require "simplecov"
 
-      # Silence output
-      SimpleCov.formatter = SimpleCov::Formatter::SimpleFormatter
-      SimpleCov.print_error_status = false
+    # Collect coverage separately
+    SimpleCov.command_name "#{ENV['COVERAGE']} #{ARGV.join(' ')} coverage"
 
-      # Ensure SimpleCov doesn't filter out all out code
-      project_root = File.expand_path("..", __dir__)
-      SimpleCov.root project_root
-    end
+    # Silence output
+    SimpleCov.formatter = SimpleCov::Formatter::SimpleFormatter
+    SimpleCov.print_error_status = false
+
+    # Ensure SimpleCov doesn't filter out all out code
+    project_root = File.expand_path("..", __dir__)
+    SimpleCov.root project_root
   end
 end

data/lib/email/provider/apple_mail.rb

@@ -4,4 +4,8 @@ class Email::Provider::AppleMail < Email::Provider::Base
   def host
     "imap.mail.me.com"
   end
+
+  def sets_seen_flags_on_fetch?
+    true
+  end
 end

data/lib/email/provider/base.rb

@@ -3,6 +3,12 @@ class Email::Provider; end
 
 class Email::Provider::Base
   def options
+    # rubocop:disable Naming/VariableNumber
     {port: 993, ssl: {ssl_version: :TLSv1_2}}
+    # rubocop:enable Naming/VariableNumber
+  end
+
+  def sets_seen_flags_on_fetch?
+    false
   end
 end

data/lib/email/provider/purelymail.rb

@@ -0,0 +1,11 @@
+require "email/provider/base"
+
+class Email::Provider::Purelymail < Email::Provider::Base
+  def host
+    "mailserver.purelymail.com"
+  end
+
+  def sets_seen_flags_on_fetch?
+    true
+  end
+end

data/lib/email/provider/unknown.rb

@@ -6,6 +6,8 @@ class Email::Provider::Unknown < Email::Provider::Base
   end
 
   def options
+    # rubocop:disable Naming/VariableNumber
     {port: 993, ssl: {ssl_version: :TLSv1_2}}
+    # rubocop:enable Naming/VariableNumber
   end
 end

data/lib/email/provider.rb

@@ -1,12 +1,14 @@
 require "email/provider/apple_mail"
 require "email/provider/fastmail"
 require "email/provider/gmail"
+require "email/provider/purelymail"
 require "email/provider/unknown"
 
 module Email; end
 
 class Email::Provider
   def self.for_address(address)
+    # rubocop:disable Lint/DuplicateBranch
     case
     when address.end_with?("@fastmail.com")
       Email::Provider::Fastmail.new
@@ -20,8 +22,11 @@ class Email::Provider
       Email::Provider::AppleMail.new
     when address.end_with?("@me.com")
       Email::Provider::AppleMail.new
+    when address.end_with?("@purelymail.com")
+      Email::Provider::Purelymail.new
     else
       Email::Provider::Unknown.new
     end
+    # rubocop:enable Lint/DuplicateBranch
   end
 end

data/lib/imap/backup/account/connection/backup_folders.rb

@@ -0,0 +1,27 @@
+module Imap::Backup
+  class Account; end
+  class Account::Connection; end
+
+  class Account::Connection::BackupFolders
+    attr_reader :account
+    attr_reader :client
+
+    def initialize(client:, account:)
+      @client = client
+      @account = account
+    end
+
+    def run
+      names =
+        if account.folders&.any?
+          account.folders.map { |af| af[:name] }
+        else
+          Account::Connection::FolderNames.new(client: client, account: account).run
+        end
+
+      names.map do |name|
+        Account::Folder.new(account.connection, name)
+      end
+    end
+  end
+end

data/lib/imap/backup/account/connection/client_factory.rb

@@ -0,0 +1,55 @@
+require "email/provider"
+require "retry_on_error"
+
+module Imap::Backup
+  class Account::Connection::ClientFactory
+    include RetryOnError
+
+    LOGIN_RETRY_CLASSES = [EOFError, Errno::ECONNRESET, SocketError].freeze
+
+    attr_reader :account
+
+    def initialize(account:)
+      @account = account
+      @provider = nil
+      @server = nil
+    end
+
+    def run
+      retry_on_error(errors: LOGIN_RETRY_CLASSES) do
+        options = provider_options
+        Logger.logger.debug(
+          "Creating IMAP instance: #{server}, options: #{options.inspect}"
+        )
+        client =
+          if provider.is_a?(Email::Provider::AppleMail)
+            Client::AppleMail.new(server, options)
+          else
+            Client::Default.new(server, options)
+          end
+        Logger.logger.debug "Logging in: #{account.username}/#{masked_password}"
+        client.login(account.username, account.password)
+        Logger.logger.debug "Login complete"
+        client
+      end
+    end
+
+    private
+
+    def masked_password
+      account.password.gsub(/./, "x")
+    end
+
+    def provider
+      @provider ||= Email::Provider.for_address(account.username)
+    end
+
+    def provider_options
+      provider.options.merge(account.connection_options || {})
+    end
+
+    def server
+      @server ||= account.server || provider.host
+    end
+  end
+end

data/lib/imap/backup/account/connection/folder_names.rb

@@ -0,0 +1,26 @@
+module Imap::Backup
+  class Account; end
+  class Account::Connection; end
+
+  class Account::Connection::FolderNames
+    attr_reader :account
+    attr_reader :client
+
+    def initialize(client:, account:)
+      @client = client
+      @account = account
+    end
+
+    def run
+      folder_names = client.list
+
+      if folder_names.empty?
+        message = "Unable to get folder list for account #{account.username}"
+        Logger.logger.info message
+        raise message
+      end
+
+      folder_names
+    end
+  end
+end

data/lib/imap/backup/account/connection.rb

@@ -1,17 +1,14 @@
 require "imap/backup/client/apple_mail"
 require "imap/backup/client/default"
+require "imap/backup/account/connection/backup_folders"
+require "imap/backup/account/connection/client_factory"
+require "imap/backup/account/connection/folder_names"
 require "imap/backup/serializer/directory"
 
-require "retry_on_error"
-
 module Imap::Backup
   class Account; end
 
   class Account::Connection
-    include RetryOnError
-
-    LOGIN_RETRY_CLASSES = [EOFError, Errno::ECONNRESET, SocketError].freeze
-
     attr_reader :account
 
     def initialize(account)
@@ -20,34 +17,12 @@ module Imap::Backup
     end
 
     def folder_names
-      @folder_names ||=
-        begin
-          folder_names = client.list
-
-          if folder_names.empty?
-            message = "Unable to get folder list for account #{account.username}"
-            Imap::Backup::Logger.logger.info message
-            raise message
-          end
-
-          folder_names
-        end
+      @folder_names ||= Account::Connection::FolderNames.new(client: client, account: account).run
     end
 
     def backup_folders
       @backup_folders ||=
-        begin
-          names =
-            if account.folders&.any?
-              account.folders.map { |af| af[:name] }
-            else
-              folder_names
-            end
-
-          names.map do |name|
-            Account::Folder.new(self, name)
-          end
-        end
+        Account::Connection::BackupFolders.new(client: client, account: account).run
     end
 
     def status
@@ -59,18 +34,21 @@ module Imap::Backup
     end
 
     def run_backup
-      Imap::Backup::Logger.logger.debug "Running backup of account: #{account.username}"
+      Logger.logger.debug "Running backup of account: #{account.username}"
       # start the connection so we get logging messages in the right order
       client
       ensure_account_folder
       each_folder do |folder, serializer|
         next if !folder.exist?
 
-        Imap::Backup::Logger.logger.debug "[#{folder.name}] running backup"
+        Logger.logger.debug "[#{folder.name}] running backup"
         serializer.apply_uid_validity(folder.uid_validity)
         begin
           Downloader.new(
-            folder, serializer, multi_fetch_size: account.multi_fetch_size
+            folder,
+            serializer,
+            multi_fetch_size: account.multi_fetch_size,
+            reset_seen_flags_after_fetch: account.reset_seen_flags_after_fetch
           ).run
         rescue Net::IMAP::ByeResponseError
           reconnect
@@ -95,7 +73,7 @@ module Imap::Backup
 
     def restore
       local_folders do |serializer, folder|
-        restore_folder serializer, folder
+        Uploader.new(folder, serializer).run
       end
     end
 
@@ -112,32 +90,11 @@ module Imap::Backup
       @backup_folders = nil
       @client = nil
       @folder_names = nil
-      @provider = nil
-      @server = nil
     end
 
+    # TODO: make this private
     def client
-      @client ||=
-        retry_on_error(errors: LOGIN_RETRY_CLASSES) do
-          options = provider_options
-          Imap::Backup::Logger.logger.debug(
-            "Creating IMAP instance: #{server}, options: #{options.inspect}"
-          )
-          client =
-            if provider.is_a?(Email::Provider::AppleMail)
-              Client::AppleMail.new(server, options)
-            else
-              Client::Default.new(server, options)
-            end
-          Imap::Backup::Logger.logger.debug "Logging in: #{account.username}/#{masked_password}"
-          client.login(account.username, account.password)
-          Imap::Backup::Logger.logger.debug "Login complete"
-          client
-        end
-    end
-
-    def server
-      @server ||= account.server || provider.host
+      @client ||= Account::Connection::ClientFactory.new(account: account).run
     end
 
     private
@@ -149,51 +106,14 @@ module Imap::Backup
       end
     end
 
-    def restore_folder(serializer, folder)
-      existing_uids = folder.uids
-      if existing_uids.any?
-        Imap::Backup::Logger.logger.debug(
-          "There's already a '#{folder.name}' folder with emails"
-        )
-        new_name = serializer.apply_uid_validity(folder.uid_validity)
-        old_name = serializer.folder
-        if new_name
-          Imap::Backup::Logger.logger.debug(
-            "Backup '#{old_name}' renamed and restored to '#{new_name}'"
-          )
-          new_serializer = Serializer.new(account.local_path, new_name)
-          new_folder = Account::Folder.new(self, new_name)
-          new_folder.create
-          new_serializer.force_uid_validity(new_folder.uid_validity)
-          Uploader.new(new_folder, new_serializer).run
-        else
-          Uploader.new(folder, serializer).run
-        end
-      else
-        folder.create
-        serializer.force_uid_validity(folder.uid_validity)
-        Uploader.new(folder, serializer).run
-      end
-    end
-
     def ensure_account_folder
+      raise "The backup path for #{account.username} is not set" if !account.local_path
+
       Utils.make_folder(
         File.dirname(account.local_path),
         File.basename(account.local_path),
         Serializer::Directory::DIRECTORY_PERMISSIONS
       )
     end
-
-    def masked_password
-      account.password.gsub(/./, "x")
-    end
-
-    def provider
-      @provider ||= Email::Provider.for_address(account.username)
-    end
-
-    def provider_options
-      provider.options.merge(account.connection_options || {})
-    end
   end
 end

data/lib/imap/backup/account/folder.rb

@@ -12,7 +12,8 @@ module Imap::Backup
     include RetryOnError
 
     BODY_ATTRIBUTE = "BODY[]".freeze
-    UID_FETCH_RETRY_CLASSES = [EOFError].freeze
+    UID_FETCH_RETRY_CLASSES = [EOFError, Errno::ECONNRESET, IOError].freeze
+    APPEND_RETRY_CLASSES = [Net::IMAP::BadResponseError].freeze
 
     attr_reader :connection
     attr_reader :name
@@ -64,7 +65,7 @@ module Imap::Backup
         in `search_internal` in stdlib net/imap.rb.
         This is caused by `@responses["SEARCH"] being unset/undefined
       MESSAGE
-      Imap::Backup::Logger.logger.warn message
+      Logger.logger.warn message
       []
     end
 
@@ -91,29 +92,50 @@ module Imap::Backup
     def append(message)
       body = message.imap_body
       date = message.date&.to_time
-      response = client.append(utf7_encoded_name, body, nil, date)
-      extract_uid(response)
+      retry_on_error(errors: APPEND_RETRY_CLASSES, limit: 3) do
+        response = client.append(utf7_encoded_name, body, nil, date)
+        extract_uid(response)
+      end
     end
 
-    def clear
-      existing = uids
+    def set_flags(uids, flags)
       # Use read-write access, via `select`
       client.select(utf7_encoded_name)
-      client.uid_store(existing, "+FLAGS", [:Deleted])
+      client.uid_store(uids, "+FLAGS", flags)
+    end
+
+    def unset_flags(uids, flags)
+      client.select(utf7_encoded_name)
+      client.uid_store(uids, "-FLAGS", flags)
+    end
+
+    def clear
+      set_flags(uids, [:Deleted])
       client.expunge
     end
 
+    def unseen(uids)
+      messages = uids.map(&:to_s).join(",")
+      examine
+      client.uid_search([messages, "UNSEEN"])
+    rescue NoMethodError
+      # Apple Mail returns an empty response when searches have no results
+      []
+    rescue FolderNotFound
+      nil
+    end
+
     private
 
     def examine
       client.examine(utf7_encoded_name)
     rescue Net::IMAP::NoResponseError
-      Imap::Backup::Logger.logger.warn "Folder '#{name}' does not exist on server"
+      Logger.logger.warn "Folder '#{name}' does not exist on server"
       raise FolderNotFound, "Folder '#{name}' does not exist on server"
     end
 
     def extract_uid(response)
-      @uid_validity, uid = response.data.code.data.split(" ").map(&:to_i)
+      @uid_validity, uid = response.data.code.data.split.map(&:to_i)
       uid
     end
 

data/lib/imap/backup/account.rb

@@ -8,6 +8,7 @@ module Imap::Backup
     attr_reader :folders
     attr_reader :server
     attr_reader :connection_options
+    attr_reader :reset_seen_flags_after_fetch
     attr_reader :changes
 
     def initialize(options)
@@ -18,12 +19,14 @@ module Imap::Backup
       @server = options[:server]
       @connection_options = options[:connection_options]
       @multi_fetch_size = options[:multi_fetch_size]
+      @reset_seen_flags_after_fetch = options[:reset_seen_flags_after_fetch]
+      @connection = nil
       @changes = {}
       @marked_for_deletion = false
     end
 
     def connection
-      Account::Connection.new(self)
+      @connection ||= Account::Connection.new(self)
     end
 
     def valid?
@@ -47,15 +50,15 @@ module Imap::Backup
     end
 
     def to_h
-      h = {
-        username: @username,
-        password: @password,
-      }
+      h = {username: @username, password: @password}
       h[:local_path] = @local_path if @local_path
       h[:folders] = @folders if @folders
       h[:server] = @server if @server
       h[:connection_options] = @connection_options if @connection_options
       h[:multi_fetch_size] = multi_fetch_size if @multi_fetch_size
+      if @reset_seen_flags_after_fetch
+        h[:reset_seen_flags_after_fetch] = @reset_seen_flags_after_fetch
+      end
       h
     end
 
@@ -73,6 +76,7 @@ module Imap::Backup
 
     def folders=(value)
       raise "folders must be an Array" if !value.is_a?(Array)
+
       update(:folders, value)
     end
 
@@ -100,6 +104,10 @@ module Imap::Backup
       update(:multi_fetch_size, parsed)
     end
 
+    def reset_seen_flags_after_fetch=(value)
+      update(:reset_seen_flags_after_fetch, value)
+    end
+
     private
 
     def update(field, value)
@@ -113,9 +121,10 @@ module Imap::Backup
         end
       else
         current = instance_variable_get(key)
-        changes[field] = {from: current, to: value}
+        changes[field] = {from: current, to: value} if value != current
       end
 
+      @connection = nil
       instance_variable_set(key, value)
     end
   end

data/lib/imap/backup/cli/backup.rb

@@ -12,9 +12,7 @@ module Imap::Backup
 
     no_commands do
       def run
-        each_connection(account_names) do |connection|
-          connection.run_backup
-        end
+        each_connection(account_names, &:run_backup)
       end
     end
   end

data/lib/imap/backup/cli/helpers.rb

@@ -1,35 +1,37 @@
 require "imap/backup"
 require "imap/backup/cli/accounts"
 
-module Imap::Backup::CLI::Helpers
-  def symbolized(options)
-    options.each.with_object({}) do |(k, v), acc|
-      key = k.gsub("-", "_").intern
-      acc[key] = v
+module Imap::Backup
+  module CLI::Helpers
+    def symbolized(options)
+      options.each.with_object({}) do |(k, v), acc|
+        key = k.gsub("-", "_").intern
+        acc[key] = v
+      end
     end
-  end
 
-  def account(email)
-    accounts = Imap::Backup::CLI::Accounts.new
-    account = accounts.find { |a| a.username == email }
-    raise "#{email} is not a configured account" if !account
+    def account(email)
+      accounts = CLI::Accounts.new
+      account = accounts.find { |a| a.username == email }
+      raise "#{email} is not a configured account" if !account
 
-    account
-  end
+      account
+    end
 
-  def connection(email)
-    account = account(email)
+    def connection(email)
+      account = account(email)
 
-    Imap::Backup::Account::Connection.new(account)
-  end
+      Account::Connection.new(account)
+    end
 
-  def each_connection(names)
-    accounts = Imap::Backup::CLI::Accounts.new(names)
+    def each_connection(names)
+      accounts = CLI::Accounts.new(names)
 
-    accounts.each do |account|
-      yield account.connection
+      accounts.each do |account|
+        yield account.connection
+      end
+    rescue ConfigurationNotFound
+      raise "imap-backup is not configured. Run `imap-backup setup`"
     end
-  rescue Imap::Backup::ConfigurationNotFound
-    raise "imap-backup is not configured. Run `imap-backup setup`"
   end
 end