imap-backup 5.2.0 → 6.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 164962561883b82372b32826843681c6a802e88982f4906778758a073e1479d8
-  data.tar.gz: 8a71ad18952f675f61bc6342a3a9a60c2fc71797444142c2af5f98ef18ac13e9
+  metadata.gz: 6d4b9386097e77af34706992fb42b6ff9edd3a62a3cfb98e0b3363d864ee75dc
+  data.tar.gz: d72ec8493ea2436ccb033f2957d44236b71d3c607e25b15b0b755e93f011da10
 SHA512:
-  metadata.gz: 600ff4b72954cecfb2037b9ef12b16029248a6c6e27b2b801661b5c746264a8aed7a99aa689cbeecc8e89f571b9f4c26350669e582bc087a518981bceb677141
-  data.tar.gz: b706026f0b8231f10dfbb0a0c39162ccdf6a8671c513b6cbf637352dcd8d99987a3f93d2b523db7f4a8e3ff7178877dc3d84c1ad506245a9773330abea2a792f
+  metadata.gz: ab38ba548fa009c6f718f8da9b9fbb971751297eac03d2839e14f247c06d6858a155350291619090ef8165ffeb865674a19ef51c13dbf1220928106abfc79fa5
+  data.tar.gz: ad4e17a2b6b0b5a9ee1cc4737c324981069525a129018278514e51e8664008c71fc610539df58a2da590084bb7e3c46cd635fe1409ba0b53ad56bd241edfb9c8

data/README.md

@@ -1,3 +1,8 @@
+![Version](https://img.shields.io/gem/v/imap-backup?label=Version&logo=rubygems)
+[![Build Status](https://github.com/joeyates/imap-backup/actions/workflows/main.yml/badge.svg)][CI Status]
+![Coverage](https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/joeyates/b54fe758bfb405c04bef72dad293d707/raw/coverage.json)
+![License](https://img.shields.io/github/license/joeyates/imap-backup?color=brightgreen&label=License)
+
 # imap-backup
 
 *Backup GMail (or other IMAP) accounts to disk*
@@ -5,10 +10,12 @@
   * [Source Code]
   * [API documentation]
   * [Rubygem]
+  * [CI Status]
 
 [Source Code]: https://github.com/joeyates/imap-backup "Source code at GitHub"
-[API documentation]: http://rubydoc.info/gems/imap-backup/frames "RDoc API Documentation at Rubydoc.info"
-[Rubygem]: http://rubygems.org/gems/imap-backup "Ruby gem at rubygems.org"
+[API documentation]: https://rubydoc.info/gems/imap-backup/frames "RDoc API Documentation at Rubydoc.info"
+[Rubygem]: https://rubygems.org/gems/imap-backup "Ruby gem at rubygems.org"
+[CI Status]: https://github.com/joeyates/imap-backup/actions/workflows/main.yml
 
 # Installation
 

data/docs/development.md

@@ -1,10 +1,10 @@
 # Testing
 
-## Integration Tests
+## Feature Specs
 
-Integration tests (feature specs) are run against a local IMAP server
-controlled by Docker Compose, which needs to be started
-before running the test suite.
+Specs under `specs/features` are integration specs run against a local IMAP server
+controlled by Docker Compose.
+Before running the test suite, it needs to be started:
 
 ```sh
 $ docker-compose up -d
@@ -26,6 +26,12 @@ or
 $ rspec --tag ~docker
 ```
 
+### Debugging
+
+The feature specs are run 'out of process' via the Aruba gem.
+In order to see debugging output from the process,
+use `last_command_started.output`.
+
 ## Access Docker imap server
 
 ```ruby

data/imap-backup.gemspec

@@ -18,7 +18,6 @@ Gem::Specification.new do |gem|
   gem.files         += %w[LICENSE README.md]
 
   gem.executables   = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) }
-  gem.test_files    = Dir.glob("spec/**/*{.rb,.yml}")
   gem.require_paths = ["lib"]
   gem.required_ruby_version = ">= 2.5"
 
@@ -34,4 +33,9 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency "rspec", ">= 3.0.0"
   gem.add_development_dependency "rubocop-rspec"
   gem.add_development_dependency "simplecov"
+  gem.add_development_dependency "yard"
+
+  gem.metadata = {
+    "rubygems_mfa_required" => "true"
+  }
 end

data/lib/cli_coverage.rb

@@ -1,18 +1,18 @@
 class CliCoverage
   def self.conditionally_activate
-    if ENV["COVERAGE"]
-      require "simplecov"
+    return if !ENV["COVERAGE"]
 
-      # Collect coverage separately
-      SimpleCov.command_name "#{ENV['COVERAGE']} coverage"
+    require "simplecov"
 
-      # Silence output
-      SimpleCov.formatter = SimpleCov::Formatter::SimpleFormatter
-      SimpleCov.print_error_status = false
+    # Collect coverage separately
+    SimpleCov.command_name "#{ENV['COVERAGE']} #{ARGV.join(' ')} coverage"
 
-      # Ensure SimpleCov doesn't filter out all out code
-      project_root = File.expand_path("..", __dir__)
-      SimpleCov.root project_root
-    end
+    # Silence output
+    SimpleCov.formatter = SimpleCov::Formatter::SimpleFormatter
+    SimpleCov.print_error_status = false
+
+    # Ensure SimpleCov doesn't filter out all out code
+    project_root = File.expand_path("..", __dir__)
+    SimpleCov.root project_root
   end
 end

data/lib/email/provider/base.rb

@@ -3,6 +3,8 @@ class Email::Provider; end
 
 class Email::Provider::Base
   def options
+    # rubocop:disable Naming/VariableNumber
     {port: 993, ssl: {ssl_version: :TLSv1_2}}
+    # rubocop:enable Naming/VariableNumber
   end
 end

data/lib/email/provider/unknown.rb

@@ -6,6 +6,8 @@ class Email::Provider::Unknown < Email::Provider::Base
   end
 
   def options
+    # rubocop:disable Naming/VariableNumber
     {port: 993, ssl: {ssl_version: :TLSv1_2}}
+    # rubocop:enable Naming/VariableNumber
   end
 end

data/lib/email/provider.rb

@@ -7,6 +7,7 @@ module Email; end
 
 class Email::Provider
   def self.for_address(address)
+    # rubocop:disable Lint/DuplicateBranch
     case
     when address.end_with?("@fastmail.com")
       Email::Provider::Fastmail.new
@@ -23,5 +24,6 @@ class Email::Provider
     else
       Email::Provider::Unknown.new
     end
+    # rubocop:enable Lint/DuplicateBranch
   end
 end

data/lib/imap/backup/account/connection/backup_folders.rb

@@ -0,0 +1,27 @@
+module Imap::Backup
+  class Account; end
+  class Account::Connection; end
+
+  class Account::Connection::BackupFolders
+    attr_reader :account
+    attr_reader :client
+
+    def initialize(client:, account:)
+      @client = client
+      @account = account
+    end
+
+    def run
+      names =
+        if account.folders&.any?
+          account.folders.map { |af| af[:name] }
+        else
+          Account::Connection::FolderNames.new(client: client, account: account).run
+        end
+
+      names.map do |name|
+        Account::Folder.new(account.connection, name)
+      end
+    end
+  end
+end

data/lib/imap/backup/account/connection/client_factory.rb

@@ -0,0 +1,54 @@
+require "retry_on_error"
+
+module Imap::Backup
+  class Account::Connection::ClientFactory
+    include RetryOnError
+
+    LOGIN_RETRY_CLASSES = [EOFError, Errno::ECONNRESET, SocketError].freeze
+
+    attr_reader :account
+
+    def initialize(account:)
+      @account = account
+      @provider = nil
+      @server = nil
+    end
+
+    def run
+      retry_on_error(errors: LOGIN_RETRY_CLASSES) do
+        options = provider_options
+        Logger.logger.debug(
+          "Creating IMAP instance: #{server}, options: #{options.inspect}"
+        )
+        client =
+          if provider.is_a?(Email::Provider::AppleMail)
+            Client::AppleMail.new(server, options)
+          else
+            Client::Default.new(server, options)
+          end
+        Logger.logger.debug "Logging in: #{account.username}/#{masked_password}"
+        client.login(account.username, account.password)
+        Logger.logger.debug "Login complete"
+        client
+      end
+    end
+
+    private
+
+    def masked_password
+      account.password.gsub(/./, "x")
+    end
+
+    def provider
+      @provider ||= Email::Provider.for_address(account.username)
+    end
+
+    def provider_options
+      provider.options.merge(account.connection_options || {})
+    end
+
+    def server
+      @server ||= account.server || provider.host
+    end
+  end
+end

data/lib/imap/backup/account/connection/folder_names.rb

@@ -0,0 +1,26 @@
+module Imap::Backup
+  class Account; end
+  class Account::Connection; end
+
+  class Account::Connection::FolderNames
+    attr_reader :account
+    attr_reader :client
+
+    def initialize(client:, account:)
+      @client = client
+      @account = account
+    end
+
+    def run
+      folder_names = client.list
+
+      if folder_names.empty?
+        message = "Unable to get folder list for account #{account.username}"
+        Logger.logger.info message
+        raise message
+      end
+
+      folder_names
+    end
+  end
+end

data/lib/imap/backup/account/connection.rb

@@ -1,16 +1,15 @@
+require "email/provider"
 require "imap/backup/client/apple_mail"
 require "imap/backup/client/default"
-
-require "retry_on_error"
+require "imap/backup/account/connection/backup_folders"
+require "imap/backup/account/connection/client_factory"
+require "imap/backup/account/connection/folder_names"
+require "imap/backup/serializer/directory"
 
 module Imap::Backup
   class Account; end
 
   class Account::Connection
-    include RetryOnError
-
-    LOGIN_RETRY_CLASSES = [EOFError, Errno::ECONNRESET, SocketError].freeze
-
     attr_reader :account
 
     def initialize(account)
@@ -19,57 +18,35 @@ module Imap::Backup
     end
 
     def folder_names
-      @folder_names ||=
-        begin
-          folder_names = client.list
-
-          if folder_names.empty?
-            message = "Unable to get folder list for account #{account.username}"
-            Imap::Backup::Logger.logger.info message
-            raise message
-          end
-
-          folder_names
-        end
+      @folder_names ||= Account::Connection::FolderNames.new(client: client, account: account).run
     end
 
     def backup_folders
       @backup_folders ||=
-        begin
-          names =
-            if account.folders&.any?
-              account.folders.map { |af| af[:name] }
-            else
-              folder_names
-            end
-
-          names.map do |name|
-            Account::Folder.new(self, name)
-          end
-        end
+        Account::Connection::BackupFolders.new(client: client, account: account).run
     end
 
     def status
       ensure_account_folder
       backup_folders.map do |folder|
-        s = Serializer::Mbox.new(account.local_path, folder.name)
+        s = Serializer.new(account.local_path, folder.name)
         {name: folder.name, local: s.uids, remote: folder.uids}
       end
     end
 
     def run_backup
-      Imap::Backup::Logger.logger.debug "Running backup of account: #{account.username}"
+      Logger.logger.debug "Running backup of account: #{account.username}"
       # start the connection so we get logging messages in the right order
       client
       ensure_account_folder
       each_folder do |folder, serializer|
         next if !folder.exist?
 
-        Imap::Backup::Logger.logger.debug "[#{folder.name}] running backup"
+        Logger.logger.debug "[#{folder.name}] running backup"
         serializer.apply_uid_validity(folder.uid_validity)
         begin
           Downloader.new(
-            folder, serializer, block_size: config.download_block_size
+            folder, serializer, multi_fetch_size: account.multi_fetch_size
           ).run
         rescue Net::IMAP::ByeResponseError
           reconnect
@@ -86,7 +63,7 @@ module Imap::Backup
       base = Pathname.new(account.local_path)
       Pathname.glob(glob) do |path|
         name = path.relative_path_from(base).to_s[0..-6]
-        serializer = Serializer::Mbox.new(account.local_path, name)
+        serializer = Serializer.new(account.local_path, name)
         folder = Account::Folder.new(self, name)
         yield serializer, folder
       end
@@ -94,7 +71,7 @@ module Imap::Backup
 
     def restore
       local_folders do |serializer, folder|
-        restore_folder serializer, folder
+        Uploader.new(folder, serializer).run
       end
     end
 
@@ -110,94 +87,29 @@ module Imap::Backup
     def reset
       @backup_folders = nil
       @client = nil
-      @config = nil
       @folder_names = nil
-      @provider = nil
-      @server = nil
     end
 
+    # TODO: make this private
     def client
-      @client ||=
-        retry_on_error(errors: LOGIN_RETRY_CLASSES) do
-          options = provider_options
-          Imap::Backup::Logger.logger.debug(
-            "Creating IMAP instance: #{server}, options: #{options.inspect}"
-          )
-          client =
-            if provider.is_a?(Email::Provider::AppleMail)
-              Client::AppleMail.new(server, options)
-            else
-              Client::Default.new(server, options)
-            end
-          Imap::Backup::Logger.logger.debug "Logging in: #{account.username}/#{masked_password}"
-          client.login(account.username, account.password)
-          Imap::Backup::Logger.logger.debug "Login complete"
-          client
-        end
-    end
-
-    def server
-      @server ||= account.server || provider.host
+      @client ||= Account::Connection::ClientFactory.new(account: account).run
     end
 
     private
 
     def each_folder
       backup_folders.each do |folder|
-        serializer = Serializer::Mbox.new(account.local_path, folder.name)
+        serializer = Serializer.new(account.local_path, folder.name)
         yield folder, serializer
       end
     end
 
-    def restore_folder(serializer, folder)
-      existing_uids = folder.uids
-      if existing_uids.any?
-        Imap::Backup::Logger.logger.debug(
-          "There's already a '#{folder.name}' folder with emails"
-        )
-        new_name = serializer.apply_uid_validity(folder.uid_validity)
-        old_name = serializer.folder
-        if new_name
-          Imap::Backup::Logger.logger.debug(
-            "Backup '#{old_name}' renamed and restored to '#{new_name}'"
-          )
-          new_serializer = Serializer::Mbox.new(account.local_path, new_name)
-          new_folder = Account::Folder.new(self, new_name)
-          new_folder.create
-          new_serializer.force_uid_validity(new_folder.uid_validity)
-          Uploader.new(new_folder, new_serializer).run
-        else
-          Uploader.new(folder, serializer).run
-        end
-      else
-        folder.create
-        serializer.force_uid_validity(folder.uid_validity)
-        Uploader.new(folder, serializer).run
-      end
-    end
-
     def ensure_account_folder
       Utils.make_folder(
         File.dirname(account.local_path),
         File.basename(account.local_path),
-        Serializer::DIRECTORY_PERMISSIONS
+        Serializer::Directory::DIRECTORY_PERMISSIONS
       )
     end
-
-    def masked_password
-      account.password.gsub(/./, "x")
-    end
-
-    def provider
-      @provider ||= Email::Provider.for_address(account.username)
-    end
-
-    def provider_options
-      provider.options.merge(account.connection_options || {})
-    end
-
-    def config
-      @config ||= Configuration.new
-    end
   end
 end

data/lib/imap/backup/account/folder.rb

@@ -12,7 +12,8 @@ module Imap::Backup
     include RetryOnError
 
     BODY_ATTRIBUTE = "BODY[]".freeze
-    UID_FETCH_RETRY_CLASSES = [EOFError].freeze
+    UID_FETCH_RETRY_CLASSES = [EOFError, Errno::ECONNRESET, IOError].freeze
+    APPEND_RETRY_CLASSES = [Net::IMAP::BadResponseError].freeze
 
     attr_reader :connection
     attr_reader :name
@@ -64,7 +65,7 @@ module Imap::Backup
         in `search_internal` in stdlib net/imap.rb.
         This is caused by `@responses["SEARCH"] being unset/undefined
       MESSAGE
-      Imap::Backup::Logger.logger.warn message
+      Logger.logger.warn message
       []
     end
 
@@ -91,8 +92,10 @@ module Imap::Backup
     def append(message)
       body = message.imap_body
       date = message.date&.to_time
-      response = client.append(utf7_encoded_name, body, nil, date)
-      extract_uid(response)
+      retry_on_error(errors: APPEND_RETRY_CLASSES, limit: 3) do
+        response = client.append(utf7_encoded_name, body, nil, date)
+        extract_uid(response)
+      end
     end
 
     def clear
@@ -108,12 +111,12 @@ module Imap::Backup
     def examine
       client.examine(utf7_encoded_name)
     rescue Net::IMAP::NoResponseError
-      Imap::Backup::Logger.logger.warn "Folder '#{name}' does not exist on server"
+      Logger.logger.warn "Folder '#{name}' does not exist on server"
       raise FolderNotFound, "Folder '#{name}' does not exist on server"
     end
 
     def extract_uid(response)
-      @uid_validity, uid = response.data.code.data.split(" ").map(&:to_i)
+      @uid_validity, uid = response.data.code.data.split.map(&:to_i)
       uid
     end
 

data/lib/imap/backup/account.rb

@@ -1,5 +1,7 @@
 module Imap::Backup
   class Account
+    DEFAULT_MULTI_FETCH_SIZE = 1
+
     attr_reader :username
     attr_reader :password
     attr_reader :local_path
@@ -7,7 +9,6 @@ module Imap::Backup
     attr_reader :server
     attr_reader :connection_options
     attr_reader :changes
-    attr_reader :marked_for_deletion
 
     def initialize(options)
       @username = options[:username]
@@ -16,27 +17,29 @@ module Imap::Backup
       @folders = options[:folders]
       @server = options[:server]
       @connection_options = options[:connection_options]
+      @multi_fetch_size = options[:multi_fetch_size]
+      @connection = nil
       @changes = {}
       @marked_for_deletion = false
     end
 
     def connection
-      Account::Connection.new(self)
+      @connection ||= Account::Connection.new(self)
     end
 
     def valid?
-      username && password
+      username && password ? true : false
     end
 
     def modified?
       changes.any?
     end
 
-    def clear_changes!
+    def clear_changes
       @changes = {}
     end
 
-    def mark_for_deletion!
+    def mark_for_deletion
       @marked_for_deletion = true
     end
 
@@ -45,14 +48,12 @@ module Imap::Backup
     end
 
     def to_h
-      h = {
-        username: @username,
-        password: @password,
-      }
+      h = {username: @username, password: @password}
       h[:local_path] = @local_path if @local_path
       h[:folders] = @folders if @folders
       h[:server] = @server if @server
       h[:connection_options] = @connection_options if @connection_options
+      h[:multi_fetch_size] = multi_fetch_size if @multi_fetch_size
       h
     end
 
@@ -70,6 +71,7 @@ module Imap::Backup
 
     def folders=(value)
       raise "folders must be an Array" if !value.is_a?(Array)
+
       update(:folders, value)
     end
 
@@ -82,20 +84,38 @@ module Imap::Backup
       update(:connection_options, parsed)
     end
 
+    def multi_fetch_size
+      int = @multi_fetch_size.to_i
+      if int.positive?
+        int
+      else
+        DEFAULT_MULTI_FETCH_SIZE
+      end
+    end
+
+    def multi_fetch_size=(value)
+      parsed = value.to_i
+      parsed = DEFAULT_MULTI_FETCH_SIZE if !parsed.positive?
+      update(:multi_fetch_size, parsed)
+    end
+
     private
 
     def update(field, value)
+      key = :"@#{field}"
       if changes[field]
         change = changes[field]
-        changes.delete(field) if change[:from] == value
+        if change[:from] == value
+          changes.delete(field)
+        else
+          change[:to] = value
+        end
+      else
+        current = instance_variable_get(key)
+        changes[field] = {from: current, to: value} if value != current
       end
-      set_field!(field, value)
-    end
 
-    def set_field!(field, value)
-      key = :"@#{field}"
-      current = instance_variable_get(key)
-      changes[field] = {from: current, to: value}
+      @connection = nil
       instance_variable_set(key, value)
     end
   end

data/lib/imap/backup/cli/backup.rb

@@ -12,9 +12,7 @@ module Imap::Backup
 
     no_commands do
       def run
-        each_connection(account_names) do |connection|
-          connection.run_backup
-        end
+        each_connection(account_names, &:run_backup)
       end
     end
   end

data/lib/imap/backup/cli/folders.rb

@@ -13,15 +13,15 @@ module Imap::Backup
     no_commands do
       def run
         each_connection(account_names) do |connection|
-          puts connection.account.username
+          Kernel.puts connection.account.username
           # TODO: Make folder_names private once this command
           # has been removed.
           folders = connection.folder_names
           if folders.nil?
-            warn "Unable to list account folders"
+            Kernel.warn "Unable to list account folders"
             return false
           end
-          folders.each { |f| puts "\t#{f}" }
+          folders.each { |f| Kernel.puts "\t#{f}" }
         end
       end
     end