imap-backup 16.4.0 → 16.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9ec15a683d70eadabab4271afcd2231c723344e79dde55fadeea091056201fc8
-  data.tar.gz: 946f1074220df2af38c92a5ff7f5dda8bcdcef3dc20f48d9f2c8206e29a8cd6e
+  metadata.gz: fc471fce2bd514e0eee4cec922e8f368df0b2fb99f8b89b2d852e07799a2be77
+  data.tar.gz: fd563c2ee648d5e384a10cdfc79a7d1d5be128bdce5908711396d414c801173f
 SHA512:
-  metadata.gz: 7c33112eb20f168c453f0cc52e86497c7a44459afa718ac12ff1ca71cf990d9566514598aae25fc22117fb30478db649bb76a1aff697d40dabb788cc54fa1508
-  data.tar.gz: ea2dd0c673c603c71708ba550cbdf98c218c51b8ff076ba031d40a7ae33f80ff830ada659b7cc300e8511cc3da207665e6f2114c231e985a0b716e2844f7fe7a
+  metadata.gz: 36088caab383a7ff2528d8ad83813f58918abda4d80a56b0f2b2f1f45d5fc840934d1bc31e1907de11cc665f5aecb5b12152bb8dee686fb656294eb55440897e
+  data.tar.gz: 389a82b7755342dd591fd0e2d45d760eb8350f697d46414d8d90776624bd0ee7263acfe163ab1e8dd42c869d12a544c2d5faa4656d4ef5c802f4991b9f33c1b4

data/README.md

@@ -25,102 +25,13 @@ The {file:CHANGELOG.md CHANGELOG} has a history of the changes to the program.
 * Restartable - calculate start point based on already downloaded messages
 * Standalone - do not rely on an email client or MTA
 
-# Repository
+# Development
 
-After cloning the repo, run the following command to get
-better `git blame` output:
-
-```sh
-git config --local blame.ignoreRevsFile .git-blame-ignore-revs
-```
+See the [development documentation](/docs/development.md).
 
 # Testing
 
-## Feature Specs
-
-Specs under `specs/features` are integration specs.
-Some of these specs run against two local IMAP servers
-controlled by Podman (or Docker) Compose.
-
-Start them before running the test suite
-
-```sh
-$ podman-compose -f dev/compose.yml up -d
-```
-
-or, with Docker
-
-```sh
-$ docker-compose -f dev/compose.yml up -d
-```
-
-Then, run all specs
-
-```sh
-$ rspec
-```
-
-To exclude container-based tests
-
-```sh
-$ rspec --tag ~container
-```
-
-To run **just** the feature specs
-
-```sh
-rspec spec/features/**/*_spec.rb
-```
-
-## Full Test Run
-
-The full test run includes RSpec specs **and** Rubocop checks
-
-```sh
-rake
-```
-
-# Test Debugging
-
-The feature specs are run 'out of process' via the Aruba gem.
-In order to see debugging output from the process,
-use `last_command_started.output`.
-
-# Older Rubies
-
-A Containerfile is available to allow testing with all available Ruby versions,
-see the README in the `dev` directory.
-
-# Performance Specs
-
-```sh
-PERFORMANCE=1 rspec --order=defined
-```
-
-Beware: the performance spec (just backup for now) takes a very
-long time to run, approximately 24 hours!
-
-# Access Docker imap server
-
-```ruby
-require "net/imap"
-require_relative "spec/features/support/30_email_server_helpers"
-
-include EmailServerHelpers
-
-test_connection = test_server_connection_parameters
-
-test_imap = Net::IMAP.new(test_connection[:server], test_connection[:connection_options])
-test_imap.login(test_connection[:username], test_connection[:password])
-
-message = "From: #{test_connection[:username]}\nSubject: Some Subject\n\nHello!\n"
-response = test_imap.append("INBOX", message, nil, nil)
-
-test_imap.examine("INBOX")
-uids = test_imap.uid_search(["ALL"]).sort
-
-fetch_data_items = test_imap.uid_fetch(uids, ["BODY[]"])
-```
+See the [testing documentation](/docs/testing.md).
 
 # Contributing
 

data/docs/development.md

@@ -0,0 +1,8 @@
+# Repository
+
+After cloning the repo, run the following command to get
+better `git blame` output:
+
+```sh
+git config --local blame.ignoreRevsFile .git-blame-ignore-revs
+```

data/docs/testing.md

@@ -0,0 +1,93 @@
+# Setup
+
+Specs under `spec/features` are integration specs.
+Some of these specs run against two local IMAP servers
+controlled by Podman (or Docker) Compose.
+
+Start them before running the test suite
+
+```sh
+$ podman-compose -f dev/compose.yml up -d
+```
+
+or, with Docker
+
+```sh
+$ docker-compose -f dev/compose.yml up -d
+```
+
+# Invocations
+
+Run all specs
+
+```sh
+$ rake spec
+```
+
+Run **just** the unit specs
+
+```sh
+$ rake spec_unit
+```
+
+Run **just** the feature specs
+
+```sh
+$ rake spec_feature
+```
+
+To exclude the slow container-based tests
+
+```sh
+$ rake spec_non_container
+```
+
+## Full Test Run
+
+The full test run includes RSpec specs **and** Rubocop checks
+
+```sh
+rake test
+```
+
+# Debugging
+
+The feature specs are run 'out of process' via the Aruba gem.
+In order to see debugging output from the process,
+use `last_command_started.output`.
+
+# Older Rubies
+
+A Containerfile is available to allow testing with all available Ruby versions,
+see the README in the `dev` directory.
+
+# Performance Specs
+
+```sh
+PERFORMANCE=1 rspec --order=defined
+```
+
+Beware: the performance spec (just backup for now) takes a very
+long time to run, approximately 24 hours!
+
+# Access Docker imap server
+
+```ruby
+require "net/imap"
+require_relative "spec/features/support/30_email_server_helpers"
+
+include EmailServerHelpers
+
+test_connection = test_server_connection_parameters
+
+test_imap = Net::IMAP.new(test_connection[:server], test_connection[:connection_options])
+test_imap.login(test_connection[:username], test_connection[:password])
+
+message = "From: #{test_connection[:username]}\nSubject: Some Subject\n\nHello!\n"
+response = test_imap.append("INBOX", message, nil, nil)
+
+test_imap.examine("INBOX")
+uids = test_imap.uid_search(["ALL"]).sort
+
+fetch_data_items = test_imap.uid_fetch(uids, ["BODY[]"])
+```

data/imap-backup.gemspec

@@ -30,7 +30,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency "rake"
   gem.add_dependency "sys-proctable"
   gem.add_dependency "thor", "~> 1.1"
-  gem.add_dependency "thunderbird", "0.3.0"
+  gem.add_dependency "thunderbird", "~> 0.6.0"
 
   gem.metadata = {
     "rubygems_mfa_required" => "true"

data/lib/imap/backup/account/backup.rb

@@ -1,6 +1,5 @@
 require "imap/backup/account/backup_folders"
 require "imap/backup/account/folder_backup"
-require "imap/backup/account/folder_ensurer"
 require "imap/backup/account/local_only_folder_deleter"
 require "imap/backup/account/locker"
 
@@ -11,6 +10,8 @@ module Imap::Backup
 
   # Carries out the backup of the configured folders of the account
   class Account::Backup
+    # @param account [Account] the account to back up
+    # @param refresh [Boolean] true to refresh folder metadata even if mirror is disabled
     def initialize(account:, refresh: false)
       @account = account
       @refresh = refresh
@@ -23,7 +24,7 @@ module Imap::Backup
       # start the connection so we get logging messages in the right order
       account.client.login
 
-      ensure_folder
+      ensure_directory
       delete_local_only_folders if account.mirror_mode
 
       if backup_folders.none?
@@ -51,8 +52,8 @@ module Imap::Backup
       Account::LocalOnlyFolderDeleter.new(account: account).run
     end
 
-    def ensure_folder
-      Account::FolderEnsurer.new(account: account).run
+    def ensure_directory
+      Serializer::DirectoryMaker.new(files_path: account.files_path).run
     end
 
     def locker

data/lib/imap/backup/account/backup_folders.rb

@@ -9,6 +9,8 @@ module Imap::Backup
   class Account::BackupFolders
     include Enumerable
 
+    # @param client [Client::AutomaticLoginWrapper] the IMAP client that lists folders
+    # @param account [Account] the account whose folders are being backed up
     def initialize(client:, account:)
       @client = client
       @account = account

data/lib/imap/backup/account/client_factory.rb

@@ -10,6 +10,7 @@ module Imap::Backup
 
   # Returns an IMAP client set up for the supplied account
   class Account::ClientFactory
+    # @param account [Account] the account whose credentials configure the client
     def initialize(account:)
       @account = account
     end

data/lib/imap/backup/account/folder.rb

@@ -23,6 +23,8 @@ module Imap::Backup
     # @return [String] the name of the folder
     attr_reader :name
 
+    # @param client [Client::Default] the IMAP client used to talk to the server
+    # @param name [String] the UTF-7 encoded folder name
     def initialize(client, name)
       @client = client
       @name = name

data/lib/imap/backup/account/folder_backup.rb

@@ -4,6 +4,7 @@ require "imap/backup/flag_refresher"
 require "imap/backup/local_only_message_deleter"
 require "imap/backup/logger"
 require "imap/backup/serializer"
+require "imap/backup/serializer/files/path"
 
 module Imap; end
 
@@ -12,6 +13,9 @@ module Imap::Backup
 
   # Implements backup for a single folder
   class Account::FolderBackup
+    # @param account [Account] the account that owns the folder
+    # @param folder [Account::Folder] the online folder being backed up
+    # @param refresh [Boolean] true to refresh metadata regardless of mirror mode
     def initialize(account:, folder:, refresh: false)
       @account = account
       @folder = folder
@@ -83,7 +87,12 @@ module Imap::Backup
     end
 
     def raw_serializer
-      @raw_serializer ||= Serializer.new(account.local_path, folder.name)
+      @raw_serializer ||= begin
+        path = Serializer::Files::Path.new(
+          base_path: account.local_path, folder_name: folder.name
+        )
+        Serializer.new(files_path: path)
+      end
     end
   end
 end

data/lib/imap/backup/account/folder_mapper.rb

@@ -3,6 +3,7 @@ require "pathname"
 
 require "imap/backup/account/folder"
 require "imap/backup/serializer"
+require "imap/backup/serializer/files/path"
 
 module Imap; end
 
@@ -11,6 +12,12 @@ module Imap::Backup
 
   # Implements a folder enumerator for backed-up accounts
   class Account::FolderMapper
+    # @param account [Account] the account whose local folders are being iterated
+    # @param destination [Account] the destination account
+    # @param destination_delimiter [String] the delimiter to use for destination folder names
+    # @param destination_prefix [String] a prefix applied to destination folder names
+    # @param source_delimiter [String] the delimiter used in the source account
+    # @param source_prefix [String] a prefix applied to source folder names
     def initialize(
       account:,
       destination:,
@@ -38,7 +45,10 @@ module Imap::Backup
       glob = File.join(source_local_path, "**", "*.imap")
       Pathname.glob(glob) do |path|
         name = source_folder_name(path)
-        serializer = Serializer.new(source_local_path, name)
+        path = Serializer::Files::Path.new(
+          base_path: source_local_path, folder_name: name
+        )
+        serializer = Serializer.new(files_path: path)
         folder = destination_folder_for_path(name)
         block.call(serializer, folder)
       end

data/lib/imap/backup/account/local_only_folder_deleter.rb

@@ -10,6 +10,7 @@ module Imap::Backup
   # This is used in mirror mode, where local copies are only kept as long as they
   # exist on the server.
   class Account::LocalOnlyFolderDeleter
+    # @param account [Account] the account whose serialized folders are being pruned
     def initialize(account:)
       @account = account
     end

data/lib/imap/backup/account/locker.rb

@@ -1,4 +1,3 @@
-require "imap/backup/account/folder_ensurer"
 require "imap/backup/lockfile"
 
 module Imap; end
@@ -9,6 +8,7 @@ module Imap::Backup
   class Account::Locker
     attr_reader :account
 
+    # @param account [Account] the account whose backup must be locked
     def initialize(account:)
       @account = account
     end
@@ -24,10 +24,14 @@ module Imap::Backup
         Logger.logger.info("Stale lockfile '#{account.lockfile_path}' found. Removing it.")
         lockfile.remove
       else
-        Account::FolderEnsurer.new(account: account).run
+        Serializer::DirectoryMaker.new(files_path: account.files_path).run
       end
 
-      lockfile.with_lock do
+      begin
+        lockfile.with_lock do
+          block.call
+        end
+      rescue Lockfile::ProcessStartTimeUnavailableError
         block.call
       end
     end

data/lib/imap/backup/account/restore.rb

@@ -9,6 +9,9 @@ module Imap::Backup
 
   # Restores all backed up folders to the server
   class Account::Restore
+    # @param account [Account] the account whose backups will be restored
+    # @param delimiter [String] the destination folder delimiter
+    # @param prefix [String] a prefix applied to restored folder names
     def initialize(account:, delimiter: "/", prefix: "")
       @account = account
       @destination_delimiter = delimiter

data/lib/imap/backup/account/serialized_folders.rb

@@ -1,8 +1,9 @@
 require "pathname"
 
 require "imap/backup/account/folder"
-require "imap/backup/account/folder_ensurer"
 require "imap/backup/serializer"
+require "imap/backup/serializer/directory_maker"
+require "imap/backup/serializer/files/path"
 
 module Imap; end
 
@@ -13,6 +14,7 @@ module Imap::Backup
   class Account::SerializedFolders
     include Enumerable
 
+    # @param account [Account] the account whose serialized folders are iterated
     def initialize(account:)
       @account = account
     end
@@ -26,7 +28,10 @@ module Imap::Backup
 
       glob.each do |path|
         name = path.relative_path_from(base).to_s[0..-6]
-        serializer = Serializer.new(account.local_path, name)
+        files_path = Serializer::Files::Path.new(
+          base_path: account.local_path, folder_name: name
+        )
+        serializer = Serializer.new(files_path: files_path)
         folder = Account::Folder.new(account.client, name)
         block.call(serializer, folder)
       end
@@ -40,7 +45,10 @@ module Imap::Backup
 
       glob.each do |path|
         name = path.relative_path_from(base).to_s[0..-6]
-        serializer = Serializer.new(account.local_path, name)
+        files_path = Serializer::Files::Path.new(
+          base_path: account.local_path, folder_name: name
+        )
+        serializer = Serializer.new(files_path: files_path)
         block.call(serializer)
       end
     end
@@ -68,7 +76,10 @@ module Imap::Backup
 
     def glob
       @glob ||= begin
-        Account::FolderEnsurer.new(account: account).run
+        files_path = Serializer::Files::Path.new(
+          base_path: account.local_path, folder_name: nil
+        )
+        Serializer::DirectoryMaker.new(files_path: files_path).run
 
         pattern = File.join(account.local_path, "**", "*.imap")
         Pathname.glob(pattern)

data/lib/imap/backup/account.rb

@@ -4,7 +4,6 @@ require "imap/backup/account/client_factory"
 
 module Imap; end
 
-# rubocop:disable Metrics/ClassLength
 module Imap::Backup
   # Contains the attributes relating to an email account.
   class Account
@@ -65,6 +64,25 @@ module Imap::Backup
     # @return [String] one of "active" (the default), "archived", or "offline"
     attr_reader :status
 
+    # @param options [Hash] Account attributes
+    # @option opts [String] :username the username of the account (usually the same as the email
+    #   address)
+    # @option opts [String] :password the password of the account
+    # @option opts [String] :local_path the path where backups will be saved
+    # @option opts [Array<String>] :folders (nil) the list of folders that have been configured for
+    #   the Account
+    # @option opts [Boolean] :folder_blacklist (false) whether the folders attribute is a blacklist
+    # @option opts [Boolean] :mirror_mode (false) whether to run in mirror mode
+    # @option opts [String] :server the address of the IMAP server
+    # @option opts [Hash] :connection_options (nil) additional connection options for the IMAP
+    #   server
+    # @option opts [String] :download_strategy (nil) the name of the download strategy to adopt
+    #   during backups
+    # @option opts [Integer] :multi_fetch_size (nil) the number of emails to fetch from the IMAP
+    #   server at a time
+    # @option opts [Boolean] :reset_seen_flags_after_fetch (false) whether to reset seen flags after
+    #   fetching messages
+    # @option opts [String] :status ("active") the status of the account
     def initialize(options)
       check_options!(options)
       @username = options[:username]
@@ -131,19 +149,19 @@ module Imap::Backup
     # @return [Hash] all Account data for serialization
     def to_h
       h = {
-        username: @username,
-        password: @password,
+        username: username,
+        password: password,
         status: status
       }
-      h[:local_path] = @local_path if @local_path
-      h[:folders] = @folders if @folders
-      h[:folder_blacklist] = true if @folder_blacklist
-      h[:mirror_mode] = true if @mirror_mode
-      h[:server] = @server if @server
-      h[:connection_options] = @connection_options if connection_options
+      h[:local_path] = local_path if local_path
+      h[:folders] = folders if folders
+      h[:folder_blacklist] = true if folder_blacklist
+      h[:mirror_mode] = true if mirror_mode
+      h[:server] = server if server
+      h[:connection_options] = connection_options if connection_options
       h[:multi_fetch_size] = multi_fetch_size
-      if @reset_seen_flags_after_fetch
-        h[:reset_seen_flags_after_fetch] = @reset_seen_flags_after_fetch
+      if reset_seen_flags_after_fetch
+        h[:reset_seen_flags_after_fetch] = reset_seen_flags_after_fetch
       end
       h
     end
@@ -169,6 +187,11 @@ module Imap::Backup
       update(:local_path, value)
     end
 
+    # @return [Serializer::Files::Path] the base files path for the account
+    def files_path
+      Serializer::Files::Path.new(base_path: local_path, folder_name: nil)
+    end
+
     # @raise [RuntimeError] if the local_path is not set
     # @return [String] the path to the lockfile for the account
     def lockfile_path
@@ -216,12 +239,11 @@ module Imap::Backup
     def connection_options=(value)
       # Ensure we've loaded the connection_options
       connection_options
-      parsed =
-        if value == ""
-          nil
-        else
-          JSON.parse(value, symbolize_names: true)
-        end
+      parsed = if value == ""
+                 nil
+               else
+                 JSON.parse(value, symbolize_names: true)
+               end
       update(:connection_options, parsed)
     end
 

data/lib/imap/backup/cli/backup.rb

@@ -15,6 +15,10 @@ module Imap::Backup
     include Thor::Actions
     include CLI::Helpers
 
+    # @param options [Hash] CLI options controlling output
+    # @option opts [String] :config (nil) the path to the configuration file
+    # @option opts [String] :erb_configuration (nil) the path to the ERB configuration file
+    # @option opts [Boolean] :refresh (false) whether to force refresh of folder metadata
     def initialize(options)
       super([])
       @options = options

data/lib/imap/backup/cli/local/check.rb

@@ -14,6 +14,11 @@ module Imap::Backup
   class CLI::Local::Check
     include CLI::Helpers
 
+    # @param options [Hash] CLI options controlling output
+    # @option opts [String] :config (nil) the path to the configuration file
+    # @option opts [String] :erb_configuration (nil) the path to the ERB configuration file
+    # @option opts [Boolean] :delete_corrupt (false) whether to delete corrupt folders
+    # @option opts [String] :format ("text") the output format, either "text" or "json"
     def initialize(options)
       @options = options
     end

data/lib/imap/backup/cli/options.rb

@@ -70,6 +70,7 @@ module Imap::Backup
       }
     ].freeze
 
+    # @param base [Thor] the command class receiving the options
     def initialize(base:)
       @base = base
     end

data/lib/imap/backup/cli/restore.rb

@@ -14,6 +14,13 @@ module Imap::Backup
     include Thor::Actions
     include CLI::Helpers
 
+    # @param email [String, nil] optional email address identifying the account to restore
+    # @param options [Hash] CLI options controlling output
+    # @option opts [String] :config (nil) the path to the configuration file
+    # @option opts [String] :erb_configuration (nil) the path to the ERB configuration file
+    # @option opts [Array<String>] :accounts (nil) the accounts to restore
+    # @option opts [String] :delimiter ("/") the destination folder delimiter
+    # @option opts [String] :prefix ("") a prefix applied to restored folder names
     def initialize(email = nil, options)
       super([])
       @email = email
@@ -30,9 +37,6 @@ module Imap::Backup
         when email && !options.key?(:accounts)
           account = account(config, email)
           restore(account, **restore_options)
-        when !email && !options.key?(:accounts)
-          Logger.logger.info "Calling restore without an EMAIL parameter is deprecated"
-          config.accounts.each { |a| restore(a) }
         when email && options.key?(:accounts)
           raise "Missing EMAIL parameter"
         when !email && options.key?(:accounts)
@@ -41,6 +45,9 @@ module Imap::Backup
             "please pass a single EMAIL parameter"
           )
           requested_accounts(config).each { |a| restore(a) }
+        else
+          Logger.logger.info "Calling restore without an EMAIL parameter is deprecated"
+          config.accounts.each { |a| restore(a) }
         end
       end
     end

data/lib/imap/backup/cli/setup.rb

@@ -13,6 +13,9 @@ module Imap::Backup
     include Thor::Actions
     include CLI::Helpers
 
+    # @param options [Hash] CLI options controlling output
+    # @option opts [String] :config (nil) the path to the configuration file
+    # @option opts [String] :erb_configuration (nil) the path to the ERB configuration file
     def initialize(options)
       super([])
       @options = options

data/lib/imap/backup/cli/single/backup.rb

@@ -12,6 +12,26 @@ module Imap::Backup
 
   # Runs a backup without relying on existing configuration
   class CLI::Single::Backup
+    # @param options [Hash] CLI options controlling output
+    # @option opts [String] :config (nil) the path to the configuration file
+    # @option opts [String] :erb_configuration (nil) the path to the ERB configuration file
+    # @option opts [String] :email the email address identifying the account to backup
+    # @option opts [String] :password (nil) the password for the account
+    # @option opts [String] :password_environment_variable (nil) the name of an environment variable
+    #   containing the password
+    # @option opts [String] :password_file (nil) the path to a file containing the password
+    # @option opts [String] :server the IMAP server address
+    # @option opts [String] :download_strategy (nil) the download strategy to use, either "delay"
+    #   or "direct"
+    # @option opts [Boolean] :folder_blacklist (false) whether to treat folder list as a blacklist
+    # @option opts [Array<String>] :folder ([]) the folders to backup
+    # @option opts [String] :path (nil) the local path to store the backup
+    # @option opts [Boolean] :mirror (false) whether to run in mirror mode
+    # @option opts [Integer] :multi_fetch_size (nil) the number of messages to fetch in a single
+    #   operation
+    # @option opts [Boolean] :refresh (false) whether to force refresh of folder metadata
+    # @option opts [Boolean] :reset_seen_flags_after_fetch (false) whether to reset seen flags
+    #   after fetching messages
     def initialize(options)
       @options = options
       @password = nil