imap-backup 16.3.0 → 16.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1dbb91a882b2679944560fe5d0f16fd11b59dce626386626d151463a8354a981
-  data.tar.gz: 7b57013410b24ed4625a0405cc9b950f6cdf528e5d50d51b2919a81048f92947
+  metadata.gz: f3ffe8d1bd32587d59e12a514b005d68175244e2cb73a9d138dd3a5d273c785f
+  data.tar.gz: 62d6364641799f1ce20466affc34052f8614a2a2d5633f53a05b4cac31a4e0f3
 SHA512:
-  metadata.gz: '04508eb2303ed44b9fdd8cd361958ed66eb5279a91dc310e10e404432a9e504094b4ff2d3972a645dc35ea2eff47f79f408466e3cf23cf6ac761ef1e5d3601fe'
-  data.tar.gz: a96050615e046863111e79a3350c38dc3d6e3e05f7be967fdd5003e295c1d896f07b48d0aad719c0d2571f0f7aa315833f343bb7b0432c0e3cef556f915b1cfc
+  metadata.gz: f4fdd4254948ef5dd7fba17e3683aa13bf84a19a77ffd935bad3f830355d4bf0d14ff48c731be2542ce56332ed8b4903b1b7971a16e7e947f4406ba4473b4a44
+  data.tar.gz: 4ca3ca9bb9eda64a9b8b9a6cc0889e5815f446e2a798f762f1d55e228928e99988c186bdaad4dfd7d033cd21667de7d906ea3cbd1f11d947dc77d68f4bbd1090

data/README.md

@@ -25,102 +25,13 @@ The {file:CHANGELOG.md CHANGELOG} has a history of the changes to the program.
 * Restartable - calculate start point based on already downloaded messages
 * Standalone - do not rely on an email client or MTA
 
-# Repository
+# Development
 
-After cloning the repo, run the following command to get
-better `git blame` output:
-
-```sh
-git config --local blame.ignoreRevsFile .git-blame-ignore-revs
-```
+See the [development documentation](/docs/development.md).
 
 # Testing
 
-## Feature Specs
-
-Specs under `specs/features` are integration specs.
-Some of these specs run against two local IMAP servers
-controlled by Podman (or Docker) Compose.
-
-Start them before running the test suite
-
-```sh
-$ podman-compose -f dev/compose.yml up -d
-```
-
-or, with Docker
-
-```sh
-$ docker-compose -f dev/compose.yml up -d
-```
-
-Then, run all specs
-
-```sh
-$ rspec
-```
-
-To exclude container-based tests
-
-```sh
-$ rspec --tag ~container
-```
-
-To run **just** the feature specs
-
-```sh
-rspec spec/features/**/*_spec.rb
-```
-
-## Full Test Run
-
-The full test run includes RSpec specs **and** Rubocop checks
-
-```sh
-rake
-```
-
-# Test Debugging
-
-The feature specs are run 'out of process' via the Aruba gem.
-In order to see debugging output from the process,
-use `last_command_started.output`.
-
-# Older Rubies
-
-A Containerfile is available to allow testing with all available Ruby versions,
-see the README in the `dev` directory.
-
-# Performance Specs
-
-```sh
-PERFORMANCE=1 rspec --order=defined
-```
-
-Beware: the performance spec (just backup for now) takes a very
-long time to run, approximately 24 hours!
-
-# Access Docker imap server
-
-```ruby
-require "net/imap"
-require_relative "spec/features/support/30_email_server_helpers"
-
-include EmailServerHelpers
-
-test_connection = test_server_connection_parameters
-
-test_imap = Net::IMAP.new(test_connection[:server], test_connection[:connection_options])
-test_imap.login(test_connection[:username], test_connection[:password])
-
-message = "From: #{test_connection[:username]}\nSubject: Some Subject\n\nHello!\n"
-response = test_imap.append("INBOX", message, nil, nil)
-
-test_imap.examine("INBOX")
-uids = test_imap.uid_search(["ALL"]).sort
-
-fetch_data_items = test_imap.uid_fetch(uids, ["BODY[]"])
-```
+See the [testing documentation](/docs/testing.md).
 
 # Contributing
 

data/docs/development.md

@@ -0,0 +1,8 @@
+# Repository
+
+After cloning the repo, run the following command to get
+better `git blame` output:
+
+```sh
+git config --local blame.ignoreRevsFile .git-blame-ignore-revs
+```

data/docs/testing.md

@@ -0,0 +1,93 @@
+# Setup
+
+Specs under `spec/features` are integration specs.
+Some of these specs run against two local IMAP servers
+controlled by Podman (or Docker) Compose.
+
+Start them before running the test suite
+
+```sh
+$ podman-compose -f dev/compose.yml up -d
+```
+
+or, with Docker
+
+```sh
+$ docker-compose -f dev/compose.yml up -d
+```
+
+# Invocations
+
+Run all specs
+
+```sh
+$ rake spec
+```
+
+Run **just** the unit specs
+
+```sh
+$ rake spec_unit
+```
+
+Run **just** the feature specs
+
+```sh
+$ rake spec_feature
+```
+
+To exclude the slow container-based tests
+
+```sh
+$ rake spec_non_container
+```
+
+## Full Test Run
+
+The full test run includes RSpec specs **and** Rubocop checks
+
+```sh
+rake test
+```
+
+# Debugging
+
+The feature specs are run 'out of process' via the Aruba gem.
+In order to see debugging output from the process,
+use `last_command_started.output`.
+
+# Older Rubies
+
+A Containerfile is available to allow testing with all available Ruby versions,
+see the README in the `dev` directory.
+
+# Performance Specs
+
+```sh
+PERFORMANCE=1 rspec --order=defined
+```
+
+Beware: the performance spec (just backup for now) takes a very
+long time to run, approximately 24 hours!
+
+# Access Docker imap server
+
+```ruby
+require "net/imap"
+require_relative "spec/features/support/30_email_server_helpers"
+
+include EmailServerHelpers
+
+test_connection = test_server_connection_parameters
+
+test_imap = Net::IMAP.new(test_connection[:server], test_connection[:connection_options])
+test_imap.login(test_connection[:username], test_connection[:password])
+
+message = "From: #{test_connection[:username]}\nSubject: Some Subject\n\nHello!\n"
+response = test_imap.append("INBOX", message, nil, nil)
+
+test_imap.examine("INBOX")
+uids = test_imap.uid_search(["ALL"]).sort
+
+fetch_data_items = test_imap.uid_fetch(uids, ["BODY[]"])
+```

data/imap-backup.gemspec

@@ -18,18 +18,19 @@ Gem::Specification.new do |gem|
 
   gem.executables   = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) }
   gem.require_paths = ["lib"]
-  gem.required_ruby_version = ">= 3.0"
+  gem.required_ruby_version = ">= 3.2"
 
-  gem.add_runtime_dependency "highline"
-  gem.add_runtime_dependency "logger"
-  gem.add_runtime_dependency "mail", "2.7.1"
-  gem.add_runtime_dependency "net-imap", ">= 0.3.2"
-  gem.add_runtime_dependency "net-smtp"
-  gem.add_runtime_dependency "os"
-  gem.add_runtime_dependency "ostruct"
-  gem.add_runtime_dependency "rake"
-  gem.add_runtime_dependency "thor", "~> 1.1"
-  gem.add_runtime_dependency "thunderbird", "0.3.0"
+  gem.add_dependency "highline"
+  gem.add_dependency "logger"
+  gem.add_dependency "mail", "2.7.1"
+  gem.add_dependency "net-imap", ">= 0.3.2"
+  gem.add_dependency "net-smtp"
+  gem.add_dependency "os"
+  gem.add_dependency "ostruct"
+  gem.add_dependency "rake"
+  gem.add_dependency "sys-proctable"
+  gem.add_dependency "thor", "~> 1.1"
+  gem.add_dependency "thunderbird", "0.3.0"
 
   gem.metadata = {
     "rubygems_mfa_required" => "true"

data/lib/imap/backup/account/backup.rb

@@ -2,6 +2,7 @@ require "imap/backup/account/backup_folders"
 require "imap/backup/account/folder_backup"
 require "imap/backup/account/folder_ensurer"
 require "imap/backup/account/local_only_folder_deleter"
+require "imap/backup/account/locker"
 
 module Imap; end
 
@@ -10,6 +11,8 @@ module Imap::Backup
 
   # Carries out the backup of the configured folders of the account
   class Account::Backup
+    # @param account [Account] the account to back up
+    # @param refresh [Boolean] true to refresh folder metadata even if mirror is disabled
     def initialize(account:, refresh: false)
       @account = account
       @refresh = refresh
@@ -22,19 +25,17 @@ module Imap::Backup
       # start the connection so we get logging messages in the right order
       account.client.login
 
-      run_pre_backup_tasks
-      backup_folders = Account::BackupFolders.new(
-        client: account.client, account: account
-      ).to_a
+      ensure_folder
+      delete_local_only_folders if account.mirror_mode
+
       if backup_folders.none?
         Logger.logger.warn "No folders found to backup for account '#{account.username}'"
         return
       end
-      Logger.logger.debug "Starting backup of #{backup_folders.count} folders"
-      backup_folders.each do |folder|
-        Account::FolderBackup.new(account: account, folder: folder, refresh: refresh).run
+
+      locker.with_lock do
+        perform_backup
       end
-      Logger.logger.debug "Backup of account '#{account.username}' complete"
     end
 
     private
@@ -42,9 +43,30 @@ module Imap::Backup
     attr_reader :account
     attr_reader :refresh
 
-    def run_pre_backup_tasks
+    def backup_folders
+      @backup_folders ||= Account::BackupFolders.new(
+        client: account.client, account: account
+      ).to_a
+    end
+
+    def delete_local_only_folders
+      Account::LocalOnlyFolderDeleter.new(account: account).run
+    end
+
+    def ensure_folder
       Account::FolderEnsurer.new(account: account).run
-      Account::LocalOnlyFolderDeleter.new(account: account).run if account.mirror_mode
+    end
+
+    def locker
+      @locker ||= Account::Locker.new(account: account)
+    end
+
+    def perform_backup
+      Logger.logger.debug "Starting backup of #{backup_folders.count} folders"
+      backup_folders.each do |folder|
+        Account::FolderBackup.new(account: account, folder: folder, refresh: refresh).run
+      end
+      Logger.logger.debug "Backup of account '#{account.username}' complete"
     end
   end
 end

data/lib/imap/backup/account/backup_folders.rb

@@ -9,6 +9,8 @@ module Imap::Backup
   class Account::BackupFolders
     include Enumerable
 
+    # @param client [Client::AutomaticLoginWrapper] the IMAP client that lists folders
+    # @param account [Account] the account whose folders are being backed up
     def initialize(client:, account:)
       @client = client
       @account = account

data/lib/imap/backup/account/client_factory.rb

@@ -10,6 +10,7 @@ module Imap::Backup
 
   # Returns an IMAP client set up for the supplied account
   class Account::ClientFactory
+    # @param account [Account] the account whose credentials configure the client
     def initialize(account:)
       @account = account
     end

data/lib/imap/backup/account/folder.rb

@@ -23,6 +23,8 @@ module Imap::Backup
     # @return [String] the name of the folder
     attr_reader :name
 
+    # @param client [Client::Default] the IMAP client used to talk to the server
+    # @param name [String] the UTF-7 encoded folder name
     def initialize(client, name)
       @client = client
       @name = name
@@ -184,6 +186,13 @@ module Imap::Backup
     CREATE_RETRY_CLASSES = [::Net::IMAP::BadResponseError].freeze
     EXAMINE_RETRY_CLASSES = [::Net::IMAP::BadResponseError].freeze
     PERMITTED_FLAGS = %i(Answered Draft Flagged Seen).freeze
+    private_constant :BODY_ATTRIBUTE,
+                     :UID_FETCH_RETRY_CLASSES,
+                     :UID_SEARCH_RETRY_CLASSES,
+                     :APPEND_RETRY_CLASSES,
+                     :CREATE_RETRY_CLASSES,
+                     :EXAMINE_RETRY_CLASSES,
+                     :PERMITTED_FLAGS
 
     def examine
       client.examine(utf7_encoded_name)

data/lib/imap/backup/account/folder_backup.rb

@@ -12,6 +12,9 @@ module Imap::Backup
 
   # Implements backup for a single folder
   class Account::FolderBackup
+    # @param account [Account] the account that owns the folder
+    # @param folder [Account::Folder] the online folder being backed up
+    # @param refresh [Boolean] true to refresh metadata regardless of mirror mode
     def initialize(account:, folder:, refresh: false)
       @account = account
       @folder = folder

data/lib/imap/backup/account/folder_ensurer.rb

@@ -8,6 +8,7 @@ module Imap::Backup
 
   # Handles creation of directories for backup storage
   class Account::FolderEnsurer
+    # @param account [Account] the account to check
     def initialize(account:)
       @account = account
     end

data/lib/imap/backup/account/folder_mapper.rb

@@ -11,6 +11,12 @@ module Imap::Backup
 
   # Implements a folder enumerator for backed-up accounts
   class Account::FolderMapper
+    # @param account [Account] the account whose local folders are being iterated
+    # @param destination [Account] the destination account
+    # @param destination_delimiter [String] the delimiter to use for destination folder names
+    # @param destination_prefix [String] a prefix applied to destination folder names
+    # @param source_delimiter [String] the delimiter used in the source account
+    # @param source_prefix [String] a prefix applied to source folder names
     def initialize(
       account:,
       destination:,

data/lib/imap/backup/account/local_only_folder_deleter.rb

@@ -10,6 +10,7 @@ module Imap::Backup
   # This is used in mirror mode, where local copies are only kept as long as they
   # exist on the server.
   class Account::LocalOnlyFolderDeleter
+    # @param account [Account] the account whose serialized folders are being pruned
     def initialize(account:)
       @account = account
     end

data/lib/imap/backup/account/locker.rb

@@ -0,0 +1,40 @@
+require "imap/backup/account/folder_ensurer"
+require "imap/backup/lockfile"
+
+module Imap; end
+
+module Imap::Backup
+  class Account; end
+
+  class Account::Locker
+    attr_reader :account
+
+    # @param account [Account] the account whose backup must be locked
+    def initialize(account:)
+      @account = account
+    end
+
+    def with_lock(&block)
+      lockfile = Lockfile.new(path: account.lockfile_path)
+      if lockfile.exists?
+        if !lockfile.stale?
+          raise Lockfile::LockfileExistsError,
+                "Lockfile '#{account.lockfile_path}' exists and is not stale."
+        end
+
+        Logger.logger.info("Stale lockfile '#{account.lockfile_path}' found. Removing it.")
+        lockfile.remove
+      else
+        Account::FolderEnsurer.new(account: account).run
+      end
+
+      begin
+        lockfile.with_lock do
+          block.call
+        end
+      rescue Lockfile::ProcessStartTimeUnavailableError
+        block.call
+      end
+    end
+  end
+end

data/lib/imap/backup/account/restore.rb

@@ -1,4 +1,5 @@
 require "imap/backup/account/folder_mapper"
+require "imap/backup/account/locker"
 require "imap/backup/uploader"
 
 module Imap; end
@@ -8,17 +9,23 @@ module Imap::Backup
 
   # Restores all backed up folders to the server
   class Account::Restore
+    # @param account [Account] the account whose backups will be restored
+    # @param delimiter [String] the destination folder delimiter
+    # @param prefix [String] a prefix applied to restored folder names
     def initialize(account:, delimiter: "/", prefix: "")
       @account = account
       @destination_delimiter = delimiter
       @destination_prefix = prefix
+      @locker = nil
     end
 
     # Runs the restore operation
     # @return [void]
     def run
-      folders.each do |serializer, folder|
-        Uploader.new(folder, serializer).run
+      locker.with_lock do
+        folders.each do |serializer, folder|
+          Uploader.new(folder, serializer).run
+        end
       end
     end
 
@@ -40,5 +47,9 @@ module Imap::Backup
     def folders
       Account::FolderMapper.new(**enumerator_options)
     end
+
+    def locker
+      @locker ||= Account::Locker.new(account: account)
+    end
   end
 end

data/lib/imap/backup/account/serialized_folders.rb

@@ -13,6 +13,7 @@ module Imap::Backup
   class Account::SerializedFolders
     include Enumerable
 
+    # @param account [Account] the account whose serialized folders are iterated
     def initialize(account:)
       @account = account
     end

data/lib/imap/backup/account.rb

@@ -4,6 +4,7 @@ require "imap/backup/account/client_factory"
 
 module Imap; end
 
+# rubocop:disable Metrics/ClassLength
 module Imap::Backup
   # Contains the attributes relating to an email account.
   class Account
@@ -64,6 +65,25 @@ module Imap::Backup
     # @return [String] one of "active" (the default), "archived", or "offline"
     attr_reader :status
 
+    # @param options [Hash] Account attributes
+    # @option opts [String] :username the username of the account (usually the same as the email
+    #   address)
+    # @option opts [String] :password the password of the account
+    # @option opts [String] :local_path the path where backups will be saved
+    # @option opts [Array<String>] :folders (nil) the list of folders that have been configured for
+    #   the Account
+    # @option opts [Boolean] :folder_blacklist (false) whether the folders attribute is a blacklist
+    # @option opts [Boolean] :mirror_mode (false) whether to run in mirror mode
+    # @option opts [String] :server the address of the IMAP server
+    # @option opts [Hash] :connection_options (nil) additional connection options for the IMAP
+    #   server
+    # @option opts [String] :download_strategy (nil) the name of the download strategy to adopt
+    #   during backups
+    # @option opts [Integer] :multi_fetch_size (nil) the number of emails to fetch from the IMAP
+    #   server at a time
+    # @option opts [Boolean] :reset_seen_flags_after_fetch (false) whether to reset seen flags after
+    #   fetching messages
+    # @option opts [String] :status ("active") the status of the account
     def initialize(options)
       check_options!(options)
       @username = options[:username]
@@ -130,19 +150,19 @@ module Imap::Backup
     # @return [Hash] all Account data for serialization
     def to_h
       h = {
-        username: @username,
-        password: @password,
+        username: username,
+        password: password,
         status: status
       }
-      h[:local_path] = @local_path if @local_path
-      h[:folders] = @folders if @folders
-      h[:folder_blacklist] = true if @folder_blacklist
-      h[:mirror_mode] = true if @mirror_mode
-      h[:server] = @server if @server
-      h[:connection_options] = @connection_options if connection_options
+      h[:local_path] = local_path if local_path
+      h[:folders] = folders if folders
+      h[:folder_blacklist] = true if folder_blacklist
+      h[:mirror_mode] = true if mirror_mode
+      h[:server] = server if server
+      h[:connection_options] = connection_options if connection_options
       h[:multi_fetch_size] = multi_fetch_size
-      if @reset_seen_flags_after_fetch
-        h[:reset_seen_flags_after_fetch] = @reset_seen_flags_after_fetch
+      if reset_seen_flags_after_fetch
+        h[:reset_seen_flags_after_fetch] = reset_seen_flags_after_fetch
       end
       h
     end
@@ -168,6 +188,14 @@ module Imap::Backup
       update(:local_path, value)
     end
 
+    # @raise [RuntimeError] if the local_path is not set
+    # @return [String] the path to the lockfile for the account
+    def lockfile_path
+      raise "local_path is not set" if !local_path
+
+      File.join(local_path, "imap-backup.lock")
+    end
+
     # @raise [RuntimeError] if the supplied value is not an Array
     # @return [void]
     def folders=(value)
@@ -207,12 +235,11 @@ module Imap::Backup
     def connection_options=(value)
       # Ensure we've loaded the connection_options
       connection_options
-      parsed =
-        if value == ""
-          nil
-        else
-          JSON.parse(value, symbolize_names: true)
-        end
+      parsed = if value == ""
+                 nil
+               else
+                 JSON.parse(value, symbolize_names: true)
+               end
       update(:connection_options, parsed)
     end
 
@@ -283,14 +310,19 @@ module Imap::Backup
 
     attr_reader :changes
 
-    REQUIRED_ATTRIBUTES = %i[password username].freeze
+    REQUIRED_ATTRIBUTES = %i[password username local_path].freeze
     OPTIONAL_ATTRIBUTES = %i[
-      connection_options download_strategy folders folder_blacklist local_path mirror_mode
+      connection_options download_strategy folders folder_blacklist mirror_mode
       multi_fetch_size reset_seen_flags_after_fetch server status
     ].freeze
     KNOWN_ATTRIBUTES = REQUIRED_ATTRIBUTES + OPTIONAL_ATTRIBUTES
     VALID_STATUSES = %w[active archived offline].freeze
     DEFAULT_STATUS = "active".freeze
+    private_constant :REQUIRED_ATTRIBUTES,
+                     :OPTIONAL_ATTRIBUTES,
+                     :KNOWN_ATTRIBUTES,
+                     :VALID_STATUSES,
+                     :DEFAULT_STATUS
 
     def check_options!(options)
       missing_required = REQUIRED_ATTRIBUTES - options.keys

data/lib/imap/backup/cli/backup.rb

@@ -15,6 +15,10 @@ module Imap::Backup
     include Thor::Actions
     include CLI::Helpers
 
+    # @param options [Hash] CLI options controlling output
+    # @option opts [String] :config (nil) the path to the configuration file
+    # @option opts [String] :erb_configuration (nil) the path to the ERB configuration file
+    # @option opts [Boolean] :refresh (false) whether to force refresh of folder metadata
     def initialize(options)
       super([])
       @options = options
@@ -64,6 +68,8 @@ module Imap::Backup
       case exception
       when Net::IMAP::NoResponseError, Errno::ECONNREFUSED
         111
+      when Lockfile::LockfileExistsError
+        112
       else
         1
       end

data/lib/imap/backup/cli/local/check.rb

@@ -14,6 +14,11 @@ module Imap::Backup
   class CLI::Local::Check
     include CLI::Helpers
 
+    # @param options [Hash] CLI options controlling output
+    # @option opts [String] :config (nil) the path to the configuration file
+    # @option opts [String] :erb_configuration (nil) the path to the ERB configuration file
+    # @option opts [Boolean] :delete_corrupt (false) whether to delete corrupt folders
+    # @option opts [String] :format ("text") the output format, either "text" or "json"
     def initialize(options)
       @options = options
     end

data/lib/imap/backup/cli/local.rb

@@ -141,6 +141,7 @@ module Imap::Backup
     private
 
     MAX_SUBJECT = 60
+    private_constant :MAX_SUBJECT
 
     def list_emails_as_json(serializer)
       emails = serializer.each_message.map do |message|

data/lib/imap/backup/cli/options.rb

@@ -70,6 +70,7 @@ module Imap::Backup
       }
     ].freeze
 
+    # @param base [Thor] the command class receiving the options
     def initialize(base:)
       @base = base
     end