imap-backup 16.3.0 → 16.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1dbb91a882b2679944560fe5d0f16fd11b59dce626386626d151463a8354a981
-  data.tar.gz: 7b57013410b24ed4625a0405cc9b950f6cdf528e5d50d51b2919a81048f92947
+  metadata.gz: 9ec15a683d70eadabab4271afcd2231c723344e79dde55fadeea091056201fc8
+  data.tar.gz: 946f1074220df2af38c92a5ff7f5dda8bcdcef3dc20f48d9f2c8206e29a8cd6e
 SHA512:
-  metadata.gz: '04508eb2303ed44b9fdd8cd361958ed66eb5279a91dc310e10e404432a9e504094b4ff2d3972a645dc35ea2eff47f79f408466e3cf23cf6ac761ef1e5d3601fe'
-  data.tar.gz: a96050615e046863111e79a3350c38dc3d6e3e05f7be967fdd5003e295c1d896f07b48d0aad719c0d2571f0f7aa315833f343bb7b0432c0e3cef556f915b1cfc
+  metadata.gz: 7c33112eb20f168c453f0cc52e86497c7a44459afa718ac12ff1ca71cf990d9566514598aae25fc22117fb30478db649bb76a1aff697d40dabb788cc54fa1508
+  data.tar.gz: ea2dd0c673c603c71708ba550cbdf98c218c51b8ff076ba031d40a7ae33f80ff830ada659b7cc300e8511cc3da207665e6f2114c231e985a0b716e2844f7fe7a

data/imap-backup.gemspec

@@ -18,18 +18,19 @@ Gem::Specification.new do |gem|
 
   gem.executables   = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) }
   gem.require_paths = ["lib"]
-  gem.required_ruby_version = ">= 3.0"
+  gem.required_ruby_version = ">= 3.2"
 
-  gem.add_runtime_dependency "highline"
-  gem.add_runtime_dependency "logger"
-  gem.add_runtime_dependency "mail", "2.7.1"
-  gem.add_runtime_dependency "net-imap", ">= 0.3.2"
-  gem.add_runtime_dependency "net-smtp"
-  gem.add_runtime_dependency "os"
-  gem.add_runtime_dependency "ostruct"
-  gem.add_runtime_dependency "rake"
-  gem.add_runtime_dependency "thor", "~> 1.1"
-  gem.add_runtime_dependency "thunderbird", "0.3.0"
+  gem.add_dependency "highline"
+  gem.add_dependency "logger"
+  gem.add_dependency "mail", "2.7.1"
+  gem.add_dependency "net-imap", ">= 0.3.2"
+  gem.add_dependency "net-smtp"
+  gem.add_dependency "os"
+  gem.add_dependency "ostruct"
+  gem.add_dependency "rake"
+  gem.add_dependency "sys-proctable"
+  gem.add_dependency "thor", "~> 1.1"
+  gem.add_dependency "thunderbird", "0.3.0"
 
   gem.metadata = {
     "rubygems_mfa_required" => "true"

data/lib/imap/backup/account/backup.rb

@@ -2,6 +2,7 @@ require "imap/backup/account/backup_folders"
 require "imap/backup/account/folder_backup"
 require "imap/backup/account/folder_ensurer"
 require "imap/backup/account/local_only_folder_deleter"
+require "imap/backup/account/locker"
 
 module Imap; end
 
@@ -22,19 +23,17 @@ module Imap::Backup
       # start the connection so we get logging messages in the right order
       account.client.login
 
-      run_pre_backup_tasks
-      backup_folders = Account::BackupFolders.new(
-        client: account.client, account: account
-      ).to_a
+      ensure_folder
+      delete_local_only_folders if account.mirror_mode
+
       if backup_folders.none?
         Logger.logger.warn "No folders found to backup for account '#{account.username}'"
         return
       end
-      Logger.logger.debug "Starting backup of #{backup_folders.count} folders"
-      backup_folders.each do |folder|
-        Account::FolderBackup.new(account: account, folder: folder, refresh: refresh).run
+
+      locker.with_lock do
+        perform_backup
       end
-      Logger.logger.debug "Backup of account '#{account.username}' complete"
     end
 
     private
@@ -42,9 +41,30 @@ module Imap::Backup
     attr_reader :account
     attr_reader :refresh
 
-    def run_pre_backup_tasks
+    def backup_folders
+      @backup_folders ||= Account::BackupFolders.new(
+        client: account.client, account: account
+      ).to_a
+    end
+
+    def delete_local_only_folders
+      Account::LocalOnlyFolderDeleter.new(account: account).run
+    end
+
+    def ensure_folder
       Account::FolderEnsurer.new(account: account).run
-      Account::LocalOnlyFolderDeleter.new(account: account).run if account.mirror_mode
+    end
+
+    def locker
+      @locker ||= Account::Locker.new(account: account)
+    end
+
+    def perform_backup
+      Logger.logger.debug "Starting backup of #{backup_folders.count} folders"
+      backup_folders.each do |folder|
+        Account::FolderBackup.new(account: account, folder: folder, refresh: refresh).run
+      end
+      Logger.logger.debug "Backup of account '#{account.username}' complete"
     end
   end
 end

data/lib/imap/backup/account/folder.rb

@@ -184,6 +184,13 @@ module Imap::Backup
     CREATE_RETRY_CLASSES = [::Net::IMAP::BadResponseError].freeze
     EXAMINE_RETRY_CLASSES = [::Net::IMAP::BadResponseError].freeze
     PERMITTED_FLAGS = %i(Answered Draft Flagged Seen).freeze
+    private_constant :BODY_ATTRIBUTE,
+                     :UID_FETCH_RETRY_CLASSES,
+                     :UID_SEARCH_RETRY_CLASSES,
+                     :APPEND_RETRY_CLASSES,
+                     :CREATE_RETRY_CLASSES,
+                     :EXAMINE_RETRY_CLASSES,
+                     :PERMITTED_FLAGS
 
     def examine
       client.examine(utf7_encoded_name)

data/lib/imap/backup/account/locker.rb

@@ -0,0 +1,35 @@
+require "imap/backup/account/folder_ensurer"
+require "imap/backup/lockfile"
+
+module Imap; end
+
+module Imap::Backup
+  class Account; end
+
+  class Account::Locker
+    attr_reader :account
+
+    def initialize(account:)
+      @account = account
+    end
+
+    def with_lock(&block)
+      lockfile = Lockfile.new(path: account.lockfile_path)
+      if lockfile.exists?
+        if !lockfile.stale?
+          raise Lockfile::LockfileExistsError,
+                "Lockfile '#{account.lockfile_path}' exists and is not stale."
+        end
+
+        Logger.logger.info("Stale lockfile '#{account.lockfile_path}' found. Removing it.")
+        lockfile.remove
+      else
+        Account::FolderEnsurer.new(account: account).run
+      end
+
+      lockfile.with_lock do
+        block.call
+      end
+    end
+  end
+end

data/lib/imap/backup/account/restore.rb

@@ -1,4 +1,5 @@
 require "imap/backup/account/folder_mapper"
+require "imap/backup/account/locker"
 require "imap/backup/uploader"
 
 module Imap; end
@@ -12,13 +13,16 @@ module Imap::Backup
       @account = account
       @destination_delimiter = delimiter
       @destination_prefix = prefix
+      @locker = nil
     end
 
     # Runs the restore operation
     # @return [void]
     def run
-      folders.each do |serializer, folder|
-        Uploader.new(folder, serializer).run
+      locker.with_lock do
+        folders.each do |serializer, folder|
+          Uploader.new(folder, serializer).run
+        end
       end
     end
 
@@ -40,5 +44,9 @@ module Imap::Backup
     def folders
       Account::FolderMapper.new(**enumerator_options)
     end
+
+    def locker
+      @locker ||= Account::Locker.new(account: account)
+    end
   end
 end

data/lib/imap/backup/account.rb

@@ -4,6 +4,7 @@ require "imap/backup/account/client_factory"
 
 module Imap; end
 
+# rubocop:disable Metrics/ClassLength
 module Imap::Backup
   # Contains the attributes relating to an email account.
   class Account
@@ -168,6 +169,14 @@ module Imap::Backup
       update(:local_path, value)
     end
 
+    # @raise [RuntimeError] if the local_path is not set
+    # @return [String] the path to the lockfile for the account
+    def lockfile_path
+      raise "local_path is not set" if !local_path
+
+      File.join(local_path, "imap-backup.lock")
+    end
+
     # @raise [RuntimeError] if the supplied value is not an Array
     # @return [void]
     def folders=(value)
@@ -283,14 +292,19 @@ module Imap::Backup
 
     attr_reader :changes
 
-    REQUIRED_ATTRIBUTES = %i[password username].freeze
+    REQUIRED_ATTRIBUTES = %i[password username local_path].freeze
     OPTIONAL_ATTRIBUTES = %i[
-      connection_options download_strategy folders folder_blacklist local_path mirror_mode
+      connection_options download_strategy folders folder_blacklist mirror_mode
       multi_fetch_size reset_seen_flags_after_fetch server status
     ].freeze
     KNOWN_ATTRIBUTES = REQUIRED_ATTRIBUTES + OPTIONAL_ATTRIBUTES
     VALID_STATUSES = %w[active archived offline].freeze
     DEFAULT_STATUS = "active".freeze
+    private_constant :REQUIRED_ATTRIBUTES,
+                     :OPTIONAL_ATTRIBUTES,
+                     :KNOWN_ATTRIBUTES,
+                     :VALID_STATUSES,
+                     :DEFAULT_STATUS
 
     def check_options!(options)
       missing_required = REQUIRED_ATTRIBUTES - options.keys

data/lib/imap/backup/cli/backup.rb

@@ -64,6 +64,8 @@ module Imap::Backup
       case exception
       when Net::IMAP::NoResponseError, Errno::ECONNREFUSED
         111
+      when Lockfile::LockfileExistsError
+        112
       else
         1
       end

data/lib/imap/backup/cli/local.rb

@@ -141,6 +141,7 @@ module Imap::Backup
     private
 
     MAX_SUBJECT = 60
+    private_constant :MAX_SUBJECT
 
     def list_emails_as_json(serializer)
       emails = serializer.each_message.map do |message|

data/lib/imap/backup/cli/remote.rb

@@ -109,9 +109,12 @@ module Imap::Backup
       Kernel.puts list.to_json
     end
 
+    NAMESPACE_TEMPLATE = "%-10<name>s %-10<prefix>s %<delim>s".freeze
+    private_constant :NAMESPACE_TEMPLATE
+
     def list_namespaces(namespaces)
       Kernel.puts format(
-        "%-10<name>s %-10<prefix>s %<delim>s",
+        NAMESPACE_TEMPLATE,
         {name: "Name", prefix: "Prefix", delim: "Delimiter"}
       )
       list_namespace namespaces, :personal
@@ -122,7 +125,7 @@ module Imap::Backup
     def list_namespace(namespaces, name)
       info = namespace_info(namespaces.send(name).first, quote: true)
       if info
-        Kernel.puts format("%-10<name>s %-10<prefix>s %<delim>s", name: name, **info)
+        Kernel.puts format(NAMESPACE_TEMPLATE, name: name, **info)
       else
         Kernel.puts format("%-10<name>s (Not defined)", name: name)
       end

data/lib/imap/backup/cli/restore.rb

@@ -50,8 +50,8 @@ module Imap::Backup
     attr_reader :email
     attr_reader :options
 
-    def restore(account, **options)
-      restore = Account::Restore.new(account: account, **options)
+    def restore(account, **)
+      restore = Account::Restore.new(account: account, **)
       restore.run
     end
 

data/lib/imap/backup/cli/stats.rb

@@ -37,6 +37,7 @@ module Imap::Backup
       {name: :local, width: 8, alignment: :right}
     ].freeze
     ALIGNMENT_FORMAT_SYMBOL = {left: "-", right: " "}.freeze
+    private_constant :TEXT_COLUMNS, :ALIGNMENT_FORMAT_SYMBOL
 
     attr_reader :email
     attr_reader :options

data/lib/imap/backup/cli/transfer.rb

@@ -1,4 +1,5 @@
 require "imap/backup/account/folder_mapper"
+require "imap/backup/account/locker"
 require "imap/backup/cli/backup"
 require "imap/backup/cli/helpers"
 require "imap/backup/logger"
@@ -23,9 +24,11 @@ module Imap::Backup
       @automatic_namespaces = nil
       @config_path = nil
       @destination_delimiter = nil
+      @destination_locker = nil
       @destination_prefix = nil
       @reset = nil
       @source_delimiter = nil
+      @source_locker = nil
       @source_prefix = nil
     end
 
@@ -34,22 +37,19 @@ module Imap::Backup
     #     or the source and destination accounts are the same,
     #     or either of the accounts is not configured,
     #     or incompatible namespace/delimiter parameters have been supplied
+    #     or one or both of the accounts is locked by another process.
     #   @return [void]
     def run
       raise "Unknown action '#{action}'" if !ACTIONS.include?(action)
 
       process_options!
       warn_if_source_account_is_not_in_mirror_mode if action == :mirror
+
       run_backup if %i(copy mirror).include?(action)
 
-      folders.each do |serializer, folder|
-        case action
-        when :copy
-          Mirror.new(serializer, folder, reset: false).run
-        when :migrate
-          Migrator.new(serializer, folder, reset: reset).run
-        when :mirror
-          Mirror.new(serializer, folder, reset: true).run
+      source_locker.with_lock do
+        destination_locker.with_lock do
+          perform_action_on_folders
         end
       end
     end
@@ -68,6 +68,19 @@ module Imap::Backup
     attr_reader :source_email
     attr_accessor :source_prefix
 
+    def perform_action_on_folders
+      folders.each do |serializer, folder|
+        case action
+        when :copy
+          Mirror.new(serializer, folder, reset: false).run
+        when :migrate
+          Migrator.new(serializer, folder, reset: reset).run
+        when :mirror
+          Mirror.new(serializer, folder, reset: true).run
+        end
+      end
+    end
+
     def process_options!
       self.automatic_namespaces = options[:automatic_namespaces] || false
       self.config_path = options[:config]
@@ -175,5 +188,13 @@ module Imap::Backup
     def source_account
       config.accounts.find { |a| a.username == source_email }
     end
+
+    def source_locker
+      @source_locker ||= Account::Locker.new(account: source_account)
+    end
+
+    def destination_locker
+      @destination_locker ||= Account::Locker.new(account: destination_account)
+    end
   end
 end

data/lib/imap/backup/cli/utils.rb

@@ -33,15 +33,10 @@ module Imap::Backup
       Logger.setup_logging options
       config = load_config(**options)
       account = account(config, email)
+      locker ||= Account::Locker.new(account: account)
 
-      backup_folders = Account::BackupFolders.new(
-        client: account.client, account: account
-      )
-      backup_folders.each do |folder|
-        next if !folder.exist?
-
-        serializer = Serializer.new(account.local_path, folder.name)
-        do_ignore_folder_history(folder, serializer)
+      locker.with_lock do
+        ignore_account_history(account)
       end
     end
 
@@ -99,8 +94,21 @@ module Imap::Backup
     private
 
     FAKE_EMAIL = "fake@email.com".freeze
+    private_constant :FAKE_EMAIL
+
+    def ignore_account_history(account)
+      backup_folders = Account::BackupFolders.new(
+        client: account.client, account: account
+      )
+      backup_folders.each do |folder|
+        next if !folder.exist?
+
+        serializer = Serializer.new(account.local_path, folder.name)
+        ignore_folder_history(folder, serializer)
+      end
+    end
 
-    def do_ignore_folder_history(folder, serializer)
+    def ignore_folder_history(folder, serializer)
       uids = folder.uids - serializer.uids
       Logger.logger.info "Folder '#{folder.name}' - #{uids.length} messages"
 

data/lib/imap/backup/client/default.rb

@@ -13,6 +13,7 @@ module Imap::Backup
   # Tracks the latest folder selection in order to avoid repeated calls
   class Client::Default
     extend Forwardable
+
     def_delegators :imap, *%i(
       append authenticate capability create expunge namespace
       responses uid_fetch uid_search uid_store

data/lib/imap/backup/configuration.rb

@@ -68,7 +68,10 @@ module Imap::Backup
         ensure_loaded!
         accounts = data[:accounts].map do |attr|
           Account.new(attr)
-        end
+        rescue ArgumentError => e
+          Logger.logger.error("Skipping invalid account in config: #{e.message}")
+          nil
+        end.compact
         inject_global_attributes(accounts)
       end
     end
@@ -108,8 +111,6 @@ module Imap::Backup
 
     private
 
-    VERSION_2_1 = "2.1".freeze
-
     attr_reader :pathname
 
     def ensure_loaded!

data/lib/imap/backup/lockfile.rb

@@ -0,0 +1,78 @@
+require "sys/proctable"
+require "json"
+
+module Imap; end
+
+module Imap::Backup
+  class Lockfile
+    # An error that is thrown if a lockfile already exists
+    class LockfileExistsError < StandardError; end
+
+    attr_reader :path
+
+    # Initializes a new Lockfile instance.
+    # @param path [String] the path to the lockfile
+    def initialize(path:)
+      @path = path
+    end
+
+    # Creates the lockfile, yields to the given block
+    # and ensures the lockfile is removed afterwards.
+    def with_lock(&block)
+      raise LockfileExistsError, "Lockfile already exists at #{path}" if exists?
+
+      begin
+        create
+        block.call
+      ensure
+        remove
+      end
+    end
+
+    # Checks if the lockfile exists.
+    # @return [Boolean] true if the lockfile exists, false otherwise
+    def exists?
+      File.exist?(path)
+    end
+
+    # Removes the lockfile.
+    def remove
+      FileUtils.rm_f(path)
+    end
+
+    # Checks if the lockfile is stale (i.e., the process that created it is no longer running).
+    # @return [Boolean] true if the lockfile is stale, false otherwise
+    def stale?
+      return false if !exists?
+
+      file_content = File.read(path)
+      data = JSON.parse(file_content, symbolize_names: true)
+      pid = data[:pid]
+      starttime = data[:starttime]
+      proc_table_entry = Sys::ProcTable.ps(pid: pid)
+
+      return true if proc_table_entry.nil?
+
+      proc_table_entry.starttime != starttime
+    end
+
+    private
+
+    def create
+      pid = Process.pid
+      proc_table_entry = Sys::ProcTable.ps(pid: pid)
+
+      raise "Unable to get process info for PID #{pid}" if proc_table_entry.nil?
+
+      starttime = proc_table_entry.starttime
+
+      data = {
+        pid: pid,
+        starttime: starttime
+      }
+
+      json_data = JSON.generate(data)
+      File.write(path, json_data)
+    end
+  end
+end

data/lib/imap/backup/mirror.rb

@@ -11,10 +11,10 @@ module Imap::Backup
       @reset = reset
     end
 
-    # If necessary, reates the destination folder,
+    # If necessary, creates the destination folder,
     # then deletes any messages in the destination folder
     # that are not in the local store,
-    # sets existing messages' flas
+    # sets existing messages' flags
     # then appends any missing messages
     # and saves the mapping file
     # @return [void]
@@ -29,6 +29,7 @@ module Imap::Backup
     private
 
     CHUNK_SIZE = 100
+    private_constant :CHUNK_SIZE
 
     attr_reader :serializer
     attr_reader :folder

data/lib/imap/backup/serializer/mbox.rb

@@ -109,7 +109,7 @@ module Imap::Backup
     # Sets the mailbox file's updated time to the current time
     # @return [void]
     def touch
-      File.open(pathname, "a") {}
+      FileUtils.touch(pathname)
     end
 
     private

data/lib/imap/backup/serializer/permission_checker.rb

@@ -22,7 +22,7 @@ module Imap::Backup
       return nil if actual.nil?
 
       mask = ~limit & 0o777
-      return if (actual & mask).zero?
+      return if actual.nobits?(mask)
 
       message = format(
         "Permissions on '%<filename>s' " \

data/lib/imap/backup/serializer/version2_migrator.rb

@@ -101,7 +101,7 @@ module Imap::Backup
           end
         end
 
-        next if lines.count.zero?
+        next if lines.none?
 
         message = {
           uid: uids[messages.count],

data/lib/imap/backup/setup/asker.rb

@@ -56,6 +56,7 @@ module Imap::Backup
     private
 
     EMAIL_MATCHER = /^[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]+$/i
+    private_constant :EMAIL_MATCHER
 
     attr_reader :highline
   end

data/lib/imap/backup/setup.rb

@@ -91,6 +91,7 @@ module Imap::Backup
       Imap::Backup::Account.new(
         username: username,
         password: "",
+        local_path: nil,
         folders: []
       ).tap do |a|
         provider = Imap::Backup::Email::Provider.for_address(username)

data/lib/imap/backup/thunderbird/mailbox_exporter.rb

@@ -51,6 +51,7 @@ module Imap::Backup
     private
 
     EXPORT_PREFIX = "imap-backup".freeze
+    private_constant :EXPORT_PREFIX
 
     attr_reader :email
     attr_reader :serializer

data/lib/imap/backup/version.rb

@@ -4,7 +4,7 @@ module Imap::Backup
   # @private
   MAJOR    = 16
   # @private
-  MINOR    = 3
+  MINOR    = 4
   # @private
   REVISION = 0
   # @private

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: imap-backup
 version: !ruby/object:Gem::Version
-  version: 16.3.0
+  version: 16.4.0
 platform: ruby
 authors:
 - Joe Yates
@@ -121,6 +121,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: sys-proctable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -173,6 +187,7 @@ files:
 - lib/imap/backup/account/folder_ensurer.rb
 - lib/imap/backup/account/folder_mapper.rb
 - lib/imap/backup/account/local_only_folder_deleter.rb
+- lib/imap/backup/account/locker.rb
 - lib/imap/backup/account/restore.rb
 - lib/imap/backup/account/serialized_folders.rb
 - lib/imap/backup/cli.rb
@@ -207,6 +222,7 @@ files:
 - lib/imap/backup/file_mode.rb
 - lib/imap/backup/flag_refresher.rb
 - lib/imap/backup/local_only_message_deleter.rb
+- lib/imap/backup/lockfile.rb
 - lib/imap/backup/logger.rb
 - lib/imap/backup/migrator.rb
 - lib/imap/backup/mirror.rb
@@ -254,14 +270,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '3.0'
+      version: '3.2'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.7
+rubygems_version: 4.0.3
 specification_version: 4
 summary: Backup GMail (or other IMAP) accounts to disk
 test_files: []