imap-backup 15.0.1 → 15.0.3.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3504dad0e3f788bcfa95b38065e0ff4035239886461689c478a2e991f5e06cf9
-  data.tar.gz: 7d1151b5fb25f2d84ae2013f8f9f74fee7f998ca36b29333bf44151e52726376
+  metadata.gz: ef0c787a6f2a954caddfc9a5c329333781ad650bf1782feb28cc69e74aeba077
+  data.tar.gz: 23387da9df55557ab56dd46bdeef6608f7af4e80434a02c21767d080598583cb
 SHA512:
-  metadata.gz: 3ec40ec413bd6bedce02d4c823ae1654c06585223532d1260bf01fe4ef0e8ba6e0b8c25c7adf0d2b39f8ff29c5199251d07fb9234a75a3ac50856075c91ccc52
-  data.tar.gz: d31020db8b37295d8d7209758161ce4fe5d16869a6976f3538e12b2d7a74f3cb363328d0dcd79ce665a2b113cf7f6289794ee328ded6e8a73cb544c4d0897428
+  metadata.gz: 7432a214ec8dd20758786550785be876f208f1bc9b76f7f60f75b30abf800b2bc4c9fca655370576fff0dea4675d1a298c4c3e8743449774ae90d914aa635805
+  data.tar.gz: 461488fe00ea29917415845c2fd7a231ffc6f3646638c1ad64c23f74f96acc5c2581f7fb3adc04a3410d5dbec67eb36a417d4b02b31e827ab95870172ab52922

data/lib/imap/backup/account/backup.rb

@@ -18,27 +18,33 @@ module Imap::Backup
     # Runs the backup
     # @return [void]
     def run
-      Logger.logger.info "Running backup of account: #{account.username}"
+      Logger.logger.info "Running backup of account '#{account.username}'"
       # start the connection so we get logging messages in the right order
       account.client.login
 
-      Account::FolderEnsurer.new(account: account).run
-      Account::LocalOnlyFolderDeleter.new(account: account).run if account.mirror_mode
+      run_pre_backup_tasks
       backup_folders = Account::BackupFolders.new(
         client: account.client, account: account
-      )
+      ).to_a
       if backup_folders.none?
-        Logger.logger.warn "Account #{account.username}: No folders found to backup"
+        Logger.logger.warn "No folders found to backup for account '#{account.username}'"
         return
       end
+      Logger.logger.debug "Starting backup of #{backup_folders.count} folders"
       backup_folders.each do |folder|
         Account::FolderBackup.new(account: account, folder: folder, refresh: refresh).run
       end
+      Logger.logger.debug "Backup of account '#{account.username}' complete"
     end
 
     private
 
     attr_reader :account
     attr_reader :refresh
+
+    def run_pre_backup_tasks
+      Account::FolderEnsurer.new(account: account).run
+      Account::LocalOnlyFolderDeleter.new(account: account).run if account.mirror_mode
+    end
   end
 end

data/lib/imap/backup/account/folder.rb

@@ -31,19 +31,15 @@ module Imap::Backup
 
     # @raise any error that occurs more than 10 times
     def exist?
-      previous_level = Imap::Backup::Logger.logger.level
-      previous_debug = Net::IMAP.debug
-      Imap::Backup::Logger.logger.level = ::Logger::Severity::UNKNOWN
-      Net::IMAP.debug = false
+      Logger.logger.debug "Checking whether folder '#{name}' exists"
       retry_on_error(errors: EXAMINE_RETRY_CLASSES) do
         examine
       end
+      Logger.logger.debug "Folder '#{name}' exists"
       true
     rescue FolderNotFound
+      Logger.logger.debug "Folder '#{name}' does not exist"
       false
-    ensure
-      Imap::Backup::Logger.logger.level = previous_level
-      Net::IMAP.debug = previous_debug
     end
 
     # Creates the folder on the server
@@ -69,8 +65,11 @@ module Imap::Backup
     # @raise any error that occurs more than 10 times
     # @return [Array<Integer>] the folders message UIDs
     def uids
+      Logger.logger.debug "Fetching UIDs for folder '#{name}'"
       examine
-      client.uid_search(["ALL"]).sort
+      result = client.uid_search(["ALL"]).sort
+      Logger.logger.debug "#{result.count} UIDs found for folder '#{name}'"
+      result
     rescue FolderNotFound
       []
     rescue NoMethodError

data/lib/imap/backup/account/folder_backup.rb

@@ -22,11 +22,11 @@ module Imap::Backup
     # @raise [RuntimeError] if the configured download strategy is incorrect
     # @return [void]
     def run
+      Logger.logger.debug "Running backup for folder '#{folder.name}'"
+
       folder_ok = folder_ok?
       return if !folder_ok
 
-      Logger.logger.debug "[#{folder.name}] running backup"
-
       serializer.apply_uid_validity(folder.uid_validity)
 
       serializer.transaction do
@@ -36,6 +36,7 @@ module Imap::Backup
       # After the transaction the serializer will have any appended messages
       # so we can check differences between the server and the local backup
       LocalOnlyMessageDeleter.new(folder, raw_serializer).run if account.mirror_mode
+      Logger.logger.debug "Backup for folder '#{folder.name}' complete"
     end
 
     private
@@ -46,10 +47,13 @@ module Imap::Backup
 
     def folder_ok?
       begin
-        return false if !folder.exist?
+        if !folder.exist?
+          Logger.logger.info "Skipping backup for folder '#{folder.name}' as it does not exist"
+          return false
+        end
       rescue Encoding::UndefinedConversionError
         message = "Skipping backup for '#{folder.name}' " \
-                  "as it is not UTF-7 encoded correctly"
+                  "as it's name is not UTF-7 encoded correctly"
         Logger.logger.info message
         return false
       end

data/lib/imap/backup/account/folder_mapper.rb

@@ -32,7 +32,7 @@ module Imap::Backup
     # @yieldparam serializer [Serializer] the folder's serializer
     # @yieldparam folder [Account::Folder] the online folder
     # @return [Enumerator, void]
-    def each
+    def each(&block)
       return enum_for(:each) if !block_given?
 
       glob = File.join(source_local_path, "**", "*.imap")
@@ -40,7 +40,7 @@ module Imap::Backup
         name = source_folder_name(path)
         serializer = Serializer.new(source_local_path, name)
         folder = destination_folder_for_path(name)
-        yield serializer, folder
+        block.call(serializer, folder)
       end
     end
 

data/lib/imap/backup/cli/backup.rb

@@ -24,6 +24,7 @@ module Imap::Backup
     #   @return [void]
     no_commands do
       def run
+        Logger.logger.debug "Loading configuration"
         config = load_config(**options)
         exit_code = nil
         accounts = requested_accounts(config)
@@ -31,6 +32,7 @@ module Imap::Backup
           Logger.logger.warn "No matching accounts found to backup"
           return
         end
+        Logger.logger.debug "Starting backup of #{accounts.count} accounts"
         accounts.each do |account|
           backup = Account::Backup.new(account: account, refresh: refresh)
           backup.run
@@ -43,6 +45,7 @@ module Imap::Backup
           Logger.logger.error message
           next
         end
+        Logger.logger.debug "Backup complete"
         exit(exit_code) if exit_code
       end
     end

data/lib/imap/backup/cli/helpers.rb

@@ -1,5 +1,6 @@
 require "thor"
 
+require "imap/backup/cli/options"
 require "imap/backup/configuration"
 require "imap/backup/configuration_not_found"
 
@@ -11,67 +12,8 @@ module Imap::Backup
   # Provides helper methods for CLI classes
   module CLI::Helpers
     def self.included(base)
-      base.class_eval do
-        def self.accounts_option
-          method_option(
-            "accounts",
-            type: :string,
-            desc: "a comma-separated list of accounts (defaults to all configured accounts)",
-            aliases: ["-a"]
-          )
-        end
-
-        def self.config_option
-          method_option(
-            "config",
-            type: :string,
-            desc: "supply the configuration file path (default: ~/.imap-backup/config.json)",
-            aliases: ["-c"]
-          )
-        end
-
-        def self.format_option
-          method_option(
-            "format",
-            type: :string,
-            desc: "the output type, 'text' for plain text or 'json'",
-            aliases: ["-f"]
-          )
-        end
-
-        def self.quiet_option
-          method_option(
-            "quiet",
-            type: :boolean,
-            desc: "silence all output",
-            aliases: ["-q"]
-          )
-        end
-
-        def self.refresh_option
-          method_option(
-            "refresh",
-            type: :boolean,
-            desc: "in the default 'keep all emails' mode, " \
-                  "updates flags for messages that are already downloaded",
-            aliases: ["-r"]
-          )
-        end
-
-        def self.verbose_option
-          method_option(
-            "verbose",
-            type: :boolean,
-            desc:
-              "increase the amount of logging. " \
-              "Without this option, the program gives minimal output. " \
-              "Using this option once gives more detailed output. " \
-              "Whereas, using this option twice also shows all IMAP network calls",
-            aliases: ["-v"],
-            repeatable: true
-          )
-        end
-      end
+      options = CLI::Options.new(base: base)
+      options.define_options
     end
 
     # Processes command-line parameters
@@ -81,7 +23,7 @@ module Imap::Backup
     def options
       @symbolized_options ||= # rubocop:disable Naming/MemoizedInstanceVariableName
         begin
-          options = super()
+          options = super
           options.each.with_object({}) do |(k, v), acc|
             key =
               if k.is_a?(String)

data/lib/imap/backup/cli/options.rb

@@ -0,0 +1,74 @@
+require "thor"
+
+module Imap; end
+
+module Imap::Backup
+  class CLI < Thor; end
+
+  # Defines option methods for CLI classes
+  class CLI::Options
+    attr_reader :base
+
+    # Options common to many commands
+    OPTIONS = [
+      {
+        name: "accounts",
+        parameters: {
+          type: :string, aliases: ["-a"],
+          desc: "a comma-separated list of accounts (defaults to all configured accounts)"
+        }
+      },
+      {
+        name: "config",
+        parameters: {
+          type: :string, aliases: ["-c"],
+          desc: "supply the configuration file path (default: ~/.imap-backup/config.json)"
+        }
+      },
+      {
+        name: "format",
+        parameters: {
+          type: :string, desc: "the output type, 'text' for plain text or 'json'", aliases: ["-f"]
+        }
+      },
+      {
+        name: "quiet",
+        parameters: {
+          type: :boolean, desc: "silence all output", aliases: ["-q"]
+        }
+      },
+      {
+        name: "refresh",
+        parameters: {
+          type: :boolean, aliases: ["-r"],
+          desc: "in the default 'keep all emails' mode, " \
+                "updates flags for messages that are already downloaded"
+        }
+      },
+      {
+        name: "verbose",
+        parameters: {
+          type: :boolean, aliases: ["-v"], repeatable: true,
+          desc: "increase the amount of logging. " \
+                "Without this option, the program gives minimal output. " \
+                "Using this option once gives more detailed output. " \
+                "Whereas, using this option twice also shows all IMAP network calls"
+        }
+      }
+    ].freeze
+
+    def initialize(base:)
+      @base = base
+    end
+
+    def define_options
+      OPTIONS.each do |option|
+        base.singleton_class.class_eval do
+          define_method("#{option[:name]}_option") do
+            method_option(option[:name], **option[:parameters])
+          end
+        end
+      end
+    end
+  end
+end

data/lib/imap/backup/client/default.rb

@@ -27,6 +27,7 @@ module Imap::Backup
     # @return [Array<String>] the account folders
     def list
       root = provider_root
+      Logger.logger.debug "Listing all account folders"
       mailbox_lists = imap.list(root, "*")
 
       return [] if mailbox_lists.nil?
@@ -105,7 +106,9 @@ module Imap::Backup
     # in the reference.
     def provider_root
       @provider_root ||= begin
+        Logger.logger.debug "Fetching provider root"
         root_info = imap.list("", "")[0]
+        Logger.logger.debug "Provider root is '#{root_info.name}'"
         root_info.name
       end
     end

data/lib/imap/backup/downloader.rb

@@ -19,7 +19,19 @@ module Imap::Backup
     # Runs the downloader
     # @return [void]
     def run
-      info("#{uids.count} new messages") if uids.any?
+      debug("#{serializer_uids.count} already messages already downloaded")
+      debug("#{folder_uids.count} messages on server")
+      local_only_count = (serializer_uids - folder_uids).count
+      if local_only_count.positive?
+        debug("#{local_only_count} downloaded messages no longer on server")
+      end
+
+      if uids.none?
+        debug("no new messages on server — skipping")
+        return
+      end
+
+      info("#{uids.count} new messages")
 
       uids.each_slice(multi_fetch_size).with_index do |block, i|
         multifetch_failed = download_block(block, i)
@@ -62,8 +74,8 @@ module Imap::Backup
         end
       if uids_and_bodies.nil?
         if multi_fetch_size > 1
-          uids = block.join(", ")
-          debug("Multi fetch failed for UIDs #{uids}, switching to single fetches")
+          uid_list = block.join(", ")
+          debug("Multi fetch failed for UIDs #{uid_list}, switching to single fetches")
           return true
         else
           debug("Fetch failed for UID #{block[0]} - skipping")
@@ -96,8 +108,16 @@ module Imap::Backup
       error(e)
     end
 
+    def folder_uids
+      @folder_uids ||= folder.uids
+    end
+
+    def serializer_uids
+      @serializer_uids ||= serializer.uids
+    end
+
     def uids
-      @uids ||= folder.uids - serializer.uids
+      @uids ||= folder_uids - serializer_uids
     end
 
     def debug(message)

data/lib/imap/backup/flag_refresher.rb

@@ -28,6 +28,13 @@ module Imap::Backup
 
     def refresh_block(uids)
       uids_and_flags = folder.fetch_multi(uids, ["FLAGS"])
+      if !uids_and_flags
+        Logger.logger.debug(
+          "[#{folder.name}] failed to fetch flags for #{uids} - " \
+          "cannot refresh flags"
+        )
+        return
+      end
       uids_and_flags.each do |uid_and_flags|
         uid = uid_and_flags[:uid]
         flags = uid_and_flags[:flags]

data/lib/imap/backup/logger.rb

@@ -51,11 +51,11 @@ module Imap::Backup
     # Wraps a block, filtering output to standard error,
     # hidng passwords and outputs the results to standard out
     # @return [void]
-    def self.sanitize_stderr
+    def self.sanitize_stderr(&block)
       sanitizer = Text::Sanitizer.new($stdout)
       previous_stderr = $stderr
       $stderr = sanitizer
-      yield
+      block.call
     ensure
       sanitizer.flush
       $stderr = previous_stderr

data/lib/imap/backup/retry_on_error.rb

@@ -13,9 +13,9 @@ module Imap::Backup
     # @param on_error [Proc] a block to call when an error occurs
     # @raise any error ocurring more than `limit` times
     # @return the result of any successful completion of the block
-    def retry_on_error(errors:, limit: 10, on_error: nil)
+    def retry_on_error(errors:, limit: 10, on_error: nil, &block)
       tries ||= 1
-      yield
+      block.call
     rescue *errors => e
       if tries < limit
         message = "#{e}, attempt #{tries} of #{limit}"

data/lib/imap/backup/serializer/message_enumerator.rb

@@ -14,14 +14,14 @@ module Imap::Backup
     # @param uids [Array<Integer>] the message UIDs of the messages to iterate over
     # @yieldparam message [Serializer::Message]
     # @return [void]
-    def run(uids:)
+    def run(uids:, &block)
       uids.each do |uid_maybe_string|
         uid = uid_maybe_string.to_i
         message = imap.get(uid)
 
         next if !message
 
-        yield message
+        block.call(message)
       end
     end
 

data/lib/imap/backup/version.rb

@@ -6,9 +6,9 @@ module Imap::Backup
   # @private
   MINOR    = 0
   # @private
-  REVISION = 1
+  REVISION = 3
   # @private
-  PRE      = nil
+  PRE      = "rc1".freeze
   # The application version
   VERSION  = [MAJOR, MINOR, REVISION, PRE].compact.map(&:to_s).join(".")
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: imap-backup
 version: !ruby/object:Gem::Version
-  version: 15.0.1
+  version: 15.0.3.rc1
 platform: ruby
 authors:
 - Joe Yates
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-06-04 00:00:00.000000000 Z
+date: 2024-11-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: highline
@@ -153,6 +153,7 @@ files:
 - lib/imap/backup/cli/helpers.rb
 - lib/imap/backup/cli/local.rb
 - lib/imap/backup/cli/local/check.rb
+- lib/imap/backup/cli/options.rb
 - lib/imap/backup/cli/remote.rb
 - lib/imap/backup/cli/restore.rb
 - lib/imap/backup/cli/setup.rb