imap-backup 12.0.0 → 12.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 22cd34d9d0940821677b2007473b2608421dda521dd47f212346385eba4ea6de
-  data.tar.gz: 3f249c818af7260fa15f8ed1509f2151876bddb28544f401c5264d9c74be1f74
+  metadata.gz: ed7a9fd869f7b1ea66f5dc0816a4ce35d77c150bcf8170e2d778e893fda43e9d
+  data.tar.gz: e75c535f849eb326906c1965a1e86507676209cf41cae5b98ad62e894912d9b8
 SHA512:
-  metadata.gz: 8773c5878b741fb116bfc617b79b91f536c10c2406668f6ad0f544f2383e9215d163d37b7e8beec6e237310b72455bac63561e3d03b171cb4b11ca37624d6851
-  data.tar.gz: 3f6113d064ef19931eb32e1c71876e704ec3898f5657154fabe5626f13805ec1b209e3ff56ec2bd4c407f54cb22e0a590064f2ee9713b0dddfd5836c0371e1f9
+  metadata.gz: 7ced0838fae828c771e831441fc3caf71b0465f4da630c3580e1ff842c687e51ad92d7e1913e7f8d48e9bb2c2db2eeee60eea3e2dcb464fda2b51ffc9252d8ea
+  data.tar.gz: d751a6fb0e2eebc107f539cb80609d353bb0c07126fb09e5001bea69476ca9eaa5f1ffce7e6bab27f7efdc149be213997d20c741cc2b60bf3ccf6a3fd0bd810c

data/README.md

@@ -113,7 +113,7 @@ These are activated via two settings:
 As with all performance tweaks, there are trade-offs.
 If you are using a small virtual server or Raspberry Pi
 to run your backups, you will probably want to leave
-the deafult settings.
+the default settings.
 If, on the other hand, you are using a computer with a
 fair bit of RAM, and you are dealing with a *lot* of email,
 then changing these settings may be worthwhile.
@@ -122,16 +122,17 @@ then changing these settings may be worthwhile.
 
 This setting affects all account backups.
 
-When not set, each message is written to disk, one at a time.
-Doing so means the message itself is appended to the MBox file,
-but more importantly, the JSON metadata is rewritten to disk
-from scratch.
+By default, `imap-backup` uses the "delay metadata" strategy.
+As messages are being backed-up, the message *text*
+is written to disk, while the related metadata is stored in memory.
 
-When in use, all of a mailboxes unbackupped messages are
-downloaded first, and then written to disk just once.
+While this uses a little more memory, it avoids rewiting a growing JSON
+file for every message, speeding things up and reducing disk wear.
 
-This speeds up backup as the metadata file is not rewritten
-after each message is added, but it potentially uses much more memory.
+The alternative strategy, called "direct", writes everything to disk
+as it is received. This method is slower, but has the advantage
+of using slightly less memory, which may be important on very
+resource-limited systems, like Raspberry Pis.
 
 ## Multi-fetch Size
 
@@ -140,12 +141,11 @@ By default, during backup, each message is downloaded one-by-one.
 Using this setting, you can download chunks of emails at a time,
 potentially speeding up the process.
 
-If you're not using "Delayed downlaod writes",
-using multi-fetch *will* mean that the backup process will use
-more memory - equivalent to the size of the greater number
-of messages downloaded at a time.
+Using multi-fetch *will* mean that the backup process will use
+more memory - equivalent to the size of the groups of messages
+that are downloaded.
 
-This behaviour may also exceed limits on your email provider,
+This behaviour may also exceed the rate limits on your email provider,
 so it's best to check before cranking it up!
 
 # Troubleshooting

data/docs/development.md

@@ -6,25 +6,26 @@
 
 # Development
 
-A setup for developing under any available Ruby version is
-available in the container directory.
+A Dockerfile is available to allow testing with all available Ruby versions,
+see the `dev/container` directory.
 
 # Testing
 
 ## Feature Specs
 
-Specs under `specs/features` are integration specs run against a local IMAP server
-controlled by Docker Compose.
-Before running the test suite, it needs to be started:
+Specs under `specs/features` are integration specs run against
+two local IMAP servers controlled by Docker Compose.
+
+Start them before running the test suite
 
 ```sh
-$ docker-compose up -d
+$ docker-compose -f dev/docker-compose.yml up -d
 ```
 
 or, with Podman
 
 ```sh
-$ podman-compose -f docker-compose.yml up -d
+$ podman-compose -f dev/docker-compose.yml up -d
 ```
 
 ```sh
@@ -62,24 +63,23 @@ use `last_command_started.output`.
 
 ```ruby
 require "net/imap"
+require_relative "spec/features/support/30_email_server_helpers"
 
-imap = Net::IMAP.new("localhost", {port: 8993, ssl: {verify_mode: 0}})
-username = "address@example.com"
-imap.login(username, "pass")
+include EmailServerHelpers
 
-message = "From: #{username}\nSubject: Some Subject\n\nHello!\n"
-response = imap.append("INBOX", message, nil, nil)
+test_connection = test_server_connection_parameters
 
-imap.examine("INBOX")
-uids = imap.uid_search(["ALL"]).sort
+test_imap = Net::IMAP.new(test_connection[:server], test_connection[:connection_options])
+test_imap.login(test_connection[:username], test_connection[:password])
 
-fetch_data_items = imap.uid_fetch(uids, ["BODY[]"])
-```
+message = "From: #{test_connection[:username]}\nSubject: Some Subject\n\nHello!\n"
+response = test_imap.append("INBOX", message, nil, nil)
 
-# Older Ruby Versions
+test_imap.examine("INBOX")
+uids = test_imap.uid_search(["ALL"]).sort
 
-Dockerfiles are available for all the supported Ruby versions,
-see the `container` directory.
+fetch_data_items = test_imap.uid_fetch(uids, ["BODY[]"])
+```
 
 # Contributing
 

data/lib/imap/backup/cli/folder_enumerator.rb

@@ -30,7 +30,7 @@ module Imap::Backup
       Pathname.glob(glob) do |path|
         name = source_folder_name(path)
         serializer = Serializer.new(source_local_path, name)
-        folder = destination_folder_for(name)
+        folder = destination_folder_for_path(name)
         yield serializer, folder
       end
     end
@@ -55,8 +55,8 @@ module Imap::Backup
         end
     end
 
-    def destination_folder_for(name)
-      parts = name.split(source_delimiter)
+    def destination_folder_for_path(name)
+      parts = name.split("/")
       no_source_prefix =
         if source_prefix_clipped != "" && parts.first == source_prefix_clipped
           parts[1..]

data/lib/imap/backup/cli/transfer.rb

@@ -0,0 +1,163 @@
+require "imap/backup/cli/backup"
+require "imap/backup/cli/helpers"
+require "imap/backup/cli/folder_enumerator"
+require "imap/backup/migrator"
+require "imap/backup/mirror"
+
+module Imap; end
+
+module Imap::Backup
+  class CLI::Transfer < Thor
+    include Thor::Actions
+    include CLI::Helpers
+
+    ACTIONS = %i(migrate mirror).freeze
+
+    attr_reader :action
+    attr_accessor :automatic_namespaces
+    attr_accessor :config_path
+    attr_accessor :destination_delimiter
+    attr_reader :destination_email
+    attr_accessor :destination_prefix
+    attr_reader :options
+    attr_accessor :reset
+    attr_accessor :source_delimiter
+    attr_reader :source_email
+    attr_accessor :source_prefix
+
+    def initialize(action, source_email, destination_email, options)
+      super([])
+      @action = action
+      @source_email = source_email
+      @destination_email = destination_email
+      @options = options
+      @automatic_namespaces = nil
+      @config_path = nil
+      @destination_delimiter = nil
+      @destination_prefix = nil
+      @reset = nil
+      @source_delimiter = nil
+      @source_prefix = nil
+    end
+
+    no_commands do
+      def run
+        raise "Unknown action '#{action}'" if !ACTIONS.include?(action)
+
+        process_options!
+        prepare_mirror if action == :mirror
+
+        folders.each do |serializer, folder|
+          case action
+          when :migrate
+            Migrator.new(serializer, folder, reset: reset).run
+          when :mirror
+            Mirror.new(serializer, folder).run
+          end
+        end
+      end
+
+      def process_options!
+        self.automatic_namespaces = options[:automatic_namespaces] || false
+        self.config_path = options[:config]
+        self.destination_delimiter = options[:destination_delimiter]
+        self.destination_prefix = options[:destination_prefix]
+        self.source_delimiter = options[:source_delimiter]
+        self.source_prefix = options[:source_prefix]
+        self.reset = options[:reset] || false
+        check_accounts!
+        choose_prefixes_and_delimiters!
+      end
+
+      def check_accounts!
+        if destination_email == source_email
+          raise "Source and destination accounts cannot be the same!"
+        end
+
+        raise "Account '#{destination_email}' does not exist" if !destination_account
+
+        raise "Account '#{source_email}' does not exist" if !source_account
+      end
+
+      def choose_prefixes_and_delimiters!
+        if automatic_namespaces
+          ensure_no_prefix_or_delimiter_parameters!
+          query_servers_for_settings
+        else
+          add_prefix_and_delimiter_defaults
+        end
+      end
+
+      def ensure_no_prefix_or_delimiter_parameters!
+        if destination_delimiter
+          raise "--automatic-namespaces is incompatible with --destination-delimiter"
+        end
+        if destination_prefix
+          raise "--automatic-namespaces is incompatible with --destination-prefix"
+        end
+        raise "--automatic-namespaces is incompatible with --source-delimiter" if source_delimiter
+        raise "--automatic-namespaces is incompatible with --source-prefix" if source_prefix
+      end
+
+      def query_servers_for_settings
+        self.destination_prefix, self.destination_delimiter = account_settings(destination_account)
+        self.source_prefix, self.source_delimiter = account_settings(source_account)
+      end
+
+      def account_settings(account)
+        namespaces = account.client.namespace
+        personal = namespaces.personal.first
+        [personal.prefix, personal.delim]
+      end
+
+      def add_prefix_and_delimiter_defaults
+        self.destination_delimiter ||= "/"
+        self.destination_prefix ||= ""
+        self.source_delimiter ||= "/"
+        self.source_prefix ||= ""
+      end
+
+      def prepare_mirror
+        warn_if_source_account_is_not_in_mirror_mode
+
+        CLI::Backup.new(config: config_path, accounts: source_email).run
+      end
+
+      def warn_if_source_account_is_not_in_mirror_mode
+        return if source_account.mirror_mode
+
+        message =
+          "The account '#{source_account.username}' " \
+          "is not set up to make mirror backups"
+        Logger.logger.warn message
+      end
+
+      def config
+        @config ||= load_config(config: config_path)
+      end
+
+      def enumerator_options
+        {
+          destination: destination_account,
+          destination_delimiter: destination_delimiter,
+          destination_prefix: destination_prefix,
+          source: source_account,
+          source_delimiter: source_delimiter,
+          source_prefix: source_prefix
+        }
+      end
+
+      def folders
+        CLI::FolderEnumerator.new(**enumerator_options)
+      end
+
+      def destination_account
+        config.accounts.find { |a| a.username == destination_email }
+      end
+
+      def source_account
+        config.accounts.find { |a| a.username == source_email }
+      end
+    end
+  end
+end

data/lib/imap/backup/cli.rb

@@ -10,18 +10,38 @@ module Imap::Backup
     autoload :Backup, "imap/backup/cli/backup"
     autoload :Folders, "imap/backup/cli/folders"
     autoload :Local, "imap/backup/cli/local"
-    autoload :Migrate, "imap/backup/cli/migrate"
-    autoload :Mirror, "imap/backup/cli/mirror"
     autoload :Remote, "imap/backup/cli/remote"
     autoload :Restore, "imap/backup/cli/restore"
     autoload :Setup, "imap/backup/cli/setup"
     autoload :Stats, "imap/backup/cli/stats"
+    autoload :Transfer, "imap/backup/cli/transfer"
     autoload :Utils, "imap/backup/cli/utils"
 
     include Helpers
 
     VERSION_ARGUMENTS = %w(-v --version).freeze
 
+    NAMESPACE_CONFIGURATION_DESCRIPTION = <<~DESC.freeze
+      Some IMAP servers use namespaces (i.e. prefixes like "INBOX"),
+      while others, while others concatenate the names of subfolders
+      with a charater ("delimiter") other than "/".
+
+      In these cases there are two choices.
+
+      You can use the `--automatic-namespaces` option.
+      This wil query the source and detination servers for their
+      namespace configuration and will adapt paths accordingly.
+      This option requires that both the source and destination
+      servers are available and work with the provided parameters
+      and authentication.
+
+      If automatic configuration does not work as desired, there are the
+      `--source-prefix=`, `--source-delimiter=`,
+      `--destination-prefix=` and `--destination-delimiter=` parameters.
+      To check what values you should use, check the output of the
+      `imap-backup remote namespaces EMAIL` command.
+    DESC
+
     default_task :backup
 
     def self.start(*args)
@@ -83,19 +103,22 @@ module Imap::Backup
       All emails which have been backed up for the "source account" (SOURCE_EMAIL) are
       uploaded to the "destination account" (DESTINATION_EMAIL).
 
-      When one or other account has namespaces (i.e. prefixes like "INBOX"),
-      use the `--source-prefix=` and/or `--destination-prefix=` options.
+      Some configuration may be necessary, as follows:
 
-      When one or other account uses a delimiter other than `/` (i.e. `.`),
-      use the `--source-delimiter=` and/or `--destination-delimiter=` options.
+      #{NAMESPACE_CONFIGURATION_DESCRIPTION}
 
-      If you you want to delete existing emails in destination folders,
+      Finally, if you want to delete existing emails in destination folders,
       use the `--reset` option. In this case, all existing emails are
       deleted before uploading the migrated emails.
     DESC
     config_option
     quiet_option
     verbose_option
+    method_option(
+      "automatic-namespaces",
+      type: :boolean,
+      desc: "automatically choose delimiters and prefixes"
+    )
     method_option(
       "destination-delimiter",
       type: :string,
@@ -126,7 +149,7 @@ module Imap::Backup
     )
     def migrate(source_email, destination_email)
       non_logging_options = Imap::Backup::Logger.setup_logging(options)
-      Migrate.new(source_email, destination_email, **non_logging_options).run
+      Transfer.new(:migrate, source_email, destination_email, non_logging_options).run
     end
 
     desc(
@@ -140,7 +163,7 @@ module Imap::Backup
       If a folder list is configured for the SOURCE_EMAIL account,
       only the folders indicated by the setting are copied.
 
-      First, runs the download of the SOURCE_EMAIL account.
+      First, it runs the download of the SOURCE_EMAIL account.
       If the SOURCE_EMAIL account is **not** configured to be in 'mirror' mode,
       a warning is printed.
 
@@ -152,6 +175,11 @@ module Imap::Backup
     config_option
     quiet_option
     verbose_option
+    method_option(
+      "automatic-namespaces",
+      type: :boolean,
+      desc: "automatically choose delimiters and prefixes"
+    )
     method_option(
       "destination-delimiter",
       type: :string,
@@ -176,7 +204,7 @@ module Imap::Backup
     )
     def mirror(source_email, destination_email)
       non_logging_options = Imap::Backup::Logger.setup_logging(options)
-      Mirror.new(source_email, destination_email, **non_logging_options).run
+      Transfer.new(:mirror, source_email, destination_email, non_logging_options).run
     end
 
     desc "remote SUBCOMMAND [OPTIONS]", "View info about online accounts"

data/lib/imap/backup/version.rb

@@ -2,7 +2,7 @@ module Imap; end
 
 module Imap::Backup
   MAJOR    = 12
-  MINOR    = 0
+  MINOR    = 1
   REVISION = 0
   PRE      = nil
   VERSION  = [MAJOR, MINOR, REVISION, PRE].compact.map(&:to_s).join(".")

data/lib/retry_on_error.rb

@@ -1,11 +1,12 @@
 module RetryOnError
-  def retry_on_error(errors:, limit: 10)
+  def retry_on_error(errors:, limit: 10, on_error: nil)
     tries ||= 1
     yield
   rescue *errors => e
     if tries < limit
       message = "#{e}, attempt #{tries} of #{limit}"
       Imap::Backup::Logger.logger.debug message
+      on_error&.call
       tries += 1
       retry
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: imap-backup
 version: !ruby/object:Gem::Version
-  version: 12.0.0
+  version: 12.1.0
 platform: ruby
 authors:
 - Joe Yates
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-09-19 00:00:00.000000000 Z
+date: 2023-09-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: highline
@@ -247,12 +247,11 @@ files:
 - lib/imap/backup/cli/helpers.rb
 - lib/imap/backup/cli/local.rb
 - lib/imap/backup/cli/local/check.rb
-- lib/imap/backup/cli/migrate.rb
-- lib/imap/backup/cli/mirror.rb
 - lib/imap/backup/cli/remote.rb
 - lib/imap/backup/cli/restore.rb
 - lib/imap/backup/cli/setup.rb
 - lib/imap/backup/cli/stats.rb
+- lib/imap/backup/cli/transfer.rb
 - lib/imap/backup/cli/utils.rb
 - lib/imap/backup/client/apple_mail.rb
 - lib/imap/backup/client/automatic_login_wrapper.rb

data/lib/imap/backup/cli/migrate.rb

@@ -1,87 +0,0 @@
-require "imap/backup/cli/folder_enumerator"
-require "imap/backup/migrator"
-
-module Imap; end
-
-module Imap::Backup
-  class CLI::Migrate < Thor
-    include Thor::Actions
-    include CLI::Helpers
-
-    attr_reader :destination_delimiter
-    attr_reader :destination_email
-    attr_reader :destination_prefix
-    attr_reader :config_path
-    attr_reader :reset
-    attr_reader :source_delimiter
-    attr_reader :source_email
-    attr_reader :source_prefix
-
-    def initialize(
-      source_email,
-      destination_email,
-      config: nil,
-      destination_delimiter: "/",
-      destination_prefix: "",
-      reset: false,
-      source_delimiter: "/",
-      source_prefix: ""
-    )
-      super([])
-      @destination_delimiter = destination_delimiter
-      @destination_email = destination_email
-      @destination_prefix = destination_prefix
-      @config_path = config
-      @reset = reset
-      @source_delimiter = source_delimiter
-      @source_email = source_email
-      @source_prefix = source_prefix
-    end
-
-    no_commands do
-      def run
-        check_accounts!
-        folders.each do |serializer, folder|
-          Migrator.new(serializer, folder, reset: reset).run
-        end
-      end
-
-      def check_accounts!
-        if destination_email == source_email
-          raise "Source and destination accounts cannot be the same!"
-        end
-
-        raise "Account '#{destination_email}' does not exist" if !destination_account
-
-        raise "Account '#{source_email}' does not exist" if !source_account
-      end
-
-      def config
-        @config ||= load_config(config: config_path)
-      end
-
-      def enumerator_options
-        {
-          destination: destination_account,
-          destination_delimiter: destination_delimiter,
-          destination_prefix: destination_prefix,
-          source: source_account,
-          source_delimiter: source_delimiter,
-          source_prefix: source_prefix
-        }
-      end
-
-      def folders
-        CLI::FolderEnumerator.new(**enumerator_options)
-      end
-
-      def destination_account
-        config.accounts.find { |a| a.username == destination_email }
-      end
-
-      def source_account
-        config.accounts.find { |a| a.username == source_email }
-      end
-    end
-  end
-end

data/lib/imap/backup/cli/mirror.rb

@@ -1,97 +0,0 @@
-require "imap/backup/cli/folder_enumerator"
-require "imap/backup/mirror"
-
-module Imap; end
-
-module Imap::Backup
-  class CLI::Mirror < Thor
-    include Thor::Actions
-    include CLI::Helpers
-
-    attr_reader :destination_delimiter
-    attr_reader :destination_email
-    attr_reader :destination_prefix
-    attr_reader :config_path
-    attr_reader :source_delimiter
-    attr_reader :source_email
-    attr_reader :source_prefix
-
-    def initialize(
-      source_email,
-      destination_email,
-      config: nil,
-      destination_delimiter: "/",
-      destination_prefix: "",
-      source_delimiter: "/",
-      source_prefix: ""
-    )
-      super([])
-      @destination_delimiter = destination_delimiter
-      @destination_email = destination_email
-      @destination_prefix = destination_prefix
-      @config_path = config
-      @source_delimiter = source_delimiter
-      @source_email = source_email
-      @source_prefix = source_prefix
-    end
-
-    no_commands do
-      def run
-        check_accounts!
-        warn_if_source_account_is_not_in_mirror_mode
-
-        CLI::Backup.new(config: config_path, accounts: source_email).run
-
-        folders.each do |serializer, folder|
-          Mirror.new(serializer, folder).run
-        end
-      end
-
-      def check_accounts!
-        if destination_email == source_email
-          raise "Source and destination accounts cannot be the same!"
-        end
-
-        raise "Account '#{destination_email}' does not exist" if !destination_account
-
-        raise "Account '#{source_email}' does not exist" if !source_account
-      end
-
-      def warn_if_source_account_is_not_in_mirror_mode
-        return if source_account.mirror_mode
-
-        message =
-          "The account '#{source_account.username}' " \
-          "is not set up to make mirror backups"
-        Logger.logger.warn message
-      end
-
-      def config
-        @config = load_config(config: config_path)
-      end
-
-      def enumerator_options
-        {
-          destination: destination_account,
-          destination_delimiter: destination_delimiter,
-          destination_prefix: destination_prefix,
-          source: source_account,
-          source_delimiter: source_delimiter,
-          source_prefix: source_prefix
-        }
-      end
-
-      def folders
-        CLI::FolderEnumerator.new(**enumerator_options)
-      end
-
-      def destination_account
-        config.accounts.find { |a| a.username == destination_email }
-      end
-
-      def source_account
-        config.accounts.find { |a| a.username == source_email }
-      end
-    end
-  end
-end