imap-backup 13.4.0 → 14.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b20d641221d0ad7716b1d5a52f28c80929197d4d22bcfdaab574061f5ba7640a
-  data.tar.gz: 3c6ae086014f6ec6568eb9c226e17928877f1d972898bd3ec0e139ba09983cfb
+  metadata.gz: 0cf96ff7b006e9e09f0ec63c75f594879a43be7065a90a35909f128077ff1a06
+  data.tar.gz: deb1c54af3238a9f8fea9f8cdb084acfde187590dc621f5280a2798dd48426cc
 SHA512:
-  metadata.gz: c85149c357c98f822dfe057e096094613f47f2ce31c039757798641145b158550b8ca43cd3e261fa655b769c850ddc08a8a333aaca24193d3c18c1392d4fd526
-  data.tar.gz: fdf5867043ad025ee8080f968f4d0c11df8b4fd2fd95c31bc2f68c34e0b79865a4ae5e7b700876185356c0f03aba9719bcd5cfe68cb1380a56a144b393f0d101
+  metadata.gz: f170fcdfbad3ebe441d3860eec5123cd2e165749e9fb64824e9e5db1948ba2ff01fb4d8d52da6ce3ad1d607d2235b0bc58f670b95b78a57595b34b039471a689
+  data.tar.gz: ad9180ddaf1ed6698627fc42bfb415cbf12abf5e6a9459483ec2c69c5bc6ef14c8465c95c5fd98934bb1d5bdbaf8c0a9c52bec27e0d6a7c34ff2fae7158b4aa7

data/README.md

@@ -2,6 +2,7 @@
 [![Build Status](https://github.com/joeyates/imap-backup/actions/workflows/main.yml/badge.svg)][CI Status]
 ![Coverage](https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/joeyates/b54fe758bfb405c04bef72dad293d707/raw/coverage.json)
 ![License](https://img.shields.io/github/license/joeyates/imap-backup?color=brightgreen&label=License)
+[![Stars](https://img.shields.io/github/stars/joeyates/imap-backup?style=social)][GitHub Stars]
 
 # imap-backup
 
@@ -11,7 +12,7 @@ Backup, restore and migrate email accounts.
 
 ```sh
 brew install imap-backup # for macOS
-gem install imap-backup # for Linux
+gem install imap-backup --no-document # for Linux
 imap-backup setup
 imap-backup
 ```
@@ -35,6 +36,8 @@ See below for a [full list of commands](#Commands).
 
 ## Homebrew (macOS)
 
+![Homebrew installs](https://img.shields.io/homebrew/installs/dm/imap-backup?label=Homebrew%20installs)
+
 If you have [Homebrew](https://brew.sh/), do this:
 
 ```sh
@@ -44,7 +47,7 @@ brew install imap-backup
 ## As a Ruby Gem
 
 ```sh
-gem install imap-backup
+gem install imap-backup --no-document
 ```
 
 If that doesn't work, see the [detailed installation instructions](docs/installation/rubygem.md).
@@ -55,8 +58,10 @@ If you want to use imap-backup directly from the source code, see [here](docs/in
 
 # Setup
 
-As a first step, you need to add accounts via a menu-driven command
-line program:
+Normally you will want to backup a number of email accounts.
+Doing so requires the creation of a config file.
+
+You do this via a menu-driven command line program:
 
 Run:
 
@@ -64,6 +69,10 @@ Run:
 imap-backup setup
 ```
 
+As an alternative, if you only want to backup a single account,
+you can pass all the necessary parameters directly to the `single backup` command
+(see the [`single backup`](docs/commands/single-backup.md) docs).
+
 ## GMail
 
 To use imap-backup with GMail, Office 365 and other services that require
@@ -80,24 +89,25 @@ imap-backup
 
 Alternatively, add it to your crontab.
 
-Backups can also be inspected, for example via [local show](docs/commands/local-show.md)
-and exported via [utils export-to-thunderbird](docs/commands/utils-export-to-thunderbird.md).
+Backups can also be inspected, for example via [`local show`](docs/commands/local-show.md)
+and exported via [`utils export-to-thunderbird`](docs/commands/utils-export-to-thunderbird.md).
 
 # Commands
 
-* [backup](docs/commands/backup.md)
-* [local accounts](docs/commands/local-accounts.md)
-* [local check](docs/commands/local-check.md)
-* [local folders](docs/commands/local-folders.md)
-* [local list](docs/commands/local-list.md)
-* [local show](docs/commands/local-show.md)
-* [migrate](docs/commands/migrate.md)
-* [mirror](docs/commands/mirror.md)
-* [remote folders](docs/commands/remote-folders.md)
-* [restore](docs/commands/restore.md)
-* [setup](docs/commands/setup.md)
-* [utils export-to-thunderbird](docs/commands/utils-export-to-thunderbird.md)
-* [utils ignore-history](docs/commands/utils-ignore-history.md)
+* [`backup`](docs/commands/backup.md)
+* [`local accounts`](docs/commands/local-accounts.md)
+* [`local check`](docs/commands/local-check.md)
+* [`local folders`](docs/commands/local-folders.md)
+* [`local list`](docs/commands/local-list.md)
+* [`local show`](docs/commands/local-show.md)
+* [`migrate`](docs/commands/migrate.md)
+* [`mirror`](docs/commands/mirror.md)
+* [`remote folders`](docs/commands/remote-folders.md)
+* [`restore`](docs/commands/restore.md)
+* [`setup`](docs/commands/setup.md)
+* [`single backup`](docs/commands/single-backup.md)
+* [`utils export-to-thunderbird`](docs/commands/utils-export-to-thunderbird.md)
+* [`utils ignore-history`](docs/commands/utils-ignore-history.md)
 
 For a full list of available commands, run
 
@@ -118,46 +128,10 @@ to improve backup speed.
 
 These are activated via two settings:
 
-* Global setting "Delay download writes"
-* Account setting "Multi-fetch size"
-
-As with all performance tweaks, there are trade-offs.
-If you are using a small virtual server or Raspberry Pi
-to run your backups, you will probably want to leave
-the default settings.
-If, on the other hand, you are using a computer with a
-fair bit of RAM, and you are dealing with a *lot* of email,
-then changing these settings may be worthwhile.
-
-## Delay download writes
-
-This setting affects all account backups.
-
-By default, `imap-backup` uses the "delay metadata" strategy.
-As messages are being backed-up, the message *text*
-is written to disk, while the related metadata is stored in memory.
+* Global setting "Delay download writes",
+* Account setting "Multi-fetch size".
 
-While this uses a little more memory, it avoids rewiting a growing JSON
-file for every message, speeding things up and reducing disk wear.
-
-The alternative strategy, called "direct", writes everything to disk
-as it is received. This method is slower, but has the advantage
-of using slightly less memory, which may be important on very
-resource-limited systems, like Raspberry Pis.
-
-## Multi-fetch Size
-
-By default, during backup, each message is downloaded one-by-one.
-
-Using this setting, you can download chunks of emails at a time,
-potentially speeding up the process.
-
-Using multi-fetch *will* mean that the backup process will use
-more memory - equivalent to the size of the groups of messages
-that are downloaded.
-
-This behaviour may also exceed the rate limits on your email provider,
-so it's best to check before cranking it up!
+See [the performance document](docs/performance.md) for more information.
 
 # Troubleshooting
 
@@ -168,10 +142,12 @@ If you have problems:
 
 # Development
 
+![Activity](https://img.shields.io/github/last-commit/joeyates/imap-backup/main)
+
 See the [Development documentation](./docs/development.md) for notes
 on development and testing.
 
-See [the CHANGELOG](./CHANGELOG.md) to a list of changes that have been
+See [the CHANGELOG](./CHANGELOG.md) for a list of changes that have been
 made in each release.
 
 * [Source Code]
@@ -180,6 +156,7 @@ made in each release.
 * [CI Status]
 
 [Source Code]: https://github.com/joeyates/imap-backup "Source code at GitHub"
+[GitHub Stars]: https://github.com/joeyates/imap-backup/stargazers "GitHub Stars"
 [Code Documentation]: https://rubydoc.info/gems/imap-backup/frames "Code Documentation at Rubydoc.info"
 [Rubygem]: https://rubygems.org/gems/imap-backup "Ruby gem at rubygems.org"
 [CI Status]: https://github.com/joeyates/imap-backup/actions/workflows/main.yml

data/docs/development.md

@@ -1,6 +1,15 @@
+# Repository
+
+After cloning the repo, run the following command to get
+better `git blame` output:
+
+```sh
+git config --local blame.ignoreRevsFile .git-blame-ignore-revs
+```
+
 # Design Goals
 
-* Secure - use a local file protected by permissions
+* Secure - use a local configuration file protected by permissions
 * Restartable - calculate start point based on already downloaded messages
 * Standalone - do not rely on an email client or MTA
 

data/docs/performance.md

@@ -0,0 +1,53 @@
+# Performace
+
+The two performance-related settings are "Download strategy",
+which is a global setting,
+and "Multi-fetch size", which is an Account-level setting.
+
+As with all performance tweaks, there are trade-offs.
+
+# Overview
+
+The defaults, which suit most machines and plays nice with servers is:
+
+* Download strategy: "delay writing metadata",
+* Multi-fetch size: 1.
+
+If you are using a resource-limited machine like 
+a small virtual server or Raspberry Pi
+to run your backups, you can change "Download strategy".
+
+If your email provider supports it,
+and you don't have tight memeory limits,
+increase "Multi-fetch size" for faster backups.
+
+# Delay download writes
+
+This is a global setting, affecting all account backups.
+
+By default, `imap-backup` uses the "delay writing metadata" strategy.
+As messages are being backed-up, the message *text*
+is written to disk, while the related metadata is stored in memory.
+
+While this uses a *little* more memory, it avoids rewiting a growing JSON
+file for every message, speeding things up and reducing disk wear.
+
+The alternative strategy, called "write straight to disk",
+writes everything to disk as it is received.
+This method is slower, but has the advantage
+of using slightly less memory, which may be important on very
+resource-limited systems, like Raspberry Pis.
+
+# Multi-fetch Size
+
+By default, during backup, each message is downloaded one-by-one.
+
+Using this setting, you can download chunks of emails at a time,
+potentially speeding up the process.
+
+Using multi-fetch *will* mean that the backup process will use
+more memory - equivalent to the size of the groups of messages
+that are downloaded.
+
+This behaviour may also exceed the rate limits on your email provider,
+so it's best to check before cranking it up!

data/lib/imap/backup/cli/helpers.rb

@@ -47,6 +47,16 @@ module Imap::Backup
           )
         end
 
+        def self.refresh_option
+          method_option(
+            "refresh",
+            type: :boolean,
+            desc: "in the default 'keep all emails' mode, " \
+                  "updates flags for messages that are already downloaded",
+            aliases: ["-r"]
+          )
+        end
+
         def self.verbose_option
           method_option(
             "verbose",

data/lib/imap/backup/cli/local.rb

@@ -3,6 +3,7 @@ require "thor"
 require "imap/backup/account/serialized_folders"
 require "imap/backup/cli/helpers"
 require "imap/backup/cli/local/check"
+require "imap/backup/logger"
 
 module Imap; end
 

data/lib/imap/backup/cli/restore.rb

@@ -1,6 +1,7 @@
 require "thor"
 
 require "imap/backup/cli/helpers"
+require "imap/backup/logger"
 
 module Imap; end
 

data/lib/imap/backup/cli/single/backup.rb

@@ -0,0 +1,131 @@
+require "thor"
+
+require "imap/backup/account"
+require "imap/backup/account/backup"
+require "imap/backup/configuration"
+
+module Imap; end
+
+module Imap::Backup
+  class CLI < Thor; end
+  class CLI::Single < Thor; end
+
+  class CLI::Single::Backup
+    attr_reader :options
+    attr_reader :password
+
+    def initialize(options)
+      @options = options
+      @password = nil
+    end
+
+    def run
+      process_options!
+      account = Account.new(
+        username: email,
+        password: password,
+        server: server,
+        download_strategy: download_strategy,
+        folder_blacklist: folder_blacklist,
+        local_path: local_path,
+        mirror: mirror,
+        reset_seen_flags_after_fetch: reset_seen_flags_after_fetch
+      )
+      account.connection_options = connection_options if connection_options
+      account.folders = folders if folders.any?
+      account.multi_fetch_size = multi_fetch_size if multi_fetch_size
+      backup = Account::Backup.new(account: account, refresh: refresh)
+      backup.run
+    end
+
+    private
+
+    def process_options!
+      if !email
+        raise Thor::RequiredArgumentMissingError,
+              "No value provided for required options '--email'"
+      end
+      if !server
+        raise Thor::RequiredArgumentMissingError,
+              "No value provided for required options '--server'"
+      end
+      handle_password_options!
+    end
+
+    def handle_password_options!
+      plain = options[:password]
+      env = options[:password_environment_variable]
+      file = options[:password_file]
+      case [plain, env, file]
+      when [nil, nil, nil]
+        raise Thor::RequiredArgumentMissingError,
+              "Supply one of the --password... parameters"
+      when [plain, nil, nil]
+        @password = plain
+      when [nil, env, nil]
+        @password = ENV.fetch(env)
+      when [nil, nil, file]
+        @password = File.read(file).gsub(/\n$/, "")
+      else
+        raise ArgumentError, "Supply only one of the --password... parameters"
+      end
+    end
+
+    def connection_options
+      options[:connection_options]
+    end
+
+    def download_strategy
+      @download_strategy =
+        case options[:download_strategy]
+        when nil
+          Configuration::DEFAULT_STRATEGY
+        when "delay"
+          "delay_metadata"
+        when "direct"
+          "direct"
+        else
+          raise ArgumentError, "Unknown download_strategy: '#{options[:download_strategy]}'"
+        end
+    end
+
+    def email
+      options[:email]
+    end
+
+    def folder_blacklist
+      options[:folder_blacklist] ? true : false
+    end
+
+    def folders
+      @folders ||= options[:folder] || []
+    end
+
+    def local_path
+      return options[:path] if options.key?(:path)
+
+      for_account = email.tr("@", "_")
+      File.join(Dir.pwd, for_account)
+    end
+
+    def mirror
+      options[:mirror] ? true : false
+    end
+
+    def multi_fetch_size
+      options[:multi_fetch_size]
+    end
+
+    def refresh
+      options[:refresh] ? true : false
+    end
+
+    def reset_seen_flags_after_fetch
+      options[:reset_seen_flags_after_fetch] ? true : false
+    end
+
+    def server
+      options[:server]
+    end
+  end
+end

data/lib/imap/backup/cli/single.rb

@@ -0,0 +1,179 @@
+require "thor"
+
+require "imap/backup/logger"
+require "imap/backup/cli/helpers"
+require "imap/backup/cli/single/backup"
+
+module Imap; end
+
+module Imap::Backup
+  class CLI < Thor; end
+
+  class CLI::Single < Thor
+    include CLI::Helpers
+
+    desc "backup", "Backup a single email account based on command-line parameters"
+    long_desc <<~DESC
+      This is a "stand-alone" backup command that doesn't require
+      a configuration file.
+
+      At a minimum, you need to supply the email, the server and the
+      password. (There are three ways of specifying the password)
+
+      $ imap-backup single backup
+        --email me@example.com
+        --password MyS3kr1t
+        --server imap.example.com
+
+      Instead of supplying the password directly on the command line,
+      there are two alternatives.
+      You can set an environment variable (with any name) to your
+      password, then pass the name of the environment variable.
+
+      For example, if MY_IMAP_PASSWORD is set to your password,
+
+      $ imap-backup single backup
+        --email me@example.com
+        --password-environment-variable MY_IMAP_PASSWORD
+        --server imap.example.com
+
+      Alternatively, you can supply the name of a file that contains
+      the password.
+
+      For example, in `~/imap-password`:
+
+      `MyS3kr1t`
+
+      $ imap-backup single backup
+        --email me@example.com
+        --password-file ~/imap-password
+        --server imap.example.com
+
+      If you need to use an insecure connection (this normally happens
+      when running an OAuth2 proxy), you can specify server connection options
+      in JSON:
+
+      $ imap-backup single backup
+        --email me@example.com
+        --password MyS3kr1t
+        --server imap.example.com
+        --connection-options '{"ssl":{"verify_mode":0}}'
+    DESC
+    method_option(
+      "email",
+      type: :string,
+      desc: "the email address",
+      aliases: ["-e"]
+    )
+    method_option(
+      "server",
+      type: :string,
+      desc: "the address of the IMAP server",
+      aliases: ["-s"]
+    )
+    method_option(
+      "password",
+      type: :string,
+      desc: "your password. " \
+            "As an alternative, use the --password-environment-variable " \
+            "or --password-file parameter. " \
+            "You need to pass exactly one of these parameters. " \
+            "If you pass more than one of the --password... parameters together " \
+            "you will get an error.",
+      aliases: ["-p"]
+    )
+    method_option(
+      "password-environment-variable",
+      type: :string,
+      desc: "an environment variable that is set to your password",
+      aliases: ["-e"]
+    )
+    method_option(
+      "password-file",
+      type: :string,
+      desc: "a file containing your password. " \
+            "Note that to make it easier to create such files, " \
+            "trailing newlines will be removed. " \
+            "If you happen to have a password that ends in a newline (!), " \
+            "you can't use this parameter.",
+      aliases: ["-W"]
+    )
+    method_option(
+      "path",
+      type: "string",
+      desc: "the path of the directory where backups are to be saved. " \
+            "If the directory does not exists, it will be created. " \
+            "If not set, this is set to a diretory under the current path " \
+            "which is derived from the username, by replacing '@' with '_'.",
+      aliases: ["-P"]
+    )
+    method_option(
+      "folder",
+      type: :string,
+      desc: "a folder (this option can be given any number of times). " \
+            "By default, all of an account's folders are backed up. " \
+            "If you supply any --folder parameters, " \
+            "only **those** folders are backed up. " \
+            "See also --folder-blacklist.",
+      repeatable: true,
+      aliases: ["-F"]
+    )
+    method_option(
+      "folder-blacklist",
+      type: :boolean,
+      desc: "if this option is given, the list of --folders specified " \
+            "will treated as a blacklist - " \
+            "those folders will be skipped and " \
+            "all others will be backed up.",
+      default: false,
+      aliases: ["-b"]
+    )
+    method_option(
+      "mirror",
+      type: :boolean,
+      desc: "if this option is given, " \
+            "emails that are removed from the server " \
+            "will be removed from the local backup.",
+      aliases: ["-m"]
+    )
+    method_option(
+      "multi-fetch-size",
+      type: :numeric,
+      desc: "the number of emails to download at a time",
+      default: 1,
+      aliases: ["-n"]
+    )
+    method_option(
+      "connection-options",
+      type: :string,
+      desc: "an optional JSON string with options for the IMAP connection",
+      aliases: ["-o"]
+    )
+    method_option(
+      "download-strategy",
+      type: :string,
+      desc: "the download strategy to adopt. " \
+            "For details, see the documentation for this setting " \
+            "in the setup program.",
+      enum: %w(delay direct),
+      default: "delay",
+      aliases: ["-S"]
+    )
+    refresh_option
+    method_option(
+      "reset-seen-flags-after-fetch",
+      type: :boolean,
+      desc: "reset 'Seen' flags after backup. " \
+            "For details, see the documentation for this setting " \
+            "in the setup program.",
+      aliases: ["-R"]
+    )
+    quiet_option
+    verbose_option
+    def backup
+      non_logging_options = Imap::Backup::Logger.setup_logging(options)
+      direct = Backup.new(non_logging_options)
+      direct.run
+    end
+  end
+end

data/lib/imap/backup/cli.rb

@@ -14,6 +14,7 @@ module Imap::Backup
     autoload :Remote, "imap/backup/cli/remote"
     autoload :Restore, "imap/backup/cli/restore"
     autoload :Setup, "imap/backup/cli/setup"
+    autoload :Single, "imap/backup/cli/single"
     autoload :Stats, "imap/backup/cli/stats"
     autoload :Transfer, "imap/backup/cli/transfer"
     autoload :Utils, "imap/backup/cli/utils"
@@ -78,13 +79,8 @@ module Imap::Backup
     accounts_option
     config_option
     quiet_option
+    refresh_option
     verbose_option
-    method_option(
-      "refresh",
-      type: :boolean,
-      desc: "in 'keep all emails' mode, update flags for messages that are already downloaded",
-      aliases: ["-r"]
-    )
     def backup
       non_logging_options = Imap::Backup::Logger.setup_logging(options)
       Backup.new(non_logging_options).run
@@ -239,6 +235,9 @@ module Imap::Backup
       CLI::Setup.new(non_logging_options).run
     end
 
+    desc "single SUBCOMMAND [OPTIONS]", "Run actions on a single account"
+    subcommand "single", Single
+
     desc "stats EMAIL [OPTIONS]", "Print stats for each account folder"
     long_desc <<~DESC
       For each account folder, lists three counts of emails:

data/lib/imap/backup/downloader.rb

@@ -96,18 +96,24 @@ module Imap::Backup
         debug("uid: #{uid} (#{index}/#{uids.count}) - #{body.size} bytes")
         serializer.append uid, body, flags
       end
+    rescue StandardError => e
+      error(e)
     end
 
     def uids
       @uids ||= folder.uids - serializer.uids
     end
 
-    def info(message)
-      Logger.logger.info("[#{folder.name}] #{message}")
-    end
-
     def debug(message)
       Logger.logger.debug("[#{folder.name}] #{message}")
     end
+
+    def error(message)
+      Logger.logger.error("[#{folder.name}] #{message}")
+    end
+
+    def info(message)
+      Logger.logger.info("[#{folder.name}] #{message}")
+    end
   end
 end

data/lib/imap/backup/logger.rb

@@ -1,3 +1,4 @@
+require "net/imap"
 require "logger"
 require "singleton"
 

data/lib/imap/backup/migrator.rb

@@ -1,3 +1,5 @@
+require "imap/backup/logger"
+
 module Imap; end
 
 module Imap::Backup

data/lib/imap/backup/setup/account.rb

@@ -1,4 +1,5 @@
 require "imap/backup/setup/account/header"
+require "imap/backup/setup/asker"
 require "imap/backup/setup/backup_path"
 require "imap/backup/setup/connection_tester"
 require "imap/backup/setup/email"

data/lib/imap/backup/setup/email.rb

@@ -1,4 +1,5 @@
 require "email/provider"
+require "imap/backup/setup/asker"
 
 module Imap; end
 

data/lib/imap/backup/version.rb

@@ -1,8 +1,8 @@
 module Imap; end
 
 module Imap::Backup
-  MAJOR    = 13
-  MINOR    = 4
+  MAJOR    = 14
+  MINOR    = 1
   REVISION = 0
   PRE      = nil
   VERSION  = [MAJOR, MINOR, REVISION, PRE].compact.map(&:to_s).join(".")

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: imap-backup
 version: !ruby/object:Gem::Version
-  version: 13.4.0
+  version: 14.1.0
 platform: ruby
 authors:
 - Joe Yates
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-10-09 00:00:00.000000000 Z
+date: 2023-10-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: highline
@@ -136,6 +136,7 @@ files:
 - docs/delimiters-and-prefixes.md
 - docs/development.md
 - docs/migrate-server-keep-address.md
+- docs/performance.md
 - imap-backup.gemspec
 - lib/cli_coverage.rb
 - lib/email/mboxrd/message.rb
@@ -165,6 +166,8 @@ files:
 - lib/imap/backup/cli/remote.rb
 - lib/imap/backup/cli/restore.rb
 - lib/imap/backup/cli/setup.rb
+- lib/imap/backup/cli/single.rb
+- lib/imap/backup/cli/single/backup.rb
 - lib/imap/backup/cli/stats.rb
 - lib/imap/backup/cli/transfer.rb
 - lib/imap/backup/cli/utils.rb