imap-backup 3.4.0 → 4.0.0.rc3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 44a7f8e8754461ed51afdbd5711ba809ad6b0ed68c0f40ff95a42688d85881b9
-  data.tar.gz: a70019109a179506f520b969bb03de78fa67fca66001b182af99427b7a02ecdd
+  metadata.gz: 956fde3bc96dc3acd872d51905a88fa9478d9b5cd7022cf55f8e73db769d8ef7
+  data.tar.gz: 28d653dc4e1cfebaed0a730c0231ec13a3c3c04e1a8514352183b11f152db065
 SHA512:
-  metadata.gz: 5055eea6a69b674f02f2b50c0423df197a4c312171aa04a19b183ce784047699815ac603d8550d28f78ada6f2aa4a904d74d5b7bef8f6ea10633de68752f7a65
-  data.tar.gz: 7849285c5b94136b7d5f620eac770630ea4e7cf988b42cc7079021dac95163c271f5fd68d5b620ac9d36326be142557c617459ba890755920c181110cf99531a
+  metadata.gz: 75643fb06176ab9017d0a7673bb428df4907e1350075cf4e3429702b42f03f8a6ef8a2be1095af3b6f4032ebd682d165e05d94b2e3930397ecb31ad95b2be665
+  data.tar.gz: e703041a7ed4e38ced9ef1b762fb3f2ebaf178cd7beb88a204624f6dd7fc53c8f9b9635942018c0f02abfa664db90e52c17eccf3278251e23395072153fe99da

data/.circleci/config.yml

@@ -27,7 +27,7 @@ jobs:
         type: string
     environment:
       BUNDLE_PATH: ./vendor/bundle
-      DOCKER_IMAP_PORT: 993
+      DOCKER_IMAP_SERVER: 993
     docker:
       - image: "cimg/ruby:<< parameters.ruby_version >>"
       - image: antespi/docker-imap-devel:latest
@@ -48,4 +48,4 @@ workflows:
       - test:
           matrix:
             parameters:
-              ruby_version: ["2.4", "2.5", "2.6", "2.7"]
+              ruby_version: ["2.5", "2.6", "2.7"]

data/.rubocop_todo.yml

@@ -63,7 +63,7 @@ Metrics/AbcSize:
 # Offense count: 3
 # Configuration parameters: CountComments, CountAsOne.
 Metrics/ClassLength:
-  Max: 171
+  Max: 172
 
 # Offense count: 19
 # Configuration parameters: CountComments, CountAsOne, ExcludedMethods, IgnoredMethods.

data/CHANGELOG.md

@@ -0,0 +1,19 @@
+# Change Log
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](http://keepachangelog.com/)
+and this project adheres to [Semantic Versioning](http://semver.org/).
+
+## [4.0.0.rc2] - 2021-11-18
+
+### Removed
+
+* GMail OAuth2 support. Tokens only last a few days, so this authentication
+  method is not usable for automated backups.
+
+## [4.0.0.rc1] - 2021-11-17
+
+### Added
+
+* `local` commands to list accounts, folders and emails and to view single
+  emails.

data/README.md

@@ -20,20 +20,24 @@
 
 To use imap-backup with GMail, you will need to enable 'App passwords' on your account.
 
-## GMail OAuth2
+# Installation
+
+```shell
+$ gem install 'imap-backup'
+```
 
-GMail OAuth2 authentication is supported, but as GMail's policy requires
-users to set up an application specific to their account, the feature
-is disabled by default.
+# Commands
 
-You will need to set the environment variable IMAP_BACKUP_ENABLE_GMAIL_OAUTH2.
+For a full list, run
 
-To set it up, [follow the HOWTO](docs/setting-up-gmail-with-oauth2.md).
+```
+$ imap-backup help
+```
 
-# Installation
+For more information about a command, run
 
 ```shell
-$ gem install 'imap-backup'
+$ imap-backup help COMMAND
 ```
 
 # Setup
@@ -148,6 +152,16 @@ Each folder is saved to an mbox file.
 Alongside each mbox is a file with extension '.imap', which lists the source IMAP
 UIDs to allow a full restore.
 
+# Local commands
+
+There a various commands for viewing local backup status.
+
+To view the list, use
+
+```shell
+$ imap_backup help local
+```
+
 # Troubleshooting
 
 If you have problems:

data/bin/imap-backup

@@ -1,98 +1,8 @@
 #!/usr/bin/env ruby
-require "optparse"
 
 $LOAD_PATH.unshift(File.expand_path("../lib/", __dir__))
-require "imap/backup"
+require "imap/backup/cli"
 
-KNOWN_COMMANDS = [
-  {name: "setup",   help: "Create/edit the configuration file"},
-  {name: "backup",  help: "Do the backup (default)"},
-  {name: "folders", help: "List folders for all (or selected) accounts"},
-  {name: "restore", help: "Restore emails to server"},
-  {name: "status",  help: "List count of non backed-up emails per folder"},
-  {name: "help",    help: "Show usage"}
-].freeze
+Imap::Backup::Configuration::List.new.setup_logging
 
-options = {command: "backup"}
-parser = OptionParser.new do |opts|
-  opts.banner = "Usage: #{$PROGRAM_NAME} [options] COMMAND"
-
-  opts.separator ""
-  opts.separator "Commands:"
-  KNOWN_COMMANDS.each do |command|
-    opts.separator format("\t%- 20<name>s %<help>s", command)
-  end
-  opts.separator ""
-  opts.separator "Common options:"
-
-  opts.on(
-    "-a",
-    "--accounts ACCOUNT1[,ACCOUNT2,...]",
-    Array,
-    "only these accounts"
-  ) do |account|
-    options[:accounts] = account
-  end
-
-  opts.on_tail("-h", "--help", "Show usage") do
-    puts opts
-    exit
-  end
-
-  opts.on_tail("--version", "Show version") do
-    puts Imap::Backup::VERSION
-    exit
-  end
-end
-parser.parse!
-
-options[:command] = ARGV.shift if !ARGV.empty?
-
-# rubocop:disable Style/IfUnlessModifier
-if KNOWN_COMMANDS.find { |c| c[:name] == options[:command] }.nil?
-  raise "Unknown command '#{options[:command]}'"
-end
-
-# rubocop:enable Style/IfUnlessModifier
-
-if options[:command] == "help"
-  puts parser
-  exit
-end
-
-begin
-  configuration = Imap::Backup::Configuration::List.new(options[:accounts])
-rescue Imap::Backup::ConfigurationNotFound
-  Imap::Backup::Configuration::Setup.new.run
-  exit
-end
-
-configuration.setup_logging
-
-case options[:command]
-when "setup"
-  Imap::Backup::Configuration::Setup.new.run
-when "backup"
-  configuration.each_connection(&:run_backup)
-when "folders"
-  configuration.each_connection do |connection|
-    puts connection.username
-    folders = connection.folders
-    if folders.nil?
-      warn "Unable to list account folders"
-      exit 1
-    end
-    folders.each { |f| puts "\t#{f}" }
-  end
-when "restore"
-  configuration.each_connection(&:restore)
-when "status"
-  configuration.each_connection do |connection|
-    puts connection.username
-    folders = connection.status
-    folders.each do |f|
-      missing_locally = f[:remote] - f[:local]
-      puts "#{f[:name]}: #{missing_locally.size}"
-    end
-  end
-end
+Imap::Backup::CLI.start(ARGV)

data/imap-backup

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift(File.expand_path("../lib/", __dir__))
+require "imap/backup/cli"
+
+Imap::Backup::CLI.start(ARGV)
+
+__END__
+

data/imap-backup.gemspec

@@ -13,14 +13,13 @@ Gem::Specification.new do |gem|
   gem.executables   = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^spec/})
   gem.require_paths = ["lib"]
-  gem.required_ruby_version = [">= 2.4.0"]
+  gem.required_ruby_version = ">= 2.5"
   gem.version = Imap::Backup::VERSION
 
-  gem.add_runtime_dependency "gmail_xoauth"
-  gem.add_runtime_dependency "googleauth"
   gem.add_runtime_dependency "highline"
   gem.add_runtime_dependency "mail"
   gem.add_runtime_dependency "rake"
+  gem.add_runtime_dependency "thor", "~> 1.1"
 
   gem.add_development_dependency "codeclimate-test-reporter", "~> 0.4.8"
   if RUBY_ENGINE == "jruby"

data/lib/email/mboxrd/message.rb

@@ -36,12 +36,12 @@ module Email::Mboxrd
       supplied_body.gsub(/(?<!\r)\n/, "\r\n")
     end
 
-    private
-
     def parsed
       @parsed ||= Mail.new(supplied_body)
     end
 
+    private
+
     def from
       @from ||=
         begin

data/lib/imap/backup/account/connection.rb

@@ -1,15 +1,11 @@
 require "net/imap"
-require "gmail_xoauth"
 
-require "gmail/authenticator"
 require "retry_on_error"
 
 module Imap::Backup
   module Account; end
 
   class Account::Connection
-    class InvalidGmailOauth2RefreshToken < StandardError; end
-
     include RetryOnError
 
     LOGIN_RETRY_CLASSES = [EOFError, Errno::ECONNRESET, SocketError].freeze
@@ -73,6 +69,19 @@ module Imap::Backup
       end
     end
 
+    def local_folders
+      return enum_for(:local_folders) if !block_given?
+
+      glob = File.join(local_path, "**", "*.imap")
+      base = Pathname.new(local_path)
+      Pathname.glob(glob) do |path|
+        name = path.relative_path_from(base).to_s[0..-6]
+        serializer = Serializer::Mbox.new(local_path, name)
+        folder = Account::Folder.new(self, name)
+        yield serializer, folder
+      end
+    end
+
     def restore
       local_folders do |serializer, folder|
         restore_folder serializer, folder
@@ -80,7 +89,7 @@ module Imap::Backup
     end
 
     def disconnect
-      imap.disconnect
+      imap.disconnect if @imap
     end
 
     def reconnect
@@ -96,17 +105,8 @@ module Imap::Backup
             "Creating IMAP instance: #{server}, options: #{options.inspect}"
           )
           imap = Net::IMAP.new(server, options)
-          if use_gmail_oauth2? && Gmail::Authenticator.refresh_token?(password)
-            authenticator = Gmail::Authenticator.new(email: username, token: password)
-            credentials = authenticator.credentials
-            raise InvalidGmailOauth2RefreshToken if !credentials
-
-            Imap::Backup.logger.debug "Logging in with OAuth2 token: #{username}"
-            imap.authenticate("XOAUTH2", username, credentials.access_token)
-          else
-            Imap::Backup.logger.debug "Logging in: #{username}/#{masked_password}"
-            imap.login(username, password)
-          end
+          Imap::Backup.logger.debug "Logging in: #{username}/#{masked_password}"
+          imap.login(username, password)
           Imap::Backup.logger.debug "Login complete"
           imap
         end
@@ -168,23 +168,6 @@ module Imap::Backup
       password.gsub(/./, "x")
     end
 
-    def use_gmail_oauth2?
-      # TODO: test use of ENV
-      server == Email::Provider::GMAIL_IMAP_SERVER &&
-        ENV["IMAP_BACKUP_ENABLE_GMAIL_OAUTH2"]
-    end
-
-    def local_folders
-      glob = File.join(local_path, "**", "*.imap")
-      base = Pathname.new(local_path)
-      Pathname.glob(glob) do |path|
-        name = path.relative_path_from(base).to_s[0..-6]
-        serializer = Serializer::Mbox.new(local_path, name)
-        folder = Account::Folder.new(self, name)
-        yield serializer, folder
-      end
-    end
-
     def backup_folders
       @backup_folders ||=
         begin

data/lib/imap/backup/cli/backup.rb

@@ -0,0 +1,21 @@
+module Imap::Backup
+  class CLI::Backup < Thor
+    include Thor::Actions
+    include CLI::Helpers
+
+    attr_reader :account_names
+
+    def initialize(options)
+      super([])
+      @account_names = (options[:accounts] || "").split(",")
+    end
+
+    no_commands do
+      def run
+        each_connection(account_names) do |connection|
+          connection.run_backup
+        end
+      end
+    end
+  end
+end

data/lib/imap/backup/cli/folders.rb

@@ -0,0 +1,27 @@
+module Imap::Backup
+  class CLI::Folders < Thor
+    include Thor::Actions
+    include CLI::Helpers
+
+    attr_reader :account_names
+
+    def initialize(options)
+      super([])
+      @account_names = (options[:accounts] || "").split(",")
+    end
+
+    no_commands do
+      def run
+        each_connection(account_names) do |connection|
+          puts connection.username
+          folders = connection.folders
+          if folders.nil?
+            warn "Unable to list account folders"
+            return false
+          end
+          folders.each { |f| puts "\t#{f}" }
+        end
+      end
+    end
+  end
+end

data/lib/imap/backup/cli/helpers.rb

@@ -0,0 +1,20 @@
+require "imap/backup"
+
+module Imap::Backup::CLI::Helpers
+  def symbolized(options)
+    options.each.with_object({}) { |(k, v), acc| acc[k.intern] = v }
+  end
+
+  def each_connection(names)
+    begin
+      connections = Imap::Backup::Configuration::List.new(names)
+    rescue Imap::Backup::ConfigurationNotFound
+      raise "imap-backup is not configured. Run `imap-backup setup`"
+      return
+    end
+
+    connections.each_connection do |connection|
+      yield connection
+    end
+  end
+end

data/lib/imap/backup/cli/local.rb

@@ -0,0 +1,70 @@
+module Imap::Backup
+  class CLI::Local < Thor
+    include Thor::Actions
+    include CLI::Helpers
+
+    desc "accounts", "List locally backed-up accounts"
+    def accounts
+      connections = Imap::Backup::Configuration::List.new
+      connections.accounts.each { |a| puts a[:username] }
+    end
+
+    desc "folders EMAIL", "List account folders"
+    def folders(email)
+      connections = Imap::Backup::Configuration::List.new
+      account = connections.accounts.find { |a| a[:username] == email }
+      raise "#{email} is not a configured account" if !account
+
+      account_connection = Imap::Backup::Account::Connection.new(account)
+      account_connection.local_folders.each do |_s, f|
+        puts %("#{f.name}")
+      end
+    end
+
+    desc "emails EMAIL FOLDER", "List emails in a folder"
+    def emails(email, folder_name)
+      connections = Imap::Backup::Configuration::List.new
+      account = connections.accounts.find { |a| a[:username] == email }
+      raise "#{email} is not a configured account" if !account
+
+      account_connection = Imap::Backup::Account::Connection.new(account)
+      folder_serializer, folder = account_connection.local_folders.find do |(_s, f)|
+        f.name == folder_name
+      end
+      raise "Folder '#{folder_name}' not found" if !folder_serializer
+
+      max_subject = 60
+      puts format("%-10<uid>s  %-#{max_subject}<subject>s - %<date>s", {uid: "UID", subject: "Subject", date: "Date"})
+      puts "-" * (12 + max_subject + 28)
+
+      uids = folder_serializer.uids
+
+      folder_serializer.each_message(uids).map do |uid, message|
+        m = {uid: uid, date: message.parsed.date.to_s, subject: message.parsed.subject}
+        if m[:subject].length > max_subject
+          puts format("% 10<uid>u: %.#{max_subject - 3}<subject>s... - %<date>s", m)
+        else
+          puts format("% 10<uid>u: %-#{max_subject}<subject>s - %<date>s", m)
+        end
+      end
+    end
+
+    desc "email EMAIL FOLDER UID", "Show an email"
+    def email(email, folder_name, uid)
+      connections = Imap::Backup::Configuration::List.new
+      account = connections.accounts.find { |a| a[:username] == email }
+      raise "#{email} is not a configured account" if !account
+
+      account_connection = Imap::Backup::Account::Connection.new(account)
+      folder_serializer, _folder = account_connection.local_folders.find do |(_s, f)|
+        f.name == folder_name
+      end
+      raise "Folder '#{folder_name}' not found" if !folder_serializer
+
+      loaded_message = folder_serializer.load(uid)
+      raise "Message #{uid} not found in folder '#{folder_name}'" if !loaded_message
+
+      puts loaded_message.supplied_body
+    end
+  end
+end

data/lib/imap/backup/cli/restore.rb

@@ -0,0 +1,19 @@
+module Imap::Backup
+  class CLI::Restore < Thor
+    include Thor::Actions
+    include CLI::Helpers
+
+    attr_reader :account_names
+
+    def initialize(options)
+      super([])
+      @account_names = (options[:accounts] || "").split(",")
+    end
+
+    no_commands do
+      def run
+        each_connection(account_names) { |connection| connection.restore }
+      end
+    end
+  end
+end

data/lib/imap/backup/cli/setup.rb

@@ -0,0 +1,13 @@
+class Imap::Backup::CLI::Setup < Thor
+  include Thor::Actions
+
+  def initialize()
+    super([])
+  end
+
+  no_commands do
+    def run
+      Imap::Backup::Configuration::Setup.new.run
+    end
+  end
+end

data/lib/imap/backup/cli/status.rb

@@ -0,0 +1,26 @@
+module Imap::Backup
+  class CLI::Status < Thor
+    include Thor::Actions
+    include CLI::Helpers
+
+    attr_reader :account_names
+
+    def initialize(options)
+      super([])
+      @account_names = (options[:accounts] || "").split(",")
+    end
+
+    no_commands do
+      def run
+        each_connection(account_names) do |connection|
+          puts connection.username
+          folders = connection.status
+          folders.each do |f|
+            missing_locally = f[:remote] - f[:local]
+            puts "#{f[:name]}: #{missing_locally.size}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/imap/backup/cli.rb

@@ -0,0 +1,89 @@
+require "thor"
+
+module Imap::Backup
+  class CLI < Thor
+    require "imap/backup/cli/helpers"
+
+    autoload :Backup, "imap/backup/cli/backup"
+    autoload :Folders, "imap/backup/cli/folders"
+    autoload :Local, "imap/backup/cli/local"
+    autoload :Remote, "imap/backup/cli/remote"
+    autoload :Restore, "imap/backup/cli/restore"
+    autoload :Setup, "imap/backup/cli/setup"
+    autoload :Status, "imap/backup/cli/status"
+
+    include Helpers
+
+    default_task :backup
+
+    def self.exit_on_failure?
+      true
+    end
+
+    def self.accounts_option
+      method_option(
+        "accounts",
+        type: :string,
+        banner: "a comma-separated list of accounts (defaults to all configured accounts)",
+        aliases: ["-a"]
+      )
+    end
+
+    desc "backup [OPTIONS]", "Run the backup"
+    long_desc <<~DESC
+      Downloads any emails not yet present locally.
+      Runs the backup for each configured account,
+      or for those requested via the --accounts option.
+      By default all folders, are backed up.
+      The setup tool can be used to choose a specific list of folders to back up.
+    DESC
+    accounts_option
+    def backup
+      Backup.new(symbolized(options)).run
+    end
+
+    desc "folders [OPTIONS]", "This command is deprecated, use `imap-backup remote folders ACCOUNT`"
+    long_desc <<~DESC
+      Lists all folders of all configured accounts.
+      This command is deprecated.
+      Instead, use a combination of `imap-backup local accounts` to get the list of accounts,
+      and `imap-backup remote folders ACCOUNT` to get the folder list.
+    DESC
+    accounts_option
+    def folders
+      Folders.new(symbolized(options)).run
+    end
+
+    desc "restore [OPTIONS]", "This command is deprecated, use `imap-backup restore ACCOUNT`"
+    long_desc <<~DESC
+      By default, restores all local emails to their respective servers.
+      This command is deprecated.
+      Instead, use `imap-backup restore ACCOUNT` to restore a single account.
+    DESC
+    accounts_option
+    def restore
+      Restore.new(symbolized(options)).run
+    end
+
+    desc "setup", "Configure imap-backup"
+    long_desc <<~DESC
+      A menu-driven command-line application used to configure imap-backup.
+      Configure email accounts to back up.
+    DESC
+    def setup
+      Setup.new().run
+    end
+
+    desc "status", "Show backup status"
+    long_desc <<~DESC
+      For each configured account and folder, lists the number of emails yet to be downloaded.
+    DESC
+    accounts_option
+    def status
+      Status.new(symbolized(options)).run
+    end
+
+    desc "local SUBCOMMAND [OPTIONS]", "View local info"
+    subcommand "local", Local
+  end
+end

data/lib/imap/backup/configuration/account.rb

@@ -68,12 +68,7 @@ module Imap::Backup
 
     def modify_password(menu)
       menu.choice("modify password") do
-        password =
-          if use_gmail_oauth2?(account)
-            Configuration::GmailOauth2.new(account).run
-          else
-            Configuration::Asker.password
-          end
+        password = Configuration::Asker.password
 
         if !password.nil?
           account[:password] = password
@@ -82,11 +77,6 @@ module Imap::Backup
       end
     end
 
-    def use_gmail_oauth2?(account)
-      account[:server] == Email::Provider::GMAIL_IMAP_SERVER &&
-        ENV["IMAP_BACKUP_ENABLE_GMAIL_OAUTH2"]
-    end
-
     def modify_server(menu)
       menu.choice("modify server") do
         server = highline.ask("server: ")