imap-backup 3.2.1 → 3.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 41b597de5360d295f1e974e5014c7b8004749298b4332db083aea95e5aa0815a
-  data.tar.gz: 95da2ebb094473eae711c85b177b6beb10153cfb1f3074942165599f40d54f63
+  metadata.gz: 79d29fc2a0bdce3edeeb819767c8c76ab319573ce1391e26cbfcb03df6f7d0b3
+  data.tar.gz: f0c169eb81dff4cdfe793c53dfd5317425c737042fe69c65be30a6df186e5057
 SHA512:
-  metadata.gz: 063bcef552b10a3cdff22d7c92346466533bea168b37d258db4277fe8165edb38967194933b5fd1d17cc5a245f721bcd14201f3f6e604092a39ea17fd4002978
-  data.tar.gz: 9121fda4183536a3a9856ad9fc6e0c1a416d31b44b5dc5e88cb884e3c843cfb668754d2a30fb024399c35aa0137d66ec2143cc2eb5df075b3445120348ae6fd9
+  metadata.gz: 4007a0ef75a42e3ad327c50bc24613a66ecf1b4f814d9f4cb91b65717199fd6a5521a8518691e56f33e7788007fa3b57a065d9383e7e0ced47d373fd7d1ab955
+  data.tar.gz: 4e04086163be13d87de681d91aa7c1664a11a49fc44d089110e67e8c2d89b22ec6811902d34ee6f00e7a3e41ac21d04ea245997c5a7921c1c5990208e8989bd1

data/.circleci/config.yml

@@ -48,4 +48,4 @@ workflows:
       - test:
           matrix:
             parameters:
-              ruby_version: ["2.4", "2.5", "2.6", "2.7"]
+              ruby_version: ["2.5", "2.6", "2.7"]

data/.rubocop_todo.yml

@@ -1,42 +1,129 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2021-01-09 09:21:34 UTC using RuboCop version 0.89.1.
+# on 2021-09-21 15:30:34 UTC using RuboCop version 1.21.0.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
 # versions of RuboCop, may require this file to be generated again.
 
-# Offense count: 11
-# Configuration parameters: IgnoredMethods.
-Metrics/AbcSize:
-  Max: 33
+# Offense count: 1
+# Cop supports --auto-correct.
+Layout/ElseAlignment:
+  Exclude:
+    - 'lib/imap/backup/configuration/gmail_oauth2.rb'
 
 # Offense count: 2
-# Configuration parameters: CountComments, CountAsOne, ExcludedMethods.
-# ExcludedMethods: refine
-Metrics/BlockLength:
-  Max: 138
+# Cop supports --auto-correct.
+# Configuration parameters: EmptyLineBetweenMethodDefs, EmptyLineBetweenClassDefs, EmptyLineBetweenModuleDefs, AllowAdjacentOneLineDefs, NumberOfEmptyLines.
+Layout/EmptyLineBetweenDefs:
+  Exclude:
+    - 'lib/google/auth/stores/in_memory_token_store.rb'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyleAlignWith, Severity.
+# SupportedStylesAlignWith: keyword, variable, start_of_line
+Layout/EndAlignment:
+  Exclude:
+    - 'lib/imap/backup/configuration/gmail_oauth2.rb'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: Width, IgnoredPatterns.
+Layout/IndentationWidth:
+  Exclude:
+    - 'lib/imap/backup/configuration/gmail_oauth2.rb'
+
+# Offense count: 34
+# Configuration parameters: AllowedMethods.
+# AllowedMethods: enums
+Lint/ConstantDefinitionInBlock:
+  Exclude:
+    - 'lib/imap/backup/configuration/asker.rb'
+    - 'spec/unit/gmail/authenticator_spec.rb'
+    - 'spec/unit/google/auth/stores/in_memory_token_store_spec.rb'
+    - 'spec/unit/imap/backup/account/connection_spec.rb'
+    - 'spec/unit/imap/backup/account/folder_spec.rb'
+    - 'spec/unit/imap/backup/configuration/account_spec.rb'
+    - 'spec/unit/imap/backup/configuration/gmail_oauth2_spec.rb'
 
 # Offense count: 2
+# Cop supports --auto-correct.
+# Configuration parameters: AllowComments.
+Lint/UselessMethodDefinition:
+  Exclude:
+    - 'lib/imap/backup/configuration/account.rb'
+    - 'lib/imap/backup/configuration/asker.rb'
+
+# Offense count: 11
+# Configuration parameters: IgnoredMethods, CountRepeatedAttributes.
+Metrics/AbcSize:
+  Max: 33
+
+# Offense count: 3
 # Configuration parameters: CountComments, CountAsOne.
 Metrics/ClassLength:
-  Max: 167
+  Max: 171
 
-# Offense count: 17
-# Configuration parameters: CountComments, CountAsOne, ExcludedMethods.
+# Offense count: 19
+# Configuration parameters: CountComments, CountAsOne, ExcludedMethods, IgnoredMethods.
 Metrics/MethodLength:
-  Max: 25
+  Max: 26
 
 # Offense count: 2
 # Configuration parameters: CountComments, CountAsOne.
 Metrics/ModuleLength:
-  Max: 141
+  Max: 145
+
+# Offense count: 2
+# Configuration parameters: EnforcedStyle, CheckMethodNames, CheckSymbols, AllowedIdentifiers.
+# SupportedStyles: snake_case, normalcase, non_integer
+# AllowedIdentifiers: capture3, iso8601, rfc1123_date, rfc822, rfc2822, rfc3339
+Naming/VariableNumber:
+  Exclude:
+    - 'lib/email/provider.rb'
+    - 'spec/unit/email/provider_spec.rb'
 
-# Offense count: 200
+# Offense count: 209
 # Configuration parameters: AllowSubject.
 RSpec/MultipleMemoizedHelpers:
   Max: 16
 
-# Offense count: 1
+# Offense count: 49
 RSpec/NestedGroups:
   Max: 6
+
+# Offense count: 8
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle.
+# SupportedStyles: nested, compact
+Style/ClassAndModuleChildren:
+  Exclude:
+    - 'lib/email/mboxrd/message.rb'
+    - 'lib/imap/backup/downloader.rb'
+    - 'lib/imap/backup/serializer.rb'
+    - 'lib/imap/backup/serializer/mbox.rb'
+    - 'lib/imap/backup/serializer/mbox_enumerator.rb'
+    - 'lib/imap/backup/serializer/mbox_store.rb'
+    - 'lib/imap/backup/uploader.rb'
+    - 'lib/imap/backup/utils.rb'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+Style/RedundantBegin:
+  Exclude:
+    - 'lib/imap/backup/account/connection.rb'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: MinSize, WordRegex.
+# SupportedStyles: percent, brackets
+Style/WordArray:
+  EnforcedStyle: percent
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
+# URISchemes: http, https
+Layout/LineLength:
+  Max: 133

data/README.md

@@ -16,11 +16,19 @@
 [Rubygem]: http://rubygems.org/gems/imap-backup "Ruby gem at rubygems.org"
 [Continuous Integration]: https://circleci.com/gh/joeyates/imap-backup "Build status by CirceCI"
 
-## GMail
+# GMail
 
-GMail OAuth2 authentication is supported.
+To use imap-backup with GMail, you will need to enable 'App passwords' on your account.
 
-To set it up, [follow the HOWTO](docs/setting-up-gmail.md).
+## GMail OAuth2
+
+GMail OAuth2 authentication is supported, but as GMail's policy requires
+users to set up an application specific to their account, the feature
+is disabled by default.
+
+You will need to set the environment variable IMAP_BACKUP_ENABLE_GMAIL_OAUTH2.
+
+To set it up, [follow the HOWTO](docs/setting-up-gmail-with-oauth2.md).
 
 # Installation
 
@@ -212,6 +220,12 @@ $ rake
 
 To exclude Docker-based tests:
 
+```sh
+rake no-docker
+```
+
+or
+
 ```sh
 $ rspec --tag ~docker
 ```

data/Rakefile

@@ -6,6 +6,12 @@ RSpec::Core::RakeTask.new do |t|
   t.pattern = "spec/**/*_spec.rb"
 end
 
+desc "Run RSpec examples, excluding ones relying on Docker IMAP"
+RSpec::Core::RakeTask.new("no-docker") do |t|
+  t.pattern = "spec/**/*_spec.rb"
+  t.rspec_opts = "--tag ~docker"
+end
+
 RuboCop::RakeTask.new
 
 task default: :spec

data/imap-backup.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |gem|
   gem.executables   = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^spec/})
   gem.require_paths = ["lib"]
-  gem.required_ruby_version = [">= 2.4.0"]
+  gem.required_ruby_version = ">= 2.5"
   gem.version = Imap::Backup::VERSION
 
   gem.add_runtime_dependency "gmail_xoauth"

data/lib/imap/backup/account/connection.rb

@@ -64,7 +64,12 @@ module Imap::Backup
 
         Imap::Backup.logger.debug "[#{folder.name}] running backup"
         serializer.apply_uid_validity(folder.uid_validity)
-        Downloader.new(folder, serializer).run
+        begin
+          Downloader.new(folder, serializer).run
+        rescue Net::IMAP::ByeResponseError
+          reconnect
+          retry
+        end
       end
     end
 
@@ -91,7 +96,7 @@ module Imap::Backup
             "Creating IMAP instance: #{server}, options: #{options.inspect}"
           )
           imap = Net::IMAP.new(server, options)
-          if gmail? && Gmail::Authenticator.refresh_token?(password)
+          if use_gmail_oauth2? && Gmail::Authenticator.refresh_token?(password)
             authenticator = Gmail::Authenticator.new(email: username, token: password)
             credentials = authenticator.credentials
             raise InvalidGmailOauth2RefreshToken if !credentials
@@ -163,8 +168,10 @@ module Imap::Backup
       password.gsub(/./, "x")
     end
 
-    def gmail?
-      server == Email::Provider::GMAIL_IMAP_SERVER
+    def use_gmail_oauth2?
+      # TODO: test use of ENV
+      server == Email::Provider::GMAIL_IMAP_SERVER &&
+        ENV["IMAP_BACKUP_ENABLE_GMAIL_OAUTH2"]
     end
 
     def local_folders

data/lib/imap/backup/configuration/account.rb

@@ -69,7 +69,7 @@ module Imap::Backup
     def modify_password(menu)
       menu.choice("modify password") do
         password =
-          if account[:server] == Email::Provider::GMAIL_IMAP_SERVER
+          if use_gmail_oauth2?(account)
             Configuration::GmailOauth2.new(account).run
           else
             Configuration::Asker.password
@@ -82,6 +82,11 @@ module Imap::Backup
       end
     end
 
+    def use_gmail_oauth2?(account)
+      account[:server] == Email::Provider::GMAIL_IMAP_SERVER &&
+        ENV["IMAP_BACKUP_ENABLE_GMAIL_OAUTH2"]
+    end
+
     def modify_server(menu)
       menu.choice("modify server") do
         server = highline.ask("server: ")

data/lib/imap/backup/configuration/gmail_oauth2.rb

@@ -8,7 +8,7 @@ module Imap::Backup
       You need to authorize imap_backup to get access to your email.
       To do so, please follow the instructions here:
 
-      https://github.com/joeyates/imap-backup/blob/main/docs/setting-up-gmail.md
+      https://github.com/joeyates/imap-backup/blob/main/docs/setting-up-gmail-with-oauth2.md
 
     BANNER
 
@@ -26,8 +26,20 @@ module Imap::Backup
     def run
       Kernel.system("clear")
       Kernel.puts BANNER
-      @client_id = highline.ask("client_id: ")
-      @client_secret = highline.ask("client_secret: ")
+
+      keep = if token.valid?
+        highline.agree("Use existing client info?")
+      else
+        false
+      end
+
+      if keep
+        @client_id = token.client_id
+        @client_secret = token.client_secret
+      else
+        @client_id = highline.ask("client_id: ")
+        @client_secret = highline.ask("client_secret: ")
+      end
 
       Kernel.puts <<~MESSAGE
 
@@ -46,9 +58,9 @@ module Imap::Backup
 
       raise "Failed" if !@credentials
 
-      token = JSON.parse(token_store.load(email))
-      token["client_secret"] = client_secret
-      token.to_json
+      new_token = JSON.parse(token_store.load(email))
+      new_token["client_secret"] = client_secret
+      new_token.to_json
     end
 
     private
@@ -57,6 +69,14 @@ module Imap::Backup
       account[:username]
     end
 
+    def password
+      account[:password]
+    end
+
+    def token
+      @token ||= Gmail::Authenticator::ImapBackupToken.new(password)
+    end
+
     def highline
       Configuration::Setup.highline
     end

data/lib/imap/backup/configuration/list.rb

@@ -12,6 +12,7 @@ module Imap::Backup
       return if !config_exists?
 
       Imap::Backup.setup_logging config
+      Net::IMAP.debug = config.debug?
     end
 
     def each_connection

data/lib/imap/backup/version.rb

@@ -2,7 +2,7 @@ module Imap; end
 
 module Imap::Backup
   MAJOR    = 3
-  MINOR    = 2
+  MINOR    = 4
   REVISION = 1
   PRE      = nil
   VERSION  = [MAJOR, MINOR, REVISION, PRE].compact.map(&:to_s).join(".")

data/spec/fixtures/connection.yml

@@ -2,6 +2,6 @@
 :username: 'address@example.org'
 :password: 'pass'
 :connection_options:
-  :port: <%= ENV.fetch("DOCKER_IMAP_SERVER", 993) %>
+  :port: <%= ENV.fetch("DOCKER_IMAP_SERVER", 8993) %>
   :ssl:
     :verify_mode: 0

data/spec/unit/imap/backup/account/connection_spec.rb

@@ -111,7 +111,15 @@ describe Imap::Backup::Account::Connection do
             with(email: USERNAME, token: PASSWORD) { authenticator }
       end
 
-      context "when the password is our copy of a GMail refresh token" do
+      context "when the password is our copy of a GMail refresh token and  the environment IMAP_BACKUP_ENABLE_GMAIL_OAUTH2 is set" do
+        before do
+          ENV["IMAP_BACKUP_ENABLE_GMAIL_OAUTH2"] = "1"
+        end
+
+        after do
+          ENV.delete("IMAP_BACKUP_ENABLE_GMAIL_OAUTH2")
+        end
+
         it "uses the OAuth2 access_token to authenticate" do
           subject.imap
 
@@ -288,6 +296,24 @@ describe Imap::Backup::Account::Connection do
       end
     end
 
+    context "when the IMAP session expires" do
+      before do
+        data = OpenStruct.new(data: "Session expired")
+        response = OpenStruct.new(data: data)
+        outcomes = [
+          -> { raise Net::IMAP::ByeResponseError, response },
+          -> { nil }
+        ]
+        allow(downloader).to receive(:run) { outcomes.shift.call }
+      end
+
+      it "reconnects" do
+        expect(downloader).to receive(:run).exactly(:twice)
+
+        subject.run_backup
+      end
+    end
+
     context "when run" do
       before { subject.run_backup }
 

data/spec/unit/imap/backup/configuration/account_spec.rb

@@ -214,16 +214,10 @@ describe Imap::Backup::Configuration::Account do
 
     describe "choosing 'modify password'" do
       let(:new_password) { "new_password" }
-      let(:gmail_oauth2) do
-        instance_double(Imap::Backup::Configuration::GmailOauth2, run: nil)
-      end
 
       before do
         allow(Imap::Backup::Configuration::Asker).
           to receive(:password) { new_password }
-        allow(Imap::Backup::Configuration::GmailOauth2).
-          to receive(:new).
-            with(account) { gmail_oauth2 }
         subject.run
         menu.choices["modify password"].call
       end
@@ -245,14 +239,49 @@ describe Imap::Backup::Configuration::Account do
 
         include_examples "it doesn't flag the account as modified"
       end
+    end
 
-      context "when the server is for GMail" do
-        let(:current_server) { GMAIL_IMAP_SERVER }
+    describe "choosing 'modify password' when the server is for GMail" do
+      let(:new_password) { "new_password" }
+      let(:current_server) { GMAIL_IMAP_SERVER }
+      let(:gmail_oauth2) do
+        instance_double(Imap::Backup::Configuration::GmailOauth2, run: nil)
+      end
+
+      before do
+        allow(Imap::Backup::Configuration::Asker).
+          to receive(:password) { new_password }
+        allow(Imap::Backup::Configuration::GmailOauth2).
+          to receive(:new).
+            with(account) { gmail_oauth2 }
+      end
+
+      context "when the environment IMAP_BACKUP_ENABLE_GMAIL_OAUTH2 is set" do
+        before do
+          ENV["IMAP_BACKUP_ENABLE_GMAIL_OAUTH2"] = "1"
+          subject.run
+          menu.choices["modify password"].call
+        end
+
+        after do
+          ENV.delete("IMAP_BACKUP_ENABLE_GMAIL_OAUTH2")
+        end
 
         it "sets up GMail OAuth2" do
           expect(gmail_oauth2).to have_received(:run)
         end
       end
+
+      context "when the environment IMAP_BACKUP_ENABLE_GMAIL_OAUTH2 is not set" do
+        before do
+          subject.run
+          menu.choices["modify password"].call
+        end
+
+        it "sets up GMail OAuth2" do
+          expect(gmail_oauth2).to_not have_received(:run)
+        end
+      end
     end
 
     describe "choosing 'modify server'" do

data/spec/unit/imap/backup/configuration/gmail_oauth2_spec.rb

@@ -1,6 +1,9 @@
 describe Imap::Backup::Configuration::GmailOauth2 do
   include HighLineTestHelpers
 
+  CLIENT_ID = "my_client_id".freeze
+  CLIENT_SECRET = "my_client_secret".freeze
+
   subject { described_class.new(account) }
 
   let(:authorization_url) { "some long authorization_url" }
@@ -11,6 +14,7 @@ describe Imap::Backup::Configuration::GmailOauth2 do
   let(:input) { highline_streams[0] }
   let(:output) { highline_streams[1] }
   let(:account) { {} }
+  let(:user_input) { %W(my_client_id\n my_secret\n my_code\n) }
 
   let(:authorizer) do
     instance_double(
@@ -25,23 +29,31 @@ describe Imap::Backup::Configuration::GmailOauth2 do
       load: json_token
     )
   end
+  let(:token) do
+    instance_double(
+      Gmail::Authenticator::ImapBackupToken,
+      valid?: valid,
+      client_id: CLIENT_ID,
+      client_secret: CLIENT_SECRET
+    )
+  end
+  let(:valid) { false }
 
   before do
     allow(Google::Auth::UserAuthorizer).
       to receive(:new) { authorizer }
     allow(Google::Auth::Stores::InMemoryTokenStore).
       to receive(:new) { token_store }
+    allow(Gmail::Authenticator::ImapBackupToken).
+      to receive(:new) { token }
 
     allow(highline).to receive(:ask).and_call_original
+    allow(highline).to receive(:agree).and_call_original
 
     allow(Kernel).to receive(:system)
     allow(Kernel).to receive(:puts)
 
-    allow(input).to receive(:gets).and_return(
-      "my_client_id\n",
-      "my_secret\n",
-      "my_code\n"
-    )
+    allow(input).to receive(:gets).and_return(*user_input)
   end
 
   describe "#run" do
@@ -80,5 +92,30 @@ describe Imap::Backup::Configuration::GmailOauth2 do
     it "includes the client_secret in the credentials" do
       expect(result).to match('"client_secret":"my_secret"')
     end
+
+    context "when the account already has client info" do
+      let(:valid) { true }
+      let(:user_input) { %W(yes\n) }
+
+      it "requests confirmation of client info" do
+        expect(highline).to have_received(:agree).with("Use existing client info?")
+      end
+
+      context "when yhe user says 'no'" do
+        let(:user_input) { %W(no\n) }
+
+        it "requests client_id" do
+          expect(highline).to have_received(:ask).with("client_id: ")
+        end
+
+        it "requests client_secret" do
+          expect(highline).to have_received(:ask).with("client_secret: ")
+        end
+
+        it "requests the success code" do
+          expect(highline).to have_received(:ask).with("success code: ")
+        end
+      end
+    end
   end
 end

data/spec/unit/imap/backup/configuration/setup_spec.rb

@@ -43,7 +43,7 @@ describe Imap::Backup::Configuration::Setup do
     describe "main menu" do
       before { subject.run }
 
-      %w(add\ account save\ and\ exit exit\ without\ saving).each do |choice|
+      ["add account", "save and exit", "exit without saving"].each do |choice|
         it "includes #{choice}" do
           expect(output.string).to include(choice)
         end

data/spec/unit/imap/backup/utils_spec.rb

@@ -12,7 +12,6 @@ describe Imap::Backup::Utils do
   describe ".check_permissions" do
     let(:requested) { 0o345 }
 
-    # rubocop:disable RSpec/EmptyExampleGroup
     context "with existing files" do
       [
         [0o100, "less than the limit", true],
@@ -37,7 +36,6 @@ describe Imap::Backup::Utils do
         end
       end
     end
-    # rubocop:enable RSpec/EmptyExampleGroup
 
     context "with non-existent files" do
       let(:exists) { false }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: imap-backup
 version: !ruby/object:Gem::Version
-  version: 3.2.1
+  version: 3.4.1
 platform: ruby
 authors:
 - Joe Yates
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-02-17 00:00:00.000000000 Z
+date: 2021-11-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: gmail_xoauth
@@ -210,7 +210,7 @@ files:
 - docs/26-type-code-into-imap-backup.png
 - docs/27-success.png
 - docs/docker-imap.md
-- docs/setting-up-gmail.md
+- docs/setting-up-gmail-with-oauth2.md
 - imap-backup.gemspec
 - lib/email/mboxrd/message.rb
 - lib/email/provider.rb
@@ -283,14 +283,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.4.0
+      version: '2.5'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Backup GMail (or other IMAP) accounts to disk