imap-backup 2.2.2 → 3.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. checksums.yaml +4 -4
  2. data/.circleci/config.yml +51 -0
  3. data/.rubocop.yml +2 -2
  4. data/.rubocop_todo.yml +10 -26
  5. data/README.md +6 -13
  6. data/bin/imap-backup +4 -1
  7. data/docs/01-credentials-screen.png +0 -0
  8. data/docs/02-new-project.png +0 -0
  9. data/docs/03-initial-credentials-for-project.png +0 -0
  10. data/docs/04-credential-type-selection.png +0 -0
  11. data/docs/05-cant-create-without-consent-setup.png +0 -0
  12. data/docs/06-user-type-selection.png +0 -0
  13. data/docs/07-consent-screen-form.png +0 -0
  14. data/docs/08-app-scopes.png +0 -0
  15. data/docs/09-scope-selection.png +0 -0
  16. data/docs/10-updated-app-scopes.png +0 -0
  17. data/docs/11-test-users.png +0 -0
  18. data/docs/12-add-users.png +0 -0
  19. data/docs/13-create-oauth-client.png +0 -0
  20. data/docs/14-application-details.png +0 -0
  21. data/docs/16-initial-menu.png +0 -0
  22. data/docs/17-inputting-the-email-address.png +0 -0
  23. data/docs/18-choose-password.png +0 -0
  24. data/docs/19-supply-client-info.png +0 -0
  25. data/docs/20-choose-gmail-account.png +0 -0
  26. data/docs/21-accept-warnings.png +0 -0
  27. data/docs/22-grant-access.png +0 -0
  28. data/docs/24-confirm-choices.png +0 -0
  29. data/docs/25-success-code.png +0 -0
  30. data/docs/26-type-code-into-imap-backup.png +0 -0
  31. data/docs/27-success.png +0 -0
  32. data/docs/setting-up-gmail.md +166 -0
  33. data/imap-backup.gemspec +3 -8
  34. data/lib/email/mboxrd/message.rb +2 -0
  35. data/lib/email/provider.rb +3 -1
  36. data/lib/gmail/authenticator.rb +160 -0
  37. data/lib/google/auth/stores/in_memory_token_store.rb +9 -0
  38. data/lib/imap/backup.rb +2 -1
  39. data/lib/imap/backup/account/connection.rb +67 -35
  40. data/lib/imap/backup/account/folder.rb +18 -6
  41. data/lib/imap/backup/configuration/account.rb +9 -1
  42. data/lib/imap/backup/configuration/folder_chooser.rb +14 -15
  43. data/lib/imap/backup/configuration/gmail_oauth2.rb +82 -0
  44. data/lib/imap/backup/configuration/setup.rb +4 -1
  45. data/lib/imap/backup/configuration/store.rb +12 -12
  46. data/lib/imap/backup/serializer/mbox_store.rb +2 -2
  47. data/lib/imap/backup/version.rb +4 -3
  48. data/lib/retry_on_error.rb +14 -0
  49. data/spec/features/backup_spec.rb +1 -1
  50. data/spec/fixtures/connection.yml +1 -1
  51. data/spec/support/fixtures.rb +7 -2
  52. data/spec/unit/gmail/authenticator_spec.rb +138 -0
  53. data/spec/unit/google/auth/stores/in_memory_token_store_spec.rb +15 -0
  54. data/spec/unit/imap/backup/account/connection_spec.rb +146 -53
  55. data/spec/unit/imap/backup/account/folder_spec.rb +42 -10
  56. data/spec/unit/imap/backup/configuration/account_spec.rb +37 -24
  57. data/spec/unit/imap/backup/configuration/folder_chooser_spec.rb +7 -13
  58. data/spec/unit/imap/backup/configuration/gmail_oauth2_spec.rb +84 -0
  59. data/spec/unit/imap/backup/configuration/setup_spec.rb +51 -31
  60. metadata +83 -9
  61. data/.travis.yml +0 -25
@@ -56,9 +56,11 @@ module Imap::Backup
56
56
  )
57
57
  else
58
58
  account[:username] = username
59
+ # rubocop:disable Style/IfUnlessModifier
59
60
  if account[:server].nil? || (account[:server] == "")
60
61
  account[:server] = default_server(username)
61
62
  end
63
+ # rubocop:enable Style/IfUnlessModifier
62
64
  account[:modified] = true
63
65
  end
64
66
  end
@@ -66,7 +68,13 @@ module Imap::Backup
66
68
 
67
69
  def modify_password(menu)
68
70
  menu.choice("modify password") do
69
- password = Configuration::Asker.password
71
+ password =
72
+ if account[:server] == Email::Provider::GMAIL_IMAP_SERVER
73
+ Configuration::GmailOauth2.new(account).run
74
+ else
75
+ Configuration::Asker.password
76
+ end
77
+
70
78
  if !password.nil?
71
79
  account[:password] = password
72
80
  account[:modified] = true
@@ -15,7 +15,7 @@ module Imap::Backup
15
15
  return
16
16
  end
17
17
 
18
- if folders.nil?
18
+ if imap_folders.nil?
19
19
  Imap::Backup.logger.warn "Unable to get folder list"
20
20
  highline.ask "Press a key "
21
21
  return
@@ -44,29 +44,28 @@ module Imap::Backup
44
44
  end
45
45
 
46
46
  def add_folders(menu)
47
- folders.each do |folder|
48
- name = folder.name
49
- mark = selected?(name) ? "+" : "-"
50
- menu.choice("#{mark} #{name}") do
51
- toggle_selection name
47
+ imap_folders.each do |folder|
48
+ mark = selected?(folder) ? "+" : "-"
49
+ menu.choice("#{mark} #{folder}") do
50
+ toggle_selection folder
52
51
  end
53
52
  end
54
53
  end
55
54
 
56
55
  def selected?(folder_name)
57
- backup_folders = account[:folders]
58
- return false if backup_folders.nil?
56
+ config_folders = account[:folders]
57
+ return false if config_folders.nil?
59
58
 
60
- backup_folders.find { |f| f[:name] == folder_name }
59
+ config_folders.find { |f| f[:name] == folder_name }
61
60
  end
62
61
 
63
62
  def remove_missing
64
63
  removed = []
65
- backup_folders = []
64
+ config_folders = []
66
65
  account[:folders].each do |f|
67
- found = folders.find { |folder| folder.name == f[:name] }
66
+ found = imap_folders.find { |folder| folder == f[:name] }
68
67
  if found
69
- backup_folders << f
68
+ config_folders << f
70
69
  else
71
70
  removed << f[:name]
72
71
  end
@@ -74,7 +73,7 @@ module Imap::Backup
74
73
 
75
74
  return if removed.empty?
76
75
 
77
- account[:folders] = backup_folders
76
+ account[:folders] = config_folders
78
77
  account[:modified] = true
79
78
 
80
79
  Kernel.puts <<~MESSAGE
@@ -101,8 +100,8 @@ module Imap::Backup
101
100
  nil
102
101
  end
103
102
 
104
- def folders
105
- @folders ||= connection.folders
103
+ def imap_folders
104
+ @imap_folders ||= connection.folders
106
105
  end
107
106
 
108
107
  def highline
@@ -0,0 +1,82 @@
1
+ module Imap::Backup
2
+ module Configuration; end
3
+
4
+ class Configuration::GmailOauth2
5
+ BANNER = <<~BANNER.freeze
6
+ GMail OAuth2 Setup
7
+
8
+ You need to authorize imap_backup to get access to your email.
9
+ To do so, please follow the instructions here:
10
+
11
+ https://github.com/joeyates/imap-backup/blob/main/docs/setting-up-gmail.md
12
+
13
+ BANNER
14
+
15
+ GMAIL_READ_SCOPE = "https://mail.google.com/".freeze
16
+ OOB_URI = "urn:ietf:wg:oauth:2.0:oob".freeze
17
+
18
+ attr_reader :account
19
+ attr_reader :client_id
20
+ attr_reader :client_secret
21
+
22
+ def initialize(account)
23
+ @account = account
24
+ end
25
+
26
+ def run
27
+ Kernel.system("clear")
28
+ Kernel.puts BANNER
29
+ @client_id = highline.ask("client_id: ")
30
+ @client_secret = highline.ask("client_secret: ")
31
+
32
+ Kernel.puts <<~MESSAGE
33
+
34
+ Open the following URL in your browser
35
+
36
+ #{authorization_url}
37
+
38
+ Then copy the success code
39
+
40
+ MESSAGE
41
+
42
+ @code = highline.ask("success code: ")
43
+ @credentials = authorizer.get_and_store_credentials_from_code(
44
+ user_id: email, code: @code, base_url: OOB_URI
45
+ )
46
+
47
+ raise "Failed" if !@credentials
48
+
49
+ token = JSON.parse(token_store.load(email))
50
+ token["client_secret"] = client_secret
51
+ token.to_json
52
+ end
53
+
54
+ private
55
+
56
+ def email
57
+ account[:username]
58
+ end
59
+
60
+ def highline
61
+ Configuration::Setup.highline
62
+ end
63
+
64
+ def auth_client_id
65
+ @auth_client_id = Google::Auth::ClientId.new(client_id, client_secret)
66
+ end
67
+
68
+ def authorizer
69
+ @authorizer ||= Google::Auth::UserAuthorizer.new(
70
+ auth_client_id, GMAIL_READ_SCOPE, token_store
71
+ )
72
+ end
73
+
74
+ def token_store
75
+ @token_store ||= Google::Auth::Stores::InMemoryTokenStore.new
76
+ end
77
+
78
+ def authorization_url
79
+ authorizer.get_authorization_url(base_url: OOB_URI)
80
+ end
81
+ end
82
+ end
@@ -75,7 +75,10 @@ module Imap::Backup
75
75
  password: "",
76
76
  local_path: File.join(config.path, username.tr("@", "_")),
77
77
  folders: []
78
- }
78
+ }.tap do |c|
79
+ server = Email::Provider.for_address(username)
80
+ c[:server] = server.host if server.host
81
+ end
79
82
  end
80
83
 
81
84
  def edit_account(username)
@@ -51,18 +51,18 @@ module Imap::Backup
51
51
  private
52
52
 
53
53
  def data
54
- return @data if @data
55
-
56
- if File.exist?(pathname)
57
- Utils.check_permissions pathname, 0o600
58
- contents = File.read(pathname)
59
- @data = JSON.parse(contents, symbolize_names: true)
60
- else
61
- @data = {accounts: []}
62
- end
63
- @data[:debug] = false unless @data.include?(:debug)
64
- @data[:debug] = false unless [true, false].include?(@data[:debug])
65
- @data
54
+ @data ||=
55
+ begin
56
+ if File.exist?(pathname)
57
+ Utils.check_permissions pathname, 0o600
58
+ contents = File.read(pathname)
59
+ data = JSON.parse(contents, symbolize_names: true)
60
+ else
61
+ data = {accounts: []}
62
+ end
63
+ data[:debug] = data.key?(:debug) ? data[:debug] == true : false
64
+ data
65
+ end
66
66
  end
67
67
 
68
68
  def remove_modified_flags
@@ -161,7 +161,7 @@ module Imap::Backup
161
161
  def load_nth(index)
162
162
  enumerator = Serializer::MboxEnumerator.new(mbox_pathname)
163
163
  enumerator.each.with_index do |raw, i|
164
- next unless i == index
164
+ next if i != index
165
165
 
166
166
  return Email::Mboxrd::Message.from_serialized(raw)
167
167
  end
@@ -169,7 +169,7 @@ module Imap::Backup
169
169
  end
170
170
 
171
171
  def imap_looks_like_json?
172
- return false unless imap_exist?
172
+ return false if !imap_exist?
173
173
 
174
174
  content = File.read(imap_pathname)
175
175
  content.start_with?("{")
@@ -1,8 +1,9 @@
1
1
  module Imap; end
2
2
 
3
3
  module Imap::Backup
4
- MAJOR = 2
4
+ MAJOR = 3
5
5
  MINOR = 2
6
- REVISION = 2
7
- VERSION = [MAJOR, MINOR, REVISION].compact.map(&:to_s).join(".")
6
+ REVISION = 1
7
+ PRE = nil
8
+ VERSION = [MAJOR, MINOR, REVISION, PRE].compact.map(&:to_s).join(".")
8
9
  end
@@ -0,0 +1,14 @@
1
+ module RetryOnError
2
+ def retry_on_error(errors:, limit: 10)
3
+ tries ||= 1
4
+ yield
5
+ rescue *errors => e
6
+ if tries < limit
7
+ message = "#{e}, attempt #{tries} of #{limit}"
8
+ Imap::Backup.logger.debug message
9
+ tries += 1
10
+ retry
11
+ end
12
+ raise e
13
+ end
14
+ end
@@ -38,7 +38,7 @@ RSpec.describe "backup", type: :feature, docker: true do
38
38
  end
39
39
 
40
40
  it "saves a file version" do
41
- expect(imap_metadata[:version].to_s).to match(/^[0-9\.]$/)
41
+ expect(imap_metadata[:version].to_s).to match(/^[0-9.]$/)
42
42
  end
43
43
 
44
44
  it "records IMAP ids" do
@@ -2,6 +2,6 @@
2
2
  :username: 'address@example.org'
3
3
  :password: 'pass'
4
4
  :connection_options:
5
- :port: 8993
5
+ :port: <%= ENV.fetch("DOCKER_IMAP_SERVER", 993) %>
6
6
  :ssl:
7
7
  :verify_mode: 0
@@ -1,6 +1,11 @@
1
+ require "erb"
2
+ require "yaml"
3
+
1
4
  def fixture(name)
2
5
  spec_root = File.expand_path("..", File.dirname(__FILE__))
3
6
  fixture_path = File.join(spec_root, "fixtures", "#{name}.yml")
4
- fixture = File.read(fixture_path)
5
- YAML.safe_load(fixture, [Symbol])
7
+ content = File.read(fixture_path)
8
+ template = ERB.new(content)
9
+ yaml = template.result(binding)
10
+ YAML.safe_load(yaml, [Symbol])
6
11
  end
@@ -0,0 +1,138 @@
1
+ require "gmail/authenticator"
2
+ require "googleauth"
3
+
4
+ describe Gmail::Authenticator do
5
+ ACCESS_TOKEN = "access_token".freeze
6
+ AUTHORIZATION_URL = "authorization_url".freeze
7
+ CLIENT_ID = "client_id".freeze
8
+ CLIENT_SECRET = "client_secret".freeze
9
+ CODE = "code".freeze
10
+ CREDENTIALS = "credentials".freeze
11
+ EMAIL = "email".freeze
12
+ EXPIRATION_TIME_MILLIS = "expiration_time_millis".freeze
13
+ GMAIL_READ_SCOPE = "https://mail.google.com/".freeze
14
+ IMAP_BACKUP_TOKEN = "imap_backup_token".freeze
15
+ OOB_URI = "urn:ietf:wg:oauth:2.0:oob".freeze
16
+ REFRESH_TOKEN = "refresh_token".freeze
17
+
18
+ subject { described_class.new(**params) }
19
+
20
+ let(:params) do
21
+ {
22
+ email: EMAIL,
23
+ token: IMAP_BACKUP_TOKEN
24
+ }
25
+ end
26
+
27
+ let(:authorizer) do
28
+ instance_double(Google::Auth::UserAuthorizer)
29
+ end
30
+
31
+ let(:imap_backup_token) do
32
+ instance_double(
33
+ Gmail::Authenticator::ImapBackupToken,
34
+ access_token: ACCESS_TOKEN,
35
+ client_id: CLIENT_ID,
36
+ client_secret: CLIENT_SECRET,
37
+ expiration_time_millis: EXPIRATION_TIME_MILLIS,
38
+ refresh_token: REFRESH_TOKEN,
39
+ valid?: true
40
+ )
41
+ end
42
+
43
+ let(:token_store) do
44
+ instance_double(Google::Auth::Stores::InMemoryTokenStore)
45
+ end
46
+
47
+ let(:credentials) do
48
+ instance_double(Google::Auth::UserRefreshCredentials, refresh!: true)
49
+ end
50
+
51
+ let(:expired) { false }
52
+
53
+ before do
54
+ allow(Google::Auth::UserAuthorizer).
55
+ to receive(:new).
56
+ with(
57
+ instance_of(Google::Auth::ClientId),
58
+ GMAIL_READ_SCOPE,
59
+ token_store
60
+ ) { authorizer }
61
+ allow(authorizer).to receive(:get_authorization_url).
62
+ with(base_url: OOB_URI) { AUTHORIZATION_URL }
63
+ allow(authorizer).to receive(:get_credentials).
64
+ with(EMAIL) { credentials }
65
+ allow(authorizer).to receive(:get_credentials_from_code).
66
+ with(user_id: EMAIL, code: CODE, base_url: OOB_URI) { CREDENTIALS }
67
+
68
+ allow(Google::Auth::UserRefreshCredentials).
69
+ to receive(:new) { credentials }
70
+ allow(credentials).to receive(:expired?) { expired }
71
+
72
+ allow(Google::Auth::Stores::InMemoryTokenStore).
73
+ to receive(:new) { token_store }
74
+ allow(token_store).to receive(:store).
75
+ with(EMAIL, anything) # TODO: use a JSON matcher
76
+ allow(Gmail::Authenticator::ImapBackupToken).
77
+ to receive(:new).
78
+ with(IMAP_BACKUP_TOKEN) { imap_backup_token }
79
+ end
80
+
81
+ describe "#initialize" do
82
+ [:email, :token].each do |param|
83
+ context "parameter #{param}" do
84
+ let(:params) { super().dup.reject { |k| k == param } }
85
+
86
+ it "is expected" do
87
+ expect { subject }.to raise_error(
88
+ ArgumentError, /missing keyword: :?#{param}/
89
+ )
90
+ end
91
+ end
92
+ end
93
+ end
94
+
95
+ describe "#credentials" do
96
+ let!(:result) { subject.credentials }
97
+
98
+ it "attempts to get credentials" do
99
+ expect(authorizer).to have_received(:get_credentials)
100
+ end
101
+
102
+ it "returns the result" do
103
+ expect(result).to eq(credentials)
104
+ end
105
+
106
+ context "when the access_token has expired" do
107
+ let(:expired) { true }
108
+
109
+ it "refreshes it" do
110
+ expect(credentials).to have_received(:refresh!)
111
+ end
112
+ end
113
+ end
114
+
115
+ describe "#authorization_url" do
116
+ let!(:result) { subject.authorization_url }
117
+
118
+ it "requests an authorization URL" do
119
+ expect(authorizer).to have_received(:get_authorization_url)
120
+ end
121
+
122
+ it "returns the result" do
123
+ expect(result).to eq(AUTHORIZATION_URL)
124
+ end
125
+ end
126
+
127
+ describe "#credentials_from_code" do
128
+ let!(:result) { subject.credentials_from_code(CODE) }
129
+
130
+ it "requests credentials" do
131
+ expect(authorizer).to have_received(:get_credentials_from_code)
132
+ end
133
+
134
+ it "returns credentials" do
135
+ expect(result).to eq(CREDENTIALS)
136
+ end
137
+ end
138
+ end
@@ -0,0 +1,15 @@
1
+ require "google/auth/stores/in_memory_token_store"
2
+
3
+ describe Google::Auth::Stores::InMemoryTokenStore do
4
+ KEY = "key".freeze
5
+ VALUE = "value".freeze
6
+
7
+ subject { described_class.new }
8
+
9
+ describe "#load" do
10
+ it "returns an item's value" do
11
+ subject[KEY] = VALUE
12
+ expect(subject.load(KEY)).to eq(VALUE)
13
+ end
14
+ end
15
+ end