imap-backup 2.2.2 → 3.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.circleci/config.yml +51 -0
- data/.rubocop.yml +2 -2
- data/.rubocop_todo.yml +10 -26
- data/README.md +6 -13
- data/bin/imap-backup +4 -1
- data/docs/01-credentials-screen.png +0 -0
- data/docs/02-new-project.png +0 -0
- data/docs/03-initial-credentials-for-project.png +0 -0
- data/docs/04-credential-type-selection.png +0 -0
- data/docs/05-cant-create-without-consent-setup.png +0 -0
- data/docs/06-user-type-selection.png +0 -0
- data/docs/07-consent-screen-form.png +0 -0
- data/docs/08-app-scopes.png +0 -0
- data/docs/09-scope-selection.png +0 -0
- data/docs/10-updated-app-scopes.png +0 -0
- data/docs/11-test-users.png +0 -0
- data/docs/12-add-users.png +0 -0
- data/docs/13-create-oauth-client.png +0 -0
- data/docs/14-application-details.png +0 -0
- data/docs/16-initial-menu.png +0 -0
- data/docs/17-inputting-the-email-address.png +0 -0
- data/docs/18-choose-password.png +0 -0
- data/docs/19-supply-client-info.png +0 -0
- data/docs/20-choose-gmail-account.png +0 -0
- data/docs/21-accept-warnings.png +0 -0
- data/docs/22-grant-access.png +0 -0
- data/docs/24-confirm-choices.png +0 -0
- data/docs/25-success-code.png +0 -0
- data/docs/26-type-code-into-imap-backup.png +0 -0
- data/docs/27-success.png +0 -0
- data/docs/setting-up-gmail.md +166 -0
- data/imap-backup.gemspec +3 -8
- data/lib/email/mboxrd/message.rb +2 -0
- data/lib/email/provider.rb +3 -1
- data/lib/gmail/authenticator.rb +160 -0
- data/lib/google/auth/stores/in_memory_token_store.rb +9 -0
- data/lib/imap/backup.rb +2 -1
- data/lib/imap/backup/account/connection.rb +67 -35
- data/lib/imap/backup/account/folder.rb +18 -6
- data/lib/imap/backup/configuration/account.rb +9 -1
- data/lib/imap/backup/configuration/folder_chooser.rb +14 -15
- data/lib/imap/backup/configuration/gmail_oauth2.rb +82 -0
- data/lib/imap/backup/configuration/setup.rb +4 -1
- data/lib/imap/backup/configuration/store.rb +12 -12
- data/lib/imap/backup/serializer/mbox_store.rb +2 -2
- data/lib/imap/backup/version.rb +4 -3
- data/lib/retry_on_error.rb +14 -0
- data/spec/features/backup_spec.rb +1 -1
- data/spec/fixtures/connection.yml +1 -1
- data/spec/support/fixtures.rb +7 -2
- data/spec/unit/gmail/authenticator_spec.rb +138 -0
- data/spec/unit/google/auth/stores/in_memory_token_store_spec.rb +15 -0
- data/spec/unit/imap/backup/account/connection_spec.rb +146 -53
- data/spec/unit/imap/backup/account/folder_spec.rb +42 -10
- data/spec/unit/imap/backup/configuration/account_spec.rb +37 -24
- data/spec/unit/imap/backup/configuration/folder_chooser_spec.rb +7 -13
- data/spec/unit/imap/backup/configuration/gmail_oauth2_spec.rb +84 -0
- data/spec/unit/imap/backup/configuration/setup_spec.rb +51 -31
- metadata +83 -9
- data/.travis.yml +0 -25
data/lib/email/mboxrd/message.rb
CHANGED
@@ -10,9 +10,11 @@ module Email::Mboxrd
|
|
10
10
|
cleaned = serialized.gsub(/^>(>*From)/, "\\1")
|
11
11
|
# Serialized messages in this format *should* start with a line
|
12
12
|
# From xxx yy zz
|
13
|
+
# rubocop:disable Style/IfUnlessModifier
|
13
14
|
if cleaned.start_with?("From ")
|
14
15
|
cleaned = cleaned.sub(/^From .*[\r\n]*/, "")
|
15
16
|
end
|
17
|
+
# rubocop:enable Style/IfUnlessModifier
|
16
18
|
new(cleaned)
|
17
19
|
end
|
18
20
|
|
data/lib/email/provider.rb
CHANGED
@@ -1,6 +1,8 @@
|
|
1
1
|
module Email; end
|
2
2
|
|
3
3
|
class Email::Provider
|
4
|
+
GMAIL_IMAP_SERVER = "imap.gmail.com".freeze
|
5
|
+
|
4
6
|
def self.for_address(address)
|
5
7
|
case
|
6
8
|
when address.end_with?("@fastmail.com")
|
@@ -27,7 +29,7 @@ class Email::Provider
|
|
27
29
|
def host
|
28
30
|
case provider
|
29
31
|
when :gmail
|
30
|
-
|
32
|
+
GMAIL_IMAP_SERVER
|
31
33
|
when :fastmail
|
32
34
|
"imap.fastmail.com"
|
33
35
|
end
|
@@ -0,0 +1,160 @@
|
|
1
|
+
require "googleauth"
|
2
|
+
require "google/auth/stores/in_memory_token_store"
|
3
|
+
|
4
|
+
module Gmail; end
|
5
|
+
|
6
|
+
class Gmail::Authenticator
|
7
|
+
class MalformedImapBackupToken < StandardError; end
|
8
|
+
|
9
|
+
class ImapBackupToken
|
10
|
+
attr_reader :token
|
11
|
+
|
12
|
+
def self.from(
|
13
|
+
access_token:,
|
14
|
+
client_id:,
|
15
|
+
client_secret:,
|
16
|
+
expiration_time_millis:,
|
17
|
+
refresh_token:
|
18
|
+
)
|
19
|
+
{
|
20
|
+
access_token: access_token,
|
21
|
+
client_id: client_id,
|
22
|
+
client_secret: client_secret,
|
23
|
+
expiration_time_millis: expiration_time_millis,
|
24
|
+
refresh_token: refresh_token
|
25
|
+
}.to_json
|
26
|
+
end
|
27
|
+
|
28
|
+
def initialize(token)
|
29
|
+
@token = token
|
30
|
+
end
|
31
|
+
|
32
|
+
def valid?
|
33
|
+
return false if !body
|
34
|
+
return false if !access_token
|
35
|
+
return false if !client_id
|
36
|
+
return false if !client_secret
|
37
|
+
return false if !expiration_time_millis
|
38
|
+
return false if !refresh_token
|
39
|
+
|
40
|
+
true
|
41
|
+
end
|
42
|
+
|
43
|
+
def access_token
|
44
|
+
body["access_token"]
|
45
|
+
end
|
46
|
+
|
47
|
+
def client_id
|
48
|
+
body["client_id"]
|
49
|
+
end
|
50
|
+
|
51
|
+
def client_secret
|
52
|
+
body["client_secret"]
|
53
|
+
end
|
54
|
+
|
55
|
+
def expiration_time_millis
|
56
|
+
body["expiration_time_millis"]
|
57
|
+
end
|
58
|
+
|
59
|
+
def refresh_token
|
60
|
+
body["refresh_token"]
|
61
|
+
end
|
62
|
+
|
63
|
+
private
|
64
|
+
|
65
|
+
def body
|
66
|
+
@body ||= JSON.parse(token)
|
67
|
+
rescue JSON::ParserError
|
68
|
+
nil
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
GMAIL_READ_SCOPE = "https://mail.google.com/".freeze
|
73
|
+
OOB_URI = "urn:ietf:wg:oauth:2.0:oob".freeze
|
74
|
+
|
75
|
+
attr_reader :email
|
76
|
+
attr_reader :token
|
77
|
+
|
78
|
+
def self.refresh_token?(text)
|
79
|
+
ImapBackupToken.new(text).valid?
|
80
|
+
end
|
81
|
+
|
82
|
+
def initialize(email:, token:)
|
83
|
+
@email = email
|
84
|
+
@token = token
|
85
|
+
end
|
86
|
+
|
87
|
+
def authorization_url
|
88
|
+
authorizer.get_authorization_url(base_url: OOB_URI)
|
89
|
+
end
|
90
|
+
|
91
|
+
def credentials
|
92
|
+
authorizer.get_credentials(email).tap do |c|
|
93
|
+
c.refresh! if c.expired?
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
def credentials_from_code(code)
|
98
|
+
authorizer.get_credentials_from_code(
|
99
|
+
user_id: email,
|
100
|
+
code: code,
|
101
|
+
base_url: OOB_URI
|
102
|
+
)
|
103
|
+
end
|
104
|
+
|
105
|
+
private
|
106
|
+
|
107
|
+
def auth_client_id
|
108
|
+
@auth_client_id = Google::Auth::ClientId.new(client_id, client_secret)
|
109
|
+
end
|
110
|
+
|
111
|
+
def authorizer
|
112
|
+
@authorizer ||= Google::Auth::UserAuthorizer.new(
|
113
|
+
auth_client_id, GMAIL_READ_SCOPE, token_store
|
114
|
+
)
|
115
|
+
end
|
116
|
+
|
117
|
+
def access_token
|
118
|
+
imap_backup_token.access_token
|
119
|
+
end
|
120
|
+
|
121
|
+
def client_id
|
122
|
+
imap_backup_token.client_id
|
123
|
+
end
|
124
|
+
|
125
|
+
def client_secret
|
126
|
+
imap_backup_token.client_secret
|
127
|
+
end
|
128
|
+
|
129
|
+
def expiration_time_millis
|
130
|
+
imap_backup_token.expiration_time_millis
|
131
|
+
end
|
132
|
+
|
133
|
+
def refresh_token
|
134
|
+
imap_backup_token.refresh_token
|
135
|
+
end
|
136
|
+
|
137
|
+
def imap_backup_token
|
138
|
+
@imap_backup_token ||=
|
139
|
+
ImapBackupToken.new(token).tap do |t|
|
140
|
+
raise MalformedImapBackupToken if !t.valid?
|
141
|
+
end
|
142
|
+
end
|
143
|
+
|
144
|
+
def store_token
|
145
|
+
{
|
146
|
+
"client_id" => client_id,
|
147
|
+
"access_token" => access_token,
|
148
|
+
"refresh_token" => refresh_token,
|
149
|
+
"scope": [GMAIL_READ_SCOPE],
|
150
|
+
"expiration_time_millis": expiration_time_millis
|
151
|
+
}.to_json
|
152
|
+
end
|
153
|
+
|
154
|
+
def token_store
|
155
|
+
@token_store ||=
|
156
|
+
Google::Auth::Stores::InMemoryTokenStore.new.tap do |t|
|
157
|
+
t.store(email, store_token)
|
158
|
+
end
|
159
|
+
end
|
160
|
+
end
|
data/lib/imap/backup.rb
CHANGED
@@ -7,6 +7,7 @@ require "imap/backup/configuration/account"
|
|
7
7
|
require "imap/backup/configuration/asker"
|
8
8
|
require "imap/backup/configuration/connection_tester"
|
9
9
|
require "imap/backup/configuration/folder_chooser"
|
10
|
+
require "imap/backup/configuration/gmail_oauth2"
|
10
11
|
require "imap/backup/configuration/list"
|
11
12
|
require "imap/backup/configuration/setup"
|
12
13
|
require "imap/backup/configuration/store"
|
@@ -28,7 +29,7 @@ module Imap::Backup
|
|
28
29
|
attr_reader :logger
|
29
30
|
|
30
31
|
def initialize
|
31
|
-
@logger = ::Logger.new(
|
32
|
+
@logger = ::Logger.new($stdout)
|
32
33
|
end
|
33
34
|
end
|
34
35
|
|
@@ -1,9 +1,19 @@
|
|
1
1
|
require "net/imap"
|
2
|
+
require "gmail_xoauth"
|
3
|
+
|
4
|
+
require "gmail/authenticator"
|
5
|
+
require "retry_on_error"
|
2
6
|
|
3
7
|
module Imap::Backup
|
4
8
|
module Account; end
|
5
9
|
|
6
10
|
class Account::Connection
|
11
|
+
class InvalidGmailOauth2RefreshToken < StandardError; end
|
12
|
+
|
13
|
+
include RetryOnError
|
14
|
+
|
15
|
+
LOGIN_RETRY_CLASSES = [EOFError, Errno::ECONNRESET, SocketError].freeze
|
16
|
+
|
7
17
|
attr_reader :connection_options
|
8
18
|
attr_reader :local_path
|
9
19
|
attr_reader :password
|
@@ -13,7 +23,7 @@ module Imap::Backup
|
|
13
23
|
@username = options[:username]
|
14
24
|
@password = options[:password]
|
15
25
|
@local_path = options[:local_path]
|
16
|
-
@
|
26
|
+
@config_folders = options[:folders]
|
17
27
|
@server = options[:server]
|
18
28
|
@connection_options = options[:connection_options] || {}
|
19
29
|
@folders = nil
|
@@ -24,21 +34,24 @@ module Imap::Backup
|
|
24
34
|
@folders ||=
|
25
35
|
begin
|
26
36
|
root = provider_root
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
37
|
+
mailbox_lists = imap.list(root, "*")
|
38
|
+
|
39
|
+
if mailbox_lists.nil?
|
40
|
+
message = "Unable to get folder list for account #{username}"
|
41
|
+
Imap::Backup.logger.info message
|
42
|
+
raise message
|
32
43
|
end
|
33
|
-
|
44
|
+
|
45
|
+
utf7_encoded = mailbox_lists.map(&:name)
|
46
|
+
utf7_encoded.map { |n| Net::IMAP.decode_utf7(n) }
|
34
47
|
end
|
35
48
|
end
|
36
49
|
|
37
50
|
def status
|
38
|
-
backup_folders.map do |
|
39
|
-
f = Account::Folder.new(self,
|
40
|
-
s = Serializer::Mbox.new(local_path,
|
41
|
-
{name:
|
51
|
+
backup_folders.map do |backup_folder|
|
52
|
+
f = Account::Folder.new(self, backup_folder[:name])
|
53
|
+
s = Serializer::Mbox.new(local_path, backup_folder[:name])
|
54
|
+
{name: backup_folder[:name], local: s.uids, remote: f.uids}
|
42
55
|
end
|
43
56
|
end
|
44
57
|
|
@@ -71,25 +84,42 @@ module Imap::Backup
|
|
71
84
|
end
|
72
85
|
|
73
86
|
def imap
|
74
|
-
|
87
|
+
@imap ||=
|
88
|
+
retry_on_error(errors: LOGIN_RETRY_CLASSES) do
|
89
|
+
options = provider_options
|
90
|
+
Imap::Backup.logger.debug(
|
91
|
+
"Creating IMAP instance: #{server}, options: #{options.inspect}"
|
92
|
+
)
|
93
|
+
imap = Net::IMAP.new(server, options)
|
94
|
+
if gmail? && Gmail::Authenticator.refresh_token?(password)
|
95
|
+
authenticator = Gmail::Authenticator.new(email: username, token: password)
|
96
|
+
credentials = authenticator.credentials
|
97
|
+
raise InvalidGmailOauth2RefreshToken if !credentials
|
98
|
+
|
99
|
+
Imap::Backup.logger.debug "Logging in with OAuth2 token: #{username}"
|
100
|
+
imap.authenticate("XOAUTH2", username, credentials.access_token)
|
101
|
+
else
|
102
|
+
Imap::Backup.logger.debug "Logging in: #{username}/#{masked_password}"
|
103
|
+
imap.login(username, password)
|
104
|
+
end
|
105
|
+
Imap::Backup.logger.debug "Login complete"
|
106
|
+
imap
|
107
|
+
end
|
108
|
+
end
|
75
109
|
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
@
|
81
|
-
Imap::Backup.logger.debug "Logging in: #{username}/#{masked_password}"
|
82
|
-
@imap.login(username, password)
|
83
|
-
Imap::Backup.logger.debug "Login complete"
|
84
|
-
@imap
|
110
|
+
def server
|
111
|
+
return @server if @server
|
112
|
+
return nil if provider.nil?
|
113
|
+
|
114
|
+
@server = provider.host
|
85
115
|
end
|
86
116
|
|
87
117
|
private
|
88
118
|
|
89
119
|
def each_folder
|
90
|
-
backup_folders.each do |
|
91
|
-
folder = Account::Folder.new(self,
|
92
|
-
serializer = Serializer::Mbox.new(local_path,
|
120
|
+
backup_folders.each do |backup_folder|
|
121
|
+
folder = Account::Folder.new(self, backup_folder[:name])
|
122
|
+
serializer = Serializer::Mbox.new(local_path, backup_folder[:name])
|
93
123
|
yield folder, serializer
|
94
124
|
end
|
95
125
|
end
|
@@ -133,10 +163,8 @@ module Imap::Backup
|
|
133
163
|
password.gsub(/./, "x")
|
134
164
|
end
|
135
165
|
|
136
|
-
def
|
137
|
-
|
138
|
-
|
139
|
-
(folders || []).map { |f| {name: f.name} }
|
166
|
+
def gmail?
|
167
|
+
server == Email::Provider::GMAIL_IMAP_SERVER
|
140
168
|
end
|
141
169
|
|
142
170
|
def local_folders
|
@@ -150,15 +178,19 @@ module Imap::Backup
|
|
150
178
|
end
|
151
179
|
end
|
152
180
|
|
153
|
-
def
|
154
|
-
@
|
181
|
+
def backup_folders
|
182
|
+
@backup_folders ||=
|
183
|
+
begin
|
184
|
+
if @config_folders&.any?
|
185
|
+
@config_folders
|
186
|
+
else
|
187
|
+
folders.map { |name| {name: name} }
|
188
|
+
end
|
189
|
+
end
|
155
190
|
end
|
156
191
|
|
157
|
-
def
|
158
|
-
|
159
|
-
return nil if provider.nil?
|
160
|
-
|
161
|
-
@server = provider.host
|
192
|
+
def provider
|
193
|
+
@provider ||= Email::Provider.for_address(username)
|
162
194
|
end
|
163
195
|
|
164
196
|
def provider_options
|
@@ -1,5 +1,7 @@
|
|
1
1
|
require "forwardable"
|
2
2
|
|
3
|
+
require "retry_on_error"
|
4
|
+
|
3
5
|
module Imap::Backup
|
4
6
|
module Account; end
|
5
7
|
|
@@ -7,8 +9,10 @@ module Imap::Backup
|
|
7
9
|
|
8
10
|
class Account::Folder
|
9
11
|
extend Forwardable
|
12
|
+
include RetryOnError
|
10
13
|
|
11
14
|
REQUESTED_ATTRIBUTES = %w[RFC822 FLAGS INTERNALDATE].freeze
|
15
|
+
UID_FETCH_RETRY_CLASSES = [EOFError].freeze
|
12
16
|
|
13
17
|
attr_reader :connection
|
14
18
|
attr_reader :name
|
@@ -36,7 +40,7 @@ module Imap::Backup
|
|
36
40
|
def create
|
37
41
|
return if exist?
|
38
42
|
|
39
|
-
imap.create(
|
43
|
+
imap.create(utf7_encoded_name)
|
40
44
|
end
|
41
45
|
|
42
46
|
def uid_validity
|
@@ -66,7 +70,10 @@ module Imap::Backup
|
|
66
70
|
|
67
71
|
def fetch(uid)
|
68
72
|
examine
|
69
|
-
fetch_data_items =
|
73
|
+
fetch_data_items =
|
74
|
+
retry_on_error(errors: UID_FETCH_RETRY_CLASSES) do
|
75
|
+
imap.uid_fetch([uid.to_i], REQUESTED_ATTRIBUTES)
|
76
|
+
end
|
70
77
|
return nil if fetch_data_items.nil?
|
71
78
|
|
72
79
|
fetch_data_item = fetch_data_items[0]
|
@@ -81,22 +88,27 @@ module Imap::Backup
|
|
81
88
|
def append(message)
|
82
89
|
body = message.imap_body
|
83
90
|
date = message.date&.to_time
|
84
|
-
response = imap.append(
|
91
|
+
response = imap.append(utf7_encoded_name, body, nil, date)
|
85
92
|
extract_uid(response)
|
86
93
|
end
|
87
94
|
|
88
95
|
private
|
89
96
|
|
90
97
|
def examine
|
91
|
-
imap.examine(
|
98
|
+
imap.examine(utf7_encoded_name)
|
92
99
|
rescue Net::IMAP::NoResponseError
|
93
|
-
Imap::Backup.logger.warn "Folder '#{name}' does not exist"
|
94
|
-
raise FolderNotFound, "Folder '#{name}' does not exist"
|
100
|
+
Imap::Backup.logger.warn "Folder '#{name}' does not exist on server"
|
101
|
+
raise FolderNotFound, "Folder '#{name}' does not exist on server"
|
95
102
|
end
|
96
103
|
|
97
104
|
def extract_uid(response)
|
98
105
|
@uid_validity, uid = response.data.code.data.split(" ").map(&:to_i)
|
99
106
|
uid
|
100
107
|
end
|
108
|
+
|
109
|
+
def utf7_encoded_name
|
110
|
+
@utf7_encoded_name ||=
|
111
|
+
Net::IMAP.encode_utf7(name).force_encoding("ASCII-8BIT")
|
112
|
+
end
|
101
113
|
end
|
102
114
|
end
|