imap-backup 2.2.2 → 3.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. checksums.yaml +4 -4
  2. data/.circleci/config.yml +51 -0
  3. data/.rubocop.yml +2 -2
  4. data/.rubocop_todo.yml +10 -26
  5. data/README.md +6 -13
  6. data/bin/imap-backup +4 -1
  7. data/docs/01-credentials-screen.png +0 -0
  8. data/docs/02-new-project.png +0 -0
  9. data/docs/03-initial-credentials-for-project.png +0 -0
  10. data/docs/04-credential-type-selection.png +0 -0
  11. data/docs/05-cant-create-without-consent-setup.png +0 -0
  12. data/docs/06-user-type-selection.png +0 -0
  13. data/docs/07-consent-screen-form.png +0 -0
  14. data/docs/08-app-scopes.png +0 -0
  15. data/docs/09-scope-selection.png +0 -0
  16. data/docs/10-updated-app-scopes.png +0 -0
  17. data/docs/11-test-users.png +0 -0
  18. data/docs/12-add-users.png +0 -0
  19. data/docs/13-create-oauth-client.png +0 -0
  20. data/docs/14-application-details.png +0 -0
  21. data/docs/16-initial-menu.png +0 -0
  22. data/docs/17-inputting-the-email-address.png +0 -0
  23. data/docs/18-choose-password.png +0 -0
  24. data/docs/19-supply-client-info.png +0 -0
  25. data/docs/20-choose-gmail-account.png +0 -0
  26. data/docs/21-accept-warnings.png +0 -0
  27. data/docs/22-grant-access.png +0 -0
  28. data/docs/24-confirm-choices.png +0 -0
  29. data/docs/25-success-code.png +0 -0
  30. data/docs/26-type-code-into-imap-backup.png +0 -0
  31. data/docs/27-success.png +0 -0
  32. data/docs/setting-up-gmail.md +166 -0
  33. data/imap-backup.gemspec +3 -8
  34. data/lib/email/mboxrd/message.rb +2 -0
  35. data/lib/email/provider.rb +3 -1
  36. data/lib/gmail/authenticator.rb +160 -0
  37. data/lib/google/auth/stores/in_memory_token_store.rb +9 -0
  38. data/lib/imap/backup.rb +2 -1
  39. data/lib/imap/backup/account/connection.rb +67 -35
  40. data/lib/imap/backup/account/folder.rb +18 -6
  41. data/lib/imap/backup/configuration/account.rb +9 -1
  42. data/lib/imap/backup/configuration/folder_chooser.rb +14 -15
  43. data/lib/imap/backup/configuration/gmail_oauth2.rb +82 -0
  44. data/lib/imap/backup/configuration/setup.rb +4 -1
  45. data/lib/imap/backup/configuration/store.rb +12 -12
  46. data/lib/imap/backup/serializer/mbox_store.rb +2 -2
  47. data/lib/imap/backup/version.rb +4 -3
  48. data/lib/retry_on_error.rb +14 -0
  49. data/spec/features/backup_spec.rb +1 -1
  50. data/spec/fixtures/connection.yml +1 -1
  51. data/spec/support/fixtures.rb +7 -2
  52. data/spec/unit/gmail/authenticator_spec.rb +138 -0
  53. data/spec/unit/google/auth/stores/in_memory_token_store_spec.rb +15 -0
  54. data/spec/unit/imap/backup/account/connection_spec.rb +146 -53
  55. data/spec/unit/imap/backup/account/folder_spec.rb +42 -10
  56. data/spec/unit/imap/backup/configuration/account_spec.rb +37 -24
  57. data/spec/unit/imap/backup/configuration/folder_chooser_spec.rb +7 -13
  58. data/spec/unit/imap/backup/configuration/gmail_oauth2_spec.rb +84 -0
  59. data/spec/unit/imap/backup/configuration/setup_spec.rb +51 -31
  60. metadata +83 -9
  61. data/.travis.yml +0 -25
@@ -10,9 +10,11 @@ module Email::Mboxrd
10
10
  cleaned = serialized.gsub(/^>(>*From)/, "\\1")
11
11
  # Serialized messages in this format *should* start with a line
12
12
  # From xxx yy zz
13
+ # rubocop:disable Style/IfUnlessModifier
13
14
  if cleaned.start_with?("From ")
14
15
  cleaned = cleaned.sub(/^From .*[\r\n]*/, "")
15
16
  end
17
+ # rubocop:enable Style/IfUnlessModifier
16
18
  new(cleaned)
17
19
  end
18
20
 
@@ -1,6 +1,8 @@
1
1
  module Email; end
2
2
 
3
3
  class Email::Provider
4
+ GMAIL_IMAP_SERVER = "imap.gmail.com".freeze
5
+
4
6
  def self.for_address(address)
5
7
  case
6
8
  when address.end_with?("@fastmail.com")
@@ -27,7 +29,7 @@ class Email::Provider
27
29
  def host
28
30
  case provider
29
31
  when :gmail
30
- "imap.gmail.com"
32
+ GMAIL_IMAP_SERVER
31
33
  when :fastmail
32
34
  "imap.fastmail.com"
33
35
  end
@@ -0,0 +1,160 @@
1
+ require "googleauth"
2
+ require "google/auth/stores/in_memory_token_store"
3
+
4
+ module Gmail; end
5
+
6
+ class Gmail::Authenticator
7
+ class MalformedImapBackupToken < StandardError; end
8
+
9
+ class ImapBackupToken
10
+ attr_reader :token
11
+
12
+ def self.from(
13
+ access_token:,
14
+ client_id:,
15
+ client_secret:,
16
+ expiration_time_millis:,
17
+ refresh_token:
18
+ )
19
+ {
20
+ access_token: access_token,
21
+ client_id: client_id,
22
+ client_secret: client_secret,
23
+ expiration_time_millis: expiration_time_millis,
24
+ refresh_token: refresh_token
25
+ }.to_json
26
+ end
27
+
28
+ def initialize(token)
29
+ @token = token
30
+ end
31
+
32
+ def valid?
33
+ return false if !body
34
+ return false if !access_token
35
+ return false if !client_id
36
+ return false if !client_secret
37
+ return false if !expiration_time_millis
38
+ return false if !refresh_token
39
+
40
+ true
41
+ end
42
+
43
+ def access_token
44
+ body["access_token"]
45
+ end
46
+
47
+ def client_id
48
+ body["client_id"]
49
+ end
50
+
51
+ def client_secret
52
+ body["client_secret"]
53
+ end
54
+
55
+ def expiration_time_millis
56
+ body["expiration_time_millis"]
57
+ end
58
+
59
+ def refresh_token
60
+ body["refresh_token"]
61
+ end
62
+
63
+ private
64
+
65
+ def body
66
+ @body ||= JSON.parse(token)
67
+ rescue JSON::ParserError
68
+ nil
69
+ end
70
+ end
71
+
72
+ GMAIL_READ_SCOPE = "https://mail.google.com/".freeze
73
+ OOB_URI = "urn:ietf:wg:oauth:2.0:oob".freeze
74
+
75
+ attr_reader :email
76
+ attr_reader :token
77
+
78
+ def self.refresh_token?(text)
79
+ ImapBackupToken.new(text).valid?
80
+ end
81
+
82
+ def initialize(email:, token:)
83
+ @email = email
84
+ @token = token
85
+ end
86
+
87
+ def authorization_url
88
+ authorizer.get_authorization_url(base_url: OOB_URI)
89
+ end
90
+
91
+ def credentials
92
+ authorizer.get_credentials(email).tap do |c|
93
+ c.refresh! if c.expired?
94
+ end
95
+ end
96
+
97
+ def credentials_from_code(code)
98
+ authorizer.get_credentials_from_code(
99
+ user_id: email,
100
+ code: code,
101
+ base_url: OOB_URI
102
+ )
103
+ end
104
+
105
+ private
106
+
107
+ def auth_client_id
108
+ @auth_client_id = Google::Auth::ClientId.new(client_id, client_secret)
109
+ end
110
+
111
+ def authorizer
112
+ @authorizer ||= Google::Auth::UserAuthorizer.new(
113
+ auth_client_id, GMAIL_READ_SCOPE, token_store
114
+ )
115
+ end
116
+
117
+ def access_token
118
+ imap_backup_token.access_token
119
+ end
120
+
121
+ def client_id
122
+ imap_backup_token.client_id
123
+ end
124
+
125
+ def client_secret
126
+ imap_backup_token.client_secret
127
+ end
128
+
129
+ def expiration_time_millis
130
+ imap_backup_token.expiration_time_millis
131
+ end
132
+
133
+ def refresh_token
134
+ imap_backup_token.refresh_token
135
+ end
136
+
137
+ def imap_backup_token
138
+ @imap_backup_token ||=
139
+ ImapBackupToken.new(token).tap do |t|
140
+ raise MalformedImapBackupToken if !t.valid?
141
+ end
142
+ end
143
+
144
+ def store_token
145
+ {
146
+ "client_id" => client_id,
147
+ "access_token" => access_token,
148
+ "refresh_token" => refresh_token,
149
+ "scope": [GMAIL_READ_SCOPE],
150
+ "expiration_time_millis": expiration_time_millis
151
+ }.to_json
152
+ end
153
+
154
+ def token_store
155
+ @token_store ||=
156
+ Google::Auth::Stores::InMemoryTokenStore.new.tap do |t|
157
+ t.store(email, store_token)
158
+ end
159
+ end
160
+ end
@@ -0,0 +1,9 @@
1
+ module Google; end
2
+ module Google::Auth; end
3
+ module Google::Auth::Stores; end
4
+
5
+ class Google::Auth::Stores::InMemoryTokenStore < Hash
6
+ def load(id)
7
+ self[id]
8
+ end
9
+ end
data/lib/imap/backup.rb CHANGED
@@ -7,6 +7,7 @@ require "imap/backup/configuration/account"
7
7
  require "imap/backup/configuration/asker"
8
8
  require "imap/backup/configuration/connection_tester"
9
9
  require "imap/backup/configuration/folder_chooser"
10
+ require "imap/backup/configuration/gmail_oauth2"
10
11
  require "imap/backup/configuration/list"
11
12
  require "imap/backup/configuration/setup"
12
13
  require "imap/backup/configuration/store"
@@ -28,7 +29,7 @@ module Imap::Backup
28
29
  attr_reader :logger
29
30
 
30
31
  def initialize
31
- @logger = ::Logger.new(STDOUT)
32
+ @logger = ::Logger.new($stdout)
32
33
  end
33
34
  end
34
35
 
@@ -1,9 +1,19 @@
1
1
  require "net/imap"
2
+ require "gmail_xoauth"
3
+
4
+ require "gmail/authenticator"
5
+ require "retry_on_error"
2
6
 
3
7
  module Imap::Backup
4
8
  module Account; end
5
9
 
6
10
  class Account::Connection
11
+ class InvalidGmailOauth2RefreshToken < StandardError; end
12
+
13
+ include RetryOnError
14
+
15
+ LOGIN_RETRY_CLASSES = [EOFError, Errno::ECONNRESET, SocketError].freeze
16
+
7
17
  attr_reader :connection_options
8
18
  attr_reader :local_path
9
19
  attr_reader :password
@@ -13,7 +23,7 @@ module Imap::Backup
13
23
  @username = options[:username]
14
24
  @password = options[:password]
15
25
  @local_path = options[:local_path]
16
- @backup_folders = options[:folders]
26
+ @config_folders = options[:folders]
17
27
  @server = options[:server]
18
28
  @connection_options = options[:connection_options] || {}
19
29
  @folders = nil
@@ -24,21 +34,24 @@ module Imap::Backup
24
34
  @folders ||=
25
35
  begin
26
36
  root = provider_root
27
- @folders = imap.list(root, "*")
28
- if @folders.nil?
29
- Imap::Backup.logger.warn(
30
- "Unable to get folder list for account #{username}"
31
- )
37
+ mailbox_lists = imap.list(root, "*")
38
+
39
+ if mailbox_lists.nil?
40
+ message = "Unable to get folder list for account #{username}"
41
+ Imap::Backup.logger.info message
42
+ raise message
32
43
  end
33
- @folders
44
+
45
+ utf7_encoded = mailbox_lists.map(&:name)
46
+ utf7_encoded.map { |n| Net::IMAP.decode_utf7(n) }
34
47
  end
35
48
  end
36
49
 
37
50
  def status
38
- backup_folders.map do |folder|
39
- f = Account::Folder.new(self, folder[:name])
40
- s = Serializer::Mbox.new(local_path, folder[:name])
41
- {name: folder[:name], local: s.uids, remote: f.uids}
51
+ backup_folders.map do |backup_folder|
52
+ f = Account::Folder.new(self, backup_folder[:name])
53
+ s = Serializer::Mbox.new(local_path, backup_folder[:name])
54
+ {name: backup_folder[:name], local: s.uids, remote: f.uids}
42
55
  end
43
56
  end
44
57
 
@@ -71,25 +84,42 @@ module Imap::Backup
71
84
  end
72
85
 
73
86
  def imap
74
- return @imap unless @imap.nil?
87
+ @imap ||=
88
+ retry_on_error(errors: LOGIN_RETRY_CLASSES) do
89
+ options = provider_options
90
+ Imap::Backup.logger.debug(
91
+ "Creating IMAP instance: #{server}, options: #{options.inspect}"
92
+ )
93
+ imap = Net::IMAP.new(server, options)
94
+ if gmail? && Gmail::Authenticator.refresh_token?(password)
95
+ authenticator = Gmail::Authenticator.new(email: username, token: password)
96
+ credentials = authenticator.credentials
97
+ raise InvalidGmailOauth2RefreshToken if !credentials
98
+
99
+ Imap::Backup.logger.debug "Logging in with OAuth2 token: #{username}"
100
+ imap.authenticate("XOAUTH2", username, credentials.access_token)
101
+ else
102
+ Imap::Backup.logger.debug "Logging in: #{username}/#{masked_password}"
103
+ imap.login(username, password)
104
+ end
105
+ Imap::Backup.logger.debug "Login complete"
106
+ imap
107
+ end
108
+ end
75
109
 
76
- options = provider_options
77
- Imap::Backup.logger.debug(
78
- "Creating IMAP instance: #{server}, options: #{options.inspect}"
79
- )
80
- @imap = Net::IMAP.new(server, options)
81
- Imap::Backup.logger.debug "Logging in: #{username}/#{masked_password}"
82
- @imap.login(username, password)
83
- Imap::Backup.logger.debug "Login complete"
84
- @imap
110
+ def server
111
+ return @server if @server
112
+ return nil if provider.nil?
113
+
114
+ @server = provider.host
85
115
  end
86
116
 
87
117
  private
88
118
 
89
119
  def each_folder
90
- backup_folders.each do |folder_info|
91
- folder = Account::Folder.new(self, folder_info[:name])
92
- serializer = Serializer::Mbox.new(local_path, folder_info[:name])
120
+ backup_folders.each do |backup_folder|
121
+ folder = Account::Folder.new(self, backup_folder[:name])
122
+ serializer = Serializer::Mbox.new(local_path, backup_folder[:name])
93
123
  yield folder, serializer
94
124
  end
95
125
  end
@@ -133,10 +163,8 @@ module Imap::Backup
133
163
  password.gsub(/./, "x")
134
164
  end
135
165
 
136
- def backup_folders
137
- return @backup_folders if @backup_folders && !@backup_folders.empty?
138
-
139
- (folders || []).map { |f| {name: f.name} }
166
+ def gmail?
167
+ server == Email::Provider::GMAIL_IMAP_SERVER
140
168
  end
141
169
 
142
170
  def local_folders
@@ -150,15 +178,19 @@ module Imap::Backup
150
178
  end
151
179
  end
152
180
 
153
- def provider
154
- @provider ||= Email::Provider.for_address(username)
181
+ def backup_folders
182
+ @backup_folders ||=
183
+ begin
184
+ if @config_folders&.any?
185
+ @config_folders
186
+ else
187
+ folders.map { |name| {name: name} }
188
+ end
189
+ end
155
190
  end
156
191
 
157
- def server
158
- return @server if @server
159
- return nil if provider.nil?
160
-
161
- @server = provider.host
192
+ def provider
193
+ @provider ||= Email::Provider.for_address(username)
162
194
  end
163
195
 
164
196
  def provider_options
@@ -1,5 +1,7 @@
1
1
  require "forwardable"
2
2
 
3
+ require "retry_on_error"
4
+
3
5
  module Imap::Backup
4
6
  module Account; end
5
7
 
@@ -7,8 +9,10 @@ module Imap::Backup
7
9
 
8
10
  class Account::Folder
9
11
  extend Forwardable
12
+ include RetryOnError
10
13
 
11
14
  REQUESTED_ATTRIBUTES = %w[RFC822 FLAGS INTERNALDATE].freeze
15
+ UID_FETCH_RETRY_CLASSES = [EOFError].freeze
12
16
 
13
17
  attr_reader :connection
14
18
  attr_reader :name
@@ -36,7 +40,7 @@ module Imap::Backup
36
40
  def create
37
41
  return if exist?
38
42
 
39
- imap.create(name)
43
+ imap.create(utf7_encoded_name)
40
44
  end
41
45
 
42
46
  def uid_validity
@@ -66,7 +70,10 @@ module Imap::Backup
66
70
 
67
71
  def fetch(uid)
68
72
  examine
69
- fetch_data_items = imap.uid_fetch([uid.to_i], REQUESTED_ATTRIBUTES)
73
+ fetch_data_items =
74
+ retry_on_error(errors: UID_FETCH_RETRY_CLASSES) do
75
+ imap.uid_fetch([uid.to_i], REQUESTED_ATTRIBUTES)
76
+ end
70
77
  return nil if fetch_data_items.nil?
71
78
 
72
79
  fetch_data_item = fetch_data_items[0]
@@ -81,22 +88,27 @@ module Imap::Backup
81
88
  def append(message)
82
89
  body = message.imap_body
83
90
  date = message.date&.to_time
84
- response = imap.append(name, body, nil, date)
91
+ response = imap.append(utf7_encoded_name, body, nil, date)
85
92
  extract_uid(response)
86
93
  end
87
94
 
88
95
  private
89
96
 
90
97
  def examine
91
- imap.examine(name)
98
+ imap.examine(utf7_encoded_name)
92
99
  rescue Net::IMAP::NoResponseError
93
- Imap::Backup.logger.warn "Folder '#{name}' does not exist"
94
- raise FolderNotFound, "Folder '#{name}' does not exist"
100
+ Imap::Backup.logger.warn "Folder '#{name}' does not exist on server"
101
+ raise FolderNotFound, "Folder '#{name}' does not exist on server"
95
102
  end
96
103
 
97
104
  def extract_uid(response)
98
105
  @uid_validity, uid = response.data.code.data.split(" ").map(&:to_i)
99
106
  uid
100
107
  end
108
+
109
+ def utf7_encoded_name
110
+ @utf7_encoded_name ||=
111
+ Net::IMAP.encode_utf7(name).force_encoding("ASCII-8BIT")
112
+ end
101
113
  end
102
114
  end