imap-backup 16.2.0 → 16.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/docs/performance.md +1 -1
- data/imap-backup.gemspec +12 -11
- data/lib/imap/backup/account/backup.rb +30 -10
- data/lib/imap/backup/account/folder.rb +10 -1
- data/lib/imap/backup/account/locker.rb +35 -0
- data/lib/imap/backup/account/restore.rb +10 -2
- data/lib/imap/backup/account.rb +17 -3
- data/lib/imap/backup/cli/backup.rb +2 -0
- data/lib/imap/backup/cli/helpers.rb +69 -2
- data/lib/imap/backup/cli/local.rb +6 -0
- data/lib/imap/backup/cli/migrate.rb +1 -0
- data/lib/imap/backup/cli/mirror.rb +1 -0
- data/lib/imap/backup/cli/options.rb +13 -0
- data/lib/imap/backup/cli/remote.rb +8 -2
- data/lib/imap/backup/cli/restore.rb +2 -2
- data/lib/imap/backup/cli/single.rb +1 -1
- data/lib/imap/backup/cli/stats.rb +1 -0
- data/lib/imap/backup/cli/transfer.rb +29 -8
- data/lib/imap/backup/cli/utils.rb +19 -9
- data/lib/imap/backup/cli.rb +4 -0
- data/lib/imap/backup/client/default.rb +1 -0
- data/lib/imap/backup/configuration.rb +4 -3
- data/lib/imap/backup/lockfile.rb +78 -0
- data/lib/imap/backup/mirror.rb +3 -2
- data/lib/imap/backup/serializer/mbox.rb +1 -1
- data/lib/imap/backup/serializer/permission_checker.rb +1 -1
- data/lib/imap/backup/serializer/version2_migrator.rb +1 -1
- data/lib/imap/backup/setup/asker.rb +1 -0
- data/lib/imap/backup/setup.rb +1 -0
- data/lib/imap/backup/thunderbird/mailbox_exporter.rb +1 -0
- data/lib/imap/backup/version.rb +1 -1
- metadata +20 -7
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 9ec15a683d70eadabab4271afcd2231c723344e79dde55fadeea091056201fc8
|
|
4
|
+
data.tar.gz: 946f1074220df2af38c92a5ff7f5dda8bcdcef3dc20f48d9f2c8206e29a8cd6e
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 7c33112eb20f168c453f0cc52e86497c7a44459afa718ac12ff1ca71cf990d9566514598aae25fc22117fb30478db649bb76a1aff697d40dabb788cc54fa1508
|
|
7
|
+
data.tar.gz: ea2dd0c673c603c71708ba550cbdf98c218c51b8ff076ba031d40a7ae33f80ff830ada659b7cc300e8511cc3da207665e6f2114c231e985a0b716e2844f7fe7a
|
data/docs/performance.md
CHANGED
|
@@ -43,7 +43,7 @@ By default, during backup, each message is downloaded one-by-one.
|
|
|
43
43
|
Using this setting, you can download chunks of emails at a time,
|
|
44
44
|
potentially speeding up the process.
|
|
45
45
|
|
|
46
|
-
Using multi-fetch
|
|
46
|
+
Using multi-fetch means that the backup process *will* use
|
|
47
47
|
more memory - equivalent to the size of the groups of messages
|
|
48
48
|
that are downloaded.
|
|
49
49
|
|
data/imap-backup.gemspec
CHANGED
|
@@ -18,18 +18,19 @@ Gem::Specification.new do |gem|
|
|
|
18
18
|
|
|
19
19
|
gem.executables = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) }
|
|
20
20
|
gem.require_paths = ["lib"]
|
|
21
|
-
gem.required_ruby_version = ">= 3.
|
|
21
|
+
gem.required_ruby_version = ">= 3.2"
|
|
22
22
|
|
|
23
|
-
gem.
|
|
24
|
-
gem.
|
|
25
|
-
gem.
|
|
26
|
-
gem.
|
|
27
|
-
gem.
|
|
28
|
-
gem.
|
|
29
|
-
gem.
|
|
30
|
-
gem.
|
|
31
|
-
gem.
|
|
32
|
-
gem.
|
|
23
|
+
gem.add_dependency "highline"
|
|
24
|
+
gem.add_dependency "logger"
|
|
25
|
+
gem.add_dependency "mail", "2.7.1"
|
|
26
|
+
gem.add_dependency "net-imap", ">= 0.3.2"
|
|
27
|
+
gem.add_dependency "net-smtp"
|
|
28
|
+
gem.add_dependency "os"
|
|
29
|
+
gem.add_dependency "ostruct"
|
|
30
|
+
gem.add_dependency "rake"
|
|
31
|
+
gem.add_dependency "sys-proctable"
|
|
32
|
+
gem.add_dependency "thor", "~> 1.1"
|
|
33
|
+
gem.add_dependency "thunderbird", "0.3.0"
|
|
33
34
|
|
|
34
35
|
gem.metadata = {
|
|
35
36
|
"rubygems_mfa_required" => "true"
|
|
@@ -2,6 +2,7 @@ require "imap/backup/account/backup_folders"
|
|
|
2
2
|
require "imap/backup/account/folder_backup"
|
|
3
3
|
require "imap/backup/account/folder_ensurer"
|
|
4
4
|
require "imap/backup/account/local_only_folder_deleter"
|
|
5
|
+
require "imap/backup/account/locker"
|
|
5
6
|
|
|
6
7
|
module Imap; end
|
|
7
8
|
|
|
@@ -22,19 +23,17 @@ module Imap::Backup
|
|
|
22
23
|
# start the connection so we get logging messages in the right order
|
|
23
24
|
account.client.login
|
|
24
25
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
).to_a
|
|
26
|
+
ensure_folder
|
|
27
|
+
delete_local_only_folders if account.mirror_mode
|
|
28
|
+
|
|
29
29
|
if backup_folders.none?
|
|
30
30
|
Logger.logger.warn "No folders found to backup for account '#{account.username}'"
|
|
31
31
|
return
|
|
32
32
|
end
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
33
|
+
|
|
34
|
+
locker.with_lock do
|
|
35
|
+
perform_backup
|
|
36
36
|
end
|
|
37
|
-
Logger.logger.debug "Backup of account '#{account.username}' complete"
|
|
38
37
|
end
|
|
39
38
|
|
|
40
39
|
private
|
|
@@ -42,9 +41,30 @@ module Imap::Backup
|
|
|
42
41
|
attr_reader :account
|
|
43
42
|
attr_reader :refresh
|
|
44
43
|
|
|
45
|
-
def
|
|
44
|
+
def backup_folders
|
|
45
|
+
@backup_folders ||= Account::BackupFolders.new(
|
|
46
|
+
client: account.client, account: account
|
|
47
|
+
).to_a
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def delete_local_only_folders
|
|
51
|
+
Account::LocalOnlyFolderDeleter.new(account: account).run
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def ensure_folder
|
|
46
55
|
Account::FolderEnsurer.new(account: account).run
|
|
47
|
-
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def locker
|
|
59
|
+
@locker ||= Account::Locker.new(account: account)
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def perform_backup
|
|
63
|
+
Logger.logger.debug "Starting backup of #{backup_folders.count} folders"
|
|
64
|
+
backup_folders.each do |folder|
|
|
65
|
+
Account::FolderBackup.new(account: account, folder: folder, refresh: refresh).run
|
|
66
|
+
end
|
|
67
|
+
Logger.logger.debug "Backup of account '#{account.username}' complete"
|
|
48
68
|
end
|
|
49
69
|
end
|
|
50
70
|
end
|
|
@@ -184,6 +184,13 @@ module Imap::Backup
|
|
|
184
184
|
CREATE_RETRY_CLASSES = [::Net::IMAP::BadResponseError].freeze
|
|
185
185
|
EXAMINE_RETRY_CLASSES = [::Net::IMAP::BadResponseError].freeze
|
|
186
186
|
PERMITTED_FLAGS = %i(Answered Draft Flagged Seen).freeze
|
|
187
|
+
private_constant :BODY_ATTRIBUTE,
|
|
188
|
+
:UID_FETCH_RETRY_CLASSES,
|
|
189
|
+
:UID_SEARCH_RETRY_CLASSES,
|
|
190
|
+
:APPEND_RETRY_CLASSES,
|
|
191
|
+
:CREATE_RETRY_CLASSES,
|
|
192
|
+
:EXAMINE_RETRY_CLASSES,
|
|
193
|
+
:PERMITTED_FLAGS
|
|
187
194
|
|
|
188
195
|
def examine
|
|
189
196
|
client.examine(utf7_encoded_name)
|
|
@@ -196,7 +203,9 @@ module Imap::Backup
|
|
|
196
203
|
def extract_uid(response)
|
|
197
204
|
uid_data = response.data.code.data
|
|
198
205
|
@uid_validity = uid_data.uidvalidity
|
|
199
|
-
|
|
206
|
+
# With net-imap >= 0.6, assigned_uids is a `Net::IMAP::SequenceSet`
|
|
207
|
+
uids = uid_data.assigned_uids.to_a
|
|
208
|
+
uids.to_a.first
|
|
200
209
|
end
|
|
201
210
|
|
|
202
211
|
def utf7_encoded_name
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
require "imap/backup/account/folder_ensurer"
|
|
2
|
+
require "imap/backup/lockfile"
|
|
3
|
+
|
|
4
|
+
module Imap; end
|
|
5
|
+
|
|
6
|
+
module Imap::Backup
|
|
7
|
+
class Account; end
|
|
8
|
+
|
|
9
|
+
class Account::Locker
|
|
10
|
+
attr_reader :account
|
|
11
|
+
|
|
12
|
+
def initialize(account:)
|
|
13
|
+
@account = account
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def with_lock(&block)
|
|
17
|
+
lockfile = Lockfile.new(path: account.lockfile_path)
|
|
18
|
+
if lockfile.exists?
|
|
19
|
+
if !lockfile.stale?
|
|
20
|
+
raise Lockfile::LockfileExistsError,
|
|
21
|
+
"Lockfile '#{account.lockfile_path}' exists and is not stale."
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
Logger.logger.info("Stale lockfile '#{account.lockfile_path}' found. Removing it.")
|
|
25
|
+
lockfile.remove
|
|
26
|
+
else
|
|
27
|
+
Account::FolderEnsurer.new(account: account).run
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
lockfile.with_lock do
|
|
31
|
+
block.call
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
require "imap/backup/account/folder_mapper"
|
|
2
|
+
require "imap/backup/account/locker"
|
|
2
3
|
require "imap/backup/uploader"
|
|
3
4
|
|
|
4
5
|
module Imap; end
|
|
@@ -12,13 +13,16 @@ module Imap::Backup
|
|
|
12
13
|
@account = account
|
|
13
14
|
@destination_delimiter = delimiter
|
|
14
15
|
@destination_prefix = prefix
|
|
16
|
+
@locker = nil
|
|
15
17
|
end
|
|
16
18
|
|
|
17
19
|
# Runs the restore operation
|
|
18
20
|
# @return [void]
|
|
19
21
|
def run
|
|
20
|
-
|
|
21
|
-
|
|
22
|
+
locker.with_lock do
|
|
23
|
+
folders.each do |serializer, folder|
|
|
24
|
+
Uploader.new(folder, serializer).run
|
|
25
|
+
end
|
|
22
26
|
end
|
|
23
27
|
end
|
|
24
28
|
|
|
@@ -40,5 +44,9 @@ module Imap::Backup
|
|
|
40
44
|
def folders
|
|
41
45
|
Account::FolderMapper.new(**enumerator_options)
|
|
42
46
|
end
|
|
47
|
+
|
|
48
|
+
def locker
|
|
49
|
+
@locker ||= Account::Locker.new(account: account)
|
|
50
|
+
end
|
|
43
51
|
end
|
|
44
52
|
end
|
data/lib/imap/backup/account.rb
CHANGED
|
@@ -4,6 +4,7 @@ require "imap/backup/account/client_factory"
|
|
|
4
4
|
|
|
5
5
|
module Imap; end
|
|
6
6
|
|
|
7
|
+
# rubocop:disable Metrics/ClassLength
|
|
7
8
|
module Imap::Backup
|
|
8
9
|
# Contains the attributes relating to an email account.
|
|
9
10
|
class Account
|
|
@@ -57,7 +58,7 @@ module Imap::Backup
|
|
|
57
58
|
# mark messages as '\Seen' when accessed).
|
|
58
59
|
# @return [Boolean]
|
|
59
60
|
attr_reader :reset_seen_flags_after_fetch
|
|
60
|
-
# The status of the account - controls backup and migration
|
|
61
|
+
# The status of the account - controls backup and migration behaviour
|
|
61
62
|
# "active" - the account is available for backup and migration,
|
|
62
63
|
# "archived" - the account is available for migration, but not backup,
|
|
63
64
|
# "offline" - the account is not available for backup or migration.
|
|
@@ -168,6 +169,14 @@ module Imap::Backup
|
|
|
168
169
|
update(:local_path, value)
|
|
169
170
|
end
|
|
170
171
|
|
|
172
|
+
# @raise [RuntimeError] if the local_path is not set
|
|
173
|
+
# @return [String] the path to the lockfile for the account
|
|
174
|
+
def lockfile_path
|
|
175
|
+
raise "local_path is not set" if !local_path
|
|
176
|
+
|
|
177
|
+
File.join(local_path, "imap-backup.lock")
|
|
178
|
+
end
|
|
179
|
+
|
|
171
180
|
# @raise [RuntimeError] if the supplied value is not an Array
|
|
172
181
|
# @return [void]
|
|
173
182
|
def folders=(value)
|
|
@@ -283,14 +292,19 @@ module Imap::Backup
|
|
|
283
292
|
|
|
284
293
|
attr_reader :changes
|
|
285
294
|
|
|
286
|
-
REQUIRED_ATTRIBUTES = %i[password username].freeze
|
|
295
|
+
REQUIRED_ATTRIBUTES = %i[password username local_path].freeze
|
|
287
296
|
OPTIONAL_ATTRIBUTES = %i[
|
|
288
|
-
connection_options download_strategy folders folder_blacklist
|
|
297
|
+
connection_options download_strategy folders folder_blacklist mirror_mode
|
|
289
298
|
multi_fetch_size reset_seen_flags_after_fetch server status
|
|
290
299
|
].freeze
|
|
291
300
|
KNOWN_ATTRIBUTES = REQUIRED_ATTRIBUTES + OPTIONAL_ATTRIBUTES
|
|
292
301
|
VALID_STATUSES = %w[active archived offline].freeze
|
|
293
302
|
DEFAULT_STATUS = "active".freeze
|
|
303
|
+
private_constant :REQUIRED_ATTRIBUTES,
|
|
304
|
+
:OPTIONAL_ATTRIBUTES,
|
|
305
|
+
:KNOWN_ATTRIBUTES,
|
|
306
|
+
:VALID_STATUSES,
|
|
307
|
+
:DEFAULT_STATUS
|
|
294
308
|
|
|
295
309
|
def check_options!(options)
|
|
296
310
|
missing_required = REQUIRED_ATTRIBUTES - options.keys
|
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
require "erb"
|
|
2
|
+
require "json"
|
|
3
|
+
require "tempfile"
|
|
1
4
|
require "thor"
|
|
2
5
|
|
|
3
6
|
require "imap/backup/cli/options"
|
|
@@ -19,7 +22,7 @@ module Imap::Backup
|
|
|
19
22
|
# @return [String] a description of the namespace configuration
|
|
20
23
|
NAMESPACE_CONFIGURATION_DESCRIPTION = <<~DESC.freeze
|
|
21
24
|
Some IMAP servers use namespaces (i.e. prefixes like "INBOX"),
|
|
22
|
-
while others
|
|
25
|
+
while others concatenate the names of subfolders
|
|
23
26
|
with a charater ("delimiter") other than "/".
|
|
24
27
|
|
|
25
28
|
In these cases there are two choices.
|
|
@@ -60,9 +63,24 @@ module Imap::Backup
|
|
|
60
63
|
|
|
61
64
|
# Loads the application configuration
|
|
62
65
|
# @raise [ConfigurationNotFound] if the configuration file does not exist
|
|
66
|
+
# @raise [RuntimeError] if both config and erb_configuration are provided
|
|
67
|
+
# @raise [RuntimeError] if ERB template has syntax errors
|
|
68
|
+
# @raise [RuntimeError] if ERB template renders invalid JSON
|
|
63
69
|
# @return [Configuration]
|
|
64
70
|
def load_config(**options)
|
|
65
|
-
|
|
71
|
+
config_path = options[:config]
|
|
72
|
+
erb_config_path = options[:erb_configuration]
|
|
73
|
+
|
|
74
|
+
# Check mutual exclusivity
|
|
75
|
+
if config_path && erb_config_path
|
|
76
|
+
raise "Cannot specify both --config and --erb-configuration options"
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# Handle ERB configuration
|
|
80
|
+
return load_erb_config(erb_config_path, options) if erb_config_path
|
|
81
|
+
|
|
82
|
+
# Handle regular JSON configuration
|
|
83
|
+
path = config_path
|
|
66
84
|
require_exists = options.key?(:require_exists) ? options[:require_exists] : true
|
|
67
85
|
if require_exists
|
|
68
86
|
exists = Configuration.exist?(path: path)
|
|
@@ -94,5 +112,54 @@ module Imap::Backup
|
|
|
94
112
|
config.accounts
|
|
95
113
|
end
|
|
96
114
|
end
|
|
115
|
+
|
|
116
|
+
private
|
|
117
|
+
|
|
118
|
+
# Processes an ERB template and loads it as configuration
|
|
119
|
+
# @raise [ConfigurationNotFound] if the ERB file does not exist
|
|
120
|
+
# @raise [RuntimeError] if ERB processing fails
|
|
121
|
+
# @raise [RuntimeError] if rendered output is invalid JSON
|
|
122
|
+
# @return [Configuration]
|
|
123
|
+
def load_erb_config(erb_path, _options)
|
|
124
|
+
# Check if file exists
|
|
125
|
+
unless File.exist?(erb_path)
|
|
126
|
+
raise ConfigurationNotFound, "ERB configuration file '#{erb_path}' not found"
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
begin
|
|
130
|
+
# Read and process ERB template
|
|
131
|
+
erb_content = File.read(erb_path)
|
|
132
|
+
erb = ERB.new(erb_content)
|
|
133
|
+
rendered_json = erb.result
|
|
134
|
+
rescue SyntaxError => e
|
|
135
|
+
raise "ERB template has syntax error: #{e.message}"
|
|
136
|
+
rescue StandardError => e
|
|
137
|
+
raise "Error processing ERB template: #{e.message}"
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
# Validate rendered JSON
|
|
141
|
+
begin
|
|
142
|
+
JSON.parse(rendered_json)
|
|
143
|
+
rescue JSON::ParserError => e
|
|
144
|
+
raise "ERB template rendered invalid JSON: #{e.message}"
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
# Create temporary file with rendered JSON
|
|
148
|
+
temp_file = Tempfile.new(["config", ".json"])
|
|
149
|
+
begin
|
|
150
|
+
temp_file.write(rendered_json)
|
|
151
|
+
temp_file.flush
|
|
152
|
+
temp_file.close
|
|
153
|
+
|
|
154
|
+
# Load configuration from temporary file
|
|
155
|
+
config = Configuration.new(path: temp_file.path)
|
|
156
|
+
# Force loading of data before temp file is deleted
|
|
157
|
+
config.accounts
|
|
158
|
+
config
|
|
159
|
+
ensure
|
|
160
|
+
# Clean up temporary file
|
|
161
|
+
temp_file&.unlink
|
|
162
|
+
end
|
|
163
|
+
end
|
|
97
164
|
end
|
|
98
165
|
end
|
|
@@ -17,6 +17,7 @@ module Imap::Backup
|
|
|
17
17
|
|
|
18
18
|
desc "accounts [OPTIONS]", "List locally backed-up accounts"
|
|
19
19
|
config_option
|
|
20
|
+
erb_configuration_option
|
|
20
21
|
format_option
|
|
21
22
|
quiet_option
|
|
22
23
|
verbose_option
|
|
@@ -44,6 +45,7 @@ module Imap::Backup
|
|
|
44
45
|
)
|
|
45
46
|
accounts_option
|
|
46
47
|
config_option
|
|
48
|
+
erb_configuration_option
|
|
47
49
|
format_option
|
|
48
50
|
quiet_option
|
|
49
51
|
verbose_option
|
|
@@ -56,6 +58,7 @@ module Imap::Backup
|
|
|
56
58
|
|
|
57
59
|
desc "folders EMAIL [OPTIONS]", "List backed up folders"
|
|
58
60
|
config_option
|
|
61
|
+
erb_configuration_option
|
|
59
62
|
format_option
|
|
60
63
|
quiet_option
|
|
61
64
|
verbose_option
|
|
@@ -78,6 +81,7 @@ module Imap::Backup
|
|
|
78
81
|
|
|
79
82
|
desc "list EMAIL FOLDER [OPTIONS]", "List emails in a folder"
|
|
80
83
|
config_option
|
|
84
|
+
erb_configuration_option
|
|
81
85
|
format_option
|
|
82
86
|
quiet_option
|
|
83
87
|
verbose_option
|
|
@@ -108,6 +112,7 @@ module Imap::Backup
|
|
|
108
112
|
the UID.
|
|
109
113
|
DESC
|
|
110
114
|
config_option
|
|
115
|
+
erb_configuration_option
|
|
111
116
|
format_option
|
|
112
117
|
quiet_option
|
|
113
118
|
verbose_option
|
|
@@ -136,6 +141,7 @@ module Imap::Backup
|
|
|
136
141
|
private
|
|
137
142
|
|
|
138
143
|
MAX_SUBJECT = 60
|
|
144
|
+
private_constant :MAX_SUBJECT
|
|
139
145
|
|
|
140
146
|
def list_emails_as_json(serializer)
|
|
141
147
|
emails = serializer.each_message.map do |message|
|
|
@@ -25,6 +25,19 @@ module Imap::Backup
|
|
|
25
25
|
desc: "supply the configuration file path (default: ~/.imap-backup/config.json)"
|
|
26
26
|
}
|
|
27
27
|
},
|
|
28
|
+
{
|
|
29
|
+
name: "erb_configuration",
|
|
30
|
+
parameters: {
|
|
31
|
+
type: :string,
|
|
32
|
+
desc:
|
|
33
|
+
"supply an ERB template file path for configuration. " \
|
|
34
|
+
"This is an alternative to the --config option. " \
|
|
35
|
+
"The file will be processed with ERB before being parsed as JSON. " \
|
|
36
|
+
"This allows use of environment variables and other dynamic content. " \
|
|
37
|
+
"Note that using this option is a potential security risk as it allows " \
|
|
38
|
+
"execution of arbitrary code."
|
|
39
|
+
}
|
|
40
|
+
},
|
|
28
41
|
{
|
|
29
42
|
name: "format",
|
|
30
43
|
parameters: {
|
|
@@ -15,6 +15,7 @@ module Imap::Backup
|
|
|
15
15
|
|
|
16
16
|
desc "folders EMAIL [OPTIONS]", "List account folders"
|
|
17
17
|
config_option
|
|
18
|
+
erb_configuration_option
|
|
18
19
|
format_option
|
|
19
20
|
quiet_option
|
|
20
21
|
verbose_option
|
|
@@ -36,6 +37,7 @@ module Imap::Backup
|
|
|
36
37
|
Lists the IMAP capabilities supported by the IMAP server.
|
|
37
38
|
DESC
|
|
38
39
|
config_option
|
|
40
|
+
erb_configuration_option
|
|
39
41
|
format_option
|
|
40
42
|
quiet_option
|
|
41
43
|
verbose_option
|
|
@@ -57,6 +59,7 @@ module Imap::Backup
|
|
|
57
59
|
the `imap-backup migrate` and `imap-backup mirror` commands.
|
|
58
60
|
DESC
|
|
59
61
|
config_option
|
|
62
|
+
erb_configuration_option
|
|
60
63
|
format_option
|
|
61
64
|
quiet_option
|
|
62
65
|
verbose_option
|
|
@@ -106,9 +109,12 @@ module Imap::Backup
|
|
|
106
109
|
Kernel.puts list.to_json
|
|
107
110
|
end
|
|
108
111
|
|
|
112
|
+
NAMESPACE_TEMPLATE = "%-10<name>s %-10<prefix>s %<delim>s".freeze
|
|
113
|
+
private_constant :NAMESPACE_TEMPLATE
|
|
114
|
+
|
|
109
115
|
def list_namespaces(namespaces)
|
|
110
116
|
Kernel.puts format(
|
|
111
|
-
|
|
117
|
+
NAMESPACE_TEMPLATE,
|
|
112
118
|
{name: "Name", prefix: "Prefix", delim: "Delimiter"}
|
|
113
119
|
)
|
|
114
120
|
list_namespace namespaces, :personal
|
|
@@ -119,7 +125,7 @@ module Imap::Backup
|
|
|
119
125
|
def list_namespace(namespaces, name)
|
|
120
126
|
info = namespace_info(namespaces.send(name).first, quote: true)
|
|
121
127
|
if info
|
|
122
|
-
Kernel.puts format(
|
|
128
|
+
Kernel.puts format(NAMESPACE_TEMPLATE, name: name, **info)
|
|
123
129
|
else
|
|
124
130
|
Kernel.puts format("%-10<name>s (Not defined)", name: name)
|
|
125
131
|
end
|
|
@@ -50,8 +50,8 @@ module Imap::Backup
|
|
|
50
50
|
attr_reader :email
|
|
51
51
|
attr_reader :options
|
|
52
52
|
|
|
53
|
-
def restore(account, **
|
|
54
|
-
restore = Account::Restore.new(account: account, **
|
|
53
|
+
def restore(account, **)
|
|
54
|
+
restore = Account::Restore.new(account: account, **)
|
|
55
55
|
restore.run
|
|
56
56
|
end
|
|
57
57
|
|
|
@@ -105,7 +105,7 @@ module Imap::Backup
|
|
|
105
105
|
type: "string",
|
|
106
106
|
desc: "the path of the directory where backups are to be saved. " \
|
|
107
107
|
"If the directory does not exist, it will be created. " \
|
|
108
|
-
"If not set, this is set to a
|
|
108
|
+
"If not set, this is set to a directory under the current path " \
|
|
109
109
|
"which is derived from the username, by replacing '@' with '_'." \
|
|
110
110
|
"If this path parameter is not indicated, " \
|
|
111
111
|
"the default is the current directory plus the email " \
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
require "imap/backup/account/folder_mapper"
|
|
2
|
+
require "imap/backup/account/locker"
|
|
2
3
|
require "imap/backup/cli/backup"
|
|
3
4
|
require "imap/backup/cli/helpers"
|
|
4
5
|
require "imap/backup/logger"
|
|
@@ -23,9 +24,11 @@ module Imap::Backup
|
|
|
23
24
|
@automatic_namespaces = nil
|
|
24
25
|
@config_path = nil
|
|
25
26
|
@destination_delimiter = nil
|
|
27
|
+
@destination_locker = nil
|
|
26
28
|
@destination_prefix = nil
|
|
27
29
|
@reset = nil
|
|
28
30
|
@source_delimiter = nil
|
|
31
|
+
@source_locker = nil
|
|
29
32
|
@source_prefix = nil
|
|
30
33
|
end
|
|
31
34
|
|
|
@@ -34,22 +37,19 @@ module Imap::Backup
|
|
|
34
37
|
# or the source and destination accounts are the same,
|
|
35
38
|
# or either of the accounts is not configured,
|
|
36
39
|
# or incompatible namespace/delimiter parameters have been supplied
|
|
40
|
+
# or one or both of the accounts is locked by another process.
|
|
37
41
|
# @return [void]
|
|
38
42
|
def run
|
|
39
43
|
raise "Unknown action '#{action}'" if !ACTIONS.include?(action)
|
|
40
44
|
|
|
41
45
|
process_options!
|
|
42
46
|
warn_if_source_account_is_not_in_mirror_mode if action == :mirror
|
|
47
|
+
|
|
43
48
|
run_backup if %i(copy mirror).include?(action)
|
|
44
49
|
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
Mirror.new(serializer, folder, reset: false).run
|
|
49
|
-
when :migrate
|
|
50
|
-
Migrator.new(serializer, folder, reset: reset).run
|
|
51
|
-
when :mirror
|
|
52
|
-
Mirror.new(serializer, folder, reset: true).run
|
|
50
|
+
source_locker.with_lock do
|
|
51
|
+
destination_locker.with_lock do
|
|
52
|
+
perform_action_on_folders
|
|
53
53
|
end
|
|
54
54
|
end
|
|
55
55
|
end
|
|
@@ -68,6 +68,19 @@ module Imap::Backup
|
|
|
68
68
|
attr_reader :source_email
|
|
69
69
|
attr_accessor :source_prefix
|
|
70
70
|
|
|
71
|
+
def perform_action_on_folders
|
|
72
|
+
folders.each do |serializer, folder|
|
|
73
|
+
case action
|
|
74
|
+
when :copy
|
|
75
|
+
Mirror.new(serializer, folder, reset: false).run
|
|
76
|
+
when :migrate
|
|
77
|
+
Migrator.new(serializer, folder, reset: reset).run
|
|
78
|
+
when :mirror
|
|
79
|
+
Mirror.new(serializer, folder, reset: true).run
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
|
|
71
84
|
def process_options!
|
|
72
85
|
self.automatic_namespaces = options[:automatic_namespaces] || false
|
|
73
86
|
self.config_path = options[:config]
|
|
@@ -175,5 +188,13 @@ module Imap::Backup
|
|
|
175
188
|
def source_account
|
|
176
189
|
config.accounts.find { |a| a.username == source_email }
|
|
177
190
|
end
|
|
191
|
+
|
|
192
|
+
def source_locker
|
|
193
|
+
@source_locker ||= Account::Locker.new(account: source_account)
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
def destination_locker
|
|
197
|
+
@destination_locker ||= Account::Locker.new(account: destination_account)
|
|
198
|
+
end
|
|
178
199
|
end
|
|
179
200
|
end
|
|
@@ -23,6 +23,7 @@ module Imap::Backup
|
|
|
23
23
|
"Skip downloading emails up to today for all configured folders"
|
|
24
24
|
)
|
|
25
25
|
config_option
|
|
26
|
+
erb_configuration_option
|
|
26
27
|
quiet_option
|
|
27
28
|
verbose_option
|
|
28
29
|
# Creates fake downloaded emails so that only the account's future emails
|
|
@@ -32,15 +33,10 @@ module Imap::Backup
|
|
|
32
33
|
Logger.setup_logging options
|
|
33
34
|
config = load_config(**options)
|
|
34
35
|
account = account(config, email)
|
|
36
|
+
locker ||= Account::Locker.new(account: account)
|
|
35
37
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
)
|
|
39
|
-
backup_folders.each do |folder|
|
|
40
|
-
next if !folder.exist?
|
|
41
|
-
|
|
42
|
-
serializer = Serializer.new(account.local_path, folder.name)
|
|
43
|
-
do_ignore_folder_history(folder, serializer)
|
|
38
|
+
locker.with_lock do
|
|
39
|
+
ignore_account_history(account)
|
|
44
40
|
end
|
|
45
41
|
end
|
|
46
42
|
|
|
@@ -52,6 +48,7 @@ module Imap::Backup
|
|
|
52
48
|
DOC
|
|
53
49
|
)
|
|
54
50
|
config_option
|
|
51
|
+
erb_configuration_option
|
|
55
52
|
quiet_option
|
|
56
53
|
verbose_option
|
|
57
54
|
method_option(
|
|
@@ -97,8 +94,21 @@ module Imap::Backup
|
|
|
97
94
|
private
|
|
98
95
|
|
|
99
96
|
FAKE_EMAIL = "fake@email.com".freeze
|
|
97
|
+
private_constant :FAKE_EMAIL
|
|
98
|
+
|
|
99
|
+
def ignore_account_history(account)
|
|
100
|
+
backup_folders = Account::BackupFolders.new(
|
|
101
|
+
client: account.client, account: account
|
|
102
|
+
)
|
|
103
|
+
backup_folders.each do |folder|
|
|
104
|
+
next if !folder.exist?
|
|
105
|
+
|
|
106
|
+
serializer = Serializer.new(account.local_path, folder.name)
|
|
107
|
+
ignore_folder_history(folder, serializer)
|
|
108
|
+
end
|
|
109
|
+
end
|
|
100
110
|
|
|
101
|
-
def
|
|
111
|
+
def ignore_folder_history(folder, serializer)
|
|
102
112
|
uids = folder.uids - serializer.uids
|
|
103
113
|
Logger.logger.info "Folder '#{folder.name}' - #{uids.length} messages"
|
|
104
114
|
|
data/lib/imap/backup/cli.rb
CHANGED
|
@@ -64,6 +64,7 @@ module Imap::Backup
|
|
|
64
64
|
DESC
|
|
65
65
|
accounts_option
|
|
66
66
|
config_option
|
|
67
|
+
erb_configuration_option
|
|
67
68
|
quiet_option
|
|
68
69
|
refresh_option
|
|
69
70
|
verbose_option
|
|
@@ -101,6 +102,7 @@ module Imap::Backup
|
|
|
101
102
|
#{Helpers::NAMESPACE_CONFIGURATION_DESCRIPTION}
|
|
102
103
|
DESC
|
|
103
104
|
config_option
|
|
105
|
+
erb_configuration_option
|
|
104
106
|
quiet_option
|
|
105
107
|
verbose_option
|
|
106
108
|
method_option(
|
|
@@ -153,6 +155,7 @@ module Imap::Backup
|
|
|
153
155
|
DESC
|
|
154
156
|
accounts_option
|
|
155
157
|
config_option
|
|
158
|
+
erb_configuration_option
|
|
156
159
|
quiet_option
|
|
157
160
|
verbose_option
|
|
158
161
|
method_option(
|
|
@@ -203,6 +206,7 @@ module Imap::Backup
|
|
|
203
206
|
on the server).
|
|
204
207
|
DESC
|
|
205
208
|
config_option
|
|
209
|
+
erb_configuration_option
|
|
206
210
|
format_option
|
|
207
211
|
quiet_option
|
|
208
212
|
verbose_option
|
|
@@ -13,6 +13,7 @@ module Imap::Backup
|
|
|
13
13
|
# Tracks the latest folder selection in order to avoid repeated calls
|
|
14
14
|
class Client::Default
|
|
15
15
|
extend Forwardable
|
|
16
|
+
|
|
16
17
|
def_delegators :imap, *%i(
|
|
17
18
|
append authenticate capability create expunge namespace
|
|
18
19
|
responses uid_fetch uid_search uid_store
|
|
@@ -68,7 +68,10 @@ module Imap::Backup
|
|
|
68
68
|
ensure_loaded!
|
|
69
69
|
accounts = data[:accounts].map do |attr|
|
|
70
70
|
Account.new(attr)
|
|
71
|
-
|
|
71
|
+
rescue ArgumentError => e
|
|
72
|
+
Logger.logger.error("Skipping invalid account in config: #{e.message}")
|
|
73
|
+
nil
|
|
74
|
+
end.compact
|
|
72
75
|
inject_global_attributes(accounts)
|
|
73
76
|
end
|
|
74
77
|
end
|
|
@@ -108,8 +111,6 @@ module Imap::Backup
|
|
|
108
111
|
|
|
109
112
|
private
|
|
110
113
|
|
|
111
|
-
VERSION_2_1 = "2.1".freeze
|
|
112
|
-
|
|
113
114
|
attr_reader :pathname
|
|
114
115
|
|
|
115
116
|
def ensure_loaded!
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
require "sys/proctable"
|
|
2
|
+
require "json"
|
|
3
|
+
|
|
4
|
+
module Imap; end
|
|
5
|
+
|
|
6
|
+
module Imap::Backup
|
|
7
|
+
class Lockfile
|
|
8
|
+
# An error that is thrown if a lockfile already exists
|
|
9
|
+
class LockfileExistsError < StandardError; end
|
|
10
|
+
|
|
11
|
+
attr_reader :path
|
|
12
|
+
|
|
13
|
+
# Initializes a new Lockfile instance.
|
|
14
|
+
# @param path [String] the path to the lockfile
|
|
15
|
+
def initialize(path:)
|
|
16
|
+
@path = path
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# Creates the lockfile, yields to the given block
|
|
20
|
+
# and ensures the lockfile is removed afterwards.
|
|
21
|
+
def with_lock(&block)
|
|
22
|
+
raise LockfileExistsError, "Lockfile already exists at #{path}" if exists?
|
|
23
|
+
|
|
24
|
+
begin
|
|
25
|
+
create
|
|
26
|
+
block.call
|
|
27
|
+
ensure
|
|
28
|
+
remove
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# Checks if the lockfile exists.
|
|
33
|
+
# @return [Boolean] true if the lockfile exists, false otherwise
|
|
34
|
+
def exists?
|
|
35
|
+
File.exist?(path)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Removes the lockfile.
|
|
39
|
+
def remove
|
|
40
|
+
FileUtils.rm_f(path)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Checks if the lockfile is stale (i.e., the process that created it is no longer running).
|
|
44
|
+
# @return [Boolean] true if the lockfile is stale, false otherwise
|
|
45
|
+
def stale?
|
|
46
|
+
return false if !exists?
|
|
47
|
+
|
|
48
|
+
file_content = File.read(path)
|
|
49
|
+
data = JSON.parse(file_content, symbolize_names: true)
|
|
50
|
+
pid = data[:pid]
|
|
51
|
+
starttime = data[:starttime]
|
|
52
|
+
proc_table_entry = Sys::ProcTable.ps(pid: pid)
|
|
53
|
+
|
|
54
|
+
return true if proc_table_entry.nil?
|
|
55
|
+
|
|
56
|
+
proc_table_entry.starttime != starttime
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
private
|
|
60
|
+
|
|
61
|
+
def create
|
|
62
|
+
pid = Process.pid
|
|
63
|
+
proc_table_entry = Sys::ProcTable.ps(pid: pid)
|
|
64
|
+
|
|
65
|
+
raise "Unable to get process info for PID #{pid}" if proc_table_entry.nil?
|
|
66
|
+
|
|
67
|
+
starttime = proc_table_entry.starttime
|
|
68
|
+
|
|
69
|
+
data = {
|
|
70
|
+
pid: pid,
|
|
71
|
+
starttime: starttime
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
json_data = JSON.generate(data)
|
|
75
|
+
File.write(path, json_data)
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
end
|
data/lib/imap/backup/mirror.rb
CHANGED
|
@@ -11,10 +11,10 @@ module Imap::Backup
|
|
|
11
11
|
@reset = reset
|
|
12
12
|
end
|
|
13
13
|
|
|
14
|
-
# If necessary,
|
|
14
|
+
# If necessary, creates the destination folder,
|
|
15
15
|
# then deletes any messages in the destination folder
|
|
16
16
|
# that are not in the local store,
|
|
17
|
-
# sets existing messages'
|
|
17
|
+
# sets existing messages' flags
|
|
18
18
|
# then appends any missing messages
|
|
19
19
|
# and saves the mapping file
|
|
20
20
|
# @return [void]
|
|
@@ -29,6 +29,7 @@ module Imap::Backup
|
|
|
29
29
|
private
|
|
30
30
|
|
|
31
31
|
CHUNK_SIZE = 100
|
|
32
|
+
private_constant :CHUNK_SIZE
|
|
32
33
|
|
|
33
34
|
attr_reader :serializer
|
|
34
35
|
attr_reader :folder
|
data/lib/imap/backup/setup.rb
CHANGED
data/lib/imap/backup/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: imap-backup
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 16.
|
|
4
|
+
version: 16.4.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Joe Yates
|
|
8
|
-
autorequire:
|
|
9
8
|
bindir: bin
|
|
10
9
|
cert_chain: []
|
|
11
|
-
date:
|
|
10
|
+
date: 1980-01-02 00:00:00.000000000 Z
|
|
12
11
|
dependencies:
|
|
13
12
|
- !ruby/object:Gem::Dependency
|
|
14
13
|
name: highline
|
|
@@ -122,6 +121,20 @@ dependencies:
|
|
|
122
121
|
- - ">="
|
|
123
122
|
- !ruby/object:Gem::Version
|
|
124
123
|
version: '0'
|
|
124
|
+
- !ruby/object:Gem::Dependency
|
|
125
|
+
name: sys-proctable
|
|
126
|
+
requirement: !ruby/object:Gem::Requirement
|
|
127
|
+
requirements:
|
|
128
|
+
- - ">="
|
|
129
|
+
- !ruby/object:Gem::Version
|
|
130
|
+
version: '0'
|
|
131
|
+
type: :runtime
|
|
132
|
+
prerelease: false
|
|
133
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
134
|
+
requirements:
|
|
135
|
+
- - ">="
|
|
136
|
+
- !ruby/object:Gem::Version
|
|
137
|
+
version: '0'
|
|
125
138
|
- !ruby/object:Gem::Dependency
|
|
126
139
|
name: thor
|
|
127
140
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -174,6 +187,7 @@ files:
|
|
|
174
187
|
- lib/imap/backup/account/folder_ensurer.rb
|
|
175
188
|
- lib/imap/backup/account/folder_mapper.rb
|
|
176
189
|
- lib/imap/backup/account/local_only_folder_deleter.rb
|
|
190
|
+
- lib/imap/backup/account/locker.rb
|
|
177
191
|
- lib/imap/backup/account/restore.rb
|
|
178
192
|
- lib/imap/backup/account/serialized_folders.rb
|
|
179
193
|
- lib/imap/backup/cli.rb
|
|
@@ -208,6 +222,7 @@ files:
|
|
|
208
222
|
- lib/imap/backup/file_mode.rb
|
|
209
223
|
- lib/imap/backup/flag_refresher.rb
|
|
210
224
|
- lib/imap/backup/local_only_message_deleter.rb
|
|
225
|
+
- lib/imap/backup/lockfile.rb
|
|
211
226
|
- lib/imap/backup/logger.rb
|
|
212
227
|
- lib/imap/backup/migrator.rb
|
|
213
228
|
- lib/imap/backup/mirror.rb
|
|
@@ -248,7 +263,6 @@ licenses:
|
|
|
248
263
|
- MIT
|
|
249
264
|
metadata:
|
|
250
265
|
rubygems_mfa_required: 'true'
|
|
251
|
-
post_install_message:
|
|
252
266
|
rdoc_options: []
|
|
253
267
|
require_paths:
|
|
254
268
|
- lib
|
|
@@ -256,15 +270,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
256
270
|
requirements:
|
|
257
271
|
- - ">="
|
|
258
272
|
- !ruby/object:Gem::Version
|
|
259
|
-
version: '3.
|
|
273
|
+
version: '3.2'
|
|
260
274
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
261
275
|
requirements:
|
|
262
276
|
- - ">="
|
|
263
277
|
- !ruby/object:Gem::Version
|
|
264
278
|
version: '0'
|
|
265
279
|
requirements: []
|
|
266
|
-
rubygems_version:
|
|
267
|
-
signing_key:
|
|
280
|
+
rubygems_version: 4.0.3
|
|
268
281
|
specification_version: 4
|
|
269
282
|
summary: Backup GMail (or other IMAP) accounts to disk
|
|
270
283
|
test_files: []
|