imap-backup 1.4.2 → 2.0.0.rc1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rspec-all +2 -0
- data/README.md +36 -3
- data/bin/imap-backup +5 -0
- data/imap-backup.gemspec +7 -0
- data/lib/email/mboxrd/message.rb +13 -10
- data/lib/email/provider.rb +11 -2
- data/lib/imap/backup.rb +2 -1
- data/lib/imap/backup/account/connection.rb +41 -0
- data/lib/imap/backup/account/folder.rb +42 -2
- data/lib/imap/backup/serializer.rb +6 -0
- data/lib/imap/backup/serializer/mbox.rb +63 -83
- data/lib/imap/backup/serializer/mbox_store.rb +214 -0
- data/lib/imap/backup/uploader.rb +26 -0
- data/lib/imap/backup/utils.rb +3 -3
- data/lib/imap/backup/version.rb +5 -4
- data/spec/features/backup_spec.rb +85 -7
- data/spec/features/restore_spec.rb +112 -0
- data/spec/features/support/backup_directory.rb +21 -1
- data/spec/features/support/email_server.rb +66 -3
- data/spec/features/support/shared/message_fixtures.rb +2 -0
- data/spec/fixtures/connection.yml +2 -3
- data/spec/unit/account/connection_spec.rb +19 -5
- data/spec/unit/configuration/account_spec.rb +2 -1
- data/spec/unit/email/provider_spec.rb +1 -5
- data/spec/unit/serializer/mbox_spec.rb +52 -89
- data/spec/unit/serializer/mbox_store_spec.rb +117 -0
- data/spec/unit/utils_spec.rb +3 -3
- metadata +17 -8
- data/lib/imap/backup/serializer/base.rb +0 -16
- data/spec/unit/serializer/base_spec.rb +0 -19
@@ -0,0 +1,214 @@
|
|
1
|
+
require "json"
|
2
|
+
|
3
|
+
require "email/mboxrd/message"
|
4
|
+
|
5
|
+
module Imap::Backup
|
6
|
+
class Serializer::MboxStore
|
7
|
+
CURRENT_VERSION = 2
|
8
|
+
|
9
|
+
attr_reader :folder
|
10
|
+
attr_reader :path
|
11
|
+
attr_reader :loaded
|
12
|
+
|
13
|
+
def initialize(path, folder)
|
14
|
+
@path = path
|
15
|
+
@folder = folder
|
16
|
+
@loaded = false
|
17
|
+
@uids = nil
|
18
|
+
@uid_validity = nil
|
19
|
+
end
|
20
|
+
|
21
|
+
def exist?
|
22
|
+
mbox_exist? && imap_exist?
|
23
|
+
end
|
24
|
+
|
25
|
+
def uid_validity
|
26
|
+
do_load if !loaded
|
27
|
+
@uid_validity
|
28
|
+
end
|
29
|
+
|
30
|
+
def uid_validity=(value)
|
31
|
+
do_load if !loaded
|
32
|
+
@uid_validity = value
|
33
|
+
@uids ||= []
|
34
|
+
write_imap_file
|
35
|
+
end
|
36
|
+
|
37
|
+
def uids
|
38
|
+
do_load if !loaded
|
39
|
+
@uids || []
|
40
|
+
end
|
41
|
+
|
42
|
+
def add(uid, message)
|
43
|
+
do_load if !loaded
|
44
|
+
raise "Can't add messages without uid_validity" if !uid_validity
|
45
|
+
uid = uid.to_i
|
46
|
+
if uids.include?(uid)
|
47
|
+
Imap::Backup.logger.debug(
|
48
|
+
"[#{folder}] message #{uid} already downloaded - skipping"
|
49
|
+
)
|
50
|
+
return
|
51
|
+
end
|
52
|
+
|
53
|
+
body = message["RFC822"]
|
54
|
+
mboxrd_message = Email::Mboxrd::Message.new(body)
|
55
|
+
mbox = nil
|
56
|
+
begin
|
57
|
+
mbox = File.open(mbox_pathname, "ab")
|
58
|
+
mbox.write mboxrd_message.to_serialized
|
59
|
+
@uids << uid
|
60
|
+
write_imap_file
|
61
|
+
rescue => e
|
62
|
+
message = <<-ERROR.gsub(/^\s*/m, "")
|
63
|
+
[#{folder}] failed to save message #{uid}:
|
64
|
+
#{body}. #{e}:
|
65
|
+
#{e.backtrace.join("\n")}"
|
66
|
+
ERROR
|
67
|
+
Imap::Backup.logger.warn message
|
68
|
+
ensure
|
69
|
+
mbox.close if mbox
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
def load(uid)
|
74
|
+
do_load if !loaded
|
75
|
+
message_index = uids.find_index(uid)
|
76
|
+
return nil if message_index.nil?
|
77
|
+
load_nth(message_index)
|
78
|
+
end
|
79
|
+
|
80
|
+
def update_uid(old, new)
|
81
|
+
index = uids.find_index(old.to_i)
|
82
|
+
return if index.nil?
|
83
|
+
uids[index] = new.to_i
|
84
|
+
write_imap_file
|
85
|
+
end
|
86
|
+
|
87
|
+
def reset
|
88
|
+
@uids = nil
|
89
|
+
@uid_validity = nil
|
90
|
+
@loaded = false
|
91
|
+
delete_files
|
92
|
+
write_blank_mbox_file
|
93
|
+
end
|
94
|
+
|
95
|
+
def rename(new_name)
|
96
|
+
new_mbox_pathname = absolute_path(new_name + ".mbox")
|
97
|
+
new_imap_pathname = absolute_path(new_name + ".imap")
|
98
|
+
File.rename(mbox_pathname, new_mbox_pathname)
|
99
|
+
File.rename(imap_pathname, new_imap_pathname)
|
100
|
+
@folder = new_name
|
101
|
+
end
|
102
|
+
|
103
|
+
def relative_path
|
104
|
+
File.dirname(folder)
|
105
|
+
end
|
106
|
+
|
107
|
+
private
|
108
|
+
|
109
|
+
def do_load
|
110
|
+
data = imap_data
|
111
|
+
if data
|
112
|
+
@uids = data[:uids].map(&:to_i).sort
|
113
|
+
@uid_validity = data[:uid_validity]
|
114
|
+
@loaded = true
|
115
|
+
else
|
116
|
+
reset
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
def imap_data
|
121
|
+
return nil if !imap_ok?
|
122
|
+
|
123
|
+
imap_data = nil
|
124
|
+
|
125
|
+
begin
|
126
|
+
imap_data = JSON.parse(File.read(imap_pathname), symbolize_names: true)
|
127
|
+
rescue JSON::ParserError
|
128
|
+
return nil
|
129
|
+
end
|
130
|
+
|
131
|
+
return nil if !imap_data.has_key?(:uids)
|
132
|
+
return nil if !imap_data[:uids].is_a?(Array)
|
133
|
+
|
134
|
+
imap_data
|
135
|
+
end
|
136
|
+
|
137
|
+
def imap_ok?
|
138
|
+
return false if !exist?
|
139
|
+
return false if !imap_looks_like_json?
|
140
|
+
true
|
141
|
+
end
|
142
|
+
|
143
|
+
def load_nth(index)
|
144
|
+
each_mbox_message.with_index do |raw, i|
|
145
|
+
next unless i == index
|
146
|
+
return Email::Mboxrd::Message.from_serialized(raw)
|
147
|
+
end
|
148
|
+
nil
|
149
|
+
end
|
150
|
+
|
151
|
+
def each_mbox_message
|
152
|
+
Enumerator.new do |e|
|
153
|
+
File.open(mbox_pathname) do |f|
|
154
|
+
lines = []
|
155
|
+
|
156
|
+
while line = f.gets
|
157
|
+
if line.start_with?("From ")
|
158
|
+
e.yield lines.join if lines.count > 0
|
159
|
+
lines = [line]
|
160
|
+
else
|
161
|
+
lines << line
|
162
|
+
end
|
163
|
+
end
|
164
|
+
e.yield lines.join if lines.count > 0
|
165
|
+
end
|
166
|
+
end
|
167
|
+
end
|
168
|
+
|
169
|
+
def imap_looks_like_json?
|
170
|
+
return false unless imap_exist?
|
171
|
+
content = File.read(imap_pathname)
|
172
|
+
content.start_with?("{")
|
173
|
+
end
|
174
|
+
|
175
|
+
def write_imap_file
|
176
|
+
imap_data = {
|
177
|
+
version: CURRENT_VERSION,
|
178
|
+
uid_validity: @uid_validity,
|
179
|
+
uids: @uids
|
180
|
+
}
|
181
|
+
content = imap_data.to_json
|
182
|
+
File.open(imap_pathname, "w") { |f| f.write content }
|
183
|
+
end
|
184
|
+
|
185
|
+
def write_blank_mbox_file
|
186
|
+
File.open(mbox_pathname, "w") { |f| f.write "" }
|
187
|
+
end
|
188
|
+
|
189
|
+
def delete_files
|
190
|
+
File.unlink(imap_pathname) if imap_exist?
|
191
|
+
File.unlink(mbox_pathname) if mbox_exist?
|
192
|
+
end
|
193
|
+
|
194
|
+
def mbox_exist?
|
195
|
+
File.exist?(mbox_pathname)
|
196
|
+
end
|
197
|
+
|
198
|
+
def imap_exist?
|
199
|
+
File.exist?(imap_pathname)
|
200
|
+
end
|
201
|
+
|
202
|
+
def absolute_path(relative_path)
|
203
|
+
File.join(path, relative_path)
|
204
|
+
end
|
205
|
+
|
206
|
+
def mbox_pathname
|
207
|
+
absolute_path(folder + ".mbox")
|
208
|
+
end
|
209
|
+
|
210
|
+
def imap_pathname
|
211
|
+
absolute_path(folder + ".imap")
|
212
|
+
end
|
213
|
+
end
|
214
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
module Imap::Backup
|
2
|
+
class Uploader
|
3
|
+
attr_reader :folder
|
4
|
+
attr_reader :serializer
|
5
|
+
|
6
|
+
def initialize(folder, serializer)
|
7
|
+
@folder = folder
|
8
|
+
@serializer = serializer
|
9
|
+
end
|
10
|
+
|
11
|
+
def run
|
12
|
+
missing_uids.each do |uid|
|
13
|
+
message = serializer.load(uid)
|
14
|
+
next if message.nil?
|
15
|
+
new_uid = folder.append(message)
|
16
|
+
serializer.update_uid(uid, new_uid)
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
private
|
21
|
+
|
22
|
+
def missing_uids
|
23
|
+
serializer.uids - folder.uids
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
data/lib/imap/backup/utils.rb
CHANGED
@@ -3,7 +3,7 @@ require "fileutils"
|
|
3
3
|
module Imap::Backup
|
4
4
|
module Utils
|
5
5
|
def self.check_permissions(filename, limit)
|
6
|
-
actual =
|
6
|
+
actual = mode(filename)
|
7
7
|
return nil if actual.nil?
|
8
8
|
mask = ~limit & 0o777
|
9
9
|
if actual & mask != 0
|
@@ -16,8 +16,8 @@ module Imap::Backup
|
|
16
16
|
end
|
17
17
|
end
|
18
18
|
|
19
|
-
def self.
|
20
|
-
return nil
|
19
|
+
def self.mode(filename)
|
20
|
+
return nil if !File.exist?(filename)
|
21
21
|
|
22
22
|
stat = File.stat(filename)
|
23
23
|
stat.mode & 0o777
|
data/lib/imap/backup/version.rb
CHANGED
@@ -1,8 +1,9 @@
|
|
1
1
|
module Imap; end
|
2
2
|
|
3
3
|
module Imap::Backup
|
4
|
-
MAJOR =
|
5
|
-
MINOR =
|
6
|
-
REVISION =
|
7
|
-
|
4
|
+
MAJOR = 2
|
5
|
+
MINOR = 0
|
6
|
+
REVISION = 0
|
7
|
+
PRE = "rc1"
|
8
|
+
VERSION = [MAJOR, MINOR, REVISION, PRE].compact.map(&:to_s).join(".")
|
8
9
|
end
|
@@ -7,21 +7,99 @@ RSpec.describe "backup", type: :feature, docker: true do
|
|
7
7
|
let(:messages_as_mbox) do
|
8
8
|
message_as_mbox_entry(msg1) + message_as_mbox_entry(msg2)
|
9
9
|
end
|
10
|
-
let(:folder) { "
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
10
|
+
let(:folder) { "my-stuff" }
|
11
|
+
let(:email1) { send_email folder, msg1 }
|
12
|
+
let(:email2) { send_email folder, msg2 }
|
13
|
+
let!(:pre) { }
|
14
|
+
let!(:setup) do
|
15
|
+
server_create_folder folder
|
16
|
+
email1
|
17
|
+
email2
|
18
|
+
connection.run_backup
|
15
19
|
end
|
16
20
|
|
17
21
|
after do
|
18
22
|
FileUtils.rm_rf local_backup_path
|
19
23
|
delete_emails folder
|
24
|
+
server_delete_folder folder
|
25
|
+
connection.disconnect
|
20
26
|
end
|
21
27
|
|
22
28
|
it "downloads messages" do
|
23
|
-
connection.run_backup
|
24
|
-
|
25
29
|
expect(mbox_content(folder)).to eq(messages_as_mbox)
|
26
30
|
end
|
31
|
+
|
32
|
+
context "IMAP metadata" do
|
33
|
+
let(:imap_metadata) { imap_parsed(folder) }
|
34
|
+
let(:folder_uids) { server_uids(folder) }
|
35
|
+
|
36
|
+
it "saves IMAP metadata in a JSON file" do
|
37
|
+
expect { imap_metadata }.to_not raise_error
|
38
|
+
end
|
39
|
+
|
40
|
+
it "saves a file version" do
|
41
|
+
expect(imap_metadata[:version].to_s).to match(/^[0-9\.]$/)
|
42
|
+
end
|
43
|
+
|
44
|
+
it "records IMAP ids" do
|
45
|
+
expect(imap_metadata[:uids]).to eq(folder_uids)
|
46
|
+
end
|
47
|
+
|
48
|
+
it "records uid_validity" do
|
49
|
+
expect(imap_metadata[:uid_validity]).to eq(server_uid_validity(folder))
|
50
|
+
end
|
51
|
+
|
52
|
+
context "when uid_validity does not match" do
|
53
|
+
let(:new_name) { "NEWNAME" }
|
54
|
+
let(:email3) { send_email folder, msg3 }
|
55
|
+
let!(:pre) do
|
56
|
+
server_create_folder folder
|
57
|
+
email3
|
58
|
+
@original_folder_uid_validity = server_uid_validity(folder)
|
59
|
+
connection.run_backup
|
60
|
+
server_rename_folder folder, new_name
|
61
|
+
end
|
62
|
+
let(:renamed_folder) { folder + "." + @original_folder_uid_validity.to_s }
|
63
|
+
|
64
|
+
after do
|
65
|
+
server_delete_folder new_name
|
66
|
+
end
|
67
|
+
|
68
|
+
it "renames the old backup" do
|
69
|
+
expect(mbox_content(renamed_folder)).to eq(message_as_mbox_entry(msg3))
|
70
|
+
end
|
71
|
+
|
72
|
+
it "downloads messages" do
|
73
|
+
expect(mbox_content(folder)).to eq(messages_as_mbox)
|
74
|
+
end
|
75
|
+
|
76
|
+
context "when a renamed local backup exists" do
|
77
|
+
let!(:pre) do
|
78
|
+
super()
|
79
|
+
File.write(imap_path(renamed_folder), "existing imap")
|
80
|
+
File.write(mbox_path(renamed_folder), "existing mbox")
|
81
|
+
end
|
82
|
+
|
83
|
+
it "moves the old backup to a uniquely named directory" do
|
84
|
+
renamed = folder + "." + @original_folder_uid_validity.to_s + ".1"
|
85
|
+
expect(mbox_content(renamed)).to eq(message_as_mbox_entry(msg3))
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
context "when an unversioned .imap file is found" do
|
91
|
+
let!(:pre) do
|
92
|
+
File.open(imap_path(folder), "w") { |f| f.write "old format imap" }
|
93
|
+
File.open(mbox_path(folder), "w") { |f| f.write "old format emails" }
|
94
|
+
end
|
95
|
+
|
96
|
+
it "replaces the .imap file with a versioned JSON file" do
|
97
|
+
expect(imap_metadata[:uids]).to eq(folder_uids)
|
98
|
+
end
|
99
|
+
|
100
|
+
it "does the download" do
|
101
|
+
expect(mbox_content(folder)).to eq(messages_as_mbox)
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
27
105
|
end
|
@@ -0,0 +1,112 @@
|
|
1
|
+
require "features/helper"
|
2
|
+
|
3
|
+
RSpec.describe "restore", type: :feature, docker: true do
|
4
|
+
include_context "imap-backup connection"
|
5
|
+
include_context "message-fixtures"
|
6
|
+
|
7
|
+
let(:messages_as_mbox) do
|
8
|
+
message_as_mbox_entry(msg1) + message_as_mbox_entry(msg2)
|
9
|
+
end
|
10
|
+
let(:messages_as_server_messages) do
|
11
|
+
[message_as_server_message(msg1), message_as_server_message(msg2)]
|
12
|
+
end
|
13
|
+
let(:message_uids) { [msg1[:uid], msg2[:uid]] }
|
14
|
+
let(:existing_imap_content) { imap_data(uid_validity, message_uids).to_json }
|
15
|
+
let(:folder) { "my-stuff" }
|
16
|
+
let(:uid_validity) { 1234 }
|
17
|
+
|
18
|
+
let!(:pre) {}
|
19
|
+
let!(:setup) do
|
20
|
+
File.write(imap_path(folder), existing_imap_content)
|
21
|
+
File.write(mbox_path(folder), messages_as_mbox)
|
22
|
+
connection.restore
|
23
|
+
end
|
24
|
+
let(:cleanup) do
|
25
|
+
FileUtils.rm_rf local_backup_path
|
26
|
+
server_delete_folder folder
|
27
|
+
connection.disconnect
|
28
|
+
end
|
29
|
+
|
30
|
+
after { cleanup }
|
31
|
+
|
32
|
+
context "when the folder doesn't exist" do
|
33
|
+
it "restores messages" do
|
34
|
+
messages = server_messages(folder).map { |m| server_message_to_body(m) }
|
35
|
+
expect(messages).to eq(messages_as_server_messages)
|
36
|
+
end
|
37
|
+
|
38
|
+
it "updates local uids to match the new server ones" do
|
39
|
+
updated_imap_content = imap_parsed(folder)
|
40
|
+
expect(server_uids(folder)).to eq(updated_imap_content[:uids])
|
41
|
+
end
|
42
|
+
|
43
|
+
it "sets the backup uid_validity to match the new folder" do
|
44
|
+
updated_imap_content = imap_parsed(folder)
|
45
|
+
expect(updated_imap_content[:uid_validity]).
|
46
|
+
to eq(server_uid_validity(folder))
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
context "when the folder exists" do
|
51
|
+
let(:email3) { send_email folder, msg3 }
|
52
|
+
|
53
|
+
context "when the uid_validity matches" do
|
54
|
+
let(:pre) do
|
55
|
+
server_create_folder folder
|
56
|
+
email3
|
57
|
+
uid_validity
|
58
|
+
end
|
59
|
+
let(:messages_as_server_messages) do
|
60
|
+
[
|
61
|
+
message_as_server_message(msg3),
|
62
|
+
message_as_server_message(msg1),
|
63
|
+
message_as_server_message(msg2)
|
64
|
+
]
|
65
|
+
end
|
66
|
+
let(:uid_validity) { server_uid_validity(folder) }
|
67
|
+
|
68
|
+
it "appends to the existing folder" do
|
69
|
+
messages = server_messages(folder).map { |m| server_message_to_body(m) }
|
70
|
+
expect(messages).to eq(messages_as_server_messages)
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
context "when the uid_validity doesn't match" do
|
75
|
+
let(:pre) do
|
76
|
+
server_create_folder folder
|
77
|
+
email3
|
78
|
+
end
|
79
|
+
let(:new_folder) { "#{folder}.#{uid_validity}" }
|
80
|
+
let(:cleanup) do
|
81
|
+
server_delete_folder new_folder
|
82
|
+
super()
|
83
|
+
end
|
84
|
+
|
85
|
+
it "sets the backup uid_validity to match the new folder" do
|
86
|
+
updated_imap_content = imap_parsed(new_folder)
|
87
|
+
expect(updated_imap_content[:uid_validity]).
|
88
|
+
to eq(server_uid_validity(new_folder))
|
89
|
+
end
|
90
|
+
|
91
|
+
it "renames the backup" do
|
92
|
+
expect(mbox_content(new_folder)).to eq(messages_as_mbox)
|
93
|
+
end
|
94
|
+
|
95
|
+
it "leaves the existing folder as is" do
|
96
|
+
messages = server_messages(folder).map { |m| server_message_to_body(m) }
|
97
|
+
expect(messages).to eq([message_as_server_message(msg3)])
|
98
|
+
end
|
99
|
+
|
100
|
+
it "creates the new folder" do
|
101
|
+
expect(server_folders.map(&:name)).to include(new_folder)
|
102
|
+
end
|
103
|
+
|
104
|
+
it "uploads to the new folder" do
|
105
|
+
messages = server_messages(new_folder).map do |m|
|
106
|
+
server_message_to_body(m)
|
107
|
+
end
|
108
|
+
expect(messages).to eq(messages_as_server_messages)
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|