imap-backup 1.4.2 → 2.0.0.rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,214 @@
1
+ require "json"
2
+
3
+ require "email/mboxrd/message"
4
+
5
+ module Imap::Backup
6
+ class Serializer::MboxStore
7
+ CURRENT_VERSION = 2
8
+
9
+ attr_reader :folder
10
+ attr_reader :path
11
+ attr_reader :loaded
12
+
13
+ def initialize(path, folder)
14
+ @path = path
15
+ @folder = folder
16
+ @loaded = false
17
+ @uids = nil
18
+ @uid_validity = nil
19
+ end
20
+
21
+ def exist?
22
+ mbox_exist? && imap_exist?
23
+ end
24
+
25
+ def uid_validity
26
+ do_load if !loaded
27
+ @uid_validity
28
+ end
29
+
30
+ def uid_validity=(value)
31
+ do_load if !loaded
32
+ @uid_validity = value
33
+ @uids ||= []
34
+ write_imap_file
35
+ end
36
+
37
+ def uids
38
+ do_load if !loaded
39
+ @uids || []
40
+ end
41
+
42
+ def add(uid, message)
43
+ do_load if !loaded
44
+ raise "Can't add messages without uid_validity" if !uid_validity
45
+ uid = uid.to_i
46
+ if uids.include?(uid)
47
+ Imap::Backup.logger.debug(
48
+ "[#{folder}] message #{uid} already downloaded - skipping"
49
+ )
50
+ return
51
+ end
52
+
53
+ body = message["RFC822"]
54
+ mboxrd_message = Email::Mboxrd::Message.new(body)
55
+ mbox = nil
56
+ begin
57
+ mbox = File.open(mbox_pathname, "ab")
58
+ mbox.write mboxrd_message.to_serialized
59
+ @uids << uid
60
+ write_imap_file
61
+ rescue => e
62
+ message = <<-ERROR.gsub(/^\s*/m, "")
63
+ [#{folder}] failed to save message #{uid}:
64
+ #{body}. #{e}:
65
+ #{e.backtrace.join("\n")}"
66
+ ERROR
67
+ Imap::Backup.logger.warn message
68
+ ensure
69
+ mbox.close if mbox
70
+ end
71
+ end
72
+
73
+ def load(uid)
74
+ do_load if !loaded
75
+ message_index = uids.find_index(uid)
76
+ return nil if message_index.nil?
77
+ load_nth(message_index)
78
+ end
79
+
80
+ def update_uid(old, new)
81
+ index = uids.find_index(old.to_i)
82
+ return if index.nil?
83
+ uids[index] = new.to_i
84
+ write_imap_file
85
+ end
86
+
87
+ def reset
88
+ @uids = nil
89
+ @uid_validity = nil
90
+ @loaded = false
91
+ delete_files
92
+ write_blank_mbox_file
93
+ end
94
+
95
+ def rename(new_name)
96
+ new_mbox_pathname = absolute_path(new_name + ".mbox")
97
+ new_imap_pathname = absolute_path(new_name + ".imap")
98
+ File.rename(mbox_pathname, new_mbox_pathname)
99
+ File.rename(imap_pathname, new_imap_pathname)
100
+ @folder = new_name
101
+ end
102
+
103
+ def relative_path
104
+ File.dirname(folder)
105
+ end
106
+
107
+ private
108
+
109
+ def do_load
110
+ data = imap_data
111
+ if data
112
+ @uids = data[:uids].map(&:to_i).sort
113
+ @uid_validity = data[:uid_validity]
114
+ @loaded = true
115
+ else
116
+ reset
117
+ end
118
+ end
119
+
120
+ def imap_data
121
+ return nil if !imap_ok?
122
+
123
+ imap_data = nil
124
+
125
+ begin
126
+ imap_data = JSON.parse(File.read(imap_pathname), symbolize_names: true)
127
+ rescue JSON::ParserError
128
+ return nil
129
+ end
130
+
131
+ return nil if !imap_data.has_key?(:uids)
132
+ return nil if !imap_data[:uids].is_a?(Array)
133
+
134
+ imap_data
135
+ end
136
+
137
+ def imap_ok?
138
+ return false if !exist?
139
+ return false if !imap_looks_like_json?
140
+ true
141
+ end
142
+
143
+ def load_nth(index)
144
+ each_mbox_message.with_index do |raw, i|
145
+ next unless i == index
146
+ return Email::Mboxrd::Message.from_serialized(raw)
147
+ end
148
+ nil
149
+ end
150
+
151
+ def each_mbox_message
152
+ Enumerator.new do |e|
153
+ File.open(mbox_pathname) do |f|
154
+ lines = []
155
+
156
+ while line = f.gets
157
+ if line.start_with?("From ")
158
+ e.yield lines.join if lines.count > 0
159
+ lines = [line]
160
+ else
161
+ lines << line
162
+ end
163
+ end
164
+ e.yield lines.join if lines.count > 0
165
+ end
166
+ end
167
+ end
168
+
169
+ def imap_looks_like_json?
170
+ return false unless imap_exist?
171
+ content = File.read(imap_pathname)
172
+ content.start_with?("{")
173
+ end
174
+
175
+ def write_imap_file
176
+ imap_data = {
177
+ version: CURRENT_VERSION,
178
+ uid_validity: @uid_validity,
179
+ uids: @uids
180
+ }
181
+ content = imap_data.to_json
182
+ File.open(imap_pathname, "w") { |f| f.write content }
183
+ end
184
+
185
+ def write_blank_mbox_file
186
+ File.open(mbox_pathname, "w") { |f| f.write "" }
187
+ end
188
+
189
+ def delete_files
190
+ File.unlink(imap_pathname) if imap_exist?
191
+ File.unlink(mbox_pathname) if mbox_exist?
192
+ end
193
+
194
+ def mbox_exist?
195
+ File.exist?(mbox_pathname)
196
+ end
197
+
198
+ def imap_exist?
199
+ File.exist?(imap_pathname)
200
+ end
201
+
202
+ def absolute_path(relative_path)
203
+ File.join(path, relative_path)
204
+ end
205
+
206
+ def mbox_pathname
207
+ absolute_path(folder + ".mbox")
208
+ end
209
+
210
+ def imap_pathname
211
+ absolute_path(folder + ".imap")
212
+ end
213
+ end
214
+ end
@@ -0,0 +1,26 @@
1
+ module Imap::Backup
2
+ class Uploader
3
+ attr_reader :folder
4
+ attr_reader :serializer
5
+
6
+ def initialize(folder, serializer)
7
+ @folder = folder
8
+ @serializer = serializer
9
+ end
10
+
11
+ def run
12
+ missing_uids.each do |uid|
13
+ message = serializer.load(uid)
14
+ next if message.nil?
15
+ new_uid = folder.append(message)
16
+ serializer.update_uid(uid, new_uid)
17
+ end
18
+ end
19
+
20
+ private
21
+
22
+ def missing_uids
23
+ serializer.uids - folder.uids
24
+ end
25
+ end
26
+ end
@@ -3,7 +3,7 @@ require "fileutils"
3
3
  module Imap::Backup
4
4
  module Utils
5
5
  def self.check_permissions(filename, limit)
6
- actual = stat(filename)
6
+ actual = mode(filename)
7
7
  return nil if actual.nil?
8
8
  mask = ~limit & 0o777
9
9
  if actual & mask != 0
@@ -16,8 +16,8 @@ module Imap::Backup
16
16
  end
17
17
  end
18
18
 
19
- def self.stat(filename)
20
- return nil unless File.exist?(filename)
19
+ def self.mode(filename)
20
+ return nil if !File.exist?(filename)
21
21
 
22
22
  stat = File.stat(filename)
23
23
  stat.mode & 0o777
@@ -1,8 +1,9 @@
1
1
  module Imap; end
2
2
 
3
3
  module Imap::Backup
4
- MAJOR = 1
5
- MINOR = 4
6
- REVISION = 2
7
- VERSION = [MAJOR, MINOR, REVISION].map(&:to_s).join(".")
4
+ MAJOR = 2
5
+ MINOR = 0
6
+ REVISION = 0
7
+ PRE = "rc1"
8
+ VERSION = [MAJOR, MINOR, REVISION, PRE].compact.map(&:to_s).join(".")
8
9
  end
@@ -7,21 +7,99 @@ RSpec.describe "backup", type: :feature, docker: true do
7
7
  let(:messages_as_mbox) do
8
8
  message_as_mbox_entry(msg1) + message_as_mbox_entry(msg2)
9
9
  end
10
- let(:folder) { "INBOX" }
11
-
12
- before do
13
- send_email folder, msg1
14
- send_email folder, msg2
10
+ let(:folder) { "my-stuff" }
11
+ let(:email1) { send_email folder, msg1 }
12
+ let(:email2) { send_email folder, msg2 }
13
+ let!(:pre) { }
14
+ let!(:setup) do
15
+ server_create_folder folder
16
+ email1
17
+ email2
18
+ connection.run_backup
15
19
  end
16
20
 
17
21
  after do
18
22
  FileUtils.rm_rf local_backup_path
19
23
  delete_emails folder
24
+ server_delete_folder folder
25
+ connection.disconnect
20
26
  end
21
27
 
22
28
  it "downloads messages" do
23
- connection.run_backup
24
-
25
29
  expect(mbox_content(folder)).to eq(messages_as_mbox)
26
30
  end
31
+
32
+ context "IMAP metadata" do
33
+ let(:imap_metadata) { imap_parsed(folder) }
34
+ let(:folder_uids) { server_uids(folder) }
35
+
36
+ it "saves IMAP metadata in a JSON file" do
37
+ expect { imap_metadata }.to_not raise_error
38
+ end
39
+
40
+ it "saves a file version" do
41
+ expect(imap_metadata[:version].to_s).to match(/^[0-9\.]$/)
42
+ end
43
+
44
+ it "records IMAP ids" do
45
+ expect(imap_metadata[:uids]).to eq(folder_uids)
46
+ end
47
+
48
+ it "records uid_validity" do
49
+ expect(imap_metadata[:uid_validity]).to eq(server_uid_validity(folder))
50
+ end
51
+
52
+ context "when uid_validity does not match" do
53
+ let(:new_name) { "NEWNAME" }
54
+ let(:email3) { send_email folder, msg3 }
55
+ let!(:pre) do
56
+ server_create_folder folder
57
+ email3
58
+ @original_folder_uid_validity = server_uid_validity(folder)
59
+ connection.run_backup
60
+ server_rename_folder folder, new_name
61
+ end
62
+ let(:renamed_folder) { folder + "." + @original_folder_uid_validity.to_s }
63
+
64
+ after do
65
+ server_delete_folder new_name
66
+ end
67
+
68
+ it "renames the old backup" do
69
+ expect(mbox_content(renamed_folder)).to eq(message_as_mbox_entry(msg3))
70
+ end
71
+
72
+ it "downloads messages" do
73
+ expect(mbox_content(folder)).to eq(messages_as_mbox)
74
+ end
75
+
76
+ context "when a renamed local backup exists" do
77
+ let!(:pre) do
78
+ super()
79
+ File.write(imap_path(renamed_folder), "existing imap")
80
+ File.write(mbox_path(renamed_folder), "existing mbox")
81
+ end
82
+
83
+ it "moves the old backup to a uniquely named directory" do
84
+ renamed = folder + "." + @original_folder_uid_validity.to_s + ".1"
85
+ expect(mbox_content(renamed)).to eq(message_as_mbox_entry(msg3))
86
+ end
87
+ end
88
+ end
89
+
90
+ context "when an unversioned .imap file is found" do
91
+ let!(:pre) do
92
+ File.open(imap_path(folder), "w") { |f| f.write "old format imap" }
93
+ File.open(mbox_path(folder), "w") { |f| f.write "old format emails" }
94
+ end
95
+
96
+ it "replaces the .imap file with a versioned JSON file" do
97
+ expect(imap_metadata[:uids]).to eq(folder_uids)
98
+ end
99
+
100
+ it "does the download" do
101
+ expect(mbox_content(folder)).to eq(messages_as_mbox)
102
+ end
103
+ end
104
+ end
27
105
  end
@@ -0,0 +1,112 @@
1
+ require "features/helper"
2
+
3
+ RSpec.describe "restore", type: :feature, docker: true do
4
+ include_context "imap-backup connection"
5
+ include_context "message-fixtures"
6
+
7
+ let(:messages_as_mbox) do
8
+ message_as_mbox_entry(msg1) + message_as_mbox_entry(msg2)
9
+ end
10
+ let(:messages_as_server_messages) do
11
+ [message_as_server_message(msg1), message_as_server_message(msg2)]
12
+ end
13
+ let(:message_uids) { [msg1[:uid], msg2[:uid]] }
14
+ let(:existing_imap_content) { imap_data(uid_validity, message_uids).to_json }
15
+ let(:folder) { "my-stuff" }
16
+ let(:uid_validity) { 1234 }
17
+
18
+ let!(:pre) {}
19
+ let!(:setup) do
20
+ File.write(imap_path(folder), existing_imap_content)
21
+ File.write(mbox_path(folder), messages_as_mbox)
22
+ connection.restore
23
+ end
24
+ let(:cleanup) do
25
+ FileUtils.rm_rf local_backup_path
26
+ server_delete_folder folder
27
+ connection.disconnect
28
+ end
29
+
30
+ after { cleanup }
31
+
32
+ context "when the folder doesn't exist" do
33
+ it "restores messages" do
34
+ messages = server_messages(folder).map { |m| server_message_to_body(m) }
35
+ expect(messages).to eq(messages_as_server_messages)
36
+ end
37
+
38
+ it "updates local uids to match the new server ones" do
39
+ updated_imap_content = imap_parsed(folder)
40
+ expect(server_uids(folder)).to eq(updated_imap_content[:uids])
41
+ end
42
+
43
+ it "sets the backup uid_validity to match the new folder" do
44
+ updated_imap_content = imap_parsed(folder)
45
+ expect(updated_imap_content[:uid_validity]).
46
+ to eq(server_uid_validity(folder))
47
+ end
48
+ end
49
+
50
+ context "when the folder exists" do
51
+ let(:email3) { send_email folder, msg3 }
52
+
53
+ context "when the uid_validity matches" do
54
+ let(:pre) do
55
+ server_create_folder folder
56
+ email3
57
+ uid_validity
58
+ end
59
+ let(:messages_as_server_messages) do
60
+ [
61
+ message_as_server_message(msg3),
62
+ message_as_server_message(msg1),
63
+ message_as_server_message(msg2)
64
+ ]
65
+ end
66
+ let(:uid_validity) { server_uid_validity(folder) }
67
+
68
+ it "appends to the existing folder" do
69
+ messages = server_messages(folder).map { |m| server_message_to_body(m) }
70
+ expect(messages).to eq(messages_as_server_messages)
71
+ end
72
+ end
73
+
74
+ context "when the uid_validity doesn't match" do
75
+ let(:pre) do
76
+ server_create_folder folder
77
+ email3
78
+ end
79
+ let(:new_folder) { "#{folder}.#{uid_validity}" }
80
+ let(:cleanup) do
81
+ server_delete_folder new_folder
82
+ super()
83
+ end
84
+
85
+ it "sets the backup uid_validity to match the new folder" do
86
+ updated_imap_content = imap_parsed(new_folder)
87
+ expect(updated_imap_content[:uid_validity]).
88
+ to eq(server_uid_validity(new_folder))
89
+ end
90
+
91
+ it "renames the backup" do
92
+ expect(mbox_content(new_folder)).to eq(messages_as_mbox)
93
+ end
94
+
95
+ it "leaves the existing folder as is" do
96
+ messages = server_messages(folder).map { |m| server_message_to_body(m) }
97
+ expect(messages).to eq([message_as_server_message(msg3)])
98
+ end
99
+
100
+ it "creates the new folder" do
101
+ expect(server_folders.map(&:name)).to include(new_folder)
102
+ end
103
+
104
+ it "uploads to the new folder" do
105
+ messages = server_messages(new_folder).map do |m|
106
+ server_message_to_body(m)
107
+ end
108
+ expect(messages).to eq(messages_as_server_messages)
109
+ end
110
+ end
111
+ end
112
+ end