imap-backup 2.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rspec-all +2 -0
- data/.rubocop.yml +10 -1
- data/.travis.yml +1 -0
- data/README.md +1 -1
- data/Rakefile +0 -1
- data/bin/imap-backup +3 -9
- data/imap-backup.gemspec +5 -5
- data/lib/email/mboxrd/message.rb +2 -2
- data/lib/imap/backup/account/connection.rb +11 -3
- data/lib/imap/backup/account/folder.rb +11 -6
- data/lib/imap/backup/configuration/account.rb +7 -7
- data/lib/imap/backup/configuration/asker.rb +2 -1
- data/lib/imap/backup/configuration/connection_tester.rb +1 -1
- data/lib/imap/backup/configuration/folder_chooser.rb +32 -5
- data/lib/imap/backup/configuration/list.rb +2 -0
- data/lib/imap/backup/configuration/setup.rb +2 -1
- data/lib/imap/backup/configuration/store.rb +3 -6
- data/lib/imap/backup/downloader.rb +8 -7
- data/lib/imap/backup/serializer/mbox.rb +2 -1
- data/lib/imap/backup/serializer/mbox_store.rb +14 -6
- data/lib/imap/backup/uploader.rb +1 -0
- data/lib/imap/backup/utils.rb +11 -9
- data/lib/imap/backup/version.rb +1 -1
- data/spec/features/backup_spec.rb +6 -5
- data/spec/features/support/backup_directory.rb +5 -5
- data/spec/features/support/email_server.rb +11 -8
- data/spec/features/support/shared/connection_context.rb +2 -2
- data/spec/support/fixtures.rb +1 -1
- data/spec/support/higline_test_helpers.rb +1 -1
- data/spec/unit/email/mboxrd/message_spec.rb +51 -42
- data/spec/unit/email/provider_spec.rb +0 -2
- data/spec/unit/imap/backup/account/connection_spec.rb +18 -11
- data/spec/unit/imap/backup/account/folder_spec.rb +26 -12
- data/spec/unit/imap/backup/configuration/account_spec.rb +22 -19
- data/spec/unit/imap/backup/configuration/asker_spec.rb +30 -31
- data/spec/unit/imap/backup/configuration/connection_tester_spec.rb +16 -13
- data/spec/unit/imap/backup/configuration/folder_chooser_spec.rb +45 -18
- data/spec/unit/imap/backup/configuration/list_spec.rb +8 -13
- data/spec/unit/imap/backup/configuration/setup_spec.rb +36 -30
- data/spec/unit/imap/backup/configuration/store_spec.rb +7 -4
- data/spec/unit/imap/backup/downloader_spec.rb +11 -7
- data/spec/unit/imap/backup/serializer/mbox_spec.rb +2 -5
- data/spec/unit/imap/backup/serializer/mbox_store_spec.rb +4 -4
- data/spec/unit/imap/backup/uploader_spec.rb +0 -2
- data/spec/unit/imap/backup/utils_spec.rb +1 -3
- metadata +6 -6
@@ -52,6 +52,7 @@ module Imap::Backup
|
|
52
52
|
|
53
53
|
def data
|
54
54
|
return @data if @data
|
55
|
+
|
55
56
|
if File.exist?(pathname)
|
56
57
|
Utils.check_permissions pathname, 0o600
|
57
58
|
contents = File.read(pathname)
|
@@ -73,12 +74,8 @@ module Imap::Backup
|
|
73
74
|
end
|
74
75
|
|
75
76
|
def mkdir_private(path)
|
76
|
-
if !File.directory?(path)
|
77
|
-
|
78
|
-
end
|
79
|
-
if Utils::mode(path) != 0o700
|
80
|
-
FileUtils.chmod 0o700, path
|
81
|
-
end
|
77
|
+
FileUtils.mkdir(path) if !File.directory?(path)
|
78
|
+
FileUtils.chmod(0o700, path) if Utils.mode(path) != 0o700
|
82
79
|
end
|
83
80
|
end
|
84
81
|
end
|
@@ -4,22 +4,23 @@ module Imap::Backup
|
|
4
4
|
attr_reader :serializer
|
5
5
|
|
6
6
|
def initialize(folder, serializer)
|
7
|
-
@folder
|
7
|
+
@folder = folder
|
8
|
+
@serializer = serializer
|
8
9
|
end
|
9
10
|
|
10
11
|
def run
|
11
12
|
uids = folder.uids - serializer.uids
|
12
|
-
|
13
|
-
|
13
|
+
count = uids.count
|
14
|
+
Imap::Backup.logger.debug "[#{folder.name}] #{count} new messages"
|
15
|
+
uids.each.with_index do |uid, i|
|
14
16
|
message = folder.fetch(uid)
|
17
|
+
log_prefix = "[#{folder.name}] uid: #{uid} (#{i + 1}/#{count}) -"
|
15
18
|
if message.nil?
|
16
|
-
Imap::Backup.logger.debug(
|
17
|
-
"[#{folder.name}] #{uid} - not available - skipped"
|
18
|
-
)
|
19
|
+
Imap::Backup.logger.debug("#{log_prefix} not available - skipped")
|
19
20
|
next
|
20
21
|
end
|
21
22
|
Imap::Backup.logger.debug(
|
22
|
-
"
|
23
|
+
"#{log_prefix} #{message['RFC822'].size} bytes"
|
23
24
|
)
|
24
25
|
serializer.save(uid, message)
|
25
26
|
end
|
@@ -27,6 +27,7 @@ module Imap::Backup
|
|
27
27
|
new_name = "#{folder}.#{existing_uid_validity}#{extra}"
|
28
28
|
test_store = Serializer::MboxStore.new(path, new_name)
|
29
29
|
break if !test_store.exist?
|
30
|
+
|
30
31
|
digit ||= 0
|
31
32
|
digit += 1
|
32
33
|
end
|
@@ -84,7 +85,7 @@ module Imap::Backup
|
|
84
85
|
end
|
85
86
|
|
86
87
|
if Imap::Backup::Utils.mode(full_path) !=
|
87
|
-
|
88
|
+
Serializer::DIRECTORY_PERMISSIONS
|
88
89
|
FileUtils.chmod Serializer::DIRECTORY_PERMISSIONS, full_path
|
89
90
|
end
|
90
91
|
end
|
@@ -42,6 +42,7 @@ module Imap::Backup
|
|
42
42
|
def add(uid, message)
|
43
43
|
do_load if !loaded
|
44
44
|
raise "Can't add messages without uid_validity" if !uid_validity
|
45
|
+
|
45
46
|
uid = uid.to_i
|
46
47
|
if uids.include?(uid)
|
47
48
|
Imap::Backup.logger.debug(
|
@@ -58,7 +59,7 @@ module Imap::Backup
|
|
58
59
|
mbox.write mboxrd_message.to_serialized
|
59
60
|
@uids << uid
|
60
61
|
write_imap_file
|
61
|
-
rescue => e
|
62
|
+
rescue StandardError => e
|
62
63
|
message = <<-ERROR.gsub(/^\s*/m, "")
|
63
64
|
[#{folder}] failed to save message #{uid}:
|
64
65
|
#{body}. #{e}:
|
@@ -66,7 +67,7 @@ module Imap::Backup
|
|
66
67
|
ERROR
|
67
68
|
Imap::Backup.logger.warn message
|
68
69
|
ensure
|
69
|
-
mbox
|
70
|
+
mbox&.close
|
70
71
|
end
|
71
72
|
end
|
72
73
|
|
@@ -74,12 +75,14 @@ module Imap::Backup
|
|
74
75
|
do_load if !loaded
|
75
76
|
message_index = uids.find_index(uid)
|
76
77
|
return nil if message_index.nil?
|
78
|
+
|
77
79
|
load_nth(message_index)
|
78
80
|
end
|
79
81
|
|
80
82
|
def update_uid(old, new)
|
81
83
|
index = uids.find_index(old.to_i)
|
82
84
|
return if index.nil?
|
85
|
+
|
83
86
|
uids[index] = new.to_i
|
84
87
|
write_imap_file
|
85
88
|
end
|
@@ -128,7 +131,7 @@ module Imap::Backup
|
|
128
131
|
return nil
|
129
132
|
end
|
130
133
|
|
131
|
-
return nil if !imap_data.
|
134
|
+
return nil if !imap_data.key?(:uids)
|
132
135
|
return nil if !imap_data[:uids].is_a?(Array)
|
133
136
|
|
134
137
|
imap_data
|
@@ -137,12 +140,14 @@ module Imap::Backup
|
|
137
140
|
def imap_ok?
|
138
141
|
return false if !exist?
|
139
142
|
return false if !imap_looks_like_json?
|
143
|
+
|
140
144
|
true
|
141
145
|
end
|
142
146
|
|
143
147
|
def load_nth(index)
|
144
148
|
each_mbox_message.with_index do |raw, i|
|
145
149
|
next unless i == index
|
150
|
+
|
146
151
|
return Email::Mboxrd::Message.from_serialized(raw)
|
147
152
|
end
|
148
153
|
nil
|
@@ -153,21 +158,24 @@ module Imap::Backup
|
|
153
158
|
File.open(mbox_pathname) do |f|
|
154
159
|
lines = []
|
155
160
|
|
156
|
-
|
161
|
+
loop do
|
162
|
+
line = f.gets
|
163
|
+
break if !line
|
157
164
|
if line.start_with?("From ")
|
158
|
-
e.yield lines.join if lines.count
|
165
|
+
e.yield lines.join if lines.count.positive?
|
159
166
|
lines = [line]
|
160
167
|
else
|
161
168
|
lines << line
|
162
169
|
end
|
163
170
|
end
|
164
|
-
e.yield lines.join if lines.count
|
171
|
+
e.yield lines.join if lines.count.positive?
|
165
172
|
end
|
166
173
|
end
|
167
174
|
end
|
168
175
|
|
169
176
|
def imap_looks_like_json?
|
170
177
|
return false unless imap_exist?
|
178
|
+
|
171
179
|
content = File.read(imap_pathname)
|
172
180
|
content.start_with?("{")
|
173
181
|
end
|
data/lib/imap/backup/uploader.rb
CHANGED
data/lib/imap/backup/utils.rb
CHANGED
@@ -5,15 +5,16 @@ module Imap::Backup
|
|
5
5
|
def self.check_permissions(filename, limit)
|
6
6
|
actual = mode(filename)
|
7
7
|
return nil if actual.nil?
|
8
|
+
|
8
9
|
mask = ~limit & 0o777
|
9
|
-
if actual & mask
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
10
|
+
return if (actual & mask).zero?
|
11
|
+
|
12
|
+
message = format(
|
13
|
+
"Permissions on '%<filename>s' " \
|
14
|
+
"should be 0%<limit>o, not 0%<actual>o",
|
15
|
+
filename: filename, limit: limit, actual: actual
|
16
|
+
)
|
17
|
+
raise message
|
17
18
|
end
|
18
19
|
|
19
20
|
def self.mode(filename)
|
@@ -25,7 +26,8 @@ module Imap::Backup
|
|
25
26
|
|
26
27
|
def self.make_folder(base_path, path, permissions)
|
27
28
|
parts = path.split("/")
|
28
|
-
return if parts.
|
29
|
+
return if parts.empty?
|
30
|
+
|
29
31
|
full_path = File.join(base_path, path)
|
30
32
|
FileUtils.mkdir_p full_path
|
31
33
|
path = base_path
|
data/lib/imap/backup/version.rb
CHANGED
@@ -10,7 +10,7 @@ RSpec.describe "backup", type: :feature, docker: true do
|
|
10
10
|
let(:folder) { "my-stuff" }
|
11
11
|
let(:email1) { send_email folder, msg1 }
|
12
12
|
let(:email2) { send_email folder, msg2 }
|
13
|
-
let!(:pre) {
|
13
|
+
let!(:pre) {}
|
14
14
|
let!(:setup) do
|
15
15
|
server_create_folder folder
|
16
16
|
email1
|
@@ -29,7 +29,7 @@ RSpec.describe "backup", type: :feature, docker: true do
|
|
29
29
|
expect(mbox_content(folder)).to eq(messages_as_mbox)
|
30
30
|
end
|
31
31
|
|
32
|
-
|
32
|
+
describe "IMAP metadata" do
|
33
33
|
let(:imap_metadata) { imap_parsed(folder) }
|
34
34
|
let(:folder_uids) { server_uids(folder) }
|
35
35
|
|
@@ -52,14 +52,15 @@ RSpec.describe "backup", type: :feature, docker: true do
|
|
52
52
|
context "when uid_validity does not match" do
|
53
53
|
let(:new_name) { "NEWNAME" }
|
54
54
|
let(:email3) { send_email folder, msg3 }
|
55
|
+
let(:original_folder_uid_validity) { server_uid_validity(folder) }
|
55
56
|
let!(:pre) do
|
56
57
|
server_create_folder folder
|
57
58
|
email3
|
58
|
-
|
59
|
+
original_folder_uid_validity
|
59
60
|
connection.run_backup
|
60
61
|
server_rename_folder folder, new_name
|
61
62
|
end
|
62
|
-
let(:renamed_folder) { folder + "." +
|
63
|
+
let(:renamed_folder) { folder + "." + original_folder_uid_validity.to_s }
|
63
64
|
|
64
65
|
after do
|
65
66
|
server_delete_folder new_name
|
@@ -81,7 +82,7 @@ RSpec.describe "backup", type: :feature, docker: true do
|
|
81
82
|
end
|
82
83
|
|
83
84
|
it "moves the old backup to a uniquely named directory" do
|
84
|
-
renamed = folder + "." +
|
85
|
+
renamed = folder + "." + original_folder_uid_validity.to_s + ".1"
|
85
86
|
expect(mbox_content(renamed)).to eq(message_as_mbox_entry(msg3))
|
86
87
|
end
|
87
88
|
end
|
@@ -3,12 +3,12 @@ module BackupDirectoryHelpers
|
|
3
3
|
from = fixture("connection")[:username]
|
4
4
|
subject = options[:subject]
|
5
5
|
body = options[:body]
|
6
|
-
body_and_headers =
|
7
|
-
From: #{from}
|
8
|
-
Subject: #{subject}
|
6
|
+
body_and_headers = <<~BODY
|
7
|
+
From: #{from}
|
8
|
+
Subject: #{subject}
|
9
9
|
|
10
|
-
#{body}
|
11
|
-
|
10
|
+
#{body}
|
11
|
+
BODY
|
12
12
|
|
13
13
|
"From #{from}\n#{body_and_headers}\n"
|
14
14
|
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
module EmailServerHelpers
|
2
|
-
REQUESTED_ATTRIBUTES =
|
3
|
-
DEFAULT_EMAIL = "address@example.org"
|
2
|
+
REQUESTED_ATTRIBUTES = %w(RFC822 FLAGS INTERNALDATE).freeze
|
3
|
+
DEFAULT_EMAIL = "address@example.org".freeze
|
4
4
|
|
5
5
|
def send_email(folder, options)
|
6
6
|
message = message_as_server_message(options)
|
@@ -11,13 +11,14 @@ module EmailServerHelpers
|
|
11
11
|
from = options[:from] || DEFAULT_EMAIL
|
12
12
|
subject = options[:subject]
|
13
13
|
body = options[:body]
|
14
|
-
message = <<-EOT.gsub("\n", "\r\n")
|
15
|
-
From: #{from}
|
16
|
-
Subject: #{subject}
|
17
14
|
|
18
|
-
|
15
|
+
<<~MESSAGE.gsub("\n", "\r\n")
|
16
|
+
From: #{from}
|
17
|
+
Subject: #{subject}
|
19
18
|
|
20
|
-
|
19
|
+
#{body}
|
20
|
+
|
21
|
+
MESSAGE
|
21
22
|
end
|
22
23
|
|
23
24
|
def server_messages(folder)
|
@@ -32,8 +33,10 @@ Subject: #{subject}
|
|
32
33
|
|
33
34
|
def server_fetch_email(folder, uid)
|
34
35
|
examine folder
|
36
|
+
|
35
37
|
fetch_data_items = imap.uid_fetch([uid.to_i], REQUESTED_ATTRIBUTES)
|
36
38
|
return nil if fetch_data_items.nil?
|
39
|
+
|
37
40
|
fetch_data_item = fetch_data_items[0]
|
38
41
|
attributes = fetch_data_item.attr
|
39
42
|
attributes["RFC822"].force_encoding("utf-8")
|
@@ -43,7 +46,7 @@ Subject: #{subject}
|
|
43
46
|
def delete_emails(folder)
|
44
47
|
imap.select(folder)
|
45
48
|
uids = imap.uid_search(["ALL"]).sort
|
46
|
-
imap.store(1
|
49
|
+
imap.store(1..uids.size, "+FLAGS", [:Deleted])
|
47
50
|
imap.expunge
|
48
51
|
end
|
49
52
|
|
@@ -3,10 +3,10 @@ shared_context "imap-backup connection" do
|
|
3
3
|
let(:default_connection) { fixture("connection") }
|
4
4
|
let(:backup_folders) { nil }
|
5
5
|
let(:connection_options) do
|
6
|
-
default_connection.merge(
|
6
|
+
default_connection.merge(
|
7
7
|
local_path: local_backup_path,
|
8
8
|
folders: backup_folders
|
9
|
-
|
9
|
+
)
|
10
10
|
end
|
11
11
|
let(:connection) { Imap::Backup::Account::Connection.new(connection_options) }
|
12
12
|
end
|
data/spec/support/fixtures.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
module HighLineTestHelpers
|
2
2
|
def prepare_highline
|
3
|
-
@input =
|
3
|
+
@input = instance_double(IO, eof?: false, gets: "q\n")
|
4
4
|
@output = StringIO.new
|
5
5
|
Imap::Backup::Configuration::Setup.highline = HighLine.new(@input, @output)
|
6
6
|
[@input, @output]
|
@@ -1,73 +1,82 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
msg_bad_from =
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
1
|
+
msg_no_from = <<~NO_FROM
|
2
|
+
Delivered-To: you@example.com
|
3
|
+
From: example <www.example.com>
|
4
|
+
To: FirstName LastName <you@example.com>
|
5
|
+
Subject: Re: no subject
|
6
|
+
NO_FROM
|
7
|
+
|
8
|
+
msg_bad_from = <<~BAD_FROM
|
9
|
+
Delivered-To: you@example.com
|
10
|
+
from: "FirstName LastName (TEXT)" <"TEXT*" <no-reply@example.com>>
|
11
|
+
To: FirstName LastName <you@example.com>
|
12
|
+
Subject: Re: no subject
|
13
|
+
BAD_FROM
|
14
|
+
|
15
|
+
msg_no_from_but_return_path = <<~RETURN_PATH
|
16
|
+
Delivered-To: you@example.com
|
17
|
+
From: example <www.example.com>
|
18
|
+
To: FirstName LastName <you@example.com>
|
19
|
+
Return-Path: <me@example.com>
|
20
|
+
Subject: Re: no subject
|
21
|
+
RETURN_PATH
|
22
|
+
|
23
|
+
msg_no_from_but_sender = <<~NOT_SENDER
|
24
|
+
Delivered-To: you@example.com
|
25
|
+
To: FirstName LastName <you@example.com>
|
26
|
+
Subject: Re: no subject
|
27
|
+
Sender: FistName LastName <me@example.com>
|
28
|
+
NOT_SENDER
|
23
29
|
|
24
30
|
describe Email::Mboxrd::Message do
|
31
|
+
subject { described_class.new(message_body) }
|
32
|
+
|
25
33
|
let(:from) { "me@example.com" }
|
26
34
|
let(:date) { DateTime.new(2012, 12, 13, 18, 23, 45) }
|
27
35
|
let(:message_body) do
|
28
|
-
|
36
|
+
instance_double(String, clone: cloned_message_body, force_encoding: nil)
|
29
37
|
end
|
30
38
|
let(:cloned_message_body) do
|
31
39
|
"Foo\nBar\nFrom at the beginning of the line\n>>From quoted"
|
32
40
|
end
|
33
41
|
let(:msg_good) do
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
42
|
+
<<~GOOD
|
43
|
+
Delivered-To: you@example.com
|
44
|
+
From: Foo <foo@example.com>
|
45
|
+
To: FirstName LastName <you@example.com>
|
46
|
+
Date: #{date.rfc822}
|
47
|
+
Subject: Re: no subject
|
48
|
+
GOOD
|
39
49
|
end
|
40
50
|
|
41
51
|
let(:msg_bad_date) do
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
52
|
+
<<~BAD
|
53
|
+
Delivered-To: you@example.com
|
54
|
+
From: Foo <foo@example.com>
|
55
|
+
To: FirstName LastName <you@example.com>
|
56
|
+
Date: Mon,5 May 2014 08:97:99 GMT
|
57
|
+
Subject: Re: no subject
|
58
|
+
BAD
|
47
59
|
end
|
48
60
|
|
49
|
-
subject { described_class.new(message_body) }
|
50
|
-
|
51
61
|
describe ".from_serialized" do
|
52
62
|
let(:serialized_message) { "From foo@a.com\n#{imap_message}" }
|
53
63
|
let(:imap_message) { "Delivered-To: me@example.com\nFrom Me\n" }
|
54
|
-
|
55
|
-
before { @result = described_class.from_serialized(serialized_message) }
|
64
|
+
let!(:result) { described_class.from_serialized(serialized_message) }
|
56
65
|
|
57
66
|
it "returns the message" do
|
58
|
-
expect(
|
67
|
+
expect(result).to be_a(described_class)
|
59
68
|
end
|
60
69
|
|
61
70
|
it "removes one level of > before From" do
|
62
|
-
expect(
|
71
|
+
expect(result.supplied_body).to eq(imap_message)
|
63
72
|
end
|
64
73
|
end
|
65
74
|
|
66
75
|
context "#to_serialized" do
|
67
|
-
let(:mail) {
|
76
|
+
let(:mail) { instance_double(Mail::Message, from: [from], date: date) }
|
68
77
|
|
69
78
|
before do
|
70
|
-
allow(Mail).to receive(:new).with(cloned_message_body)
|
79
|
+
allow(Mail).to receive(:new).with(cloned_message_body) { mail }
|
71
80
|
end
|
72
81
|
|
73
82
|
it "does not modify the message" do
|
@@ -113,7 +122,7 @@ Subject: Re: no subject|
|
|
113
122
|
end
|
114
123
|
end
|
115
124
|
|
116
|
-
context "when original message 'from' is a
|
125
|
+
context "when original message 'from' is not a well-formed address" do
|
117
126
|
let(:message_body) { msg_bad_from }
|
118
127
|
|
119
128
|
it "'from' is empty string" do
|