imap-backup 5.0.0 → 6.0.0.rc2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +8 -7
  3. data/bin/imap-backup +4 -0
  4. data/docs/development.md +10 -4
  5. data/imap-backup.gemspec +2 -7
  6. data/lib/cli_coverage.rb +18 -0
  7. data/lib/imap/backup/account/connection.rb +7 -11
  8. data/lib/imap/backup/account/folder.rb +0 -16
  9. data/lib/imap/backup/account.rb +31 -11
  10. data/lib/imap/backup/cli/folders.rb +3 -3
  11. data/lib/imap/backup/cli/migrate.rb +3 -3
  12. data/lib/imap/backup/cli/restore.rb +20 -4
  13. data/lib/imap/backup/cli/utils.rb +2 -2
  14. data/lib/imap/backup/cli.rb +6 -7
  15. data/lib/imap/backup/configuration.rb +1 -11
  16. data/lib/imap/backup/downloader.rb +13 -9
  17. data/lib/imap/backup/serializer/directory.rb +37 -0
  18. data/lib/imap/backup/serializer/imap.rb +120 -0
  19. data/lib/imap/backup/serializer/mbox.rb +23 -94
  20. data/lib/imap/backup/serializer/mbox_enumerator.rb +2 -0
  21. data/lib/imap/backup/serializer.rb +180 -3
  22. data/lib/imap/backup/setup/account.rb +52 -29
  23. data/lib/imap/backup/setup/helpers.rb +1 -1
  24. data/lib/imap/backup/thunderbird/mailbox_exporter.rb +1 -1
  25. data/lib/imap/backup/version.rb +2 -2
  26. data/lib/imap/backup.rb +0 -1
  27. data/spec/features/backup_spec.rb +22 -29
  28. data/spec/features/restore_spec.rb +8 -6
  29. data/spec/features/support/aruba.rb +12 -3
  30. data/spec/features/support/backup_directory.rb +0 -4
  31. data/spec/features/support/email_server.rb +0 -1
  32. data/spec/spec_helper.rb +4 -9
  33. data/spec/unit/imap/backup/account/connection_spec.rb +36 -8
  34. data/spec/unit/imap/backup/account/folder_spec.rb +18 -16
  35. data/spec/unit/imap/backup/account_spec.rb +246 -0
  36. data/spec/unit/imap/backup/cli/accounts_spec.rb +12 -1
  37. data/spec/unit/imap/backup/cli/backup_spec.rb +19 -0
  38. data/spec/unit/imap/backup/cli/folders_spec.rb +39 -0
  39. data/spec/unit/imap/backup/cli/local_spec.rb +26 -7
  40. data/spec/unit/imap/backup/cli/migrate_spec.rb +80 -0
  41. data/spec/unit/imap/backup/cli/restore_spec.rb +67 -0
  42. data/spec/unit/imap/backup/cli/setup_spec.rb +17 -0
  43. data/spec/unit/imap/backup/cli/utils_spec.rb +68 -5
  44. data/spec/unit/imap/backup/cli_spec.rb +93 -0
  45. data/spec/unit/imap/backup/client/apple_mail_spec.rb +9 -0
  46. data/spec/unit/imap/backup/configuration_spec.rb +2 -2
  47. data/spec/unit/imap/backup/downloader_spec.rb +60 -8
  48. data/spec/unit/imap/backup/logger_spec.rb +1 -1
  49. data/spec/unit/imap/backup/migrator_spec.rb +1 -1
  50. data/spec/unit/imap/backup/sanitizer_spec.rb +42 -0
  51. data/spec/unit/imap/backup/serializer/directory_spec.rb +37 -0
  52. data/spec/unit/imap/backup/serializer/imap_spec.rb +218 -0
  53. data/spec/unit/imap/backup/serializer/mbox_spec.rb +62 -183
  54. data/spec/unit/imap/backup/serializer_spec.rb +296 -0
  55. data/spec/unit/imap/backup/setup/account_spec.rb +120 -25
  56. data/spec/unit/imap/backup/setup/helpers_spec.rb +15 -0
  57. data/spec/unit/imap/backup/thunderbird/mailbox_exporter_spec.rb +116 -0
  58. data/spec/unit/imap/backup/uploader_spec.rb +1 -1
  59. data/spec/unit/retry_on_error_spec.rb +34 -0
  60. metadata +44 -37
  61. data/lib/imap/backup/serializer/mbox_store.rb +0 -217
  62. data/lib/thunderbird/install.rb +0 -16
  63. data/lib/thunderbird/local_folder.rb +0 -65
  64. data/lib/thunderbird/profile.rb +0 -30
  65. data/lib/thunderbird/profiles.rb +0 -71
  66. data/lib/thunderbird/subdirectory.rb +0 -93
  67. data/lib/thunderbird/subdirectory_placeholder.rb +0 -21
  68. data/lib/thunderbird.rb +0 -14
  69. data/spec/gather_rspec_coverage.rb +0 -1
  70. data/spec/unit/imap/backup/serializer/mbox_store_spec.rb +0 -329
@@ -0,0 +1,120 @@
1
+ require "json"
2
+
3
+ module Imap::Backup
4
+ class Serializer::Imap
5
+ CURRENT_VERSION = 2
6
+
7
+ attr_reader :folder_path
8
+ attr_reader :loaded
9
+
10
+ def initialize(folder_path)
11
+ @folder_path = folder_path
12
+ @loaded = false
13
+ @uid_validity = nil
14
+ @uids = nil
15
+ end
16
+
17
+ def append(uid)
18
+ uids << uid
19
+ save
20
+ end
21
+
22
+ def exist?
23
+ File.exist?(pathname)
24
+ end
25
+
26
+ def include?(uid)
27
+ uids.include?(uid)
28
+ end
29
+
30
+ def index(uid)
31
+ uids.find_index(uid)
32
+ end
33
+
34
+ def rename(new_path)
35
+ if exist?
36
+ old_pathname = pathname
37
+ @folder_path = new_path
38
+ File.rename(old_pathname, pathname)
39
+ else
40
+ @folder_path = new_path
41
+ end
42
+ end
43
+
44
+ def uid_validity
45
+ ensure_loaded
46
+ @uid_validity
47
+ end
48
+
49
+ def uid_validity=(value)
50
+ ensure_loaded
51
+ @uid_validity = value
52
+ @uids ||= []
53
+ save
54
+ end
55
+
56
+ def uids
57
+ ensure_loaded
58
+ @uids || []
59
+ end
60
+
61
+ def update_uid(old, new)
62
+ index = uids.find_index(old.to_i)
63
+ return if index.nil?
64
+
65
+ uids[index] = new.to_i
66
+ save
67
+ end
68
+
69
+ private
70
+
71
+ def pathname
72
+ "#{folder_path}.imap"
73
+ end
74
+
75
+ def ensure_loaded
76
+ return if loaded
77
+
78
+ data = load
79
+ if data
80
+ @uids = data[:uids].map(&:to_i)
81
+ @uid_validity = data[:uid_validity]
82
+ else
83
+ @uids = []
84
+ @uid_validity = nil
85
+ end
86
+ @loaded = true
87
+ end
88
+
89
+ def load
90
+ return nil if !exist?
91
+
92
+ data = nil
93
+ begin
94
+ content = File.read(pathname)
95
+ data = JSON.parse(content, symbolize_names: true)
96
+ rescue JSON::ParserError
97
+ return nil
98
+ end
99
+
100
+ return nil if !data.key?(:uids)
101
+ return nil if !data[:uids].is_a?(Array)
102
+
103
+ data
104
+ end
105
+
106
+ def save
107
+ ensure_loaded
108
+
109
+ raise "Cannot save metadata without a uid_validity" if !uid_validity
110
+
111
+ data = {
112
+ version: CURRENT_VERSION,
113
+ uid_validity: @uid_validity,
114
+ uids: @uids
115
+ }
116
+ content = data.to_json
117
+ File.open(pathname, "w") { |f| f.write content }
118
+ end
119
+ end
120
+ end
@@ -1,115 +1,44 @@
1
- require "forwardable"
2
-
3
- require "imap/backup/serializer/mbox_store"
4
-
5
1
  module Imap::Backup
6
2
  class Serializer::Mbox
7
- extend Forwardable
8
- def_delegators :store, :mbox_pathname
3
+ attr_reader :folder_path
9
4
 
10
- attr_reader :path
11
- attr_reader :folder
12
-
13
- def initialize(path, folder)
14
- @path = path
15
- @folder = folder
5
+ def initialize(folder_path)
6
+ @folder_path = folder_path
16
7
  end
17
8
 
18
- def apply_uid_validity(value)
19
- case
20
- when store.uid_validity.nil?
21
- store.uid_validity = value
22
- nil
23
- when store.uid_validity == value
24
- # NOOP
25
- nil
26
- else
27
- apply_new_uid_validity value
9
+ def append(message)
10
+ File.open(pathname, "ab") do |file|
11
+ file.write message
28
12
  end
29
13
  end
30
14
 
31
- def force_uid_validity(value)
32
- store.uid_validity = value
15
+ def exist?
16
+ File.exist?(pathname)
33
17
  end
34
18
 
35
- def uids
36
- store.uids
37
- end
38
-
39
- def load(uid)
40
- store.load(uid)
41
- end
42
-
43
- def each_message(uids)
44
- store.each_message(uids)
45
- end
46
-
47
- def save(uid, message)
48
- store.add(uid, message)
49
- end
50
-
51
- def rename(new_name)
52
- @folder = new_name
53
- store.rename new_name
54
- end
19
+ def length
20
+ return nil if !exist?
55
21
 
56
- def update_uid(old, new)
57
- store.update_uid old, new
22
+ File.stat(pathname).size
58
23
  end
59
24
 
60
- private
61
-
62
- def store
63
- @store ||=
64
- begin
65
- create_containing_directory
66
- Serializer::MboxStore.new(path, folder)
67
- end
25
+ def pathname
26
+ "#{folder_path}.mbox"
68
27
  end
69
28
 
70
- def apply_new_uid_validity(value)
71
- digit = 0
72
- new_name = nil
73
- loop do
74
- extra = digit.zero? ? "" : ".#{digit}"
75
- new_name = "#{folder}.#{store.uid_validity}#{extra}"
76
- test_store = Serializer::MboxStore.new(path, new_name)
77
- break if !test_store.exist?
78
-
79
- digit += 1
29
+ def rename(new_path)
30
+ if exist?
31
+ old_pathname = pathname
32
+ @folder_path = new_path
33
+ File.rename(old_pathname, pathname)
34
+ else
35
+ @folder_path = new_path
80
36
  end
81
- rename_store new_name, value
82
- end
83
-
84
- def rename_store(new_name, value)
85
- store.rename new_name
86
- @store = nil
87
- store.uid_validity = value
88
- new_name
89
- end
90
-
91
- def relative_path
92
- File.dirname(folder)
93
37
  end
94
38
 
95
- def containing_directory
96
- File.join(path, relative_path)
97
- end
98
-
99
- def full_path
100
- File.expand_path(containing_directory)
101
- end
102
-
103
- def create_containing_directory
104
- if !File.directory?(full_path)
105
- Utils.make_folder(
106
- path, relative_path, Serializer::DIRECTORY_PERMISSIONS
107
- )
108
- end
109
-
110
- if Utils.mode(full_path) !=
111
- Serializer::DIRECTORY_PERMISSIONS
112
- FileUtils.chmod Serializer::DIRECTORY_PERMISSIONS, full_path
39
+ def rewind(length)
40
+ File.open(pathname, File::RDWR | File::CREAT, 0o644) do |f|
41
+ f.truncate(length)
113
42
  end
114
43
  end
115
44
  end
@@ -1,4 +1,6 @@
1
1
  module Imap::Backup
2
+ class Serializer; end
3
+
2
4
  class Serializer::MboxEnumerator
3
5
  attr_reader :mbox_pathname
4
6
 
@@ -1,6 +1,183 @@
1
+ require "forwardable"
2
+
3
+ require "email/mboxrd/message"
4
+ require "imap/backup/serializer/imap"
5
+ require "imap/backup/serializer/mbox"
6
+ require "imap/backup/serializer/mbox_enumerator"
7
+
1
8
  module Imap::Backup
2
- module Serializer
3
- DIRECTORY_PERMISSIONS = 0o700
4
- FILE_PERMISSIONS = 0o600
9
+ class Serializer
10
+ extend Forwardable
11
+
12
+ def_delegator :mbox, :pathname, :mbox_pathname
13
+ def_delegators :imap, :uid_validity, :uids, :update_uid
14
+
15
+ attr_reader :folder
16
+ attr_reader :path
17
+
18
+ def initialize(path, folder)
19
+ @path = path
20
+ @folder = folder
21
+ end
22
+
23
+ def apply_uid_validity(value)
24
+ case
25
+ when uid_validity.nil?
26
+ imap.uid_validity = value
27
+ nil
28
+ when uid_validity == value
29
+ # NOOP
30
+ nil
31
+ else
32
+ apply_new_uid_validity value
33
+ end
34
+ end
35
+
36
+ def force_uid_validity(value)
37
+ imap.uid_validity = value
38
+ end
39
+
40
+ def append(uid, message)
41
+ raise "Can't add messages without uid_validity" if !imap.uid_validity
42
+
43
+ uid = uid.to_i
44
+ if imap.include?(uid)
45
+ Logger.logger.debug(
46
+ "[#{folder}] message #{uid} already downloaded - skipping"
47
+ )
48
+ return
49
+ end
50
+
51
+ do_append uid, message
52
+ end
53
+
54
+ def load(uid_maybe_string)
55
+ uid = uid_maybe_string.to_i
56
+ message_index = imap.index(uid)
57
+ return nil if message_index.nil?
58
+
59
+ load_nth(message_index)
60
+ end
61
+
62
+ def load_nth(index)
63
+ enumerator = Serializer::MboxEnumerator.new(mbox.pathname)
64
+ enumerator.each.with_index do |raw, i|
65
+ next if i != index
66
+
67
+ return Email::Mboxrd::Message.from_serialized(raw)
68
+ end
69
+ nil
70
+ end
71
+
72
+ def each_message(required_uids)
73
+ return enum_for(:each_message, required_uids) if !block_given?
74
+
75
+ indexes = required_uids.each.with_object({}) do |uid_maybe_string, acc|
76
+ uid = uid_maybe_string.to_i
77
+ index = imap.index(uid)
78
+ acc[index] = uid if index
79
+ end
80
+ enumerator = Serializer::MboxEnumerator.new(mbox.pathname)
81
+ enumerator.each.with_index do |raw, i|
82
+ uid = indexes[i]
83
+ next if !uid
84
+
85
+ yield uid, Email::Mboxrd::Message.from_serialized(raw)
86
+ end
87
+ end
88
+
89
+ def rename(new_name)
90
+ # Initialize so we get memoized instances with the correct folder_path
91
+ mbox
92
+ imap
93
+ @folder = new_name
94
+ ensure_containing_directory
95
+ mbox.rename folder_path
96
+ imap.rename folder_path
97
+ end
98
+
99
+ private
100
+
101
+ def do_append(uid, message)
102
+ mboxrd_message = Email::Mboxrd::Message.new(message)
103
+ initial = mbox.length || 0
104
+ mbox_appended = false
105
+ begin
106
+ mbox.append mboxrd_message.to_serialized
107
+ mbox_appended = true
108
+ imap.append uid
109
+ rescue StandardError => e
110
+ mbox.rewind(initial) if mbox_appended
111
+
112
+ message = <<-ERROR.gsub(/^\s*/m, "")
113
+ [#{folder}] failed to append message #{uid}:
114
+ #{message}. #{e}:
115
+ #{e.backtrace.join("\n")}"
116
+ ERROR
117
+ Logger.logger.warn message
118
+ end
119
+ end
120
+
121
+ def mbox
122
+ @mbox ||=
123
+ begin
124
+ ensure_containing_directory
125
+ Serializer::Mbox.new(folder_path)
126
+ end
127
+ end
128
+
129
+ def imap
130
+ @imap ||=
131
+ begin
132
+ ensure_containing_directory
133
+ Serializer::Imap.new(folder_path)
134
+ end
135
+ end
136
+
137
+ def folder_path
138
+ folder_path_for(path, folder)
139
+ end
140
+
141
+ def folder_path_for(path, folder)
142
+ relative = File.join(path, folder)
143
+ File.expand_path(relative)
144
+ end
145
+
146
+ def ensure_containing_directory
147
+ relative = File.dirname(folder)
148
+ directory = Serializer::Directory.new(path, relative)
149
+ directory.ensure_exists
150
+ end
151
+
152
+ def apply_new_uid_validity(value)
153
+ new_name = rename_existing_folder
154
+ # Clear memoization so we get empty data
155
+ @mbox = nil
156
+ @imap = nil
157
+ imap.uid_validity = value
158
+
159
+ new_name
160
+ end
161
+
162
+ def rename_existing_folder
163
+ digit = 0
164
+ new_name = nil
165
+ loop do
166
+ extra = digit.zero? ? "" : "-#{digit}"
167
+ new_name = "#{folder}-#{imap.uid_validity}#{extra}"
168
+ new_folder_path = folder_path_for(path, new_name)
169
+ test_mbox = Serializer::Mbox.new(new_folder_path)
170
+ test_imap = Serializer::Imap.new(new_folder_path)
171
+ break if !test_mbox.exist? && !test_imap.exist?
172
+
173
+ digit += 1
174
+ end
175
+
176
+ previous = folder
177
+ rename(new_name)
178
+ @folder = previous
179
+
180
+ new_name
181
+ end
5
182
  end
6
183
  end
@@ -26,6 +26,7 @@ module Imap::Backup
26
26
  modify_password menu
27
27
  modify_backup_path menu
28
28
  choose_folders menu
29
+ modify_multi_fetch_size menu
29
30
  modify_server menu
30
31
  modify_connection_options menu
31
32
  test_connection menu
@@ -37,21 +38,30 @@ module Imap::Backup
37
38
 
38
39
  def header(menu)
39
40
  modified = account.modified? ? "*" : ""
40
- connection_options =
41
- if account.connection_options
42
- escaped =
43
- JSON.generate(account.connection_options).
44
- gsub('"', '\"')
45
- "\n connection options #{escaped}"
46
- end
41
+
42
+ if account.multi_fetch_size > 1
43
+ multi_fetch_size = "\nmulti-fetch #{account.multi_fetch_size}"
44
+ end
45
+
46
+ if account.connection_options
47
+ escaped =
48
+ JSON.generate(account.connection_options)
49
+ connection_options =
50
+ "\nconnection options '#{escaped}'"
51
+ space = " " * 12
52
+ else
53
+ connection_options = nil
54
+ space = " " * 4
55
+ end
56
+
47
57
  menu.header = <<~HEADER.chomp
48
58
  #{helpers.title_prefix} Account#{modified}
49
59
 
50
- email #{account.username}
51
- password #{masked_password}
52
- path #{account.local_path}
53
- folders #{folders.map { |f| f[:name] }.join(', ')}
54
- server #{account.server}#{connection_options}
60
+ email #{space}#{account.username}
61
+ password#{space}#{masked_password}
62
+ path #{space}#{account.local_path}
63
+ folders #{space}#{folders.map { |f| f[:name] }.join(', ')}#{multi_fetch_size}
64
+ server #{space}#{account.server}#{connection_options}
55
65
 
56
66
  Choose an action
57
67
  HEADER
@@ -70,12 +80,10 @@ module Imap::Backup
70
80
  )
71
81
  else
72
82
  account.username = username
73
- # rubocop:disable Style/IfUnlessModifier
74
83
  default = default_server(username)
75
84
  if default && (account.server.nil? || (account.server == ""))
76
85
  account.server = default
77
86
  end
78
- # rubocop:enable Style/IfUnlessModifier
79
87
  end
80
88
  end
81
89
  end
@@ -88,20 +96,6 @@ module Imap::Backup
88
96
  end
89
97
  end
90
98
 
91
- def modify_server(menu)
92
- menu.choice("modify server") do
93
- server = highline.ask("server: ")
94
- account.server = server if !server.nil?
95
- end
96
- end
97
-
98
- def modify_connection_options(menu)
99
- menu.choice("modify connection options") do
100
- connection_options = highline.ask("connections options (as JSON): ")
101
- account.connection_options = connection_options if !connection_options.nil?
102
- end
103
- end
104
-
105
99
  def path_modification_validator(path)
106
100
  same = config.accounts.find do |a|
107
101
  a.username != account.username && a.local_path == path
@@ -130,6 +124,35 @@ module Imap::Backup
130
124
  end
131
125
  end
132
126
 
127
+ def modify_multi_fetch_size(menu)
128
+ menu.choice("modify multi-fetch size (number of emails to fetch at a time)") do
129
+ size = highline.ask("size: ")
130
+ int = size.to_i
131
+ account.multi_fetch_size = int if int.positive?
132
+ end
133
+ end
134
+
135
+ def modify_server(menu)
136
+ menu.choice("modify server") do
137
+ server = highline.ask("server: ")
138
+ account.server = server if !server.nil?
139
+ end
140
+ end
141
+
142
+ def modify_connection_options(menu)
143
+ menu.choice("modify connection options") do
144
+ connection_options = highline.ask("connections options (as JSON): ")
145
+ if !connection_options.nil?
146
+ begin
147
+ account.connection_options = connection_options
148
+ rescue JSON::ParserError
149
+ Kernel.puts "Malformed JSON, please try again"
150
+ highline.ask "Press a key "
151
+ end
152
+ end
153
+ end
154
+ end
155
+
133
156
  def test_connection(menu)
134
157
  menu.choice("test connection") do
135
158
  result = Setup::ConnectionTester.new(account).test
@@ -141,7 +164,7 @@ module Imap::Backup
141
164
  def delete_account(menu)
142
165
  menu.choice("delete") do
143
166
  if highline.agree("Are you sure? (y/n) ")
144
- account.mark_for_deletion!
167
+ account.mark_for_deletion
145
168
  throw :done
146
169
  end
147
170
  end
@@ -9,7 +9,7 @@ module Imap::Backup
9
9
  end
10
10
 
11
11
  def version
12
- Version::VERSION
12
+ VERSION
13
13
  end
14
14
  end
15
15
  end
@@ -19,7 +19,7 @@ module Imap::Backup
19
19
 
20
20
  def run
21
21
  local_folder_ok = local_folder.set_up
22
- return if !local_folder_ok
22
+ return false if !local_folder_ok
23
23
 
24
24
  if local_folder.msf_exists?
25
25
  if force
@@ -1,9 +1,9 @@
1
1
  module Imap; end
2
2
 
3
3
  module Imap::Backup
4
- MAJOR = 5
4
+ MAJOR = 6
5
5
  MINOR = 0
6
6
  REVISION = 0
7
- PRE = nil
7
+ PRE = "rc2".freeze
8
8
  VERSION = [MAJOR, MINOR, REVISION, PRE].compact.map(&:to_s).join(".")
9
9
  end
data/lib/imap/backup.rb CHANGED
@@ -8,7 +8,6 @@ require "imap/backup/downloader"
8
8
  require "imap/backup/logger"
9
9
  require "imap/backup/uploader"
10
10
  require "imap/backup/serializer"
11
- require "imap/backup/serializer/mbox"
12
11
  require "imap/backup/setup"
13
12
  require "imap/backup/setup/account"
14
13
  require "imap/backup/setup/asker"