rgrove-larch 1.0.2 → 1.0.2.1
Sign up to get free protection for your applications and to get access to all the features.
- data/HISTORY +8 -0
- data/bin/larch +7 -6
- data/lib/larch/db/account.rb +12 -0
- data/lib/larch/db/mailbox.rb +12 -0
- data/lib/larch/db/message.rb +6 -0
- data/lib/larch/db/migrate/001_create_schema.rb +42 -0
- data/lib/larch/imap/mailbox.rb +280 -81
- data/lib/larch/imap.rb +18 -7
- data/lib/larch/version.rb +1 -1
- data/lib/larch.rb +69 -22
- metadata +30 -6
data/HISTORY
CHANGED
@@ -1,6 +1,14 @@
|
|
1
1
|
Larch History
|
2
2
|
================================================================================
|
3
3
|
|
4
|
+
Version 1.1.0 (git)
|
5
|
+
* Mailbox and message state information is now stored in a local SQLite
|
6
|
+
database, which allows Larch to resync and resume interrupted syncs much
|
7
|
+
more quickly without having to rescan all messages. As a result, SQLite 3 is
|
8
|
+
now a dependency.
|
9
|
+
* Progress information is now displayed regularly while scanning large
|
10
|
+
mailboxes.
|
11
|
+
|
4
12
|
Version 1.0.2 (2009-08-05)
|
5
13
|
* Fixed a bug that caused Larch to try to set the read-only \Recent flag on
|
6
14
|
the destination server.
|
data/bin/larch
CHANGED
@@ -22,7 +22,7 @@ EOS
|
|
22
22
|
opt :from, "URI of the source IMAP server.", :short => '-f', :type => :string, :required => true
|
23
23
|
opt :to, "URI of the destination IMAP server.", :short => '-t', :type => :string, :required => true
|
24
24
|
|
25
|
-
text "\
|
25
|
+
text "\nSync Options:"
|
26
26
|
opt :all, "Copy all folders recursively", :short => :none
|
27
27
|
opt :all_subscribed, "Copy all subscribed folders recursively", :short => :none
|
28
28
|
opt :exclude, "List of mailbox names/patterns that shouldn't be copied", :short => :none, :type => :strings, :multi => true
|
@@ -35,7 +35,8 @@ EOS
|
|
35
35
|
opt :to_user, "Destination server username (default: prompt)", :short => :none, :type => :string
|
36
36
|
|
37
37
|
text "\nGeneral Options:"
|
38
|
-
opt :
|
38
|
+
opt :database, "Specify a non-default message database to use", :short => :none, :default => File.join('~', '.larch', 'larch.db')
|
39
|
+
opt :dry_run, "Don't actually make any changes", :short => '-n'
|
39
40
|
opt :fast_scan, "Use a faster (but less accurate) method to scan mailboxes. This may result in messages being re-copied.", :short => :none
|
40
41
|
opt :max_retries, "Maximum number of times to retry after a recoverable error", :short => :none, :default => 3
|
41
42
|
opt :no_create_folder, "Don't create destination folders that don't already exist", :short => :none
|
@@ -104,10 +105,10 @@ EOS
|
|
104
105
|
uri_to.password ||= CGI.escape(ask("Destination password (#{uri_to.host}): ") {|q| q.echo = false })
|
105
106
|
|
106
107
|
# Go go go!
|
107
|
-
init(
|
108
|
-
options[:
|
109
|
-
|
110
|
-
options[:
|
108
|
+
init(options[:database],
|
109
|
+
:exclude => options[:exclude] ? options[:exclude].flatten : [],
|
110
|
+
:exclude_file => options[:exclude_file],
|
111
|
+
:log_level => options[:verbosity]
|
111
112
|
)
|
112
113
|
|
113
114
|
Net::IMAP.debug = true if @log.level == :insane
|
@@ -0,0 +1,42 @@
|
|
1
|
+
class CreateSchema < Sequel::Migration
|
2
|
+
def down
|
3
|
+
drop_table :accounts, :mailboxes, :messages
|
4
|
+
end
|
5
|
+
|
6
|
+
def up
|
7
|
+
create_table :accounts do
|
8
|
+
primary_key :id
|
9
|
+
text :hostname, :null => false
|
10
|
+
text :username, :null => false
|
11
|
+
|
12
|
+
unique [:hostname, :username]
|
13
|
+
end
|
14
|
+
|
15
|
+
create_table :mailboxes do
|
16
|
+
primary_key :id
|
17
|
+
foreign_key :account_id, :table => :accounts
|
18
|
+
text :name, :null => false
|
19
|
+
text :delim, :null => false
|
20
|
+
text :attr, :null => false, :default => ''
|
21
|
+
integer :subscribed, :null => false, :default => 0
|
22
|
+
integer :uidvalidity
|
23
|
+
integer :uidnext
|
24
|
+
|
25
|
+
unique [:account_id, :name, :uidvalidity]
|
26
|
+
end
|
27
|
+
|
28
|
+
create_table :messages do
|
29
|
+
primary_key :id
|
30
|
+
foreign_key :mailbox_id, :table => :mailboxes
|
31
|
+
integer :uid, :null => false
|
32
|
+
text :guid, :null => false
|
33
|
+
text :message_id
|
34
|
+
integer :rfc822_size, :null => false
|
35
|
+
integer :internaldate, :null => false
|
36
|
+
text :flags, :null => false, :default => ''
|
37
|
+
|
38
|
+
index :guid
|
39
|
+
unique [:mailbox_id, :uid]
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
data/lib/larch/imap/mailbox.rb
CHANGED
@@ -2,7 +2,10 @@ module Larch; class IMAP
|
|
2
2
|
|
3
3
|
# Represents an IMAP mailbox.
|
4
4
|
class Mailbox
|
5
|
-
attr_reader :attr, :delim, :imap, :name, :state
|
5
|
+
attr_reader :attr, :db_mailbox, :delim, :imap, :name, :state, :subscribed
|
6
|
+
|
7
|
+
# Maximum number of message headers to fetch with a single IMAP command.
|
8
|
+
FETCH_BLOCK_SIZE = 1024
|
6
9
|
|
7
10
|
# Regex to capture a Message-Id header.
|
8
11
|
REGEX_MESSAGE_ID = /message-id\s*:\s*(\S+)/i
|
@@ -15,12 +18,11 @@ class Mailbox
|
|
15
18
|
|
16
19
|
@imap = imap
|
17
20
|
@name = name
|
21
|
+
@name_utf7 = Net::IMAP.encode_utf7(@name)
|
18
22
|
@delim = delim
|
19
23
|
@subscribed = subscribed
|
20
24
|
@attr = attr.flatten
|
21
25
|
|
22
|
-
@ids = {}
|
23
|
-
@last_id = 0
|
24
26
|
@last_scan = nil
|
25
27
|
@mutex = Mutex.new
|
26
28
|
|
@@ -28,6 +30,23 @@ class Mailbox
|
|
28
30
|
# open and read-only), or :selected (mailbox open and read-write).
|
29
31
|
@state = :closed
|
30
32
|
|
33
|
+
# Create/update this mailbox in the database.
|
34
|
+
mb_data = {
|
35
|
+
:name => @name,
|
36
|
+
:delim => @delim,
|
37
|
+
:attr => @attr.map{|a| a.to_s }.join(','),
|
38
|
+
:subscribed => @subscribed ? 1 : 0
|
39
|
+
}
|
40
|
+
|
41
|
+
@db_mailbox = imap.db_account.mailboxes_dataset.filter(:name => @name).first
|
42
|
+
|
43
|
+
if @db_mailbox
|
44
|
+
@db_mailbox.update(mb_data)
|
45
|
+
else
|
46
|
+
@db_mailbox = Database::Mailbox.create(mb_data)
|
47
|
+
imap.db_account.add_mailbox(@db_mailbox)
|
48
|
+
end
|
49
|
+
|
31
50
|
# Create private convenience methods (debug, info, warn, etc.) to make
|
32
51
|
# logging easier.
|
33
52
|
Logger::LEVELS.each_key do |level|
|
@@ -46,68 +65,71 @@ class Mailbox
|
|
46
65
|
# +false+ if the message already exists in the mailbox.
|
47
66
|
def append(message)
|
48
67
|
raise ArgumentError, "must provide a Larch::IMAP::Message object" unless message.is_a?(Larch::IMAP::Message)
|
49
|
-
return false if
|
68
|
+
return false if has_guid?(message.guid)
|
50
69
|
|
51
70
|
@imap.safely do
|
52
71
|
unless imap_select(!!@imap.options[:create_mailbox])
|
53
72
|
raise Larch::IMAP::Error, "mailbox cannot contain messages: #{@name}"
|
54
73
|
end
|
55
74
|
|
56
|
-
debug "appending message: #{message.
|
75
|
+
debug "appending message: #{message.guid}"
|
57
76
|
|
58
77
|
# The \Recent flag is read-only, so we shouldn't try to set it at the
|
59
78
|
# destination.
|
60
79
|
flags = message.flags.dup
|
61
80
|
flags.delete(:Recent)
|
62
81
|
|
63
|
-
@imap.conn.append(@
|
82
|
+
@imap.conn.append(@name_utf7, message.rfc822, flags, message.internaldate) unless @imap.options[:dry_run]
|
64
83
|
end
|
65
84
|
|
66
85
|
true
|
67
86
|
end
|
68
87
|
alias << append
|
69
88
|
|
70
|
-
# Iterates through
|
71
|
-
# provided block.
|
72
|
-
def
|
89
|
+
# Iterates through messages in this mailbox, yielding the Larch message guid
|
90
|
+
# of each to the provided block.
|
91
|
+
def each_guid
|
73
92
|
scan
|
74
|
-
@
|
93
|
+
@db_mailbox.messages.each {|db_message| yield db_message.guid }
|
75
94
|
end
|
76
95
|
|
77
|
-
#
|
78
|
-
|
96
|
+
# Returns a Larch::IMAP::Message struct representing the message with the
|
97
|
+
# specified Larch _guid_, or +nil+ if the specified guid was not found in this
|
98
|
+
# mailbox.
|
99
|
+
def fetch(guid, peek = false)
|
79
100
|
scan
|
80
|
-
raise Larch::IMAP::MessageNotFoundError, "message not found: #{message_id}" unless uid = @ids[message_id]
|
81
101
|
|
82
|
-
|
83
|
-
|
84
|
-
|
102
|
+
unless db_message = @db_mailbox.messages_dataset.filter(:guid => guid).first
|
103
|
+
warn "message not found in local db: #{guid}"
|
104
|
+
return nil
|
105
|
+
end
|
85
106
|
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
107
|
+
debug "#{peek ? 'peeking at' : 'fetching'} message: #{guid}"
|
108
|
+
|
109
|
+
imap_uid_fetch([db_message.uid], [(peek ? 'BODY.PEEK[]' : 'BODY[]'), 'FLAGS', 'INTERNALDATE', 'ENVELOPE']) do |fetch_data|
|
110
|
+
data = fetch_data.first
|
111
|
+
check_response_fields(data, 'BODY[]', 'FLAGS', 'INTERNALDATE', 'ENVELOPE')
|
91
112
|
|
92
|
-
|
93
|
-
|
113
|
+
return Message.new(guid, data.attr['ENVELOPE'], data.attr['BODY[]'],
|
114
|
+
data.attr['FLAGS'], Time.parse(data.attr['INTERNALDATE']))
|
115
|
+
end
|
94
116
|
|
95
|
-
|
96
|
-
|
117
|
+
warn "message not found on server: #{guid}"
|
118
|
+
return nil
|
97
119
|
end
|
98
120
|
alias [] fetch
|
99
121
|
|
100
|
-
# Returns +true+ if a message with the specified Larch
|
101
|
-
#
|
102
|
-
def
|
122
|
+
# Returns +true+ if a message with the specified Larch guid exists in this
|
123
|
+
# mailbox, +false+ otherwise.
|
124
|
+
def has_guid?(guid)
|
103
125
|
scan
|
104
|
-
@
|
126
|
+
@db_mailbox.messages_dataset.filter(:guid => guid).count > 0
|
105
127
|
end
|
106
128
|
|
107
129
|
# Gets the number of messages in this mailbox.
|
108
130
|
def length
|
109
131
|
scan
|
110
|
-
@
|
132
|
+
@db_mailbox.messages_dataset.count
|
111
133
|
end
|
112
134
|
alias size length
|
113
135
|
|
@@ -124,52 +146,149 @@ class Mailbox
|
|
124
146
|
# Fetches message headers from this mailbox.
|
125
147
|
def scan
|
126
148
|
return if @last_scan && (Time.now - @last_scan) < SCAN_INTERVAL
|
149
|
+
first_scan = @last_scan.nil?
|
150
|
+
@mutex.synchronize { @last_scan = Time.now }
|
127
151
|
|
152
|
+
# Compare the mailbox's current status with its last known status.
|
128
153
|
begin
|
129
|
-
return unless
|
154
|
+
return unless status = imap_status('MESSAGES', 'UIDNEXT', 'UIDVALIDITY')
|
130
155
|
rescue Error => e
|
131
156
|
return if @imap.options[:create_mailbox]
|
132
157
|
raise
|
133
158
|
end
|
134
159
|
|
135
|
-
|
136
|
-
|
137
|
-
return if last_id == @last_id
|
160
|
+
flag_range = nil
|
161
|
+
full_range = nil
|
138
162
|
|
139
|
-
|
140
|
-
|
163
|
+
if @db_mailbox.uidvalidity && @db_mailbox.uidnext &&
|
164
|
+
status['UIDVALIDITY'] == @db_mailbox.uidvalidity
|
141
165
|
|
142
|
-
|
143
|
-
|
166
|
+
# The UIDVALIDITY is the same as what we saw last time we scanned at this
|
167
|
+
# mailbox, which means that all the existing messages in the database are
|
168
|
+
# still valid. We only need to request headers for new messages.
|
169
|
+
#
|
170
|
+
# If this is the first scan of this mailbox during this Larch session,
|
171
|
+
# then we'll also update the flags of all messages in the mailbox.
|
172
|
+
|
173
|
+
flag_range = 1...@db_mailbox.uidnext if first_scan
|
174
|
+
full_range = @db_mailbox.uidnext...status['UIDNEXT']
|
144
175
|
|
145
|
-
fields = if @imap.options[:fast_scan]
|
146
|
-
['UID', 'RFC822.SIZE', 'INTERNALDATE']
|
147
176
|
else
|
148
|
-
|
177
|
+
|
178
|
+
# The UIDVALIDITY has changed or this is the first time we've scanned this
|
179
|
+
# mailbox (ever). Either way, all existing messages in the database are no
|
180
|
+
# longer valid, so we have to throw them out and re-request everything.
|
181
|
+
|
182
|
+
@db_mailbox.remove_all_messages
|
183
|
+
full_range = 1...status['UIDNEXT']
|
149
184
|
end
|
150
185
|
|
151
|
-
|
152
|
-
|
186
|
+
@db_mailbox.update(:uidvalidity => status['UIDVALIDITY'])
|
187
|
+
|
188
|
+
return unless flag_range || full_range.last - full_range.first > 0
|
189
|
+
|
190
|
+
# Open the mailbox for read-only access.
|
191
|
+
return unless imap_examine
|
192
|
+
|
193
|
+
if flag_range && flag_range.last - flag_range.first > 0
|
194
|
+
info "fetching latest message flags..."
|
195
|
+
|
196
|
+
expected_uids = {}
|
197
|
+
@db_mailbox.messages.each {|db_message| expected_uids[db_message.uid] = true }
|
153
198
|
|
154
|
-
|
155
|
-
|
156
|
-
|
199
|
+
imap_uid_fetch(flag_range, "(UID FLAGS)", 16384) do |fetch_data|
|
200
|
+
Larch.db.transaction do
|
201
|
+
fetch_data.each do |data|
|
202
|
+
check_response_fields(data, 'UID', 'FLAGS')
|
203
|
+
expected_uids.delete(data.attr['UID'])
|
204
|
+
|
205
|
+
@db_mailbox.messages_dataset.filter(:uid => data.attr['UID']).
|
206
|
+
update(:flags => data.attr['FLAGS'].map{|f| f.to_s }.join(','))
|
207
|
+
end
|
208
|
+
end
|
157
209
|
end
|
158
210
|
|
159
|
-
|
160
|
-
|
161
|
-
|
211
|
+
# Any UIDs that are in the database but weren't in the response have been
|
212
|
+
# deleted from the server, so we need to delete them from the database as
|
213
|
+
# well.
|
214
|
+
unless expected_uids.empty?
|
215
|
+
debug "removing #{expected_uids.length} deleted messages from the database..."
|
216
|
+
|
217
|
+
Larch.db.transaction do
|
218
|
+
expected_uids.each do |uid|
|
219
|
+
@db_mailbox.messages_dataset.filter(:uid => uid).destroy
|
220
|
+
end
|
221
|
+
end
|
162
222
|
end
|
223
|
+
end
|
224
|
+
|
225
|
+
if full_range && full_range.last - full_range.first > 0
|
226
|
+
start = @db_mailbox.messages_dataset.count + 1
|
227
|
+
total = status['MESSAGES']
|
228
|
+
fetched = 0
|
229
|
+
progress = 0
|
230
|
+
|
231
|
+
show_progress = total - start > FETCH_BLOCK_SIZE * 4
|
232
|
+
|
233
|
+
info "fetching message headers #{start} through #{total}..."
|
234
|
+
|
235
|
+
begin
|
236
|
+
last_good_uid = nil
|
237
|
+
|
238
|
+
imap_uid_fetch(full_range, "(UID BODY.PEEK[HEADER.FIELDS (MESSAGE-ID)] RFC822.SIZE INTERNALDATE FLAGS)") do |fetch_data|
|
239
|
+
check_response_fields(fetch_data, 'UID', 'RFC822.SIZE', 'INTERNALDATE', 'FLAGS')
|
240
|
+
|
241
|
+
Larch.db.transaction do
|
242
|
+
fetch_data.each do |data|
|
243
|
+
uid = data.attr['UID']
|
244
|
+
|
245
|
+
message_data = {
|
246
|
+
:guid => create_guid(data),
|
247
|
+
:uid => uid,
|
248
|
+
:message_id => parse_message_id(data.attr['BODY[HEADER.FIELDS (MESSAGE-ID)]']),
|
249
|
+
:rfc822_size => data.attr['RFC822.SIZE'].to_i,
|
250
|
+
:internaldate => Time.parse(data.attr['INTERNALDATE']).to_i,
|
251
|
+
:flags => data.attr['FLAGS'].map{|f| f.to_s }.join(',')
|
252
|
+
}
|
253
|
+
|
254
|
+
@db_mailbox.add_message(Database::Message.create(message_data))
|
255
|
+
last_good_uid = uid
|
256
|
+
end
|
257
|
+
|
258
|
+
@db_mailbox.update(:uidnext => last_good_uid + 1)
|
259
|
+
end
|
260
|
+
|
261
|
+
if show_progress
|
262
|
+
fetched += fetch_data.length
|
263
|
+
last_progress = progress
|
264
|
+
progress = ((100 / (total - start).to_f) * fetched).round
|
265
|
+
|
266
|
+
info "#{progress}% complete" if progress > last_progress
|
267
|
+
end
|
268
|
+
end
|
163
269
|
|
164
|
-
|
270
|
+
rescue => e
|
271
|
+
# Set this mailbox's uidnext value to the last known good UID that was
|
272
|
+
# stored in the database, plus 1. This will allow Larch to resume where
|
273
|
+
# the error occurred on the next attempt rather than having to start over.
|
274
|
+
@db_mailbox.update(:uidnext => last_good_uid + 1) if last_good_uid
|
275
|
+
raise
|
276
|
+
end
|
165
277
|
end
|
278
|
+
|
279
|
+
@db_mailbox.update(:uidnext => status['UIDNEXT'])
|
280
|
+
return
|
166
281
|
end
|
167
282
|
|
168
283
|
# Subscribes to this mailbox.
|
169
284
|
def subscribe(force = false)
|
170
|
-
return if subscribed? && !force
|
171
|
-
|
285
|
+
return false if subscribed? && !force
|
286
|
+
|
287
|
+
@imap.safely { @imap.conn.subscribe(@name_utf7) } unless @imap.options[:dry_run]
|
172
288
|
@mutex.synchronize { @subscribed = true }
|
289
|
+
@db_mailbox.update(:subscribed => 1)
|
290
|
+
|
291
|
+
true
|
173
292
|
end
|
174
293
|
|
175
294
|
# Returns +true+ if this mailbox is subscribed, +false+ otherwise.
|
@@ -179,35 +298,55 @@ class Mailbox
|
|
179
298
|
|
180
299
|
# Unsubscribes from this mailbox.
|
181
300
|
def unsubscribe(force = false)
|
182
|
-
return unless subscribed? || force
|
183
|
-
|
301
|
+
return false unless subscribed? || force
|
302
|
+
|
303
|
+
@imap.safely { @imap.conn.unsubscribe(@name_utf7) } unless @imap.options[:dry_run]
|
184
304
|
@mutex.synchronize { @subscribed = false }
|
305
|
+
@db_mailbox.update(:subscribed => 0)
|
306
|
+
|
307
|
+
true
|
185
308
|
end
|
186
309
|
|
187
310
|
private
|
188
311
|
|
189
|
-
#
|
190
|
-
#
|
312
|
+
# Checks the specified Net::IMAP::FetchData object and raises a
|
313
|
+
# Larch::IMAP::Error unless it contains all the specified _fields_.
|
314
|
+
#
|
315
|
+
# _data_ can be a single object or an Array of objects; if it's an Array, then
|
316
|
+
# only the first object in the Array will be checked.
|
317
|
+
def check_response_fields(data, *fields)
|
318
|
+
check_data = data.is_a?(Array) ? data.first : data
|
319
|
+
|
320
|
+
fields.each do |f|
|
321
|
+
raise Error, "required data not in IMAP response: #{f}" unless check_data.attr.has_key?(f)
|
322
|
+
end
|
323
|
+
|
324
|
+
true
|
325
|
+
end
|
326
|
+
|
327
|
+
# Creates a globally unique id suitable for identifying a specific message
|
328
|
+
# on any mail server (we hope) based on the given IMAP FETCH _data_.
|
191
329
|
#
|
192
330
|
# If the given message data includes a valid Message-Id header, then that will
|
193
331
|
# be used to generate an MD5 hash. Otherwise, the hash will be generated based
|
194
332
|
# on the message's RFC822.SIZE and INTERNALDATE.
|
195
|
-
def
|
196
|
-
['
|
197
|
-
|
198
|
-
end
|
199
|
-
|
200
|
-
if data.attr['BODY[HEADER.FIELDS (MESSAGE-ID)]'] =~ REGEX_MESSAGE_ID
|
201
|
-
Digest::MD5.hexdigest($1)
|
333
|
+
def create_guid(data)
|
334
|
+
if message_id = parse_message_id(data.attr['BODY[HEADER.FIELDS (MESSAGE-ID)]'])
|
335
|
+
Digest::MD5.hexdigest(message_id)
|
202
336
|
else
|
337
|
+
check_response_fields(data, 'RFC822.SIZE', 'INTERNALDATE')
|
338
|
+
|
203
339
|
Digest::MD5.hexdigest(sprintf('%d%d', data.attr['RFC822.SIZE'],
|
204
340
|
Time.parse(data.attr['INTERNALDATE']).to_i))
|
205
341
|
end
|
206
342
|
end
|
207
343
|
|
208
|
-
# Examines
|
344
|
+
# Examines this mailbox. If _force_ is true, the mailbox will be examined even
|
209
345
|
# if it is already selected (which isn't necessary unless you want to ensure
|
210
346
|
# that it's in a read-only state).
|
347
|
+
#
|
348
|
+
# Returns +false+ if this mailbox cannot be examined, which may be the case if
|
349
|
+
# the \Noselect attribute is set.
|
211
350
|
def imap_examine(force = false)
|
212
351
|
return false if @attr.include?(:Noselect)
|
213
352
|
return true if @state == :examined || (!force && @state == :selected)
|
@@ -217,7 +356,7 @@ class Mailbox
|
|
217
356
|
@mutex.synchronize { @state = :closed }
|
218
357
|
|
219
358
|
debug "examining mailbox"
|
220
|
-
@imap.conn.examine(@
|
359
|
+
@imap.conn.examine(@name_utf7)
|
221
360
|
|
222
361
|
@mutex.synchronize { @state = :examined }
|
223
362
|
|
@@ -229,18 +368,12 @@ class Mailbox
|
|
229
368
|
return true
|
230
369
|
end
|
231
370
|
|
232
|
-
# Fetches the specified _fields_ for the specified _set_ of message sequence
|
233
|
-
# ids (either a Range or an Array of ids).
|
234
|
-
def imap_fetch(set, fields)
|
235
|
-
@imap.safely do
|
236
|
-
imap_examine
|
237
|
-
@imap.conn.fetch(set, fields)
|
238
|
-
end
|
239
|
-
end
|
240
|
-
|
241
371
|
# Selects the mailbox if it is not already selected. If the mailbox does not
|
242
372
|
# exist and _create_ is +true+, it will be created. Otherwise, a
|
243
373
|
# Larch::IMAP::Error will be raised.
|
374
|
+
#
|
375
|
+
# Returns +false+ if this mailbox cannot be selected, which may be the case if
|
376
|
+
# the \Noselect attribute is set.
|
244
377
|
def imap_select(create = false)
|
245
378
|
return false if @attr.include?(:Noselect)
|
246
379
|
return true if @state == :selected
|
@@ -250,7 +383,7 @@ class Mailbox
|
|
250
383
|
@mutex.synchronize { @state = :closed }
|
251
384
|
|
252
385
|
debug "selecting mailbox"
|
253
|
-
@imap.conn.select(@
|
386
|
+
@imap.conn.select(@name_utf7)
|
254
387
|
|
255
388
|
@mutex.synchronize { @state = :selected }
|
256
389
|
|
@@ -260,7 +393,7 @@ class Mailbox
|
|
260
393
|
info "creating mailbox: #{@name}"
|
261
394
|
|
262
395
|
begin
|
263
|
-
@imap.conn.create(@
|
396
|
+
@imap.conn.create(@name_utf7) unless @imap.options[:dry_run]
|
264
397
|
retry
|
265
398
|
rescue => e
|
266
399
|
raise Error, "unable to create mailbox: #{e.message}"
|
@@ -271,15 +404,81 @@ class Mailbox
|
|
271
404
|
return true
|
272
405
|
end
|
273
406
|
|
274
|
-
#
|
275
|
-
#
|
276
|
-
|
407
|
+
# Sends an IMAP STATUS command and returns the status of the requested
|
408
|
+
# attributes. Supported attributes include:
|
409
|
+
#
|
410
|
+
# - MESSAGES
|
411
|
+
# - RECENT
|
412
|
+
# - UIDNEXT
|
413
|
+
# - UIDVALIDITY
|
414
|
+
# - UNSEEN
|
415
|
+
def imap_status(*attr)
|
277
416
|
@imap.safely do
|
278
|
-
|
279
|
-
|
417
|
+
begin
|
418
|
+
debug "getting mailbox status"
|
419
|
+
@imap.conn.status(@name_utf7, attr)
|
420
|
+
rescue Net::IMAP::NoResponseError => e
|
421
|
+
raise Error, "unable to get status of mailbox: #{e.message}"
|
422
|
+
end
|
423
|
+
end
|
424
|
+
end
|
425
|
+
|
426
|
+
# Fetches the specified _fields_ for the specified _set_ of UIDs, which can be
|
427
|
+
# a number, Range, or Array of UIDs.
|
428
|
+
#
|
429
|
+
# If _set_ is a number, an Array containing a single Net::IMAP::FetchData
|
430
|
+
# object will be yielded to the given block.
|
431
|
+
#
|
432
|
+
# If _set_ is a Range or Array of UIDs, Arrays of up to <i>block_size</i>
|
433
|
+
# Net::IMAP::FetchData objects will be yielded until all requested messages
|
434
|
+
# have been fetched.
|
435
|
+
#
|
436
|
+
# However, if _set_ is a Range with an end value of -1, a single Array
|
437
|
+
# containing all requested messages will be yielded, since it's impossible to
|
438
|
+
# divide an infinite range into finite blocks.
|
439
|
+
def imap_uid_fetch(set, fields, block_size = FETCH_BLOCK_SIZE, &block) # :yields: fetch_data
|
440
|
+
if set.is_a?(Numeric) || (set.is_a?(Range) && set.last < 0)
|
441
|
+
data = @imap.safely do
|
442
|
+
imap_examine
|
443
|
+
@imap.conn.uid_fetch(set, fields)
|
444
|
+
end
|
445
|
+
|
446
|
+
yield data unless data.nil?
|
447
|
+
end
|
448
|
+
|
449
|
+
blocks = []
|
450
|
+
pos = 0
|
451
|
+
|
452
|
+
if set.is_a?(Array)
|
453
|
+
while pos < set.length
|
454
|
+
blocks += set[pos, block_size]
|
455
|
+
pos += block_size
|
456
|
+
end
|
457
|
+
|
458
|
+
elsif set.is_a?(Range)
|
459
|
+
pos = set.first - 1
|
460
|
+
|
461
|
+
while pos < set.last
|
462
|
+
blocks << ((pos + 1)..[set.last, pos += block_size].min)
|
463
|
+
end
|
464
|
+
end
|
465
|
+
|
466
|
+
blocks.each do |block|
|
467
|
+
data = @imap.safely do
|
468
|
+
imap_examine
|
469
|
+
@imap.conn.uid_fetch(block, fields)
|
470
|
+
end
|
471
|
+
|
472
|
+
yield data unless data.nil?
|
280
473
|
end
|
281
474
|
end
|
282
475
|
|
476
|
+
# Parses a Message-Id header out of _str_ and returns it, or +nil+ if _str_
|
477
|
+
# doesn't contain a valid Message-Id header.
|
478
|
+
def parse_message_id(str)
|
479
|
+
return str =~ REGEX_MESSAGE_ID ? $1 : nil
|
480
|
+
end
|
481
|
+
|
283
482
|
end
|
284
483
|
|
285
484
|
end; end
|
data/lib/larch/imap.rb
CHANGED
@@ -6,19 +6,19 @@ module Larch
|
|
6
6
|
# required reading if you're doing anything with IMAP in Ruby:
|
7
7
|
# http://sup.rubyforge.org
|
8
8
|
class IMAP
|
9
|
-
attr_reader :conn, :options
|
9
|
+
attr_reader :conn, :db_account, :options
|
10
10
|
|
11
11
|
# URI format validation regex.
|
12
12
|
REGEX_URI = URI.regexp(['imap', 'imaps'])
|
13
13
|
|
14
14
|
# Larch::IMAP::Message represents a transferable IMAP message which can be
|
15
15
|
# passed between Larch::IMAP instances.
|
16
|
-
Message = Struct.new(:
|
16
|
+
Message = Struct.new(:guid, :envelope, :rfc822, :flags, :internaldate)
|
17
17
|
|
18
18
|
# Initializes a new Larch::IMAP instance that will connect to the specified
|
19
19
|
# IMAP URI.
|
20
20
|
#
|
21
|
-
# In addition to the URI, the following options may
|
21
|
+
# In addition to the URI, the following options may be specified:
|
22
22
|
#
|
23
23
|
# [:create_mailbox]
|
24
24
|
# If +true+, mailboxes that don't already exist will be created if
|
@@ -62,9 +62,14 @@ class IMAP
|
|
62
62
|
|
63
63
|
raise ArgumentError, "must provide a username and password" unless @uri.user && @uri.password
|
64
64
|
|
65
|
-
@conn
|
66
|
-
@mailboxes
|
67
|
-
@mutex
|
65
|
+
@conn = nil
|
66
|
+
@mailboxes = {}
|
67
|
+
@mutex = Mutex.new
|
68
|
+
|
69
|
+
@db_account = Database::Account.find_or_create(
|
70
|
+
:hostname => host,
|
71
|
+
:username => username
|
72
|
+
)
|
68
73
|
|
69
74
|
# Create private convenience methods (debug, info, warn, etc.) to make
|
70
75
|
# logging easier.
|
@@ -170,7 +175,8 @@ class IMAP
|
|
170
175
|
begin
|
171
176
|
yield
|
172
177
|
|
173
|
-
rescue Errno::
|
178
|
+
rescue Errno::ECONNABORTED,
|
179
|
+
Errno::ECONNRESET,
|
174
180
|
Errno::ENOTCONN,
|
175
181
|
Errno::EPIPE,
|
176
182
|
Errno::ETIMEDOUT,
|
@@ -332,6 +338,11 @@ class IMAP
|
|
332
338
|
subscribed.any?{|s| s.name == mb.name}, mb.attr)
|
333
339
|
end
|
334
340
|
end
|
341
|
+
|
342
|
+
# Remove mailboxes that no longer exist from the database.
|
343
|
+
@db_account.mailboxes.each do |db_mailbox|
|
344
|
+
db_mailbox.destroy unless @mailboxes.has_key?(db_mailbox.name)
|
345
|
+
end
|
335
346
|
end
|
336
347
|
|
337
348
|
end
|
data/lib/larch/version.rb
CHANGED
data/lib/larch.rb
CHANGED
@@ -4,10 +4,14 @@ $:.uniq!
|
|
4
4
|
|
5
5
|
require 'cgi'
|
6
6
|
require 'digest/md5'
|
7
|
+
require 'fileutils'
|
7
8
|
require 'net/imap'
|
8
9
|
require 'time'
|
9
10
|
require 'uri'
|
10
11
|
|
12
|
+
require 'sequel'
|
13
|
+
require 'sequel/extensions/migration'
|
14
|
+
|
11
15
|
require 'larch/errors'
|
12
16
|
require 'larch/imap'
|
13
17
|
require 'larch/imap/mailbox'
|
@@ -17,16 +21,23 @@ require 'larch/version'
|
|
17
21
|
module Larch
|
18
22
|
|
19
23
|
class << self
|
20
|
-
attr_reader :log, :exclude
|
24
|
+
attr_reader :db, :log, :exclude
|
21
25
|
|
22
26
|
EXCLUDE_COMMENT = /#.*$/
|
23
27
|
EXCLUDE_REGEX = /^\s*\/(.*)\/\s*/
|
24
28
|
GLOB_PATTERNS = {'*' => '.*', '?' => '.'}
|
29
|
+
LIB_DIR = File.join(File.dirname(File.expand_path(__FILE__)), 'larch')
|
30
|
+
|
31
|
+
def init(database, config = {})
|
32
|
+
@config = {
|
33
|
+
:exclude => [],
|
34
|
+
:log_level => :info
|
35
|
+
}.merge(config)
|
25
36
|
|
26
|
-
|
27
|
-
@
|
37
|
+
@log = Logger.new(@config[:log_level])
|
38
|
+
@db = open_db(database)
|
28
39
|
|
29
|
-
@exclude = exclude.map do |e|
|
40
|
+
@exclude = @config[:exclude].map do |e|
|
30
41
|
if e =~ EXCLUDE_REGEX
|
31
42
|
Regexp.new($1, Regexp::IGNORECASE)
|
32
43
|
else
|
@@ -34,7 +45,7 @@ module Larch
|
|
34
45
|
end
|
35
46
|
end
|
36
47
|
|
37
|
-
load_exclude_file(exclude_file) if exclude_file
|
48
|
+
load_exclude_file(@config[:exclude_file]) if @config[:exclude_file]
|
38
49
|
|
39
50
|
# Stats
|
40
51
|
@copied = 0
|
@@ -59,7 +70,7 @@ module Larch
|
|
59
70
|
mailbox_to = imap_to.mailbox(mailbox_from.name, mailbox_from.delim)
|
60
71
|
mailbox_to.subscribe if mailbox_from.subscribed?
|
61
72
|
|
62
|
-
copy_messages(imap_from, mailbox_from, imap_to, mailbox_to)
|
73
|
+
copy_messages(imap_from, mailbox_from.name, imap_to, mailbox_to.name)
|
63
74
|
end
|
64
75
|
|
65
76
|
rescue => e
|
@@ -79,13 +90,12 @@ module Larch
|
|
79
90
|
@failed = 0
|
80
91
|
@total = 0
|
81
92
|
|
82
|
-
|
83
|
-
|
93
|
+
from_mb_name = imap_from.uri_mailbox || 'INBOX'
|
94
|
+
to_mb_name = imap_to.uri_mailbox || 'INBOX'
|
84
95
|
|
85
|
-
return if excluded?(
|
96
|
+
return if excluded?(from_mb_name) || excluded?(to_mb_name)
|
86
97
|
|
87
|
-
copy_messages(imap_from,
|
88
|
-
imap_to.mailbox(to_name))
|
98
|
+
copy_messages(imap_from, from_mb_name, imap_to, to_mb_name)
|
89
99
|
|
90
100
|
imap_from.disconnect
|
91
101
|
imap_to.disconnect
|
@@ -97,32 +107,70 @@ module Larch
|
|
97
107
|
summary
|
98
108
|
end
|
99
109
|
|
110
|
+
# Opens a connection to the Larch message database, creating it if
|
111
|
+
# necessary.
|
112
|
+
def open_db(database)
|
113
|
+
filename = File.expand_path(database)
|
114
|
+
directory = File.dirname(filename)
|
115
|
+
|
116
|
+
unless File.exist?(directory)
|
117
|
+
FileUtils.mkdir_p(directory)
|
118
|
+
File.chmod(0700, directory)
|
119
|
+
end
|
120
|
+
|
121
|
+
begin
|
122
|
+
db = Sequel.connect("sqlite://#{filename}")
|
123
|
+
db.test_connection
|
124
|
+
rescue => e
|
125
|
+
@log.fatal "unable to open message database: #{e}"
|
126
|
+
abort
|
127
|
+
end
|
128
|
+
|
129
|
+
# Ensure that the database schema is up to date.
|
130
|
+
migration_dir = File.join(LIB_DIR, 'db', 'migrate')
|
131
|
+
|
132
|
+
unless Sequel::Migrator.get_current_migration_version(db) ==
|
133
|
+
Sequel::Migrator.latest_migration_version(migration_dir)
|
134
|
+
begin
|
135
|
+
Sequel::Migrator.apply(db, migration_dir)
|
136
|
+
rescue => e
|
137
|
+
@log.fatal "unable to migrate message database: #{e}"
|
138
|
+
abort
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
require 'larch/db/message'
|
143
|
+
require 'larch/db/mailbox'
|
144
|
+
require 'larch/db/account'
|
145
|
+
|
146
|
+
db
|
147
|
+
end
|
148
|
+
|
100
149
|
def summary
|
101
150
|
@log.info "#{@copied} message(s) copied, #{@failed} failed, #{@total - @copied - @failed} untouched out of #{@total} total"
|
102
151
|
end
|
103
152
|
|
104
153
|
private
|
105
154
|
|
106
|
-
def copy_messages(imap_from,
|
155
|
+
def copy_messages(imap_from, mb_name_from, imap_to, mb_name_to)
|
107
156
|
raise ArgumentError, "imap_from must be a Larch::IMAP instance" unless imap_from.is_a?(IMAP)
|
108
|
-
raise ArgumentError, "mailbox_from must be a Larch::IMAP::Mailbox instance" unless mailbox_from.is_a?(IMAP::Mailbox)
|
109
157
|
raise ArgumentError, "imap_to must be a Larch::IMAP instance" unless imap_to.is_a?(IMAP)
|
110
|
-
raise ArgumentError, "mailbox_to must be a Larch::IMAP::Mailbox instance" unless mailbox_to.is_a?(IMAP::Mailbox)
|
111
158
|
|
112
|
-
return if excluded?(
|
159
|
+
return if excluded?(mb_name_from) || excluded?(mb_name_to)
|
113
160
|
|
114
|
-
@log.info "copying messages from #{imap_from.host}/#{
|
161
|
+
@log.info "copying messages from #{imap_from.host}/#{mb_name_from} to #{imap_to.host}/#{mb_name_to}"
|
115
162
|
|
116
|
-
imap_from.
|
117
|
-
imap_to.connect
|
163
|
+
mailbox_from = imap_from.mailbox(mb_name_from)
|
118
164
|
|
119
165
|
@total += mailbox_from.length
|
120
166
|
|
121
|
-
|
122
|
-
|
167
|
+
mailbox_to = imap_to.mailbox(mb_name_to)
|
168
|
+
|
169
|
+
mailbox_from.each_guid do |guid|
|
170
|
+
next if mailbox_to.has_guid?(guid)
|
123
171
|
|
124
172
|
begin
|
125
|
-
msg = mailbox_from.peek(
|
173
|
+
next unless msg = mailbox_from.peek(guid)
|
126
174
|
|
127
175
|
if msg.envelope.from
|
128
176
|
env_from = msg.envelope.from.first
|
@@ -137,7 +185,6 @@ module Larch
|
|
137
185
|
@copied += 1
|
138
186
|
|
139
187
|
rescue Larch::IMAP::Error => e
|
140
|
-
# TODO: Keep failed message envelopes in a buffer for later output?
|
141
188
|
@failed += 1
|
142
189
|
@log.error e.message
|
143
190
|
next
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rgrove-larch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.2
|
4
|
+
version: 1.0.2.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ryan Grove
|
@@ -9,8 +9,8 @@ autorequire:
|
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
|
12
|
-
date: 2009-
|
13
|
-
default_executable:
|
12
|
+
date: 2009-08-17 00:00:00 -07:00
|
13
|
+
default_executable: larch
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: highline
|
@@ -22,6 +22,26 @@ dependencies:
|
|
22
22
|
- !ruby/object:Gem::Version
|
23
23
|
version: 1.5.0
|
24
24
|
version:
|
25
|
+
- !ruby/object:Gem::Dependency
|
26
|
+
name: sequel
|
27
|
+
type: :runtime
|
28
|
+
version_requirement:
|
29
|
+
version_requirements: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ~>
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 3.3.0
|
34
|
+
version:
|
35
|
+
- !ruby/object:Gem::Dependency
|
36
|
+
name: sqlite3-ruby
|
37
|
+
type: :runtime
|
38
|
+
version_requirement:
|
39
|
+
version_requirements: !ruby/object:Gem::Requirement
|
40
|
+
requirements:
|
41
|
+
- - ~>
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
version: 1.2.5
|
44
|
+
version:
|
25
45
|
- !ruby/object:Gem::Dependency
|
26
46
|
name: trollop
|
27
47
|
type: :runtime
|
@@ -45,12 +65,16 @@ files:
|
|
45
65
|
- LICENSE
|
46
66
|
- README.rdoc
|
47
67
|
- bin/larch
|
48
|
-
- lib/larch.rb
|
68
|
+
- lib/larch/db/account.rb
|
69
|
+
- lib/larch/db/mailbox.rb
|
70
|
+
- lib/larch/db/message.rb
|
71
|
+
- lib/larch/db/migrate/001_create_schema.rb
|
49
72
|
- lib/larch/errors.rb
|
50
|
-
- lib/larch/imap.rb
|
51
73
|
- lib/larch/imap/mailbox.rb
|
74
|
+
- lib/larch/imap.rb
|
52
75
|
- lib/larch/logger.rb
|
53
76
|
- lib/larch/version.rb
|
77
|
+
- lib/larch.rb
|
54
78
|
has_rdoc: false
|
55
79
|
homepage: http://github.com/rgrove/larch/
|
56
80
|
licenses:
|
@@ -76,7 +100,7 @@ requirements: []
|
|
76
100
|
rubyforge_project:
|
77
101
|
rubygems_version: 1.3.5
|
78
102
|
signing_key:
|
79
|
-
specification_version:
|
103
|
+
specification_version: 3
|
80
104
|
summary: Larch syncs messages from one IMAP server to another. Awesomely.
|
81
105
|
test_files: []
|
82
106
|
|