domma-cms 1.28.3 → 1.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -0
- package/admin/js/lib/contacts-arrange.js +1 -1
- package/docs/DommaFeatures.md +5 -3
- package/package.json +4 -1
- package/plugins/_lib/admin/mail/body-menu.js +2 -0
- package/plugins/_lib/admin/mail/cms-host.js +22 -0
- package/plugins/_lib/admin/mail/compose-window.js +1027 -1021
- package/plugins/_lib/admin/mail/contacts.js +10 -8
- package/plugins/_lib/admin/mail/diagnostics-section.js +2 -0
- package/plugins/_lib/admin/mail/drag-drop.js +46 -0
- package/plugins/_lib/admin/mail/folder-tree.js +2 -0
- package/plugins/_lib/admin/mail/font-controls.js +6 -4
- package/plugins/_lib/admin/mail/fonts.js +1 -1
- package/plugins/_lib/admin/mail/host-context.js +87 -0
- package/plugins/_lib/admin/mail/identity.js +2 -0
- package/plugins/_lib/admin/mail/image-senders-section.js +2 -0
- package/plugins/_lib/admin/mail/keyboard.js +2 -0
- package/plugins/_lib/admin/mail/mail.css +1 -1
- package/plugins/_lib/admin/mail/mail.html +4 -0
- package/plugins/_lib/admin/mail/message-actions.js +2 -0
- package/plugins/_lib/admin/mail/oauth-panel.js +2 -0
- package/plugins/_lib/admin/mail/panes.js +2 -0
- package/plugins/_lib/admin/mail/reader-kit.js +2 -0
- package/plugins/_lib/admin/mail/reader-runtime.js +6534 -0
- package/plugins/_lib/admin/mail/reader-view.js +9 -6380
- package/plugins/_lib/admin/mail/resizable.js +2 -0
- package/plugins/_lib/admin/mail/restore-spam.js +13 -0
- package/plugins/_lib/admin/mail/rules.js +14 -0
- package/plugins/_lib/admin/mail/scheduling.js +4 -1
- package/plugins/_lib/admin/mail/section-kit.js +4 -1
- package/plugins/_lib/admin/mail/security-note.js +2 -0
- package/plugins/_lib/admin/mail/settings-section.js +11 -10
- package/plugins/_lib/admin/mail/templates.js +5 -2
- package/plugins/_lib/admin/mail/threads.js +2 -0
- package/plugins/_lib/admin/mail/vacation.js +16 -2
- package/plugins/_lib/admin/mail/when-closed.js +24 -2
- package/plugins/_lib/admin/ui/dates.js +2 -0
- package/plugins/_lib/admin/ui/resizable.js +7 -5
- package/plugins/_lib/contacts/mailContact.js +69 -0
- package/plugins/_lib/contacts/model.js +199 -0
- package/plugins/_lib/domma-mail-contacts.lock.json +10 -0
- package/plugins/_lib/domma-mail.lock.json +85 -0
- package/plugins/_lib/games/plays.js +15 -3
- package/plugins/_lib/mail/accountFileStore.js +66 -0
- package/plugins/_lib/mail/accountRepository.js +617 -0
- package/plugins/_lib/mail/accountRoutes.js +130 -0
- package/plugins/_lib/mail/accounts.js +66 -940
- package/plugins/_lib/mail/automationLock.js +10 -0
- package/plugins/_lib/mail/bodyTokenProvider.js +36 -0
- package/plugins/_lib/mail/bodyTokens.js +5 -99
- package/plugins/_lib/mail/compose.js +5 -3
- package/plugins/_lib/mail/defaults.js +2 -0
- package/plugins/_lib/mail/envelope.js +3 -1
- package/plugins/_lib/mail/folderActions.js +83 -0
- package/plugins/_lib/mail/folderRuntime.js +109 -0
- package/plugins/_lib/mail/folders.js +4 -9
- package/plugins/_lib/mail/imapPool.js +11 -389
- package/plugins/_lib/mail/imapPoolRuntime.js +396 -0
- package/plugins/_lib/mail/mbox.js +2 -0
- package/plugins/_lib/mail/message-action.js +2 -0
- package/plugins/_lib/mail/messageSecurity.js +4 -111
- package/plugins/_lib/mail/messageSecurityProvider.js +116 -0
- package/plugins/_lib/mail/networkAddresses.js +77 -0
- package/plugins/_lib/mail/networkPolicy.js +62 -0
- package/plugins/_lib/mail/oauth.js +3 -451
- package/plugins/_lib/mail/oauthAccounts.js +178 -0
- package/plugins/_lib/mail/oauthRouteRuntime.js +282 -0
- package/plugins/_lib/mail/oauthRoutes.js +7 -289
- package/plugins/_lib/mail/oauthRuntime.js +456 -0
- package/plugins/_lib/mail/pollSchedule.js +2 -0
- package/plugins/_lib/mail/poller.js +4 -136
- package/plugins/_lib/mail/pollerRuntime.js +150 -0
- package/plugins/_lib/mail/priority.js +2 -0
- package/plugins/_lib/mail/publicMailbox.js +27 -0
- package/plugins/_lib/mail/readRoutes.js +14 -827
- package/plugins/_lib/mail/readRuntime.js +701 -0
- package/plugins/_lib/mail/render.js +2 -0
- package/plugins/_lib/mail/ruleRunner.js +27 -10
- package/plugins/_lib/mail/scheduler.js +11 -279
- package/plugins/_lib/mail/schedulerRuntime.js +259 -0
- package/plugins/_lib/mail/secretProvider.js +112 -0
- package/plugins/_lib/mail/secretbox.js +12 -225
- package/plugins/_lib/mail/send.js +21 -537
- package/plugins/_lib/mail/sieve.js +211 -0
- package/plugins/_lib/mail/sieveHost.js +24 -0
- package/plugins/_lib/mail/smtpService.js +553 -0
- package/plugins/_lib/mail/snoozeRuntime.js +60 -0
- package/plugins/_lib/mail/store.js +122 -64
- package/plugins/_lib/mail/subscriptions.js +11 -0
- package/plugins/_lib/mail/sync.js +64 -290
- package/plugins/_lib/mail/syncPlan.js +2 -0
- package/plugins/_lib/mail/syncRuntime.js +255 -0
- package/plugins/_lib/mail/syncSelection.js +2 -0
- package/plugins/_lib/mail/unreadBadge.js +2 -0
- package/plugins/_lib/mail/unsubscribe.js +2 -0
- package/plugins/_lib/mail/vacationRunner.js +3 -116
- package/plugins/_lib/mail/vacationRuntime.js +70 -0
- package/plugins/_lib/mail/write.js +44 -13
- package/server/routes/api/contacts.js +2 -196
|
@@ -1,295 +1,69 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Bringing a folder's local mirror up to date.
|
|
3
|
-
*
|
|
4
|
-
* The decision lives in `syncPlan.js`, which is pure and tested on its own.
|
|
5
|
-
* This is the half that talks to IMAP and MongoDB and carries the plan out.
|
|
6
|
-
*
|
|
7
|
-
* ## Still read-only
|
|
8
|
-
*
|
|
9
|
-
* Sync opens mailboxes exactly as the reader does - `EXAMINE`, and fetches
|
|
10
|
-
* through `BODY.PEEK[]` - so mirroring a mailbox never marks anything read.
|
|
11
|
-
* Writing flags is a separate, deliberate operation that does not belong here.
|
|
12
|
-
*
|
|
13
|
-
* ## Envelopes only
|
|
14
|
-
*
|
|
15
|
-
* Nothing in this module fetches a message body. The mirror is what a list
|
|
16
|
-
* view needs; bodies are fetched per view and rendered through the sanitiser,
|
|
17
|
-
* so a compromised store leaks metadata rather than correspondence.
|
|
18
|
-
*
|
|
19
|
-
* @module _lib/mail/sync
|
|
20
|
-
*/
|
|
1
|
+
/** CMS automation hooks for the shared mirror engine. */
|
|
21
2
|
import * as accounts from './accounts.js';
|
|
3
|
+
import {withMailAutomation} from './automationLock.js';
|
|
4
|
+
import {planActions} from '../admin/mail/rules.js';
|
|
22
5
|
import {applyRules, eligible} from './ruleRunner.js';
|
|
23
6
|
import {runVacation} from './vacationRunner.js';
|
|
24
|
-
import {
|
|
25
|
-
import {
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
}) {
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
// had deliberately left alone.
|
|
83
|
-
plan.fetchFrom
|
|
84
|
-
);
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
if (plan.flagScan !== 'none') {
|
|
88
|
-
result.flagged = await scanFlags(client, store, accountId, path, plan);
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
if (plan.reconcile) {
|
|
92
|
-
result.removed = await reconcile(client, store, accountId, path);
|
|
93
|
-
}
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
await store.saveFolder(accountId, path, {
|
|
97
|
-
uidValidity: String(server.uidValidity),
|
|
98
|
-
uidNext: Number(server.uidNext),
|
|
99
|
-
exists: Number(server.exists),
|
|
100
|
-
highestModseq: server.highestModseq ? String(server.highestModseq) : null,
|
|
101
|
-
lastSyncAt: new Date(now),
|
|
102
|
-
...(plan.mode === 'full' || plan.reconcile ? {lastReconcileAt: now} : {})
|
|
103
|
-
});
|
|
104
|
-
|
|
105
|
-
return result;
|
|
106
|
-
}
|
|
107
|
-
|
|
108
|
-
/**
|
|
109
|
-
* Fetch envelopes for a UID range, writing them in batches.
|
|
110
|
-
*
|
|
111
|
-
* Batched so a mailbox with tens of thousands of messages does not build one
|
|
112
|
-
* enormous array before anything is stored.
|
|
113
|
-
*
|
|
114
|
-
* @param {object} client
|
|
115
|
-
* @param {object} store
|
|
116
|
-
* @param {string} accountId
|
|
117
|
-
* @param {string} path
|
|
118
|
-
* @param {string} range
|
|
119
|
-
* @param {number} batchSize
|
|
120
|
-
* @param {object[]|null} [collect] - records are pushed here when given
|
|
121
|
-
* @param {number} [collectLimit]
|
|
122
|
-
* @param {number} [collectFrom] - only collect UIDs at or above this
|
|
123
|
-
* @returns {Promise<number>}
|
|
124
|
-
*/
|
|
125
|
-
async function fetchRange(
|
|
126
|
-
client, store, accountId, path, range, batchSize,
|
|
127
|
-
collect = null, collectLimit = 200, collectFrom = 0
|
|
128
|
-
) {
|
|
129
|
-
let batch = [];
|
|
130
|
-
let written = 0;
|
|
131
|
-
|
|
132
|
-
for await (const message of client.fetch(range, ENVELOPE_QUERY, {uid: true})) {
|
|
133
|
-
const record = toRecord(message);
|
|
134
|
-
// Collected only when it is genuinely new. See the note at the call
|
|
135
|
-
// site: an IMAP `n:*` range can hand back a UID below `n`.
|
|
136
|
-
if (collect && collect.length < collectLimit && record.uid >= collectFrom) {
|
|
137
|
-
collect.push(record);
|
|
138
|
-
}
|
|
139
|
-
batch.push(record);
|
|
140
|
-
if (batch.length >= batchSize) {
|
|
141
|
-
await store.putMessages(accountId, path, batch);
|
|
142
|
-
written += batch.length;
|
|
143
|
-
batch = [];
|
|
144
|
-
}
|
|
145
|
-
}
|
|
146
|
-
|
|
147
|
-
if (batch.length) {
|
|
148
|
-
await store.putMessages(accountId, path, batch);
|
|
149
|
-
written += batch.length;
|
|
150
|
-
}
|
|
151
|
-
return written;
|
|
152
|
-
}
|
|
153
|
-
|
|
154
|
-
/**
|
|
155
|
-
* Bring stored flags back in line with the server's.
|
|
156
|
-
*
|
|
157
|
-
* @param {object} client
|
|
158
|
-
* @param {object} store
|
|
159
|
-
* @param {string} accountId
|
|
160
|
-
* @param {string} path
|
|
161
|
-
* @param {object} plan
|
|
162
|
-
* @returns {Promise<number>}
|
|
163
|
-
*/
|
|
164
|
-
async function scanFlags(client, store, accountId, path, plan) {
|
|
165
|
-
const query = {uid: true, flags: true};
|
|
166
|
-
const options = {uid: true};
|
|
167
|
-
// CONDSTORE lets the server send only what moved, which on a large folder
|
|
168
|
-
// is the difference between a few rows and all of them.
|
|
169
|
-
if (plan.flagScan === 'changed' && plan.changedSince) {
|
|
170
|
-
options.changedSince = BigInt(plan.changedSince);
|
|
171
|
-
}
|
|
172
|
-
|
|
173
|
-
const updates = [];
|
|
174
|
-
for await (const message of client.fetch('1:*', query, options)) {
|
|
175
|
-
updates.push({uid: message.uid, flags: [...(message.flags ?? [])]});
|
|
176
|
-
}
|
|
177
|
-
return store.setFlags(accountId, path, updates);
|
|
178
|
-
}
|
|
179
|
-
|
|
180
|
-
/**
|
|
181
|
-
* Drop messages the server no longer lists.
|
|
182
|
-
*
|
|
183
|
-
* @param {object} client
|
|
184
|
-
* @param {object} store
|
|
185
|
-
* @param {string} accountId
|
|
186
|
-
* @param {string} path
|
|
187
|
-
* @returns {Promise<number>}
|
|
188
|
-
*/
|
|
189
|
-
async function reconcile(client, store, accountId, path) {
|
|
190
|
-
const onServer = [];
|
|
191
|
-
for await (const message of client.fetch('1:*', {uid: true}, {uid: true})) {
|
|
192
|
-
onServer.push(message.uid);
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
const stored = await store.storedUids(accountId, path);
|
|
196
|
-
const gone = missingUids(stored, onServer);
|
|
197
|
-
return store.removeMessages(accountId, path, gone);
|
|
198
|
-
}
|
|
199
|
-
|
|
200
|
-
/**
|
|
201
|
-
* Mirror every folder one mailbox has chosen.
|
|
202
|
-
*
|
|
203
|
-
* Shared by the manual "sync now" route and the background poller, so the two
|
|
204
|
-
* cannot drift into disagreeing about what a sync pass means.
|
|
205
|
-
*
|
|
206
|
-
* @param {{pool: object, store: object, account: object, config: object}} deps
|
|
207
|
-
* @returns {Promise<{selection: string, folders: Object.<string, object>}>}
|
|
208
|
-
*/
|
|
209
|
-
export async function syncAccount({pool, store, account, config}) {
|
|
210
|
-
const connection = accounts.toConnection(account);
|
|
211
|
-
const poolKey = `${account.userId}:${account.id}`;
|
|
212
|
-
|
|
213
|
-
const list = await pool.withClient(poolKey, connection, client => client.list());
|
|
214
|
-
const folders = list
|
|
215
|
-
.filter(box => !box.flags?.has('\\Noselect'))
|
|
216
|
-
.map(box => ({path: box.path, specialUse: box.specialUse ?? null}));
|
|
217
|
-
|
|
218
|
-
const {paths, source} = resolveSyncFolders(account, folders);
|
|
219
|
-
const results = {};
|
|
220
|
-
|
|
221
|
-
// Switched on, and not in a switched-off group.
|
|
222
|
-
const rules = accounts.rulesToRun
|
|
223
|
-
? accounts.rulesToRun(account)
|
|
224
|
-
: accounts.rulesToPublic(account).filter(rule => rule.enabled);
|
|
225
|
-
const ruleFolders = new Set(rules.length ? accounts.ruleFolders(account) : []);
|
|
226
|
-
|
|
227
|
-
// The away message answers mail arriving in the same folders rules run
|
|
228
|
-
// over - which is the inbox by default. Answering mail that arrived in
|
|
229
|
-
// Sent, or in an archive a rule had just filed it into, is not what
|
|
230
|
-
// "I am away" means.
|
|
231
|
-
const away = accounts.vacationToPublic(account);
|
|
232
|
-
const awayFolders = new Set(away.enabled ? accounts.ruleFolders(account) : []);
|
|
233
|
-
|
|
234
|
-
for (const folderPath of paths) {
|
|
235
|
-
const filed = ruleFolders.has(folderPath) || awayFolders.has(folderPath);
|
|
236
|
-
try {
|
|
237
|
-
results[folderPath] = await pool.withMailbox(poolKey, connection, folderPath,
|
|
238
|
-
client => syncFolder(client, store, {
|
|
239
|
-
accountId: account.id,
|
|
240
|
-
path: folderPath,
|
|
241
|
-
batchSize: config.syncBatchSize,
|
|
242
|
-
collectAdded: filed
|
|
243
|
-
}));
|
|
244
|
-
} catch (err) {
|
|
245
|
-
// One bad folder must not abandon the rest of the mailbox.
|
|
246
|
-
results[folderPath] = {mode: 'error', reason: err.message};
|
|
247
|
-
continue;
|
|
248
|
-
}
|
|
249
|
-
|
|
250
|
-
// Outside the `withMailbox` above, deliberately: filing a message
|
|
251
|
-
// opens its destination on the same connection, and ImapFlow queues
|
|
252
|
-
// mailbox locks per client - doing it inside would wait on a lock
|
|
253
|
-
// this very call is holding.
|
|
254
|
-
if (!filed) continue;
|
|
255
|
-
const candidates = eligible(results[folderPath].addedMessages, results[folderPath], account.rulesFrom ?? null);
|
|
256
|
-
if (!candidates.length) continue;
|
|
257
|
-
|
|
258
|
-
if (ruleFolders.has(folderPath)) {
|
|
259
|
-
try {
|
|
260
|
-
results[folderPath].rules = await applyRules({
|
|
261
|
-
pool, store, account, poolKey, connection,
|
|
262
|
-
folder: folderPath, messages: candidates, rules
|
|
263
|
-
});
|
|
264
|
-
} catch (err) {
|
|
265
|
-
results[folderPath].rules = {error: err.message};
|
|
7
|
+
import {describeError} from './imapPoolRuntime.js';
|
|
8
|
+
import {createSyncService} from './syncRuntime.js';
|
|
9
|
+
export {syncFolder} from './syncRuntime.js';
|
|
10
|
+
|
|
11
|
+
const service = createSyncService({
|
|
12
|
+
accounts,
|
|
13
|
+
withAccountLock: withMailAutomation,
|
|
14
|
+
describeError,
|
|
15
|
+
async prepareAccount({pool, store, account, config, poolKey, connection}) {
|
|
16
|
+
const serverFiltering = accounts.sieveToPublic(account).filtering;
|
|
17
|
+
const rules = serverFiltering ? [] : accounts.rulesToRun(account);
|
|
18
|
+
const ruleFolders = new Set(rules.length ? accounts.ruleFolders(account) : []);
|
|
19
|
+
|
|
20
|
+
// The away message answers mail arriving in the same folders rules run
|
|
21
|
+
// over - which is the inbox by default. Answering mail that arrived in
|
|
22
|
+
// Sent, or in an archive a rule had just filed it into, is not what
|
|
23
|
+
// "I am away" means.
|
|
24
|
+
const away = accounts.vacationToPublic(account);
|
|
25
|
+
const awayBudget = {remaining: 10};
|
|
26
|
+
const awayFolders = new Set(!serverFiltering && away.enabled ? accounts.ruleFolders(account) : []);
|
|
27
|
+
|
|
28
|
+
return {
|
|
29
|
+
collectAdded: path => ruleFolders.has(path) || awayFolders.has(path),
|
|
30
|
+
async afterFolder({path: folderPath, result}) {
|
|
31
|
+
const filed = ruleFolders.has(folderPath) || awayFolders.has(folderPath);
|
|
32
|
+
if (!filed) return;
|
|
33
|
+
const candidates = eligible(result.addedMessages, result, account.rulesFrom ?? null);
|
|
34
|
+
if (!candidates.length) return;
|
|
35
|
+
|
|
36
|
+
if (ruleFolders.has(folderPath)) {
|
|
37
|
+
try {
|
|
38
|
+
result.rules = await applyRules({
|
|
39
|
+
pool, store, account, poolKey, connection,
|
|
40
|
+
folder: folderPath, messages: candidates, rules
|
|
41
|
+
});
|
|
42
|
+
} catch (err) {
|
|
43
|
+
result.rules = {error: err.message};
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
// After the rules, on the same candidates. A message a rule has just
|
|
48
|
+
// filed into Junk should not also get a cheerful note saying when its
|
|
49
|
+
// sender can expect a reply.
|
|
50
|
+
if (awayFolders.has(folderPath)) {
|
|
51
|
+
const junked = new Set(candidates.filter(message => planActions(rules, message).actions.some(action => ['junk', 'delete', 'move'].includes(action.type))).map(message => message.uid));
|
|
52
|
+
try {
|
|
53
|
+
result.away = await runVacation({
|
|
54
|
+
pool, store, account, poolKey, connection,
|
|
55
|
+
smtp: accounts.toSmtpConnection(account),
|
|
56
|
+
folder: folderPath, uidValidity: (await store.getFolder(account.id, folderPath))?.uidValidity, budget: awayBudget,
|
|
57
|
+
messages: candidates.filter(message => !junked.has(message.uid)),
|
|
58
|
+
config,
|
|
59
|
+
addresses: accounts.sendingAddresses(account)
|
|
60
|
+
});
|
|
61
|
+
} catch (err) {
|
|
62
|
+
result.away = {error: err.message};
|
|
63
|
+
}
|
|
64
|
+
}
|
|
266
65
|
}
|
|
267
|
-
}
|
|
268
|
-
|
|
269
|
-
// After the rules, on the same candidates. A message a rule has just
|
|
270
|
-
// filed into Junk should not also get a cheerful note saying when its
|
|
271
|
-
// sender can expect a reply.
|
|
272
|
-
if (awayFolders.has(folderPath)) {
|
|
273
|
-
const junked = new Set(
|
|
274
|
-
(results[folderPath].rules?.actions ?? []).map(action => action.uid)
|
|
275
|
-
);
|
|
276
|
-
try {
|
|
277
|
-
results[folderPath].away = await runVacation({
|
|
278
|
-
pool, store, account, poolKey, connection,
|
|
279
|
-
smtp: accounts.toSmtpConnection(account),
|
|
280
|
-
messages: candidates.filter(message => !junked.has(message.uid)),
|
|
281
|
-
config,
|
|
282
|
-
addresses: accounts.sendingAddresses(account)
|
|
283
|
-
});
|
|
284
|
-
} catch (err) {
|
|
285
|
-
results[folderPath].away = {error: err.message};
|
|
286
|
-
}
|
|
287
|
-
}
|
|
66
|
+
};
|
|
288
67
|
}
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
// reported back to a diagnostics screen as a wall of envelopes.
|
|
292
|
-
for (const result of Object.values(results)) delete result.addedMessages;
|
|
293
|
-
|
|
294
|
-
return {selection: source, folders: results};
|
|
295
|
-
}
|
|
68
|
+
});
|
|
69
|
+
export const syncAccount = service.syncAccount;
|
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
// Source: @domma-mail/core 0.18.0, packages/core/src/mail/syncRuntime.js.
|
|
2
|
+
// Update with domma-mail/scripts/sync-cms.mjs; edit the shared source.
|
|
3
|
+
import {missingUids, planFolderSync} from './syncPlan.js';
|
|
4
|
+
import {resolveSyncFolders} from './syncSelection.js';
|
|
5
|
+
import {ENVELOPE_QUERY, toRecord} from './envelope.js';
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Sync one folder.
|
|
9
|
+
*
|
|
10
|
+
* @param {object} client - a connected ImapFlow, mailbox already locked
|
|
11
|
+
* @param {object} store
|
|
12
|
+
* `collectAdded` asks for the records of whatever was newly fetched, so the
|
|
13
|
+
* caller can run filing rules over them. Off by default and capped, because
|
|
14
|
+
* the whole point of batching is not to hold a folder in memory - a mailbox
|
|
15
|
+
* catching up on ten thousand messages must not build a ten thousand element
|
|
16
|
+
* array to decide none of them matched a rule.
|
|
17
|
+
*
|
|
18
|
+
* @param {{accountId: string, path: string, batchSize?: number, now?: number,
|
|
19
|
+
* collectAdded?: boolean, collectLimit?: number}} options
|
|
20
|
+
* @returns {Promise<{mode: string, reason: string, added: number, flagged: number,
|
|
21
|
+
* removed: number, addedMessages: object[]}>}
|
|
22
|
+
*/
|
|
23
|
+
export async function syncFolder(client, store, {
|
|
24
|
+
accountId, path, batchSize = 500, now = Date.now(),
|
|
25
|
+
collectAdded = false, collectLimit = 200, selectionKey
|
|
26
|
+
}) {
|
|
27
|
+
const mailbox = client.mailbox;
|
|
28
|
+
const server = {
|
|
29
|
+
uidValidity: mailbox.uidValidity,
|
|
30
|
+
uidNext: mailbox.uidNext,
|
|
31
|
+
exists: mailbox.exists,
|
|
32
|
+
highestModseq: mailbox.highestModseq ?? null
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
const stored = await store.getFolder(accountId, path);
|
|
36
|
+
const plan = planFolderSync(stored, server, {now});
|
|
37
|
+
|
|
38
|
+
const result = {mode: plan.mode, reason: plan.reason, added: 0, flagged: 0, removed: 0, addedMessages: []};
|
|
39
|
+
if (plan.mode === 'none') {
|
|
40
|
+
if (selectionKey !== undefined && (stored?.selectionKey !== selectionKey || stored?.active === false)) await store.saveFolder(accountId, path, {selectionKey, active: true});
|
|
41
|
+
return result;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
// Never collected on a full resync: that is the whole folder arriving
|
|
45
|
+
// again, not new mail, and filing it would move an archive.
|
|
46
|
+
const collect = collectAdded && plan.mode !== 'full' ? result.addedMessages : null;
|
|
47
|
+
|
|
48
|
+
if (plan.mode === 'full') {
|
|
49
|
+
// The stored UIDs are meaningless now, so they go before anything new
|
|
50
|
+
// arrives - leaving them would mix two generations of the same folder.
|
|
51
|
+
await store.clearFolder(accountId, path);
|
|
52
|
+
result.added = await fetchRange(client, store, accountId, path, '1:*', batchSize);
|
|
53
|
+
} else {
|
|
54
|
+
if (plan.fetchFrom !== null) {
|
|
55
|
+
result.added = await fetchRange(
|
|
56
|
+
client, store, accountId, path, `${plan.fetchFrom}:*`, batchSize,
|
|
57
|
+
collect, collectLimit,
|
|
58
|
+
// The floor is the point. In an IMAP UID range `*` is the
|
|
59
|
+
// HIGHEST EXISTING UID, not infinity - so `101:*` on a folder
|
|
60
|
+
// whose highest UID is 100 is read as `100:101` and returns
|
|
61
|
+
// UID 100, a message that was already here. Harmless for the
|
|
62
|
+
// mirror, which just rewrites a row it already had; not
|
|
63
|
+
// harmless for rules, which would fire again on mail the user
|
|
64
|
+
// had deliberately left alone.
|
|
65
|
+
plan.fetchFrom
|
|
66
|
+
);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
if (plan.flagScan !== 'none') {
|
|
70
|
+
result.flagged = await scanFlags(client, store, accountId, path, plan);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
if (plan.reconcile) {
|
|
74
|
+
result.removed = await reconcile(client, store, accountId, path);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
await store.saveFolder(accountId, path, {
|
|
79
|
+
...(selectionKey === undefined ? {} : {selectionKey, active: true}),
|
|
80
|
+
uidValidity: String(server.uidValidity),
|
|
81
|
+
uidNext: Number(server.uidNext),
|
|
82
|
+
exists: Number(server.exists),
|
|
83
|
+
highestModseq: server.highestModseq ? String(server.highestModseq) : null,
|
|
84
|
+
lastSyncAt: new Date(now),
|
|
85
|
+
...(plan.mode === 'full' || plan.reconcile ? {lastReconcileAt: now} : {})
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
return result;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Fetch envelopes for a UID range, writing them in batches.
|
|
93
|
+
*
|
|
94
|
+
* Batched so a mailbox with tens of thousands of messages does not build one
|
|
95
|
+
* enormous array before anything is stored.
|
|
96
|
+
*
|
|
97
|
+
* @param {object} client
|
|
98
|
+
* @param {object} store
|
|
99
|
+
* @param {string} accountId
|
|
100
|
+
* @param {string} path
|
|
101
|
+
* @param {string} range
|
|
102
|
+
* @param {number} batchSize
|
|
103
|
+
* @param {object[]|null} [collect] - records are pushed here when given
|
|
104
|
+
* @param {number} [collectLimit]
|
|
105
|
+
* @param {number} [collectFrom] - only collect UIDs at or above this
|
|
106
|
+
* @returns {Promise<number>}
|
|
107
|
+
*/
|
|
108
|
+
async function fetchRange(
|
|
109
|
+
client, store, accountId, path, range, batchSize,
|
|
110
|
+
collect = null, collectLimit = 200, collectFrom = 0
|
|
111
|
+
) {
|
|
112
|
+
let batch = [];
|
|
113
|
+
let written = 0;
|
|
114
|
+
|
|
115
|
+
for await (const message of client.fetch(range, ENVELOPE_QUERY, {uid: true})) {
|
|
116
|
+
const record = toRecord(message);
|
|
117
|
+
// Collected only when it is genuinely new. See the note at the call
|
|
118
|
+
// site: an IMAP `n:*` range can hand back a UID below `n`.
|
|
119
|
+
if (collect && collect.length < collectLimit && record.uid >= collectFrom) {
|
|
120
|
+
collect.push(record);
|
|
121
|
+
}
|
|
122
|
+
batch.push(record);
|
|
123
|
+
if (batch.length >= batchSize) {
|
|
124
|
+
await store.putMessages(accountId, path, batch);
|
|
125
|
+
written += batch.length;
|
|
126
|
+
batch = [];
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
if (batch.length) {
|
|
131
|
+
await store.putMessages(accountId, path, batch);
|
|
132
|
+
written += batch.length;
|
|
133
|
+
}
|
|
134
|
+
return written;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Bring stored flags back in line with the server's.
|
|
139
|
+
*
|
|
140
|
+
* @param {object} client
|
|
141
|
+
* @param {object} store
|
|
142
|
+
* @param {string} accountId
|
|
143
|
+
* @param {string} path
|
|
144
|
+
* @param {object} plan
|
|
145
|
+
* @returns {Promise<number>}
|
|
146
|
+
*/
|
|
147
|
+
async function scanFlags(client, store, accountId, path, plan) {
|
|
148
|
+
const query = {uid: true, flags: true};
|
|
149
|
+
const options = {uid: true};
|
|
150
|
+
// CONDSTORE lets the server send only what moved, which on a large folder
|
|
151
|
+
// is the difference between a few rows and all of them.
|
|
152
|
+
if (plan.flagScan === 'changed' && plan.changedSince) {
|
|
153
|
+
options.changedSince = BigInt(plan.changedSince);
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
const updates = [];
|
|
157
|
+
for await (const message of client.fetch('1:*', query, options)) {
|
|
158
|
+
updates.push({uid: message.uid, flags: [...(message.flags ?? [])]});
|
|
159
|
+
}
|
|
160
|
+
return store.setFlags(accountId, path, updates);
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* Drop messages the server no longer lists.
|
|
165
|
+
*
|
|
166
|
+
* @param {object} client
|
|
167
|
+
* @param {object} store
|
|
168
|
+
* @param {string} accountId
|
|
169
|
+
* @param {string} path
|
|
170
|
+
* @returns {Promise<number>}
|
|
171
|
+
*/
|
|
172
|
+
async function reconcile(client, store, accountId, path) {
|
|
173
|
+
const onServer = [];
|
|
174
|
+
for await (const message of client.fetch('1:*', {uid: true}, {uid: true})) {
|
|
175
|
+
onServer.push(message.uid);
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
const stored = await store.storedUids(accountId, path);
|
|
179
|
+
const gone = missingUids(stored, onServer);
|
|
180
|
+
return store.removeMessages(accountId, path, gone);
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/**
|
|
184
|
+
* Bind mirror coordination to one host's accounts and optional automation.
|
|
185
|
+
* prepareAccount returns collectAdded(path) and afterFolder({path, result}).
|
|
186
|
+
* The latter runs after the IMAP lock is released, so host filing rules can
|
|
187
|
+
* safely open destination folders on the same connection.
|
|
188
|
+
*/
|
|
189
|
+
export function createSyncService({accounts, withAccountLock, prepareAccount = async () => ({}), describeError = () => 'This folder could not be synchronised.'}) {
|
|
190
|
+
const queues = new Map();
|
|
191
|
+
const active = new Set();
|
|
192
|
+
let stopped = false;
|
|
193
|
+
|
|
194
|
+
function exclusive(account, action) {
|
|
195
|
+
if (stopped) return Promise.reject(new Error('Mail synchronisation is stopping.'));
|
|
196
|
+
let run;
|
|
197
|
+
if (withAccountLock) run = Promise.resolve().then(() => withAccountLock(account.id, action));
|
|
198
|
+
else {
|
|
199
|
+
const key = `${account.userId}:${account.id}`;
|
|
200
|
+
run = (queues.get(key) ?? Promise.resolve()).catch(() => {}).then(action);
|
|
201
|
+
queues.set(key, run);
|
|
202
|
+
run.finally(() => { if (queues.get(key) === run) queues.delete(key); }).catch(() => {});
|
|
203
|
+
}
|
|
204
|
+
active.add(run);
|
|
205
|
+
run.finally(() => active.delete(run)).catch(() => {});
|
|
206
|
+
return run;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
async function syncAccount({pool, store, account: requested, config = {}}) {
|
|
210
|
+
return exclusive(requested, async () => {
|
|
211
|
+
if (stopped) return {selection: 'none', folders: {}, skipped: 'stopping'};
|
|
212
|
+
const account = await accounts.getAccount(requested.userId, requested.id);
|
|
213
|
+
if (!account) return {selection: 'none', folders: {}, skipped: 'mailbox-removed'};
|
|
214
|
+
const connection = accounts.toConnection(account);
|
|
215
|
+
const poolKey = `${account.userId}:${account.id}`;
|
|
216
|
+
const list = await pool.withClient(poolKey, connection, client => client.list());
|
|
217
|
+
const folders = list.filter(box => !box.flags?.has('\\Noselect'))
|
|
218
|
+
.map(box => ({path: box.path, specialUse: box.specialUse ?? null}));
|
|
219
|
+
const {paths, source} = resolveSyncFolders(account, folders);
|
|
220
|
+
const automation = await prepareAccount({pool, store, account, config, poolKey, connection, isStopping: () => stopped}) ?? {};
|
|
221
|
+
const results = {};
|
|
222
|
+
if (typeof store.listFolders === 'function') {
|
|
223
|
+
const selected = new Set(paths);
|
|
224
|
+
for (const previous of await store.listFolders(account.id)) {
|
|
225
|
+
if (selected.has(previous.path)) continue;
|
|
226
|
+
await store.clearFolder(account.id, previous.path);
|
|
227
|
+
await store.saveFolder(account.id, previous.path, {lastSyncAt: null, active: false});
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
for (const path of paths) {
|
|
231
|
+
if (stopped) break;
|
|
232
|
+
try {
|
|
233
|
+
results[path] = await pool.withMailbox(poolKey, connection, path, client => syncFolder(client, store, {
|
|
234
|
+
accountId: account.id, path, batchSize: config.syncBatchSize,
|
|
235
|
+
selectionKey: JSON.stringify(account.syncFolders ?? null),
|
|
236
|
+
collectAdded: automation.collectAdded?.(path) === true
|
|
237
|
+
}));
|
|
238
|
+
} catch (error) {
|
|
239
|
+
results[path] = {mode: 'error', reason: describeError(error)};
|
|
240
|
+
continue;
|
|
241
|
+
}
|
|
242
|
+
// Hooks may move mail, so this is deliberately outside withMailbox.
|
|
243
|
+
await automation.afterFolder?.({path, result: results[path]});
|
|
244
|
+
delete results[path].addedMessages;
|
|
245
|
+
}
|
|
246
|
+
return {selection: source, folders: results};
|
|
247
|
+
});
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
return {
|
|
251
|
+
syncAccount,
|
|
252
|
+
exclusive,
|
|
253
|
+
async stop() { stopped = true; await Promise.allSettled([...active]); }
|
|
254
|
+
};
|
|
255
|
+
}
|