domma-cms 0.54.2 → 0.66.0

package/plugins/_lib/mail/store.js

@@ -0,0 +1,1002 @@
+/**
+ * The synchronised message store.
+ *
+ * `mail-reader` keeps nothing: every folder listing and every message is
+ * fetched from IMAP on demand, which is honest but slow and makes search and
+ * threading impossible. `email-pro` keeps a local mirror of the envelopes and
+ * flags so the list opens instantly, search can run locally, and a flag change
+ * can be applied optimistically and reconciled afterwards.
+ *
+ * Only envelopes, flags and structure live here - never message bodies.
+ * Bodies stay on the server and are fetched and rendered per view, so the
+ * store holds metadata about someone's mail rather than the mail itself.
+ *
+ * ## mongodb is optional
+ *
+ * It is in the engine's `optionalDependencies`, so a free-tier site may not
+ * have it installed at all. The driver is therefore imported lazily, inside
+ * `connect()`, and never at module load - importing it at the top would break
+ * every file-mode site the moment this module was referenced.
+ *
+ * @module _lib/mail/store
+ */
+
+/** Collection names. Prefixed like the rest of the CMS's Mongo collections. */
+const FOLDERS = 'cms_mail_folders';
+const MESSAGES = 'cms_mail_messages';
+/** Messages waiting to be sent - scheduled, or held briefly so Send can be taken back. */
+const OUTBOX = 'cms_mail_outbox';
+/** Messages put out of sight until a time. */
+const SNOOZED = 'cms_mail_snoozed';
+/** Who the away message has already answered, and when. */
+const AUTOREPLIES = 'cms_mail_autoreplies';
+
+/**
+ * The most a queued message may weigh, in bytes of base64.
+ *
+ * BSON caps a document at 16MB and a queued send carries its attachments
+ * inside one. Sized well under that so the envelope, the recipients and the
+ * body cannot push an otherwise-legal message over the edge - a rejection at
+ * the driver would be an unsendable message sitting in a queue.
+ */
+export const MAX_QUEUED_BYTES = 12_000_000;
+
+/**
+ * How long a claimed row may sit before another tick may take it back.
+ *
+ * A claim flips a row to `sending` or `waking`, and only the worker finishing
+ * moves it out again - so a process that is restarted mid-claim leaves the
+ * row in a state nothing ever matches again. It is then in no list, has no
+ * button, and the message is neither sent nor reported as failed.
+ *
+ * Generous, because the point is to recover from a dead process rather than
+ * to race a slow mail server: a send that genuinely takes four minutes must
+ * not be picked up twice.
+ */
+const CLAIM_STALE_MS = 5 * 60_000;
+
+/**
+ * How many messages per conversation a page of threads carries, on average.
+ *
+ * A ceiling on the second query rather than a per-thread limit, which Mongo
+ * cannot express in one find. The conversation's true size comes from the
+ * grouping and is reported regardless; this only bounds how much is sent to
+ * a browser drawing fifty rows.
+ */
+const THREAD_MESSAGE_CAP = 20;
+
+/**
+ * A short random id for a queue row.
+ *
+ * Not `randomUUID`: this module is imported by a plugin that may run on a
+ * site without `mongodb` installed, and keeping the import surface to
+ * nothing but the driver means it can be loaded and inspected anywhere.
+ *
+ * @returns {string}
+ */
+function randomId() {
+    return `${Math.random().toString(36).slice(2, 12)}${Date.now().toString(36)}`;
+}
+
+/**
+ * The stable id for "this mailbox has answered this address".
+ *
+ * Lower-cased, because a sender writing from Sam@ and sam@ is one person and
+ * must not be told twice.
+ *
+ * @param   {string} accountId
+ * @param   {string} address
+ * @returns {string}
+ */
+function autoReplyId(accountId, address) {
+    return `${accountId}:${String(address ?? '').trim().toLowerCase()}`;
+}
+
+/**
+ * The document out of a findOneAndUpdate result.
+ *
+ * Driver 6 returns the document itself; drivers before it wrapped it in
+ * `{value, ok}`. Checked by looking for an `_id` rather than by reaching for
+ * `.value` first, because a document that happened to have a field called
+ * `value` would otherwise be unwrapped into whatever that field held.
+ *
+ * @param   {object|null} result
+ * @returns {object|null}
+ */
+function claimed(result) {
+    if (!result) return null;
+    if (result._id !== undefined) return result;
+    return result.value?._id !== undefined ? result.value : null;
+}
+
+/**
+ * The first message id out of a header that may hold several.
+ *
+ * @param   {string|string[]|null|undefined} value
+ * @returns {string|null}
+ */
+function firstId(value) {
+    if (Array.isArray(value)) return value[0] ?? null;
+    const match = String(value ?? '').trim().split(/\s+/)[0];
+    return match || null;
+}
+
+/**
+ * Compose the stable id for a folder record.
+ *
+ * @param   {string} accountId
+ * @param   {string} path
+ * @returns {string}
+ */
+export function folderId(accountId, path) {
+    return `${accountId}:${path}`;
+}
+
+/**
+ * Compose the stable id for a message record.
+ *
+ * UID is unique per folder per UIDVALIDITY, so the id is only stable while
+ * UIDVALIDITY holds - which is exactly the condition a resync checks.
+ *
+ * @param   {string} accountId
+ * @param   {string} path
+ * @param   {number} uid
+ * @returns {string}
+ */
+export function messageId(accountId, path, uid) {
+    return `${accountId}:${path}:${uid}`;
+}
+
+/**
+ * Create a store bound to one connection.
+ *
+ * @param   {{uri: string, database: string}} connection
+ * @returns {object} store API
+ */
+export function createStore(connection) {
+    let client = null;
+    let db = null;
+    let connecting = null;
+
+    /**
+     * Connect once, lazily, and make sure the indexes exist.
+     *
+     * @returns {Promise<object>} the database handle
+     */
+    async function connect() {
+        if (db) return db;
+        if (connecting) return connecting;
+
+        connecting = (async () => {
+            const {MongoClient} = await import('mongodb');
+            client = new MongoClient(connection.uri, {serverSelectionTimeoutMS: 8000});
+            await client.connect();
+            db = client.db(connection.database);
+            await ensureIndexes(db);
+            return db;
+        })();
+
+        try {
+            return await connecting;
+        } finally {
+            connecting = null;
+        }
+    }
+
+    /**
+     * Indexes the sync and the reader depend on.
+     *
+     * Creating them is idempotent, and doing it on connect means a store that
+     * has just been provisioned works without a separate migration step.
+     *
+     * @param   {object} database
+     * @returns {Promise<void>}
+     */
+    async function ensureIndexes(database) {
+        await database.collection(FOLDERS).createIndex({accountId: 1});
+        await Promise.all([
+            // The list view: newest first within a folder.
+            database.collection(MESSAGES).createIndex({accountId: 1, folder: 1, date: -1}),
+            // Sync reconciliation works in UID order.
+            database.collection(MESSAGES).createIndex({accountId: 1, folder: 1, uid: 1}),
+            // Threading, once replies are being matched up.
+            database.collection(MESSAGES).createIndex({accountId: 1, messageId: 1}),
+            // Listing a folder by conversation.
+            database.collection(MESSAGES).createIndex({accountId: 1, folder: 1, threadRoot: 1}),
+            // Search narrows by mailbox first, then scans. Fine at the volumes
+            // a mirrored selection reaches; a mailbox with hundreds of
+            // thousands of messages would want a text index instead, at the
+            // cost of substring matching, which is what people actually type.
+            database.collection(MESSAGES).createIndex({accountId: 1, date: -1}),
+            // The two queues. The worker asks "what is due" across every
+            // mailbox on every tick, so `at` leads; the listing screens ask
+            // per mailbox.
+            database.collection(OUTBOX).createIndex({status: 1, at: 1}),
+            database.collection(OUTBOX).createIndex({accountId: 1, createdAt: -1}),
+            database.collection(SNOOZED).createIndex({status: 1, at: 1}),
+            database.collection(SNOOZED).createIndex({accountId: 1, at: 1}),
+            database.collection(AUTOREPLIES).createIndex({accountId: 1, at: -1})
+        ]);
+    }
+
+    /**
+     * Folder sync state, or null when this folder has never been synced.
+     *
+     * @param   {string} accountId
+     * @param   {string} path
+     * @returns {Promise<object|null>}
+     */
+    async function getFolder(accountId, path) {
+        const database = await connect();
+        return database.collection(FOLDERS).findOne({_id: folderId(accountId, path)});
+    }
+
+    /**
+     * Record what a folder looked like after a sync pass.
+     *
+     * @param   {string} accountId
+     * @param   {string} path
+     * @param   {object} state
+     * @returns {Promise<void>}
+     */
+    async function saveFolder(accountId, path, state) {
+        const database = await connect();
+        await database.collection(FOLDERS).updateOne(
+            {_id: folderId(accountId, path)},
+            {$set: {...state, accountId, path, updatedAt: new Date()}},
+            {upsert: true}
+        );
+    }
+
+    /**
+     * Store a batch of envelopes, replacing any already held for those UIDs.
+     *
+     * @param   {string} accountId
+     * @param   {string} path
+     * @param   {object[]} messages
+     * @returns {Promise<number>} how many were written
+     */
+    async function putMessages(accountId, path, messages) {
+        if (!messages.length) return 0;
+        const database = await connect();
+        const now = new Date();
+        const rooted = await inheritThreadRoots(database, accountId, messages);
+        const operations = rooted.map(message => ({
+            replaceOne: {
+                filter: {_id: messageId(accountId, path, message.uid)},
+                replacement: {
+                    ...message,
+                    _id: messageId(accountId, path, message.uid),
+                    accountId,
+                    folder: path,
+                    fetchedAt: now
+                },
+                upsert: true
+            }
+        }));
+        const result = await database.collection(MESSAGES).bulkWrite(operations, {ordered: false});
+        return (result.upsertedCount ?? 0) + (result.modifiedCount ?? 0);
+    }
+
+    /**
+     * Give a reply the same thread root as the message it answers.
+     *
+     * `toRecord` works out a root from the headers alone, which is right
+     * whenever `References` carries the whole chain. Plenty of clients put
+     * only the immediate parent in it, and those replies each get a
+     * different root - so a conversation of six arrives as six conversations.
+     *
+     * One indexed lookup per batch fixes it: whatever the parent is filed
+     * under, the reply is filed under too. Parents inside the same batch are
+     * resolved first, because a batch is very often an entire thread
+     * arriving at once and neither half is in the database yet.
+     *
+     * @param   {object} database
+     * @param   {string} accountId
+     * @param   {object[]} messages
+     * @returns {Promise<object[]>}
+     */
+    async function inheritThreadRoots(database, accountId, messages) {
+        const wanted = new Set();
+        for (const message of messages) {
+            const parent = firstId(message.inReplyTo);
+            // Already the root of its own chain - nothing to inherit.
+            if (parent && parent !== message.messageId) wanted.add(parent);
+        }
+        if (!wanted.size) return messages;
+
+        const known = new Map();
+        // Anything arriving in this same batch, first.
+        for (const message of messages) {
+            if (message.messageId && message.threadRoot) known.set(message.messageId, message.threadRoot);
+        }
+
+        const missing = [...wanted].filter(id => !known.has(id));
+        if (missing.length) {
+            const rows = await database.collection(MESSAGES)
+                .find({accountId, messageId: {$in: missing}},
+                    {projection: {messageId: 1, threadRoot: 1, _id: 0}})
+                .toArray();
+            for (const row of rows) {
+                if (row.messageId && row.threadRoot) known.set(row.messageId, row.threadRoot);
+            }
+        }
+
+        return messages.map(message => {
+            const parent = firstId(message.inReplyTo);
+            const inherited = parent ? known.get(parent) : null;
+            return inherited ? {...message, threadRoot: inherited} : message;
+        });
+    }
+
+    /**
+     * Apply flag changes without refetching the envelope.
+     *
+     * @param   {string} accountId
+     * @param   {string} path
+     * @param   {{uid: number, flags: string[]}[]} updates
+     * @returns {Promise<number>}
+     */
+    async function setFlags(accountId, path, updates) {
+        if (!updates.length) return 0;
+        const database = await connect();
+        const operations = updates.map(({uid, flags}) => ({
+            updateOne: {
+                filter: {_id: messageId(accountId, path, uid)},
+                update: {$set: {flags, seen: flags.includes('\\Seen'), flagged: flags.includes('\\Flagged')}}
+            }
+        }));
+        const result = await database.collection(MESSAGES).bulkWrite(operations, {ordered: false});
+        return result.modifiedCount ?? 0;
+    }
+
+    /**
+     * Forget messages that are no longer on the server.
+     *
+     * @param   {string} accountId
+     * @param   {string} path
+     * @param   {number[]} uids
+     * @returns {Promise<number>}
+     */
+    async function removeMessages(accountId, path, uids) {
+        if (!uids.length) return 0;
+        const database = await connect();
+        const result = await database.collection(MESSAGES).deleteMany({
+            _id: {$in: uids.map(uid => messageId(accountId, path, uid))}
+        });
+        return result.deletedCount ?? 0;
+    }
+
+    /**
+     * Drop everything held for a folder.
+     *
+     * Used when UIDVALIDITY changes, which means every UID we hold now refers
+     * to a different message - the one case where throwing the mirror away is
+     * the correct response rather than a failure.
+     *
+     * @param   {string} accountId
+     * @param   {string} path
+     * @returns {Promise<number>}
+     */
+    async function clearFolder(accountId, path) {
+        const database = await connect();
+        const result = await database.collection(MESSAGES).deleteMany({accountId, folder: path});
+        return result.deletedCount ?? 0;
+    }
+
+    /**
+     * Every UID held for a folder, ascending - the left-hand side of the
+     * reconciliation against what the server still has.
+     *
+     * @param   {string} accountId
+     * @param   {string} path
+     * @returns {Promise<number[]>}
+     */
+    async function storedUids(accountId, path) {
+        const database = await connect();
+        const rows = await database.collection(MESSAGES)
+            .find({accountId, folder: path}, {projection: {uid: 1, _id: 0}})
+            .sort({uid: 1})
+            .toArray();
+        return rows.map(row => row.uid);
+    }
+
+    /**
+     * One page of a folder, newest first.
+     *
+     * @param   {string} accountId
+     * @param   {string} path
+     * @param   {{page?: number, limit?: number}} [options]
+     * @returns {Promise<{messages: object[], total: number}>}
+     */
+    async function listMessages(accountId, path, {page = 1, limit = 50} = {}) {
+        const database = await connect();
+        const collection = database.collection(MESSAGES);
+        const filter = {accountId, folder: path};
+        const [messages, total] = await Promise.all([
+            collection.find(filter, {projection: {_id: 0}})
+                .sort({date: -1, uid: -1})
+                .skip((page - 1) * limit)
+                .limit(limit)
+                .toArray(),
+            collection.countDocuments(filter)
+        ]);
+        return {messages, total};
+    }
+
+    /**
+     * Search the mirror.
+     *
+     * Substring, case-insensitive, across the fields a person searches by.
+     * Deliberately not a Mongo text index: that matches whole words, so
+     * searching "domma" would not find "dommajs", and typing a fragment is
+     * what people actually do.
+     *
+     * Only mirrored folders can be searched - the rest of the mailbox is not
+     * here. The caller is told which folders were covered so it can say so
+     * rather than quietly returning less than the user expected.
+     *
+     * @param   {string} accountId
+     * @param   {{query: string, folders?: string[]|null, page?: number, limit?: number}} options
+     * @returns {Promise<{messages: object[], total: number, folders: string[]}>}
+     */
+    async function searchMessages(accountId, {query, folders = null, page = 1, limit = 50}) {
+        const database = await connect();
+        const collection = database.collection(MESSAGES);
+
+        const trimmed = String(query ?? '').trim();
+        if (!trimmed) return {messages: [], total: 0, folders: []};
+
+        // Escaped: a search for "a.b" or "c++" is a search, not a pattern.
+        const pattern = new RegExp(trimmed.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'i');
+
+        const filter = {
+            accountId,
+            $or: [
+                {subject: pattern},
+                {'from.address': pattern},
+                {'from.name': pattern},
+                {'to.address': pattern},
+                {'to.name': pattern}
+            ]
+        };
+        if (Array.isArray(folders) && folders.length) filter.folder = {$in: folders};
+
+        const [messages, total, covered] = await Promise.all([
+            collection.find(filter, {projection: {_id: 0}})
+                .sort({date: -1, uid: -1})
+                .skip((page - 1) * limit)
+                .limit(limit)
+                .toArray(),
+            collection.countDocuments(filter),
+            collection.distinct('folder', {accountId})
+        ]);
+
+        return {messages, total, folders: covered};
+    }
+
+    /**
+     * Remove everything belonging to one mailbox.
+     *
+     * Called when a mailbox is removed or its owner is deleted: the mirror
+     * must not outlive the account any more than the password does.
+     *
+     * @param   {string} accountId
+     * @returns {Promise<void>}
+     */
+    async function forgetAccount(accountId) {
+        const database = await connect();
+        // Every collection, not just the two that existed first. A queued
+        // send outliving its mailbox would try to send from credentials that
+        // no longer exist, and the auto-reply log is a record of who wrote to
+        // a mailbox that has been removed.
+        await Promise.all([
+            database.collection(MESSAGES).deleteMany({accountId}),
+            database.collection(FOLDERS).deleteMany({accountId}),
+            database.collection(OUTBOX).deleteMany({accountId}),
+            database.collection(SNOOZED).deleteMany({accountId}),
+            database.collection(AUTOREPLIES).deleteMany({accountId})
+        ]);
+    }
+
+    /**
+     * @returns {Promise<void>}
+     */
+    async function close() {
+        const open = client;
+        client = null;
+        db = null;
+        if (open) await open.close().catch(() => {});
+    }
+
+    /**
+     * One page of a folder, grouped into conversations.
+     *
+     * The page is of CONVERSATIONS, not of messages, which is why this cannot
+     * be done by grouping the output of `listMessages`: fifty messages might
+     * be four conversations or fifty, and a list that sometimes shows four
+     * rows is not a list. The grouping therefore happens in the database,
+     * over the whole folder, and the page is taken from the groups.
+     *
+     * @param   {string} accountId
+     * @param   {string} path
+     * @param   {{page?: number, limit?: number}} [options]
+     * @returns {Promise<{threads: object[], total: number}>}
+     */
+    async function listThreads(accountId, path, {page = 1, limit = 50} = {}) {
+        const database = await connect();
+        const collection = database.collection(MESSAGES);
+        const match = {accountId, folder: path};
+
+        await backfillThreadRoots(collection, accountId, path);
+
+        // A row with no root at all is its own conversation. Grouping them
+        // all under null would collapse every message the server gave no
+        // Message-ID into one heap.
+        const rootOf = {$ifNull: ['$threadRoot', {$concat: ['\u0000uid-', {$toString: '$uid'}]}]};
+
+        // Two queries, deliberately, rather than one that pushes whole
+        // documents into each group.
+        //
+        // `{$push: '$$ROOT'}` builds one array per conversation in memory and
+        // then pages the result - so a folder collecting automated mail under
+        // a single thread root (CI alerts, a long mailing list) assembles a
+        // document larger than BSON's 16MB ceiling and the whole view 500s.
+        // Grouping to counts alone is bounded by the number of conversations;
+        // the messages for the page are then a plain indexed find.
+        const summaries = await collection.aggregate([
+            {$match: match},
+            {$group: {
+                _id: rootOf,
+                latestDate: {$max: '$date'},
+                count: {$sum: 1},
+                unseen: {$sum: {$cond: [{$eq: ['$seen', false]}, 1, 0]}},
+                flagged: {$max: {$cond: ['$flagged', 1, 0]}},
+                hasAttachment: {$max: {$cond: ['$hasAttachment', 1, 0]}}
+            }},
+            {$sort: {latestDate: -1, _id: 1}},
+            {$skip: (page - 1) * limit},
+            {$limit: limit}
+        ], {allowDiskUse: true}).toArray();
+
+        const totals = await collection.aggregate([
+            {$match: match},
+            {$group: {_id: rootOf}},
+            {$count: 'total'}
+        ], {allowDiskUse: true}).toArray();
+
+        if (!summaries.length) return {threads: [], total: totals[0]?.total ?? 0};
+
+        // The messages behind this page's conversations, newest first and
+        // capped. The cap is a memory bound, not a correctness one: `count`
+        // above is authoritative, so a conversation of twelve thousand still
+        // SAYS twelve thousand - it just does not hand all of them to a
+        // browser that is drawing fifty rows.
+        const roots = summaries.map(row => row._id);
+        const plain = roots.filter(root => !String(root).startsWith('\u0000'));
+        const rows = await collection
+            .find(
+                plain.length === roots.length
+                    ? {...match, threadRoot: {$in: plain}}
+                    // Some of this page's conversations are rootless rows,
+                    // which are identified by UID rather than by root.
+                    : {
+                        ...match,
+                        $or: [
+                            ...(plain.length ? [{threadRoot: {$in: plain}}] : []),
+                            {uid: {$in: roots
+                                .filter(root => String(root).startsWith('\u0000'))
+                                .map(root => Number(String(root).slice(String(root).indexOf('-') + 1)))
+                                .filter(Number.isFinite)}}
+                        ]
+                    },
+                {projection: {_id: 0}}
+            )
+            .sort({date: -1, uid: -1})
+            .limit(limit * THREAD_MESSAGE_CAP)
+            .toArray();
+
+        const byRoot = new Map(roots.map(root => [String(root), []]));
+        for (const row of rows) {
+            const key = row.threadRoot ?? `\u0000uid-${row.uid}`;
+            if (byRoot.has(String(key))) byRoot.get(String(key)).push(row);
+        }
+
+        const threads = summaries.map(summary => {
+            const messages = byRoot.get(String(summary._id)) ?? [];
+            return {
+                root: summary._id,
+                messages,
+                latest: messages[0] ?? null,
+                count: summary.count,
+                unseen: summary.unseen,
+                flagged: summary.flagged === 1,
+                hasAttachment: summary.hasAttachment === 1
+            };
+        // A conversation whose messages all fell outside the cap has nothing
+        // to draw, and a row with no latest message is a row with no subject
+        // and no date.
+        }).filter(thread => thread.latest);
+
+        return {threads, total: totals[0]?.total ?? 0};
+    }
+
+    // -----------------------------------------------------------------------
+    // The queues
+    // -----------------------------------------------------------------------
+
+    /**
+     * Put a message in the outbox.
+     *
+     * @param   {object} record
+     * @returns {Promise<string>} the queue id
+     */
+    async function queueSend(record) {
+        const database = await connect();
+        const id = `out-${randomId()}`;
+        await database.collection(OUTBOX).insertOne({
+            ...record,
+            _id: id,
+            status: 'pending',
+            attempts: 0,
+            createdAt: new Date()
+        });
+        return id;
+    }
+
+    /**
+     * What is waiting, or recently went, for one mailbox.
+     *
+     * Sent and failed entries are kept for a while rather than deleted the
+     * moment they go: "did that actually send?" is the question the screen
+     * exists to answer, and an empty list answers it ambiguously.
+     *
+     * @param   {string} accountId
+     * @param   {{limit?: number}} [options]
+     * @returns {Promise<object[]>}
+     */
+    async function listQueuedSends(accountId, {limit = 50} = {}) {
+        const database = await connect();
+        return database.collection(OUTBOX)
+            .find({accountId}, {projection: {'draft.attachments.content': 0}})
+            .sort({createdAt: -1})
+            .limit(limit)
+            .toArray();
+    }
+
+    /**
+     * Claim the queued sends that are due.
+     *
+     * Claimed one at a time with a conditional update, so two processes - or
+     * two ticks that overlap because one was slow - cannot both pick up the
+     * same message and send it twice. Sending twice is the one failure this
+     * queue must not have: a duplicate cannot be recalled.
+     *
+     * @param   {Date|number} [now]
+     * @param   {number} [max]
+     * @returns {Promise<object[]>}
+     */
+    async function claimDueSends(now = Date.now(), max = 10) {
+        const database = await connect();
+        const at = new Date(now instanceof Date ? now.getTime() : now);
+        const due = [];
+
+        const stale = new Date(at.getTime() - CLAIM_STALE_MS);
+
+        for (let i = 0; i < max; i += 1) {
+            const result = await database.collection(OUTBOX).findOneAndUpdate(
+                {
+                    $or: [
+                        {status: 'pending', at: {$lte: at}},
+                        // Reclaimed. A row left `sending` by a restart matches
+                        // nothing otherwise, and the message is then neither
+                        // sent nor reported - see CLAIM_STALE_MS. The attempt
+                        // counter still applies, so a send that kills the
+                        // process every time is given up on rather than
+                        // retried forever.
+                        {status: 'sending', claimedAt: {$lte: stale}}
+                    ]
+                },
+                {$set: {status: 'sending', claimedAt: new Date()}, $inc: {attempts: 1}},
+                {sort: {at: 1}, returnDocument: 'after'}
+            );
+            const doc = claimed(result);
+            if (!doc) break;
+            due.push(doc);
+        }
+        return due;
+    }
+
+    /**
+     * Record what became of a queued send.
+     *
+     * @param   {string} id
+     * @param   {object} patch
+     * @returns {Promise<void>}
+     */
+    async function finishSend(id, patch) {
+        const database = await connect();
+        await database.collection(OUTBOX).updateOne({_id: id}, {$set: {...patch, finishedAt: new Date()}});
+    }
+
+    /**
+     * Take a message back out of the outbox.
+     *
+     * Only while it is still pending. A message already claimed by the worker
+     * is on its way to the mail server, and reporting it cancelled would be
+     * the one lie this feature cannot afford.
+     *
+     * @param   {string} userId
+     * @param   {string} id
+     * @returns {Promise<object|null>} the cancelled record, or null
+     */
+    async function cancelSend(userId, id) {
+        const database = await connect();
+        const result = await database.collection(OUTBOX).findOneAndUpdate(
+            {_id: id, userId, status: 'pending'},
+            {$set: {status: 'cancelled', finishedAt: new Date()}},
+            {returnDocument: 'after'}
+        );
+        return claimed(result);
+    }
+
+    /**
+     * One queued send, if it belongs to this user.
+     *
+     * @param   {string} userId
+     * @param   {string} id
+     * @returns {Promise<object|null>}
+     */
+    async function getQueuedSend(userId, id) {
+        const database = await connect();
+        return database.collection(OUTBOX).findOne({_id: id, userId});
+    }
+
+    /**
+     * How much unread mail a mailbox holds, across its mirrored folders.
+     *
+     * Answered from the mirror rather than from IMAP, which is what makes it
+     * cheap enough to ask for on every sidebar render: it is a counting query
+     * against an index, not a connection to a mail server.
+     *
+     * @param   {string} accountId
+     * @param   {string[]|null} [folders] - which to count, or null for all mirrored
+     * @returns {Promise<number>}
+     */
+    async function countUnread(accountId, folders = null) {
+        const database = await connect();
+        const filter = {accountId, seen: false};
+        if (Array.isArray(folders) && folders.length) filter.folder = {$in: folders};
+        return database.collection(MESSAGES).countDocuments(filter);
+    }
+
+    /** Folders already backfilled in this process, so it runs at most once each. */
+    const backfilled = new Set();
+
+    /**
+     * Give rows written before `threadRoot` existed one.
+     *
+     * A mirror built by an earlier version has no thread root on any row, so
+     * every message would group as its own conversation - which is not a
+     * feature arriving quietly, it is a feature that visibly does not work on
+     * the mail somebody already has.
+     *
+     * The root is derived from what those rows DO carry: whatever the message
+     * is a reply to, or itself. `References` was not stored either, so a deep
+     * chain from a client that sends only the immediate parent still lands on
+     * several roots - which the page repair in `groupThreads` then joins.
+     * Between them the old mail groups; new mail is exact.
+     *
+     * Run on the read path rather than the sync, because this is exactly the
+     * moment it matters, and it is guarded so a folder pays for it once per
+     * process. After the first pass the filter matches nothing.
+     *
+     * @param   {object} collection
+     * @param   {string} accountId
+     * @param   {string} path
+     * @returns {Promise<void>}
+     */
+    async function backfillThreadRoots(collection, accountId, path) {
+        const key = `${accountId}:${path}`;
+        if (backfilled.has(key)) return;
+        backfilled.add(key);
+
+        try {
+            await collection.updateMany(
+                {accountId, folder: path, threadRoot: {$in: [null, undefined]}},
+                [{$set: {threadRoot: {$ifNull: ['$inReplyTo', '$messageId']}}}]
+            );
+        } catch {
+            // A backfill that cannot run is a folder that groups less well,
+            // not a folder that cannot be listed. The aggregation handles a
+            // missing root on its own.
+            backfilled.delete(key);
+        }
+    }
+
+    // -----------------------------------------------------------------------
+    // The away message's memory
+    // -----------------------------------------------------------------------
+
+    /**
+     * When this mailbox last answered a given address automatically.
+     *
+     * The whole point of the once-per-sender rule: somebody who writes four
+     * times in an afternoon is told once.
+     *
+     * @param   {string} accountId
+     * @param   {string} address
+     * @returns {Promise<string|null>} an ISO timestamp, or null
+     */
+    async function lastAutoReply(accountId, address) {
+        const database = await connect();
+        const row = await database.collection(AUTOREPLIES)
+            .findOne({_id: autoReplyId(accountId, address)});
+        return row?.at ? new Date(row.at).toISOString() : null;
+    }
+
+    /**
+     * Remember that it has been answered.
+     *
+     * @param   {string} accountId
+     * @param   {string} address
+     * @param   {Date} [at]
+     * @returns {Promise<void>}
+     */
+    async function noteAutoReply(accountId, address, at = new Date()) {
+        const database = await connect();
+        await database.collection(AUTOREPLIES).updateOne(
+            {_id: autoReplyId(accountId, address)},
+            {$set: {accountId, address: String(address).toLowerCase(), at}},
+            {upsert: true}
+        );
+    }
+
+    /**
+     * Who has been answered recently - the responder's audit trail.
+     *
+     * It exists because the replies are deliberately NOT filed in Sent: a
+     * holiday can produce a hundred of them, and a Sent folder that is mostly
+     * "Out of office" is one nobody can find anything in. Without this there
+     * would be no record at all of what went out on your behalf.
+     *
+     * @param   {string} accountId
+     * @param   {{limit?: number}} [options]
+     * @returns {Promise<object[]>}
+     */
+    async function listAutoReplies(accountId, {limit = 50} = {}) {
+        const database = await connect();
+        return database.collection(AUTOREPLIES)
+            .find({accountId}, {projection: {_id: 0}})
+            .sort({at: -1})
+            .limit(limit)
+            .toArray();
+    }
+
+    /**
+     * Forget who has been answered, so everybody is told again.
+     *
+     * Wanted whenever the away message is turned on for a NEW absence: the
+     * people who wrote during the last one should not be silently skipped
+     * because they were told about that one.
+     *
+     * @param   {string} accountId
+     * @returns {Promise<number>}
+     */
+    async function clearAutoReplies(accountId) {
+        const database = await connect();
+        const result = await database.collection(AUTOREPLIES).deleteMany({accountId});
+        return result.deletedCount ?? 0;
+    }
+
+    /**
+     * Note that a message has been put away until later.
+     *
+     * @param   {object} record
+     * @returns {Promise<string>}
+     */
+    async function queueSnooze(record) {
+        const database = await connect();
+        const id = `snz-${randomId()}`;
+        await database.collection(SNOOZED).insertOne({
+            ...record, _id: id, status: 'pending', createdAt: new Date()
+        });
+        return id;
+    }
+
+    /**
+     * Everything this mailbox has put away, soonest first.
+     *
+     * Everything NOT finished, rather than everything pending. A row that is
+     * mid-wake, or that failed, still has a message sitting in the snooze
+     * folder - and a list that quietly omitted those would be a list that
+     * said "nothing is snoozed" while the mail was still parked.
+     *
+     * @param   {string} accountId
+     * @returns {Promise<object[]>}
+     */
+    async function listSnoozed(accountId) {
+        const database = await connect();
+        return database.collection(SNOOZED)
+            .find({accountId, status: {$nin: ['woken', 'cancelled']}})
+            .sort({at: 1})
+            .toArray();
+    }
+
+    /**
+     * Claim the snoozes that have come due. Same one-at-a-time claim as sends.
+     *
+     * @param   {Date|number} [now]
+     * @param   {number} [max]
+     * @returns {Promise<object[]>}
+     */
+    async function claimDueSnoozes(now = Date.now(), max = 20) {
+        const database = await connect();
+        const at = new Date(now instanceof Date ? now.getTime() : now);
+        const due = [];
+
+        const stale = new Date(at.getTime() - CLAIM_STALE_MS);
+
+        for (let i = 0; i < max; i += 1) {
+            const result = await database.collection(SNOOZED).findOneAndUpdate(
+                {
+                    $or: [
+                        {status: 'pending', at: {$lte: at}},
+                        // Same reclaim as the outbox. Worse here if it is
+                        // missed: `listSnoozed` only shows pending rows, so a
+                        // row stuck at `waking` vanishes from the Snoozed list
+                        // while the message sits in a folder nobody reads.
+                        {status: 'waking', claimedAt: {$lte: stale}}
+                    ]
+                },
+                {$set: {status: 'waking', claimedAt: new Date()}, $inc: {attempts: 1}},
+                {sort: {at: 1}, returnDocument: 'after'}
+            );
+            const doc = claimed(result);
+            if (!doc) break;
+            due.push(doc);
+        }
+        return due;
+    }
+
+    /**
+     * @param   {string} id
+     * @param   {object} patch
+     * @returns {Promise<void>}
+     */
+    async function finishSnooze(id, patch) {
+        const database = await connect();
+        await database.collection(SNOOZED).updateOne({_id: id}, {$set: {...patch, finishedAt: new Date()}});
+    }
+
+    /**
+     * Bring one back early.
+     *
+     * @param   {string} userId
+     * @param   {string} id
+     * @returns {Promise<object|null>}
+     */
+    async function wakeSnooze(userId, id) {
+        const database = await connect();
+        const result = await database.collection(SNOOZED).findOneAndUpdate(
+            // A failed row is exactly the one somebody most wants to retry by
+            // hand, so it is eligible too - it is reset to pending and the
+            // attempt counter cleared.
+            {_id: id, userId, status: {$in: ['pending', 'failed']}},
+            {$set: {at: new Date(0), status: 'pending', attempts: 0, wokeEarly: true}},
+            {returnDocument: 'after'}
+        );
+        return claimed(result);
+    }
+
+    return {
+        connect,
+        getFolder, saveFolder,
+        putMessages, setFlags, removeMessages, clearFolder,
+        storedUids, listMessages, listThreads, searchMessages, countUnread, forgetAccount,
+        queueSend, listQueuedSends, claimDueSends, finishSend, cancelSend, getQueuedSend,
+        queueSnooze, listSnoozed, claimDueSnoozes, finishSnooze, wakeSnooze,
+        lastAutoReply, noteAutoReply, listAutoReplies, clearAutoReplies,
+        close
+    };
+}