domma-cms 0.54.2 → 0.66.0

package/plugins/_lib/mail/imapPool.js

@@ -0,0 +1,293 @@
+/**
+ * IMAP connection pool for the Mail Reader plugin.
+ *
+ * IMAP is unlike anything else the CMS talks to: the connection is long-lived,
+ * stateful, and handles exactly one command at a time. Opening a fresh
+ * connection per request would mean a TLS handshake and a LOGIN for every
+ * click, and most mail servers rate-limit that fairly aggressively.
+ *
+ * So connections are cached per user and reaped when idle. All mailbox access
+ * goes through `withMailbox()`, which takes ImapFlow's own mailbox lock, so
+ * two overlapping requests from the same user queue rather than corrupting
+ * each other's command stream.
+ *
+ * ## Read-only, twice over
+ *
+ * Every mailbox is opened `{readOnly: true}` and ImapFlow fetches with
+ * `BODY.PEEK[]` (see its commands/fetch.js). Either alone would stop the
+ * server setting `\Seen` as a side effect of reading a message; phase 1 has
+ * both, because a reader that silently marks a user's mail as read on their
+ * phone is worse than no reader at all. Do not relax this without making
+ * mark-as-read a deliberate, user-visible action.
+ *
+ * **Single-process only.** The pool is module state, exactly like the messages
+ * plugin's SSE registry. Running the site across workers would give each
+ * worker its own connections and blow through per-user connection limits.
+ *
+ * @module mail-reader/services/imapPool
+ */
+import {ImapFlow} from 'imapflow';
+
+/**
+ * Identify a connection's target. A change to any of these means the cached
+ * connection is for a different account and must be thrown away.
+ *
+ * @param   {object} connection
+ * @returns {string}
+ */
+function fingerprint(connection) {
+    return [connection.host, connection.port, connection.secure, connection.auth.user].join('|');
+}
+
+/**
+ * Create an IMAP connection pool.
+ *
+ * @param   {{connectionIdleMs: number, maxConnections: number, connectionTimeoutMs: number, allowInsecureTLS: boolean}} config
+ * @returns {object} pool API
+ */
+export function createPool(config) {
+    /** @type {Map<string, {client: ImapFlow, fingerprint: string, lastUsed: number}>} */
+    const pool = new Map();
+
+    /** In-flight connects, so two parallel requests do not open two connections. */
+    const pending = new Map();
+
+    /**
+     * Build the ImapFlow options for a connection.
+     *
+     * @param   {object} connection
+     * @returns {object}
+     */
+    function clientOptions(connection) {
+        return {
+            host: connection.host,
+            port: connection.port,
+            secure: connection.secure,
+            auth: connection.auth,
+            logger: false,
+            // A dead mail server should fail the request, not hang the admin.
+            greetingTimeout: config.connectionTimeoutMs,
+            socketTimeout: config.connectionTimeoutMs,
+            connectionTimeout: config.connectionTimeoutMs,
+            tls: {rejectUnauthorized: !config.allowInsecureTLS}
+        };
+    }
+
+    /**
+     * Drop a pooled connection, closing it best-effort.
+     *
+     * @param   {string} userId
+     * @returns {void}
+     */
+    function evict(userId) {
+        const entry = pool.get(userId);
+        if (!entry) return;
+        pool.delete(userId);
+        entry.client.logout().catch(() => entry.client.close());
+    }
+
+    /**
+     * Get a live, authenticated client for a user.
+     *
+     * @param   {string} userId
+     * @param   {object} connection
+     * @returns {Promise<ImapFlow>}
+     */
+    async function acquire(userId, connection) {
+        const fp = fingerprint(connection);
+        const existing = pool.get(userId);
+
+        if (existing && existing.fingerprint === fp && existing.client.usable) {
+            existing.lastUsed = Date.now();
+            return existing.client;
+        }
+        // Stale, or the account details changed under it.
+        if (existing) evict(userId);
+
+        const inFlight = pending.get(userId);
+        if (inFlight) return inFlight;
+
+        if (pool.size >= config.maxConnections) {
+            // Evict the least recently used rather than refusing the request -
+            // the cap exists to bound resource use, not to lock people out.
+            const oldest = [...pool.entries()].sort((a, b) => a[1].lastUsed - b[1].lastUsed)[0];
+            if (oldest) evict(oldest[0]);
+        }
+
+        const connect = (async () => {
+            const client = new ImapFlow(clientOptions(connection));
+            // ImapFlow emits 'error' on an already-failed connection; without a
+            // listener that becomes an unhandled event and takes the process out.
+            client.on('error', () => evict(userId));
+            client.on('close', () => {
+                if (pool.get(userId)?.client === client) pool.delete(userId);
+            });
+            await client.connect();
+            pool.set(userId, {client, fingerprint: fp, lastUsed: Date.now()});
+            return client;
+        })();
+
+        pending.set(userId, connect);
+        try {
+            return await connect;
+        } finally {
+            pending.delete(userId);
+        }
+    }
+
+    /**
+     * Run a function against a user's connection, no mailbox selected.
+     *
+     * @param   {string} userId
+     * @param   {object} connection
+     * @param   {(client: ImapFlow) => Promise<any>} fn
+     * @returns {Promise<any>}
+     */
+    async function withClient(userId, connection, fn) {
+        const client = await acquire(userId, connection);
+        try {
+            const result = await fn(client);
+            const entry = pool.get(userId);
+            if (entry) entry.lastUsed = Date.now();
+            return result;
+        } catch (err) {
+            evict(userId);
+            throw err;
+        }
+    }
+
+    /**
+     * Run a function with a mailbox open and locked.
+     *
+     * Read-only unless asked otherwise, and the asking is deliberate: the free
+     * reader's entire guarantee is that looking at mail never changes it, and
+     * a default of read-write would quietly break that the first time a shared
+     * helper forgot the option. Only the Pro edition's write operations pass
+     * `writable`, and only for the one call that needs it.
+     *
+     * @param   {string} userId
+     * @param   {object} connection
+     * @param   {string} mailbox
+     * @param   {(client: ImapFlow, mailbox: object) => Promise<any>} fn
+     * @param   {{writable?: boolean}} [options]
+     * @returns {Promise<any>}
+     */
+    async function withMailbox(userId, connection, mailbox, fn, {writable = false} = {}) {
+        return withClient(userId, connection, async (client) => {
+            const lock = await client.getMailboxLock(mailbox, {readOnly: !writable});
+            try {
+                return await fn(client, client.mailbox);
+            } finally {
+                lock.release();
+            }
+        });
+    }
+
+    /**
+     * One-shot connect used by the "Test connection" button.
+     *
+     * Deliberately not pooled: it runs against details that have not been
+     * saved yet, and a failed attempt should leave nothing behind.
+     *
+     * @param   {object} connection
+     * @returns {Promise<{ok: true, greeting: string|null} | {ok: false, error: string}>}
+     */
+    async function test(connection) {
+        const client = new ImapFlow(clientOptions(connection));
+        client.on('error', () => {});
+        try {
+            await client.connect();
+            const greeting = client.serverInfo?.greeting ?? null;
+            await client.logout();
+            return {ok: true, greeting};
+        } catch (err) {
+            try {
+                client.close();
+            } catch {
+                // Already down - nothing to close.
+            }
+            return {ok: false, error: describeError(err)};
+        }
+    }
+
+    /**
+     * Close a specific user's connection - used when their account changes or
+     * is deleted, so the next request does not reuse the old credentials.
+     *
+     * @param   {string} userId
+     * @returns {void}
+     */
+    function release(userId) {
+        evict(userId);
+    }
+
+    /**
+     * Close every connection a user holds, whatever mailbox it belongs to.
+     *
+     * Keys are `userId:accountId`, so this is a prefix sweep. Used when a user
+     * is deleted - their sockets should not outlive their account either.
+     *
+     * @param   {string} userId
+     * @returns {void}
+     */
+    function releaseUser(userId) {
+        const prefix = `${userId}:`;
+        for (const key of [...pool.keys()]) {
+            if (key === userId || key.startsWith(prefix)) evict(key);
+        }
+    }
+
+    /**
+     * Close everything. Called on plugin disable and server shutdown.
+     *
+     * @returns {void}
+     */
+    function closeAll() {
+        for (const userId of [...pool.keys()]) evict(userId);
+    }
+
+    const reaper = setInterval(() => {
+        const cutoff = Date.now() - config.connectionIdleMs;
+        for (const [userId, entry] of pool) {
+            if (entry.lastUsed < cutoff) evict(userId);
+        }
+    }, Math.max(30_000, Math.floor(config.connectionIdleMs / 2)));
+    // Never hold the process open on account of an idle mail connection.
+    reaper.unref?.();
+
+    return {withClient, withMailbox, test, release, releaseUser, closeAll, get size() {
+        return pool.size;
+    }};
+}
+
+/**
+ * Turn an IMAP/network error into something worth showing a user.
+ *
+ * ImapFlow's own messages are decent but its auth failures surface as raw
+ * server responses, which vary by server and read like a stack trace.
+ *
+ * @param   {Error & {code?: string, authenticationFailed?: boolean}} err
+ * @returns {string}
+ */
+export function describeError(err) {
+    if (!err) return 'Unknown error.';
+    if (err.authenticationFailed) return 'The mail server rejected that username and password.';
+
+    switch (err.code) {
+        case 'ENOTFOUND':
+        case 'EAI_AGAIN':
+            return 'That mail server hostname could not be resolved.';
+        case 'ECONNREFUSED':
+            return 'The mail server refused the connection on that port.';
+        case 'ETIMEDOUT':
+        case 'ECONNRESET':
+            return 'The mail server did not respond in time.';
+        case 'CERT_HAS_EXPIRED':
+        case 'DEPTH_ZERO_SELF_SIGNED_CERT':
+        case 'SELF_SIGNED_CERT_IN_CHAIN':
+        case 'ERR_TLS_CERT_ALTNAME_INVALID':
+            return 'The mail server’s TLS certificate could not be verified.';
+        default:
+            return err.message || 'The mail server connection failed.';
+    }
+}

package/plugins/_lib/mail/mbox.js

@@ -0,0 +1,74 @@
+/**
+ * Exporting a folder as mbox.
+ *
+ * The point of this feature is that it is not a feature of this plugin: an
+ * mbox file opens in Thunderbird, in mutt, in Apple Mail and in anything else
+ * that has ever handled mail, so a mailbox mirrored here is not a mailbox
+ * held hostage here. That matters more for a CMS plugin than for a mail
+ * client, because the plugin can be disabled.
+ *
+ * ## The "From " line, and why escaping it matters
+ *
+ * mbox separates messages with a line beginning `From `. Any line in a message
+ * body that also begins `From ` would therefore split it in two - so those
+ * lines get a `>` in front, which is the mboxrd convention. Readers undo it on
+ * the way back in. Getting this wrong does not corrupt the file visibly; it
+ * silently turns one message into two, one of which has no headers.
+ *
+ * @module _lib/mail/mbox
+ */
+import {bareAddress} from '../admin/mail/identity.js';
+
+/**
+ * The separator line that introduces a message.
+ *
+ * The address is the envelope sender, and the date format is asctime - not
+ * RFC 5322, and not ISO. This line is not a header; it predates all of them,
+ * and readers parse it by shape.
+ *
+ * @param   {string} address
+ * @param   {Date} date
+ * @returns {string}
+ */
+export function fromLine(address, date = new Date()) {
+    // Through `bareAddress`, not by stripping the punctuation: removing every
+    // space and angle bracket from `Sales <s@e.com>` gives `Saless@e.com`,
+    // which is a separator line naming an address that does not exist. The
+    // remaining strip is a backstop for anything `bareAddress` leaves that
+    // would still break the line.
+    const clean = bareAddress(address).replace(/\s+/g, '') || 'MAILER-DAEMON';
+    const when = Number.isNaN(date?.getTime?.()) ? new Date() : date;
+    // asctime, in C locale, always: "Sun Sep 20 14:30:00 2026".
+    const asctime = when.toUTCString()
+        .replace(/^(\w{3}), (\d{2}) (\w{3}) (\d{4}) (\d{2}:\d{2}:\d{2}) GMT$/, '$1 $3 $2 $5 $4');
+    return `From ${clean} ${asctime}`;
+}
+
+/**
+ * Escape the body lines that would otherwise look like a separator.
+ *
+ * mboxrd: `From ` becomes `>From `, and an already-escaped `>From ` becomes
+ * `>>From `, so the transformation can be undone exactly.
+ *
+ * @param   {string} text
+ * @returns {string}
+ */
+export function escapeFromLines(text) {
+    return String(text ?? '').replace(/^(>*From )/gm, '>$1');
+}
+
+/**
+ * One message, as it appears in an mbox file.
+ *
+ * @param   {Buffer|string} source - the raw RFC822 message
+ * @param   {{address?: string, date?: Date}} [envelope]
+ * @returns {string}
+ */
+export function mboxEntry(source, {address = '', date = new Date()} = {}) {
+    const text = Buffer.isBuffer(source) ? source.toString('utf8') : String(source ?? '');
+    // Normalised to LF. mbox is a line-oriented text format and mixing CRLF
+    // bodies into it is what makes some readers show a blank line between
+    // every line of every message.
+    const body = escapeFromLines(text.replace(/\r\n/g, '\n'));
+    return `${fromLine(address, date)}\n${body}${body.endsWith('\n') ? '' : '\n'}\n`;
+}

package/plugins/_lib/mail/pollSchedule.js

@@ -0,0 +1,79 @@
+/**
+ * Deciding which mailboxes are due a poll.
+ *
+ * Pure, and separate from the timer that acts on it, because the interesting
+ * behaviour here is not "call sync every N minutes" - it is what happens when
+ * a mail server is down, or slow, or rate-limiting, and the obvious
+ * implementation keeps hammering it every N minutes forever.
+ *
+ * @module _lib/mail/pollSchedule
+ */
+
+/** Multiplier applied per consecutive failure. */
+const BACKOFF_FACTOR = 2;
+
+/** However bad things get, try again at least this often. */
+export const MAX_BACKOFF_MS = 3_600_000;
+
+/**
+ * How long to wait before the next attempt for one mailbox.
+ *
+ * Doubles per consecutive failure and is capped, so a mailbox whose server is
+ * down backs off towards hourly instead of knocking every few minutes for a
+ * week - which is how an IP ends up rate-limited or blocked.
+ *
+ * @param   {number} intervalMs - the mailbox's configured interval
+ * @param   {number} failures - consecutive failures so far
+ * @returns {number}
+ */
+export function backoffFor(intervalMs, failures) {
+    if (!failures) return intervalMs;
+    const grown = intervalMs * Math.pow(BACKOFF_FACTOR, Math.min(failures, 10));
+    return Math.min(grown, MAX_BACKOFF_MS);
+}
+
+/**
+ * Which mailboxes should be polled now.
+ *
+ * A mailbox never polled before is due immediately: the alternative is that
+ * adding one shows an empty list until the first interval elapses, which
+ * reads as broken.
+ *
+ * @param   {object[]} accounts
+ * @param   {Map<string, {lastAttemptAt?: number, failures?: number, running?: boolean}>} state
+ * @param   {{now: number, defaultMinutes: number}} options
+ * @returns {{id: string, waitedMs: number|null}[]} due mailboxes, most overdue first
+ */
+export function selectDuePolls(accounts, state, {now, defaultMinutes}) {
+    const due = [];
+
+    for (const account of accounts) {
+        const entry = state.get(account.id) ?? {};
+
+        // A pass already running must never be started again: two syncs of one
+        // folder race on the same UID range and the later write wins twice.
+        if (entry.running) continue;
+
+        const minutes = Number.isFinite(account.pollIntervalMinutes) && account.pollIntervalMinutes > 0
+            ? account.pollIntervalMinutes
+            : defaultMinutes;
+        const intervalMs = minutes * 60_000;
+
+        if (!entry.lastAttemptAt) {
+            due.push({id: account.id, waitedMs: null});
+            continue;
+        }
+
+        const wait = backoffFor(intervalMs, entry.failures ?? 0);
+        const waited = now - entry.lastAttemptAt;
+        if (waited >= wait) due.push({id: account.id, waitedMs: waited});
+    }
+
+    // Never-polled mailboxes first, then whoever has waited longest - so a
+    // backlog drains fairly rather than always favouring the same mailbox.
+    return due.sort((a, b) => {
+        if (a.waitedMs === null) return b.waitedMs === null ? 0 : -1;
+        if (b.waitedMs === null) return 1;
+        return b.waitedMs - a.waitedMs;
+    });
+}

package/plugins/_lib/mail/poller.js

@@ -0,0 +1,135 @@
+/**
+ * The background poller.
+ *
+ * One ticker for the whole plugin rather than a timer per mailbox: mailboxes
+ * come and go while the process runs, and a per-mailbox timer means tracking
+ * and cancelling each one. A single tick asks `selectDuePolls()` who is due
+ * and works through them.
+ *
+ * The scheduling decision - including the backoff that stops a dead mail
+ * server being knocked every few minutes for a week - lives in
+ * `pollSchedule.js`, pure and tested there.
+ *
+ * @module _lib/mail/poller
+ */
+import {selectDuePolls} from './pollSchedule.js';
+
+/** How often the ticker looks for work. Finer than any poll interval. */
+const TICK_MS = 30_000;
+
+/**
+ * Create a poller.
+ *
+ * @param   {{listAccounts: Function, syncAccount: Function, defaultMinutes: number,
+ *            logger: object, tickMs?: number}} deps
+ * @returns {{start: Function, stop: Function, state: Function, pollNow: Function}}
+ */
+export function createPoller({listAccounts, syncAccount, defaultMinutes, logger, tickMs = TICK_MS}) {
+    /** @type {Map<string, {lastAttemptAt?: number, failures?: number, running?: boolean, lastError?: string|null}>} */
+    const state = new Map();
+    let timer = null;
+    let stopped = false;
+
+    /**
+     * Sync one mailbox, recording whether it worked.
+     *
+     * @param   {object} account
+     * @returns {Promise<void>}
+     */
+    async function runOne(account) {
+        const entry = state.get(account.id) ?? {};
+        state.set(account.id, {...entry, running: true, lastAttemptAt: Date.now()});
+
+        try {
+            const result = await syncAccount(account);
+            state.set(account.id, {
+                lastAttemptAt: Date.now(), failures: 0, running: false, lastError: null,
+                lastResult: result
+            });
+        } catch (err) {
+            const failures = (entry.failures ?? 0) + 1;
+            state.set(account.id, {
+                lastAttemptAt: Date.now(), failures, running: false, lastError: err.message
+            });
+            // Warn, not error: a mail server being unreachable is an ordinary
+            // condition, and the backoff already handles it.
+            logger.warn(
+                `[mail] poll failed for mailbox ${account.id} (${failures} in a row): ${err.message}`
+            );
+        }
+    }
+
+    /**
+     * One pass: find who is due and sync them, one at a time.
+     *
+     * Sequential on purpose. Polling every mailbox at once means opening every
+     * connection at once, which is exactly the burst a mail server throttles.
+     *
+     * @returns {Promise<void>}
+     */
+    async function tick() {
+        if (stopped) return;
+
+        let accounts;
+        try {
+            accounts = await listAccounts();
+        } catch (err) {
+            logger.warn(`[mail] could not list mailboxes to poll: ${err.message}`);
+            return;
+        }
+
+        // Forget state for mailboxes that have been removed, so the map does
+        // not grow for the life of the process.
+        const live = new Set(accounts.map(a => a.id));
+        for (const id of [...state.keys()]) {
+            if (!live.has(id)) state.delete(id);
+        }
+
+        const due = selectDuePolls(accounts, state, {now: Date.now(), defaultMinutes});
+        for (const {id} of due) {
+            if (stopped) return;
+            const account = accounts.find(a => a.id === id);
+            if (account) await runOne(account);
+        }
+    }
+
+    return {
+        /**
+         * @returns {void}
+         */
+        start() {
+            if (timer) return;
+            stopped = false;
+            timer = setInterval(() => {
+                tick().catch(err => logger.warn(`[mail] poll tick failed: ${err.message}`));
+            }, tickMs);
+            // Never hold the process open for a poll.
+            timer.unref?.();
+        },
+
+        /**
+         * @returns {void}
+         */
+        stop() {
+            stopped = true;
+            if (timer) clearInterval(timer);
+            timer = null;
+        },
+
+        /**
+         * Run a pass immediately, for a manual "sync now".
+         *
+         * @returns {Promise<void>}
+         */
+        pollNow: tick,
+
+        /**
+         * What the poller currently believes about each mailbox.
+         *
+         * @returns {object}
+         */
+        state() {
+            return Object.fromEntries(state);
+        }
+    };
+}