domma-cms 0.54.2 → 0.66.0

package/plugins/_lib/mail/envelope.js

@@ -0,0 +1,161 @@
+/**
+ * The shape the mirror holds one message in.
+ *
+ * Its own module because two things write mirror rows - the sync engine on a
+ * poll, and a move ingesting the message it just placed in the destination -
+ * and two definitions of "a stored message" would drift. A row written by one
+ * path and read by the other has to be indistinguishable.
+ *
+ * @module _lib/mail/envelope
+ */
+import {threadRoot} from '../admin/mail/threads.js';
+import {looksAutomated} from '../admin/mail/vacation.js';
+import {readPriority} from './priority.js';
+
+/**
+ * What a fetch asks for when it wants a full envelope.
+ *
+ * Priority is not IN the envelope - there is no such field - so the three
+ * headers that carry it are fetched alongside. A handful of bytes per message,
+ * and the alternative is a second round trip per row.
+ *
+ * `references` and `list-id` are here for the same reason. IMAP's ENVELOPE
+ * carries `Message-ID` and `In-Reply-To` but not `References`, which is the
+ * header a conversation is actually reconstructed from; and `List-Id` is the
+ * one thing that reliably distinguishes a mailing list from a person, which
+ * is most of what filing rules are written about.
+ */
+export const ENVELOPE_QUERY = {
+    uid: true,
+    envelope: true,
+    flags: true,
+    size: true,
+    bodyStructure: true,
+    headers: [
+        'x-priority', 'importance', 'x-msmail-priority', 'references', 'list-id',
+        // What tells an away message not to answer. Fetched here rather than
+        // per message later: the responder runs over new mail on every poll,
+        // and a second round trip per message to read three headers would be
+        // the most expensive part of the feature.
+        'auto-submitted', 'precedence', 'return-path'
+    ]
+};
+
+/**
+ * Does this structure carry something a reader would call an attachment?
+ *
+ * @param   {object} node
+ * @returns {boolean}
+ */
+export function hasAttachment(node) {
+    if (!node) return false;
+    if (node.disposition === 'attachment') return true;
+    return (node.childNodes ?? []).some(hasAttachment);
+}
+
+/**
+ * Turn an ImapFlow message into the shape the store holds.
+ *
+ * @param   {object} message
+ * @returns {object}
+ */
+export function toRecord(message) {
+    const flags = [...(message.flags ?? [])];
+    const envelope = message.envelope ?? {};
+    const addresses = list => (list ?? []).map(a => ({name: a.name ?? '', address: a.address ?? ''}));
+    const headers = parseHeaderBlock(message.headers);
+
+    const record = {
+        uid: message.uid,
+        seq: message.seq,
+        subject: envelope.subject ?? '',
+        from: addresses(envelope.from),
+        to: addresses(envelope.to),
+        // Carried so an away message can tell "addressed to me" from
+        // "delivered to me" - a Bcc'd blast is exactly what must not be
+        // answered five hundred times.
+        cc: addresses(envelope.cc),
+        date: envelope.date ?? null,
+        size: message.size ?? 0,
+        flags,
+        seen: flags.includes('\\Seen'),
+        flagged: flags.includes('\\Flagged'),
+        answered: flags.includes('\\Answered'),
+        hasAttachment: hasAttachment(message.bodyStructure),
+        messageId: envelope.messageId ?? null,
+        inReplyTo: envelope.inReplyTo ?? null,
+        // Stored as it arrived - a space-separated string - and normalised
+        // by whoever reads it. Splitting here would make the stored shape
+        // depend on which server sent it.
+        references: typeof headers.get === 'function'
+            ? (headers.get('references') ?? null)
+            : (headers.references ?? null),
+        // The bare list id, without the angle brackets and without the
+        // descriptive part some senders put in front of it, so a rule can be
+        // written about "github.com" rather than about
+        // "My List <list.github.com>".
+        listId: readListId(headers),
+        // Decided once, at fetch time, so the responder is a field read rather
+        // than a header parse. This single boolean is what closes the mail
+        // loop: our own replies carry `Auto-Submitted: auto-replied`, so they
+        // arrive at the far end already marked as not worth answering.
+        automated: looksAutomated(headers),
+        priority: readPriority(headers)
+    };
+
+    // Computed here rather than by the caller, so every writer of a mirror
+    // row agrees about it - the sync engine and the move that ingests a
+    // message it has just placed are two different code paths writing the
+    // same collection.
+    return {...record, threadRoot: threadRoot(record)};
+}
+
+/**
+ * The List-Id header, reduced to the bit worth matching on.
+ *
+ * `List-Id: Dev Discussion <dev.example.com>` answers `dev.example.com`. A
+ * sender that gives only the bracketed part, or only a bare token, answers
+ * that instead.
+ *
+ * @param   {Map|object} headers
+ * @returns {string}
+ */
+export function readListId(headers) {
+    const raw = typeof headers?.get === 'function'
+        ? headers.get('list-id')
+        : headers?.['list-id'];
+    const value = String(raw ?? '').trim();
+    if (!value) return '';
+    const angled = value.match(/<([^>]+)>/);
+    return (angled ? angled[1] : value).trim();
+}
+
+/**
+ * Turn imapflow's raw header buffer into something `readPriority` can read.
+ *
+ * A `headers` fetch hands back the raw bytes of the requested block, not a
+ * parsed map - so this does the one line of parsing needed, rather than
+ * pulling in a MIME parser to read three fields.
+ *
+ * @param   {Buffer|string|Map|object|null} raw
+ * @returns {Object.<string, string>}
+ */
+export function parseHeaderBlock(raw) {
+    if (!raw) return {};
+    // Already a map or object from another path - hand it straight on.
+    if (typeof raw.get === 'function' || (typeof raw === 'object' && !Buffer.isBuffer(raw)
+        && typeof raw !== 'string')) {
+        return raw;
+    }
+
+    const out = {};
+    // Unfold first: a header may continue onto the next line indented, and
+    // splitting naively would lose the continuation.
+    const text = String(raw).replace(/\r?\n[ \t]+/g, ' ');
+    for (const line of text.split(/\r?\n/)) {
+        const at = line.indexOf(':');
+        if (at === -1) continue;
+        out[line.slice(0, at).trim().toLowerCase()] = line.slice(at + 1).trim();
+    }
+    return out;
+}

package/plugins/_lib/mail/folders.js

@@ -0,0 +1,192 @@
+/**
+ * Creating, renaming and deleting mailboxes.
+ *
+ * The IMAP calls are one line each; what is worth having in its own module is
+ * everything around them. A mailbox path is a string with a server-chosen
+ * delimiter in it, so "create a folder called X inside Y" is path arithmetic
+ * that goes wrong quietly - a name containing the delimiter silently creates a
+ * nested folder, and renaming a parent moves every child with it whether the
+ * caller expected that or not.
+ *
+ * Pure, so the arithmetic and the refusals can be tested without a server.
+ *
+ * @module _lib/mail/folders
+ */
+
+/**
+ * Folders the server has told us it uses for something.
+ *
+ * Deleting or renaming one of these does not tidy a mailbox, it breaks it:
+ * the server files sent mail in Sent by SPECIAL-USE, and a client that cannot
+ * find Trash falls back to expunging.
+ */
+const PROTECTED_USES = new Set([
+    '\\Inbox', '\\Sent', '\\Drafts', '\\Trash', '\\Junk', '\\Archive', '\\All', '\\Flagged'
+]);
+
+/** The longest a single folder name may be. Servers vary; this is generous. */
+const MAX_NAME = 100;
+
+/**
+ * Is this a usable name for one folder?
+ *
+ * The delimiter is refused rather than escaped. Someone typing `Work.2026`
+ * into a "folder name" box means a folder with a dot in it, but the server
+ * would read it as `2026` inside `Work` - so the honest answer is to say the
+ * character cannot be used rather than to quietly create something else.
+ *
+ * @param   {string} name
+ * @param   {string} delimiter - the server's hierarchy separator
+ * @returns {{ok: true} | {ok: false, error: string}}
+ */
+export function validateFolderName(name, delimiter) {
+    const trimmed = String(name ?? '').trim();
+    if (!trimmed) return {ok: false, error: 'A folder needs a name.'};
+    if (trimmed.length > MAX_NAME) {
+        return {ok: false, error: `A folder name cannot be longer than ${MAX_NAME} characters.`};
+    }
+    if (delimiter && trimmed.includes(delimiter)) {
+        return {
+            ok: false,
+            error: `A folder name cannot contain "${delimiter}" - that is how this server separates folders. Create it inside another folder instead.`
+        };
+    }
+    // Control characters are not typed on purpose and break the protocol line.
+    if (/[\u0000-\u001f\u007f]/.test(trimmed)) {
+        return {ok: false, error: 'A folder name cannot contain control characters.'};
+    }
+    return {ok: true};
+}
+
+/**
+ * The full path a new folder would have.
+ *
+ * @param   {string|null} parentPath - null or '' for a top-level folder
+ * @param   {string} name
+ * @param   {string} delimiter
+ * @returns {string}
+ */
+export function buildFolderPath(parentPath, name, delimiter) {
+    const leaf = String(name ?? '').trim();
+    const parent = String(parentPath ?? '').trim();
+    if (!parent) return leaf;
+    // A server with no delimiter has a flat namespace, so there is no "inside".
+    if (!delimiter) return leaf;
+    return `${parent}${delimiter}${leaf}`;
+}
+
+/**
+ * The path a folder would have after being renamed, keeping it where it is.
+ *
+ * Renaming is a MOVE in IMAP - there is no rename verb - so the new path has
+ * to carry the same parent, or "rename Contracts to Deals" would move it to
+ * the top level.
+ *
+ * @param   {string} path - the folder's current full path
+ * @param   {string} name - the new leaf name
+ * @param   {string} delimiter
+ * @returns {string}
+ */
+export function renameTarget(path, name, delimiter) {
+    const current = String(path ?? '');
+    const cut = delimiter ? current.lastIndexOf(delimiter) : -1;
+    const parent = cut === -1 ? '' : current.slice(0, cut);
+    return buildFolderPath(parent, name, delimiter);
+}
+
+/**
+ * May this folder be renamed or deleted?
+ *
+ * @param   {{path: string, specialUse?: string|null}} folder
+ * @param   {{action?: string}} [options]
+ * @returns {{ok: true} | {ok: false, error: string}}
+ */
+export function canModifyFolder(folder, {action = 'change'} = {}) {
+    const path = String(folder?.path ?? '').trim();
+    if (!path) return {ok: false, error: 'No folder was given.'};
+
+    // INBOX is INBOX on every server, whatever it reports for SPECIAL-USE.
+    if (path.toUpperCase() === 'INBOX') {
+        return {ok: false, error: `The Inbox cannot be ${action}d.`};
+    }
+    if (folder?.specialUse && PROTECTED_USES.has(folder.specialUse)) {
+        return {
+            ok: false,
+            error: `This server uses that folder for ${describeUse(folder.specialUse)}, so it cannot be ${action}d.`
+        };
+    }
+    return {ok: true};
+}
+
+/**
+ * A SPECIAL-USE attribute in words.
+ *
+ * @param   {string} use
+ * @returns {string}
+ */
+function describeUse(use) {
+    return {
+        '\\Sent': 'sent mail',
+        '\\Drafts': 'drafts',
+        '\\Trash': 'deleted mail',
+        '\\Junk': 'spam',
+        '\\Archive': 'archived mail',
+        '\\All': 'all mail',
+        '\\Flagged': 'starred mail'
+    }[use] ?? 'something';
+}
+
+/**
+ * The folders that would disappear along with this one.
+ *
+ * IMAP deletes one mailbox at a time, and a server may refuse to delete a
+ * parent that still has children - but more importantly, someone deleting
+ * `Work` should be told it takes `Work.Contracts.2026` with it BEFORE they
+ * agree to it, not after.
+ *
+ * @param   {string} path
+ * @param   {{path: string}[]} folders - every folder on the account
+ * @param   {string} delimiter
+ * @returns {string[]} descendant paths, deepest first
+ */
+export function descendantsOf(path, folders, delimiter) {
+    if (!delimiter) return [];
+    const prefix = `${path}${delimiter}`;
+    return (folders ?? [])
+        .map(f => f.path)
+        .filter(p => p.startsWith(prefix))
+        // Deepest first, so a server that refuses to delete a non-empty
+        // parent is never asked to.
+        .sort((a, b) => b.split(delimiter).length - a.split(delimiter).length || b.localeCompare(a));
+}
+
+/**
+ * Work out which mailboxes are actually subscribed.
+ *
+ * Harder than it looks, and the reason this is a function rather than a
+ * property read. imapflow reports an UNSUBSCRIBED mailbox as
+ * `subscribed: undefined` - not `false` - so a `!== false` test reads every
+ * hidden folder as visible, which is exactly how this first shipped.
+ *
+ * Reading `=== true` instead has the opposite failure: a server that does not
+ * return the SUBSCRIBED attribute at all reports `undefined` for everything,
+ * and every folder would read as hidden. So the attribute is trusted only
+ * when the server demonstrably uses it - if not one mailbox claims to be
+ * subscribed, the server is not answering the question and everything is
+ * treated as visible.
+ *
+ * `list({subscribed: true})` is not a way out: on a LIST-EXTENDED server it
+ * came back with all 104 mailboxes including the unsubscribed one.
+ *
+ * @param   {{path: string, subscribed?: boolean}[]} boxes
+ * @returns {Map<string, boolean>} path -> visible
+ */
+export function resolveSubscriptions(boxes) {
+    const list = boxes ?? [];
+    const serverAnswers = list.some(box => box.subscribed === true);
+    const out = new Map();
+    for (const box of list) {
+        out.set(box.path, serverAnswers ? box.subscribed === true : true);
+    }
+    return out;
+}

package/plugins/_lib/mail/handoff.js

@@ -0,0 +1,274 @@
+/**
+ * Getting mail out of the mailbox and into the CMS.
+ *
+ * The thing a mail plugin inside a content management system can do that a
+ * standalone client cannot: an attachment becomes a media file, and a message
+ * becomes a draft page or a collection entry. Every other feature here is one
+ * a webmail client would have; these are the ones that are the point of it
+ * living in Domma at all.
+ *
+ * Pure, and separate from the routes that use it, because the rules about
+ * what may be written into the public media directory are the security
+ * boundary of the whole feature and need testing without a mail server.
+ *
+ * @module _lib/mail/handoff
+ */
+
+/**
+ * Attachment types that may be saved into the media library.
+ *
+ * Deliberately NARROWER than the CMS's own upload allowlist, in two places:
+ *
+ *   - **No SVG.** An SVG is a script container, and `/media/` is served from
+ *     the site's own origin. The CMS allows one on a deliberate upload by an
+ *     administrator who chose the file; this is a file a stranger emailed in,
+ *     and one click between the two is not enough separation.
+ *   - **No HTML.** For the same reason, more obviously.
+ *
+ * Anything not listed is refused rather than saved under a safe extension:
+ * renaming a file does not change what it is, and a media library full of
+ * `.bin` is not a media library.
+ */
+export const SAVEABLE_TYPES = new Map([
+    ['image/jpeg', 'jpg'], ['image/jpg', 'jpg'], ['image/png', 'png'],
+    ['image/gif', 'gif'], ['image/webp', 'webp'], ['image/x-icon', 'ico'],
+    ['image/bmp', 'bmp'], ['image/tiff', 'tiff'],
+    ['application/pdf', 'pdf'], ['text/plain', 'txt'], ['text/csv', 'csv'],
+    ['application/msword', 'doc'],
+    ['application/vnd.openxmlformats-officedocument.wordprocessingml.document', 'docx'],
+    ['application/vnd.ms-excel', 'xls'],
+    ['application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', 'xlsx'],
+    ['video/mp4', 'mp4'], ['video/webm', 'webm'], ['video/ogg', 'ogv'],
+    ['audio/mpeg', 'mp3'], ['audio/mp3', 'mp3'], ['audio/ogg', 'oga'],
+    ['audio/wav', 'wav'], ['audio/webm', 'weba']
+]);
+
+/**
+ * The media type of an attachment, without its parameters.
+ *
+ * @param   {*} value
+ * @returns {string}
+ */
+export function mediaType(value) {
+    return String(value ?? '').toLowerCase().split(';')[0].trim();
+}
+
+/**
+ * The name this attachment will be saved under.
+ *
+ * The extension comes from the ALLOWED TYPE, never from the sender's
+ * filename, and that is the whole point of this function.
+ *
+ * `/media/` is served by `@fastify/static`, which sets `Content-Type` from
+ * the file extension. The declared MIME type and the filename are two
+ * separate fields, both chosen by whoever sent the message - so an
+ * attachment declared `image/png` and named `logo.svg` passed the type check
+ * and then landed on disk as `.svg`, served from the site's own origin as
+ * `image/svg+xml`. An SVG is a script container and that origin is where the
+ * admin's token lives. `nosniff` does not help: it stops the browser
+ * sniffing AWAY from a declared type, not the extension declaring one.
+ *
+ * Rewriting rather than refusing, because a mislabelled extension is far more
+ * often a sender being careless than a sender being hostile - and a file
+ * saved as the type it actually claims to be is both safe and correct.
+ *
+ * @param   {string} filename
+ * @param   {string} contentType
+ * @returns {string}
+ */
+export function mediaNameFor(filename, contentType) {
+    const extension = SAVEABLE_TYPES.get(mediaType(contentType));
+    const base = safeName(filename);
+    // Everything before the last dot, so `report.final.pdf` keeps its stem.
+    const dot = base.lastIndexOf('.');
+    const stem = (dot > 0 ? base.slice(0, dot) : base) || 'attachment';
+    return extension ? `${stem}.${extension}` : stem;
+}
+
+/**
+ * Make an attachment's filename safe to write.
+ *
+ * The same transformation the media route uses - `path.basename` then a
+ * character allowlist - spelled out rather than imported, because this module
+ * is deliberately free of node built-ins so it can be tested and reasoned
+ * about on its own. A name that reduces to nothing gets one.
+ *
+ * @param   {string} name
+ * @returns {string}
+ */
+export function safeName(name) {
+    const base = String(name ?? '')
+        // Everything up to and including the last separator, either kind.
+        .replace(/^.*[\\/]/, '')
+        .replace(/[^a-zA-Z0-9._-]/g, '_')
+        // A leading dot makes a hidden file; a name of only dots makes none.
+        .replace(/^\.+/, '');
+    return base.slice(0, 120) || 'attachment';
+}
+
+/**
+ * A name that is not already taken, by adding a counter before the extension.
+ *
+ * `saveMedia` overwrites silently, so a second `invoice.pdf` from a second
+ * sender would replace the first. Anything derived from mail has to assume
+ * the name is not unique, because it was chosen by somebody else.
+ *
+ * @param   {string} name
+ * @param   {Set<string>|string[]} taken
+ * @returns {string}
+ */
+export function uniqueName(name, taken) {
+    const used = taken instanceof Set ? taken : new Set(taken ?? []);
+    const clean = safeName(name);
+    if (!used.has(clean)) return clean;
+
+    const dot = clean.lastIndexOf('.');
+    const stem = dot > 0 ? clean.slice(0, dot) : clean;
+    const ext = dot > 0 ? clean.slice(dot) : '';
+
+    for (let n = 2; n < 1000; n += 1) {
+        const candidate = `${stem}-${n}${ext}`;
+        if (!used.has(candidate)) return candidate;
+    }
+    // A thousand files of one name is not a case worth a cleverer answer.
+    return `${stem}-${Date.now()}${ext}`;
+}
+
+/**
+ * May this attachment be saved into the media library?
+ *
+ * @param   {{filename?: string, contentType?: string, size?: number}} attachment
+ * @param   {{maxBytes?: number}} [limits]
+ * @returns {{ok: true} | {ok: false, error: string}}
+ */
+export function canSave(attachment, {maxBytes = 20_000_000} = {}) {
+    const type = mediaType(attachment?.contentType);
+    if (!SAVEABLE_TYPES.has(type)) {
+        return {
+            ok: false,
+            error: `${type || 'That file type'} cannot be saved to the media library. Images, documents, audio and video can.`
+        };
+    }
+    if (Number(attachment?.size ?? 0) > maxBytes) {
+        return {ok: false, error: 'That attachment is too large for the media library.'};
+    }
+    return {ok: true};
+}
+
+/**
+ * A URL path for a page made from a message.
+ *
+ * Under a folder rather than at the root: a mailbox can produce a great many
+ * of these, and a site whose top level fills up with them is a site nobody
+ * will use the feature on twice. Dated, because two messages very often share
+ * a subject.
+ *
+ * @param   {string} subject
+ * @param   {Date} [date]
+ * @param   {string} [prefix]
+ * @returns {string}
+ */
+export function pagePathFor(subject, date = new Date(), prefix = 'from-mail') {
+    const slug = String(subject ?? '')
+        .toLowerCase()
+        .replace(/^((re|fwd|fw)\s*:\s*)+/i, '')
+        .replace(/[^a-z0-9]+/g, '-')
+        .replace(/^-+|-+$/g, '')
+        .slice(0, 60) || 'message';
+
+    const when = Number.isNaN(date?.getTime?.()) ? new Date() : date;
+    const stamp = when.toISOString().slice(0, 10);
+    return `/${prefix}/${stamp}-${slug}`;
+}
+
+/**
+ * Turn a message's plain text into a Markdown body.
+ *
+ * The TEXT part, never the HTML. Mail HTML is table layouts, inline styles
+ * and tracking, and embedding it in a Markdown file produces a page that is
+ * unusable to edit and, because a blank line ends an HTML block in Markdown,
+ * frequently one that renders its own markup as prose halfway down. The text
+ * part is what the message actually says.
+ *
+ * Quoted material is dropped: a page made from the fourth message in a thread
+ * should be that message, not it and the three under it.
+ *
+ * @param   {string} text
+ * @returns {string}
+ */
+export function textToMarkdown(text) {
+    const lines = String(text ?? '').replace(/\r\n/g, '\n').split('\n');
+
+    const kept = [];
+    for (const line of lines) {
+        // The quote and everything after it. Attribution lines vary too much
+        // to match reliably, but the first `>` is dependable and is where the
+        // original message starts in every client that quotes at all.
+        if (/^\s*>/.test(line)) break;
+        // The plain-text signature delimiter, likewise.
+        if (/^--\s*$/.test(line)) break;
+        kept.push(line);
+    }
+
+    // The attribution line sits ABOVE the first `>`, so breaking on the quote
+    // leaves it behind - "On Monday, Sam wrote:" as the last line of a page
+    // about something else. Dropped from the end, along with the blank line
+    // that introduced it, and only from the end: the same words in the middle
+    // of a message are the sender quoting someone in prose.
+    while (kept.length) {
+        const last = kept[kept.length - 1].trim();
+        if (!last) { kept.pop(); continue; }
+        if (/wrote:$/i.test(last) || /^-{2,}\s*(original|forwarded) message/i.test(last)) {
+            kept.pop();
+            continue;
+        }
+        break;
+    }
+
+    return kept
+        .join('\n')
+        // Markdown treats a single newline as a space, so a hard-wrapped mail
+        // would reflow into one paragraph. Blank-line-separated blocks are
+        // paragraphs; single breaks inside them keep their line.
+        .replace(/[ \t]+$/gm, '')
+        .replace(/\n{3,}/g, '\n\n')
+        .trim();
+}
+
+/**
+ * Match a message onto a collection's fields, by what they are called.
+ *
+ * A guess, offered to the user rather than applied behind their back - the
+ * route hands the mapping back and the screen shows it, so a field called
+ * `body` that was meant for something else is visible before anything is
+ * written.
+ *
+ * @param   {Array<{name: string, type?: string}>} fields
+ * @returns {Object.<string, string>} field name -> message property
+ */
+export function guessFieldMapping(fields) {
+    const candidates = {
+        subject: ['subject', 'title', 'name', 'heading'],
+        body: ['body', 'content', 'message', 'text', 'description', 'notes'],
+        fromName: ['from', 'sender', 'author', 'contact', 'fullname', 'name'],
+        fromEmail: ['email', 'emailaddress', 'from_email', 'fromemail', 'sender_email'],
+        date: ['date', 'received', 'sent', 'createdat', 'when']
+    };
+
+    const mapping = {};
+    const used = new Set();
+
+    // In this order, so `name` goes to the subject on a collection that has
+    // both `name` and `from` rather than to the sender.
+    for (const [property, names] of Object.entries(candidates)) {
+        for (const field of fields ?? []) {
+            const key = String(field?.name ?? '').toLowerCase().replace(/[^a-z]/g, '');
+            if (used.has(field.name)) continue;
+            if (!names.includes(key)) continue;
+            mapping[field.name] = property;
+            used.add(field.name);
+            break;
+        }
+    }
+    return mapping;
+}