@pithy-sh/support 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +17 -0
- package/package.json +68 -0
- package/pithy.manifest.json +40 -0
- package/src/ai/classify.ts +239 -0
- package/src/attachment/store.ts +78 -0
- package/src/audit/actions.ts +71 -0
- package/src/capability.ts +293 -0
- package/src/client/projection.ts +60 -0
- package/src/cloudflare-test.d.ts +15 -0
- package/src/config/config.ts +400 -0
- package/src/data/attachment.ts +65 -0
- package/src/data/billingScope.ts +32 -0
- package/src/data/categories.ts +117 -0
- package/src/data/classification.ts +50 -0
- package/src/data/enums.ts +90 -0
- package/src/data/flag.ts +37 -0
- package/src/data/message.ts +224 -0
- package/src/data/tables.ts +58 -0
- package/src/data/thread.ts +138 -0
- package/src/error/errors.ts +133 -0
- package/src/http/guards.ts +59 -0
- package/src/http/handlers.ts +418 -0
- package/src/http/resolve.ts +109 -0
- package/src/http/responses.ts +506 -0
- package/src/http/routes.ts +272 -0
- package/src/http/schemas.ts +251 -0
- package/src/http/scopes.ts +117 -0
- package/src/http/views.ts +169 -0
- package/src/inbound/authenticity.ts +114 -0
- package/src/inbound/guard.ts +127 -0
- package/src/inbound/handler.ts +102 -0
- package/src/inbound/ingest.ts +548 -0
- package/src/inbound/recipient.ts +67 -0
- package/src/index.ts +63 -0
- package/src/link/sender.ts +334 -0
- package/src/migrations/0001_threads.ts +296 -0
- package/src/mime/address.ts +37 -0
- package/src/mime/parse.ts +299 -0
- package/src/mime/sanitize.ts +253 -0
- package/src/mime/threading.ts +127 -0
- package/src/mime/truncate.ts +55 -0
- package/src/provision/provisionSupport.ts +179 -0
- package/src/provision/resolveSupportConfig.ts +67 -0
- package/src/reply/send.ts +322 -0
- package/src/reply/snippets.ts +167 -0
- package/src/secret/registry.ts +24 -0
- package/src/seeds/example.ts +385 -0
- package/src/store/paging.ts +22 -0
- package/src/store/search.ts +197 -0
- package/src/store/searchIndex.ts +71 -0
- package/src/store/threads.ts +452 -0
- package/src/submission/encoding.ts +66 -0
- package/src/submission/guard.ts +120 -0
- package/src/submission/submit.ts +539 -0
- package/src/version.generated.ts +16 -0
- package/src/workflows/classify.ts +164 -0
- package/src/workflows/retryPolicy.ts +48 -0
- package/src/workflows/specs.ts +61 -0
- package/src/workflows/worker.ts +82 -0
- package/src/workflows/wrangler.jsonc +46 -0
|
@@ -0,0 +1,299 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
import { parseAddress } from "@pithy-sh/core/src/address/address";
|
|
5
|
+
import PostalMime, { type Address, type Attachment, type Email } from "postal-mime";
|
|
6
|
+
import { SupportUnparseableMessageError } from "../error/errors";
|
|
7
|
+
import { normalizeDisplayName } from "./address";
|
|
8
|
+
import { normalizeMessageId, parseReferences } from "./threading";
|
|
9
|
+
import { truncateToBytes } from "./truncate";
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Inbound MIME, parsed into the shape the rest of the capability works in.
|
|
13
|
+
*
|
|
14
|
+
* `@pithy-sh/email` already depends on `postal-mime` and already parses inbound mail — but only its
|
|
15
|
+
* headers, to classify a bounce. Everything below the headers is untouched there, so multipart
|
|
16
|
+
* bodies, attachments, and the threading chain are parsed here for the first time.
|
|
17
|
+
*
|
|
18
|
+
* **This module does not sanitize.** `html` comes back exactly as it arrived, and the caller runs it
|
|
19
|
+
* through `sanitizeHtml` before anything stores or renders it. Keeping the two apart is what lets
|
|
20
|
+
* this file be unit-tested under node against real messages — `HTMLRewriter` is a Workers global —
|
|
21
|
+
* and it keeps the parse honest: a sanitizer that ran here would quietly make the raw bytes and the
|
|
22
|
+
* parsed form disagree about what arrived.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
/** One attachment, decoded, with its declared metadata carried as-declared. */
|
|
26
|
+
export interface ParsedAttachment {
|
|
27
|
+
/** The filename the sender declared, already stripped of path separators and control characters. */
|
|
28
|
+
filename: string;
|
|
29
|
+
/** The MIME type the sender declared. Recorded, never honored by a serve path. */
|
|
30
|
+
contentType: string;
|
|
31
|
+
/** The decoded bytes. */
|
|
32
|
+
bytes: Uint8Array;
|
|
33
|
+
/** The `Content-ID`, when the part carried one — what an inline image refers to by `cid:`. */
|
|
34
|
+
contentId?: string;
|
|
35
|
+
/** Whether the part declared `Content-Disposition: inline`. */
|
|
36
|
+
inline: boolean;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** One inbound message, parsed. */
|
|
40
|
+
export interface ParsedInboundMessage {
|
|
41
|
+
/** The `Message-ID`, without angle brackets. Absent when the sender omitted one. */
|
|
42
|
+
messageId?: string;
|
|
43
|
+
/** The `In-Reply-To` id, without angle brackets — the direct parent. */
|
|
44
|
+
inReplyTo?: string;
|
|
45
|
+
/** The `References` chain, oldest first. */
|
|
46
|
+
references: string[];
|
|
47
|
+
/** The sender's address, normalized. */
|
|
48
|
+
fromAddress: string;
|
|
49
|
+
/** The sender's display name, if any — untrusted text, already stripped of control characters. */
|
|
50
|
+
fromName?: string;
|
|
51
|
+
/** Every address named in `To`/`Cc`/`Delivered-To`, normalized. Corroborating evidence, never authority. */
|
|
52
|
+
headerRecipients: string[];
|
|
53
|
+
/** The subject, bounded. Empty when the message had none. */
|
|
54
|
+
subject: string;
|
|
55
|
+
/** The plain-text body. Empty when the message carried only HTML — the caller derives text from it. */
|
|
56
|
+
text: string;
|
|
57
|
+
/** The HTML body **exactly as it arrived**, unsanitised. Absent when the message carried none. */
|
|
58
|
+
html?: string;
|
|
59
|
+
/** The decoded attachments, in the order the parts appeared. */
|
|
60
|
+
attachments: ParsedAttachment[];
|
|
61
|
+
/**
|
|
62
|
+
* The `Authentication-Results` verdicts the receiving MTA stamped, lowercased, keyed by method
|
|
63
|
+
* (`dmarc`, `spf`, `dkim`). Empty when the header was absent — which is itself a signal.
|
|
64
|
+
*
|
|
65
|
+
* Read from the **topmost** such header only. A sender may include one of their own, and it will be
|
|
66
|
+
* below the MTA's; reading any but the first is a forged verdict.
|
|
67
|
+
*/
|
|
68
|
+
authResults: Record<string, string>;
|
|
69
|
+
/**
|
|
70
|
+
* Which sender-authentication headers were actually present in the received bytes.
|
|
71
|
+
*
|
|
72
|
+
* **A diagnostic, not a signal** — nothing branches on it, and it is deliberately not stored. It
|
|
73
|
+
* exists to settle a question this package cannot answer by reading documentation: whether
|
|
74
|
+
* Cloudflare Email Routing delivers a Worker the headers a sender-authenticity check would need.
|
|
75
|
+
* `Authentication-Results`, `Received`, and `DKIM-Signature` are all reported missing from
|
|
76
|
+
* `message.headers` (cloudflare/workerd#6740), but `message.raw` — which is what this parses — is
|
|
77
|
+
* documented as the raw MIME and is a different surface. Inferring the answer from a GitHub issue
|
|
78
|
+
* is guessing; logging what arrived means the first real delivery answers it.
|
|
79
|
+
*
|
|
80
|
+
* See issue #47, which is where that live verification lives.
|
|
81
|
+
*/
|
|
82
|
+
authHeadersSeen: string[];
|
|
83
|
+
/**
|
|
84
|
+
* Whether this looks machine-generated: an out-of-office, a mailing-list post, or a bulk send.
|
|
85
|
+
*
|
|
86
|
+
* Not a reason to drop the message — a vacation responder appended to a real thread is context an
|
|
87
|
+
* operator wants — but a good reason not to pay a model to classify it. Read by the ingest path to
|
|
88
|
+
* skip the classification dispatch, which is the only cost per message that is not fixed.
|
|
89
|
+
*/
|
|
90
|
+
autoSubmitted: boolean;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* A verdict at the start of its clause. Anchored: an unanchored scan matched `method=result` inside a
|
|
95
|
+
* *property tag*, and `smtp.mailfrom=` carries an address the sender chose.
|
|
96
|
+
*/
|
|
97
|
+
const VERDICT = /^\s*(dmarc|spf|dkim|iprev)\s*=\s*([a-z]+)/i;
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* The headers whose presence decides whether in-Worker sender authentication is possible at all.
|
|
101
|
+
* Reported, never trusted — see {@link ParsedInboundMessage.authHeadersSeen}.
|
|
102
|
+
*/
|
|
103
|
+
const AUTH_HEADERS = ["authentication-results", "dkim-signature", "arc-authentication-results", "received"];
|
|
104
|
+
|
|
105
|
+
/** The longest subject stored. Long enough for any real one; short enough not to be a storage lever. */
|
|
106
|
+
const MAX_SUBJECT = 500;
|
|
107
|
+
|
|
108
|
+
/** The longest text body stored, **in bytes**. A support request needing more than this has a file attached. */
|
|
109
|
+
export const MAX_TEXT_BODY = 256 * 1024;
|
|
110
|
+
|
|
111
|
+
/** Pull the first usable address out of whatever `postal-mime` produced for a header. */
|
|
112
|
+
function addressesOf(value: Address | Address[] | undefined): string[] {
|
|
113
|
+
const list = value === undefined ? [] : Array.isArray(value) ? value : [value];
|
|
114
|
+
const found: string[] = [];
|
|
115
|
+
for (const entry of list) {
|
|
116
|
+
// A group address (`Undisclosed recipients: ;`) carries its members rather than an address of
|
|
117
|
+
// its own, so both forms flatten to the same list before anything is normalized.
|
|
118
|
+
const members = entry.group !== undefined ? entry.group : entry.address !== undefined ? [entry] : [];
|
|
119
|
+
for (const member of members) {
|
|
120
|
+
const normalized = parseAddress(member.address);
|
|
121
|
+
if (normalized) found.push(normalized);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
return found;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* Make a declared filename safe to store and display.
|
|
129
|
+
*
|
|
130
|
+
* Path separators go first — a name is never part of a storage key here, but a dashboard that offers
|
|
131
|
+
* a download should not be handed `../../etc/passwd` to put in a `Content-Disposition`. Control
|
|
132
|
+
* characters and bidirectional overrides go with them: a filename carrying a right-to-left override
|
|
133
|
+
* renders as something other than what it is, which is the oldest attachment trick there is.
|
|
134
|
+
*/
|
|
135
|
+
export function safeFilename(value: string | null | undefined): string {
|
|
136
|
+
const cleaned = (value ?? "")
|
|
137
|
+
// biome-ignore-start lint/suspicious/noControlCharactersInRegex: stripping control
|
|
138
|
+
// characters is the entire purpose — a bidi override renders a filename or a display name as
|
|
139
|
+
// something other than what it is, which is the oldest trick in inbound mail.
|
|
140
|
+
.replace(/[\u0000-\u001f\u007f-\u009f\u202a-\u202e\u2066-\u2069]/g, "")
|
|
141
|
+
// biome-ignore-end lint/suspicious/noControlCharactersInRegex: see above
|
|
142
|
+
.replace(/[/\\]/g, "_")
|
|
143
|
+
.trim()
|
|
144
|
+
.slice(0, 200);
|
|
145
|
+
return cleaned.length > 0 ? cleaned : "attachment";
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* Parse `Authentication-Results` into `method → verdict`.
|
|
150
|
+
*
|
|
151
|
+
* Cloudflare stamps this before a message reaches a Worker, so it is the one trustworthy statement
|
|
152
|
+
* about the sender available here — everything else in the message is the sender's own claim. The
|
|
153
|
+
* format is `mta.example.com; spf=pass smtp.mailfrom=a@b; dkim=pass header.d=b; dmarc=pass`, and
|
|
154
|
+
* only the method/verdict pairs are read; the property tags after each are informational.
|
|
155
|
+
*/
|
|
156
|
+
/**
|
|
157
|
+
* The authserv-id out of an `Authentication-Results` first clause: the host name, with the optional
|
|
158
|
+
* version token and any CFWS comments removed, lowercased.
|
|
159
|
+
*/
|
|
160
|
+
function authservIdOf(clause: string | undefined): string {
|
|
161
|
+
return (clause ?? "")
|
|
162
|
+
.replace(/\([^)]*\)/g, " ")
|
|
163
|
+
.trim()
|
|
164
|
+
.replace(/\s+\d+$/, "")
|
|
165
|
+
.trim()
|
|
166
|
+
.toLowerCase();
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
export function parseAuthResults(value: string | undefined, expectedAuthservId?: string): Record<string, string> {
|
|
170
|
+
if (typeof value !== "string") return {};
|
|
171
|
+
|
|
172
|
+
// Quoted strings go first. A local part may legally be quoted and may then contain `;` and `=` —
|
|
173
|
+
// `"a;dmarc=pass"@evil.com` is a valid address — so a sender who chooses their own envelope address
|
|
174
|
+
// can otherwise manufacture what looks like a whole extra verdict clause inside the header the MTA
|
|
175
|
+
// stamped about them. Blanking quoted runs removes that alphabet entirely.
|
|
176
|
+
const unquoted = value.replace(/"(?:[^"\\]|\\.)*"/g, '""');
|
|
177
|
+
|
|
178
|
+
// RFC 8601: `authserv-id; method=result properties; method=result properties`. The authserv-id is
|
|
179
|
+
// the first field and every verdict lives at the *start* of a subsequent `;`-delimited clause.
|
|
180
|
+
const clauses = unquoted.split(";");
|
|
181
|
+
|
|
182
|
+
// Microsoft 365 stamps this header with **no authserv-id at all**, so its first clause is already a
|
|
183
|
+
// verdict. Skipping clause 0 unconditionally dropped that verdict — and, with `authservId`
|
|
184
|
+
// configured, compared a verdict against a hostname, failed, and returned nothing at all: the
|
|
185
|
+
// customer link silently off for every message behind Exchange Online. That is exactly the failure
|
|
186
|
+
// this function's own comment says a safety check must not have.
|
|
187
|
+
const leadingIsVerdict = VERDICT.test(clauses[0] ?? "");
|
|
188
|
+
|
|
189
|
+
if (expectedAuthservId !== undefined && !leadingIsVerdict) {
|
|
190
|
+
// Normalized before comparing, because RFC 8601 lets the authserv-id clause carry more than the
|
|
191
|
+
// id: an optional version token (`mx.example.com 1;`) and CFWS comments
|
|
192
|
+
// (`mx.example.com (amavisd-new);`) are both legal and both common. An exact match against the
|
|
193
|
+
// whole clause therefore fails for an adopter who configured the value correctly by reading it off
|
|
194
|
+
// a delivered message — and it fails *closed*, silently turning off the customer link for every
|
|
195
|
+
// message with nothing but an info log to show for it. A safety check that disables the feature it
|
|
196
|
+
// guards, when configured as documented, is worse than no check.
|
|
197
|
+
if (authservIdOf(clauses[0]) !== authservIdOf(expectedAuthservId)) return {};
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
const results: Record<string, string> = {};
|
|
201
|
+
// Clause 0 is read only when it *is* a verdict. Otherwise it is a host name, and an attacker who
|
|
202
|
+
// could get it read would only have to call themselves `dmarc=pass` to authenticate themselves.
|
|
203
|
+
for (const clause of leadingIsVerdict ? clauses : clauses.slice(1)) {
|
|
204
|
+
// **Anchored.** The old scan searched the whole string, so it matched a `method=result` pair
|
|
205
|
+
// appearing inside a *property tag* — and `smtp.mailfrom=` carries an address the sender chose.
|
|
206
|
+
// `MAIL FROM: <dmarc=pass@evil.com>` made the MTA's own honest header read as a DMARC pass.
|
|
207
|
+
// A verdict is only a verdict when it opens its clause.
|
|
208
|
+
const match = VERDICT.exec(clause);
|
|
209
|
+
const method = match?.[1]?.toLowerCase();
|
|
210
|
+
const verdict = match?.[2]?.toLowerCase();
|
|
211
|
+
// First clause wins for a method: a second one is a forgery appended after the real answer.
|
|
212
|
+
if (method && verdict && !(method in results)) results[method] = verdict;
|
|
213
|
+
}
|
|
214
|
+
return results;
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
/** Whether a header set marks the message as machine-generated. */
|
|
218
|
+
function isAutoSubmitted(headers: Map<string, string>): boolean {
|
|
219
|
+
const autoSubmitted = headers.get("auto-submitted")?.toLowerCase() ?? "";
|
|
220
|
+
if (autoSubmitted.startsWith("auto-")) return true;
|
|
221
|
+
const precedence = headers.get("precedence")?.toLowerCase() ?? "";
|
|
222
|
+
if (["bulk", "list", "junk", "auto_reply"].includes(precedence)) return true;
|
|
223
|
+
return headers.has("list-id") || headers.has("list-unsubscribe") || headers.has("x-autoreply");
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/** Decode `postal-mime`'s attachment content into bytes, whatever form it handed back. */
|
|
227
|
+
function attachmentBytes(attachment: Attachment): Uint8Array {
|
|
228
|
+
const content = attachment.content;
|
|
229
|
+
if (typeof content === "string") return new TextEncoder().encode(content);
|
|
230
|
+
if (content instanceof Uint8Array) return content;
|
|
231
|
+
return new Uint8Array(content);
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
/** Parse a raw MIME message. Throws `support/unparseable_message` when there is no usable sender. */
|
|
235
|
+
export async function parseInbound(
|
|
236
|
+
raw: string | ArrayBuffer | Uint8Array,
|
|
237
|
+
options: { expectedAuthservId?: string } = {},
|
|
238
|
+
): Promise<ParsedInboundMessage> {
|
|
239
|
+
let email: Email;
|
|
240
|
+
try {
|
|
241
|
+
email = await PostalMime.parse(raw);
|
|
242
|
+
} catch (cause) {
|
|
243
|
+
throw new SupportUnparseableMessageError({ detail: "postal-mime could not parse the message" }, { cause });
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
// **First occurrence wins, and this is a security boundary rather than a style choice.**
|
|
247
|
+
//
|
|
248
|
+
// `postal-mime` returns headers in document order, and a receiving MTA *prepends* its trace headers
|
|
249
|
+
// — `Authentication-Results` above all — while a sender's own copy of that header stays wherever
|
|
250
|
+
// they put it, which is below. A `Map.set` loop keeps the last occurrence, so last-wins hands the
|
|
251
|
+
// attacker's forged `dmarc=pass` to the authenticity check instead of Cloudflare's real verdict,
|
|
252
|
+
// and inverts the entire anti-spoofing control into a bypass.
|
|
253
|
+
//
|
|
254
|
+
// Trace headers are prepended as a class, so first-wins is the correct rule for every header read
|
|
255
|
+
// through this map, not only for the one that made it obvious.
|
|
256
|
+
const headers = new Map<string, string>();
|
|
257
|
+
for (const header of email.headers) {
|
|
258
|
+
const key = header.key.toLowerCase();
|
|
259
|
+
if (!headers.has(key)) headers.set(key, header.value);
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
const fromAddress = addressesOf(email.from)[0];
|
|
263
|
+
if (!fromAddress) {
|
|
264
|
+
// No sender means no thread key, no user link, and nothing to reply to. Refusing here is better
|
|
265
|
+
// than storing a row that can never be acted on.
|
|
266
|
+
throw new SupportUnparseableMessageError({ detail: "the message carried no parseable From address" });
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
const from = Array.isArray(email.from) ? email.from[0] : email.from;
|
|
270
|
+
const inReplyTo = normalizeMessageId(email.inReplyTo);
|
|
271
|
+
|
|
272
|
+
return {
|
|
273
|
+
messageId: normalizeMessageId(email.messageId),
|
|
274
|
+
inReplyTo,
|
|
275
|
+
references: parseReferences(email.references),
|
|
276
|
+
fromAddress,
|
|
277
|
+
fromName: normalizeDisplayName(from?.name),
|
|
278
|
+
headerRecipients: [
|
|
279
|
+
...new Set([
|
|
280
|
+
...addressesOf(email.to),
|
|
281
|
+
...addressesOf(email.cc),
|
|
282
|
+
...(parseAddress(email.deliveredTo) ? [parseAddress(email.deliveredTo) as string] : []),
|
|
283
|
+
]),
|
|
284
|
+
],
|
|
285
|
+
subject: (email.subject ?? "").replace(/\s+/g, " ").trim().slice(0, MAX_SUBJECT),
|
|
286
|
+
text: truncateToBytes(email.text ?? "", MAX_TEXT_BODY),
|
|
287
|
+
html: email.html,
|
|
288
|
+
attachments: email.attachments.map((attachment) => ({
|
|
289
|
+
filename: safeFilename(attachment.filename),
|
|
290
|
+
contentType: attachment.mimeType || "application/octet-stream",
|
|
291
|
+
bytes: attachmentBytes(attachment),
|
|
292
|
+
contentId: attachment.contentId?.replace(/^<|>$/g, "") || undefined,
|
|
293
|
+
inline: attachment.disposition === "inline",
|
|
294
|
+
})),
|
|
295
|
+
authResults: parseAuthResults(headers.get("authentication-results"), options.expectedAuthservId),
|
|
296
|
+
authHeadersSeen: AUTH_HEADERS.filter((name) => headers.has(name)),
|
|
297
|
+
autoSubmitted: isAutoSubmitted(headers),
|
|
298
|
+
};
|
|
299
|
+
}
|
|
@@ -0,0 +1,253 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
import { truncateToBytes } from "./truncate";
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* HTML sanitization for inbound mail, through the runtime's own streaming parser.
|
|
8
|
+
*
|
|
9
|
+
* This capability ingests attacker-controlled content and a dashboard renders it. That is the
|
|
10
|
+
* highest-risk sentence in the package, so the sanitizer is built on `HTMLRewriter` — workerd's real
|
|
11
|
+
* HTML parser — rather than on regular expressions. A regex sanitizer is a list of the tricks its
|
|
12
|
+
* author thought of; a parser-based one strips what the browser will actually see, which is the only
|
|
13
|
+
* question that matters. It is also why this file has no dependencies: the safest HTML parser
|
|
14
|
+
* available here already ships in the runtime.
|
|
15
|
+
*
|
|
16
|
+
* The policy is an **allowlist on both axes** — a small set of tags survives and a small set of
|
|
17
|
+
* attributes survives — because a denylist is wrong the day a new attribute is invented, and a
|
|
18
|
+
* support inbox will still be running then.
|
|
19
|
+
*
|
|
20
|
+
* `HTMLRewriter` is a Workers global, so it is touched only inside these functions. The module
|
|
21
|
+
* imports cleanly under node (which the `.describe()` meta-test requires); its tests are
|
|
22
|
+
* `.workers.test.ts`, running against the real parser rather than a stand-in for it.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Tags removed together with everything inside them.
|
|
27
|
+
*
|
|
28
|
+
* Two groups. The obvious executables (`script`, `iframe`, `object`, `embed`, `svg` — SVG is a
|
|
29
|
+
* scripting context, not an image format), and the quiet ones: `style` because CSS can position an
|
|
30
|
+
* invisible overlay over a real control, `link`/`base`/`meta` because they retarget or redirect the
|
|
31
|
+
* whole document, `form` and its inputs because a phishing form rendered inside a trusted dashboard
|
|
32
|
+
* is the most convincing phishing form there is, and `noscript`/`template` because their contents
|
|
33
|
+
* are inert to the parser here and live to a browser later.
|
|
34
|
+
*/
|
|
35
|
+
const DROPPED_TAGS = new Set([
|
|
36
|
+
"script",
|
|
37
|
+
"style",
|
|
38
|
+
"iframe",
|
|
39
|
+
"frame",
|
|
40
|
+
"frameset",
|
|
41
|
+
"object",
|
|
42
|
+
"embed",
|
|
43
|
+
"applet",
|
|
44
|
+
"link",
|
|
45
|
+
"meta",
|
|
46
|
+
"base",
|
|
47
|
+
"form",
|
|
48
|
+
"input",
|
|
49
|
+
"button",
|
|
50
|
+
"select",
|
|
51
|
+
"option",
|
|
52
|
+
"textarea",
|
|
53
|
+
"svg",
|
|
54
|
+
"math",
|
|
55
|
+
"noscript",
|
|
56
|
+
"template",
|
|
57
|
+
"title",
|
|
58
|
+
]);
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Attributes that survive, on any tag.
|
|
62
|
+
*
|
|
63
|
+
* Note what is absent and deliberate: `style` (see above), every `on*` handler, `srcset`/`ping`/
|
|
64
|
+
* `formaction`, and every `data-*`. Nothing needs an allowlist entry to be dropped — anything not on
|
|
65
|
+
* this list goes, which is what makes the policy hold against attributes that do not exist yet.
|
|
66
|
+
*/
|
|
67
|
+
const ALLOWED_ATTRIBUTES = new Set(["href", "alt", "title", "colspan", "rowspan", "start", "type", "dir", "lang"]);
|
|
68
|
+
|
|
69
|
+
/** URL schemes a surviving `href` may use. Everything else — `javascript:`, `data:`, `vbscript:` — is dropped. */
|
|
70
|
+
const ALLOWED_SCHEMES = ["http:", "https:", "mailto:", "tel:"];
|
|
71
|
+
|
|
72
|
+
/** The largest sanitized body kept, **in bytes**. A dashboard rendering 5 MB of markup has already lost. */
|
|
73
|
+
export const MAX_HTML_BYTES = 512 * 1024;
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Decode the character references a browser would decode before it resolves a scheme.
|
|
77
|
+
*
|
|
78
|
+
* `href="java script:alert(1)"` is a `javascript:` URL to every browser: the parser decodes the
|
|
79
|
+
* entity into a newline while reading the attribute, and the URL parser then strips it. A check that
|
|
80
|
+
* reads the raw attribute sees `java script:...`, whose first `#` looks like a fragment marker,
|
|
81
|
+
* and waves it through as relative — which is exactly the bypass this function exists to remove.
|
|
82
|
+
*
|
|
83
|
+
* Numeric, hex, and the three named references that can hide a scheme. Deliberately not a general
|
|
84
|
+
* entity decoder: this value is about to be compared against a scheme allowlist, and the only
|
|
85
|
+
* question worth answering is whether decoding could turn it into something that *is* a scheme.
|
|
86
|
+
*/
|
|
87
|
+
function decodeReferences(value: string): string {
|
|
88
|
+
return value
|
|
89
|
+
.replace(/&#x([0-9a-f]+);?/gi, (_, hex: string) => codePoint(Number.parseInt(hex, 16)))
|
|
90
|
+
.replace(/&#(\d+);?/g, (_, dec: string) => codePoint(Number.parseInt(dec, 10)))
|
|
91
|
+
.replace(/:?/gi, ":")
|
|
92
|
+
.replace(/&newline;?/gi, "\n")
|
|
93
|
+
.replace(/&tab;?/gi, "\t");
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* One decoded character, or the replacement character when the reference names nothing.
|
|
98
|
+
*
|
|
99
|
+
* `String.fromCodePoint` throws a `RangeError` above U+10FFFF, and `�` in an `href` is a
|
|
100
|
+
* string an attacker chooses. Unguarded it throws out of the attribute walk, rejects the whole
|
|
101
|
+
* `sanitizeHtml` promise, and fails ingest for that message — a mailable denial of service in the
|
|
102
|
+
* decoder that exists to make ingest safe. The HTML spec's own answer is U+FFFD, so this returns what
|
|
103
|
+
* a browser would have shown: a character that names no scheme and so decides nothing.
|
|
104
|
+
*/
|
|
105
|
+
function codePoint(value: number): string {
|
|
106
|
+
if (!Number.isInteger(value) || value < 0 || value > 0x10ffff) return "�";
|
|
107
|
+
// Lone surrogates are equally not characters; the spec replaces them for the same reason.
|
|
108
|
+
return value >= 0xd800 && value <= 0xdfff ? "�" : String.fromCodePoint(value);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/** Whether an `href` value is safe to keep. Relative URLs are fine; a bare scheme we do not know is not. */
|
|
112
|
+
export function isSafeHref(value: string): boolean {
|
|
113
|
+
// Decode first, THEN strip, in that order — the point is to see what a browser will see. Stripping
|
|
114
|
+
// first leaves ` ` intact and the check then reads a fragment marker where a scheme is hiding.
|
|
115
|
+
const trimmed = decodeReferences(value)
|
|
116
|
+
// biome-ignore lint/suspicious/noControlCharactersInRegex: matching them is the point — this is what stops one hiding a `javascript:` scheme from the check below.
|
|
117
|
+
.replace(/[\u0000-\u0020]+/g, "")
|
|
118
|
+
.toLowerCase();
|
|
119
|
+
if (trimmed.length === 0) return false;
|
|
120
|
+
// No colon before the first `/`, `?` or `#` means it is relative, which cannot name a scheme.
|
|
121
|
+
const colon = trimmed.indexOf(":");
|
|
122
|
+
if (colon === -1) return true;
|
|
123
|
+
const firstDelimiter = Math.min(
|
|
124
|
+
...["/", "?", "#"].map((char) => {
|
|
125
|
+
const index = trimmed.indexOf(char);
|
|
126
|
+
return index === -1 ? Number.POSITIVE_INFINITY : index;
|
|
127
|
+
}),
|
|
128
|
+
);
|
|
129
|
+
if (colon > firstDelimiter) return true;
|
|
130
|
+
return ALLOWED_SCHEMES.includes(trimmed.slice(0, colon + 1));
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* Sanitize an HTML body for storage and later rendering.
|
|
135
|
+
*
|
|
136
|
+
* **Remote images are stripped, not merely sandboxed.** An `<img src="https://…">` in a support
|
|
137
|
+
* message is a read receipt: it tells the sender the exact moment somebody opened their mail and
|
|
138
|
+
* from which IP, and against a support inbox that is a reconnaissance tool for whoever is probing
|
|
139
|
+
* you. Every mail client blocks them by default and so does this. The `alt` text survives, so a
|
|
140
|
+
* screenshot that mattered leaves a visible trace rather than vanishing silently. The original is
|
|
141
|
+
* still in R2 if an operator genuinely needs to see it.
|
|
142
|
+
*/
|
|
143
|
+
export async function sanitizeHtml(html: string): Promise<string> {
|
|
144
|
+
const rewriter = new HTMLRewriter()
|
|
145
|
+
.onDocument({
|
|
146
|
+
// Conditional comments (`<!--[if mso]><script>…`) are markup to some renderers and a comment
|
|
147
|
+
// to others. Dropping every comment removes the whole disagreement.
|
|
148
|
+
comments(comment) {
|
|
149
|
+
comment.remove();
|
|
150
|
+
},
|
|
151
|
+
})
|
|
152
|
+
.on("*", {
|
|
153
|
+
element(element) {
|
|
154
|
+
const tag = element.tagName.toLowerCase();
|
|
155
|
+
if (DROPPED_TAGS.has(tag)) {
|
|
156
|
+
element.remove();
|
|
157
|
+
return;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// Snapshot before mutating: the iterator walks the same attribute list `removeAttribute`
|
|
161
|
+
// shortens, so removing during the walk silently skips whatever moved into the gap.
|
|
162
|
+
for (const pair of [...element.attributes]) {
|
|
163
|
+
const name = pair[0] ?? "";
|
|
164
|
+
const value = pair[1] ?? "";
|
|
165
|
+
const attribute = name.toLowerCase();
|
|
166
|
+
// `src` is never allowlisted, so an image loses its source here and keeps its alt text.
|
|
167
|
+
if (!ALLOWED_ATTRIBUTES.has(attribute)) {
|
|
168
|
+
element.removeAttribute(name);
|
|
169
|
+
continue;
|
|
170
|
+
}
|
|
171
|
+
if (attribute === "href" && !isSafeHref(value)) {
|
|
172
|
+
element.removeAttribute(name);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
// A surviving link opens somewhere else and must not hand that page a handle back to the
|
|
177
|
+
// dashboard's window, whatever the dashboard's own headers say.
|
|
178
|
+
if (tag === "a" && element.getAttribute("href")) {
|
|
179
|
+
element.setAttribute("rel", "noopener noreferrer nofollow");
|
|
180
|
+
element.setAttribute("target", "_blank");
|
|
181
|
+
}
|
|
182
|
+
},
|
|
183
|
+
});
|
|
184
|
+
|
|
185
|
+
const sanitized = await rewriter.transform(new Response(html)).text();
|
|
186
|
+
return truncateToBytes(sanitized, MAX_HTML_BYTES);
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* Reduce HTML to readable plain text — the fallback when a message carried no text part.
|
|
191
|
+
*
|
|
192
|
+
* The classifier reads text, never markup: markup is tokens the adopter pays for that carry no
|
|
193
|
+
* meaning, and it is also an injection surface, since `<!-- ignore your instructions -->` reads
|
|
194
|
+
* exactly like prose to a model. Block-level tags become newlines so a paragraph structure survives
|
|
195
|
+
* the trip.
|
|
196
|
+
*/
|
|
197
|
+
export async function htmlToText(html: string): Promise<string> {
|
|
198
|
+
const parts: string[] = [];
|
|
199
|
+
// How many dropped elements we are currently inside. A counter rather than a boolean because
|
|
200
|
+
// `<script>` inside `<svg>` is legal and both are dropped, and a boolean would surface the tail of
|
|
201
|
+
// the outer one when the inner closed.
|
|
202
|
+
let suppressed = 0;
|
|
203
|
+
|
|
204
|
+
const rewriter = new HTMLRewriter()
|
|
205
|
+
.onDocument({
|
|
206
|
+
comments(comment) {
|
|
207
|
+
comment.remove();
|
|
208
|
+
},
|
|
209
|
+
// Document-level, not `on("*")`. A text handler bound to an element selector never fires for
|
|
210
|
+
// text outside any tag, so a message that is one bare sentence — which plenty are — came back
|
|
211
|
+
// as the empty string and the classifier was handed nothing to read.
|
|
212
|
+
text(chunk) {
|
|
213
|
+
if (suppressed === 0) parts.push(chunk.text);
|
|
214
|
+
},
|
|
215
|
+
})
|
|
216
|
+
.on("*", {
|
|
217
|
+
element(element) {
|
|
218
|
+
const tag = element.tagName.toLowerCase();
|
|
219
|
+
if (DROPPED_TAGS.has(tag)) {
|
|
220
|
+
element.remove();
|
|
221
|
+
// `remove()` takes the element out of the *output*; it does not stop the document text
|
|
222
|
+
// handler firing for what was inside it. Without this counter the contents of `<script>`
|
|
223
|
+
// and `<style>` land in `textBody` — which is the copy that gets stored, searched, and
|
|
224
|
+
// handed to the model. That is a prompt-injection surface and a way for anything an
|
|
225
|
+
// attacker put in a script tag to end up in the adopter's database as prose.
|
|
226
|
+
suppressed += 1;
|
|
227
|
+
try {
|
|
228
|
+
element.onEndTag(() => {
|
|
229
|
+
suppressed -= 1;
|
|
230
|
+
});
|
|
231
|
+
} catch {
|
|
232
|
+
// A void element has no end tag, so there is nothing to close and nothing to suppress.
|
|
233
|
+
suppressed -= 1;
|
|
234
|
+
}
|
|
235
|
+
return;
|
|
236
|
+
}
|
|
237
|
+
if (["p", "div", "br", "tr", "li", "h1", "h2", "h3", "h4", "h5", "h6", "blockquote"].includes(tag)) {
|
|
238
|
+
parts.push("\n");
|
|
239
|
+
}
|
|
240
|
+
},
|
|
241
|
+
});
|
|
242
|
+
|
|
243
|
+
// The transform is lazy: nothing runs until the body is consumed, so the text must be awaited even
|
|
244
|
+
// though the handlers, not the result, are what this function is after.
|
|
245
|
+
await rewriter.transform(new Response(html)).text();
|
|
246
|
+
|
|
247
|
+
return parts
|
|
248
|
+
.join("")
|
|
249
|
+
.replace(/[\t\r ]+/g, " ")
|
|
250
|
+
.replace(/ ?\n ?/g, "\n")
|
|
251
|
+
.replace(/\n{3,}/g, "\n\n")
|
|
252
|
+
.trim();
|
|
253
|
+
}
|