@saypsst/next 0.0.0-stage → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +190 -2
- package/dist/chunk-2NMIB353.js +65 -0
- package/dist/chunk-ELYUDQHH.js +42 -0
- package/dist/chunk-QE3BBW25.js +18 -0
- package/dist/chunk-X3GT7AOS.js +593 -0
- package/dist/chunk-ZELUPPQB.js +95 -0
- package/dist/config-LILlPYgW.d.ts +55 -0
- package/dist/index.d.ts +7 -0
- package/dist/index.js +2 -0
- package/dist/llms.d.ts +8 -0
- package/dist/llms.js +16 -0
- package/dist/page.d.ts +41 -0
- package/dist/page.js +190 -0
- package/dist/proxy.d.ts +18 -0
- package/dist/proxy.js +112 -0
- package/dist/sitemap.d.ts +11 -0
- package/dist/sitemap.js +19 -0
- package/dist/style.d.ts +8 -0
- package/dist/style.js +12 -0
- package/dist/verify.d.ts +12 -0
- package/dist/verify.js +17 -0
- package/package.json +73 -4
|
@@ -0,0 +1,593 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
|
|
3
|
+
// ../content/src/blocks.ts
|
|
4
|
+
|
|
5
|
+
// ../content/src/safe.ts
|
|
6
|
+
var HTML_ESCAPES = {
|
|
7
|
+
"&": "&",
|
|
8
|
+
"<": "<",
|
|
9
|
+
">": ">",
|
|
10
|
+
'"': """,
|
|
11
|
+
"'": "'"
|
|
12
|
+
};
|
|
13
|
+
function escapeHtml(value) {
|
|
14
|
+
return value.replace(/[&<>"']/g, (c) => HTML_ESCAPES[c] ?? c);
|
|
15
|
+
}
|
|
16
|
+
var XML_FORBIDDEN = new RegExp("[\\u0000-\\u0008\\u000B\\u000C\\u000E-\\u001F\\uFFFE\\uFFFF]", "g");
|
|
17
|
+
function escapeXml(value) {
|
|
18
|
+
return escapeHtml(value).replace(XML_FORBIDDEN, "");
|
|
19
|
+
}
|
|
20
|
+
var LS = new RegExp(String.fromCharCode(8232), "g");
|
|
21
|
+
var PS = new RegExp(String.fromCharCode(8233), "g");
|
|
22
|
+
function serializeJsonForScript(value) {
|
|
23
|
+
return JSON.stringify(value).replace(/</g, "\\u003c").replace(/>/g, "\\u003e").replace(/&/g, "\\u0026").replace(LS, "\\u2028").replace(PS, "\\u2029");
|
|
24
|
+
}
|
|
25
|
+
var FORBIDDEN_TEXT = new RegExp("[\\u0000-\\u0008\\u000B\\u000C\\u000E-\\u001F\\u007F\\u202A-\\u202E\\u2066-\\u2069]");
|
|
26
|
+
var PRINTABLE_ASCII = /^[!-~]+$/;
|
|
27
|
+
var UNSAFE_URL_CHARS = /["'<>\\`{}|^]/;
|
|
28
|
+
var MAX_URL_LENGTH = 2048;
|
|
29
|
+
function parseHttpsUrl(value) {
|
|
30
|
+
if (value.length > MAX_URL_LENGTH) return null;
|
|
31
|
+
if (!PRINTABLE_ASCII.test(value) || UNSAFE_URL_CHARS.test(value)) return null;
|
|
32
|
+
if (!/^https:\/\/[^/?#]/i.test(value)) return null;
|
|
33
|
+
let url;
|
|
34
|
+
try {
|
|
35
|
+
url = new URL(value);
|
|
36
|
+
} catch {
|
|
37
|
+
return null;
|
|
38
|
+
}
|
|
39
|
+
if (url.protocol !== "https:" || url.username !== "" || url.password !== "" || url.hostname === "") return null;
|
|
40
|
+
return url;
|
|
41
|
+
}
|
|
42
|
+
function isHttpsUrl(value) {
|
|
43
|
+
return parseHttpsUrl(value) !== null;
|
|
44
|
+
}
|
|
45
|
+
function isSameSiteRelative(value) {
|
|
46
|
+
if (value.length > MAX_URL_LENGTH) return false;
|
|
47
|
+
if (!PRINTABLE_ASCII.test(value) || UNSAFE_URL_CHARS.test(value)) return false;
|
|
48
|
+
if (!value.startsWith("/") || value.startsWith("//")) return false;
|
|
49
|
+
try {
|
|
50
|
+
return new URL(value, "https://base.invalid").host === "base.invalid";
|
|
51
|
+
} catch {
|
|
52
|
+
return false;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
function isSafeHref(value) {
|
|
56
|
+
return isSameSiteRelative(value) || isHttpsUrl(value);
|
|
57
|
+
}
|
|
58
|
+
var BARE_HOST = /^(?=.{1,253}$)([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z]{2,63}$/;
|
|
59
|
+
var PATH_PREFIX = /^\/[a-z0-9][a-z0-9-]*(\/[a-z0-9][a-z0-9-]*)*$/;
|
|
60
|
+
var SITE_PATH = /^\/[A-Za-z0-9._~\-/]*$/;
|
|
61
|
+
|
|
62
|
+
// ../content/src/blocks.ts
|
|
63
|
+
var CONTENT_SCHEMA_VERSION = 1;
|
|
64
|
+
var EDGE_IMAGE_HOST = "edge.saypsst.com";
|
|
65
|
+
function safeText(max, min = 1) {
|
|
66
|
+
return z.string().min(min).max(max).refine((s) => !FORBIDDEN_TEXT.test(s), "control or bidi-override characters are not allowed");
|
|
67
|
+
}
|
|
68
|
+
var safeHref = z.string().max(2048).refine(isSafeHref, "href must be https: or a same-site relative path");
|
|
69
|
+
var httpsUrl = z.string().max(2048).refine(isHttpsUrl, "url must be https:");
|
|
70
|
+
var inlineSchema = z.discriminatedUnion("type", [
|
|
71
|
+
z.strictObject({ type: z.literal("text"), text: safeText(2e3) }),
|
|
72
|
+
z.strictObject({ type: z.literal("strong"), text: safeText(500) }),
|
|
73
|
+
z.strictObject({ type: z.literal("em"), text: safeText(500) }),
|
|
74
|
+
z.strictObject({ type: z.literal("link"), text: safeText(300), href: safeHref })
|
|
75
|
+
]);
|
|
76
|
+
var inlines = z.array(inlineSchema).min(1).max(100);
|
|
77
|
+
var cell = safeText(500, 0);
|
|
78
|
+
var columns = z.array(safeText(120)).min(1).max(12);
|
|
79
|
+
function tableLike(shape) {
|
|
80
|
+
return z.strictObject({
|
|
81
|
+
caption: safeText(300),
|
|
82
|
+
columns,
|
|
83
|
+
rows: z.array(z.array(cell).min(1).max(12)).min(1).max(100),
|
|
84
|
+
...shape
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
function rowsMatchColumns(v) {
|
|
88
|
+
return v.rows.every((r) => r.length === v.columns.length);
|
|
89
|
+
}
|
|
90
|
+
var ROW_MSG = "every row must have exactly one cell per column";
|
|
91
|
+
function imageSrc(allowedHosts) {
|
|
92
|
+
return z.string().max(2048).refine((src) => {
|
|
93
|
+
const url = parseHttpsUrl(src);
|
|
94
|
+
if (!url || url.port !== "") return false;
|
|
95
|
+
if (allowedHosts === "any") return true;
|
|
96
|
+
return allowedHosts.map((h) => h.toLowerCase()).includes(url.hostname.toLowerCase());
|
|
97
|
+
}, "image src must be https: on an allowlisted host");
|
|
98
|
+
}
|
|
99
|
+
function createPageDocumentSchema(options) {
|
|
100
|
+
const blockSchema = z.discriminatedUnion("type", [
|
|
101
|
+
z.strictObject({ type: z.literal("heading"), level: z.union([z.literal(2), z.literal(3)]), text: safeText(200) }),
|
|
102
|
+
z.strictObject({ type: z.literal("paragraph"), inline: inlines }),
|
|
103
|
+
z.strictObject({ type: z.literal("list"), ordered: z.boolean(), items: z.array(inlines).min(1).max(50) }),
|
|
104
|
+
tableLike({ type: z.literal("table") }).refine(rowsMatchColumns, ROW_MSG),
|
|
105
|
+
z.strictObject({
|
|
106
|
+
type: z.literal("faq"),
|
|
107
|
+
items: z.array(z.strictObject({ q: safeText(300), a: inlines })).min(1).max(30)
|
|
108
|
+
}),
|
|
109
|
+
tableLike({ type: z.literal("comparison"), notes: z.array(safeText(500)).max(20) }).refine(
|
|
110
|
+
rowsMatchColumns,
|
|
111
|
+
ROW_MSG
|
|
112
|
+
),
|
|
113
|
+
// An official facts page's facts (ladder M3): each a labelled value with its unit and the https page stating it,
|
|
114
|
+
// rendered as a definition list. `currency` (ISO 4217) only on a price; JSON-LD names an `Offer` for it only when
|
|
115
|
+
// the visible value or unit states that currency.
|
|
116
|
+
z.strictObject({
|
|
117
|
+
type: z.literal("facts"),
|
|
118
|
+
items: z.array(
|
|
119
|
+
z.strictObject({
|
|
120
|
+
label: safeText(120),
|
|
121
|
+
value: safeText(200),
|
|
122
|
+
unit: safeText(40).optional(),
|
|
123
|
+
currency: z.string().regex(/^[A-Z]{3}$/, "currency must be an ISO 4217 code").optional(),
|
|
124
|
+
source: httpsUrl
|
|
125
|
+
})
|
|
126
|
+
).min(1).max(30)
|
|
127
|
+
}),
|
|
128
|
+
z.strictObject({ type: z.literal("cta"), label: safeText(80), href: safeHref }),
|
|
129
|
+
z.strictObject({
|
|
130
|
+
type: z.literal("image"),
|
|
131
|
+
src: imageSrc(options.allowedImageHosts),
|
|
132
|
+
alt: safeText(300),
|
|
133
|
+
width: z.number().int().min(1).max(1e4),
|
|
134
|
+
height: z.number().int().min(1).max(1e4)
|
|
135
|
+
}),
|
|
136
|
+
z.strictObject({
|
|
137
|
+
type: z.literal("citation"),
|
|
138
|
+
title: safeText(300),
|
|
139
|
+
url: httpsUrl,
|
|
140
|
+
accessed: z.iso.date()
|
|
141
|
+
})
|
|
142
|
+
]);
|
|
143
|
+
return z.strictObject({
|
|
144
|
+
v: z.literal(CONTENT_SCHEMA_VERSION),
|
|
145
|
+
kind: z.enum(["about", "llms_txt", "guide"]),
|
|
146
|
+
title: safeText(120),
|
|
147
|
+
description: safeText(320),
|
|
148
|
+
h1: safeText(160),
|
|
149
|
+
lede: safeText(600),
|
|
150
|
+
blocks: z.array(blockSchema).min(1).max(200),
|
|
151
|
+
sources: z.array(z.strictObject({ title: safeText(300), url: httpsUrl })).max(50),
|
|
152
|
+
updatedAt: z.iso.datetime()
|
|
153
|
+
});
|
|
154
|
+
}
|
|
155
|
+
createPageDocumentSchema({ allowedImageHosts: [EDGE_IMAGE_HOST] });
|
|
156
|
+
var pageDocumentStructureSchema = createPageDocumentSchema({ allowedImageHosts: "any" });
|
|
157
|
+
function parsePageDocument(input, site) {
|
|
158
|
+
return createPageDocumentSchema({ allowedImageHosts: [site.domain, EDGE_IMAGE_HOST] }).safeParse(input);
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
// ../content/src/hash.ts
|
|
162
|
+
function canonicalJson(value) {
|
|
163
|
+
if (value === null) return "null";
|
|
164
|
+
switch (typeof value) {
|
|
165
|
+
case "string":
|
|
166
|
+
case "boolean":
|
|
167
|
+
return JSON.stringify(value);
|
|
168
|
+
case "number":
|
|
169
|
+
if (!Number.isFinite(value)) throw new TypeError("canonicalJson: non-finite number");
|
|
170
|
+
return JSON.stringify(value);
|
|
171
|
+
case "object": {
|
|
172
|
+
if (Array.isArray(value)) return `[${value.map((v) => v === void 0 ? "null" : canonicalJson(v)).join(",")}]`;
|
|
173
|
+
const entries = Object.entries(value).filter(([, v]) => v !== void 0).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0);
|
|
174
|
+
return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${canonicalJson(v)}`).join(",")}}`;
|
|
175
|
+
}
|
|
176
|
+
default:
|
|
177
|
+
throw new TypeError(`canonicalJson: unsupported type ${typeof value}`);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
var K = new Uint32Array([
|
|
181
|
+
1116352408,
|
|
182
|
+
1899447441,
|
|
183
|
+
3049323471,
|
|
184
|
+
3921009573,
|
|
185
|
+
961987163,
|
|
186
|
+
1508970993,
|
|
187
|
+
2453635748,
|
|
188
|
+
2870763221,
|
|
189
|
+
3624381080,
|
|
190
|
+
310598401,
|
|
191
|
+
607225278,
|
|
192
|
+
1426881987,
|
|
193
|
+
1925078388,
|
|
194
|
+
2162078206,
|
|
195
|
+
2614888103,
|
|
196
|
+
3248222580,
|
|
197
|
+
3835390401,
|
|
198
|
+
4022224774,
|
|
199
|
+
264347078,
|
|
200
|
+
604807628,
|
|
201
|
+
770255983,
|
|
202
|
+
1249150122,
|
|
203
|
+
1555081692,
|
|
204
|
+
1996064986,
|
|
205
|
+
2554220882,
|
|
206
|
+
2821834349,
|
|
207
|
+
2952996808,
|
|
208
|
+
3210313671,
|
|
209
|
+
3336571891,
|
|
210
|
+
3584528711,
|
|
211
|
+
113926993,
|
|
212
|
+
338241895,
|
|
213
|
+
666307205,
|
|
214
|
+
773529912,
|
|
215
|
+
1294757372,
|
|
216
|
+
1396182291,
|
|
217
|
+
1695183700,
|
|
218
|
+
1986661051,
|
|
219
|
+
2177026350,
|
|
220
|
+
2456956037,
|
|
221
|
+
2730485921,
|
|
222
|
+
2820302411,
|
|
223
|
+
3259730800,
|
|
224
|
+
3345764771,
|
|
225
|
+
3516065817,
|
|
226
|
+
3600352804,
|
|
227
|
+
4094571909,
|
|
228
|
+
275423344,
|
|
229
|
+
430227734,
|
|
230
|
+
506948616,
|
|
231
|
+
659060556,
|
|
232
|
+
883997877,
|
|
233
|
+
958139571,
|
|
234
|
+
1322822218,
|
|
235
|
+
1537002063,
|
|
236
|
+
1747873779,
|
|
237
|
+
1955562222,
|
|
238
|
+
2024104815,
|
|
239
|
+
2227730452,
|
|
240
|
+
2361852424,
|
|
241
|
+
2428436474,
|
|
242
|
+
2756734187,
|
|
243
|
+
3204031479,
|
|
244
|
+
3329325298
|
|
245
|
+
]);
|
|
246
|
+
var rotr = (x, n) => x >>> n | x << 32 - n;
|
|
247
|
+
function sha256Hex(text) {
|
|
248
|
+
const data = new TextEncoder().encode(text);
|
|
249
|
+
const bitLen = data.length * 8;
|
|
250
|
+
const padded = new Uint8Array(data.length + 9 + 63 >> 6 << 6);
|
|
251
|
+
padded.set(data);
|
|
252
|
+
padded[data.length] = 128;
|
|
253
|
+
const view = new DataView(padded.buffer);
|
|
254
|
+
view.setUint32(padded.length - 8, Math.floor(bitLen / 4294967296));
|
|
255
|
+
view.setUint32(padded.length - 4, bitLen >>> 0);
|
|
256
|
+
const h = new Uint32Array([
|
|
257
|
+
1779033703,
|
|
258
|
+
3144134277,
|
|
259
|
+
1013904242,
|
|
260
|
+
2773480762,
|
|
261
|
+
1359893119,
|
|
262
|
+
2600822924,
|
|
263
|
+
528734635,
|
|
264
|
+
1541459225
|
|
265
|
+
]);
|
|
266
|
+
const w = new Uint32Array(64);
|
|
267
|
+
for (let off = 0; off < padded.length; off += 64) {
|
|
268
|
+
for (let i = 0; i < 16; i++) w[i] = view.getUint32(off + i * 4);
|
|
269
|
+
for (let i = 16; i < 64; i++) {
|
|
270
|
+
const w15 = w[i - 15] ?? 0;
|
|
271
|
+
const w2 = w[i - 2] ?? 0;
|
|
272
|
+
const s0 = rotr(w15, 7) ^ rotr(w15, 18) ^ w15 >>> 3;
|
|
273
|
+
const s1 = rotr(w2, 17) ^ rotr(w2, 19) ^ w2 >>> 10;
|
|
274
|
+
w[i] = (w[i - 16] ?? 0) + s0 + (w[i - 7] ?? 0) + s1 >>> 0;
|
|
275
|
+
}
|
|
276
|
+
let [a, b, c, d, e, f, g, hh] = [h[0] ?? 0, h[1] ?? 0, h[2] ?? 0, h[3] ?? 0, h[4] ?? 0, h[5] ?? 0, h[6] ?? 0, h[7] ?? 0];
|
|
277
|
+
for (let i = 0; i < 64; i++) {
|
|
278
|
+
const S1 = rotr(e, 6) ^ rotr(e, 11) ^ rotr(e, 25);
|
|
279
|
+
const ch = e & f ^ ~e & g;
|
|
280
|
+
const t1 = hh + S1 + ch + (K[i] ?? 0) + (w[i] ?? 0) >>> 0;
|
|
281
|
+
const S0 = rotr(a, 2) ^ rotr(a, 13) ^ rotr(a, 22);
|
|
282
|
+
const maj = a & b ^ a & c ^ b & c;
|
|
283
|
+
const t2 = S0 + maj >>> 0;
|
|
284
|
+
hh = g;
|
|
285
|
+
g = f;
|
|
286
|
+
f = e;
|
|
287
|
+
e = d + t1 >>> 0;
|
|
288
|
+
d = c;
|
|
289
|
+
c = b;
|
|
290
|
+
b = a;
|
|
291
|
+
a = t1 + t2 >>> 0;
|
|
292
|
+
}
|
|
293
|
+
const next = [a, b, c, d, e, f, g, hh];
|
|
294
|
+
for (let i = 0; i < 8; i++) h[i] = (h[i] ?? 0) + (next[i] ?? 0) >>> 0;
|
|
295
|
+
}
|
|
296
|
+
return Array.from(h, (x) => x.toString(16).padStart(8, "0")).join("");
|
|
297
|
+
}
|
|
298
|
+
function hashDocument(doc) {
|
|
299
|
+
return sha256Hex(canonicalJson(doc));
|
|
300
|
+
}
|
|
301
|
+
function shortHash(hash) {
|
|
302
|
+
return hash.slice(0, 12);
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
// ../content/src/jsonld.ts
|
|
306
|
+
function inlineText(inline) {
|
|
307
|
+
return inline.map((i) => i.text).join("");
|
|
308
|
+
}
|
|
309
|
+
function factValue(f) {
|
|
310
|
+
return f.unit ? `${f.value} ${f.unit}` : f.value;
|
|
311
|
+
}
|
|
312
|
+
var CURRENCY_MARKS = { USD: ["$", "US$"], EUR: ["\u20AC"], GBP: ["\xA3"], JPY: ["\xA5"], INR: ["\u20B9"], CAD: ["C$", "CA$"], AUD: ["A$", "AU$"], CHF: ["CHF"] };
|
|
313
|
+
function priceOf(value) {
|
|
314
|
+
const nums = value.match(/\d(?:\d|[., ](?=\d))*/g) ?? [];
|
|
315
|
+
const raw = nums[0];
|
|
316
|
+
if (nums.length !== 1 || !raw) return null;
|
|
317
|
+
const d = /[.,](\d{1,2})$/.exec(raw);
|
|
318
|
+
const head = d ? raw.slice(0, d.index) : raw;
|
|
319
|
+
if (!/^\d+$/.test(head) && !/^[1-9]\d{0,2}([., ])\d{3}(?:\1\d{3})*$/.test(head)) return null;
|
|
320
|
+
const int = head.replace(/\D/g, "");
|
|
321
|
+
return d ? `${int}.${d[1]}` : int;
|
|
322
|
+
}
|
|
323
|
+
function offerOf(f, url) {
|
|
324
|
+
if (!f.currency) return null;
|
|
325
|
+
const shown = factValue(f);
|
|
326
|
+
const marks = [f.currency, ...CURRENCY_MARKS[f.currency] ?? []];
|
|
327
|
+
if (!marks.some((m) => m === "$" ? /(^|[^A-Za-z])\$/.test(shown) : shown.includes(m))) return null;
|
|
328
|
+
const price = priceOf(f.value);
|
|
329
|
+
if (!price) return null;
|
|
330
|
+
return { "@type": "Offer", name: f.label, price, priceCurrency: f.currency, url };
|
|
331
|
+
}
|
|
332
|
+
function buildJsonLd(doc, ctx) {
|
|
333
|
+
const origin = `https://${ctx.domain}`;
|
|
334
|
+
const url = `${origin}${ctx.canonicalPath}`;
|
|
335
|
+
const main = doc.kind === "about" ? { "@type": "AboutPage", "@id": `${url}#page`, url, name: doc.title, description: doc.description } : {
|
|
336
|
+
"@type": "Article",
|
|
337
|
+
"@id": `${url}#page`,
|
|
338
|
+
url,
|
|
339
|
+
headline: doc.h1,
|
|
340
|
+
description: doc.description,
|
|
341
|
+
dateModified: doc.updatedAt,
|
|
342
|
+
mainEntityOfPage: url,
|
|
343
|
+
author: { "@type": "Organization", name: ctx.siteName, url: origin },
|
|
344
|
+
publisher: { "@type": "Organization", name: ctx.siteName, url: origin }
|
|
345
|
+
};
|
|
346
|
+
if (doc.kind === "about") main["dateModified"] = doc.updatedAt;
|
|
347
|
+
const graph = [
|
|
348
|
+
main,
|
|
349
|
+
{
|
|
350
|
+
"@type": "BreadcrumbList",
|
|
351
|
+
itemListElement: [
|
|
352
|
+
{ "@type": "ListItem", position: 1, name: ctx.siteName, item: `${origin}/` },
|
|
353
|
+
{ "@type": "ListItem", position: 2, name: doc.title, item: url }
|
|
354
|
+
]
|
|
355
|
+
}
|
|
356
|
+
];
|
|
357
|
+
const faqs = doc.blocks.filter((b) => b.type === "faq").flatMap((b) => b.items);
|
|
358
|
+
if (faqs.length > 0) {
|
|
359
|
+
graph.push({
|
|
360
|
+
"@type": "FAQPage",
|
|
361
|
+
mainEntity: faqs.map((f) => ({
|
|
362
|
+
"@type": "Question",
|
|
363
|
+
name: f.q,
|
|
364
|
+
acceptedAnswer: { "@type": "Answer", text: inlineText(f.a) }
|
|
365
|
+
}))
|
|
366
|
+
});
|
|
367
|
+
}
|
|
368
|
+
for (const b of doc.blocks) {
|
|
369
|
+
if (b.type !== "facts") continue;
|
|
370
|
+
for (const f of b.items) {
|
|
371
|
+
const offer = offerOf(f, url);
|
|
372
|
+
if (offer) graph.push(offer);
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
return { "@context": "https://schema.org", "@graph": graph };
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
// ../content/src/render-html.ts
|
|
379
|
+
var ContentError = class extends Error {
|
|
380
|
+
constructor(message) {
|
|
381
|
+
super(message);
|
|
382
|
+
this.name = "ContentError";
|
|
383
|
+
}
|
|
384
|
+
};
|
|
385
|
+
function assertSite(site, pages) {
|
|
386
|
+
if (!BARE_HOST.test(site.domain)) throw new ContentError("invalid domain");
|
|
387
|
+
for (const p of pages) {
|
|
388
|
+
if (!SITE_PATH.test(p.path) || p.path.startsWith("//") || p.path.includes("..")) throw new ContentError("invalid page path");
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
function renderSitemap(site, pages) {
|
|
392
|
+
assertSite(site, pages);
|
|
393
|
+
const urls = pages.map((p) => {
|
|
394
|
+
const lastmod = p.updatedAt && /^\d{4}-\d{2}-\d{2}/.test(p.updatedAt) ? `<lastmod>${escapeXml(p.updatedAt)}</lastmod>` : "";
|
|
395
|
+
return `<url><loc>${escapeXml(`https://${site.domain}${p.path}`)}</loc>${lastmod}</url>`;
|
|
396
|
+
}).join("\n");
|
|
397
|
+
return `<?xml version="1.0" encoding="UTF-8"?>
|
|
398
|
+
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
|
|
399
|
+
${urls}
|
|
400
|
+
</urlset>
|
|
401
|
+
`;
|
|
402
|
+
}
|
|
403
|
+
var publicIdSchema = z.string().regex(/^site_[a-z2-7]{16}$/);
|
|
404
|
+
var sitePathSchema = z.string().max(512).regex(SITE_PATH).refine((p) => !p.startsWith("//") && !p.includes(".."), "invalid path");
|
|
405
|
+
var siteModeSchema = z.enum(["next", "vercel_static", "proxy"]);
|
|
406
|
+
var agentClassSchema = z.enum(["training", "search", "user_fetch", "other"]);
|
|
407
|
+
var referrerClassSchema = z.enum([
|
|
408
|
+
"chatgpt",
|
|
409
|
+
"perplexity",
|
|
410
|
+
"gemini",
|
|
411
|
+
"copilot",
|
|
412
|
+
"claude",
|
|
413
|
+
"grok",
|
|
414
|
+
"mistral",
|
|
415
|
+
"deepseek",
|
|
416
|
+
"meta",
|
|
417
|
+
"you",
|
|
418
|
+
"other",
|
|
419
|
+
"none"
|
|
420
|
+
]);
|
|
421
|
+
var hitSourceSchema = z.enum(["edge", "proxy"]);
|
|
422
|
+
var publishStateSchema = z.enum(["published", "verified"]);
|
|
423
|
+
var hashSchema = z.string().regex(/^[0-9a-f]{64}$/);
|
|
424
|
+
z.strictObject({ path: sitePathSchema });
|
|
425
|
+
var pageResponseSchema = z.strictObject({
|
|
426
|
+
v: z.literal(CONTENT_SCHEMA_VERSION),
|
|
427
|
+
publicId: publicIdSchema,
|
|
428
|
+
path: sitePathSchema,
|
|
429
|
+
hash: hashSchema,
|
|
430
|
+
state: publishStateSchema,
|
|
431
|
+
publishedAt: z.iso.datetime(),
|
|
432
|
+
/** Structure-checked here; `renderPageHtml` enforces the image host allowlist. */
|
|
433
|
+
document: pageDocumentStructureSchema
|
|
434
|
+
});
|
|
435
|
+
var manifestResponseSchema = z.strictObject({
|
|
436
|
+
v: z.literal(CONTENT_SCHEMA_VERSION),
|
|
437
|
+
site: z.strictObject({
|
|
438
|
+
publicId: publicIdSchema,
|
|
439
|
+
domain: z.string().regex(BARE_HOST),
|
|
440
|
+
mode: siteModeSchema,
|
|
441
|
+
prefix: z.string().regex(PATH_PREFIX),
|
|
442
|
+
claimedRootPaths: z.array(z.enum(["/about", "/llms.txt"])).max(2),
|
|
443
|
+
/** Brand name for Open Graph and JSON-LD. Optional so older edges still parse. */
|
|
444
|
+
siteName: z.string().min(1).max(120).optional()
|
|
445
|
+
}),
|
|
446
|
+
pages: z.array(
|
|
447
|
+
z.strictObject({
|
|
448
|
+
path: sitePathSchema,
|
|
449
|
+
kind: z.enum(["about", "llms_txt", "guide"]),
|
|
450
|
+
title: z.string().min(1).max(120),
|
|
451
|
+
description: z.string().max(320),
|
|
452
|
+
hash: hashSchema,
|
|
453
|
+
state: publishStateSchema,
|
|
454
|
+
updatedAt: z.iso.datetime()
|
|
455
|
+
})
|
|
456
|
+
).max(1e3),
|
|
457
|
+
/** Withdrawn paths: answer 410 and drop from the sitemap. */
|
|
458
|
+
withdrawn: z.array(sitePathSchema).max(1e3)
|
|
459
|
+
});
|
|
460
|
+
var hitRequestSchema = z.strictObject({
|
|
461
|
+
v: z.literal(CONTENT_SCHEMA_VERSION),
|
|
462
|
+
publicId: publicIdSchema,
|
|
463
|
+
path: sitePathSchema,
|
|
464
|
+
agent: z.string().min(1).max(64).nullable(),
|
|
465
|
+
agentClass: agentClassSchema,
|
|
466
|
+
referrerClass: referrerClassSchema,
|
|
467
|
+
source: hitSourceSchema
|
|
468
|
+
});
|
|
469
|
+
z.strictObject({
|
|
470
|
+
v: z.literal(CONTENT_SCHEMA_VERSION),
|
|
471
|
+
error: z.enum(["not_found", "gone", "unavailable", "bad_request", "rate_limited"]),
|
|
472
|
+
message: z.string().max(300),
|
|
473
|
+
retryAfterSeconds: z.number().int().min(1).max(3600).optional()
|
|
474
|
+
});
|
|
475
|
+
|
|
476
|
+
// ../content/src/bots.ts
|
|
477
|
+
var AI_AGENT_RULES = [
|
|
478
|
+
{ agent: "ChatGPT-User", agentClass: "user_fetch", pattern: /chatgpt-user/i },
|
|
479
|
+
{ agent: "OAI-SearchBot", agentClass: "search", pattern: /oai-searchbot/i },
|
|
480
|
+
{ agent: "GPTBot", agentClass: "training", pattern: /gptbot/i },
|
|
481
|
+
{ agent: "Claude-SearchBot", agentClass: "search", pattern: /claude-searchbot/i },
|
|
482
|
+
{ agent: "Claude-User", agentClass: "user_fetch", pattern: /claude-user/i },
|
|
483
|
+
{ agent: "ClaudeBot", agentClass: "training", pattern: /claudebot/i },
|
|
484
|
+
{ agent: "Perplexity-User", agentClass: "user_fetch", pattern: /perplexity-user/i },
|
|
485
|
+
{ agent: "PerplexityBot", agentClass: "search", pattern: /perplexitybot/i },
|
|
486
|
+
{ agent: "Google-CloudVertexBot", agentClass: "search", pattern: /google-cloudvertexbot/i },
|
|
487
|
+
{ agent: "Applebot-Extended", agentClass: "training", pattern: /applebot-extended/i },
|
|
488
|
+
{ agent: "Meta-ExternalAgent", agentClass: "training", pattern: /meta-externalagent/i },
|
|
489
|
+
{ agent: "Bytespider", agentClass: "training", pattern: /bytespider/i },
|
|
490
|
+
{ agent: "CCBot", agentClass: "training", pattern: /ccbot/i },
|
|
491
|
+
{ agent: "Amazonbot", agentClass: "training", pattern: /amazonbot/i },
|
|
492
|
+
{ agent: "MistralAI-User", agentClass: "user_fetch", pattern: /mistralai-user/i },
|
|
493
|
+
{ agent: "MistralAI-Index", agentClass: "search", pattern: /mistralai-index/i },
|
|
494
|
+
{ agent: "DuckAssistBot", agentClass: "search", pattern: /duckassistbot/i },
|
|
495
|
+
{ agent: "Google-Agent", agentClass: "user_fetch", pattern: /google-agent/i },
|
|
496
|
+
{ agent: "Meta-ExternalFetcher", agentClass: "user_fetch", pattern: /meta-externalfetcher/i },
|
|
497
|
+
{ agent: "meta-webindexer", agentClass: "search", pattern: /meta-webindexer/i },
|
|
498
|
+
{ agent: "YouBot", agentClass: "search", pattern: /youbot/i },
|
|
499
|
+
{ agent: "Amzn-SearchBot", agentClass: "search", pattern: /amzn-searchbot/i },
|
|
500
|
+
{ agent: "Amzn-User", agentClass: "user_fetch", pattern: /amzn-user/i },
|
|
501
|
+
{ agent: "Claude-Agent", agentClass: "user_fetch", pattern: /claude-agent/i }
|
|
502
|
+
];
|
|
503
|
+
new Set(AI_AGENT_RULES.map((r) => r.agent));
|
|
504
|
+
function classifyUserAgent(userAgent) {
|
|
505
|
+
if (typeof userAgent === "string" && userAgent.length > 0 && userAgent.length <= 2048) {
|
|
506
|
+
for (const rule of AI_AGENT_RULES) {
|
|
507
|
+
if (rule.pattern.test(userAgent)) return { agent: rule.agent, agentClass: rule.agentClass };
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
return { agent: null, agentClass: "other" };
|
|
511
|
+
}
|
|
512
|
+
var REFERRER_HOSTS = [
|
|
513
|
+
{ host: "chatgpt.com", cls: "chatgpt" },
|
|
514
|
+
{ host: "chat.openai.com", cls: "chatgpt" },
|
|
515
|
+
{ host: "perplexity.ai", cls: "perplexity" },
|
|
516
|
+
{ host: "gemini.google.com", cls: "gemini" },
|
|
517
|
+
{ host: "bard.google.com", cls: "gemini" },
|
|
518
|
+
{ host: "copilot.microsoft.com", cls: "copilot" },
|
|
519
|
+
{ host: "copilot.cloud.microsoft", cls: "copilot" },
|
|
520
|
+
{ host: "claude.ai", cls: "claude" },
|
|
521
|
+
{ host: "grok.com", cls: "grok" },
|
|
522
|
+
{ host: "chat.mistral.ai", cls: "mistral" },
|
|
523
|
+
{ host: "chat.deepseek.com", cls: "deepseek" },
|
|
524
|
+
{ host: "meta.ai", cls: "meta" },
|
|
525
|
+
{ host: "you.com", cls: "you" }
|
|
526
|
+
];
|
|
527
|
+
var REFERRER_PATHS = [
|
|
528
|
+
{ host: "x.com", pathPrefix: "/i/grok", cls: "grok" },
|
|
529
|
+
{ host: "twitter.com", pathPrefix: "/i/grok", cls: "grok" }
|
|
530
|
+
];
|
|
531
|
+
function classifyReferrer(referrer) {
|
|
532
|
+
if (typeof referrer !== "string" || referrer.length === 0 || referrer.length > 2048) return "none";
|
|
533
|
+
let url;
|
|
534
|
+
try {
|
|
535
|
+
url = new URL(referrer);
|
|
536
|
+
} catch {
|
|
537
|
+
return "none";
|
|
538
|
+
}
|
|
539
|
+
const host = url.hostname.toLowerCase();
|
|
540
|
+
if (host === "") return "none";
|
|
541
|
+
for (const { host: h, cls } of REFERRER_HOSTS) {
|
|
542
|
+
if (host === h || host.endsWith(`.${h}`)) return cls;
|
|
543
|
+
}
|
|
544
|
+
const bare = host.replace(/^www\./, "");
|
|
545
|
+
const path = url.pathname.toLowerCase();
|
|
546
|
+
for (const { host: h, pathPrefix, cls } of REFERRER_PATHS) {
|
|
547
|
+
if (bare === h && (path === pathPrefix || path.startsWith(`${pathPrefix}/`))) return cls;
|
|
548
|
+
}
|
|
549
|
+
return "other";
|
|
550
|
+
}
|
|
551
|
+
var AI_UTM_RULES = {
|
|
552
|
+
chatgpt: "chatgpt",
|
|
553
|
+
"chatgpt.com": "chatgpt",
|
|
554
|
+
openai: "chatgpt",
|
|
555
|
+
perplexity: "perplexity",
|
|
556
|
+
"perplexity.ai": "perplexity",
|
|
557
|
+
gemini: "gemini",
|
|
558
|
+
"gemini.google.com": "gemini",
|
|
559
|
+
copilot: "copilot",
|
|
560
|
+
"copilot.microsoft.com": "copilot",
|
|
561
|
+
claude: "claude",
|
|
562
|
+
"claude.ai": "claude",
|
|
563
|
+
grok: "grok",
|
|
564
|
+
mistral: "mistral",
|
|
565
|
+
lechat: "mistral",
|
|
566
|
+
deepseek: "deepseek",
|
|
567
|
+
metaai: "meta",
|
|
568
|
+
youcom: "you"
|
|
569
|
+
};
|
|
570
|
+
function classifyUtmSource(utmSource) {
|
|
571
|
+
if (typeof utmSource !== "string" || utmSource.length === 0 || utmSource.length > 200) return null;
|
|
572
|
+
const v = utmSource.trim().toLowerCase();
|
|
573
|
+
if (Object.hasOwn(AI_UTM_RULES, v)) return AI_UTM_RULES[v] ?? null;
|
|
574
|
+
for (const { host, cls } of REFERRER_HOSTS) {
|
|
575
|
+
if (v === host || v.endsWith(`.${host}`)) return cls;
|
|
576
|
+
}
|
|
577
|
+
return null;
|
|
578
|
+
}
|
|
579
|
+
function classifyArrival(referrer, utmSource) {
|
|
580
|
+
const fromReferrer = classifyReferrer(referrer);
|
|
581
|
+
if (fromReferrer !== "other" && fromReferrer !== "none") return fromReferrer;
|
|
582
|
+
return classifyUtmSource(utmSource) ?? fromReferrer;
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
// src/content.ts
|
|
586
|
+
var siteShape = manifestResponseSchema.shape.site.shape;
|
|
587
|
+
var domainSchema = siteShape.domain;
|
|
588
|
+
var prefixSchema = siteShape.prefix;
|
|
589
|
+
|
|
590
|
+
// src/version.ts
|
|
591
|
+
var PSST_NEXT_VERSION = "0.3.0";
|
|
592
|
+
|
|
593
|
+
export { CONTENT_SCHEMA_VERSION, ContentError, PSST_NEXT_VERSION, buildJsonLd, classifyArrival, classifyUserAgent, domainSchema, factValue, hashDocument, hitRequestSchema, isSafeHref, manifestResponseSchema, pageResponseSchema, parsePageDocument, prefixSchema, publicIdSchema, renderSitemap, serializeJsonForScript, shortHash, sitePathSchema };
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import { pageResponseSchema, PSST_NEXT_VERSION, manifestResponseSchema } from './chunk-X3GT7AOS.js';
|
|
2
|
+
import { cache } from 'react';
|
|
3
|
+
|
|
4
|
+
var PsstUnavailableError = class extends Error {
|
|
5
|
+
constructor(message, options) {
|
|
6
|
+
super(`PSST: ${message}`, options);
|
|
7
|
+
this.name = "PsstUnavailableError";
|
|
8
|
+
}
|
|
9
|
+
};
|
|
10
|
+
var RENDER_TIMEOUT_MS = 8e3;
|
|
11
|
+
var MAX_JSON_BYTES = 2 * 1024 * 1024;
|
|
12
|
+
var USER_AGENT = `psst-next/${PSST_NEXT_VERSION}`;
|
|
13
|
+
async function edgeGet(url, accept, timeoutMs) {
|
|
14
|
+
try {
|
|
15
|
+
return await fetch(url, {
|
|
16
|
+
method: "GET",
|
|
17
|
+
headers: { accept, "user-agent": USER_AGENT },
|
|
18
|
+
redirect: "error",
|
|
19
|
+
signal: AbortSignal.timeout(timeoutMs)
|
|
20
|
+
});
|
|
21
|
+
} catch (cause) {
|
|
22
|
+
throw new PsstUnavailableError(`edge unreachable (${url})`, { cause });
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
async function readCapped(res, maxBytes) {
|
|
26
|
+
const declared = Number(res.headers.get("content-length") ?? "0");
|
|
27
|
+
if (declared > maxBytes) {
|
|
28
|
+
await res.body?.cancel().catch(() => void 0);
|
|
29
|
+
throw new PsstUnavailableError(`edge response too large (${declared} bytes)`);
|
|
30
|
+
}
|
|
31
|
+
if (!res.body) return "";
|
|
32
|
+
const reader = res.body.getReader();
|
|
33
|
+
const chunks = [];
|
|
34
|
+
let size = 0;
|
|
35
|
+
try {
|
|
36
|
+
for (; ; ) {
|
|
37
|
+
const { done, value } = await reader.read();
|
|
38
|
+
if (done) break;
|
|
39
|
+
size += value.byteLength;
|
|
40
|
+
if (size > maxBytes) {
|
|
41
|
+
await reader.cancel().catch(() => void 0);
|
|
42
|
+
throw new PsstUnavailableError(`edge response too large (more than ${maxBytes} bytes)`);
|
|
43
|
+
}
|
|
44
|
+
chunks.push(value);
|
|
45
|
+
}
|
|
46
|
+
} catch (cause) {
|
|
47
|
+
if (cause instanceof PsstUnavailableError) throw cause;
|
|
48
|
+
throw new PsstUnavailableError("edge response body could not be read", { cause });
|
|
49
|
+
}
|
|
50
|
+
const buf = new Uint8Array(size);
|
|
51
|
+
let offset = 0;
|
|
52
|
+
for (const chunk of chunks) {
|
|
53
|
+
buf.set(chunk, offset);
|
|
54
|
+
offset += chunk.byteLength;
|
|
55
|
+
}
|
|
56
|
+
return new TextDecoder().decode(buf);
|
|
57
|
+
}
|
|
58
|
+
async function readJson(res) {
|
|
59
|
+
const text = await readCapped(res, MAX_JSON_BYTES);
|
|
60
|
+
try {
|
|
61
|
+
return JSON.parse(text);
|
|
62
|
+
} catch (cause) {
|
|
63
|
+
throw new PsstUnavailableError("edge returned invalid JSON", { cause });
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
async function loadPage(edge, site, path) {
|
|
67
|
+
const url = `${edge}/v1/sites/${encodeURIComponent(site)}/pages?path=${encodeURIComponent(path)}`;
|
|
68
|
+
const res = await edgeGet(url, "application/json", RENDER_TIMEOUT_MS);
|
|
69
|
+
if (res.status === 404) return { status: "not_found" };
|
|
70
|
+
if (res.status === 410) return { status: "gone" };
|
|
71
|
+
if (res.status !== 200) throw new PsstUnavailableError(`edge answered ${res.status} for ${path}`);
|
|
72
|
+
const parsed = pageResponseSchema.safeParse(await readJson(res));
|
|
73
|
+
if (!parsed.success) throw new PsstUnavailableError(`edge page response failed validation for ${path}`);
|
|
74
|
+
if (parsed.data.publicId !== site || parsed.data.path !== path) {
|
|
75
|
+
throw new PsstUnavailableError(`edge answered for another page (${parsed.data.publicId}${parsed.data.path})`);
|
|
76
|
+
}
|
|
77
|
+
return { status: "ok", page: parsed.data };
|
|
78
|
+
}
|
|
79
|
+
var fetchPage = cache((config, path) => loadPage(config.edge, config.site, path));
|
|
80
|
+
async function loadManifest(edge, site, timeoutMs) {
|
|
81
|
+
const res = await edgeGet(`${edge}/v1/sites/${encodeURIComponent(site)}/manifest`, "application/json", timeoutMs);
|
|
82
|
+
if (res.status !== 200) throw new PsstUnavailableError(`edge answered ${res.status} for the manifest`);
|
|
83
|
+
const parsed = manifestResponseSchema.safeParse(await readJson(res));
|
|
84
|
+
if (!parsed.success) throw new PsstUnavailableError("edge manifest failed validation");
|
|
85
|
+
if (parsed.data.site.publicId !== site) throw new PsstUnavailableError("edge answered with another site's manifest");
|
|
86
|
+
return parsed.data;
|
|
87
|
+
}
|
|
88
|
+
function fetchManifest(config, timeoutMs = RENDER_TIMEOUT_MS) {
|
|
89
|
+
return loadManifest(config.edge, config.site, timeoutMs);
|
|
90
|
+
}
|
|
91
|
+
async function fetchEdgeFile(config, path, accept) {
|
|
92
|
+
return edgeGet(`${config.edge}/s/${encodeURIComponent(config.site)}${path}`, accept, RENDER_TIMEOUT_MS);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
export { PsstUnavailableError, USER_AGENT, fetchEdgeFile, fetchManifest, fetchPage, readCapped };
|