@wxxb789/anc 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +295 -0
- package/astro.config.mjs +255 -0
- package/bin/anc.mjs +798 -0
- package/package.json +93 -0
- package/public/_headers +45 -0
- package/public/favicon.ico +0 -0
- package/public/favicon.svg +9 -0
- package/public/fonts/Temml.woff2 +0 -0
- package/scripts/build-snapshot.js +69 -0
- package/scripts/build-wasm.js +67 -0
- package/scripts/build-workspace.js +50 -0
- package/scripts/copy-snapshot.js +48 -0
- package/scripts/copy-wasm.js +57 -0
- package/scripts/emit-redirects.js +64 -0
- package/scripts/generate-corpus.js +421 -0
- package/scripts/init-repository.js +249 -0
- package/scripts/load-config.js +975 -0
- package/scripts/markdown-to-artifact.js +1286 -0
- package/scripts/preview-site.js +364 -0
- package/scripts/publish-set-review.js +157 -0
- package/scripts/resolve-links.js +640 -0
- package/scripts/run-pagefind.js +125 -0
- package/scripts/scan-residue.js +1016 -0
- package/scripts/scan-secrets.js +268 -0
- package/scripts/seed-gitignore.js +410 -0
- package/scripts/snapshot-rows.js +205 -0
- package/scripts/validate-content.js +95 -0
- package/scripts/vendor-provenance.js +307 -0
- package/scripts/verify-output-inventory.js +307 -0
- package/scripts/write-report.js +351 -0
- package/scripts/write-snapshot.js +136 -0
- package/src/components/CollectionExplorer.astro +149 -0
- package/src/components/FacetIndex.astro +45 -0
- package/src/components/LinkedNotes.astro +94 -0
- package/src/components/NoteGraph.astro +434 -0
- package/src/components/NoteList.astro +54 -0
- package/src/components/SiteMetadata.astro +143 -0
- package/src/components/TableOfContents.astro +69 -0
- package/src/components/TagBrowser.astro +70 -0
- package/src/layouts/Layout.astro +305 -0
- package/src/lib/artifact-source.js +72 -0
- package/src/lib/build-bindings.js +11 -0
- package/src/lib/collection-navigation.js +145 -0
- package/src/lib/content.js +92 -0
- package/src/lib/diagram-mode.js +18 -0
- package/src/lib/graph-layout.js +219 -0
- package/src/lib/graph-selection.js +84 -0
- package/src/lib/graph.js +42 -0
- package/src/lib/link-resolution.js +389 -0
- package/src/lib/markdown.js +1469 -0
- package/src/lib/math-mode.js +31 -0
- package/src/lib/math-output.js +131 -0
- package/src/lib/math.js +165 -0
- package/src/lib/mermaid-environment.js +475 -0
- package/src/lib/mermaid-render.js +769 -0
- package/src/lib/preview-model.js +96 -0
- package/src/lib/relations.js +242 -0
- package/src/lib/rendered-marker.js +55 -0
- package/src/lib/route-path.js +192 -0
- package/src/lib/routes.js +342 -0
- package/src/lib/schema.js +578 -0
- package/src/lib/site.js +578 -0
- package/src/lib/snapshot-contract.js +216 -0
- package/src/lib/snapshot-operations.js +147 -0
- package/src/lib/snapshot-queries.js +216 -0
- package/src/lib/snapshot-reader.js +210 -0
- package/src/lib/snapshot.js +233 -0
- package/src/lib/sqlite-warning.js +22 -0
- package/src/lib/sqlite.js +14 -0
- package/src/lib/tag-browser-model.js +18 -0
- package/src/lib/translations.js +533 -0
- package/src/lib/wasm-asset.js +47 -0
- package/src/lib/worker-protocol.js +93 -0
- package/src/pages/404.astro +35 -0
- package/src/pages/about.astro +100 -0
- package/src/pages/collections/[slug].astro +36 -0
- package/src/pages/collections/index.astro +33 -0
- package/src/pages/graph.astro +64 -0
- package/src/pages/index.astro +33 -0
- package/src/pages/notes/[slug].astro +509 -0
- package/src/pages/privacy.astro +121 -0
- package/src/pages/private.astro +59 -0
- package/src/pages/recent.astro +33 -0
- package/src/pages/robots.txt.js +15 -0
- package/src/pages/rss.xml.js +21 -0
- package/src/pages/sitemap.xml.js +20 -0
- package/src/pages/tags/[tag].astro +56 -0
- package/src/pages/tags/index.astro +37 -0
- package/src/scripts/code-copy.js +45 -0
- package/src/scripts/diagram-disabled.js +16 -0
- package/src/scripts/diagram.js +322 -0
- package/src/scripts/graph-client.js +222 -0
- package/src/scripts/graph-draw.js +166 -0
- package/src/scripts/link-preview.js +334 -0
- package/src/scripts/math-disabled.js +18 -0
- package/src/scripts/math.js +133 -0
- package/src/scripts/preferences.js +123 -0
- package/src/scripts/search-dialog.js +651 -0
- package/src/scripts/snapshot-client.js +157 -0
- package/src/scripts/snapshot-worker.js +341 -0
- package/src/scripts/tag-browser.js +153 -0
- package/src/scripts/theme-init.js +35 -0
- package/src/scripts/toc-active.js +38 -0
- package/src/styles/code.css +152 -0
- package/src/styles/diagram.css +149 -0
- package/src/styles/global.css +1739 -0
- package/src/styles/graph.css +317 -0
- package/src/styles/math.css +376 -0
- package/src/styles/tokens.css +171 -0
- package/tsconfig.json +25 -0
|
@@ -0,0 +1,578 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Authoritative schema for the generated public content artifact.
|
|
3
|
+
*
|
|
4
|
+
* The artifact is produced from the user's notes by the shipped producer, or by
|
|
5
|
+
* this repository's fixture build, and is never hand-edited. Validation is
|
|
6
|
+
* deliberately strict: unknown fields are rejected so a producer change is a
|
|
7
|
+
* loud build failure rather than a silent privacy leak.
|
|
8
|
+
*/
|
|
9
|
+
import { compareSlugs, isNoteSlug, SLUG_MAX_BYTES } from "./route-path.js";
|
|
10
|
+
export const SCHEMA_VERSION = 1;
|
|
11
|
+
/** Top-level route segments the site owns; a note slug may not collide with them. */
|
|
12
|
+
export const RESERVED_SLUGS = new Set([
|
|
13
|
+
'notes',
|
|
14
|
+
'tags',
|
|
15
|
+
'collections',
|
|
16
|
+
'recent',
|
|
17
|
+
'graph',
|
|
18
|
+
'search',
|
|
19
|
+
'about',
|
|
20
|
+
'privacy',
|
|
21
|
+
// Where a link to a withheld note lands. Reserved for a reason the others are
|
|
22
|
+
// not: `markdown.ts`'s `INTERNAL_HREF` rewrites every single-segment
|
|
23
|
+
// `/<slug>/` href through `routeForSlug`, so a note published under this slug
|
|
24
|
+
// would silently capture every withheld link on the site and send readers to
|
|
25
|
+
// itself. See `WITHHELD_ROUTE` in `src/lib/route-path.ts`, and the gate in
|
|
26
|
+
// `tests/route-model.test.ts` that holds the mechanism rather than the
|
|
27
|
+
// membership.
|
|
28
|
+
'private',
|
|
29
|
+
'404',
|
|
30
|
+
'rss',
|
|
31
|
+
'sitemap',
|
|
32
|
+
'pagefind',
|
|
33
|
+
'data',
|
|
34
|
+
'wasm',
|
|
35
|
+
]);
|
|
36
|
+
/**
|
|
37
|
+
* A note slug: `isNoteSlug` from `route-path.ts`, the one definition every
|
|
38
|
+
* consumer shares.
|
|
39
|
+
*
|
|
40
|
+
* Unicode letters, numbers, and marks in hyphen-separated runs — `日记-今天`,
|
|
41
|
+
* `projects-观点`, `three-laws` — lowercase, NFC, with no invisible code point,
|
|
42
|
+
* no leading, trailing, or doubled hyphen, and at most `SLUG_MAX_BYTES` UTF-8
|
|
43
|
+
* bytes. The doubled-hyphen rule exists because a slug and a collection are used
|
|
44
|
+
* *verbatim* as public route segments, and `src/lib/routes.ts` rejects a doubled
|
|
45
|
+
* separator in any route key. The two rules are asserted to agree in
|
|
46
|
+
* `tests/route-model.test.ts`.
|
|
47
|
+
*/
|
|
48
|
+
export function isSlug(value) {
|
|
49
|
+
return isNoteSlug(value);
|
|
50
|
+
}
|
|
51
|
+
/** ISO 8601 calendar date, optionally with a time and explicit offset. */
|
|
52
|
+
const DATE = /^\d{4}-\d{2}-\d{2}(?:T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2}))?$/;
|
|
53
|
+
/** BCP 47 shape, e.g. `en`, `zh-CN`, `zh-Hans-CN`. */
|
|
54
|
+
const LANGUAGE = /^[A-Za-z]{2,3}(?:-[A-Za-z0-9]{2,8})*$/;
|
|
55
|
+
export function isLanguageTag(value) {
|
|
56
|
+
return LANGUAGE.test(value);
|
|
57
|
+
}
|
|
58
|
+
/** Joins emoji sequences into one glyph, so it is allowed inside a tag or alias. */
|
|
59
|
+
const ZERO_WIDTH_JOINER = String.fromCharCode(0x200d);
|
|
60
|
+
export function isSafeTagOrAlias(value) {
|
|
61
|
+
return !value.includes('/') && !/[\p{Cc}\p{Cf}]/u.test(value.replaceAll(ZERO_WIDTH_JOINER, ''));
|
|
62
|
+
}
|
|
63
|
+
const STATUS_VALUES = new Set(['published', 'tombstone']);
|
|
64
|
+
const REQUIRED_STRINGS = ['slug', 'title', 'excerpt', 'markdown'];
|
|
65
|
+
const OPTIONAL_FIELDS = [
|
|
66
|
+
'public_id',
|
|
67
|
+
'created',
|
|
68
|
+
'updated',
|
|
69
|
+
'language',
|
|
70
|
+
'tags',
|
|
71
|
+
'collection',
|
|
72
|
+
'status',
|
|
73
|
+
'aliases',
|
|
74
|
+
'description',
|
|
75
|
+
// Relationship authorities are **not** serialized in the packaged artifact:
|
|
76
|
+
// the snapshot is the public relation model, and the renderer reads it. They
|
|
77
|
+
// remain accepted here because this repository's own producer handoff and its
|
|
78
|
+
// committed fixture corpora still carry them, and because a mixed artifact is
|
|
79
|
+
// a producer defect worth naming.
|
|
80
|
+
'outgoing',
|
|
81
|
+
'backlinks',
|
|
82
|
+
];
|
|
83
|
+
const KNOWN_FIELDS = new Set([
|
|
84
|
+
...REQUIRED_STRINGS,
|
|
85
|
+
...OPTIONAL_FIELDS,
|
|
86
|
+
]);
|
|
87
|
+
/**
|
|
88
|
+
* Size ceilings, stated per field.
|
|
89
|
+
*
|
|
90
|
+
* Without these the contract bounds nothing: a 3 MB `markdown` body and 5,000
|
|
91
|
+
* tags both pass, and 5,000 tags on one entry is 5,000 public routes. A limit
|
|
92
|
+
* that fails validation is a message naming the field; the same input without
|
|
93
|
+
* one is a build that either takes minutes or emits a route map the host
|
|
94
|
+
* rejects.
|
|
95
|
+
*
|
|
96
|
+
* Lengths are counted in UTF-16 code units, which is what `String.length`
|
|
97
|
+
* returns. The number is a size ceiling, not a grapheme count, so counting
|
|
98
|
+
* astral characters as two is the honest direction: an emoji really does cost
|
|
99
|
+
* twice a Latin letter to store and to serve.
|
|
100
|
+
*
|
|
101
|
+
* Each ceiling is set an order of magnitude above anything the corpus plausibly
|
|
102
|
+
* produces, so it catches a runaway producer rather than constraining an author.
|
|
103
|
+
*/
|
|
104
|
+
const STRING_LIMITS = {
|
|
105
|
+
// A path segment. The binding bound is `SLUG_MAX_BYTES` (UTF-8 bytes, a
|
|
106
|
+
// filesystem name limit), enforced by `isSlug`; this UTF-16 ceiling is the
|
|
107
|
+
// same number and can never be the tighter of the two.
|
|
108
|
+
slug: SLUG_MAX_BYTES,
|
|
109
|
+
// A `<title>`, a card heading, and a search result. Search engines truncate a
|
|
110
|
+
// title well below this.
|
|
111
|
+
title: 300,
|
|
112
|
+
// The `<meta name="description">`, the card body, and the hover preview.
|
|
113
|
+
excerpt: 1000,
|
|
114
|
+
description: 1000,
|
|
115
|
+
// Requirements section 18 warns above 250 KB of uncompressed article HTML,
|
|
116
|
+
// and rendering Markdown only adds markup, so the source is bounded below it.
|
|
117
|
+
markdown: 200_000,
|
|
118
|
+
// An opaque public identifier, not prose.
|
|
119
|
+
public_id: 128,
|
|
120
|
+
// BCP 47 language tags are capped at 35 characters by the registry itself.
|
|
121
|
+
language: 35,
|
|
122
|
+
collection: 128,
|
|
123
|
+
};
|
|
124
|
+
/** Array ceilings: how many members, and how long each member may be. */
|
|
125
|
+
const ARRAY_LIMITS = {
|
|
126
|
+
// Every tag is a public route, so an unbounded tag list is an unbounded page
|
|
127
|
+
// count. A note carrying more than this is a classification failure, not a
|
|
128
|
+
// note. There is no matching ceiling on the *corpus* — see the note below
|
|
129
|
+
// on why the entry limit was removed.
|
|
130
|
+
tags: { items: 50, itemChars: 128 },
|
|
131
|
+
// Aliases are bounded public display/search/preview metadata.
|
|
132
|
+
aliases: { items: 50, itemChars: 300 },
|
|
133
|
+
// `outgoing` and `backlinks` deliberately have no count ceiling. The old 500
|
|
134
|
+
// was a serialized-array limit, and the SQLite projection is the relationship
|
|
135
|
+
// authority now: a hub with thousands of backlinks must stay fully reachable,
|
|
136
|
+
// and a graph drawing bound must never reject a DB fact. A producer defect is
|
|
137
|
+
// caught by the edge's own resolution and foreign-key checks instead.
|
|
138
|
+
};
|
|
139
|
+
/** Size ceilings for a field, exposed so a consumer can state the same number. */
|
|
140
|
+
export const FIELD_LIMITS = { strings: STRING_LIMITS, arrays: ARRAY_LIMITS };
|
|
141
|
+
/**
|
|
142
|
+
* **There is no ceiling on how many entries an artifact may carry, and the
|
|
143
|
+
* removal of one is the deliberate part.**
|
|
144
|
+
*
|
|
145
|
+
* A `MAX_ENTRIES = 900` lived here and refused a corpus past it. Measured
|
|
146
|
+
* against the shipped binary, a generated repository of 1,000 notes failed the
|
|
147
|
+
* build outright: `entries: has 957 entries, over the limit of 900`. An ordinary
|
|
148
|
+
* Obsidian vault passes 900 notes in a year or two, so the tool refused the
|
|
149
|
+
* corpus it exists to publish.
|
|
150
|
+
*
|
|
151
|
+
* Both premises the number rested on had already stopped being true, and its own
|
|
152
|
+
* comment said so:
|
|
153
|
+
*
|
|
154
|
+
* - *"An artifact past this is an exporter fault — the allowlist is
|
|
155
|
+
* hand-curated."* There is no allowlist. Under default-publish every Markdown
|
|
156
|
+
* file in a stranger's repository is an entry, and a stranger does not curate
|
|
157
|
+
* them.
|
|
158
|
+
* - *"900 rather than a round 1,000 to leave headroom under Cloudflare Pages'
|
|
159
|
+
* ceiling of 2,000 static redirect rules … TK-12 deleted those rules, so today
|
|
160
|
+
* the ceiling is not binding."* The host limit it protected is not reached,
|
|
161
|
+
* because `REDIRECT_RULES` is a hand-written literal that cannot grow with the
|
|
162
|
+
* corpus.
|
|
163
|
+
*
|
|
164
|
+
* So it was retained on a hypothetical — that a future rename might make rules
|
|
165
|
+
* derived again — while the fault it named could not occur. A cap kept for a
|
|
166
|
+
* mechanism that does not exist is a cap that fails real users to protect a
|
|
167
|
+
* theoretical one.
|
|
168
|
+
*
|
|
169
|
+
* **What is lost, stated plainly.** A producer that runs away now exhausts
|
|
170
|
+
* memory instead of failing with a message. That trade was made knowingly: the
|
|
171
|
+
* limit did not distinguish a runaway producer from an ordinary vault, so it
|
|
172
|
+
* caught the second far more often than the first. If a guard is wanted later it
|
|
173
|
+
* should be an order of magnitude above any real corpus and named for what it
|
|
174
|
+
* catches, rather than a number chosen for a host constraint that no longer
|
|
175
|
+
* applies.
|
|
176
|
+
*
|
|
177
|
+
* Two comments elsewhere quote this ceiling as the bound on their own timings —
|
|
178
|
+
* `src/lib/relations.ts` and `src/lib/graph.ts`. Their measurements stand as
|
|
179
|
+
* measurements *at 900 entries*; they no longer describe the largest corpus this
|
|
180
|
+
* tool accepts, and both say so.
|
|
181
|
+
*/
|
|
182
|
+
function checkLimits(value, label, issues) {
|
|
183
|
+
for (const [field, max] of Object.entries(STRING_LIMITS)) {
|
|
184
|
+
const item = value[field];
|
|
185
|
+
if (typeof item === 'string' && item.length > max) {
|
|
186
|
+
issues.push(`${label}.${field}: is ${item.length} characters, over the ${max} character limit`);
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
for (const [field, { items, itemChars }] of Object.entries(ARRAY_LIMITS)) {
|
|
190
|
+
const list = value[field];
|
|
191
|
+
if (!Array.isArray(list))
|
|
192
|
+
continue;
|
|
193
|
+
if (list.length > items) {
|
|
194
|
+
issues.push(`${label}.${field}: has ${list.length} entries, over the limit of ${items}`);
|
|
195
|
+
}
|
|
196
|
+
if (itemChars === undefined)
|
|
197
|
+
continue;
|
|
198
|
+
for (const [position, member] of list.entries()) {
|
|
199
|
+
if (typeof member === 'string' && member.length > itemChars) {
|
|
200
|
+
issues.push(`${label}.${field}[${position}]: is ${member.length} characters, ` +
|
|
201
|
+
`over the ${itemChars} character limit`);
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
export function contentLimitIssues(value, label) {
|
|
207
|
+
const issues = [];
|
|
208
|
+
checkLimits(value, label, issues);
|
|
209
|
+
return issues;
|
|
210
|
+
}
|
|
211
|
+
/**
|
|
212
|
+
* Markers that must never reach a public artifact. Scanned against each entry's
|
|
213
|
+
* string values so that no field, present or future, can smuggle one through.
|
|
214
|
+
*
|
|
215
|
+
* These are deliberately NOT matched against a whitespace-stripped form: doing so
|
|
216
|
+
* would join unrelated lines and turn prose such as "Option A:" above "/usr/bin"
|
|
217
|
+
* into an apparent drive path.
|
|
218
|
+
*
|
|
219
|
+
* **`[[` used to be on this list and is not, and the removal is TK-27's.** It was
|
|
220
|
+
* never a privacy rule — `[[` discloses nothing — it was a proxy for "the
|
|
221
|
+
* producer forgot to resolve something", and it was safe only while the producer
|
|
222
|
+
* resolved every wikilink against a closed allowlist. Under default-publish the
|
|
223
|
+
* proxy is false in both directions: a link that resolves to nothing now
|
|
224
|
+
* degrades to text by design, and a note *documenting* wikilink syntax inside a
|
|
225
|
+
* code fence is content that must publish. Measured on the shipped binary, a
|
|
226
|
+
* two-note corpus containing one fenced `[[documented syntax]]` failed the build
|
|
227
|
+
* with "forbidden unresolved [[wikilink]]".
|
|
228
|
+
*
|
|
229
|
+
* The invariant that replaces it lives where the information is, in
|
|
230
|
+
* `scripts/resolve-links.ts`: every wikilink *node* the parser found has a
|
|
231
|
+
* recorded resolution, so a `[[` surviving in `markdown` is by construction
|
|
232
|
+
* inside a fence, inside inline code, or escaped. The residue scan over `dist/`
|
|
233
|
+
* keeps a narrowed form of the old rule, because there a stray `[[` outside a
|
|
234
|
+
* code region means the degradation itself failed — a producer defect rather
|
|
235
|
+
* than a user's.
|
|
236
|
+
*/
|
|
237
|
+
const STRUCTURAL_MARKERS = [
|
|
238
|
+
[/msw\//i, 'private "msw/" path marker'],
|
|
239
|
+
[/(?<![A-Za-z])[A-Za-z]:[\\/]/, 'absolute local path'],
|
|
240
|
+
];
|
|
241
|
+
/**
|
|
242
|
+
* Schemes a browser resolves after discarding whitespace anywhere in the URL, so
|
|
243
|
+
* these are matched against a whitespace-stripped form too. Mirrors the
|
|
244
|
+
* producer's own rule. `file:` carries a lookbehind so that ordinary words
|
|
245
|
+
* ending in "file" ("profile:", "makefile:") followed by "//" stay readable.
|
|
246
|
+
*/
|
|
247
|
+
const UNSAFE_SCHEMES = [
|
|
248
|
+
[/javascript:/i, 'javascript: URL'],
|
|
249
|
+
[/vbscript:/i, 'vbscript: URL'],
|
|
250
|
+
[/(?<![A-Za-z])file:\/\//i, 'file:// URL'],
|
|
251
|
+
];
|
|
252
|
+
/**
|
|
253
|
+
* A `data:` URL whose media type is not an allowlisted base64 raster image.
|
|
254
|
+
*
|
|
255
|
+
* Anchored on the `,` that separates a data URL's media type from its payload,
|
|
256
|
+
* so prose such as "Response data: see below" is not mistaken for one. The media
|
|
257
|
+
* type is optional because `data:,` and `data:;base64,` are both legal and both
|
|
258
|
+
* default to `text/plain`, and it is matched as "anything up to the comma" so a
|
|
259
|
+
* parameter or a wildcard type cannot carry a payload past the gate.
|
|
260
|
+
*
|
|
261
|
+
* Interior whitespace is excluded: a media type containing a space does not
|
|
262
|
+
* parse, so `data:text /html,x` degrades to `text/plain` in a browser, and
|
|
263
|
+
* excluding it is what keeps a sentence like "Sample data: see docs/guide, then
|
|
264
|
+
* run it" readable. Whitespace immediately after `data:` or before the comma IS
|
|
265
|
+
* stripped by the browser, so `stripDataUrlPadding` removes it before this runs.
|
|
266
|
+
*/
|
|
267
|
+
const DISALLOWED_DATA_URI = [
|
|
268
|
+
/data:(?!image\/(?:png|jpe?g|gif|webp|avif);base64,)(?:,|[^,\s]*(?:\/|;base64)[^,\s]*,)/i,
|
|
269
|
+
'non-image data: URL',
|
|
270
|
+
];
|
|
271
|
+
/**
|
|
272
|
+
* Drop the whitespace a browser trims from a data URL's media type (fetch spec:
|
|
273
|
+
* the media type is trimmed before parsing), so `data: text/html,` and
|
|
274
|
+
* `data:text/html ,` are judged as the `text/html` they actually resolve to.
|
|
275
|
+
*/
|
|
276
|
+
function stripDataUrlPadding(text) {
|
|
277
|
+
return text.replace(/(data:)[ \t]+/gi, '$1').replace(/[ \t]+,/g, ',');
|
|
278
|
+
}
|
|
279
|
+
/**
|
|
280
|
+
* Characters a URL parser discards from anywhere in a URL (WHATWG URL: tab, LF,
|
|
281
|
+
* CR) plus the invisible format characters that could hide inside a marker. A
|
|
282
|
+
* literal space is deliberately NOT removed — browsers do not strip it, so
|
|
283
|
+
* removing it here would fuse ordinary prose into an apparent URL.
|
|
284
|
+
*/
|
|
285
|
+
const URL_IGNORED = /[\t\n\r\p{Cf}]+/gu;
|
|
286
|
+
/** Invisible characters only: safe to remove before looking for a literal marker. */
|
|
287
|
+
const INVISIBLE = /\p{Cf}+/gu;
|
|
288
|
+
const NAMED_ENTITIES = {
|
|
289
|
+
amp: '&',
|
|
290
|
+
apos: "'",
|
|
291
|
+
colon: ':',
|
|
292
|
+
gt: '>',
|
|
293
|
+
lt: '<',
|
|
294
|
+
newline: '\n',
|
|
295
|
+
quot: '"',
|
|
296
|
+
sol: '/',
|
|
297
|
+
tab: '\t',
|
|
298
|
+
};
|
|
299
|
+
export class ContentValidationError extends Error {
|
|
300
|
+
issues;
|
|
301
|
+
constructor(source, issues) {
|
|
302
|
+
super(`${source}: ${issues.length} content contract violation${issues.length === 1 ? '' : 's'}\n` +
|
|
303
|
+
issues.map((issue) => ` - ${issue}`).join('\n'));
|
|
304
|
+
this.name = 'ContentValidationError';
|
|
305
|
+
this.issues = issues;
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
function isPlainObject(value) {
|
|
309
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
310
|
+
}
|
|
311
|
+
/** Decode the entity forms a browser would resolve inside an attribute value. */
|
|
312
|
+
function decodeEntities(value) {
|
|
313
|
+
return value.replace(/&(?:#(\d+)|#[xX]([0-9a-fA-F]+)|([A-Za-z][A-Za-z0-9]*));?/g, (match, decimal, hex, name) => {
|
|
314
|
+
const code = decimal !== undefined ? Number(decimal) : hex !== undefined ? parseInt(hex, 16) : NaN;
|
|
315
|
+
if (Number.isFinite(code))
|
|
316
|
+
return code <= 0x10ffff ? String.fromCodePoint(code) : match;
|
|
317
|
+
return (name !== undefined ? NAMED_ENTITIES[name.toLowerCase()] : undefined) ?? match;
|
|
318
|
+
});
|
|
319
|
+
}
|
|
320
|
+
function scan(text, rules, found) {
|
|
321
|
+
for (const [pattern, label] of rules)
|
|
322
|
+
if (pattern.test(text))
|
|
323
|
+
found.add(label);
|
|
324
|
+
}
|
|
325
|
+
/**
|
|
326
|
+
* Every string reachable in the entry, so no field can hide a marker in a nested
|
|
327
|
+
* value. Iterative rather than recursive, and pushing one element at a time
|
|
328
|
+
* rather than spreading: a deeply nested or very wide artifact must fail
|
|
329
|
+
* validation with a contract error, not crash the build.
|
|
330
|
+
*/
|
|
331
|
+
function* strings(value) {
|
|
332
|
+
const stack = [value];
|
|
333
|
+
while (stack.length > 0) {
|
|
334
|
+
const current = stack.pop();
|
|
335
|
+
if (typeof current === 'string')
|
|
336
|
+
yield current;
|
|
337
|
+
else if (Array.isArray(current))
|
|
338
|
+
for (const item of current)
|
|
339
|
+
stack.push(item);
|
|
340
|
+
else if (typeof current === 'object' && current !== null) {
|
|
341
|
+
for (const [key, item] of Object.entries(current)) {
|
|
342
|
+
yield key;
|
|
343
|
+
stack.push(item);
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
function checkPrivacy(entry, label, issues) {
|
|
349
|
+
const found = new Set();
|
|
350
|
+
// Scan the parsed string values, not the JSON encoding of the entry: in JSON
|
|
351
|
+
// text a newline is the two characters "\" + "n", which would make ordinary
|
|
352
|
+
// prose such as "Option A:\n- foo" look like the drive path "A:\".
|
|
353
|
+
for (const value of strings(entry)) {
|
|
354
|
+
const decoded = decodeEntities(value);
|
|
355
|
+
// Structural markers get the invisible characters removed, so `msw<ZWJ>/x`
|
|
356
|
+
// cannot hide a private path that a browser would still render as `msw/x`.
|
|
357
|
+
// They do NOT get line breaks removed: joining lines would turn prose such
|
|
358
|
+
// as "Option A:" above "/usr/bin" into an apparent drive path.
|
|
359
|
+
const visible = decoded.replace(INVISIBLE, '');
|
|
360
|
+
// Scheme rules additionally drop the tab/CR/LF a URL parser ignores, so
|
|
361
|
+
// `java\tscript:` inside one URL is caught. This mirrors the producer.
|
|
362
|
+
const compact = decoded.replace(URL_IGNORED, '');
|
|
363
|
+
for (const text of [value, decoded, visible])
|
|
364
|
+
scan(text, STRUCTURAL_MARKERS, found);
|
|
365
|
+
for (const text of [value, decoded, compact]) {
|
|
366
|
+
scan(text, UNSAFE_SCHEMES, found);
|
|
367
|
+
scan(stripDataUrlPadding(text), [DISALLOWED_DATA_URI], found);
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
for (const marker of [...found].sort())
|
|
371
|
+
issues.push(`${label}: forbidden ${marker} in serialized content`);
|
|
372
|
+
}
|
|
373
|
+
export function contentPrivacyIssues(value, label) {
|
|
374
|
+
const issues = [];
|
|
375
|
+
checkPrivacy(value, label, issues);
|
|
376
|
+
return issues;
|
|
377
|
+
}
|
|
378
|
+
function checkStringArray(value, label, issues, options) {
|
|
379
|
+
if (!Array.isArray(value)) {
|
|
380
|
+
issues.push(`${label}: must be an array of strings`);
|
|
381
|
+
return false;
|
|
382
|
+
}
|
|
383
|
+
const clean = value.filter((item, index) => {
|
|
384
|
+
const ok = typeof item === 'string' && item.trim() !== '' && item === item.trim();
|
|
385
|
+
if (!ok)
|
|
386
|
+
issues.push(`${label}[${index}]: must be a non-empty string without surrounding whitespace`);
|
|
387
|
+
return ok;
|
|
388
|
+
});
|
|
389
|
+
if (clean.length !== value.length)
|
|
390
|
+
return false;
|
|
391
|
+
if (new Set(clean).size !== clean.length)
|
|
392
|
+
issues.push(`${label}: must not contain duplicates`);
|
|
393
|
+
// Slug lists are in canonical slug order (code points, the order SQLite's
|
|
394
|
+
// BINARY collation uses), not JavaScript's UTF-16 `<`; see `compareSlugs`.
|
|
395
|
+
if (options.sorted && clean.some((item, index) => index > 0 && compareSlugs(clean[index - 1], item) > 0)) {
|
|
396
|
+
issues.push(`${label}: must be sorted in ascending order`);
|
|
397
|
+
}
|
|
398
|
+
return true;
|
|
399
|
+
}
|
|
400
|
+
function checkEntry(value, index, issues) {
|
|
401
|
+
if (!isPlainObject(value)) {
|
|
402
|
+
issues.push(`entries[${index}]: must be an object`);
|
|
403
|
+
return undefined;
|
|
404
|
+
}
|
|
405
|
+
const slug = value['slug'];
|
|
406
|
+
const label = typeof slug === 'string' ? `entries[${index}] (slug "${slug}")` : `entries[${index}]`;
|
|
407
|
+
const before = issues.length;
|
|
408
|
+
for (const field of Object.keys(value)) {
|
|
409
|
+
if (!KNOWN_FIELDS.has(field))
|
|
410
|
+
issues.push(`${label}.${field}: unknown field is not allowed`);
|
|
411
|
+
}
|
|
412
|
+
for (const field of REQUIRED_STRINGS) {
|
|
413
|
+
const item = value[field];
|
|
414
|
+
if (typeof item !== 'string')
|
|
415
|
+
issues.push(`${label}.${field}: is required and must be a string`);
|
|
416
|
+
else if (field !== 'excerpt' && item.trim() === '')
|
|
417
|
+
issues.push(`${label}.${field}: must not be empty`);
|
|
418
|
+
}
|
|
419
|
+
if (typeof slug === 'string') {
|
|
420
|
+
if (!isSlug(slug)) {
|
|
421
|
+
issues.push(`${label}.slug: must be lowercase letters, digits, or marks in single-hyphen runs, ` +
|
|
422
|
+
`NFC, at most ${SLUG_MAX_BYTES} UTF-8 bytes`);
|
|
423
|
+
}
|
|
424
|
+
else if (RESERVED_SLUGS.has(slug)) {
|
|
425
|
+
issues.push(`${label}.slug: collides with reserved route segment "${slug}"`);
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
for (const field of ['outgoing', 'backlinks']) {
|
|
429
|
+
const item = value[field];
|
|
430
|
+
if (item === undefined)
|
|
431
|
+
continue;
|
|
432
|
+
checkStringArray(item, `${label}.${field}`, issues, { sorted: true });
|
|
433
|
+
}
|
|
434
|
+
for (const field of ['public_id', 'collection', 'description']) {
|
|
435
|
+
const item = value[field];
|
|
436
|
+
if (item !== undefined && (typeof item !== 'string' || item.trim() === '')) {
|
|
437
|
+
issues.push(`${label}.${field}: must be a non-empty string when present`);
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
if (typeof value['collection'] === 'string' && !isSlug(value['collection'])) {
|
|
441
|
+
issues.push(`${label}.collection: must be lowercase letters, digits, or marks in single-hyphen runs, ` +
|
|
442
|
+
`NFC, at most ${SLUG_MAX_BYTES} UTF-8 bytes`);
|
|
443
|
+
}
|
|
444
|
+
for (const field of ['created', 'updated']) {
|
|
445
|
+
const item = value[field];
|
|
446
|
+
if (item !== undefined && (typeof item !== 'string' || !DATE.test(item) || Number.isNaN(Date.parse(item)))) {
|
|
447
|
+
issues.push(`${label}.${field}: must be an ISO 8601 date when present`);
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
const language = value['language'];
|
|
451
|
+
if (language !== undefined && (typeof language !== 'string' || !isLanguageTag(language))) {
|
|
452
|
+
issues.push(`${label}.language: must be a BCP 47 language tag when present`);
|
|
453
|
+
}
|
|
454
|
+
const status = value['status'];
|
|
455
|
+
if (status !== undefined && (typeof status !== 'string' || !STATUS_VALUES.has(status))) {
|
|
456
|
+
issues.push(`${label}.status: must be one of ${[...STATUS_VALUES].join(', ')} when present`);
|
|
457
|
+
}
|
|
458
|
+
// Tags become `/tags/<tag>/` route keys; aliases enter display, search, and
|
|
459
|
+
// preview payloads but deliberately not link resolution. A "/" would break
|
|
460
|
+
// tag routing and could carry a private path fragment in either field; a control
|
|
461
|
+
// or format character (bidi overrides, soft hyphens) could disguise either.
|
|
462
|
+
// U+200D is exempt: it joins emoji sequences such as a person-at-keyboard glyph.
|
|
463
|
+
for (const field of ['tags', 'aliases']) {
|
|
464
|
+
const item = value[field];
|
|
465
|
+
if (item === undefined)
|
|
466
|
+
continue;
|
|
467
|
+
if (checkStringArray(item, `${label}.${field}`, issues, { sorted: false })) {
|
|
468
|
+
for (const [position, member] of item.entries()) {
|
|
469
|
+
if (!isSafeTagOrAlias(member)) {
|
|
470
|
+
issues.push(`${label}.${field}[${position}]: must not contain "/" or a control character`);
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
// Size before content, and an oversized entry stops here: a 3 MB body is
|
|
476
|
+
// rejected on its length rather than scanned four times over first, which is
|
|
477
|
+
// what makes "a contract error, not a slow build" true rather than
|
|
478
|
+
// aspirational. Only a *size* issue short-circuits — an entry with an unknown
|
|
479
|
+
// field or a bad slug is still privacy-scanned, so no violation goes
|
|
480
|
+
// unreported for an entry that would otherwise have been merely malformed.
|
|
481
|
+
const beforeLimits = issues.length;
|
|
482
|
+
issues.push(...contentLimitIssues(value, label));
|
|
483
|
+
if (issues.length !== beforeLimits)
|
|
484
|
+
return undefined;
|
|
485
|
+
issues.push(...contentPrivacyIssues(value, label));
|
|
486
|
+
return issues.length === before ? value : undefined;
|
|
487
|
+
}
|
|
488
|
+
function checkCorpus(entries, issues, edges) {
|
|
489
|
+
const bySlug = new Map();
|
|
490
|
+
for (const entry of entries) {
|
|
491
|
+
if (bySlug.has(entry.slug))
|
|
492
|
+
issues.push(`entries: duplicate slug "${entry.slug}"`);
|
|
493
|
+
else
|
|
494
|
+
bySlug.set(entry.slug, entry);
|
|
495
|
+
}
|
|
496
|
+
// No alias is checked across entries. `docs/core-design/content-semantics.md`
|
|
497
|
+
// owns the rule: aliases are "not alternate resolver targets, routes, globally
|
|
498
|
+
// unique names, or graph nodes", and "the same alias may belong to different
|
|
499
|
+
// notes" — so a repeated alias, or an alias that equals another note's slug,
|
|
500
|
+
// is metadata, not a conflict. The schema still enforces per-note uniqueness
|
|
501
|
+
// in the SQLite projection's `(node_id, alias)` key.
|
|
502
|
+
if (!edges)
|
|
503
|
+
return;
|
|
504
|
+
const expectedBacklinks = new Map(entries.map((entry) => [entry.slug, []]));
|
|
505
|
+
for (const entry of entries) {
|
|
506
|
+
const label = `entries (slug "${entry.slug}")`;
|
|
507
|
+
for (const target of entry.outgoing) {
|
|
508
|
+
if (target === entry.slug)
|
|
509
|
+
issues.push(`${label}.outgoing: must not link to itself`);
|
|
510
|
+
else if (!bySlug.has(target))
|
|
511
|
+
issues.push(`${label}.outgoing: "${target}" does not resolve to a published entry`);
|
|
512
|
+
else
|
|
513
|
+
expectedBacklinks.get(target)?.push(entry.slug);
|
|
514
|
+
}
|
|
515
|
+
for (const source of entry.backlinks) {
|
|
516
|
+
if (source === entry.slug)
|
|
517
|
+
issues.push(`${label}.backlinks: must not link to itself`);
|
|
518
|
+
else if (!bySlug.has(source))
|
|
519
|
+
issues.push(`${label}.backlinks: "${source}" does not resolve to a published entry`);
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
for (const entry of entries) {
|
|
523
|
+
const expected = (expectedBacklinks.get(entry.slug) ?? []).sort(compareSlugs);
|
|
524
|
+
const actual = [...entry.backlinks].sort(compareSlugs);
|
|
525
|
+
if (expected.length !== actual.length || expected.some((slug, index) => slug !== actual[index])) {
|
|
526
|
+
issues.push(`entries (slug "${entry.slug}").backlinks: must be the exact inverse of outgoing links ` +
|
|
527
|
+
`(expected [${expected.join(', ')}], got [${entry.backlinks.join(', ')}])`);
|
|
528
|
+
}
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
/**
|
|
532
|
+
* Validate a parsed content artifact.
|
|
533
|
+
*
|
|
534
|
+
* @throws {ContentValidationError} listing every violation, each naming the
|
|
535
|
+
* offending entry and field.
|
|
536
|
+
*/
|
|
537
|
+
export function validateArtifact(data, source = 'content artifact') {
|
|
538
|
+
const issues = [];
|
|
539
|
+
if (!isPlainObject(data))
|
|
540
|
+
throw new ContentValidationError(source, ['artifact: must be an object']);
|
|
541
|
+
for (const field of Object.keys(data)) {
|
|
542
|
+
if (field !== 'version' && field !== 'entries')
|
|
543
|
+
issues.push(`${field}: unknown field is not allowed`);
|
|
544
|
+
}
|
|
545
|
+
if (data['version'] !== SCHEMA_VERSION) {
|
|
546
|
+
issues.push(`version: must be ${SCHEMA_VERSION}, got ${JSON.stringify(data['version'])}`);
|
|
547
|
+
}
|
|
548
|
+
const rawEntries = data['entries'];
|
|
549
|
+
if (!Array.isArray(rawEntries))
|
|
550
|
+
issues.push('entries: must be an array');
|
|
551
|
+
else {
|
|
552
|
+
const valid = [];
|
|
553
|
+
for (const [index, entry] of rawEntries.entries()) {
|
|
554
|
+
const checked = checkEntry(entry, index, issues);
|
|
555
|
+
if (checked)
|
|
556
|
+
valid.push(checked);
|
|
557
|
+
}
|
|
558
|
+
const declared = rawEntries.map((entry) => isPlainObject(entry) && 'outgoing' in entry && 'backlinks' in entry);
|
|
559
|
+
const all = declared.every(Boolean);
|
|
560
|
+
const none = declared.every((value) => !value);
|
|
561
|
+
if (!all && !none) {
|
|
562
|
+
issues.push('entries: outgoing/backlinks must be declared on every entry or on none of them');
|
|
563
|
+
}
|
|
564
|
+
if (valid.length === rawEntries.length && (all || none))
|
|
565
|
+
checkCorpus(valid, issues, all);
|
|
566
|
+
}
|
|
567
|
+
if (issues.length > 0)
|
|
568
|
+
throw new ContentValidationError(source, issues);
|
|
569
|
+
const artifact = data;
|
|
570
|
+
// An artifact with no serialized relation authorities is the packaged target:
|
|
571
|
+
// the snapshot is the public relation model, so the in-memory entries start
|
|
572
|
+
// with empty lists and the build hydrates them from the finalized DB.
|
|
573
|
+
for (const entry of artifact.entries) {
|
|
574
|
+
entry.outgoing ??= [];
|
|
575
|
+
entry.backlinks ??= [];
|
|
576
|
+
}
|
|
577
|
+
return artifact;
|
|
578
|
+
}
|