@wxxb789/anc 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +295 -0
- package/astro.config.mjs +255 -0
- package/bin/anc.mjs +798 -0
- package/package.json +93 -0
- package/public/_headers +45 -0
- package/public/favicon.ico +0 -0
- package/public/favicon.svg +9 -0
- package/public/fonts/Temml.woff2 +0 -0
- package/scripts/build-snapshot.js +69 -0
- package/scripts/build-wasm.js +67 -0
- package/scripts/build-workspace.js +50 -0
- package/scripts/copy-snapshot.js +48 -0
- package/scripts/copy-wasm.js +57 -0
- package/scripts/emit-redirects.js +64 -0
- package/scripts/generate-corpus.js +421 -0
- package/scripts/init-repository.js +249 -0
- package/scripts/load-config.js +975 -0
- package/scripts/markdown-to-artifact.js +1286 -0
- package/scripts/preview-site.js +364 -0
- package/scripts/publish-set-review.js +157 -0
- package/scripts/resolve-links.js +640 -0
- package/scripts/run-pagefind.js +125 -0
- package/scripts/scan-residue.js +1016 -0
- package/scripts/scan-secrets.js +268 -0
- package/scripts/seed-gitignore.js +410 -0
- package/scripts/snapshot-rows.js +205 -0
- package/scripts/validate-content.js +95 -0
- package/scripts/vendor-provenance.js +307 -0
- package/scripts/verify-output-inventory.js +307 -0
- package/scripts/write-report.js +351 -0
- package/scripts/write-snapshot.js +136 -0
- package/src/components/CollectionExplorer.astro +149 -0
- package/src/components/FacetIndex.astro +45 -0
- package/src/components/LinkedNotes.astro +94 -0
- package/src/components/NoteGraph.astro +434 -0
- package/src/components/NoteList.astro +54 -0
- package/src/components/SiteMetadata.astro +143 -0
- package/src/components/TableOfContents.astro +69 -0
- package/src/components/TagBrowser.astro +70 -0
- package/src/layouts/Layout.astro +305 -0
- package/src/lib/artifact-source.js +72 -0
- package/src/lib/build-bindings.js +11 -0
- package/src/lib/collection-navigation.js +145 -0
- package/src/lib/content.js +92 -0
- package/src/lib/diagram-mode.js +18 -0
- package/src/lib/graph-layout.js +219 -0
- package/src/lib/graph-selection.js +84 -0
- package/src/lib/graph.js +42 -0
- package/src/lib/link-resolution.js +389 -0
- package/src/lib/markdown.js +1469 -0
- package/src/lib/math-mode.js +31 -0
- package/src/lib/math-output.js +131 -0
- package/src/lib/math.js +165 -0
- package/src/lib/mermaid-environment.js +475 -0
- package/src/lib/mermaid-render.js +769 -0
- package/src/lib/preview-model.js +96 -0
- package/src/lib/relations.js +242 -0
- package/src/lib/rendered-marker.js +55 -0
- package/src/lib/route-path.js +192 -0
- package/src/lib/routes.js +342 -0
- package/src/lib/schema.js +578 -0
- package/src/lib/site.js +578 -0
- package/src/lib/snapshot-contract.js +216 -0
- package/src/lib/snapshot-operations.js +147 -0
- package/src/lib/snapshot-queries.js +216 -0
- package/src/lib/snapshot-reader.js +210 -0
- package/src/lib/snapshot.js +233 -0
- package/src/lib/sqlite-warning.js +22 -0
- package/src/lib/sqlite.js +14 -0
- package/src/lib/tag-browser-model.js +18 -0
- package/src/lib/translations.js +533 -0
- package/src/lib/wasm-asset.js +47 -0
- package/src/lib/worker-protocol.js +93 -0
- package/src/pages/404.astro +35 -0
- package/src/pages/about.astro +100 -0
- package/src/pages/collections/[slug].astro +36 -0
- package/src/pages/collections/index.astro +33 -0
- package/src/pages/graph.astro +64 -0
- package/src/pages/index.astro +33 -0
- package/src/pages/notes/[slug].astro +509 -0
- package/src/pages/privacy.astro +121 -0
- package/src/pages/private.astro +59 -0
- package/src/pages/recent.astro +33 -0
- package/src/pages/robots.txt.js +15 -0
- package/src/pages/rss.xml.js +21 -0
- package/src/pages/sitemap.xml.js +20 -0
- package/src/pages/tags/[tag].astro +56 -0
- package/src/pages/tags/index.astro +37 -0
- package/src/scripts/code-copy.js +45 -0
- package/src/scripts/diagram-disabled.js +16 -0
- package/src/scripts/diagram.js +322 -0
- package/src/scripts/graph-client.js +222 -0
- package/src/scripts/graph-draw.js +166 -0
- package/src/scripts/link-preview.js +334 -0
- package/src/scripts/math-disabled.js +18 -0
- package/src/scripts/math.js +133 -0
- package/src/scripts/preferences.js +123 -0
- package/src/scripts/search-dialog.js +651 -0
- package/src/scripts/snapshot-client.js +157 -0
- package/src/scripts/snapshot-worker.js +341 -0
- package/src/scripts/tag-browser.js +153 -0
- package/src/scripts/theme-init.js +35 -0
- package/src/scripts/toc-active.js +38 -0
- package/src/styles/code.css +152 -0
- package/src/styles/diagram.css +149 -0
- package/src/styles/global.css +1739 -0
- package/src/styles/graph.css +317 -0
- package/src/styles/math.css +376 -0
- package/src/styles/tokens.css +171 -0
- package/tsconfig.json +25 -0
|
@@ -0,0 +1,1016 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Privacy and security residue scan over the built artifact.
|
|
3
|
+
*
|
|
4
|
+
* This is the last gate before `dist/` becomes what readers receive, and it is
|
|
5
|
+
* the only one that judges the *shipped bytes* rather than an input to them.
|
|
6
|
+
* `scripts/validate-content.ts` scans the content artifact, and
|
|
7
|
+
* `tests/built-routes.test.ts` scans the pages this repository authors for
|
|
8
|
+
* structural disclosure — neither sees the stylesheet, the bundled scripts, the
|
|
9
|
+
* files copied verbatim out of `public/`, or anything a future build step
|
|
10
|
+
* writes. Requirements section 19.1 asks for a scan of the built output, and
|
|
11
|
+
* until this existed it was performed by an agent typing `grep` and recording
|
|
12
|
+
* the result in a report. That is advice, not a gate.
|
|
13
|
+
*
|
|
14
|
+
* Three properties make it evidence rather than decoration:
|
|
15
|
+
*
|
|
16
|
+
* 1. **It fails closed on novelty.** Every file is classified as text, binary,
|
|
17
|
+
* or a carrier it knows how to open. A file matching none of those fails the
|
|
18
|
+
* scan by name, so a build step that starts emitting a new kind of file
|
|
19
|
+
* cannot have it silently pass unscanned. **A carrier it can open but not
|
|
20
|
+
* read through is the same case**: a database whose table yields no text is
|
|
21
|
+
* reported, not skipped.
|
|
22
|
+
* 2. **It refuses to pass vacuously.** An empty or missing `dist/`, a `dist/`
|
|
23
|
+
* with no scannable text in it, or a database holding no rows, is a failure
|
|
24
|
+
* rather than zero findings.
|
|
25
|
+
* 3. **It reports every finding**, not the first, so one run names the whole
|
|
26
|
+
* problem.
|
|
27
|
+
*
|
|
28
|
+
* **A file is judged by its bytes, not by its name.** Gzip and SQLite are both
|
|
29
|
+
* recognised by their magic, so a compressed member and a database are opened
|
|
30
|
+
* wherever they appear and under whatever extension. An earlier version anchored
|
|
31
|
+
* the inflate to `pagefind/*.pf_fragment`, which was safe only while every
|
|
32
|
+
* compressed member in the artifact was third-party — a build step compressing
|
|
33
|
+
* this site's own content would have shipped it behind a layer this scan could
|
|
34
|
+
* not open.
|
|
35
|
+
*
|
|
36
|
+
* **What it does not cover**, stated here so the gate is not mistaken for the
|
|
37
|
+
* whole of requirements section 19.1. That list has eight items; this scanner
|
|
38
|
+
* closes five — path markers, unresolved wikilinks, absolute local paths, source
|
|
39
|
+
* maps, and unsafe link schemes. The remaining three are covered by separate
|
|
40
|
+
* publication-event, provenance, and credential-rule owners:
|
|
41
|
+
*
|
|
42
|
+
* - *Slugs absent from the reviewed publish set* are enforced by `review` plus
|
|
43
|
+
* `build --release`, because they are a publication-event property rather than
|
|
44
|
+
* a byte pattern.
|
|
45
|
+
* - *Unexpected routes or assets* is enforced by
|
|
46
|
+
* `scripts/verify-output-inventory.ts`, because it is a route-model and
|
|
47
|
+
* provenance property rather than a byte pattern.
|
|
48
|
+
* - *Secrets* are enforced by `scripts/scan-secrets.ts` using the exact pinned
|
|
49
|
+
* Gitleaks rule set. A credential pattern set has a different false-positive
|
|
50
|
+
* profile, so duplicating it here with an entropy heuristic would be weaker.
|
|
51
|
+
*/
|
|
52
|
+
import { readFileSync, readdirSync, statSync } from 'node:fs';
|
|
53
|
+
import { gunzipSync } from 'node:zlib';
|
|
54
|
+
import { fileURLToPath } from 'node:url';
|
|
55
|
+
import { basename, extname, join, relative, sep } from 'node:path';
|
|
56
|
+
import { BuildFailure } from "./write-report.js";
|
|
57
|
+
import { readVendorProvenance } from "./vendor-provenance.js";
|
|
58
|
+
import { DIAGRAM_MODE } from "../src/lib/diagram-mode.js";
|
|
59
|
+
import { MATH_MODE } from "../src/lib/math-mode.js";
|
|
60
|
+
import { MARKED_ATTRIBUTE, RENDERED_MATH } from "../src/lib/rendered-marker.js";
|
|
61
|
+
import { SNAPSHOT_DIRECTORY, SNAPSHOT_FILE_PATTERN } from "../src/lib/snapshot.js";
|
|
62
|
+
import { databaseText, isGzip, isSqlite } from "./snapshot-rows.js";
|
|
63
|
+
const DIST = fileURLToPath(new URL('../dist', import.meta.url));
|
|
64
|
+
/**
|
|
65
|
+
* What must never reach the built site.
|
|
66
|
+
*
|
|
67
|
+
* The first three mirror `STRUCTURAL_MARKERS` in `src/lib/schema.ts` and the
|
|
68
|
+
* next three mirror its `UNSAFE_SCHEMES`, restated rather than imported: those
|
|
69
|
+
* constants are module-private there, and the two scans judge different
|
|
70
|
+
* corpora. The schema scans the *content artifact's* parsed string values
|
|
71
|
+
* before a build and rejects the artifact; this scans the *bytes of every file
|
|
72
|
+
* the site serves* after one.
|
|
73
|
+
*
|
|
74
|
+
* They are matched against decoded forms as well as raw bytes, for the reason
|
|
75
|
+
* this scan exists at all. An earlier version skipped that on the argument that
|
|
76
|
+
* the schema had already rejected the encoded forms — which is false for
|
|
77
|
+
* exactly the files this gate was written to cover. The schema only ever reads
|
|
78
|
+
* `src/data/content.json`; a file copied verbatim out of `public/`, a
|
|
79
|
+
* stylesheet, or anything a future build step emits passes it untouched. So
|
|
80
|
+
* `msw/secret` in a `public/` file reached `dist/` and this scan reported
|
|
81
|
+
* nothing. See `normalizedForms`.
|
|
82
|
+
*
|
|
83
|
+
* The last two are this scan's own, because only a built artifact can carry
|
|
84
|
+
* them: a source map reference (requirements section 19.1, "source maps
|
|
85
|
+
* containing private data") would embed absolute paths from the build machine,
|
|
86
|
+
* and a home-directory path is the POSIX shape of the drive-letter marker the
|
|
87
|
+
* schema already catches.
|
|
88
|
+
*/
|
|
89
|
+
const RESIDUE_RULES = [
|
|
90
|
+
[/msw\//i, 'private "msw/" path marker'],
|
|
91
|
+
[/\[\[/, 'unresolved [[wikilink]]'],
|
|
92
|
+
[/(?<![A-Za-z])[A-Za-z]:[\\/]/, 'absolute local path'],
|
|
93
|
+
[/(?<![A-Za-z])\/(?:Users|home)\/[A-Za-z0-9._-]+\//, 'absolute home-directory path'],
|
|
94
|
+
[/javascript:/i, 'javascript: URL'],
|
|
95
|
+
[/vbscript:/i, 'vbscript: URL'],
|
|
96
|
+
[/(?<![A-Za-z])file:\/\//i, 'file:// URL'],
|
|
97
|
+
[/data:(?!(?:image\\?\/(?:png|jpe?g|gif|webp|avif)|image\\\/\(\?:png\|jpe\?g\|gif\|webp\|avif\));base64,)(?:,|[^,\s]*(?:\/|;base64)[^,\s]*,)/i, 'non-image data: URL'],
|
|
98
|
+
[/sourceMappingURL/, 'source map reference'],
|
|
99
|
+
];
|
|
100
|
+
/**
|
|
101
|
+
* The one rule that does not apply inside a code region, and why exactly one.
|
|
102
|
+
*
|
|
103
|
+
* `[[` is not a privacy marker — it discloses nothing — it is a producer
|
|
104
|
+
* self-check: TK-27 resolves every wikilink *node*, so a `[[` reaching output
|
|
105
|
+
* from prose means the degradation failed. That reading is only true outside
|
|
106
|
+
* `<code>` and `<pre>`. A note documenting Obsidian syntax is content, and its
|
|
107
|
+
* fence renders as `<code class="language-text">[[not a link]]</code>` —
|
|
108
|
+
* measured through the shipped renderer, brackets intact and unescaped. The
|
|
109
|
+
* artifact-level version of this rule was deleted for that reason
|
|
110
|
+
* (`src/lib/schema.ts`); here it is narrowed instead, because over built output
|
|
111
|
+
* the rule still catches something real that nothing else does.
|
|
112
|
+
*
|
|
113
|
+
* Every other rule keeps applying everywhere. An absolute path or a
|
|
114
|
+
* `javascript:` URL inside a fence is exactly as much of a disclosure as one in
|
|
115
|
+
* a paragraph — more, if anything, because a reader is likelier to copy it.
|
|
116
|
+
*/
|
|
117
|
+
const CODE_EXEMPT = new Set(['unresolved [[wikilink]]']);
|
|
118
|
+
/**
|
|
119
|
+
* The rules that do not apply inside a **math** region, and why this is not
|
|
120
|
+
* {@link CODE_EXEMPT} with another member.
|
|
121
|
+
*
|
|
122
|
+
* Real TeX trips two of these rules by writing ordinary mathematics. `f:\mathbb{R}`
|
|
123
|
+
* and `X:\Omega` — a letter, a colon, a backslash — is the commonest way to
|
|
124
|
+
* typeset a function's signature, and it is byte-identical to a Windows drive
|
|
125
|
+
* path. `k[[x]]` is the ring of formal power series and `\mathbb{E}[[X\mid
|
|
126
|
+
* \mathcal{F}]]` a conditional expectation, both of which spell `[[`.
|
|
127
|
+
*
|
|
128
|
+
* **Widening `CODE_EXEMPT` instead was measured and is wrong.** Adding
|
|
129
|
+
* `absolute local path` there exempts *every* code region, and a real host path
|
|
130
|
+
* in a `language-bash` fence — `cd C:\Users\alice\vault\private.md`, a pasted
|
|
131
|
+
* shell transcript, which is the likeliest way a genuine host path reaches
|
|
132
|
+
* `dist/` at all — goes from FIRES to CLEAN. Measured through this scanner, not
|
|
133
|
+
* reasoned about. `FRAGMENT_EXEMPT` fails the same way in the other direction:
|
|
134
|
+
* it would take a path split by markup with it, which is the TK-29 defect the
|
|
135
|
+
* fragment pass exists for.
|
|
136
|
+
*
|
|
137
|
+
* So the exemption is keyed on the minted marker, which is the narrowest
|
|
138
|
+
* carrier that distinguishes "an author typeset a function" from "an author
|
|
139
|
+
* pasted a terminal". `unresolved [[wikilink]]` is already exempt in every code
|
|
140
|
+
* region via `CODE_EXEMPT`, so this set holds only the path rule; it is spelled
|
|
141
|
+
* as a set rather than a literal because the *reason* generalises to any rule a
|
|
142
|
+
* TeX corpus can spell by accident, and the next one should join it here rather
|
|
143
|
+
* than widening a different set.
|
|
144
|
+
*
|
|
145
|
+
* **Scoped to client math mode, because in build-time mode it protects nothing
|
|
146
|
+
* and costs something.** Measured: with `MATH_MODE` at `build-time` every TeX
|
|
147
|
+
* subject above renders to MathML — `f:\mathbb{R}` becomes
|
|
148
|
+
* `<mi>f</mi><mo>:</mo>…` and `k[[x]]` becomes fence elements — so none trips a
|
|
149
|
+
* rule and the exemption is never reached. Only under client rendering, where
|
|
150
|
+
* the TeX source itself ships, do the raw bytes reach the page.
|
|
151
|
+
*
|
|
152
|
+
* **The key used to be author-reachable, and that is what the minted marker
|
|
153
|
+
* fixed.** A note writing a fence labelled `language-math` produces
|
|
154
|
+
* `<code class="language-math">`, and so does raw HTML the sanitizer's
|
|
155
|
+
* `allowedClasses.code: ['language-*']` preserves — so an author could put a
|
|
156
|
+
* real host path inside one and this exemption passed it. Measured with
|
|
157
|
+
* `MATH_MODE` at `client`, where the exemption is live: a fence around
|
|
158
|
+
* `cd C:\Users\alice\vault\private.md` went **CLEAN**, and against
|
|
159
|
+
* {@link RENDERED_MARKER} it **FIRES**.
|
|
160
|
+
*
|
|
161
|
+
* **The scope stays anyway**, because the two reasons are independent: the
|
|
162
|
+
* marker makes the key unforgeable, and the mode scope means that in the shipped
|
|
163
|
+
* configuration there is no exemption to reach at all. Removing either would be
|
|
164
|
+
* a widening.
|
|
165
|
+
*
|
|
166
|
+
* **What remains, and it is not closed by the marker.** The exemption's *subject*
|
|
167
|
+
* is still an author's own TeX, so an author can put a path-shaped string in a
|
|
168
|
+
* genuine expression: measured, `$$D:/vault/clients/acme/renewal.md$$` is valid
|
|
169
|
+
* TeX, renders, and goes CLEAN, as does `\texttt{…}` around anything. The
|
|
170
|
+
* backslash form throws at the renderer, so the Windows spelling is blocked and
|
|
171
|
+
* the forward-slash one is not. That is the rule protecting an author from their
|
|
172
|
+
* own paths being disabled by that same author, deliberately, inside content
|
|
173
|
+
* they wrote — a different thing from a *forged* marker, and bounded the same
|
|
174
|
+
* way: nobody else's disclosure is at stake.
|
|
175
|
+
*/
|
|
176
|
+
const MATH_EXEMPT = MATH_MODE === 'client' ? new Set(['absolute local path']) : new Set();
|
|
177
|
+
/**
|
|
178
|
+
* The contents of every `code` element the renderer marked as math, blanked.
|
|
179
|
+
*
|
|
180
|
+
* Structurally a sibling of {@link withoutCodeRegions} rather than a fresh
|
|
181
|
+
* regex, and deliberately: blanked rather than removed so byte offsets are
|
|
182
|
+
* unchanged, non-greedy, tag-anchored on both ends, and **failing closed on
|
|
183
|
+
* every ambiguity** — an unmarked region is left alone and therefore scanned.
|
|
184
|
+
*
|
|
185
|
+
* **One ambiguity it does not fail closed on, measured rather than claimed.** A
|
|
186
|
+
* marked `<code>` left unclosed swallows up to the *next* `</code>`, so a real
|
|
187
|
+
* region after it is blanked too. That is inherited from `withoutCodeRegions`'s
|
|
188
|
+
* shape and is unreachable through this pipeline — the renderer emits balanced
|
|
189
|
+
* tags and the sanitizer rebuilds them — but the honest description of the
|
|
190
|
+
* regex is that it trusts its input to be well formed.
|
|
191
|
+
*
|
|
192
|
+
* The marker is required here, where `withoutCodeRegions` deliberately requires
|
|
193
|
+
* no class — because the two are answering different questions. That function
|
|
194
|
+
* asks "is this a code region at all", and a body can legitimately write a bare
|
|
195
|
+
* `<code>`; this one asks "did a *renderer* produce this", which only the minted
|
|
196
|
+
* marker answers. A bare `<code>` containing a drive path is a pasted path and
|
|
197
|
+
* must still fail.
|
|
198
|
+
*/
|
|
199
|
+
function withoutMathRegions(text) {
|
|
200
|
+
return text.replace(/<code(\s[^>]*)?>([\s\S]*?)<\/code>/gi, (whole, attributes, body) => {
|
|
201
|
+
// **Keyed on the minted marker, not on the language class.** A note can
|
|
202
|
+
// write a fence labelled `language-math` and produce
|
|
203
|
+
// `<code class="language-math">` itself — measured, a host path inside one
|
|
204
|
+
// went FIRES → CLEAN, which is this exemption disabled by the author it
|
|
205
|
+
// exists to protect. `RENDERED_MARKER` is written at
|
|
206
|
+
// `src/lib/markdown.ts`'s substitution step, *after* the sanitizer, so no
|
|
207
|
+
// author byte can carry it; an attempt to write it is stripped, and an
|
|
208
|
+
// attempt to write the substitution token makes the build throw.
|
|
209
|
+
//
|
|
210
|
+
// **Fails closed**: an unmarked region is left alone and therefore
|
|
211
|
+
// scanned. That is the direction an exemption has to err in, and it is
|
|
212
|
+
// what makes a renderer whose output shape changes — or a marker that
|
|
213
|
+
// failed to mint — cost a false positive rather than a silent hole.
|
|
214
|
+
// **Anchored at an attribute boundary, not merely contained.** A bare
|
|
215
|
+
// `includes` matches the marker inside *another* attribute's value —
|
|
216
|
+
// `class="x data-rendered="math""` — which is a shape this pipeline cannot
|
|
217
|
+
// emit, because the sanitizer entity-escapes an inner quote. Relying on
|
|
218
|
+
// that is relying on a layer away from here; requiring whitespace before
|
|
219
|
+
// the attribute name makes the test hold on its own.
|
|
220
|
+
if (attributes === undefined || !MARKED_ATTRIBUTE(RENDERED_MATH).test(attributes)) {
|
|
221
|
+
return whole;
|
|
222
|
+
}
|
|
223
|
+
return whole.slice(0, whole.length - body.length - '</code>'.length) + ' '.repeat(body.length) + '</code>';
|
|
224
|
+
});
|
|
225
|
+
}
|
|
226
|
+
/**
|
|
227
|
+
* The contents of every `<code>` and `<pre>` element blanked.
|
|
228
|
+
*
|
|
229
|
+
* Blanked rather than removed, so byte offsets are unchanged and the text either
|
|
230
|
+
* side of a fence cannot be joined into a marker that neither half contains —
|
|
231
|
+
* the same reason `normalizedForms` refuses a whitespace-stripped form.
|
|
232
|
+
*
|
|
233
|
+
* ## Why the tag alone, and not `class="language-…"` as well
|
|
234
|
+
*
|
|
235
|
+
* A `<code>` a *note body* wrote is exempted too, and that was tested rather
|
|
236
|
+
* than assumed. `sanitize-html` allows `code` as a raw tag, so a body containing
|
|
237
|
+
* `Text with a raw <code> tag then [[stray]] after.` renders as
|
|
238
|
+
* `<p>Text with a raw <code> tag …</code></p>` — a `<code>` element wrapping
|
|
239
|
+
* ordinary prose, with no language class. Requiring the class looked like the
|
|
240
|
+
* safer rule and is measurably worse:
|
|
241
|
+
*
|
|
242
|
+
* - **The escape it closes is unreachable.** Put that exact body through the
|
|
243
|
+
* real producer and the traversal parses `[[stray]]` as a link node and
|
|
244
|
+
* degrades it, so the artifact reads `then stray after.` and the rendered page
|
|
245
|
+
* contains no `[[` at all. A note cannot carry a stray `[[` into prose in the
|
|
246
|
+
* first place — that is what this rule is checking for, and it is checking for
|
|
247
|
+
* a *producer* defect, not for anything a body can arrange.
|
|
248
|
+
* - **It breaks a case that is real.** Inline code renders as a bare
|
|
249
|
+
* `<code>[[syntax]]</code>` with no class — measured — so requiring the class
|
|
250
|
+
* makes ``Inline `[[syntax]]` here`` fail the build. Documenting wikilink
|
|
251
|
+
* syntax inline is exactly as legitimate as documenting it in a fence.
|
|
252
|
+
*
|
|
253
|
+
* So the exemption is the tag, and the reason it is safe is not that a body
|
|
254
|
+
* cannot write the tag: it is that a body cannot produce the *marker* this rule
|
|
255
|
+
* looks for. If a future change lets one, this comment is where to start.
|
|
256
|
+
*
|
|
257
|
+
* Non-greedy, and tag-name-anchored on both ends: `<pre` matches `<pre>` and
|
|
258
|
+
* `<pre tabindex="0">` alike, which is what this pipeline emits, while `</pre>`
|
|
259
|
+
* closes only a `pre`. Nesting is not handled and does not need to be — `code`
|
|
260
|
+
* inside `pre` is the only nesting that occurs here, and blanking the outer
|
|
261
|
+
* covers the inner.
|
|
262
|
+
*
|
|
263
|
+
* **Every ambiguous shape fails closed**, measured: an unclosed `<code>`, a
|
|
264
|
+
* `</code>` inside an attribute value, and a stray between two fences are all
|
|
265
|
+
* still reported. That is the direction an exemption has to err in.
|
|
266
|
+
*/
|
|
267
|
+
function withoutCodeRegions(text) {
|
|
268
|
+
return text.replace(/<(pre|code)(\s[^>]*)?>([\s\S]*?)<\/\1>/gi, (whole, tag, _attributes, body) => whole.slice(0, whole.length - body.length - `</${tag}>`.length) +
|
|
269
|
+
' '.repeat(body.length) +
|
|
270
|
+
`</${tag}>`);
|
|
271
|
+
}
|
|
272
|
+
/**
|
|
273
|
+
* The forms of one file's text a browser could resolve a marker out of.
|
|
274
|
+
*
|
|
275
|
+
* Three, mirroring `checkPrivacy` in `src/lib/schema.ts` and for the same
|
|
276
|
+
* reasons. The raw bytes, because that is what ships. The entity-decoded text,
|
|
277
|
+
* because `/` is a `/` to every HTML parser and `msw/secret` is the
|
|
278
|
+
* `msw/` marker wearing a hat. And the decoded text with format characters
|
|
279
|
+
* removed, because a zero-width joiner between `msw` and `/` is invisible in
|
|
280
|
+
* the rendered page and defeats a literal match.
|
|
281
|
+
*
|
|
282
|
+
* A whitespace-stripped form is deliberately absent: joining lines turns
|
|
283
|
+
* ordinary prose such as "Option A:" above "/usr/bin" into an apparent drive
|
|
284
|
+
* path, which is the false positive the schema documents avoiding.
|
|
285
|
+
*/
|
|
286
|
+
function normalizedForms(text) {
|
|
287
|
+
const decoded = decodeHtmlEntities(text);
|
|
288
|
+
return [text, decoded, decoded.replace(/\p{Cf}+/gu, '')];
|
|
289
|
+
}
|
|
290
|
+
/**
|
|
291
|
+
* Decode the numeric and named entity forms a browser resolves.
|
|
292
|
+
*
|
|
293
|
+
* Deliberately small: the named set is the handful whose expansion produces a
|
|
294
|
+
* character that appears in a marker — a separator, a scheme's colon, or a
|
|
295
|
+
* bracket. Decoding every named entity in HTML would need a table this file has
|
|
296
|
+
* no reason to carry, and the ones omitted cannot manufacture a marker.
|
|
297
|
+
*
|
|
298
|
+
* The trailing `;` is optional, matching `decodeEntities` in
|
|
299
|
+
* `src/lib/schema.ts`, which is the reference implementation for what counts as
|
|
300
|
+
* a marker. HTML5 decodes a numeric reference without it — the missing
|
|
301
|
+
* semicolon is a parse error that still yields the character — so
|
|
302
|
+
* `msw/ secret` renders as the private path marker. An earlier version of
|
|
303
|
+
* this file required the semicolon and let that through, on a comment claiming
|
|
304
|
+
* the schema had the same gap. It does not, and never did: the two decoders
|
|
305
|
+
* differed by exactly this character, and the scan was the weaker one.
|
|
306
|
+
*
|
|
307
|
+
* Double encoding is deliberately *not* followed. `&#x2F;` renders as the
|
|
308
|
+
* literal text `/` in a browser, not as `/`, so decoding twice would
|
|
309
|
+
* invent a marker the reader never sees.
|
|
310
|
+
*/
|
|
311
|
+
function decodeHtmlEntities(text) {
|
|
312
|
+
const named = {
|
|
313
|
+
amp: '&',
|
|
314
|
+
colon: ':',
|
|
315
|
+
lsqb: '[',
|
|
316
|
+
sol: '/',
|
|
317
|
+
rsqb: ']',
|
|
318
|
+
bsol: '\\',
|
|
319
|
+
tab: '\t',
|
|
320
|
+
newline: '\n',
|
|
321
|
+
};
|
|
322
|
+
return text.replace(/&(?:#(\d+)|#[xX]([0-9a-fA-F]+)|([A-Za-z][A-Za-z0-9]*));?/g, (match, decimal, hex, name) => {
|
|
323
|
+
const code = decimal === undefined ? (hex === undefined ? NaN : parseInt(hex, 16)) : Number(decimal);
|
|
324
|
+
if (Number.isFinite(code))
|
|
325
|
+
return code <= 0x10ffff ? String.fromCodePoint(code) : match;
|
|
326
|
+
return (name === undefined ? undefined : named[name.toLowerCase()]) ?? match;
|
|
327
|
+
});
|
|
328
|
+
}
|
|
329
|
+
/**
|
|
330
|
+
* File kinds whose bytes are text, and are therefore scanned.
|
|
331
|
+
*
|
|
332
|
+
* `.xml` is here ahead of the first feed or sitemap: scanning a format that
|
|
333
|
+
* does not exist yet costs nothing and errs toward coverage, whereas omitting
|
|
334
|
+
* it would put the first one in the unclassified bucket.
|
|
335
|
+
*/
|
|
336
|
+
const TEXT_EXTENSIONS = new Set([
|
|
337
|
+
'.css',
|
|
338
|
+
'.html',
|
|
339
|
+
'.js',
|
|
340
|
+
'.json',
|
|
341
|
+
'.svg',
|
|
342
|
+
'.txt',
|
|
343
|
+
'.xml',
|
|
344
|
+
]);
|
|
345
|
+
/** Extensionless files Cloudflare Pages reads. Both are text and both ship. */
|
|
346
|
+
const TEXT_NAMES = new Set(['_headers', '_redirects']);
|
|
347
|
+
/**
|
|
348
|
+
* File kinds whose bytes are not text, so a marker search over them reports
|
|
349
|
+
* coincidences rather than residue.
|
|
350
|
+
*
|
|
351
|
+
* Mirrors the `binary` list in `.gitattributes`, plus `.jpeg` — which that file
|
|
352
|
+
* omits because nothing in the tree uses the spelling, and which is listed here
|
|
353
|
+
* because an unlisted extension would fail the scan as unclassified rather than
|
|
354
|
+
* be treated as the image it is. These are not *unscanned by
|
|
355
|
+
* accident* — they are declared, which is what separates them from a file kind
|
|
356
|
+
* nobody has considered.
|
|
357
|
+
*/
|
|
358
|
+
const BINARY_EXTENSIONS = new Set([
|
|
359
|
+
'.ico',
|
|
360
|
+
'.jpg',
|
|
361
|
+
'.jpeg',
|
|
362
|
+
'.png',
|
|
363
|
+
'.wasm',
|
|
364
|
+
'.webp',
|
|
365
|
+
'.woff',
|
|
366
|
+
'.woff2',
|
|
367
|
+
]);
|
|
368
|
+
/**
|
|
369
|
+
* Pagefind's own bundle, excluded with a reason rather than by convenience —
|
|
370
|
+
* **except for its fragments, which are read after inflating.**
|
|
371
|
+
*
|
|
372
|
+
* Its runtime — `pagefind*.js`, `.css`, and two WebAssembly binaries — is
|
|
373
|
+
* third-party code this repository does not author; it legitimately contains
|
|
374
|
+
* `javascript:void(0)` in its form markup, and the compiled WebAssembly contains
|
|
375
|
+
* `[[` as a byte coincidence.
|
|
376
|
+
*
|
|
377
|
+
* ## The index was excluded on an argument that is measurably false
|
|
378
|
+
*
|
|
379
|
+
* That argument was: Pagefind indexes exactly the `data-pagefind-body` subtree
|
|
380
|
+
* of the built HTML, that HTML is scanned here in full, so nothing reaches the
|
|
381
|
+
* index without first appearing in a page this scan reads. It is wrong in one
|
|
382
|
+
* direction that matters, because **a fragment stores extracted text, not
|
|
383
|
+
* markup** — decoded, and with inline elements joined.
|
|
384
|
+
*
|
|
385
|
+
* Measured on a real packaged build, from an ordinary note body:
|
|
386
|
+
*
|
|
387
|
+
* - `A path ms**w/s**ecret here.` renders as `ms<strong>w/s</strong>ecret`. No
|
|
388
|
+
* byte sequence anywhere in `dist/` contains `msw/`; the fragment contains it.
|
|
389
|
+
* The build reported `residue scan ok: 25 files, 0 findings`.
|
|
390
|
+
* - The same split carried `/home/someone/`, `javascript:`, and
|
|
391
|
+
* `sourceMappingURL` into the fragment, each invisible to every raw read.
|
|
392
|
+
* - HTML escaping does it too, without any markup: a body containing
|
|
393
|
+
* `zzqacme&secret` ships `&` in the page and the decoded form in the
|
|
394
|
+
* fragment.
|
|
395
|
+
*
|
|
396
|
+
* So a marker split by emphasis, or merely containing an `&`, reached the
|
|
397
|
+
* shipped search index past a green scan. That is the failure mode a privacy
|
|
398
|
+
* gate may not have, and it is not hypothetical — a reader typing the marker
|
|
399
|
+
* into the site's own search box gets the page back.
|
|
400
|
+
*
|
|
401
|
+
* ## Why only the fragments
|
|
402
|
+
*
|
|
403
|
+
* **Because the fragment is the source the other members are derived from.**
|
|
404
|
+
* Every other member of the bundle — `.pf_index`, `.pf_meta`, `.pf_filter`, and
|
|
405
|
+
* the two `.pagefind` blobs — is built from the same extracted text a fragment
|
|
406
|
+
* stores verbatim, so any marker reaching them has already passed through a
|
|
407
|
+
* fragment this scan now reads. That argument is what makes the exclusion safe,
|
|
408
|
+
* and it survives a new rule being added to {@link RESIDUE_RULES} — which is the
|
|
409
|
+
* property the earlier reasoning here lacked.
|
|
410
|
+
*
|
|
411
|
+
* **The set is stated as "everything in the bundle that is not a fragment"
|
|
412
|
+
* rather than as a list, and that is deliberate.** An earlier version of this
|
|
413
|
+
* comment enumerated `.pf_index` and `.pf_meta` as though they were the whole
|
|
414
|
+
* excluded set, which is how `.pf_filter` — under `pagefind/filter/`, and the
|
|
415
|
+
* member that stores its values *least* digested of any — went unnamed. It is
|
|
416
|
+
* genuinely covered, because a filter value is also in its fragment's `filters`
|
|
417
|
+
* key, and this build emits none: nothing under `src/` writes
|
|
418
|
+
* `data-pagefind-filter` (`src/scripts/search-dialog.ts:482` records that too).
|
|
419
|
+
* But an auditor reading a list looks for the members on it. The code has always
|
|
420
|
+
* excluded by "not a fragment"; the comment now says the same thing.
|
|
421
|
+
*
|
|
422
|
+
* A weaker argument was written first and is recorded because it is wrong in an
|
|
423
|
+
* instructive way: that a word list is split on punctuation, so a marker cannot
|
|
424
|
+
* survive in one as a matchable string. Measured, a token of pure letters
|
|
425
|
+
* reaches `.pf_index` intact and **lowercased** — `sourceMappingXYZ` is stored
|
|
426
|
+
* as `sourcemappingxyz`. So `sourceMappingURL`, the one rule here that is a bare
|
|
427
|
+
* word, misses it only because that rule carries no `i` flag. The exclusion was
|
|
428
|
+
* resting on a regular-expression flag nobody had written down, and adding `i`
|
|
429
|
+
* to that rule — a change that reads as strictly safer — would have opened the
|
|
430
|
+
* surface silently.
|
|
431
|
+
*
|
|
432
|
+
* The two `.pagefind` blobs stay excluded, and they are not what their name
|
|
433
|
+
* suggests: measured, both begin with the gzip magic `1f 8b`, and their inflated
|
|
434
|
+
* bytes begin `pagefind` rather than the WebAssembly `\0asm`. Inflated they
|
|
435
|
+
* carry no marker; **raw** they trip the `[[` rule as a byte coincidence, which
|
|
436
|
+
* is the actual reason to keep them out.
|
|
437
|
+
*
|
|
438
|
+
* `tests/built-output.test.ts` draws the same third-party boundary for the same
|
|
439
|
+
* reason, though more loosely — it matches `pagefind` anywhere in the path.
|
|
440
|
+
* This is anchored at the root instead, because loose matching has a false-pass
|
|
441
|
+
* mode a privacy gate must not have: Pagefind's bundle is emitted to exactly
|
|
442
|
+
* `dist/pagefind/`, but a published note slugged `pagefind` builds to
|
|
443
|
+
* `dist/notes/pagefind/index.html`, and a substring test would exclude that
|
|
444
|
+
* page from the scan entirely. Anchoring costs nothing and removes the case.
|
|
445
|
+
*/
|
|
446
|
+
const THIRD_PARTY = 'pagefind';
|
|
447
|
+
/**
|
|
448
|
+
* The pinned SQLite runtime, also third-party and also anchored at the root.
|
|
449
|
+
*
|
|
450
|
+
* Its Emscripten glue legitimately contains `[[` and `file://` strings, which the
|
|
451
|
+
* residue rules would otherwise read as ANC content. It is copied verbatim from
|
|
452
|
+
* the pinned package (see `scripts/build-wasm.ts`), never authored here, and its
|
|
453
|
+
* WASM binary is a compiled artifact; `tests/built-output.test.ts` and the
|
|
454
|
+
* output inventory still bind its exact members. A note slugged `wasm` builds
|
|
455
|
+
* under `/notes/wasm/`, so anchoring keeps that page scanned.
|
|
456
|
+
*/
|
|
457
|
+
const WASM_RUNTIME = 'wasm';
|
|
458
|
+
/** The one member of that bundle whose text is this site's own content. */
|
|
459
|
+
const FRAGMENT_EXTENSION = '.pf_fragment';
|
|
460
|
+
/**
|
|
461
|
+
* The rules that do not apply to a fragment, and the one reason they do not.
|
|
462
|
+
*
|
|
463
|
+
* A fragment is extracted *text*: Pagefind strips the markup before storing it,
|
|
464
|
+
* so a fenced ` ```text\n[[not a link]] ` arrives with no `<code>` element left
|
|
465
|
+
* to exempt it. Measured — the fragment for such a note reads
|
|
466
|
+
* `"content":"Obsidian writes a link as: [[not a link]]"` while the page it came
|
|
467
|
+
* from reads `<code class="language-text">[[not a link]]</code>`. Applying the
|
|
468
|
+
* wikilink rule there would fail the build on a note documenting Obsidian
|
|
469
|
+
* syntax, which is exactly the case {@link CODE_EXEMPT} exists to permit, and it
|
|
470
|
+
* would do so with no way for the author to escape it.
|
|
471
|
+
*
|
|
472
|
+
* Dropping it costs nothing, because `[[` is the one rule that is not a privacy
|
|
473
|
+
* marker. It discloses nothing; it is a producer self-check over *rendered
|
|
474
|
+
* prose*, and the rendered prose is scanned in full. A `[[` that reaches a
|
|
475
|
+
* fragment either came from a code region — legitimate — or came from prose, in
|
|
476
|
+
* which case the page carrying that prose fails this scan first.
|
|
477
|
+
*
|
|
478
|
+
* Every other rule applies. A path or a scheme in a fragment is a disclosure
|
|
479
|
+
* wherever it came from, and the search box will hand it to a reader.
|
|
480
|
+
*
|
|
481
|
+
* **Spelled out rather than aliased to {@link CODE_EXEMPT}, though the two hold
|
|
482
|
+
* the same member today.** They encode opposite decisions: `CODE_EXEMPT`
|
|
483
|
+
* *narrows* a rule, blanking code regions while still catching the marker in
|
|
484
|
+
* prose either side, and this one *drops* a rule outright. A second rule added
|
|
485
|
+
* to `CODE_EXEMPT` for the narrowing reason would, through an alias, silently
|
|
486
|
+
* become disabled over the whole search index — and there is nothing about those
|
|
487
|
+
* two reasons that makes them co-vary. The duplicated literal is the cost of
|
|
488
|
+
* keeping one edit from meaning two things.
|
|
489
|
+
*/
|
|
490
|
+
const FRAGMENT_EXEMPT = new Set(['unresolved [[wikilink]]']);
|
|
491
|
+
/**
|
|
492
|
+
* The rules that do not apply to a database's raw bytes, and why exactly one.
|
|
493
|
+
*
|
|
494
|
+
* A database is read twice — as rows, and as bytes — and the byte pass exists
|
|
495
|
+
* for one thing only: a deleted row's payload, which survives in the file until
|
|
496
|
+
* a `VACUUM` and which no `SELECT` can reach. Its cost was measured across four
|
|
497
|
+
* corpus sizes, and it is zero until the file gets large: at 0.2 MB, 2.1 MB and
|
|
498
|
+
* 43 MB no rule matched the bytes that did not also match a row, and at 172 MB
|
|
499
|
+
* exactly one did — `[[`, struck by a **B-tree interior page's cell-pointer
|
|
500
|
+
* array**, where a run of two-byte big-endian offsets happens to spell `5b 5b`.
|
|
501
|
+
* Verified at byte 95,618,728 of a 12,000-row database of pure `x`/`y` padding:
|
|
502
|
+
* the surrounding bytes are `00 00 5f 21 b6 2c 00 00 5f 1a b6 2a`, no row
|
|
503
|
+
* contains `[[`, and the hits scale with the pointer array rather than with
|
|
504
|
+
* anything authored.
|
|
505
|
+
*
|
|
506
|
+
* So the rule is dropped over bytes and kept over rows, and that split is not a
|
|
507
|
+
* weakening: `[[` is the one rule here that is **not** a privacy marker — it
|
|
508
|
+
* discloses nothing, it is the producer's self-check that link degradation ran,
|
|
509
|
+
* and what it is a check *on* is authored text. A `[[` in a free page came from
|
|
510
|
+
* a row that was authored and then withheld; if the degradation failed, it
|
|
511
|
+
* failed on the live rows too and is caught there. Reading it off B-tree
|
|
512
|
+
* structure measures SQLite's file format, not the corpus.
|
|
513
|
+
*
|
|
514
|
+
* Every other rule applies to both. A path, a scheme or a source-map reference
|
|
515
|
+
* found in a free page is a real disclosure — that is the whole reason the byte
|
|
516
|
+
* pass exists, and none of them can be spelled by a pointer array: each requires
|
|
517
|
+
* a multi-character literal that a run of offsets does not produce, which the
|
|
518
|
+
* same sweep confirms at every size measured.
|
|
519
|
+
*
|
|
520
|
+
* **Spelled out rather than aliased to {@link FRAGMENT_EXEMPT} or
|
|
521
|
+
* {@link CODE_EXEMPT}, though all three hold the same member today**, for the
|
|
522
|
+
* reason `FRAGMENT_EXEMPT` already states: they encode three different
|
|
523
|
+
* decisions — narrow a rule, drop it over one carrier, drop it over one *pass*
|
|
524
|
+
* of one carrier — and nothing makes those three co-vary.
|
|
525
|
+
*/
|
|
526
|
+
const RAW_BYTES_EXEMPT = new Set(['unresolved [[wikilink]]']);
|
|
527
|
+
/**
|
|
528
|
+
* The forms of a Markdown body a *reader* resolves a marker out of.
|
|
529
|
+
*
|
|
530
|
+
* A database stores Markdown source, and the reader receives what a renderer
|
|
531
|
+
* makes of it. That gap is precisely the surface TK-29 closed for the search
|
|
532
|
+
* index and which storing source would otherwise reopen: a body written
|
|
533
|
+
* `ms**w/s**ecret` renders as `ms<strong>w/s</strong>ecret`, whose text is
|
|
534
|
+
* `msw/secret`, while the stored bytes contain no `msw/` anywhere. Measured
|
|
535
|
+
* both ways — through the real renderer, `ms**w/s**ecret`, `ms*w/s*ecret` and
|
|
536
|
+
* ``ms`w/s`ecret`` all reach the page as the marker.
|
|
537
|
+
*
|
|
538
|
+
* So the delimiters stripped here are the ones the shipped renderer actually
|
|
539
|
+
* consumes *between two word characters*: `*`, `_` and a backtick. That the set
|
|
540
|
+
* is measured rather than assumed matters in both directions — `ms__w/s__ecret`
|
|
541
|
+
* and `ms_w/s_ecret` were measured **not** to join, because CommonMark does not
|
|
542
|
+
* open intra-word emphasis on underscores, so including `_` is this form being
|
|
543
|
+
* deliberately wider than the renderer rather than narrower.
|
|
544
|
+
*
|
|
545
|
+
* The tag form is the same defect wearing HTML, for the day a body carries raw
|
|
546
|
+
* inline markup or a column stores rendered fragments.
|
|
547
|
+
*
|
|
548
|
+
* **Anchored between two non-delimiter, non-space characters**, which is what
|
|
549
|
+
* keeps it from inventing markers: ordinary emphasis (`**bold** at edges`) and
|
|
550
|
+
* spaced asterisks (`a * b * c`) are untouched. Swept over every tracked `.md`,
|
|
551
|
+
* `.ts` and `.astro` file in this repository — 104 files — and over every text
|
|
552
|
+
* file in a built `dist/`, **no rule matched a joined form that did not already
|
|
553
|
+
* match the raw text**. The cost it does carry is real and small:
|
|
554
|
+
* `snake_case_name` joins to `snakecasename`, which no rule here can match.
|
|
555
|
+
*
|
|
556
|
+
* **A tag is replaced by a space, not removed**, and that is not symmetry with
|
|
557
|
+
* the emphasis form — it is the one place this transform can invent a marker.
|
|
558
|
+
* Deleting the tag lets the text either side abut, so `Option A:<br>/usr/bin`
|
|
559
|
+
* becomes `Option A:/usr/bin` and trips the absolute-path rule on ordinary
|
|
560
|
+
* prose; measured, along with `<td>A:</td><td>/usr</td>`. That is precisely the
|
|
561
|
+
* false positive {@link normalizedForms} refuses a whitespace-stripped form to
|
|
562
|
+
* avoid.
|
|
563
|
+
*
|
|
564
|
+
* **Anchored inside a word**, which is what keeps the joining honest: a tag
|
|
565
|
+
* joins only where both sides are word characters, because that is the only
|
|
566
|
+
* shape a renderer joins into a single word. `ms<strong>w/s</strong>ecret`
|
|
567
|
+
* becomes `msw/secret`; `A:<br>/usr` does not become `A:/usr`, because `:` is
|
|
568
|
+
* not a word character and no reader sees those two run together.
|
|
569
|
+
*/
|
|
570
|
+
function joinedForms(text) {
|
|
571
|
+
return [
|
|
572
|
+
text.replace(/(?<=[^\s*_`])[*_`]+(?=[^\s*_`])/g, ''),
|
|
573
|
+
text.replace(/(?<=[A-Za-z0-9])<\/?[A-Za-z][A-Za-z0-9]*(?:\s[^<>]*)?>(?=[A-Za-z0-9])/g, ''),
|
|
574
|
+
text.replace(/<\/?[A-Za-z][A-Za-z0-9]*(?:\s[^<>]*)?>/g, ' '),
|
|
575
|
+
];
|
|
576
|
+
}
|
|
577
|
+
/**
|
|
578
|
+
* Every form of one database subject a marker could be resolved out of, lazily.
|
|
579
|
+
*
|
|
580
|
+
* A generator rather than an array because a database subject is the whole
|
|
581
|
+
* corpus: materialising all nine forms at once held 1.16 GB for a 129 MB file
|
|
582
|
+
* and exhausted a 4 GB heap at 304 MB. Yielded one at a time, the caller's
|
|
583
|
+
* first match ends the walk and at most one derived string is live.
|
|
584
|
+
*/
|
|
585
|
+
function* databaseForms(subject) {
|
|
586
|
+
yield* normalizedForms(subject);
|
|
587
|
+
for (const joined of joinedForms(subject))
|
|
588
|
+
yield* normalizedForms(joined);
|
|
589
|
+
}
|
|
590
|
+
/**
|
|
591
|
+
* Markdown fenced and inline code spans blanked, so `[[` inside them is exempt.
|
|
592
|
+
*
|
|
593
|
+
* The sibling of {@link withoutCodeRegions}, and it exists because that function
|
|
594
|
+
* cannot serve this carrier: it keys on `<code>` and `<pre>` elements, and a
|
|
595
|
+
* database row holds Markdown that has no elements in it yet. Measured — the
|
|
596
|
+
* same note that builds clean today, whose page carries
|
|
597
|
+
* `<code class="language-text">[[not a link]]</code>`, stores a body that
|
|
598
|
+
* survives `withoutCodeRegions` untouched and trips the wikilink rule.
|
|
599
|
+
*
|
|
600
|
+
* **The reasoning at {@link CODE_EXEMPT} is what is being carried across, not
|
|
601
|
+
* its implementation.** A note documenting Obsidian syntax is content; the
|
|
602
|
+
* `[[` rule is the producer's self-check that link degradation ran, and that
|
|
603
|
+
* reading is only true outside a code region. Which characters delimit a code
|
|
604
|
+
* region is a property of the carrier, so the rule keeps its meaning by
|
|
605
|
+
* changing its blanking to match. Dropping the rule instead — the shape
|
|
606
|
+
* {@link FRAGMENT_EXEMPT} takes — would be wrong here for the reason stated
|
|
607
|
+
* there: a fragment's prose is scanned in full on the page it came from, and a
|
|
608
|
+
* database's is not scanned anywhere else.
|
|
609
|
+
*
|
|
610
|
+
* Blanked rather than removed, for {@link withoutCodeRegions}'s reason: offsets
|
|
611
|
+
* are unchanged, so text either side of a fence cannot be joined into a marker
|
|
612
|
+
* neither half contains.
|
|
613
|
+
*
|
|
614
|
+
* ## Which shapes count as code, and why the list is longer than fences
|
|
615
|
+
*
|
|
616
|
+
* Every shape below was checked **through the shipped renderer**, and each one
|
|
617
|
+
* puts its content inside a `<code>` element on the page — so failing the build
|
|
618
|
+
* on it fails on content the site publishes cleanly, with no escape available to
|
|
619
|
+
* the author. That is the wall of false positives this exemption exists to
|
|
620
|
+
* prevent, and an earlier version of this function produced it on four shapes at
|
|
621
|
+
* once:
|
|
622
|
+
*
|
|
623
|
+
* - A **fenced block**, with up to three spaces of indent on either fence, which
|
|
624
|
+
* is what CommonMark permits.
|
|
625
|
+
* - A **fence inside a blockquote**, where every line carries a `>` prefix.
|
|
626
|
+
* - An **indented code block** — four spaces or a tab — which is ordinary
|
|
627
|
+
* Markdown and carries no fence at all.
|
|
628
|
+
* - An **inline span**, between matched backtick runs.
|
|
629
|
+
*
|
|
630
|
+
* Fences first, then indented blocks, then inline spans, so a backtick inside a
|
|
631
|
+
* fenced block cannot open a span that swallows the prose after it.
|
|
632
|
+
*
|
|
633
|
+
* **Every ambiguous shape fails closed** — an unterminated fence and an unpaired
|
|
634
|
+
* backtick both leave the text after them scanned.
|
|
635
|
+
*/
|
|
636
|
+
function withoutMarkdownCode(text) {
|
|
637
|
+
const blankLines = (block) => block.replace(/[^\n]/g, ' ');
|
|
638
|
+
return text
|
|
639
|
+
.replace(/(^|\n)([ \t]{0,3}(?:> ?)*)(```+|~~~+)([^\n]*\n)([\s\S]*?)(\n[ \t]{0,3}(?:> ?)*\3)/g, (_whole, lead, quote, fence, info, body, end) => `${lead}${quote}${fence}${info}${blankLines(body)}${end}`)
|
|
640
|
+
// An indented code block: a run of lines each starting with four spaces or a
|
|
641
|
+
// tab, after a blank line. The blank line is what separates it from the
|
|
642
|
+
// continuation of a paragraph, which is not code.
|
|
643
|
+
.replace(/(^|\n\s*?\n)((?:(?: {4}|\t)[^\n]*(?:\n|$))+)/g, (_whole, lead, block) => `${lead}${blankLines(block)}`)
|
|
644
|
+
.replace(/(`+)([^\n]+?)\1/g, (_whole, ticks, body) => `${ticks}${' '.repeat(body.length)}${ticks}`);
|
|
645
|
+
}
|
|
646
|
+
function walk(directory) {
|
|
647
|
+
const found = [];
|
|
648
|
+
for (const name of readdirSync(directory)) {
|
|
649
|
+
const path = join(directory, name);
|
|
650
|
+
if (statSync(path).isDirectory())
|
|
651
|
+
found.push(...walk(path));
|
|
652
|
+
else
|
|
653
|
+
found.push(path);
|
|
654
|
+
}
|
|
655
|
+
return found;
|
|
656
|
+
}
|
|
657
|
+
export function scanResidue(root = DIST, options = {}) {
|
|
658
|
+
const findings = [];
|
|
659
|
+
const detailed = [];
|
|
660
|
+
const scannedPaths = [];
|
|
661
|
+
let rowCount = 0;
|
|
662
|
+
let sawDatabase = false;
|
|
663
|
+
/**
|
|
664
|
+
* Record one finding.
|
|
665
|
+
*
|
|
666
|
+
* Two arrays and one call site per finding, because the split is only safe if
|
|
667
|
+
* it is exhaustive: an earlier version of this change redacted the matched
|
|
668
|
+
* text on the one line that obviously carried it and left `${root}` on the
|
|
669
|
+
* three that also did — the missing-directory line, and both vacuity guards.
|
|
670
|
+
* Under the packaged CLI `root` is the per-run `mkdtemp` workspace, whose
|
|
671
|
+
* random suffix differs between two runs of the same corpus, so those three
|
|
672
|
+
* were a host path on a public surface and a rename differential that could
|
|
673
|
+
* never be byte-equal.
|
|
674
|
+
*
|
|
675
|
+
* @param message The public half: a route this build was about to serve, and
|
|
676
|
+
* literals of this file's own source.
|
|
677
|
+
* @param detail The same finding with whatever must not be printed — the
|
|
678
|
+
* scanned root, the bytes that matched. Defaults to `message` when there is
|
|
679
|
+
* nothing to withhold.
|
|
680
|
+
*/
|
|
681
|
+
const report = (message, detail = message) => {
|
|
682
|
+
findings.push(message);
|
|
683
|
+
detailed.push(detail);
|
|
684
|
+
};
|
|
685
|
+
let files;
|
|
686
|
+
try {
|
|
687
|
+
files = walk(root);
|
|
688
|
+
}
|
|
689
|
+
catch (error) {
|
|
690
|
+
// **Vacuity throws; it is not a finding.** A finding is a fact about output
|
|
691
|
+
// that exists — this is the absence of output, and returning it as a
|
|
692
|
+
// finding made "I could not look" arrive in the same channel and the same
|
|
693
|
+
// shape as "I looked and found residue". Every caller that counts findings,
|
|
694
|
+
// tests `length`, or filters by rule name then treats an unbuilt directory
|
|
695
|
+
// as a dirty one, and the one caller that reports success reads
|
|
696
|
+
// `scannedCount` — which was 0, a number no assertion here checks.
|
|
697
|
+
//
|
|
698
|
+
// The other "could not look" cases in this file are deliberately *not*
|
|
699
|
+
// collapsed into this, and the line between them is a rule rather than a
|
|
700
|
+
// list: **a file that shipped and could not be read inside is a finding and
|
|
701
|
+
// names itself; vacuity is the case where there is no artifact at all.**
|
|
702
|
+
// Stated as a rule because a list is what `docs/gate-reading.md`'s corollary
|
|
703
|
+
// about enumerations warns against — and earned it here, since the first
|
|
704
|
+
// draft of this comment counted five such branches as four (it missed the
|
|
705
|
+
// search-index fragment, which has its own inflate because a fragment's
|
|
706
|
+
// bytes are never read into the shared gzip path).
|
|
707
|
+
// The public half names no path. `root` is a host filesystem path and this
|
|
708
|
+
// throw reaches a workflow log on a public repository, which is the
|
|
709
|
+
// disclosure the whole module exists to prevent — `BuildFailure` already
|
|
710
|
+
// carries the split, so the detail goes where the report reads it.
|
|
711
|
+
throw new BuildFailure('residue-scan-vacuous', 'the built site is missing or unreadable — run `pnpm run build` first. Nothing was ' +
|
|
712
|
+
'scanned, so a clean result would be the zero of having looked nowhere.', `${root} is missing or unreadable: ${error instanceof Error ? error.message : String(error)}`);
|
|
713
|
+
}
|
|
714
|
+
const provenanceOptions = {
|
|
715
|
+
...(options.vendoredChunksPath === undefined ? {} : { path: options.vendoredChunksPath }),
|
|
716
|
+
required: options.requireVendorProvenance ?? false,
|
|
717
|
+
};
|
|
718
|
+
const vendoredChunks = readVendorProvenance(root, provenanceOptions);
|
|
719
|
+
for (const path of files) {
|
|
720
|
+
const name = basename(path);
|
|
721
|
+
const where = relative(root, path);
|
|
722
|
+
const posixWhere = where.split(sep).join('/');
|
|
723
|
+
const extension = extname(name).toLowerCase();
|
|
724
|
+
// Anchored at the root of the built site: `pagefind/…` and nothing else.
|
|
725
|
+
// Its fragments are the one member read anyway — see {@link THIRD_PARTY}.
|
|
726
|
+
//
|
|
727
|
+
// **Both conditions, not the extension alone.** A `.pf_fragment` anywhere
|
|
728
|
+
// else is not Pagefind's — it is a file a user put in their notes, or a
|
|
729
|
+
// future build step's — and treating it as one would skip this file's
|
|
730
|
+
// fail-closed classifier for it and then report it under a location that
|
|
731
|
+
// names Pagefind, sending the reader to the wrong place. Anchored, it falls
|
|
732
|
+
// through to the unclassified branch and fails the scan by name, which is
|
|
733
|
+
// property 1 of this file's header.
|
|
734
|
+
const inBundle = where.split(sep)[0] === THIRD_PARTY;
|
|
735
|
+
// The SQLite runtime is the other declared third-party root; see
|
|
736
|
+
// {@link WASM_RUNTIME}. Every member is bound by the output inventory.
|
|
737
|
+
if (posixWhere.split('/')[0] === WASM_RUNTIME)
|
|
738
|
+
continue;
|
|
739
|
+
const isFragment = inBundle && extension === FRAGMENT_EXTENSION;
|
|
740
|
+
if (inBundle && !isFragment)
|
|
741
|
+
continue;
|
|
742
|
+
// Exact membership in build-minted provenance, never a basename or path
|
|
743
|
+
// pattern. Mixed and first-party chunks are absent and keep falling through
|
|
744
|
+
// to the full scan; an invalid or stale grant already threw above.
|
|
745
|
+
if (vendoredChunks.has(posixWhere))
|
|
746
|
+
continue;
|
|
747
|
+
// **A fragment's own filename may not be printed.** Measured: Pagefind names
|
|
748
|
+
// each fragment for a digest of the text in it, so two corpora differing
|
|
749
|
+
// only in a note's body produce `en_ad9f487` and `en_4d27cf9`. That makes
|
|
750
|
+
// the name a function of content the stream may not carry — and
|
|
751
|
+
// `tests/disclosure.test.ts`'s rename differential caught exactly that when
|
|
752
|
+
// this scan started reading fragments, which is the gate doing its job.
|
|
753
|
+
//
|
|
754
|
+
// The public half names the *surface* instead. It is a fixed literal, it is
|
|
755
|
+
// the same on every machine, and it loses nothing a reader needs: there is
|
|
756
|
+
// one search index, the fix is always in the note the marker came from, and
|
|
757
|
+
// the exact fragment is in the report's `detailed` half where the person who
|
|
758
|
+
// can act on it reads it.
|
|
759
|
+
// **A public snapshot's filename may not be printed either**, for the same
|
|
760
|
+
// reason as a fragment's: it is `site.<sha256>.sqlite`, and that digest is a
|
|
761
|
+
// function of the corpus. The rename differential caught this too — a note
|
|
762
|
+
// body carrying a planted home path lands in the snapshot's `excerpt`, and
|
|
763
|
+
// the finding named the content-addressed file. The public half names the
|
|
764
|
+
// surface instead; the digest is in the report's `detailed` half.
|
|
765
|
+
const isSnapshot = posixWhere.split('/').length === 2 &&
|
|
766
|
+
posixWhere.split('/')[0] === SNAPSHOT_DIRECTORY &&
|
|
767
|
+
SNAPSHOT_FILE_PATTERN.test(name);
|
|
768
|
+
const publicWhere = isFragment
|
|
769
|
+
? `the search index (${THIRD_PARTY}/)`
|
|
770
|
+
: isSnapshot
|
|
771
|
+
? `the site snapshot (${SNAPSHOT_DIRECTORY}/)`
|
|
772
|
+
: where;
|
|
773
|
+
// Read once, and classify on what the bytes are rather than on what the
|
|
774
|
+
// name claims. A fragment keeps its own branch below because its *location*
|
|
775
|
+
// is what identifies it; everything else is decided here.
|
|
776
|
+
const bytes = isFragment ? new Uint8Array() : readFileSync(path);
|
|
777
|
+
// **The inflate is no longer anchored to Pagefind.** It was, and that was
|
|
778
|
+
// safe only for as long as every gzip member in the artifact belonged to a
|
|
779
|
+
// third-party bundle. A build step compressing this site's own content —
|
|
780
|
+
// measured, a gzipped database hides `msw/`, `javascript:` and `C:/` from
|
|
781
|
+
// every raw read, all three visible again after inflating — would otherwise
|
|
782
|
+
// ship behind a layer this scan structurally could not open. Keying on the
|
|
783
|
+
// gzip magic covers the member that exists today and the one a future step
|
|
784
|
+
// adds, which is the corollary about enumerations in `docs/gate-reading.md`.
|
|
785
|
+
let inflated;
|
|
786
|
+
if (isGzip(bytes)) {
|
|
787
|
+
try {
|
|
788
|
+
inflated = gunzipSync(bytes);
|
|
789
|
+
}
|
|
790
|
+
catch {
|
|
791
|
+
report(`${publicWhere}: is gzipped and could not be inflated, so it shipped unscanned`, `${where}: is gzipped and could not be inflated, so it shipped unscanned`);
|
|
792
|
+
continue;
|
|
793
|
+
}
|
|
794
|
+
}
|
|
795
|
+
const payload = inflated ?? bytes;
|
|
796
|
+
// A database is recognised by its header, so `.sqlite3`, `.db` and a file
|
|
797
|
+
// with no extension at all are one case — and so a gzipped one is the same
|
|
798
|
+
// case, since the test runs on the inflated payload.
|
|
799
|
+
const isDatabase = !isFragment && isSqlite(payload);
|
|
800
|
+
if (!isFragment && !isDatabase) {
|
|
801
|
+
// A gzip member that is not a database is classified on the name it ships
|
|
802
|
+
// under, as before. Its inflated text is what gets scanned.
|
|
803
|
+
if (BINARY_EXTENSIONS.has(extension))
|
|
804
|
+
continue;
|
|
805
|
+
if (!TEXT_EXTENSIONS.has(extension) && !TEXT_NAMES.has(name)) {
|
|
806
|
+
report(`${where}: is neither declared text nor declared binary, so it shipped unscanned — ` +
|
|
807
|
+
`add its extension to TEXT_EXTENSIONS or BINARY_EXTENSIONS in scripts/scan-residue.ts`);
|
|
808
|
+
continue;
|
|
809
|
+
}
|
|
810
|
+
}
|
|
811
|
+
// A fragment is gzip, and an unreadable one is reported rather than skipped:
|
|
812
|
+
// "I could not inflate it" and "it was clean" must not be the same outcome
|
|
813
|
+
// on the surface this exclusion was just narrowed to cover.
|
|
814
|
+
//
|
|
815
|
+
// Every text this file carries, as separate subjects. One for an ordinary
|
|
816
|
+
// file; for a database, one per stored value plus one for its raw bytes.
|
|
817
|
+
// **Separate rather than concatenated**, so a marker cannot be manufactured
|
|
818
|
+
// by joining two values that neither of them contains — and so the peak
|
|
819
|
+
// memory is one copy of the corpus rather than two.
|
|
820
|
+
let subjects;
|
|
821
|
+
if (isFragment) {
|
|
822
|
+
try {
|
|
823
|
+
subjects = [{ text: gunzipSync(readFileSync(path)).toString('utf8'), raw: false }];
|
|
824
|
+
}
|
|
825
|
+
catch {
|
|
826
|
+
report(`${publicWhere}: holds a fragment that could not be inflated, so it shipped unscanned`, `${where}: is a search-index fragment that could not be inflated, so it shipped unscanned`);
|
|
827
|
+
continue;
|
|
828
|
+
}
|
|
829
|
+
}
|
|
830
|
+
else if (isDatabase) {
|
|
831
|
+
// **Rows and bytes both**, for the two opposite reasons {@link databaseText}
|
|
832
|
+
// states: a `SELECT` sees a marker split across an overflow-page boundary
|
|
833
|
+
// that no byte read can match, and the bytes carry a deleted row's payload
|
|
834
|
+
// that no `SELECT` can reach. Measured to cost nothing in false positives.
|
|
835
|
+
let read;
|
|
836
|
+
try {
|
|
837
|
+
read = databaseText(payload);
|
|
838
|
+
}
|
|
839
|
+
catch {
|
|
840
|
+
// A file wearing the SQLite magic that SQLite will not open is "could
|
|
841
|
+
// not look". Reported, never skipped — measured, both a corrupt file and
|
|
842
|
+
// a truncated one throw here rather than returning empty.
|
|
843
|
+
report(`${publicWhere}: is a database that could not be opened, so it shipped unscanned`, `${where}: is a database that could not be opened, so it shipped unscanned`);
|
|
844
|
+
// Not counted as a database for the row guard below: this one has
|
|
845
|
+
// already been reported by name, and adding "it holds no rows" to that
|
|
846
|
+
// says the same thing less precisely.
|
|
847
|
+
continue;
|
|
848
|
+
}
|
|
849
|
+
sawDatabase = true;
|
|
850
|
+
for (const table of read.unreadable) {
|
|
851
|
+
// The table's name is a schema literal the build chose, not content, so
|
|
852
|
+
// both halves may carry it — unlike a fragment's digest filename.
|
|
853
|
+
report(`${publicWhere}: holds table "${table}", whose text this scan can neither read as rows ` +
|
|
854
|
+
`nor find as bytes, so it shipped unscanned`, `${where}: holds table "${table}", whose text this scan can neither read as rows ` +
|
|
855
|
+
`nor find as bytes, so it shipped unscanned`);
|
|
856
|
+
}
|
|
857
|
+
rowCount += read.corpusRows;
|
|
858
|
+
// The raw bytes are one more subject, flagged so a rule that reads
|
|
859
|
+
// authored text can decline it — see {@link RAW_BYTES_EXEMPT}.
|
|
860
|
+
subjects = [
|
|
861
|
+
...read.values.map((value) => ({ text: value, raw: false })),
|
|
862
|
+
{ text: Buffer.from(payload).toString('utf8'), raw: true },
|
|
863
|
+
];
|
|
864
|
+
}
|
|
865
|
+
else {
|
|
866
|
+
subjects = [{ text: Buffer.from(payload).toString('utf8'), raw: false }];
|
|
867
|
+
}
|
|
868
|
+
scannedPaths.push(where);
|
|
869
|
+
for (const [pattern, what] of RESIDUE_RULES) {
|
|
870
|
+
// A fragment carries no markup, so the rule that reads code regions has
|
|
871
|
+
// nothing to exempt there and is dropped instead of being applied blind.
|
|
872
|
+
if (isFragment && FRAGMENT_EXEMPT.has(what))
|
|
873
|
+
continue;
|
|
874
|
+
// One finding per rule per *file*, not one per form and not one per row:
|
|
875
|
+
// the same marker seen raw and again decoded is one defect, and reporting
|
|
876
|
+
// it twice would make a clean fix look half-done.
|
|
877
|
+
let match;
|
|
878
|
+
for (const { text, raw } of subjects) {
|
|
879
|
+
if (raw && RAW_BYTES_EXEMPT.has(what))
|
|
880
|
+
continue;
|
|
881
|
+
// A database carries Markdown, whose code regions are delimited by
|
|
882
|
+
// fences and backticks rather than by elements — the reasoning at
|
|
883
|
+
// {@link CODE_EXEMPT} carried across a carrier that has no elements in
|
|
884
|
+
// it yet, per {@link withoutMarkdownCode}.
|
|
885
|
+
const codeExempted = !CODE_EXEMPT.has(what)
|
|
886
|
+
? text
|
|
887
|
+
: isDatabase
|
|
888
|
+
? withoutMarkdownCode(text)
|
|
889
|
+
: text.includes('<code') || text.includes('<pre')
|
|
890
|
+
? withoutCodeRegions(text)
|
|
891
|
+
: text;
|
|
892
|
+
// And then the math regions, for the rules real TeX spells by accident.
|
|
893
|
+
// Applied over the code-exempted text rather than instead of it, because
|
|
894
|
+
// the two answer different questions and a marker can be in both kinds
|
|
895
|
+
// of region — see {@link MATH_EXEMPT}. A database carries Markdown with
|
|
896
|
+
// no elements in it yet, so this pass has nothing to find there and is
|
|
897
|
+
// skipped rather than run over source it cannot key on.
|
|
898
|
+
const subject = !isDatabase && MATH_EXEMPT.has(what) && codeExempted.includes('<code')
|
|
899
|
+
? withoutMathRegions(codeExempted)
|
|
900
|
+
: codeExempted;
|
|
901
|
+
// A database stores Markdown source, and the reader receives what a
|
|
902
|
+
// renderer makes of it — so the forms a *reader* can resolve a marker
|
|
903
|
+
// out of include the ones inline markup joins. That is the surface
|
|
904
|
+
// TK-29 closed for the search index; it applies to this carrier for the
|
|
905
|
+
// same reason and is measured to add no false positive. See
|
|
906
|
+
// {@link joinedForms}.
|
|
907
|
+
//
|
|
908
|
+
// **Generated rather than collected**, so at most one derived string is
|
|
909
|
+
// live at a time. Materialising all nine of a database's forms at once
|
|
910
|
+
// held 1.16 GB for a 129 MB file and exhausted a 4 GB heap at 304 MB.
|
|
911
|
+
//
|
|
912
|
+
// **And the raw-byte subject takes none of them**, which is what keeps
|
|
913
|
+
// the peak bounded rather than merely lower: every form is a full copy
|
|
914
|
+
// of its subject — measured, 638 MB for a 142 MB file and 923 MB after
|
|
915
|
+
// one decode pass — and for the whole file that is the largest subject
|
|
916
|
+
// there is. It also buys nothing there. The byte pass exists for one
|
|
917
|
+
// thing, a deleted row's payload sitting in a free page, and those bytes
|
|
918
|
+
// are whatever the row held; an entity-encoded or emphasis-split marker
|
|
919
|
+
// is a property of *text a reader resolves*, which is the row pass's
|
|
920
|
+
// job and which the row pass does on that same text before it is
|
|
921
|
+
// deleted. Derived forms of a B-tree's raw image are forms of something
|
|
922
|
+
// no reader ever parses.
|
|
923
|
+
const forms = raw
|
|
924
|
+
? [text]
|
|
925
|
+
: isDatabase
|
|
926
|
+
? databaseForms(subject)
|
|
927
|
+
: normalizedForms(subject);
|
|
928
|
+
for (const form of forms) {
|
|
929
|
+
match = pattern.exec(form) ?? undefined;
|
|
930
|
+
if (match)
|
|
931
|
+
break;
|
|
932
|
+
}
|
|
933
|
+
if (match)
|
|
934
|
+
break;
|
|
935
|
+
}
|
|
936
|
+
if (match) {
|
|
937
|
+
// `publicWhere` is a route or a literal and `what` is a literal, but
|
|
938
|
+
// `match[0]` is whatever was found — measured, `/home/<user>/` for the
|
|
939
|
+
// home-directory rule, a real disclosure on a surface a workflow log
|
|
940
|
+
// inherits. One rule leaking is enough to withhold the echo for all nine
|
|
941
|
+
// rather than maintain a per-rule table.
|
|
942
|
+
report(`${publicWhere}: contains ${what}`, `${where}: contains ${what} (${JSON.stringify(match[0])})`);
|
|
943
|
+
}
|
|
944
|
+
}
|
|
945
|
+
}
|
|
946
|
+
// Three non-vacuity guards, because "zero findings" and "nothing was examined"
|
|
947
|
+
// are the same output. A file count alone is the weakest: a `dist/` holding
|
|
948
|
+
// one stray file satisfies it. Requiring the site's entry point ties the clean
|
|
949
|
+
// result to a build that actually produced a site.
|
|
950
|
+
//
|
|
951
|
+
// **The third exists because a file count stops measuring coverage the moment
|
|
952
|
+
// the corpus stops being files.** One database in place of a thousand pages
|
|
953
|
+
// leaves `scannedCount` at a number that no longer varies with what is
|
|
954
|
+
// published, and an instrument whose reading does not move with its subject
|
|
955
|
+
// has stopped looking — the unifying claim of `docs/gate-reading.md`. So a
|
|
956
|
+
// database that yields no row is a finding: the artifact holding the corpus
|
|
957
|
+
// was opened and gave nothing up, which is "could not look" wearing the shape
|
|
958
|
+
// of a clean result. `rowCount` is returned for the same reason `scannedCount`
|
|
959
|
+
// is — so a caller can refuse a zero — and it is only meaningful where a
|
|
960
|
+
// database was read, which is exactly when this guard applies.
|
|
961
|
+
if (scannedPaths.length === 0) {
|
|
962
|
+
report('no scannable file was found in the built site, so this scan proved nothing', `${root}: no scannable file was found, so this scan proved nothing`);
|
|
963
|
+
}
|
|
964
|
+
else if (!scannedPaths.includes('index.html')) {
|
|
965
|
+
report('the built site has no index.html at its root, so this is not a complete built site', `${root}: has no index.html at its root, so this is not a complete built site`);
|
|
966
|
+
}
|
|
967
|
+
if (sawDatabase && rowCount === 0) {
|
|
968
|
+
report('the built site ships a database holding no rows, so scanning it proved nothing', `${root}: ships a database holding no rows, so scanning it proved nothing`);
|
|
969
|
+
}
|
|
970
|
+
return { findings, detailed, scannedCount: scannedPaths.length, rowCount };
|
|
971
|
+
}
|
|
972
|
+
/**
|
|
973
|
+
* Scan a built site and fail loudly if it carries residue.
|
|
974
|
+
*
|
|
975
|
+
* The reporting half of {@link scanResidue}, exported so the packaged CLI does
|
|
976
|
+
* not restate it. Two callers formatting the same findings is two places the
|
|
977
|
+
* wording, the plural, and — the one that matters — the decision that a finding
|
|
978
|
+
* is fatal can drift apart.
|
|
979
|
+
*
|
|
980
|
+
* The message carries no `root`: measured, the packaged build's `root` is a
|
|
981
|
+
* `mkdtemp` workspace whose random suffix differs between two runs of the same
|
|
982
|
+
* corpus, so it is a host path the user never typed. The full detail, matched
|
|
983
|
+
* bytes and scanned root included, is on the thrown error for the report.
|
|
984
|
+
*
|
|
985
|
+
* @throws {BuildFailure} listing every finding, when the scan is not clean.
|
|
986
|
+
*/
|
|
987
|
+
export function assertNoResidue(root = DIST, options = { requireVendorProvenance: DIAGRAM_MODE === 'client' }) {
|
|
988
|
+
const { findings, detailed, scannedCount } = scanResidue(root, options);
|
|
989
|
+
if (findings.length > 0) {
|
|
990
|
+
const count = `${findings.length} finding${findings.length === 1 ? '' : 's'}`;
|
|
991
|
+
throw new BuildFailure('residue-scan-failed', `residue scan: ${count}\n${findings.map((finding) => ` - ${finding}`).join('\n')}`, `residue scan: ${count} in ${root}\n${detailed.map((finding) => ` - ${finding}`).join('\n')}`);
|
|
992
|
+
}
|
|
993
|
+
return scannedCount;
|
|
994
|
+
}
|
|
995
|
+
/**
|
|
996
|
+
* This repository scanning its own `dist/`, from `pnpm run build`.
|
|
997
|
+
*
|
|
998
|
+
* The detail is printed here and only here. This path never runs in a user's
|
|
999
|
+
* repository — the packaged CLI imports {@link assertNoResidue} directly — and
|
|
1000
|
+
* its `root` is this repository's own `dist/`, on a host whose operator is the
|
|
1001
|
+
* person reading the output. A maintainer fixing residue needs the bytes that
|
|
1002
|
+
* matched; a stranger's workflow log must not have them, and that caller takes
|
|
1003
|
+
* the public half off the same error.
|
|
1004
|
+
*/
|
|
1005
|
+
function main() {
|
|
1006
|
+
try {
|
|
1007
|
+
console.log(`residue scan ok: ${assertNoResidue()} files, 0 findings`);
|
|
1008
|
+
return 0;
|
|
1009
|
+
}
|
|
1010
|
+
catch (error) {
|
|
1011
|
+
console.error(error instanceof BuildFailure ? error.detail : error instanceof Error ? error.message : String(error));
|
|
1012
|
+
return 1;
|
|
1013
|
+
}
|
|
1014
|
+
}
|
|
1015
|
+
if (process.argv[1] === fileURLToPath(import.meta.url))
|
|
1016
|
+
process.exit(main());
|