@wxxb789/anc 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +295 -0
- package/astro.config.mjs +255 -0
- package/bin/anc.mjs +798 -0
- package/package.json +93 -0
- package/public/_headers +45 -0
- package/public/favicon.ico +0 -0
- package/public/favicon.svg +9 -0
- package/public/fonts/Temml.woff2 +0 -0
- package/scripts/build-snapshot.js +69 -0
- package/scripts/build-wasm.js +67 -0
- package/scripts/build-workspace.js +50 -0
- package/scripts/copy-snapshot.js +48 -0
- package/scripts/copy-wasm.js +57 -0
- package/scripts/emit-redirects.js +64 -0
- package/scripts/generate-corpus.js +421 -0
- package/scripts/init-repository.js +249 -0
- package/scripts/load-config.js +975 -0
- package/scripts/markdown-to-artifact.js +1286 -0
- package/scripts/preview-site.js +364 -0
- package/scripts/publish-set-review.js +157 -0
- package/scripts/resolve-links.js +640 -0
- package/scripts/run-pagefind.js +125 -0
- package/scripts/scan-residue.js +1016 -0
- package/scripts/scan-secrets.js +268 -0
- package/scripts/seed-gitignore.js +410 -0
- package/scripts/snapshot-rows.js +205 -0
- package/scripts/validate-content.js +95 -0
- package/scripts/vendor-provenance.js +307 -0
- package/scripts/verify-output-inventory.js +307 -0
- package/scripts/write-report.js +351 -0
- package/scripts/write-snapshot.js +136 -0
- package/src/components/CollectionExplorer.astro +149 -0
- package/src/components/FacetIndex.astro +45 -0
- package/src/components/LinkedNotes.astro +94 -0
- package/src/components/NoteGraph.astro +434 -0
- package/src/components/NoteList.astro +54 -0
- package/src/components/SiteMetadata.astro +143 -0
- package/src/components/TableOfContents.astro +69 -0
- package/src/components/TagBrowser.astro +70 -0
- package/src/layouts/Layout.astro +305 -0
- package/src/lib/artifact-source.js +72 -0
- package/src/lib/build-bindings.js +11 -0
- package/src/lib/collection-navigation.js +145 -0
- package/src/lib/content.js +92 -0
- package/src/lib/diagram-mode.js +18 -0
- package/src/lib/graph-layout.js +219 -0
- package/src/lib/graph-selection.js +84 -0
- package/src/lib/graph.js +42 -0
- package/src/lib/link-resolution.js +389 -0
- package/src/lib/markdown.js +1469 -0
- package/src/lib/math-mode.js +31 -0
- package/src/lib/math-output.js +131 -0
- package/src/lib/math.js +165 -0
- package/src/lib/mermaid-environment.js +475 -0
- package/src/lib/mermaid-render.js +769 -0
- package/src/lib/preview-model.js +96 -0
- package/src/lib/relations.js +242 -0
- package/src/lib/rendered-marker.js +55 -0
- package/src/lib/route-path.js +192 -0
- package/src/lib/routes.js +342 -0
- package/src/lib/schema.js +578 -0
- package/src/lib/site.js +578 -0
- package/src/lib/snapshot-contract.js +216 -0
- package/src/lib/snapshot-operations.js +147 -0
- package/src/lib/snapshot-queries.js +216 -0
- package/src/lib/snapshot-reader.js +210 -0
- package/src/lib/snapshot.js +233 -0
- package/src/lib/sqlite-warning.js +22 -0
- package/src/lib/sqlite.js +14 -0
- package/src/lib/tag-browser-model.js +18 -0
- package/src/lib/translations.js +533 -0
- package/src/lib/wasm-asset.js +47 -0
- package/src/lib/worker-protocol.js +93 -0
- package/src/pages/404.astro +35 -0
- package/src/pages/about.astro +100 -0
- package/src/pages/collections/[slug].astro +36 -0
- package/src/pages/collections/index.astro +33 -0
- package/src/pages/graph.astro +64 -0
- package/src/pages/index.astro +33 -0
- package/src/pages/notes/[slug].astro +509 -0
- package/src/pages/privacy.astro +121 -0
- package/src/pages/private.astro +59 -0
- package/src/pages/recent.astro +33 -0
- package/src/pages/robots.txt.js +15 -0
- package/src/pages/rss.xml.js +21 -0
- package/src/pages/sitemap.xml.js +20 -0
- package/src/pages/tags/[tag].astro +56 -0
- package/src/pages/tags/index.astro +37 -0
- package/src/scripts/code-copy.js +45 -0
- package/src/scripts/diagram-disabled.js +16 -0
- package/src/scripts/diagram.js +322 -0
- package/src/scripts/graph-client.js +222 -0
- package/src/scripts/graph-draw.js +166 -0
- package/src/scripts/link-preview.js +334 -0
- package/src/scripts/math-disabled.js +18 -0
- package/src/scripts/math.js +133 -0
- package/src/scripts/preferences.js +123 -0
- package/src/scripts/search-dialog.js +651 -0
- package/src/scripts/snapshot-client.js +157 -0
- package/src/scripts/snapshot-worker.js +341 -0
- package/src/scripts/tag-browser.js +153 -0
- package/src/scripts/theme-init.js +35 -0
- package/src/scripts/toc-active.js +38 -0
- package/src/styles/code.css +152 -0
- package/src/styles/diagram.css +149 -0
- package/src/styles/global.css +1739 -0
- package/src/styles/graph.css +317 -0
- package/src/styles/math.css +376 -0
- package/src/styles/tokens.css +171 -0
- package/tsconfig.json +25 -0
|
@@ -0,0 +1,1286 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Discovery and exclusion: a git repository of Markdown in, an artifact out.
|
|
3
|
+
*
|
|
4
|
+
* Replaces TK-24's non-recursive bridge. The walk is recursive, the exclusion
|
|
5
|
+
* rules are two mechanisms with a stated precedence, and every file the walk
|
|
6
|
+
* enumerated either becomes an entry or gets a row in the report — so
|
|
7
|
+
* `discovered === published + dropped` holds by construction rather than by
|
|
8
|
+
* three counters agreeing.
|
|
9
|
+
*
|
|
10
|
+
* ## Structural ignores are not patterns, and that is the whole of C3
|
|
11
|
+
*
|
|
12
|
+
* The plan says the shipped defaults `.git/**`, `.obsidian/**` and `**\/.*`
|
|
13
|
+
* "structurally match zero files" because `fs.globSync` cannot match a dotfile,
|
|
14
|
+
* and so fail every user's first push under the zero-match rule. **Measured on
|
|
15
|
+
* Node 24.18.1, that premise is false on both platforms.** `globSync('.obsidian/**')`
|
|
16
|
+
* returns `['.obsidian', '.obsidian/app.json']` on win32 and the same on Linux;
|
|
17
|
+
* `globSync('**\/.*')` returns `['.hidden', '.obsidian']`. There is no dotfile
|
|
18
|
+
* bug to work around and none is worked around here.
|
|
19
|
+
*
|
|
20
|
+
* The split the ticket asks for is still right, for a different reason. A
|
|
21
|
+
* shipped default matching nothing is *normal* — not every repository has an
|
|
22
|
+
* `.obsidian/` — while a user's own pattern matching nothing is a probable typo,
|
|
23
|
+
* and `drafts/**` mistyped as `draft/**` is precisely the mistype that publishes
|
|
24
|
+
* a draft silently. So the two are not two lists of globs with one rule between
|
|
25
|
+
* them: {@link structurallyIgnored} is a *predicate on a name*, evaluated during
|
|
26
|
+
* the walk and never expressed as a pattern at all, and the zero-match rule
|
|
27
|
+
* applies to the user's patterns because they are the only patterns there are.
|
|
28
|
+
* A rule that cannot be written down cannot be mistyped.
|
|
29
|
+
*
|
|
30
|
+
* Pruning during the walk rather than filtering after it is also the only
|
|
31
|
+
* affordable shape: `.git/` and `node_modules/` in a real repository are tens of
|
|
32
|
+
* thousands of files, and a walk that enumerates them to discard them pays for
|
|
33
|
+
* every one.
|
|
34
|
+
*
|
|
35
|
+
* ## Case sensitivity: exact on every platform, and `globSync` is why
|
|
36
|
+
*
|
|
37
|
+
* Every name comparison here — structural ignore, extension, glob — is
|
|
38
|
+
* byte-exact and case-sensitive on win32 and on Linux alike.
|
|
39
|
+
*
|
|
40
|
+
* The measurement that decides it is not about which behaviour is nicer. On
|
|
41
|
+
* win32, `globSync('readme.md')` returns `['readme.md']` in a directory whose
|
|
42
|
+
* only such file is `README.md` — it echoes the *pattern* back as if it were a
|
|
43
|
+
* path, and `readdirSync` does not contain that name. On Linux the same call
|
|
44
|
+
* returns `[]`. So a mistyped-case pattern is green here and red in CI, and any
|
|
45
|
+
* code treating the result as a real path acts on a filename that does not
|
|
46
|
+
* exist. Case-insensitive matching would inherit exactly that divergence.
|
|
47
|
+
*
|
|
48
|
+
* Two consequences, taken deliberately:
|
|
49
|
+
*
|
|
50
|
+
* - **Discovery never calls `globSync`.** The walk is `readdir`, which returns
|
|
51
|
+
* the names that are actually on disk. Patterns are matched with
|
|
52
|
+
* `path.matchesGlob`, measured identical on both platforms and case-sensitive
|
|
53
|
+
* on both (`matchesGlob('README.md', 'readme.md')` is `false` on win32 and on
|
|
54
|
+
* Linux).
|
|
55
|
+
* - **A differently-cased name is a different name.** `readme.md` at the root is
|
|
56
|
+
* an ordinary note; only `README.md` is structurally ignored. That is
|
|
57
|
+
* surprising exactly once, and it is the only rule that gives one answer on
|
|
58
|
+
* both platforms.
|
|
59
|
+
*
|
|
60
|
+
* ## What this file still does not do, and who owns each
|
|
61
|
+
*
|
|
62
|
+
* The link pass below derives backlinks. Frontmatter aliases remain display and
|
|
63
|
+
* search metadata rather than link targets. Tracked notes take `created` and
|
|
64
|
+
* `updated` from the first and last visible commits. Frontmatter can supply
|
|
65
|
+
* `slug`, `language`/`lang`, `description`, `tags`, and `aliases`; the first
|
|
66
|
+
* folder becomes the flat `collection`. All reuse fields
|
|
67
|
+
* and routes the site already owns. `title:` is read because the frontmatter
|
|
68
|
+
* had to be parsed and stripped regardless.
|
|
69
|
+
*
|
|
70
|
+
* ## Link resolution runs after the whole walk, and that ordering is forced
|
|
71
|
+
*
|
|
72
|
+
* TK-27 resolves links against the **full** file set — every discovered file,
|
|
73
|
+
* published or not — because a link to a note the user excluded and a link to
|
|
74
|
+
* nothing at all are different events with different fixes, and resolving
|
|
75
|
+
* against the published set only would merge them. The full set does not exist
|
|
76
|
+
* until the walk finishes, so the traversal cannot run inside the loop that
|
|
77
|
+
* builds it. See {@link resolveCorpusLinks}.
|
|
78
|
+
*/
|
|
79
|
+
import { spawnSync } from 'node:child_process';
|
|
80
|
+
import { createHash } from 'node:crypto';
|
|
81
|
+
import { readFile, readdir, mkdir, realpath, stat, writeFile } from 'node:fs/promises';
|
|
82
|
+
import { dirname, isAbsolute, matchesGlob, relative } from 'node:path';
|
|
83
|
+
import { parse as parseYaml } from 'yaml';
|
|
84
|
+
import { FIELD_LIMITS, RESERVED_SLUGS, contentLimitIssues, contentPrivacyIssues, isLanguageTag, isSafeTagOrAlias, isSlug, validateArtifact, } from "../src/lib/schema.js";
|
|
85
|
+
import { indexCorpus } from "../src/lib/link-resolution.js";
|
|
86
|
+
import { SLUG_MAX_BYTES, compareSlugs, slugSegment, utf8Length } from "../src/lib/route-path.js";
|
|
87
|
+
import { BuildFailure, bySourceThenLine } from "./write-report.js";
|
|
88
|
+
/**
|
|
89
|
+
* Names the walk never descends into or enumerates, tested against one path
|
|
90
|
+
* segment.
|
|
91
|
+
*
|
|
92
|
+
* Not a glob, and not configurable — see this file's header. Each is machine
|
|
93
|
+
* state or tool configuration rather than a note a user wrote:
|
|
94
|
+
*
|
|
95
|
+
* - **any dot-prefixed name**, which is configuration by convention, and which
|
|
96
|
+
* covers `.git/`, `.obsidian/` (workspace state, plugin data, `graph.json`),
|
|
97
|
+
* `.github/`, and `.gitignore` in one rule rather than four;
|
|
98
|
+
* - **`node_modules`**, which a notes repository acquires from a local preview
|
|
99
|
+
* and never authors.
|
|
100
|
+
*
|
|
101
|
+
* A structural ignore matching nothing is silent, because most repositories have
|
|
102
|
+
* no `.obsidian/` and a build that failed for that reason would fail on first
|
|
103
|
+
* use for everyone.
|
|
104
|
+
*/
|
|
105
|
+
function structurallyIgnored(name) {
|
|
106
|
+
return name.startsWith('.') || name === 'node_modules';
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* The one file excluded by name rather than by kind: the repository's own
|
|
110
|
+
* README.
|
|
111
|
+
*
|
|
112
|
+
* Root-level only. In a repository adopted by adding a workflow file, the root
|
|
113
|
+
* README carries the badge, the action snippet, and the install steps — it
|
|
114
|
+
* addresses the repository, not the reader. A README *inside* a folder is that
|
|
115
|
+
* folder's index, which is prose, so it is an ordinary note (and reachable here
|
|
116
|
+
* only through the nested branch, which never consults this).
|
|
117
|
+
*
|
|
118
|
+
* Byte-exact, per the header's case rule: `readme.md` is published. Stated in
|
|
119
|
+
* the one place a reader looks for it rather than left to be discovered.
|
|
120
|
+
*/
|
|
121
|
+
const REPOSITORY_README = 'README.md';
|
|
122
|
+
/**
|
|
123
|
+
* The repo-relative path, slugified per segment and joined.
|
|
124
|
+
*
|
|
125
|
+
* Each segment goes through `slugSegment` from `src/lib/route-path.ts` — NFC,
|
|
126
|
+
* Unicode lowercase, invisible code points removed, every run of anything that
|
|
127
|
+
* is not a letter, number, or mark collapsed to one hyphen — so `日记/今天.md`
|
|
128
|
+
* is `日记-今天` and `Projects/观点.md` is `projects-观点`. There is no
|
|
129
|
+
* transliteration: every transliteration is locale-dependent, and a
|
|
130
|
+
* locale-dependent URL is not deterministic output.
|
|
131
|
+
*
|
|
132
|
+
* Per segment and not over the whole string, so a directory boundary survives as
|
|
133
|
+
* a hyphen. A *folder* segment that slugifies to nothing is dropped rather than
|
|
134
|
+
* producing the doubled hyphen `src/lib/schema.ts` rejects.
|
|
135
|
+
*
|
|
136
|
+
* **When the result is not an addressable slug, the answer is a hash, not a
|
|
137
|
+
* drop.** A stem of only punctuation or emoji (`___.md`, `🌱.md`, `---.md`), a
|
|
138
|
+
* stem of only combining marks, or a path whose slug exceeds
|
|
139
|
+
* `SLUG_MAX_BYTES` gets {@link hashSlug} of its own path. Dropping it — the
|
|
140
|
+
* earlier behaviour — silently lost a note the author never excluded, under a
|
|
141
|
+
* default-publish contract; failing would refuse a repository for a filename.
|
|
142
|
+
* Only the stem is tested for emptiness, because a stem that vanished while its
|
|
143
|
+
* folder did not would otherwise take the folder's slug and collide with the
|
|
144
|
+
* folder's own index note for a reason invisible in the source.
|
|
145
|
+
*/
|
|
146
|
+
function slugFor(relativePath) {
|
|
147
|
+
// Case-sensitive, matching the extension test that admitted this path: only
|
|
148
|
+
// `.md` reaches here, so an `/i` flag would describe a case that cannot
|
|
149
|
+
// occur while contradicting this file's byte-exact rule.
|
|
150
|
+
const segments = relativePath.replace(/\.md$/, '').split('/').map(slugSegment);
|
|
151
|
+
if (segments.at(-1) === '')
|
|
152
|
+
return hashSlug(relativePath);
|
|
153
|
+
const slug = segments.filter((segment) => segment !== '').join('-');
|
|
154
|
+
return isSlug(slug) ? slug : hashSlug(relativePath);
|
|
155
|
+
}
|
|
156
|
+
/**
|
|
157
|
+
* A deterministic slug for a path that yields no addressable one.
|
|
158
|
+
*
|
|
159
|
+
* `note-` and the first ten hex digits of the SHA-256 of the NFC repo-relative
|
|
160
|
+
* path. A pure function of that one path: it does not read a sibling, a
|
|
161
|
+
* position, or a corpus size, so adding or removing another file can never move
|
|
162
|
+
* this note's URL — the same stability property tag route keys keep, and the
|
|
163
|
+
* reason a numeric disambiguating suffix is not used. NFC, so a macOS-decomposed
|
|
164
|
+
* filename and its composed spelling get one URL. Forty bits keeps an accidental
|
|
165
|
+
* collision negligible for any real repository, and a collision would still be
|
|
166
|
+
* reported as `slug-collision` rather than merge two notes.
|
|
167
|
+
*
|
|
168
|
+
* The digest is of a path this build publishes, so it names nothing withheld.
|
|
169
|
+
*/
|
|
170
|
+
function hashSlug(relativePath) {
|
|
171
|
+
return `note-${createHash('sha256').update(relativePath.normalize('NFC'), 'utf8').digest('hex').slice(0, 10)}`;
|
|
172
|
+
}
|
|
173
|
+
/**
|
|
174
|
+
* First folder as the flat collection key.
|
|
175
|
+
*
|
|
176
|
+
* The same segment derivation as a slug, so `研究/note.md` is in the `研究`
|
|
177
|
+
* collection. A folder that yields no addressable key — only punctuation or
|
|
178
|
+
* emoji, say — leaves the note uncollected rather than rejecting a repository
|
|
179
|
+
* that already publishes; a folder whose key exceeds the slug byte limit fails,
|
|
180
|
+
* because silently uncollecting it would change the site's navigation for a
|
|
181
|
+
* reason the author cannot see.
|
|
182
|
+
*/
|
|
183
|
+
function collectionFor(relativePath) {
|
|
184
|
+
const separator = relativePath.indexOf('/');
|
|
185
|
+
if (separator === -1)
|
|
186
|
+
return undefined;
|
|
187
|
+
const collection = slugSegment(relativePath.slice(0, separator));
|
|
188
|
+
if (collection === '')
|
|
189
|
+
return undefined;
|
|
190
|
+
const limits = contentLimitIssues({ collection }, 'frontmatter');
|
|
191
|
+
if (utf8Length(collection) > SLUG_MAX_BYTES) {
|
|
192
|
+
limits.push(`frontmatter.collection: is ${utf8Length(collection)} UTF-8 bytes, over the ${SLUG_MAX_BYTES} byte limit`);
|
|
193
|
+
}
|
|
194
|
+
if (limits.length > 0) {
|
|
195
|
+
throw new BuildFailure('collection-folder-too-long', 'one published note exceeds the collection field limit', `${relativePath}: ${limits.join('; ')}`);
|
|
196
|
+
}
|
|
197
|
+
return isSlug(collection) ? collection : undefined;
|
|
198
|
+
}
|
|
199
|
+
/** Read a bounded public-label list without silently coercing YAML values. */
|
|
200
|
+
function publicLabelsFor(data, field, path) {
|
|
201
|
+
const value = data?.[field];
|
|
202
|
+
if (value === undefined)
|
|
203
|
+
return undefined;
|
|
204
|
+
let reason;
|
|
205
|
+
if (!Array.isArray(value))
|
|
206
|
+
reason = `${field} must be a YAML list`;
|
|
207
|
+
else {
|
|
208
|
+
const limits = contentLimitIssues({ [field]: value }, 'frontmatter');
|
|
209
|
+
const privacy = contentPrivacyIssues({ [field]: value }, 'frontmatter');
|
|
210
|
+
if (limits.length > 0)
|
|
211
|
+
reason = limits.join('; ');
|
|
212
|
+
else if (privacy.length > 0)
|
|
213
|
+
reason = privacy.join('; ');
|
|
214
|
+
else {
|
|
215
|
+
const invalid = value.findIndex((item) => typeof item !== 'string' ||
|
|
216
|
+
item.trim() === '' ||
|
|
217
|
+
item !== item.trim() ||
|
|
218
|
+
!isSafeTagOrAlias(item));
|
|
219
|
+
if (invalid >= 0)
|
|
220
|
+
reason = `${field}[${invalid}] must be non-empty public text`;
|
|
221
|
+
else if (new Set(value).size !== value.length)
|
|
222
|
+
reason = `${field} must not contain duplicates`;
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
if (reason !== undefined) {
|
|
226
|
+
throw new BuildFailure(`invalid-${field}-frontmatter`, `frontmatter ${field} must be a YAML list of non-empty text`, `${path}: ${reason}; ${field} must not contain "/" or control characters`);
|
|
227
|
+
}
|
|
228
|
+
const labels = value;
|
|
229
|
+
return labels.length === 0 ? undefined : [...labels];
|
|
230
|
+
}
|
|
231
|
+
/**
|
|
232
|
+
* A frontmatter `slug:` override, in the same grammar a derived slug has.
|
|
233
|
+
*
|
|
234
|
+
* NFC-normalised before it is judged, so an override typed on a platform that
|
|
235
|
+
* writes decomposed text names the same URL as the composed spelling. Nothing
|
|
236
|
+
* else is rewritten: an override is the author saying exactly which URL they
|
|
237
|
+
* want, so an uppercase or punctuated one is refused rather than slugified into
|
|
238
|
+
* a URL they did not write. Unlike a derived slug, an over-long override fails
|
|
239
|
+
* rather than falling back to a hash — the author asked for that URL.
|
|
240
|
+
*/
|
|
241
|
+
function slugOverrideFor(data, path) {
|
|
242
|
+
const value = data?.['slug'];
|
|
243
|
+
if (value === undefined)
|
|
244
|
+
return undefined;
|
|
245
|
+
const slug = typeof value === 'string' ? value.normalize('NFC') : undefined;
|
|
246
|
+
if (slug === undefined || !isSlug(slug)) {
|
|
247
|
+
throw new BuildFailure('invalid-slug-frontmatter', 'frontmatter slug must be a lowercase public route key', `${path}: slug must use lowercase letters, digits, or marks in single-hyphen runs, ` +
|
|
248
|
+
`at most ${SLUG_MAX_BYTES} UTF-8 bytes`);
|
|
249
|
+
}
|
|
250
|
+
return slug;
|
|
251
|
+
}
|
|
252
|
+
function languageFor(data, path) {
|
|
253
|
+
const language = data?.['language'];
|
|
254
|
+
const lang = data?.['lang'];
|
|
255
|
+
for (const [field, value] of [
|
|
256
|
+
['language', language],
|
|
257
|
+
['lang', lang],
|
|
258
|
+
]) {
|
|
259
|
+
if (value === undefined)
|
|
260
|
+
continue;
|
|
261
|
+
if (typeof value !== 'string' ||
|
|
262
|
+
value !== value.trim() ||
|
|
263
|
+
!isLanguageTag(value) ||
|
|
264
|
+
value.length > FIELD_LIMITS.strings.language) {
|
|
265
|
+
throw new BuildFailure('invalid-language-frontmatter', 'frontmatter language must be a BCP 47 tag', `${path}: ${field} must be a BCP 47 tag such as en, zh-CN, or zh-Hans-CN`);
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
const preferred = language;
|
|
269
|
+
const short = lang;
|
|
270
|
+
if (preferred !== undefined && short !== undefined && preferred.toLowerCase() !== short.toLowerCase()) {
|
|
271
|
+
throw new BuildFailure('conflicting-language-frontmatter', 'frontmatter lang and language must name the same locale', `${path}: lang and language disagree; keep one key or make their BCP 47 tags equal`);
|
|
272
|
+
}
|
|
273
|
+
// `language` is the artifact field and therefore owns the spelling when both
|
|
274
|
+
// equivalent keys exist. BCP 47 comparison itself is case-insensitive.
|
|
275
|
+
return preferred ?? short;
|
|
276
|
+
}
|
|
277
|
+
function descriptionFor(data, path) {
|
|
278
|
+
const value = data?.['description'];
|
|
279
|
+
if (value === undefined)
|
|
280
|
+
return undefined;
|
|
281
|
+
if (typeof value !== 'string' || value.trim() === '') {
|
|
282
|
+
throw new BuildFailure('invalid-description-frontmatter', 'frontmatter description must be valid public text', `${path}: description must be a non-empty string`);
|
|
283
|
+
}
|
|
284
|
+
const limits = contentLimitIssues({ description: value }, 'frontmatter');
|
|
285
|
+
if (limits.length > 0) {
|
|
286
|
+
throw new BuildFailure('invalid-description-frontmatter', 'frontmatter description must be valid public text', `${path}: ${limits.join('; ')}`);
|
|
287
|
+
}
|
|
288
|
+
const privacy = contentPrivacyIssues({ description: value }, 'frontmatter');
|
|
289
|
+
if (privacy.length > 0) {
|
|
290
|
+
throw new BuildFailure('invalid-description-frontmatter', 'frontmatter description must be valid public text', `${path}: ${privacy.join('; ')}`);
|
|
291
|
+
}
|
|
292
|
+
return value;
|
|
293
|
+
}
|
|
294
|
+
function titleFrom(data, body, slug, path) {
|
|
295
|
+
const authored = data?.['title'];
|
|
296
|
+
if (authored !== undefined && (typeof authored !== 'string' || authored.trim() === '')) {
|
|
297
|
+
throw new BuildFailure('invalid-title-frontmatter', 'frontmatter title must be non-empty text', `${path}: title must be a non-empty string`);
|
|
298
|
+
}
|
|
299
|
+
const declared = typeof authored === 'string';
|
|
300
|
+
const title = declared ? authored.trim() : titleFor(body, slug);
|
|
301
|
+
const limits = contentLimitIssues({ title }, 'frontmatter');
|
|
302
|
+
if (limits.length > 0) {
|
|
303
|
+
throw new BuildFailure('title-too-long', 'one published note has a title over the content limit', `${path}: ${limits.join('; ')}`);
|
|
304
|
+
}
|
|
305
|
+
return { title, declared };
|
|
306
|
+
}
|
|
307
|
+
/**
|
|
308
|
+
* First and last commit dates for current paths, from one history scan.
|
|
309
|
+
*
|
|
310
|
+
* Git emits newest commits first. The first date seen for a path is `updated`;
|
|
311
|
+
* each older one replaces `created`. A shallow clone cannot establish creation,
|
|
312
|
+
* so it emits only the latest visible update rather than asserting a false first
|
|
313
|
+
* date. Dates use the committer timestamp, so a rebase may change them, and the
|
|
314
|
+
* scan follows current path names rather than identities across renames. Merge
|
|
315
|
+
* commits use Git's default no-diff view; their ordinary side commits still
|
|
316
|
+
* count, while a resolution-only edit does not move `updated`. `-z` both keeps
|
|
317
|
+
* non-ASCII names unquoted and uses NUL boundaries; names remain `/`-separated
|
|
318
|
+
* on every platform, matching the walk. Outside git, for untracked files, or
|
|
319
|
+
* after the 30-second/64 MiB bounds, the map is empty and the existing undated
|
|
320
|
+
* behavior remains.
|
|
321
|
+
*/
|
|
322
|
+
function gitDatesFor(contentDirectory, paths) {
|
|
323
|
+
const options = {
|
|
324
|
+
cwd: contentDirectory,
|
|
325
|
+
encoding: 'utf8',
|
|
326
|
+
env: { ...process.env, GIT_OPTIONAL_LOCKS: '0' },
|
|
327
|
+
maxBuffer: 64 * 1024 * 1024,
|
|
328
|
+
timeout: 30_000,
|
|
329
|
+
windowsHide: true,
|
|
330
|
+
};
|
|
331
|
+
const shallowProbe = spawnSync('git', ['rev-parse', '--is-shallow-repository'], options);
|
|
332
|
+
if (shallowProbe.status !== 0)
|
|
333
|
+
return new Map();
|
|
334
|
+
const shallow = shallowProbe.stdout.trim() === 'true';
|
|
335
|
+
const history = spawnSync('git', [
|
|
336
|
+
'--no-pager',
|
|
337
|
+
'log',
|
|
338
|
+
'-z',
|
|
339
|
+
'--format=%x1e%cI',
|
|
340
|
+
'--name-only',
|
|
341
|
+
'--relative',
|
|
342
|
+
'--',
|
|
343
|
+
'.',
|
|
344
|
+
], options);
|
|
345
|
+
if (history.status !== 0)
|
|
346
|
+
return new Map();
|
|
347
|
+
const wanted = new Set(paths);
|
|
348
|
+
const dates = new Map();
|
|
349
|
+
let commitDate;
|
|
350
|
+
for (const raw of history.stdout.split('\0')) {
|
|
351
|
+
if (raw.startsWith('\x1e')) {
|
|
352
|
+
// `%cI` is strict ISO 8601 in the committer's own offset, and that offset
|
|
353
|
+
// is a fact about where the author was, published on every page, in the
|
|
354
|
+
// feed, and in the sitemap. Normalise to UTC `Z`: the same instant, no
|
|
355
|
+
// location, and byte-stable across git versions. Second precision stays,
|
|
356
|
+
// because `recentFirst` orders by it.
|
|
357
|
+
const parsed = Date.parse(raw.slice(1).trim());
|
|
358
|
+
commitDate = Number.isNaN(parsed) ? undefined : new Date(parsed).toISOString().replace(/\.000Z$/, 'Z');
|
|
359
|
+
continue;
|
|
360
|
+
}
|
|
361
|
+
if (commitDate === undefined)
|
|
362
|
+
continue;
|
|
363
|
+
const path = raw.startsWith('\r\n') ? raw.slice(2) : raw.startsWith('\n') ? raw.slice(1) : raw;
|
|
364
|
+
if (path === '' || !wanted.has(path))
|
|
365
|
+
continue;
|
|
366
|
+
const previous = dates.get(path);
|
|
367
|
+
dates.set(path, {
|
|
368
|
+
updated: previous?.updated ?? commitDate,
|
|
369
|
+
...(shallow ? {} : { created: commitDate }),
|
|
370
|
+
});
|
|
371
|
+
}
|
|
372
|
+
return dates;
|
|
373
|
+
}
|
|
374
|
+
/**
|
|
375
|
+
* Frontmatter, the body without it, and whether the note asked not to be
|
|
376
|
+
* published.
|
|
377
|
+
*
|
|
378
|
+
* Split on the source text rather than through satteri's mdast: the renderer
|
|
379
|
+
* runs with `frontmatter: false` (`src/lib/markdown.ts`), so a block left in
|
|
380
|
+
* `markdown` renders as content — measured before this ticket, a note carrying
|
|
381
|
+
* `publish: false` published the note *and* printed `publish: false` and its
|
|
382
|
+
* frontmatter title into the page. Stripping here is what makes the field a
|
|
383
|
+
* directive instead of prose.
|
|
384
|
+
*
|
|
385
|
+
* A note that *opens* with a thematic break is the case this has to get right,
|
|
386
|
+
* and it took three attempts to find a discriminator that is not itself a bug.
|
|
387
|
+
* Measured, in order:
|
|
388
|
+
*
|
|
389
|
+
* - **Anchoring the delimiter is not enough.** `---\n\nOpens with a break.\n\n---\n`
|
|
390
|
+
* matches an anchored pattern too, because a break's own two rules look
|
|
391
|
+
* exactly like a delimited block. That reduced the note to its tail — precisely
|
|
392
|
+
* `export.py`'s defect reproduced.
|
|
393
|
+
* - **"Must parse as a YAML mapping" is not enough either.** `Status: draft note`
|
|
394
|
+
* between two breaks *is* a mapping, so an ordinary note opening `Note:` or
|
|
395
|
+
* `See: https://…` still lost its head; and `**Bold** opener.` is not valid
|
|
396
|
+
* YAML at all, so it threw `malformed-frontmatter` and **failed the whole
|
|
397
|
+
* build** on a note that has no frontmatter.
|
|
398
|
+
* - **Satteri's own `yaml` mdast node does not discriminate either**, which is
|
|
399
|
+
* worth recording because plan §2.2 prescribes it as the fix. Measured: it
|
|
400
|
+
* reports a `yaml` node with value `"\n**Bold** opener.\n"` for that input.
|
|
401
|
+
*
|
|
402
|
+
* What actually separates them is the blank line. A thematic break is followed
|
|
403
|
+
* by one; a frontmatter block opens immediately with its first key, because a
|
|
404
|
+
* leading blank line is not how anyone writes one and not what any editor emits.
|
|
405
|
+
* So `(?!\s*\r?\n)` after the opening delimiter is the whole test — and it keeps
|
|
406
|
+
* a *malformed* frontmatter block loud, because such a block still opens with a
|
|
407
|
+
* key.
|
|
408
|
+
*/
|
|
409
|
+
function frontmatterOf(source) {
|
|
410
|
+
// A byte-order mark is invisible in every editor that writes one, and left in
|
|
411
|
+
// place it kept the anchored delimiter below from matching: measured, the
|
|
412
|
+
// block — and the `publish: false` inside it — became body text and the note
|
|
413
|
+
// published. Trailing blanks after the opening delimiter, and YAML's own `...`
|
|
414
|
+
// document end as the closer, did the same.
|
|
415
|
+
const markdown = source.startsWith(BYTE_ORDER_MARK) ? source.slice(BYTE_ORDER_MARK.length) : source;
|
|
416
|
+
const match = /^---[ \t]*\r?\n(?!\s*\r?\n)([\s\S]*?)\r?\n(?:---|\.\.\.)[ \t]*(?:\r?\n|$)/.exec(markdown);
|
|
417
|
+
if (match === null) {
|
|
418
|
+
// Fail closed rather than guess. A leading block that plainly carries a
|
|
419
|
+
// `publish:` key but is not one this parser recognises — an unterminated
|
|
420
|
+
// block, most often — is an author's opt-out that would publish as prose.
|
|
421
|
+
if (UNRECOGNISED_PUBLISH_BLOCK.test(markdown)) {
|
|
422
|
+
throw new FrontmatterError('unrecognised-frontmatter', 'a leading `---` block carries `publish:` but has no closing `---` line, so the ' +
|
|
423
|
+
'flag would be ignored and the note published. See the report for the file.', 'opening `---` and a `publish:` key with no closing delimiter line');
|
|
424
|
+
}
|
|
425
|
+
return { body: markdown, data: undefined };
|
|
426
|
+
}
|
|
427
|
+
const raw = match[1] ?? '';
|
|
428
|
+
let parsed;
|
|
429
|
+
try {
|
|
430
|
+
parsed = parseYaml(raw);
|
|
431
|
+
}
|
|
432
|
+
catch (error) {
|
|
433
|
+
// Loud, and this is H2's first half. A YAML error *anywhere* in the block
|
|
434
|
+
// would otherwise take `publish: false` down with it silently: the parse
|
|
435
|
+
// fails, the field is never read, and the note the user withheld is
|
|
436
|
+
// published. Failing the whole build is the only outcome that cannot
|
|
437
|
+
// publish something by accident.
|
|
438
|
+
//
|
|
439
|
+
// No filename in the public half — a withheld note's name is the disclosure
|
|
440
|
+
// (TK-25 §2.4) and this reaches a world-readable log. The name goes to the
|
|
441
|
+
// report, where the person who can act on it reads it.
|
|
442
|
+
throw new FrontmatterError('malformed-frontmatter', 'malformed YAML frontmatter — a parse error here would silently discard a ' +
|
|
443
|
+
'`publish: false` in the same block, so the build stops. See the report for the file.', error instanceof Error ? error.message : String(error));
|
|
444
|
+
}
|
|
445
|
+
// A scalar or a list opened with no blank line — `---\njust a sentence\n---` —
|
|
446
|
+
// is not a frontmatter block and cannot carry `publish:`. Leave the body
|
|
447
|
+
// intact rather than deleting content the user wrote.
|
|
448
|
+
if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {
|
|
449
|
+
return { body: markdown, data: undefined };
|
|
450
|
+
}
|
|
451
|
+
return { body: markdown.slice(match[0].length), data: parsed };
|
|
452
|
+
}
|
|
453
|
+
/**
|
|
454
|
+
* A frontmatter fault, carrying the private half the report prints.
|
|
455
|
+
*
|
|
456
|
+
* Separate from {@link BuildFailure} only so the walk can attach the path of the
|
|
457
|
+
* offending file — which the thrower does not know — before it reaches the
|
|
458
|
+
* boundary as one.
|
|
459
|
+
*/
|
|
460
|
+
const BYTE_ORDER_MARK = String.fromCharCode(0xfeff);
|
|
461
|
+
/**
|
|
462
|
+
* An opening delimiter followed, anywhere after it, by a line opening with a
|
|
463
|
+
* `publish:` key. Unbounded on purpose: the caller reaches this only when no
|
|
464
|
+
* closing delimiter matched, so the block runs to the end of the file, and a
|
|
465
|
+
* long frontmatter block is valid — a bound would publish an opt-out that
|
|
466
|
+
* happened to sit below it.
|
|
467
|
+
*/
|
|
468
|
+
const UNRECOGNISED_PUBLISH_BLOCK = /^---[ \t]*\r?\n(?:[^\n]*\n)*?[ \t]*publish[ \t]*:/;
|
|
469
|
+
class FrontmatterError extends Error {
|
|
470
|
+
code;
|
|
471
|
+
detail;
|
|
472
|
+
constructor(code, message, detail) {
|
|
473
|
+
super(message);
|
|
474
|
+
this.name = 'FrontmatterError';
|
|
475
|
+
this.code = code;
|
|
476
|
+
this.detail = detail;
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
/**
|
|
480
|
+
* Whether a note asked not to be published, refusing every near-miss loudly.
|
|
481
|
+
*
|
|
482
|
+
* H2's second half, and the measurement behind it: `yaml` parses `publish: no`
|
|
483
|
+
* to the **string** `'no'`, `publish: off` to `'off'`, `publish: "false"` to
|
|
484
|
+
* `'false'`, and `publish: 0` to `0`. YAML 1.2 dropped YAML 1.1's `no`/`off`
|
|
485
|
+
* booleans, so every one of those is a user writing "do not publish" and getting
|
|
486
|
+
* a truthy non-boolean that a `=== false` test reads as consent. Each publishes
|
|
487
|
+
* the note.
|
|
488
|
+
*
|
|
489
|
+
* So the field is boolean or the build stops. There is no coercion table: a
|
|
490
|
+
* coercion table is a list of spellings that silently work, and the next
|
|
491
|
+
* spelling nobody listed publishes the note anyway. `publish: true` is accepted
|
|
492
|
+
* and does **not** override an exclusion — §2.3's asymmetry, and the reason for
|
|
493
|
+
* it is that the two failure modes are not equal. A note the user meant to
|
|
494
|
+
* publish and did not is an inconvenience they will notice; a note they meant to
|
|
495
|
+
* withhold and published is irreversible once it is on a CDN and in a search
|
|
496
|
+
* index.
|
|
497
|
+
*/
|
|
498
|
+
function publishFlag(data) {
|
|
499
|
+
if (data === undefined || !('publish' in data))
|
|
500
|
+
return undefined;
|
|
501
|
+
const value = data['publish'];
|
|
502
|
+
if (typeof value === 'boolean')
|
|
503
|
+
return value;
|
|
504
|
+
throw new FrontmatterError('non-boolean-publish-flag', 'frontmatter `publish:` must be `true` or `false` — YAML 1.2 reads `no`, `off`, ' +
|
|
505
|
+
'`"false"` and `0` as a string or a number, not as a boolean, and a note meaning ' +
|
|
506
|
+
'to withhold itself would publish. See the report for the file.', `publish: ${JSON.stringify(value)} (${typeof value})`);
|
|
507
|
+
}
|
|
508
|
+
/**
|
|
509
|
+
* A link's text, with its syntax removed — for the two fields that are read as
|
|
510
|
+
* plain text rather than rendered.
|
|
511
|
+
*
|
|
512
|
+
* `title` and `excerpt` are derived from the *rewritten* Markdown, so every
|
|
513
|
+
* internal link in them is already `[label](/route/)`. Neither field goes
|
|
514
|
+
* through the Markdown pipeline: the excerpt lands verbatim in
|
|
515
|
+
* the public snapshot, in `rss.xml`, and in the `<meta name="description">` of
|
|
516
|
+
* every page showing the card, and the title lands in `<title>` and `og:title`.
|
|
517
|
+
* Measured before this existed — a heading and a body each containing one
|
|
518
|
+
* ordinary link:
|
|
519
|
+
*
|
|
520
|
+
* ```
|
|
521
|
+
* <title>See [beta](/beta/) now · Notes</title>
|
|
522
|
+
* <meta name="description" content="See [beta](/beta/) for the numbers.">
|
|
523
|
+
* ```
|
|
524
|
+
*
|
|
525
|
+
* Brackets and a route in a reader's search result, on every build with a link
|
|
526
|
+
* in a first heading or first paragraph. This is the same defect `aab694b`
|
|
527
|
+
* fixed for code spans, in the same function, and that fix stripped backticks
|
|
528
|
+
* and stopped.
|
|
529
|
+
*
|
|
530
|
+
* **The label is kept and the destination dropped**, which is what a reader
|
|
531
|
+
* would have seen had the field been rendered. That is also why this cannot be
|
|
532
|
+
* "delete the whole construct": the label is the author's prose.
|
|
533
|
+
*
|
|
534
|
+
* **Applied repeatedly until it stops changing**, because a badge is a link
|
|
535
|
+
* whose label is an image — `[](/t/)` — and one pass leaves
|
|
536
|
+
* the outer construct behind.
|
|
537
|
+
*
|
|
538
|
+
* The label pattern admits `\]`, which `scripts/resolve-links.ts` writes when an
|
|
539
|
+
* author's own label contains a bracket; those escapes are unescaped afterwards,
|
|
540
|
+
* since a reader of a plain-text field should see the bracket rather than the
|
|
541
|
+
* backslash. A reference link (`[text][ci]`) never reaches this in its
|
|
542
|
+
* reference form when its definition is internal: `scripts/resolve-links.ts`
|
|
543
|
+
* has already rewritten it inline. What is left untouched is an external or
|
|
544
|
+
* missing-definition reference, whose target this module cannot vouch for.
|
|
545
|
+
*/
|
|
546
|
+
function withoutLinkSyntax(markdown) {
|
|
547
|
+
let text = markdown;
|
|
548
|
+
for (;;) {
|
|
549
|
+
// No whitespace and no parens in the destination, so ordinary prose such as
|
|
550
|
+
// "the array [a] (see below)" is not mistaken for a link.
|
|
551
|
+
const next = text.replace(/!?\[((?:[^\][\\]|\\.)*)\]\([^()\s]*\)/g, '$1');
|
|
552
|
+
if (next === text)
|
|
553
|
+
break;
|
|
554
|
+
text = next;
|
|
555
|
+
}
|
|
556
|
+
return text.replace(/\\([[\]])/g, '$1');
|
|
557
|
+
}
|
|
558
|
+
/**
|
|
559
|
+
* Every math expression removed, and the punctuation it stranded repaired.
|
|
560
|
+
*
|
|
561
|
+
* ## Why an excerpt carries nothing for an expression
|
|
562
|
+
*
|
|
563
|
+
* This is a judgement and the alternatives were read as a reader gets them,
|
|
564
|
+
* because the sentence on the page is the whole of the evidence:
|
|
565
|
+
*
|
|
566
|
+
* | | `The identity $$\frac{a}{b}$$ holds.` | `Consider $$…$$. It follows.` |
|
|
567
|
+
* | --- | --- | --- |
|
|
568
|
+
* | keep the TeX (today) | `The identity $$\frac{a}{b} = \sqrt{c}$$ holds.` | `Consider $$\frac{\partial \mathcal{L}}{\partial \theta} = 0$$. It follows.` |
|
|
569
|
+
* | drop it, no repair | `The identity holds.` | `Consider . It follows.` |
|
|
570
|
+
* | a `[math]` placeholder | `The identity [math] holds.` | `Consider [math]. It follows.` |
|
|
571
|
+
* | **drop and repair** | `The identity holds.` | `Consider. It follows.` |
|
|
572
|
+
*
|
|
573
|
+
* **Keeping the TeX is what ships today and it is the worst of the four.** An
|
|
574
|
+
* excerpt is plain text on a card, in `rss.xml`, and in every page's
|
|
575
|
+
* `<meta name="description">` — `\frac{\partial \mathcal{L}}{\partial \theta}`
|
|
576
|
+
* is noise in all three, and it crowds out the prose that would have told a
|
|
577
|
+
* reader what the note is about.
|
|
578
|
+
*
|
|
579
|
+
* **A placeholder was rejected**, though it reads well. `[math]` is a chrome
|
|
580
|
+
* string minted in the producer, which is outside the components TK-16's
|
|
581
|
+
* bilingual sweep covers — so a zh-CN note would carry an English marker in its
|
|
582
|
+
* own description. Translating it would put a locale lookup in a module that has
|
|
583
|
+
* no document language to look up. And it is not free of judgement either: it
|
|
584
|
+
* asserts to a reader that something was removed, which is a claim about the
|
|
585
|
+
* note the excerpt is not otherwise in the business of making.
|
|
586
|
+
*
|
|
587
|
+
* **What is dropped is a real loss, stated plainly**: a note whose sentence is
|
|
588
|
+
* mostly one expression yields a thinner excerpt than it deserves, and
|
|
589
|
+
* `$$E = mc^2$$ is the relation everyone knows.` becomes
|
|
590
|
+
* `is the relation everyone knows.` — a sentence starting mid-clause. There is
|
|
591
|
+
* no rendered form available at this stage to substitute, because the excerpt is
|
|
592
|
+
* derived from Markdown before any renderer runs. Given that, the honest options
|
|
593
|
+
* were the four above and this is the least bad.
|
|
594
|
+
*
|
|
595
|
+
* It also makes the two delimiter forms agree. A ```` ```math ```` fence already
|
|
596
|
+
* produces exactly this today — fenced code is stripped a line above — so the
|
|
597
|
+
* change is that `$$…$$` stops being the exception rather than that anything new
|
|
598
|
+
* starts happening.
|
|
599
|
+
*
|
|
600
|
+
* ## What counts as math, taken from the renderer rather than guessed
|
|
601
|
+
*
|
|
602
|
+
* `renderMarkdown` sets `singleDollarTextMath: false`, so a single `$` is not a
|
|
603
|
+
* delimiter and ordinary prices are left alone. Verified rather than trusted —
|
|
604
|
+
* measured through the real renderer: `It costs $5 and then $10 later.` sets
|
|
605
|
+
* `hasMath: false`, and `The price is $$ and rising.` does too, because an
|
|
606
|
+
* unpaired `$$` is not a span. Both survive this function unchanged.
|
|
607
|
+
*
|
|
608
|
+
* `From $$5 to $$10 in a year.` *does* parse as math — the two `$$` pair up
|
|
609
|
+
* across the prose between them — and is stripped here in consequence. That is
|
|
610
|
+
* the renderer's reading and this follows it: an excerpt disagreeing with the
|
|
611
|
+
* page about what is math would be a second, quieter defect.
|
|
612
|
+
*
|
|
613
|
+
* ## The seam
|
|
614
|
+
*
|
|
615
|
+
* Removing a span strands whatever punctuation surrounded it. A space before a
|
|
616
|
+
* comma or a stop is an artefact of the removal rather than anything the author
|
|
617
|
+
* wrote, so it is closed up; a bracket left holding nothing — `($$\alpha$$)`
|
|
618
|
+
* becoming `()` — is the same artefact in a worse form, and goes entirely.
|
|
619
|
+
*
|
|
620
|
+
* **The span is found once, and the seam is decided from its own boundaries.**
|
|
621
|
+
* That shape is forced by two defects a chained-regex version had, both found by
|
|
622
|
+
* review and both measured through the producer:
|
|
623
|
+
*
|
|
624
|
+
* - A rule that looked for the span *and* its trailing punctuation in one match
|
|
625
|
+
* backtracked past the closing `$$` to find one. Measured:
|
|
626
|
+
* `The bound is $$\alpha$$ and the limit is $$\beta$$.` came out as
|
|
627
|
+
* `The bound is.` — eight words of the author's prose eaten, on what is the
|
|
628
|
+
* ordinary shape of a sentence in a maths note. The docstring claimed the
|
|
629
|
+
* repair was "structurally unable to reach prose it did not just alter", and
|
|
630
|
+
* a lazy quantifier with a forced trailing anchor is not bounded at all.
|
|
631
|
+
* - A rule deleting `()` and `[]` across the whole string fired on notes with no
|
|
632
|
+
* math in them: `The function main() returns zero, and init() does not.`
|
|
633
|
+
* became `The function main returns zero, and init does not.`, and a GFM
|
|
634
|
+
* `- [ ]` task marker vanished. Deciding it at the removal site makes it
|
|
635
|
+
* reachable only by text this function just changed.
|
|
636
|
+
*
|
|
637
|
+
* A `while` loop with `lastIndex` rather than `replace`, because the decision
|
|
638
|
+
* needs the characters on both sides of the match and a replacer callback does
|
|
639
|
+
* not get them.
|
|
640
|
+
*/
|
|
641
|
+
function withoutMath(markdown) {
|
|
642
|
+
const span = /\$\$[\s\S]*?\$\$/g;
|
|
643
|
+
let result = '';
|
|
644
|
+
let read = 0;
|
|
645
|
+
let match;
|
|
646
|
+
while ((match = span.exec(markdown)) !== null) {
|
|
647
|
+
const before = markdown.slice(read, match.index);
|
|
648
|
+
const after = markdown.slice(span.lastIndex);
|
|
649
|
+
// Whitespace either side of the span belongs to the seam rather than to the
|
|
650
|
+
// prose, and is taken with it. What replaces the whole seam is decided by
|
|
651
|
+
// what sits immediately outside: a space, unless the removal would leave a
|
|
652
|
+
// gap before punctuation or inside a bracket, where it would be visible.
|
|
653
|
+
const leading = /\s*$/.exec(before)?.[0].length ?? 0;
|
|
654
|
+
const trailing = /^\s*/.exec(after)?.[0].length ?? 0;
|
|
655
|
+
const prose = before.slice(0, before.length - leading);
|
|
656
|
+
const next = after[trailing];
|
|
657
|
+
// A bracket pair whose entire content was the expression is an artefact of
|
|
658
|
+
// the removal, not the author's — but only when *this* removal emptied it.
|
|
659
|
+
const opensHere = /[([]$/.test(prose);
|
|
660
|
+
const closesNext = next !== undefined && /[)\]]/.test(next);
|
|
661
|
+
if (opensHere && closesNext) {
|
|
662
|
+
result += prose.slice(0, -1);
|
|
663
|
+
read = span.lastIndex + trailing + 1;
|
|
664
|
+
continue;
|
|
665
|
+
}
|
|
666
|
+
const joinTight = next !== undefined && /[.,;:!?)\]]/.test(next);
|
|
667
|
+
result += prose + (prose === '' || joinTight ? '' : ' ');
|
|
668
|
+
read = span.lastIndex + trailing;
|
|
669
|
+
}
|
|
670
|
+
const stripped = result + markdown.slice(read);
|
|
671
|
+
// **An unpaired `$$` still ships, so it is removed too.** The renderer pairs
|
|
672
|
+
// delimiters left to right and leaves a leftover one as literal text — so
|
|
673
|
+
// `A $$x$$ b $$ c, and $$y$$ done.` renders with a visible `$$`, and the loop
|
|
674
|
+
// above, which pairs the same way, leaves the same one behind. It is exactly
|
|
675
|
+
// the byte this whole function exists to keep out of a description, and the
|
|
676
|
+
// page carrying it too is a separate defect rather than a licence to ship it
|
|
677
|
+
// here. What is left is the delimiter alone; the author's surrounding words
|
|
678
|
+
// are untouched.
|
|
679
|
+
return stripped.replace(/\$\$/g, '');
|
|
680
|
+
}
|
|
681
|
+
/**
|
|
682
|
+
* The first heading's text, or the filename.
|
|
683
|
+
*
|
|
684
|
+
* A title is required and must be non-empty, and the renderer removes a leading
|
|
685
|
+
* `# Title` that matches the page title — so taking it from the body is what
|
|
686
|
+
* makes an ordinary note render with its heading once rather than twice.
|
|
687
|
+
*
|
|
688
|
+
* Link syntax is stripped for the reason {@link withoutLinkSyntax} gives: this
|
|
689
|
+
* value reaches `<title>` and `og:title` as plain text, and a heading may carry
|
|
690
|
+
* a link like any other line.
|
|
691
|
+
*
|
|
692
|
+
* **Math goes the same way, and the title path had the defect too** — checked
|
|
693
|
+
* rather than assumed, because `4cae152` found `titleFor` shared the link-syntax
|
|
694
|
+
* defect and this is the third repair of the same shape. Measured before the
|
|
695
|
+
* fix: `# The $$\alpha$$ theorem` produced the title `The $$\alpha$$ theorem`,
|
|
696
|
+
* which reaches `<title>`, `og:title`, and the browser tab.
|
|
697
|
+
*
|
|
698
|
+
* The whitespace collapse is this path's own. `excerptFor` collapses as its last
|
|
699
|
+
* step and a title never went through one, because nothing it stripped used to
|
|
700
|
+
* leave a gap mid-line — a removed expression does. Measured without it:
|
|
701
|
+
* `The theorem`, three spaces, in the browser tab.
|
|
702
|
+
*
|
|
703
|
+
* **Code spans are removed first, exactly as `excerptFor` does**, and a first
|
|
704
|
+
* version of this omitted them while its own comment defended the ordering that
|
|
705
|
+
* only the other function had. Measured: `` # Writing `$$x$$` in a note ``
|
|
706
|
+
* rendered as `Writing $$x$$ in a note` on the page — the renderer treats a
|
|
707
|
+
* backticked `$$` as code, not math — while the title came back
|
|
708
|
+
* ``Writing ` ` in a note``: the author's example eaten *and* stray backticks
|
|
709
|
+
* shipped into `<title>` and `og:title`, which is the defect `aab694b` fixed for
|
|
710
|
+
* excerpts arriving on the other surface.
|
|
711
|
+
*/
|
|
712
|
+
function titleFor(markdown, fallback) {
|
|
713
|
+
const heading = /^#\s+(.+)$/m.exec(markdown)?.[1]?.trim();
|
|
714
|
+
const stripped = heading === undefined
|
|
715
|
+
? ''
|
|
716
|
+
: withoutMath(withoutLinkSyntax(heading.replace(/`[^`\n]*`/g, '')))
|
|
717
|
+
.replace(/\s+/g, ' ')
|
|
718
|
+
.trim();
|
|
719
|
+
return stripped || fallback;
|
|
720
|
+
}
|
|
721
|
+
/**
|
|
722
|
+
* Prose, collapsed and bounded, for the card and the meta description.
|
|
723
|
+
*
|
|
724
|
+
* **Inline code is stripped as well as fenced, and the reason is a build break
|
|
725
|
+
* rather than tidiness.** An excerpt is not rendered through the Markdown
|
|
726
|
+
* pipeline — it lands verbatim in the public snapshot, in `rss.xml`, and in the
|
|
727
|
+
* `<meta name="description">` of every page that shows the card. Those surfaces
|
|
728
|
+
* carry no `<code>` element, so the residue scan's code-region exemption cannot
|
|
729
|
+
* see them, and a note writing ``Inline `[[syntax]]` is how you write it.``
|
|
730
|
+
* exited 1 with `contains unresolved [[wikilink]]` against four files. Measured
|
|
731
|
+
* on the shipped binary: a *fenced* example built clean and the inline form did
|
|
732
|
+
* not, which is the asymmetry this line removes.
|
|
733
|
+
*
|
|
734
|
+
* A note documenting wikilink syntax is this tool's own audience, so the break
|
|
735
|
+
* was on exactly the content it exists to publish.
|
|
736
|
+
*
|
|
737
|
+
* Stripped rather than exempted downstream: the excerpt is a projection of the
|
|
738
|
+
* prose, and a code span is not prose. Widening the scanner's exemption instead
|
|
739
|
+
* would have admitted a genuinely unresolved `[[wikilink]]` — the producer
|
|
740
|
+
* defect that rule exists to catch — which is the repair this rule has already
|
|
741
|
+
* had twice.
|
|
742
|
+
*
|
|
743
|
+
* **Link syntax goes the same way and for the same reason**, one commit later
|
|
744
|
+
* and found the same way: the fix above stripped backticks and stopped, so
|
|
745
|
+
* every build shipped `See [beta](/beta/) for the numbers.` into the meta
|
|
746
|
+
* description. See {@link withoutLinkSyntax}. Code spans are removed *before*
|
|
747
|
+
* links, so a note documenting `` `[label](/route/)` `` loses the span whole
|
|
748
|
+
* rather than having its example rewritten into prose.
|
|
749
|
+
*
|
|
750
|
+
* **And math, one ticket later again, found by measuring the three surfaces
|
|
751
|
+
* rather than by a build break.** `The identity $$\frac{a}{b} = \sqrt{c}$$
|
|
752
|
+
* holds.` shipped verbatim into the public snapshot, `rss.xml`, and every meta
|
|
753
|
+
* description. See {@link withoutMath} for what an excerpt carries instead and
|
|
754
|
+
* what that costs.
|
|
755
|
+
*
|
|
756
|
+
* **Ordered last, after the code strips.** A note documenting math syntax writes
|
|
757
|
+
* `` `$$x^2$$` `` or a fenced block, and the renderer treats neither as math —
|
|
758
|
+
* measured, `hasMath` is `false` for both. Stripping code first means this
|
|
759
|
+
* function never sees them, so the excerpt agrees with the page about which
|
|
760
|
+
* dollar signs were delimiters. Reversing the order would eat an author's
|
|
761
|
+
* example out of a note about writing math.
|
|
762
|
+
*/
|
|
763
|
+
function excerptFor(markdown) {
|
|
764
|
+
const prose = withoutMath(withoutLinkSyntax(markdown
|
|
765
|
+
.replace(/^#.*$/gm, '')
|
|
766
|
+
.replace(/```[\s\S]*?```/g, '')
|
|
767
|
+
.replace(/`[^`\n]*`/g, '')))
|
|
768
|
+
.replace(/\s+/g, ' ')
|
|
769
|
+
.trim();
|
|
770
|
+
return prose.length > 200 ? `${prose.slice(0, 200).trimEnd()}…` : prose;
|
|
771
|
+
}
|
|
772
|
+
/**
|
|
773
|
+
* Every file under the root, depth-first and sorted at each level, with
|
|
774
|
+
* structurally ignored names pruned.
|
|
775
|
+
*
|
|
776
|
+
* Sorted because glob and `readdir` order are both unspecified and determinism
|
|
777
|
+
* is a hard constraint — measured, win32 and Linux return different orders for
|
|
778
|
+
* the same directory, and a slug collision's winner is decided by this order.
|
|
779
|
+
* Sorting per level rather than over the whole result keeps a directory's files
|
|
780
|
+
* adjacent to it, which is what makes the collision winner predictable from the
|
|
781
|
+
* tree rather than from the flattening.
|
|
782
|
+
*
|
|
783
|
+
* Paths are joined with `/` explicitly rather than with `path.join`:
|
|
784
|
+
* `matchesGlob` is separator-literal — measured, `matchesGlob('a\\b.md', 'a/**')`
|
|
785
|
+
* is `false` on win32 — so a native-separator path would silently match no
|
|
786
|
+
* pattern at all on the platform where the user is most likely to be testing.
|
|
787
|
+
*/
|
|
788
|
+
async function walk(root, escaped, prefix = '') {
|
|
789
|
+
const entries = (await readdir(prefix === '' ? root : `${root}/${prefix}`, { withFileTypes: true })).sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
|
|
790
|
+
const found = [];
|
|
791
|
+
for (const entry of entries) {
|
|
792
|
+
if (structurallyIgnored(entry.name))
|
|
793
|
+
continue;
|
|
794
|
+
const relativePath = prefix === '' ? entry.name : `${prefix}/${entry.name}`;
|
|
795
|
+
// `isFile()` alone is not the file test, and the comment that used to sit
|
|
796
|
+
// here said it was. Measured on win32 (Node 24.18.1): a symlink to
|
|
797
|
+
// `real.md` reports `isFile() === false` and `isSymbolicLink() === true`
|
|
798
|
+
// under `withFileTypes`, so an `isFile()`-only branch put a symlinked note
|
|
799
|
+
// in **neither** `discovered` nor `dropped` — it left the partition
|
|
800
|
+
// silently, which is the one outcome the report cannot express. TK-25 §4.2
|
|
801
|
+
// records the opposite as measured; it is wrong on this platform.
|
|
802
|
+
//
|
|
803
|
+
// So a symlink is resolved by asking the filesystem what it points at.
|
|
804
|
+
// `stat` follows the link where `readdir` did not, and a broken link throws
|
|
805
|
+
// rather than answering — which is a file that is not there, so it is
|
|
806
|
+
// pruned like a directory rather than counted.
|
|
807
|
+
if (entry.isDirectory()) {
|
|
808
|
+
found.push(...(await walk(root, escaped, relativePath)));
|
|
809
|
+
continue;
|
|
810
|
+
}
|
|
811
|
+
if (entry.isFile()) {
|
|
812
|
+
found.push(relativePath);
|
|
813
|
+
continue;
|
|
814
|
+
}
|
|
815
|
+
if (!entry.isSymbolicLink())
|
|
816
|
+
continue;
|
|
817
|
+
const target = await stat(`${root}/${relativePath}`).catch(() => undefined);
|
|
818
|
+
// A symlinked *directory* is deliberately not descended into: a link may
|
|
819
|
+
// point at an ancestor, and a walk that follows one does not terminate.
|
|
820
|
+
// Enumerating it as a file would be worse than skipping it, so it is
|
|
821
|
+
// treated exactly as a real directory's own entry is — pruned, in neither
|
|
822
|
+
// column, because "the tool declined to follow" is not a fact about a file
|
|
823
|
+
// the user wrote.
|
|
824
|
+
if (target?.isFile() !== true)
|
|
825
|
+
continue;
|
|
826
|
+
// A file link is followed only while it stays inside the content root.
|
|
827
|
+
// `readFile` later follows it too, so a link to `~/private/plan.md` would
|
|
828
|
+
// publish that file's contents under the link's own name — and a release
|
|
829
|
+
// ledger records only the slug, while git never sees the target change, so
|
|
830
|
+
// an approved slug could carry different private text on every build.
|
|
831
|
+
// Resolved through `realpath` on both sides, so a root reached through a
|
|
832
|
+
// link of its own still contains its own files.
|
|
833
|
+
const resolvedRoot = await realpath(root);
|
|
834
|
+
const resolvedTarget = await realpath(`${root}/${relativePath}`).catch(() => undefined);
|
|
835
|
+
const step = resolvedTarget === undefined ? '..' : relative(resolvedRoot, resolvedTarget);
|
|
836
|
+
found.push(relativePath);
|
|
837
|
+
if (step === '' || step.startsWith('..') || isAbsolute(step))
|
|
838
|
+
escaped.add(relativePath);
|
|
839
|
+
}
|
|
840
|
+
return found;
|
|
841
|
+
}
|
|
842
|
+
/**
|
|
843
|
+
* A bare configuration filename, which is the only shape allowed onto a stream.
|
|
844
|
+
*
|
|
845
|
+
* Tighter than "no separators", and the loose version was measured admitting
|
|
846
|
+
* exactly the hazard the field exists to exclude: `/^[A-Za-z0-9._-]+$/` accepts
|
|
847
|
+
* `zzqsecret-client.md` and `zzq2026-layoffs`, which are a withheld note's
|
|
848
|
+
* filename and its stem. So the shape required is a *known configuration file
|
|
849
|
+
* extension* — a stem alone is refused, and a note's `.md` is not on the list.
|
|
850
|
+
*
|
|
851
|
+
* A caller passing anything else gets the fixed literal instead of its string,
|
|
852
|
+
* silently, because refusing the build over a diagnostic's own label would fail
|
|
853
|
+
* a run that had nothing else wrong with it.
|
|
854
|
+
*/
|
|
855
|
+
const CONFIG_LOCATION = /^[A-Za-z0-9._-]+\.(?:ts|js|mjs|cjs|json|ya?ml|toml)$/;
|
|
856
|
+
/**
|
|
857
|
+
* Whether the last pattern to mention this path re-included it.
|
|
858
|
+
*
|
|
859
|
+
* Only the default-excluded root README consults this, and it is what makes
|
|
860
|
+
* plan §2.2's "re-includable with `!README.md`" true. `excludes` cannot answer
|
|
861
|
+
* it: a path no pattern touches and a path whose last rule was a `!` both come
|
|
862
|
+
* back `false`, and those are opposite answers to "did the user ask for this
|
|
863
|
+
* file back".
|
|
864
|
+
*
|
|
865
|
+
* No `matched` argument: the recording already happened in the `excludes` call
|
|
866
|
+
* this run made for the same path, and marking a pattern twice would let a
|
|
867
|
+
* `!README.md` that matched nothing else look used.
|
|
868
|
+
*/
|
|
869
|
+
function reincluded(relativePath, patterns) {
|
|
870
|
+
let answer = false;
|
|
871
|
+
for (const pattern of patterns) {
|
|
872
|
+
const negated = pattern.startsWith('!');
|
|
873
|
+
if (matchesGlob(relativePath, negated ? pattern.slice(1) : pattern))
|
|
874
|
+
answer = negated;
|
|
875
|
+
}
|
|
876
|
+
return answer;
|
|
877
|
+
}
|
|
878
|
+
/**
|
|
879
|
+
* Which patterns matched a path, and the verdict of the last one to do so.
|
|
880
|
+
*
|
|
881
|
+
* Every match is recorded, not just the decisive one: a pattern shadowed by a
|
|
882
|
+
* later rule has still done its job of proving it was not a typo, so the
|
|
883
|
+
* zero-match check does not depend on rule order.
|
|
884
|
+
*
|
|
885
|
+
* `matched` is written through rather than returned because the caller
|
|
886
|
+
* accumulates it across every discovered path — the question the zero-match rule
|
|
887
|
+
* asks is about the corpus, not about any one file.
|
|
888
|
+
*/
|
|
889
|
+
function excludes(relativePath, patterns, matched) {
|
|
890
|
+
let verdict = false;
|
|
891
|
+
for (const [index, pattern] of patterns.entries()) {
|
|
892
|
+
const negated = pattern.startsWith('!');
|
|
893
|
+
// `matchesGlob` has no negation of its own — measured, `matchesGlob('a/b.md',
|
|
894
|
+
// '!a/**')` is `false` rather than an inversion — so the `!` is stripped here
|
|
895
|
+
// and the sense is applied to the verdict.
|
|
896
|
+
const body = negated ? pattern.slice(1) : pattern;
|
|
897
|
+
if (!matchesGlob(relativePath, body))
|
|
898
|
+
continue;
|
|
899
|
+
matched[index] = true;
|
|
900
|
+
verdict = !negated;
|
|
901
|
+
}
|
|
902
|
+
return verdict;
|
|
903
|
+
}
|
|
904
|
+
/**
|
|
905
|
+
* Walk a repository, apply both exclusion mechanisms, and say what was left out.
|
|
906
|
+
*
|
|
907
|
+
* Discovery only: the artifact is **not** validated here, because the caller has
|
|
908
|
+
* to be able to record what was discovered before the contract can reject any of
|
|
909
|
+
* it. That ordering is the whole reason a run which fails the contract still
|
|
910
|
+
* leaves a report naming the files it dropped.
|
|
911
|
+
*
|
|
912
|
+
* `backlinks` is the mechanical inverse of `outgoing`, derived in
|
|
913
|
+
* {@link resolveCorpusLinks} because `checkCorpus` rejects any artifact where it
|
|
914
|
+
* is not — so a producer emitting only one direction makes every repository with
|
|
915
|
+
* an internal link unbuildable. Deriving the array is all this ticket does with
|
|
916
|
+
* it; what a page *renders* from it is TK-28's.
|
|
917
|
+
*
|
|
918
|
+
* **Enumerate first, classify inside.** Every path {@link walk} returned is
|
|
919
|
+
* `discovered`, and leaves with either an entry or a `DroppedFile` row — so
|
|
920
|
+
* `discovered === published + dropped` is a property of the loop rather than of
|
|
921
|
+
* three counters. A structurally ignored name is in neither, for the same reason
|
|
922
|
+
* a directory is in neither: "the tool declined to look" is not a fact about a
|
|
923
|
+
* file the user wrote.
|
|
924
|
+
*
|
|
925
|
+
* **Exclusion precedence, and it is not symmetric.** Frontmatter `publish: false`
|
|
926
|
+
* is rank 1 and unconditional — a `!` re-include cannot resurrect it — and the
|
|
927
|
+
* glob rules are rank 2. The mechanism pointing toward *not* publishing is the
|
|
928
|
+
* three words inside the file itself, nearest the content, and it always wins.
|
|
929
|
+
*/
|
|
930
|
+
export async function discover(contentDirectory, options = {}) {
|
|
931
|
+
const patterns = options.exclude ?? [];
|
|
932
|
+
const matched = patterns.map(() => false);
|
|
933
|
+
/** File links whose target resolves outside the content root: dropped unread. */
|
|
934
|
+
const escaped = new Set();
|
|
935
|
+
const paths = await walk(contentDirectory, escaped);
|
|
936
|
+
const gitDates = gitDatesFor(contentDirectory, paths);
|
|
937
|
+
/** Published source paths whose derived slug belongs to the site itself. */
|
|
938
|
+
const reserved = [];
|
|
939
|
+
/** Slug to the path that claimed it, so a collision can name its winner. */
|
|
940
|
+
const claimed = new Map();
|
|
941
|
+
/** Slug to the path that produced it, for the link traversal. */
|
|
942
|
+
const pathBySlug = new Map();
|
|
943
|
+
/** Slugs whose title the author wrote in frontmatter rather than a heading. */
|
|
944
|
+
const declaredTitles = new Set();
|
|
945
|
+
const dropped = [];
|
|
946
|
+
const entries = [];
|
|
947
|
+
for (const path of paths) {
|
|
948
|
+
// **Every discovered path is offered to every pattern, before any drop.**
|
|
949
|
+
// Not a reordering for tidiness: with the `not-markdown` drop ahead of this,
|
|
950
|
+
// a non-Markdown file was `discovered` and yet never seen by a pattern, so
|
|
951
|
+
// `exclude: ['assets/**']` on a repository whose `assets/` holds only images
|
|
952
|
+
// matched nothing and failed the build. Measured — excluding an asset folder
|
|
953
|
+
// is the ordinary case in a notes repository, and it was unbuildable.
|
|
954
|
+
//
|
|
955
|
+
// Plan §2.3 states the rule this restores: "'Matched' means the pattern
|
|
956
|
+
// returned true for at least one *discovered* path, whatever the final
|
|
957
|
+
// verdict was." A file dropped for another reason is still discovered.
|
|
958
|
+
const excluded = excludes(path, patterns, matched);
|
|
959
|
+
if (!path.endsWith('.md')) {
|
|
960
|
+
dropped.push({ path, reason: 'not-markdown' });
|
|
961
|
+
continue;
|
|
962
|
+
}
|
|
963
|
+
// The root README is excluded by default and, per plan §2.2, is
|
|
964
|
+
// "re-includable with `!README.md`" — so the default yields to a user's
|
|
965
|
+
// negation rather than preceding it. `excludes` returning false for a path
|
|
966
|
+
// no pattern touched is why this cannot simply test `excluded`: the question
|
|
967
|
+
// is whether the user *said* something about this path, not what the last
|
|
968
|
+
// rule concluded about every path.
|
|
969
|
+
if (path === REPOSITORY_README && !reincluded(path, patterns)) {
|
|
970
|
+
dropped.push({ path, reason: 'repository-readme' });
|
|
971
|
+
continue;
|
|
972
|
+
}
|
|
973
|
+
// Rank 2, and the read is still below it: an excluded note's bytes are not
|
|
974
|
+
// needed, and not reading them is one fewer way for its contents to reach
|
|
975
|
+
// anything. Rank 1 overrides this below — the read happens for a published
|
|
976
|
+
// candidate only, so the two ranks are ordered by outcome rather than by
|
|
977
|
+
// which test runs first.
|
|
978
|
+
if (excluded) {
|
|
979
|
+
dropped.push({ path, reason: 'excluded-by-pattern' });
|
|
980
|
+
continue;
|
|
981
|
+
}
|
|
982
|
+
// Below the author's own exclusions, which may already cover it, and above
|
|
983
|
+
// the read: the target lies outside what the author chose to publish from,
|
|
984
|
+
// so its bytes are never read at all. See `walk`.
|
|
985
|
+
if (escaped.has(path)) {
|
|
986
|
+
dropped.push({ path, reason: 'link-outside-content' });
|
|
987
|
+
continue;
|
|
988
|
+
}
|
|
989
|
+
const source = (await readFile(`${contentDirectory}/${path}`, 'utf8')).replace(/\r\n/g, '\n');
|
|
990
|
+
let parsed;
|
|
991
|
+
let published;
|
|
992
|
+
try {
|
|
993
|
+
parsed = frontmatterOf(source);
|
|
994
|
+
published = publishFlag(parsed.data);
|
|
995
|
+
}
|
|
996
|
+
catch (error) {
|
|
997
|
+
// The thrower knows the fault and not the file; the walk knows the file
|
|
998
|
+
// and not the fault. Joining them here is what lets the public half stay
|
|
999
|
+
// nameless while the report gets a detail a user can act on.
|
|
1000
|
+
if (error instanceof FrontmatterError) {
|
|
1001
|
+
throw new BuildFailure(error.code, error.message, `${path}: ${error.detail}`);
|
|
1002
|
+
}
|
|
1003
|
+
throw error;
|
|
1004
|
+
}
|
|
1005
|
+
if (published === false) {
|
|
1006
|
+
dropped.push({ path, reason: 'excluded-by-frontmatter' });
|
|
1007
|
+
continue;
|
|
1008
|
+
}
|
|
1009
|
+
const slug = slugOverrideFor(parsed.data, path) ?? slugFor(path);
|
|
1010
|
+
if (RESERVED_SLUGS.has(slug)) {
|
|
1011
|
+
// Fatal candidates are not "dropped": no Discovery is returned and the
|
|
1012
|
+
// failure report owns these paths. Calling this an exclusion would imply
|
|
1013
|
+
// the rest of the site can still publish.
|
|
1014
|
+
reserved.push({ path, slug });
|
|
1015
|
+
continue;
|
|
1016
|
+
}
|
|
1017
|
+
// A duplicate slug fails the contract, so two files that collide would fail
|
|
1018
|
+
// the build with a schema error naming neither file. Dropping the second is
|
|
1019
|
+
// no better as an answer — plan D1 gives ambiguity to TK-27 — but it fails
|
|
1020
|
+
// understandably here rather than confusingly, and the report says which
|
|
1021
|
+
// file lost and to which. The winner is the first in the sorted walk, which
|
|
1022
|
+
// is why the walk sorts.
|
|
1023
|
+
const winner = claimed.get(slug);
|
|
1024
|
+
if (winner !== undefined) {
|
|
1025
|
+
dropped.push({ path, reason: 'slug-collision', collidedWith: winner });
|
|
1026
|
+
continue;
|
|
1027
|
+
}
|
|
1028
|
+
claimed.set(slug, path);
|
|
1029
|
+
const { title, declared: titleDeclared } = titleFrom(parsed.data, parsed.body, slug, path);
|
|
1030
|
+
const tags = publicLabelsFor(parsed.data, 'tags', path);
|
|
1031
|
+
const aliases = publicLabelsFor(parsed.data, 'aliases', path);
|
|
1032
|
+
const collection = collectionFor(path);
|
|
1033
|
+
const language = languageFor(parsed.data, path) ?? options.language;
|
|
1034
|
+
const description = descriptionFor(parsed.data, path);
|
|
1035
|
+
const dates = gitDates.get(path);
|
|
1036
|
+
entries.push({
|
|
1037
|
+
slug,
|
|
1038
|
+
...(dates?.created === undefined ? {} : { created: dates.created }),
|
|
1039
|
+
...(dates?.updated === undefined ? {} : { updated: dates.updated }),
|
|
1040
|
+
...(tags === undefined ? {} : { tags }),
|
|
1041
|
+
...(aliases === undefined ? {} : { aliases }),
|
|
1042
|
+
...(collection === undefined ? {} : { collection }),
|
|
1043
|
+
...(language === undefined ? {} : { language }),
|
|
1044
|
+
...(description === undefined ? {} : { description }),
|
|
1045
|
+
// Derived from the body as authored, and derived **again** from the
|
|
1046
|
+
// rewritten body by {@link resolveCorpusLinks}. Not merely deferred:
|
|
1047
|
+
// deriving them only here was measured wrong in the loudest possible way —
|
|
1048
|
+
// the excerpt of a note containing `[[b]]` still carried the two brackets
|
|
1049
|
+
// after the body no longer did, and `schema.ts`'s unresolved-wikilink rule
|
|
1050
|
+
// rejected the artifact naming the entry rather than the field. Deriving
|
|
1051
|
+
// them only *there* would leave a caller that never runs the traversal
|
|
1052
|
+
// with an untitled entry, which is a second way to be wrong. So both, and
|
|
1053
|
+
// the second overwrites the first.
|
|
1054
|
+
title,
|
|
1055
|
+
excerpt: excerptFor(parsed.body),
|
|
1056
|
+
markdown: parsed.body,
|
|
1057
|
+
outgoing: [],
|
|
1058
|
+
backlinks: [],
|
|
1059
|
+
});
|
|
1060
|
+
// The frontmatter title, kept apart from the derived one: the traversal
|
|
1061
|
+
// re-derives a title from the rewritten body, and it must not overwrite a
|
|
1062
|
+
// title the author wrote down.
|
|
1063
|
+
if (titleDeclared)
|
|
1064
|
+
declaredTitles.add(slug);
|
|
1065
|
+
// The path a link resolves *from*, kept beside the entry it produced rather
|
|
1066
|
+
// than inside it: `slug` is a public route and `path` is a host-relative
|
|
1067
|
+
// filename, and `schema.ts` rejects an artifact carrying an unknown field
|
|
1068
|
+
// precisely so a source path cannot ride into `dist/` on one.
|
|
1069
|
+
pathBySlug.set(slug, path);
|
|
1070
|
+
}
|
|
1071
|
+
// After the walk, because a pattern's verdict is "did it match any discovered
|
|
1072
|
+
// path", and that is not known until every path has been offered to it. Before
|
|
1073
|
+
// the artifact is written, because a typo'd exclusion is exactly the fault that
|
|
1074
|
+
// must stop a publication rather than be reported after one.
|
|
1075
|
+
const idle = matched.flatMap((hit, index) => (hit ? [] : [index]));
|
|
1076
|
+
if (idle.length > 0) {
|
|
1077
|
+
const source = options.excludeSource;
|
|
1078
|
+
const where = source !== undefined && CONFIG_LOCATION.test(source) ? source : 'the exclude list';
|
|
1079
|
+
// The index and the config location, never the pattern's text: a
|
|
1080
|
+
// gitignore-style pattern may be a bare path, so `exclude[3]` on a public log
|
|
1081
|
+
// and the text in the report is the split TK-25 §2.3 requires. The user opens
|
|
1082
|
+
// their own file at that index and reads their own pattern there.
|
|
1083
|
+
throw new BuildFailure('exclusion-pattern-matched-nothing', `${where} — ${idle.map((index) => `exclude[${index}]`).join(', ')} matched 0 files. ` +
|
|
1084
|
+
'A pattern that matches nothing is usually a typo, and a mistyped exclusion ' +
|
|
1085
|
+
'publishes what it was meant to withhold. Correct it, or delete it.', idle.map((index) => `exclude[${index}] = ${JSON.stringify(patterns[index])}`).join('; '));
|
|
1086
|
+
}
|
|
1087
|
+
// The exclusion typo wins when both faults exist: it could expose content,
|
|
1088
|
+
// while a reserved slug can only stop the build. Report every reserved path
|
|
1089
|
+
// together once the higher-risk configuration fault is clear.
|
|
1090
|
+
if (reserved.length > 0) {
|
|
1091
|
+
throw new BuildFailure('reserved-slug', `${reserved.length} published ${reserved.length === 1 ? 'note uses' : 'notes use'} reserved route slugs`, reserved
|
|
1092
|
+
.map(({ path, slug }) => `${path}: slug "${slug}" is reserved by the site; rename the source file or its directory`)
|
|
1093
|
+
.join('\n'));
|
|
1094
|
+
}
|
|
1095
|
+
// No alias check runs here. `docs/core-design/content-semantics.md` owns the
|
|
1096
|
+
// rule: aliases are display/search/preview metadata and "not alternate
|
|
1097
|
+
// resolver targets, routes, globally unique names, or graph nodes", so two
|
|
1098
|
+
// notes may carry the same alias and an alias may equal another note's slug.
|
|
1099
|
+
// The accepted YAML order still round-trips per note.
|
|
1100
|
+
return {
|
|
1101
|
+
entries,
|
|
1102
|
+
counts: { discovered: paths.length, published: entries.length, dropped: dropped.length },
|
|
1103
|
+
dropped,
|
|
1104
|
+
paths,
|
|
1105
|
+
pathBySlug,
|
|
1106
|
+
declaredTitles,
|
|
1107
|
+
};
|
|
1108
|
+
}
|
|
1109
|
+
const defaultRouteForSlug = (slug) => `/${slug}/`;
|
|
1110
|
+
/**
|
|
1111
|
+
* Discoveries this process has already resolved links for.
|
|
1112
|
+
*
|
|
1113
|
+
* A `WeakSet` rather than a flag on {@link Discovery}: the flag would be a field
|
|
1114
|
+
* of the returned shape, which every caller can see and set, and this is an
|
|
1115
|
+
* invariant of the module rather than a property of the data. Weak so a
|
|
1116
|
+
* long-running process holding many discoveries does not retain them.
|
|
1117
|
+
*/
|
|
1118
|
+
const resolved = new WeakSet();
|
|
1119
|
+
/**
|
|
1120
|
+
* Resolve every link in every published note, in one traversal per note.
|
|
1121
|
+
*
|
|
1122
|
+
* **A separate exported step rather than the tail of {@link discover}, for the
|
|
1123
|
+
* same reason {@link writeArtifact} is one: a caller has to be able to put its
|
|
1124
|
+
* own work between the two.** It also keeps the walk free of a parser — see the
|
|
1125
|
+
* dynamic import below — which TK-26's cross-platform gate depends on.
|
|
1126
|
+
*
|
|
1127
|
+
* **Resolution runs over the full file set**, every path the walk classified and
|
|
1128
|
+
* not only the published ones, so a link to an excluded note resolves to *that
|
|
1129
|
+
* note* and is reported as a publication-boundary event rather than as a broken
|
|
1130
|
+
* link. Those are different mistakes with different fixes, and the excluded-note
|
|
1131
|
+
* case is the mistyped-exclusion hazard seen from the other side. The full set
|
|
1132
|
+
* does not exist until the walk finishes, which is why this cannot run inside
|
|
1133
|
+
* the loop that builds it.
|
|
1134
|
+
*
|
|
1135
|
+
* Entries are rewritten **in place**, and each `outgoing` comes from the same
|
|
1136
|
+
* traversal that produced its rewrite — see `scripts/resolve-links.ts` for why
|
|
1137
|
+
* that is one walk rather than two. `backlinks` is derived here as the exact
|
|
1138
|
+
* inverse, because `checkCorpus` rejects an artifact where it is not.
|
|
1139
|
+
*
|
|
1140
|
+
* @returns Every link that did not simply resolve, sorted by source then line.
|
|
1141
|
+
*/
|
|
1142
|
+
export async function resolveCorpusLinks(discovery, routeForSlug = defaultRouteForSlug) {
|
|
1143
|
+
const { entries, pathBySlug, paths: allPaths, declaredTitles } = discovery;
|
|
1144
|
+
// **Called twice, this would silently corrupt the corpus**, so it refuses.
|
|
1145
|
+
// Measured: a second pass over an already-rewritten body reads `[b](/b/)` as a
|
|
1146
|
+
// link to a route rather than to a file, resolves it to nothing, degrades it
|
|
1147
|
+
// to the text `b`, empties `outgoing` and `backlinks`, and reports a spurious
|
|
1148
|
+
// `unresolved` finding. Every one of those is silent. Rewriting is not
|
|
1149
|
+
// idempotent because it cannot be — the output syntax is the input syntax —
|
|
1150
|
+
// so the guard is the honest shape rather than a defensive flourish.
|
|
1151
|
+
//
|
|
1152
|
+
// A plain `Error` rather than a `BuildFailure`: the closed set of failure
|
|
1153
|
+
// codes exists so a *user's* build can put a stable identifier on a stream
|
|
1154
|
+
// and a name in the report, and this is not a fault of any repository. It is
|
|
1155
|
+
// this module's caller calling it twice, which no corpus can cause and no
|
|
1156
|
+
// user can fix. `failureFor` files it as `unexpected-error` with the stack,
|
|
1157
|
+
// which is what a programmer error should look like in the report.
|
|
1158
|
+
if (resolved.has(discovery)) {
|
|
1159
|
+
throw new Error('link resolution ran twice over one discovery. It rewrites bodies in place, so a ' +
|
|
1160
|
+
'second pass would read its own output as new links and drop them.');
|
|
1161
|
+
}
|
|
1162
|
+
resolved.add(discovery);
|
|
1163
|
+
// Every discovered file, carrying its slug where it has one. A file with no
|
|
1164
|
+
// slug is discoverable and unpublishable, which is exactly what makes
|
|
1165
|
+
// `unpublished` distinguishable from `unresolved`.
|
|
1166
|
+
const slugByPath = new Map([...pathBySlug].map(([slug, path]) => [path, slug]));
|
|
1167
|
+
const corpus = allPaths.map((path) => ({ path, slug: slugByPath.get(path) }));
|
|
1168
|
+
const index = indexCorpus(corpus);
|
|
1169
|
+
// **Imported here rather than at the top of the file, and this is measured
|
|
1170
|
+
// rather than stylistic.** `resolve-links.ts` imports `satteri`, whose parser
|
|
1171
|
+
// is a native binding installed per platform — a win32 install carries
|
|
1172
|
+
// `@bruits/satteri-win32-x64-msvc` and nothing else. TK-26's cross-platform
|
|
1173
|
+
// gate loads *this module* under WSL to compare the two walks, and a static
|
|
1174
|
+
// import made that load throw `Cannot find native binding` before discovery
|
|
1175
|
+
// ran at all, turning a green gate red on a property this ticket does not
|
|
1176
|
+
// touch. The walk needs no parser; only the traversal does.
|
|
1177
|
+
const { resolveLinksIn } = await import("./resolve-links.js");
|
|
1178
|
+
const findings = [];
|
|
1179
|
+
for (const entry of entries) {
|
|
1180
|
+
const path = pathBySlug.get(entry.slug);
|
|
1181
|
+
// Unreachable by construction — `pathBySlug` is written for every entry pushed
|
|
1182
|
+
// — but an entry with no path cannot be resolved *from* anywhere, and
|
|
1183
|
+
// resolving it from the corpus root would silently answer tiers 2 and 5
|
|
1184
|
+
// wrong rather than not answering.
|
|
1185
|
+
if (path === undefined)
|
|
1186
|
+
continue;
|
|
1187
|
+
let result;
|
|
1188
|
+
try {
|
|
1189
|
+
result = resolveLinksIn(entry.markdown, path, index, entry.slug, routeForSlug);
|
|
1190
|
+
}
|
|
1191
|
+
catch (error) {
|
|
1192
|
+
// `applyEdits` refuses overlapping spans, which is a rewritten body that
|
|
1193
|
+
// would be neither of the two things it was built from. The public half
|
|
1194
|
+
// stays nameless — a withheld note's path is the disclosure — and the
|
|
1195
|
+
// report gets the file.
|
|
1196
|
+
throw new BuildFailure('link-rewrite-conflict', 'a note\'s links could not be rewritten unambiguously. See the report for the file.', `${path}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1197
|
+
}
|
|
1198
|
+
entry.markdown = result.markdown;
|
|
1199
|
+
// Sorted, because `checkCorpus` requires ascending order and the traversal
|
|
1200
|
+
// emits document order. Without this, `See [[zebra]] and [[apple]].` fails
|
|
1201
|
+
// the build outright — measured on the shipped binary, exit 1 with
|
|
1202
|
+
// `outgoing: must be sorted in ascending order`, while the same corpus with
|
|
1203
|
+
// the two links swapped exits 0. Nobody writes prose in slug-alphabetical
|
|
1204
|
+
// order, so this was a first build a stranger could not get past.
|
|
1205
|
+
//
|
|
1206
|
+
// The *page* still renders links in document order: this array is the edge
|
|
1207
|
+
// set, not the body, and the body is `result.markdown` above.
|
|
1208
|
+
entry.outgoing = [...result.outgoing].sort(compareSlugs);
|
|
1209
|
+
// Re-derived from the rewritten body. A title and an excerpt are
|
|
1210
|
+
// projections of what was published, and taking them from the pre-traversal
|
|
1211
|
+
// text put an unresolved `[[wikilink]]` into the excerpt of a body that no
|
|
1212
|
+
// longer had one — which `schema.ts` rejects, correctly. A title the author
|
|
1213
|
+
// wrote in frontmatter is never overwritten; only a derived one is.
|
|
1214
|
+
if (!declaredTitles.has(entry.slug))
|
|
1215
|
+
entry.title = titleFor(entry.markdown, entry.slug);
|
|
1216
|
+
entry.excerpt = excerptFor(entry.markdown);
|
|
1217
|
+
findings.push(...result.findings);
|
|
1218
|
+
}
|
|
1219
|
+
// **The mechanical inverse, and why it is here rather than left to TK-28.**
|
|
1220
|
+
// `checkCorpus` requires `backlinks` to be the exact inverse of `outgoing`
|
|
1221
|
+
// and computes precisely this list to compare against. So the moment
|
|
1222
|
+
// `outgoing` stopped being empty, an artifact without this was rejected —
|
|
1223
|
+
// measured on two notes and one link: `backlinks: must be the exact inverse
|
|
1224
|
+
// of outgoing links (expected [a], got [])`. A producer emitting half an edge
|
|
1225
|
+
// set makes every repository with a link between two notes unbuildable, which
|
|
1226
|
+
// is the ordinary case for the corpora this tool exists to serve.
|
|
1227
|
+
//
|
|
1228
|
+
// This is the derivation only. What a *page* does with a backlink — the
|
|
1229
|
+
// aside, the hover preview, the "links to this note" heading — is TK-28's,
|
|
1230
|
+
// and none of it is decided here.
|
|
1231
|
+
//
|
|
1232
|
+
// Sorted, because the contract compares sorted lists and an unsorted one
|
|
1233
|
+
// would make the artifact's bytes depend on entry order for no visible reason.
|
|
1234
|
+
const backlinks = new Map(entries.map((entry) => [entry.slug, []]));
|
|
1235
|
+
for (const entry of entries) {
|
|
1236
|
+
for (const target of entry.outgoing)
|
|
1237
|
+
backlinks.get(target)?.push(entry.slug);
|
|
1238
|
+
}
|
|
1239
|
+
for (const entry of entries)
|
|
1240
|
+
entry.backlinks = (backlinks.get(entry.slug) ?? []).sort(compareSlugs);
|
|
1241
|
+
// Sorted by where a reader would look for them, so a report reads identically
|
|
1242
|
+
// however the walk found the notes. One comparator, exported by the module
|
|
1243
|
+
// that owns the row type, because the report sorts them again on the way to
|
|
1244
|
+
// disk.
|
|
1245
|
+
return findings.sort(bySourceThenLine);
|
|
1246
|
+
}
|
|
1247
|
+
/**
|
|
1248
|
+
* Validate what discovery produced, and write it where the build will read it.
|
|
1249
|
+
*
|
|
1250
|
+
* Split from {@link discover} rather than taking a callback, so the caller can
|
|
1251
|
+
* put its own work between the two — which is exactly what the report needs: a
|
|
1252
|
+
* caller records the counts and the dropped rows, and only then asks for the
|
|
1253
|
+
* artifact. A callback would have hidden that ordering inside this module,
|
|
1254
|
+
* where nothing depends on it.
|
|
1255
|
+
*
|
|
1256
|
+
* @param options.includeEdges When false, `outgoing`/`backlinks` are validated
|
|
1257
|
+
* against the in-memory producer result but not written: the packaged target's
|
|
1258
|
+
* relation authority is the finalized snapshot, and the caller builds that
|
|
1259
|
+
* from this same in-memory `Discovery` before Astro runs. Default true, which
|
|
1260
|
+
* is the fixture/committed-corpus path.
|
|
1261
|
+
*/
|
|
1262
|
+
export async function writeArtifact(discovery, destination, options = {}) {
|
|
1263
|
+
if (discovery.entries.length === 0) {
|
|
1264
|
+
// Thrown here rather than at the end of `discover`, and that ordering is
|
|
1265
|
+
// load-bearing: a directory holding only a `.pdf` and a filename that
|
|
1266
|
+
// slugifies to nothing is exactly the corpus the report exists for, and
|
|
1267
|
+
// throwing before the caller could record the counts left that run's report
|
|
1268
|
+
// saying `aborted` with three zeroes — the one shape a reader cannot act on.
|
|
1269
|
+
//
|
|
1270
|
+
// No path in the message: the content directory is a host path the user did
|
|
1271
|
+
// not type in this form, and this reaches a workflow log. The flag spelling
|
|
1272
|
+
// is a literal of this tool's own source, identical in every run.
|
|
1273
|
+
throw new BuildFailure('no-markdown-found', 'no Markdown found — this build reads *.md from the directory named by --content');
|
|
1274
|
+
}
|
|
1275
|
+
// A source literal, not the content directory: `ContentValidationError`
|
|
1276
|
+
// prefixes its message with this, and that message reaches a stream.
|
|
1277
|
+
const artifact = validateArtifact({ version: 1, entries: discovery.entries }, 'content directory');
|
|
1278
|
+
const serialized = options.includeEdges === false
|
|
1279
|
+
? {
|
|
1280
|
+
version: artifact.version,
|
|
1281
|
+
entries: artifact.entries.map(({ outgoing: _outgoing, backlinks: _backlinks, ...entry }) => entry),
|
|
1282
|
+
}
|
|
1283
|
+
: artifact;
|
|
1284
|
+
await mkdir(dirname(destination), { recursive: true });
|
|
1285
|
+
await writeFile(destination, `${JSON.stringify(serialized, null, 2)}\n`, 'utf8');
|
|
1286
|
+
}
|