@twango/twiki 0.0.0-stage → 0.1.0-rc.0

package/site/tooling/assets.ts

@@ -0,0 +1,323 @@
+/** Sync assets, emit navigation, and report unresolved content links for prerendering. */
+
+import {
+	copyFileSync,
+	cpSync,
+	existsSync,
+	linkSync,
+	mkdirSync,
+	readdirSync,
+	readFileSync,
+	rmSync,
+	statSync,
+	unlinkSync
+} from 'node:fs';
+import path from 'node:path';
+import {
+	staticDirectory,
+	courseFilesDirectory,
+	compiledAssetsDirectory
+} from './lib/edition-paths.js';
+import { writeChanged } from './lib/output';
+import { writeTextExports } from './lib/text-exports';
+import { buildCourseFiles } from './lib/course-files';
+import { addNotebookNavigation } from '../src/lib/notebook-nav';
+import type { CourseFile } from '../src/lib/files';
+import { runtimeDataAssets, runtimeDataModuleSource } from './lib/runtime-data-modules';
+import {
+	contentEntries,
+	displayRoute,
+	getManifest,
+	resetContentCache,
+	navTree,
+	loadPage,
+	pageSlugForRoute
+} from '../src/lib/server/content';
+
+export async function prepareAssets(changedInputs?: Set<string>, development = false) {
+	resetContentCache();
+	const SITE_DIR = path.resolve(import.meta.dirname, '..');
+	const GENERATED = path.join(SITE_DIR, 'build/generated');
+	const STATIC_DIR = staticDirectory(SITE_DIR);
+	mkdirSync(STATIC_DIR, { recursive: true });
+	if (process.env.VITE_PUBLIC_EDITION === 'true') {
+		for (const name of ['fonts', 'favicon.png']) {
+			const source = path.join(SITE_DIR, 'static', name);
+			if (existsSync(source)) cpSync(source, path.join(STATIC_DIR, name), { recursive: true });
+		}
+	}
+	if (!existsSync(path.join(GENERATED, 'public-assets.json'))) {
+		mkdirSync(GENERATED, { recursive: true });
+		writeChanged(path.join(GENERATED, 'public-assets.json'), '{}');
+	}
+
+	if (!existsSync(path.join(GENERATED, 'content-manifest.json'))) {
+		throw new Error('Content must be compiled before preparing assets');
+	}
+
+	const manifest = getManifest();
+	await buildCourseFiles(
+		SITE_DIR,
+		new Set(Object.keys(manifest.pages)),
+		changedInputs,
+		development
+	);
+	if (process.env.VITE_PUBLIC_EDITION === 'true') {
+		const entries = JSON.parse(
+			readFileSync(path.join(SITE_DIR, 'src/lib/generated/file-entries.json'), 'utf8')
+		) as { course: string; file: string }[];
+		manifest.fileCourses = entries
+			.filter((entry) => entry.file === '')
+			.map((entry) => entry.course);
+		for (const course of manifest.fileCourses) {
+			if (!manifest.folders.includes(course)) manifest.folders.push(course);
+			if (!manifest.tree.children.some((node) => node.slug === course))
+				manifest.tree.children.push({
+					name: course,
+					slug: course,
+					title: course,
+					children: [],
+					pages: []
+				});
+		}
+		writeChanged(path.join(GENERATED, 'content-manifest.json'), JSON.stringify(manifest));
+	}
+	const WARN_BYTES = 20 * 1024 * 1024;
+	const FAIL_BYTES = 25 * 1024 * 1024;
+
+	// ---------------------------------------------------------------------------
+	// 1. Asset sync into static/
+	// ---------------------------------------------------------------------------
+
+	const htmlAssetSet = new Set(manifest.htmlAssets);
+	const srcRoot = compiledAssetsDirectory(SITE_DIR);
+
+	/** rel path in build/generated/assets → rel path in static/. */
+	function destRel(rel: string): string {
+		return htmlAssetSet.has(rel) ? `${rel}.html` : rel;
+	}
+
+	function* walk(dir: string): Generator<string> {
+		for (const entry of readdirSync(dir, { withFileTypes: true })) {
+			const p = path.join(dir, entry.name);
+			if (entry.isDirectory()) yield* walk(p);
+			else yield p;
+		}
+	}
+
+	let copied = 0;
+	let kept = 0;
+	let oversize = false;
+	const wanted = new Set<string>(Object.values(runtimeDataAssets));
+
+	const sources = [
+		...[...walk(srcRoot)]
+			.filter((src) => !path.relative(srcRoot, src).startsWith('_files' + path.sep))
+			.map((src) => ({ src, rel: path.relative(srcRoot, src) })),
+		...[...walk(courseFilesDirectory(SITE_DIR))].map((src) => ({
+			src,
+			rel: path.join('_files', path.relative(courseFilesDirectory(SITE_DIR), src))
+		}))
+	];
+	for (const { src, rel: relative } of sources) {
+		const rel = relative.split(path.sep).join('/');
+		const out = destRel(rel);
+		wanted.add(out);
+		const dest = path.join(STATIC_DIR, out);
+		const srcStat = statSync(src);
+		if (srcStat.size > FAIL_BYTES) {
+			console.error(
+				`FAIL: asset ${rel} is ${(srcStat.size / 1048576).toFixed(1)} MiB (>25 MiB Workers cap)`
+			);
+			oversize = true;
+		} else if (srcStat.size > WARN_BYTES) {
+			console.warn(`warn: asset ${rel} is ${(srcStat.size / 1048576).toFixed(1)} MiB (>20 MiB)`);
+		}
+		if (existsSync(dest)) {
+			const destStat = statSync(dest);
+			if (
+				destStat.ino === srcStat.ino ||
+				(destStat.size === srcStat.size && readFileSync(dest).equals(readFileSync(src)))
+			) {
+				kept++;
+				continue;
+			}
+			unlinkSync(dest);
+		}
+		mkdirSync(path.dirname(dest), { recursive: true });
+		try {
+			linkSync(src, dest); // hardlink: free, instant
+		} catch {
+			copyFileSync(src, dest);
+		}
+		copied++;
+	}
+	if (oversize) throw new Error('Assets exceed the deployment size limit');
+
+	for (const file of writeTextExports(
+		STATIC_DIR,
+		Object.keys(manifest.pages).map(loadPage),
+		wanted
+	))
+		wanted.add(file);
+
+	// Prune stale synced files (never touch the hand-placed statics).
+	const KEEP = new Set(['.gitignore', 'favicon.png']);
+	let pruned = 0;
+	for (const file of walk(STATIC_DIR)) {
+		const rel = path.relative(STATIC_DIR, file).split(path.sep).join('/');
+		if (KEEP.has(rel) || rel.startsWith('fonts/') || rel.startsWith('_og/')) continue;
+		if (!wanted.has(rel)) {
+			unlinkSync(file);
+			pruned++;
+		}
+	}
+	// Remove now-empty directories.
+	function pruneEmptyDirs(dir: string): boolean {
+		let empty = true;
+		for (const entry of readdirSync(dir, { withFileTypes: true })) {
+			const p = path.join(dir, entry.name);
+			if (entry.isDirectory()) {
+				if (pruneEmptyDirs(p)) rmSync(p, { recursive: true });
+				else empty = false;
+			} else empty = false;
+		}
+		return empty;
+	}
+	pruneEmptyDirs(STATIC_DIR);
+	console.log(`assets: pruned ${pruned} stale files`);
+
+	console.log(`assets: ${copied} synced, ${kept} up-to-date, ${wanted.size} total`);
+
+	// ---------------------------------------------------------------------------
+	// 2. Nav tree module
+	// ---------------------------------------------------------------------------
+
+	const navOut = path.join(SITE_DIR, 'src/lib/generated/nav.json');
+	mkdirSync(path.dirname(navOut), { recursive: true });
+	const navigation = navTree();
+	const fileIndexDirectory = path.join(STATIC_DIR, '_files/index');
+	for (const name of existsSync(fileIndexDirectory) ? readdirSync(fileIndexDirectory) : []) {
+		if (!name.endsWith('.json')) continue;
+		const files: CourseFile[] = JSON.parse(
+			readFileSync(path.join(STATIC_DIR, '_files/index', name), 'utf8')
+		);
+		addNotebookNavigation(navigation, name.slice(0, -5), files);
+	}
+	for (const node of navigation) {
+		if (manifest.fileCourses?.includes(node.segment) && !node.children.length)
+			node.children.push({
+				title: 'Files',
+				segment: 'files',
+				route: `/${node.segment}/files`,
+				children: []
+			});
+	}
+	writeChanged(navOut, JSON.stringify(navigation));
+	for (const [input, asset] of Object.entries(runtimeDataAssets)) {
+		const data = JSON.parse(readFileSync(path.join(path.dirname(navOut), input), 'utf8'));
+		writeChanged(path.join(STATIC_DIR, asset), runtimeDataModuleSource(data));
+	}
+	console.log(`nav: wrote src/lib/generated/nav.json`);
+
+	// ---------------------------------------------------------------------------
+	// 3. Expected-404 list for the strict prerender crawler
+	// ---------------------------------------------------------------------------
+
+	const valid = new Set<string>(['/']);
+	for (const route of contentEntries()) if (route !== '') valid.add(`/${route}`);
+	valid.add('/tags');
+	for (const tag of Object.keys(manifest.tags)) valid.add(`/tags/${tag}`);
+	for (const a of manifest.assets) valid.add(`/${a}`);
+	// html assets are linkable both extensionless (live URL) and with .html
+	for (const h of manifest.htmlAssets) {
+		valid.add(`/${h}`);
+		valid.add(`/${h}.html`);
+	}
+	valid.add('/index.xml');
+	valid.add('/sitemap.xml');
+	valid.add('/404');
+	valid.add('/favicon.png');
+
+	const expected404 = new Set<string>();
+	const attrRe = /(?:href|src)="([^"]*)"/g;
+	for (const slug of Object.keys(manifest.pages)) {
+		const doc = loadPage(slug);
+		for (const match of doc.html.matchAll(attrRe)) {
+			const raw = match[1];
+			if (!raw.startsWith('/') || raw.startsWith('//')) continue; // internal only
+			let p = raw.split('#')[0].split('?')[0];
+			if (p.length > 1 && p.endsWith('/')) p = p.slice(0, -1);
+			try {
+				p = decodeURI(p);
+			} catch {
+				/* keep raw */
+			}
+			if (p === '') p = '/';
+			if (!valid.has(p)) expected404.add(p);
+		}
+	}
+
+	const sorted = [...expected404].sort();
+	writeChanged(path.join(GENERATED, 'expected-404.json'), JSON.stringify(sorted, null, '\t'));
+	console.log(
+		`expected-404: ${sorted.length} unresolved content link targets reported to the crawler`
+	);
+
+	// Missing content anchors are reported without blocking publication.
+	const pageIds = new Map<string, Set<string>>();
+	const idRe = /id="([^"]*)"/g;
+	for (const slug of Object.keys(manifest.pages)) {
+		const doc = loadPage(slug);
+		const ids = new Set<string>();
+		for (const m of doc.html.matchAll(idRe)) ids.add(m[1]);
+		pageIds.set(slug, ids);
+	}
+
+	const missingAnchor = new Set<string>();
+	const anchorRe = /(?:href|src)="((?:\/[^"#]*)?#[^"]+)"/g;
+	for (const slug of Object.keys(manifest.pages)) {
+		const doc = loadPage(slug);
+		for (const m of doc.html.matchAll(anchorRe)) {
+			const raw = m[1];
+			if (raw.startsWith('//')) continue;
+			const hashIdx = raw.indexOf('#');
+			let target = raw.slice(0, hashIdx) || `/${displayRoute(slug)}`;
+			let id = raw.slice(hashIdx + 1);
+			if (id.startsWith('page=')) continue; // PDF page anchors pass raw
+			try {
+				target = decodeURI(target);
+				id = decodeURI(id);
+			} catch {
+				/* keep raw */
+			}
+			if (target.length > 1 && target.endsWith('/')) target = target.slice(0, -1);
+			// Map the URL path the prerenderer renders (`/` → home, `/dir` → folder
+			// index) to the canonical page slug, exactly like the catch-all route.
+			const route = target === '/' ? '' : target.slice(1);
+			const targetSlug = pageSlugForRoute(route);
+			if (!targetSlug) continue; // not a page route (asset/folder-listing/404 elsewhere)
+			const ids = pageIds.get(targetSlug);
+			if (!ids) continue;
+			// The prerenderer keys missing-id errors by the rendered path, not the slug.
+			if (!ids.has(id)) missingAnchor.add(`${target}#${id}`);
+		}
+	}
+
+	const sortedIds = [...missingAnchor].sort();
+	writeChanged(
+		path.join(GENERATED, 'expected-missing-id.json'),
+		JSON.stringify(sortedIds, null, '\t')
+	);
+	console.log(
+		`expected-missing-id: ${sortedIds.length} unresolved content anchors reported to the crawler`
+	);
+	console.log(
+		`routes: ${contentEntries().length} content + ${Object.keys(manifest.tags).length + 1} tag + 2 xml + /404`
+	);
+
+	if (!development) {
+		const { buildSocialImages } = await import('./lib/social-images');
+		await buildSocialImages(SITE_DIR, manifest);
+	}
+}

package/site/tooling/benchmark.ts

@@ -0,0 +1,191 @@
+import { spawn, spawnSync } from 'node:child_process';
+import { createWriteStream, mkdirSync, writeFileSync } from 'node:fs';
+import { createServer } from 'node:net';
+import { availableParallelism, cpus, totalmem } from 'node:os';
+import path from 'node:path';
+
+const [mode, requestedLabel] = process.argv.slice(2);
+if (mode !== 'build' && mode !== 'dev') {
+	console.error('Usage: bun tooling/benchmark.ts <build|dev> [label]');
+	process.exit(2);
+}
+const label = requestedLabel ?? `${mode}-${new Date().toISOString().replaceAll(':', '-')}`;
+if (!/^[\w.-]+$/.test(label) || label === '.' || label === '..') {
+	throw new Error('Label must contain only letters, numbers, underscores, dots, or hyphens');
+}
+const timeoutSeconds = Number(process.env.BENCHMARK_TIMEOUT_SECONDS ?? '300');
+if (!Number.isFinite(timeoutSeconds) || timeoutSeconds <= 0) {
+	throw new Error('BENCHMARK_TIMEOUT_SECONDS must be positive');
+}
+const site = path.resolve(import.meta.dirname, '..');
+const directory = path.join(site, 'build/benchmarks');
+mkdirSync(directory, { recursive: true });
+const logPath = path.join(directory, `${label}.log`);
+const resultPath = path.join(directory, `${label}.json`);
+const log = createWriteStream(logPath);
+const nodeRuntime = spawnSync('node', ['--version'], { encoding: 'utf8', timeout: 5000 });
+const actualNodeVersion = nodeRuntime.status === 0 ? nodeRuntime.stdout.trim() : undefined;
+const started = performance.now();
+const elapsed = () => Math.round((performance.now() - started) / 10) / 100;
+
+async function freePort() {
+	const server = createServer();
+	await new Promise<void>((resolve, reject) => {
+		server.once('error', reject);
+		server.listen(0, '127.0.0.1', resolve);
+	});
+	const address = server.address();
+	if (!address || typeof address === 'string') throw new Error('No benchmark port allocated');
+	await new Promise<void>((resolve, reject) =>
+		server.close((error) => (error ? reject(error) : resolve()))
+	);
+	return address.port;
+}
+
+const port = mode === 'dev' ? await freePort() : undefined;
+const args =
+	mode === 'build'
+		? ['run', process.env.WISCONSIN_BENCHMARK_STATIC === 'true' ? 'build:static' : 'build:all']
+		: ['run', 'dev', '--host', '127.0.0.1', '--port', String(port), '--strictPort'];
+const command = process.versions.bun ? process.execPath : 'bun';
+const child = spawn(command, args, {
+	cwd: site,
+	env: { ...process.env, FORCE_COLOR: '0', NO_COLOR: '1' },
+	detached: process.platform !== 'win32',
+	stdio: ['ignore', 'pipe', 'pipe']
+});
+let readySeconds: number | undefined;
+let httpSeconds: number | undefined;
+let failure: string | undefined;
+let closed = false;
+let exitCode: number | null = null;
+let exitSignal: string | null = null;
+let tail = '';
+for (const stream of [child.stdout, child.stderr]) {
+	stream.on('data', (chunk: Buffer) => {
+		log.write(chunk);
+		process.stdout.write(chunk);
+		tail = (tail + chunk.toString()).slice(-4096);
+		if (
+			mode === 'dev' &&
+			readySeconds === undefined &&
+			/\bready in\s+[\d.]+\s*(?:ms|s)/.test(tail)
+		) {
+			readySeconds = elapsed();
+		}
+	});
+}
+const completion = new Promise<void>((resolve) => {
+	child.on('error', (error) => {
+		failure = error.message;
+	});
+	child.on('close', (code, signal) => {
+		closed = true;
+		exitCode = code;
+		exitSignal = signal;
+		resolve();
+	});
+});
+function stop() {
+	if (!child.pid) return;
+	try {
+		if (process.platform === 'win32') child.kill('SIGTERM');
+		else process.kill(-child.pid, 'SIGTERM');
+	} catch (error) {
+		if ((error as NodeJS.ErrnoException).code !== 'ESRCH') throw error;
+	}
+}
+function interrupt() {
+	failure = 'Benchmark interrupted';
+	gracefulStop();
+}
+process.once('SIGINT', interrupt);
+process.once('SIGTERM', interrupt);
+const timeout = setTimeout(() => {
+	failure = `Timed out after ${timeoutSeconds}s`;
+	gracefulStop();
+}, timeoutSeconds * 1000);
+let killTimeout: ReturnType<typeof setTimeout> | undefined;
+function gracefulStop() {
+	if (killTimeout) return;
+	stop();
+	killTimeout = setTimeout(
+		() => {
+			if (!child.pid) return;
+			try {
+				if (process.platform === 'win32') child.kill('SIGKILL');
+				else process.kill(-child.pid, 'SIGKILL');
+			} catch (error) {
+				if ((error as NodeJS.ErrnoException).code !== 'ESRCH') throw error;
+			}
+		},
+		mode === 'build' &&
+			['parallel', 'isolated-serial'].includes(process.env.WISCONSIN_BUILD_MODE ?? '')
+			? 15_000
+			: 2_000
+	);
+}
+try {
+	if (mode === 'dev') {
+		while (!closed && !failure && httpSeconds === undefined) {
+			try {
+				const response = await fetch(`http://127.0.0.1:${port}/`, {
+					signal: AbortSignal.timeout(2000),
+					redirect: 'manual'
+				});
+				const html = await response.text();
+				if (
+					response.status === 200 &&
+					response.headers.get('content-type')?.includes('text/html') &&
+					/<h1\b[^>]*>[\s\S]*?<\/h1>/i.test(html) &&
+					!html.includes('<vite-error-overlay')
+				) {
+					httpSeconds = elapsed();
+				}
+			} catch {
+				/* The server may still be compiling or binding its port. */
+			}
+			if (httpSeconds === undefined) await new Promise((resolve) => setTimeout(resolve, 100));
+		}
+		if (httpSeconds === undefined && !failure)
+			failure = 'Dev server exited before serving the expected home page';
+		gracefulStop();
+	}
+	await completion;
+	if (mode === 'build' && exitCode !== 0 && !failure)
+		failure = `Build exited with ${exitCode ?? exitSignal}`;
+} finally {
+	clearTimeout(timeout);
+	if (killTimeout) clearTimeout(killTimeout);
+	process.removeListener('SIGINT', interrupt);
+	process.removeListener('SIGTERM', interrupt);
+	await new Promise<void>((resolve) => log.end(resolve));
+}
+const result = {
+	label,
+	mode,
+	recordedAt: new Date().toISOString(),
+	command: [command, ...args],
+	seconds: elapsed(),
+	readySeconds,
+	firstHttp200Seconds: httpSeconds,
+	exitCode,
+	exitSignal,
+	success: !failure,
+	failure,
+	host: {
+		platform: process.platform,
+		architecture: process.arch,
+		bun: process.versions.bun,
+		nodeCompatibilityVersion: process.versions.node,
+		actualNodeVersion,
+		cpu: cpus()[0]?.model,
+		availableCpus: availableParallelism(),
+		memoryBytes: totalmem()
+	},
+	logPath
+};
+writeFileSync(resultPath, JSON.stringify(result, null, 2) + '\n');
+console.log(`Benchmark saved: ${resultPath}`);
+console.log(JSON.stringify(result, null, 2));
+if (failure) process.exitCode = 1;