@twango/twiki 0.0.0-stage → 0.1.1

package/site/tooling/lib/edition-builds.ts

@@ -0,0 +1,184 @@
+import { spawn, type ChildProcess } from 'node:child_process';
+import { closeSync, mkdirSync, openSync, writeSync } from 'node:fs';
+import path from 'node:path';
+
+export type BuildMode = 'serial' | 'parallel' | 'isolated-serial';
+
+export function buildMode(value = process.env.WISCONSIN_BUILD_MODE): BuildMode {
+	if (value === undefined || value === 'serial') return 'serial';
+	if (value === 'parallel' || value === 'isolated-serial') return value;
+	throw new Error('WISCONSIN_BUILD_MODE must be serial, parallel or isolated-serial');
+}
+
+interface BuildOptions {
+	mode: Exclude<BuildMode, 'serial'>;
+	contentRepo: string;
+	applicationVersion: string;
+	logDirectory: string;
+	signal?: AbortSignal;
+	log?: (message: string) => void;
+	command?: string;
+	args?: string[];
+	killGraceMs?: number;
+}
+
+/** Build normal Vite projects in separate process groups; reap both before returning or rejecting. */
+export async function runEditionBuilds(
+	sites: { publicSite: string; fullSite: string },
+	options: BuildOptions
+) {
+	const log = options.log ?? console.log;
+	const processes = new Set<ChildProcess>();
+	const tasks: Promise<number>[] = [];
+	const terminations: Promise<void>[] = [];
+	const terminating = new Set<ChildProcess>();
+	let failure: Error | undefined;
+	mkdirSync(options.logDirectory, { recursive: true, mode: 0o700 });
+
+	function signalGroup(child: ChildProcess, signal: NodeJS.Signals) {
+		if (child.pid === undefined) return;
+		try {
+			if (process.platform === 'win32') child.kill(signal);
+			else process.kill(-child.pid, signal);
+		} catch (error) {
+			if ((error as NodeJS.ErrnoException).code !== 'ESRCH') throw error;
+		}
+	}
+
+	function terminate(child: ChildProcess) {
+		if (terminating.has(child) || child.pid === undefined) return;
+		terminating.add(child);
+		signalGroup(child, 'SIGTERM');
+		// Retain the deadline even if the direct child exits first: its descendants
+		// can ignore SIGTERM or close their inherited pipes before exiting.
+		terminations.push(
+			new Promise((resolve) => {
+				setTimeout(() => {
+					try {
+						signalGroup(child, 'SIGKILL');
+					} finally {
+						resolve();
+					}
+				}, options.killGraceMs ?? 2_000);
+			})
+		);
+	}
+
+	function cancel(error: Error) {
+		failure ??= error;
+		for (const child of processes) terminate(child);
+	}
+	const interrupt = () => cancel(new Error('Edition builds interrupted by SIGINT'));
+	const terminateSignal = () => cancel(new Error('Edition builds interrupted by SIGTERM'));
+	const abort = () =>
+		cancel(new Error('Edition builds aborted', { cause: options.signal?.reason }));
+	process.on('SIGINT', interrupt);
+	process.on('SIGTERM', terminateSignal);
+	options.signal?.addEventListener('abort', abort, { once: true });
+	if (options.signal?.aborted) abort();
+
+	function start(publicEdition: boolean) {
+		if (failure) throw failure;
+		const edition = publicEdition ? 'public' : 'full';
+		const started = performance.now();
+		const file = openSync(path.join(options.logDirectory, `${edition}.log`), 'wx', 0o600);
+		let child: ChildProcess;
+		try {
+			child = spawn(options.command ?? 'bun', options.args ?? ['x', 'vite', 'build'], {
+				cwd: publicEdition ? sites.publicSite : sites.fullSite,
+				detached: process.platform !== 'win32',
+				stdio: ['ignore', 'pipe', 'pipe'],
+				env: {
+					...process.env,
+					NODE_ENV: process.env.NODE_ENV ?? 'production',
+					WISCONSIN_CONTENT_REPO: options.contentRepo,
+					WISCONSIN_APPLICATION_VERSION: options.applicationVersion,
+					VITE_PUBLIC_EDITION: String(publicEdition)
+				}
+			});
+		} catch (error) {
+			closeSync(file);
+			throw error;
+		}
+		processes.add(child);
+		const task = new Promise<number>((resolve, reject) => {
+			let error: Error | undefined;
+			for (const stream of [child.stdout!, child.stderr!]) {
+				let pending = '';
+				stream.setEncoding('utf8');
+				stream.on('data', (chunk: string) => {
+					try {
+						writeSync(file, chunk);
+						pending += chunk;
+						const lines = pending.split('\n');
+						pending = lines.pop()!;
+						for (const line of lines) log(`${edition} | ${line}`);
+					} catch (cause) {
+						error = new Error(`Could not record ${edition} build output`, { cause });
+						cancel(error);
+					}
+				});
+				stream.on('end', () => {
+					try {
+						if (pending) log(`${edition} | ${pending}`);
+					} catch (cause) {
+						error = new Error(`Could not record ${edition} build output`, { cause });
+						cancel(error);
+					}
+				});
+			}
+			child.on('error', (cause) => {
+				error = new Error(`Could not start ${edition} build`, { cause });
+				cancel(error);
+			});
+			child.on('close', (code, signal) => {
+				closeSync(file);
+				if (error || code !== 0) {
+					error ??= new Error(`${edition} build failed (${signal ?? `exit ${code}`})`);
+					cancel(error);
+					reject(error);
+				} else {
+					const seconds = (performance.now() - started) / 1000;
+					try {
+						log(`build: ${edition} completed in ${seconds.toFixed(2)}s`);
+						resolve(seconds);
+					} catch (cause) {
+						error = new Error(`Could not record ${edition} build timing`, { cause });
+						cancel(error);
+						reject(error);
+					}
+				}
+				processes.delete(child);
+			});
+		});
+		tasks.push(task);
+		return task;
+	}
+
+	try {
+		let publicSeconds: number;
+		let fullSeconds: number;
+		if (options.mode === 'parallel') {
+			const results = await Promise.allSettled([start(true), start(false)]);
+			if (failure) throw failure;
+			if (results[0].status === 'rejected') throw results[0].reason;
+			if (results[1].status === 'rejected') throw results[1].reason;
+			publicSeconds = results[0].value;
+			fullSeconds = results[1].value;
+		} else {
+			publicSeconds = await start(true);
+			fullSeconds = await start(false);
+		}
+		if (failure) throw failure;
+		return { publicSeconds, fullSeconds };
+	} catch (error) {
+		cancel(error instanceof Error ? error : new Error(String(error)));
+		throw error;
+	} finally {
+		await Promise.allSettled(tasks);
+		await Promise.all(terminations);
+		process.removeListener('SIGINT', interrupt);
+		process.removeListener('SIGTERM', terminateSignal);
+		options.signal?.removeEventListener('abort', abort);
+	}
+}

package/site/tooling/lib/edition-paths.js

@@ -0,0 +1,29 @@
+import path from 'node:path';
+
+/** Keep one edition from pruning the other edition's compiled source assets. @param {string} site */
+export function compiledAssetsDirectory(site) {
+	return path.join(
+		site,
+		process.env.VITE_PUBLIC_EDITION === 'true'
+			? 'build/generated/public-assets'
+			: 'build/generated/assets'
+	);
+}
+
+/** @param {string} site */
+export function staticDirectory(site) {
+	return path.join(
+		site,
+		process.env.VITE_PUBLIC_EDITION === 'true' ? 'build/generated/public-static' : 'static'
+	);
+}
+
+/** @param {string} site */
+export function courseFilesDirectory(site) {
+	return path.join(
+		site,
+		process.env.VITE_PUBLIC_EDITION === 'true'
+			? 'build/generated/public-files'
+			: 'build/generated/assets/_files'
+	);
+}

package/site/tooling/lib/empty-tag-route.js

@@ -0,0 +1,23 @@
+import { readFileSync } from 'node:fs';
+import path from 'node:path';
+
+/** Empty tag indexes have no concrete dynamic pages for SvelteKit to prerender.
+ * @param {string} site
+ * @param {string[]} routes
+ */
+export function emptyTagRoute(site, routes) {
+	if (routes.length === 0 || !routes.every((route) => route === '/tags/[...tag]')) return false;
+	try {
+		const manifest = JSON.parse(
+			readFileSync(path.join(site, 'build/generated/content-manifest.json'), 'utf8')
+		);
+		return (
+			manifest?.tags !== null &&
+			typeof manifest?.tags === 'object' &&
+			!Array.isArray(manifest.tags) &&
+			Object.keys(manifest.tags).length === 0
+		);
+	} catch {
+		return false;
+	}
+}

package/site/tooling/lib/file-history.ts

@@ -0,0 +1,201 @@
+import { execFileSync } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import { existsSync, mkdirSync, readFileSync, writeFileSync, copyFileSync } from 'node:fs';
+import path from 'node:path';
+import { stageFingerprint } from '../fingerprint';
+import { createRevisionLookup } from './git-revisions';
+import type { CourseFile } from '../../src/lib/files';
+import type { FileBlame, FileChange, FileCommit, FileHistory } from '../../src/lib/file-history';
+
+const historyVersion = await stageFingerprint('history');
+const maxAssetBytes = 25 * 1024 * 1024;
+
+export function parseBlame(output: string): FileBlame {
+	const commits: Record<string, FileCommit> = {};
+	const ranges: FileBlame['ranges'] = [];
+	let commit: FileCommit | undefined;
+	for (const line of output.split('\n')) {
+		const header = /^([a-f0-9]{40,64}) \d+ (\d+) (\d+)$/.exec(line);
+		if (header) {
+			const [, id, start, count] = header;
+			commit = commits[id] ??= { id, author: '', date: '', subject: '' };
+			ranges.push([Number(start), Number(start) + Number(count) - 1, id]);
+		} else if (commit) {
+			if (line.startsWith('author ')) commit.author = line.slice(7);
+			else if (line.startsWith('author-time '))
+				commit.date = new Date(Number(line.slice(12)) * 1000).toISOString();
+			else if (line.startsWith('summary ')) commit.subject = line.slice(8);
+		}
+	}
+	ranges.sort((a, b) => a[0] - b[0]);
+	return { commits, ranges };
+}
+
+export function createFileHistoryBuilder(
+	repo: string,
+	output: string,
+	cache: string,
+	allowed: (file: string) => boolean,
+	policyKey = allowed.toString(),
+	retain?: (file: string) => void,
+	urlPrefix = '/_files/history'
+) {
+	if (!existsSync(path.join(repo, '.git'))) return;
+	const git = (...args: string[]) =>
+		execFileSync('git', ['-C', repo, ...args], { maxBuffer: 128 * 1024 * 1024, encoding: 'utf8' });
+	const head = git('rev-parse', 'HEAD').trim();
+	if (git('rev-parse', '--is-shallow-repository').trim() === 'true')
+		throw new Error(`Full Git history required: ${repo}`);
+	mkdirSync(cache, { recursive: true });
+	mkdirSync(path.join(output, 'history'), { recursive: true });
+	const revisionFor = createRevisionLookup(
+		head,
+		cache,
+		(file) => git('log', '-1', '--first-parent', '--format=%H', head, '--', file).trim() || head,
+		historyVersion
+	);
+	const commits = new Map<string, { commit: FileCommit; parent?: string; changes: string[] }>();
+	function details(id: string) {
+		let cached = commits.get(id);
+		if (cached) return cached;
+		const [author, date, subject, parents] = git(
+			'show',
+			'-s',
+			'--format=%an%x00%aI%x00%s%x00%P',
+			id
+		)
+			.trimEnd()
+			.split('\0');
+		const parent = parents?.split(' ')[0] || undefined;
+		const changes = (
+			parent
+				? git('diff', '--name-status', '-z', '-M', parent, id)
+				: git('diff-tree', '--root', '-r', '--name-status', '-z', '--no-commit-id', id)
+		).split('\0');
+		cached = { commit: { id, author, date, subject }, parent, changes };
+		commits.set(id, cached);
+		return cached;
+	}
+	function publish(name: string, data?: unknown) {
+		const target = path.join(cache, name);
+		if (data !== undefined) {
+			const serialized = typeof data === 'string' ? data : JSON.stringify(data);
+			if (Buffer.byteLength(serialized) > maxAssetBytes) return;
+			writeFileSync(target, serialized);
+		}
+		const destination = path.join(output, 'history', name);
+		retain?.(destination);
+		if (!existsSync(destination)) copyFileSync(target, destination);
+		return `${urlPrefix}/${name}`;
+	}
+	return (file: CourseFile, bytes: Uint8Array) => {
+		// Unrelated commits do not change this file's history or blame.
+		const revision = revisionFor(file.path);
+		const key = createHash('sha256')
+			.update(
+				`${historyVersion}\0${policyKey}\0${urlPrefix}\0${revision}\0${file.kind}\0${file.path}\0`
+			)
+			.update(bytes)
+			.digest('hex');
+		const name = `${key}.json`;
+		const cached = path.join(cache, name);
+		try {
+			const history: FileHistory = JSON.parse(readFileSync(cached, 'utf8'));
+			const assets = [history.blame, ...history.commits.map((commit) => commit.diff)].filter(
+				(url): url is string => Boolean(url)
+			);
+			if (
+				history.revision === revision &&
+				history.commits.every(
+					(commit) => allowed(commit.path) && (!commit.previousPath || allowed(commit.previousPath))
+				) &&
+				assets.every((url) => existsSync(path.join(cache, path.basename(url))))
+			) {
+				for (const url of assets) publish(path.basename(url));
+				return publish(name);
+			}
+		} catch {
+			// Missing, truncated, or incomplete records are rebuilt from Git.
+		}
+		const history: FileHistory = { revision, commits: [] };
+		const workingBlob = execFileSync('git', ['-C', repo, 'hash-object', '--stdin'], {
+			input: bytes,
+			encoding: 'utf8'
+		}).trim();
+		const committedBlob = git('ls-tree', '-z', revision, '--', file.path)
+			.split('\t')[0]
+			?.split(' ')[2];
+		const clean = workingBlob === committedBlob;
+		if (!clean)
+			history.notice =
+				'Working copy differs from the committed file. Blame is unavailable; history shows committed changes only.';
+		let currentPath = file.path;
+		let restricted = false;
+		for (const id of git(
+			'log',
+			'--follow',
+			'--first-parent',
+			'--format=%H',
+			revision,
+			'--',
+			file.path
+		)
+			.trim()
+			.split('\n')
+			.filter(Boolean)) {
+			const { commit, parent, changes } = details(id);
+			let previousPath = currentPath;
+			let added = false;
+			for (let i = 0; i < changes.length; ) {
+				const status = changes[i++];
+				const before = changes[i++];
+				const after = /^[RC]/.test(status) ? changes[i++] : before;
+				if (after !== currentPath) continue;
+				if (status.startsWith('R')) previousPath = before;
+				added = status === 'A';
+			}
+			const change: FileChange = { ...commit, path: currentPath };
+			if (previousPath !== currentPath) change.previousPath = previousPath;
+			if (!allowed(previousPath)) {
+				delete change.previousPath;
+				change.unavailable = 'Earlier history belongs to an excluded path.';
+				history.commits.push(change);
+				restricted = true;
+				break;
+			}
+			if (parent && !added) {
+				const diff = git(
+					'diff',
+					'--no-ext-diff',
+					'--no-textconv',
+					'--no-color',
+					'-M',
+					parent,
+					id,
+					'--',
+					previousPath,
+					currentPath
+				);
+				if (
+					/-----BEGIN [A-Z ]*PRIVATE KEY-----|\b(?:gh[pousr]_[A-Za-z0-9]{30,}|github_pat_[A-Za-z0-9_]{30,})/.test(
+						diff
+					)
+				) {
+					change.unavailable = 'This change contains a potential credential and was not exported.';
+				} else {
+					change.diff = publish(`${key}-${id}.diff`, diff);
+					if (!change.diff) change.unavailable = 'This diff exceeds the hosting size limit.';
+				}
+			} else change.unavailable = 'Initial version. No earlier version to compare.';
+			history.commits.push(change);
+			currentPath = previousPath;
+			if (added) break;
+		}
+		if (restricted) history.notice = 'History stops at an excluded path. Blame is unavailable.';
+		if (clean && !restricted && file.kind === 'text') {
+			const blame = parseBlame(git('blame', '--incremental', '--root', revision, '--', file.path));
+			history.blame = publish(`${key}-blame.json`, blame);
+		}
+		return publish(name, history);
+	};
+}

package/site/tooling/lib/file-icons.ts

@@ -0,0 +1,93 @@
+import { mkdirSync } from 'node:fs';
+import { writeChanged, copyChanged, pruneOutputs } from './output';
+import path from 'node:path';
+import { courseFilesDirectory } from './edition-paths.js';
+import { fileURLToPath } from 'node:url';
+import { generateManifest } from 'material-icon-theme';
+import type { Manifest } from 'material-icon-theme';
+import type { FileIconTheme, IconVariants, FolderIconVariants } from '../../src/lib/file-icons';
+
+const manifest = generateManifest();
+
+function association(
+	key: string,
+	base?: Record<string, string>,
+	override?: Record<string, string>
+) {
+	if (override && Object.hasOwn(override, key)) return override[key];
+	if (base && Object.hasOwn(base, key)) return base[key];
+}
+
+export function resolveIcon(name: string, folder = false, expanded = false, light = false) {
+	const key = name.toLowerCase();
+	const variant: Manifest = light ? (manifest.light ?? {}) : {};
+	if (folder) {
+		return expanded
+			? (association(key, manifest.folderNamesExpanded, variant.folderNamesExpanded) ??
+					variant.folderExpanded ??
+					manifest.folderExpanded!)
+			: (association(key, manifest.folderNames, variant.folderNames) ??
+					variant.folder ??
+					manifest.folder!);
+	}
+	const named = association(key, manifest.fileNames, variant.fileNames);
+	if (named) return named;
+	const parts = key.split('.');
+	for (let i = 1; i < parts.length; i++) {
+		const extension = association(
+			parts.slice(i).join('.'),
+			manifest.fileExtensions,
+			variant.fileExtensions
+		);
+		if (extension) return extension;
+	}
+	return variant.file ?? manifest.file!;
+}
+
+export function buildFileIcons(siteDir: string, files: string[]) {
+	const output = path.join(courseFilesDirectory(siteDir), 'icons');
+	const moduleDir = path.join(siteDir, 'src/lib/generated');
+	const packageDir = path.dirname(
+		fileURLToPath(import.meta.resolve('material-icon-theme/package.json'))
+	);
+	const used = new Set<string>();
+	function variants(name: string, folder = false, expanded = false): IconVariants {
+		const light = resolveIcon(name, folder, expanded, true);
+		const dark = resolveIcon(name, folder, expanded);
+		used.add(light);
+		used.add(dark);
+		return { light, dark };
+	}
+	function folderVariants(name: string): FolderIconVariants {
+		return { ...variants(name, true), expanded: variants(name, true, true) };
+	}
+	const theme: FileIconTheme = {
+		file: variants(''),
+		folder: folderVariants(''),
+		files: Object.create(null),
+		folders: Object.create(null)
+	};
+	for (const file of files) {
+		const parts = file.toLowerCase().split('/');
+		const name = parts.pop()!;
+		theme.files[name] ??= variants(name);
+		for (const folder of parts) theme.folders[folder] ??= folderVariants(folder);
+	}
+	mkdirSync(output, { recursive: true });
+	mkdirSync(moduleDir, { recursive: true });
+	for (const icon of used) {
+		const source = path.basename(manifest.iconDefinitions![icon].iconPath);
+		copyChanged(path.join(packageDir, 'icons', source), path.join(output, `${icon}.svg`));
+	}
+	copyChanged(path.join(packageDir, 'LICENSE'), path.join(output, 'LICENSE.txt'));
+	writeChanged(path.join(moduleDir, 'file-icons.json'), JSON.stringify(theme));
+	pruneOutputs(
+		output,
+		new Set(
+			[...used]
+				.map((icon) => path.join(output, `${icon}.svg`))
+				.concat(path.join(output, 'LICENSE.txt'))
+		)
+	);
+	console.log(`icons: ${used.size} local file and folder icons`);
+}

package/site/tooling/lib/file-policy.ts

@@ -0,0 +1,38 @@
+import { createHash } from 'node:crypto';
+
+const excludedDirectories = new Set([
+	'node_modules',
+	'vendor',
+	'build',
+	'dist',
+	'target',
+	'__pycache__'
+]);
+const excludedNames =
+	/^(?:credentials?|secrets?|id_rsa|id_ed25519)(?:[._-]|$)|\.(?:pem|key|p12|pfx|keystore)$/i;
+
+/** Shared path policy without loading course history or R preview rendering. */
+export function browsablePath(file: string) {
+	return (
+		!file.includes('\\') &&
+		!file.split('/').includes('publish.yaml') &&
+		!/[\x00-\x1f\x7f]/.test(file) &&
+		file
+			.split('/')
+			.every(
+				(segment) =>
+					segment.length > 0 &&
+					!segment.startsWith('.') &&
+					!excludedDirectories.has(segment.toLowerCase()) &&
+					!excludedNames.test(segment)
+			)
+	);
+}
+
+export function fileHistoryPolicyKey() {
+	return createHash('sha256')
+		.update(browsablePath.toString())
+		.update(JSON.stringify([...excludedDirectories].sort()))
+		.update(excludedNames.source)
+		.digest('hex');
+}

package/site/tooling/lib/file-search.js

@@ -0,0 +1,13 @@
+/** File names remain searchable without generating an HTML page for each path.
+ * @param {{ course: string; file: string }} entry
+ */
+export function fileSearchRecord({ course, file }) {
+	const route = `/${encodeURIComponent(course)}/files${file ? '/' + file.split('/').map(encodeURIComponent).join('/') : ''}`;
+	return {
+		url: route,
+		content: `${course} ${file}`,
+		language: 'en',
+		meta: { title: file.split('/').at(-1) || course },
+		filters: { course: [course] }
+	};
+}

package/site/tooling/lib/git-revisions.ts

@@ -0,0 +1,41 @@
+import { appendFileSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import path from 'node:path';
+
+// Keep Git's path-limited history semantics, including merges and renames.
+// A course HEAD identifies immutable answers, even when its working tree is dirty.
+export function createRevisionLookup(
+	head: string,
+	cache: string,
+	lookup: (file: string) => string,
+	version = 'v1'
+) {
+	const target = path.join(cache, `revisions-${version}-${head}.jsonl`);
+	const revisions = new Map<string, string>();
+	try {
+		for (const line of readFileSync(target, 'utf8').split('\n').filter(Boolean)) {
+			const entry: unknown = JSON.parse(line);
+			if (
+				!Array.isArray(entry) ||
+				entry.length !== 2 ||
+				typeof entry[0] !== 'string' ||
+				typeof entry[1] !== 'string' ||
+				!/^[a-f0-9]{40,64}$/.test(entry[1])
+			)
+				throw new Error('Invalid revision cache');
+			revisions.set(entry[0], entry[1]);
+		}
+	} catch {
+		// A partial write or damaged cache must never supply a history revision.
+		revisions.clear();
+		mkdirSync(cache, { recursive: true });
+		writeFileSync(target, '');
+	}
+	return (file: string) => {
+		const cached = revisions.get(file);
+		if (cached) return cached;
+		const revision = lookup(file);
+		revisions.set(file, revision);
+		appendFileSync(target, JSON.stringify([file, revision]) + '\n');
+		return revision;
+	};
+}