@twango/twiki 0.0.0-stage → 0.1.1

package/site/tooling/deploy.ts

@@ -0,0 +1,112 @@
+import { readFile, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { deploymentError, readDeploymentConfig } from './lib/deployment-config';
+import {
+	captureAssetInventory,
+	assetPath,
+	SHA256,
+	readCurrentDescriptorProvenance,
+	assetContentType
+} from './lib/deployment-manifest';
+import {
+	buildDeploymentSnapshot,
+	loadPreparedWorker,
+	prepareWorker,
+	type DeploymentSnapshot
+} from './lib/deployment-prepare';
+import { CloudflareDeploymentAPI, uploadDeployment } from './lib/deployment-upload';
+import { captureAssetHashes, flushAssetHashCache } from './lib/asset-hash-cache';
+
+/** No implicit network operation: --prepare is offline; --upload is an explicit production mutation. */
+export async function deploymentCLI(args: string[]): Promise<void> {
+	const options = new Map<string, string>();
+	let mode: '--prepare' | '--upload' | undefined;
+	for (let i = 0; i < args.length; i++) {
+		const argument = args[i];
+		if (argument === '--prepare' || argument === '--upload') {
+			if (mode) deploymentError('select exactly one deployment mode');
+			mode = argument;
+		} else if (
+			['--output', '--snapshot', '--provenance', '--site', '--worker'].includes(argument) &&
+			args[i + 1] &&
+			!args[i + 1].startsWith('--') &&
+			!options.has(argument)
+		)
+			options.set(argument, args[++i]);
+		else deploymentError('unsupported CLI arguments');
+	}
+	if (!mode) deploymentError('select --prepare or --upload explicitly');
+	const site = path.resolve(options.get('--site') ?? path.resolve(import.meta.dir, '..'));
+	const config = readDeploymentConfig(site);
+	const provenance = await readCurrentDescriptorProvenance(
+		path.resolve(site, config.assets.directory)
+	);
+	const requestedProvenance = options.get('--provenance');
+	if (
+		requestedProvenance &&
+		(!SHA256.test(requestedProvenance) || requestedProvenance !== provenance)
+	)
+		deploymentError('explicit provenance does not match current content descriptor');
+	const outputDirectory = path.resolve(
+		options.get('--output') ?? path.join(site, 'build/.svelte-kit/prepared-worker')
+	);
+	const snapshotFile = path.resolve(
+		options.get('--snapshot') ?? path.join(site, 'build/.svelte-kit/deployment.json')
+	);
+	if (mode === '--prepare') {
+		const worker = await prepareWorker(site, outputDirectory);
+		const inventory = await captureAssetInventory(
+			path.resolve(site, config.assets.directory),
+			provenance,
+			{
+				resolveAsset: async (absolute, filename) => ({
+					...captureAssetHashes(site, absolute),
+					contentType: assetContentType(filename)
+				})
+			}
+		);
+		flushAssetHashCache(site);
+		const snapshot = buildDeploymentSnapshot(worker, inventory, config, provenance);
+		await writeFile(snapshotFile, JSON.stringify(snapshot), { mode: 0o600 });
+		console.log(
+			`deployment: prepared ${worker.modules.length} Worker modules and ${Object.keys(inventory.assets).length} assets offline`
+		);
+		return;
+	}
+	const accountId = process.env.CLOUDFLARE_ACCOUNT_ID ?? config.account_id;
+	const token = process.env.CLOUDFLARE_API_TOKEN;
+	if (!accountId || !token)
+		deploymentError('explicit upload requires Cloudflare account and API credentials');
+	const workerDirectory = path.resolve(options.get('--worker') ?? outputDirectory);
+	const snapshot: DeploymentSnapshot = JSON.parse(await readFile(snapshotFile, 'utf8'));
+	if (snapshot.inventory.assets['/_content/current.json']?.sha256 !== provenance)
+		deploymentError('deployment inventory does not match current content descriptor');
+	const loadedWorker = await loadPreparedWorker(workerDirectory, config);
+	if (loadedWorker.digest !== snapshot.worker.digest)
+		deploymentError('Worker package does not match deployment snapshot');
+	await uploadDeployment(snapshot, {
+		api: new CloudflareDeploymentAPI(token),
+		accountId,
+		expectedProvenance: provenance,
+		config,
+		readModule: (module) => readFile(path.join(workerDirectory, module.filename)),
+		readAsset: (filename) => {
+			if (!assetPath(filename)) deploymentError('invalid asset path');
+			return readFile(path.join(path.resolve(site, config.assets.directory), filename.slice(1)));
+		},
+		log: console.log
+	});
+}
+
+if (import.meta.main) {
+	try {
+		await deploymentCLI(process.argv.slice(2));
+	} catch (error) {
+		console.error(
+			error instanceof Error && error.message.startsWith('deployment:')
+				? error.message
+				: 'deployment: local preparation or input validation failed'
+		);
+		process.exitCode = 1;
+	}
+}

package/site/tooling/fingerprint.ts

@@ -0,0 +1,81 @@
+import { createHash } from 'node:crypto';
+import { readFileSync } from 'node:fs';
+import { isBuiltin } from 'node:module';
+import path from 'node:path';
+import { rolldown } from 'rolldown';
+
+const site = path.resolve(import.meta.dirname, '..');
+const socialAssets = ['assets/fonts/OverusedGrotesk-SemiBold.ttf', 'assets/social-background.svg'];
+const rAssets = ['tooling/lib/knit-rmd.R'];
+
+// Runtime reads and subprocess entrypoints are not visible in the import graph.
+export const stages = {
+	parser: { entries: ['tooling/compiler.ts'], assets: [] },
+	history: { entries: ['tooling/lib/file-history.ts'], assets: [] },
+	gitdates: { entries: ['tooling/lib/lastmod.ts'], assets: [] },
+	social: { entries: ['tooling/lib/social-images.ts'], assets: socialAssets },
+	rmd: { entries: ['tooling/lib/rmd-previews.ts'], assets: rAssets },
+	files: {
+		entries: ['tooling/lib/course-files.ts', 'tooling/lib/history-worker.ts'],
+		assets: rAssets
+	},
+	pipeline: {
+		entries: ['tooling/pipeline.ts', 'tooling/lib/history-worker.ts'],
+		assets: [...socialAssets, ...rAssets]
+	}
+};
+
+export type Stage = keyof typeof stages;
+
+/** Hash portable names and bytes; never inode, mtime, or absolute checkout paths. */
+function hashFiles(root: string, stage: Stage, files: string[]) {
+	const hash = createHash('sha256').update(
+		JSON.stringify([
+			'compiler-stage-v1',
+			stage,
+			process.version,
+			process.versions.bun ?? '',
+			process.platform,
+			process.arch,
+			stages[stage]
+		])
+	);
+	for (const file of files) {
+		const bytes = readFileSync(path.join(root, file));
+		hash.update(JSON.stringify([file, bytes.length])).update(bytes);
+	}
+	return hash.digest('hex');
+}
+
+export async function stageFingerprint(stage: Stage, root = site): Promise<string> {
+	const definition = stages[stage];
+	const pkg = JSON.parse(readFileSync(path.join(root, 'package.json'), 'utf8'));
+	const packages = Object.keys({ ...pkg.dependencies, ...pkg.devDependencies });
+	const build = await rolldown({
+		cwd: root,
+		input: definition.entries.map((file) => path.join(root, file)),
+		platform: 'node',
+		// Tooling uses relative imports and must scan before SvelteKit generates its config.
+		tsconfig: false,
+		treeshake: false,
+		external: (id) =>
+			isBuiltin(id) || packages.some((pkg) => id === pkg || id.startsWith(pkg + '/')),
+		onwarn(warning) {
+			throw new Error(`Cache dependency scan: ${warning.message}`);
+		}
+	});
+	let sources: string[];
+	try {
+		// Generate in memory so Rolldown traverses imports; never execute or emit the bundle.
+		await build.generate({ format: 'esm' });
+		sources = (await build.watchFiles).map((file) =>
+			path.relative(root, file).split(path.sep).join('/')
+		);
+	} finally {
+		await build.close();
+	}
+	const files = [
+		...new Set([...sources, ...definition.assets, 'bun.lock', 'package.json', 'tsconfig.json'])
+	].sort();
+	return hashFiles(root, stage, files);
+}

package/site/tooling/lib/app-version.js

@@ -0,0 +1,92 @@
+import { execFileSync } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import { existsSync, readdirSync, readFileSync } from 'node:fs';
+import path from 'node:path';
+
+const directories = ['src', 'tooling', 'worker', 'assets', 'database'];
+const configFiles = [
+	'package.json',
+	'bun.lock',
+	'svelte.config.js',
+	'vite.config.ts',
+	'tsconfig.json',
+	'wrangler.jsonc',
+	'_headers',
+	'_redirects'
+];
+// Bun can load these before Vite starts. Include every supported mode so a
+// copied dotenv file changing during an isolated build invalidates its snapshot.
+const environmentFiles = [
+	'.env',
+	'.env.local',
+	'.env.production',
+	'.env.production.local',
+	'.env.development',
+	'.env.development.local',
+	'.env.test',
+	'.env.test.local'
+];
+
+/**
+ * Identify application code, independent of checkout paths and course revisions.
+ * Generated navigation and file icon catalogs are loaded as separate modules,
+ * so course catalog changes do not change the application bundle's version.
+ * @param {string} site
+ * @param {NodeJS.ProcessEnv} environment
+ */
+export function applicationVersion(site, environment = process.env) {
+	// Isolated edition snapshots use the canonical maintained-source inventory.
+	const snapshotVersion = environment.WISCONSIN_APPLICATION_VERSION;
+	if (snapshotVersion !== undefined) {
+		if (!/^[a-f0-9]{64}$/.test(snapshotVersion))
+			throw new Error('WISCONSIN_APPLICATION_VERSION must be a SHA-256 application version');
+		return snapshotVersion;
+	}
+	const files = new Set(configFiles);
+	/** @param {string} relative */
+	function walk(relative) {
+		if (!existsSync(path.join(site, relative))) return;
+		for (const entry of readdirSync(path.join(site, relative), { withFileTypes: true })) {
+			if (entry.name === 'node_modules' || entry.name.startsWith('.')) continue;
+			const file = `${relative}/${entry.name}`;
+			// The pinned R renderer controls content artifacts, not the application bundle.
+			if (file === 'src/lib/generated' || file === 'tooling/r-render-profile.json') continue;
+			if (entry.isDirectory()) walk(file);
+			else if (entry.isFile()) files.add(file);
+		}
+	}
+	for (const directory of directories) walk(directory);
+	// These maintained assets are also present in Git-free content-test copies.
+	files.add('static/.gitignore');
+	files.add('static/favicon.png');
+	walk('static/fonts');
+	try {
+		// Respect static/.gitignore: synced course blobs must not change the app
+		// version, while any additional maintained static asset should change it.
+		for (const file of execFileSync(
+			'git',
+			['-C', site, 'ls-files', '--cached', '--others', '--exclude-standard', '-z', '--', 'static'],
+			{ encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }
+		).split('\0')) {
+			if (file) files.add(file);
+		}
+	} catch {
+		// Scratch copies intentionally have no Git metadata.
+	}
+	for (const file of environmentFiles) if (existsSync(path.join(site, file))) files.add(file);
+	const compileEnvironment = Object.entries(environment)
+		.filter(
+			([key]) =>
+				key === 'NODE_ENV' ||
+				((key.startsWith('VITE_') || key.startsWith('PUBLIC_')) && key !== 'VITE_PUBLIC_EDITION')
+		)
+		.sort(([a], [b]) => a.localeCompare(b));
+	const hash = createHash('sha256').update(JSON.stringify(['site-app-v1', compileEnvironment]));
+	for (const file of [...files].sort()) {
+		const absolute = path.join(site, file);
+		if (!existsSync(absolute)) continue;
+		const bytes = readFileSync(absolute);
+		hash.update(JSON.stringify([file, bytes.length])).update(bytes);
+	}
+	return hash.digest('hex');
+}

package/site/tooling/lib/application-asset-urls.ts

@@ -0,0 +1,27 @@
+import type { Plugin } from 'vite';
+
+/** Keep root-mounted SSR client assets absolute without enabling mutable relative SSR paths. */
+export function applicationAssetUrls(
+	staticExport = process.env.VITE_STATIC_EXPORT === 'true'
+): Plugin {
+	return {
+		name: 'wisconsin:application-asset-urls',
+		enforce: 'post',
+		config(config) {
+			if (staticExport) return;
+			const renderBuiltUrl = config.experimental?.renderBuiltUrl;
+			return {
+				experimental: {
+					renderBuiltUrl(filename, details) {
+						// Kit 2.65 forces Vite's base to './'. With paths.relative=false,
+						// its {relative:false} result becomes './_app/...' even though Vite
+						// resolves preload dependencies against the importing module URL.
+						// The documented renderBuiltUrl hook can emit the correct root URL.
+						if (!details.ssr && details.hostType === 'js') return `/${filename}`;
+						return renderBuiltUrl?.(filename, details);
+					}
+				}
+			};
+		}
+	};
+}

package/site/tooling/lib/application-cache.ts

@@ -0,0 +1,269 @@
+import { execFileSync, spawnSync } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import {
+	constants,
+	cpSync,
+	existsSync,
+	lstatSync,
+	mkdirSync,
+	mkdtempSync,
+	readFileSync,
+	readdirSync,
+	realpathSync,
+	renameSync,
+	rmSync,
+	writeFileSync
+} from 'node:fs';
+import path from 'node:path';
+import { applicationVersion } from './app-version.js';
+
+const schema = 1;
+const requiredFiles = [
+	'worker.js',
+	'svelte-worker.js',
+	'prepared-worker/worker.json',
+	'application-assets.json'
+];
+type FileRecord = { path: string; bytes: number; sha256: string };
+export type ApplicationIdentity = {
+	identity: string;
+	applicationVersion: string;
+	runtime: {
+		node: string;
+		bun: string;
+		platform: string;
+		architecture: string;
+		packages: [string, string][];
+	};
+};
+type Manifest = ApplicationIdentity & { schema: number; files: FileRecord[] };
+export type CachedApplication = { directory: string; manifest: Manifest };
+const copyOptions = { recursive: true, mode: constants.COPYFILE_FICLONE, preserveTimestamps: true };
+const digest = (bytes: string | Buffer) => createHash('sha256').update(bytes).digest('hex');
+const within = (root: string, file: string) => file === root || file.startsWith(root + path.sep);
+
+export function applicationEnvironment(
+	environment: NodeJS.ProcessEnv = process.env
+): NodeJS.ProcessEnv {
+	const selected: NodeJS.ProcessEnv = {
+		...environment,
+		NODE_ENV: environment.NODE_ENV ?? 'production',
+		VITE_PUBLIC_EDITION: 'false',
+		VITE_STATIC_EXPORT: 'false',
+		WISCONSIN_SKIP_CONTENT: '1'
+	};
+	delete selected.WISCONSIN_APPLICATION_VERSION;
+	return selected;
+}
+
+/** Read-only identity shared by the build producer and encrypted cache transport. */
+export function applicationCacheIdentity(
+	site: string,
+	environment: NodeJS.ProcessEnv = process.env
+): ApplicationIdentity {
+	const selected = applicationEnvironment(environment);
+	const sourceVersion = applicationVersion(site, selected);
+	const declared = JSON.parse(readFileSync(path.join(site, 'package.json'), 'utf8'));
+	const packages: [string, string][] = Object.keys({
+		...declared.dependencies,
+		...declared.devDependencies
+	})
+		.sort()
+		.map((name) => {
+			try {
+				return [
+					name,
+					JSON.parse(readFileSync(path.join(site, 'node_modules', name, 'package.json'), 'utf8'))
+						.version
+				];
+			} catch {
+				return [name, 'missing'];
+			}
+		});
+	const node = spawnSync('node', ['--version'], { encoding: 'utf8', timeout: 5000 });
+	const runtime = {
+		node: node.status === 0 ? node.stdout.trim() : 'missing',
+		bun: process.versions.bun ?? 'missing',
+		platform: process.platform,
+		architecture: process.arch,
+		packages
+	};
+	const identity = digest(
+		JSON.stringify({
+			schema,
+			applicationVersion: sourceVersion,
+			runtime,
+			prerenderConcurrency: selected.WISCONSIN_PRERENDER_CONCURRENCY ?? '1'
+		})
+	);
+	return { identity, applicationVersion: sourceVersion, runtime };
+}
+
+function inventory(directory: string): FileRecord[] {
+	const root = realpathSync(directory);
+	const files: FileRecord[] = [];
+	function walk(relative: string) {
+		for (const entry of readdirSync(path.join(root, relative), { withFileTypes: true })) {
+			const file = relative ? `${relative}/${entry.name}` : entry.name;
+			const absolute = path.join(root, file);
+			if (entry.isDirectory()) walk(file);
+			else {
+				if (!entry.isFile() || realpathSync(absolute) !== absolute)
+					throw new Error('Application cache contains a non-regular file');
+				const bytes = readFileSync(absolute);
+				files.push({ path: file, bytes: bytes.length, sha256: digest(bytes) });
+			}
+		}
+	}
+	walk('');
+	return files.sort((a, b) => a.path.localeCompare(b.path));
+}
+
+function cacheDirectory(site: string, identity: ApplicationIdentity) {
+	if (!/^[a-f0-9]{64}$/.test(identity.identity))
+		throw new Error('Invalid application cache identity');
+	return path.join(site, 'build/generated/cache/application', identity.identity);
+}
+
+export function loadApplicationCache(
+	site: string,
+	identity: ApplicationIdentity
+): CachedApplication | undefined {
+	const directory = cacheDirectory(site, identity);
+	try {
+		if (realpathSync(directory) !== path.resolve(directory))
+			throw new Error('Redirected application cache');
+		const manifest: Manifest = JSON.parse(
+			readFileSync(path.join(directory, 'manifest.json'), 'utf8')
+		);
+		if (
+			manifest.schema !== schema ||
+			manifest.identity !== identity.identity ||
+			manifest.applicationVersion !== identity.applicationVersion ||
+			JSON.stringify(manifest.runtime) !== JSON.stringify(identity.runtime) ||
+			!Array.isArray(manifest.files)
+		)
+			return;
+		const actual = inventory(path.join(directory, 'kit'));
+		if (JSON.stringify(actual) !== JSON.stringify(manifest.files)) return;
+		for (const required of requiredFiles)
+			if (!actual.some((file) => file.path === required)) return;
+		return { directory, manifest };
+	} catch {
+		return;
+	}
+}
+
+/** Save a complete neutral prepared app, without any course content or search index. */
+export function saveApplicationCache(
+	site: string,
+	identity: ApplicationIdentity
+): CachedApplication {
+	const source = path.join(site, 'build/.svelte-kit');
+	const files = inventory(source);
+	for (const required of requiredFiles)
+		if (!files.some((file) => file.path === required))
+			throw new Error('Application artifact is incomplete');
+	if (
+		files.some((file) =>
+			/^cloudflare\/(?:_content|_published|pagefind|_files)(?:\/|$)/.test(file.path)
+		)
+	)
+		throw new Error('Application artifact contains generated content');
+	const directory = cacheDirectory(site, identity);
+	mkdirSync(path.dirname(directory), { recursive: true });
+	const transaction = mkdtempSync(path.join(path.dirname(directory), '.application-save-'));
+	const staged = path.join(transaction, 'application');
+	const backup = path.join(transaction, 'previous');
+	let moved = false,
+		preserve = false;
+	try {
+		mkdirSync(staged);
+		cpSync(source, path.join(staged, 'kit'), copyOptions);
+		const manifest: Manifest = { ...identity, schema, files };
+		writeFileSync(path.join(staged, 'manifest.json'), JSON.stringify(manifest));
+		if (existsSync(directory)) {
+			renameSync(directory, backup);
+			moved = true;
+		}
+		try {
+			renameSync(staged, directory);
+		} catch (error) {
+			if (moved)
+				try {
+					renameSync(backup, directory);
+				} catch (rollback) {
+					preserve = true;
+					throw new AggregateError(
+						[error, rollback],
+						'Previous application cache preserved for recovery'
+					);
+				}
+			throw error;
+		}
+		const pointer = {
+			schema,
+			identity: identity.identity,
+			files: [
+				`application/${identity.identity}/manifest.json`,
+				...files.map((file) => `application/${identity.identity}/kit/${file.path}`)
+			]
+		};
+		const current = path.join(site, 'build/generated/cache/application-current.json');
+		const temporary = path.join(transaction, 'current.json');
+		writeFileSync(temporary, JSON.stringify(pointer));
+		renameSync(temporary, current);
+		return { directory, manifest };
+	} finally {
+		if (!preserve) rmSync(transaction, { recursive: true, force: true });
+	}
+}
+
+export function stageApplication(cache: CachedApplication, destination: string) {
+	if (existsSync(destination)) throw new Error('Application staging destination already exists');
+	cpSync(path.join(cache.directory, 'kit'), destination, copyOptions);
+}
+
+/** Prevent generated course assets from becoming inputs to the neutral Vite build. */
+export function prepareApplicationStatic(site: string, destination: string) {
+	if (existsSync(destination))
+		throw new Error('Application static staging destination already exists');
+	mkdirSync(destination, { recursive: true });
+	let files: string[];
+	try {
+		files = execFileSync(
+			'git',
+			['-C', site, 'ls-files', '--cached', '--others', '--exclude-standard', '-z', '--', 'static'],
+			{ encoding: 'utf8', maxBuffer: 32 * 1024 * 1024 }
+		)
+			.split('\0')
+			.filter(Boolean);
+	} catch {
+		files = ['static/.gitignore', 'static/favicon.png'];
+		const fonts = path.join(site, 'static/fonts');
+		if (existsSync(fonts))
+			for (const entry of readdirSync(fonts, { recursive: true, withFileTypes: true }))
+				if (entry.isFile())
+					files.push(path.relative(site, path.join(entry.parentPath, entry.name)));
+	}
+	// Packaged runtimes often live under a Git-ignored .twiki directory; an empty
+	// git inventory there does not mean the engine's maintained fonts disappeared.
+	files.push('static/.gitignore', 'static/favicon.png');
+	const fonts = path.join(site, 'static/fonts');
+	if (existsSync(fonts))
+		for (const entry of readdirSync(fonts, { recursive: true, withFileTypes: true }))
+			if (entry.isFile()) files.push(path.relative(site, path.join(entry.parentPath, entry.name)));
+	const root = realpathSync(site);
+	for (const file of new Set(files)) {
+		const source = path.resolve(site, file);
+		if (!file.startsWith('static/') || !within(root, source))
+			throw new Error('Invalid maintained static path');
+		const info = lstatSync(source, { throwIfNoEntry: false });
+		if (!info) continue;
+		if (!info.isFile() || !within(root, realpathSync(source)))
+			throw new Error('Maintained static asset must be an owned regular file');
+		const target = path.join(destination, file.slice('static/'.length));
+		mkdirSync(path.dirname(target), { recursive: true });
+		cpSync(source, target, copyOptions);
+	}
+}