@twango/twiki 0.0.0-stage → 0.1.1

package/site/tooling/lib/course-files.ts

@@ -0,0 +1,281 @@
+import { execFileSync } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import {
+	existsSync,
+	lstatSync,
+	mkdirSync,
+	readFileSync,
+	realpathSync,
+	copyFileSync
+} from 'node:fs';
+import path from 'node:path';
+import { writeChanged, pruneOutputs } from './output';
+import type { CourseFile } from '../../src/lib/files';
+import {
+	resolveProject,
+	declaredContentCourses,
+	discoverSourceFiles,
+	projectIdentity
+} from './project-config';
+import { slugifyFilePath, type FilePath } from './slug';
+import { buildFileIcons } from './file-icons';
+import { buildHistories, type HistoryJob } from './history-pool';
+import { courseFilesDirectory } from './edition-paths.js';
+import { stageFingerprint } from '../fingerprint';
+import { courseFingerprint, readCourseCache, saveCourseCache } from './course-cache';
+import { publicationFilter, courseLicenseResolver } from './publishing';
+import { buildRmdPreviews } from './rmd-previews';
+import { browsablePath } from './file-policy';
+export { browsablePath, fileHistoryPolicyKey } from './file-policy';
+const inlineImages = new Set(['png', 'jpg', 'jpeg', 'gif', 'webp', 'avif']);
+export const assetLimit = 25 * 1024 * 1024;
+
+export function previewText(bytes: Uint8Array): { text: string } | undefined {
+	if (Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength).includes(0)) return;
+	try {
+		const text = new TextDecoder('utf-8', { fatal: true }).decode(bytes);
+		if (/[\x00-\x08\x0e-\x1f]/.test(text)) return;
+		return { text };
+	} catch {
+		return;
+	}
+}
+
+type CourseOutput = { files: CourseFile[]; outputs: Set<string>; notes: string };
+const previousBuilds = new Map<string, Map<string, CourseOutput>>();
+
+export async function buildCourseFiles(
+	siteDir: string,
+	noteSlugs: Set<string>,
+	changedInputs?: Set<string>,
+	development = false
+) {
+	const project = resolveProject(siteDir);
+	const repo = project.root;
+	const publicEdition = process.env.VITE_PUBLIC_EDITION === 'true';
+	const isPublished = publicationFilter(repo, project);
+	const licenseFor = courseLicenseResolver(repo, project);
+	const contentRoot = project.contentDir;
+	const output = courseFilesDirectory(siteDir);
+	const wanted = new Set<string>();
+	for (const dir of ['index', 'blobs']) mkdirSync(path.join(output, dir), { recursive: true });
+	const courses = new Map(
+		declaredContentCourses(project).map((course) => [course.name, [] as CourseFile[]])
+	);
+	const key = JSON.stringify([siteDir, projectIdentity(project), publicEdition, development]);
+	const previous = previousBuilds.get(key);
+	const active = new Set(courses.keys());
+	const outputByCourse = new Map<string, Set<string>>();
+	const noteKeys = new Map(
+		[...courses.keys()].map((course) => [
+			course,
+			JSON.stringify([...noteSlugs].filter((slug) => slug.startsWith(course + '/')).sort())
+		])
+	);
+	const paths = discoverSourceFiles(project);
+	const fingerprints = new Map<string, string>();
+	const cacheFile = (course: string) =>
+		path.join(
+			siteDir,
+			'build/generated/cache/course-files',
+			publicEdition ? 'public' : 'full',
+			`${course}.json`
+		);
+	const cacheRoot = path.join(siteDir, 'build/generated/cache');
+	const cacheContext = (course: string) => ({
+		repo,
+		contentRoot,
+		course,
+		output,
+		cache: cacheRoot
+	});
+	if (!development) {
+		const version = `${await stageFingerprint('files')}\0${publicEdition ? 'public' : 'full'}\0${project.publication}\0${project.discovery}`;
+		for (const course of courses.keys()) {
+			const tracked = paths.filter((file) => file.startsWith(`${course}/`));
+			// Let the R renderer validate its installed runtime and worksheet dependencies.
+			if (tracked.some((file) => /\.rmd$/i.test(file))) continue;
+			const fingerprint = courseFingerprint(
+				repo,
+				course,
+				tracked,
+				version,
+				noteKeys.get(course)!,
+				contentRoot
+			);
+			fingerprints.set(course, fingerprint);
+			const cached = readCourseCache(cacheFile(course), fingerprint, cacheContext(course));
+			if (!cached) continue;
+			courses.set(course, cached.files);
+			outputByCourse.set(course, cached.outputs);
+			for (const file of cached.outputs) wanted.add(file);
+			active.delete(course);
+		}
+	}
+	if (
+		development &&
+		previous &&
+		changedInputs?.size &&
+		[...changedInputs].every((file) => file.startsWith(contentRoot + path.sep))
+	) {
+		for (const course of courses.keys()) {
+			const cached = previous.get(course);
+			if (
+				!cached ||
+				cached.notes !== noteKeys.get(course) ||
+				[...changedInputs].some(
+					(file) =>
+						file === path.join(contentRoot, course) ||
+						file.startsWith(path.join(contentRoot, course) + path.sep)
+				)
+			)
+				continue;
+			if (![...cached.outputs].every(existsSync)) continue;
+			courses.set(course, cached.files);
+			outputByCourse.set(course, cached.outputs);
+			for (const file of cached.outputs) wanted.add(file);
+			active.delete(course);
+		}
+	}
+	for (const course of active) outputByCourse.set(course, new Set());
+	const retain = (course: string, file: string) => {
+		wanted.add(file);
+		outputByCourse.get(course)!.add(file);
+	};
+	let count = 0;
+	const historyJobs = new Map<string, HistoryJob>(
+		(publicEdition || development ? [] : [...active]).map((course) => [
+			course,
+			{
+				repo: path.join(contentRoot, course),
+				output,
+				cache: path.join(siteDir, 'build/generated/cache/file-history', course),
+				files: []
+			}
+		])
+	);
+	const revisions = new Map(
+		(development && !publicEdition ? [...active] : []).map((course) => [
+			course,
+			execFileSync('git', ['-C', path.join(contentRoot, course), 'rev-parse', 'HEAD'], {
+				encoding: 'utf8'
+			}).trim()
+		])
+	);
+	for (const tracked of paths) {
+		const relative = tracked;
+		const [course, ...segments] = relative.split('/');
+		const files = courses.get(course);
+		if (!files || !active.has(course) || !browsablePath(segments.join('/'))) continue;
+		const source = path.join(contentRoot, tracked);
+		if (!existsSync(source) || !lstatSync(source).isFile()) continue;
+		if (realpathSync(source) !== path.join(contentRoot, relative)) continue;
+		const size = lstatSync(source).size;
+		const file: CourseFile = {
+			path: segments.join('/'),
+			size,
+			kind: 'binary',
+			license: licenseFor(relative)
+		};
+		const extension = path.extname(source).slice(1).toLowerCase();
+		const slug = slugifyFilePath(relative as FilePath);
+		if (extension === 'md' && noteSlugs.has(slug))
+			file.note = '/' + (slug.endsWith('/index') ? slug.slice(0, -6) : slug);
+		file.locked =
+			publicEdition && (!isPublished(relative) || (extension === 'md' && !noteSlugs.has(slug)));
+		files.push(file);
+		count++;
+		if (file.locked) {
+			delete file.license;
+			continue;
+		}
+		if (size > assetLimit) continue;
+		const bytes = readFileSync(source);
+		const hash = createHash('sha256').update(bytes).digest('hex');
+		const suffix = extension === 'pdf' ? 'pdf' : inlineImages.has(extension) ? extension : 'bin';
+		const blob = `${hash}.${suffix}`;
+		file.download = `/_files/blobs/${blob}`;
+		retain(course, path.join(output, 'blobs', blob));
+		if (!existsSync(path.join(output, 'blobs', blob)))
+			copyFileSync(source, path.join(output, 'blobs', blob));
+		if (suffix === 'pdf') file.kind = 'pdf';
+		else if (inlineImages.has(suffix)) file.kind = 'image';
+		else if (previewText(bytes)) file.kind = 'text';
+		file.history =
+			development && !publicEdition
+				? `/__content/history?course=${encodeURIComponent(course)}&file=${encodeURIComponent(file.path)}&v=${revisions.get(course)}-${hash}`
+				: undefined;
+		historyJobs.get(course)?.files.push({ file, source: path.join(output, 'blobs', blob) });
+	}
+	await buildHistories(
+		[...historyJobs.values()].filter((job) => job.files.length),
+		(job, file) => retain(path.basename(job.repo), file)
+	);
+	const entries: { course: string; file: string }[] = [];
+	for (const [course, files] of courses) {
+		if (publicEdition && !files.length) continue;
+		if (active.has(course))
+			await buildRmdPreviews(siteDir, course, files, output, (file) => retain(course, file));
+		retain(course, path.join(output, 'index', `${course}.json`));
+		writeChanged(path.join(output, 'index', `${course}.json`), JSON.stringify(files));
+		if (publicEdition) {
+			const paths = new Set(['']);
+			for (const file of files) {
+				const segments = file.path.split('/');
+				for (let i = 1; i <= segments.length; i++) paths.add(segments.slice(0, i).join('/'));
+			}
+			entries.push(...[...paths].map((file) => ({ course, file })));
+		}
+	}
+	pruneOutputs(output, wanted, (file) => file === path.join(output, 'icons'));
+	if (!publicEdition && !development) {
+		const history = [...outputByCourse].flatMap(([course, outputs]) =>
+			[...outputs]
+				.filter((file) => path.dirname(file) === path.join(output, 'history'))
+				.map((file) => `file-history/${course}/${path.basename(file)}`)
+		);
+		// Keep transport selection portable when a fresh runner restores only the cache.
+		writeChanged(
+			path.join(siteDir, 'build/generated/cache/history-current.json'),
+			JSON.stringify([...new Set(history)].sort())
+		);
+	}
+	mkdirSync(path.join(siteDir, 'src/lib/generated'), { recursive: true });
+	writeChanged(path.join(siteDir, 'src/lib/generated/file-entries.json'), JSON.stringify(entries));
+	buildFileIcons(
+		siteDir,
+		[...courses.values()].flatMap((files) => files.map((file) => file.path))
+	);
+	for (const course of active) {
+		const fingerprint = fingerprints.get(course);
+		if (fingerprint)
+			saveCourseCache(
+				cacheFile(course),
+				fingerprint,
+				courses.get(course)!,
+				outputByCourse.get(course)!,
+				cacheContext(course)
+			);
+	}
+	if (!development)
+		writeChanged(
+			path.join(cacheRoot, `course-files-current-${publicEdition ? 'public' : 'full'}.json`),
+			JSON.stringify(
+				[...fingerprints.keys()]
+					.map((course) => `course-files/${publicEdition ? 'public' : 'full'}/${course}.json`)
+					.sort()
+			)
+		);
+	previousBuilds.set(
+		key,
+		new Map(
+			[...courses].map(([course, files]) => [
+				course,
+				{ files, outputs: outputByCourse.get(course)!, notes: noteKeys.get(course)! }
+			])
+		)
+	);
+	console.log(
+		`files: ${count} tracked files across ${active.size} changed courses (${courses.size - active.size} reused)`
+	);
+}

package/site/tooling/lib/currency.ts

@@ -0,0 +1,33 @@
+import type { Root } from 'mdast';
+import type { InlineMath } from 'mdast-util-math';
+import { visit } from 'unist-util-visit';
+
+/** Recognize paired prices that remark-math would silently consume as inline math. */
+export function isCurrencyMath(node: InlineMath, source: string): boolean {
+	const start = node.position?.start.offset;
+	const end = node.position?.end.offset;
+	if (start == null || end == null) return false;
+	const raw = source.slice(start, end);
+	// Only single-dollar spans starting with an amount and ending at the next amount.
+	// Explicit LaTeX and ordinary $2 + 3$ / $x$ expressions remain unchanged.
+	if (!/^\$\d/.test(raw) || !/^\d/.test(source.slice(end))) return false;
+	if (/[\\\r\n]/.test(node.value)) return false;
+	return (
+		(/\s\$$/.test(raw) && /\b[A-Za-z]{2,}\b/.test(node.value)) || /^[\d.,\s–—-]+$/.test(node.value)
+	);
+}
+
+/** Restore the exact literal text, including both dollar signs and all spacing. */
+export function preserveCurrency(source: string) {
+	return (tree: Root) => {
+		visit(tree, 'inlineMath', (node, index, parent) => {
+			if (parent && index != null && isCurrencyMath(node, source)) {
+				parent.children[index] = {
+					type: 'text',
+					value: source.slice(node.position!.start.offset!, node.position!.end.offset!),
+					position: node.position
+				};
+			}
+		});
+	};
+}

package/site/tooling/lib/deployment-config.ts

@@ -0,0 +1,222 @@
+import { createHash } from 'node:crypto';
+import { readFileSync } from 'node:fs';
+import path from 'node:path';
+import ts from 'typescript';
+
+export interface DeploymentConfig {
+	name: string;
+	main: string;
+	compatibility_date: string;
+	compatibility_flags: string[];
+	vars: Record<string, string>;
+	d1_databases: {
+		binding: string;
+		database_name: string;
+		database_id: string;
+		migrations_dir?: string;
+	}[];
+	r2_buckets?: { binding: string; bucket_name: string; jurisdiction?: string }[];
+	assets: {
+		binding: string;
+		directory: string;
+		html_handling: 'drop-trailing-slash';
+		not_found_handling: '404-page';
+		run_worker_first: true;
+	};
+	workers_dev: boolean;
+	preview_urls: false;
+	routes: { pattern: string; custom_domain: true }[];
+	account_id?: string;
+	minify?: boolean;
+	upload_source_maps?: boolean;
+}
+
+export function deploymentError(message: string): never {
+	// Messages deliberately exclude API bodies, credentials, private paths, and module contents.
+	throw new Error(`deployment: ${message}`);
+}
+
+export function record(value: unknown): value is Record<string, unknown> {
+	return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+
+export function allowedKeys(value: Record<string, unknown>, allowed: string[]): void {
+	if (Object.keys(value).some((key) => !allowed.includes(key)))
+		deploymentError('unsupported configuration');
+}
+
+export function sha256(bytes: Uint8Array | string): string {
+	return createHash('sha256').update(bytes).digest('hex');
+}
+
+function canonical(value: unknown): unknown {
+	if (Array.isArray(value)) return value.map(canonical);
+	if (record(value))
+		return Object.fromEntries(
+			Object.keys(value)
+				.sort()
+				.map((key) => [key, canonical(value[key])])
+		);
+	return value;
+}
+
+export function configSha256(config: DeploymentConfig): string {
+	return sha256(JSON.stringify(canonical(config)));
+}
+
+export function validateDeploymentConfig(value: unknown): DeploymentConfig {
+	if (!record(value)) deploymentError('invalid configuration');
+	allowedKeys(value, [
+		'$schema',
+		'name',
+		'main',
+		'compatibility_date',
+		'compatibility_flags',
+		'vars',
+		'd1_databases',
+		'r2_buckets',
+		'assets',
+		'workers_dev',
+		'preview_urls',
+		'routes',
+		'account_id',
+		'minify',
+		'upload_source_maps'
+	]);
+	if (
+		typeof value.name !== 'string' ||
+		!/^[a-z0-9][a-z0-9_-]*$/.test(value.name) ||
+		typeof value.main !== 'string' ||
+		!value.main
+	)
+		deploymentError('invalid Worker identity');
+	if (
+		typeof value.compatibility_date !== 'string' ||
+		!/^\d{4}-\d{2}-\d{2}$/.test(value.compatibility_date) ||
+		!Array.isArray(value.compatibility_flags) ||
+		value.compatibility_flags.some((flag) => typeof flag !== 'string')
+	)
+		deploymentError('invalid compatibility settings');
+	if (
+		!record(value.vars) ||
+		Object.entries(value.vars).some(([key, v]) => !bindingName(key) || typeof v !== 'string')
+	)
+		deploymentError('unsupported variables');
+	if (!Array.isArray(value.d1_databases)) deploymentError('missing D1 configuration');
+	for (const database of value.d1_databases) {
+		if (!record(database)) deploymentError('invalid D1 configuration');
+		allowedKeys(database, ['binding', 'database_name', 'database_id', 'migrations_dir']);
+		if (
+			!bindingName(database.binding) ||
+			typeof database.database_name !== 'string' ||
+			typeof database.database_id !== 'string' ||
+			!/^[0-9a-f-]{36}$/.test(database.database_id) ||
+			(database.migrations_dir !== undefined && typeof database.migrations_dir !== 'string')
+		)
+			deploymentError('invalid D1 configuration');
+	}
+	if (value.r2_buckets !== undefined) {
+		if (!Array.isArray(value.r2_buckets)) deploymentError('invalid R2 configuration');
+		for (const bucket of value.r2_buckets) {
+			if (!record(bucket)) deploymentError('invalid R2 configuration');
+			allowedKeys(bucket, ['binding', 'bucket_name', 'jurisdiction']);
+			if (
+				!bindingName(bucket.binding) ||
+				typeof bucket.bucket_name !== 'string' ||
+				!bucket.bucket_name ||
+				(bucket.jurisdiction !== undefined && typeof bucket.jurisdiction !== 'string')
+			)
+				deploymentError('invalid R2 configuration');
+		}
+	}
+	if (!record(value.assets)) deploymentError('missing asset configuration');
+	allowedKeys(value.assets, [
+		'binding',
+		'directory',
+		'html_handling',
+		'not_found_handling',
+		'run_worker_first'
+	]);
+	if (
+		!bindingName(value.assets.binding) ||
+		typeof value.assets.directory !== 'string' ||
+		!value.assets.directory ||
+		value.assets.run_worker_first !== true ||
+		value.assets.html_handling !== 'drop-trailing-slash' ||
+		value.assets.not_found_handling !== '404-page'
+	)
+		deploymentError('unsupported asset routing');
+	if (
+		typeof value.workers_dev !== 'boolean' ||
+		value.preview_urls !== false ||
+		(value.workers_dev && value.vars.TWIKI_AUTH_MODE !== 'none')
+	)
+		deploymentError('public development endpoints must be disabled');
+	if (!Array.isArray(value.routes) || (!value.routes.length && !value.workers_dev))
+		deploymentError('missing custom domains');
+	for (const route of value.routes) {
+		if (!record(route)) deploymentError('invalid custom domain');
+		allowedKeys(route, ['pattern', 'custom_domain']);
+		if (
+			route.custom_domain !== true ||
+			typeof route.pattern !== 'string' ||
+			!/^[a-z0-9.-]+$/.test(route.pattern) ||
+			!route.pattern.includes('.')
+		)
+			deploymentError('unsupported custom domain');
+	}
+	if (
+		new Set(value.routes.map((route) => (route as { pattern: string }).pattern)).size !==
+		value.routes.length
+	)
+		deploymentError('duplicate custom domains');
+	if (
+		value.account_id !== undefined &&
+		(typeof value.account_id !== 'string' || !/^[0-9a-f]{32}$/.test(value.account_id))
+	)
+		deploymentError('invalid account identity');
+	for (const key of ['minify', 'upload_source_maps'])
+		if (value[key] !== undefined && typeof value[key] !== 'boolean')
+			deploymentError('invalid build settings');
+	const config = { ...value };
+	delete config.$schema;
+	const names = [
+		...Object.keys(value.vars),
+		...value.d1_databases.map((db) => (db as { binding: string }).binding),
+		...((value.r2_buckets ?? []) as { binding: string }[]).map((bucket) => bucket.binding),
+		value.assets.binding
+	];
+	if (new Set(names).size !== names.length) deploymentError('duplicate bindings');
+	return config as unknown as DeploymentConfig;
+}
+
+export function bindingName(value: unknown): value is string {
+	return typeof value === 'string' && /^[A-Za-z_$][A-Za-z0-9_$]*$/.test(value);
+}
+
+export function readDeploymentConfig(siteRoot: string): DeploymentConfig {
+	const parsed = ts.parseConfigFileTextToJson(
+		'wrangler.jsonc',
+		readFileSync(path.join(siteRoot, 'wrangler.jsonc'), 'utf8')
+	);
+	if (parsed.error) deploymentError('invalid JSONC configuration');
+	return validateDeploymentConfig(parsed.config);
+}
+
+export function expectedBindings(config: DeploymentConfig): Record<string, unknown>[] {
+	return [
+		...Object.entries(config.vars).map(([name, text]) => ({ name, type: 'plain_text', text })),
+		...config.d1_databases.map(({ binding: name, database_id: id }) => ({ name, type: 'd1', id })),
+		...(config.r2_buckets ?? []).map(({ binding: name, bucket_name, jurisdiction }) => ({
+			name,
+			type: 'r2_bucket',
+			bucket_name,
+			...(jurisdiction === undefined ? {} : { jurisdiction })
+		})),
+		{ name: config.assets.binding, type: 'assets' }
+	];
+}
+
+export function equalJSON(a: unknown, b: unknown): boolean {
+	return JSON.stringify(canonical(a)) === JSON.stringify(canonical(b));
+}