@twango/twiki 0.0.0-stage → 0.1.0-rc.0

package/site/tooling/lib/content-cache.ts

@@ -0,0 +1,244 @@
+import { createHash } from 'node:crypto';
+import { lstatSync, mkdirSync, readFileSync, realpathSync } from 'node:fs';
+import path from 'node:path';
+import type { FullSlug } from './slug';
+import { writeChanged } from './output';
+
+export type ContentEdition = 'public' | 'full';
+export interface ContentSummary {
+	slug: FullSlug;
+	rel: string;
+	frontmatter: Record<string, unknown> & { title: string };
+	heading?: string;
+	title: string;
+	description: string;
+	tags: string[];
+	toc: { depth: number; text: string; slug: string }[];
+	draft: boolean;
+	hasMermaid: boolean;
+	wordCount: number;
+}
+export interface ContentCachePage {
+	parseKey: string;
+	summary: ContentSummary;
+	outgoing: string[];
+	transcludes: { target: string; selector?: string }[];
+	parseWarnings: string[];
+	unknownLanguages?: Record<string, number>;
+	linkWarnings: string[];
+	transclusionWarnings: string[];
+	body?: string;
+	document?: string;
+	documentKey?: string;
+}
+export interface ContentCacheIndex {
+	schema: 1;
+	renderer: string;
+	context: string;
+	pages: Record<string, ContentCachePage>;
+}
+export interface ContentBody {
+	html: string;
+	markdown: string;
+}
+
+export function contentHash(bytes: string | Uint8Array) {
+	return createHash('sha256').update(bytes).digest('hex');
+}
+
+const digest = /^[a-f0-9]{64}$/;
+export const contentObjectPath = /^content\/(?:public|full)\/(?:bodies|docs)\/[a-f0-9]{64}\.json$/;
+const strings = (value: unknown): value is string[] =>
+	Array.isArray(value) && value.every((item) => typeof item === 'string');
+
+/** Refuse symlinks at every level; restored manifests contain portable paths only. */
+function readRegular(cache: string, relative: string): Buffer | undefined {
+	try {
+		const root = realpathSync(cache);
+		if (root !== path.resolve(cache)) return;
+		const file = path.join(root, relative);
+		if (realpathSync(file) !== file || !lstatSync(file).isFile()) return;
+		return readFileSync(file);
+	} catch {
+		return;
+	}
+}
+
+function writeOwned(cache: string, relative: string, bytes: string | Uint8Array) {
+	const root = path.resolve(cache);
+	mkdirSync(root, { recursive: true });
+	if (realpathSync(root) !== root)
+		throw new Error('Content cache must have owned directory ancestors');
+	let directory = root;
+	for (const segment of relative.split('/').slice(0, -1)) {
+		directory = path.join(directory, segment);
+		const info = lstatSync(directory, { throwIfNoEntry: false });
+		if (info && (!info.isDirectory() || realpathSync(directory) !== directory))
+			throw new Error('Content cache must have owned directory ancestors');
+		if (!info) mkdirSync(directory);
+	}
+	const target = path.join(root, relative);
+	const info = lstatSync(target, { throwIfNoEntry: false });
+	if (info && (!info.isFile() || realpathSync(target) !== target))
+		throw new Error('Content cache entries must be owned regular files');
+	writeChanged(target, bytes);
+}
+
+function ownedObject(value: unknown, edition: ContentEdition, kind?: 'bodies' | 'docs') {
+	return (
+		typeof value === 'string' &&
+		contentObjectPath.test(value) &&
+		value.startsWith(`content/${edition}/${kind ?? ''}`)
+	);
+}
+
+export function readContentObject(cache: string, relative: string, edition: ContentEdition) {
+	if (!ownedObject(relative, edition)) return;
+	const bytes = readRegular(cache, relative);
+	if (!bytes || contentHash(bytes) !== path.basename(relative, '.json')) return;
+	return bytes;
+}
+
+export function readContentBody(cache: string, relative: string, edition: ContentEdition) {
+	if (!ownedObject(relative, edition, 'bodies')) return;
+	try {
+		const bytes = readContentObject(cache, relative, edition);
+		if (!bytes) return;
+		const body: unknown = JSON.parse(bytes.toString());
+		if (
+			body &&
+			typeof body === 'object' &&
+			'html' in body &&
+			typeof body.html === 'string' &&
+			'markdown' in body &&
+			typeof body.markdown === 'string'
+		)
+			return body as ContentBody;
+	} catch {
+		return;
+	}
+}
+
+function validPage(value: unknown, edition: ContentEdition): value is ContentCachePage {
+	if (!value || typeof value !== 'object') return false;
+	const page = value as ContentCachePage;
+	const summary = page.summary;
+	return (
+		digest.test(page.parseKey) &&
+		!!summary &&
+		typeof summary === 'object' &&
+		typeof summary.rel === 'string' &&
+		typeof summary.slug === 'string' &&
+		typeof summary.title === 'string' &&
+		typeof summary.description === 'string' &&
+		!!summary.frontmatter &&
+		typeof summary.frontmatter === 'object' &&
+		!Array.isArray(summary.frontmatter) &&
+		typeof summary.draft === 'boolean' &&
+		typeof summary.hasMermaid === 'boolean' &&
+		Number.isFinite(summary.wordCount) &&
+		strings(summary.tags) &&
+		Array.isArray(summary.toc) &&
+		summary.toc.every(
+			(entry) =>
+				entry &&
+				Number.isFinite(entry.depth) &&
+				typeof entry.text === 'string' &&
+				typeof entry.slug === 'string'
+		) &&
+		strings(page.outgoing) &&
+		strings(page.parseWarnings) &&
+		strings(page.linkWarnings) &&
+		strings(page.transclusionWarnings) &&
+		(page.unknownLanguages === undefined ||
+			(!!page.unknownLanguages &&
+				typeof page.unknownLanguages === 'object' &&
+				!Array.isArray(page.unknownLanguages) &&
+				Object.values(page.unknownLanguages).every(
+					(count) => Number.isInteger(count) && count > 0
+				))) &&
+		Array.isArray(page.transcludes) &&
+		page.transcludes.every(
+			(entry) =>
+				entry &&
+				typeof entry.target === 'string' &&
+				(entry.selector === undefined || typeof entry.selector === 'string')
+		) &&
+		(page.body === undefined || ownedObject(page.body, edition, 'bodies')) &&
+		(page.document === undefined || ownedObject(page.document, edition, 'docs')) &&
+		(page.documentKey === undefined || digest.test(page.documentKey))
+	);
+}
+
+export function readContentIndex(cache: string, edition: ContentEdition, renderer: string) {
+	try {
+		const bytes = readRegular(cache, `content-current-${edition}.json`);
+		if (!bytes) return;
+		const { checksum, ...index } = JSON.parse(bytes.toString());
+		if (
+			index.schema !== 1 ||
+			index.renderer !== renderer ||
+			!digest.test(index.context) ||
+			checksum !== contentHash(JSON.stringify(index)) ||
+			!index.pages ||
+			typeof index.pages !== 'object' ||
+			Array.isArray(index.pages) ||
+			!Object.entries(index.pages).every(
+				([relative, page]) => validPage(page, edition) && page.summary.rel === relative
+			)
+		)
+			return;
+		return index as ContentCacheIndex;
+	} catch {
+		return;
+	}
+}
+
+export function writeContentObject(
+	cache: string,
+	edition: ContentEdition,
+	kind: 'bodies' | 'docs',
+	bytes: string | Uint8Array
+) {
+	const relative = `content/${edition}/${kind}/${contentHash(bytes)}.json`;
+	writeOwned(cache, relative, bytes);
+	return relative;
+}
+
+/** Commit the current selection only after the compiler completed all output checks. */
+export function writeContentIndex(
+	cache: string,
+	edition: ContentEdition,
+	index: ContentCacheIndex
+) {
+	const payload = JSON.stringify(index);
+	writeOwned(
+		cache,
+		`content-current-${edition}.json`,
+		JSON.stringify({ ...index, checksum: contentHash(payload) })
+	);
+	const files = new Set<string>([`content-current-${edition}.json`]);
+	for (const page of Object.values(index.pages)) {
+		if (page.body) files.add(page.body);
+		if (page.document) files.add(page.document);
+	}
+	writeOwned(cache, `content-files-current-${edition}.json`, JSON.stringify([...files].sort()));
+}
+
+/** Earlier targets may already be expanded, so the safe reverse closure is not depth limited. */
+export function transclusionClosure(pages: Iterable<ContentCachePage>, changed: Set<string>) {
+	const records = [...pages];
+	const affected = new Set(changed);
+	let added: boolean;
+	do {
+		added = false;
+		for (const page of records) {
+			if (affected.has(page.summary.slug)) continue;
+			if (page.transcludes.some(({ target }) => affected.has(target))) {
+				affected.add(page.summary.slug);
+				added = true;
+			}
+		}
+	} while (added);
+	return affected;
+}

package/site/tooling/lib/content-urls.ts

@@ -0,0 +1,50 @@
+import { visitElements } from './visit-elements';
+import type { Root } from 'hast';
+import { simplifySlug, stripSlashes, type FullSlug } from './slug';
+
+export function htmlAssetPaths(assets: { rel: string; slug: FullSlug }[]): Map<string, string> {
+	return new Map(
+		assets
+			.filter((asset) => asset.rel.toLowerCase().endsWith('.html'))
+			.map((asset) => [`/${simplifySlug(asset.slug)}`.replace(/\/$/, ''), `/${asset.slug}.html`])
+	);
+}
+
+function isLocalContentUrl(value: string): boolean {
+	return !(
+		/^[a-zA-Z][a-zA-Z\d+.-]*:/.test(value) ||
+		value.startsWith('//') ||
+		value.startsWith('#')
+	);
+}
+
+function resolveContentUrl(
+	value: string,
+	pageUrl: URL,
+	assetPaths: ReadonlyMap<string, string>
+): string {
+	if (!isLocalContentUrl(value)) return value;
+
+	const url = new URL(value, pageUrl);
+	const route = url.pathname.replace(/\/$/, '') || '/';
+	const assetPath = assetPaths.get(decodeURIComponent(route));
+	const pathname = assetPath ? assetPath.split('/').map(encodeURIComponent).join('/') : route;
+	return pathname + url.search + url.hash;
+}
+
+export function rewriteContentUrls(
+	tree: Root,
+	slug: FullSlug,
+	assetPaths: ReadonlyMap<string, string>
+) {
+	const pagePath = stripSlashes(simplifySlug(slug), true);
+	const pageUrl = new URL(`/${pagePath}`, 'https://content.invalid');
+
+	visitElements(tree, (node) => {
+		for (const attribute of ['href', 'src']) {
+			const value = node.properties[attribute];
+			if (typeof value === 'string')
+				node.properties[attribute] = resolveContentUrl(value, pageUrl, assetPaths);
+		}
+	});
+}

package/site/tooling/lib/course-cache.ts

@@ -0,0 +1,298 @@
+import { createHash } from 'node:crypto';
+import { execFileSync } from 'node:child_process';
+import { lstatSync, mkdirSync, readFileSync, realpathSync } from 'node:fs';
+import path from 'node:path';
+import type { CourseFile } from '../../src/lib/files';
+import { writeChanged } from './output';
+import { browsablePath } from './file-policy';
+
+const digest = (bytes: string | Uint8Array) => createHash('sha256').update(bytes).digest('hex');
+const sha = /^[a-f0-9]{64}$/;
+const courseName = /^[\w-]+$/;
+const historyName = /^[a-f0-9]{64}(?:\.json|-blame\.json|-[a-f0-9]{40,64}\.diff)$/;
+const blobName = /^blobs\/[a-f0-9]{64}\.(?:bin|pdf|png|jpg|jpeg|gif|webp|avif)$/;
+type Origin =
+	| { kind: 'index' }
+	| { kind: 'source'; path: string }
+	| { kind: 'history'; cachePath: string };
+export type CourseCacheOutput = { path: string; bytes: number; sha256: string; origin: Origin };
+export type CourseCacheRecord = {
+	version: 2;
+	course: string;
+	fingerprint: string;
+	files: CourseFile[];
+	outputs: CourseCacheOutput[];
+	payloadSha256: string;
+};
+export type CourseCacheContext = {
+	repo: string;
+	contentRoot?: string;
+	course: string;
+	output: string;
+	cache: string;
+};
+
+function regularBytes(root: string, relative: string) {
+	if (
+		!relative ||
+		/[\\\x00-\x1f\x7f]/.test(relative) ||
+		relative.split('/').some((part) => !part || part === '.' || part === '..')
+	)
+		return;
+	const file = path.join(root, relative);
+	try {
+		if (!lstatSync(file).isFile() || realpathSync(file) !== path.join(realpathSync(root), relative))
+			return;
+		return readFileSync(file);
+	} catch (error) {
+		if (['ENOENT', 'ENOTDIR'].includes((error as NodeJS.ErrnoException).code ?? '')) return;
+		throw error;
+	}
+}
+
+/** Portable identity includes tracked source bytes, offline edits, deletions and publication rules. */
+export function courseFingerprint(
+	repo: string,
+	course: string,
+	tracked: string[],
+	version: string,
+	notes: string,
+	contentRoot?: string
+) {
+	const sourceRoot = contentRoot ?? path.join(repo, 'content');
+	if (!courseName.test(course)) throw new Error('Unsafe course name');
+	const hash = createHash('sha256').update(
+		JSON.stringify(['course-files-v2', course, version, notes])
+	);
+	try {
+		hash.update(
+			execFileSync('git', ['-C', path.join(sourceRoot, course), 'rev-parse', 'HEAD'], {
+				stdio: ['ignore', 'pipe', 'ignore']
+			})
+		);
+	} catch {
+		hash.update('unborn');
+	}
+	const prefix = contentRoot === undefined ? `content/${course}/` : `${course}/`;
+	const inputs = [
+		...new Set([
+			...tracked.filter(
+				(file) => file.startsWith(prefix) && browsablePath(file.slice(prefix.length))
+			),
+			`${prefix}publish.yaml`
+		])
+	].sort();
+	for (const file of inputs) {
+		const bytes = regularBytes(contentRoot === undefined ? repo : sourceRoot, file);
+		hash.update(JSON.stringify([file, bytes ? [bytes.length, digest(bytes)] : null]));
+	}
+	return hash.digest('hex');
+}
+
+function payload(saved: Omit<CourseCacheRecord, 'payloadSha256'>) {
+	return JSON.stringify([
+		saved.version,
+		saved.course,
+		saved.fingerprint,
+		saved.files,
+		saved.outputs
+	]);
+}
+
+/** Validate portable metadata before output restoration or transport selection. */
+export function parseCourseCache(value: unknown, course: string): CourseCacheRecord | undefined {
+	const saved = value as CourseCacheRecord;
+	if (
+		!saved ||
+		saved.version !== 2 ||
+		saved.course !== course ||
+		!courseName.test(course) ||
+		typeof saved.fingerprint !== 'string' ||
+		!sha.test(saved.fingerprint) ||
+		!Array.isArray(saved.files) ||
+		!Array.isArray(saved.outputs) ||
+		typeof saved.payloadSha256 !== 'string' ||
+		!sha.test(saved.payloadSha256) ||
+		digest(payload(saved)) !== saved.payloadSha256
+	)
+		return;
+	if (
+		!saved.files.every(
+			(file) =>
+				file &&
+				typeof file.path === 'string' &&
+				browsablePath(file.path) &&
+				Number.isSafeInteger(file.size) &&
+				file.size >= 0 &&
+				['text', 'image', 'pdf', 'binary'].includes(file.kind) &&
+				[file.download, file.history, file.rmdPreview, file.note].every(
+					(value) => value === undefined || typeof value === 'string'
+				)
+		)
+	)
+		return;
+	const paths = new Set<string>();
+	for (const output of saved.outputs) {
+		if (
+			!output ||
+			typeof output.path !== 'string' ||
+			!browsablePath(output.path) ||
+			paths.has(output.path) ||
+			!Number.isSafeInteger(output.bytes) ||
+			output.bytes < 0 ||
+			typeof output.sha256 !== 'string' ||
+			!sha.test(output.sha256) ||
+			!output.origin
+		)
+			return;
+		paths.add(output.path);
+		if (output.origin.kind === 'index') {
+			if (output.path !== `index/${course}.json`) return;
+		} else if (output.origin.kind === 'history') {
+			const name = output.path.slice('history/'.length);
+			if (
+				!output.path.startsWith('history/') ||
+				!historyName.test(name) ||
+				output.origin.cachePath !== `file-history/${course}/${name}`
+			)
+				return;
+		} else if (output.origin.kind === 'source') {
+			const source = output.origin.path;
+			if (
+				!blobName.test(output.path) ||
+				output.path.slice(6, 70) !== output.sha256 ||
+				typeof source !== 'string' ||
+				!browsablePath(source) ||
+				!saved.files.some(
+					(file) =>
+						!file.locked &&
+						file.path === source &&
+						file.size === output.bytes &&
+						file.download === `/_files/${output.path}`
+				)
+			)
+				return;
+		} else return;
+	}
+	if (saved.files.length && !paths.has(`index/${course}.json`)) return;
+	for (const file of saved.files) {
+		if (
+			file.download &&
+			(file.locked ||
+				!file.download.startsWith('/_files/blobs/') ||
+				!paths.has(file.download.slice('/_files/'.length)))
+		)
+			return;
+		if (
+			file.history &&
+			(file.locked ||
+				!file.history.startsWith('/_files/history/') ||
+				!paths.has(file.history.slice('/_files/'.length)))
+		)
+			return;
+		if (file.rmdPreview) return; // R courses remain owned by the renderer preflight.
+	}
+	return saved;
+}
+
+function safeParents(root: string, relative: string) {
+	for (let parent = path.dirname(relative); parent !== '.'; parent = path.dirname(parent)) {
+		const target = path.join(root, parent);
+		const stat = lstatSync(target, { throwIfNoEntry: false });
+		if (
+			stat &&
+			(!stat.isDirectory() || realpathSync(target) !== path.join(realpathSync(root), parent))
+		)
+			return false;
+	}
+	return true;
+}
+
+export function readCourseCache(file: string, fingerprint: string, context: CourseCacheContext) {
+	try {
+		if (!lstatSync(file).isFile() || realpathSync(file) !== path.resolve(file)) return;
+		const saved = parseCourseCache(JSON.parse(readFileSync(file, 'utf8')), context.course);
+		if (!saved || saved.fingerprint !== fingerprint) return;
+		const pending: CourseCacheOutput[] = [];
+		const recoveryBytes = (output: CourseCacheOutput) => {
+			if (output.origin.kind === 'index') return Buffer.from(JSON.stringify(saved.files));
+			if (output.origin.kind === 'source')
+				return regularBytes(
+					path.join(context.contentRoot ?? path.join(context.repo, 'content'), context.course),
+					output.origin.path
+				);
+			return regularBytes(context.cache, output.origin.cachePath);
+		};
+		for (const output of saved.outputs) {
+			if (!safeParents(context.output, output.path)) return;
+			const existing = regularBytes(context.output, output.path);
+			if (existing && existing.length === output.bytes && digest(existing) === output.sha256)
+				continue;
+			const bytes = recoveryBytes(output);
+			if (!bytes || bytes.length !== output.bytes || digest(bytes) !== output.sha256) return;
+			pending.push(output);
+		}
+		// Check the complete closure before writing; unavailable products rebuild normally.
+		for (const output of pending) {
+			// Recheck bytes at the write boundary without retaining whole courses in memory.
+			const bytes = recoveryBytes(output);
+			if (
+				!bytes ||
+				bytes.length !== output.bytes ||
+				digest(bytes) !== output.sha256 ||
+				!safeParents(context.output, output.path)
+			)
+				return;
+			const target = path.join(context.output, output.path);
+			mkdirSync(path.dirname(target), { recursive: true });
+			const stat = lstatSync(target, { throwIfNoEntry: false });
+			if (stat && !stat.isFile()) return;
+			writeChanged(target, bytes);
+		}
+		return {
+			files: saved.files,
+			outputs: new Set(saved.outputs.map((output) => path.join(context.output, output.path)))
+		};
+	} catch {
+		return;
+	}
+}
+
+export function saveCourseCache(
+	file: string,
+	fingerprint: string,
+	files: CourseFile[],
+	outputs: Set<string>,
+	context: CourseCacheContext
+) {
+	const records: CourseCacheOutput[] = [...outputs].sort().map((target) => {
+		const relative = path.relative(context.output, target).split(path.sep).join('/');
+		const bytes = regularBytes(context.output, relative);
+		if (!bytes) throw new Error('Invalid course cache output');
+		let origin: Origin;
+		if (relative === `index/${context.course}.json`) origin = { kind: 'index' };
+		else if (relative.startsWith('history/'))
+			origin = {
+				kind: 'history',
+				cachePath: `file-history/${context.course}/${path.basename(relative)}`
+			};
+		else {
+			const source = files.find(
+				(entry) => !entry.locked && entry.download === `/_files/${relative}`
+			);
+			if (!source) throw new Error('Course output has no tracked source');
+			origin = { kind: 'source', path: source.path };
+		}
+		return { path: relative, bytes: bytes.length, sha256: digest(bytes), origin };
+	});
+	const saved = {
+		version: 2 as const,
+		course: context.course,
+		fingerprint,
+		files,
+		outputs: records
+	};
+	const record = { ...saved, payloadSha256: digest(payload(saved)) };
+	if (!parseCourseCache(record, context.course)) throw new Error('Invalid portable course cache');
+	writeChanged(file, JSON.stringify(record));
+}