@twango/twiki 0.0.0-stage → 0.1.0-rc.0

package/site/tooling/lib/project-config.ts

@@ -0,0 +1,288 @@
+import { execFileSync } from 'node:child_process';
+import { existsSync, lstatSync, readFileSync, readdirSync, realpathSync } from 'node:fs';
+import path from 'node:path';
+import { browsablePath } from './file-policy';
+import { writeChanged } from './output';
+
+export interface SiteMetadata {
+	name: string;
+	title: string;
+	url: string;
+	description: string;
+	author: { name: string; url: string; github: string; email: string };
+	locale: string;
+	language: string;
+}
+export interface ProjectConfig {
+	version: 1;
+	contentDir?: string;
+	discovery?: 'filesystem' | 'git';
+	publication?: 'private' | 'public' | 'policies';
+	site?: Partial<Omit<SiteMetadata, 'author'>> & { author?: Partial<SiteMetadata['author']> };
+	cloudflare?: Record<string, unknown>;
+	analytics?: { siteId: string; scriptUrl?: string; endpoint?: string };
+}
+export interface ResolvedProject {
+	root: string;
+	contentDir: string;
+	configFile?: string;
+	discovery: 'filesystem' | 'git';
+	publication: 'private' | 'public' | 'policies';
+	site: SiteMetadata;
+	cloudflare?: Record<string, unknown>;
+	analytics?: ProjectConfig['analytics'];
+}
+const defaults: SiteMetadata = {
+	name: 'twiki',
+	title: 'twiki',
+	url: 'http://localhost:5173',
+	description: 'Notes, ideas, and reference material.',
+	author: { name: '', url: '', github: '', email: '' },
+	locale: 'en_US',
+	language: 'en'
+};
+const isObject = (value: unknown): value is Record<string, unknown> =>
+	!!value && typeof value === 'object' && !Array.isArray(value);
+export function parseProjectConfig(value: unknown): ProjectConfig {
+	if (!isObject(value) || value.version !== 1)
+		throw new Error('Twiki config must be a JSON object with version: 1');
+	if (value.contentDir !== undefined && (typeof value.contentDir !== 'string' || !value.contentDir))
+		throw new Error('contentDir must be a nonempty path');
+	if (
+		value.discovery !== undefined &&
+		(typeof value.discovery !== 'string' || !['filesystem', 'git'].includes(value.discovery))
+	)
+		throw new Error('discovery must be filesystem or git');
+	if (
+		value.publication !== undefined &&
+		(typeof value.publication !== 'string' ||
+			!['private', 'public', 'policies'].includes(value.publication))
+	)
+		throw new Error('publication must be private, public or policies');
+	if (value.site !== undefined) {
+		if (!isObject(value.site)) throw new Error('site must be an object');
+		for (const [name, field] of Object.entries(value.site)) {
+			if (name === 'author') {
+				if (
+					!isObject(field) ||
+					Object.entries(field).some(
+						([key, text]) =>
+							!['name', 'url', 'github', 'email'].includes(key) || typeof text !== 'string'
+					)
+				)
+					throw new Error('site.author must contain text fields');
+			} else if (
+				!['name', 'title', 'url', 'description', 'locale', 'language'].includes(name) ||
+				typeof field !== 'string'
+			)
+				throw new Error(`Unknown or invalid site field: ${name}`);
+		}
+		if (
+			value.site.url !== undefined &&
+			!['https:', 'http:'].includes(new URL(String(value.site.url)).protocol)
+		)
+			throw new Error('site.url must use HTTP or HTTPS');
+	}
+	if (value.cloudflare !== undefined && !isObject(value.cloudflare))
+		throw new Error('cloudflare must be an object');
+	if (
+		value.analytics !== undefined &&
+		(!isObject(value.analytics) ||
+			typeof value.analytics.siteId !== 'string' ||
+			[value.analytics.scriptUrl, value.analytics.endpoint].some(
+				(x) => x !== undefined && typeof x !== 'string'
+			))
+	)
+		throw new Error('analytics must contain a siteId and optional scriptUrl/endpoint');
+	return value as unknown as ProjectConfig;
+}
+/** All writable engine state belongs to siteDir; project/content roots are read-only inputs. */
+export function resolveProject(
+	siteDir = process.cwd(),
+	repoOverride?: string,
+	environment: NodeJS.ProcessEnv = process.env
+): ResolvedProject {
+	const explicit =
+		!!environment.TWIKI_PROJECT_ROOT ||
+		!!environment.TWIKI_CONFIG_FILE ||
+		!!environment.TWIKI_CONTENT_DIR;
+	const requestedRoot = path.resolve(
+		repoOverride ??
+			environment.TWIKI_PROJECT_ROOT ??
+			environment.WISCONSIN_CONTENT_REPO ??
+			path.dirname(siteDir)
+	);
+	const root = existsSync(requestedRoot) ? realpathSync(requestedRoot) : requestedRoot;
+	const requested = environment.TWIKI_CONFIG_FILE;
+	if (requested && !path.isAbsolute(requested))
+		throw new Error('TWIKI_CONFIG_FILE must be absolute');
+	const configFile =
+		requested ??
+		(existsSync(path.join(root, 'twiki.config.json'))
+			? path.join(root, 'twiki.config.json')
+			: undefined);
+	const config = configFile
+		? parseProjectConfig(JSON.parse(readFileSync(configFile, 'utf8')))
+		: { version: 1 as const };
+	const selectedContent = path.resolve(
+		root,
+		environment.TWIKI_CONTENT_DIR ?? config.contentDir ?? (explicit ? '.' : 'content')
+	);
+	const contentDir = existsSync(selectedContent) ? realpathSync(selectedContent) : selectedContent;
+	return {
+		root,
+		contentDir,
+		configFile,
+		discovery: config.discovery ?? (explicit ? 'filesystem' : 'git'),
+		publication: config.publication ?? (explicit ? 'private' : 'policies'),
+		site: { ...defaults, ...config.site, author: { ...defaults.author, ...config.site?.author } },
+		cloudflare: config.cloudflare,
+		analytics: config.analytics
+	};
+}
+export function writeProjectSiteConfig(siteDir: string, project = resolveProject(siteDir)) {
+	writeChanged(
+		path.join(siteDir, 'src/lib/site-config.json'),
+		JSON.stringify(project.site, null, 2) + '\n'
+	);
+}
+export function projectIdentity(project: ResolvedProject) {
+	return JSON.stringify([
+		project.root,
+		project.contentDir,
+		project.discovery,
+		project.publication,
+		project.site,
+		project.cloudflare,
+		project.analytics
+	]);
+}
+export function gitRepositoryRoot(directory: string): string | undefined {
+	try {
+		return execFileSync('git', ['-C', directory, 'rev-parse', '--show-toplevel'], {
+			encoding: 'utf8',
+			stdio: 'pipe'
+		}).trim();
+	} catch {
+		return;
+	}
+}
+export function gitAvailable(directory: string) {
+	return gitRepositoryRoot(directory) !== undefined;
+}
+export function contentWithinProject(project: ResolvedProject) {
+	const relative = path.relative(project.root, project.contentDir);
+	return relative !== '..' && !relative.startsWith('..' + path.sep) && !path.isAbsolute(relative);
+}
+export function contentRelative(project: ResolvedProject, file: string) {
+	const relative = path.relative(project.contentDir, file).split(path.sep).join('/');
+	return relative && relative !== '..' && !relative.startsWith('../') && !path.isAbsolute(relative)
+		? relative
+		: undefined;
+}
+export interface ContentCourse {
+	name: string;
+	directory: string;
+}
+/** Only explicit submodules are course units; folders in a plain vault remain ordinary folders. */
+export function declaredContentCourses(project: ResolvedProject): ContentCourse[] {
+	const modules = path.join(project.root, '.gitmodules');
+	if (project.discovery !== 'git' || !existsSync(modules)) return [];
+	let records: string;
+	try {
+		records = execFileSync(
+			'git',
+			['config', '-z', '--file', modules, '--get-regexp', '^submodule\\..*\\.path$'],
+			{ encoding: 'utf8', stdio: 'pipe' }
+		);
+	} catch (error) {
+		if ((error as { status?: number }).status === 1) return [];
+		throw error;
+	}
+	const courses: ContentCourse[] = [];
+	for (const record of records.split('\0').filter(Boolean)) {
+		const directory = path.resolve(project.root, record.slice(record.indexOf('\n') + 1));
+		const name = contentRelative(project, directory);
+		if (name && !name.includes('/') && /^[\w-]+$/.test(name)) courses.push({ name, directory });
+	}
+	return courses.sort((a, b) => a.name.localeCompare(b.name));
+}
+const projectFiles = new Set([
+	'twiki.config.json',
+	'twiki.config.ts',
+	'package.json',
+	'bun.lock',
+	'bun.lockb',
+	'package-lock.json',
+	'pnpm-lock.yaml',
+	'yarn.lock'
+]);
+function sourceVisible(relative: string) {
+	return (
+		browsablePath(relative) ||
+		(relative.endsWith('/publish.yaml') &&
+			browsablePath(relative.slice(0, -'publish.yaml'.length) + 'publication.yaml')) ||
+		relative === 'publish.yaml'
+	);
+}
+export function projectSourceVisible(project: ResolvedProject, relative: string) {
+	return (
+		sourceVisible(relative) &&
+		!(
+			path.resolve(project.contentDir) === path.resolve(project.root) && projectFiles.has(relative)
+		) &&
+		!(
+			project.configFile &&
+			path.resolve(project.contentDir, relative) === path.resolve(project.configFile)
+		)
+	);
+}
+/** Regular, contained inputs only: never follow symlinks or include engine/editor hidden state. */
+export function discoverSourceFiles(project: ResolvedProject): string[] {
+	if (!existsSync(project.contentDir)) throw new Error('Content directory does not exist');
+	const root = realpathSync(project.contentDir);
+	const paths = new Set<string>();
+	const add = (relative: string) => {
+		if (!projectSourceVisible(project, relative)) return;
+		const file = path.join(project.contentDir, relative);
+		try {
+			if (lstatSync(file).isFile() && realpathSync(file) === path.join(root, relative))
+				paths.add(relative);
+		} catch (error) {
+			if (!['ENOENT', 'ENOTDIR'].includes((error as NodeJS.ErrnoException).code ?? '')) throw error;
+		}
+	};
+	if (project.discovery === 'git') {
+		if (!gitAvailable(project.contentDir))
+			throw new Error('Git discovery requires a Git repository containing the content directory');
+		const tracked = (directory: string) =>
+			execFileSync('git', ['-C', directory, 'ls-files', '-z', '--recurse-submodules'], {
+				maxBuffer: 1 << 28
+			})
+				.toString()
+				.split('\0')
+				.filter(Boolean);
+		for (const relative of tracked(project.contentDir)) add(relative);
+		for (const course of declaredContentCourses(project)) {
+			if (!existsSync(path.join(course.directory, '.git'))) continue;
+			for (const relative of tracked(course.directory)) add(`${course.name}/${relative}`);
+		}
+	} else {
+		const walk = (directory: string, prefix = '') => {
+			for (const entry of readdirSync(directory, { withFileTypes: true }).sort((a, b) =>
+				a.name.localeCompare(b.name)
+			)) {
+				const relative = prefix + entry.name;
+				if (!sourceVisible(relative)) continue;
+				if (entry.isDirectory()) walk(path.join(directory, entry.name), relative + '/');
+				else if (entry.isFile()) add(relative);
+			}
+		};
+		walk(project.contentDir);
+	}
+	return [...paths].sort();
+}
+
+export function projectConfig(repoRoot?: string, environment: NodeJS.ProcessEnv = process.env) {
+	return resolveProject(process.cwd(), repoRoot, environment);
+}

package/site/tooling/lib/public-assets.ts

@@ -0,0 +1,38 @@
+import { readdirSync } from 'node:fs';
+import path from 'node:path';
+import { fileRoute } from '../../src/lib/files';
+import { publicFileBrowserShell } from '../../worker/publication';
+
+export function publicAssetManifest(
+	directory: string,
+	pages: Record<string, string>,
+	fileEntries: { course: string; file: string }[] = []
+) {
+	const assets: Record<string, string> = {};
+	function walk(relative = '') {
+		for (const entry of readdirSync(path.join(directory, relative), { withFileTypes: true })) {
+			const file = relative ? `${relative}/${entry.name}` : entry.name;
+			if (entry.isDirectory()) walk(file);
+			else if (
+				!file.startsWith('.') &&
+				!['_headers', '_redirects', '404.html'].includes(file) &&
+				!file.endsWith('.html')
+			) {
+				const url = '/' + file.split('/').map(encodeURIComponent).join('/');
+				assets[url] = `/_published${url}`;
+			}
+		}
+	}
+	walk();
+	for (const [route, file] of Object.entries(pages)) {
+		if (route === '/404') continue;
+		const target = file === 'index.html' ? '' : file.replace(/\.html$/, '');
+		const url = route.split('/').map(encodeURIComponent).join('/');
+		assets[url] = target
+			? `/_published/${target.split('/').map(encodeURIComponent).join('/')}`
+			: '/_published';
+	}
+	for (const { course, file } of fileEntries)
+		assets[fileRoute(course, file)] = publicFileBrowserShell;
+	return assets;
+}

package/site/tooling/lib/public-links.ts

@@ -0,0 +1,56 @@
+import { site } from '../../src/lib/config';
+import { visitElements } from './visit-elements';
+import type { Root, Element } from 'hast';
+
+export function protectPublicLinks(
+	tree: Root,
+	page: string,
+	allowed: Set<string>,
+	catalog = allowed,
+	origin = site.url
+) {
+	const permitted = (value: string, targets = allowed) => {
+		const url = new URL(value, new URL('/' + page, origin));
+		if (url.origin !== new URL(origin).origin) return true;
+		const target = decodeURIComponent(url.pathname).replace(/^\/|\/$/g, '');
+		return (
+			targets.has(target) ||
+			targets.has(`${target}/index`) ||
+			(target === '' && targets.has('index'))
+		);
+	};
+	visitElements(tree, (node) => {
+		const classes = node.properties.className;
+		if (node.tagName === 'blockquote' && Array.isArray(classes) && classes.includes('transclude')) {
+			const link = node.children[0] as Element | undefined;
+			if (typeof link?.properties?.href === 'string' && !permitted(link.properties.href))
+				throw new Error(
+					`${page}: transclusion requires explicit publication: ${link.properties.href}`
+				);
+		}
+		for (const attribute of ['src', 'poster', 'data', 'srcSet']) {
+			const value = node.properties[attribute];
+			if (typeof value !== 'string') continue;
+			const urls =
+				attribute === 'srcSet'
+					? value.split(',').map((part) => part.trim().split(/\s+/)[0])
+					: [value];
+			for (const url of urls)
+				if (!permitted(url))
+					throw new Error(`${page}: embedded asset requires explicit publication: ${url}`);
+		}
+		if (
+			typeof node.properties.href === 'string' &&
+			!permitted(node.properties.href, node.tagName === 'a' ? catalog : allowed)
+		) {
+			if (node.tagName !== 'a')
+				throw new Error(
+					`${page}: linked resource requires explicit publication: ${node.properties.href}`
+				);
+			delete node.properties.href;
+			delete node.properties['data-slug'];
+			delete node.properties.className;
+			node.tagName = 'span';
+		}
+	});
+}

package/site/tooling/lib/publishing.ts

@@ -0,0 +1,156 @@
+import { existsSync, lstatSync, readFileSync, realpathSync } from 'node:fs';
+import { execFileSync } from 'node:child_process';
+import path from 'node:path';
+import { parseDocument } from 'yaml';
+import {
+	declaredContentCourses,
+	projectSourceVisible,
+	resolveProject,
+	type ResolvedProject
+} from './project-config';
+import type { ContentLicense } from '../../src/lib/content-license';
+
+export interface PublishPolicy {
+	include: string[];
+	exclude: string[];
+	license?: ContentLicense;
+}
+
+function parseLicense(value: unknown): ContentLicense {
+	if (!value || typeof value !== 'object' || Array.isArray(value))
+		throw new Error('license must be a mapping');
+	const fields = value as Record<string, unknown>;
+	if (
+		Object.keys(fields).some(
+			(key) => !['name', 'url', 'attribution', 'source', 'changes'].includes(key)
+		)
+	)
+		throw new Error('Unknown license field');
+	for (const key of [
+		'name',
+		'url',
+		'attribution',
+		'source',
+		...(Object.hasOwn(fields, 'changes') ? ['changes'] : [])
+	]) {
+		if (typeof fields[key] !== 'string' || !(fields[key] as string).trim())
+			throw new Error(`license.${key} must be nonempty text`);
+	}
+	for (const key of ['url', 'source']) {
+		const url = new URL(fields[key] as string);
+		if (url.protocol !== 'https:' || url.username || url.password)
+			throw new Error(`license.${key} must be an HTTPS URL without credentials`);
+	}
+	return fields as unknown as ContentLicense;
+}
+
+export function parsePublishPolicy(source: string): PublishPolicy {
+	const document = parseDocument(source, { uniqueKeys: true, merge: false });
+	if (document.errors.length || document.warnings.length)
+		throw new Error(
+			[...document.errors, ...document.warnings].map((error) => error.message).join('\n')
+		);
+	const value = document.toJS({ maxAliasCount: 0 });
+	if (!value || typeof value !== 'object' || Array.isArray(value))
+		throw new Error('Expected a mapping with include and optional exclude lists');
+	if (Object.keys(value).some((key) => !['include', 'exclude', 'license'].includes(key)))
+		throw new Error('Only include, exclude, and license are supported');
+	for (const key of ['include', 'exclude']) {
+		const patterns = Object.hasOwn(value, key) ? value[key] : key === 'exclude' ? [] : undefined;
+		if (!Array.isArray(patterns)) throw new Error(`${key} must be a list of paths`);
+		for (const pattern of patterns) {
+			if (
+				typeof pattern !== 'string' ||
+				!pattern ||
+				/[\\\x00-\x1f\x7f{}\[\]()!:#]/.test(pattern) ||
+				pattern
+					.split('/')
+					.some(
+						(part: string) =>
+							!part || part.startsWith('.') || (part.includes('**') && part !== '**')
+					)
+			)
+				throw new Error(
+					`Invalid ${key} pattern: ${JSON.stringify(pattern)}; use relative paths with *, **, or ?`
+				);
+		}
+	}
+	return {
+		include: value.include,
+		exclude: value.exclude ?? [],
+		...(Object.hasOwn(value, 'license') ? { license: parseLicense(value.license) } : {})
+	};
+}
+
+export function publicationDecision(file: string, policy?: PublishPolicy) {
+	if (
+		file
+			.split('/')
+			.some((segment) => !segment || segment.startsWith('.') || segment === 'publish.yaml') ||
+		file.includes('\\')
+	)
+		return { public: false, reason: 'Reserved path' };
+	if (!policy) return { public: false, reason: 'No publish.yaml' };
+	const excluded = policy.exclude.find((pattern) => path.posix.matchesGlob(file, pattern));
+	if (excluded) return { public: false, reason: `Excluded by ${excluded}` };
+	const included = policy.include.find((pattern) => path.posix.matchesGlob(file, pattern));
+	return included
+		? { public: true, reason: `Included by ${included}` }
+		: { public: false, reason: 'Not included' };
+}
+
+export function coursePolicies(repo: string, project = resolveProject(process.cwd(), repo)) {
+	const policies = new Map<string, PublishPolicy>();
+	for (const course of declaredContentCourses(project)) {
+		const file = path.join(course.directory, 'publish.yaml');
+		if (!existsSync(file)) continue;
+		try {
+			if (!lstatSync(file).isFile() || realpathSync(file) !== file)
+				throw new Error('Policy must be a regular file');
+			execFileSync(
+				'git',
+				['-C', course.directory, 'ls-files', '--error-unmatch', '--', 'publish.yaml'],
+				{ stdio: 'pipe' }
+			);
+			policies.set(course.name, parsePublishPolicy(readFileSync(file, 'utf8')));
+		} catch (error) {
+			throw new Error(`${file}: invalid or untracked publication policy`, { cause: error });
+		}
+	}
+	return policies;
+}
+
+export function publicationResolver(
+	repo: string,
+	project: ResolvedProject = resolveProject(process.cwd(), repo)
+) {
+	if (project.publication === 'public')
+		return (relative: string) => ({
+			public:
+				projectSourceVisible(project, relative) && !relative.split('/').includes('publish.yaml'),
+			reason: 'Project publication is public; reserved paths stay private.'
+		});
+	if (project.publication === 'private')
+		return (_relative: string) => ({ public: false, reason: 'Project publication is private.' });
+	const policies = coursePolicies(repo, project);
+	return (relative: string) => {
+		const [course, ...segments] = relative.split('/');
+		if (!segments.length)
+			return {
+				public: false,
+				reason: 'Root notes are private; publish.yaml applies to course content.'
+			};
+		const decision = publicationDecision(segments.join('/'), policies.get(course));
+		return { ...decision, reason: `${course}/publish.yaml: ${decision.reason}` };
+	};
+}
+
+export function publicationFilter(repo: string, project = resolveProject(process.cwd(), repo)) {
+	const resolve = publicationResolver(repo, project);
+	return (relative: string) => resolve(relative).public;
+}
+
+export function courseLicenseResolver(repo: string, project = resolveProject(process.cwd(), repo)) {
+	const policies = coursePolicies(repo, project);
+	return (relative: string) => policies.get(relative.split('/')[0])?.license;
+}