@twango/twiki 0.0.0-stage → 0.1.0-rc.0

package/site/src/lib/server/content-provider.ts

@@ -0,0 +1,266 @@
+import type { ContentManifest, NavNode, PageDoc } from '../types';
+import type { FileIconTheme } from '../file-icons';
+import { createContentModel } from './content-model';
+
+export interface ContentContext {
+	edition: 'public' | 'full';
+	snapshot: string;
+	applicationVersion: string;
+}
+
+export interface ContentProvider {
+	context: ContentContext;
+	manifest(): Promise<ContentManifest>;
+	page(slug: string): Promise<PageDoc>;
+	nav(): Promise<NavNode[]>;
+	icons(): Promise<FileIconTheme>;
+	model(): Promise<ReturnType<typeof createContentModel>>;
+}
+
+type Assets = { fetch(request: Request): Promise<Response> };
+type ContentPlatform = { env: { ASSETS: Assets; WISCONSIN_CONTENT_CONTEXT?: ContentContext } };
+const providers = new WeakMap<Assets, Map<string, ContentProvider>>();
+const releaseProvider = new WeakMap<ContentProvider, () => void>();
+const digest = /^[a-f0-9]{64}$/;
+const memoryBudget = 16 * 1024 * 1024;
+const entryOverhead = 512;
+const maximumPendingReads = 32;
+const maximumActiveReads = 2;
+type CacheEntry = {
+	promise: Promise<unknown>;
+	cost: number;
+	pending: boolean;
+	remove: () => void;
+};
+// The budget belongs to the isolate, rather than each binding or snapshot.
+const retained = new Map<CacheEntry, true>();
+let retainedBytes = 0;
+let pendingReads = 0;
+let activeReads = 0;
+const readQueue: Array<() => void> = [];
+
+function discard(entry: CacheEntry) {
+	if (retained.delete(entry)) retainedBytes -= entry.cost;
+	entry.remove();
+}
+
+function touch(entry: CacheEntry) {
+	if (retained.delete(entry)) retained.set(entry, true);
+}
+
+function trim() {
+	for (const entry of retained.keys()) {
+		if (retainedBytes <= memoryBudget) break;
+		if (!entry.pending) discard(entry);
+	}
+}
+
+function readBounded(read: () => Promise<unknown>): Promise<unknown> {
+	if (pendingReads >= maximumPendingReads)
+		return Promise.reject(new Error('Content reads temporarily busy'));
+	pendingReads++;
+	return new Promise((resolve, reject) => {
+		const start = () => {
+			activeReads++;
+			void Promise.resolve()
+				.then(read)
+				.then(resolve, reject)
+				.finally(() => {
+					pendingReads--;
+					activeReads--;
+					readQueue.shift()?.();
+				});
+		};
+		if (activeReads < maximumActiveReads) start();
+		else readQueue.push(start);
+	});
+}
+
+function validContext(context: ContentContext | undefined): context is ContentContext {
+	return (
+		!!context &&
+		(context.edition === 'public' || context.edition === 'full') &&
+		typeof context.snapshot === 'string' &&
+		typeof context.applicationVersion === 'string' &&
+		digest.test(context.snapshot) &&
+		digest.test(context.applicationVersion)
+	);
+}
+
+/** Only canonical compiler slugs become internal asset paths. */
+function pageFile(slug: string) {
+	if (
+		!slug ||
+		slug.includes('\\') ||
+		slug.includes('\0') ||
+		slug.split('/').some((segment) => !segment || segment === '.' || segment === '..')
+	)
+		throw new Error('Invalid content slug');
+	return `pages/${slug.split('/').map(encodeURIComponent).join('/')}.json`;
+}
+
+function provider(
+	context: ContentContext,
+	read: (file: string) => Promise<unknown>
+): ContentProvider {
+	const cache = new Map<string, CacheEntry>();
+	function json<T>(file: string): Promise<T> {
+		const cached = cache.get(file);
+		if (cached) {
+			cache.delete(file);
+			cache.set(file, cached);
+			touch(cached);
+			return cached.promise as Promise<T>;
+		}
+		if (pendingReads >= maximumPendingReads)
+			return Promise.reject(new Error('Content reads temporarily busy'));
+		const entry: CacheEntry = {
+			promise: readBounded(() => read(file)),
+			cost: entryOverhead,
+			pending: true,
+			remove: () => {
+				if (cache.get(file) === entry) cache.delete(file);
+			}
+		};
+		entry.promise = entry.promise.then(
+			(value) => {
+				entry.pending = false;
+				if (cache.get(file) !== entry) return value;
+				// Conservative parsed-object estimate; oversized documents remain usable
+				// by active requests but are never kept for subsequent requests.
+				const cost = JSON.stringify(value).length * 3 + entryOverhead;
+				if (cost > memoryBudget) discard(entry);
+				else {
+					retainedBytes += cost - entry.cost;
+					entry.cost = cost;
+					trim();
+				}
+				return value;
+			},
+			(error) => {
+				discard(entry);
+				throw error;
+			}
+		);
+		cache.set(file, entry);
+		retained.set(entry, true);
+		retainedBytes += entry.cost;
+		// A secondary entry bound also limits bookkeeping for many tiny documents.
+		for (const candidate of cache.values()) {
+			if (cache.size <= 64) break;
+			if (!candidate.pending) discard(candidate);
+		}
+		trim();
+		return entry.promise as Promise<T>;
+	}
+	const result: ContentProvider = {
+		context,
+		manifest: () => json<ContentManifest>('manifest.json'),
+		async page(slug) {
+			const file = pageFile(slug);
+			const manifest = await result.manifest();
+			if (!Object.hasOwn(manifest.pages, slug)) throw new Error('Unknown content page');
+			const page = await json<PageDoc>(file);
+			if (page.slug !== slug) throw new Error('Content page does not match snapshot');
+			if (page.publication.public !== manifest.pages[slug].publication.public)
+				throw new Error('Content publication does not match snapshot');
+			if (
+				context.edition === 'public' &&
+				!page.publication.public &&
+				(!page.locked || page.html || page.markdown || page.description)
+			)
+				throw new Error('Private content in public snapshot');
+			return page;
+		},
+		nav: () => json<NavNode[]>('nav.json'),
+		icons: () => json<FileIconTheme>('file-icons.json'),
+		// Keep projections request-local so they cannot retain an evicted manifest.
+		model: () => result.manifest().then(createContentModel)
+	};
+	releaseProvider.set(result, () => {
+		for (const entry of cache.values()) discard(entry);
+	});
+	return result;
+}
+
+/** Application-only builds may render a 404 shell without a content checkout. */
+export function emptyContentProvider(): ContentProvider {
+	const manifest: ContentManifest = {
+		generatedAt: '',
+		counts: {},
+		pages: {},
+		folders: [],
+		tags: {},
+		assets: [],
+		htmlAssets: [],
+		tree: { name: '', slug: '', title: '', children: [], pages: [] },
+		graph: { nodes: [], links: [] }
+	};
+	const file = { light: 'file', dark: 'file' };
+	const folder = {
+		light: 'folder',
+		dark: 'folder',
+		expanded: { light: 'folder-open', dark: 'folder-open' }
+	};
+	return provider(
+		{ edition: 'public', snapshot: 'application-shell', applicationVersion: 'application-shell' },
+		async (fileName) => {
+			if (fileName === 'manifest.json') return manifest;
+			if (fileName === 'nav.json') return [];
+			if (fileName === 'file-icons.json') return { file, folder, files: {}, folders: {} };
+			throw new Error('No content in application-only build');
+		}
+	);
+}
+
+/** Context is supplied by the authenticated Worker, never by a client header or URL. */
+export function assetContentProvider(platform: ContentPlatform): ContentProvider {
+	const context = platform.env.WISCONSIN_CONTENT_CONTEXT;
+	if (!validContext(context)) throw new Error('Missing trusted content context');
+	const assets = platform.env.ASSETS;
+	if (!assets || typeof assets.fetch !== 'function')
+		throw new Error('Missing content assets binding');
+	let snapshots = providers.get(assets);
+	if (!snapshots) providers.set(assets, (snapshots = new Map()));
+	const key = `${context.edition}:${context.snapshot}:${context.applicationVersion}`;
+	const cached = snapshots.get(key);
+	if (cached) {
+		snapshots.delete(key);
+		snapshots.set(key, cached);
+		return cached;
+	}
+	const selected = Object.freeze({ ...context });
+	const root = `/_content/${selected.edition}/${selected.snapshot}/`;
+	const result = provider(selected, async (file) => {
+		const response = await assets.fetch(
+			new Request(`https://wisconsin-content.internal${root}${file}`)
+		);
+		if (response.status !== 200) throw new Error('Content snapshot unavailable');
+		return response.json();
+	});
+	snapshots.set(key, result);
+	// Keep a bounded number of snapshots without sharing public/full data.
+	if (snapshots.size > 2) {
+		const oldest = snapshots.keys().next().value!;
+		releaseProvider.get(snapshots.get(oldest)!)?.();
+		snapshots.delete(oldest);
+	}
+	return result;
+}
+
+/** Async filesystem provider is exclusively selected by local development/static export. */
+export function localContentProvider(
+	site: string,
+	edition: ContentContext['edition']
+): ContentProvider {
+	return provider({ edition, snapshot: 'local', applicationVersion: 'local' }, async (file) => {
+		const { readFile } = await import('node:fs/promises');
+		const relative =
+			file === 'manifest.json'
+				? 'build/generated/content-manifest.json'
+				: file === 'nav.json' || file === 'file-icons.json'
+					? `src/lib/generated/${file}`
+					: `build/generated/${file.split('/').map(decodeURIComponent).join('/')}`;
+		return JSON.parse(await readFile(`${site}/${relative}`, 'utf8'));
+	});
+}

package/site/src/lib/server/content.ts

@@ -0,0 +1,277 @@
+/**
+ * Cached readers for generated content used during prerendering and development.
+ * Keep imports relative: the content pipeline also loads this module outside SvelteKit.
+ */
+
+import { existsSync, readFileSync } from 'node:fs';
+import path from 'node:path';
+import type {
+	ContentManifest,
+	FolderEntry,
+	FolderListing,
+	ManifestPage,
+	NavNode,
+	PageDoc,
+	TreeNode
+} from '../types';
+import { site } from '../config';
+
+/** `vite`/`bun` are always invoked from `site/`, so cwd-relative is stable. */
+const GENERATED_DIR = path.resolve(process.cwd(), 'build/generated');
+
+let manifestCache: ContentManifest | undefined;
+
+export function resetContentCache() {
+	manifestCache = undefined;
+	pageDocCache.clear();
+}
+
+export function getManifest(): ContentManifest {
+	if (!manifestCache) {
+		const file = path.join(GENERATED_DIR, 'content-manifest.json');
+		if (!existsSync(file)) {
+			throw new Error(`Missing ${file} — run \`bun run content:prepare\` in site/.`);
+		}
+		manifestCache = JSON.parse(readFileSync(file, 'utf-8')) as ContentManifest;
+	}
+	return manifestCache;
+}
+
+/**
+ * Canonical slug → display route (no leading/trailing slash; `''` = home).
+ * Trailing `index` segments are trimmed from URLs.
+ */
+export function displayRoute(slug: string): string {
+	if (slug === 'index') return '';
+	if (slug.endsWith('/index')) return slug.slice(0, -'/index'.length);
+	return slug;
+}
+
+/** Display route → canonical page slug, or undefined if no page lives there. */
+export function pageSlugForRoute(route: string): string | undefined {
+	const { pages } = getManifest();
+	if (route === '') return pages['index'] ? 'index' : undefined;
+	if (pages[route] && !route.endsWith('/index')) return route;
+	if (pages[`${route}/index`]) return `${route}/index`;
+	// A literal `…/index` URL is not canonical (the live site never emits it).
+	return undefined;
+}
+
+const pageDocCache = new Map<string, PageDoc>();
+
+/** Read a full page document (rendered HTML + TOC + backlinks) by slug. */
+export function loadPage(slug: string): PageDoc {
+	let doc = pageDocCache.get(slug);
+	if (!doc) {
+		const file = path.join(GENERATED_DIR, 'pages', `${slug}.json`);
+		doc = JSON.parse(readFileSync(file, 'utf-8')) as PageDoc;
+		pageDocCache.set(slug, doc);
+	}
+	return doc;
+}
+
+/** Folders that have no index page and therefore get a generated listing. */
+export function listingFolders(): string[] {
+	const m = getManifest();
+	return m.folders.filter((f) => !m.pages[`${f}/index`] && !m.pages[f]);
+}
+
+/**
+ * Every route served by the catch-all `[...slug]` page route:
+ * all page display routes (including `''` = home) + generated folder listings.
+ * Tag routes live under `src/routes/tags/` and are not included here.
+ */
+export function contentEntries(): string[] {
+	const m = getManifest();
+	const routes = Object.keys(m.pages).map(displayRoute);
+	return [...new Set(['', ...routes, ...listingFolders()])];
+}
+
+/** Natural sort, Quartz Explorer-style (numeric, case-insensitive). */
+const collator = new Intl.Collator(undefined, { numeric: true, sensitivity: 'base' });
+
+function findTreeNode(folder: string): TreeNode | undefined {
+	let node: TreeNode | undefined = getManifest().tree;
+	if (folder === '') return node;
+	for (const segment of folder.split('/')) {
+		node = node?.children.find((c) => c.name === segment);
+		if (!node) return undefined;
+	}
+	return node;
+}
+
+function toNavNode(node: TreeNode): NavNode {
+	const indexSlug = `${node.slug}/index`;
+	const folders = node.children.map(toNavNode).sort((a, b) => collator.compare(a.title, b.title));
+	const pages = node.pages
+		.filter((p) => p.slug !== indexSlug)
+		.map(
+			(p): NavNode => ({
+				locked: getManifest().pages[p.slug]?.locked,
+				title: p.title,
+				route: `/${displayRoute(p.slug)}`,
+				children: [],
+				segment: p.slug.split('/').pop() ?? p.slug
+			})
+		)
+		.sort((a, b) => collator.compare(a.title, b.title));
+	return {
+		locked: getManifest().pages[indexSlug]?.locked,
+		// Explorer shows folder names (matching the live Quartz explorer), not
+		// index-page titles — those can be long ("Midterm 1 - Practice Questions").
+		title: node.name,
+		route: `/${node.slug}`, // every folder has a route: index page or listing
+		children: [...folders, ...pages],
+		segment: node.name
+	};
+}
+
+let navCache: NavNode[] | undefined;
+
+/** Sidebar Explorer tree: course folders first, then root pages (minus home). */
+export function navTree(): NavNode[] {
+	if (navCache) return navCache;
+	const root = getManifest().tree;
+	const folders = root.children.map(toNavNode).sort((a, b) => collator.compare(a.title, b.title));
+	const rootPages = root.pages
+		.filter((p) => p.slug !== 'index')
+		.map(
+			(p): NavNode => ({
+				title: p.title,
+				route: `/${displayRoute(p.slug)}`,
+				children: [],
+				segment: p.slug
+			})
+		)
+		.sort((a, b) => collator.compare(a.title, b.title));
+	navCache = [...folders, ...rootPages];
+	return navCache;
+}
+
+function countPages(node: TreeNode): number {
+	return node.pages.length + node.children.reduce((n, c) => n + countPages(c), 0);
+}
+
+/** Listing data for a folder without its own index page. */
+export function folderListing(folder: string): FolderListing | undefined {
+	const m = getManifest();
+	if (folder !== '' && !m.folders.includes(folder)) return undefined;
+	const node = findTreeNode(folder);
+	if (!node) return undefined;
+	const subfolders: FolderEntry[] = node.children
+		.map((c) => ({ title: c.name, route: `/${c.slug}`, isFolder: true }))
+		.sort((a, b) => collator.compare(a.title, b.title));
+	const pages: FolderEntry[] = node.pages
+		.filter((p) => p.slug !== `${folder}/index`)
+		.map((p) => ({
+			title: p.title,
+			route: `/${displayRoute(p.slug)}`,
+			isFolder: false,
+			modified: m.pages[p.slug]?.dates.modified
+		}))
+		// Quartz folder pages list by date (newest first), then alphabetical.
+		.sort(
+			(a, b) =>
+				Date.parse(b.modified ?? '0') - Date.parse(a.modified ?? '0') ||
+				collator.compare(a.title, b.title)
+		);
+	return {
+		folder,
+		name: folder === '' ? site.name : (folder.split('/').pop() ?? folder),
+		entries: [
+			...subfolders,
+			...pages,
+			...(m.fileCourses?.includes(folder)
+				? [{ title: 'Files', route: `/${folder}/files`, isFolder: true }]
+				: [])
+		],
+		pageCount: countPages(node)
+	};
+}
+
+/** All tags (case-sensitive) with page counts, alphabetical. */
+export function tagIndex(): { tag: string; count: number }[] {
+	const { tags } = getManifest();
+	return Object.keys(tags)
+		.sort((a, b) => collator.compare(a, b))
+		.map((tag) => ({ tag, count: tags[tag].length }));
+}
+
+/** Pages for one tag (case-sensitive), newest first. */
+export function tagPages(tag: string): ManifestPage[] | undefined {
+	const m = getManifest();
+	const slugs = m.tags[tag];
+	if (!slugs) return undefined;
+	return slugs
+		.map((s) => m.pages[s])
+		.filter((p): p is ManifestPage => !!p)
+		.sort(
+			(a, b) =>
+				Date.parse(b.dates.modified) - Date.parse(a.dates.modified) ||
+				collator.compare(a.title, b.title)
+		);
+}
+
+function escapeXml(s: string): string {
+	return s
+		.replaceAll('&', '&amp;')
+		.replaceAll('<', '&lt;')
+		.replaceAll('>', '&gt;')
+		.replaceAll('"', '&quot;');
+}
+
+function absoluteUrl(route: string): string {
+	return route === '' ? `${site.url}/` : `${site.url}/${route}`;
+}
+
+/** RSS feed matching the live `/index.xml` shape: latest 10 by modified date. */
+export function rssXml(): string {
+	const m = getManifest();
+	const items = Object.values(m.pages)
+		.filter((page) => !page.locked)
+		.sort((a, b) => Date.parse(b.dates.modified) - Date.parse(a.dates.modified))
+		.slice(0, 10)
+		.map((p) => {
+			const url = absoluteUrl(displayRoute(p.slug));
+			const description = p.description.replaceAll(']]>', ']]&gt;');
+			return `<item>
+    <title>${escapeXml(p.title)}</title>
+    <link>${url}</link>
+    <guid>${url}</guid>
+    <description><![CDATA[ ${description} ]]></description>
+    ${p.dates.published || p.dates.modified ? `<pubDate>${new Date(p.dates.published || p.dates.modified).toUTCString()}</pubDate>` : ''}
+  </item>`;
+		})
+		.join('');
+	return `<?xml version="1.0" encoding="UTF-8" ?>
+<rss version="2.0">
+    <channel>
+      <title>${escapeXml(site.title)}</title>
+      <link>${site.url}</link>
+      <description>Last 10 notes on ${escapeXml(site.title)}</description>
+      <generator>${site.url}</generator>
+      ${items}
+    </channel>
+  </rss>`;
+}
+
+/** Sitemap matching the live `/sitemap.xml` shape: every page, ISO lastmod. */
+export function sitemapXml(): string {
+	const m = getManifest();
+	const urls = Object.values(m.pages)
+		.filter((page) => !page.locked)
+		.map(
+			(p) => `<url>
+    <loc>${escapeXml(absoluteUrl(displayRoute(p.slug)))}</loc>
+    ${p.dates.modified ? `<lastmod>${p.dates.modified}</lastmod>` : ''}
+  </url>`
+		)
+		.join('');
+	const folders = m.fileCourses
+		? contentEntries()
+				.filter((route) => !pageSlugForRoute(route))
+				.map((route) => `<url><loc>${escapeXml(absoluteUrl(route))}</loc></url>`)
+				.join('')
+		: '';
+	return `<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:xhtml="http://www.w3.org/1999/xhtml">${urls}${folders}</urlset>`;
+}

package/site/src/lib/server/runtime-content.ts

@@ -0,0 +1,34 @@
+import { building, dev } from '$app/environment';
+import { error } from '@sveltejs/kit';
+import {
+	assetContentProvider,
+	emptyContentProvider,
+	localContentProvider
+} from './content-provider';
+
+export function requestContent(platform: App.Platform | undefined) {
+	if (building && import.meta.env.VITE_STATIC_EXPORT !== 'true') return emptyContentProvider();
+	if (dev || building)
+		return localContentProvider(
+			process.cwd(),
+			import.meta.env.VITE_PUBLIC_EDITION === 'true' ? 'public' : 'full'
+		);
+	if (!platform) error(503, 'Content unavailable');
+	try {
+		const content = assetContentProvider(platform);
+		const available = <T>(read: Promise<T>): Promise<T> =>
+			read.catch(() => {
+				error(503, 'Content unavailable');
+			});
+		return {
+			context: content.context,
+			manifest: () => available(content.manifest()),
+			page: (slug: string) => available(content.page(slug)),
+			nav: () => available(content.nav()),
+			icons: () => available(content.icons()),
+			model: () => available(content.model())
+		};
+	} catch {
+		error(503, 'Content unavailable');
+	}
+}

package/site/src/lib/site-config.json

@@ -0,0 +1,9 @@
+{
+	"name": "twiki",
+	"url": "http://localhost:5173",
+	"title": "twiki",
+	"description": "Notes, ideas, and reference material.",
+	"author": { "name": "", "url": "", "github": "", "email": "" },
+	"locale": "en_US",
+	"language": "en"
+}

package/site/src/lib/social-image.ts

@@ -0,0 +1,13 @@
+import { site } from './config';
+
+export const socialImageSize = { width: 1200, height: 630 };
+
+export function socialImagePath(slug?: string) {
+	return slug
+		? `/_og/notes/${slug.split('/').map(encodeURIComponent).join('/')}.png`
+		: '/_og/default.png';
+}
+
+export function socialImageUrl(slug?: string) {
+	return site.url + socialImagePath(slug);
+}