@twango/twiki 0.0.0-stage → 0.1.0-rc.0

package/site/tooling/lib/slug.ts

@@ -0,0 +1,377 @@
+/**
+ * URL and wikilink utilities ported from the Wisconsin Quartz fork's path.ts.
+ * Ambiguous links prefer the longest shared directory prefix, then the
+ * shallowest path, then lexicographic order.
+ */
+import { slug as slugAnchor } from 'github-slugger';
+import type { Element as HastElement } from 'hast';
+
+const clone = <T>(x: T): T => structuredClone(x);
+
+/// Utility type to simulate nominal types in TypeScript
+type SlugLike<T> = string & { __brand: T };
+
+/** Cannot be relative and must have a file extension. */
+export type FilePath = SlugLike<'filepath'>;
+export function isFilePath(s: string): s is FilePath {
+	const validStart = !s.startsWith('.');
+	return validStart && _hasFileExtension(s);
+}
+
+/** Cannot be relative and may not have leading or trailing slashes. It can have `index` as it's last segment. Use this wherever possible is it's the most 'general' interpretation of a slug. */
+export type FullSlug = SlugLike<'full'>;
+export function isFullSlug(s: string): s is FullSlug {
+	const validStart = !(s.startsWith('.') || s.startsWith('/'));
+	const validEnding = !s.endsWith('/');
+	return validStart && validEnding && !containsForbiddenCharacters(s);
+}
+
+/** Shouldn't be a relative path and shouldn't have `/index` as an ending or a file extension. It _can_ however have a trailing slash to indicate a folder path. */
+export type SimpleSlug = SlugLike<'simple'>;
+export function isSimpleSlug(s: string): s is SimpleSlug {
+	const validStart = !(s.startsWith('.') || (s.length > 1 && s.startsWith('/')));
+	const validEnding = !endsWith(s, 'index');
+	return validStart && !containsForbiddenCharacters(s) && validEnding && !_hasFileExtension(s);
+}
+
+/** Can be found on `href`s but can also be constructed for client-side navigation (e.g. search and graph) */
+export type RelativeURL = SlugLike<'relative'>;
+export function isRelativeURL(s: string): s is RelativeURL {
+	const validStart = /^\.{1,2}/.test(s);
+	const validEnding = !endsWith(s, 'index');
+	return validStart && validEnding && !['.md', '.html'].includes(getFileExtension(s) ?? '');
+}
+
+export function isAbsoluteURL(s: string): boolean {
+	try {
+		new URL(s);
+	} catch {
+		return false;
+	}
+	return true;
+}
+
+export function sluggify(s: string): string {
+	return s
+		.split('/')
+		.map((segment) =>
+			segment
+				.replace(/\s/g, '-')
+				.replace(/&/g, '-and-')
+				.replace(/%/g, '-percent')
+				.replace(/\?/g, '')
+				.replace(/#/g, '')
+		)
+		.join('/') // always use / as sep
+		.replace(/\/$/, '');
+}
+
+export function slugifyFilePath(fp: FilePath, excludeExt?: boolean): FullSlug {
+	fp = stripSlashes(fp) as FilePath;
+	let ext = getFileExtension(fp);
+	const withoutFileExt = fp.replace(new RegExp(ext + '$'), '');
+	if (excludeExt || ['.md', '.html', undefined].includes(ext)) {
+		ext = '';
+	}
+
+	let slug = sluggify(withoutFileExt);
+
+	// treat _index as index
+	if (endsWith(slug, '_index')) {
+		slug = slug.replace(/_index$/, 'index');
+	}
+
+	return (slug + ext) as FullSlug;
+}
+
+export function simplifySlug(fp: FullSlug): SimpleSlug {
+	const res = stripSlashes(trimSuffix(fp, 'index'), true);
+	return (res.length === 0 ? '/' : res) as SimpleSlug;
+}
+
+export function transformInternalLink(link: string): RelativeURL {
+	let [fplike, anchor] = splitAnchor(decodeURI(link));
+
+	const folderPath = isFolderPath(fplike);
+	let segments = fplike.split('/').filter((x) => x.length > 0);
+	let prefix = segments.filter(isRelativeSegment).join('/');
+	let fp = segments.filter((seg) => !isRelativeSegment(seg) && seg !== '').join('/');
+
+	// manually add ext here as we want to not strip 'index' if it has an extension
+	const simpleSlug = simplifySlug(slugifyFilePath(fp as FilePath));
+	const joined = joinSegments(stripSlashes(prefix), stripSlashes(simpleSlug));
+	const trail = folderPath ? '/' : '';
+	const res = (_addRelativeToStart(joined) + trail + anchor) as RelativeURL;
+	return res;
+}
+
+const _rebaseHastElement = (
+	el: HastElement,
+	attr: string,
+	curBase: FullSlug,
+	newBase: FullSlug
+) => {
+	if (el.properties?.[attr]) {
+		if (!isRelativeURL(String(el.properties[attr]))) {
+			return;
+		}
+
+		const rel = joinSegments(
+			resolveRelative(curBase, newBase),
+			'..',
+			el.properties[attr] as string
+		);
+		el.properties[attr] = rel;
+	}
+};
+
+export function normalizeHastElement(rawEl: HastElement, curBase: FullSlug, newBase: FullSlug) {
+	const el = clone(rawEl); // clone so we dont modify the original page
+	_rebaseHastElement(el, 'src', curBase, newBase);
+	_rebaseHastElement(el, 'href', curBase, newBase);
+	if (el.children) {
+		el.children = el.children.map((child) =>
+			normalizeHastElement(child as HastElement, curBase, newBase)
+		);
+	}
+
+	return el;
+}
+
+// resolve /a/b/c to ../..
+export function pathToRoot(slug: FullSlug): RelativeURL {
+	let rootPath = slug
+		.split('/')
+		.filter((x) => x !== '')
+		.slice(0, -1)
+		.map((_) => '..')
+		.join('/');
+
+	if (rootPath.length === 0) {
+		rootPath = '.';
+	}
+
+	return rootPath as RelativeURL;
+}
+
+export function resolveRelative(current: FullSlug, target: FullSlug | SimpleSlug): RelativeURL {
+	const res = joinSegments(pathToRoot(current), simplifySlug(target as FullSlug)) as RelativeURL;
+	return res;
+}
+
+export function splitAnchor(link: string): [string, string] {
+	let [fp, anchor] = link.split('#', 2);
+	if (fp.endsWith('.pdf')) {
+		return [fp, anchor === undefined ? '' : `#${anchor}`];
+	}
+	anchor = anchor === undefined ? '' : '#' + slugAnchor(anchor);
+	return [fp, anchor];
+}
+
+export function slugTag(tag: string) {
+	return tag
+		.split('/')
+		.map((tagSegment) => sluggify(tagSegment))
+		.join('/');
+}
+
+export function joinSegments(...args: string[]): string {
+	if (args.length === 0) {
+		return '';
+	}
+
+	let joined = args
+		.filter((segment) => segment !== '' && segment !== '/')
+		.map((segment) => stripSlashes(segment))
+		.join('/');
+
+	// if the first segment starts with a slash, add it back
+	if (args[0].startsWith('/')) {
+		joined = '/' + joined;
+	}
+
+	// if the last segment is a folder, add a trailing slash
+	if (args[args.length - 1].endsWith('/')) {
+		joined = joined + '/';
+	}
+
+	return joined;
+}
+
+export function getAllSegmentPrefixes(tags: string): string[] {
+	const segments = tags.split('/');
+	const results: string[] = [];
+	for (let i = 0; i < segments.length; i++) {
+		results.push(segments.slice(0, i + 1).join('/'));
+	}
+	return results;
+}
+
+export interface TransformOptions {
+	strategy: 'absolute' | 'relative' | 'shortest';
+	allSlugs: readonly FullSlug[];
+	slugIndex?: {
+		files: ReadonlyMap<string, readonly FullSlug[]>;
+		folders: ReadonlyMap<string, readonly FullSlug[]>;
+	};
+}
+
+/** Index each path suffix once for a corpus snapshot, preserving candidate order.
+ * Create new options when the corpus changes; no index survives between builds.
+ */
+export function createTransformOptions(
+	allSlugs: readonly FullSlug[],
+	strategy: TransformOptions['strategy'] = 'shortest'
+): TransformOptions {
+	const options: TransformOptions = { strategy, allSlugs: Object.freeze([...allSlugs]) };
+	if (strategy !== 'shortest') return options;
+	const files = new Map<string, FullSlug[]>();
+	const folders = new Map<string, FullSlug[]>();
+	const add = (index: Map<string, FullSlug[]>, candidate: string, slug: FullSlug) => {
+		while (true) {
+			const matches = index.get(candidate);
+			if (matches) matches.push(slug);
+			else index.set(candidate, [slug]);
+			const slash = candidate.indexOf('/');
+			if (slash < 0) break;
+			candidate = candidate.slice(slash + 1);
+		}
+	};
+	for (const slug of options.allSlugs) {
+		add(files, slug, slug);
+		add(folders, stripSlashes(simplifySlug(slug)), slug);
+	}
+	options.slugIndex = { files, folders };
+	return options;
+}
+
+export function transformLink(src: FullSlug, target: string, opts: TransformOptions): RelativeURL {
+	// Explicit relative links are relative to the source note, not the vault root.
+	if (opts.strategy !== 'relative' && /^\.\.?\//.test(target)) {
+		const [relativePath, anchor] = splitAnchor(decodeURI(target));
+		const parts = src.split('/').slice(0, -1);
+		for (const part of relativePath.split('/')) {
+			if (part === '..') parts.pop();
+			else if (part && part !== '.') parts.push(part);
+		}
+		const resolved = slugifyFilePath(parts.join('/') as FilePath);
+		return (resolveRelative(src, resolved) + anchor) as RelativeURL;
+	}
+	let targetSlug = transformInternalLink(target);
+
+	if (opts.strategy === 'relative') {
+		return targetSlug as RelativeURL;
+	} else {
+		const folderTail = isFolderPath(splitAnchor(targetSlug)[0]) ? '/' : '';
+		const canonicalSlug = stripSlashes(targetSlug.slice('.'.length));
+		let [targetCanonical, targetAnchor] = splitAnchor(canonicalSlug);
+		targetCanonical = stripSlashes(targetCanonical);
+
+		if (opts.strategy === 'shortest') {
+			// match by full path suffix on a "/" boundary; for basename-only targets
+			// this is equivalent to matching the last path segment, and for
+			// path-prefix wikilinks like [[folder/file]] it finds slugs whose path
+			// ends in folder/file (mirrors Obsidian's shortest-path resolution).
+			const matchingFileNames = opts.slugIndex
+				? ((folderTail ? opts.slugIndex.folders : opts.slugIndex.files).get(targetCanonical) ?? [])
+				: opts.allSlugs.filter((slug) => {
+						const candidate = folderTail ? stripSlashes(simplifySlug(slug)) : slug;
+						return candidate === targetCanonical || candidate.endsWith('/' + targetCanonical);
+					});
+
+			// only match, just use it
+			if (matchingFileNames.length === 1) {
+				const targetSlug = matchingFileNames[0];
+				return (resolveRelative(src, targetSlug) + targetAnchor) as RelativeURL;
+			}
+
+			// multiple matches: mirror Obsidian's "nearest" resolution.
+			// Prefer the candidate with the longest shared directory prefix to src;
+			// break ties by shallowest path, then lexicographically for stability.
+			if (matchingFileNames.length > 1) {
+				const srcDir = src.split('/').slice(0, -1);
+				const best = matchingFileNames
+					.map((slug) => {
+						const dir = slug.split('/').slice(0, -1);
+						let shared = 0;
+						while (
+							shared < srcDir.length &&
+							shared < dir.length &&
+							srcDir[shared] === dir[shared]
+						) {
+							shared++;
+						}
+						return { slug, shared, depth: dir.length };
+					})
+					.sort(
+						(a, b) => b.shared - a.shared || a.depth - b.depth || a.slug.localeCompare(b.slug)
+					)[0];
+				return (resolveRelative(src, best.slug) + targetAnchor) as RelativeURL;
+			}
+		}
+
+		// if it's not unique, then it's the absolute path from the vault root
+		return (joinSegments(pathToRoot(src), canonicalSlug) + folderTail) as RelativeURL;
+	}
+}
+
+// path helpers
+export function isFolderPath(fplike: string): boolean {
+	return (
+		fplike.endsWith('/') ||
+		endsWith(fplike, 'index') ||
+		endsWith(fplike, 'index.md') ||
+		endsWith(fplike, 'index.html')
+	);
+}
+
+export function endsWith(s: string, suffix: string): boolean {
+	return s === suffix || s.endsWith('/' + suffix);
+}
+
+export function trimSuffix(s: string, suffix: string): string {
+	if (endsWith(s, suffix)) {
+		s = s.slice(0, -suffix.length);
+	}
+	return s;
+}
+
+function containsForbiddenCharacters(s: string): boolean {
+	return s.includes(' ') || s.includes('#') || s.includes('?') || s.includes('&');
+}
+
+function _hasFileExtension(s: string): boolean {
+	return getFileExtension(s) !== undefined;
+}
+
+export function getFileExtension(s: string): string | undefined {
+	return s.match(/\.[A-Za-z0-9]+$/)?.[0];
+}
+
+function isRelativeSegment(s: string): boolean {
+	return /^\.{0,2}$/.test(s);
+}
+
+export function stripSlashes(s: string, onlyStripPrefix?: boolean): string {
+	if (s.startsWith('/')) {
+		s = s.substring(1);
+	}
+
+	if (!onlyStripPrefix && s.endsWith('/')) {
+		s = s.slice(0, -1);
+	}
+
+	return s;
+}
+
+function _addRelativeToStart(s: string): string {
+	if (s === '') {
+		s = '.';
+	}
+
+	if (!s.startsWith('.')) {
+		s = joinSegments('.', s);
+	}
+
+	return s;
+}

package/site/tooling/lib/social-images.ts

@@ -0,0 +1,210 @@
+import satori from 'satori';
+import { stageFingerprint } from '../fingerprint';
+import { Resvg } from '@resvg/resvg-js';
+import { createHash } from 'node:crypto';
+import {
+	copyFileSync,
+	existsSync,
+	mkdirSync,
+	readdirSync,
+	readFileSync,
+	rmSync,
+	writeFileSync
+} from 'node:fs';
+import path from 'node:path';
+import { staticDirectory } from './edition-paths.js';
+import type { ContentManifest, ManifestPage } from '../../src/lib/types';
+import { site } from '../../src/lib/config';
+import { socialImagePath, socialImageSize } from '../../src/lib/social-image';
+import { writeChanged } from './output';
+
+const fontPath = path.resolve(
+	import.meta.dirname,
+	'../../assets/fonts/OverusedGrotesk-SemiBold.ttf'
+);
+const font = readFileSync(fontPath);
+const background = readFileSync(
+	path.resolve(import.meta.dirname, '../../assets/social-background.svg')
+);
+const backgroundImage = `data:image/svg+xml;base64,${background.toString('base64')}`;
+const rendererKey = await stageFingerprint('social');
+
+interface SocialCard {
+	title: string;
+	subtitle?: string;
+	label: string;
+}
+
+function shorten(text: string, length: number) {
+	const normalized = text.replace(/\s+/g, ' ').trim();
+	return normalized.length <= length ? normalized : normalized.slice(0, length - 1).trimEnd() + '…';
+}
+
+export async function renderSocialImage(card: SocialCard) {
+	const title = shorten(card.title, 200);
+	const svg = await satori(
+		{
+			type: 'div',
+			props: {
+				style: {
+					position: 'relative',
+					overflow: 'hidden',
+					display: 'flex',
+					flexDirection: 'column',
+					width: '100%',
+					height: '100%',
+					background: '#1a1916',
+					color: '#e8e5df',
+					padding: '64px 72px',
+					fontFamily: 'Overused Grotesk',
+					fontWeight: 600
+				},
+				children: [
+					{
+						type: 'img',
+						props: {
+							src: backgroundImage,
+							width: socialImageSize.width,
+							height: socialImageSize.height,
+							style: { position: 'absolute', top: 0, left: 0 }
+						}
+					},
+					{
+						type: 'div',
+						props: {
+							style: { color: '#e68578', fontSize: 24 },
+							children: shorten(card.label, 70)
+						}
+					},
+					{
+						type: 'div',
+						props: {
+							style: {
+								display: 'flex',
+								flex: 1,
+								flexDirection: 'column',
+								justifyContent: 'center'
+							},
+							children: [
+								{
+									type: 'div',
+									props: {
+										style: {
+											fontSize: title.length > 130 ? 48 : title.length > 80 ? 60 : 80,
+											lineHeight: 1.08,
+											letterSpacing: '-1.5px',
+											overflowWrap: 'anywhere'
+										},
+										children: title
+									}
+								},
+								{
+									type: 'div',
+									props: {
+										style: { marginTop: 24, fontSize: 26, color: '#7a7568' },
+										children: shorten(card.subtitle ?? '', 90)
+									}
+								}
+							]
+						}
+					},
+					{
+						type: 'div',
+						props: {
+							style: {
+								display: 'flex',
+								justifyContent: 'space-between',
+								fontSize: 20,
+								color: '#7a7568'
+							},
+							children: [
+								{ type: 'span', props: { children: site.name } },
+								{ type: 'span', props: { children: new URL(site.url).host } }
+							]
+						}
+					}
+				]
+			}
+		},
+		{
+			...socialImageSize,
+			fonts: [{ name: 'Overused Grotesk', data: font, weight: 600, style: 'normal' }]
+		}
+	);
+	return new Resvg(svg, { font: { loadSystemFonts: false } }).render().asPng();
+}
+
+function courseLabel(slug: string) {
+	const course = slug.split('/')[0];
+	const match = /^(sp|su|fa|wi)(\d{2})-(.+)$/.exec(course);
+	if (!match) return course === 'des-inv' ? 'Design Innovation Lab' : 'Course notes';
+	const term = { sp: 'Spring', su: 'Summer', fa: 'Fall', wi: 'Winter' }[match[1]];
+	const code = match[3].replace(/([a-z])(\d)/, '$1 $2').toUpperCase();
+	return `${code} · ${term} 20${match[2]}`;
+}
+
+export function socialCard(
+	page: Pick<ManifestPage, 'slug' | 'title' | 'heading' | 'relativePath'>
+) {
+	return {
+		slug: page.slug,
+		title: page.heading?.trim() || page.title,
+		subtitle: path.posix.basename(page.relativePath),
+		label: courseLabel(page.slug)
+	};
+}
+
+export const defaultSocialCard = {
+	slug: undefined,
+	title: site.title,
+	label: 'Notes'
+};
+
+export async function cachedSocialImage(siteDir: string, card: SocialCard & { slug?: string }) {
+	const cache = path.join(siteDir, 'build/generated/cache/social-titles');
+	mkdirSync(cache, { recursive: true });
+	const key = createHash('sha256').update(rendererKey).update(JSON.stringify(card)).digest('hex');
+	const cached = path.join(cache, key + '.png');
+	const rendered = !existsSync(cached);
+	if (rendered) writeFileSync(cached, await renderSocialImage(card));
+	return { cached, rendered };
+}
+
+export async function buildSocialImages(siteDir: string, manifest: Pick<ContentManifest, 'pages'>) {
+	const output = path.join(staticDirectory(siteDir), '_og');
+	mkdirSync(output, { recursive: true });
+	const wanted = new Set<string>();
+	const current = new Set<string>();
+	let rendered = 0;
+	const cards = [defaultSocialCard, ...Object.values(manifest.pages).map(socialCard)];
+	for (const card of cards) {
+		const result = await cachedSocialImage(siteDir, card);
+		const cached = result.cached;
+		current.add(
+			path.relative(path.join(siteDir, 'build/generated/cache'), cached).split(path.sep).join('/')
+		);
+		if (result.rendered) rendered++;
+		const destination = path.join(
+			staticDirectory(siteDir),
+			decodeURIComponent(socialImagePath(card.slug))
+		);
+		mkdirSync(path.dirname(destination), { recursive: true });
+		wanted.add(destination);
+		if (!existsSync(destination) || !readFileSync(destination).equals(readFileSync(cached)))
+			copyFileSync(cached, destination);
+	}
+	for (const file of readdirSync(output, { recursive: true, withFileTypes: true })) {
+		const filename = path.join(file.parentPath, file.name);
+		if (file.isFile() && !wanted.has(filename)) rmSync(filename);
+	}
+	writeChanged(
+		path.join(
+			siteDir,
+			`build/generated/cache/social-titles-current-${process.env.VITE_PUBLIC_EDITION === 'true' ? 'public' : 'full'}.json`
+		),
+		JSON.stringify([...current].sort())
+	);
+	console.log(
+		`social images: ${cards.length} title-only cards (${rendered} rendered, ${cards.length - rendered} cached)`
+	);
+}

package/site/tooling/lib/text-exports.ts

@@ -0,0 +1,51 @@
+import { mkdirSync } from 'node:fs';
+import { writeChanged } from './output';
+import path from 'node:path';
+import type { PageDoc } from '../../src/lib/types';
+import { site } from '../../src/lib/config';
+import { textExportUrl } from '../../src/lib/text-exports';
+
+/** Called separately for each edition; locked outlines never become text exports. */
+export function writeTextExports(directory: string, pages: PageDoc[], assets = new Set<string>()) {
+	const written = new Set<string>();
+	const index = [
+		`# ${site.title}`,
+		'',
+		`> ${site.description}`,
+		'',
+		'Notes are available as UTF-8 Markdown (.md) and plain-text Markdown (.txt).',
+		'Obsidian wikilinks and LaTeX math are preserved. Restricted notes require site access.',
+		'',
+		'## Notes',
+		''
+	];
+	for (const page of [...pages].sort((a, b) => a.slug.localeCompare(b.slug))) {
+		if (page.locked || page.markdown === undefined) continue;
+		const route = page.slug === 'index' ? '' : page.slug.replace(/\/index$/, '');
+		const source = `${site.url}/${route.split('/').map(encodeURIComponent).join('/')}`;
+		const text = `# ${page.title}\n\nSource: ${source}\n\n${page.markdown.trim()}\n`;
+		for (const format of ['md', 'txt'] as const) {
+			const file = path.join(
+				directory,
+				decodeURIComponent(textExportUrl(page.slug, format)).slice(1)
+			);
+			if (
+				assets.has(path.relative(directory, file)) ||
+				written.has(path.relative(directory, file)) ||
+				path.relative(directory, file) === 'llms.txt'
+			)
+				throw new Error(`Text export conflicts with an existing asset: ${page.slug}.${format}`);
+			if (Buffer.byteLength(text) > 25 * 1024 * 1024)
+				throw new Error(`Text export exceeds deployment size limit: ${page.slug}`);
+			mkdirSync(path.dirname(file), { recursive: true });
+			writeChanged(file, text);
+			written.add(path.relative(directory, file));
+		}
+		const title = page.title.replace(/[\r\n]/g, ' ').replace(/[\\[\]]/g, '\\$&');
+		index.push(`- [${title}](${site.url}${textExportUrl(page.slug)})`);
+	}
+	if (assets.has('llms.txt')) throw new Error('Text export conflicts with llms.txt');
+	writeChanged(path.join(directory, 'llms.txt'), index.join('\n') + '\n');
+	written.add('llms.txt');
+	return written;
+}

package/site/tooling/lib/visit-elements.ts

@@ -0,0 +1,17 @@
+import type { Element, Root } from 'hast';
+import { SKIP } from 'unist-util-visit';
+
+/** Preorder HAST traversal for visitors that only mutate the current element.
+ * Child mutations are visited immediately; SKIP prunes the current subtree.
+ * Sibling replacement and index/ancestor callbacks still need unist-util-visit.
+ */
+export function visitElements(
+	tree: Root | Element,
+	visitor: (node: Element) => void | typeof SKIP
+) {
+	if (tree.type === 'element' && visitor(tree) === SKIP) return;
+	for (let index = 0; index < tree.children.length; index++) {
+		const child = tree.children[index];
+		if (child.type === 'element') visitElements(child, visitor);
+	}
+}