@twango/twiki 0.0.0-stage → 0.1.1

package/site/tooling/lib/history-pool.ts

@@ -0,0 +1,74 @@
+import { spawn } from 'node:child_process';
+import { availableParallelism } from 'node:os';
+import path from 'node:path';
+import type { CourseFile } from '../../src/lib/files';
+
+export type HistoryJob = {
+	repo: string;
+	output: string;
+	cache: string;
+	files: { file: CourseFile; source: string }[];
+};
+
+export async function buildHistories(
+	jobs: HistoryJob[],
+	retain: (job: HistoryJob, file: string) => void
+) {
+	if (!jobs.length) return;
+	const requested = Number(
+		process.env.WISCONSIN_HISTORY_WORKERS ?? Math.min(8, availableParallelism())
+	);
+	if (!Number.isInteger(requested) || requested < 1 || requested > 48)
+		throw new Error('WISCONSIN_HISTORY_WORKERS must be an integer from 1 to 48');
+	const started = performance.now();
+	let next = 0;
+	let failure: unknown;
+	const results = await Promise.allSettled(
+		Array.from({ length: Math.min(requested, jobs.length) }, async () => {
+			while (!failure && next < jobs.length) {
+				const job = jobs[next++];
+				try {
+					const result = await new Promise<{ histories: (string | null)[]; outputs: string[] }>(
+						(resolve, reject) => {
+							const child = spawn('bun', [path.join(import.meta.dirname, 'history-worker.ts')], {
+								stdio: ['pipe', 'pipe', 'pipe']
+							});
+							let stdout = '',
+								stderr = '';
+							child.stdout.setEncoding('utf8').on('data', (chunk) => {
+								stdout += chunk;
+							});
+							child.stderr.setEncoding('utf8').on('data', (chunk) => {
+								stderr += chunk;
+							});
+							child.on('error', reject);
+							child.stdin.on('error', reject);
+							child.on('close', (code) => {
+								if (code !== 0)
+									return reject(new Error(`History worker failed for ${job.repo}: ${stderr}`));
+								try {
+									resolve(JSON.parse(stdout));
+								} catch (error) {
+									reject(error);
+								}
+							});
+							child.stdin.end(JSON.stringify(job));
+						}
+					);
+					job.files.forEach(({ file }, i) => {
+						file.history = result.histories[i] ?? undefined;
+					});
+					for (const output of result.outputs) retain(job, output);
+				} catch (error) {
+					failure = error;
+					throw error;
+				}
+			}
+		})
+	);
+	const failed = results.find((result) => result.status === 'rejected');
+	if (failed?.status === 'rejected') throw failed.reason;
+	console.log(
+		`history: ${jobs.length} courses with ${Math.min(requested, jobs.length)} workers in ${((performance.now() - started) / 1000).toFixed(2)}s`
+	);
+}

package/site/tooling/lib/history-worker.ts

@@ -0,0 +1,22 @@
+import { readFileSync } from 'node:fs';
+import { createFileHistoryBuilder } from './file-history';
+import { browsablePath, fileHistoryPolicyKey } from './file-policy';
+import type { CourseFile } from '../../src/lib/files';
+
+const job: {
+	repo: string;
+	output: string;
+	cache: string;
+	files: { file: CourseFile; source: string }[];
+} = JSON.parse(readFileSync(0, 'utf8'));
+const outputs = new Set<string>();
+const build = createFileHistoryBuilder(
+	job.repo,
+	job.output,
+	job.cache,
+	browsablePath,
+	fileHistoryPolicyKey(),
+	(file) => outputs.add(file)
+);
+const histories = job.files.map(({ file, source }) => build?.(file, readFileSync(source)) ?? null);
+process.stdout.write(JSON.stringify({ histories, outputs: [...outputs] }));

package/site/tooling/lib/knit-rmd.R

@@ -0,0 +1,38 @@
+args <- commandArgs(trailingOnly = TRUE)
+input <- normalizePath(args[[1]])
+output <- args[[2]]
+figures <- args[[3]]
+
+library(knitr)
+render_markdown()
+opts_knit$set(root.dir = dirname(input))
+opts_chunk$set(dev = "png", fig.path = paste0(figures, "/"), error = FALSE, cache = FALSE)
+# Keep plots as Markdown even when a worksheet sets HTML/PDF sizing options.
+# The site renderer handles typography, math, and responsive image sizing.
+knit_hooks$set(plot = function(x, options) {
+  caption <- if (is.null(options$fig.cap)) paste("Plot from", options$label) else options$fig.cap
+  caption <- gsub("[\r\n]", " ", paste(caption, collapse = " "))
+  paste0("\n\n![", caption, "](", x, ")\n\n")
+})
+set.seed(1)
+# Worksheets often include setup instructions for their interactive session.
+# Builds use packages provisioned beforehand, without updating shared libraries.
+# This binding applies to unqualified worksheet calls, not utils::install.packages.
+worksheet <- new.env(parent = globalenv())
+worksheet$install.packages <- function(pkgs, lib = .libPaths(), ...) {
+  if (missing(pkgs) || !is.character(pkgs) || anyNA(pkgs) ||
+      any(!grepl("^[A-Za-z][A-Za-z0-9.]*$", pkgs))) {
+    stop("Worksheet package setup requires names of preinstalled R packages.", call. = FALSE)
+  }
+  available <- rownames(utils::installed.packages(lib.loc = lib))
+  missing_packages <- setdiff(pkgs, available)
+  if (length(missing_packages)) {
+    stop(paste0(
+      "Missing preinstalled R packages: ", paste(missing_packages, collapse = ", "),
+      ". Install worksheet dependencies in the build environment before rendering; ",
+      "worksheet package installation is disabled."
+    ), call. = FALSE)
+  }
+  invisible(NULL)
+}
+knit(input, output = output, envir = worksheet, quiet = TRUE)

package/site/tooling/lib/lastmod.ts

@@ -0,0 +1,201 @@
+import fs from 'node:fs';
+import { stageFingerprint } from '../fingerprint';
+import path from 'node:path';
+import { execFileSync } from 'node:child_process';
+import {
+	resolveProject,
+	declaredContentCourses,
+	contentRelative,
+	gitRepositoryRoot
+} from './project-config';
+
+const datesVersion = await stageFingerprint('gitdates');
+
+export interface SubmoduleInfo {
+	path: string;
+	fullPath: string;
+}
+
+export function parseGitmodules(gitmodulesPath: string): SubmoduleInfo[] {
+	try {
+		const content = fs.readFileSync(gitmodulesPath, 'utf8');
+		const submodules: SubmoduleInfo[] = [];
+		const lines = content.split('\n');
+		let currentPath = '';
+
+		for (const line of lines) {
+			const trimmed = line.trim();
+			if (trimmed.startsWith('path = ')) {
+				currentPath = trimmed.substring(7).trim();
+				if (currentPath) {
+					submodules.push({
+						path: currentPath,
+						fullPath: path.resolve(path.dirname(gitmodulesPath), currentPath)
+					});
+				}
+			}
+		}
+		return submodules.sort((a, b) => b.fullPath.length - a.fullPath.length);
+	} catch {
+		return [];
+	}
+}
+
+function git(repo: string, args: string[], maxBuffer = 1024 * 1024 * 512): string {
+	return execFileSync('git', ['-C', repo, ...args], { encoding: 'utf8', maxBuffer });
+}
+
+export function repositoryDates(repoDir: string): GitDateMap {
+	const out = git(repoDir, [
+		'-c',
+		'core.quotepath=false',
+		'log',
+		'--reverse',
+		'-m',
+		'--first-parent',
+		'--format=%H %ct',
+		'--name-status',
+		'-M'
+	]);
+	const dates: GitDateMap = { created: {}, modified: {} };
+	let currentEpoch: number | null = null;
+	for (const line of out.split('\n')) {
+		if (line === '') continue;
+		const m = line.match(/^[0-9a-f]{40} (\d+)$/);
+		if (m) {
+			currentEpoch = Number(m[1]);
+			continue;
+		}
+		const [status, file, renamed] = line.split('\t');
+		if (currentEpoch === null || !file) continue;
+		if (status.startsWith('R') && renamed) {
+			dates.created[renamed] = dates.created[file] ?? currentEpoch;
+			dates.modified[renamed] = currentEpoch;
+			delete dates.created[file];
+			delete dates.modified[file];
+		} else if (status === 'D') {
+			delete dates.created[file];
+			delete dates.modified[file];
+		} else {
+			if (status === 'A' || dates.created[file] === undefined) dates.created[file] = currentEpoch;
+			dates.modified[file] = currentEpoch;
+		}
+	}
+	return dates;
+}
+
+function headSha(repoDir: string): string {
+	return git(repoDir, ['rev-parse', 'HEAD']).trim();
+}
+
+export interface GitDateMap {
+	created: Record<string, number>;
+	modified: Record<string, number>;
+}
+
+export function buildGitDateMap(
+	repoRoot: string,
+	cacheDir: string,
+	project = resolveProject(process.cwd(), repoRoot)
+): GitDateMap {
+	fs.mkdirSync(cacheDir, { recursive: true });
+	const submodules = declaredContentCourses(project);
+
+	const created: Record<string, number> = {};
+	const modified: Record<string, number> = {};
+
+	const loadRepo = (repoDir: string, cacheKey: string): GitDateMap => {
+		let head: string;
+		try {
+			head = headSha(repoDir);
+		} catch {
+			return { created: {}, modified: {} };
+		}
+		const cacheFile = path.join(cacheDir, `gitdates-v3-${cacheKey}-${datesVersion}-${head}.json`);
+		if (fs.existsSync(cacheFile)) {
+			try {
+				const saved = JSON.parse(fs.readFileSync(cacheFile, 'utf8'));
+				if (!saved.created || !saved.modified) throw new Error('Invalid Git date cache');
+				return saved;
+			} catch {
+				/* fall through to rebuild */
+			}
+		}
+		const dates = repositoryDates(repoDir);
+		for (const f of fs.readdirSync(cacheDir)) {
+			if (f.startsWith(`gitdates-v3-${cacheKey}-`)) fs.rmSync(path.join(cacheDir, f));
+		}
+		fs.writeFileSync(cacheFile, JSON.stringify(dates));
+		return dates;
+	};
+	for (const sub of submodules) {
+		const name = sub.name;
+		if (!fs.existsSync(path.join(sub.directory, '.git'))) continue;
+		const dates = loadRepo(sub.directory, name);
+		for (const [p, epoch] of Object.entries(dates.modified)) {
+			modified[`${name}/${p}`] = epoch;
+			created[`${name}/${p}`] = dates.created[p];
+		}
+	}
+	const dateRoot = gitRepositoryRoot(project.contentDir) ?? repoRoot;
+	const superDates =
+		project.discovery === 'git'
+			? loadRepo(dateRoot, 'superproject')
+			: { created: {}, modified: {} };
+	for (const [p, epoch] of Object.entries(superDates.modified)) {
+		const rel = contentRelative(project, path.join(dateRoot, p));
+		if (rel) {
+			if (!submodules.some((s) => rel === s.name || rel.startsWith(s.name + '/'))) {
+				modified[rel] = epoch;
+				created[rel] = superDates.created[p];
+			}
+		}
+	}
+
+	return { created, modified };
+}
+
+export function coerceDate(
+	file: string,
+	value: unknown,
+	warn?: (message: string) => void
+): Date | undefined {
+	if (value === undefined || value === null || value === '') return;
+	const dateOnly = typeof value === 'string' && /^\d{4}-\d{2}-\d{2}$/.test(value);
+	const supported =
+		value instanceof Date ||
+		typeof value === 'number' ||
+		(typeof value === 'string' &&
+			(dateOnly || /^\d{4}-\d{2}-\d{2}T.*(?:Z|[+-]\d{2}:\d{2})$/.test(value)));
+	const date = supported ? new Date(value as string | number) : new Date(NaN);
+	if (
+		!Number.isFinite(date.getTime()) ||
+		date.getTime() > Date.now() ||
+		(dateOnly && date.toISOString().slice(0, 10) !== value)
+	) {
+		warn?.(`${file}: ignored invalid or future date ${String(value)}`);
+		return;
+	}
+	return date;
+}
+
+export function resolveDates({
+	relativePath,
+	frontmatter,
+	gitDates,
+	warn
+}: {
+	relativePath: string;
+	frontmatter: Record<string, unknown>;
+	gitDates: GitDateMap;
+	warn?: (message: string) => void;
+}) {
+	const fromGit = (kind: 'created' | 'modified') => {
+		const epoch = gitDates[kind][relativePath];
+		return epoch === undefined ? undefined : coerceDate(relativePath, epoch * 1000, warn);
+	};
+	const created = coerceDate(relativePath, frontmatter.created, warn) ?? fromGit('created');
+	const modified = coerceDate(relativePath, frontmatter.modified, warn) ?? fromGit('modified');
+	const published = coerceDate(relativePath, frontmatter.published, warn);
+	return { created, modified, published };
+}

package/site/tooling/lib/markdown-files.ts

@@ -0,0 +1,51 @@
+import { execFileSync } from 'node:child_process';
+import { lstatSync, existsSync, readdirSync } from 'node:fs';
+import path from 'node:path';
+
+/** Include tracked and untracked, non-ignored Markdown, descending into submodules. */
+export function markdownFiles(inputs: string[], extension = /\.md$/i): string[] {
+	const files = new Set<string>();
+	function collect(input: string) {
+		const absolute = path.resolve(input);
+		const stat = lstatSync(absolute);
+		if (stat.isSymbolicLink()) return;
+		if (stat.isFile()) {
+			if (!extension.test(absolute)) throw new Error(`Expected Markdown: ${input}`);
+			files.add(absolute);
+			return;
+		}
+		let names: string[];
+		try {
+			names = execFileSync(
+				'git',
+				['-C', absolute, 'ls-files', '--cached', '--others', '--exclude-standard', '-z'],
+				{ encoding: 'utf8', maxBuffer: 32 * 1024 * 1024 }
+			).split('\0');
+		} catch (error) {
+			// A plain Obsidian folder is a supported source; permission failures remain errors.
+			const detail = String((error as { stderr?: unknown }).stderr ?? '');
+			if (!detail.includes('not a git repository')) throw error;
+			for (const name of readdirSync(absolute)) {
+				if (name.startsWith('.') || ['node_modules', 'build', 'dist'].includes(name)) continue;
+				const full = path.join(absolute, name),
+					entry = lstatSync(full);
+				if (entry.isDirectory()) collect(full);
+				else if (entry.isFile() && extension.test(name)) files.add(full);
+			}
+			return;
+		}
+		for (const name of new Set(names.filter(Boolean))) {
+			const full = path.join(absolute, name);
+			if (!existsSync(full)) continue; // tracked deletion in the working tree
+			const entry = lstatSync(full);
+			if (entry.isDirectory()) {
+				if (!existsSync(path.join(full, '.git'))) {
+					throw new Error(`Submodule is not initialized: ${full}`);
+				}
+				collect(full);
+			} else if (entry.isFile() && extension.test(name)) files.add(full);
+		}
+	}
+	inputs.forEach(collect);
+	return [...files].sort();
+}