@twango/twiki 0.0.0-stage → 0.1.0-rc.0

package/site/tooling/lib/deployment-upload.ts

@@ -0,0 +1,293 @@
+import {
+	configSha256,
+	deploymentError,
+	equalJSON,
+	record,
+	type DeploymentConfig
+} from './deployment-config';
+import {
+	ASSET_HASH,
+	validateAssetInventory,
+	verifyAssetBytes,
+	type DeploymentAsset
+} from './deployment-manifest';
+import {
+	validatePreparedWorker,
+	verifyModuleBytes,
+	type DeploymentSnapshot,
+	type PreparedModule
+} from './deployment-prepare';
+
+export interface DeploymentAPI {
+	request(endpoint: string, init?: RequestInit, bearer?: string): Promise<unknown>;
+}
+
+/** Official API, bounded retry, sanitized errors. Caller must explicitly invoke --upload. */
+export class CloudflareDeploymentAPI implements DeploymentAPI {
+	constructor(
+		private readonly token: string,
+		private readonly fetcher: (input: string, init: RequestInit) => Promise<Response> = fetch,
+		private readonly sleep: (milliseconds: number) => Promise<void> = (milliseconds) =>
+			new Promise((resolve) => setTimeout(resolve, milliseconds))
+	) {
+		if (!token || /[\r\n]/.test(token)) deploymentError('missing API credential');
+	}
+
+	async request(endpoint: string, init: RequestInit = {}, bearer = this.token): Promise<unknown> {
+		if (!endpoint.startsWith('/accounts/') || endpoint.includes('://') || /[\r\n]/.test(endpoint))
+			deploymentError('invalid API endpoint');
+		for (let attempt = 0; attempt < 4; attempt++) {
+			let response: Response;
+			try {
+				response = await this.fetcher(`https://api.cloudflare.com/client/v4${endpoint}`, {
+					...init,
+					redirect: 'error',
+					signal: AbortSignal.timeout(30_000),
+					headers: {
+						...Object.fromEntries(new Headers(init.headers)),
+						Authorization: `Bearer ${bearer}`
+					}
+				});
+			} catch {
+				if (attempt === 3) deploymentError('API transport failed');
+				await this.sleep(Math.min(1000 * 2 ** attempt, 10_000));
+				continue;
+			}
+			if (response.status === 429 || response.status >= 500) {
+				if (attempt === 3) deploymentError(`API transient failure (${response.status})`);
+				const retryAfter = response.headers.get('Retry-After');
+				const seconds = retryAfter === null ? NaN : Number(retryAfter);
+				const delay = Number.isFinite(seconds)
+					? seconds * 1000
+					: retryAfter
+						? Date.parse(retryAfter) - Date.now()
+						: NaN;
+				await response.body?.cancel();
+				await this.sleep(
+					Math.max(0, Math.min(Number.isFinite(delay) ? delay : 1000 * 2 ** attempt, 30_000))
+				);
+				continue;
+			}
+			if (!response.ok) deploymentError(`API request failed (${response.status})`);
+			let envelope: unknown;
+			try {
+				envelope = await response.json();
+			} catch {
+				deploymentError('invalid API response');
+			}
+			if (!record(envelope) || envelope.success !== true || !('result' in envelope))
+				deploymentError('API operation failed');
+			// Domain listings must be complete; fail instead of silently accepting a truncated collection.
+			if (
+				/\/workers\/domains(?:\?|$)/.test(endpoint) &&
+				record(envelope.result_info) &&
+				typeof envelope.result_info.total_pages === 'number' &&
+				envelope.result_info.total_pages > 1
+			)
+				deploymentError('paginated domain listing unsupported');
+			return envelope.result;
+		}
+		deploymentError('API retries exhausted');
+	}
+}
+
+export interface DeploymentBodyProvider {
+	readModule(module: PreparedModule): Promise<Uint8Array>;
+	readAsset(filename: string, asset: DeploymentAsset): Promise<Uint8Array>;
+}
+
+function jsonRequest(method: string, body: unknown): RequestInit {
+	return { method, headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) };
+}
+
+async function verifyRouting(
+	api: DeploymentAPI,
+	accountPrefix: string,
+	workerPrefix: string,
+	config: DeploymentConfig
+): Promise<void> {
+	const [domains, subdomain] = await Promise.all([
+		api.request(
+			`${accountPrefix}/workers/domains?service=${encodeURIComponent(config.name)}&environment=production`
+		),
+		api.request(`${workerPrefix}/subdomain`)
+	]);
+	if (
+		!Array.isArray(domains) ||
+		domains.some(
+			(domain) =>
+				!record(domain) ||
+				typeof domain.hostname !== 'string' ||
+				domain.service !== config.name ||
+				domain.environment !== 'production'
+		)
+	)
+		deploymentError('invalid custom domain response');
+	if (
+		!equalJSON(
+			domains.map((domain) => domain.hostname).sort(),
+			config.routes.map((route) => route.pattern).sort()
+		)
+	)
+		deploymentError('custom domains differ; explicit route provisioning is required');
+	if (!record(subdomain) || subdomain.enabled !== false || subdomain.previews_enabled !== false)
+		deploymentError('remote development endpoints must already be disabled');
+}
+
+export interface UploadDeploymentOptions extends DeploymentBodyProvider {
+	api: DeploymentAPI;
+	accountId: string;
+	expectedProvenance: string;
+	config: DeploymentConfig;
+	log?: (message: string) => void;
+}
+
+/** Upload requested bodies only. An authenticated cached inventory can supply the complete manifest. */
+export async function uploadDeployment(
+	snapshot: DeploymentSnapshot,
+	options: UploadDeploymentOptions
+): Promise<{ uploadedAssets: number; versionId?: string }> {
+	const { api, config } = options;
+	if (
+		!/^[0-9a-f]{32}$/.test(options.accountId) ||
+		(config.account_id && config.account_id !== options.accountId)
+	)
+		deploymentError('account identity mismatch');
+	if (
+		snapshot.schema !== 1 ||
+		snapshot.provenance !== options.expectedProvenance ||
+		snapshot.configSha256 !== configSha256(config)
+	)
+		deploymentError('stale deployment snapshot');
+	const inventory = validateAssetInventory(snapshot.inventory, options.expectedProvenance);
+	const worker = validatePreparedWorker(snapshot.worker, config);
+	const modules: [PreparedModule, Uint8Array][] = [];
+	// Validate every Worker dependency before performing any remote mutation.
+	for (const module of worker.modules) {
+		let bytes: Uint8Array;
+		try {
+			bytes = await options.readModule(module);
+		} catch {
+			deploymentError('prepared Worker module is unavailable');
+		}
+		verifyModuleBytes(module, bytes);
+		modules.push([module, bytes]);
+	}
+	const accountPrefix = `/accounts/${options.accountId}`;
+	const workerPrefix = `${accountPrefix}/workers/scripts/${encodeURIComponent(config.name)}`;
+	await verifyRouting(api, accountPrefix, workerPrefix, config);
+	const manifest = Object.fromEntries(
+		Object.entries(inventory.assets).map(([filename, { hash, size }]) => [filename, { hash, size }])
+	);
+	const byHash = new Map<string, [string, DeploymentAsset]>();
+	for (const [filename, asset] of Object.entries(inventory.assets)) {
+		const existing = byHash.get(asset.hash);
+		if (
+			existing &&
+			(existing[1].sha256 !== asset.sha256 ||
+				existing[1].size !== asset.size ||
+				existing[1].contentType !== asset.contentType)
+		)
+			deploymentError('ambiguous asset hash');
+		byHash.set(asset.hash, [filename, asset]);
+	}
+	const session = await api.request(
+		`${workerPrefix}/assets-upload-session`,
+		jsonRequest('POST', { manifest })
+	);
+	if (
+		!record(session) ||
+		typeof session.jwt !== 'string' ||
+		!session.jwt ||
+		!Array.isArray(session.buckets)
+	)
+		deploymentError('invalid asset upload session');
+	const buckets: string[][] = [];
+	const seen = new Set<string>();
+	for (const bucket of session.buckets) {
+		if (!Array.isArray(bucket) || !bucket.length) deploymentError('invalid requested asset bucket');
+		for (const hash of bucket) {
+			if (typeof hash !== 'string' || !ASSET_HASH.test(hash) || !byHash.has(hash) || seen.has(hash))
+				deploymentError('invalid requested asset hash');
+			seen.add(hash);
+		}
+		buckets.push(bucket);
+	}
+	const uploadToken = session.jwt;
+	let completionToken = buckets.length === 0 ? uploadToken : '';
+	let cursor = 0;
+	let failed = false;
+	async function uploadBuckets(): Promise<void> {
+		while (!failed && cursor < buckets.length) {
+			const bucket = buckets[cursor++];
+			try {
+				const payload = new FormData();
+				for (const hash of bucket) {
+					const [filename, asset] = byHash.get(hash)!;
+					let bytes: Uint8Array;
+					try {
+						bytes = await options.readAsset(filename, asset);
+					} catch {
+						deploymentError('requested asset body is unavailable');
+					}
+					verifyAssetBytes(filename, asset, bytes);
+					payload.append(
+						hash,
+						new File([Buffer.from(bytes).toString('base64')], hash, { type: asset.contentType }),
+						hash
+					);
+				}
+				const result = await api.request(
+					`${accountPrefix}/workers/assets/upload?base64=true`,
+					{ method: 'POST', body: payload },
+					uploadToken
+				);
+				if (
+					!record(result) ||
+					(result.jwt !== undefined && (typeof result.jwt !== 'string' || !result.jwt))
+				)
+					deploymentError('invalid asset completion response');
+				if (typeof result.jwt === 'string') {
+					if (completionToken && completionToken !== result.jwt)
+						deploymentError('conflicting asset completion tokens');
+					completionToken = result.jwt;
+				}
+			} catch (error) {
+				failed = true;
+				throw error;
+			}
+		}
+	}
+	const results = await Promise.allSettled(
+		Array.from({ length: Math.min(3, buckets.length) }, uploadBuckets)
+	);
+	for (const result of results) if (result.status === 'rejected') throw result.reason;
+	if (!completionToken) deploymentError('asset upload did not complete');
+	const metadata = {
+		...worker.metadata,
+		keep_bindings: ['secret_text', 'secret_key'],
+		assets: {
+			jwt: completionToken,
+			config: {
+				html_handling: config.assets.html_handling,
+				not_found_handling: config.assets.not_found_handling,
+				run_worker_first: true,
+				...inventory.controlFiles
+			}
+		}
+	};
+	const payload = new FormData();
+	payload.set('metadata', JSON.stringify(metadata));
+	for (const [module, bytes] of modules)
+		payload.set(module.name, new File([bytes as BlobPart], module.name, { type: module.type }));
+	const deployment = await api.request(`${workerPrefix}?bindings_inherit=strict`, {
+		method: 'PUT',
+		body: payload
+	});
+	if (!record(deployment)) deploymentError('invalid Worker deployment result');
+	// Verify endpoint/domain state again; never create, remove, or retarget domains silently.
+	await verifyRouting(api, accountPrefix, workerPrefix, config);
+	const versionId = typeof deployment.version_id === 'string' ? deployment.version_id : undefined;
+	options.log?.(`deployment: uploaded ${seen.size} asset bodies; Worker API accepted deployment`);
+	return { uploadedAssets: seen.size, ...(versionId ? { versionId } : {}) };
+}

package/site/tooling/lib/dev-git-state.ts

@@ -0,0 +1,130 @@
+import { execFileSync } from 'node:child_process';
+import { existsSync, watch, type FSWatcher } from 'node:fs';
+import path from 'node:path';
+import {
+	resolveProject,
+	declaredContentCourses,
+	gitAvailable,
+	contentWithinProject
+} from './project-config';
+
+export interface ContentRepository {
+	directory: string;
+	head: string;
+	gitDirectory: string;
+	commonDirectory: string;
+	index: string;
+	config: string;
+	worktreeConfig: string;
+	tracked: Buffer;
+}
+
+/** Read local Git metadata; never initialize submodules or fetch remote content. */
+export function contentGit(...args: string[]) {
+	return execFileSync('git', args, { maxBuffer: 128 * 1024 * 1024 });
+}
+
+/** Course directories declared by the root, matching compiler discovery without loading renderers. */
+export function declaredCourseDirectories(repo: string): string[] {
+	return declaredContentCourses(resolveProject(process.cwd(), repo)).map(
+		(course) => course.directory
+	);
+}
+
+/** Follow initialized declared courses and indexed gitlinks, including nested projects. */
+export function contentRepositories(repo: string): ContentRepository[] {
+	const project = resolveProject(process.cwd(), repo);
+	if (project.discovery !== 'git') return [];
+	const root = path.resolve(repo);
+	const pending = [
+		...(contentWithinProject(project) && gitAvailable(root)
+			? [root]
+			: gitAvailable(project.contentDir)
+				? [project.contentDir]
+				: []),
+		...declaredCourseDirectories(root).filter((directory) =>
+			existsSync(path.join(directory, '.git'))
+		)
+	];
+	const repositories: ContentRepository[] = [];
+	const visited = new Set<string>();
+	for (let position = 0; position < pending.length; position++) {
+		const directory = pending[position];
+		if (visited.has(directory)) continue;
+		visited.add(directory);
+		const [gitDirectory, commonDirectory, index, config, worktreeConfig] = contentGit(
+			'-C',
+			directory,
+			'rev-parse',
+			'--path-format=absolute',
+			'--git-dir',
+			'--git-common-dir',
+			'--git-path',
+			'index',
+			'--git-path',
+			'config',
+			'--git-path',
+			'config.worktree'
+		)
+			.toString()
+			.trimEnd()
+			.split('\n');
+		let head = 'unborn';
+		try {
+			head = contentGit('-C', directory, 'rev-parse', '--verify', '--quiet', 'HEAD')
+				.toString()
+				.trim();
+		} catch {
+			/* Empty repositories still have tracked content. */
+		}
+		const tracked = contentGit(
+			'-C',
+			directory,
+			'ls-files',
+			'-z',
+			'--stage',
+			...(directory === root ? ['--', path.relative(root, project.contentDir) || '.'] : [])
+		);
+		repositories.push({
+			directory,
+			head,
+			gitDirectory,
+			commonDirectory,
+			index,
+			config,
+			worktreeConfig,
+			tracked
+		});
+		for (const entry of tracked.toString().split('\0')) {
+			if (!entry.startsWith('160000 ')) continue;
+			const file = entry.slice(entry.indexOf('\t') + 1);
+			const nested = path.join(directory, file);
+			if (existsSync(path.join(nested, '.git'))) pending.push(nested);
+		}
+	}
+	return repositories;
+}
+
+type GitDirectoryWatch = (
+	directory: string,
+	options: { recursive: boolean },
+	listener: (event: string, file: string | Buffer | null) => void
+) => FSWatcher;
+
+/** Missing metadata directories can race removal; resource or permission failures must remain visible. */
+export function watchContentGitDirectory(
+	directory: string,
+	recursive: boolean,
+	files: () => Set<string>,
+	invalidate: () => void,
+	watchDirectory: GitDirectoryWatch = watch
+) {
+	try {
+		return watchDirectory(directory, { recursive }, (_event, file) => {
+			// Node may omit the filename, so conservatively reconcile the repository.
+			if (file === null || recursive || files().has(String(file))) invalidate();
+		});
+	} catch (error) {
+		if (!['ENOENT', 'ENOTDIR'].includes((error as NodeJS.ErrnoException).code ?? '')) throw error;
+	}
+}

package/site/tooling/lib/dev-history.ts

@@ -0,0 +1,86 @@
+import { existsSync, readFileSync, realpathSync } from 'node:fs';
+import path from 'node:path';
+import type { IncomingMessage, ServerResponse } from 'node:http';
+import type { CourseFile } from '../../src/lib/files';
+import { assetLimit, browsablePath, fileHistoryPolicyKey } from './course-files';
+import { createFileHistoryBuilder } from './file-history';
+import { parseGitmodules } from './lastmod';
+
+const assetPrefix = '/__content/history-assets';
+
+/** Dev-only: create history when its panel is opened, outside the generated static tree. */
+export function serveDevHistory(
+	request: IncomingMessage,
+	response: ServerResponse,
+	site: string,
+	repo: string
+) {
+	const url = new URL(request.url ?? '/', 'http://localhost');
+	if (url.pathname !== '/__content/history' && !url.pathname.startsWith(assetPrefix + '/'))
+		return false;
+	response.setHeader('Cache-Control', 'no-store');
+	if (process.env.VITE_PUBLIC_EDITION === 'true') {
+		response.statusCode = 404;
+		response.end();
+		return true;
+	}
+	if (request.method !== 'GET' && request.method !== 'HEAD') {
+		response.statusCode = 405;
+		response.end();
+		return true;
+	}
+	const output = path.join(site, 'build/generated/dev-history');
+	try {
+		let target: string;
+		if (url.pathname.startsWith(assetPrefix + '/')) {
+			const name = url.pathname.slice(assetPrefix.length + 1);
+			if (!/^[a-f0-9]{64}(?:-(?:blame|[a-f0-9]{40,64}))?\.(?:json|diff)$/.test(name))
+				throw new Error('Not found');
+			target = path.join(output, 'history', name);
+		} else {
+			const course = url.searchParams.get('course') ?? '';
+			const file = url.searchParams.get('file') ?? '';
+			const module = parseGitmodules(path.join(repo, '.gitmodules')).find(
+				(entry) => entry.path === `content/${course}`
+			);
+			if (!module || !/^[\w-]+$/.test(course) || !browsablePath(file)) throw new Error('Not found');
+			const index: CourseFile[] = JSON.parse(
+				readFileSync(
+					path.join(site, 'build/generated/assets/_files/index', `${course}.json`),
+					'utf8'
+				)
+			);
+			const entry = index.find(
+				(entry) =>
+					entry.path === file && !entry.locked && entry.history?.startsWith('/__content/history?')
+			);
+			if (!entry) throw new Error('Not found');
+			const source = path.join(module.fullPath, file);
+			if (realpathSync(source) !== source) throw new Error('Not found');
+			const bytes = readFileSync(source);
+			if (bytes.length > assetLimit) throw new Error('Not found');
+			const build = createFileHistoryBuilder(
+				module.fullPath,
+				output,
+				path.join(site, 'build/generated/cache/file-history', course),
+				browsablePath,
+				fileHistoryPolicyKey(),
+				undefined,
+				assetPrefix
+			);
+			const asset = build?.(entry, bytes);
+			if (!asset) throw new Error('Not found');
+			target = path.join(output, 'history', path.basename(asset));
+		}
+		if (!existsSync(target)) throw new Error('Not found');
+		response.setHeader(
+			'Content-Type',
+			target.endsWith('.json') ? 'application/json' : 'text/plain; charset=utf-8'
+		);
+		response.end(request.method === 'HEAD' ? undefined : readFileSync(target));
+	} catch {
+		response.statusCode = 404;
+		response.end('History unavailable');
+	}
+	return true;
+}

package/site/tooling/lib/dev-social-images.ts

@@ -0,0 +1,38 @@
+import { readFileSync } from 'node:fs';
+import path from 'node:path';
+import type { IncomingMessage, ServerResponse } from 'node:http';
+import type { ContentManifest } from '../../src/lib/types';
+import { socialImagePath } from '../../src/lib/social-image';
+import { cachedSocialImage, defaultSocialCard, socialCard } from './social-images';
+
+export async function serveDevSocialImage(
+	request: IncomingMessage,
+	response: ServerResponse,
+	site: string
+) {
+	const url = new URL(request.url ?? '/', 'http://localhost');
+	if (!url.pathname.startsWith('/_og/')) return false;
+	response.setHeader('Cache-Control', 'no-store');
+	if (request.method !== 'GET' && request.method !== 'HEAD') {
+		response.statusCode = 405;
+		response.end();
+		return true;
+	}
+	const manifest: ContentManifest = JSON.parse(
+		readFileSync(path.join(site, 'build/generated/content-manifest.json'), 'utf8')
+	);
+	const page = Object.values(manifest.pages).find(
+		(page) => socialImagePath(page.slug) === url.pathname
+	);
+	const card =
+		url.pathname === socialImagePath() ? defaultSocialCard : page ? socialCard(page) : undefined;
+	if (!card) {
+		response.statusCode = 404;
+		response.end();
+		return true;
+	}
+	const { cached } = await cachedSocialImage(site, card);
+	response.setHeader('Content-Type', 'image/png');
+	response.end(request.method === 'HEAD' ? undefined : readFileSync(cached));
+	return true;
+}

package/site/tooling/lib/dev-state.ts

@@ -0,0 +1,107 @@
+import { createHash } from 'node:crypto';
+import {
+	existsSync,
+	lstatSync,
+	readFileSync,
+	readdirSync,
+	readlinkSync,
+	type BigIntStats
+} from 'node:fs';
+import path from 'node:path';
+import { contentGit, contentRepositories } from './dev-git-state';
+import {
+	resolveProject,
+	discoverSourceFiles,
+	projectIdentity,
+	gitAvailable,
+	contentWithinProject
+} from './project-config';
+
+function fileStat(file: string): BigIntStats | null {
+	try {
+		return lstatSync(file, { bigint: true });
+	} catch (error) {
+		if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null;
+		throw error;
+	}
+}
+
+function stamp(file: string, stat = fileStat(file)) {
+	if (!stat) return null;
+	return [
+		String(stat.mode),
+		String(stat.ino),
+		String(stat.size),
+		String(stat.mtimeNs),
+		String(stat.ctimeNs),
+		stat.isSymbolicLink() ? readlinkSync(file) : ''
+	];
+}
+
+/** Validate tracked paths, working-copy edits, index changes and Git dates without reading large assets. */
+export function contentInputFingerprint(repo: string, pipeline: string) {
+	const project = resolveProject(process.cwd(), repo);
+	const hash = createHash('sha256').update(pipeline).update(projectIdentity(project));
+	for (const file of discoverSourceFiles(project)) {
+		if (project.discovery === 'filesystem') {
+			const bytes = readFileSync(path.join(project.contentDir, file));
+			hash.update(JSON.stringify([file, bytes.length])).update(bytes);
+		} else hash.update(JSON.stringify([file, stamp(path.join(project.contentDir, file))]));
+	}
+	if (project.configFile) hash.update(readFileSync(project.configFile));
+	const modules = path.join(repo, '.gitmodules');
+	hash.update(existsSync(modules) ? readFileSync(modules) : '');
+	for (const repository of contentRepositories(repo)) {
+		const { directory, head, tracked, config, worktreeConfig } = repository;
+		hash.update(directory).update(head).update(tracked);
+		// Registration and worktree-specific config can change recursive visibility
+		// without changing a commit or any tracked working-copy file.
+		hash.update(JSON.stringify([stamp(config), stamp(worktreeConfig)]));
+		const modules = path.join(directory, '.gitmodules');
+		hash.update(existsSync(modules) ? readFileSync(modules) : '');
+		for (const entry of tracked.toString().split('\0').filter(Boolean)) {
+			const file = entry.slice(entry.indexOf('\t') + 1);
+			hash.update(JSON.stringify([file, stamp(path.join(directory, file))]));
+		}
+	}
+	// Git's recursive index is authoritative about which initialized submodules
+	// are enabled, including nested registration changes.
+	if (project.discovery === 'git' && contentWithinProject(project) && gitAvailable(repo))
+		hash.update(
+			contentGit(
+				'-C',
+				repo,
+				'ls-files',
+				'-z',
+				'--stage',
+				'--recurse-submodules',
+				'--',
+				path.relative(repo, project.contentDir) || '.'
+			)
+		);
+	return hash.digest('hex');
+}
+
+/** A production build or missing/modified output invalidates the saved dev result. */
+export function contentOutputFingerprint(site: string) {
+	const hash = createHash('sha256');
+	function walk(file: string) {
+		const stat = fileStat(file);
+		if (stat?.isDirectory()) {
+			for (const name of readdirSync(file).sort()) walk(path.join(file, name));
+		} else hash.update(JSON.stringify([path.relative(site, file), stamp(file, stat)]));
+	}
+	for (const file of [
+		'build/generated/content-manifest.json',
+		'build/generated/expected-404.json',
+		'build/generated/expected-missing-id.json',
+		'build/generated/public-routes.json',
+		'build/generated/public-assets.json',
+		'build/generated/pages',
+		'build/generated/assets',
+		'src/lib/generated',
+		'static'
+	])
+		walk(path.join(site, file));
+	return hash.digest('hex');
+}