@twango/twiki 0.0.0-stage → 0.1.0-rc.0

package/site/tooling/lib/deployment-manifest.ts

@@ -0,0 +1,274 @@
+import { lstat, readFile, readdir } from 'node:fs/promises';
+import path from 'node:path';
+import { hash } from 'blake3-wasm';
+import ignore from 'ignore';
+import mime from 'mime';
+import { deploymentError, record, sha256 } from './deployment-config';
+
+export interface DeploymentAsset {
+	hash: string;
+	size: number;
+	sha256: string;
+	contentType: string;
+}
+
+export interface AssetInventory {
+	schema: 1;
+	provenance: string;
+	complete: true;
+	assets: Record<string, DeploymentAsset>;
+	controlFiles: { _headers?: string; _redirects?: string };
+	digest: string;
+}
+
+export const SHA256 = /^[0-9a-f]{64}$/;
+export const ASSET_HASH = /^[0-9a-f]{32}$/;
+export const MAX_ASSET_BYTES = 25 * 1024 * 1024;
+export const WRANGLER_ASSET_HASH_VERSION = 'wrangler-4.100-blake3-base64-extension-v1';
+
+export function assetPath(value: string): boolean {
+	return (
+		value.startsWith('/') &&
+		value.length > 1 &&
+		!/[\\\0\r\n]/.test(value) &&
+		value
+			.slice(1)
+			.split('/')
+			.every((segment) => segment !== '' && segment !== '.' && segment !== '..')
+	);
+}
+
+export function wranglerAssetHash(bytes: Uint8Array, filename: string): string {
+	// Matches Wrangler 4.100.0 hashFile: BLAKE3(base64(contents) + extension), first 128 bits.
+	return hash(Buffer.from(bytes).toString('base64') + path.extname(filename).slice(1))
+		.toString('hex')
+		.slice(0, 32);
+}
+
+export function assetContentType(filename: string): string {
+	const type = mime.getType(filename);
+	return type?.startsWith('text/') ? `${type}; charset=utf-8` : (type ?? 'application/null');
+}
+
+export function deploymentAsset(bytes: Uint8Array, filename: string): DeploymentAsset {
+	if (!assetPath(filename) || bytes.byteLength > MAX_ASSET_BYTES)
+		deploymentError('invalid asset product');
+	return {
+		hash: wranglerAssetHash(bytes, filename),
+		size: bytes.byteLength,
+		sha256: sha256(bytes),
+		contentType: assetContentType(filename)
+	};
+}
+
+export function createAssetInventory(
+	assets: Record<string, DeploymentAsset>,
+	provenance: string,
+	controlFiles: AssetInventory['controlFiles'] = {}
+): AssetInventory {
+	const inventory: AssetInventory = {
+		schema: 1,
+		complete: true,
+		provenance,
+		assets,
+		controlFiles,
+		digest: ''
+	};
+	inventory.digest = inventoryDigest(inventory);
+	return validateAssetInventory(inventory, provenance);
+}
+
+export interface VerifiedInventoryMerge {
+	pieces: { owner: string; inventory: AssetInventory }[];
+	/** Exact identity and inventory digest from the current authenticated product manifest. */
+	expectedProducts: Record<string, { provenance: string; digest: string }>;
+	/** Complete current output ledger, including precedence decisions for overlapping products. */
+	ownership: Record<string, { owner: string; sha256: string; size: number }>;
+	provenance: string;
+	controlFiles: AssetInventory['controlFiles'];
+}
+
+/**
+ * Reuse hashes only from verified product inventories, never from file timestamps alone.
+ * The caller must authenticate expectedProducts and the complete current ownership ledger.
+ * Missing/revoked products and stale byte identities fail closed. No filesystem reads occur.
+ */
+export function mergeVerifiedInventories(input: VerifiedInventoryMerge): AssetInventory {
+	const products = new Map<string, AssetInventory>();
+	for (const { owner, inventory } of input.pieces) {
+		const expected = input.expectedProducts[owner];
+		if (!expected || products.has(owner) || inventory.digest !== expected.digest)
+			deploymentError('unexpected product inventory');
+		products.set(owner, validateAssetInventory(inventory, expected.provenance));
+	}
+	if (products.size !== Object.keys(input.expectedProducts).length)
+		deploymentError('missing product inventory');
+	const assets: Record<string, DeploymentAsset> = Object.create(null);
+	for (const [filename, source] of Object.entries(input.ownership)) {
+		if (!assetPath(filename) || !SHA256.test(source.sha256) || !Number.isSafeInteger(source.size))
+			deploymentError('invalid ownership ledger');
+		const asset = products.get(source.owner)?.assets[filename];
+		if (!asset || asset.sha256 !== source.sha256 || asset.size !== source.size)
+			deploymentError('product inventory does not cover current output');
+		assets[filename] = asset;
+	}
+	return createAssetInventory(assets, input.provenance, input.controlFiles);
+}
+
+export function inventoryDigest(
+	inventory: Pick<AssetInventory, 'provenance' | 'assets' | 'controlFiles'>
+): string {
+	return sha256(
+		JSON.stringify([
+			inventory.provenance,
+			Object.entries(inventory.assets).sort(([a], [b]) => a.localeCompare(b)),
+			inventory.controlFiles._headers ?? null,
+			inventory.controlFiles._redirects ?? null
+		])
+	);
+}
+
+export function validateAssetInventory(value: unknown, expectedProvenance: string): AssetInventory {
+	if (
+		!record(value) ||
+		value.schema !== 1 ||
+		value.complete !== true ||
+		value.provenance !== expectedProvenance ||
+		!SHA256.test(expectedProvenance) ||
+		!record(value.assets) ||
+		!record(value.controlFiles)
+	)
+		deploymentError('stale or incomplete asset inventory');
+	if (
+		Object.keys(value.controlFiles).some((key) => !['_headers', '_redirects'].includes(key)) ||
+		Object.values(value.controlFiles).some((v) => typeof v !== 'string')
+	)
+		deploymentError('invalid asset control files');
+	for (const [filename, asset] of Object.entries(value.assets)) {
+		if (
+			!assetPath(filename) ||
+			!record(asset) ||
+			typeof asset.hash !== 'string' ||
+			!ASSET_HASH.test(asset.hash) ||
+			typeof asset.sha256 !== 'string' ||
+			!SHA256.test(asset.sha256) ||
+			!Number.isSafeInteger(asset.size) ||
+			(asset.size as number) < 0 ||
+			(asset.size as number) > MAX_ASSET_BYTES ||
+			typeof asset.contentType !== 'string' ||
+			/[\r\n]/.test(asset.contentType) ||
+			asset.contentType !== assetContentType(filename)
+		)
+			deploymentError('invalid asset inventory');
+		if (
+			filename === '/_headers' ||
+			filename === '/_redirects' ||
+			filename === '/.assetsignore' ||
+			filename === '/_worker.js'
+		)
+			deploymentError('invalid asset inventory control entry');
+	}
+	const inventory = value as unknown as AssetInventory;
+	if (inventory.digest !== inventoryDigest(inventory))
+		deploymentError('asset inventory integrity failed');
+	return inventory;
+}
+
+export function verifyAssetBytes(
+	filename: string,
+	asset: DeploymentAsset,
+	bytes: Uint8Array
+): void {
+	if (
+		bytes.byteLength !== asset.size ||
+		sha256(bytes) !== asset.sha256 ||
+		wranglerAssetHash(bytes, filename) !== asset.hash
+	)
+		deploymentError('asset body integrity failed');
+}
+
+export function currentDescriptorProvenance(bytes: Uint8Array): string {
+	let descriptor: unknown;
+	try {
+		descriptor = JSON.parse(Buffer.from(bytes).toString('utf8'));
+	} catch {
+		deploymentError('invalid current content descriptor');
+	}
+	if (
+		!record(descriptor) ||
+		descriptor.schemaVersion !== 1 ||
+		typeof descriptor.applicationVersion !== 'string' ||
+		!SHA256.test(descriptor.applicationVersion) ||
+		!record(descriptor.snapshots) ||
+		typeof descriptor.snapshots.public !== 'string' ||
+		!SHA256.test(descriptor.snapshots.public) ||
+		typeof descriptor.snapshots.full !== 'string' ||
+		!SHA256.test(descriptor.snapshots.full)
+	)
+		deploymentError('invalid current content descriptor');
+	return sha256(bytes);
+}
+
+export async function readCurrentDescriptorProvenance(assetDirectory: string): Promise<string> {
+	return currentDescriptorProvenance(
+		await readFile(path.join(assetDirectory, '_content/current.json'))
+	);
+}
+
+/** Always hashes actual bytes. A restored authenticated inventory may be reused by its product provenance. */
+export async function captureAssetInventory(
+	assetDirectory: string,
+	provenance: string,
+	options: {
+		resolveAsset?: (absoluteFilename: string, assetPath: string) => Promise<DeploymentAsset>;
+	} = {}
+): Promise<AssetInventory> {
+	if (!SHA256.test(provenance)) deploymentError('invalid inventory provenance');
+	const controlFiles: AssetInventory['controlFiles'] = {};
+	for (const filename of ['_headers', '_redirects'] as const) {
+		try {
+			if (!(await lstat(path.join(assetDirectory, filename))).isFile())
+				deploymentError('invalid asset control file');
+			controlFiles[filename] = await readFile(path.join(assetDirectory, filename), 'utf8');
+		} catch (error) {
+			if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
+		}
+	}
+	let patterns = '';
+	try {
+		if (!(await lstat(path.join(assetDirectory, '.assetsignore'))).isFile())
+			deploymentError('invalid asset ignore file');
+		patterns = await readFile(path.join(assetDirectory, '.assetsignore'), 'utf8');
+	} catch (error) {
+		if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
+	}
+	const ignorer = ignore().add([
+		'/.assetsignore',
+		'/_headers',
+		'/_redirects',
+		...patterns.split('\n')
+	]);
+	const assets: Record<string, DeploymentAsset> = Object.create(null);
+	async function visit(directory: string, prefix: string): Promise<void> {
+		for (const entry of await readdir(directory, { withFileTypes: true })) {
+			const relative = `${prefix}${entry.name}`;
+			if (ignorer.test(relative).ignored) continue;
+			if (entry.isSymbolicLink()) deploymentError('asset symlinks are unsupported');
+			if (entry.isDirectory()) await visit(path.join(directory, entry.name), `${relative}/`);
+			else if (entry.isFile()) {
+				const filename = `/${relative}`;
+				if (!assetPath(filename) || relative === '_worker.js')
+					deploymentError('invalid asset path');
+				if (options.resolveAsset)
+					assets[filename] = await options.resolveAsset(path.join(directory, entry.name), filename);
+				else
+					assets[filename] = deploymentAsset(
+						await readFile(path.join(directory, entry.name)),
+						filename
+					);
+			} else deploymentError('unsupported asset file');
+		}
+	}
+	await visit(assetDirectory, '');
+	return createAssetInventory(assets, provenance, controlFiles);
+}

package/site/tooling/lib/deployment-prepare.ts

@@ -0,0 +1,325 @@
+import { spawnSync } from 'node:child_process';
+import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import {
+	configSha256,
+	deploymentError,
+	equalJSON,
+	expectedBindings,
+	readDeploymentConfig,
+	record,
+	sha256,
+	type DeploymentConfig
+} from './deployment-config';
+import {
+	SHA256,
+	assetPath,
+	validateAssetInventory,
+	type AssetInventory
+} from './deployment-manifest';
+
+export interface PreparedModule {
+	name: string;
+	type: string;
+	sha256: string;
+	size: number;
+	filename: string;
+}
+
+export interface PreparedWorker {
+	schema: 1;
+	configSha256: string;
+	wranglerVersion: string;
+	metadata: Record<string, unknown>;
+	modules: PreparedModule[];
+	digest: string;
+}
+
+export interface DeploymentSnapshot {
+	schema: 1;
+	provenance: string;
+	configSha256: string;
+	worker: PreparedWorker;
+	inventory: AssetInventory;
+}
+
+export function workerDigest(worker: Omit<PreparedWorker, 'digest'> | PreparedWorker): string {
+	return sha256(
+		JSON.stringify([
+			worker.schema,
+			worker.configSha256,
+			worker.wranglerVersion,
+			worker.metadata,
+			worker.modules
+		])
+	);
+}
+
+export function validatePreparedWorker(value: unknown, config: DeploymentConfig): PreparedWorker {
+	if (
+		!record(value) ||
+		value.schema !== 1 ||
+		value.configSha256 !== configSha256(config) ||
+		typeof value.wranglerVersion !== 'string' ||
+		!record(value.metadata) ||
+		!Array.isArray(value.modules)
+	)
+		deploymentError('incompatible prepared Worker');
+	const metadata = value.metadata;
+	if (
+		typeof metadata.main_module !== 'string' ||
+		!equalJSON(metadata.compatibility_flags, config.compatibility_flags) ||
+		metadata.compatibility_date !== config.compatibility_date ||
+		!Array.isArray(metadata.bindings)
+	)
+		deploymentError('prepared Worker metadata mismatch');
+	const bindings = [...metadata.bindings].sort((a, b) =>
+		String(a.name).localeCompare(String(b.name))
+	);
+	if (
+		!equalJSON(
+			bindings,
+			expectedBindings(config).sort((a, b) => String(a.name).localeCompare(String(b.name)))
+		)
+	)
+		deploymentError('prepared Worker binding mismatch');
+	if (!equalJSON(metadata.keep_bindings, ['secret_text', 'secret_key']))
+		deploymentError('prepared Worker must preserve secrets');
+	const supported = [
+		'main_module',
+		'bindings',
+		'compatibility_date',
+		'compatibility_flags',
+		'keep_bindings',
+		'logpush',
+		'observability'
+	];
+	if (
+		Object.keys(metadata).some((key) => !supported.includes(key)) ||
+		(metadata.logpush !== undefined && metadata.logpush !== false)
+	)
+		deploymentError('unsupported prepared Worker metadata');
+	// Wrangler normalizes observability even when it is absent from the source configuration.
+	if (
+		metadata.observability !== undefined &&
+		(!record(metadata.observability) ||
+			metadata.observability.enabled !== false ||
+			Object.keys(metadata.observability).some((key) => key !== 'enabled'))
+	)
+		deploymentError('unsupported prepared Worker observability');
+	const names = new Set<string>();
+	for (const module of value.modules) {
+		if (
+			!record(module) ||
+			typeof module.name !== 'string' ||
+			!assetPath(`/${module.name}`) ||
+			names.has(module.name) ||
+			typeof module.sha256 !== 'string' ||
+			!SHA256.test(module.sha256) ||
+			module.filename !== `modules/${module.sha256}` ||
+			!Number.isSafeInteger(module.size) ||
+			(module.size as number) < 0 ||
+			![
+				'application/javascript+module',
+				'application/javascript',
+				'text/plain',
+				'application/wasm',
+				'application/octet-stream',
+				'application/source-map'
+			].includes(module.type as string)
+		)
+			deploymentError('invalid prepared Worker module');
+		names.add(module.name);
+	}
+	if (!names.has(metadata.main_module) || !value.modules.length)
+		deploymentError('prepared Worker entry module is missing');
+	const worker = value as unknown as PreparedWorker;
+	if (worker.digest !== workerDigest(worker)) deploymentError('prepared Worker integrity failed');
+	return worker;
+}
+
+/** Parse Wrangler's actual serialized upload form; no module/default dependency inference. */
+export async function capturePreparedWorker(
+	multipart: Uint8Array,
+	config: DeploymentConfig,
+	version: string,
+	outputDirectory: string
+): Promise<PreparedWorker> {
+	const firstLine = Buffer.from(multipart).subarray(0, 200).toString().split('\r\n')[0];
+	if (!/^--[-a-zA-Z0-9]+$/.test(firstLine)) deploymentError('invalid Wrangler multipart boundary');
+	// Bun's multipart parser rewrites .js part MIME types. Preserve Wrangler's exact raw headers.
+	const buffer = Buffer.from(multipart);
+	const boundary = Buffer.from(`\r\n${firstLine}`);
+	const parts: { name: string; type: string; bytes: Uint8Array }[] = [];
+	let cursor = firstLine.length + 2;
+	let closed = false;
+	while (cursor < buffer.length) {
+		const headersEnd = buffer.indexOf('\r\n\r\n', cursor);
+		if (headersEnd < 0 || headersEnd - cursor > 16_384)
+			deploymentError('invalid Wrangler multipart headers');
+		const headers = buffer.subarray(cursor, headersEnd).toString('utf8');
+		const disposition = headers.split('\r\n').find((line) => /^Content-Disposition: /i.test(line));
+		const nameMatch = disposition?.match(/; name=("(?:[^"\\]|\\.)*")/);
+		if (!nameMatch) deploymentError('invalid Wrangler multipart disposition');
+		let name: string;
+		try {
+			name = JSON.parse(nameMatch[1]);
+		} catch {
+			deploymentError('invalid Wrangler module name');
+		}
+		const type =
+			headers
+				.split('\r\n')
+				.find((line) => /^Content-Type: /i.test(line))
+				?.slice('Content-Type: '.length) ?? '';
+		const bodyStart = headersEnd + 4;
+		const bodyEnd = buffer.indexOf(boundary, bodyStart);
+		if (bodyEnd < 0) deploymentError('invalid Wrangler multipart body');
+		parts.push({ name, type, bytes: buffer.subarray(bodyStart, bodyEnd) });
+		cursor = bodyEnd + boundary.length;
+		if (buffer.subarray(cursor, cursor + 2).toString() === '--') {
+			if (!/^--(?:\r\n)?$/.test(buffer.subarray(cursor).toString()))
+				deploymentError('invalid Wrangler multipart trailer');
+			closed = true;
+			break;
+		}
+		if (buffer.subarray(cursor, cursor + 2).toString() !== '\r\n')
+			deploymentError('invalid Wrangler multipart separator');
+		cursor += 2;
+	}
+	if (!closed || parts.filter((part) => part.name === 'metadata').length !== 1)
+		deploymentError('Wrangler metadata is missing');
+	const rawMetadata = Buffer.from(parts.find((part) => part.name === 'metadata')!.bytes).toString(
+		'utf8'
+	);
+	let metadata: Record<string, unknown>;
+	try {
+		metadata = JSON.parse(rawMetadata);
+	} catch {
+		deploymentError('invalid Wrangler metadata');
+	}
+	if (!record(metadata) || !Array.isArray(metadata.bindings))
+		deploymentError('invalid Wrangler metadata');
+	metadata.keep_bindings = ['secret_text', 'secret_key'];
+	const modules: PreparedModule[] = [];
+	const bodies = new Map<string, Uint8Array>();
+	for (const { name, type, bytes } of parts) {
+		if (name === 'metadata') continue;
+		const digest = sha256(bytes);
+		modules.push({
+			name,
+			type,
+			sha256: digest,
+			size: bytes.byteLength,
+			filename: `modules/${digest}`
+		});
+		bodies.set(digest, bytes);
+	}
+	const worker: PreparedWorker = {
+		schema: 1,
+		configSha256: configSha256(config),
+		wranglerVersion: version,
+		metadata,
+		modules,
+		digest: ''
+	};
+	worker.digest = workerDigest(worker);
+	validatePreparedWorker(worker, config);
+	await mkdir(path.join(outputDirectory, 'modules'), { recursive: true });
+	for (const [digest, body] of bodies)
+		await writeFile(path.join(outputDirectory, 'modules', digest), body, { mode: 0o600 });
+	await writeFile(path.join(outputDirectory, 'worker.json'), JSON.stringify(worker), {
+		mode: 0o600
+	});
+	return worker;
+}
+
+export function verifyModuleBytes(module: PreparedModule, bytes: Uint8Array): void {
+	if (bytes.byteLength !== module.size || sha256(bytes) !== module.sha256)
+		deploymentError('prepared Worker module integrity failed');
+}
+
+export async function loadPreparedWorker(
+	outputDirectory: string,
+	config?: DeploymentConfig
+): Promise<PreparedWorker> {
+	const worker: PreparedWorker = JSON.parse(
+		await readFile(path.join(outputDirectory, 'worker.json'), 'utf8')
+	);
+	// Callers normally supply config. Derivation from the site directory is only a convenience for canonical layout.
+	validatePreparedWorker(
+		worker,
+		config ?? readDeploymentConfig(path.resolve(outputDirectory, '../../..'))
+	);
+	for (const module of worker.modules)
+		verifyModuleBytes(module, await readFile(path.join(outputDirectory, module.filename)));
+	return worker;
+}
+
+/** Offline, credential-free Wrangler build. Does not register/upload assets or call account APIs. */
+export async function prepareWorker(
+	siteRoot: string,
+	outputDirectory: string
+): Promise<PreparedWorker> {
+	siteRoot = path.resolve(siteRoot);
+	outputDirectory = path.resolve(outputDirectory);
+	const config = readDeploymentConfig(siteRoot);
+	const version = JSON.parse(
+		await readFile(path.join(siteRoot, 'node_modules/wrangler/package.json'), 'utf8')
+	).version;
+	await mkdir(outputDirectory, { recursive: true });
+	const offlineConfig = {
+		...config,
+		main: path.resolve(siteRoot, config.main),
+		assets: { ...config.assets, directory: path.resolve(siteRoot, config.assets.directory) }
+	};
+	delete offlineConfig.account_id;
+	// Do not load the site's dotenv files or inherit Cloudflare credentials into Wrangler's process.
+	const configFile = path.join(outputDirectory, 'offline-wrangler.json');
+	const multipartFile = path.join(outputDirectory, 'worker.multipart');
+	await writeFile(configFile, JSON.stringify(offlineConfig), { mode: 0o600 });
+	const env = Object.fromEntries(
+		Object.entries(process.env).filter(([key]) => !/^(CLOUDFLARE_|CF_|WRANGLER_)/.test(key))
+	);
+	env.WRANGLER_SEND_METRICS = 'false';
+	env.WRANGLER_LOG = 'error';
+	env.WRANGLER_LOG_PATH = path.join(outputDirectory, 'wrangler.log');
+	const result = spawnSync(
+		process.env.NODE_BINARY ?? 'node',
+		[
+			path.join(siteRoot, 'node_modules/wrangler/bin/wrangler.js'),
+			'deploy',
+			'--config',
+			configFile,
+			'--dry-run',
+			'--outdir',
+			path.join(outputDirectory, 'wrangler'),
+			'--outfile',
+			multipartFile
+		],
+		{ cwd: outputDirectory, env, encoding: 'utf8', maxBuffer: 1024 * 1024 * 8 }
+	);
+	try {
+		if (result.status !== 0) deploymentError('offline Wrangler preparation failed');
+		return await capturePreparedWorker(
+			await readFile(multipartFile),
+			config,
+			version,
+			outputDirectory
+		);
+	} finally {
+		await rm(configFile, { force: true });
+		await rm(path.join(outputDirectory, 'wrangler.log'), { force: true });
+	}
+}
+
+export function buildDeploymentSnapshot(
+	worker: PreparedWorker,
+	inventory: AssetInventory,
+	config: DeploymentConfig,
+	provenance: string
+): DeploymentSnapshot {
+	validatePreparedWorker(worker, config);
+	validateAssetInventory(inventory, provenance);
+	return { schema: 1, configSha256: configSha256(config), provenance, worker, inventory };
+}