@serverkgg/bridge 0.0.1 → 0.0.2

package/src/runtime/bridgeEgress.ts

@@ -0,0 +1,360 @@
+import {
+	BRIDGE_EGRESS_FRAME_LIMIT,
+	BRIDGE_EGRESS_SOCKET_NAME,
+	type BridgeConfig,
+	type BridgeEgressDownloadResult,
+	type BridgeEgressFetchResult,
+	BridgeEgressMethod,
+	type BridgeEgressRequest,
+	type BridgeEgressResponse,
+} from "../protocol";
+
+const CONNECT_RETRY_MS = 10_000;
+
+const CALL_TIMEOUT_MS = 20_000;
+
+const DOWNLOAD_TIMEOUT_MS = 300_000;
+
+const QUEUE_HIGH_WATER_BYTES = 8 * 1024 * 1024;
+
+const NEWLINE = 0x0a;
+
+export class BridgeEgressUnavailableError extends Error {
+	constructor(message: string) {
+		super(message);
+
+		this.name = "BridgeEgressUnavailableError";
+	}
+}
+
+export class BridgeEgressCallError extends Error {
+	readonly status: number | null;
+
+	constructor(status: number | null, message: string) {
+		super(message);
+
+		this.name = "BridgeEgressCallError";
+		this.status = status;
+	}
+}
+
+export interface BridgeEgressCallOptions {
+	headers?: Record<string, string>;
+	timeoutMs?: number;
+	cache?: boolean;
+	cacheSeconds?: number;
+	refresh?: boolean;
+}
+
+export interface BridgeEgressDownloadOptions extends BridgeEgressCallOptions {
+	digest?: string;
+	sizeBytes?: number;
+}
+
+export interface BridgeEgressDownload extends BridgeEgressDownloadResult {
+	body: AsyncIterable<Uint8Array>;
+}
+
+export interface BridgeEgress {
+	fetch(url: string, options?: BridgeEgressCallOptions): Promise<BridgeEgressFetchResult>;
+	download(url: string, options?: BridgeEgressDownloadOptions): Promise<BridgeEgressDownload>;
+}
+
+const createByteStream = () => {
+	const chunks: Uint8Array[] = [];
+
+	let queued = 0;
+	let done = false;
+	let failure: unknown = null;
+	let wake: (() => void) | null = null;
+	let drain: (() => void) | null = null;
+
+	const signal = () => {
+		const resume = wake;
+
+		wake = null;
+		resume?.();
+	};
+
+	return {
+		get queued() {
+			return queued;
+		},
+
+		push(chunk: Uint8Array) {
+			chunks.push(chunk);
+			queued += chunk.byteLength;
+			signal();
+		},
+
+		finish() {
+			done = true;
+			signal();
+		},
+
+		fail(error: unknown) {
+			failure = error;
+			done = true;
+			signal();
+		},
+
+		onDrain(listener: () => void) {
+			drain = listener;
+		},
+
+		async *stream(): AsyncGenerator<Uint8Array> {
+			while (true) {
+				const chunk = chunks.shift();
+
+				if (chunk) {
+					queued -= chunk.byteLength;
+
+					if (queued < QUEUE_HIGH_WATER_BYTES) {
+						drain?.();
+					}
+
+					yield chunk;
+
+					continue;
+				}
+
+				if (failure) {
+					throw failure;
+				}
+
+				if (done) {
+					return;
+				}
+
+				await new Promise<void>((resolve) => {
+					wake = resolve;
+				});
+			}
+		},
+	};
+};
+
+export const createBridgeEgress = (config: BridgeConfig): BridgeEgress => {
+	const path = `${config.run}/${BRIDGE_EGRESS_SOCKET_NAME}`;
+
+	let blockedUntil = 0;
+
+	const frameOf = (method: BridgeEgressMethod, url: string, options: BridgeEgressDownloadOptions) => {
+		const request: BridgeEgressRequest = {
+			id: 1,
+			token: config.token,
+			method,
+			url,
+			...options,
+		};
+
+		return `${JSON.stringify(request)}\n`;
+	};
+
+	const connect = async <Data>(handlers: Bun.SocketHandler<Data>) => {
+		if (performance.now() < blockedUntil) {
+			throw new BridgeEgressUnavailableError("the egress channel is not available");
+		}
+
+		try {
+			return await Bun.connect<Data>({
+				unix: path,
+				socket: handlers,
+			});
+		} catch {
+			blockedUntil = performance.now() + CONNECT_RETRY_MS;
+
+			throw new BridgeEgressUnavailableError("the egress channel is not available");
+		}
+	};
+
+	return {
+		async fetch(url, options = {}) {
+			let settle: (response: BridgeEgressResponse<BridgeEgressFetchResult>) => void = () => undefined;
+			let fail: (error: Error) => void = () => undefined;
+
+			const pending = new Promise<BridgeEgressResponse<BridgeEgressFetchResult>>((resolve, reject) => {
+				settle = resolve;
+				fail = reject;
+			});
+
+			let buffer = "";
+
+			const socket = await connect<undefined>({
+				data(_connection, chunk) {
+					buffer += chunk.toString();
+
+					if (buffer.length > BRIDGE_EGRESS_FRAME_LIMIT) {
+						fail(new BridgeEgressUnavailableError("the egress reply is too large"));
+
+						return;
+					}
+
+					const frame = buffer.split("\n").at(0) ?? "";
+
+					if (buffer.includes("\n") && frame.trim().length > 0) {
+						try {
+							settle(JSON.parse(frame) as BridgeEgressResponse<BridgeEgressFetchResult>);
+						} catch {
+							fail(new BridgeEgressUnavailableError("the egress channel sent a malformed reply"));
+						}
+					}
+				},
+
+				error(_connection, error) {
+					fail(new BridgeEgressUnavailableError(error.message));
+				},
+
+				close() {
+					fail(new BridgeEgressUnavailableError("the egress channel closed the connection"));
+				},
+			});
+
+			try {
+				socket.write(frameOf(BridgeEgressMethod.Fetch, url, options));
+
+				const response = await Promise.race([
+					pending,
+					Bun.sleep(options.timeoutMs ?? CALL_TIMEOUT_MS).then(() => {
+						throw new BridgeEgressUnavailableError("the egress channel timed out");
+					}),
+				]);
+
+				if (!response.ok) {
+					throw new BridgeEgressCallError(response.status, response.error);
+				}
+
+				return response.result;
+			} finally {
+				socket.end();
+			}
+		},
+
+		async download(url, options = {}) {
+			let settle: (response: BridgeEgressResponse<BridgeEgressDownloadResult>) => void = () => undefined;
+			let fail: (error: Error) => void = () => undefined;
+
+			const pending = new Promise<BridgeEgressResponse<BridgeEgressDownloadResult>>((resolve, reject) => {
+				settle = resolve;
+				fail = reject;
+			});
+
+			const stream = createByteStream();
+
+			let header: string | null = null;
+			let headerBuffer = "";
+			let received = 0;
+			let expected: number | null = null;
+
+			const socket = await connect<undefined>({
+				data(connection, chunk) {
+					let body = chunk;
+
+					if (header === null) {
+						const boundary = chunk.indexOf(NEWLINE);
+
+						if (boundary === -1) {
+							headerBuffer += chunk.toString();
+
+							if (headerBuffer.length > BRIDGE_EGRESS_FRAME_LIMIT) {
+								fail(new BridgeEgressUnavailableError("the egress reply is too large"));
+							}
+
+							return;
+						}
+
+						header = headerBuffer + chunk.subarray(0, boundary).toString();
+
+						try {
+							const response = JSON.parse(header) as BridgeEgressResponse<BridgeEgressDownloadResult>;
+
+							settle(response);
+
+							if (!response.ok) {
+								return;
+							}
+
+							expected = response.result.sizeBytes;
+						} catch {
+							fail(new BridgeEgressUnavailableError("the egress channel sent a malformed reply"));
+
+							return;
+						}
+
+						body = chunk.subarray(boundary + 1);
+					}
+
+					if (body.byteLength === 0) {
+						return;
+					}
+
+					received += body.byteLength;
+					stream.push(new Uint8Array(body));
+
+					if (stream.queued >= QUEUE_HIGH_WATER_BYTES) {
+						connection.pause();
+					}
+
+					if (expected !== null && expected > 0 && received >= expected) {
+						stream.finish();
+					}
+				},
+
+				error(_connection, error) {
+					fail(new BridgeEgressUnavailableError(error.message));
+					stream.fail(new BridgeEgressUnavailableError(error.message));
+				},
+
+				close() {
+					fail(new BridgeEgressUnavailableError("the egress channel closed the connection"));
+
+					if (expected !== null && expected > 0 && received < expected) {
+						stream.fail(new BridgeEgressUnavailableError("the egress channel closed mid-transfer"));
+
+						return;
+					}
+
+					stream.finish();
+				},
+			});
+
+			stream.onDrain(() => {
+				socket.resume();
+			});
+
+			try {
+				socket.write(frameOf(BridgeEgressMethod.Download, url, options));
+
+				const response = await Promise.race([
+					pending,
+					Bun.sleep(options.timeoutMs ?? DOWNLOAD_TIMEOUT_MS).then(() => {
+						throw new BridgeEgressUnavailableError("the egress channel timed out");
+					}),
+				]);
+
+				if (!response.ok) {
+					socket.end();
+
+					throw new BridgeEgressCallError(response.status, response.error);
+				}
+
+				const body = async function* () {
+					try {
+						yield* stream.stream();
+					} finally {
+						socket.end();
+					}
+				};
+
+				return {
+					...response.result,
+					body: body(),
+				};
+			} catch (error) {
+				socket.end();
+
+				throw error;
+			}
+		},
+	};
+};

package/src/runtime/bridgeExec.ts

@@ -8,10 +8,20 @@ const OUTPUT_LIMIT = 256 * 1024;
 
 const EXIT_GRACE_MS = 3000;
 
-const BASE_ENVIRONMENT = {
-	HOME: "/home/container",
-	LANG: "C.UTF-8",
-	PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+const FALLBACK_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin";
+
+const inheritedPath = () => {
+	const declared = process.env.PATH ?? "";
+
+	return declared.length > 0 ? declared : FALLBACK_PATH;
+};
+
+const baseEnvironment = () => {
+	return {
+		HOME: "/home/container",
+		LANG: "C.UTF-8",
+		PATH: inheritedPath(),
+	};
 };
 
 export class BridgeExecError extends Error {
@@ -92,7 +102,7 @@ export const createBridgeExec = (target: BridgeExecTarget) => {
 		const child = Bun.spawn(argv, {
 			cwd: target.cwd,
 			env: {
-				...BASE_ENVIRONMENT,
+				...baseEnvironment(),
 				HOME: target.cwd,
 				...environment,
 			},

package/src/runtime/bridgeFiles.ts

@@ -2,7 +2,9 @@ import type { Dirent } from "node:fs";
 import { chown, mkdir, readdir, rename, rm, writeFile } from "node:fs/promises";
 import type { BridgeDownloadOptions, BridgeFileEntry, BridgeFiles } from "../context";
 import { compileGlob } from "../utils";
+import { type BridgeEgress, BridgeEgressCallError, BridgeEgressUnavailableError } from "./bridgeEgress";
 import type { BridgeLogger } from "./bridgeLog";
+import { USER_AGENT } from "./bridgeNet";
 import { BridgePathError, resolveWithin, toSegments } from "./bridgePath";
 import { isPrivileged } from "./bridgePrivilege";
 import { readZip } from "./bridgeZip";
@@ -21,8 +23,6 @@ const WALK_LIMIT = 4096;
 
 const MILESTONE_STEP = 25;
 
-const USER_AGENT = "serverk-bridge/1.0 (serverk.gg)";
-
 const DIGEST_LENGTHS: Record<string, number> = {
 	sha1: 40,
 	sha256: 64,
@@ -33,9 +33,15 @@ export interface BridgeFilesOptions {
 	root: string;
 	uid: number;
 	gid: number;
+	egress: BridgeEgress;
 	onLog: BridgeLogger;
 }
 
+interface DownloadSource {
+	body: AsyncIterable<Uint8Array>;
+	totalBytes: number | null;
+}
+
 const parseDigest = (digest: string) => {
 	const [algo, expected] = digest.split(":");
 
@@ -169,6 +175,49 @@ export const createBridgeFiles = (options: BridgeFilesOptions): BridgeFiles => {
 		await ensureDirectory(segments.slice(0, -1).join("/"));
 	};
 
+	const openSource = async (remote: URL, downloadOptions: BridgeDownloadOptions): Promise<DownloadSource> => {
+		if (downloadOptions.cache !== false) {
+			try {
+				const shared = await options.egress.download(remote.href, {
+					digest: downloadOptions.digest,
+					sizeBytes: downloadOptions.sizeBytes,
+				});
+
+				return {
+					body: shared.body,
+					totalBytes: shared.sizeBytes > 0 ? shared.sizeBytes : (downloadOptions.sizeBytes ?? null),
+				};
+			} catch (error) {
+				if (error instanceof BridgeEgressCallError) {
+					throw new BridgePathError("missing", `${remote.host} answered ${error.status ?? error.message}`);
+				}
+
+				if (!(error instanceof BridgeEgressUnavailableError)) {
+					throw error;
+				}
+			}
+		}
+
+		const response = await fetch(remote, {
+			headers: {
+				"user-agent": USER_AGENT,
+			},
+			redirect: "error",
+			signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS),
+		});
+
+		if (!response.ok || !response.body) {
+			throw new BridgePathError("missing", `${remote.host} answered ${response.status}`);
+		}
+
+		const declared = Number(response.headers.get("content-length"));
+
+		return {
+			body: response.body,
+			totalBytes: Number.isSafeInteger(declared) && declared > 0 ? declared : (downloadOptions.sizeBytes ?? null),
+		};
+	};
+
 	return {
 		async read(path) {
 			const resolved = await resolveWithin(options.root, path);
@@ -262,7 +311,7 @@ export const createBridgeFiles = (options: BridgeFilesOptions): BridgeFiles => {
 			const remote = new URL(url);
 
 			if (remote.protocol !== "https:") {
-				throw new BridgePathError("invalid", `url "${url}" is not https`);
+				throw new BridgePathError("invalid", `${remote.host} is not https`);
 			}
 
 			const limit = downloadOptions.sizeBytes ?? DOWNLOAD_LIMIT;
@@ -271,26 +320,14 @@ export const createBridgeFiles = (options: BridgeFilesOptions): BridgeFiles => {
 				throw new BridgePathError("invalid", `download of ${limit} bytes exceeds the limit`);
 			}
 
-			const response = await fetch(url, {
-				headers: {
-					"user-agent": USER_AGENT,
-				},
-				redirect: "follow",
-				signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS),
-			});
-
-			if (!response.ok || !response.body) {
-				throw new BridgePathError("missing", `${url} answered ${response.status}`);
-			}
+			const source = await openSource(remote, downloadOptions);
 
 			await ensureParent(path);
 
 			const resolved = await resolveWithin(options.root, path);
 			const staging = `${resolved.target}.download`;
 
-			const declared = Number(response.headers.get("content-length"));
-			const totalBytes =
-				Number.isSafeInteger(declared) && declared > 0 ? declared : (downloadOptions.sizeBytes ?? null);
+			const totalBytes = source.totalBytes;
 
 			const hasher = downloadOptions.digest ? new Bun.CryptoHasher(parseDigest(downloadOptions.digest).algo) : null;
 			const writer = Bun.file(staging).writer();
@@ -305,11 +342,11 @@ export const createBridgeFiles = (options: BridgeFilesOptions): BridgeFiles => {
 			let milestone = 0;
 
 			try {
-				for await (const chunk of response.body) {
+				for await (const chunk of source.body) {
 					readBytes += chunk.byteLength;
 
 					if (readBytes > limit) {
-						throw new BridgePathError("invalid", `${url} returned more than ${limit} bytes`);
+						throw new BridgePathError("invalid", `${remote.host} returned more than ${limit} bytes`);
 					}
 
 					hasher?.update(chunk);
@@ -336,7 +373,7 @@ export const createBridgeFiles = (options: BridgeFilesOptions): BridgeFiles => {
 				if (downloadOptions.sizeBytes !== undefined && readBytes !== downloadOptions.sizeBytes) {
 					throw new BridgePathError(
 						"invalid",
-						`${url} returned ${readBytes} bytes, expected ${downloadOptions.sizeBytes}`,
+						`${remote.host} returned ${readBytes} bytes, expected ${downloadOptions.sizeBytes}`,
 					);
 				}
 
@@ -344,7 +381,7 @@ export const createBridgeFiles = (options: BridgeFilesOptions): BridgeFiles => {
 					const { expected } = parseDigest(downloadOptions.digest);
 
 					if (hasher.digest("hex") !== expected) {
-						throw new BridgePathError("invalid", `${url} failed digest verification`);
+						throw new BridgePathError("invalid", `${remote.host} failed digest verification`);
 					}
 				}
 			} catch (error) {

package/src/runtime/bridgeLog.ts

@@ -1,15 +1,38 @@
+import { BRIDGE_LOG_SOURCE } from "../protocol/bridgeLine";
+import { BridgeTerminalLevel } from "../terminal";
+
 export type BridgeLogger = (message: string, fields?: Record<string, unknown>) => void;
 
-export const log: BridgeLogger = (message, fields = {}) => {
+export interface BridgeLeveledLogger extends BridgeLogger {
+	warn: BridgeLogger;
+	error: BridgeLogger;
+}
+
+const write = (level: BridgeTerminalLevel, message: string, fields: Record<string, unknown>) => {
 	process.stdout.write(
 		`${JSON.stringify({
-			source: "bridge",
-			message,
 			...fields,
+			source: BRIDGE_LOG_SOURCE,
+			level,
+			message,
 		})}\n`,
 	);
 };
 
+export const log: BridgeLeveledLogger = Object.assign(
+	(message: string, fields: Record<string, unknown> = {}) => {
+		write(BridgeTerminalLevel.Info, message, fields);
+	},
+	{
+		warn: (message: string, fields: Record<string, unknown> = {}) => {
+			write(BridgeTerminalLevel.Warn, message, fields);
+		},
+		error: (message: string, fields: Record<string, unknown> = {}) => {
+			write(BridgeTerminalLevel.Error, message, fields);
+		},
+	},
+);
+
 export const errorText = (error: unknown) => {
 	return error instanceof Error ? error.message : String(error);
 };

package/src/runtime/bridgeNet.ts

@@ -1,50 +1,103 @@
 import type { BridgeNet, BridgeRequestOptions } from "../context";
+import { BridgeNetError } from "../core";
+import { type BridgeEgress, BridgeEgressCallError, BridgeEgressUnavailableError } from "./bridgeEgress";
 
 const REQUEST_TIMEOUT_MS = 15_000;
 
 const RESPONSE_LIMIT = 8 * 1024 * 1024;
 
-const USER_AGENT = "serverk-bridge/1.0 (serverk.gg)";
+export const USER_AGENT = "serverk-bridge/1.0 (serverk.gg)";
 
-export class BridgeNetError extends Error {
-	readonly status: number | null;
+const LINE_BREAK = /[\r\n]/;
 
-	constructor(status: number | null, message: string) {
-		super(message);
+export const safeHeaders = (headers: Record<string, string> = {}) => {
+	const checked: Record<string, string> = {};
 
-		this.name = "BridgeNetError";
-		this.status = status;
+	for (const [key, value] of Object.entries(headers)) {
+		if (LINE_BREAK.test(key) || LINE_BREAK.test(value)) {
+			throw new BridgeNetError(null, "a header contains a line break");
+		}
+
+		checked[key] = value;
 	}
-}
 
-export const createBridgeNet = (): BridgeNet => {
-	const request = async (url: string, options: BridgeRequestOptions = {}) => {
-		const remote = new URL(url);
+	return checked;
+};
 
-		if (remote.protocol !== "https:") {
-			throw new BridgeNetError(null, `"${url}" is not https`);
+const readCapped = async (response: Response, host: string) => {
+	if (!response.body) {
+		return "";
+	}
+
+	const decoder = new TextDecoder();
+	const parts: string[] = [];
+
+	let read = 0;
+
+	for await (const chunk of response.body) {
+		read += chunk.byteLength;
+
+		if (read > RESPONSE_LIMIT) {
+			throw new BridgeNetError(null, `${host} returned more than ${RESPONSE_LIMIT} bytes`);
 		}
 
-		const response = await fetch(url, {
+		parts.push(
+			decoder.decode(chunk, {
+				stream: true,
+			}),
+		);
+	}
+
+	parts.push(decoder.decode());
+
+	return parts.join("");
+};
+
+export const createBridgeNet = (egress: BridgeEgress): BridgeNet => {
+	const direct = async (remote: URL, headers: Record<string, string>, timeoutMs: number) => {
+		const response = await fetch(remote, {
 			headers: {
+				...headers,
 				"user-agent": USER_AGENT,
-				...options.headers,
 			},
-			redirect: "follow",
-			signal: AbortSignal.timeout(options.timeoutMs ?? REQUEST_TIMEOUT_MS),
+			redirect: "error",
+			signal: AbortSignal.timeout(timeoutMs),
 		});
 
 		if (!response.ok) {
-			throw new BridgeNetError(response.status, `${url} answered ${response.status}`);
+			throw new BridgeNetError(response.status, `${remote.host} answered ${response.status}`);
 		}
 
-		const body = await response.text();
+		return await readCapped(response, remote.host);
+	};
+
+	const request = async (url: string, options: BridgeRequestOptions = {}) => {
+		const remote = new URL(url);
+
+		if (remote.protocol !== "https:") {
+			throw new BridgeNetError(null, `${remote.host} is not https`);
+		}
+
+		const headers = safeHeaders(options.headers);
+
+		try {
+			const result = await egress.fetch(url, {
+				...options,
+				headers,
+			});
+
+			return result.body;
+		} catch (error) {
+			if (error instanceof BridgeEgressCallError) {
+				throw new BridgeNetError(error.status, error.message);
+			}
 
-		if (body.length > RESPONSE_LIMIT) {
-			throw new BridgeNetError(null, `${url} returned more than ${RESPONSE_LIMIT} bytes`);
+			if (!(error instanceof BridgeEgressUnavailableError)) {
+				throw error;
+			}
 		}
 
-		return body;
+		return await direct(remote, headers, options.timeoutMs ?? REQUEST_TIMEOUT_MS);
 	};
 
 	return {