@kubb/studio 5.3.2 → 5.3.4

package/README.md

@@ -128,6 +128,10 @@ if (finished.status === 'failed') throw new Error(finished.error)
 const snapshot = finished.snapshot
 ```
 
+A snapshot job packs the tarball on the agent, not on Studio. The agent `PUT`s an empty request to
+a path Studio provides, gets back a redirect to a short-lived storage URL, and uploads the tarball
+there. The storage URL never crosses the WebSocket.
+
 ## Protocol
 
 `@kubb/studio/protocol` holds the WebSocket message types shared by both ends, so the agent and

package/dist/index.cjs

@@ -8,18 +8,21 @@ let node_child_process = require("node:child_process");
 let node_fs_promises = require("node:fs/promises");
 let node_path = require("node:path");
 node_path = require_rolldown_runtime.__toESM(node_path, 1);
+let _kubb_core = require("@kubb/core");
 let ofetch = require("ofetch");
 let node_crypto = require("node:crypto");
 let unstorage = require("unstorage");
 let unstorage_drivers_fs = require("unstorage/drivers/fs");
 unstorage_drivers_fs = require_rolldown_runtime.__toESM(unstorage_drivers_fs, 1);
-let _kubb_core = require("@kubb/core");
 let tinyexec = require("tinyexec");
 let magicast = require("magicast");
 let node_fs = require("node:fs");
 let node_module = require("node:module");
 let node_url = require("node:url");
 let remeda = require("remeda");
+let node_os = require("node:os");
+let node_zlib = require("node:zlib");
+let tsdown = require("tsdown");
 let ws = require("ws");
 ws = require_rolldown_runtime.__toESM(ws, 1);
 let node_timers_promises = require("node:timers/promises");
@@ -561,19 +564,20 @@ async function runRegistration({ token, studioUrl, poolSize }) {
 * Called on process termination or server close. A failed notify is logged and swallowed: the
 * local socket is already gone, and failing teardown must not block shutdown or reconnect.
 */
-async function disconnect({ sessionId, token, studioUrl, slug }) {
+async function disconnect({ sessionId, token, studioUrl, slug, logLevel }) {
 	const url = `${studioUrl}/api/agent/sessions/${sessionId}/disconnect`;
 	const tag = slug ?? "agent";
+	const canLog = logLevel !== void 0 && logLevel > _kubb_core.logLevel.silent;
 	try {
 		await (0, ofetch.ofetch)(url, {
 			method: "POST",
 			headers: { Authorization: `Bearer ${token}` }
 		});
-		console.log((0, node_util.styleText)("green", `[${tag}] Disconnected from Studio`));
+		if (canLog) console.error((0, node_util.styleText)("green", `[${tag}] Disconnected from Studio`));
 	} catch (error) {
 		const statusCode = error?.statusCode;
 		if (statusCode !== void 0 && statusCode >= 400 && statusCode < 500) return;
-		console.warn((0, node_util.styleText)("yellow", `[${tag}] Failed to notify Studio of disconnection: ${getErrorMessage(error)}`));
+		if (canLog) console.warn((0, node_util.styleText)("yellow", `[${tag}] Failed to notify Studio of disconnection: ${getErrorMessage(error)}`));
 	}
 }
 /**
@@ -610,18 +614,41 @@ async function createJob({ studioUrl, token, type, agentId, name, version, confi
 	return job;
 }
 /**
-* Polls `GET /api/jobs/{id}` until the job reaches `success` or `failed`.
+* A job runs a generation and packs a tarball, so it is never done the instant it is queued.
+*/
+const INITIAL_POLL_DELAY_MS = 2e3;
+/**
+* Slowest the poll backs off to. Requests per run are roughly `timeoutMs` divided by this, and
+* every concurrent run on the same organization key draws on one budget.
+*/
+const MAX_POLL_INTERVAL_MS = 3e4;
+/**
+* Polls `GET /api/jobs/{id}` until the job reaches `success` or `failed`, waiting
+* {@link INITIAL_POLL_DELAY_MS} first and doubling up to {@link MAX_POLL_INTERVAL_MS} so a long
+* job stays inside the API key's rate limit.
 *
 * A `failed` job resolves normally. Check `job.status` and `job.error`. Throws only when the
 * deadline passes before Studio finishes.
 */
 async function waitForJob({ studioUrl, token, id, timeoutMs = 6e4 }) {
 	const deadline = Date.now() + timeoutMs;
+	let interval = INITIAL_POLL_DELAY_MS;
 	for (;;) {
-		const { job } = await (0, ofetch.ofetch)(`${studioUrl}/api/jobs/${id}`, { headers: { "x-api-key": token } });
-		if (job.status === "success" || job.status === "failed") return job;
+		await new Promise((resolve) => setTimeout(resolve, Math.max(Math.min(interval, deadline - Date.now()), 0)));
 		if (Date.now() >= deadline) throw new Error("Timed out waiting for the Studio job");
-		await new Promise((resolve) => setTimeout(resolve, 1e3));
+		interval = Math.min(interval * 2, MAX_POLL_INTERVAL_MS);
+		try {
+			const { job } = await (0, ofetch.ofetch)(`${studioUrl}/api/jobs/${id}`, {
+				headers: { "x-api-key": token },
+				retry: false
+			});
+			if (job.status === "success" || job.status === "failed") return job;
+		} catch (error) {
+			const response = error.response;
+			if (response?.status !== 429) throw error;
+			const retryAfter = response._data?.data?.tryAgainIn;
+			interval = Math.max(interval, typeof retryAfter === "number" && Number.isFinite(retryAfter) && retryAfter > 0 ? retryAfter : MAX_POLL_INTERVAL_MS);
+		}
 	}
 }
 /**
@@ -651,7 +678,7 @@ async function createAgent({ studioUrl, token, name, machineToken }) {
 }
 //#endregion
 //#region package.json
-var version = "5.3.2";
+var version = "5.3.4";
 //#endregion
 //#region src/hooks.ts
 /**
@@ -1584,6 +1611,134 @@ async function generate({ config, hooks }) {
 	}
 }
 //#endregion
+//#region src/snapshotPackage.ts
+const gzipAsync = (0, node_util.promisify)(node_zlib.gzip);
+/**
+* Maps a generated file's path to its place inside the tarball, stripping everything before a
+* `src`/`dist` segment and any `..`/empty path segment so a crafted file name cannot escape the
+* `package/` root.
+*/
+function packagePath(filePath) {
+	const normalized = filePath.replaceAll("\\", "/");
+	return `package/${(normalized.match(/\/(?:src|dist)\/.*$/)?.[0].slice(1) ?? normalized.replace(/^\/+/, "")).split("/").filter((part) => part && part !== "." && part !== "..").join("/")}`;
+}
+/**
+* Splits a tarball entry path into the legacy 100-byte `name` field and, when the path does not
+* fit, the 155-byte USTAR `prefix` field that extends it. Throws rather than silently truncating
+* a path the format cannot address (max 256 bytes: 100 name + 1 separator + 155 prefix).
+*/
+function splitEntryPath(path) {
+	if (Buffer.byteLength(path, "utf8") <= 100) return {
+		name: path,
+		prefix: ""
+	};
+	for (let i = path.length - 1; i >= 0; i--) {
+		if (path[i] !== "/") continue;
+		const prefix = path.slice(0, i);
+		const name = path.slice(i + 1);
+		if (Buffer.byteLength(prefix, "utf8") <= 155 && Buffer.byteLength(name, "utf8") <= 100) return {
+			name,
+			prefix
+		};
+	}
+	throw new Error(`Snapshot path is too long for a tar entry: ${path}`);
+}
+function header(path, size) {
+	const { name, prefix } = splitEntryPath(path);
+	const value = Buffer.alloc(512);
+	value.write(name, 0, "utf8");
+	value.write("0000644\0", 100, "ascii");
+	value.write("0000000\0", 108, "ascii");
+	value.write("0000000\0", 116, "ascii");
+	value.write(`${size.toString(8).padStart(11, "0")}\0`, 124, "ascii");
+	value.write(`${Math.floor(Date.now() / 1e3).toString(8).padStart(11, "0")}\0`, 136, "ascii");
+	value.fill(32, 148, 156);
+	value.write("0", 156, "ascii");
+	value.write("ustar\0", 257, "ascii");
+	value.write("00", 263, "ascii");
+	value.write("0000000\0", 265, "ascii");
+	value.write("0000000\0", 297, "ascii");
+	value.write(prefix, 345, "utf8");
+	const checksum = [...value].reduce((sum, byte) => sum + byte, 0);
+	value.write(`${checksum.toString(8).padStart(6, "0")}\0 `, 148, "ascii");
+	return value;
+}
+/**
+* Packs a generation's files into a gzipped, npm-installable tarball: a `package/` root with the
+* generated sources, a `tsdown`-built `dist/` (esm + cjs), and a `package.json` manifest.
+*/
+async function createSnapshotPackage(files, packageInfo) {
+	const root = await (0, node_fs_promises.mkdtemp)((0, node_path.join)((0, node_os.tmpdir)(), "kubb-snapshot-"));
+	const dist = (0, node_path.join)(root, "dist");
+	const resolvedPaths = Object.entries(files).map(([name, content]) => ({
+		name,
+		content,
+		target: packagePath(name)
+	}));
+	const targetOwners = /* @__PURE__ */ new Map();
+	for (const { name, target } of resolvedPaths) {
+		const owner = targetOwners.get(target);
+		if (owner) throw new Error(`Snapshot has two generated files that sanitize to the same path "${target}": "${owner}" and "${name}"`);
+		targetOwners.set(target, name);
+	}
+	try {
+		await (0, node_fs_promises.mkdir)(dist);
+		await Promise.all(resolvedPaths.map(async ({ content, target }) => {
+			const path = (0, node_path.join)(root, target.slice(8));
+			await (0, node_fs_promises.mkdir)((0, node_path.join)(path, ".."), { recursive: true });
+			await (0, node_fs_promises.writeFile)(path, content);
+		}));
+		const sourceEntries = resolvedPaths.filter(({ name }) => /\.(?:[cm]?[jt]sx?)$/.test(name)).map(({ target }) => (0, node_path.join)(root, target.slice(8)));
+		if (sourceEntries.length) await (0, tsdown.build)({
+			entry: sourceEntries,
+			outDir: dist,
+			format: ["esm", "cjs"],
+			dts: false,
+			sourcemap: false,
+			unbundle: true,
+			report: false,
+			logLevel: "silent",
+			fixedExtension: true
+		});
+		const builtEntries = await Promise.all((await Array.fromAsync((0, node_fs_promises.glob)("**/*", {
+			cwd: dist,
+			withFileTypes: true
+		}))).filter((entry) => entry.isFile()).map(async (entry) => {
+			const filePath = (0, node_path.join)(entry.parentPath, entry.name);
+			return [`package/dist/${(0, node_path.relative)(dist, filePath).split(node_path.sep).join("/")}`, await (0, node_fs_promises.readFile)(filePath, "utf8")];
+		}));
+		const entries = {
+			"package/package.json": JSON.stringify({
+				...packageInfo,
+				type: "module",
+				main: "./dist/index.cjs",
+				module: "./dist/index.mjs",
+				exports: { ".": {
+					import: "./dist/index.mjs",
+					require: "./dist/index.cjs"
+				} }
+			}, null, 2),
+			...Object.fromEntries(resolvedPaths.map(({ content, target }) => [target, content])),
+			...Object.fromEntries(builtEntries)
+		};
+		const chunks = [];
+		for (const [name, content] of Object.entries(entries)) {
+			const bytes = Buffer.from(content);
+			chunks.push(header(name, bytes.length), bytes, Buffer.alloc((512 - bytes.length % 512) % 512));
+		}
+		const bytes = await gzipAsync(Buffer.concat([...chunks, Buffer.alloc(1024)]));
+		return {
+			bytes,
+			integrity: `sha512-${(0, node_crypto.createHash)("sha512").update(bytes).digest("base64")}`
+		};
+	} finally {
+		await (0, node_fs_promises.rm)(root, {
+			recursive: true,
+			force: true
+		});
+	}
+}
+//#endregion
 //#region src/ws.ts
 /**
 * How many generated files are read from storage at once when building the
@@ -1679,7 +1834,7 @@ function sendErrorMessage(ws$4, error, jobId) {
 /**
 * Forwards selected Kubb lifecycle events to Studio as data messages for the active session.
 */
-function setupEventsStream(ws$5, hooks, jobId) {
+function setupEventsStream(ws$5, hooks, jobId, options = {}) {
 	const unhooks = [];
 	/**
 	* Registers a listener and keeps its remover, so one generation's listeners come off the session
@@ -1793,11 +1948,12 @@ function setupEventsStream(ws$5, hooks, jobId) {
 				if (content !== null) files[relativeStoragePath(config.root, path)] = content;
 			}
 		});
+		options.onGenerationEnd?.(files);
 		sendDataMessage({
 			type: "kubb:generation:end",
 			data: [{
 				config,
-				storage: files,
+				storage: options.skipStorage ? {} : files,
 				peerDependencies,
 				missingDependencies
 			}]
@@ -1914,15 +2070,15 @@ function applyStudioDefaults(options) {
 * socket, its hook emitter, or its session id alive for the length of the retry interval.
 */
 function reconnect(options) {
-	const { signal, retryInterval, onTokenRejected } = options;
+	const { signal, retryInterval, onTokenRejected, logLevel } = options;
 	if (signal?.aborted) return;
-	console.info((0, node_util.styleText)("dim", `Retrying connection in ${retryInterval}ms to Kubb Studio ...`));
+	if (logLevel !== void 0 && logLevel > _kubb_core.logLevel.silent) console.error((0, node_util.styleText)("dim", `Retrying connection in ${retryInterval}ms to Kubb Studio ...`));
 	const cancel = () => clearTimeout(timer);
 	const timer = setTimeout(() => {
 		signal?.removeEventListener("abort", cancel);
 		if (signal?.aborted) return;
 		new StudioSession(options).connect().catch((error) => {
-			console.error((0, node_util.styleText)("red", `Reconnect attempt to Kubb Studio failed: ${getErrorMessage(error)}`));
+			if (logLevel !== void 0 && logLevel > _kubb_core.logLevel.silent) console.error((0, node_util.styleText)("red", `Reconnect attempt to Kubb Studio failed: ${getErrorMessage(error)}`));
 			if (error instanceof InvalidAgentTokenError) {
 				onTokenRejected?.(error);
 				return;
@@ -1957,6 +2113,7 @@ var StudioSession = class {
 	#heartbeatTimer;
 	#readyTimer;
 	#lastPongAt = Date.now();
+	#lastGeneration;
 	constructor(options) {
 		this.#options = applyStudioDefaults(options);
 	}
@@ -2139,7 +2296,7 @@ var StudioSession = class {
 	* `#disposed` keeps the close event from running this twice, and a shutdown from reconnecting.
 	*/
 	async #end({ reason, retry }) {
-		const { studioUrl, token } = this.#options;
+		const { studioUrl, token, logLevel } = this.#options;
 		if (this.#disposed) return;
 		this.#disposed = true;
 		if (reason === "shutdown" && this.#ws) sendAgentMessage(this.#ws, {
@@ -2151,7 +2308,8 @@ var StudioSession = class {
 			sessionId: this.#session.sessionId,
 			studioUrl,
 			token,
-			slug: this.#session.slug
+			slug: this.#session.slug,
+			logLevel
 		}).catch(() => {});
 		if (retry) reconnect(this.#options);
 	}
@@ -2210,6 +2368,9 @@ var StudioSession = class {
 			case "studio:save":
 				await this.#handleSave(ws, data, command);
 				return;
+			case "studio:snapshot":
+				await this.#handleSnapshot(ws, data, command);
+				return;
 		}
 	}
 	async #handleGenerate(ws, data, command) {
@@ -2233,7 +2394,13 @@ var StudioSession = class {
 				await this.#warn(`Ignored the spec from Studio; set ${remedy} to generate from it`);
 			}
 			const resolvedPlugins = plugins ?? config.plugins;
-			const detach = [setupHookListener(this.#hooks, root), setupEventsStream(ws, this.#hooks, data.jobId)];
+			let generatedFiles;
+			const detach = [setupHookListener(this.#hooks, root), setupEventsStream(ws, this.#hooks, data.jobId, {
+				skipStorage: client?.kind === "ci",
+				onGenerationEnd: (files) => {
+					generatedFiles = files;
+				}
+			})];
 			try {
 				await generate({
 					config: {
@@ -2254,6 +2421,7 @@ var StudioSession = class {
 				});
 			} finally {
 				for (const remove of detach) remove();
+				this.#lastGeneration = generatedFiles;
 			}
 			await this.#hooks.callHook("studio:command:end", {
 				command,
@@ -2322,6 +2490,68 @@ var StudioSession = class {
 			refuse(getErrorMessage(error));
 		}
 	}
+	async #handleSnapshot(ws, data, command) {
+		const refuse = (message) => sendAgentMessage(ws, {
+			type: "agent:snapshot",
+			jobId: data.jobId,
+			payload: {
+				status: "error",
+				message
+			}
+		});
+		if (this.#isSandbox) {
+			await this.#warn("Ignored snapshot: a sandbox agent has no project to build a package from");
+			refuse("a sandbox agent has no project to build a package from");
+			return;
+		}
+		const { name, version, peerDependencies, uploadPath } = data.payload;
+		if (!name || !version || !uploadPath) {
+			await this.#warn("Ignored snapshot: the message was missing required fields");
+			refuse("the message was missing required fields");
+			return;
+		}
+		const files = this.#lastGeneration;
+		if (!files) {
+			await this.#warn("Ignored snapshot: no prior generation to pack");
+			refuse("no prior generation exists to pack, run a generation first");
+			return;
+		}
+		try {
+			const { bytes, integrity } = await createSnapshotPackage(files, {
+				name,
+				version,
+				peerDependencies: peerDependencies ?? {}
+			});
+			const { token, studioUrl } = this.#options;
+			const redirect = await fetch(new URL(uploadPath, studioUrl), {
+				method: "PUT",
+				headers: { Authorization: `Bearer ${token}` },
+				redirect: "manual"
+			});
+			const storageUrl = redirect.headers.get("location");
+			if (redirect.status !== 307 || !storageUrl) throw new Error(`Studio did not provide a storage URL (status ${redirect.status})`);
+			const response = await fetch(storageUrl, {
+				method: "PUT",
+				body: new Uint8Array(bytes)
+			});
+			if (!response.ok) throw new Error(`Snapshot upload failed with status ${response.status}`);
+			sendAgentMessage(ws, {
+				type: "agent:snapshot",
+				jobId: data.jobId,
+				payload: {
+					status: "ok",
+					integrity
+				}
+			});
+			await this.#hooks.callHook("studio:command:end", {
+				command,
+				info: `packed ${Object.keys(files).length} file${Object.keys(files).length === 1 ? "" : "s"}`
+			});
+		} catch (error) {
+			await this.#hooks.callHook("studio:error", { error: toError(error) });
+			refuse(getErrorMessage(error));
+		}
+	}
 };
 //#endregion
 //#region src/client.ts