@kubb/studio 5.3.11 → 5.3.13

package/dist/index.js

@@ -1,12 +1,12 @@
 import { t as __name } from "./rolldown-runtime-CRm0XQPb.js";
-import { generationEventTypes } from "./protocol.js";
+import { GENERATION_GONE_MESSAGE, generationEventTypes } from "./protocol.js";
 import { createRequire } from "node:module";
 import { promisify, styleText } from "node:util";
 import process$1 from "node:process";
 import { spawn } from "node:child_process";
 import { glob, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
 import path, { isAbsolute, join, relative, resolve, sep } from "node:path";
-import { Diagnostics, Hookable, createKubb, fsStorage, logLevel, memoryStorage } from "@kubb/core";
+import { Diagnostics, Hookable, cacheStorage, createKubb, fsStorage, logLevel, memoryStorage } from "@kubb/core";
 import { FetchError, ofetch } from "ofetch";
 import { createHash, hash, randomBytes } from "node:crypto";
 import { createStorage } from "unstorage";
@@ -44,8 +44,27 @@ const agentDefaults = {
 	maxHeartbeatIntervalMs: 6e4,
 	/** How long a heartbeat ping may take before the session is treated as dead. */
 	heartbeatTimeoutMs: 1e4,
-	poolSize: 1
+	poolSize: 1,
+	maxGenerations: 8,
+	maxGenerationsMb: 100,
+	maxSnapshotMb: 50
 };
+function positiveNumber(value) {
+	const parsed = Number(value);
+	return value && Number.isFinite(parsed) && parsed > 0 ? parsed : void 0;
+}
+/**
+* How many generations an agent keeps and how large they may get, read from
+* `KUBB_AGENT_MAX_GENERATIONS`, `KUBB_AGENT_MAX_GENERATIONS_MB` and `KUBB_AGENT_MAX_SNAPSHOT_MB`.
+* An unset or invalid value keeps the default.
+*/
+function resolveGenerationLimits(env = process.env) {
+	return {
+		maxCount: Math.max(1, Math.floor(positiveNumber(env.KUBB_AGENT_MAX_GENERATIONS) ?? agentDefaults.maxGenerations)),
+		maxMb: positiveNumber(env.KUBB_AGENT_MAX_GENERATIONS_MB) ?? agentDefaults.maxGenerationsMb,
+		maxSnapshotMb: positiveNumber(env.KUBB_AGENT_MAX_SNAPSHOT_MB) ?? agentDefaults.maxSnapshotMb
+	};
+}
 //#endregion
 //#region ../../internals/utils/src/casing.ts
 /**
@@ -676,7 +695,7 @@ async function createAgent({ studioUrl, token, name, machineToken }) {
 }
 //#endregion
 //#region package.json
-var version = "5.3.11";
+var version = "5.3.13";
 //#endregion
 //#region src/hooks.ts
 /**
@@ -1765,6 +1784,116 @@ async function createSnapshotPackage(files, packageInfo) {
 	}
 }
 //#endregion
+//#region src/generations.ts
+const READ_CONCURRENCY = 50;
+const MB = 1048576;
+const INDEX_KEY = "studio/generations.json";
+const hashOf = (content) => createHash("sha1").update(content).digest("hex").slice(0, 16);
+/**
+* What the output directory holds on disk before a run. `undefined` when `outputPath` is not a real
+* subdirectory of `root` (listing the root would take in every source file) or holds more than `maxFiles`.
+*/
+async function listDisk({ root, outputPath, maxFiles }) {
+	const outputDir = resolve(root, outputPath);
+	const fromRoot = relative(resolve(root), outputDir);
+	if (!fromRoot || fromRoot.startsWith("..") || fromRoot.startsWith(sep)) return void 0;
+	const storage = fsStorage();
+	const keys = await storage.readKeys(outputDir);
+	if (keys.length > maxFiles) return void 0;
+	const paths = keys.filter((key) => !key.split("/").includes("node_modules")).map((key) => relative(resolve(root), resolve(outputDir, key)).replaceAll("\\", "/"));
+	return {
+		storage,
+		root,
+		paths: new Set(paths)
+	};
+}
+/**
+* Keeps recent generations by job id in `storage`, with an index next to them so a store on disk
+* survives a restart. A generation is only ever looked up by its job id, so on a pooled sandbox one
+* tenant never reaches another's output. The oldest go past `maxCount` or `maxMb`, but the newest
+* always stays.
+*/
+function createGenerationStore({ storage, maxCount, maxMb }) {
+	let index;
+	const dirOf = (jobId) => `studio/generations/${hashOf(jobId)}/`;
+	async function load() {
+		if (index) return index;
+		const stored = await storage.readItem(INDEX_KEY).catch(() => null);
+		try {
+			index = stored ? JSON.parse(stored) : [];
+		} catch {
+			index = [];
+		}
+		return index;
+	}
+	/**
+	* Copies `files` into the store as one set of `jobId`. Above `maxSetMb` only the hashes are kept.
+	*/
+	async function keep({ jobId, source, files, maxSetMb }) {
+		const maxSetBytes = maxSetMb * MB;
+		const hashes = {};
+		let bytes = 0;
+		await inParallel({
+			items: [...files.paths],
+			limit: READ_CONCURRENCY,
+			run: async (path) => {
+				if (path.split("/").includes("..")) return;
+				const content = await files.storage.readItem(resolve(files.root, path));
+				if (content === null) return;
+				hashes[path] = hashOf(content);
+				bytes += Buffer.byteLength(content);
+				if (bytes <= maxSetBytes) await storage.writeItem(`${dirOf(jobId)}${source}/${path}`, content);
+			}
+		});
+		if (bytes > maxSetBytes) {
+			await storage.empty(`${dirOf(jobId)}${source}/`);
+			return {
+				paths: [],
+				hashes,
+				bytes: 0
+			};
+		}
+		return {
+			paths: Object.keys(hashes),
+			hashes,
+			bytes
+		};
+	}
+	async function drop(jobId) {
+		await storage.empty(dirOf(jobId));
+	}
+	return {
+		keep,
+		drop,
+		get: async (jobId) => (await load()).find((generation) => generation.jobId === jobId),
+		latest: async () => (await load()).at(-1),
+		async add(generation) {
+			const entries = (await load()).filter((entry) => entry.jobId !== generation.jobId);
+			entries.push(generation);
+			const weight = () => entries.reduce((sum, { output, disk }) => sum + output.bytes + (disk?.bytes ?? 0), 0);
+			while (entries.length > 1 && (entries.length > maxCount || weight() > maxMb * MB)) await drop(entries.shift().jobId);
+			index = entries;
+			await storage.writeItem(INDEX_KEY, JSON.stringify(entries));
+		},
+		/**
+		* Reads the requested paths the set holds, skipping any it does not.
+		*/
+		async read({ generation, source, paths }) {
+			const kept = new Set(generation[source]?.paths);
+			const files = {};
+			await inParallel({
+				items: paths.filter((path) => kept.has(path)),
+				limit: READ_CONCURRENCY,
+				run: async (path) => {
+					const content = await storage.readItem(`${dirOf(generation.jobId)}${source}/${path}`);
+					if (content !== null) files[path] = content;
+				}
+			});
+			return files;
+		}
+	};
+}
+//#endregion
 //#region src/ws.ts
 /**
 * How long the initial handshake may take before the socket is closed and the reconnect loop
@@ -1775,12 +1904,6 @@ const require = createRequire(import.meta.url);
 function relativeStoragePath(root, filePath) {
 	return (isAbsolute(filePath) ? relative(resolve(root), filePath) : filePath).replaceAll("\\", "/");
 }
-/**
-* Inverse of {@link relativeStoragePath}: rebuilds the storage key a relative path came from.
-*/
-function absoluteStoragePath(root, relativePath) {
-	return resolve(root, relativePath);
-}
 async function resolvePeerDependencies(names) {
 	const uniqueNames = [...new Set(names.map(toPackageName))];
 	const peerDependencies = {};
@@ -1907,9 +2030,11 @@ function createGenerationStream(hooks, jobId, options = {}) {
 		const keys = await storage.readKeys();
 		const paths = new Set(keys.map((key) => relativeStoragePath(config.root, key)));
 		options.onGenerationEnd?.({
-			storage,
-			root: config.root,
-			paths,
+			output: {
+				storage,
+				root: config.root,
+				paths
+			},
 			peerDependencies,
 			missingDependencies
 		});
@@ -2060,9 +2185,9 @@ const connectWebSocketRpc = async ({ url, token, local }) => {
 //#endregion
 //#region src/StudioSession.ts
 /**
-* How many files are read from storage at once when serving `readFiles` or packing a snapshot.
+* Past this many files in the output directory, no snapshot of it is taken before a run.
 */
-const FILE_READ_CONCURRENCY = 50;
+const DISK_SNAPSHOT_MAX_FILES = 1e4;
 var GenerationRunTarget = class extends RpcTarget {
 	generationStream;
 	generationResult;
@@ -2161,6 +2286,8 @@ var StudioSession = class {
 	#isGenerating = false;
 	#heartbeatTimer;
 	#lastGeneration;
+	#store;
+	#limits = resolveGenerationLimits();
 	/**
 	* Resolves when Studio calls {@link StudioSession.connect}. `studio:ready` waits on this so the
 	* host does not queue jobs before the agent session is registered.
@@ -2176,6 +2303,18 @@ var StudioSession = class {
 	get #isSandbox() {
 		return this.#session?.isSandbox === true;
 	}
+	/**
+	* Kept in the project's cache directory, so it survives a restart, except on a sandbox: its pool
+	* sessions run every tenant's jobs, so it keeps them in memory. Looked up by job id only.
+	*/
+	get #generations() {
+		this.#store ??= createGenerationStore({
+			storage: this.#isSandbox ? memoryStorage() : cacheStorage({ root: this.#options.root }),
+			maxCount: this.#limits.maxCount,
+			maxMb: this.#limits.maxMb
+		});
+		return this.#store;
+	}
 	get #canWrite() {
 		return !this.#isSandbox && this.#options.permissions.allowWrite;
 	}
@@ -2386,6 +2525,17 @@ var StudioSession = class {
 			}
 			const resolvedPlugins = plugins ?? config.plugins;
 			this.#lastGeneration = void 0;
+			const diskFiles = this.#hasProjectOnDisk ? await listDisk({
+				root,
+				outputPath: config.output.path,
+				maxFiles: DISK_SNAPSHOT_MAX_FILES
+			}) : void 0;
+			const disk = diskFiles ? await this.#generations.keep({
+				jobId: data.jobId,
+				source: "disk",
+				files: diskFiles,
+				maxSetMb: this.#limits.maxSnapshotMb
+			}) : void 0;
 			const detach = [setupHookListener(this.#hooks, root, controller.signal)];
 			try {
 				await generate({
@@ -2406,6 +2556,9 @@ var StudioSession = class {
 					hooks: this.#hooks,
 					signal: controller.signal
 				});
+			} catch (error) {
+				await this.#generations.drop(data.jobId);
+				throw error;
 			} finally {
 				for (const remove of detach) remove();
 			}
@@ -2413,11 +2566,31 @@ var StudioSession = class {
 				command,
 				info: `${resolvedPlugins.length} plugin${resolvedPlugins.length === 1 ? "" : "s"}, ${this.#canWrite ? "written to disk" : "in memory"}${inputOverride !== void 0 ? ", from a Studio spec" : ""}`
 			});
-			const files = [...this.#lastGeneration?.paths ?? []];
+			const generation = this.#lastGeneration;
+			const output = generation ? await this.#generations.keep({
+				jobId: data.jobId,
+				source: "output",
+				files: generation.output,
+				maxSetMb: this.#limits.maxMb
+			}) : void 0;
+			if (generation && output) {
+				if (!output.paths.length && Object.keys(output.hashes).length) await this.#warn("Kept only the hashes of this generation: its output is too large to keep");
+				const { peerDependencies, missingDependencies } = generation;
+				await this.#generations.add({
+					jobId: data.jobId,
+					output,
+					disk,
+					peerDependencies,
+					missingDependencies
+				});
+			}
+			const files = [...generation?.output.paths ?? []];
 			return {
 				status: "success",
 				files,
-				fileCount: files.length
+				fileCount: files.length,
+				hashes: output?.hashes ?? {},
+				disk: disk ? { hashes: disk.hashes } : void 0
 			};
 		} finally {
 			this.#isGenerating = false;
@@ -2472,20 +2645,16 @@ var StudioSession = class {
 		if (this.#isSandbox) return this.#refuse("Ignored snapshot: a sandbox agent has no project to build a package from", "A sandbox agent has no project to build a package from");
 		const { name, version, bundledDependencies, uploadPath } = data;
 		if (!name || !version || !uploadPath) return this.#refuse("Ignored snapshot: the message was missing required fields", "The request was missing required fields");
-		const generation = this.#lastGeneration;
+		const generation = await this.#generations.latest();
 		if (!generation) return this.#refuse("Ignored snapshot: no prior generation to pack", "No prior generation exists to pack, run a generation first");
 		const bundled = new Set(bundledDependencies ?? []);
 		const missing = generation.missingDependencies.filter((dependency) => !bundled.has(dependency));
 		if (missing.length) return this.#refuse(`Ignored snapshot: missing dependencies: ${missing.join(", ")}`, `Missing dependencies: ${missing.join(", ")}`);
 		try {
-			const files = {};
-			await inParallel({
-				items: [...generation.paths],
-				limit: FILE_READ_CONCURRENCY,
-				run: async (relativePath) => {
-					const content = await generation.storage.readItem(absoluteStoragePath(generation.root, relativePath));
-					if (content !== null) files[relativePath] = content;
-				}
+			const files = await this.#generations.read({
+				generation,
+				source: "output",
+				paths: generation.output.paths
 			});
 			const { bytes, integrity } = await createSnapshotPackage(files, {
 				name,
@@ -2523,6 +2692,13 @@ var StudioSession = class {
 			throw error;
 		}
 	}
+	/**
+	* An agent with a project on disk can show a run against what its output directory held before.
+	* A sandbox agent has no project.
+	*/
+	get #hasProjectOnDisk() {
+		return !this.#isSandbox && this.#canRead;
+	}
 	async readFiles(data) {
 		const command = "readFiles";
 		await this.#hooks.callHook("studio:command:start", { command });
@@ -2535,17 +2711,15 @@ var StudioSession = class {
 		if (!Array.isArray(data.paths)) return this.#refuse("Ignored files: the message carried no paths", "The request carried no paths");
 		const { paths } = data;
 		if (paths.length > 50) return this.#refuse(`Ignored files: requested ${paths.length} paths, more than the 50 allowed per request`, `At most 50 paths may be requested at once`);
-		const generation = this.#lastGeneration;
-		if (!generation) return this.#refuse("Ignored files: no prior generation to read from", "No prior generation to read from, run a generation first");
-		const requested = paths.filter((path) => generation.paths.has(path));
-		const files = {};
-		await inParallel({
-			items: requested,
-			limit: FILE_READ_CONCURRENCY,
-			run: async (path) => {
-				const content = await generation.storage.readItem(absoluteStoragePath(generation.root, path));
-				if (content !== null) files[path] = content;
-			}
+		if (typeof data.jobId !== "string" || !data.jobId) return this.#refuse("Ignored files: the message named no job", "The request named no generation job");
+		const generation = await this.#generations.get(data.jobId);
+		if (!generation) return this.#refuse(`Ignored files: job ${data.jobId} is not kept on this agent`, GENERATION_GONE_MESSAGE);
+		const source = data.source === "disk" ? "disk" : "output";
+		if (!generation[source]) return this.#refuse("Ignored files: that job has no snapshot of the files on disk", "This agent kept no snapshot of the files on disk for that job");
+		const files = await this.#generations.read({
+			generation,
+			source,
+			paths
 		});
 		await this.#hooks.callHook("studio:command:end", {
 			command,