@kubb/studio 5.3.12 → 5.3.14

package/dist/index.d.ts

@@ -1,5 +1,5 @@
 import { t as __name } from "./rolldown-runtime-CRm0XQPb.js";
-import { AgentApi, AgentPermissions, ClientInfo, ConfigEdit, ConnectMessagePayload, FileChange, GenerateInput, GenerateResult, GenerationEvent, GenerationEventPayloads, GenerationEventType, GenerationRun, PublishSnapshotInput, PublishSnapshotResult, RpcConnection, RpcConnector, StudioApi, generationEventTypes } from "./protocol.js";
+import { AgentApi, AgentPermissions, ClientInfo, ConfigEdit, ConnectMessagePayload, GenerateInput, GenerateResult, GenerationEvent, GenerationEventPayloads, GenerationEventType, GenerationRun, PublishSnapshotInput, PublishSnapshotResult, RpcConnection, RpcConnector, StudioApi, generationEventTypes } from "./protocol.js";
 import { Config, Hookable, KubbHooks } from "@kubb/core";
 import { Storage } from "unstorage";
 //#region src/api.d.ts
@@ -554,5 +554,5 @@ export declare function pollForPairingToken({ studioUrl, session, signal }: Poll
  */
 export declare const connectWebSocketRpc: RpcConnector;
 //#endregion
-export { type AgentApi, type Client, type ClientOptions, type ConfigEdit, type ConnectMessagePayload, type ConnectionOptions, type FileChange, type GenerateInput, type GenerateResult, type GenerationEvent, type GenerationEventPayloads, type GenerationEventType, type GenerationRun, type PublishSnapshotInput, type PublishSnapshotResult, type RpcConnection, type RpcConnector, type StudioAgent, type StudioApi, type StudioConnectedContext, type StudioJob, type StudioJobStatus, type StudioSnapshot, generationEventTypes };
+export { type AgentApi, type Client, type ClientOptions, type ConfigEdit, type ConnectMessagePayload, type ConnectionOptions, type GenerateInput, type GenerateResult, type GenerationEvent, type GenerationEventPayloads, type GenerationEventType, type GenerationRun, type PublishSnapshotInput, type PublishSnapshotResult, type RpcConnection, type RpcConnector, type StudioAgent, type StudioApi, type StudioConnectedContext, type StudioJob, type StudioJobStatus, type StudioSnapshot, generationEventTypes };
 //# sourceMappingURL=index.d.ts.map

package/dist/index.js

@@ -1,12 +1,12 @@
 import { t as __name } from "./rolldown-runtime-CRm0XQPb.js";
-import { generationEventTypes } from "./protocol.js";
+import { GENERATION_GONE_MESSAGE, generationEventTypes } from "./protocol.js";
 import { createRequire } from "node:module";
 import { promisify, styleText } from "node:util";
 import process$1 from "node:process";
 import { spawn } from "node:child_process";
 import { glob, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
 import path, { isAbsolute, join, relative, resolve, sep } from "node:path";
-import { Diagnostics, Hookable, createKubb, fsStorage, logLevel, memoryStorage } from "@kubb/core";
+import { Diagnostics, Hookable, cacheStorage, createKubb, fsStorage, logLevel, memoryStorage } from "@kubb/core";
 import { FetchError, ofetch } from "ofetch";
 import { createHash, hash, randomBytes } from "node:crypto";
 import { createStorage } from "unstorage";
@@ -44,8 +44,27 @@ const agentDefaults = {
 	maxHeartbeatIntervalMs: 6e4,
 	/** How long a heartbeat ping may take before the session is treated as dead. */
 	heartbeatTimeoutMs: 1e4,
-	poolSize: 1
+	poolSize: 1,
+	maxGenerations: 8,
+	maxGenerationsMb: 100,
+	maxSnapshotMb: 50
 };
+function positiveNumber(value) {
+	const parsed = Number(value);
+	return value && Number.isFinite(parsed) && parsed > 0 ? parsed : void 0;
+}
+/**
+* How many generations an agent keeps and how large they may get, read from
+* `KUBB_AGENT_MAX_GENERATIONS`, `KUBB_AGENT_MAX_GENERATIONS_MB` and `KUBB_AGENT_MAX_SNAPSHOT_MB`.
+* An unset or invalid value keeps the default.
+*/
+function resolveGenerationLimits(env = process.env) {
+	return {
+		maxCount: Math.max(1, Math.floor(positiveNumber(env.KUBB_AGENT_MAX_GENERATIONS) ?? agentDefaults.maxGenerations)),
+		maxMb: positiveNumber(env.KUBB_AGENT_MAX_GENERATIONS_MB) ?? agentDefaults.maxGenerationsMb,
+		maxSnapshotMb: positiveNumber(env.KUBB_AGENT_MAX_SNAPSHOT_MB) ?? agentDefaults.maxSnapshotMb
+	};
+}
 //#endregion
 //#region ../../internals/utils/src/casing.ts
 /**
@@ -676,7 +695,7 @@ async function createAgent({ studioUrl, token, name, machineToken }) {
 }
 //#endregion
 //#region package.json
-var version = "5.3.12";
+var version = "5.3.14";
 //#endregion
 //#region src/hooks.ts
 /**
@@ -1765,6 +1784,116 @@ async function createSnapshotPackage(files, packageInfo) {
 	}
 }
 //#endregion
+//#region src/generations.ts
+const READ_CONCURRENCY = 50;
+const MB = 1048576;
+const INDEX_KEY = "studio/generations.json";
+const hashOf = (content) => createHash("sha1").update(content).digest("hex").slice(0, 16);
+/**
+* What the output directory holds on disk before a run. `undefined` when `outputPath` is not a real
+* subdirectory of `root` (listing the root would take in every source file) or holds more than `maxFiles`.
+*/
+async function listDisk({ root, outputPath, maxFiles }) {
+	const outputDir = resolve(root, outputPath);
+	const fromRoot = relative(resolve(root), outputDir);
+	if (!fromRoot || fromRoot.startsWith("..") || fromRoot.startsWith(sep)) return void 0;
+	const storage = fsStorage();
+	const keys = await storage.readKeys(outputDir);
+	if (keys.length > maxFiles) return void 0;
+	const paths = keys.filter((key) => !key.split("/").includes("node_modules")).map((key) => relative(resolve(root), resolve(outputDir, key)).replaceAll("\\", "/"));
+	return {
+		storage,
+		root,
+		paths: new Set(paths)
+	};
+}
+/**
+* Keeps recent generations by job id in `storage`, with an index next to them so a store on disk
+* survives a restart. A generation is only ever looked up by its job id, so on a pooled sandbox one
+* tenant never reaches another's output. The oldest go past `maxCount` or `maxMb`, but the newest
+* always stays.
+*/
+function createGenerationStore({ storage, maxCount, maxMb }) {
+	let index;
+	const dirOf = (jobId) => `studio/generations/${hashOf(jobId)}/`;
+	async function load() {
+		if (index) return index;
+		const stored = await storage.readItem(INDEX_KEY).catch(() => null);
+		try {
+			index = stored ? JSON.parse(stored) : [];
+		} catch {
+			index = [];
+		}
+		return index;
+	}
+	/**
+	* Copies `files` into the store as one set of `jobId`. Above `maxSetMb` only the hashes are kept.
+	*/
+	async function keep({ jobId, source, files, maxSetMb }) {
+		const maxSetBytes = maxSetMb * MB;
+		const hashes = {};
+		let bytes = 0;
+		await inParallel({
+			items: [...files.paths],
+			limit: READ_CONCURRENCY,
+			run: async (path) => {
+				if (path.split("/").includes("..")) return;
+				const content = await files.storage.readItem(resolve(files.root, path));
+				if (content === null) return;
+				hashes[path] = hashOf(content);
+				bytes += Buffer.byteLength(content);
+				if (bytes <= maxSetBytes) await storage.writeItem(`${dirOf(jobId)}${source}/${path}`, content);
+			}
+		});
+		if (bytes > maxSetBytes) {
+			await storage.empty(`${dirOf(jobId)}${source}/`);
+			return {
+				paths: [],
+				hashes,
+				bytes: 0
+			};
+		}
+		return {
+			paths: Object.keys(hashes),
+			hashes,
+			bytes
+		};
+	}
+	async function drop(jobId) {
+		await storage.empty(dirOf(jobId));
+	}
+	return {
+		keep,
+		drop,
+		get: async (jobId) => (await load()).find((generation) => generation.jobId === jobId),
+		latest: async () => (await load()).at(-1),
+		async add(generation) {
+			const entries = (await load()).filter((entry) => entry.jobId !== generation.jobId);
+			entries.push(generation);
+			const weight = () => entries.reduce((sum, { output, disk }) => sum + output.bytes + (disk?.bytes ?? 0), 0);
+			while (entries.length > 1 && (entries.length > maxCount || weight() > maxMb * MB)) await drop(entries.shift().jobId);
+			index = entries;
+			await storage.writeItem(INDEX_KEY, JSON.stringify(entries));
+		},
+		/**
+		* Reads the requested paths the set holds, skipping any it does not.
+		*/
+		async read({ generation, source, paths }) {
+			const kept = new Set(generation[source]?.paths);
+			const files = {};
+			await inParallel({
+				items: paths.filter((path) => kept.has(path)),
+				limit: READ_CONCURRENCY,
+				run: async (path) => {
+					const content = await storage.readItem(`${dirOf(generation.jobId)}${source}/${path}`);
+					if (content !== null) files[path] = content;
+				}
+			});
+			return files;
+		}
+	};
+}
+//#endregion
 //#region src/ws.ts
 /**
 * How long the initial handshake may take before the socket is closed and the reconnect loop
@@ -1775,12 +1904,6 @@ const require = createRequire(import.meta.url);
 function relativeStoragePath(root, filePath) {
 	return (isAbsolute(filePath) ? relative(resolve(root), filePath) : filePath).replaceAll("\\", "/");
 }
-/**
-* Inverse of {@link relativeStoragePath}: rebuilds the storage key a relative path came from.
-*/
-function absoluteStoragePath(root, relativePath) {
-	return resolve(root, relativePath);
-}
 async function resolvePeerDependencies(names) {
 	const uniqueNames = [...new Set(names.map(toPackageName))];
 	const peerDependencies = {};
@@ -1906,10 +2029,12 @@ function createGenerationStream(hooks, jobId, options = {}) {
 		const { peerDependencies, missingDependencies } = await resolvePeerDependencies(config.plugins.map(({ name }) => name));
 		const keys = await storage.readKeys();
 		const paths = new Set(keys.map((key) => relativeStoragePath(config.root, key)));
-		if ((status ?? "success") === "success") options.onGenerationEnd?.({
-			storage,
-			root: config.root,
-			paths,
+		options.onGenerationEnd?.({
+			output: {
+				storage,
+				root: config.root,
+				paths
+			},
 			peerDependencies,
 			missingDependencies
 		});
@@ -2060,39 +2185,9 @@ const connectWebSocketRpc = async ({ url, token, local }) => {
 //#endregion
 //#region src/StudioSession.ts
 /**
-* How many files are read from storage at once when serving `readFiles` or packing a snapshot.
+* Past this many files in the output directory, no snapshot of it is taken before a run.
 */
-const FILE_READ_CONCURRENCY = 50;
-/**
-* Reads every file a run produced back out of its storage.
-*/
-async function readSnapshot(generation) {
-	const snapshot = /* @__PURE__ */ new Map();
-	await inParallel({
-		items: [...generation.paths],
-		limit: FILE_READ_CONCURRENCY,
-		run: async (path) => {
-			const content = await generation.storage.readItem(absoluteStoragePath(generation.root, path));
-			if (content !== null) snapshot.set(path, content);
-		}
-	});
-	return snapshot;
-}
-/**
-* How each path differs between two runs. Paths with identical content are left out.
-*/
-function diffSnapshots(previous, current) {
-	const changes = {};
-	for (const [path, content] of current) {
-		if (!previous.has(path)) {
-			changes[path] = "added";
-			continue;
-		}
-		if (previous.get(path) !== content) changes[path] = "changed";
-	}
-	for (const path of previous.keys()) if (!current.has(path)) changes[path] = "removed";
-	return changes;
-}
+const DISK_SNAPSHOT_MAX_FILES = 1e4;
 var GenerationRunTarget = class extends RpcTarget {
 	generationStream;
 	generationResult;
@@ -2191,7 +2286,8 @@ var StudioSession = class {
 	#isGenerating = false;
 	#heartbeatTimer;
 	#lastGeneration;
-	#previousGeneration;
+	#store;
+	#limits = resolveGenerationLimits();
 	/**
 	* Resolves when Studio calls {@link StudioSession.connect}. `studio:ready` waits on this so the
 	* host does not queue jobs before the agent session is registered.
@@ -2207,6 +2303,18 @@ var StudioSession = class {
 	get #isSandbox() {
 		return this.#session?.isSandbox === true;
 	}
+	/**
+	* Kept in the project's cache directory, so it survives a restart, except on a sandbox: its pool
+	* sessions run every tenant's jobs, so it keeps them in memory. Looked up by job id only.
+	*/
+	get #generations() {
+		this.#store ??= createGenerationStore({
+			storage: this.#isSandbox ? memoryStorage() : cacheStorage({ root: this.#options.root }),
+			maxCount: this.#limits.maxCount,
+			maxMb: this.#limits.maxMb
+		});
+		return this.#store;
+	}
 	get #canWrite() {
 		return !this.#isSandbox && this.#options.permissions.allowWrite;
 	}
@@ -2416,8 +2524,18 @@ var StudioSession = class {
 				await this.#warn(`Ignored the spec from Studio; set ${remedy} to generate from it`);
 			}
 			const resolvedPlugins = plugins ?? config.plugins;
-			if (this.#lastGeneration) this.#previousGeneration = await readSnapshot(this.#lastGeneration);
 			this.#lastGeneration = void 0;
+			const diskFiles = this.#hasProjectOnDisk ? await listDisk({
+				root,
+				outputPath: config.output.path,
+				maxFiles: DISK_SNAPSHOT_MAX_FILES
+			}) : void 0;
+			const disk = diskFiles ? await this.#generations.keep({
+				jobId: data.jobId,
+				source: "disk",
+				files: diskFiles,
+				maxSetMb: this.#limits.maxSnapshotMb
+			}) : void 0;
 			const detach = [setupHookListener(this.#hooks, root, controller.signal)];
 			try {
 				await generate({
@@ -2438,6 +2556,9 @@ var StudioSession = class {
 					hooks: this.#hooks,
 					signal: controller.signal
 				});
+			} catch (error) {
+				await this.#generations.drop(data.jobId);
+				throw error;
 			} finally {
 				for (const remove of detach) remove();
 			}
@@ -2446,18 +2567,30 @@ var StudioSession = class {
 				info: `${resolvedPlugins.length} plugin${resolvedPlugins.length === 1 ? "" : "s"}, ${this.#canWrite ? "written to disk" : "in memory"}${inputOverride !== void 0 ? ", from a Studio spec" : ""}`
 			});
 			const generation = this.#lastGeneration;
-			const files = [...generation?.paths ?? []];
-			const previous = this.#previousGeneration;
-			if (!generation || !previous) return {
-				status: "success",
-				files,
-				fileCount: files.length
-			};
+			const output = generation ? await this.#generations.keep({
+				jobId: data.jobId,
+				source: "output",
+				files: generation.output,
+				maxSetMb: this.#limits.maxMb
+			}) : void 0;
+			if (generation && output) {
+				if (!output.paths.length && Object.keys(output.hashes).length) await this.#warn("Kept only the hashes of this generation: its output is too large to keep");
+				const { peerDependencies, missingDependencies } = generation;
+				await this.#generations.add({
+					jobId: data.jobId,
+					output,
+					disk,
+					peerDependencies,
+					missingDependencies
+				});
+			}
+			const files = [...generation?.output.paths ?? []];
 			return {
 				status: "success",
 				files,
 				fileCount: files.length,
-				changes: diffSnapshots(previous, await readSnapshot(generation))
+				hashes: output?.hashes ?? {},
+				disk: disk ? { hashes: disk.hashes } : void 0
 			};
 		} finally {
 			this.#isGenerating = false;
@@ -2512,20 +2645,16 @@ var StudioSession = class {
 		if (this.#isSandbox) return this.#refuse("Ignored snapshot: a sandbox agent has no project to build a package from", "A sandbox agent has no project to build a package from");
 		const { name, version, bundledDependencies, uploadPath } = data;
 		if (!name || !version || !uploadPath) return this.#refuse("Ignored snapshot: the message was missing required fields", "The request was missing required fields");
-		const generation = this.#lastGeneration;
+		const generation = await this.#generations.latest();
 		if (!generation) return this.#refuse("Ignored snapshot: no prior generation to pack", "No prior generation exists to pack, run a generation first");
 		const bundled = new Set(bundledDependencies ?? []);
 		const missing = generation.missingDependencies.filter((dependency) => !bundled.has(dependency));
 		if (missing.length) return this.#refuse(`Ignored snapshot: missing dependencies: ${missing.join(", ")}`, `Missing dependencies: ${missing.join(", ")}`);
 		try {
-			const files = {};
-			await inParallel({
-				items: [...generation.paths],
-				limit: FILE_READ_CONCURRENCY,
-				run: async (relativePath) => {
-					const content = await generation.storage.readItem(absoluteStoragePath(generation.root, relativePath));
-					if (content !== null) files[relativePath] = content;
-				}
+			const files = await this.#generations.read({
+				generation,
+				source: "output",
+				paths: generation.output.paths
 			});
 			const { bytes, integrity } = await createSnapshotPackage(files, {
 				name,
@@ -2563,6 +2692,13 @@ var StudioSession = class {
 			throw error;
 		}
 	}
+	/**
+	* An agent with a project on disk can show a run against what its output directory held before.
+	* A sandbox agent has no project.
+	*/
+	get #hasProjectOnDisk() {
+		return !this.#isSandbox && this.#canRead;
+	}
 	async readFiles(data) {
 		const command = "readFiles";
 		await this.#hooks.callHook("studio:command:start", { command });
@@ -2575,27 +2711,15 @@ var StudioSession = class {
 		if (!Array.isArray(data.paths)) return this.#refuse("Ignored files: the message carried no paths", "The request carried no paths");
 		const { paths } = data;
 		if (paths.length > 50) return this.#refuse(`Ignored files: requested ${paths.length} paths, more than the 50 allowed per request`, `At most 50 paths may be requested at once`);
-		if (data.revision === "previous") {
-			const previous = this.#previousGeneration;
-			if (!previous) return this.#refuse("Ignored files: no previous generation to read from", "No previous generation to compare against");
-			const files = Object.fromEntries(paths.filter((path) => previous.has(path)).map((path) => [path, previous.get(path)]));
-			await this.#hooks.callHook("studio:command:end", {
-				command,
-				info: `read ${Object.keys(files).length}/${paths.length} requested file${paths.length === 1 ? "" : "s"} from the previous run`
-			});
-			return { files };
-		}
-		const generation = this.#lastGeneration;
-		if (!generation) return this.#refuse("Ignored files: no prior generation to read from", "No prior generation to read from, run a generation first");
-		const requested = paths.filter((path) => generation.paths.has(path));
-		const files = {};
-		await inParallel({
-			items: requested,
-			limit: FILE_READ_CONCURRENCY,
-			run: async (path) => {
-				const content = await generation.storage.readItem(absoluteStoragePath(generation.root, path));
-				if (content !== null) files[path] = content;
-			}
+		if (typeof data.jobId !== "string" || !data.jobId) return this.#refuse("Ignored files: the message named no job", "The request named no generation job");
+		const generation = await this.#generations.get(data.jobId);
+		if (!generation) return this.#refuse(`Ignored files: job ${data.jobId} is not kept on this agent`, GENERATION_GONE_MESSAGE);
+		const source = data.source === "disk" ? "disk" : "output";
+		if (!generation[source]) return this.#refuse("Ignored files: that job has no snapshot of the files on disk", "This agent kept no snapshot of the files on disk for that job");
+		const files = await this.#generations.read({
+			generation,
+			source,
+			paths
 		});
 		await this.#hooks.callHook("studio:command:end", {
 			command,