@mastra/daytona 0.9.0-alpha.1 → 0.9.0-alpha.2

package/CHANGELOG.md

@@ -1,5 +1,26 @@
 # @mastra/daytona
 
+## 0.9.0-alpha.2
+
+### Minor Changes
+
+- Added computer-use support to `DaytonaSandbox`. Workspaces backed by Daytona can now take screenshots, control the mouse and keyboard, inspect the display, and open a noVNC viewer through the standard computer tools. ([#21701](https://github.com/mastra-ai/mastra/pull/21701))
+
+  ```typescript
+  const sandbox = new DaytonaSandbox();
+  await sandbox.start();
+
+  await sandbox.computer.leftClick(100, 200);
+  const screenshot = await sandbox.computer.screenshot();
+  ```
+
+  Desktop services start lazily on the first computer operation. Set `computerUse: false` to disable the capability or `computerUse: { autoStart: false }` to manage those services directly.
+
+### Patch Changes
+
+- Updated dependencies [[`48ef1f1`](https://github.com/mastra-ai/mastra/commit/48ef1f1d24eedafbb07f64e659a81b52b67b8bf6), [`63796ba`](https://github.com/mastra-ai/mastra/commit/63796ba0fda60253be17535e68f6bbbf1e6ffa09), [`3c19dce`](https://github.com/mastra-ai/mastra/commit/3c19dcef8e73062a80627a4927eae3ec11145afd)]:
+  - @mastra/core@1.62.0-alpha.12
+
 ## 0.9.0-alpha.1
 
 ### Minor Changes

package/dist/index.cjs

@@ -618,6 +618,8 @@ function validateMountPath(mountPath) {
 }
 /** Allowlist for marker filenames from ls output — e.g. "mount-abc123" */
 const SAFE_MARKER_NAME = /^mount-[a-z0-9]+$/;
+/** Default port of the noVNC web viewer started by Daytona computer use. */
+const DEFAULT_NOVNC_PORT = 6080;
 /** Patterns indicating the sandbox is dead/gone (@daytonaio/sdk@0.143.0). */
 const SANDBOX_DEAD_PATTERNS = [
 	/sandbox is not running/i,
@@ -690,6 +692,9 @@ var DaytonaSandbox = class DaytonaSandbox extends _mastra_core_workspace.MastraS
 	_createdAt = null;
 	_workingDir = null;
 	_isRetrying = false;
+	_computerUseStarted = null;
+	computerUseAutoStart;
+	noVncPort;
 	timeout;
 	language;
 	resources;
@@ -742,6 +747,10 @@ var DaytonaSandbox = class DaytonaSandbox extends _mastra_core_workspace.MastraS
 			...options.target !== void 0 && { target: options.target }
 		};
 		this._constructorOptions = { ...options };
+		const computerUseOption = options.computerUse ?? true;
+		this.computerUseAutoStart = typeof computerUseOption === "object" ? computerUseOption.autoStart ?? true : true;
+		this.noVncPort = typeof computerUseOption === "object" ? computerUseOption.noVncPort ?? DEFAULT_NOVNC_PORT : DEFAULT_NOVNC_PORT;
+		if (computerUseOption !== false) this.computer = this.createComputer();
 	}
 	generateId() {
 		return `daytona-sandbox-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
@@ -876,6 +885,7 @@ var DaytonaSandbox = class DaytonaSandbox extends _mastra_core_workspace.MastraS
 			await this._daytona.stop(this._sandbox);
 		} catch {}
 		this._sandbox = null;
+		this._computerUseStarted = null;
 	}
 	/**
 	* Destroy the Daytona sandbox and clean up all resources.
@@ -892,6 +902,7 @@ var DaytonaSandbox = class DaytonaSandbox extends _mastra_core_workspace.MastraS
 		this._sandbox = null;
 		this._daytonaSandboxId = void 0;
 		this._daytona = null;
+		this._computerUseStarted = null;
 		this.mounts?.clear();
 	}
 	/**
@@ -955,6 +966,96 @@ var DaytonaSandbox = class DaytonaSandbox extends _mastra_core_workspace.MastraS
 		})));
 	}
 	/**
+	* Ensure the Daytona computer use processes (Xvfb, xfce4, x11vnc, noVNC)
+	* are running. Memoized per attached sandbox; the memo is reset when the
+	* sandbox stops, dies, or is destroyed so a fresh sandbox restarts them.
+	*/
+	async ensureComputerUseStarted() {
+		if (!this.computerUseAutoStart) return;
+		if (!this._computerUseStarted) this._computerUseStarted = this.daytona.computerUse.start().then(() => void 0).catch((error) => {
+			this._computerUseStarted = null;
+			throw error;
+		});
+		return this._computerUseStarted;
+	}
+	/**
+	* Build the {@link SandboxComputer} capability backed by Daytona's
+	* computer use API (`sandbox.computerUse`).
+	*
+	* Every operation ensures the sandbox is running and the desktop processes
+	* are started, and retries once if the sandbox died (mirroring command
+	* execution behavior).
+	*/
+	createComputer() {
+		const run = async (fn) => {
+			await this.ensureRunning();
+			return this.retryOnDead(async () => {
+				await this.ensureComputerUseStarted();
+				return fn(this.daytona.computerUse);
+			});
+		};
+		return {
+			screenshot: async () => {
+				const response = await run((computerUse) => computerUse.screenshot.takeFullScreen());
+				if (!response.screenshot) throw new Error(`${LOG_PREFIX} Daytona returned an empty screenshot response`);
+				return {
+					data: new Uint8Array(Buffer.from(response.screenshot, "base64")),
+					mediaType: "image/png"
+				};
+			},
+			leftClick: async (x, y) => {
+				await run((computerUse) => computerUse.mouse.click(x, y, "left"));
+			},
+			rightClick: async (x, y) => {
+				await run((computerUse) => computerUse.mouse.click(x, y, "right"));
+			},
+			doubleClick: async (x, y) => {
+				await run((computerUse) => computerUse.mouse.click(x, y, "left", true));
+			},
+			moveMouse: async (x, y) => {
+				await run((computerUse) => computerUse.mouse.move(x, y));
+			},
+			drag: async (from, to) => {
+				await run((computerUse) => computerUse.mouse.drag(from.x, from.y, to.x, to.y));
+			},
+			scroll: async (direction, amount) => {
+				await run(async (computerUse) => {
+					const position = await computerUse.mouse.getPosition();
+					return computerUse.mouse.scroll(position.x ?? 0, position.y ?? 0, direction, amount);
+				});
+			},
+			type: async (text) => {
+				await run((computerUse) => computerUse.keyboard.type(text));
+			},
+			press: async (key) => {
+				await run((computerUse) => Array.isArray(key) ? computerUse.keyboard.hotkey(key.join("+")) : computerUse.keyboard.press(key));
+			},
+			getScreenSize: async () => {
+				const info = await run((computerUse) => computerUse.display.getInfo());
+				const display = info.displays?.find((d) => d.isActive) ?? info.displays?.[0];
+				if (!display || display.width === void 0 || display.height === void 0) throw new Error(`${LOG_PREFIX} Daytona did not return display information`);
+				return {
+					width: display.width,
+					height: display.height
+				};
+			},
+			getCursorPosition: async () => {
+				const position = await run((computerUse) => computerUse.mouse.getPosition());
+				return {
+					x: position.x ?? 0,
+					y: position.y ?? 0
+				};
+			},
+			streamUrl: async () => {
+				try {
+					return (await run(() => this.daytona.getPreviewLink(this.noVncPort)))?.url ?? null;
+				} catch {
+					return null;
+				}
+			}
+		};
+	}
+	/**
 	* Mount a filesystem at a path in the sandbox.
 	* Uses FUSE tools (s3fs, gcsfuse) to mount cloud storage.
 	*/
@@ -1389,6 +1490,7 @@ var DaytonaSandbox = class DaytonaSandbox extends _mastra_core_workspace.MastraS
 	*/
 	handleSandboxTimeout() {
 		this._sandbox = null;
+		this._computerUseStarted = null;
 		if (this.mounts) {
 			for (const [path, entry] of this.mounts.entries) if (entry.state === "mounted" || entry.state === "mounting") this.mounts.set(path, { state: "pending" });
 		}