@agimon-ai/po-computer-use 0.0.0-alpha.0

package/dist/index.mjs

@@ -0,0 +1,129 @@
+import { o as COMPUTER_USE_GUIDANCE, r as ComputerScriptRunner, s as COMPUTER_USE_TOOL_NAMES, t as DefaultComputerUseExtensionService } from "./extensionService-DXvF7esO.mjs";
+import path from "node:path";
+import { loadDoomConfig } from "@agimon-ai/po-config";
+//#region src/services/sessionApiClient/index.ts
+/** Uses the injected host-owned service. Absence never triggers transport or credential discovery. */
+function createComputerUseSessionClient(host) {
+	return host;
+}
+//#endregion
+//#region src/services/dependencies/index.ts
+const COMPUTER_USE_SCRIPT_PATHS_ENV = "PO_COMPUTER_USE_SCRIPT_PATHS";
+function createComputerUseDependencies(overrides = {}) {
+	const client = overrides.client ?? createComputerUseSessionClient();
+	const allowedScriptPaths = (process.env["PO_COMPUTER_USE_SCRIPT_PATHS"] ?? "").split(path.delimiter).filter((entry) => entry.length > 0);
+	return {
+		service: overrides.service ?? new DefaultComputerUseExtensionService(client),
+		enabled: overrides.enabled ?? ((cwd) => loadDoomConfig(cwd ?? process.env.PI_PROJECT_ROOT ?? process.cwd()).computerUse?.enabled === true),
+		...client === void 0 ? {} : { client },
+		...overrides.scriptRunner !== void 0 ? { scriptRunner: overrides.scriptRunner } : client === void 0 ? {} : { scriptRunner: new ComputerScriptRunner({
+			client,
+			allowedScriptPaths
+		}) }
+	};
+}
+//#endregion
+//#region src/models/computerUseState.ts
+function requireRequest(state, requestId) {
+	if (state.requestId !== requestId) throw new Error("The computer-use activation request is stale.");
+}
+function next(state, patch) {
+	return {
+		revision: state.revision + 1,
+		identity: state.identity,
+		...patch
+	};
+}
+function initialComputerUseState(identity) {
+	return {
+		revision: 0,
+		phase: "inactive",
+		identity
+	};
+}
+function reduceComputerUseState(state, event) {
+	if (event.type === "fail") return next(state, {
+		phase: "failed",
+		failure: {
+			code: event.code,
+			message: event.message
+		},
+		...state.requestId === void 0 ? {} : { requestId: state.requestId },
+		...state.target === void 0 ? {} : { target: state.target },
+		...state.grant === void 0 ? {} : { grant: state.grant }
+	});
+	if (event.type === "reset") {
+		if (state.phase !== "failed") throw new Error("Only a failed computer-use state can be reset.");
+		return next(state, { phase: "inactive" });
+	}
+	switch (state.phase) {
+		case "inactive":
+			if (event.type !== "request") throw new Error("Computer use is inactive.");
+			return next(state, {
+				phase: "requesting",
+				requestId: event.requestId,
+				target: event.target
+			});
+		case "requesting":
+			if (event.type !== "await_confirmation") throw new Error("Computer use is preparing an activation request.");
+			requireRequest(state, event.requestId);
+			return next(state, {
+				phase: "awaiting_confirmation",
+				requestId: event.requestId,
+				target: state.target
+			});
+		case "awaiting_confirmation":
+			if (event.type === "deny") {
+				requireRequest(state, event.requestId);
+				return next(state, { phase: "inactive" });
+			}
+			if (event.type !== "confirm") throw new Error("Computer use is waiting for confirmation.");
+			requireRequest(state, event.requestId);
+			return next(state, {
+				phase: "activating",
+				requestId: event.requestId,
+				target: state.target
+			});
+		case "activating":
+			if (event.type !== "activated") throw new Error("Computer use is activating.");
+			requireRequest(state, event.requestId);
+			return next(state, {
+				phase: "active",
+				requestId: event.requestId,
+				target: state.target,
+				grant: event.grant
+			});
+		case "active":
+			if (event.type !== "stop") throw new Error("Computer use is active.");
+			return next(state, {
+				phase: "stopping",
+				requestId: state.requestId,
+				target: state.target,
+				grant: state.grant
+			});
+		case "stopping":
+			if (event.type !== "stopped") throw new Error("Computer use is stopping.");
+			return next(state, { phase: "inactive" });
+		case "failed": throw new Error("Computer use must be reset before another transition.");
+	}
+}
+//#endregion
+//#region src/services/traceRedaction/index.ts
+function redactComputerUseTrace(input) {
+	return {
+		sequence: input.sequence,
+		timestamp: input.timestamp,
+		runId: input.runId,
+		bundleId: input.bundleId,
+		actionKind: input.action.kind,
+		...input.role === void 0 ? {} : { role: input.role },
+		...input.action.kind === "set_value" ? { textLength: input.action.value.length } : {},
+		durationMs: input.durationMs,
+		outcome: input.outcome,
+		...input.errorCode === void 0 ? {} : { errorCode: input.errorCode }
+	};
+}
+//#endregion
+export { COMPUTER_USE_GUIDANCE, COMPUTER_USE_SCRIPT_PATHS_ENV, COMPUTER_USE_TOOL_NAMES, ComputerScriptRunner, DefaultComputerUseExtensionService, createComputerUseDependencies, createComputerUseSessionClient, initialComputerUseState, redactComputerUseTrace, reduceComputerUseState };
+
+//# sourceMappingURL=index.mjs.map

package/dist/index.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.mjs","names":[],"sources":["../src/services/sessionApiClient/index.ts","../src/services/dependencies/index.ts","../src/models/computerUseState.ts","../src/services/traceRedaction/index.ts"],"sourcesContent":["import type { ComputerUseSessionClient } from '../../types/computerUse';\n\nexport type { ComputerUseSessionClient } from '../../types/computerUse';\n\n/** Uses the injected host-owned service. Absence never triggers transport or credential discovery. */\nexport function createComputerUseSessionClient(host?: ComputerUseSessionClient): ComputerUseSessionClient | undefined {\n  return host;\n}\n","import path from 'node:path';\n\nimport { loadDoomConfig } from '@agimon-ai/po-config';\n\nimport type { ComputerUseExtensionDependencies } from '../../types/extension';\nimport { ComputerScriptRunner } from '../computerScriptRunner';\nimport { DefaultComputerUseExtensionService } from '../extensionService';\nimport { createComputerUseSessionClient } from '../sessionApiClient';\n\nexport const COMPUTER_USE_SCRIPT_PATHS_ENV = 'PO_COMPUTER_USE_SCRIPT_PATHS';\nexport function createComputerUseDependencies(\n  overrides: Partial<ComputerUseExtensionDependencies> = {},\n): ComputerUseExtensionDependencies {\n  const client = overrides.client ?? createComputerUseSessionClient();\n  const allowedScriptPaths = (process.env[COMPUTER_USE_SCRIPT_PATHS_ENV] ?? '')\n    .split(path.delimiter)\n    .filter((entry) => entry.length > 0);\n  return {\n    service: overrides.service ?? new DefaultComputerUseExtensionService(client),\n    enabled:\n      overrides.enabled ??\n      ((cwd) => loadDoomConfig(cwd ?? process.env.PI_PROJECT_ROOT ?? process.cwd()).computerUse?.enabled === true),\n    ...(client === undefined ? {} : { client }),\n    ...(overrides.scriptRunner !== undefined\n      ? { scriptRunner: overrides.scriptRunner }\n      : client === undefined\n        ? {}\n        : { scriptRunner: new ComputerScriptRunner({ client, allowedScriptPaths }) }),\n  };\n}\n","import type { ComputerUseState, ComputerUseStateEvent } from '../types/computerUse';\n\nfunction requireRequest(state: ComputerUseState, requestId: string): void {\n  if (state.requestId !== requestId) throw new Error('The computer-use activation request is stale.');\n}\n\nfunction next(state: ComputerUseState, patch: Omit<ComputerUseState, 'revision' | 'identity'>): ComputerUseState {\n  return {\n    revision: state.revision + 1,\n    identity: state.identity,\n    ...patch,\n  };\n}\n\nexport function initialComputerUseState(identity: ComputerUseState['identity']): ComputerUseState {\n  return { revision: 0, phase: 'inactive', identity };\n}\n\nexport function reduceComputerUseState(state: ComputerUseState, event: ComputerUseStateEvent): ComputerUseState {\n  if (event.type === 'fail') {\n    return next(state, {\n      phase: 'failed',\n      failure: { code: event.code, message: event.message },\n      ...(state.requestId === undefined ? {} : { requestId: state.requestId }),\n      ...(state.target === undefined ? {} : { target: state.target }),\n      ...(state.grant === undefined ? {} : { grant: state.grant }),\n    });\n  }\n\n  if (event.type === 'reset') {\n    if (state.phase !== 'failed') throw new Error('Only a failed computer-use state can be reset.');\n    return next(state, { phase: 'inactive' });\n  }\n\n  switch (state.phase) {\n    case 'inactive': {\n      if (event.type !== 'request') throw new Error('Computer use is inactive.');\n      return next(state, {\n        phase: 'requesting',\n        requestId: event.requestId,\n        target: event.target,\n      });\n    }\n    case 'requesting': {\n      if (event.type !== 'await_confirmation') throw new Error('Computer use is preparing an activation request.');\n      requireRequest(state, event.requestId);\n      return next(state, {\n        phase: 'awaiting_confirmation',\n        requestId: event.requestId,\n        target: state.target,\n      });\n    }\n    case 'awaiting_confirmation': {\n      if (event.type === 'deny') {\n        requireRequest(state, event.requestId);\n        return next(state, { phase: 'inactive' });\n      }\n      if (event.type !== 'confirm') throw new Error('Computer use is waiting for confirmation.');\n      requireRequest(state, event.requestId);\n      return next(state, {\n        phase: 'activating',\n        requestId: event.requestId,\n        target: state.target,\n      });\n    }\n    case 'activating': {\n      if (event.type !== 'activated') throw new Error('Computer use is activating.');\n      requireRequest(state, event.requestId);\n      return next(state, {\n        phase: 'active',\n        requestId: event.requestId,\n        target: state.target,\n        grant: event.grant,\n      });\n    }\n    case 'active': {\n      if (event.type !== 'stop') throw new Error('Computer use is active.');\n      return next(state, {\n        phase: 'stopping',\n        requestId: state.requestId,\n        target: state.target,\n        grant: state.grant,\n      });\n    }\n    case 'stopping': {\n      if (event.type !== 'stopped') throw new Error('Computer use is stopping.');\n      return next(state, { phase: 'inactive' });\n    }\n    case 'failed':\n      throw new Error('Computer use must be reset before another transition.');\n  }\n}\n","import type { ComputerUseAction, ComputerUseFailureCode } from '../../types/computerUse';\n\nexport interface ComputerUseTraceInput {\n  readonly sequence: number;\n  readonly timestamp: number;\n  readonly runId: string;\n  readonly bundleId: string;\n  readonly action: ComputerUseAction;\n  readonly role?: string;\n  readonly durationMs: number;\n  readonly outcome: 'succeeded' | 'failed' | 'uncertain';\n  readonly errorCode?: ComputerUseFailureCode;\n}\n\nexport interface ComputerUseTraceRecord {\n  readonly sequence: number;\n  readonly timestamp: number;\n  readonly runId: string;\n  readonly bundleId: string;\n  readonly actionKind: ComputerUseAction['kind'];\n  readonly role?: string;\n  readonly textLength?: number;\n  readonly durationMs: number;\n  readonly outcome: ComputerUseTraceInput['outcome'];\n  readonly errorCode?: ComputerUseFailureCode;\n}\n\nexport function redactComputerUseTrace(input: ComputerUseTraceInput): ComputerUseTraceRecord {\n  return {\n    sequence: input.sequence,\n    timestamp: input.timestamp,\n    runId: input.runId,\n    bundleId: input.bundleId,\n    actionKind: input.action.kind,\n    ...(input.role === undefined ? {} : { role: input.role }),\n    ...(input.action.kind === 'set_value' ? { textLength: input.action.value.length } : {}),\n    durationMs: input.durationMs,\n    outcome: input.outcome,\n    ...(input.errorCode === undefined ? {} : { errorCode: input.errorCode }),\n  };\n}\n"],"mappings":";;;;;AAKA,SAAgB,+BAA+B,MAAuE;CACpH,OAAO;AACT;;;ACEA,MAAa,gCAAgC;AAC7C,SAAgB,8BACd,YAAuD,CAAC,GACtB;CAClC,MAAM,SAAS,UAAU,UAAU,+BAA+B;CAClE,MAAM,sBAAsB,QAAQ,IAAA,mCAAsC,GAAA,CACvE,MAAM,KAAK,SAAS,CAAC,CACrB,QAAQ,UAAU,MAAM,SAAS,CAAC;CACrC,OAAO;EACL,SAAS,UAAU,WAAW,IAAI,mCAAmC,MAAM;EAC3E,SACE,UAAU,aACR,QAAQ,eAAe,OAAO,QAAQ,IAAI,mBAAmB,QAAQ,IAAI,CAAC,CAAC,CAAC,aAAa,YAAY;EACzG,GAAI,WAAW,KAAA,IAAY,CAAC,IAAI,EAAE,OAAO;EACzC,GAAI,UAAU,iBAAiB,KAAA,IAC3B,EAAE,cAAc,UAAU,aAAa,IACvC,WAAW,KAAA,IACT,CAAC,IACD,EAAE,cAAc,IAAI,qBAAqB;GAAE;GAAQ;EAAmB,CAAC,EAAE;CACjF;AACF;;;AC3BA,SAAS,eAAe,OAAyB,WAAyB;CACxE,IAAI,MAAM,cAAc,WAAW,MAAM,IAAI,MAAM,+CAA+C;AACpG;AAEA,SAAS,KAAK,OAAyB,OAA0E;CAC/G,OAAO;EACL,UAAU,MAAM,WAAW;EAC3B,UAAU,MAAM;EAChB,GAAG;CACL;AACF;AAEA,SAAgB,wBAAwB,UAA0D;CAChG,OAAO;EAAE,UAAU;EAAG,OAAO;EAAY;CAAS;AACpD;AAEA,SAAgB,uBAAuB,OAAyB,OAAgD;CAC9G,IAAI,MAAM,SAAS,QACjB,OAAO,KAAK,OAAO;EACjB,OAAO;EACP,SAAS;GAAE,MAAM,MAAM;GAAM,SAAS,MAAM;EAAQ;EACpD,GAAI,MAAM,cAAc,KAAA,IAAY,CAAC,IAAI,EAAE,WAAW,MAAM,UAAU;EACtE,GAAI,MAAM,WAAW,KAAA,IAAY,CAAC,IAAI,EAAE,QAAQ,MAAM,OAAO;EAC7D,GAAI,MAAM,UAAU,KAAA,IAAY,CAAC,IAAI,EAAE,OAAO,MAAM,MAAM;CAC5D,CAAC;CAGH,IAAI,MAAM,SAAS,SAAS;EAC1B,IAAI,MAAM,UAAU,UAAU,MAAM,IAAI,MAAM,gDAAgD;EAC9F,OAAO,KAAK,OAAO,EAAE,OAAO,WAAW,CAAC;CAC1C;CAEA,QAAQ,MAAM,OAAd;EACE,KAAK;GACH,IAAI,MAAM,SAAS,WAAW,MAAM,IAAI,MAAM,2BAA2B;GACzE,OAAO,KAAK,OAAO;IACjB,OAAO;IACP,WAAW,MAAM;IACjB,QAAQ,MAAM;GAChB,CAAC;EAEH,KAAK;GACH,IAAI,MAAM,SAAS,sBAAsB,MAAM,IAAI,MAAM,kDAAkD;GAC3G,eAAe,OAAO,MAAM,SAAS;GACrC,OAAO,KAAK,OAAO;IACjB,OAAO;IACP,WAAW,MAAM;IACjB,QAAQ,MAAM;GAChB,CAAC;EAEH,KAAK;GACH,IAAI,MAAM,SAAS,QAAQ;IACzB,eAAe,OAAO,MAAM,SAAS;IACrC,OAAO,KAAK,OAAO,EAAE,OAAO,WAAW,CAAC;GAC1C;GACA,IAAI,MAAM,SAAS,WAAW,MAAM,IAAI,MAAM,2CAA2C;GACzF,eAAe,OAAO,MAAM,SAAS;GACrC,OAAO,KAAK,OAAO;IACjB,OAAO;IACP,WAAW,MAAM;IACjB,QAAQ,MAAM;GAChB,CAAC;EAEH,KAAK;GACH,IAAI,MAAM,SAAS,aAAa,MAAM,IAAI,MAAM,6BAA6B;GAC7E,eAAe,OAAO,MAAM,SAAS;GACrC,OAAO,KAAK,OAAO;IACjB,OAAO;IACP,WAAW,MAAM;IACjB,QAAQ,MAAM;IACd,OAAO,MAAM;GACf,CAAC;EAEH,KAAK;GACH,IAAI,MAAM,SAAS,QAAQ,MAAM,IAAI,MAAM,yBAAyB;GACpE,OAAO,KAAK,OAAO;IACjB,OAAO;IACP,WAAW,MAAM;IACjB,QAAQ,MAAM;IACd,OAAO,MAAM;GACf,CAAC;EAEH,KAAK;GACH,IAAI,MAAM,SAAS,WAAW,MAAM,IAAI,MAAM,2BAA2B;GACzE,OAAO,KAAK,OAAO,EAAE,OAAO,WAAW,CAAC;EAE1C,KAAK,UACH,MAAM,IAAI,MAAM,uDAAuD;CAC3E;AACF;;;AChEA,SAAgB,uBAAuB,OAAsD;CAC3F,OAAO;EACL,UAAU,MAAM;EAChB,WAAW,MAAM;EACjB,OAAO,MAAM;EACb,UAAU,MAAM;EAChB,YAAY,MAAM,OAAO;EACzB,GAAI,MAAM,SAAS,KAAA,IAAY,CAAC,IAAI,EAAE,MAAM,MAAM,KAAK;EACvD,GAAI,MAAM,OAAO,SAAS,cAAc,EAAE,YAAY,MAAM,OAAO,MAAM,OAAO,IAAI,CAAC;EACrF,YAAY,MAAM;EAClB,SAAS,MAAM;EACf,GAAI,MAAM,cAAc,KAAA,IAAY,CAAC,IAAI,EAAE,WAAW,MAAM,UAAU;CACxE;AACF"}

package/docs/computer-scripts.md

@@ -0,0 +1,72 @@
+# Author computer functions
+
+`computer_exec` runs an ordinary `run` function against the application authorized by the current Desktop grant. Write the function and any helpers using the existing file tools, then call `computer_exec` with `scriptPath` and JSON `input`. A function does not need its own extension package or registered agent tool.
+
+## Restricted execution, the default
+
+The Desktop-backed server admits scripts below the session working directory. Entry files and relative helpers may use `.ts`, `.js`, or `.mjs`. TypeScript must use erasable syntax. The loader resolves real paths and rejects helpers that escape this directory, including through symlinks. Inline source, package imports, and `node:` imports are not supported in restricted execution.
+
+Generated code runs inside a separate QuickJS WebAssembly runtime in a child process. Only the program callbacks, JSON input, bounded logger, and portable cancellation signal are exposed. Node globals, environment variables, filesystem access, networking, subprocesses, and host constructors are not exposed to the function. Relative helpers execute with the same restrictions. This is not permission to control another application: every program operation still passes through the current session's Desktop grant checks.
+
+## Function contract
+
+```ts
+import type { ComputerScriptProgram, ComputerScriptRun } from '@agimon-ai/po-computer-use';
+
+async function pressByLabel(program: ComputerScriptProgram, label: string) {
+  const snapshot = await program.observe();
+  const matches = snapshot.elements.filter(
+    (element) => element.label === label && element.enabled && !element.secure && element.actions.includes('press'),
+  );
+  if (matches.length !== 1) throw new Error(`Expected one pressable element named ${label}.`);
+  await program.act({
+    kind: 'press',
+    snapshotId: snapshot.snapshotId,
+    elementRef: matches[0]!.ref,
+  });
+}
+
+export const run: ComputerScriptRun = async ({ context: { program }, input, logger, signal }) => {
+  if (typeof input !== 'object' || input === null || !('label' in input) || typeof input.label !== 'string') {
+    throw new Error('input.label must be a string.');
+  }
+  signal.throwIfAborted();
+  await pressByLabel(program, input.label);
+  logger.info('Pressed the requested element.');
+  return { pressed: input.label };
+};
+```
+
+Type-only imports are erased before execution. Runtime helpers can be moved into neighboring files and imported with explicit relative paths such as `./press.ts`.
+
+`program.observe()` returns current semantic state without a screenshot by default. `program.observe({ includeScreenshot: true })` additionally captures an image. `program.act(action)` supports `press`, `focus`, `set_value`, and `scroll` against current snapshot IDs and element refs. Calls are serialized even when the function starts several promises. Re-observe after changes; serialization does not make old snapshots valid.
+
+Use normal local variables and function arguments for working state within a chain. An observation is a snapshot, not a mutable application instance. Actual application changes must use `program.act`. Each invocation starts a fresh worker; module globals do not persist across invocations.
+
+The portable `signal` supports `aborted` and `throwIfAborted()`. Restricted functions should not assume the rest of the Node AbortSignal or timer APIs exists. An external timeout can terminate synchronous infinite loops as well as pending asynchronous work.
+
+## Tool result and limits
+
+By default, successful `computer_exec` returns the function's JSON result, bounded logs, a fresh semantic observation, and execution metrics. Pass `includeScreenshot: true` to the tool to include a final image. Images use image content blocks, not base64 embedded in model-facing observation text. Avoid returning image data inside the function's own result.
+
+The default execution deadline is 30 seconds. Input and text output are bounded to 256 KiB, with the final semantic observation counted against output. The final tool formatter also checks the complete text response. Images have a separate 6 MiB decoded-size ceiling. Restricted execution limits its JavaScript heap to 32 MiB, stack to 512 KiB, helper graph to 32 modules, individual source files to 256 KiB, total source to 512 KiB, and outstanding program operations to 64.
+
+On failure, `computer_exec` reports completed actions and whether an action's outcome is uncertain when that information is available. Cancellation prevents queued actions from starting and attempts to abort the current request. An interrupted request may already have changed the application, so do not automatically retry the entire chain or repeat the uncertain action. Re-establish authorized, fresh state before continuing.
+
+Metrics describe this invocation's action count, observation count, elapsed milliseconds, and counted text-output bytes. They are not a measurement of model tokens or native application reliability.
+
+## Explicitly trusted Node scripts
+
+Full Node execution remains available only with `trusted: true` and an entry path explicitly included in the host-configured `PO_COMPUTER_USE_SCRIPT_PATHS`. The value is a platform-delimited list of exact paths, not directories or globs:
+
+```bash
+export PO_COMPUTER_USE_SCRIPT_PATHS="$PWD/scripts/reviewed.ts"
+```
+
+Trusted scripts use normal file-based module loading, including relative imports. They are **not sandboxed** and inherit the Pi process environment and permissions. Review the entrypoint and every dependency before allowing them, and protect those files against unreviewed modification. The path allowlist is not a content signature and does not approve subsequent edits or dependency changes. Setting an allowlisted path alone does not switch execution out of restricted mode.
+
+## Desktop ownership
+
+Mode selection, global opt-in, and a native application grant are separate. A selected mode is not authority. Before granting control, the owning Desktop renderer must authorize the request, and the agent must be idle with no queued prompts. The native host then asks for confirmation of the application window and duration.
+
+Once Desktop claims a session for this workflow, browser and remote MCP clients cannot drive that session, including connections attached before the claim. The claim remains pinned for the server lifetime, even after stopping control or losing Desktop, so another client cannot take over a session that handled Desktop capabilities. Grants and claims are not restored from persisted session data.

package/llms.txt

@@ -0,0 +1,12 @@
+# @agimon-ai/po-computer-use
+
+Session-scoped semantic computer control through Po Desktop.
+
+## Guidance
+
+- [Use Computer Use](./src/prompts/po-use-computer-use/SKILL.md): Active-mode safety, observation, action, confirmation, and shutdown guidance.
+
+## Resources
+
+- [Package README](./README.md): Installation, activation workflow, tools, API boundaries, and development commands.
+- [Author computer scripts](./docs/computer-scripts.md): Trusted script authorization, runtime contract, limits, and example.

package/package.json

@@ -0,0 +1,147 @@
+{
+  "name": "@agimon-ai/po-computer-use",
+  "version": "0.0.0-alpha.0",
+  "description": "Session scoped semantic computer control through the Po Desktop capability.",
+  "keywords": [
+    "ai",
+    "coding-agent",
+    "computer-use",
+    "developer-tools",
+    "pi-coding-agent",
+    "pi-extension",
+    "pi-package",
+    "po",
+    "typescript"
+  ],
+  "homepage": "https://agimon.ai",
+  "bugs": {
+    "url": "https://github.com/AgiFlow/po/issues"
+  },
+  "license": "MIT",
+  "author": "Vuong Ngo",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/AgiFlow/po.git",
+    "directory": "packages/minor/po-computer-use"
+  },
+  "files": [
+    "dist",
+    "docs",
+    "src/types/computerUse.ts",
+    "src/types/computerUseApi.ts",
+    "src/prompts",
+    "llms.txt",
+    "README.md",
+    "LICENSE",
+    "package.json"
+  ],
+  "type": "module",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.mjs",
+  "types": "./dist/index.d.mts",
+  "jsnext:main": "./dist/index.mjs",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.mts",
+      "import": "./dist/index.mjs",
+      "require": "./dist/index.cjs"
+    },
+    "./extensions/server": {
+      "types": "./dist/extensions/server.d.mts",
+      "import": "./dist/extensions/server.mjs",
+      "require": "./dist/extensions/server.cjs"
+    },
+    "./extensions/web": {
+      "import": "./dist/extensions/web.mjs"
+    },
+    "./package.json": "./package.json"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "@agimon-ai/po-config": "0.0.0-alpha.0",
+    "@agimon-ai/po-core": "0.0.0-alpha.0",
+    "@agimon-ai/po-minor-mode": "0.0.0-alpha.0",
+    "@agimon-ai/po-web-components": "0.0.0-alpha.0",
+    "@agimon-ai/po-web-security": "0.0.0-alpha.0",
+    "@deepseek-ai/cordis": "4.0.4",
+    "quickjs-emscripten": "0.32.0",
+    "typebox": "1.3.34"
+  },
+  "devDependencies": {
+    "@agimon-ai/po-build": "0.0.0-alpha.0",
+    "@agimon-ai/vibe-lint-plugin-doom-extension": "0.0.0-alpha.0",
+    "@earendil-works/pi-coding-agent": "1.0.2",
+    "@tanstack/react-store": "0.11.2",
+    "@tanstack/store": "0.11.2",
+    "@types/node": "26.6.4",
+    "@types/react": "19.3.0",
+    "@vitest/coverage-v8": "5.0.3",
+    "react": "19.3.0",
+    "react-dom": "19.3.0",
+    "tsdown": "0.23.0",
+    "typescript": "7.0.2",
+    "vitest": "5.0.3"
+  },
+  "peerDependencies": {
+    "@earendil-works/pi-coding-agent": "1.0.2"
+  },
+  "peerDependenciesMeta": {
+    "@earendil-works/pi-coding-agent": {
+      "optional": true
+    }
+  },
+  "engines": {
+    "node": ">=24"
+  },
+  "poMcp": {
+    "entry": "./generated/mcp.ts",
+    "dist": "./dist/extensions/mcp.mjs",
+    "scopes": [
+      "session"
+    ],
+    "ui": {
+      "entry": "./generated/mcp-ui.ts",
+      "dist": "./dist/extensions/mcp-ui.mjs",
+      "widgets": [
+        "@agimon-ai/po-computer-use/computer_action",
+        "@agimon-ai/po-computer-use/computer_exec",
+        "@agimon-ai/po-computer-use/computer_state"
+      ]
+    }
+  },
+  "poServer": {
+    "entry": "./generated/server.ts",
+    "dist": "./dist/extensions/server.mjs",
+    "scopes": [
+      "global",
+      "workspace",
+      "session"
+    ],
+    "contracts": {
+      "entry": "./src/exports/apiContracts.ts",
+      "dist": "./dist/apiContracts.mjs"
+    }
+  },
+  "poWeb": {
+    "pluginId": "computer-use",
+    "channels": [
+      "computer_use"
+    ],
+    "client": "./dist/extensions/web.mjs",
+    "scopes": [
+      "global",
+      "workspace",
+      "session"
+    ]
+  },
+  "scripts": {
+    "build": "tsdown && tsdown --config tsdown.mcp.config.ts",
+    "build:mcp": "tsdown --config tsdown.mcp.config.ts",
+    "test": "vitest --run",
+    "typecheck": "tsc --noEmit && tsc --noEmit -p tsconfig.web.json && tsc --noEmit -p tsconfig.mcp.json",
+    "lint": "oxlint . && oxfmt . --check",
+    "fixcode": "oxlint . --fix && oxfmt ."
+  }
+}

package/src/prompts/po-use-computer-use/SKILL.md

@@ -0,0 +1,34 @@
+---
+name: po-use-computer-use
+description: 'Use Desktop-authorized semantic computer control and compose restricted application functions.'
+---
+
+# Use Computer Use
+
+This guidance applies only while this session holds an active application grant from its owning Po Desktop instance. A selected minor mode is not a grant. Normal browsers and remote MCP clients cannot obtain Desktop authority.
+
+## Observe and act
+
+Call `computer_state` before choosing an action. Use `includeScreenshot: false` for semantic-only inspection when an image is unnecessary. Use only the snapshot IDs and element refs returned by the current observation. `computer_action` supports `press`, `focus`, `set_value`, and `scroll`. Never infer coordinates, inspect another application, or bypass secure elements.
+
+Re-observe after actions that change the interface. Check the expected outcome rather than replaying stale refs or sleeping for a guessed interval. An observation is a snapshot: editing its fields does not mutate the application.
+
+## Compose reusable functions
+
+For predictable multi-step work, write a `.ts`, `.js`, or `.mjs` file below the session working directory using the existing file tools. Export `run({ context: { program }, input, logger, signal })`. Invoke it through `computer_exec` with `scriptPath` and JSON `input`. This batches local actions into one agent tool call; it does not require registering a new tool per function.
+
+The authorized `program` exposes `observe(options?)` and `act(action)`. Keep working state in ordinary variables and pass it to helper functions. Helpers can be imported with explicit relative paths within the working directory. Re-observe inside the chain whenever a previous action may have invalidated a snapshot.
+
+Generated functions run with restricted capabilities, not as Node programs. Do not import `node:` modules, installed packages, network APIs, or host constructors. Type-only imports are erased. Cancellation supports `signal.aborted` and `signal.throwIfAborted()`. Each invocation starts fresh; do not rely on globals persisting.
+
+`program.observe()` and the final `computer_exec` observation omit screenshots by default. Request `includeScreenshot: true` only where an image is needed. Return compact JSON, not screenshot base64 or entire repeated observations. Keep logs small. The default deadline is 30 seconds and bounded text output includes the final semantic observation.
+
+Full Node execution is a separate trusted path. Never request `trusted: true` to evade a restricted-runtime error. It requires an explicitly host-allowlisted entry and reviewed dependencies; those programs are not sandboxed.
+
+## Stop and recover
+
+Failures report completed actions and uncertain outcomes where available. Never automatically retry an uncertain action or replay a partially completed chain. Stop, inspect the failure, and re-establish authorized fresh state before continuing.
+
+Activation requires Desktop's global opt-in, the selected mode, an idle agent without queued prompts, and explicit native confirmation of the target window and duration. Once Desktop claims the session, browser and remote MCP clients cannot drive it for the rest of the server lifetime. Grants are not restored across restarts.
+
+Stop Computer Use when the requested task is complete. Expiry, cancellation, mode disable, global disable, and Desktop disconnection must not be bypassed.

package/src/types/computerUse.ts

@@ -0,0 +1,132 @@
+import type { ComputerUseSessionView } from './computerUseApi';
+
+export interface ComputerUseSessionClient {
+  state(signal?: AbortSignal): Promise<ComputerUseSessionView>;
+  subscribeStatus?(listener: (state: ComputerUseSessionView) => void): () => void;
+  observe(signal?: AbortSignal, options?: ComputerUseObservationOptions): Promise<ComputerUseObservation>;
+  act(action: ComputerUseAction, signal?: AbortSignal): Promise<unknown>;
+  stop(signal?: AbortSignal): Promise<ComputerUseSessionView>;
+}
+
+export interface ComputerUseObservationOptions {
+  includeScreenshot?: boolean;
+}
+
+export const COMPUTER_USE_PHASES = [
+  'inactive',
+  'requesting',
+  'awaiting_confirmation',
+  'activating',
+  'active',
+  'stopping',
+  'failed',
+] as const;
+
+export type ComputerUsePhase = (typeof COMPUTER_USE_PHASES)[number];
+
+export interface ComputerUseSessionIdentity {
+  readonly sessionId: string;
+  readonly runtimeId: string;
+  readonly modeEpoch: string;
+}
+
+export interface ComputerUseTarget {
+  readonly bundleId: string;
+  readonly applicationName: string;
+  readonly processId: number;
+  readonly windowId: string;
+  readonly windowTitle: string;
+}
+
+export interface ComputerUseGrant {
+  readonly runId: string;
+  readonly grantId: string;
+  readonly hostGeneration: string;
+  readonly targetGeneration: string;
+  readonly expiresAt: number;
+}
+
+export interface ComputerUseElement {
+  readonly ref: string;
+  readonly role: string;
+  readonly label?: string;
+  readonly value?: string;
+  readonly enabled: boolean;
+  readonly secure: boolean;
+  readonly actions: readonly ComputerUseActionKind[];
+}
+
+export interface ComputerUseObservation {
+  readonly runId: string;
+  readonly snapshotId: string;
+  readonly targetGeneration: string;
+  readonly applicationName: string;
+  readonly bundleId: string;
+  readonly windowTitle: string;
+  readonly elements: readonly ComputerUseElement[];
+  readonly screenshot?: {
+    readonly mimeType: 'image/png';
+    readonly data: string;
+  };
+}
+
+export type ComputerUseActionKind = 'press' | 'focus' | 'set_value' | 'scroll';
+
+export type ComputerUseAction =
+  | {
+      readonly kind: 'press' | 'focus';
+      readonly snapshotId: string;
+      readonly elementRef: string;
+    }
+  | {
+      readonly kind: 'set_value';
+      readonly snapshotId: string;
+      readonly elementRef: string;
+      readonly value: string;
+    }
+  | {
+      readonly kind: 'scroll';
+      readonly snapshotId: string;
+      readonly elementRef: string;
+      readonly direction: 'up' | 'down' | 'left' | 'right';
+      readonly amount: 'line' | 'page';
+    };
+
+export type ComputerUseFailureCode =
+  | 'busy_in_another_session'
+  | 'desktop_unavailable'
+  | 'permission_required_on_mac'
+  | 'confirmation_denied'
+  | 'confirmation_expired'
+  | 'target_invalid'
+  | 'target_lost'
+  | 'recording_failed'
+  | 'trace_failed'
+  | 'grant_expired'
+  | 'stale_request'
+  | 'outcome_uncertain'
+  | 'internal_error';
+
+export interface ComputerUseState {
+  readonly revision: number;
+  readonly phase: ComputerUsePhase;
+  readonly identity: ComputerUseSessionIdentity;
+  readonly requestId?: string;
+  readonly target?: ComputerUseTarget;
+  readonly grant?: ComputerUseGrant;
+  readonly failure?: {
+    readonly code: ComputerUseFailureCode;
+    readonly message: string;
+  };
+}
+
+export type ComputerUseStateEvent =
+  | { readonly type: 'request'; readonly requestId: string; readonly target: ComputerUseTarget }
+  | { readonly type: 'await_confirmation'; readonly requestId: string }
+  | { readonly type: 'confirm'; readonly requestId: string }
+  | { readonly type: 'deny'; readonly requestId: string }
+  | { readonly type: 'activated'; readonly requestId: string; readonly grant: ComputerUseGrant }
+  | { readonly type: 'stop' }
+  | { readonly type: 'stopped' }
+  | { readonly type: 'fail'; readonly code: ComputerUseFailureCode; readonly message: string }
+  | { readonly type: 'reset' };

package/src/types/computerUseApi.ts

@@ -0,0 +1,83 @@
+import type { DoomApiCaller } from '@agimon-ai/po-core/packageApi';
+
+export const API_BASE_PATH = 'computer-use';
+export const computerUseChannelType = 'computer_use_state';
+export const COMPUTER_USE_MODE_ID = 'computer-use';
+export const COMPUTER_USE_STATUS_KEY = 'doom-computer-use';
+export const COMPUTER_USE_MODE_STATUS_KEY = 'doom-computer-use-mode';
+export const COMPUTER_USE_MAX_DURATION_MS = 1_800 * 1_000;
+export const COMPUTER_USE_DEFAULT_DURATION_MS = 300 * 1_000;
+export const COMPUTER_USE_CONFIRMATION_WINDOW_MS = 2 * 60 * 1_000;
+
+/**
+ * Every path this API answers on, below its mount.
+ *
+ * The typed route table in `./apiRoutes` is built from this, the dispatcher
+ * reads that table, and the hub channel addresses the broker with these same
+ * strings, so a route cannot move on one side alone.
+ */
+export const COMPUTER_USE_ROUTES = {
+  activate: '/activate',
+  artifact: '/artifact',
+  agentState: '/agent/state',
+  agentObserve: '/agent/observe',
+  agentAction: '/agent/action',
+  agentStop: '/agent/stop',
+  hubState: '/hub/state',
+  hubActivation: '/hub/activation',
+  hubAuthorization: '/hub/authorization',
+  hubNext: '/hub/next',
+  hubComplete: '/hub/complete',
+  hubStop: '/hub/stop',
+} as const;
+
+export const COMPUTER_USE_WAKE_LIMIT = 1_000_000;
+
+export interface ComputerUseArtifactView {
+  readonly artifactId: string;
+  readonly status: 'ready' | 'failed';
+  readonly sizeBytes?: number;
+  readonly downloadUrl?: string;
+  readonly previewUrl?: string;
+  readonly actionCount?: number;
+  readonly completedAt?: string;
+  readonly failure?: { readonly code: string; readonly message: string };
+}
+
+export interface ComputerUseSessionView {
+  readonly sessionId: string;
+  readonly revision: number;
+  readonly wake: number;
+  readonly phase: 'inactive' | 'awaiting_confirmation' | 'activating' | 'active' | 'stopping' | 'failed';
+  readonly requestId?: string;
+  readonly target?: Record<string, unknown>;
+  readonly durationMs?: number;
+  readonly expiresAt?: number;
+  readonly failure?: { readonly code: string; readonly message: string };
+  readonly artifact?: ComputerUseArtifactView;
+}
+
+export interface ComputerUseActivationRequest {
+  readonly requestId: string;
+  readonly target?: Record<string, unknown>;
+  readonly durationSeconds: number;
+  readonly createdAt: number;
+  readonly confirmationExpiresAt: number;
+  readonly caller: DoomApiCaller;
+}
+
+export interface ComputerUseBrokerRequest {
+  readonly id: string;
+  readonly operation: 'observe' | 'act';
+  readonly payload?: unknown;
+  readonly grantId: string;
+  readonly sequence?: number;
+}
+
+export interface ComputerUseChannelPayload {
+  readonly state: ComputerUseSessionView;
+  readonly targets: readonly Record<string, unknown>[];
+  readonly busy?: true;
+}
+
+export type ComputerUseBrowserCommand = { readonly action: 'status' | 'targets' | 'stop' | 'artifacts' };