@agimon-ai/doompi-computer-use 0.0.1-alpha.2 → 0.0.1-alpha.21

package/dist/src/services/computerUseRuntime/index.mjs

@@ -0,0 +1,299 @@
+import { COMPUTER_USE_GUIDANCE } from "../../constants/computerUse.mjs";
+import { COMPUTER_USE_MODE_ID, COMPUTER_USE_MODE_STATUS_KEY, COMPUTER_USE_STATUS_KEY } from "../../types/computerUseApi.mjs";
+import { createComputerUseCommand } from "../computerUseCommand/index.mjs";
+import { computerUseRestriction, modeState } from "../../models/computerUseMode.mjs";
+import { defineMinorMode, piMinorModes } from "@agimon-ai/doompi-minor-mode";
+//#region src/services/computerUseRuntime/index.ts
+const PACKAGE_SOURCE = "@agimon-ai/doompi-computer-use";
+function createComputerUseRuntime(dependencies, signal) {
+	const client = dependencies.client;
+	let state;
+	let enabled = false;
+	let restriction = computerUseRestriction(false);
+	const restrictionListeners = /* @__PURE__ */ new Set();
+	const reconcileTools = (active) => {
+		restriction = computerUseRestriction(active);
+		for (const listener of restrictionListeners) listener();
+	};
+	let globallyEnabled = false;
+	let mode;
+	const modeListeners = /* @__PURE__ */ new Set();
+	let timer;
+	let activeContext;
+	let publishedMode = "";
+	let publishedModeStatus;
+	let publishedActivityStatus;
+	const publish = () => {
+		const phase = state?.phase ?? "inactive";
+		const projection = `${enabled}:${phase}:${state?.revision ?? "none"}`;
+		if (mode !== void 0 && projection !== publishedMode) {
+			mode.publish();
+			publishedMode = projection;
+		}
+		const modeStatus = state === void 0 ? void 0 : enabled || phase !== "inactive" ? phase : "";
+		if (activeContext !== void 0 && modeStatus !== publishedModeStatus) {
+			activeContext.ui.setStatus(COMPUTER_USE_MODE_STATUS_KEY, modeStatus);
+			publishedModeStatus = modeStatus;
+		}
+		const activityStatus = enabled || phase !== "inactive" ? `computer use: ${phase}` : void 0;
+		if (activeContext !== void 0 && activityStatus !== publishedActivityStatus) {
+			activeContext.ui.setStatus(COMPUTER_USE_STATUS_KEY, activityStatus);
+			publishedActivityStatus = activityStatus;
+		}
+	};
+	const refresh = async () => {
+		if (signal.aborted) return;
+		let nextGloballyEnabled = false;
+		try {
+			nextGloballyEnabled = await dependencies.enabled?.() === true;
+		} catch {
+			nextGloballyEnabled = false;
+		}
+		if (signal.aborted) return;
+		if (nextGloballyEnabled !== globallyEnabled) {
+			globallyEnabled = nextGloballyEnabled;
+			if (!globallyEnabled) {
+				if (client !== void 0 && state !== void 0 && state.phase !== "inactive" && state.phase !== "failed") try {
+					await client.stop();
+				} catch {}
+				enabled = false;
+				state = void 0;
+				reconcileTools(false);
+			}
+			publishedMode = "";
+			for (const listener of modeListeners) listener();
+		}
+		if (!globallyEnabled || client === void 0) {
+			state = void 0;
+			reconcileTools(false);
+			publish();
+			return;
+		}
+		try {
+			const nextState = await client.state(signal);
+			if (signal.aborted) return;
+			state = nextState;
+			reconcileTools(state.phase === "active");
+			publish();
+		} catch {
+			state = void 0;
+			reconcileTools(false);
+			publish();
+		}
+	};
+	mode = defineMinorMode({
+		descriptor: {
+			source: PACKAGE_SOURCE,
+			id: COMPUTER_USE_MODE_ID,
+			label: "Computer Use",
+			description: "Session-scoped semantic control through DoomPi Desktop.",
+			order: 450,
+			actions: [
+				{
+					id: "activate",
+					label: "Activate",
+					description: "Open the cockpit activation workflow and native confirmation.",
+					contexts: ["tui", "headless"],
+					parameters: []
+				},
+				{
+					id: "deactivate",
+					label: "Deactivate",
+					description: "Stop this session computer-use run.",
+					contexts: ["tui", "headless"],
+					parameters: []
+				},
+				{
+					id: "doctor",
+					label: "Doctor",
+					description: "Report Desktop availability and session state.",
+					contexts: ["tui", "headless"],
+					parameters: []
+				}
+			]
+		},
+		state: () => modeState(state, enabled),
+		async handleAction(_runtime, actionId, _argumentsValue, execution) {
+			activeContext = execution.context;
+			if (actionId === "activate") {
+				if (await dependencies.enabled?.() !== true) {
+					await refresh();
+					throw new Error("Enable computer use in global settings first.");
+				}
+				if (client === void 0) throw new Error("DoomPi Desktop computer use is unavailable.");
+				enabled = true;
+				await refresh();
+				publish();
+				return { message: "Computer use is ready to configure in Activity." };
+			}
+			if (actionId === "doctor") {
+				await refresh();
+				return { message: client === void 0 ? "DoomPi Desktop session API is unavailable." : `Computer use is ${state?.phase.replaceAll("_", " ") ?? "unavailable"}.` };
+			}
+			if (actionId === "deactivate") {
+				if (client === void 0) throw new Error("DoomPi Desktop computer use is unavailable.");
+				if (state !== void 0 && state.phase !== "inactive" && state.phase !== "failed") await client.stop(execution.signal);
+				enabled = false;
+				await refresh();
+				publish();
+				return { message: "Computer use is off." };
+			}
+			throw new Error(`Unknown computer-use action: ${actionId}`);
+		}
+	}).createOwner(void 0);
+	return {
+		commands: [createComputerUseCommand(dependencies.service)],
+		tools: [
+			{
+				name: "computer_state",
+				label: "Computer State",
+				description: "Observe the authorized application window and return its semantic accessibility state.",
+				promptSnippet: "Observe the authorized application before choosing a semantic action",
+				parameters: {
+					type: "object",
+					properties: {},
+					additionalProperties: false
+				},
+				async execute(_toolCallId, _params, signal) {
+					if (!globallyEnabled || client === void 0 || state?.phase !== "active") throw new Error("Computer use is not active for this session.");
+					const observation = await client.observe(signal);
+					return {
+						content: [{
+							type: "text",
+							text: JSON.stringify(observation, null, 2)
+						}],
+						details: observation
+					};
+				}
+			},
+			{
+				name: "computer_action",
+				label: "Computer Action",
+				description: "Perform one constrained semantic action in the authorized application window.",
+				promptSnippet: "Perform one action using a current computer_state snapshot and element ref",
+				parameters: {
+					type: "object",
+					properties: {
+						kind: {
+							type: "string",
+							enum: [
+								"press",
+								"focus",
+								"set_value",
+								"scroll"
+							]
+						},
+						snapshotId: { type: "string" },
+						elementRef: { type: "string" },
+						value: { type: "string" },
+						direction: {
+							type: "string",
+							enum: [
+								"up",
+								"down",
+								"left",
+								"right"
+							]
+						},
+						amount: {
+							type: "string",
+							enum: ["line", "page"]
+						}
+					},
+					required: [
+						"kind",
+						"snapshotId",
+						"elementRef"
+					],
+					additionalProperties: false
+				},
+				async execute(_toolCallId, params, signal) {
+					if (!globallyEnabled || client === void 0 || state?.phase !== "active") throw new Error("Computer use is not active for this session.");
+					const result = await client.act(params, signal);
+					return {
+						content: [{
+							type: "text",
+							text: JSON.stringify(result, null, 2)
+						}],
+						details: result
+					};
+				}
+			},
+			{
+				name: "computer_exec",
+				label: "Computer Script",
+				description: "Run a trusted, explicitly allowed local TypeScript script against the authorized application session.",
+				promptSnippet: "Run a trusted reusable computer script by its explicitly allowed path and JSON input",
+				parameters: {
+					type: "object",
+					properties: {
+						scriptPath: { type: "string" },
+						input: {}
+					},
+					required: ["scriptPath", "input"],
+					additionalProperties: false
+				},
+				async execute(_toolCallId, params, signal) {
+					if (!globallyEnabled || client === void 0 || state?.phase !== "active") throw new Error("Computer use is not active for this session.");
+					if (dependencies.scriptRunner === void 0) throw new Error("Computer script execution is unavailable.");
+					const input = params;
+					const result = await dependencies.scriptRunner.execute(input.scriptPath, input.input, signal);
+					return {
+						content: [{
+							type: "text",
+							text: JSON.stringify(result, null, 2)
+						}],
+						details: result
+					};
+				}
+			}
+		],
+		services: [piMinorModes({
+			snapshot: () => globallyEnabled ? [mode] : [],
+			subscribe(listener) {
+				modeListeners.add(listener);
+				return () => {
+					modeListeners.delete(listener);
+				};
+			}
+		})],
+		toolRestrictions: [{
+			source: PACKAGE_SOURCE,
+			restrict: (incoming, available) => restriction(incoming, available),
+			subscribe(listener) {
+				restrictionListeners.add(listener);
+				return () => {
+					restrictionListeners.delete(listener);
+				};
+			}
+		}],
+		events: {
+			before_agent_start: (event) => globallyEnabled && state?.phase === "active" ? { systemPrompt: `${event.systemPrompt}\n\n${COMPUTER_USE_GUIDANCE}` } : void 0,
+			session_start: (_event, context) => {
+				if (signal.aborted) return;
+				activeContext = context;
+				reconcileTools(false);
+				refresh();
+				if (timer) clearInterval(timer);
+				timer = setInterval(() => void refresh(), 250);
+				timer.unref?.();
+			}
+		},
+		onStop() {
+			if (timer) clearInterval(timer);
+			timer = void 0;
+		},
+		onDispose() {
+			if (timer) clearInterval(timer);
+			timer = void 0;
+			reconcileTools(false);
+			activeContext?.ui.setStatus(COMPUTER_USE_MODE_STATUS_KEY, void 0);
+			activeContext?.ui.setStatus(COMPUTER_USE_STATUS_KEY, void 0);
+			activeContext = void 0;
+		}
+	};
+}
+//#endregion
+export { COMPUTER_USE_MODE_ID, createComputerUseRuntime };
+
+//# sourceMappingURL=index.mjs.map

package/dist/src/services/computerUseRuntime/index.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.mjs","names":[],"sources":["../../../../src/services/computerUseRuntime/index.ts"],"sourcesContent":["import { type PiPluginContributions } from '@agimon-ai/doompi-core/pi-extension';\nimport { type DoomToolRestriction } from '@agimon-ai/doompi-core/tool-surface';\nimport { piMinorModes } from '@agimon-ai/doompi-minor-mode';\nimport { defineMinorMode, type MinorModeOwner, type MinorModeOwnerActionContext } from '@agimon-ai/doompi-minor-mode';\nimport type { ExtensionAPI, ExtensionContext } from '@earendil-works/pi-coding-agent';\n\nimport { COMPUTER_USE_GUIDANCE } from '../../constants/computerUse';\nimport type { ComputerUseAction } from '../../types/computerUse';\nimport {\n  COMPUTER_USE_MODE_ID,\n  COMPUTER_USE_MODE_STATUS_KEY,\n  COMPUTER_USE_STATUS_KEY,\n  type ComputerUseSessionView,\n} from '../../types/computerUseApi';\nimport type { ComputerUseExtensionDependencies } from '../../types/extension';\nimport { createComputerUseCommand } from '../computerUseCommand';\n\nconst PACKAGE_SOURCE = '@agimon-ai/doompi-computer-use';\nexport { COMPUTER_USE_MODE_ID };\nimport { modeState, computerUseRestriction } from '../../models/computerUseMode';\n\nexport function createComputerUseRuntime(\n  dependencies: ComputerUseExtensionDependencies,\n  signal: AbortSignal,\n): PiPluginContributions<ComputerUseExtensionDependencies> & {\n  tools: readonly Parameters<ExtensionAPI['registerTool']>[0][];\n} {\n  const client = dependencies.client;\n  let state: ComputerUseSessionView | undefined;\n  let enabled = false;\n  let restriction: DoomToolRestriction = computerUseRestriction(false);\n  const restrictionListeners = new Set<() => void>();\n  const reconcileTools = (active: boolean): void => {\n    restriction = computerUseRestriction(active);\n    for (const listener of restrictionListeners) listener();\n  };\n  let globallyEnabled = false;\n  let mode: MinorModeOwner<MinorModeOwnerActionContext<ExtensionContext>>;\n  const modeListeners = new Set<() => void>();\n  let timer: ReturnType<typeof setInterval> | undefined;\n  let activeContext: ExtensionContext | undefined;\n  let publishedMode = '';\n  let publishedModeStatus: string | undefined;\n  let publishedActivityStatus: string | undefined;\n  const publish = (): void => {\n    const phase = state?.phase ?? 'inactive';\n    const projection = `${enabled}:${phase}:${state?.revision ?? 'none'}`;\n    if (mode !== undefined && projection !== publishedMode) {\n      mode.publish();\n      publishedMode = projection;\n    }\n    const modeStatus = state === undefined ? undefined : enabled || phase !== 'inactive' ? phase : '';\n    if (activeContext !== undefined && modeStatus !== publishedModeStatus) {\n      activeContext.ui.setStatus(COMPUTER_USE_MODE_STATUS_KEY, modeStatus);\n      publishedModeStatus = modeStatus;\n    }\n    const activityStatus = enabled || phase !== 'inactive' ? `computer use: ${phase}` : undefined;\n    if (activeContext !== undefined && activityStatus !== publishedActivityStatus) {\n      activeContext.ui.setStatus(COMPUTER_USE_STATUS_KEY, activityStatus);\n      publishedActivityStatus = activityStatus;\n    }\n  };\n  const refresh = async (): Promise<void> => {\n    if (signal.aborted) return;\n    let nextGloballyEnabled = false;\n    try {\n      nextGloballyEnabled = (await dependencies.enabled?.()) === true;\n    } catch {\n      nextGloballyEnabled = false;\n    }\n    if (signal.aborted) return;\n    if (nextGloballyEnabled !== globallyEnabled) {\n      globallyEnabled = nextGloballyEnabled;\n      if (!globallyEnabled) {\n        if (client !== undefined && state !== undefined && state.phase !== 'inactive' && state.phase !== 'failed') {\n          try {\n            await client.stop();\n          } catch {\n            // The global opt-in is still authoritative even when remote cleanup fails.\n          }\n        }\n        enabled = false;\n        state = undefined;\n        reconcileTools(false);\n      }\n      publishedMode = '';\n      for (const listener of modeListeners) listener();\n    }\n    if (!globallyEnabled || client === undefined) {\n      state = undefined;\n      reconcileTools(false);\n      publish();\n      return;\n    }\n    try {\n      const nextState = await client.state(signal);\n      if (signal.aborted) return;\n      state = nextState;\n      reconcileTools(state.phase === 'active');\n      publish();\n    } catch {\n      state = undefined;\n      reconcileTools(false);\n      publish();\n    }\n  };\n  mode = defineMinorMode<undefined, MinorModeOwnerActionContext<ExtensionContext>>({\n    descriptor: {\n      source: PACKAGE_SOURCE,\n      id: COMPUTER_USE_MODE_ID,\n      label: 'Computer Use',\n      description: 'Session-scoped semantic control through DoomPi Desktop.',\n      order: 450,\n      actions: [\n        {\n          id: 'activate',\n          label: 'Activate',\n          description: 'Open the cockpit activation workflow and native confirmation.',\n          contexts: ['tui', 'headless'],\n          parameters: [],\n        },\n        {\n          id: 'deactivate',\n          label: 'Deactivate',\n          description: 'Stop this session computer-use run.',\n          contexts: ['tui', 'headless'],\n          parameters: [],\n        },\n        {\n          id: 'doctor',\n          label: 'Doctor',\n          description: 'Report Desktop availability and session state.',\n          contexts: ['tui', 'headless'],\n          parameters: [],\n        },\n      ],\n    },\n    state: () => modeState(state, enabled),\n    async handleAction(_runtime, actionId, _argumentsValue, execution) {\n      activeContext = execution.context;\n      if (actionId === 'activate') {\n        if ((await dependencies.enabled?.()) !== true) {\n          await refresh();\n          throw new Error('Enable computer use in global settings first.');\n        }\n        if (client === undefined) throw new Error('DoomPi Desktop computer use is unavailable.');\n        enabled = true;\n        await refresh();\n        publish();\n        return { message: 'Computer use is ready to configure in Activity.' };\n      }\n      if (actionId === 'doctor') {\n        await refresh();\n        return {\n          message:\n            client === undefined\n              ? 'DoomPi Desktop session API is unavailable.'\n              : `Computer use is ${state?.phase.replaceAll('_', ' ') ?? 'unavailable'}.`,\n        };\n      }\n      if (actionId === 'deactivate') {\n        if (client === undefined) throw new Error('DoomPi Desktop computer use is unavailable.');\n        if (state !== undefined && state.phase !== 'inactive' && state.phase !== 'failed')\n          await client.stop(execution.signal);\n        enabled = false;\n        await refresh();\n        publish();\n        return { message: 'Computer use is off.' };\n      }\n      throw new Error(`Unknown computer-use action: ${actionId}`);\n    },\n  }).createOwner(undefined);\n  return {\n    commands: [createComputerUseCommand(dependencies.service)],\n    tools: [\n      {\n        name: 'computer_state',\n        label: 'Computer State',\n        description: 'Observe the authorized application window and return its semantic accessibility state.',\n        promptSnippet: 'Observe the authorized application before choosing a semantic action',\n        parameters: { type: 'object', properties: {}, additionalProperties: false },\n        async execute(_toolCallId, _params, signal) {\n          if (!globallyEnabled || client === undefined || state?.phase !== 'active')\n            throw new Error('Computer use is not active for this session.');\n          const observation = await client.observe(signal);\n          return { content: [{ type: 'text', text: JSON.stringify(observation, null, 2) }], details: observation };\n        },\n      },\n      {\n        name: 'computer_action',\n        label: 'Computer Action',\n        description: 'Perform one constrained semantic action in the authorized application window.',\n        promptSnippet: 'Perform one action using a current computer_state snapshot and element ref',\n        parameters: {\n          type: 'object',\n          properties: {\n            kind: { type: 'string', enum: ['press', 'focus', 'set_value', 'scroll'] },\n            snapshotId: { type: 'string' },\n            elementRef: { type: 'string' },\n            value: { type: 'string' },\n            direction: { type: 'string', enum: ['up', 'down', 'left', 'right'] },\n            amount: { type: 'string', enum: ['line', 'page'] },\n          },\n          required: ['kind', 'snapshotId', 'elementRef'],\n          additionalProperties: false,\n        },\n        async execute(_toolCallId, params, signal) {\n          if (!globallyEnabled || client === undefined || state?.phase !== 'active')\n            throw new Error('Computer use is not active for this session.');\n          const result = await client.act(params as unknown as ComputerUseAction, signal);\n          return { content: [{ type: 'text', text: JSON.stringify(result, null, 2) }], details: result };\n        },\n      },\n      {\n        name: 'computer_exec',\n        label: 'Computer Script',\n        description:\n          'Run a trusted, explicitly allowed local TypeScript script against the authorized application session.',\n        promptSnippet: 'Run a trusted reusable computer script by its explicitly allowed path and JSON input',\n        parameters: {\n          type: 'object',\n          properties: {\n            scriptPath: { type: 'string' },\n            input: {},\n          },\n          required: ['scriptPath', 'input'],\n          additionalProperties: false,\n        },\n        async execute(_toolCallId, params, signal) {\n          if (!globallyEnabled || client === undefined || state?.phase !== 'active')\n            throw new Error('Computer use is not active for this session.');\n          if (dependencies.scriptRunner === undefined) throw new Error('Computer script execution is unavailable.');\n          const input = params as { scriptPath: string; input: unknown };\n          const result = await dependencies.scriptRunner.execute(input.scriptPath, input.input, signal);\n          return { content: [{ type: 'text', text: JSON.stringify(result, null, 2) }], details: result };\n        },\n      },\n    ],\n    services: [\n      piMinorModes({\n        snapshot: () => (globallyEnabled ? [mode] : []),\n        subscribe(listener) {\n          modeListeners.add(listener);\n          return () => {\n            modeListeners.delete(listener);\n          };\n        },\n      }),\n    ],\n    toolRestrictions: [\n      {\n        source: PACKAGE_SOURCE,\n        restrict: (incoming, available) => restriction(incoming, available),\n        subscribe(listener) {\n          restrictionListeners.add(listener);\n          return () => {\n            restrictionListeners.delete(listener);\n          };\n        },\n      },\n    ],\n    events: {\n      before_agent_start: (event) =>\n        globallyEnabled && state?.phase === 'active'\n          ? { systemPrompt: `${event.systemPrompt}\\n\\n${COMPUTER_USE_GUIDANCE}` }\n          : undefined,\n      session_start: (_event, context) => {\n        if (signal.aborted) return;\n        activeContext = context;\n        reconcileTools(false);\n        void refresh();\n        if (timer) clearInterval(timer);\n        timer = setInterval(() => void refresh(), 250);\n        timer.unref?.();\n      },\n    },\n    onStop() {\n      if (timer) clearInterval(timer);\n      timer = undefined;\n    },\n    onDispose() {\n      if (timer) clearInterval(timer);\n      timer = undefined;\n      reconcileTools(false);\n      activeContext?.ui.setStatus(COMPUTER_USE_MODE_STATUS_KEY, undefined);\n      activeContext?.ui.setStatus(COMPUTER_USE_STATUS_KEY, undefined);\n      activeContext = undefined;\n    },\n  };\n}\n"],"mappings":";;;;;;AAiBA,MAAM,iBAAiB;AAIvB,SAAgB,yBACd,cACA,QAGA;CACA,MAAM,SAAS,aAAa;CAC5B,IAAI;CACJ,IAAI,UAAU;CACd,IAAI,cAAmC,uBAAuB,KAAK;CACnE,MAAM,uCAAuB,IAAI,IAAgB;CACjD,MAAM,kBAAkB,WAA0B;EAChD,cAAc,uBAAuB,MAAM;EAC3C,KAAK,MAAM,YAAY,sBAAsB,SAAS;CACxD;CACA,IAAI,kBAAkB;CACtB,IAAI;CACJ,MAAM,gCAAgB,IAAI,IAAgB;CAC1C,IAAI;CACJ,IAAI;CACJ,IAAI,gBAAgB;CACpB,IAAI;CACJ,IAAI;CACJ,MAAM,gBAAsB;EAC1B,MAAM,QAAQ,OAAO,SAAS;EAC9B,MAAM,aAAa,GAAG,QAAQ,GAAG,MAAM,GAAG,OAAO,YAAY;EAC7D,IAAI,SAAS,KAAA,KAAa,eAAe,eAAe;GACtD,KAAK,QAAQ;GACb,gBAAgB;EAClB;EACA,MAAM,aAAa,UAAU,KAAA,IAAY,KAAA,IAAY,WAAW,UAAU,aAAa,QAAQ;EAC/F,IAAI,kBAAkB,KAAA,KAAa,eAAe,qBAAqB;GACrE,cAAc,GAAG,UAAU,8BAA8B,UAAU;GACnE,sBAAsB;EACxB;EACA,MAAM,iBAAiB,WAAW,UAAU,aAAa,iBAAiB,UAAU,KAAA;EACpF,IAAI,kBAAkB,KAAA,KAAa,mBAAmB,yBAAyB;GAC7E,cAAc,GAAG,UAAU,yBAAyB,cAAc;GAClE,0BAA0B;EAC5B;CACF;CACA,MAAM,UAAU,YAA2B;EACzC,IAAI,OAAO,SAAS;EACpB,IAAI,sBAAsB;EAC1B,IAAI;GACF,sBAAuB,MAAM,aAAa,UAAU,MAAO;EAC7D,QAAQ;GACN,sBAAsB;EACxB;EACA,IAAI,OAAO,SAAS;EACpB,IAAI,wBAAwB,iBAAiB;GAC3C,kBAAkB;GAClB,IAAI,CAAC,iBAAiB;IACpB,IAAI,WAAW,KAAA,KAAa,UAAU,KAAA,KAAa,MAAM,UAAU,cAAc,MAAM,UAAU,UAC/F,IAAI;KACF,MAAM,OAAO,KAAK;IACpB,QAAQ,CAER;IAEF,UAAU;IACV,QAAQ,KAAA;IACR,eAAe,KAAK;GACtB;GACA,gBAAgB;GAChB,KAAK,MAAM,YAAY,eAAe,SAAS;EACjD;EACA,IAAI,CAAC,mBAAmB,WAAW,KAAA,GAAW;GAC5C,QAAQ,KAAA;GACR,eAAe,KAAK;GACpB,QAAQ;GACR;EACF;EACA,IAAI;GACF,MAAM,YAAY,MAAM,OAAO,MAAM,MAAM;GAC3C,IAAI,OAAO,SAAS;GACpB,QAAQ;GACR,eAAe,MAAM,UAAU,QAAQ;GACvC,QAAQ;EACV,QAAQ;GACN,QAAQ,KAAA;GACR,eAAe,KAAK;GACpB,QAAQ;EACV;CACF;CACA,OAAO,gBAA0E;EAC/E,YAAY;GACV,QAAQ;GACR,IAAI;GACJ,OAAO;GACP,aAAa;GACb,OAAO;GACP,SAAS;IACP;KACE,IAAI;KACJ,OAAO;KACP,aAAa;KACb,UAAU,CAAC,OAAO,UAAU;KAC5B,YAAY,CAAC;IACf;IACA;KACE,IAAI;KACJ,OAAO;KACP,aAAa;KACb,UAAU,CAAC,OAAO,UAAU;KAC5B,YAAY,CAAC;IACf;IACA;KACE,IAAI;KACJ,OAAO;KACP,aAAa;KACb,UAAU,CAAC,OAAO,UAAU;KAC5B,YAAY,CAAC;IACf;GACF;EACF;EACA,aAAa,UAAU,OAAO,OAAO;EACrC,MAAM,aAAa,UAAU,UAAU,iBAAiB,WAAW;GACjE,gBAAgB,UAAU;GAC1B,IAAI,aAAa,YAAY;IAC3B,IAAK,MAAM,aAAa,UAAU,MAAO,MAAM;KAC7C,MAAM,QAAQ;KACd,MAAM,IAAI,MAAM,+CAA+C;IACjE;IACA,IAAI,WAAW,KAAA,GAAW,MAAM,IAAI,MAAM,6CAA6C;IACvF,UAAU;IACV,MAAM,QAAQ;IACd,QAAQ;IACR,OAAO,EAAE,SAAS,kDAAkD;GACtE;GACA,IAAI,aAAa,UAAU;IACzB,MAAM,QAAQ;IACd,OAAO,EACL,SACE,WAAW,KAAA,IACP,+CACA,mBAAmB,OAAO,MAAM,WAAW,KAAK,GAAG,KAAK,cAAc,GAC9E;GACF;GACA,IAAI,aAAa,cAAc;IAC7B,IAAI,WAAW,KAAA,GAAW,MAAM,IAAI,MAAM,6CAA6C;IACvF,IAAI,UAAU,KAAA,KAAa,MAAM,UAAU,cAAc,MAAM,UAAU,UACvE,MAAM,OAAO,KAAK,UAAU,MAAM;IACpC,UAAU;IACV,MAAM,QAAQ;IACd,QAAQ;IACR,OAAO,EAAE,SAAS,uBAAuB;GAC3C;GACA,MAAM,IAAI,MAAM,gCAAgC,UAAU;EAC5D;CACF,CAAC,CAAC,CAAC,YAAY,KAAA,CAAS;CACxB,OAAO;EACL,UAAU,CAAC,yBAAyB,aAAa,OAAO,CAAC;EACzD,OAAO;GACL;IACE,MAAM;IACN,OAAO;IACP,aAAa;IACb,eAAe;IACf,YAAY;KAAE,MAAM;KAAU,YAAY,CAAC;KAAG,sBAAsB;IAAM;IAC1E,MAAM,QAAQ,aAAa,SAAS,QAAQ;KAC1C,IAAI,CAAC,mBAAmB,WAAW,KAAA,KAAa,OAAO,UAAU,UAC/D,MAAM,IAAI,MAAM,8CAA8C;KAChE,MAAM,cAAc,MAAM,OAAO,QAAQ,MAAM;KAC/C,OAAO;MAAE,SAAS,CAAC;OAAE,MAAM;OAAQ,MAAM,KAAK,UAAU,aAAa,MAAM,CAAC;MAAE,CAAC;MAAG,SAAS;KAAY;IACzG;GACF;GACA;IACE,MAAM;IACN,OAAO;IACP,aAAa;IACb,eAAe;IACf,YAAY;KACV,MAAM;KACN,YAAY;MACV,MAAM;OAAE,MAAM;OAAU,MAAM;QAAC;QAAS;QAAS;QAAa;OAAQ;MAAE;MACxE,YAAY,EAAE,MAAM,SAAS;MAC7B,YAAY,EAAE,MAAM,SAAS;MAC7B,OAAO,EAAE,MAAM,SAAS;MACxB,WAAW;OAAE,MAAM;OAAU,MAAM;QAAC;QAAM;QAAQ;QAAQ;OAAO;MAAE;MACnE,QAAQ;OAAE,MAAM;OAAU,MAAM,CAAC,QAAQ,MAAM;MAAE;KACnD;KACA,UAAU;MAAC;MAAQ;MAAc;KAAY;KAC7C,sBAAsB;IACxB;IACA,MAAM,QAAQ,aAAa,QAAQ,QAAQ;KACzC,IAAI,CAAC,mBAAmB,WAAW,KAAA,KAAa,OAAO,UAAU,UAC/D,MAAM,IAAI,MAAM,8CAA8C;KAChE,MAAM,SAAS,MAAM,OAAO,IAAI,QAAwC,MAAM;KAC9E,OAAO;MAAE,SAAS,CAAC;OAAE,MAAM;OAAQ,MAAM,KAAK,UAAU,QAAQ,MAAM,CAAC;MAAE,CAAC;MAAG,SAAS;KAAO;IAC/F;GACF;GACA;IACE,MAAM;IACN,OAAO;IACP,aACE;IACF,eAAe;IACf,YAAY;KACV,MAAM;KACN,YAAY;MACV,YAAY,EAAE,MAAM,SAAS;MAC7B,OAAO,CAAC;KACV;KACA,UAAU,CAAC,cAAc,OAAO;KAChC,sBAAsB;IACxB;IACA,MAAM,QAAQ,aAAa,QAAQ,QAAQ;KACzC,IAAI,CAAC,mBAAmB,WAAW,KAAA,KAAa,OAAO,UAAU,UAC/D,MAAM,IAAI,MAAM,8CAA8C;KAChE,IAAI,aAAa,iBAAiB,KAAA,GAAW,MAAM,IAAI,MAAM,2CAA2C;KACxG,MAAM,QAAQ;KACd,MAAM,SAAS,MAAM,aAAa,aAAa,QAAQ,MAAM,YAAY,MAAM,OAAO,MAAM;KAC5F,OAAO;MAAE,SAAS,CAAC;OAAE,MAAM;OAAQ,MAAM,KAAK,UAAU,QAAQ,MAAM,CAAC;MAAE,CAAC;MAAG,SAAS;KAAO;IAC/F;GACF;EACF;EACA,UAAU,CACR,aAAa;GACX,gBAAiB,kBAAkB,CAAC,IAAI,IAAI,CAAC;GAC7C,UAAU,UAAU;IAClB,cAAc,IAAI,QAAQ;IAC1B,aAAa;KACX,cAAc,OAAO,QAAQ;IAC/B;GACF;EACF,CAAC,CACH;EACA,kBAAkB,CAChB;GACE,QAAQ;GACR,WAAW,UAAU,cAAc,YAAY,UAAU,SAAS;GAClE,UAAU,UAAU;IAClB,qBAAqB,IAAI,QAAQ;IACjC,aAAa;KACX,qBAAqB,OAAO,QAAQ;IACtC;GACF;EACF,CACF;EACA,QAAQ;GACN,qBAAqB,UACnB,mBAAmB,OAAO,UAAU,WAChC,EAAE,cAAc,GAAG,MAAM,aAAa,MAAM,wBAAwB,IACpE,KAAA;GACN,gBAAgB,QAAQ,YAAY;IAClC,IAAI,OAAO,SAAS;IACpB,gBAAgB;IAChB,eAAe,KAAK;IACpB,QAAa;IACb,IAAI,OAAO,cAAc,KAAK;IAC9B,QAAQ,kBAAkB,KAAK,QAAQ,GAAG,GAAG;IAC7C,MAAM,QAAQ;GAChB;EACF;EACA,SAAS;GACP,IAAI,OAAO,cAAc,KAAK;GAC9B,QAAQ,KAAA;EACV;EACA,YAAY;GACV,IAAI,OAAO,cAAc,KAAK;GAC9B,QAAQ,KAAA;GACR,eAAe,KAAK;GACpB,eAAe,GAAG,UAAU,8BAA8B,KAAA,CAAS;GACnE,eAAe,GAAG,UAAU,yBAAyB,KAAA,CAAS;GAC9D,gBAAgB,KAAA;EAClB;CACF;AACF"}

package/dist/src/services/computerUseServer/index.cjs

@@ -0,0 +1,350 @@
+const require_runtime = require("../../../_virtual/_rolldown/runtime.cjs");
+const require_computerUse = require("../../constants/computerUse.cjs");
+const require_index = require("../sessionApiClient/index.cjs");
+const require_index$1 = require("../computerScriptRunner/index.cjs");
+const require_index$2 = require("../extensionService/index.cjs");
+const require_computerUseApi = require("../../types/computerUseApi.cjs");
+const require_computerUseMode = require("../../models/computerUseMode.cjs");
+let node_path = require("node:path");
+node_path = require_runtime.__toESM(node_path, 1);
+let _agimon_ai_doompi_minor_mode = require("@agimon-ai/doompi-minor-mode");
+//#region src/services/computerUseServer/index.ts
+/** This package's identity for mode, activity and restriction ownership, not a status key. */
+const SOURCE = "@agimon-ai/doompi-computer-use";
+const SCRIPT_PATHS_ENV = "DOOMPI_COMPUTER_USE_SCRIPT_PATHS";
+const GUIDANCE = "[COMPUTER USE ACTIVE]\nUse computer_state before acting and after any action that may change the interface. Use only element refs and snapshot ids returned by computer_state. Never infer coordinates or bypass secure elements.";
+function output(value, isError = false) {
+	return {
+		content: [{
+			type: "text",
+			text: typeof value === "string" ? value : JSON.stringify(value, null, 2)
+		}],
+		details: value,
+		...isError ? { isError: true } : {}
+	};
+}
+function observationOutput(value) {
+	return {
+		content: [{
+			type: "text",
+			text: JSON.stringify({
+				...value,
+				screenshot: void 0
+			}, null, 2)
+		}, {
+			type: "image",
+			data: value.screenshot.data,
+			mimeType: value.screenshot.mimeType
+		}],
+		details: value
+	};
+}
+function createComputerUseServer(host) {
+	const client = require_index.createComputerUseSessionClient();
+	const allowedScriptPaths = (host.context.environment[SCRIPT_PATHS_ENV] ?? "").split(node_path.default.delimiter).filter((entry) => entry.length > 0);
+	const dependencies = {
+		client,
+		scriptRunner: client === void 0 ? void 0 : new require_index$1.ComputerScriptRunner({
+			client,
+			allowedScriptPaths
+		}),
+		service: new require_index$2.DefaultComputerUseExtensionService(client)
+	};
+	let state;
+	let stopActivity;
+	let modeOwner;
+	const modeSelected = () => (host.context.selection.state?.["minor-mode"] ?? []).includes(require_computerUseApi.COMPUTER_USE_MODE_ID);
+	const publishMode = () => modeOwner?.publish();
+	const applyState = (next) => {
+		state = next;
+		host.context.client.setStatus(require_computerUseApi.COMPUTER_USE_STATUS_KEY, next === void 0 || next.phase === "inactive" ? void 0 : `computer use: ${next.phase}`);
+		publishMode();
+	};
+	const selectMode = async (enabled) => {
+		const modes = (host.context.selection.state?.["minor-mode"] ?? []).filter((mode) => mode !== require_computerUseApi.COMPUTER_USE_MODE_ID);
+		await host.changeSelection({
+			axis: "state",
+			key: "minor-mode",
+			values: enabled ? [...modes, require_computerUseApi.COMPUTER_USE_MODE_ID] : modes
+		});
+		publishMode();
+	};
+	const refresh = async (signal) => {
+		if (!client) {
+			applyState(void 0);
+			return;
+		}
+		try {
+			applyState(await client.state(signal));
+		} catch {
+			applyState(void 0);
+		}
+	};
+	const requireActive = () => {
+		if (!client || state?.phase !== "active") throw new Error("Computer use is not active for this session.");
+		return client;
+	};
+	modeOwner = (0, _agimon_ai_doompi_minor_mode.defineMinorMode)({
+		descriptor: {
+			source: SOURCE,
+			id: require_computerUseApi.COMPUTER_USE_MODE_ID,
+			label: "Computer Use",
+			description: "Session-scoped semantic control through DoomPi Desktop.",
+			order: 450,
+			actions: [
+				{
+					id: "activate",
+					label: "Activate",
+					description: "Enable computer use for this session.",
+					contexts: ["headless"],
+					parameters: []
+				},
+				{
+					id: "deactivate",
+					label: "Deactivate",
+					description: "Stop this session computer-use run.",
+					contexts: ["headless"],
+					parameters: []
+				},
+				{
+					id: "doctor",
+					label: "Doctor",
+					description: "Report Desktop availability and session state.",
+					contexts: ["headless"],
+					parameters: []
+				}
+			]
+		},
+		state: () => require_computerUseMode.modeState(state, modeSelected()),
+		async handleAction(_runtime, actionId, _argumentsValue, execution) {
+			execution.signal.throwIfAborted();
+			if (actionId === "activate") {
+				if (!client) throw new Error("DoomPi Desktop computer use is unavailable.");
+				await selectMode(true);
+				await refresh(execution.signal);
+				return { message: "Computer use activated." };
+			}
+			if (actionId === "doctor") {
+				await refresh(execution.signal);
+				return { message: client ? `Computer use is ${state?.phase ?? "unavailable"}.` : "DoomPi Desktop session API is unavailable." };
+			}
+			if (actionId === "deactivate") {
+				if (client && state?.phase !== void 0 && state.phase !== "inactive" && state.phase !== "failed") await client.stop(execution.signal);
+				await selectMode(false);
+				await refresh(execution.signal);
+				return { message: "Computer use deactivated." };
+			}
+			throw new Error(`Unknown computer-use action: ${actionId}`);
+		}
+	}).createOwner(void 0);
+	return {
+		services: [(0, _agimon_ai_doompi_minor_mode.serverMinorModes)([modeOwner])],
+		toolRestrictions: [{
+			when: { state: { "minor-mode": require_computerUseApi.COMPUTER_USE_MODE_ID } },
+			allowedTools: [...require_computerUse.COMPUTER_USE_TOOL_NAMES]
+		}],
+		activities: [{
+			when: {
+				state: { "minor-mode": require_computerUseApi.COMPUTER_USE_MODE_ID },
+				attribution: {
+					kind: "minor",
+					mode: require_computerUseApi.COMPUTER_USE_MODE_ID
+				}
+			},
+			name: SOURCE,
+			async start() {
+				await refresh();
+				const unsubscribe = client?.subscribeStatus?.(applyState);
+				stopActivity = () => {
+					unsubscribe?.();
+					applyState(void 0);
+				};
+				return () => {
+					stopActivity?.();
+					stopActivity = void 0;
+				};
+			}
+		}],
+		tools: [
+			{
+				when: {
+					state: { "minor-mode": require_computerUseApi.COMPUTER_USE_MODE_ID },
+					attribution: {
+						kind: "minor",
+						mode: require_computerUseApi.COMPUTER_USE_MODE_ID
+					}
+				},
+				name: "computer_state",
+				label: "Computer State",
+				description: "Observe the authorized application window and return semantic accessibility state.",
+				parameters: {
+					type: "object",
+					properties: {},
+					additionalProperties: false
+				},
+				executionMode: "serial",
+				async execute(_toolCallId, _parameters, signal) {
+					try {
+						return observationOutput(await requireActive().observe(signal));
+					} catch (error) {
+						return output(error instanceof Error ? error.message : String(error), true);
+					}
+				}
+			},
+			{
+				when: {
+					state: { "minor-mode": require_computerUseApi.COMPUTER_USE_MODE_ID },
+					attribution: {
+						kind: "minor",
+						mode: require_computerUseApi.COMPUTER_USE_MODE_ID
+					}
+				},
+				name: "computer_action",
+				label: "Computer Action",
+				description: "Perform one constrained semantic action in the authorized application window.",
+				parameters: {
+					type: "object",
+					properties: {
+						kind: {
+							type: "string",
+							enum: [
+								"press",
+								"focus",
+								"set_value",
+								"scroll"
+							]
+						},
+						snapshotId: {
+							type: "string",
+							minLength: 1
+						},
+						elementRef: {
+							type: "string",
+							minLength: 1
+						},
+						value: { type: "string" },
+						direction: {
+							type: "string",
+							enum: [
+								"up",
+								"down",
+								"left",
+								"right"
+							]
+						},
+						amount: {
+							type: "string",
+							enum: ["line", "page"]
+						}
+					},
+					required: [
+						"kind",
+						"snapshotId",
+						"elementRef"
+					],
+					additionalProperties: false
+				},
+				executionMode: "serial",
+				async execute(_toolCallId, parameters, signal) {
+					try {
+						const result = await requireActive().act(parameters, signal);
+						await refresh(signal);
+						return output(result);
+					} catch (error) {
+						return output(error instanceof Error ? error.message : String(error), true);
+					}
+				}
+			},
+			{
+				when: {
+					state: { "minor-mode": require_computerUseApi.COMPUTER_USE_MODE_ID },
+					attribution: {
+						kind: "minor",
+						mode: require_computerUseApi.COMPUTER_USE_MODE_ID
+					}
+				},
+				name: "computer_exec",
+				label: "Computer Script",
+				description: "Run a trusted, explicitly allowed local TypeScript script against the authorized application session.",
+				parameters: {
+					type: "object",
+					properties: {
+						scriptPath: {
+							type: "string",
+							minLength: 1
+						},
+						input: {}
+					},
+					required: ["scriptPath", "input"],
+					additionalProperties: false
+				},
+				executionMode: "serial",
+				async execute(_toolCallId, parameters, signal) {
+					try {
+						if (!dependencies.scriptRunner) throw new Error("Computer script execution is unavailable.");
+						const input = parameters;
+						return output(await dependencies.scriptRunner.execute(input.scriptPath, input.input, signal));
+					} catch (error) {
+						return output(error instanceof Error ? error.message : String(error), true);
+					}
+				}
+			}
+		],
+		commands: [{
+			name: require_computerUse.COMMAND_NAME,
+			description: require_computerUse.COMMAND_DESCRIPTION,
+			async execute(args, execution) {
+				const result = await dependencies.service.execute();
+				await execution.client.notify({
+					body: result.message,
+					level: result.level
+				});
+				if (args.trim() === "deactivate" && client && state?.phase !== "inactive" && state?.phase !== "failed") {
+					await client.stop();
+					await host.changeSelection({
+						axis: "state",
+						key: "minor-mode",
+						values: (execution.selection.state?.["minor-mode"] ?? []).filter((mode) => mode !== require_computerUseApi.COMPUTER_USE_MODE_ID)
+					});
+					await refresh();
+				}
+			}
+		}],
+		hooks: [{
+			when: {
+				state: { "minor-mode": require_computerUseApi.COMPUTER_USE_MODE_ID },
+				attribution: {
+					kind: "minor",
+					mode: require_computerUseApi.COMPUTER_USE_MODE_ID
+				}
+			},
+			event: "before_agent_start",
+			handle(event) {
+				if (state?.phase !== "active") return void 0;
+				return { systemPrompt: `${typeof event.systemPrompt === "string" ? event.systemPrompt : ""}\n\n${GUIDANCE}`.trim() };
+			}
+		}, {
+			when: {
+				state: { "minor-mode": require_computerUseApi.COMPUTER_USE_MODE_ID },
+				attribution: {
+					kind: "minor",
+					mode: require_computerUseApi.COMPUTER_USE_MODE_ID
+				}
+			},
+			event: "session_shutdown",
+			handle: async () => {
+				stopActivity?.();
+				if (client && state?.phase !== "inactive" && state?.phase !== "failed") await client.stop();
+			}
+		}],
+		onDispose() {
+			stopActivity?.();
+			stopActivity = void 0;
+			host.context.client.setStatus(require_computerUseApi.COMPUTER_USE_STATUS_KEY, void 0);
+		}
+	};
+}
+//#endregion
+exports.createComputerUseServer = createComputerUseServer;
+
+//# sourceMappingURL=index.cjs.map