@agimon-ai/po-computer-use 0.0.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +49 -0
- package/dist/apiContracts.cjs +242 -0
- package/dist/apiContracts.cjs.map +1 -0
- package/dist/apiContracts.d.mts +53 -0
- package/dist/apiContracts.d.mts.map +1 -0
- package/dist/apiContracts.mjs +237 -0
- package/dist/apiContracts.mjs.map +1 -0
- package/dist/extensionService-Bp0q295i.cjs +504 -0
- package/dist/extensionService-Bp0q295i.cjs.map +1 -0
- package/dist/extensionService-DXvF7esO.mjs +434 -0
- package/dist/extensionService-DXvF7esO.mjs.map +1 -0
- package/dist/extensions/mcp-ui.mjs +47 -0
- package/dist/extensions/mcp-ui.mjs.map +1 -0
- package/dist/extensions/mcp.cjs +66 -0
- package/dist/extensions/mcp.cjs.map +1 -0
- package/dist/extensions/mcp.d.mts +5 -0
- package/dist/extensions/mcp.d.mts.map +1 -0
- package/dist/extensions/mcp.mjs +61 -0
- package/dist/extensions/mcp.mjs.map +1 -0
- package/dist/extensions/server.cjs +1394 -0
- package/dist/extensions/server.cjs.map +1 -0
- package/dist/extensions/server.d.mts +5 -0
- package/dist/extensions/server.d.mts.map +1 -0
- package/dist/extensions/server.mjs +1388 -0
- package/dist/extensions/server.mjs.map +1 -0
- package/dist/extensions/web.mjs +771 -0
- package/dist/extensions/web.mjs.map +1 -0
- package/dist/index.cjs +140 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.mts +325 -0
- package/dist/index.d.mts.map +1 -0
- package/dist/index.mjs +129 -0
- package/dist/index.mjs.map +1 -0
- package/docs/computer-scripts.md +72 -0
- package/llms.txt +12 -0
- package/package.json +147 -0
- package/src/prompts/po-use-computer-use/SKILL.md +34 -0
- package/src/types/computerUse.ts +132 -0
- package/src/types/computerUseApi.ts +83 -0
|
@@ -0,0 +1,1388 @@
|
|
|
1
|
+
import { a as COMMAND_NAME, i as COMMAND_DESCRIPTION, n as ComputerScriptExecutionError, r as ComputerScriptRunner, t as DefaultComputerUseExtensionService } from "../extensionService-DXvF7esO.mjs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { composeServerProviderOverrides, defineServerPlugin } from "@agimon-ai/po-core/serverFacet";
|
|
4
|
+
import { composeRootHooks, defineChannel, defineRoot, defineRoutedContribution } from "@agimon-ai/po-core/extensionFile";
|
|
5
|
+
import { defineMinorMode, serverMinorModes } from "@agimon-ai/po-minor-mode";
|
|
6
|
+
import { randomUUID } from "node:crypto";
|
|
7
|
+
import { doomApiCallerFrom } from "@agimon-ai/po-core/packageApi";
|
|
8
|
+
import { apiResponse, defineApiRoutes } from "@agimon-ai/po-core/web";
|
|
9
|
+
//#region src/types/computerUseApi.ts
|
|
10
|
+
const API_BASE_PATH = "computer-use";
|
|
11
|
+
const computerUseChannelType = "computer_use_state";
|
|
12
|
+
const COMPUTER_USE_MODE_ID = "computer-use";
|
|
13
|
+
const COMPUTER_USE_STATUS_KEY = "doom-computer-use";
|
|
14
|
+
const COMPUTER_USE_CONFIRMATION_WINDOW_MS = 12e4;
|
|
15
|
+
/**
|
|
16
|
+
* Every path this API answers on, below its mount.
|
|
17
|
+
*
|
|
18
|
+
* The typed route table in `./apiRoutes` is built from this, the dispatcher
|
|
19
|
+
* reads that table, and the hub channel addresses the broker with these same
|
|
20
|
+
* strings, so a route cannot move on one side alone.
|
|
21
|
+
*/
|
|
22
|
+
const COMPUTER_USE_ROUTES = {
|
|
23
|
+
activate: "/activate",
|
|
24
|
+
artifact: "/artifact",
|
|
25
|
+
agentState: "/agent/state",
|
|
26
|
+
agentObserve: "/agent/observe",
|
|
27
|
+
agentAction: "/agent/action",
|
|
28
|
+
agentStop: "/agent/stop",
|
|
29
|
+
hubState: "/hub/state",
|
|
30
|
+
hubActivation: "/hub/activation",
|
|
31
|
+
hubAuthorization: "/hub/authorization",
|
|
32
|
+
hubNext: "/hub/next",
|
|
33
|
+
hubComplete: "/hub/complete",
|
|
34
|
+
hubStop: "/hub/stop"
|
|
35
|
+
};
|
|
36
|
+
const COMPUTER_USE_WAKE_LIMIT = 1e6;
|
|
37
|
+
//#endregion
|
|
38
|
+
//#region src/services/mcpTools/index.ts
|
|
39
|
+
const COMPUTER_USE_MCP_TOOLS_SERVICE = "doom/computer-use-mcp-tools";
|
|
40
|
+
function mountMcpTools(tools) {
|
|
41
|
+
return (context) => {
|
|
42
|
+
context.plugin((providerContext) => {
|
|
43
|
+
providerContext.provide(COMPUTER_USE_MCP_TOOLS_SERVICE, {
|
|
44
|
+
get: (name) => tools.find((tool) => tool.name === name),
|
|
45
|
+
names: () => tools.map((tool) => tool.name)
|
|
46
|
+
});
|
|
47
|
+
});
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
//#endregion
|
|
51
|
+
//#region src/models/computerUseMode.ts
|
|
52
|
+
function modeState(state, enabled = false) {
|
|
53
|
+
const phase = state?.phase ?? "inactive";
|
|
54
|
+
const modeEnabled = enabled || phase !== "inactive" && phase !== "failed";
|
|
55
|
+
return {
|
|
56
|
+
activation: modeEnabled ? phase === "stopping" ? "deactivating" : phase === "awaiting_confirmation" || phase === "activating" ? "activating" : "active" : "inactive",
|
|
57
|
+
condition: phase === "awaiting_confirmation" ? "blocked" : phase === "failed" ? "failed" : phase === "activating" || phase === "stopping" ? "queued" : "ready",
|
|
58
|
+
...modeEnabled && phase === "inactive" ? { detail: "configure computer use in Activity" } : {},
|
|
59
|
+
...phase === "awaiting_confirmation" ? { detail: "awaiting confirmation in Activity" } : {},
|
|
60
|
+
actions: [
|
|
61
|
+
{
|
|
62
|
+
id: "activate",
|
|
63
|
+
enabled: !modeEnabled,
|
|
64
|
+
...!modeEnabled ? {} : { disabledReason: "Computer use is already enabled." }
|
|
65
|
+
},
|
|
66
|
+
{
|
|
67
|
+
id: "deactivate",
|
|
68
|
+
enabled: modeEnabled,
|
|
69
|
+
...modeEnabled ? {} : { disabledReason: "Computer use is not enabled." }
|
|
70
|
+
},
|
|
71
|
+
{
|
|
72
|
+
id: "doctor",
|
|
73
|
+
enabled: true
|
|
74
|
+
}
|
|
75
|
+
]
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
//#endregion
|
|
79
|
+
//#region src/services/computerToolOutput/index.ts
|
|
80
|
+
const MAX_TEXT_BYTES = 262144;
|
|
81
|
+
const MAX_IMAGE_DATA_LENGTH = 8388608;
|
|
82
|
+
function content(value, observation) {
|
|
83
|
+
const text = JSON.stringify(value);
|
|
84
|
+
if (Buffer.byteLength(text) > MAX_TEXT_BYTES) throw new Error("Computer-use text response exceeds the size limit.");
|
|
85
|
+
const screenshot = observation.screenshot;
|
|
86
|
+
if (screenshot && (screenshot.mimeType !== "image/png" || screenshot.data.length > MAX_IMAGE_DATA_LENGTH)) throw new Error("Computer-use screenshot exceeds the image limit.");
|
|
87
|
+
return [{
|
|
88
|
+
type: "text",
|
|
89
|
+
text
|
|
90
|
+
}, ...screenshot ? [{
|
|
91
|
+
type: "image",
|
|
92
|
+
mimeType: screenshot.mimeType,
|
|
93
|
+
data: screenshot.data
|
|
94
|
+
}] : []];
|
|
95
|
+
}
|
|
96
|
+
function computerObservationOutput(observation) {
|
|
97
|
+
return {
|
|
98
|
+
content: content({
|
|
99
|
+
...observation,
|
|
100
|
+
screenshot: void 0
|
|
101
|
+
}, observation),
|
|
102
|
+
details: observation
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
function computerScriptOutput(result) {
|
|
106
|
+
try {
|
|
107
|
+
return {
|
|
108
|
+
content: content({
|
|
109
|
+
...result,
|
|
110
|
+
observation: {
|
|
111
|
+
...result.observation,
|
|
112
|
+
screenshot: void 0
|
|
113
|
+
}
|
|
114
|
+
}, result.observation),
|
|
115
|
+
details: result
|
|
116
|
+
};
|
|
117
|
+
} catch (error) {
|
|
118
|
+
if (result.metrics === void 0) throw error;
|
|
119
|
+
throw new ComputerScriptExecutionError(error instanceof Error ? error.message : String(error), result.metrics.actions, false);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
function computerScriptFailure(error) {
|
|
123
|
+
const details = {
|
|
124
|
+
error: (error instanceof Error ? error.message : String(error)).slice(0, 1024),
|
|
125
|
+
...error instanceof ComputerScriptExecutionError ? {
|
|
126
|
+
completedActions: error.completedActions,
|
|
127
|
+
outcomeUncertain: error.outcomeUncertain
|
|
128
|
+
} : {}
|
|
129
|
+
};
|
|
130
|
+
return {
|
|
131
|
+
content: [{
|
|
132
|
+
type: "text",
|
|
133
|
+
text: JSON.stringify(details)
|
|
134
|
+
}],
|
|
135
|
+
details,
|
|
136
|
+
isError: true
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
//#endregion
|
|
140
|
+
//#region src/types/apiRoutes.ts
|
|
141
|
+
/**
|
|
142
|
+
* This package's routes, as data.
|
|
143
|
+
*
|
|
144
|
+
* No scope and no base path: the build reads both off
|
|
145
|
+
* `src/extensions/workspaces/sessions/(backend)/api/computer-use/`, so the
|
|
146
|
+
* mount is stated once, by the folder that creates it.
|
|
147
|
+
*
|
|
148
|
+
* The paths come from `COMPUTER_USE_ROUTES` rather than being written out
|
|
149
|
+
* again. That table already addressed the broker from the hub channel and from
|
|
150
|
+
* the broker's own dispatcher, and a second copy here is exactly the
|
|
151
|
+
* disagreement this file exists to prevent. The dispatcher now reads this
|
|
152
|
+
* table, so the page, the broker and the hub all resolve one set of strings.
|
|
153
|
+
*
|
|
154
|
+
* Activation, status, cancellation and recording reads are reachable from a browser. The `/agent/` and `/hub/` routes
|
|
155
|
+
* each demand a host-issued bearer token and answer 404 without one; they are
|
|
156
|
+
* declared here because they are this API's surface, not because a page calls
|
|
157
|
+
* them.
|
|
158
|
+
*/
|
|
159
|
+
var apiRoutes_default = defineApiRoutes({
|
|
160
|
+
artifact: {
|
|
161
|
+
method: "GET",
|
|
162
|
+
path: COMPUTER_USE_ROUTES.artifact,
|
|
163
|
+
query: ["artifactId", "offset"],
|
|
164
|
+
raw: true
|
|
165
|
+
},
|
|
166
|
+
activationState: {
|
|
167
|
+
method: "GET",
|
|
168
|
+
path: COMPUTER_USE_ROUTES.activate,
|
|
169
|
+
response: apiResponse()
|
|
170
|
+
},
|
|
171
|
+
cancelActivation: {
|
|
172
|
+
method: "DELETE",
|
|
173
|
+
path: COMPUTER_USE_ROUTES.activate,
|
|
174
|
+
response: apiResponse()
|
|
175
|
+
},
|
|
176
|
+
activate: {
|
|
177
|
+
method: "POST",
|
|
178
|
+
path: COMPUTER_USE_ROUTES.activate,
|
|
179
|
+
response: apiResponse()
|
|
180
|
+
},
|
|
181
|
+
agentState: {
|
|
182
|
+
method: "GET",
|
|
183
|
+
path: COMPUTER_USE_ROUTES.agentState,
|
|
184
|
+
response: apiResponse()
|
|
185
|
+
},
|
|
186
|
+
/** Answers whatever the admitted Desktop provider returned, which is runtime-defined JSON. */
|
|
187
|
+
agentObserve: {
|
|
188
|
+
method: "POST",
|
|
189
|
+
path: COMPUTER_USE_ROUTES.agentObserve,
|
|
190
|
+
response: apiResponse()
|
|
191
|
+
},
|
|
192
|
+
agentAction: {
|
|
193
|
+
method: "POST",
|
|
194
|
+
path: COMPUTER_USE_ROUTES.agentAction,
|
|
195
|
+
response: apiResponse()
|
|
196
|
+
},
|
|
197
|
+
agentStop: {
|
|
198
|
+
method: "POST",
|
|
199
|
+
path: COMPUTER_USE_ROUTES.agentStop,
|
|
200
|
+
response: apiResponse()
|
|
201
|
+
},
|
|
202
|
+
hubState: {
|
|
203
|
+
method: "GET",
|
|
204
|
+
path: COMPUTER_USE_ROUTES.hubState,
|
|
205
|
+
response: apiResponse()
|
|
206
|
+
},
|
|
207
|
+
/** Null until a reader has confirmed one; reading it moves the session to `activating`. */
|
|
208
|
+
hubActivation: {
|
|
209
|
+
method: "GET",
|
|
210
|
+
path: COMPUTER_USE_ROUTES.hubActivation,
|
|
211
|
+
response: apiResponse()
|
|
212
|
+
},
|
|
213
|
+
hubAuthorization: {
|
|
214
|
+
method: "GET",
|
|
215
|
+
path: COMPUTER_USE_ROUTES.hubAuthorization,
|
|
216
|
+
response: apiResponse()
|
|
217
|
+
},
|
|
218
|
+
hubNext: {
|
|
219
|
+
method: "GET",
|
|
220
|
+
path: COMPUTER_USE_ROUTES.hubNext,
|
|
221
|
+
response: apiResponse()
|
|
222
|
+
},
|
|
223
|
+
hubComplete: {
|
|
224
|
+
method: "POST",
|
|
225
|
+
path: COMPUTER_USE_ROUTES.hubComplete,
|
|
226
|
+
response: apiResponse()
|
|
227
|
+
},
|
|
228
|
+
hubStop: {
|
|
229
|
+
method: "POST",
|
|
230
|
+
path: COMPUTER_USE_ROUTES.hubStop,
|
|
231
|
+
response: apiResponse()
|
|
232
|
+
}
|
|
233
|
+
});
|
|
234
|
+
//#endregion
|
|
235
|
+
//#region src/services/computerUseApi/index.ts
|
|
236
|
+
function jsonError(message, status) {
|
|
237
|
+
return Response.json({ error: message }, { status });
|
|
238
|
+
}
|
|
239
|
+
function record$1(value) {
|
|
240
|
+
return typeof value === "object" && value !== null && !Array.isArray(value) ? value : void 0;
|
|
241
|
+
}
|
|
242
|
+
async function body(request) {
|
|
243
|
+
try {
|
|
244
|
+
return record$1(await request.json()) ?? {};
|
|
245
|
+
} catch {
|
|
246
|
+
return {};
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
function semanticAction(value) {
|
|
250
|
+
if (!(typeof value.snapshotId === "string" && value.snapshotId.length > 0 && typeof value.elementRef === "string" && value.elementRef.length > 0)) return false;
|
|
251
|
+
const allowed = (keys) => Object.keys(value).every((key) => keys.includes(key));
|
|
252
|
+
if (value.kind === "press" || value.kind === "focus") return allowed([
|
|
253
|
+
"kind",
|
|
254
|
+
"snapshotId",
|
|
255
|
+
"elementRef"
|
|
256
|
+
]);
|
|
257
|
+
if (value.kind === "set_value") return typeof value.value === "string" && allowed([
|
|
258
|
+
"kind",
|
|
259
|
+
"snapshotId",
|
|
260
|
+
"elementRef",
|
|
261
|
+
"value"
|
|
262
|
+
]);
|
|
263
|
+
if (value.kind === "scroll") return [
|
|
264
|
+
"up",
|
|
265
|
+
"down",
|
|
266
|
+
"left",
|
|
267
|
+
"right"
|
|
268
|
+
].includes(value.direction) && (value.amount === "line" || value.amount === "page") && allowed([
|
|
269
|
+
"kind",
|
|
270
|
+
"snapshotId",
|
|
271
|
+
"elementRef",
|
|
272
|
+
"direction",
|
|
273
|
+
"amount"
|
|
274
|
+
]);
|
|
275
|
+
return false;
|
|
276
|
+
}
|
|
277
|
+
function artifactView(value) {
|
|
278
|
+
const input = record$1(value);
|
|
279
|
+
if (typeof input?.artifactId !== "string" || input.status !== "ready" && input.status !== "failed") return void 0;
|
|
280
|
+
const safeUrl = (candidate) => typeof candidate === "string" && candidate.startsWith("/api/") ? candidate : void 0;
|
|
281
|
+
const downloadUrl = safeUrl(input.downloadUrl);
|
|
282
|
+
const previewUrl = safeUrl(input.previewUrl);
|
|
283
|
+
const artifactFailure = record$1(input.failure);
|
|
284
|
+
const safeFailure = typeof artifactFailure?.code === "string" && typeof artifactFailure.message === "string" ? {
|
|
285
|
+
code: artifactFailure.code.slice(0, 128),
|
|
286
|
+
message: artifactFailure.message.slice(0, 512)
|
|
287
|
+
} : void 0;
|
|
288
|
+
return {
|
|
289
|
+
artifactId: input.artifactId,
|
|
290
|
+
status: input.status,
|
|
291
|
+
...typeof input.sizeBytes === "number" && Number.isSafeInteger(input.sizeBytes) && input.sizeBytes > 0 ? { sizeBytes: input.sizeBytes } : {},
|
|
292
|
+
...downloadUrl === void 0 ? {} : { downloadUrl },
|
|
293
|
+
...previewUrl === void 0 ? {} : { previewUrl },
|
|
294
|
+
...typeof input.actionCount === "number" ? { actionCount: input.actionCount } : {},
|
|
295
|
+
...typeof input.completedAt === "string" ? { completedAt: input.completedAt } : {},
|
|
296
|
+
...safeFailure === void 0 ? {} : { failure: safeFailure }
|
|
297
|
+
};
|
|
298
|
+
}
|
|
299
|
+
var ComputerUseRequestBroker = class ComputerUseRequestBroker {
|
|
300
|
+
sessionId;
|
|
301
|
+
internalToken;
|
|
302
|
+
hubToken;
|
|
303
|
+
directEvents;
|
|
304
|
+
requestTimeoutMs;
|
|
305
|
+
revision = 0;
|
|
306
|
+
wake = 0;
|
|
307
|
+
phase = "inactive";
|
|
308
|
+
activation;
|
|
309
|
+
grantId;
|
|
310
|
+
expiresAt;
|
|
311
|
+
actionSequence = 0;
|
|
312
|
+
expiryTimer;
|
|
313
|
+
failure;
|
|
314
|
+
artifact;
|
|
315
|
+
pending;
|
|
316
|
+
closed = false;
|
|
317
|
+
desktop;
|
|
318
|
+
beforeActivate;
|
|
319
|
+
listeners = /* @__PURE__ */ new Set();
|
|
320
|
+
unsubscribeDesktop;
|
|
321
|
+
static DEFAULT_REQUEST_TIMEOUT_MS = 3e4;
|
|
322
|
+
constructor(options) {
|
|
323
|
+
this.sessionId = options.sessionId ?? "unknown";
|
|
324
|
+
this.internalToken = options.internalToken;
|
|
325
|
+
this.hubToken = options.hubToken;
|
|
326
|
+
this.directEvents = options.directEvents;
|
|
327
|
+
this.requestTimeoutMs = options.requestTimeoutMs ?? ComputerUseRequestBroker.DEFAULT_REQUEST_TIMEOUT_MS;
|
|
328
|
+
this.desktop = options.desktop;
|
|
329
|
+
this.beforeActivate = options.beforeActivate;
|
|
330
|
+
this.unsubscribeDesktop = options.desktop?.subscribe(() => {
|
|
331
|
+
clearTimeout(this.expiryTimer);
|
|
332
|
+
if (!options.desktop?.available) {
|
|
333
|
+
this.rejectPending("Desktop disconnected.");
|
|
334
|
+
this.phase = "failed";
|
|
335
|
+
this.grantId = void 0;
|
|
336
|
+
this.failure = {
|
|
337
|
+
code: "desktop_unavailable",
|
|
338
|
+
message: "Desktop disconnected."
|
|
339
|
+
};
|
|
340
|
+
} else if (this.phase !== "inactive" && this.phase !== "failed") {
|
|
341
|
+
this.phase = "stopping";
|
|
342
|
+
this.rejectPending("Control was stopped by Desktop.");
|
|
343
|
+
}
|
|
344
|
+
this.changed();
|
|
345
|
+
});
|
|
346
|
+
}
|
|
347
|
+
state() {
|
|
348
|
+
return Object.freeze({
|
|
349
|
+
sessionId: this.sessionId,
|
|
350
|
+
revision: this.revision,
|
|
351
|
+
wake: this.wake,
|
|
352
|
+
phase: this.phase,
|
|
353
|
+
...this.activation === void 0 ? {} : {
|
|
354
|
+
requestId: this.activation.requestId,
|
|
355
|
+
target: this.activation.target,
|
|
356
|
+
durationMs: this.activation.durationSeconds * 1e3
|
|
357
|
+
},
|
|
358
|
+
...this.expiresAt === void 0 ? {} : { expiresAt: this.expiresAt },
|
|
359
|
+
...this.failure === void 0 ? {} : { failure: this.failure },
|
|
360
|
+
...this.artifact === void 0 ? {} : { artifact: this.artifact }
|
|
361
|
+
});
|
|
362
|
+
}
|
|
363
|
+
/** Tools and the API share this broker. No socket, credential discovery, or second state store. */
|
|
364
|
+
sessionClient() {
|
|
365
|
+
const result = async (response) => {
|
|
366
|
+
const value = await response.json();
|
|
367
|
+
if (!response.ok) throw new Error(value.error ?? `Computer-use request failed (${response.status}).`);
|
|
368
|
+
return value;
|
|
369
|
+
};
|
|
370
|
+
const requireDesktop = () => {
|
|
371
|
+
if (this.closed || !this.desktop?.available || this.desktop.enabled === false) throw new Error("Desktop computer use is unavailable.");
|
|
372
|
+
};
|
|
373
|
+
return {
|
|
374
|
+
state: async () => this.state(),
|
|
375
|
+
subscribeStatus: (listener) => {
|
|
376
|
+
this.listeners.add(listener);
|
|
377
|
+
return () => {
|
|
378
|
+
this.listeners.delete(listener);
|
|
379
|
+
};
|
|
380
|
+
},
|
|
381
|
+
observe: async (signal, options = {}) => {
|
|
382
|
+
requireDesktop();
|
|
383
|
+
return result(await this.enqueue("observe", options, signal));
|
|
384
|
+
},
|
|
385
|
+
act: async (action, signal) => {
|
|
386
|
+
requireDesktop();
|
|
387
|
+
if (!semanticAction(action)) throw new Error("A valid semantic action is required.");
|
|
388
|
+
return result(await this.enqueue("act", action, signal));
|
|
389
|
+
},
|
|
390
|
+
stop: async (signal) => {
|
|
391
|
+
signal?.throwIfAborted();
|
|
392
|
+
if (this.phase !== "inactive" && this.phase !== "failed") {
|
|
393
|
+
this.phase = "stopping";
|
|
394
|
+
this.rejectPending("Computer use was stopped.");
|
|
395
|
+
this.changed();
|
|
396
|
+
}
|
|
397
|
+
return this.state();
|
|
398
|
+
}
|
|
399
|
+
};
|
|
400
|
+
}
|
|
401
|
+
changed() {
|
|
402
|
+
this.revision += 1;
|
|
403
|
+
this.wake = (this.wake + 1) % COMPUTER_USE_WAKE_LIMIT;
|
|
404
|
+
const state = this.state();
|
|
405
|
+
for (const listener of this.listeners) listener(state);
|
|
406
|
+
this.directEvents.publish(computerUseChannelType, this.sessionId, state);
|
|
407
|
+
}
|
|
408
|
+
authorized(request, token) {
|
|
409
|
+
return token !== void 0 && request.headers.get("authorization") === `Bearer ${token}`;
|
|
410
|
+
}
|
|
411
|
+
async requestActivation(request) {
|
|
412
|
+
if (!this.desktop?.available || !(this.desktop.authorizeActivation?.(request.headers) ?? this.desktop.authorize(request.headers))) return jsonError("Not found.", 404);
|
|
413
|
+
if (this.desktop.enabled === false) return jsonError("Enable computer use in global Desktop settings first.", 409);
|
|
414
|
+
const stamped = doomApiCallerFrom(request.headers);
|
|
415
|
+
const caller = stamped?.locality === "remote" ? stamped : {
|
|
416
|
+
locality: "local",
|
|
417
|
+
stepUp: "not-required"
|
|
418
|
+
};
|
|
419
|
+
if (this.phase !== "inactive" && this.phase !== "failed") return jsonError("Computer use is already busy.", 409);
|
|
420
|
+
const input = await body(request);
|
|
421
|
+
const target = record$1(input.target);
|
|
422
|
+
const durationMs = input.durationMs;
|
|
423
|
+
if (target === void 0 && (input.target !== void 0 || caller.locality !== "remote") || target !== void 0 && (typeof target.windowId !== "string" || typeof target.bundleId !== "string")) return jsonError("A Desktop target is required, or a remote request must leave selection to the Mac.", 400);
|
|
424
|
+
if (typeof durationMs !== "number" || !Number.isInteger(durationMs) || durationMs < 1e3 || durationMs > 18e5) return jsonError("The requested duration is invalid.", 400);
|
|
425
|
+
try {
|
|
426
|
+
this.desktop.claim(request.headers);
|
|
427
|
+
await this.beforeActivate?.();
|
|
428
|
+
} catch (error) {
|
|
429
|
+
return jsonError(error instanceof Error ? error.message : String(error), 409);
|
|
430
|
+
}
|
|
431
|
+
if (this.closed || !this.desktop.available || this.phase !== "inactive" && this.phase !== "failed") return jsonError("Computer use is unavailable or already busy.", 409);
|
|
432
|
+
const createdAt = Date.now();
|
|
433
|
+
this.activation = Object.freeze({
|
|
434
|
+
requestId: randomUUID(),
|
|
435
|
+
...target === void 0 ? {} : { target: Object.freeze({ ...target }) },
|
|
436
|
+
durationSeconds: durationMs / 1e3,
|
|
437
|
+
createdAt,
|
|
438
|
+
confirmationExpiresAt: createdAt + COMPUTER_USE_CONFIRMATION_WINDOW_MS,
|
|
439
|
+
caller: Object.freeze({ ...caller })
|
|
440
|
+
});
|
|
441
|
+
this.phase = "awaiting_confirmation";
|
|
442
|
+
this.failure = void 0;
|
|
443
|
+
this.artifact = void 0;
|
|
444
|
+
this.changed();
|
|
445
|
+
return Response.json(this.state(), { status: 202 });
|
|
446
|
+
}
|
|
447
|
+
rejectPending(message) {
|
|
448
|
+
const pending = this.pending;
|
|
449
|
+
if (pending === void 0) return;
|
|
450
|
+
this.pending = void 0;
|
|
451
|
+
pending.dispose();
|
|
452
|
+
pending.reject(new Error(message));
|
|
453
|
+
}
|
|
454
|
+
stopAfterUncertainRequest(message) {
|
|
455
|
+
if (this.phase === "active") {
|
|
456
|
+
this.phase = "stopping";
|
|
457
|
+
this.changed();
|
|
458
|
+
}
|
|
459
|
+
this.rejectPending(message);
|
|
460
|
+
}
|
|
461
|
+
async enqueue(operation, payload, signal) {
|
|
462
|
+
if (this.phase !== "active" || this.grantId === void 0) return jsonError("Computer use is not active.", 409);
|
|
463
|
+
if (this.pending !== void 0) return jsonError("Another computer-use request is live.", 409);
|
|
464
|
+
if (signal?.aborted === true) return jsonError("The computer-use request was cancelled.", 499);
|
|
465
|
+
const sequence = operation === "act" ? ++this.actionSequence : void 0;
|
|
466
|
+
const request = {
|
|
467
|
+
id: randomUUID(),
|
|
468
|
+
operation,
|
|
469
|
+
grantId: this.grantId,
|
|
470
|
+
...sequence === void 0 ? {} : { sequence },
|
|
471
|
+
...payload === void 0 ? {} : { payload }
|
|
472
|
+
};
|
|
473
|
+
const completion = new Promise((resolve, reject) => {
|
|
474
|
+
const onAbort = () => this.stopAfterUncertainRequest("The computer-use request was cancelled.");
|
|
475
|
+
const timer = setTimeout(() => this.stopAfterUncertainRequest("The computer-use request timed out."), this.requestTimeoutMs);
|
|
476
|
+
timer.unref?.();
|
|
477
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
478
|
+
this.pending = {
|
|
479
|
+
request,
|
|
480
|
+
resolve,
|
|
481
|
+
reject,
|
|
482
|
+
dispose: () => {
|
|
483
|
+
clearTimeout(timer);
|
|
484
|
+
signal?.removeEventListener("abort", onAbort);
|
|
485
|
+
}
|
|
486
|
+
};
|
|
487
|
+
});
|
|
488
|
+
this.changed();
|
|
489
|
+
try {
|
|
490
|
+
return Response.json(await completion);
|
|
491
|
+
} catch (error) {
|
|
492
|
+
return jsonError(error instanceof Error ? error.message : String(error), 502);
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
async fetch(request) {
|
|
496
|
+
if (this.closed) return jsonError("Computer-use broker is closed.", 503);
|
|
497
|
+
const url = new URL(request.url);
|
|
498
|
+
const path = url.pathname;
|
|
499
|
+
if (request.method === "GET" && path === apiRoutes_default.artifact.path) {
|
|
500
|
+
if (!(this.desktop?.authorizeRecording?.(request.headers) ?? this.desktop?.authorize(request.headers)) || !this.desktop?.fetchRecording) return jsonError("Not found.", 404);
|
|
501
|
+
const artifactId = url.searchParams.get("artifactId") ?? "";
|
|
502
|
+
const offsetText = url.searchParams.get("offset") ?? "";
|
|
503
|
+
const offset = Number(offsetText);
|
|
504
|
+
if (!/^[0-9]+$/u.test(offsetText) || !Number.isSafeInteger(offset) || offset < 0 || offset > 134217728) return jsonError("Invalid recording offset.", 400);
|
|
505
|
+
return this.desktop.fetchRecording(artifactId, `bytes=${offset}-${offset + 1048576 - 1}`);
|
|
506
|
+
}
|
|
507
|
+
if (path === apiRoutes_default.activate.path && request.method === "GET") {
|
|
508
|
+
if (!this.desktop?.authorizePending?.(request.headers) && !(this.desktop?.authorizeRecording?.(request.headers) ?? this.desktop?.authorize(request.headers))) return jsonError("Not found.", 404);
|
|
509
|
+
return Response.json(this.state());
|
|
510
|
+
}
|
|
511
|
+
if (path === apiRoutes_default.activate.path && request.method === "DELETE") {
|
|
512
|
+
if (!this.desktop?.authorizePending?.(request.headers)) return jsonError("Not found.", 404);
|
|
513
|
+
if (this.phase !== "awaiting_confirmation" && this.phase !== "activating") return jsonError("No activation is pending.", 409);
|
|
514
|
+
this.phase = "stopping";
|
|
515
|
+
this.rejectPending("The activation was cancelled.");
|
|
516
|
+
this.changed();
|
|
517
|
+
return Response.json(this.state(), { status: 202 });
|
|
518
|
+
}
|
|
519
|
+
if (request.method === "POST" && path === apiRoutes_default.activate.path) return this.requestActivation(request);
|
|
520
|
+
const agent = path.startsWith("/agent/");
|
|
521
|
+
const hub = path.startsWith("/hub/");
|
|
522
|
+
if (agent && !this.authorized(request, this.internalToken) || hub && !this.authorized(request, this.hubToken)) return jsonError("Not found.", 404);
|
|
523
|
+
if (request.method === "GET" && (path === apiRoutes_default.agentState.path || path === apiRoutes_default.hubState.path)) return Response.json(this.state());
|
|
524
|
+
if (request.method === "POST" && path === apiRoutes_default.agentObserve.path) return this.enqueue("observe", void 0, request.signal);
|
|
525
|
+
if (request.method === "POST" && path === apiRoutes_default.agentAction.path) {
|
|
526
|
+
const action = await body(request);
|
|
527
|
+
if (!semanticAction(action)) return jsonError("A valid semantic action is required.", 400);
|
|
528
|
+
return this.enqueue("act", action, request.signal);
|
|
529
|
+
}
|
|
530
|
+
if (request.method === "POST" && path === apiRoutes_default.agentStop.path) {
|
|
531
|
+
if (this.phase !== "active") return jsonError("Computer use is not active.", 409);
|
|
532
|
+
this.phase = "stopping";
|
|
533
|
+
this.rejectPending("The computer-use request was stopped.");
|
|
534
|
+
this.changed();
|
|
535
|
+
return Response.json(this.state(), { status: 202 });
|
|
536
|
+
}
|
|
537
|
+
if (request.method === "GET" && path === apiRoutes_default.hubActivation.path) {
|
|
538
|
+
if (this.activation === void 0 || this.phase !== "awaiting_confirmation") return Response.json(null);
|
|
539
|
+
this.phase = "activating";
|
|
540
|
+
this.changed();
|
|
541
|
+
return Response.json(this.activation);
|
|
542
|
+
}
|
|
543
|
+
if (request.method === "GET" && path === apiRoutes_default.hubAuthorization.path) return Response.json(this.grantId === void 0 ? null : {
|
|
544
|
+
grantId: this.grantId,
|
|
545
|
+
expiresAt: this.expiresAt
|
|
546
|
+
});
|
|
547
|
+
if (request.method === "GET" && path === apiRoutes_default.hubNext.path) return Response.json(this.pending?.request ?? null);
|
|
548
|
+
if (request.method === "POST" && path === apiRoutes_default.hubComplete.path) {
|
|
549
|
+
const input = await body(request);
|
|
550
|
+
const id = typeof input.id === "string" ? input.id : void 0;
|
|
551
|
+
if (this.pending === void 0 || this.pending.request.id !== id) return jsonError("The request is stale.", 409);
|
|
552
|
+
const pending = this.pending;
|
|
553
|
+
this.pending = void 0;
|
|
554
|
+
pending.dispose();
|
|
555
|
+
if (typeof input.error === "string") {
|
|
556
|
+
this.phase = "stopping";
|
|
557
|
+
pending.reject(new Error(input.error));
|
|
558
|
+
} else pending.resolve(input.result);
|
|
559
|
+
this.changed();
|
|
560
|
+
return Response.json(this.state());
|
|
561
|
+
}
|
|
562
|
+
if (request.method === "POST" && path === apiRoutes_default.hubStop.path) {
|
|
563
|
+
const input = await body(request);
|
|
564
|
+
if (this.phase === "activating") {
|
|
565
|
+
if (typeof input.error === "string") {
|
|
566
|
+
this.phase = "failed";
|
|
567
|
+
this.failure = {
|
|
568
|
+
code: "desktop_unavailable",
|
|
569
|
+
message: input.error.slice(0, 512)
|
|
570
|
+
};
|
|
571
|
+
this.activation = void 0;
|
|
572
|
+
} else {
|
|
573
|
+
const host = record$1(input.host);
|
|
574
|
+
if (typeof host?.grantId !== "string" || !Number.isFinite(host.expiresAt) || host.expiresAt <= Date.now()) return jsonError("Desktop did not issue a complete grant.", 502);
|
|
575
|
+
const selectedTarget = record$1(host.target);
|
|
576
|
+
if (this.activation !== void 0 && selectedTarget !== void 0) this.activation = Object.freeze({
|
|
577
|
+
...this.activation,
|
|
578
|
+
target: Object.freeze({ ...selectedTarget })
|
|
579
|
+
});
|
|
580
|
+
this.grantId = host.grantId;
|
|
581
|
+
this.expiresAt = host.expiresAt;
|
|
582
|
+
this.actionSequence = 0;
|
|
583
|
+
this.phase = "active";
|
|
584
|
+
clearTimeout(this.expiryTimer);
|
|
585
|
+
this.expiryTimer = setTimeout(() => {
|
|
586
|
+
if (this.closed || this.phase !== "active") return;
|
|
587
|
+
this.phase = "stopping";
|
|
588
|
+
this.rejectPending("The computer-use grant expired.");
|
|
589
|
+
this.changed();
|
|
590
|
+
}, Math.max(0, this.expiresAt - Date.now()));
|
|
591
|
+
this.expiryTimer.unref?.();
|
|
592
|
+
}
|
|
593
|
+
} else {
|
|
594
|
+
clearTimeout(this.expiryTimer);
|
|
595
|
+
this.rejectPending("The computer-use session stopped.");
|
|
596
|
+
this.phase = "inactive";
|
|
597
|
+
this.grantId = void 0;
|
|
598
|
+
this.expiresAt = void 0;
|
|
599
|
+
this.activation = void 0;
|
|
600
|
+
this.artifact = artifactView(input.artifact) ?? this.artifact;
|
|
601
|
+
}
|
|
602
|
+
this.changed();
|
|
603
|
+
return Response.json(this.state());
|
|
604
|
+
}
|
|
605
|
+
return jsonError("Not found.", 404);
|
|
606
|
+
}
|
|
607
|
+
close() {
|
|
608
|
+
if (this.closed) return;
|
|
609
|
+
this.closed = true;
|
|
610
|
+
clearTimeout(this.expiryTimer);
|
|
611
|
+
this.unsubscribeDesktop?.();
|
|
612
|
+
this.listeners.clear();
|
|
613
|
+
this.rejectPending("Computer-use broker closed.");
|
|
614
|
+
this.grantId = void 0;
|
|
615
|
+
this.expiresAt = void 0;
|
|
616
|
+
}
|
|
617
|
+
};
|
|
618
|
+
function createComputerUseApi(options) {
|
|
619
|
+
return new ComputerUseRequestBroker(options);
|
|
620
|
+
}
|
|
621
|
+
//#endregion
|
|
622
|
+
//#region src/services/computerUseServer/index.ts
|
|
623
|
+
/** This package's identity for mode, activity and restriction ownership, not a status key. */
|
|
624
|
+
const SOURCE = "@agimon-ai/po-computer-use";
|
|
625
|
+
const SCRIPT_PATHS_ENV = "PO_COMPUTER_USE_SCRIPT_PATHS";
|
|
626
|
+
const GUIDANCE = "[COMPUTER USE ACTIVE]\nUse computer_state before acting and after any action that may change the interface. Use only element refs and snapshot ids returned by computer_state. Never infer coordinates or bypass secure elements.";
|
|
627
|
+
function output(value, isError = false) {
|
|
628
|
+
return {
|
|
629
|
+
content: [{
|
|
630
|
+
type: "text",
|
|
631
|
+
text: typeof value === "string" ? value : JSON.stringify(value, null, 2)
|
|
632
|
+
}],
|
|
633
|
+
details: value,
|
|
634
|
+
...isError ? { isError: true } : {}
|
|
635
|
+
};
|
|
636
|
+
}
|
|
637
|
+
function createComputerUseServer(host, apiContext) {
|
|
638
|
+
if (!apiContext?.computerUse?.available || !apiContext.directEvents) return {
|
|
639
|
+
tools: [],
|
|
640
|
+
commands: []
|
|
641
|
+
};
|
|
642
|
+
const desktop = apiContext.computerUse;
|
|
643
|
+
const broker = createComputerUseApi({
|
|
644
|
+
sessionId: host.context.sessionId,
|
|
645
|
+
internalToken: apiContext.internalToken,
|
|
646
|
+
hubToken: apiContext.hubToken,
|
|
647
|
+
directEvents: apiContext.directEvents,
|
|
648
|
+
desktop,
|
|
649
|
+
beforeActivate: async () => {
|
|
650
|
+
if (!modeSelected()) throw new Error("Enable computer use before authorizing an application.");
|
|
651
|
+
const activity = await host.context.session.activity();
|
|
652
|
+
if (!activity.isIdle || activity.hasPendingMessages) throw new Error("Wait for the agent and its queued prompts to finish before granting computer use.");
|
|
653
|
+
}
|
|
654
|
+
});
|
|
655
|
+
const client = broker.sessionClient();
|
|
656
|
+
const allowedScriptPaths = (host.context.environment[SCRIPT_PATHS_ENV] ?? "").split(path.delimiter).filter((entry) => entry.length > 0);
|
|
657
|
+
const dependencies = {
|
|
658
|
+
client,
|
|
659
|
+
scriptRunner: new ComputerScriptRunner({
|
|
660
|
+
client,
|
|
661
|
+
allowedScriptPaths,
|
|
662
|
+
scriptRoot: host.context.cwd
|
|
663
|
+
}),
|
|
664
|
+
service: new DefaultComputerUseExtensionService(client)
|
|
665
|
+
};
|
|
666
|
+
let state = broker.state();
|
|
667
|
+
const restrictionListeners = /* @__PURE__ */ new Set();
|
|
668
|
+
let stopActivity;
|
|
669
|
+
let modeOwner;
|
|
670
|
+
const modeSelected = () => (host.context.selection.state?.["minor-mode"] ?? []).includes(COMPUTER_USE_MODE_ID);
|
|
671
|
+
const publishMode = () => modeOwner?.publish();
|
|
672
|
+
const applyState = (next) => {
|
|
673
|
+
state = next;
|
|
674
|
+
host.context.client.setStatus(COMPUTER_USE_STATUS_KEY, next === void 0 || next.phase === "inactive" ? void 0 : `computer use: ${next.phase}`);
|
|
675
|
+
publishMode();
|
|
676
|
+
for (const listener of restrictionListeners) listener();
|
|
677
|
+
};
|
|
678
|
+
const selectMode = async (enabled) => {
|
|
679
|
+
const modes = (host.context.selection.state?.["minor-mode"] ?? []).filter((mode) => mode !== COMPUTER_USE_MODE_ID);
|
|
680
|
+
await host.changeSelection({
|
|
681
|
+
axis: "state",
|
|
682
|
+
key: "minor-mode",
|
|
683
|
+
values: enabled ? [...modes, COMPUTER_USE_MODE_ID] : modes
|
|
684
|
+
});
|
|
685
|
+
publishMode();
|
|
686
|
+
};
|
|
687
|
+
const refresh = async (signal) => {
|
|
688
|
+
if (!client) {
|
|
689
|
+
applyState(void 0);
|
|
690
|
+
return;
|
|
691
|
+
}
|
|
692
|
+
try {
|
|
693
|
+
applyState(await client.state(signal));
|
|
694
|
+
} catch {
|
|
695
|
+
applyState(void 0);
|
|
696
|
+
}
|
|
697
|
+
};
|
|
698
|
+
const requireActive = () => {
|
|
699
|
+
if (!desktop.available || desktop.enabled === false || !modeSelected() || state?.phase !== "active") throw new Error("Computer use is not active for this session.");
|
|
700
|
+
return client;
|
|
701
|
+
};
|
|
702
|
+
modeOwner = defineMinorMode({
|
|
703
|
+
descriptor: {
|
|
704
|
+
source: SOURCE,
|
|
705
|
+
id: COMPUTER_USE_MODE_ID,
|
|
706
|
+
label: "Computer Use",
|
|
707
|
+
description: "Session-scoped semantic control through Po Desktop.",
|
|
708
|
+
order: 450,
|
|
709
|
+
actions: [
|
|
710
|
+
{
|
|
711
|
+
id: "activate",
|
|
712
|
+
label: "Activate",
|
|
713
|
+
description: "Enable computer use for this session.",
|
|
714
|
+
contexts: ["headless"],
|
|
715
|
+
parameters: []
|
|
716
|
+
},
|
|
717
|
+
{
|
|
718
|
+
id: "deactivate",
|
|
719
|
+
label: "Deactivate",
|
|
720
|
+
description: "Stop this session computer-use run.",
|
|
721
|
+
contexts: ["headless"],
|
|
722
|
+
parameters: []
|
|
723
|
+
},
|
|
724
|
+
{
|
|
725
|
+
id: "doctor",
|
|
726
|
+
label: "Doctor",
|
|
727
|
+
description: "Report Desktop availability and session state.",
|
|
728
|
+
contexts: ["headless"],
|
|
729
|
+
parameters: []
|
|
730
|
+
}
|
|
731
|
+
]
|
|
732
|
+
},
|
|
733
|
+
state: () => {
|
|
734
|
+
const mode = modeState(state, modeSelected());
|
|
735
|
+
return desktop.enabled === false ? {
|
|
736
|
+
...mode,
|
|
737
|
+
condition: "blocked",
|
|
738
|
+
detail: "Enable computer use in global Desktop settings.",
|
|
739
|
+
actions: mode.actions.map((action) => ({
|
|
740
|
+
...action,
|
|
741
|
+
enabled: false,
|
|
742
|
+
disabledReason: "Enable computer use in global Desktop settings."
|
|
743
|
+
}))
|
|
744
|
+
} : mode;
|
|
745
|
+
},
|
|
746
|
+
async handleAction(_runtime, actionId, _argumentsValue, execution) {
|
|
747
|
+
execution.signal.throwIfAborted();
|
|
748
|
+
if (actionId === "activate") {
|
|
749
|
+
if (!desktop.available || desktop.enabled === false) throw new Error("Enable computer use in global Desktop settings first.");
|
|
750
|
+
await selectMode(true);
|
|
751
|
+
await refresh(execution.signal);
|
|
752
|
+
return { message: "Computer use is ready to configure in Activity." };
|
|
753
|
+
}
|
|
754
|
+
if (actionId === "doctor") {
|
|
755
|
+
await refresh(execution.signal);
|
|
756
|
+
return { message: client ? `Computer use is ${state?.phase ?? "unavailable"}.` : "Po Desktop session API is unavailable." };
|
|
757
|
+
}
|
|
758
|
+
if (actionId === "deactivate") {
|
|
759
|
+
if (client && state?.phase !== void 0 && state.phase !== "inactive" && state.phase !== "failed") await client.stop(execution.signal);
|
|
760
|
+
await selectMode(false);
|
|
761
|
+
await refresh(execution.signal);
|
|
762
|
+
return { message: "Computer use deactivated." };
|
|
763
|
+
}
|
|
764
|
+
throw new Error(`Unknown computer-use action: ${actionId}`);
|
|
765
|
+
}
|
|
766
|
+
}).createOwner(void 0);
|
|
767
|
+
const unsubscribeState = client.subscribeStatus?.(applyState);
|
|
768
|
+
return {
|
|
769
|
+
api: [{
|
|
770
|
+
basePath: API_BASE_PATH,
|
|
771
|
+
start: () => broker
|
|
772
|
+
}],
|
|
773
|
+
services: [(context) => {
|
|
774
|
+
const fiber = context.plugin(serverMinorModes([modeOwner]));
|
|
775
|
+
context.effect(() => desktop.subscribe(() => {
|
|
776
|
+
if (!desktop.available) fiber.dispose();
|
|
777
|
+
}));
|
|
778
|
+
}],
|
|
779
|
+
toolRestrictions: [{
|
|
780
|
+
source: SOURCE,
|
|
781
|
+
restrict: () => ({ excludedTools: desktop.available && desktop.enabled !== false && modeSelected() && state?.phase === "active" ? [] : [
|
|
782
|
+
"computer_state",
|
|
783
|
+
"computer_action",
|
|
784
|
+
"computer_exec"
|
|
785
|
+
] }),
|
|
786
|
+
subscribe: (listener) => {
|
|
787
|
+
restrictionListeners.add(listener);
|
|
788
|
+
return () => {
|
|
789
|
+
restrictionListeners.delete(listener);
|
|
790
|
+
};
|
|
791
|
+
}
|
|
792
|
+
}],
|
|
793
|
+
activities: [{
|
|
794
|
+
when: {
|
|
795
|
+
state: { "minor-mode": COMPUTER_USE_MODE_ID },
|
|
796
|
+
attribution: {
|
|
797
|
+
kind: "minor",
|
|
798
|
+
mode: COMPUTER_USE_MODE_ID
|
|
799
|
+
}
|
|
800
|
+
},
|
|
801
|
+
name: SOURCE,
|
|
802
|
+
async start() {
|
|
803
|
+
await refresh();
|
|
804
|
+
stopActivity = () => {
|
|
805
|
+
client.stop();
|
|
806
|
+
};
|
|
807
|
+
return () => {
|
|
808
|
+
stopActivity?.();
|
|
809
|
+
stopActivity = void 0;
|
|
810
|
+
};
|
|
811
|
+
}
|
|
812
|
+
}],
|
|
813
|
+
tools: [
|
|
814
|
+
{
|
|
815
|
+
when: {
|
|
816
|
+
state: { "minor-mode": COMPUTER_USE_MODE_ID },
|
|
817
|
+
attribution: {
|
|
818
|
+
kind: "minor",
|
|
819
|
+
mode: COMPUTER_USE_MODE_ID
|
|
820
|
+
}
|
|
821
|
+
},
|
|
822
|
+
name: "computer_state",
|
|
823
|
+
label: "Computer State",
|
|
824
|
+
description: "Observe the authorized application window and return semantic accessibility state.",
|
|
825
|
+
parameters: {
|
|
826
|
+
type: "object",
|
|
827
|
+
properties: { includeScreenshot: { type: "boolean" } },
|
|
828
|
+
additionalProperties: false
|
|
829
|
+
},
|
|
830
|
+
executionMode: "serial",
|
|
831
|
+
async execute(_toolCallId, _parameters, signal) {
|
|
832
|
+
try {
|
|
833
|
+
return computerObservationOutput(await requireActive().observe(signal, { includeScreenshot: _parameters.includeScreenshot !== false }));
|
|
834
|
+
} catch (error) {
|
|
835
|
+
return output(error instanceof Error ? error.message : String(error), true);
|
|
836
|
+
}
|
|
837
|
+
}
|
|
838
|
+
},
|
|
839
|
+
{
|
|
840
|
+
when: {
|
|
841
|
+
state: { "minor-mode": COMPUTER_USE_MODE_ID },
|
|
842
|
+
attribution: {
|
|
843
|
+
kind: "minor",
|
|
844
|
+
mode: COMPUTER_USE_MODE_ID
|
|
845
|
+
}
|
|
846
|
+
},
|
|
847
|
+
name: "computer_action",
|
|
848
|
+
label: "Computer Action",
|
|
849
|
+
description: "Perform one constrained semantic action in the authorized application window.",
|
|
850
|
+
parameters: {
|
|
851
|
+
type: "object",
|
|
852
|
+
properties: {
|
|
853
|
+
kind: {
|
|
854
|
+
type: "string",
|
|
855
|
+
enum: [
|
|
856
|
+
"press",
|
|
857
|
+
"focus",
|
|
858
|
+
"set_value",
|
|
859
|
+
"scroll"
|
|
860
|
+
]
|
|
861
|
+
},
|
|
862
|
+
snapshotId: {
|
|
863
|
+
type: "string",
|
|
864
|
+
minLength: 1
|
|
865
|
+
},
|
|
866
|
+
elementRef: {
|
|
867
|
+
type: "string",
|
|
868
|
+
minLength: 1
|
|
869
|
+
},
|
|
870
|
+
value: { type: "string" },
|
|
871
|
+
direction: {
|
|
872
|
+
type: "string",
|
|
873
|
+
enum: [
|
|
874
|
+
"up",
|
|
875
|
+
"down",
|
|
876
|
+
"left",
|
|
877
|
+
"right"
|
|
878
|
+
]
|
|
879
|
+
},
|
|
880
|
+
amount: {
|
|
881
|
+
type: "string",
|
|
882
|
+
enum: ["line", "page"]
|
|
883
|
+
}
|
|
884
|
+
},
|
|
885
|
+
required: [
|
|
886
|
+
"kind",
|
|
887
|
+
"snapshotId",
|
|
888
|
+
"elementRef"
|
|
889
|
+
],
|
|
890
|
+
additionalProperties: false
|
|
891
|
+
},
|
|
892
|
+
executionMode: "serial",
|
|
893
|
+
async execute(_toolCallId, parameters, signal) {
|
|
894
|
+
try {
|
|
895
|
+
const result = await requireActive().act(parameters, signal);
|
|
896
|
+
await refresh(signal);
|
|
897
|
+
return output(result);
|
|
898
|
+
} catch (error) {
|
|
899
|
+
return output(error instanceof Error ? error.message : String(error), true);
|
|
900
|
+
}
|
|
901
|
+
}
|
|
902
|
+
},
|
|
903
|
+
{
|
|
904
|
+
when: {
|
|
905
|
+
state: { "minor-mode": COMPUTER_USE_MODE_ID },
|
|
906
|
+
attribution: {
|
|
907
|
+
kind: "minor",
|
|
908
|
+
mode: COMPUTER_USE_MODE_ID
|
|
909
|
+
}
|
|
910
|
+
},
|
|
911
|
+
name: "computer_exec",
|
|
912
|
+
label: "Computer Script",
|
|
913
|
+
description: "Run a reusable function with only the authorized program API. Relative helpers are supported. Full Node requires trusted:true and an explicitly allowlisted path.",
|
|
914
|
+
parameters: {
|
|
915
|
+
type: "object",
|
|
916
|
+
properties: {
|
|
917
|
+
scriptPath: {
|
|
918
|
+
type: "string",
|
|
919
|
+
minLength: 1
|
|
920
|
+
},
|
|
921
|
+
input: {},
|
|
922
|
+
trusted: { type: "boolean" },
|
|
923
|
+
includeScreenshot: { type: "boolean" }
|
|
924
|
+
},
|
|
925
|
+
required: ["scriptPath", "input"],
|
|
926
|
+
additionalProperties: false
|
|
927
|
+
},
|
|
928
|
+
executionMode: "serial",
|
|
929
|
+
async execute(_toolCallId, parameters, signal) {
|
|
930
|
+
try {
|
|
931
|
+
requireActive();
|
|
932
|
+
if (!dependencies.scriptRunner) throw new Error("Computer script execution is unavailable.");
|
|
933
|
+
const input = parameters;
|
|
934
|
+
return computerScriptOutput(await dependencies.scriptRunner.execute(input.scriptPath, input.input, signal, input));
|
|
935
|
+
} catch (error) {
|
|
936
|
+
return computerScriptFailure(error);
|
|
937
|
+
}
|
|
938
|
+
}
|
|
939
|
+
}
|
|
940
|
+
],
|
|
941
|
+
commands: [{
|
|
942
|
+
name: COMMAND_NAME,
|
|
943
|
+
description: COMMAND_DESCRIPTION,
|
|
944
|
+
async execute(args, execution) {
|
|
945
|
+
const result = await dependencies.service.execute();
|
|
946
|
+
await execution.client.notify({
|
|
947
|
+
body: result.message,
|
|
948
|
+
level: result.level
|
|
949
|
+
});
|
|
950
|
+
if (args.trim() === "deactivate" && client && state?.phase !== "inactive" && state?.phase !== "failed") {
|
|
951
|
+
await client.stop();
|
|
952
|
+
await host.changeSelection({
|
|
953
|
+
axis: "state",
|
|
954
|
+
key: "minor-mode",
|
|
955
|
+
values: (execution.selection.state?.["minor-mode"] ?? []).filter((mode) => mode !== COMPUTER_USE_MODE_ID)
|
|
956
|
+
});
|
|
957
|
+
await refresh();
|
|
958
|
+
}
|
|
959
|
+
}
|
|
960
|
+
}],
|
|
961
|
+
hooks: [{
|
|
962
|
+
when: {
|
|
963
|
+
state: { "minor-mode": COMPUTER_USE_MODE_ID },
|
|
964
|
+
attribution: {
|
|
965
|
+
kind: "minor",
|
|
966
|
+
mode: COMPUTER_USE_MODE_ID
|
|
967
|
+
}
|
|
968
|
+
},
|
|
969
|
+
event: "before_agent_start",
|
|
970
|
+
handle(event) {
|
|
971
|
+
if (state?.phase !== "active") return void 0;
|
|
972
|
+
return { systemPrompt: `${typeof event.systemPrompt === "string" ? event.systemPrompt : ""}\n\n${GUIDANCE}`.trim() };
|
|
973
|
+
}
|
|
974
|
+
}, {
|
|
975
|
+
when: {
|
|
976
|
+
state: { "minor-mode": COMPUTER_USE_MODE_ID },
|
|
977
|
+
attribution: {
|
|
978
|
+
kind: "minor",
|
|
979
|
+
mode: COMPUTER_USE_MODE_ID
|
|
980
|
+
}
|
|
981
|
+
},
|
|
982
|
+
event: "session_shutdown",
|
|
983
|
+
handle: async () => {
|
|
984
|
+
stopActivity?.();
|
|
985
|
+
if (client && state?.phase !== "inactive" && state?.phase !== "failed") await client.stop();
|
|
986
|
+
}
|
|
987
|
+
}],
|
|
988
|
+
onDispose() {
|
|
989
|
+
stopActivity?.();
|
|
990
|
+
stopActivity = void 0;
|
|
991
|
+
unsubscribeState?.();
|
|
992
|
+
broker.close();
|
|
993
|
+
restrictionListeners.clear();
|
|
994
|
+
host.context.client.setStatus(COMPUTER_USE_STATUS_KEY, void 0);
|
|
995
|
+
}
|
|
996
|
+
};
|
|
997
|
+
}
|
|
998
|
+
//#endregion
|
|
999
|
+
//#region src/extensions/workspaces/sessions/(backend)/_runtime/index.server.ts
|
|
1000
|
+
var index_server_default = (({ agent, host }) => agent ? createComputerUseServer(agent, host.context) : {
|
|
1001
|
+
tools: [],
|
|
1002
|
+
commands: []
|
|
1003
|
+
});
|
|
1004
|
+
//#endregion
|
|
1005
|
+
//#region src/extensions/workspaces/sessions/(backend)/root.server.ts
|
|
1006
|
+
var root_server_default = defineRoot((context) => {
|
|
1007
|
+
const value = index_server_default(context);
|
|
1008
|
+
return {
|
|
1009
|
+
value,
|
|
1010
|
+
services: [...value.services ?? [], ...value.tools?.length ? [mountMcpTools(value.tools)] : []],
|
|
1011
|
+
activities: value.activities,
|
|
1012
|
+
onStop: value.onStop,
|
|
1013
|
+
onDispose: value.onDispose
|
|
1014
|
+
};
|
|
1015
|
+
});
|
|
1016
|
+
//#endregion
|
|
1017
|
+
//#region src/services/webComputerUseAvailability/index.ts
|
|
1018
|
+
var MissingComputerUseApiError = class extends Error {};
|
|
1019
|
+
function computerUseSessionApiError(status, value) {
|
|
1020
|
+
const error = typeof value === "object" && value !== null && !Array.isArray(value) ? value.error : void 0;
|
|
1021
|
+
const message = typeof error === "string" ? error : `HTTP ${String(status)}`;
|
|
1022
|
+
return status === 404 && message === `No API 'computer-use' in this session.` ? new MissingComputerUseApiError(message) : new Error(message);
|
|
1023
|
+
}
|
|
1024
|
+
//#endregion
|
|
1025
|
+
//#region src/services/webComputerUseChannel/index.ts
|
|
1026
|
+
/** Computer use runs on the Po Desktop host, so every other client declines it. */
|
|
1027
|
+
const DESKTOP_UNAVAILABLE = "Po Desktop computer use is unavailable.";
|
|
1028
|
+
function record(value) {
|
|
1029
|
+
return typeof value === "object" && value !== null && !Array.isArray(value) ? value : void 0;
|
|
1030
|
+
}
|
|
1031
|
+
function command(value) {
|
|
1032
|
+
const input = record(value);
|
|
1033
|
+
return input?.action === "status" || input?.action === "targets" || input?.action === "stop" || input?.action === "artifacts" ? { action: input.action } : void 0;
|
|
1034
|
+
}
|
|
1035
|
+
async function responseJson(response) {
|
|
1036
|
+
const value = await response.json();
|
|
1037
|
+
if (!response.ok) throw computerUseSessionApiError(response.status, value);
|
|
1038
|
+
return value;
|
|
1039
|
+
}
|
|
1040
|
+
function sessionState(value) {
|
|
1041
|
+
const input = record(value);
|
|
1042
|
+
if (typeof input?.sessionId !== "string" || typeof input.revision !== "number" || typeof input.wake !== "number" || ![
|
|
1043
|
+
"inactive",
|
|
1044
|
+
"awaiting_confirmation",
|
|
1045
|
+
"activating",
|
|
1046
|
+
"active",
|
|
1047
|
+
"stopping",
|
|
1048
|
+
"failed"
|
|
1049
|
+
].includes(input.phase)) return void 0;
|
|
1050
|
+
return input;
|
|
1051
|
+
}
|
|
1052
|
+
function createComputerUseChannel() {
|
|
1053
|
+
let receiveCommand;
|
|
1054
|
+
return {
|
|
1055
|
+
frameType: computerUseChannelType,
|
|
1056
|
+
lifecycle: "hub",
|
|
1057
|
+
start(host) {
|
|
1058
|
+
const scopes = /* @__PURE__ */ new Map();
|
|
1059
|
+
const latest = /* @__PURE__ */ new Map();
|
|
1060
|
+
const targets = /* @__PURE__ */ new Map();
|
|
1061
|
+
const grants = /* @__PURE__ */ new Map();
|
|
1062
|
+
const subscriptions = /* @__PURE__ */ new Map();
|
|
1063
|
+
const processing = /* @__PURE__ */ new Set();
|
|
1064
|
+
const rerun = /* @__PURE__ */ new Map();
|
|
1065
|
+
const requests = /* @__PURE__ */ new Map();
|
|
1066
|
+
let activeSessionId;
|
|
1067
|
+
let closed = false;
|
|
1068
|
+
const sessionRequest = async (scope, path, method = "GET", value) => host.requestSessionApi(scope, {
|
|
1069
|
+
basePath: API_BASE_PATH,
|
|
1070
|
+
path,
|
|
1071
|
+
method,
|
|
1072
|
+
...value === void 0 ? {} : { body: JSON.stringify(value) }
|
|
1073
|
+
});
|
|
1074
|
+
const publish = (scope, state) => {
|
|
1075
|
+
if (state.phase === "active" || state.phase === "activating" || state.phase === "stopping") activeSessionId = scope.sessionId;
|
|
1076
|
+
else if (activeSessionId === scope.sessionId) {
|
|
1077
|
+
activeSessionId = void 0;
|
|
1078
|
+
grants.delete(scope.sessionId);
|
|
1079
|
+
}
|
|
1080
|
+
const payload = {
|
|
1081
|
+
state,
|
|
1082
|
+
targets: targets.get(scope.sessionId) ?? [],
|
|
1083
|
+
...activeSessionId !== void 0 && activeSessionId !== scope.sessionId ? { busy: true } : {}
|
|
1084
|
+
};
|
|
1085
|
+
latest.set(scope.sessionId, payload);
|
|
1086
|
+
host.publish(scope.sessionId, payload);
|
|
1087
|
+
};
|
|
1088
|
+
const refresh = async (scope) => {
|
|
1089
|
+
const state = sessionState(await responseJson(await sessionRequest(scope, COMPUTER_USE_ROUTES.hubState)));
|
|
1090
|
+
if (state === void 0) throw new Error("Computer-use API returned an invalid session state.");
|
|
1091
|
+
publish(scope, state);
|
|
1092
|
+
return state;
|
|
1093
|
+
};
|
|
1094
|
+
const completePending = async (scope, pending) => {
|
|
1095
|
+
if (!host.computerUse?.available) throw new Error(DESKTOP_UNAVAILABLE);
|
|
1096
|
+
const controller = new AbortController();
|
|
1097
|
+
requests.set(scope.sessionId, controller);
|
|
1098
|
+
try {
|
|
1099
|
+
const result = await host.computerUse.request(scope, {
|
|
1100
|
+
operation: pending.operation,
|
|
1101
|
+
signal: controller.signal,
|
|
1102
|
+
payload: pending.operation === "act" ? {
|
|
1103
|
+
grantId: pending.grantId,
|
|
1104
|
+
sequence: pending.sequence,
|
|
1105
|
+
action: pending.payload
|
|
1106
|
+
} : {
|
|
1107
|
+
grantId: pending.grantId,
|
|
1108
|
+
includeScreenshot: record(pending.payload)?.includeScreenshot !== false
|
|
1109
|
+
}
|
|
1110
|
+
});
|
|
1111
|
+
await responseJson(await sessionRequest(scope, COMPUTER_USE_ROUTES.hubComplete, "POST", {
|
|
1112
|
+
id: pending.id,
|
|
1113
|
+
result
|
|
1114
|
+
}));
|
|
1115
|
+
} catch (error) {
|
|
1116
|
+
await responseJson(await sessionRequest(scope, COMPUTER_USE_ROUTES.hubComplete, "POST", {
|
|
1117
|
+
id: pending.id,
|
|
1118
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1119
|
+
}));
|
|
1120
|
+
} finally {
|
|
1121
|
+
if (requests.get(scope.sessionId) === controller) requests.delete(scope.sessionId);
|
|
1122
|
+
}
|
|
1123
|
+
};
|
|
1124
|
+
const stopSession = async (scope) => {
|
|
1125
|
+
requests.get(scope.sessionId)?.abort();
|
|
1126
|
+
const authorization = await responseJson(await sessionRequest(scope, COMPUTER_USE_ROUTES.hubAuthorization));
|
|
1127
|
+
let artifact;
|
|
1128
|
+
if (authorization !== null && host.computerUse?.available) try {
|
|
1129
|
+
artifact = record(await host.computerUse.request(scope, {
|
|
1130
|
+
operation: "stop",
|
|
1131
|
+
payload: { grantId: authorization.grantId }
|
|
1132
|
+
}))?.artifact;
|
|
1133
|
+
} catch (error) {
|
|
1134
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1135
|
+
artifact = {
|
|
1136
|
+
artifactId: randomUUID(),
|
|
1137
|
+
status: "failed",
|
|
1138
|
+
failure: {
|
|
1139
|
+
code: "recording_failed",
|
|
1140
|
+
message: message.slice(0, 512)
|
|
1141
|
+
}
|
|
1142
|
+
};
|
|
1143
|
+
host.onNotice(`computer-use Desktop stop was already unavailable (${message})`);
|
|
1144
|
+
}
|
|
1145
|
+
await responseJson(await sessionRequest(scope, COMPUTER_USE_ROUTES.hubStop, "POST", { artifact }));
|
|
1146
|
+
grants.delete(scope.sessionId);
|
|
1147
|
+
if (activeSessionId === scope.sessionId) activeSessionId = void 0;
|
|
1148
|
+
};
|
|
1149
|
+
const processScope = async (scope, announced) => {
|
|
1150
|
+
if (closed || !scopes.has(scope.sessionId)) return;
|
|
1151
|
+
if (processing.has(scope.sessionId)) {
|
|
1152
|
+
rerun.set(scope.sessionId, announced);
|
|
1153
|
+
return;
|
|
1154
|
+
}
|
|
1155
|
+
processing.add(scope.sessionId);
|
|
1156
|
+
try {
|
|
1157
|
+
const state = announced ?? await refresh(scope);
|
|
1158
|
+
if (state.phase === "active") {
|
|
1159
|
+
if (!grants.has(scope.sessionId)) {
|
|
1160
|
+
const authorization = await responseJson(await sessionRequest(scope, COMPUTER_USE_ROUTES.hubAuthorization));
|
|
1161
|
+
if (authorization !== null) grants.set(scope.sessionId, authorization.grantId);
|
|
1162
|
+
}
|
|
1163
|
+
const expired = typeof state.expiresAt === "number" && state.expiresAt <= Date.now();
|
|
1164
|
+
let owned = false;
|
|
1165
|
+
if (!expired && host.computerUse?.available) try {
|
|
1166
|
+
owned = record(await host.computerUse.request(scope, { operation: "status" }))?.ownedBySession === true;
|
|
1167
|
+
} catch {
|
|
1168
|
+
owned = false;
|
|
1169
|
+
}
|
|
1170
|
+
if (expired || !owned) {
|
|
1171
|
+
await stopSession(scope);
|
|
1172
|
+
await refresh(scope);
|
|
1173
|
+
return;
|
|
1174
|
+
}
|
|
1175
|
+
}
|
|
1176
|
+
if (state.phase === "awaiting_confirmation" && (activeSessionId === void 0 || activeSessionId === scope.sessionId)) {
|
|
1177
|
+
const activation = await responseJson(await sessionRequest(scope, COMPUTER_USE_ROUTES.hubActivation));
|
|
1178
|
+
if (activation !== null && !host.computerUse?.available) {
|
|
1179
|
+
await responseJson(await sessionRequest(scope, COMPUTER_USE_ROUTES.hubStop, "POST", { error: DESKTOP_UNAVAILABLE }));
|
|
1180
|
+
await refresh(scope);
|
|
1181
|
+
return;
|
|
1182
|
+
}
|
|
1183
|
+
if (activation !== null && host.computerUse?.available) {
|
|
1184
|
+
activeSessionId = scope.sessionId;
|
|
1185
|
+
const controller = new AbortController();
|
|
1186
|
+
requests.set(scope.sessionId, controller);
|
|
1187
|
+
try {
|
|
1188
|
+
const hostResult = await host.computerUse.request(scope, {
|
|
1189
|
+
operation: "activate",
|
|
1190
|
+
payload: activation,
|
|
1191
|
+
signal: controller.signal
|
|
1192
|
+
});
|
|
1193
|
+
await responseJson(await sessionRequest(scope, COMPUTER_USE_ROUTES.hubStop, "POST", { host: hostResult }));
|
|
1194
|
+
const grantId = record(hostResult)?.grantId;
|
|
1195
|
+
if (typeof grantId === "string") grants.set(scope.sessionId, grantId);
|
|
1196
|
+
} catch (error) {
|
|
1197
|
+
activeSessionId = void 0;
|
|
1198
|
+
await responseJson(await sessionRequest(scope, COMPUTER_USE_ROUTES.hubStop, "POST", { error: error instanceof Error ? error.message : String(error) }));
|
|
1199
|
+
} finally {
|
|
1200
|
+
if (requests.get(scope.sessionId) === controller) requests.delete(scope.sessionId);
|
|
1201
|
+
}
|
|
1202
|
+
await refresh(scope);
|
|
1203
|
+
}
|
|
1204
|
+
}
|
|
1205
|
+
if (state.phase === "stopping") {
|
|
1206
|
+
await stopSession(scope);
|
|
1207
|
+
await refresh(scope);
|
|
1208
|
+
}
|
|
1209
|
+
const pending = await responseJson(await sessionRequest(scope, COMPUTER_USE_ROUTES.hubNext));
|
|
1210
|
+
if (pending !== null) await completePending(scope, pending);
|
|
1211
|
+
} catch (error) {
|
|
1212
|
+
if (error instanceof MissingComputerUseApiError) latest.delete(scope.sessionId);
|
|
1213
|
+
else host.onNotice(`computer-use state handling failed for ${scope.sessionId} (${error instanceof Error ? error.message : String(error)})`);
|
|
1214
|
+
} finally {
|
|
1215
|
+
processing.delete(scope.sessionId);
|
|
1216
|
+
if (!closed && scopes.has(scope.sessionId) && rerun.has(scope.sessionId)) {
|
|
1217
|
+
const next = rerun.get(scope.sessionId);
|
|
1218
|
+
rerun.delete(scope.sessionId);
|
|
1219
|
+
queueMicrotask(() => {
|
|
1220
|
+
processScope(scope, next);
|
|
1221
|
+
});
|
|
1222
|
+
}
|
|
1223
|
+
}
|
|
1224
|
+
};
|
|
1225
|
+
const onState = (scope, payload) => {
|
|
1226
|
+
const state = sessionState(payload);
|
|
1227
|
+
if (state === void 0 || state.sessionId !== scope.sessionId) return;
|
|
1228
|
+
if (state.phase === "stopping" || state.phase === "failed" || state.phase === "inactive") requests.get(scope.sessionId)?.abort();
|
|
1229
|
+
publish(scope, state);
|
|
1230
|
+
processScope(scope, state);
|
|
1231
|
+
};
|
|
1232
|
+
receiveCommand = (scope, payload) => {
|
|
1233
|
+
const parsed = command(payload);
|
|
1234
|
+
if (parsed === void 0) return;
|
|
1235
|
+
(async () => {
|
|
1236
|
+
try {
|
|
1237
|
+
if (parsed.action === "targets") {
|
|
1238
|
+
const result = host.computerUse?.available ? await host.computerUse.request(scope, { operation: "targets" }) : [];
|
|
1239
|
+
targets.set(scope.sessionId, Array.isArray(result) ? result.filter((item) => record(item) !== void 0) : []);
|
|
1240
|
+
} else if (parsed.action === "stop") await stopSession(scope);
|
|
1241
|
+
await refresh(scope);
|
|
1242
|
+
processScope(scope);
|
|
1243
|
+
} catch (error) {
|
|
1244
|
+
if (error instanceof MissingComputerUseApiError) latest.delete(scope.sessionId);
|
|
1245
|
+
else host.onNotice(`computer-use command failed for ${scope.sessionId} (${error instanceof Error ? error.message : String(error)})`);
|
|
1246
|
+
}
|
|
1247
|
+
})();
|
|
1248
|
+
};
|
|
1249
|
+
return {
|
|
1250
|
+
payloadFor: (scope) => latest.get(scope.sessionId),
|
|
1251
|
+
sessionAdded(scope) {
|
|
1252
|
+
scopes.set(scope.sessionId, scope);
|
|
1253
|
+
const unsubscribe = host.directEvents.subscribe(computerUseChannelType, scope.sessionId, (payload) => onState(scope, payload), { replayLatest: true });
|
|
1254
|
+
subscriptions.set(scope.sessionId, unsubscribe);
|
|
1255
|
+
processScope(scope);
|
|
1256
|
+
},
|
|
1257
|
+
sessionRemoved(sessionId) {
|
|
1258
|
+
requests.get(sessionId)?.abort();
|
|
1259
|
+
requests.delete(sessionId);
|
|
1260
|
+
rerun.delete(sessionId);
|
|
1261
|
+
const scope = scopes.get(sessionId);
|
|
1262
|
+
const grantId = grants.get(sessionId);
|
|
1263
|
+
subscriptions.get(sessionId)?.();
|
|
1264
|
+
subscriptions.delete(sessionId);
|
|
1265
|
+
if (scope !== void 0 && grantId !== void 0 && host.computerUse?.available) host.computerUse.request(scope, {
|
|
1266
|
+
operation: "stop",
|
|
1267
|
+
payload: { grantId }
|
|
1268
|
+
}).catch((error) => host.onNotice(`computer-use session cleanup failed (${error instanceof Error ? error.message : String(error)})`));
|
|
1269
|
+
scopes.delete(sessionId);
|
|
1270
|
+
latest.delete(sessionId);
|
|
1271
|
+
targets.delete(sessionId);
|
|
1272
|
+
grants.delete(sessionId);
|
|
1273
|
+
processing.delete(sessionId);
|
|
1274
|
+
if (activeSessionId === sessionId) activeSessionId = void 0;
|
|
1275
|
+
},
|
|
1276
|
+
close() {
|
|
1277
|
+
closed = true;
|
|
1278
|
+
for (const controller of requests.values()) controller.abort();
|
|
1279
|
+
requests.clear();
|
|
1280
|
+
rerun.clear();
|
|
1281
|
+
for (const unsubscribe of subscriptions.values()) unsubscribe();
|
|
1282
|
+
subscriptions.clear();
|
|
1283
|
+
if (host.computerUse?.available) for (const [sessionId, grantId] of grants) {
|
|
1284
|
+
const scope = scopes.get(sessionId);
|
|
1285
|
+
if (scope !== void 0) host.computerUse.request(scope, {
|
|
1286
|
+
operation: "stop",
|
|
1287
|
+
payload: { grantId }
|
|
1288
|
+
}).catch(() => void 0);
|
|
1289
|
+
}
|
|
1290
|
+
scopes.clear();
|
|
1291
|
+
latest.clear();
|
|
1292
|
+
targets.clear();
|
|
1293
|
+
grants.clear();
|
|
1294
|
+
}
|
|
1295
|
+
};
|
|
1296
|
+
},
|
|
1297
|
+
receive(scope, payload, connection) {
|
|
1298
|
+
if (connection.desktopAuthorized !== true && connection.computerUseAuthorized !== true) return;
|
|
1299
|
+
receiveCommand?.(scope, payload);
|
|
1300
|
+
}
|
|
1301
|
+
};
|
|
1302
|
+
}
|
|
1303
|
+
//#endregion
|
|
1304
|
+
//#region src/extensions/(backend)/channel/computer-use.server.ts
|
|
1305
|
+
var computer_use_server_default$2 = defineChannel(createComputerUseChannel);
|
|
1306
|
+
//#endregion
|
|
1307
|
+
//#region src/extensions/workspaces/sessions/(backend)/api/computer-use/route.server.ts
|
|
1308
|
+
var route_server_default = defineRoutedContribution((context) => context.root.api?.find((api) => api.basePath === "computer-use"), { cardinality: "optional" });
|
|
1309
|
+
//#endregion
|
|
1310
|
+
//#region src/extensions/workspaces/sessions/(backend)/command/computer-use.server.ts
|
|
1311
|
+
var computer_use_server_default$1 = defineRoutedContribution((context) => context.root.commands.find((command) => command.name === "computer-use"), { cardinality: "optional" });
|
|
1312
|
+
//#endregion
|
|
1313
|
+
//#region src/extensions/workspaces/sessions/(backend)/hook/before-agent-start.server.ts
|
|
1314
|
+
var before_agent_start_server_default = defineRoutedContribution((context) => context.root.hooks?.find((hook) => hook.event === "before_agent_start"), { cardinality: "optional" });
|
|
1315
|
+
//#endregion
|
|
1316
|
+
//#region src/extensions/workspaces/sessions/(backend)/hook/session-shutdown.server.ts
|
|
1317
|
+
var session_shutdown_server_default = defineRoutedContribution((context) => context.root.hooks?.find((hook) => hook.event === "session_shutdown"), { cardinality: "optional" });
|
|
1318
|
+
//#endregion
|
|
1319
|
+
//#region src/extensions/workspaces/sessions/(backend)/tool-restriction/computer-use.server.ts
|
|
1320
|
+
var computer_use_server_default = defineRoutedContribution((context) => context.root.toolRestrictions?.[0], { cardinality: "optional" });
|
|
1321
|
+
//#endregion
|
|
1322
|
+
//#region src/extensions/workspaces/sessions/(backend)/tool/00-computer-state.server.ts
|
|
1323
|
+
var _00_computer_state_server_default = defineRoutedContribution((context) => context.root.tools.find((tool) => tool.name === "computer_state"), { cardinality: "optional" });
|
|
1324
|
+
//#endregion
|
|
1325
|
+
//#region src/extensions/workspaces/sessions/(backend)/tool/10-computer-action.server.ts
|
|
1326
|
+
var _10_computer_action_server_default = defineRoutedContribution((context) => context.root.tools.find((tool) => tool.name === "computer_action"), { cardinality: "optional" });
|
|
1327
|
+
//#endregion
|
|
1328
|
+
//#region src/extensions/workspaces/sessions/(backend)/tool/20-computer-exec.server.ts
|
|
1329
|
+
var _20_computer_exec_server_default = defineRoutedContribution((context) => context.root.tools.find((tool) => tool.name === "computer_exec"), { cardinality: "optional" });
|
|
1330
|
+
//#endregion
|
|
1331
|
+
//#region generated/server.ts
|
|
1332
|
+
const at = (value, context) => typeof value === "function" ? value(context) : value;
|
|
1333
|
+
/* v8 ignore next -- generated normalization branch */
|
|
1334
|
+
const defined = (values) => values.filter((value) => value !== void 0);
|
|
1335
|
+
const via = (identity, value) => value === void 0 ? void 0 : {
|
|
1336
|
+
...identity,
|
|
1337
|
+
...value
|
|
1338
|
+
};
|
|
1339
|
+
const facet = defineServerPlugin({
|
|
1340
|
+
name: "@agimon-ai/po-computer-use",
|
|
1341
|
+
providerOverrides: composeServerProviderOverrides(root_server_default),
|
|
1342
|
+
global: () => ({ get channels() {
|
|
1343
|
+
return [() => via({ frameType: "computer_use" }, computer_use_server_default$2())];
|
|
1344
|
+
} }),
|
|
1345
|
+
workspace: () => ({ get channels() {
|
|
1346
|
+
return [() => via({ frameType: "computer_use" }, computer_use_server_default$2())];
|
|
1347
|
+
} }),
|
|
1348
|
+
session: async (context) => {
|
|
1349
|
+
const scopeSession = await root_server_default(context);
|
|
1350
|
+
const contextSession = {
|
|
1351
|
+
...context,
|
|
1352
|
+
root: scopeSession.value
|
|
1353
|
+
};
|
|
1354
|
+
return {
|
|
1355
|
+
...composeRootHooks(scopeSession),
|
|
1356
|
+
get api() {
|
|
1357
|
+
return defined([at(route_server_default, contextSession)]);
|
|
1358
|
+
},
|
|
1359
|
+
get channels() {
|
|
1360
|
+
return [() => via({ frameType: "computer_use" }, computer_use_server_default$2())];
|
|
1361
|
+
},
|
|
1362
|
+
get commands() {
|
|
1363
|
+
return defined([via({ name: "computer-use" }, at(computer_use_server_default$1, contextSession))]);
|
|
1364
|
+
},
|
|
1365
|
+
get hooks() {
|
|
1366
|
+
return defined([via({ event: "before_agent_start" }, at(before_agent_start_server_default, contextSession)), via({ event: "session_shutdown" }, at(session_shutdown_server_default, contextSession))]);
|
|
1367
|
+
},
|
|
1368
|
+
get toolRestrictions() {
|
|
1369
|
+
return defined([at(computer_use_server_default, contextSession)]);
|
|
1370
|
+
},
|
|
1371
|
+
get tools() {
|
|
1372
|
+
return defined([
|
|
1373
|
+
via({ name: "00_computer_state" }, at(_00_computer_state_server_default, contextSession)),
|
|
1374
|
+
via({ name: "10_computer_action" }, at(_10_computer_action_server_default, contextSession)),
|
|
1375
|
+
via({ name: "20_computer_exec" }, at(_20_computer_exec_server_default, contextSession))
|
|
1376
|
+
]);
|
|
1377
|
+
},
|
|
1378
|
+
services: [...scopeSession.services ?? []],
|
|
1379
|
+
get activities() {
|
|
1380
|
+
return [...scopeSession.activities ?? []];
|
|
1381
|
+
}
|
|
1382
|
+
};
|
|
1383
|
+
}
|
|
1384
|
+
});
|
|
1385
|
+
//#endregion
|
|
1386
|
+
export { facet as default, facet };
|
|
1387
|
+
|
|
1388
|
+
//# sourceMappingURL=server.mjs.map
|