dsh-click 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/LICENSE +201 -0
- package/README.es.md +174 -0
- package/README.hi.md +174 -0
- package/README.md +174 -0
- package/README.pt.md +174 -0
- package/README.zh.md +174 -0
- package/SECURITY.md +38 -0
- package/THIRD_PARTY_NOTICES.md +20 -0
- package/cordis.patch.yml +53 -0
- package/lib/index.js +1810 -0
- package/lib/types/actions.d.ts +94 -0
- package/lib/types/actions.d.ts.map +1 -0
- package/lib/types/config.d.ts +119 -0
- package/lib/types/config.d.ts.map +1 -0
- package/lib/types/events.d.ts +111 -0
- package/lib/types/events.d.ts.map +1 -0
- package/lib/types/index.d.ts +39 -0
- package/lib/types/index.d.ts.map +1 -0
- package/lib/types/observe.d.ts +76 -0
- package/lib/types/observe.d.ts.map +1 -0
- package/lib/types/platform/runner.d.ts +40 -0
- package/lib/types/platform/runner.d.ts.map +1 -0
- package/lib/types/platform/selection.d.ts +44 -0
- package/lib/types/platform/selection.d.ts.map +1 -0
- package/lib/types/platform/types.d.ts +220 -0
- package/lib/types/platform/types.d.ts.map +1 -0
- package/lib/types/sanitize.d.ts +45 -0
- package/lib/types/sanitize.d.ts.map +1 -0
- package/lib/types/tools.d.ts +113 -0
- package/lib/types/tools.d.ts.map +1 -0
- package/lib/types/version.d.ts +13 -0
- package/lib/types/version.d.ts.map +1 -0
- package/lib/types/vision.d.ts +22 -0
- package/lib/types/vision.d.ts.map +1 -0
- package/native/win32/dsh-click-helper.ps1 +870 -0
- package/package.json +143 -0
package/lib/index.js
ADDED
|
@@ -0,0 +1,1810 @@
|
|
|
1
|
+
import z from "@deepseek-ai/schemastery";
|
|
2
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
3
|
+
import { existsSync } from "node:fs";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import { fileURLToPath } from "node:url";
|
|
6
|
+
import { defineTool } from "@deepseek-ai/dsh-tools";
|
|
7
|
+
//#region src/events.ts
|
|
8
|
+
/** The observation audit event type. */
|
|
9
|
+
const OBSERVED_EVENT = "dsh-click/observed";
|
|
10
|
+
/** The action audit event type. */
|
|
11
|
+
const ACTION_EVENT = "dsh-click/action";
|
|
12
|
+
//#endregion
|
|
13
|
+
//#region src/sanitize.ts
|
|
14
|
+
/**
|
|
15
|
+
* Pure display/audit redaction for everything dsh-click shows to the model or
|
|
16
|
+
* writes to a session log: window titles, executable paths, and free text that
|
|
17
|
+
* may contain credentials. Every function is pure and total on string input;
|
|
18
|
+
* extreme cases are covered by `tests/sanitize.spec.ts`.
|
|
19
|
+
*
|
|
20
|
+
* @module dsh-click/sanitize
|
|
21
|
+
*/
|
|
22
|
+
/** Control characters replaced by a space (tab) or removed entirely (the rest). */
|
|
23
|
+
const CONTROL_CHARACTERS = /[\u0000-\u0008\u000B-\u000C\u000E-\u001F\u007F]/gu;
|
|
24
|
+
/** Truncation marker appended when a string is cut to `maxLength`. */
|
|
25
|
+
const TRUNCATION_MARKER = "…";
|
|
26
|
+
/** Credential-shaped `name=value` / `name: value` pairs; values stop at control characters too. */
|
|
27
|
+
const SECRET_ASSIGNMENT = /((?:api[_-]?key|token|secret|password|passwd)\s*[=:])\s*[^\s"'<>\u0000-\u001F\u007F]+/giu;
|
|
28
|
+
/** JSON Web Tokens (three base64url segments separated by dots). */
|
|
29
|
+
const JWT_PATTERN = /\beyJ[A-Za-z0-9_-]{4,}\.[A-Za-z0-9_-]{4,}\.[A-Za-z0-9_-]{4,}\b/gu;
|
|
30
|
+
/** `Bearer` credentials. */
|
|
31
|
+
const BEARER_PATTERN = /(\bbearer\s+)[A-Za-z0-9._~+/=-]{8,}/giu;
|
|
32
|
+
/**
|
|
33
|
+
* Strip control characters and truncate to `maxLength`.
|
|
34
|
+
*
|
|
35
|
+
* @param text - the raw string.
|
|
36
|
+
* @param maxLength - maximum returned length (including the truncation marker).
|
|
37
|
+
* @returns the sanitized string.
|
|
38
|
+
*/
|
|
39
|
+
function sanitizeText(text, maxLength) {
|
|
40
|
+
const cleaned = text.replace(CONTROL_CHARACTERS, "").replace(/\t+/gu, " ");
|
|
41
|
+
if (cleaned.length <= maxLength) return cleaned;
|
|
42
|
+
return cleaned.slice(0, Math.max(0, maxLength - 1)) + TRUNCATION_MARKER;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Sanitize a filesystem path for display: control characters removed, and the
|
|
46
|
+
* TAIL (filename side) preserved when truncating, since the head is the part
|
|
47
|
+
* that usually repeats.
|
|
48
|
+
*
|
|
49
|
+
* @param path - the raw path.
|
|
50
|
+
* @param maxLength - maximum returned length (including the truncation marker).
|
|
51
|
+
* @returns the sanitized path.
|
|
52
|
+
*/
|
|
53
|
+
function sanitizePath(path, maxLength) {
|
|
54
|
+
const cleaned = path.replace(CONTROL_CHARACTERS, "").replace(/\t+/gu, " ");
|
|
55
|
+
if (cleaned.length <= maxLength) return cleaned;
|
|
56
|
+
return TRUNCATION_MARKER + cleaned.slice(cleaned.length - (maxLength - 1));
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Redact credential-shaped fragments from free text before it reaches a log or
|
|
60
|
+
* display surface: `key=`/`key:` assignments, JWTs, and bearer tokens.
|
|
61
|
+
*
|
|
62
|
+
* @param text - the raw text.
|
|
63
|
+
* @returns the redacted text.
|
|
64
|
+
*/
|
|
65
|
+
function redactSensitive(text) {
|
|
66
|
+
return text.replace(SECRET_ASSIGNMENT, "$1[redacted]").replace(JWT_PATTERN, "[redacted]").replace(BEARER_PATTERN, "$1[redacted]");
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* The one entry point for model-visible strings: redact first (secret values
|
|
70
|
+
* stop at control characters), then sanitize (control characters removed,
|
|
71
|
+
* tabs collapsed, length capped).
|
|
72
|
+
*
|
|
73
|
+
* @param text - the raw string.
|
|
74
|
+
* @param maxLength - truncation length.
|
|
75
|
+
* @returns the sanitized, redacted string.
|
|
76
|
+
*/
|
|
77
|
+
function sanitizeVisible(text, maxLength) {
|
|
78
|
+
return sanitizeText(redactSensitive(text), maxLength);
|
|
79
|
+
}
|
|
80
|
+
//#endregion
|
|
81
|
+
//#region src/platform/types.ts
|
|
82
|
+
/** Error with a stable machine-routing code, model-readable message included. */
|
|
83
|
+
var DshClickError = class extends Error {
|
|
84
|
+
code;
|
|
85
|
+
/** @param message - human- and model-readable failure text. */
|
|
86
|
+
/** @param code - stable code for policy and diagnostics. */
|
|
87
|
+
constructor(message, code) {
|
|
88
|
+
super(message);
|
|
89
|
+
this.code = code;
|
|
90
|
+
this.name = "DshClickError";
|
|
91
|
+
}
|
|
92
|
+
};
|
|
93
|
+
/** Assert one helper response field is a finite number; throws on violation. */
|
|
94
|
+
function expectNumber(record, field, op) {
|
|
95
|
+
const value = record[field];
|
|
96
|
+
if (typeof value !== "number" || !Number.isFinite(value)) throw new DshClickError(`helper "${op}" returned a non-numeric "${field}" field`, "BAD_HELPER_RESPONSE");
|
|
97
|
+
return value;
|
|
98
|
+
}
|
|
99
|
+
/** Assert one helper response field is a string; throws on violation. */
|
|
100
|
+
function expectString(record, field, op) {
|
|
101
|
+
const value = record[field];
|
|
102
|
+
if (typeof value !== "string") throw new DshClickError(`helper "${op}" returned a non-string "${field}" field`, "BAD_HELPER_RESPONSE");
|
|
103
|
+
return value;
|
|
104
|
+
}
|
|
105
|
+
/** Assert an entire helper payload is a plain record; throws on violation. */
|
|
106
|
+
function expectRecordValue(value, op) {
|
|
107
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) throw new DshClickError(`helper "${op}" returned a non-object payload`, "BAD_HELPER_RESPONSE");
|
|
108
|
+
return value;
|
|
109
|
+
}
|
|
110
|
+
/** Parse the helper's `processBefore`/`processAfter` facts at the wire boundary. */
|
|
111
|
+
function expectProcessFacts(value, op) {
|
|
112
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) throw new DshClickError(`helper "${op}" returned malformed process facts`, "BAD_HELPER_RESPONSE");
|
|
113
|
+
const record = value;
|
|
114
|
+
const pid = expectNumber(record, "pid", op);
|
|
115
|
+
const executablePath = record["executablePath"];
|
|
116
|
+
if (executablePath !== null && typeof executablePath !== "string") throw new DshClickError(`helper "${op}" returned malformed process facts`, "BAD_HELPER_RESPONSE");
|
|
117
|
+
return {
|
|
118
|
+
pid,
|
|
119
|
+
executablePath
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
//#endregion
|
|
123
|
+
//#region src/actions.ts
|
|
124
|
+
/** Approval failure detail, model-readable. */
|
|
125
|
+
const APPROVAL_DETAIL = {
|
|
126
|
+
rejected: "rejected by the approval answerer",
|
|
127
|
+
cancelled: "cancelled while waiting for approval",
|
|
128
|
+
unavailable: "no approval answerer is available (fail closed)"
|
|
129
|
+
};
|
|
130
|
+
/** Executes mutating desktop actions behind the full safety boundary. */
|
|
131
|
+
var ActionExecutor = class {
|
|
132
|
+
deps;
|
|
133
|
+
/** @param deps - executor dependencies. */
|
|
134
|
+
constructor(deps) {
|
|
135
|
+
this.deps = deps;
|
|
136
|
+
}
|
|
137
|
+
/** The mounting context. */
|
|
138
|
+
get ctx() {
|
|
139
|
+
return this.deps.ctx;
|
|
140
|
+
}
|
|
141
|
+
/** The resolved config. */
|
|
142
|
+
get config() {
|
|
143
|
+
return this.deps.config;
|
|
144
|
+
}
|
|
145
|
+
/** Append one action audit event; a failed append never changes the outcome. */
|
|
146
|
+
audit(exec, event) {
|
|
147
|
+
const session = exec.agent?.session;
|
|
148
|
+
if (session === void 0) return;
|
|
149
|
+
try {
|
|
150
|
+
session.append(ACTION_EVENT, event);
|
|
151
|
+
} catch {}
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* The staleness boundary: resolve the cited observation, re-observe the
|
|
155
|
+
* window right now, and compare.
|
|
156
|
+
*
|
|
157
|
+
* @param observationId - the observation the action cites.
|
|
158
|
+
* @param windowId - the window the action addresses.
|
|
159
|
+
* @param signal - cancellation.
|
|
160
|
+
* @returns the cited record and the fresh snapshot.
|
|
161
|
+
*/
|
|
162
|
+
async requireFreshWindow(observationId, windowId, signal) {
|
|
163
|
+
const record = this.deps.observations.get(observationId);
|
|
164
|
+
if (record === void 0) throw new DshClickError(`unknown observation "${observationId}" — run screen_read or screen_shot again and cite the returned observationId`, "UNKNOWN_OBSERVATION");
|
|
165
|
+
if (record.windowId !== windowId) throw new DshClickError(`observation ${observationId} belongs to window ${record.windowId}, not ${windowId}`, "UNKNOWN_OBSERVATION");
|
|
166
|
+
const fresh = await this.deps.backend.snapshot(windowId, signal);
|
|
167
|
+
const verdict = this.deps.observations.verify(record, fresh);
|
|
168
|
+
if (!verdict.ok) throw new DshClickError(`${verdict.detail} — run screen_read or screen_shot again before acting`, verdict.code);
|
|
169
|
+
return {
|
|
170
|
+
record,
|
|
171
|
+
fresh
|
|
172
|
+
};
|
|
173
|
+
}
|
|
174
|
+
/**
|
|
175
|
+
* The approval gate. Matchers run against the sanitized window title and
|
|
176
|
+
* the executable path; a match skips the ask (still audited as allowlist).
|
|
177
|
+
* With no matcher, a missing approval service or a missing calling agent
|
|
178
|
+
* fails closed; the real approval service is asked otherwise.
|
|
179
|
+
*
|
|
180
|
+
* @param exec - the calling tool execution.
|
|
181
|
+
* @param toolName - the tool asking (audit + prompt identity).
|
|
182
|
+
* @param subject - what the action targets, for the reason and the matchers.
|
|
183
|
+
* @param signal - cancellation.
|
|
184
|
+
* @returns which gate allowed the action.
|
|
185
|
+
*/
|
|
186
|
+
async gate(exec, toolName, subject, signal) {
|
|
187
|
+
if (!this.config.requireApproval) return "none";
|
|
188
|
+
if (this.config.autoApproveMatchers.some((matcher) => subject.title !== null && matcher.test(subject.title) || subject.executablePath !== null && matcher.test(subject.executablePath))) return "allowlist";
|
|
189
|
+
const approval = this.ctx.get("approval");
|
|
190
|
+
if (approval === void 0) throw new DshClickError("approval is required but the approval service is not mounted — mount @deepseek-ai/dsh-user-approval or allowlist this window", "APPROVAL_UNAVAILABLE");
|
|
191
|
+
if (exec.agent === void 0) throw new DshClickError("approval is required but no agent owns this call — refusing (fail closed)", "APPROVAL_UNAVAILABLE");
|
|
192
|
+
const target = subject.title ?? subject.executablePath ?? "unknown target";
|
|
193
|
+
const outcome = await approval.request({
|
|
194
|
+
agent: exec.agent,
|
|
195
|
+
toolName,
|
|
196
|
+
callId: exec.callId,
|
|
197
|
+
reason: `${toolName} on ${sanitizeVisible(target, this.config.maxTextLength)}`,
|
|
198
|
+
signal
|
|
199
|
+
});
|
|
200
|
+
if (outcome !== "allowed-once") throw new DshClickError(`action denied by approval: ${APPROVAL_DETAIL[outcome] ?? outcome}`, "APPROVAL_DENIED");
|
|
201
|
+
return "approval";
|
|
202
|
+
}
|
|
203
|
+
/**
|
|
204
|
+
* Compare the process identity captured before and after an action.
|
|
205
|
+
*
|
|
206
|
+
* @param before - facts captured pre-action.
|
|
207
|
+
* @param after - facts captured post-action.
|
|
208
|
+
*/
|
|
209
|
+
assertProcessUnchanged(before, after) {
|
|
210
|
+
if (before.pid !== after.pid || before.executablePath !== after.executablePath) throw new DshClickError(`the target process changed during the action (before: pid ${before.pid} ${before.executablePath ?? "?"}; after: pid ${after.pid} ${after.executablePath ?? "?"}) — the action was delivered to a different process identity and must be reviewed`, "PROCESS_CHANGED");
|
|
211
|
+
}
|
|
212
|
+
/**
|
|
213
|
+
* Run one window-scoped mutating action through the shared boundary:
|
|
214
|
+
* freshness, approval, perform, process check, audit.
|
|
215
|
+
*
|
|
216
|
+
* @param toolName - the calling tool name.
|
|
217
|
+
* @param exec - the calling execution.
|
|
218
|
+
* @param observationId - the cited observation.
|
|
219
|
+
* @param windowId - the target window.
|
|
220
|
+
* @param run - the backend call (receives the resolved focus flag).
|
|
221
|
+
* @returns the validated backend outcome.
|
|
222
|
+
*/
|
|
223
|
+
async perform(toolName, exec, observationId, windowId, run) {
|
|
224
|
+
let approved = "none";
|
|
225
|
+
let observationIdAudited;
|
|
226
|
+
let windowIdAudited;
|
|
227
|
+
try {
|
|
228
|
+
const { record } = await this.requireFreshWindow(observationId, windowId, exec.signal);
|
|
229
|
+
observationIdAudited = observationId;
|
|
230
|
+
windowIdAudited = windowId;
|
|
231
|
+
approved = await this.gate(exec, toolName, {
|
|
232
|
+
title: record.title,
|
|
233
|
+
executablePath: record.executablePath
|
|
234
|
+
}, exec.signal);
|
|
235
|
+
const outcome = await run(this.config.focusFallback === "allow");
|
|
236
|
+
this.assertProcessUnchanged(outcome.processBefore, outcome.processAfter);
|
|
237
|
+
this.audit(exec, {
|
|
238
|
+
tool: toolName,
|
|
239
|
+
approved,
|
|
240
|
+
outcome: "ok",
|
|
241
|
+
observationId: observationIdAudited,
|
|
242
|
+
windowId: windowIdAudited,
|
|
243
|
+
processBefore: outcome.processBefore,
|
|
244
|
+
processAfter: outcome.processAfter,
|
|
245
|
+
...outcome.restored !== void 0 ? { restored: outcome.restored } : {},
|
|
246
|
+
...outcome.detail !== void 0 ? { detail: outcome.detail } : {}
|
|
247
|
+
});
|
|
248
|
+
return outcome;
|
|
249
|
+
} catch (error) {
|
|
250
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
251
|
+
this.audit(exec, {
|
|
252
|
+
tool: toolName,
|
|
253
|
+
approved,
|
|
254
|
+
outcome: "error",
|
|
255
|
+
...observationIdAudited !== void 0 ? { observationId: observationIdAudited } : {},
|
|
256
|
+
...windowIdAudited !== void 0 ? { windowId: windowIdAudited } : {},
|
|
257
|
+
detail: message
|
|
258
|
+
});
|
|
259
|
+
throw error;
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
/**
|
|
263
|
+
* Gate and launch one application. No window observation exists yet, so the
|
|
264
|
+
* gate runs against the requested name/path alone and the launch outcome's
|
|
265
|
+
* process facts are the post-identity proof.
|
|
266
|
+
*
|
|
267
|
+
* @param exec - the calling execution.
|
|
268
|
+
* @param name - the app name or executable path.
|
|
269
|
+
* @param args - launch arguments.
|
|
270
|
+
* @returns the launch outcome.
|
|
271
|
+
*/
|
|
272
|
+
async launch(exec, name, args) {
|
|
273
|
+
const toolName = "app_launch";
|
|
274
|
+
let approved = "none";
|
|
275
|
+
try {
|
|
276
|
+
approved = await this.gate(exec, toolName, {
|
|
277
|
+
title: null,
|
|
278
|
+
executablePath: name
|
|
279
|
+
}, exec.signal);
|
|
280
|
+
const outcome = await this.deps.backend.launch(name, args, exec.signal);
|
|
281
|
+
this.audit(exec, {
|
|
282
|
+
tool: toolName,
|
|
283
|
+
approved,
|
|
284
|
+
outcome: "ok",
|
|
285
|
+
processAfter: {
|
|
286
|
+
pid: outcome.processId,
|
|
287
|
+
executablePath: outcome.executablePath
|
|
288
|
+
},
|
|
289
|
+
detail: name
|
|
290
|
+
});
|
|
291
|
+
return outcome;
|
|
292
|
+
} catch (error) {
|
|
293
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
294
|
+
this.audit(exec, {
|
|
295
|
+
tool: toolName,
|
|
296
|
+
approved,
|
|
297
|
+
outcome: "error",
|
|
298
|
+
detail: message
|
|
299
|
+
});
|
|
300
|
+
throw error;
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
};
|
|
304
|
+
//#endregion
|
|
305
|
+
//#region src/config.ts
|
|
306
|
+
/**
|
|
307
|
+
* Plugin configuration and its explicit resolve step. `resolveConfig` re-judges
|
|
308
|
+
* every default and bound so programmatic construction that bypasses
|
|
309
|
+
* Schemastery normalization still fails loud instead of running with hidden
|
|
310
|
+
* defaults (the explicit-resolve contract).
|
|
311
|
+
*
|
|
312
|
+
* @module dsh-click/config
|
|
313
|
+
*/
|
|
314
|
+
/** Default per-helper-call timeout in milliseconds. */
|
|
315
|
+
const DEFAULT_HELPER_TIMEOUT_MS = 3e4;
|
|
316
|
+
/** Ceiling for one helper call: a screenshot of a large desktop can take a while. */
|
|
317
|
+
const MAX_HELPER_TIMEOUT_MS = 3e5;
|
|
318
|
+
/** Default cap on one helper response in bytes (PNG screenshots are base64-heavy). */
|
|
319
|
+
const DEFAULT_MAX_HELPER_OUTPUT_BYTES = 25165824;
|
|
320
|
+
/** Ceiling on one helper response. */
|
|
321
|
+
const MAX_HELPER_OUTPUT_BYTES = 67108864;
|
|
322
|
+
/** Default longest side of a captured screenshot in pixels (larger captures downscale). */
|
|
323
|
+
const DEFAULT_MAX_SCREENSHOT_SIDE = 2560;
|
|
324
|
+
const MAX_SCREENSHOT_SIDE = 7680;
|
|
325
|
+
/** Default maximum age of an observation that an action may still base on. */
|
|
326
|
+
const DEFAULT_MAX_OBSERVATION_AGE_MS = 3e4;
|
|
327
|
+
/** Bounds for the observation age. */
|
|
328
|
+
const MIN_OBSERVATION_AGE_MS = 1e3;
|
|
329
|
+
const MAX_OBSERVATION_AGE_MS = 6e5;
|
|
330
|
+
/** Ceiling on the element cap. */
|
|
331
|
+
const MAX_ELEMENTS = 2e3;
|
|
332
|
+
/** Ceiling on the sanitization truncation length. */
|
|
333
|
+
const MAX_TEXT_LENGTH = 1e4;
|
|
334
|
+
/** Schemastery schema for loader-validated configuration. */
|
|
335
|
+
const Config = z.object({
|
|
336
|
+
requireApproval: z.boolean().default(true),
|
|
337
|
+
autoApproveWindows: z.array(z.string()).default([]),
|
|
338
|
+
focusFallback: z.union(["never", "allow"]).default("never"),
|
|
339
|
+
imageMode: z.union(["auto", "text"]).default("auto"),
|
|
340
|
+
helperTimeoutMs: z.number().min(1).max(MAX_HELPER_TIMEOUT_MS).default(DEFAULT_HELPER_TIMEOUT_MS),
|
|
341
|
+
maxHelperOutputBytes: z.number().min(1024).max(MAX_HELPER_OUTPUT_BYTES).default(DEFAULT_MAX_HELPER_OUTPUT_BYTES),
|
|
342
|
+
maxScreenshotSide: z.number().min(320).max(MAX_SCREENSHOT_SIDE).default(DEFAULT_MAX_SCREENSHOT_SIDE),
|
|
343
|
+
staleCheckPixels: z.boolean().default(true),
|
|
344
|
+
maxObservationAgeMs: z.number().min(MIN_OBSERVATION_AGE_MS).max(MAX_OBSERVATION_AGE_MS).default(DEFAULT_MAX_OBSERVATION_AGE_MS),
|
|
345
|
+
maxCachedObservations: z.number().min(1).max(64).default(8),
|
|
346
|
+
maxElements: z.number().min(1).max(MAX_ELEMENTS).default(500),
|
|
347
|
+
maxTreeDepth: z.number().min(1).max(64).default(32),
|
|
348
|
+
maxTextLength: z.number().min(16).max(MAX_TEXT_LENGTH).default(200),
|
|
349
|
+
rollbackEnabled: z.boolean().default(true)
|
|
350
|
+
});
|
|
351
|
+
/** Throw the standard fail-loud config error for one invalid field. */
|
|
352
|
+
function invalid(field, detail) {
|
|
353
|
+
throw new Error(`dsh-click: config.${field} ${detail}`);
|
|
354
|
+
}
|
|
355
|
+
/** Compile one allowlist pattern; an invalid regex fails at load, not at match time. */
|
|
356
|
+
function compileMatcher(source, index) {
|
|
357
|
+
try {
|
|
358
|
+
return new RegExp(source, "iu");
|
|
359
|
+
} catch {
|
|
360
|
+
invalid("autoApproveWindows", `entry ${index} (${JSON.stringify(source)}) is not a valid regular expression`);
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
/**
|
|
364
|
+
* Resolve raw config to the runtime policy, re-validating defaults and bounds.
|
|
365
|
+
*
|
|
366
|
+
* @param config - raw loader config; `undefined` for a bare row.
|
|
367
|
+
* @returns the frozen resolved config.
|
|
368
|
+
*/
|
|
369
|
+
function resolveConfig(config) {
|
|
370
|
+
const requireApproval = config?.requireApproval ?? true;
|
|
371
|
+
if (typeof requireApproval !== "boolean") invalid("requireApproval", "must be a boolean");
|
|
372
|
+
const autoApproveMatchers = (config?.autoApproveWindows ?? []).map(compileMatcher);
|
|
373
|
+
const focusFallback = config?.focusFallback ?? "never";
|
|
374
|
+
if (focusFallback !== "never" && focusFallback !== "allow") invalid("focusFallback", "must be \"never\" or \"allow\"");
|
|
375
|
+
const imageMode = config?.imageMode ?? "auto";
|
|
376
|
+
if (imageMode !== "auto" && imageMode !== "text") invalid("imageMode", "must be \"auto\" or \"text\"");
|
|
377
|
+
const helperTimeoutMs = config?.helperTimeoutMs ?? 3e4;
|
|
378
|
+
if (!Number.isFinite(helperTimeoutMs) || helperTimeoutMs < 1 || helperTimeoutMs > 3e5) invalid("helperTimeoutMs", `must be a finite number between 1 and ${MAX_HELPER_TIMEOUT_MS}`);
|
|
379
|
+
const maxHelperOutputBytes = config?.maxHelperOutputBytes ?? 25165824;
|
|
380
|
+
if (!Number.isInteger(maxHelperOutputBytes) || maxHelperOutputBytes < 1024 || maxHelperOutputBytes > 67108864) invalid("maxHelperOutputBytes", `must be an integer between 1024 and ${MAX_HELPER_OUTPUT_BYTES}`);
|
|
381
|
+
const maxScreenshotSide = config?.maxScreenshotSide ?? 2560;
|
|
382
|
+
if (!Number.isInteger(maxScreenshotSide) || maxScreenshotSide < 320 || maxScreenshotSide > 7680) invalid("maxScreenshotSide", `must be an integer between 320 and ${MAX_SCREENSHOT_SIDE}`);
|
|
383
|
+
const staleCheckPixels = config?.staleCheckPixels ?? true;
|
|
384
|
+
if (typeof staleCheckPixels !== "boolean") invalid("staleCheckPixels", "must be a boolean");
|
|
385
|
+
const maxObservationAgeMs = config?.maxObservationAgeMs ?? 3e4;
|
|
386
|
+
if (!Number.isFinite(maxObservationAgeMs) || maxObservationAgeMs < 1e3 || maxObservationAgeMs > 6e5) invalid("maxObservationAgeMs", `must be a finite number between ${MIN_OBSERVATION_AGE_MS} and ${MAX_OBSERVATION_AGE_MS}`);
|
|
387
|
+
const maxCachedObservations = config?.maxCachedObservations ?? 8;
|
|
388
|
+
if (!Number.isInteger(maxCachedObservations) || maxCachedObservations < 1 || maxCachedObservations > 64) invalid("maxCachedObservations", `must be an integer between 1 and 64`);
|
|
389
|
+
const maxElements = config?.maxElements ?? 500;
|
|
390
|
+
if (!Number.isInteger(maxElements) || maxElements < 1 || maxElements > 2e3) invalid("maxElements", `must be an integer between 1 and ${MAX_ELEMENTS}`);
|
|
391
|
+
const maxTreeDepth = config?.maxTreeDepth ?? 32;
|
|
392
|
+
if (!Number.isInteger(maxTreeDepth) || maxTreeDepth < 1 || maxTreeDepth > 64) invalid("maxTreeDepth", `must be an integer between 1 and 64`);
|
|
393
|
+
const maxTextLength = config?.maxTextLength ?? 200;
|
|
394
|
+
if (!Number.isInteger(maxTextLength) || maxTextLength < 16 || maxTextLength > 1e4) invalid("maxTextLength", `must be an integer between 16 and ${MAX_TEXT_LENGTH}`);
|
|
395
|
+
const rollbackEnabled = config?.rollbackEnabled ?? true;
|
|
396
|
+
if (typeof rollbackEnabled !== "boolean") invalid("rollbackEnabled", "must be a boolean");
|
|
397
|
+
return Object.freeze({
|
|
398
|
+
requireApproval,
|
|
399
|
+
autoApproveMatchers: Object.freeze(autoApproveMatchers),
|
|
400
|
+
focusFallback,
|
|
401
|
+
imageMode,
|
|
402
|
+
helperTimeoutMs,
|
|
403
|
+
maxHelperOutputBytes,
|
|
404
|
+
maxScreenshotSide,
|
|
405
|
+
staleCheckPixels,
|
|
406
|
+
maxObservationAgeMs,
|
|
407
|
+
maxCachedObservations,
|
|
408
|
+
maxElements,
|
|
409
|
+
maxTreeDepth,
|
|
410
|
+
maxTextLength,
|
|
411
|
+
rollbackEnabled
|
|
412
|
+
});
|
|
413
|
+
}
|
|
414
|
+
//#endregion
|
|
415
|
+
//#region src/observe.ts
|
|
416
|
+
/**
|
|
417
|
+
* Observation cache and the staleness boundary. Every observation stores the
|
|
418
|
+
* window's identity facts plus content hashes; before any action the executor
|
|
419
|
+
* re-observes the window and this store compares. Identity, tree, or pixel
|
|
420
|
+
* drift — or an expired observation — refuses the action, so the model must
|
|
421
|
+
* re-observe instead of acting on a screen that changed.
|
|
422
|
+
*
|
|
423
|
+
* @module dsh-click/observe
|
|
424
|
+
*/
|
|
425
|
+
/**
|
|
426
|
+
* Hash the observable state of one window into a stable observation id.
|
|
427
|
+
*
|
|
428
|
+
* @param snapshot - the snapshot facts.
|
|
429
|
+
* @returns a hex observation id.
|
|
430
|
+
*/
|
|
431
|
+
function observationIdOf(snapshot) {
|
|
432
|
+
const material = JSON.stringify([
|
|
433
|
+
snapshot.windowId,
|
|
434
|
+
snapshot.processId,
|
|
435
|
+
snapshot.executablePath,
|
|
436
|
+
snapshot.title,
|
|
437
|
+
snapshot.className,
|
|
438
|
+
snapshot.rect,
|
|
439
|
+
snapshot.treeHash,
|
|
440
|
+
snapshot.shotHash
|
|
441
|
+
]);
|
|
442
|
+
return createHash("sha256").update(material).digest("hex").slice(0, 32);
|
|
443
|
+
}
|
|
444
|
+
/** Bounded LRU cache of observations, plus the staleness verdict logic. */
|
|
445
|
+
var ObservationStore = class {
|
|
446
|
+
config;
|
|
447
|
+
entries = /* @__PURE__ */ new Map();
|
|
448
|
+
/** @param config - resolved caps (cache size, max age, pixel check). */
|
|
449
|
+
constructor(config) {
|
|
450
|
+
this.config = config;
|
|
451
|
+
}
|
|
452
|
+
/**
|
|
453
|
+
* Cache one observation and return it.
|
|
454
|
+
*
|
|
455
|
+
* @param snapshot - the snapshot facts it came from.
|
|
456
|
+
* @returns the stored record (id = hash of the state).
|
|
457
|
+
*/
|
|
458
|
+
record(snapshot) {
|
|
459
|
+
const record = {
|
|
460
|
+
id: observationIdOf(snapshot),
|
|
461
|
+
windowId: snapshot.windowId,
|
|
462
|
+
processId: snapshot.processId,
|
|
463
|
+
executablePath: snapshot.executablePath,
|
|
464
|
+
title: snapshot.title,
|
|
465
|
+
className: snapshot.className,
|
|
466
|
+
rect: { ...snapshot.rect },
|
|
467
|
+
treeHash: snapshot.treeHash,
|
|
468
|
+
shotHash: snapshot.shotHash,
|
|
469
|
+
elementCount: snapshot.elementCount,
|
|
470
|
+
observedAt: Date.now()
|
|
471
|
+
};
|
|
472
|
+
this.entries.delete(record.id);
|
|
473
|
+
this.entries.set(record.id, record);
|
|
474
|
+
while (this.entries.size > this.config.maxCachedObservations) {
|
|
475
|
+
const oldest = this.entries.keys().next().value;
|
|
476
|
+
if (oldest === void 0) break;
|
|
477
|
+
this.entries.delete(oldest);
|
|
478
|
+
}
|
|
479
|
+
return record;
|
|
480
|
+
}
|
|
481
|
+
/**
|
|
482
|
+
* Look one observation up by id.
|
|
483
|
+
*
|
|
484
|
+
* @param id - the observation id an action cites.
|
|
485
|
+
* @returns the record, or undefined when unknown or evicted.
|
|
486
|
+
*/
|
|
487
|
+
get(id) {
|
|
488
|
+
return this.entries.get(id);
|
|
489
|
+
}
|
|
490
|
+
/**
|
|
491
|
+
* Compare a fresh re-observation against the cited record. The verdict is
|
|
492
|
+
* `ok` only when identity (window/pid/exe/title/class/rect), the tree hash,
|
|
493
|
+
* and — when `staleCheckPixels` is on — the pixel hash all still match, and
|
|
494
|
+
* the record is not older than `maxObservationAgeMs`.
|
|
495
|
+
*
|
|
496
|
+
* @param record - the observation the action cites.
|
|
497
|
+
* @param fresh - the re-observation captured immediately before the action.
|
|
498
|
+
* @param now - wall-clock now (injectable for tests).
|
|
499
|
+
* @returns the verdict.
|
|
500
|
+
*/
|
|
501
|
+
verify(record, fresh, now = Date.now()) {
|
|
502
|
+
if (now - record.observedAt > this.config.maxObservationAgeMs) return {
|
|
503
|
+
ok: false,
|
|
504
|
+
code: "EXPIRED",
|
|
505
|
+
detail: `observation ${record.id} is older than ${this.config.maxObservationAgeMs} ms`
|
|
506
|
+
};
|
|
507
|
+
if (fresh.windowId !== record.windowId || fresh.processId !== record.processId || fresh.executablePath !== record.executablePath || fresh.title !== record.title || fresh.className !== record.className || fresh.rect.x !== record.rect.x || fresh.rect.y !== record.rect.y || fresh.rect.width !== record.rect.width || fresh.rect.height !== record.rect.height) return {
|
|
508
|
+
ok: false,
|
|
509
|
+
code: "STALE_IDENTITY",
|
|
510
|
+
detail: `window ${record.windowId} no longer matches observation ${record.id} (title/class/process or rect changed)`
|
|
511
|
+
};
|
|
512
|
+
if (fresh.treeHash !== record.treeHash) return {
|
|
513
|
+
ok: false,
|
|
514
|
+
code: "STALE_TREE",
|
|
515
|
+
detail: `the accessibility tree of window ${record.windowId} changed since observation ${record.id}`
|
|
516
|
+
};
|
|
517
|
+
if (this.config.staleCheckPixels && fresh.shotHash !== record.shotHash) return {
|
|
518
|
+
ok: false,
|
|
519
|
+
code: "STALE_PIXELS",
|
|
520
|
+
detail: `the pixels of window ${record.windowId} changed since observation ${record.id}`
|
|
521
|
+
};
|
|
522
|
+
return { ok: true };
|
|
523
|
+
}
|
|
524
|
+
};
|
|
525
|
+
//#endregion
|
|
526
|
+
//#region src/version.ts
|
|
527
|
+
/**
|
|
528
|
+
* The plugin version, hardcoded on purpose: it rides the helper-process
|
|
529
|
+
* protocol handshake, so it must track `package.json` or released helpers
|
|
530
|
+
* report a stale plugin. Bumping `package.json` without touching this file
|
|
531
|
+
* fails `tests/version.spec.ts`, and `scripts/release.mjs` bumps both.
|
|
532
|
+
*
|
|
533
|
+
* @module dsh-click/version
|
|
534
|
+
*/
|
|
535
|
+
/** Plugin version advertised in the helper-protocol handshake. */
|
|
536
|
+
const VERSION = "0.1.0";
|
|
537
|
+
/** The helper protocol revision; bump only on incompatible wire changes. */
|
|
538
|
+
const HELPER_PROTOCOL_VERSION = 1;
|
|
539
|
+
//#endregion
|
|
540
|
+
//#region src/platform/runner.ts
|
|
541
|
+
/**
|
|
542
|
+
* The Windows backend: every operation spawns the native PowerShell helper
|
|
543
|
+
* (`native/win32/dsh-click-helper.ps1`) through the harness's `ctx.subprocess`
|
|
544
|
+
* seam. The request travels as JSON on stdin; the response is JSON on stdout
|
|
545
|
+
* and is validated field-by-field at this wire boundary. Each call is
|
|
546
|
+
* deadline-bounded (config `helperTimeoutMs`) and aborts with the caller's
|
|
547
|
+
* signal (the subprocess service escalates to tree termination).
|
|
548
|
+
*
|
|
549
|
+
* @module dsh-click/platform/runner
|
|
550
|
+
*/
|
|
551
|
+
/** Locate the shipped helper script from either source or bundle layout. */
|
|
552
|
+
function findHelperPath() {
|
|
553
|
+
const override = process.env["DSH_CLICK_HELPER"];
|
|
554
|
+
if (override !== void 0 && override !== "") return override;
|
|
555
|
+
let dir = path.dirname(fileURLToPath(import.meta.url));
|
|
556
|
+
for (let depth = 0; depth < 8; depth += 1) {
|
|
557
|
+
const candidate = path.join(dir, "native", "win32", "dsh-click-helper.ps1");
|
|
558
|
+
if (existsSync(candidate)) return candidate;
|
|
559
|
+
dir = path.dirname(dir);
|
|
560
|
+
}
|
|
561
|
+
throw new DshClickError("cannot locate native/win32/dsh-click-helper.ps1 next to the package", "HELPER_MISSING");
|
|
562
|
+
}
|
|
563
|
+
/** Validate a helper rect at the wire boundary. */
|
|
564
|
+
function expectRect(value, op) {
|
|
565
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) throw new DshClickError(`helper "${op}" returned a malformed rect`, "BAD_HELPER_RESPONSE");
|
|
566
|
+
const record = value;
|
|
567
|
+
return {
|
|
568
|
+
x: expectNumber(record, "x", op),
|
|
569
|
+
y: expectNumber(record, "y", op),
|
|
570
|
+
width: expectNumber(record, "width", op),
|
|
571
|
+
height: expectNumber(record, "height", op)
|
|
572
|
+
};
|
|
573
|
+
}
|
|
574
|
+
/** Validate one helper window record. */
|
|
575
|
+
function expectWindow(value, op) {
|
|
576
|
+
const record = expectRecordValue(value, op);
|
|
577
|
+
const processIdRaw = record["processId"];
|
|
578
|
+
const processId = typeof processIdRaw === "number" ? processIdRaw : null;
|
|
579
|
+
const executablePath = record["executablePath"];
|
|
580
|
+
if (executablePath !== null && typeof executablePath !== "string") throw new DshClickError(`helper "${op}" returned malformed executablePath`, "BAD_HELPER_RESPONSE");
|
|
581
|
+
return {
|
|
582
|
+
windowId: expectNumber(record, "windowId", op),
|
|
583
|
+
processId,
|
|
584
|
+
title: expectString(record, "title", op),
|
|
585
|
+
className: expectString(record, "className", op),
|
|
586
|
+
rect: expectRect(record["rect"], op),
|
|
587
|
+
executablePath,
|
|
588
|
+
visible: record["visible"] !== false
|
|
589
|
+
};
|
|
590
|
+
}
|
|
591
|
+
/** Validate one helper snapshot record. */
|
|
592
|
+
function expectSnapshot(value, op) {
|
|
593
|
+
const record = expectRecordValue(value, op);
|
|
594
|
+
const executablePath = record["executablePath"];
|
|
595
|
+
if (executablePath !== null && typeof executablePath !== "string") throw new DshClickError(`helper "${op}" returned malformed executablePath`, "BAD_HELPER_RESPONSE");
|
|
596
|
+
return {
|
|
597
|
+
windowId: expectNumber(record, "windowId", op),
|
|
598
|
+
processId: expectNumber(record, "processId", op),
|
|
599
|
+
executablePath,
|
|
600
|
+
title: expectString(record, "title", op),
|
|
601
|
+
className: expectString(record, "className", op),
|
|
602
|
+
rect: expectRect(record["rect"], op),
|
|
603
|
+
foreground: record["foreground"] === true,
|
|
604
|
+
treeHash: expectString(record, "treeHash", op),
|
|
605
|
+
shotHash: expectString(record, "shotHash", op),
|
|
606
|
+
elementCount: expectNumber(record, "elementCount", op)
|
|
607
|
+
};
|
|
608
|
+
}
|
|
609
|
+
/** Validate one helper element record. */
|
|
610
|
+
function expectElement(value, op) {
|
|
611
|
+
const record = expectRecordValue(value, op);
|
|
612
|
+
const patterns = record["patterns"];
|
|
613
|
+
if (!Array.isArray(patterns) || patterns.some((item) => typeof item !== "string")) throw new DshClickError(`helper "${op}" returned malformed element patterns`, "BAD_HELPER_RESPONSE");
|
|
614
|
+
return {
|
|
615
|
+
elementId: expectString(record, "elementId", op),
|
|
616
|
+
controlType: expectString(record, "controlType", op),
|
|
617
|
+
name: expectString(record, "name", op),
|
|
618
|
+
automationId: expectString(record, "automationId", op),
|
|
619
|
+
rect: expectRect(record["rect"], op),
|
|
620
|
+
enabled: record["enabled"] === true,
|
|
621
|
+
patterns
|
|
622
|
+
};
|
|
623
|
+
}
|
|
624
|
+
/** Validate one helper pixel hint. */
|
|
625
|
+
function expectPixel(value, op) {
|
|
626
|
+
const record = expectRecordValue(value, op);
|
|
627
|
+
return {
|
|
628
|
+
label: expectString(record, "label", op),
|
|
629
|
+
x: expectNumber(record, "x", op),
|
|
630
|
+
y: expectNumber(record, "y", op),
|
|
631
|
+
color: expectString(record, "color", op)
|
|
632
|
+
};
|
|
633
|
+
}
|
|
634
|
+
/** Validate one helper action outcome. */
|
|
635
|
+
function expectActionOutcome(value, op) {
|
|
636
|
+
const record = expectRecordValue(value, op);
|
|
637
|
+
const delivered = expectString(record, "delivered", op);
|
|
638
|
+
if (delivered !== "uia" && delivered !== "posted" && delivered !== "none") throw new DshClickError(`helper "${op}" returned an unknown delivered mechanism`, "BAD_HELPER_RESPONSE");
|
|
639
|
+
const restored = record["restored"];
|
|
640
|
+
if (restored !== void 0 && typeof restored !== "boolean") throw new DshClickError(`helper "${op}" returned a non-boolean restored flag`, "BAD_HELPER_RESPONSE");
|
|
641
|
+
const detail = record["detail"];
|
|
642
|
+
if (detail !== void 0 && typeof detail !== "string") throw new DshClickError(`helper "${op}" returned a non-string detail`, "BAD_HELPER_RESPONSE");
|
|
643
|
+
return {
|
|
644
|
+
windowId: expectNumber(record, "windowId", op),
|
|
645
|
+
action: expectString(record, "action", op),
|
|
646
|
+
delivered,
|
|
647
|
+
processBefore: expectProcessFacts(record["processBefore"], op),
|
|
648
|
+
processAfter: expectProcessFacts(record["processAfter"], op),
|
|
649
|
+
...restored !== void 0 ? { restored } : {},
|
|
650
|
+
...detail !== void 0 ? { detail } : {}
|
|
651
|
+
};
|
|
652
|
+
}
|
|
653
|
+
/** The Windows helper backend over `ctx.subprocess`. */
|
|
654
|
+
var HelperBackend = class {
|
|
655
|
+
config;
|
|
656
|
+
subprocess;
|
|
657
|
+
/** Resolved powershell executable path, cached across calls. */
|
|
658
|
+
powershell;
|
|
659
|
+
available = true;
|
|
660
|
+
/** @param config - resolved limits for timeout and output size. */
|
|
661
|
+
/** @param subprocess - the subprocess service that runs the helper. */
|
|
662
|
+
constructor(config, subprocess) {
|
|
663
|
+
this.config = config;
|
|
664
|
+
this.subprocess = subprocess;
|
|
665
|
+
}
|
|
666
|
+
platform = "win32";
|
|
667
|
+
/** Run one helper operation and return its validated result payload. */
|
|
668
|
+
async invoke(op, args, signal) {
|
|
669
|
+
const request = {
|
|
670
|
+
protocol: 1,
|
|
671
|
+
pluginVersion: VERSION,
|
|
672
|
+
requestId: randomUUID(),
|
|
673
|
+
op,
|
|
674
|
+
args
|
|
675
|
+
};
|
|
676
|
+
const helperPath = findHelperPath();
|
|
677
|
+
const executable = await this.resolvePowershell();
|
|
678
|
+
const deadline = AbortSignal.timeout(this.config.helperTimeoutMs);
|
|
679
|
+
const fused = signal === void 0 ? deadline : AbortSignal.any([signal, deadline]);
|
|
680
|
+
const handle = this.subprocess.spawn({
|
|
681
|
+
argv: [
|
|
682
|
+
executable,
|
|
683
|
+
"-NoProfile",
|
|
684
|
+
"-NonInteractive",
|
|
685
|
+
"-ExecutionPolicy",
|
|
686
|
+
"Bypass",
|
|
687
|
+
"-File",
|
|
688
|
+
helperPath
|
|
689
|
+
],
|
|
690
|
+
cwd: process.cwd(),
|
|
691
|
+
stdio: {
|
|
692
|
+
stdin: { data: JSON.stringify(request) },
|
|
693
|
+
stdout: { maxBytes: this.config.maxHelperOutputBytes },
|
|
694
|
+
stderr: { maxBytes: 65536 }
|
|
695
|
+
},
|
|
696
|
+
graceMs: 5e3,
|
|
697
|
+
signal: fused
|
|
698
|
+
});
|
|
699
|
+
const outcome = await handle.done;
|
|
700
|
+
const stdoutText = handle.collected.stdout?.readFrom(0).text ?? "";
|
|
701
|
+
const stderrText = handle.collected.stderr?.readFrom(0).text ?? "";
|
|
702
|
+
if (outcome.exitCode !== 0) {
|
|
703
|
+
const tail = stderrText.trim().split(/\r?\n/u).slice(-3).join(" | ");
|
|
704
|
+
throw new DshClickError(`helper "${op}" failed (exit ${outcome.exitCode}${tail === "" ? "" : `: ${tail}`})`, "HELPER_FAILED");
|
|
705
|
+
}
|
|
706
|
+
let response;
|
|
707
|
+
try {
|
|
708
|
+
response = JSON.parse(stdoutText);
|
|
709
|
+
} catch {
|
|
710
|
+
throw new DshClickError(`helper "${op}" produced ${stdoutText.length === 0 ? "no stdout" : "malformed JSON on stdout"}`, "BAD_HELPER_RESPONSE");
|
|
711
|
+
}
|
|
712
|
+
if (typeof response !== "object" || response === null || typeof response.ok !== "boolean") throw new DshClickError(`helper "${op}" returned a malformed response envelope`, "BAD_HELPER_RESPONSE");
|
|
713
|
+
if (!response.ok) {
|
|
714
|
+
const error = response.error;
|
|
715
|
+
if (typeof error !== "object" || error === null || typeof error.code !== "string" || typeof error.message !== "string") throw new DshClickError(`helper "${op}" returned a malformed error envelope`, "BAD_HELPER_RESPONSE");
|
|
716
|
+
throw new DshClickError(`helper "${op}": ${error.message}`, error.code);
|
|
717
|
+
}
|
|
718
|
+
return response.result;
|
|
719
|
+
}
|
|
720
|
+
/** Resolve (and cache) the powershell executable through the subprocess seam. */
|
|
721
|
+
resolvePowershell() {
|
|
722
|
+
this.powershell ??= this.subprocess.resolveExecutable("powershell.exe").catch((error) => {
|
|
723
|
+
this.powershell = void 0;
|
|
724
|
+
throw new DshClickError(`cannot resolve powershell.exe for the desktop helper: ${error instanceof Error ? error.message : String(error)}`, "HELPER_MISSING");
|
|
725
|
+
});
|
|
726
|
+
return this.powershell;
|
|
727
|
+
}
|
|
728
|
+
async listWindows(signal) {
|
|
729
|
+
const result = await this.invoke("windows", {}, signal);
|
|
730
|
+
if (!Array.isArray(result)) throw new DshClickError("helper \"windows\" returned a non-array result", "BAD_HELPER_RESPONSE");
|
|
731
|
+
return result.map((item) => expectWindow(item, "windows"));
|
|
732
|
+
}
|
|
733
|
+
async shot(ref, maxSide, signal) {
|
|
734
|
+
const record = expectRecordValue(await this.invoke("shot", {
|
|
735
|
+
target: ref,
|
|
736
|
+
maxSide,
|
|
737
|
+
maxElements: this.config.maxElements,
|
|
738
|
+
maxDepth: this.config.maxTreeDepth
|
|
739
|
+
}, signal), "shot");
|
|
740
|
+
return {
|
|
741
|
+
pngBase64: expectString(record, "pngBase64", "shot"),
|
|
742
|
+
width: expectNumber(record, "width", "shot"),
|
|
743
|
+
height: expectNumber(record, "height", "shot"),
|
|
744
|
+
snapshot: expectSnapshot(record["snapshot"], "shot")
|
|
745
|
+
};
|
|
746
|
+
}
|
|
747
|
+
async tree(ref, maxElements, maxDepth, includePixels, signal) {
|
|
748
|
+
const record = expectRecordValue(await this.invoke("tree", {
|
|
749
|
+
target: ref,
|
|
750
|
+
maxElements,
|
|
751
|
+
maxDepth,
|
|
752
|
+
includePixels
|
|
753
|
+
}, signal), "tree");
|
|
754
|
+
const elements = record["elements"];
|
|
755
|
+
const pixels = record["pixels"];
|
|
756
|
+
if (!Array.isArray(elements) || !Array.isArray(pixels)) throw new DshClickError("helper \"tree\" returned malformed elements/pixels arrays", "BAD_HELPER_RESPONSE");
|
|
757
|
+
return {
|
|
758
|
+
snapshot: expectSnapshot(record["snapshot"], "tree"),
|
|
759
|
+
elements: elements.map((item) => expectElement(item, "tree")),
|
|
760
|
+
pixels: pixels.map((item) => expectPixel(item, "tree"))
|
|
761
|
+
};
|
|
762
|
+
}
|
|
763
|
+
async snapshot(windowId, signal) {
|
|
764
|
+
return expectSnapshot(await this.invoke("snapshot", {
|
|
765
|
+
windowId,
|
|
766
|
+
maxElements: this.config.maxElements,
|
|
767
|
+
maxDepth: this.config.maxTreeDepth
|
|
768
|
+
}, signal), "snapshot");
|
|
769
|
+
}
|
|
770
|
+
async click(request, focusFallback, signal) {
|
|
771
|
+
return expectActionOutcome(await this.invoke("click", {
|
|
772
|
+
request,
|
|
773
|
+
focusFallback
|
|
774
|
+
}, signal), "click");
|
|
775
|
+
}
|
|
776
|
+
async type(request, focusFallback, signal) {
|
|
777
|
+
return expectActionOutcome(await this.invoke("type", {
|
|
778
|
+
request,
|
|
779
|
+
focusFallback
|
|
780
|
+
}, signal), "type");
|
|
781
|
+
}
|
|
782
|
+
async scroll(request, focusFallback, signal) {
|
|
783
|
+
return expectActionOutcome(await this.invoke("scroll", {
|
|
784
|
+
request,
|
|
785
|
+
focusFallback
|
|
786
|
+
}, signal), "scroll");
|
|
787
|
+
}
|
|
788
|
+
async key(request, focusFallback, signal) {
|
|
789
|
+
return expectActionOutcome(await this.invoke("key", {
|
|
790
|
+
request,
|
|
791
|
+
focusFallback
|
|
792
|
+
}, signal), "key");
|
|
793
|
+
}
|
|
794
|
+
async apps(signal) {
|
|
795
|
+
const result = await this.invoke("apps", {}, signal);
|
|
796
|
+
if (!Array.isArray(result)) throw new DshClickError("helper \"apps\" returned a non-array result", "BAD_HELPER_RESPONSE");
|
|
797
|
+
return result.map((item) => {
|
|
798
|
+
const record = expectRecordValue(item, "apps");
|
|
799
|
+
const windows = record["windows"];
|
|
800
|
+
if (!Array.isArray(windows)) throw new DshClickError("helper \"apps\" returned malformed windows", "BAD_HELPER_RESPONSE");
|
|
801
|
+
const executablePath = record["executablePath"];
|
|
802
|
+
if (executablePath !== null && typeof executablePath !== "string") throw new DshClickError("helper \"apps\" returned malformed executablePath", "BAD_HELPER_RESPONSE");
|
|
803
|
+
return {
|
|
804
|
+
processId: expectNumber(record, "processId", "apps"),
|
|
805
|
+
name: expectString(record, "name", "apps"),
|
|
806
|
+
executablePath,
|
|
807
|
+
windows: windows.map((windowInfo) => expectWindow(windowInfo, "apps"))
|
|
808
|
+
};
|
|
809
|
+
});
|
|
810
|
+
}
|
|
811
|
+
async launch(name, args, signal) {
|
|
812
|
+
const record = expectRecordValue(await this.invoke("launch", {
|
|
813
|
+
name,
|
|
814
|
+
args
|
|
815
|
+
}, signal), "launch");
|
|
816
|
+
const executablePath = record["executablePath"];
|
|
817
|
+
if (executablePath !== null && typeof executablePath !== "string") throw new DshClickError("helper \"launch\" returned malformed executablePath", "BAD_HELPER_RESPONSE");
|
|
818
|
+
return {
|
|
819
|
+
processId: expectNumber(record, "processId", "launch"),
|
|
820
|
+
executablePath
|
|
821
|
+
};
|
|
822
|
+
}
|
|
823
|
+
};
|
|
824
|
+
//#endregion
|
|
825
|
+
//#region src/platform/selection.ts
|
|
826
|
+
/** A backend that cannot act: every operation fails closed with the given reason. */
|
|
827
|
+
var UnavailableBackend = class {
|
|
828
|
+
platform;
|
|
829
|
+
reason;
|
|
830
|
+
/** @param platform - the platform id this stub stands in for. */
|
|
831
|
+
/** @param reason - model-readable reason why the backend cannot act. */
|
|
832
|
+
constructor(platform, reason) {
|
|
833
|
+
this.platform = platform;
|
|
834
|
+
this.reason = reason;
|
|
835
|
+
}
|
|
836
|
+
available = false;
|
|
837
|
+
get unavailableReason() {
|
|
838
|
+
return this.reason;
|
|
839
|
+
}
|
|
840
|
+
/** Throw the shared backend-unavailable failure. */
|
|
841
|
+
refuse() {
|
|
842
|
+
throw new DshClickError(`dsh-click backend unavailable on ${this.platform}: ${this.reason}`, "BACKEND_UNAVAILABLE");
|
|
843
|
+
}
|
|
844
|
+
listWindows() {
|
|
845
|
+
this.refuse();
|
|
846
|
+
}
|
|
847
|
+
shot() {
|
|
848
|
+
this.refuse();
|
|
849
|
+
}
|
|
850
|
+
tree() {
|
|
851
|
+
this.refuse();
|
|
852
|
+
}
|
|
853
|
+
snapshot() {
|
|
854
|
+
this.refuse();
|
|
855
|
+
}
|
|
856
|
+
click() {
|
|
857
|
+
this.refuse();
|
|
858
|
+
}
|
|
859
|
+
type() {
|
|
860
|
+
this.refuse();
|
|
861
|
+
}
|
|
862
|
+
scroll() {
|
|
863
|
+
this.refuse();
|
|
864
|
+
}
|
|
865
|
+
key() {
|
|
866
|
+
this.refuse();
|
|
867
|
+
}
|
|
868
|
+
apps() {
|
|
869
|
+
this.refuse();
|
|
870
|
+
}
|
|
871
|
+
launch() {
|
|
872
|
+
this.refuse();
|
|
873
|
+
}
|
|
874
|
+
};
|
|
875
|
+
/** The reserved-but-unimplemented backends, by platform. */
|
|
876
|
+
const RESERVED_BACKENDS = {
|
|
877
|
+
darwin: "macOS backend is reserved but not implemented yet (Windows-first); only Windows can act today",
|
|
878
|
+
linux: "Linux backend is reserved but not implemented yet (Windows-first); only Windows can act today"
|
|
879
|
+
};
|
|
880
|
+
/**
|
|
881
|
+
* Select the backend for this deployment. Windows needs `ctx.subprocess` to
|
|
882
|
+
* run the native helper; every other platform is reserved and fails closed.
|
|
883
|
+
*
|
|
884
|
+
* @param ctx - the mounting context.
|
|
885
|
+
* @param config - resolved plugin config (helper limits).
|
|
886
|
+
* @returns the active backend (possibly unavailable).
|
|
887
|
+
*/
|
|
888
|
+
function createBackend(ctx, config) {
|
|
889
|
+
if (process.platform === "win32") {
|
|
890
|
+
const subprocess = ctx.get("subprocess");
|
|
891
|
+
if (subprocess === void 0) return new UnavailableBackend("win32", "ctx.subprocess is not mounted — required to run the native Windows helper");
|
|
892
|
+
return new HelperBackend(config, subprocess);
|
|
893
|
+
}
|
|
894
|
+
return new UnavailableBackend(process.platform, RESERVED_BACKENDS[process.platform] ?? "unsupported platform");
|
|
895
|
+
}
|
|
896
|
+
//#endregion
|
|
897
|
+
//#region src/vision.ts
|
|
898
|
+
/**
|
|
899
|
+
* Resolve whether the calling agent's current model declares image input.
|
|
900
|
+
*
|
|
901
|
+
* @param agent - the calling agent (may be undefined).
|
|
902
|
+
* @param ctx - the mounting context.
|
|
903
|
+
* @param signal - cancellation for the model-info lookup.
|
|
904
|
+
* @returns true only when the resolved model declares the `image` modality.
|
|
905
|
+
*/
|
|
906
|
+
async function sessionAcceptsImages(agent, ctx, signal) {
|
|
907
|
+
if (agent === void 0) return false;
|
|
908
|
+
const provider = agent.options.provider;
|
|
909
|
+
const model = agent.options.model;
|
|
910
|
+
if (provider === void 0 || model === void 0) return false;
|
|
911
|
+
const llm = ctx.get("llm");
|
|
912
|
+
if (llm === void 0) return false;
|
|
913
|
+
try {
|
|
914
|
+
return (await llm.resolveModelInfo(provider, model, signal)).inputModalities?.includes("image") ?? false;
|
|
915
|
+
} catch {
|
|
916
|
+
return false;
|
|
917
|
+
}
|
|
918
|
+
}
|
|
919
|
+
//#endregion
|
|
920
|
+
//#region src/tools.ts
|
|
921
|
+
/** Sanitize one snapshot into the model-visible window value. */
|
|
922
|
+
function observedWindow(snapshot, maxTextLength) {
|
|
923
|
+
return {
|
|
924
|
+
windowId: snapshot.windowId,
|
|
925
|
+
processId: snapshot.processId,
|
|
926
|
+
title: sanitizeVisible(snapshot.title, maxTextLength),
|
|
927
|
+
className: sanitizeVisible(snapshot.className, maxTextLength),
|
|
928
|
+
executablePath: snapshot.executablePath === null ? null : sanitizePath(snapshot.executablePath, maxTextLength),
|
|
929
|
+
rect: { ...snapshot.rect },
|
|
930
|
+
foreground: snapshot.foreground
|
|
931
|
+
};
|
|
932
|
+
}
|
|
933
|
+
/** Sanitize one accessibility element into the model-visible form. */
|
|
934
|
+
function observedElement(element, maxTextLength) {
|
|
935
|
+
return {
|
|
936
|
+
elementId: element.elementId,
|
|
937
|
+
controlType: sanitizeVisible(element.controlType, 64),
|
|
938
|
+
name: sanitizeVisible(element.name, maxTextLength),
|
|
939
|
+
automationId: sanitizeVisible(element.automationId, 64),
|
|
940
|
+
rect: element.rect,
|
|
941
|
+
enabled: element.enabled,
|
|
942
|
+
patterns: element.patterns
|
|
943
|
+
};
|
|
944
|
+
}
|
|
945
|
+
/** Sanitize one pixel hint into the model-visible form. */
|
|
946
|
+
function observedPixel(pixel, maxTextLength) {
|
|
947
|
+
return {
|
|
948
|
+
label: sanitizeVisible(pixel.label, maxTextLength),
|
|
949
|
+
x: pixel.x,
|
|
950
|
+
y: pixel.y,
|
|
951
|
+
color: pixel.color
|
|
952
|
+
};
|
|
953
|
+
}
|
|
954
|
+
/** Sanitize one window listing entry. */
|
|
955
|
+
function observedWindowInfo(info, maxTextLength) {
|
|
956
|
+
return {
|
|
957
|
+
windowId: info.windowId,
|
|
958
|
+
processId: info.processId,
|
|
959
|
+
title: sanitizeVisible(info.title, maxTextLength),
|
|
960
|
+
className: sanitizeVisible(info.className, maxTextLength),
|
|
961
|
+
rect: info.rect,
|
|
962
|
+
executablePath: info.executablePath === null ? null : sanitizePath(info.executablePath, maxTextLength),
|
|
963
|
+
visible: info.visible
|
|
964
|
+
};
|
|
965
|
+
}
|
|
966
|
+
/** Append the `dsh-click/observed` audit event; a failed append is swallowed. */
|
|
967
|
+
function auditObservation(exec, event) {
|
|
968
|
+
const session = exec.agent?.session;
|
|
969
|
+
if (session === void 0) return;
|
|
970
|
+
try {
|
|
971
|
+
session.append(OBSERVED_EVENT, event);
|
|
972
|
+
} catch {}
|
|
973
|
+
}
|
|
974
|
+
/** One-line window identity for render text. */
|
|
975
|
+
function windowLine(window) {
|
|
976
|
+
return `"${window.title}" (windowId ${window.windowId}, pid ${window.processId}, ${window.executablePath ?? "unknown executable"})`;
|
|
977
|
+
}
|
|
978
|
+
/** Refuse an unusable backend before any backend call. */
|
|
979
|
+
function backendGuard(services) {
|
|
980
|
+
if (!services.backend.available) throw new Error(`dsh-click backend unavailable on ${services.backend.platform}: ${services.backend.unavailableReason ?? "unknown reason"}`);
|
|
981
|
+
}
|
|
982
|
+
/** Plain-text fallback description for a captured screenshot. */
|
|
983
|
+
function shotDescription(window, width, height) {
|
|
984
|
+
return `${width}x${height} screenshot of ${windowLine(window)}; run screen_read on the same window for the structured element list and pixel positions.`;
|
|
985
|
+
}
|
|
986
|
+
/**
|
|
987
|
+
* `screen_shot` — capture the addressed window (or the primary screen).
|
|
988
|
+
* In `imageMode: 'auto'` the result carries an image attachment when the
|
|
989
|
+
* current model accepts images; otherwise a text description keeps text-only
|
|
990
|
+
* models working.
|
|
991
|
+
*
|
|
992
|
+
* @param services - runtime services.
|
|
993
|
+
* @returns the tool definition.
|
|
994
|
+
*/
|
|
995
|
+
function screenShotTool(services) {
|
|
996
|
+
const { ctx, config, backend, observations } = services;
|
|
997
|
+
return defineTool({
|
|
998
|
+
name: "screen_shot",
|
|
999
|
+
description: "Capture a screenshot of a desktop window (or the primary screen when no window is given). Returns an observationId that later actions cite in `basedOn`. When the current model accepts images the result includes the image; otherwise it includes a text description. Read-only: never needs approval.",
|
|
1000
|
+
parameters: {
|
|
1001
|
+
target: {
|
|
1002
|
+
type: "object",
|
|
1003
|
+
description: "Which window to capture (windowId, windowTitle, or processId); omitted = the foreground window.",
|
|
1004
|
+
properties: {
|
|
1005
|
+
windowId: {
|
|
1006
|
+
type: "integer",
|
|
1007
|
+
description: "Native window handle from app_list or screen_read."
|
|
1008
|
+
},
|
|
1009
|
+
windowTitle: {
|
|
1010
|
+
type: "string",
|
|
1011
|
+
description: "Visible window title (matched case-insensitively by substring)."
|
|
1012
|
+
},
|
|
1013
|
+
processId: {
|
|
1014
|
+
type: "integer",
|
|
1015
|
+
description: "Owning process id."
|
|
1016
|
+
}
|
|
1017
|
+
},
|
|
1018
|
+
additionalProperties: false
|
|
1019
|
+
},
|
|
1020
|
+
maxSide: {
|
|
1021
|
+
type: "integer",
|
|
1022
|
+
description: "Longest side in pixels; larger captures are downscaled."
|
|
1023
|
+
}
|
|
1024
|
+
},
|
|
1025
|
+
output: {
|
|
1026
|
+
schema: {
|
|
1027
|
+
type: "object",
|
|
1028
|
+
properties: {
|
|
1029
|
+
ok: {
|
|
1030
|
+
type: "boolean",
|
|
1031
|
+
const: true
|
|
1032
|
+
},
|
|
1033
|
+
observationId: { type: "string" },
|
|
1034
|
+
window: {
|
|
1035
|
+
type: "object",
|
|
1036
|
+
properties: {
|
|
1037
|
+
windowId: { type: "integer" },
|
|
1038
|
+
processId: { type: "integer" },
|
|
1039
|
+
title: { type: "string" },
|
|
1040
|
+
className: { type: "string" },
|
|
1041
|
+
executablePath: { oneOf: [{ type: "string" }, { type: "null" }] },
|
|
1042
|
+
rect: {
|
|
1043
|
+
type: "object",
|
|
1044
|
+
properties: {
|
|
1045
|
+
x: { type: "integer" },
|
|
1046
|
+
y: { type: "integer" },
|
|
1047
|
+
width: { type: "integer" },
|
|
1048
|
+
height: { type: "integer" }
|
|
1049
|
+
},
|
|
1050
|
+
additionalProperties: false
|
|
1051
|
+
},
|
|
1052
|
+
foreground: { type: "boolean" }
|
|
1053
|
+
},
|
|
1054
|
+
additionalProperties: false
|
|
1055
|
+
},
|
|
1056
|
+
image: {
|
|
1057
|
+
type: "object",
|
|
1058
|
+
properties: {
|
|
1059
|
+
attachmentId: { type: "string" },
|
|
1060
|
+
mediaType: { type: "string" },
|
|
1061
|
+
bytes: { type: "integer" },
|
|
1062
|
+
width: { type: "integer" },
|
|
1063
|
+
height: { type: "integer" },
|
|
1064
|
+
name: { type: "string" }
|
|
1065
|
+
},
|
|
1066
|
+
additionalProperties: false
|
|
1067
|
+
},
|
|
1068
|
+
description: { type: "string" }
|
|
1069
|
+
},
|
|
1070
|
+
additionalProperties: false
|
|
1071
|
+
},
|
|
1072
|
+
render(_args, value) {
|
|
1073
|
+
const result = value;
|
|
1074
|
+
const blocks = [{
|
|
1075
|
+
type: "text",
|
|
1076
|
+
text: `Screenshot of ${windowLine(result.window)} captured.\nobservationId: ${result.observationId}\n${result.image === void 0 ? result.description : "Image attached; cite this observationId in later actions."}`
|
|
1077
|
+
}];
|
|
1078
|
+
if (result.image !== void 0) blocks.push({
|
|
1079
|
+
type: "image",
|
|
1080
|
+
attachment: result.image
|
|
1081
|
+
});
|
|
1082
|
+
return blocks;
|
|
1083
|
+
}
|
|
1084
|
+
},
|
|
1085
|
+
timeoutMs: config.helperTimeoutMs + 15e3,
|
|
1086
|
+
async execute(args, exec) {
|
|
1087
|
+
backendGuard(services);
|
|
1088
|
+
const target = args.target ?? {};
|
|
1089
|
+
const requestedSide = args.maxSide;
|
|
1090
|
+
const maxSide = requestedSide === void 0 ? config.maxScreenshotSide : Math.min(requestedSide, config.maxScreenshotSide);
|
|
1091
|
+
const shot = await backend.shot(target, maxSide, exec.signal);
|
|
1092
|
+
const sanitized = observedWindow(shot.snapshot, config.maxTextLength);
|
|
1093
|
+
const record = observations.record(shot.snapshot);
|
|
1094
|
+
const description = shotDescription(sanitized, shot.width, shot.height);
|
|
1095
|
+
let image;
|
|
1096
|
+
const attachments = ctx.get("attachments");
|
|
1097
|
+
if (config.imageMode !== "text" && attachments !== void 0 && await sessionAcceptsImages(exec.agent, ctx, exec.signal)) try {
|
|
1098
|
+
image = await attachments.saveImage({
|
|
1099
|
+
data: Buffer.from(shot.pngBase64, "base64"),
|
|
1100
|
+
mediaType: "image/png",
|
|
1101
|
+
name: `dsh-click-${record.id.slice(0, 12)}.png`
|
|
1102
|
+
});
|
|
1103
|
+
} catch {
|
|
1104
|
+
image = void 0;
|
|
1105
|
+
}
|
|
1106
|
+
auditObservation(exec, {
|
|
1107
|
+
observationId: record.id,
|
|
1108
|
+
windowId: shot.snapshot.windowId,
|
|
1109
|
+
processId: shot.snapshot.processId,
|
|
1110
|
+
executablePath: sanitized.executablePath,
|
|
1111
|
+
windowTitle: sanitized.title,
|
|
1112
|
+
elementCount: shot.snapshot.elementCount,
|
|
1113
|
+
...image !== void 0 ? { image: {
|
|
1114
|
+
attachmentId: image.attachmentId,
|
|
1115
|
+
mediaType: image.mediaType,
|
|
1116
|
+
bytes: image.bytes,
|
|
1117
|
+
width: image.width,
|
|
1118
|
+
height: image.height
|
|
1119
|
+
} } : {}
|
|
1120
|
+
});
|
|
1121
|
+
return {
|
|
1122
|
+
ok: true,
|
|
1123
|
+
observationId: record.id,
|
|
1124
|
+
window: sanitized,
|
|
1125
|
+
...image !== void 0 ? { image } : {},
|
|
1126
|
+
description
|
|
1127
|
+
};
|
|
1128
|
+
}
|
|
1129
|
+
});
|
|
1130
|
+
}
|
|
1131
|
+
/**
|
|
1132
|
+
* `screen_read` — the structured observation for text-only models: the
|
|
1133
|
+
* window's accessibility tree plus pixel-location hints. The returned
|
|
1134
|
+
* elementIds are what click/type/scroll address.
|
|
1135
|
+
*
|
|
1136
|
+
* @param services - runtime services.
|
|
1137
|
+
* @returns the tool definition.
|
|
1138
|
+
*/
|
|
1139
|
+
function screenReadTool(services) {
|
|
1140
|
+
const { config, backend, observations } = services;
|
|
1141
|
+
return defineTool({
|
|
1142
|
+
name: "screen_read",
|
|
1143
|
+
description: "Read a desktop window as structured text: its accessibility tree (element ids, types, names, rectangles, supported patterns) plus pixel-location hints with colors. Returns an observationId that later actions cite in `basedOn`; elements are addressed by their elementId. Read-only: never needs approval.",
|
|
1144
|
+
parameters: {
|
|
1145
|
+
target: {
|
|
1146
|
+
type: "object",
|
|
1147
|
+
description: "Which window to read (windowId, windowTitle, or processId); omitted = the foreground window.",
|
|
1148
|
+
properties: {
|
|
1149
|
+
windowId: {
|
|
1150
|
+
type: "integer",
|
|
1151
|
+
description: "Native window handle from app_list or screen_read."
|
|
1152
|
+
},
|
|
1153
|
+
windowTitle: {
|
|
1154
|
+
type: "string",
|
|
1155
|
+
description: "Visible window title (matched case-insensitively by substring)."
|
|
1156
|
+
},
|
|
1157
|
+
processId: {
|
|
1158
|
+
type: "integer",
|
|
1159
|
+
description: "Owning process id."
|
|
1160
|
+
}
|
|
1161
|
+
},
|
|
1162
|
+
additionalProperties: false
|
|
1163
|
+
},
|
|
1164
|
+
includePixels: {
|
|
1165
|
+
type: "boolean",
|
|
1166
|
+
description: "Include pixel-location hints (default true)."
|
|
1167
|
+
}
|
|
1168
|
+
},
|
|
1169
|
+
output: {
|
|
1170
|
+
schema: {
|
|
1171
|
+
type: "object",
|
|
1172
|
+
properties: {
|
|
1173
|
+
ok: {
|
|
1174
|
+
type: "boolean",
|
|
1175
|
+
const: true
|
|
1176
|
+
},
|
|
1177
|
+
observationId: { type: "string" },
|
|
1178
|
+
window: {
|
|
1179
|
+
type: "object",
|
|
1180
|
+
properties: {
|
|
1181
|
+
windowId: { type: "integer" },
|
|
1182
|
+
processId: { type: "integer" },
|
|
1183
|
+
title: { type: "string" },
|
|
1184
|
+
className: { type: "string" },
|
|
1185
|
+
executablePath: { oneOf: [{ type: "string" }, { type: "null" }] },
|
|
1186
|
+
rect: {
|
|
1187
|
+
type: "object",
|
|
1188
|
+
properties: {
|
|
1189
|
+
x: { type: "integer" },
|
|
1190
|
+
y: { type: "integer" },
|
|
1191
|
+
width: { type: "integer" },
|
|
1192
|
+
height: { type: "integer" }
|
|
1193
|
+
},
|
|
1194
|
+
additionalProperties: false
|
|
1195
|
+
},
|
|
1196
|
+
foreground: { type: "boolean" }
|
|
1197
|
+
},
|
|
1198
|
+
additionalProperties: false
|
|
1199
|
+
},
|
|
1200
|
+
elements: {
|
|
1201
|
+
type: "array",
|
|
1202
|
+
items: {
|
|
1203
|
+
type: "object",
|
|
1204
|
+
properties: {
|
|
1205
|
+
elementId: { type: "string" },
|
|
1206
|
+
controlType: { type: "string" },
|
|
1207
|
+
name: { type: "string" },
|
|
1208
|
+
automationId: { type: "string" },
|
|
1209
|
+
rect: {
|
|
1210
|
+
type: "object",
|
|
1211
|
+
properties: {
|
|
1212
|
+
x: { type: "integer" },
|
|
1213
|
+
y: { type: "integer" },
|
|
1214
|
+
width: { type: "integer" },
|
|
1215
|
+
height: { type: "integer" }
|
|
1216
|
+
},
|
|
1217
|
+
additionalProperties: false
|
|
1218
|
+
},
|
|
1219
|
+
enabled: { type: "boolean" },
|
|
1220
|
+
patterns: {
|
|
1221
|
+
type: "array",
|
|
1222
|
+
items: { type: "string" }
|
|
1223
|
+
}
|
|
1224
|
+
},
|
|
1225
|
+
additionalProperties: false
|
|
1226
|
+
}
|
|
1227
|
+
},
|
|
1228
|
+
pixels: {
|
|
1229
|
+
type: "array",
|
|
1230
|
+
items: {
|
|
1231
|
+
type: "object",
|
|
1232
|
+
properties: {
|
|
1233
|
+
label: { type: "string" },
|
|
1234
|
+
x: { type: "integer" },
|
|
1235
|
+
y: { type: "integer" },
|
|
1236
|
+
color: { type: "string" }
|
|
1237
|
+
},
|
|
1238
|
+
additionalProperties: false
|
|
1239
|
+
}
|
|
1240
|
+
}
|
|
1241
|
+
},
|
|
1242
|
+
additionalProperties: false
|
|
1243
|
+
},
|
|
1244
|
+
render(_args, value) {
|
|
1245
|
+
const result = value;
|
|
1246
|
+
const lines = [
|
|
1247
|
+
`Window read: ${windowLine(result.window)}`,
|
|
1248
|
+
`observationId: ${result.observationId}`,
|
|
1249
|
+
`${result.elements.length} element(s):`
|
|
1250
|
+
];
|
|
1251
|
+
for (const element of result.elements) lines.push(`- [${element.elementId}] ${element.controlType} ${JSON.stringify(element.name)} at (${element.rect.x}, ${element.rect.y}) ${element.rect.width}x${element.rect.height}${element.enabled ? "" : " (disabled)"}${element.patterns.length > 0 ? ` patterns: ${element.patterns.join(",")}` : ""}`);
|
|
1252
|
+
if (result.pixels.length > 0) {
|
|
1253
|
+
lines.push(`${result.pixels.length} pixel hint(s):`);
|
|
1254
|
+
for (const pixel of result.pixels) lines.push(`- ${pixel.label} at (${pixel.x}, ${pixel.y}) ${pixel.color}`);
|
|
1255
|
+
}
|
|
1256
|
+
lines.push("Actions (click/type/scroll/key) must cite this observationId in `basedOn` and address elements by elementId.");
|
|
1257
|
+
return [{
|
|
1258
|
+
type: "text",
|
|
1259
|
+
text: lines.join("\n")
|
|
1260
|
+
}];
|
|
1261
|
+
}
|
|
1262
|
+
},
|
|
1263
|
+
timeoutMs: config.helperTimeoutMs + 15e3,
|
|
1264
|
+
async execute(args, exec) {
|
|
1265
|
+
backendGuard(services);
|
|
1266
|
+
const parsed = args;
|
|
1267
|
+
const tree = await backend.tree(parsed.target ?? {}, config.maxElements, config.maxTreeDepth, parsed.includePixels ?? true, exec.signal);
|
|
1268
|
+
const sanitized = observedWindow(tree.snapshot, config.maxTextLength);
|
|
1269
|
+
const record = observations.record(tree.snapshot);
|
|
1270
|
+
auditObservation(exec, {
|
|
1271
|
+
observationId: record.id,
|
|
1272
|
+
windowId: tree.snapshot.windowId,
|
|
1273
|
+
processId: tree.snapshot.processId,
|
|
1274
|
+
executablePath: sanitized.executablePath,
|
|
1275
|
+
windowTitle: sanitized.title,
|
|
1276
|
+
elementCount: tree.elements.length
|
|
1277
|
+
});
|
|
1278
|
+
return {
|
|
1279
|
+
ok: true,
|
|
1280
|
+
observationId: record.id,
|
|
1281
|
+
window: sanitized,
|
|
1282
|
+
elements: tree.elements.map((element) => observedElement(element, config.maxTextLength)),
|
|
1283
|
+
pixels: tree.pixels.map((pixel) => observedPixel(pixel, config.maxTextLength))
|
|
1284
|
+
};
|
|
1285
|
+
}
|
|
1286
|
+
});
|
|
1287
|
+
}
|
|
1288
|
+
/** The shared window-action output schema (click/scroll/key). */
|
|
1289
|
+
function actionOutputSchema(withRestored) {
|
|
1290
|
+
return {
|
|
1291
|
+
type: "object",
|
|
1292
|
+
properties: {
|
|
1293
|
+
ok: {
|
|
1294
|
+
type: "boolean",
|
|
1295
|
+
const: true
|
|
1296
|
+
},
|
|
1297
|
+
windowId: { type: "integer" },
|
|
1298
|
+
delivered: {
|
|
1299
|
+
type: "string",
|
|
1300
|
+
enum: [
|
|
1301
|
+
"uia",
|
|
1302
|
+
"posted",
|
|
1303
|
+
"none"
|
|
1304
|
+
]
|
|
1305
|
+
},
|
|
1306
|
+
process: {
|
|
1307
|
+
type: "object",
|
|
1308
|
+
properties: {
|
|
1309
|
+
before: {
|
|
1310
|
+
type: "object",
|
|
1311
|
+
properties: {
|
|
1312
|
+
pid: { type: "integer" },
|
|
1313
|
+
executablePath: { oneOf: [{ type: "string" }, { type: "null" }] }
|
|
1314
|
+
},
|
|
1315
|
+
additionalProperties: false
|
|
1316
|
+
},
|
|
1317
|
+
after: {
|
|
1318
|
+
type: "object",
|
|
1319
|
+
properties: {
|
|
1320
|
+
pid: { type: "integer" },
|
|
1321
|
+
executablePath: { oneOf: [{ type: "string" }, { type: "null" }] }
|
|
1322
|
+
},
|
|
1323
|
+
additionalProperties: false
|
|
1324
|
+
}
|
|
1325
|
+
},
|
|
1326
|
+
additionalProperties: false
|
|
1327
|
+
},
|
|
1328
|
+
detail: { type: "string" },
|
|
1329
|
+
...withRestored ? { restored: { type: "boolean" } } : {}
|
|
1330
|
+
},
|
|
1331
|
+
additionalProperties: false
|
|
1332
|
+
};
|
|
1333
|
+
}
|
|
1334
|
+
/** The cited-observation parameters shared by the four action tools. */
|
|
1335
|
+
const basedOnParameters = { basedOn: {
|
|
1336
|
+
type: "object",
|
|
1337
|
+
description: "The observation this action is based on; the action fails if the screen changed since that observation.",
|
|
1338
|
+
properties: {
|
|
1339
|
+
observationId: {
|
|
1340
|
+
type: "string",
|
|
1341
|
+
description: "observationId returned by screen_shot or screen_read.",
|
|
1342
|
+
required: true
|
|
1343
|
+
},
|
|
1344
|
+
windowId: {
|
|
1345
|
+
type: "integer",
|
|
1346
|
+
description: "windowId of the observed window.",
|
|
1347
|
+
required: true
|
|
1348
|
+
}
|
|
1349
|
+
},
|
|
1350
|
+
additionalProperties: false,
|
|
1351
|
+
required: true
|
|
1352
|
+
} };
|
|
1353
|
+
/** One-line action summary for render text. */
|
|
1354
|
+
function actionLine(toolName, value) {
|
|
1355
|
+
const before = value.process.before;
|
|
1356
|
+
const after = value.process.after;
|
|
1357
|
+
return `${toolName} delivered via ${value.delivered} to window ${value.windowId}; process pid ${before.pid} → ${after.pid}${value.restored === void 0 ? "" : value.restored ? " (original text restored)" : " (text restore unavailable)"}.`;
|
|
1358
|
+
}
|
|
1359
|
+
/**
|
|
1360
|
+
* `click` — click an element (by elementId) or a coordinate inside the
|
|
1361
|
+
* observed window. Never steals foreground focus; delivery is UIA invoke
|
|
1362
|
+
* where possible, posted window messages otherwise.
|
|
1363
|
+
*
|
|
1364
|
+
* @param services - runtime services.
|
|
1365
|
+
* @returns the tool definition.
|
|
1366
|
+
*/
|
|
1367
|
+
function clickTool(services) {
|
|
1368
|
+
const { config, backend, actions } = services;
|
|
1369
|
+
return defineTool({
|
|
1370
|
+
name: "click",
|
|
1371
|
+
description: "Click an element or a coordinate inside an observed desktop window. Requires `basedOn` (a fresh observationId from screen_shot/screen_read); the call fails if the screen changed since that observation. Never steals foreground focus. Requires approval unless the window is allowlisted.",
|
|
1372
|
+
parameters: {
|
|
1373
|
+
...basedOnParameters,
|
|
1374
|
+
target: {
|
|
1375
|
+
type: "object",
|
|
1376
|
+
description: "Exactly one of elementId or (x, y) — screen coordinates.",
|
|
1377
|
+
properties: {
|
|
1378
|
+
elementId: {
|
|
1379
|
+
type: "string",
|
|
1380
|
+
description: "Element id from screen_read."
|
|
1381
|
+
},
|
|
1382
|
+
x: {
|
|
1383
|
+
type: "integer",
|
|
1384
|
+
description: "Screen x coordinate."
|
|
1385
|
+
},
|
|
1386
|
+
y: {
|
|
1387
|
+
type: "integer",
|
|
1388
|
+
description: "Screen y coordinate."
|
|
1389
|
+
}
|
|
1390
|
+
},
|
|
1391
|
+
additionalProperties: false,
|
|
1392
|
+
required: true
|
|
1393
|
+
},
|
|
1394
|
+
button: {
|
|
1395
|
+
type: "string",
|
|
1396
|
+
enum: ["left", "right"],
|
|
1397
|
+
description: "Mouse button (default left)."
|
|
1398
|
+
}
|
|
1399
|
+
},
|
|
1400
|
+
output: {
|
|
1401
|
+
schema: actionOutputSchema(false),
|
|
1402
|
+
render(_args, value) {
|
|
1403
|
+
return [{
|
|
1404
|
+
type: "text",
|
|
1405
|
+
text: actionLine("click", value)
|
|
1406
|
+
}];
|
|
1407
|
+
}
|
|
1408
|
+
},
|
|
1409
|
+
timeoutMs: config.helperTimeoutMs + 15e3,
|
|
1410
|
+
async execute(args, exec) {
|
|
1411
|
+
backendGuard(services);
|
|
1412
|
+
const parsed = args;
|
|
1413
|
+
const target = parsed.target;
|
|
1414
|
+
const byElement = target.elementId !== void 0;
|
|
1415
|
+
if (byElement === (target.x !== void 0 && target.y !== void 0)) throw new Error("click target must name exactly one of elementId or (x, y)");
|
|
1416
|
+
const outcome = await actions.perform("click", exec, parsed.basedOn.observationId, parsed.basedOn.windowId, (focusFallback) => backend.click({
|
|
1417
|
+
windowId: parsed.basedOn.windowId,
|
|
1418
|
+
...byElement ? { elementId: target.elementId } : {
|
|
1419
|
+
x: target.x,
|
|
1420
|
+
y: target.y
|
|
1421
|
+
},
|
|
1422
|
+
button: parsed.button ?? "left"
|
|
1423
|
+
}, focusFallback, exec.signal));
|
|
1424
|
+
return {
|
|
1425
|
+
ok: true,
|
|
1426
|
+
windowId: outcome.windowId,
|
|
1427
|
+
delivered: outcome.delivered,
|
|
1428
|
+
process: {
|
|
1429
|
+
before: outcome.processBefore,
|
|
1430
|
+
after: outcome.processAfter
|
|
1431
|
+
},
|
|
1432
|
+
...outcome.detail !== void 0 ? { detail: outcome.detail } : {}
|
|
1433
|
+
};
|
|
1434
|
+
}
|
|
1435
|
+
});
|
|
1436
|
+
}
|
|
1437
|
+
/**
|
|
1438
|
+
* `type` — type text into a value-pattern element. The helper backs up the
|
|
1439
|
+
* control text first and restores it when the action fails (config
|
|
1440
|
+
* `rollbackEnabled`).
|
|
1441
|
+
*
|
|
1442
|
+
* @param services - runtime services.
|
|
1443
|
+
* @returns the tool definition.
|
|
1444
|
+
*/
|
|
1445
|
+
function typeTool(services) {
|
|
1446
|
+
const { config, backend, actions } = services;
|
|
1447
|
+
return defineTool({
|
|
1448
|
+
name: "type",
|
|
1449
|
+
description: "Type text into an editable element of an observed desktop window (addressed by elementId from screen_read; the element must expose a value pattern). Requires `basedOn`; fails if the screen changed since that observation. Never steals foreground focus. Requires approval unless the window is allowlisted. On failure the previous control text is restored when rollback is enabled.",
|
|
1450
|
+
parameters: {
|
|
1451
|
+
...basedOnParameters,
|
|
1452
|
+
elementId: {
|
|
1453
|
+
type: "string",
|
|
1454
|
+
description: "Editable element id from screen_read.",
|
|
1455
|
+
required: true
|
|
1456
|
+
},
|
|
1457
|
+
text: {
|
|
1458
|
+
type: "string",
|
|
1459
|
+
description: "The exact text to type (up to 10000 characters).",
|
|
1460
|
+
required: true
|
|
1461
|
+
}
|
|
1462
|
+
},
|
|
1463
|
+
output: {
|
|
1464
|
+
schema: actionOutputSchema(true),
|
|
1465
|
+
render(_args, value) {
|
|
1466
|
+
return [{
|
|
1467
|
+
type: "text",
|
|
1468
|
+
text: actionLine("type", value)
|
|
1469
|
+
}];
|
|
1470
|
+
}
|
|
1471
|
+
},
|
|
1472
|
+
timeoutMs: config.helperTimeoutMs + 15e3,
|
|
1473
|
+
async execute(args, exec) {
|
|
1474
|
+
backendGuard(services);
|
|
1475
|
+
const parsed = args;
|
|
1476
|
+
if (parsed.text.length > 1e4) throw new Error("type text must be at most 10000 characters");
|
|
1477
|
+
const outcome = await actions.perform("type", exec, parsed.basedOn.observationId, parsed.basedOn.windowId, (focusFallback) => backend.type({
|
|
1478
|
+
windowId: parsed.basedOn.windowId,
|
|
1479
|
+
elementId: parsed.elementId,
|
|
1480
|
+
text: parsed.text,
|
|
1481
|
+
rollback: config.rollbackEnabled
|
|
1482
|
+
}, focusFallback, exec.signal));
|
|
1483
|
+
return {
|
|
1484
|
+
ok: true,
|
|
1485
|
+
windowId: outcome.windowId,
|
|
1486
|
+
delivered: outcome.delivered,
|
|
1487
|
+
process: {
|
|
1488
|
+
before: outcome.processBefore,
|
|
1489
|
+
after: outcome.processAfter
|
|
1490
|
+
},
|
|
1491
|
+
restored: outcome.restored ?? false,
|
|
1492
|
+
...outcome.detail !== void 0 ? { detail: outcome.detail } : {}
|
|
1493
|
+
};
|
|
1494
|
+
}
|
|
1495
|
+
});
|
|
1496
|
+
}
|
|
1497
|
+
/**
|
|
1498
|
+
* `scroll` — scroll an element (scroll pattern) or the observed window
|
|
1499
|
+
* (posted wheel messages). Never steals foreground focus.
|
|
1500
|
+
*
|
|
1501
|
+
* @param services - runtime services.
|
|
1502
|
+
* @returns the tool definition.
|
|
1503
|
+
*/
|
|
1504
|
+
function scrollTool(services) {
|
|
1505
|
+
const { config, backend, actions } = services;
|
|
1506
|
+
return defineTool({
|
|
1507
|
+
name: "scroll",
|
|
1508
|
+
description: "Scroll an element (by elementId) or the observed window. Requires `basedOn`; fails if the screen changed since that observation. Never steals foreground focus. Requires approval unless the window is allowlisted.",
|
|
1509
|
+
parameters: {
|
|
1510
|
+
...basedOnParameters,
|
|
1511
|
+
elementId: {
|
|
1512
|
+
type: "string",
|
|
1513
|
+
description: "Optional scrollable element id from screen_read; omitted = scroll the window itself."
|
|
1514
|
+
},
|
|
1515
|
+
direction: {
|
|
1516
|
+
type: "string",
|
|
1517
|
+
enum: [
|
|
1518
|
+
"up",
|
|
1519
|
+
"down",
|
|
1520
|
+
"page-up",
|
|
1521
|
+
"page-down"
|
|
1522
|
+
],
|
|
1523
|
+
description: "Scroll direction.",
|
|
1524
|
+
required: true
|
|
1525
|
+
},
|
|
1526
|
+
amount: {
|
|
1527
|
+
type: "integer",
|
|
1528
|
+
description: "Number of increments (default 3)."
|
|
1529
|
+
}
|
|
1530
|
+
},
|
|
1531
|
+
output: {
|
|
1532
|
+
schema: actionOutputSchema(false),
|
|
1533
|
+
render(_args, value) {
|
|
1534
|
+
return [{
|
|
1535
|
+
type: "text",
|
|
1536
|
+
text: actionLine("scroll", value)
|
|
1537
|
+
}];
|
|
1538
|
+
}
|
|
1539
|
+
},
|
|
1540
|
+
timeoutMs: config.helperTimeoutMs + 15e3,
|
|
1541
|
+
async execute(args, exec) {
|
|
1542
|
+
backendGuard(services);
|
|
1543
|
+
const parsed = args;
|
|
1544
|
+
const amount = parsed.amount ?? 3;
|
|
1545
|
+
if (!Number.isInteger(amount) || amount < 1) throw new Error("scroll amount must be a positive integer");
|
|
1546
|
+
const outcome = await actions.perform("scroll", exec, parsed.basedOn.observationId, parsed.basedOn.windowId, (focusFallback) => backend.scroll({
|
|
1547
|
+
windowId: parsed.basedOn.windowId,
|
|
1548
|
+
...parsed.elementId !== void 0 ? { elementId: parsed.elementId } : {},
|
|
1549
|
+
direction: parsed.direction,
|
|
1550
|
+
amount
|
|
1551
|
+
}, focusFallback, exec.signal));
|
|
1552
|
+
return {
|
|
1553
|
+
ok: true,
|
|
1554
|
+
windowId: outcome.windowId,
|
|
1555
|
+
delivered: outcome.delivered,
|
|
1556
|
+
process: {
|
|
1557
|
+
before: outcome.processBefore,
|
|
1558
|
+
after: outcome.processAfter
|
|
1559
|
+
},
|
|
1560
|
+
...outcome.detail !== void 0 ? { detail: outcome.detail } : {}
|
|
1561
|
+
};
|
|
1562
|
+
}
|
|
1563
|
+
});
|
|
1564
|
+
}
|
|
1565
|
+
/**
|
|
1566
|
+
* `key` — send a key combination to the observed window as posted window
|
|
1567
|
+
* messages. Never steals foreground focus; apps that ignore posted input
|
|
1568
|
+
* fail with a clear error.
|
|
1569
|
+
*
|
|
1570
|
+
* @param services - runtime services.
|
|
1571
|
+
* @returns the tool definition.
|
|
1572
|
+
*/
|
|
1573
|
+
function keyTool(services) {
|
|
1574
|
+
const { config, backend, actions } = services;
|
|
1575
|
+
return defineTool({
|
|
1576
|
+
name: "key",
|
|
1577
|
+
description: "Send a key combination (e.g. \"Ctrl+S\", \"Enter\", \"Alt+F4\") to an observed desktop window via posted window messages. Requires `basedOn`; fails if the screen changed since that observation. Never steals foreground focus — applications that ignore posted input will not react; prefer click/type for those. Requires approval unless the window is allowlisted.",
|
|
1578
|
+
parameters: {
|
|
1579
|
+
...basedOnParameters,
|
|
1580
|
+
keys: {
|
|
1581
|
+
type: "string",
|
|
1582
|
+
description: "Key combination, e.g. \"Ctrl+S\" or \"Enter\".",
|
|
1583
|
+
required: true
|
|
1584
|
+
}
|
|
1585
|
+
},
|
|
1586
|
+
output: {
|
|
1587
|
+
schema: actionOutputSchema(false),
|
|
1588
|
+
render(_args, value) {
|
|
1589
|
+
return [{
|
|
1590
|
+
type: "text",
|
|
1591
|
+
text: actionLine("key", value)
|
|
1592
|
+
}];
|
|
1593
|
+
}
|
|
1594
|
+
},
|
|
1595
|
+
timeoutMs: config.helperTimeoutMs + 15e3,
|
|
1596
|
+
async execute(args, exec) {
|
|
1597
|
+
backendGuard(services);
|
|
1598
|
+
const parsed = args;
|
|
1599
|
+
if (parsed.keys.trim() === "") throw new Error("keys must not be empty");
|
|
1600
|
+
const outcome = await actions.perform("key", exec, parsed.basedOn.observationId, parsed.basedOn.windowId, (focusFallback) => backend.key({
|
|
1601
|
+
windowId: parsed.basedOn.windowId,
|
|
1602
|
+
keys: parsed.keys.trim()
|
|
1603
|
+
}, focusFallback, exec.signal));
|
|
1604
|
+
return {
|
|
1605
|
+
ok: true,
|
|
1606
|
+
windowId: outcome.windowId,
|
|
1607
|
+
delivered: outcome.delivered,
|
|
1608
|
+
process: {
|
|
1609
|
+
before: outcome.processBefore,
|
|
1610
|
+
after: outcome.processAfter
|
|
1611
|
+
},
|
|
1612
|
+
...outcome.detail !== void 0 ? { detail: outcome.detail } : {}
|
|
1613
|
+
};
|
|
1614
|
+
}
|
|
1615
|
+
});
|
|
1616
|
+
}
|
|
1617
|
+
/**
|
|
1618
|
+
* `app_list` — enumerate running desktop applications and their windows.
|
|
1619
|
+
* Read-only: never needs approval.
|
|
1620
|
+
*
|
|
1621
|
+
* @param services - runtime services.
|
|
1622
|
+
* @returns the tool definition.
|
|
1623
|
+
*/
|
|
1624
|
+
function appListTool(services) {
|
|
1625
|
+
const { config, backend } = services;
|
|
1626
|
+
return defineTool({
|
|
1627
|
+
name: "app_list",
|
|
1628
|
+
description: "List running desktop applications and their visible windows (windowId, title, process id, executable path). Read-only: never needs approval. Use the returned windowIds as screen_shot/screen_read targets.",
|
|
1629
|
+
parameters: {},
|
|
1630
|
+
output: {
|
|
1631
|
+
schema: {
|
|
1632
|
+
type: "object",
|
|
1633
|
+
properties: {
|
|
1634
|
+
ok: {
|
|
1635
|
+
type: "boolean",
|
|
1636
|
+
const: true
|
|
1637
|
+
},
|
|
1638
|
+
apps: {
|
|
1639
|
+
type: "array",
|
|
1640
|
+
items: {
|
|
1641
|
+
type: "object",
|
|
1642
|
+
properties: {
|
|
1643
|
+
processId: { type: "integer" },
|
|
1644
|
+
name: { type: "string" },
|
|
1645
|
+
executablePath: { oneOf: [{ type: "string" }, { type: "null" }] },
|
|
1646
|
+
windows: {
|
|
1647
|
+
type: "array",
|
|
1648
|
+
items: {
|
|
1649
|
+
type: "object",
|
|
1650
|
+
properties: {
|
|
1651
|
+
windowId: { type: "integer" },
|
|
1652
|
+
processId: { oneOf: [{ type: "integer" }, { type: "null" }] },
|
|
1653
|
+
title: { type: "string" },
|
|
1654
|
+
className: { type: "string" },
|
|
1655
|
+
rect: {
|
|
1656
|
+
type: "object",
|
|
1657
|
+
properties: {
|
|
1658
|
+
x: { type: "integer" },
|
|
1659
|
+
y: { type: "integer" },
|
|
1660
|
+
width: { type: "integer" },
|
|
1661
|
+
height: { type: "integer" }
|
|
1662
|
+
},
|
|
1663
|
+
additionalProperties: false
|
|
1664
|
+
},
|
|
1665
|
+
executablePath: { oneOf: [{ type: "string" }, { type: "null" }] },
|
|
1666
|
+
visible: { type: "boolean" }
|
|
1667
|
+
},
|
|
1668
|
+
additionalProperties: false
|
|
1669
|
+
}
|
|
1670
|
+
}
|
|
1671
|
+
},
|
|
1672
|
+
additionalProperties: false
|
|
1673
|
+
}
|
|
1674
|
+
}
|
|
1675
|
+
},
|
|
1676
|
+
additionalProperties: false
|
|
1677
|
+
},
|
|
1678
|
+
render(_args, value) {
|
|
1679
|
+
const result = value;
|
|
1680
|
+
const lines = [`${result.apps.length} running application(s) with windows:`];
|
|
1681
|
+
for (const app of result.apps) {
|
|
1682
|
+
lines.push(`- ${app.name} (pid ${app.processId}, ${app.executablePath ?? "unknown executable"})`);
|
|
1683
|
+
for (const window of app.windows) lines.push(` - [windowId ${window.windowId}] ${JSON.stringify(window.title)} at (${window.rect.x}, ${window.rect.y}) ${window.rect.width}x${window.rect.height}`);
|
|
1684
|
+
}
|
|
1685
|
+
return [{
|
|
1686
|
+
type: "text",
|
|
1687
|
+
text: lines.join("\n")
|
|
1688
|
+
}];
|
|
1689
|
+
}
|
|
1690
|
+
},
|
|
1691
|
+
timeoutMs: config.helperTimeoutMs + 15e3,
|
|
1692
|
+
async execute(_args, exec) {
|
|
1693
|
+
backendGuard(services);
|
|
1694
|
+
return {
|
|
1695
|
+
ok: true,
|
|
1696
|
+
apps: (await backend.apps(exec.signal)).map((app) => ({
|
|
1697
|
+
processId: app.processId,
|
|
1698
|
+
name: sanitizeVisible(app.name, config.maxTextLength),
|
|
1699
|
+
executablePath: app.executablePath === null ? null : sanitizePath(app.executablePath, config.maxTextLength),
|
|
1700
|
+
windows: app.windows.map((window) => observedWindowInfo(window, config.maxTextLength))
|
|
1701
|
+
}))
|
|
1702
|
+
};
|
|
1703
|
+
}
|
|
1704
|
+
});
|
|
1705
|
+
}
|
|
1706
|
+
/**
|
|
1707
|
+
* `app_launch` — launch one application by name or path. Gated by approval
|
|
1708
|
+
* (allowlist matching applies against the requested name/path).
|
|
1709
|
+
*
|
|
1710
|
+
* @param services - runtime services.
|
|
1711
|
+
* @returns the tool definition.
|
|
1712
|
+
*/
|
|
1713
|
+
function appLaunchTool(services) {
|
|
1714
|
+
const { config, actions } = services;
|
|
1715
|
+
return defineTool({
|
|
1716
|
+
name: "app_launch",
|
|
1717
|
+
description: "Launch a desktop application by name (e.g. \"notepad\") or executable path, with optional arguments. Requires approval unless requireApproval is off or the name/path is allowlisted. Returns the new process identity.",
|
|
1718
|
+
parameters: {
|
|
1719
|
+
name: {
|
|
1720
|
+
type: "string",
|
|
1721
|
+
description: "Application name (resolved through the executable search path) or full path.",
|
|
1722
|
+
required: true
|
|
1723
|
+
},
|
|
1724
|
+
args: {
|
|
1725
|
+
type: "array",
|
|
1726
|
+
items: { type: "string" },
|
|
1727
|
+
description: "Optional command-line arguments."
|
|
1728
|
+
}
|
|
1729
|
+
},
|
|
1730
|
+
output: {
|
|
1731
|
+
schema: {
|
|
1732
|
+
type: "object",
|
|
1733
|
+
properties: {
|
|
1734
|
+
ok: {
|
|
1735
|
+
type: "boolean",
|
|
1736
|
+
const: true
|
|
1737
|
+
},
|
|
1738
|
+
processId: { type: "integer" },
|
|
1739
|
+
executablePath: { oneOf: [{ type: "string" }, { type: "null" }] }
|
|
1740
|
+
},
|
|
1741
|
+
additionalProperties: false
|
|
1742
|
+
},
|
|
1743
|
+
render(_args, value) {
|
|
1744
|
+
const result = value;
|
|
1745
|
+
return [{
|
|
1746
|
+
type: "text",
|
|
1747
|
+
text: `Launched: pid ${result.processId} (${result.executablePath ?? "unknown executable"}).`
|
|
1748
|
+
}];
|
|
1749
|
+
}
|
|
1750
|
+
},
|
|
1751
|
+
timeoutMs: config.helperTimeoutMs + 15e3,
|
|
1752
|
+
async execute(args, exec) {
|
|
1753
|
+
backendGuard(services);
|
|
1754
|
+
const parsed = args;
|
|
1755
|
+
if (parsed.name.trim() === "") throw new Error("app_launch name must not be empty");
|
|
1756
|
+
const outcome = await actions.launch(exec, parsed.name.trim(), parsed.args ?? []);
|
|
1757
|
+
return {
|
|
1758
|
+
ok: true,
|
|
1759
|
+
processId: outcome.processId,
|
|
1760
|
+
executablePath: outcome.executablePath === null ? null : sanitizePath(outcome.executablePath, config.maxTextLength)
|
|
1761
|
+
};
|
|
1762
|
+
}
|
|
1763
|
+
});
|
|
1764
|
+
}
|
|
1765
|
+
/** Every tool definition, in registration order. */
|
|
1766
|
+
function allTools(services) {
|
|
1767
|
+
return [
|
|
1768
|
+
screenShotTool(services),
|
|
1769
|
+
screenReadTool(services),
|
|
1770
|
+
clickTool(services),
|
|
1771
|
+
typeTool(services),
|
|
1772
|
+
scrollTool(services),
|
|
1773
|
+
keyTool(services),
|
|
1774
|
+
appListTool(services),
|
|
1775
|
+
appLaunchTool(services)
|
|
1776
|
+
];
|
|
1777
|
+
}
|
|
1778
|
+
//#endregion
|
|
1779
|
+
//#region src/index.ts
|
|
1780
|
+
const name = "dsh-click";
|
|
1781
|
+
/** Hard services: the tool registry every contribution lands in. */
|
|
1782
|
+
const inject = ["tools"];
|
|
1783
|
+
/**
|
|
1784
|
+
* Mount the plugin: resolve config, select the platform backend, and register
|
|
1785
|
+
* every tool through `ctx.tools.register` (each registration is an effect
|
|
1786
|
+
* whose disposer removes exactly that tool on stop/HMR).
|
|
1787
|
+
*
|
|
1788
|
+
* @param ctx - context carrying the tools registry.
|
|
1789
|
+
* @param config - raw loader config; defaults applied through {@link resolveConfig}.
|
|
1790
|
+
*/
|
|
1791
|
+
function apply(ctx, config) {
|
|
1792
|
+
const resolved = resolveConfig(config);
|
|
1793
|
+
const backend = ctx.get("dsh-click/backend") ?? createBackend(ctx, resolved);
|
|
1794
|
+
const observations = new ObservationStore(resolved);
|
|
1795
|
+
const services = {
|
|
1796
|
+
ctx,
|
|
1797
|
+
config: resolved,
|
|
1798
|
+
backend,
|
|
1799
|
+
observations,
|
|
1800
|
+
actions: new ActionExecutor({
|
|
1801
|
+
ctx,
|
|
1802
|
+
config: resolved,
|
|
1803
|
+
backend,
|
|
1804
|
+
observations
|
|
1805
|
+
})
|
|
1806
|
+
};
|
|
1807
|
+
for (const tool of allTools(services)) ctx.effect(() => ctx.tools.register(tool), `dsh-click: ${tool.name} tool`);
|
|
1808
|
+
}
|
|
1809
|
+
//#endregion
|
|
1810
|
+
export { ACTION_EVENT, ActionExecutor, Config, DshClickError, HELPER_PROTOCOL_VERSION, HelperBackend, OBSERVED_EVENT, ObservationStore, UnavailableBackend, VERSION, allTools, apply, createBackend, inject, name, observationIdOf, redactSensitive, resolveConfig, sanitizePath, sanitizeText, sanitizeVisible };
|