dsh-click 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/index.js ADDED
@@ -0,0 +1,1810 @@
1
+ import z from "@deepseek-ai/schemastery";
2
+ import { createHash, randomUUID } from "node:crypto";
3
+ import { existsSync } from "node:fs";
4
+ import path from "node:path";
5
+ import { fileURLToPath } from "node:url";
6
+ import { defineTool } from "@deepseek-ai/dsh-tools";
7
+ //#region src/events.ts
8
+ /** The observation audit event type. */
9
+ const OBSERVED_EVENT = "dsh-click/observed";
10
+ /** The action audit event type. */
11
+ const ACTION_EVENT = "dsh-click/action";
12
+ //#endregion
13
+ //#region src/sanitize.ts
14
+ /**
15
+ * Pure display/audit redaction for everything dsh-click shows to the model or
16
+ * writes to a session log: window titles, executable paths, and free text that
17
+ * may contain credentials. Every function is pure and total on string input;
18
+ * extreme cases are covered by `tests/sanitize.spec.ts`.
19
+ *
20
+ * @module dsh-click/sanitize
21
+ */
22
+ /** Control characters replaced by a space (tab) or removed entirely (the rest). */
23
+ const CONTROL_CHARACTERS = /[\u0000-\u0008\u000B-\u000C\u000E-\u001F\u007F]/gu;
24
+ /** Truncation marker appended when a string is cut to `maxLength`. */
25
+ const TRUNCATION_MARKER = "…";
26
+ /** Credential-shaped `name=value` / `name: value` pairs; values stop at control characters too. */
27
+ const SECRET_ASSIGNMENT = /((?:api[_-]?key|token|secret|password|passwd)\s*[=:])\s*[^\s"'<>\u0000-\u001F\u007F]+/giu;
28
+ /** JSON Web Tokens (three base64url segments separated by dots). */
29
+ const JWT_PATTERN = /\beyJ[A-Za-z0-9_-]{4,}\.[A-Za-z0-9_-]{4,}\.[A-Za-z0-9_-]{4,}\b/gu;
30
+ /** `Bearer` credentials. */
31
+ const BEARER_PATTERN = /(\bbearer\s+)[A-Za-z0-9._~+/=-]{8,}/giu;
32
+ /**
33
+ * Strip control characters and truncate to `maxLength`.
34
+ *
35
+ * @param text - the raw string.
36
+ * @param maxLength - maximum returned length (including the truncation marker).
37
+ * @returns the sanitized string.
38
+ */
39
+ function sanitizeText(text, maxLength) {
40
+ const cleaned = text.replace(CONTROL_CHARACTERS, "").replace(/\t+/gu, " ");
41
+ if (cleaned.length <= maxLength) return cleaned;
42
+ return cleaned.slice(0, Math.max(0, maxLength - 1)) + TRUNCATION_MARKER;
43
+ }
44
+ /**
45
+ * Sanitize a filesystem path for display: control characters removed, and the
46
+ * TAIL (filename side) preserved when truncating, since the head is the part
47
+ * that usually repeats.
48
+ *
49
+ * @param path - the raw path.
50
+ * @param maxLength - maximum returned length (including the truncation marker).
51
+ * @returns the sanitized path.
52
+ */
53
+ function sanitizePath(path, maxLength) {
54
+ const cleaned = path.replace(CONTROL_CHARACTERS, "").replace(/\t+/gu, " ");
55
+ if (cleaned.length <= maxLength) return cleaned;
56
+ return TRUNCATION_MARKER + cleaned.slice(cleaned.length - (maxLength - 1));
57
+ }
58
+ /**
59
+ * Redact credential-shaped fragments from free text before it reaches a log or
60
+ * display surface: `key=`/`key:` assignments, JWTs, and bearer tokens.
61
+ *
62
+ * @param text - the raw text.
63
+ * @returns the redacted text.
64
+ */
65
+ function redactSensitive(text) {
66
+ return text.replace(SECRET_ASSIGNMENT, "$1[redacted]").replace(JWT_PATTERN, "[redacted]").replace(BEARER_PATTERN, "$1[redacted]");
67
+ }
68
+ /**
69
+ * The one entry point for model-visible strings: redact first (secret values
70
+ * stop at control characters), then sanitize (control characters removed,
71
+ * tabs collapsed, length capped).
72
+ *
73
+ * @param text - the raw string.
74
+ * @param maxLength - truncation length.
75
+ * @returns the sanitized, redacted string.
76
+ */
77
+ function sanitizeVisible(text, maxLength) {
78
+ return sanitizeText(redactSensitive(text), maxLength);
79
+ }
80
+ //#endregion
81
+ //#region src/platform/types.ts
82
+ /** Error with a stable machine-routing code, model-readable message included. */
83
+ var DshClickError = class extends Error {
84
+ code;
85
+ /** @param message - human- and model-readable failure text. */
86
+ /** @param code - stable code for policy and diagnostics. */
87
+ constructor(message, code) {
88
+ super(message);
89
+ this.code = code;
90
+ this.name = "DshClickError";
91
+ }
92
+ };
93
+ /** Assert one helper response field is a finite number; throws on violation. */
94
+ function expectNumber(record, field, op) {
95
+ const value = record[field];
96
+ if (typeof value !== "number" || !Number.isFinite(value)) throw new DshClickError(`helper "${op}" returned a non-numeric "${field}" field`, "BAD_HELPER_RESPONSE");
97
+ return value;
98
+ }
99
+ /** Assert one helper response field is a string; throws on violation. */
100
+ function expectString(record, field, op) {
101
+ const value = record[field];
102
+ if (typeof value !== "string") throw new DshClickError(`helper "${op}" returned a non-string "${field}" field`, "BAD_HELPER_RESPONSE");
103
+ return value;
104
+ }
105
+ /** Assert an entire helper payload is a plain record; throws on violation. */
106
+ function expectRecordValue(value, op) {
107
+ if (typeof value !== "object" || value === null || Array.isArray(value)) throw new DshClickError(`helper "${op}" returned a non-object payload`, "BAD_HELPER_RESPONSE");
108
+ return value;
109
+ }
110
+ /** Parse the helper's `processBefore`/`processAfter` facts at the wire boundary. */
111
+ function expectProcessFacts(value, op) {
112
+ if (typeof value !== "object" || value === null || Array.isArray(value)) throw new DshClickError(`helper "${op}" returned malformed process facts`, "BAD_HELPER_RESPONSE");
113
+ const record = value;
114
+ const pid = expectNumber(record, "pid", op);
115
+ const executablePath = record["executablePath"];
116
+ if (executablePath !== null && typeof executablePath !== "string") throw new DshClickError(`helper "${op}" returned malformed process facts`, "BAD_HELPER_RESPONSE");
117
+ return {
118
+ pid,
119
+ executablePath
120
+ };
121
+ }
122
+ //#endregion
123
+ //#region src/actions.ts
124
+ /** Approval failure detail, model-readable. */
125
+ const APPROVAL_DETAIL = {
126
+ rejected: "rejected by the approval answerer",
127
+ cancelled: "cancelled while waiting for approval",
128
+ unavailable: "no approval answerer is available (fail closed)"
129
+ };
130
+ /** Executes mutating desktop actions behind the full safety boundary. */
131
+ var ActionExecutor = class {
132
+ deps;
133
+ /** @param deps - executor dependencies. */
134
+ constructor(deps) {
135
+ this.deps = deps;
136
+ }
137
+ /** The mounting context. */
138
+ get ctx() {
139
+ return this.deps.ctx;
140
+ }
141
+ /** The resolved config. */
142
+ get config() {
143
+ return this.deps.config;
144
+ }
145
+ /** Append one action audit event; a failed append never changes the outcome. */
146
+ audit(exec, event) {
147
+ const session = exec.agent?.session;
148
+ if (session === void 0) return;
149
+ try {
150
+ session.append(ACTION_EVENT, event);
151
+ } catch {}
152
+ }
153
+ /**
154
+ * The staleness boundary: resolve the cited observation, re-observe the
155
+ * window right now, and compare.
156
+ *
157
+ * @param observationId - the observation the action cites.
158
+ * @param windowId - the window the action addresses.
159
+ * @param signal - cancellation.
160
+ * @returns the cited record and the fresh snapshot.
161
+ */
162
+ async requireFreshWindow(observationId, windowId, signal) {
163
+ const record = this.deps.observations.get(observationId);
164
+ if (record === void 0) throw new DshClickError(`unknown observation "${observationId}" — run screen_read or screen_shot again and cite the returned observationId`, "UNKNOWN_OBSERVATION");
165
+ if (record.windowId !== windowId) throw new DshClickError(`observation ${observationId} belongs to window ${record.windowId}, not ${windowId}`, "UNKNOWN_OBSERVATION");
166
+ const fresh = await this.deps.backend.snapshot(windowId, signal);
167
+ const verdict = this.deps.observations.verify(record, fresh);
168
+ if (!verdict.ok) throw new DshClickError(`${verdict.detail} — run screen_read or screen_shot again before acting`, verdict.code);
169
+ return {
170
+ record,
171
+ fresh
172
+ };
173
+ }
174
+ /**
175
+ * The approval gate. Matchers run against the sanitized window title and
176
+ * the executable path; a match skips the ask (still audited as allowlist).
177
+ * With no matcher, a missing approval service or a missing calling agent
178
+ * fails closed; the real approval service is asked otherwise.
179
+ *
180
+ * @param exec - the calling tool execution.
181
+ * @param toolName - the tool asking (audit + prompt identity).
182
+ * @param subject - what the action targets, for the reason and the matchers.
183
+ * @param signal - cancellation.
184
+ * @returns which gate allowed the action.
185
+ */
186
+ async gate(exec, toolName, subject, signal) {
187
+ if (!this.config.requireApproval) return "none";
188
+ if (this.config.autoApproveMatchers.some((matcher) => subject.title !== null && matcher.test(subject.title) || subject.executablePath !== null && matcher.test(subject.executablePath))) return "allowlist";
189
+ const approval = this.ctx.get("approval");
190
+ if (approval === void 0) throw new DshClickError("approval is required but the approval service is not mounted — mount @deepseek-ai/dsh-user-approval or allowlist this window", "APPROVAL_UNAVAILABLE");
191
+ if (exec.agent === void 0) throw new DshClickError("approval is required but no agent owns this call — refusing (fail closed)", "APPROVAL_UNAVAILABLE");
192
+ const target = subject.title ?? subject.executablePath ?? "unknown target";
193
+ const outcome = await approval.request({
194
+ agent: exec.agent,
195
+ toolName,
196
+ callId: exec.callId,
197
+ reason: `${toolName} on ${sanitizeVisible(target, this.config.maxTextLength)}`,
198
+ signal
199
+ });
200
+ if (outcome !== "allowed-once") throw new DshClickError(`action denied by approval: ${APPROVAL_DETAIL[outcome] ?? outcome}`, "APPROVAL_DENIED");
201
+ return "approval";
202
+ }
203
+ /**
204
+ * Compare the process identity captured before and after an action.
205
+ *
206
+ * @param before - facts captured pre-action.
207
+ * @param after - facts captured post-action.
208
+ */
209
+ assertProcessUnchanged(before, after) {
210
+ if (before.pid !== after.pid || before.executablePath !== after.executablePath) throw new DshClickError(`the target process changed during the action (before: pid ${before.pid} ${before.executablePath ?? "?"}; after: pid ${after.pid} ${after.executablePath ?? "?"}) — the action was delivered to a different process identity and must be reviewed`, "PROCESS_CHANGED");
211
+ }
212
+ /**
213
+ * Run one window-scoped mutating action through the shared boundary:
214
+ * freshness, approval, perform, process check, audit.
215
+ *
216
+ * @param toolName - the calling tool name.
217
+ * @param exec - the calling execution.
218
+ * @param observationId - the cited observation.
219
+ * @param windowId - the target window.
220
+ * @param run - the backend call (receives the resolved focus flag).
221
+ * @returns the validated backend outcome.
222
+ */
223
+ async perform(toolName, exec, observationId, windowId, run) {
224
+ let approved = "none";
225
+ let observationIdAudited;
226
+ let windowIdAudited;
227
+ try {
228
+ const { record } = await this.requireFreshWindow(observationId, windowId, exec.signal);
229
+ observationIdAudited = observationId;
230
+ windowIdAudited = windowId;
231
+ approved = await this.gate(exec, toolName, {
232
+ title: record.title,
233
+ executablePath: record.executablePath
234
+ }, exec.signal);
235
+ const outcome = await run(this.config.focusFallback === "allow");
236
+ this.assertProcessUnchanged(outcome.processBefore, outcome.processAfter);
237
+ this.audit(exec, {
238
+ tool: toolName,
239
+ approved,
240
+ outcome: "ok",
241
+ observationId: observationIdAudited,
242
+ windowId: windowIdAudited,
243
+ processBefore: outcome.processBefore,
244
+ processAfter: outcome.processAfter,
245
+ ...outcome.restored !== void 0 ? { restored: outcome.restored } : {},
246
+ ...outcome.detail !== void 0 ? { detail: outcome.detail } : {}
247
+ });
248
+ return outcome;
249
+ } catch (error) {
250
+ const message = error instanceof Error ? error.message : String(error);
251
+ this.audit(exec, {
252
+ tool: toolName,
253
+ approved,
254
+ outcome: "error",
255
+ ...observationIdAudited !== void 0 ? { observationId: observationIdAudited } : {},
256
+ ...windowIdAudited !== void 0 ? { windowId: windowIdAudited } : {},
257
+ detail: message
258
+ });
259
+ throw error;
260
+ }
261
+ }
262
+ /**
263
+ * Gate and launch one application. No window observation exists yet, so the
264
+ * gate runs against the requested name/path alone and the launch outcome's
265
+ * process facts are the post-identity proof.
266
+ *
267
+ * @param exec - the calling execution.
268
+ * @param name - the app name or executable path.
269
+ * @param args - launch arguments.
270
+ * @returns the launch outcome.
271
+ */
272
+ async launch(exec, name, args) {
273
+ const toolName = "app_launch";
274
+ let approved = "none";
275
+ try {
276
+ approved = await this.gate(exec, toolName, {
277
+ title: null,
278
+ executablePath: name
279
+ }, exec.signal);
280
+ const outcome = await this.deps.backend.launch(name, args, exec.signal);
281
+ this.audit(exec, {
282
+ tool: toolName,
283
+ approved,
284
+ outcome: "ok",
285
+ processAfter: {
286
+ pid: outcome.processId,
287
+ executablePath: outcome.executablePath
288
+ },
289
+ detail: name
290
+ });
291
+ return outcome;
292
+ } catch (error) {
293
+ const message = error instanceof Error ? error.message : String(error);
294
+ this.audit(exec, {
295
+ tool: toolName,
296
+ approved,
297
+ outcome: "error",
298
+ detail: message
299
+ });
300
+ throw error;
301
+ }
302
+ }
303
+ };
304
+ //#endregion
305
+ //#region src/config.ts
306
+ /**
307
+ * Plugin configuration and its explicit resolve step. `resolveConfig` re-judges
308
+ * every default and bound so programmatic construction that bypasses
309
+ * Schemastery normalization still fails loud instead of running with hidden
310
+ * defaults (the explicit-resolve contract).
311
+ *
312
+ * @module dsh-click/config
313
+ */
314
+ /** Default per-helper-call timeout in milliseconds. */
315
+ const DEFAULT_HELPER_TIMEOUT_MS = 3e4;
316
+ /** Ceiling for one helper call: a screenshot of a large desktop can take a while. */
317
+ const MAX_HELPER_TIMEOUT_MS = 3e5;
318
+ /** Default cap on one helper response in bytes (PNG screenshots are base64-heavy). */
319
+ const DEFAULT_MAX_HELPER_OUTPUT_BYTES = 25165824;
320
+ /** Ceiling on one helper response. */
321
+ const MAX_HELPER_OUTPUT_BYTES = 67108864;
322
+ /** Default longest side of a captured screenshot in pixels (larger captures downscale). */
323
+ const DEFAULT_MAX_SCREENSHOT_SIDE = 2560;
324
+ const MAX_SCREENSHOT_SIDE = 7680;
325
+ /** Default maximum age of an observation that an action may still base on. */
326
+ const DEFAULT_MAX_OBSERVATION_AGE_MS = 3e4;
327
+ /** Bounds for the observation age. */
328
+ const MIN_OBSERVATION_AGE_MS = 1e3;
329
+ const MAX_OBSERVATION_AGE_MS = 6e5;
330
+ /** Ceiling on the element cap. */
331
+ const MAX_ELEMENTS = 2e3;
332
+ /** Ceiling on the sanitization truncation length. */
333
+ const MAX_TEXT_LENGTH = 1e4;
334
+ /** Schemastery schema for loader-validated configuration. */
335
+ const Config = z.object({
336
+ requireApproval: z.boolean().default(true),
337
+ autoApproveWindows: z.array(z.string()).default([]),
338
+ focusFallback: z.union(["never", "allow"]).default("never"),
339
+ imageMode: z.union(["auto", "text"]).default("auto"),
340
+ helperTimeoutMs: z.number().min(1).max(MAX_HELPER_TIMEOUT_MS).default(DEFAULT_HELPER_TIMEOUT_MS),
341
+ maxHelperOutputBytes: z.number().min(1024).max(MAX_HELPER_OUTPUT_BYTES).default(DEFAULT_MAX_HELPER_OUTPUT_BYTES),
342
+ maxScreenshotSide: z.number().min(320).max(MAX_SCREENSHOT_SIDE).default(DEFAULT_MAX_SCREENSHOT_SIDE),
343
+ staleCheckPixels: z.boolean().default(true),
344
+ maxObservationAgeMs: z.number().min(MIN_OBSERVATION_AGE_MS).max(MAX_OBSERVATION_AGE_MS).default(DEFAULT_MAX_OBSERVATION_AGE_MS),
345
+ maxCachedObservations: z.number().min(1).max(64).default(8),
346
+ maxElements: z.number().min(1).max(MAX_ELEMENTS).default(500),
347
+ maxTreeDepth: z.number().min(1).max(64).default(32),
348
+ maxTextLength: z.number().min(16).max(MAX_TEXT_LENGTH).default(200),
349
+ rollbackEnabled: z.boolean().default(true)
350
+ });
351
+ /** Throw the standard fail-loud config error for one invalid field. */
352
+ function invalid(field, detail) {
353
+ throw new Error(`dsh-click: config.${field} ${detail}`);
354
+ }
355
+ /** Compile one allowlist pattern; an invalid regex fails at load, not at match time. */
356
+ function compileMatcher(source, index) {
357
+ try {
358
+ return new RegExp(source, "iu");
359
+ } catch {
360
+ invalid("autoApproveWindows", `entry ${index} (${JSON.stringify(source)}) is not a valid regular expression`);
361
+ }
362
+ }
363
+ /**
364
+ * Resolve raw config to the runtime policy, re-validating defaults and bounds.
365
+ *
366
+ * @param config - raw loader config; `undefined` for a bare row.
367
+ * @returns the frozen resolved config.
368
+ */
369
+ function resolveConfig(config) {
370
+ const requireApproval = config?.requireApproval ?? true;
371
+ if (typeof requireApproval !== "boolean") invalid("requireApproval", "must be a boolean");
372
+ const autoApproveMatchers = (config?.autoApproveWindows ?? []).map(compileMatcher);
373
+ const focusFallback = config?.focusFallback ?? "never";
374
+ if (focusFallback !== "never" && focusFallback !== "allow") invalid("focusFallback", "must be \"never\" or \"allow\"");
375
+ const imageMode = config?.imageMode ?? "auto";
376
+ if (imageMode !== "auto" && imageMode !== "text") invalid("imageMode", "must be \"auto\" or \"text\"");
377
+ const helperTimeoutMs = config?.helperTimeoutMs ?? 3e4;
378
+ if (!Number.isFinite(helperTimeoutMs) || helperTimeoutMs < 1 || helperTimeoutMs > 3e5) invalid("helperTimeoutMs", `must be a finite number between 1 and ${MAX_HELPER_TIMEOUT_MS}`);
379
+ const maxHelperOutputBytes = config?.maxHelperOutputBytes ?? 25165824;
380
+ if (!Number.isInteger(maxHelperOutputBytes) || maxHelperOutputBytes < 1024 || maxHelperOutputBytes > 67108864) invalid("maxHelperOutputBytes", `must be an integer between 1024 and ${MAX_HELPER_OUTPUT_BYTES}`);
381
+ const maxScreenshotSide = config?.maxScreenshotSide ?? 2560;
382
+ if (!Number.isInteger(maxScreenshotSide) || maxScreenshotSide < 320 || maxScreenshotSide > 7680) invalid("maxScreenshotSide", `must be an integer between 320 and ${MAX_SCREENSHOT_SIDE}`);
383
+ const staleCheckPixels = config?.staleCheckPixels ?? true;
384
+ if (typeof staleCheckPixels !== "boolean") invalid("staleCheckPixels", "must be a boolean");
385
+ const maxObservationAgeMs = config?.maxObservationAgeMs ?? 3e4;
386
+ if (!Number.isFinite(maxObservationAgeMs) || maxObservationAgeMs < 1e3 || maxObservationAgeMs > 6e5) invalid("maxObservationAgeMs", `must be a finite number between ${MIN_OBSERVATION_AGE_MS} and ${MAX_OBSERVATION_AGE_MS}`);
387
+ const maxCachedObservations = config?.maxCachedObservations ?? 8;
388
+ if (!Number.isInteger(maxCachedObservations) || maxCachedObservations < 1 || maxCachedObservations > 64) invalid("maxCachedObservations", `must be an integer between 1 and 64`);
389
+ const maxElements = config?.maxElements ?? 500;
390
+ if (!Number.isInteger(maxElements) || maxElements < 1 || maxElements > 2e3) invalid("maxElements", `must be an integer between 1 and ${MAX_ELEMENTS}`);
391
+ const maxTreeDepth = config?.maxTreeDepth ?? 32;
392
+ if (!Number.isInteger(maxTreeDepth) || maxTreeDepth < 1 || maxTreeDepth > 64) invalid("maxTreeDepth", `must be an integer between 1 and 64`);
393
+ const maxTextLength = config?.maxTextLength ?? 200;
394
+ if (!Number.isInteger(maxTextLength) || maxTextLength < 16 || maxTextLength > 1e4) invalid("maxTextLength", `must be an integer between 16 and ${MAX_TEXT_LENGTH}`);
395
+ const rollbackEnabled = config?.rollbackEnabled ?? true;
396
+ if (typeof rollbackEnabled !== "boolean") invalid("rollbackEnabled", "must be a boolean");
397
+ return Object.freeze({
398
+ requireApproval,
399
+ autoApproveMatchers: Object.freeze(autoApproveMatchers),
400
+ focusFallback,
401
+ imageMode,
402
+ helperTimeoutMs,
403
+ maxHelperOutputBytes,
404
+ maxScreenshotSide,
405
+ staleCheckPixels,
406
+ maxObservationAgeMs,
407
+ maxCachedObservations,
408
+ maxElements,
409
+ maxTreeDepth,
410
+ maxTextLength,
411
+ rollbackEnabled
412
+ });
413
+ }
414
+ //#endregion
415
+ //#region src/observe.ts
416
+ /**
417
+ * Observation cache and the staleness boundary. Every observation stores the
418
+ * window's identity facts plus content hashes; before any action the executor
419
+ * re-observes the window and this store compares. Identity, tree, or pixel
420
+ * drift — or an expired observation — refuses the action, so the model must
421
+ * re-observe instead of acting on a screen that changed.
422
+ *
423
+ * @module dsh-click/observe
424
+ */
425
+ /**
426
+ * Hash the observable state of one window into a stable observation id.
427
+ *
428
+ * @param snapshot - the snapshot facts.
429
+ * @returns a hex observation id.
430
+ */
431
+ function observationIdOf(snapshot) {
432
+ const material = JSON.stringify([
433
+ snapshot.windowId,
434
+ snapshot.processId,
435
+ snapshot.executablePath,
436
+ snapshot.title,
437
+ snapshot.className,
438
+ snapshot.rect,
439
+ snapshot.treeHash,
440
+ snapshot.shotHash
441
+ ]);
442
+ return createHash("sha256").update(material).digest("hex").slice(0, 32);
443
+ }
444
+ /** Bounded LRU cache of observations, plus the staleness verdict logic. */
445
+ var ObservationStore = class {
446
+ config;
447
+ entries = /* @__PURE__ */ new Map();
448
+ /** @param config - resolved caps (cache size, max age, pixel check). */
449
+ constructor(config) {
450
+ this.config = config;
451
+ }
452
+ /**
453
+ * Cache one observation and return it.
454
+ *
455
+ * @param snapshot - the snapshot facts it came from.
456
+ * @returns the stored record (id = hash of the state).
457
+ */
458
+ record(snapshot) {
459
+ const record = {
460
+ id: observationIdOf(snapshot),
461
+ windowId: snapshot.windowId,
462
+ processId: snapshot.processId,
463
+ executablePath: snapshot.executablePath,
464
+ title: snapshot.title,
465
+ className: snapshot.className,
466
+ rect: { ...snapshot.rect },
467
+ treeHash: snapshot.treeHash,
468
+ shotHash: snapshot.shotHash,
469
+ elementCount: snapshot.elementCount,
470
+ observedAt: Date.now()
471
+ };
472
+ this.entries.delete(record.id);
473
+ this.entries.set(record.id, record);
474
+ while (this.entries.size > this.config.maxCachedObservations) {
475
+ const oldest = this.entries.keys().next().value;
476
+ if (oldest === void 0) break;
477
+ this.entries.delete(oldest);
478
+ }
479
+ return record;
480
+ }
481
+ /**
482
+ * Look one observation up by id.
483
+ *
484
+ * @param id - the observation id an action cites.
485
+ * @returns the record, or undefined when unknown or evicted.
486
+ */
487
+ get(id) {
488
+ return this.entries.get(id);
489
+ }
490
+ /**
491
+ * Compare a fresh re-observation against the cited record. The verdict is
492
+ * `ok` only when identity (window/pid/exe/title/class/rect), the tree hash,
493
+ * and — when `staleCheckPixels` is on — the pixel hash all still match, and
494
+ * the record is not older than `maxObservationAgeMs`.
495
+ *
496
+ * @param record - the observation the action cites.
497
+ * @param fresh - the re-observation captured immediately before the action.
498
+ * @param now - wall-clock now (injectable for tests).
499
+ * @returns the verdict.
500
+ */
501
+ verify(record, fresh, now = Date.now()) {
502
+ if (now - record.observedAt > this.config.maxObservationAgeMs) return {
503
+ ok: false,
504
+ code: "EXPIRED",
505
+ detail: `observation ${record.id} is older than ${this.config.maxObservationAgeMs} ms`
506
+ };
507
+ if (fresh.windowId !== record.windowId || fresh.processId !== record.processId || fresh.executablePath !== record.executablePath || fresh.title !== record.title || fresh.className !== record.className || fresh.rect.x !== record.rect.x || fresh.rect.y !== record.rect.y || fresh.rect.width !== record.rect.width || fresh.rect.height !== record.rect.height) return {
508
+ ok: false,
509
+ code: "STALE_IDENTITY",
510
+ detail: `window ${record.windowId} no longer matches observation ${record.id} (title/class/process or rect changed)`
511
+ };
512
+ if (fresh.treeHash !== record.treeHash) return {
513
+ ok: false,
514
+ code: "STALE_TREE",
515
+ detail: `the accessibility tree of window ${record.windowId} changed since observation ${record.id}`
516
+ };
517
+ if (this.config.staleCheckPixels && fresh.shotHash !== record.shotHash) return {
518
+ ok: false,
519
+ code: "STALE_PIXELS",
520
+ detail: `the pixels of window ${record.windowId} changed since observation ${record.id}`
521
+ };
522
+ return { ok: true };
523
+ }
524
+ };
525
+ //#endregion
526
+ //#region src/version.ts
527
+ /**
528
+ * The plugin version, hardcoded on purpose: it rides the helper-process
529
+ * protocol handshake, so it must track `package.json` or released helpers
530
+ * report a stale plugin. Bumping `package.json` without touching this file
531
+ * fails `tests/version.spec.ts`, and `scripts/release.mjs` bumps both.
532
+ *
533
+ * @module dsh-click/version
534
+ */
535
+ /** Plugin version advertised in the helper-protocol handshake. */
536
+ const VERSION = "0.1.0";
537
+ /** The helper protocol revision; bump only on incompatible wire changes. */
538
+ const HELPER_PROTOCOL_VERSION = 1;
539
+ //#endregion
540
+ //#region src/platform/runner.ts
541
+ /**
542
+ * The Windows backend: every operation spawns the native PowerShell helper
543
+ * (`native/win32/dsh-click-helper.ps1`) through the harness's `ctx.subprocess`
544
+ * seam. The request travels as JSON on stdin; the response is JSON on stdout
545
+ * and is validated field-by-field at this wire boundary. Each call is
546
+ * deadline-bounded (config `helperTimeoutMs`) and aborts with the caller's
547
+ * signal (the subprocess service escalates to tree termination).
548
+ *
549
+ * @module dsh-click/platform/runner
550
+ */
551
+ /** Locate the shipped helper script from either source or bundle layout. */
552
+ function findHelperPath() {
553
+ const override = process.env["DSH_CLICK_HELPER"];
554
+ if (override !== void 0 && override !== "") return override;
555
+ let dir = path.dirname(fileURLToPath(import.meta.url));
556
+ for (let depth = 0; depth < 8; depth += 1) {
557
+ const candidate = path.join(dir, "native", "win32", "dsh-click-helper.ps1");
558
+ if (existsSync(candidate)) return candidate;
559
+ dir = path.dirname(dir);
560
+ }
561
+ throw new DshClickError("cannot locate native/win32/dsh-click-helper.ps1 next to the package", "HELPER_MISSING");
562
+ }
563
+ /** Validate a helper rect at the wire boundary. */
564
+ function expectRect(value, op) {
565
+ if (typeof value !== "object" || value === null || Array.isArray(value)) throw new DshClickError(`helper "${op}" returned a malformed rect`, "BAD_HELPER_RESPONSE");
566
+ const record = value;
567
+ return {
568
+ x: expectNumber(record, "x", op),
569
+ y: expectNumber(record, "y", op),
570
+ width: expectNumber(record, "width", op),
571
+ height: expectNumber(record, "height", op)
572
+ };
573
+ }
574
+ /** Validate one helper window record. */
575
+ function expectWindow(value, op) {
576
+ const record = expectRecordValue(value, op);
577
+ const processIdRaw = record["processId"];
578
+ const processId = typeof processIdRaw === "number" ? processIdRaw : null;
579
+ const executablePath = record["executablePath"];
580
+ if (executablePath !== null && typeof executablePath !== "string") throw new DshClickError(`helper "${op}" returned malformed executablePath`, "BAD_HELPER_RESPONSE");
581
+ return {
582
+ windowId: expectNumber(record, "windowId", op),
583
+ processId,
584
+ title: expectString(record, "title", op),
585
+ className: expectString(record, "className", op),
586
+ rect: expectRect(record["rect"], op),
587
+ executablePath,
588
+ visible: record["visible"] !== false
589
+ };
590
+ }
591
+ /** Validate one helper snapshot record. */
592
+ function expectSnapshot(value, op) {
593
+ const record = expectRecordValue(value, op);
594
+ const executablePath = record["executablePath"];
595
+ if (executablePath !== null && typeof executablePath !== "string") throw new DshClickError(`helper "${op}" returned malformed executablePath`, "BAD_HELPER_RESPONSE");
596
+ return {
597
+ windowId: expectNumber(record, "windowId", op),
598
+ processId: expectNumber(record, "processId", op),
599
+ executablePath,
600
+ title: expectString(record, "title", op),
601
+ className: expectString(record, "className", op),
602
+ rect: expectRect(record["rect"], op),
603
+ foreground: record["foreground"] === true,
604
+ treeHash: expectString(record, "treeHash", op),
605
+ shotHash: expectString(record, "shotHash", op),
606
+ elementCount: expectNumber(record, "elementCount", op)
607
+ };
608
+ }
609
+ /** Validate one helper element record. */
610
+ function expectElement(value, op) {
611
+ const record = expectRecordValue(value, op);
612
+ const patterns = record["patterns"];
613
+ if (!Array.isArray(patterns) || patterns.some((item) => typeof item !== "string")) throw new DshClickError(`helper "${op}" returned malformed element patterns`, "BAD_HELPER_RESPONSE");
614
+ return {
615
+ elementId: expectString(record, "elementId", op),
616
+ controlType: expectString(record, "controlType", op),
617
+ name: expectString(record, "name", op),
618
+ automationId: expectString(record, "automationId", op),
619
+ rect: expectRect(record["rect"], op),
620
+ enabled: record["enabled"] === true,
621
+ patterns
622
+ };
623
+ }
624
+ /** Validate one helper pixel hint. */
625
+ function expectPixel(value, op) {
626
+ const record = expectRecordValue(value, op);
627
+ return {
628
+ label: expectString(record, "label", op),
629
+ x: expectNumber(record, "x", op),
630
+ y: expectNumber(record, "y", op),
631
+ color: expectString(record, "color", op)
632
+ };
633
+ }
634
+ /** Validate one helper action outcome. */
635
+ function expectActionOutcome(value, op) {
636
+ const record = expectRecordValue(value, op);
637
+ const delivered = expectString(record, "delivered", op);
638
+ if (delivered !== "uia" && delivered !== "posted" && delivered !== "none") throw new DshClickError(`helper "${op}" returned an unknown delivered mechanism`, "BAD_HELPER_RESPONSE");
639
+ const restored = record["restored"];
640
+ if (restored !== void 0 && typeof restored !== "boolean") throw new DshClickError(`helper "${op}" returned a non-boolean restored flag`, "BAD_HELPER_RESPONSE");
641
+ const detail = record["detail"];
642
+ if (detail !== void 0 && typeof detail !== "string") throw new DshClickError(`helper "${op}" returned a non-string detail`, "BAD_HELPER_RESPONSE");
643
+ return {
644
+ windowId: expectNumber(record, "windowId", op),
645
+ action: expectString(record, "action", op),
646
+ delivered,
647
+ processBefore: expectProcessFacts(record["processBefore"], op),
648
+ processAfter: expectProcessFacts(record["processAfter"], op),
649
+ ...restored !== void 0 ? { restored } : {},
650
+ ...detail !== void 0 ? { detail } : {}
651
+ };
652
+ }
653
+ /** The Windows helper backend over `ctx.subprocess`. */
654
+ var HelperBackend = class {
655
+ config;
656
+ subprocess;
657
+ /** Resolved powershell executable path, cached across calls. */
658
+ powershell;
659
+ available = true;
660
+ /** @param config - resolved limits for timeout and output size. */
661
+ /** @param subprocess - the subprocess service that runs the helper. */
662
+ constructor(config, subprocess) {
663
+ this.config = config;
664
+ this.subprocess = subprocess;
665
+ }
666
+ platform = "win32";
667
+ /** Run one helper operation and return its validated result payload. */
668
+ async invoke(op, args, signal) {
669
+ const request = {
670
+ protocol: 1,
671
+ pluginVersion: VERSION,
672
+ requestId: randomUUID(),
673
+ op,
674
+ args
675
+ };
676
+ const helperPath = findHelperPath();
677
+ const executable = await this.resolvePowershell();
678
+ const deadline = AbortSignal.timeout(this.config.helperTimeoutMs);
679
+ const fused = signal === void 0 ? deadline : AbortSignal.any([signal, deadline]);
680
+ const handle = this.subprocess.spawn({
681
+ argv: [
682
+ executable,
683
+ "-NoProfile",
684
+ "-NonInteractive",
685
+ "-ExecutionPolicy",
686
+ "Bypass",
687
+ "-File",
688
+ helperPath
689
+ ],
690
+ cwd: process.cwd(),
691
+ stdio: {
692
+ stdin: { data: JSON.stringify(request) },
693
+ stdout: { maxBytes: this.config.maxHelperOutputBytes },
694
+ stderr: { maxBytes: 65536 }
695
+ },
696
+ graceMs: 5e3,
697
+ signal: fused
698
+ });
699
+ const outcome = await handle.done;
700
+ const stdoutText = handle.collected.stdout?.readFrom(0).text ?? "";
701
+ const stderrText = handle.collected.stderr?.readFrom(0).text ?? "";
702
+ if (outcome.exitCode !== 0) {
703
+ const tail = stderrText.trim().split(/\r?\n/u).slice(-3).join(" | ");
704
+ throw new DshClickError(`helper "${op}" failed (exit ${outcome.exitCode}${tail === "" ? "" : `: ${tail}`})`, "HELPER_FAILED");
705
+ }
706
+ let response;
707
+ try {
708
+ response = JSON.parse(stdoutText);
709
+ } catch {
710
+ throw new DshClickError(`helper "${op}" produced ${stdoutText.length === 0 ? "no stdout" : "malformed JSON on stdout"}`, "BAD_HELPER_RESPONSE");
711
+ }
712
+ if (typeof response !== "object" || response === null || typeof response.ok !== "boolean") throw new DshClickError(`helper "${op}" returned a malformed response envelope`, "BAD_HELPER_RESPONSE");
713
+ if (!response.ok) {
714
+ const error = response.error;
715
+ if (typeof error !== "object" || error === null || typeof error.code !== "string" || typeof error.message !== "string") throw new DshClickError(`helper "${op}" returned a malformed error envelope`, "BAD_HELPER_RESPONSE");
716
+ throw new DshClickError(`helper "${op}": ${error.message}`, error.code);
717
+ }
718
+ return response.result;
719
+ }
720
+ /** Resolve (and cache) the powershell executable through the subprocess seam. */
721
+ resolvePowershell() {
722
+ this.powershell ??= this.subprocess.resolveExecutable("powershell.exe").catch((error) => {
723
+ this.powershell = void 0;
724
+ throw new DshClickError(`cannot resolve powershell.exe for the desktop helper: ${error instanceof Error ? error.message : String(error)}`, "HELPER_MISSING");
725
+ });
726
+ return this.powershell;
727
+ }
728
+ async listWindows(signal) {
729
+ const result = await this.invoke("windows", {}, signal);
730
+ if (!Array.isArray(result)) throw new DshClickError("helper \"windows\" returned a non-array result", "BAD_HELPER_RESPONSE");
731
+ return result.map((item) => expectWindow(item, "windows"));
732
+ }
733
+ async shot(ref, maxSide, signal) {
734
+ const record = expectRecordValue(await this.invoke("shot", {
735
+ target: ref,
736
+ maxSide,
737
+ maxElements: this.config.maxElements,
738
+ maxDepth: this.config.maxTreeDepth
739
+ }, signal), "shot");
740
+ return {
741
+ pngBase64: expectString(record, "pngBase64", "shot"),
742
+ width: expectNumber(record, "width", "shot"),
743
+ height: expectNumber(record, "height", "shot"),
744
+ snapshot: expectSnapshot(record["snapshot"], "shot")
745
+ };
746
+ }
747
+ async tree(ref, maxElements, maxDepth, includePixels, signal) {
748
+ const record = expectRecordValue(await this.invoke("tree", {
749
+ target: ref,
750
+ maxElements,
751
+ maxDepth,
752
+ includePixels
753
+ }, signal), "tree");
754
+ const elements = record["elements"];
755
+ const pixels = record["pixels"];
756
+ if (!Array.isArray(elements) || !Array.isArray(pixels)) throw new DshClickError("helper \"tree\" returned malformed elements/pixels arrays", "BAD_HELPER_RESPONSE");
757
+ return {
758
+ snapshot: expectSnapshot(record["snapshot"], "tree"),
759
+ elements: elements.map((item) => expectElement(item, "tree")),
760
+ pixels: pixels.map((item) => expectPixel(item, "tree"))
761
+ };
762
+ }
763
+ async snapshot(windowId, signal) {
764
+ return expectSnapshot(await this.invoke("snapshot", {
765
+ windowId,
766
+ maxElements: this.config.maxElements,
767
+ maxDepth: this.config.maxTreeDepth
768
+ }, signal), "snapshot");
769
+ }
770
+ async click(request, focusFallback, signal) {
771
+ return expectActionOutcome(await this.invoke("click", {
772
+ request,
773
+ focusFallback
774
+ }, signal), "click");
775
+ }
776
+ async type(request, focusFallback, signal) {
777
+ return expectActionOutcome(await this.invoke("type", {
778
+ request,
779
+ focusFallback
780
+ }, signal), "type");
781
+ }
782
+ async scroll(request, focusFallback, signal) {
783
+ return expectActionOutcome(await this.invoke("scroll", {
784
+ request,
785
+ focusFallback
786
+ }, signal), "scroll");
787
+ }
788
+ async key(request, focusFallback, signal) {
789
+ return expectActionOutcome(await this.invoke("key", {
790
+ request,
791
+ focusFallback
792
+ }, signal), "key");
793
+ }
794
+ async apps(signal) {
795
+ const result = await this.invoke("apps", {}, signal);
796
+ if (!Array.isArray(result)) throw new DshClickError("helper \"apps\" returned a non-array result", "BAD_HELPER_RESPONSE");
797
+ return result.map((item) => {
798
+ const record = expectRecordValue(item, "apps");
799
+ const windows = record["windows"];
800
+ if (!Array.isArray(windows)) throw new DshClickError("helper \"apps\" returned malformed windows", "BAD_HELPER_RESPONSE");
801
+ const executablePath = record["executablePath"];
802
+ if (executablePath !== null && typeof executablePath !== "string") throw new DshClickError("helper \"apps\" returned malformed executablePath", "BAD_HELPER_RESPONSE");
803
+ return {
804
+ processId: expectNumber(record, "processId", "apps"),
805
+ name: expectString(record, "name", "apps"),
806
+ executablePath,
807
+ windows: windows.map((windowInfo) => expectWindow(windowInfo, "apps"))
808
+ };
809
+ });
810
+ }
811
+ async launch(name, args, signal) {
812
+ const record = expectRecordValue(await this.invoke("launch", {
813
+ name,
814
+ args
815
+ }, signal), "launch");
816
+ const executablePath = record["executablePath"];
817
+ if (executablePath !== null && typeof executablePath !== "string") throw new DshClickError("helper \"launch\" returned malformed executablePath", "BAD_HELPER_RESPONSE");
818
+ return {
819
+ processId: expectNumber(record, "processId", "launch"),
820
+ executablePath
821
+ };
822
+ }
823
+ };
824
+ //#endregion
825
+ //#region src/platform/selection.ts
826
+ /** A backend that cannot act: every operation fails closed with the given reason. */
827
+ var UnavailableBackend = class {
828
+ platform;
829
+ reason;
830
+ /** @param platform - the platform id this stub stands in for. */
831
+ /** @param reason - model-readable reason why the backend cannot act. */
832
+ constructor(platform, reason) {
833
+ this.platform = platform;
834
+ this.reason = reason;
835
+ }
836
+ available = false;
837
+ get unavailableReason() {
838
+ return this.reason;
839
+ }
840
+ /** Throw the shared backend-unavailable failure. */
841
+ refuse() {
842
+ throw new DshClickError(`dsh-click backend unavailable on ${this.platform}: ${this.reason}`, "BACKEND_UNAVAILABLE");
843
+ }
844
+ listWindows() {
845
+ this.refuse();
846
+ }
847
+ shot() {
848
+ this.refuse();
849
+ }
850
+ tree() {
851
+ this.refuse();
852
+ }
853
+ snapshot() {
854
+ this.refuse();
855
+ }
856
+ click() {
857
+ this.refuse();
858
+ }
859
+ type() {
860
+ this.refuse();
861
+ }
862
+ scroll() {
863
+ this.refuse();
864
+ }
865
+ key() {
866
+ this.refuse();
867
+ }
868
+ apps() {
869
+ this.refuse();
870
+ }
871
+ launch() {
872
+ this.refuse();
873
+ }
874
+ };
875
+ /** The reserved-but-unimplemented backends, by platform. */
876
+ const RESERVED_BACKENDS = {
877
+ darwin: "macOS backend is reserved but not implemented yet (Windows-first); only Windows can act today",
878
+ linux: "Linux backend is reserved but not implemented yet (Windows-first); only Windows can act today"
879
+ };
880
+ /**
881
+ * Select the backend for this deployment. Windows needs `ctx.subprocess` to
882
+ * run the native helper; every other platform is reserved and fails closed.
883
+ *
884
+ * @param ctx - the mounting context.
885
+ * @param config - resolved plugin config (helper limits).
886
+ * @returns the active backend (possibly unavailable).
887
+ */
888
+ function createBackend(ctx, config) {
889
+ if (process.platform === "win32") {
890
+ const subprocess = ctx.get("subprocess");
891
+ if (subprocess === void 0) return new UnavailableBackend("win32", "ctx.subprocess is not mounted — required to run the native Windows helper");
892
+ return new HelperBackend(config, subprocess);
893
+ }
894
+ return new UnavailableBackend(process.platform, RESERVED_BACKENDS[process.platform] ?? "unsupported platform");
895
+ }
896
+ //#endregion
897
+ //#region src/vision.ts
898
+ /**
899
+ * Resolve whether the calling agent's current model declares image input.
900
+ *
901
+ * @param agent - the calling agent (may be undefined).
902
+ * @param ctx - the mounting context.
903
+ * @param signal - cancellation for the model-info lookup.
904
+ * @returns true only when the resolved model declares the `image` modality.
905
+ */
906
+ async function sessionAcceptsImages(agent, ctx, signal) {
907
+ if (agent === void 0) return false;
908
+ const provider = agent.options.provider;
909
+ const model = agent.options.model;
910
+ if (provider === void 0 || model === void 0) return false;
911
+ const llm = ctx.get("llm");
912
+ if (llm === void 0) return false;
913
+ try {
914
+ return (await llm.resolveModelInfo(provider, model, signal)).inputModalities?.includes("image") ?? false;
915
+ } catch {
916
+ return false;
917
+ }
918
+ }
919
+ //#endregion
920
+ //#region src/tools.ts
921
+ /** Sanitize one snapshot into the model-visible window value. */
922
+ function observedWindow(snapshot, maxTextLength) {
923
+ return {
924
+ windowId: snapshot.windowId,
925
+ processId: snapshot.processId,
926
+ title: sanitizeVisible(snapshot.title, maxTextLength),
927
+ className: sanitizeVisible(snapshot.className, maxTextLength),
928
+ executablePath: snapshot.executablePath === null ? null : sanitizePath(snapshot.executablePath, maxTextLength),
929
+ rect: { ...snapshot.rect },
930
+ foreground: snapshot.foreground
931
+ };
932
+ }
933
+ /** Sanitize one accessibility element into the model-visible form. */
934
+ function observedElement(element, maxTextLength) {
935
+ return {
936
+ elementId: element.elementId,
937
+ controlType: sanitizeVisible(element.controlType, 64),
938
+ name: sanitizeVisible(element.name, maxTextLength),
939
+ automationId: sanitizeVisible(element.automationId, 64),
940
+ rect: element.rect,
941
+ enabled: element.enabled,
942
+ patterns: element.patterns
943
+ };
944
+ }
945
+ /** Sanitize one pixel hint into the model-visible form. */
946
+ function observedPixel(pixel, maxTextLength) {
947
+ return {
948
+ label: sanitizeVisible(pixel.label, maxTextLength),
949
+ x: pixel.x,
950
+ y: pixel.y,
951
+ color: pixel.color
952
+ };
953
+ }
954
+ /** Sanitize one window listing entry. */
955
+ function observedWindowInfo(info, maxTextLength) {
956
+ return {
957
+ windowId: info.windowId,
958
+ processId: info.processId,
959
+ title: sanitizeVisible(info.title, maxTextLength),
960
+ className: sanitizeVisible(info.className, maxTextLength),
961
+ rect: info.rect,
962
+ executablePath: info.executablePath === null ? null : sanitizePath(info.executablePath, maxTextLength),
963
+ visible: info.visible
964
+ };
965
+ }
966
+ /** Append the `dsh-click/observed` audit event; a failed append is swallowed. */
967
+ function auditObservation(exec, event) {
968
+ const session = exec.agent?.session;
969
+ if (session === void 0) return;
970
+ try {
971
+ session.append(OBSERVED_EVENT, event);
972
+ } catch {}
973
+ }
974
+ /** One-line window identity for render text. */
975
+ function windowLine(window) {
976
+ return `"${window.title}" (windowId ${window.windowId}, pid ${window.processId}, ${window.executablePath ?? "unknown executable"})`;
977
+ }
978
+ /** Refuse an unusable backend before any backend call. */
979
+ function backendGuard(services) {
980
+ if (!services.backend.available) throw new Error(`dsh-click backend unavailable on ${services.backend.platform}: ${services.backend.unavailableReason ?? "unknown reason"}`);
981
+ }
982
+ /** Plain-text fallback description for a captured screenshot. */
983
+ function shotDescription(window, width, height) {
984
+ return `${width}x${height} screenshot of ${windowLine(window)}; run screen_read on the same window for the structured element list and pixel positions.`;
985
+ }
986
+ /**
987
+ * `screen_shot` — capture the addressed window (or the primary screen).
988
+ * In `imageMode: 'auto'` the result carries an image attachment when the
989
+ * current model accepts images; otherwise a text description keeps text-only
990
+ * models working.
991
+ *
992
+ * @param services - runtime services.
993
+ * @returns the tool definition.
994
+ */
995
+ function screenShotTool(services) {
996
+ const { ctx, config, backend, observations } = services;
997
+ return defineTool({
998
+ name: "screen_shot",
999
+ description: "Capture a screenshot of a desktop window (or the primary screen when no window is given). Returns an observationId that later actions cite in `basedOn`. When the current model accepts images the result includes the image; otherwise it includes a text description. Read-only: never needs approval.",
1000
+ parameters: {
1001
+ target: {
1002
+ type: "object",
1003
+ description: "Which window to capture (windowId, windowTitle, or processId); omitted = the foreground window.",
1004
+ properties: {
1005
+ windowId: {
1006
+ type: "integer",
1007
+ description: "Native window handle from app_list or screen_read."
1008
+ },
1009
+ windowTitle: {
1010
+ type: "string",
1011
+ description: "Visible window title (matched case-insensitively by substring)."
1012
+ },
1013
+ processId: {
1014
+ type: "integer",
1015
+ description: "Owning process id."
1016
+ }
1017
+ },
1018
+ additionalProperties: false
1019
+ },
1020
+ maxSide: {
1021
+ type: "integer",
1022
+ description: "Longest side in pixels; larger captures are downscaled."
1023
+ }
1024
+ },
1025
+ output: {
1026
+ schema: {
1027
+ type: "object",
1028
+ properties: {
1029
+ ok: {
1030
+ type: "boolean",
1031
+ const: true
1032
+ },
1033
+ observationId: { type: "string" },
1034
+ window: {
1035
+ type: "object",
1036
+ properties: {
1037
+ windowId: { type: "integer" },
1038
+ processId: { type: "integer" },
1039
+ title: { type: "string" },
1040
+ className: { type: "string" },
1041
+ executablePath: { oneOf: [{ type: "string" }, { type: "null" }] },
1042
+ rect: {
1043
+ type: "object",
1044
+ properties: {
1045
+ x: { type: "integer" },
1046
+ y: { type: "integer" },
1047
+ width: { type: "integer" },
1048
+ height: { type: "integer" }
1049
+ },
1050
+ additionalProperties: false
1051
+ },
1052
+ foreground: { type: "boolean" }
1053
+ },
1054
+ additionalProperties: false
1055
+ },
1056
+ image: {
1057
+ type: "object",
1058
+ properties: {
1059
+ attachmentId: { type: "string" },
1060
+ mediaType: { type: "string" },
1061
+ bytes: { type: "integer" },
1062
+ width: { type: "integer" },
1063
+ height: { type: "integer" },
1064
+ name: { type: "string" }
1065
+ },
1066
+ additionalProperties: false
1067
+ },
1068
+ description: { type: "string" }
1069
+ },
1070
+ additionalProperties: false
1071
+ },
1072
+ render(_args, value) {
1073
+ const result = value;
1074
+ const blocks = [{
1075
+ type: "text",
1076
+ text: `Screenshot of ${windowLine(result.window)} captured.\nobservationId: ${result.observationId}\n${result.image === void 0 ? result.description : "Image attached; cite this observationId in later actions."}`
1077
+ }];
1078
+ if (result.image !== void 0) blocks.push({
1079
+ type: "image",
1080
+ attachment: result.image
1081
+ });
1082
+ return blocks;
1083
+ }
1084
+ },
1085
+ timeoutMs: config.helperTimeoutMs + 15e3,
1086
+ async execute(args, exec) {
1087
+ backendGuard(services);
1088
+ const target = args.target ?? {};
1089
+ const requestedSide = args.maxSide;
1090
+ const maxSide = requestedSide === void 0 ? config.maxScreenshotSide : Math.min(requestedSide, config.maxScreenshotSide);
1091
+ const shot = await backend.shot(target, maxSide, exec.signal);
1092
+ const sanitized = observedWindow(shot.snapshot, config.maxTextLength);
1093
+ const record = observations.record(shot.snapshot);
1094
+ const description = shotDescription(sanitized, shot.width, shot.height);
1095
+ let image;
1096
+ const attachments = ctx.get("attachments");
1097
+ if (config.imageMode !== "text" && attachments !== void 0 && await sessionAcceptsImages(exec.agent, ctx, exec.signal)) try {
1098
+ image = await attachments.saveImage({
1099
+ data: Buffer.from(shot.pngBase64, "base64"),
1100
+ mediaType: "image/png",
1101
+ name: `dsh-click-${record.id.slice(0, 12)}.png`
1102
+ });
1103
+ } catch {
1104
+ image = void 0;
1105
+ }
1106
+ auditObservation(exec, {
1107
+ observationId: record.id,
1108
+ windowId: shot.snapshot.windowId,
1109
+ processId: shot.snapshot.processId,
1110
+ executablePath: sanitized.executablePath,
1111
+ windowTitle: sanitized.title,
1112
+ elementCount: shot.snapshot.elementCount,
1113
+ ...image !== void 0 ? { image: {
1114
+ attachmentId: image.attachmentId,
1115
+ mediaType: image.mediaType,
1116
+ bytes: image.bytes,
1117
+ width: image.width,
1118
+ height: image.height
1119
+ } } : {}
1120
+ });
1121
+ return {
1122
+ ok: true,
1123
+ observationId: record.id,
1124
+ window: sanitized,
1125
+ ...image !== void 0 ? { image } : {},
1126
+ description
1127
+ };
1128
+ }
1129
+ });
1130
+ }
1131
+ /**
1132
+ * `screen_read` — the structured observation for text-only models: the
1133
+ * window's accessibility tree plus pixel-location hints. The returned
1134
+ * elementIds are what click/type/scroll address.
1135
+ *
1136
+ * @param services - runtime services.
1137
+ * @returns the tool definition.
1138
+ */
1139
+ function screenReadTool(services) {
1140
+ const { config, backend, observations } = services;
1141
+ return defineTool({
1142
+ name: "screen_read",
1143
+ description: "Read a desktop window as structured text: its accessibility tree (element ids, types, names, rectangles, supported patterns) plus pixel-location hints with colors. Returns an observationId that later actions cite in `basedOn`; elements are addressed by their elementId. Read-only: never needs approval.",
1144
+ parameters: {
1145
+ target: {
1146
+ type: "object",
1147
+ description: "Which window to read (windowId, windowTitle, or processId); omitted = the foreground window.",
1148
+ properties: {
1149
+ windowId: {
1150
+ type: "integer",
1151
+ description: "Native window handle from app_list or screen_read."
1152
+ },
1153
+ windowTitle: {
1154
+ type: "string",
1155
+ description: "Visible window title (matched case-insensitively by substring)."
1156
+ },
1157
+ processId: {
1158
+ type: "integer",
1159
+ description: "Owning process id."
1160
+ }
1161
+ },
1162
+ additionalProperties: false
1163
+ },
1164
+ includePixels: {
1165
+ type: "boolean",
1166
+ description: "Include pixel-location hints (default true)."
1167
+ }
1168
+ },
1169
+ output: {
1170
+ schema: {
1171
+ type: "object",
1172
+ properties: {
1173
+ ok: {
1174
+ type: "boolean",
1175
+ const: true
1176
+ },
1177
+ observationId: { type: "string" },
1178
+ window: {
1179
+ type: "object",
1180
+ properties: {
1181
+ windowId: { type: "integer" },
1182
+ processId: { type: "integer" },
1183
+ title: { type: "string" },
1184
+ className: { type: "string" },
1185
+ executablePath: { oneOf: [{ type: "string" }, { type: "null" }] },
1186
+ rect: {
1187
+ type: "object",
1188
+ properties: {
1189
+ x: { type: "integer" },
1190
+ y: { type: "integer" },
1191
+ width: { type: "integer" },
1192
+ height: { type: "integer" }
1193
+ },
1194
+ additionalProperties: false
1195
+ },
1196
+ foreground: { type: "boolean" }
1197
+ },
1198
+ additionalProperties: false
1199
+ },
1200
+ elements: {
1201
+ type: "array",
1202
+ items: {
1203
+ type: "object",
1204
+ properties: {
1205
+ elementId: { type: "string" },
1206
+ controlType: { type: "string" },
1207
+ name: { type: "string" },
1208
+ automationId: { type: "string" },
1209
+ rect: {
1210
+ type: "object",
1211
+ properties: {
1212
+ x: { type: "integer" },
1213
+ y: { type: "integer" },
1214
+ width: { type: "integer" },
1215
+ height: { type: "integer" }
1216
+ },
1217
+ additionalProperties: false
1218
+ },
1219
+ enabled: { type: "boolean" },
1220
+ patterns: {
1221
+ type: "array",
1222
+ items: { type: "string" }
1223
+ }
1224
+ },
1225
+ additionalProperties: false
1226
+ }
1227
+ },
1228
+ pixels: {
1229
+ type: "array",
1230
+ items: {
1231
+ type: "object",
1232
+ properties: {
1233
+ label: { type: "string" },
1234
+ x: { type: "integer" },
1235
+ y: { type: "integer" },
1236
+ color: { type: "string" }
1237
+ },
1238
+ additionalProperties: false
1239
+ }
1240
+ }
1241
+ },
1242
+ additionalProperties: false
1243
+ },
1244
+ render(_args, value) {
1245
+ const result = value;
1246
+ const lines = [
1247
+ `Window read: ${windowLine(result.window)}`,
1248
+ `observationId: ${result.observationId}`,
1249
+ `${result.elements.length} element(s):`
1250
+ ];
1251
+ for (const element of result.elements) lines.push(`- [${element.elementId}] ${element.controlType} ${JSON.stringify(element.name)} at (${element.rect.x}, ${element.rect.y}) ${element.rect.width}x${element.rect.height}${element.enabled ? "" : " (disabled)"}${element.patterns.length > 0 ? ` patterns: ${element.patterns.join(",")}` : ""}`);
1252
+ if (result.pixels.length > 0) {
1253
+ lines.push(`${result.pixels.length} pixel hint(s):`);
1254
+ for (const pixel of result.pixels) lines.push(`- ${pixel.label} at (${pixel.x}, ${pixel.y}) ${pixel.color}`);
1255
+ }
1256
+ lines.push("Actions (click/type/scroll/key) must cite this observationId in `basedOn` and address elements by elementId.");
1257
+ return [{
1258
+ type: "text",
1259
+ text: lines.join("\n")
1260
+ }];
1261
+ }
1262
+ },
1263
+ timeoutMs: config.helperTimeoutMs + 15e3,
1264
+ async execute(args, exec) {
1265
+ backendGuard(services);
1266
+ const parsed = args;
1267
+ const tree = await backend.tree(parsed.target ?? {}, config.maxElements, config.maxTreeDepth, parsed.includePixels ?? true, exec.signal);
1268
+ const sanitized = observedWindow(tree.snapshot, config.maxTextLength);
1269
+ const record = observations.record(tree.snapshot);
1270
+ auditObservation(exec, {
1271
+ observationId: record.id,
1272
+ windowId: tree.snapshot.windowId,
1273
+ processId: tree.snapshot.processId,
1274
+ executablePath: sanitized.executablePath,
1275
+ windowTitle: sanitized.title,
1276
+ elementCount: tree.elements.length
1277
+ });
1278
+ return {
1279
+ ok: true,
1280
+ observationId: record.id,
1281
+ window: sanitized,
1282
+ elements: tree.elements.map((element) => observedElement(element, config.maxTextLength)),
1283
+ pixels: tree.pixels.map((pixel) => observedPixel(pixel, config.maxTextLength))
1284
+ };
1285
+ }
1286
+ });
1287
+ }
1288
+ /** The shared window-action output schema (click/scroll/key). */
1289
+ function actionOutputSchema(withRestored) {
1290
+ return {
1291
+ type: "object",
1292
+ properties: {
1293
+ ok: {
1294
+ type: "boolean",
1295
+ const: true
1296
+ },
1297
+ windowId: { type: "integer" },
1298
+ delivered: {
1299
+ type: "string",
1300
+ enum: [
1301
+ "uia",
1302
+ "posted",
1303
+ "none"
1304
+ ]
1305
+ },
1306
+ process: {
1307
+ type: "object",
1308
+ properties: {
1309
+ before: {
1310
+ type: "object",
1311
+ properties: {
1312
+ pid: { type: "integer" },
1313
+ executablePath: { oneOf: [{ type: "string" }, { type: "null" }] }
1314
+ },
1315
+ additionalProperties: false
1316
+ },
1317
+ after: {
1318
+ type: "object",
1319
+ properties: {
1320
+ pid: { type: "integer" },
1321
+ executablePath: { oneOf: [{ type: "string" }, { type: "null" }] }
1322
+ },
1323
+ additionalProperties: false
1324
+ }
1325
+ },
1326
+ additionalProperties: false
1327
+ },
1328
+ detail: { type: "string" },
1329
+ ...withRestored ? { restored: { type: "boolean" } } : {}
1330
+ },
1331
+ additionalProperties: false
1332
+ };
1333
+ }
1334
+ /** The cited-observation parameters shared by the four action tools. */
1335
+ const basedOnParameters = { basedOn: {
1336
+ type: "object",
1337
+ description: "The observation this action is based on; the action fails if the screen changed since that observation.",
1338
+ properties: {
1339
+ observationId: {
1340
+ type: "string",
1341
+ description: "observationId returned by screen_shot or screen_read.",
1342
+ required: true
1343
+ },
1344
+ windowId: {
1345
+ type: "integer",
1346
+ description: "windowId of the observed window.",
1347
+ required: true
1348
+ }
1349
+ },
1350
+ additionalProperties: false,
1351
+ required: true
1352
+ } };
1353
+ /** One-line action summary for render text. */
1354
+ function actionLine(toolName, value) {
1355
+ const before = value.process.before;
1356
+ const after = value.process.after;
1357
+ return `${toolName} delivered via ${value.delivered} to window ${value.windowId}; process pid ${before.pid} → ${after.pid}${value.restored === void 0 ? "" : value.restored ? " (original text restored)" : " (text restore unavailable)"}.`;
1358
+ }
1359
+ /**
1360
+ * `click` — click an element (by elementId) or a coordinate inside the
1361
+ * observed window. Never steals foreground focus; delivery is UIA invoke
1362
+ * where possible, posted window messages otherwise.
1363
+ *
1364
+ * @param services - runtime services.
1365
+ * @returns the tool definition.
1366
+ */
1367
+ function clickTool(services) {
1368
+ const { config, backend, actions } = services;
1369
+ return defineTool({
1370
+ name: "click",
1371
+ description: "Click an element or a coordinate inside an observed desktop window. Requires `basedOn` (a fresh observationId from screen_shot/screen_read); the call fails if the screen changed since that observation. Never steals foreground focus. Requires approval unless the window is allowlisted.",
1372
+ parameters: {
1373
+ ...basedOnParameters,
1374
+ target: {
1375
+ type: "object",
1376
+ description: "Exactly one of elementId or (x, y) — screen coordinates.",
1377
+ properties: {
1378
+ elementId: {
1379
+ type: "string",
1380
+ description: "Element id from screen_read."
1381
+ },
1382
+ x: {
1383
+ type: "integer",
1384
+ description: "Screen x coordinate."
1385
+ },
1386
+ y: {
1387
+ type: "integer",
1388
+ description: "Screen y coordinate."
1389
+ }
1390
+ },
1391
+ additionalProperties: false,
1392
+ required: true
1393
+ },
1394
+ button: {
1395
+ type: "string",
1396
+ enum: ["left", "right"],
1397
+ description: "Mouse button (default left)."
1398
+ }
1399
+ },
1400
+ output: {
1401
+ schema: actionOutputSchema(false),
1402
+ render(_args, value) {
1403
+ return [{
1404
+ type: "text",
1405
+ text: actionLine("click", value)
1406
+ }];
1407
+ }
1408
+ },
1409
+ timeoutMs: config.helperTimeoutMs + 15e3,
1410
+ async execute(args, exec) {
1411
+ backendGuard(services);
1412
+ const parsed = args;
1413
+ const target = parsed.target;
1414
+ const byElement = target.elementId !== void 0;
1415
+ if (byElement === (target.x !== void 0 && target.y !== void 0)) throw new Error("click target must name exactly one of elementId or (x, y)");
1416
+ const outcome = await actions.perform("click", exec, parsed.basedOn.observationId, parsed.basedOn.windowId, (focusFallback) => backend.click({
1417
+ windowId: parsed.basedOn.windowId,
1418
+ ...byElement ? { elementId: target.elementId } : {
1419
+ x: target.x,
1420
+ y: target.y
1421
+ },
1422
+ button: parsed.button ?? "left"
1423
+ }, focusFallback, exec.signal));
1424
+ return {
1425
+ ok: true,
1426
+ windowId: outcome.windowId,
1427
+ delivered: outcome.delivered,
1428
+ process: {
1429
+ before: outcome.processBefore,
1430
+ after: outcome.processAfter
1431
+ },
1432
+ ...outcome.detail !== void 0 ? { detail: outcome.detail } : {}
1433
+ };
1434
+ }
1435
+ });
1436
+ }
1437
+ /**
1438
+ * `type` — type text into a value-pattern element. The helper backs up the
1439
+ * control text first and restores it when the action fails (config
1440
+ * `rollbackEnabled`).
1441
+ *
1442
+ * @param services - runtime services.
1443
+ * @returns the tool definition.
1444
+ */
1445
+ function typeTool(services) {
1446
+ const { config, backend, actions } = services;
1447
+ return defineTool({
1448
+ name: "type",
1449
+ description: "Type text into an editable element of an observed desktop window (addressed by elementId from screen_read; the element must expose a value pattern). Requires `basedOn`; fails if the screen changed since that observation. Never steals foreground focus. Requires approval unless the window is allowlisted. On failure the previous control text is restored when rollback is enabled.",
1450
+ parameters: {
1451
+ ...basedOnParameters,
1452
+ elementId: {
1453
+ type: "string",
1454
+ description: "Editable element id from screen_read.",
1455
+ required: true
1456
+ },
1457
+ text: {
1458
+ type: "string",
1459
+ description: "The exact text to type (up to 10000 characters).",
1460
+ required: true
1461
+ }
1462
+ },
1463
+ output: {
1464
+ schema: actionOutputSchema(true),
1465
+ render(_args, value) {
1466
+ return [{
1467
+ type: "text",
1468
+ text: actionLine("type", value)
1469
+ }];
1470
+ }
1471
+ },
1472
+ timeoutMs: config.helperTimeoutMs + 15e3,
1473
+ async execute(args, exec) {
1474
+ backendGuard(services);
1475
+ const parsed = args;
1476
+ if (parsed.text.length > 1e4) throw new Error("type text must be at most 10000 characters");
1477
+ const outcome = await actions.perform("type", exec, parsed.basedOn.observationId, parsed.basedOn.windowId, (focusFallback) => backend.type({
1478
+ windowId: parsed.basedOn.windowId,
1479
+ elementId: parsed.elementId,
1480
+ text: parsed.text,
1481
+ rollback: config.rollbackEnabled
1482
+ }, focusFallback, exec.signal));
1483
+ return {
1484
+ ok: true,
1485
+ windowId: outcome.windowId,
1486
+ delivered: outcome.delivered,
1487
+ process: {
1488
+ before: outcome.processBefore,
1489
+ after: outcome.processAfter
1490
+ },
1491
+ restored: outcome.restored ?? false,
1492
+ ...outcome.detail !== void 0 ? { detail: outcome.detail } : {}
1493
+ };
1494
+ }
1495
+ });
1496
+ }
1497
+ /**
1498
+ * `scroll` — scroll an element (scroll pattern) or the observed window
1499
+ * (posted wheel messages). Never steals foreground focus.
1500
+ *
1501
+ * @param services - runtime services.
1502
+ * @returns the tool definition.
1503
+ */
1504
+ function scrollTool(services) {
1505
+ const { config, backend, actions } = services;
1506
+ return defineTool({
1507
+ name: "scroll",
1508
+ description: "Scroll an element (by elementId) or the observed window. Requires `basedOn`; fails if the screen changed since that observation. Never steals foreground focus. Requires approval unless the window is allowlisted.",
1509
+ parameters: {
1510
+ ...basedOnParameters,
1511
+ elementId: {
1512
+ type: "string",
1513
+ description: "Optional scrollable element id from screen_read; omitted = scroll the window itself."
1514
+ },
1515
+ direction: {
1516
+ type: "string",
1517
+ enum: [
1518
+ "up",
1519
+ "down",
1520
+ "page-up",
1521
+ "page-down"
1522
+ ],
1523
+ description: "Scroll direction.",
1524
+ required: true
1525
+ },
1526
+ amount: {
1527
+ type: "integer",
1528
+ description: "Number of increments (default 3)."
1529
+ }
1530
+ },
1531
+ output: {
1532
+ schema: actionOutputSchema(false),
1533
+ render(_args, value) {
1534
+ return [{
1535
+ type: "text",
1536
+ text: actionLine("scroll", value)
1537
+ }];
1538
+ }
1539
+ },
1540
+ timeoutMs: config.helperTimeoutMs + 15e3,
1541
+ async execute(args, exec) {
1542
+ backendGuard(services);
1543
+ const parsed = args;
1544
+ const amount = parsed.amount ?? 3;
1545
+ if (!Number.isInteger(amount) || amount < 1) throw new Error("scroll amount must be a positive integer");
1546
+ const outcome = await actions.perform("scroll", exec, parsed.basedOn.observationId, parsed.basedOn.windowId, (focusFallback) => backend.scroll({
1547
+ windowId: parsed.basedOn.windowId,
1548
+ ...parsed.elementId !== void 0 ? { elementId: parsed.elementId } : {},
1549
+ direction: parsed.direction,
1550
+ amount
1551
+ }, focusFallback, exec.signal));
1552
+ return {
1553
+ ok: true,
1554
+ windowId: outcome.windowId,
1555
+ delivered: outcome.delivered,
1556
+ process: {
1557
+ before: outcome.processBefore,
1558
+ after: outcome.processAfter
1559
+ },
1560
+ ...outcome.detail !== void 0 ? { detail: outcome.detail } : {}
1561
+ };
1562
+ }
1563
+ });
1564
+ }
1565
+ /**
1566
+ * `key` — send a key combination to the observed window as posted window
1567
+ * messages. Never steals foreground focus; apps that ignore posted input
1568
+ * fail with a clear error.
1569
+ *
1570
+ * @param services - runtime services.
1571
+ * @returns the tool definition.
1572
+ */
1573
+ function keyTool(services) {
1574
+ const { config, backend, actions } = services;
1575
+ return defineTool({
1576
+ name: "key",
1577
+ description: "Send a key combination (e.g. \"Ctrl+S\", \"Enter\", \"Alt+F4\") to an observed desktop window via posted window messages. Requires `basedOn`; fails if the screen changed since that observation. Never steals foreground focus — applications that ignore posted input will not react; prefer click/type for those. Requires approval unless the window is allowlisted.",
1578
+ parameters: {
1579
+ ...basedOnParameters,
1580
+ keys: {
1581
+ type: "string",
1582
+ description: "Key combination, e.g. \"Ctrl+S\" or \"Enter\".",
1583
+ required: true
1584
+ }
1585
+ },
1586
+ output: {
1587
+ schema: actionOutputSchema(false),
1588
+ render(_args, value) {
1589
+ return [{
1590
+ type: "text",
1591
+ text: actionLine("key", value)
1592
+ }];
1593
+ }
1594
+ },
1595
+ timeoutMs: config.helperTimeoutMs + 15e3,
1596
+ async execute(args, exec) {
1597
+ backendGuard(services);
1598
+ const parsed = args;
1599
+ if (parsed.keys.trim() === "") throw new Error("keys must not be empty");
1600
+ const outcome = await actions.perform("key", exec, parsed.basedOn.observationId, parsed.basedOn.windowId, (focusFallback) => backend.key({
1601
+ windowId: parsed.basedOn.windowId,
1602
+ keys: parsed.keys.trim()
1603
+ }, focusFallback, exec.signal));
1604
+ return {
1605
+ ok: true,
1606
+ windowId: outcome.windowId,
1607
+ delivered: outcome.delivered,
1608
+ process: {
1609
+ before: outcome.processBefore,
1610
+ after: outcome.processAfter
1611
+ },
1612
+ ...outcome.detail !== void 0 ? { detail: outcome.detail } : {}
1613
+ };
1614
+ }
1615
+ });
1616
+ }
1617
+ /**
1618
+ * `app_list` — enumerate running desktop applications and their windows.
1619
+ * Read-only: never needs approval.
1620
+ *
1621
+ * @param services - runtime services.
1622
+ * @returns the tool definition.
1623
+ */
1624
+ function appListTool(services) {
1625
+ const { config, backend } = services;
1626
+ return defineTool({
1627
+ name: "app_list",
1628
+ description: "List running desktop applications and their visible windows (windowId, title, process id, executable path). Read-only: never needs approval. Use the returned windowIds as screen_shot/screen_read targets.",
1629
+ parameters: {},
1630
+ output: {
1631
+ schema: {
1632
+ type: "object",
1633
+ properties: {
1634
+ ok: {
1635
+ type: "boolean",
1636
+ const: true
1637
+ },
1638
+ apps: {
1639
+ type: "array",
1640
+ items: {
1641
+ type: "object",
1642
+ properties: {
1643
+ processId: { type: "integer" },
1644
+ name: { type: "string" },
1645
+ executablePath: { oneOf: [{ type: "string" }, { type: "null" }] },
1646
+ windows: {
1647
+ type: "array",
1648
+ items: {
1649
+ type: "object",
1650
+ properties: {
1651
+ windowId: { type: "integer" },
1652
+ processId: { oneOf: [{ type: "integer" }, { type: "null" }] },
1653
+ title: { type: "string" },
1654
+ className: { type: "string" },
1655
+ rect: {
1656
+ type: "object",
1657
+ properties: {
1658
+ x: { type: "integer" },
1659
+ y: { type: "integer" },
1660
+ width: { type: "integer" },
1661
+ height: { type: "integer" }
1662
+ },
1663
+ additionalProperties: false
1664
+ },
1665
+ executablePath: { oneOf: [{ type: "string" }, { type: "null" }] },
1666
+ visible: { type: "boolean" }
1667
+ },
1668
+ additionalProperties: false
1669
+ }
1670
+ }
1671
+ },
1672
+ additionalProperties: false
1673
+ }
1674
+ }
1675
+ },
1676
+ additionalProperties: false
1677
+ },
1678
+ render(_args, value) {
1679
+ const result = value;
1680
+ const lines = [`${result.apps.length} running application(s) with windows:`];
1681
+ for (const app of result.apps) {
1682
+ lines.push(`- ${app.name} (pid ${app.processId}, ${app.executablePath ?? "unknown executable"})`);
1683
+ for (const window of app.windows) lines.push(` - [windowId ${window.windowId}] ${JSON.stringify(window.title)} at (${window.rect.x}, ${window.rect.y}) ${window.rect.width}x${window.rect.height}`);
1684
+ }
1685
+ return [{
1686
+ type: "text",
1687
+ text: lines.join("\n")
1688
+ }];
1689
+ }
1690
+ },
1691
+ timeoutMs: config.helperTimeoutMs + 15e3,
1692
+ async execute(_args, exec) {
1693
+ backendGuard(services);
1694
+ return {
1695
+ ok: true,
1696
+ apps: (await backend.apps(exec.signal)).map((app) => ({
1697
+ processId: app.processId,
1698
+ name: sanitizeVisible(app.name, config.maxTextLength),
1699
+ executablePath: app.executablePath === null ? null : sanitizePath(app.executablePath, config.maxTextLength),
1700
+ windows: app.windows.map((window) => observedWindowInfo(window, config.maxTextLength))
1701
+ }))
1702
+ };
1703
+ }
1704
+ });
1705
+ }
1706
+ /**
1707
+ * `app_launch` — launch one application by name or path. Gated by approval
1708
+ * (allowlist matching applies against the requested name/path).
1709
+ *
1710
+ * @param services - runtime services.
1711
+ * @returns the tool definition.
1712
+ */
1713
+ function appLaunchTool(services) {
1714
+ const { config, actions } = services;
1715
+ return defineTool({
1716
+ name: "app_launch",
1717
+ description: "Launch a desktop application by name (e.g. \"notepad\") or executable path, with optional arguments. Requires approval unless requireApproval is off or the name/path is allowlisted. Returns the new process identity.",
1718
+ parameters: {
1719
+ name: {
1720
+ type: "string",
1721
+ description: "Application name (resolved through the executable search path) or full path.",
1722
+ required: true
1723
+ },
1724
+ args: {
1725
+ type: "array",
1726
+ items: { type: "string" },
1727
+ description: "Optional command-line arguments."
1728
+ }
1729
+ },
1730
+ output: {
1731
+ schema: {
1732
+ type: "object",
1733
+ properties: {
1734
+ ok: {
1735
+ type: "boolean",
1736
+ const: true
1737
+ },
1738
+ processId: { type: "integer" },
1739
+ executablePath: { oneOf: [{ type: "string" }, { type: "null" }] }
1740
+ },
1741
+ additionalProperties: false
1742
+ },
1743
+ render(_args, value) {
1744
+ const result = value;
1745
+ return [{
1746
+ type: "text",
1747
+ text: `Launched: pid ${result.processId} (${result.executablePath ?? "unknown executable"}).`
1748
+ }];
1749
+ }
1750
+ },
1751
+ timeoutMs: config.helperTimeoutMs + 15e3,
1752
+ async execute(args, exec) {
1753
+ backendGuard(services);
1754
+ const parsed = args;
1755
+ if (parsed.name.trim() === "") throw new Error("app_launch name must not be empty");
1756
+ const outcome = await actions.launch(exec, parsed.name.trim(), parsed.args ?? []);
1757
+ return {
1758
+ ok: true,
1759
+ processId: outcome.processId,
1760
+ executablePath: outcome.executablePath === null ? null : sanitizePath(outcome.executablePath, config.maxTextLength)
1761
+ };
1762
+ }
1763
+ });
1764
+ }
1765
+ /** Every tool definition, in registration order. */
1766
+ function allTools(services) {
1767
+ return [
1768
+ screenShotTool(services),
1769
+ screenReadTool(services),
1770
+ clickTool(services),
1771
+ typeTool(services),
1772
+ scrollTool(services),
1773
+ keyTool(services),
1774
+ appListTool(services),
1775
+ appLaunchTool(services)
1776
+ ];
1777
+ }
1778
+ //#endregion
1779
+ //#region src/index.ts
1780
+ const name = "dsh-click";
1781
+ /** Hard services: the tool registry every contribution lands in. */
1782
+ const inject = ["tools"];
1783
+ /**
1784
+ * Mount the plugin: resolve config, select the platform backend, and register
1785
+ * every tool through `ctx.tools.register` (each registration is an effect
1786
+ * whose disposer removes exactly that tool on stop/HMR).
1787
+ *
1788
+ * @param ctx - context carrying the tools registry.
1789
+ * @param config - raw loader config; defaults applied through {@link resolveConfig}.
1790
+ */
1791
+ function apply(ctx, config) {
1792
+ const resolved = resolveConfig(config);
1793
+ const backend = ctx.get("dsh-click/backend") ?? createBackend(ctx, resolved);
1794
+ const observations = new ObservationStore(resolved);
1795
+ const services = {
1796
+ ctx,
1797
+ config: resolved,
1798
+ backend,
1799
+ observations,
1800
+ actions: new ActionExecutor({
1801
+ ctx,
1802
+ config: resolved,
1803
+ backend,
1804
+ observations
1805
+ })
1806
+ };
1807
+ for (const tool of allTools(services)) ctx.effect(() => ctx.tools.register(tool), `dsh-click: ${tool.name} tool`);
1808
+ }
1809
+ //#endregion
1810
+ export { ACTION_EVENT, ActionExecutor, Config, DshClickError, HELPER_PROTOCOL_VERSION, HelperBackend, OBSERVED_EVENT, ObservationStore, UnavailableBackend, VERSION, allTools, apply, createBackend, inject, name, observationIdOf, redactSensitive, resolveConfig, sanitizePath, sanitizeText, sanitizeVisible };