dsh-click 0.1.4 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/README.es.md +38 -32
- package/README.hi.md +38 -32
- package/README.md +38 -32
- package/README.pt.md +38 -32
- package/README.zh.md +38 -32
- package/cordis.patch.yml +9 -0
- package/lib/index.js +432 -15
- package/lib/types/config.d.ts +26 -5
- package/lib/types/config.d.ts.map +1 -1
- package/lib/types/events.d.ts +24 -7
- package/lib/types/events.d.ts.map +1 -1
- package/lib/types/index.d.ts +6 -2
- package/lib/types/index.d.ts.map +1 -1
- package/lib/types/tools.d.ts +17 -0
- package/lib/types/tools.d.ts.map +1 -1
- package/lib/types/version.d.ts +1 -1
- package/lib/types/vision/grounding.d.ts +46 -0
- package/lib/types/vision/grounding.d.ts.map +1 -0
- package/lib/types/vision/index.d.ts +41 -0
- package/lib/types/vision/index.d.ts.map +1 -0
- package/lib/types/vision/ocr.d.ts +89 -0
- package/lib/types/vision/ocr.d.ts.map +1 -0
- package/package.json +6 -2
package/lib/index.js
CHANGED
|
@@ -1,14 +1,35 @@
|
|
|
1
|
+
import { KNOWN_SESSION_EVENT_TYPES } from "@deepseek-ai/dsh-session";
|
|
1
2
|
import z from "@deepseek-ai/schemastery";
|
|
2
3
|
import { createHash, randomUUID } from "node:crypto";
|
|
3
4
|
import { existsSync } from "node:fs";
|
|
4
|
-
import path from "node:path";
|
|
5
|
+
import path, { join } from "node:path";
|
|
5
6
|
import { fileURLToPath } from "node:url";
|
|
6
7
|
import { defineTool } from "@deepseek-ai/dsh-tools";
|
|
8
|
+
import { mkdtemp, rm, writeFile } from "node:fs/promises";
|
|
9
|
+
import { tmpdir } from "node:os";
|
|
7
10
|
//#region src/events.ts
|
|
8
11
|
/** The observation audit event type. */
|
|
9
12
|
const OBSERVED_EVENT = "dsh-click/observed";
|
|
10
13
|
/** The action audit event type. */
|
|
11
14
|
const ACTION_EVENT = "dsh-click/action";
|
|
15
|
+
/**
|
|
16
|
+
* Append one dsh-click audit event when the host can carry it safely; skip
|
|
17
|
+
* silently otherwise (the tool/call + tool/result events remain the
|
|
18
|
+
* model-visible log, so nothing model-visible is lost). See the module doc
|
|
19
|
+
* for the three host classes.
|
|
20
|
+
* @param session - the calling session.
|
|
21
|
+
* @param type - the audit event type.
|
|
22
|
+
* @param data - the audit payload.
|
|
23
|
+
*/
|
|
24
|
+
function appendAuditEvent(session, type, data) {
|
|
25
|
+
if (KNOWN_SESSION_EVENT_TYPES.has(type)) {
|
|
26
|
+
if (type === "dsh-click/observed") session.append(type, data);
|
|
27
|
+
else session.append(type, data);
|
|
28
|
+
return;
|
|
29
|
+
}
|
|
30
|
+
const append = session.append;
|
|
31
|
+
if (Function.prototype.toString.call(append).includes("ignorable")) append.call(session, type, data, { ignorable: true });
|
|
32
|
+
}
|
|
12
33
|
//#endregion
|
|
13
34
|
//#region src/sanitize.ts
|
|
14
35
|
/**
|
|
@@ -148,7 +169,7 @@ var ActionExecutor = class {
|
|
|
148
169
|
const session = exec.agent?.session;
|
|
149
170
|
if (session === void 0) return;
|
|
150
171
|
try {
|
|
151
|
-
session
|
|
172
|
+
appendAuditEvent(session, ACTION_EVENT, event);
|
|
152
173
|
} catch {}
|
|
153
174
|
}
|
|
154
175
|
/**
|
|
@@ -348,7 +369,12 @@ const Config = z.object({
|
|
|
348
369
|
maxElements: z.number().min(1).max(MAX_ELEMENTS).default(500),
|
|
349
370
|
maxTreeDepth: z.number().min(1).max(64).default(32),
|
|
350
371
|
maxTextLength: z.number().min(16).max(MAX_TEXT_LENGTH).default(200),
|
|
351
|
-
rollbackEnabled: z.boolean().default(true)
|
|
372
|
+
rollbackEnabled: z.boolean().default(true),
|
|
373
|
+
ocr: z.object({
|
|
374
|
+
enabled: z.boolean().default(true),
|
|
375
|
+
command: z.string().min(1).max(256).default("tesseract"),
|
|
376
|
+
language: z.string().min(1).max(32).default("eng")
|
|
377
|
+
})
|
|
352
378
|
});
|
|
353
379
|
/** Throw the standard fail-loud config error for one invalid field. */
|
|
354
380
|
function invalid(field, detail) {
|
|
@@ -398,6 +424,12 @@ function resolveConfig(config) {
|
|
|
398
424
|
if (!Number.isInteger(maxTextLength) || maxTextLength < 16 || maxTextLength > 1e4) invalid("maxTextLength", `must be an integer between 16 and ${MAX_TEXT_LENGTH}`);
|
|
399
425
|
const rollbackEnabled = config?.rollbackEnabled ?? true;
|
|
400
426
|
if (typeof rollbackEnabled !== "boolean") invalid("rollbackEnabled", "must be a boolean");
|
|
427
|
+
const ocrEnabled = config?.ocr?.enabled ?? true;
|
|
428
|
+
if (typeof ocrEnabled !== "boolean") invalid("ocr.enabled", "must be a boolean");
|
|
429
|
+
const ocrCommand = config?.ocr?.command ?? "tesseract";
|
|
430
|
+
if (typeof ocrCommand !== "string" || ocrCommand.length === 0 || ocrCommand.length > 256) invalid("ocr.command", "must be a non-empty string of at most 256 characters");
|
|
431
|
+
const ocrLanguage = config?.ocr?.language ?? "eng";
|
|
432
|
+
if (typeof ocrLanguage !== "string" || ocrLanguage.length === 0 || ocrLanguage.length > 32) invalid("ocr.language", "must be a non-empty string of at most 32 characters");
|
|
401
433
|
return Object.freeze({
|
|
402
434
|
requireApproval,
|
|
403
435
|
autoApproveMatchers: Object.freeze(autoApproveMatchers),
|
|
@@ -413,7 +445,12 @@ function resolveConfig(config) {
|
|
|
413
445
|
maxElements,
|
|
414
446
|
maxTreeDepth,
|
|
415
447
|
maxTextLength,
|
|
416
|
-
rollbackEnabled
|
|
448
|
+
rollbackEnabled,
|
|
449
|
+
ocr: Object.freeze({
|
|
450
|
+
enabled: ocrEnabled,
|
|
451
|
+
command: ocrCommand,
|
|
452
|
+
language: ocrLanguage
|
|
453
|
+
})
|
|
417
454
|
});
|
|
418
455
|
}
|
|
419
456
|
//#endregion
|
|
@@ -538,7 +575,7 @@ var ObservationStore = class {
|
|
|
538
575
|
* @module dsh-click/version
|
|
539
576
|
*/
|
|
540
577
|
/** Plugin version advertised in the helper-protocol handshake. */
|
|
541
|
-
const VERSION = "0.
|
|
578
|
+
const VERSION = "0.3.0";
|
|
542
579
|
/** The helper protocol revision; bump only on incompatible wire changes. */
|
|
543
580
|
const HELPER_PROTOCOL_VERSION = 1;
|
|
544
581
|
//#endregion
|
|
@@ -922,6 +959,209 @@ async function sessionAcceptsImages(agent, ctx, signal) {
|
|
|
922
959
|
}
|
|
923
960
|
}
|
|
924
961
|
//#endregion
|
|
962
|
+
//#region src/vision/ocr.ts
|
|
963
|
+
/**
|
|
964
|
+
* Optical character recognition seam for windows without a UIA tree. OCR is
|
|
965
|
+
* optional and probed at mount: the `tesseract` provider is used only when the
|
|
966
|
+
* binary resolves through the subprocess service, otherwise an unavailable
|
|
967
|
+
* provider fails closed with a model-readable reason. Recognized words carry
|
|
968
|
+
* their pixel boxes, which `screen_find` maps to screen coordinates for the
|
|
969
|
+
* coordinate-based `click` path.
|
|
970
|
+
*
|
|
971
|
+
* @module dsh-click/vision/ocr
|
|
972
|
+
*/
|
|
973
|
+
/** Bytes cap for one tesseract TSV stdout. */
|
|
974
|
+
const OCR_STDOUT_BYTES = 16777216;
|
|
975
|
+
/**
|
|
976
|
+
* Build the tesseract command line: write words as TSV to stdout.
|
|
977
|
+
* @param command - tesseract executable.
|
|
978
|
+
* @param imagePath - absolute PNG path.
|
|
979
|
+
* @param language - tesseract language tag.
|
|
980
|
+
* @returns the argv (argv[0] is the executable).
|
|
981
|
+
*/
|
|
982
|
+
function tesseractArgv(command, imagePath, language) {
|
|
983
|
+
return [
|
|
984
|
+
command,
|
|
985
|
+
imagePath,
|
|
986
|
+
"stdout",
|
|
987
|
+
"-l",
|
|
988
|
+
language,
|
|
989
|
+
"tsv"
|
|
990
|
+
];
|
|
991
|
+
}
|
|
992
|
+
/**
|
|
993
|
+
* Parse tesseract TSV output into word boxes. Non-word rows, empty text, and
|
|
994
|
+
* malformed numeric fields are skipped; a hostile or truncated stdout cannot
|
|
995
|
+
* produce bogus coordinates.
|
|
996
|
+
* @param tsv - tesseract TSV text from stdout.
|
|
997
|
+
* @returns the recognized words.
|
|
998
|
+
*/
|
|
999
|
+
function parseTesseractTsv(tsv) {
|
|
1000
|
+
const words = [];
|
|
1001
|
+
for (const line of tsv.split(/\r?\n/u)) {
|
|
1002
|
+
if (line === "" || line.startsWith("level")) continue;
|
|
1003
|
+
const columns = line.split(" ");
|
|
1004
|
+
if (columns.length < 12 || columns[0] !== "5") continue;
|
|
1005
|
+
const left = Number(columns[6]);
|
|
1006
|
+
const top = Number(columns[7]);
|
|
1007
|
+
const width = Number(columns[8]);
|
|
1008
|
+
const height = Number(columns[9]);
|
|
1009
|
+
const confidence = Number(columns[10]);
|
|
1010
|
+
const text = columns[11] ?? "";
|
|
1011
|
+
if (text.trim() === "") continue;
|
|
1012
|
+
if (!Number.isFinite(left) || !Number.isFinite(top) || !Number.isFinite(width) || !Number.isFinite(height)) continue;
|
|
1013
|
+
words.push({
|
|
1014
|
+
text: text.trim(),
|
|
1015
|
+
box: {
|
|
1016
|
+
x: left,
|
|
1017
|
+
y: top,
|
|
1018
|
+
width,
|
|
1019
|
+
height
|
|
1020
|
+
},
|
|
1021
|
+
confidence: Number.isFinite(confidence) ? confidence : 0
|
|
1022
|
+
});
|
|
1023
|
+
}
|
|
1024
|
+
return words;
|
|
1025
|
+
}
|
|
1026
|
+
/** A provider that cannot run: `recognize` fails closed with the given reason. */
|
|
1027
|
+
var UnavailableOcrProvider = class {
|
|
1028
|
+
id;
|
|
1029
|
+
reason;
|
|
1030
|
+
available = false;
|
|
1031
|
+
/** @param id - provider id. @param reason - model-readable unavailability reason. */
|
|
1032
|
+
constructor(id, reason) {
|
|
1033
|
+
this.id = id;
|
|
1034
|
+
this.reason = reason;
|
|
1035
|
+
}
|
|
1036
|
+
get unavailableReason() {
|
|
1037
|
+
return this.reason;
|
|
1038
|
+
}
|
|
1039
|
+
recognize(_pngBase64, _signal) {
|
|
1040
|
+
return Promise.reject(new DshClickError(`OCR unavailable: ${this.reason}`, "OCR_UNAVAILABLE"));
|
|
1041
|
+
}
|
|
1042
|
+
};
|
|
1043
|
+
/** The tesseract OCR provider over `ctx.subprocess`. */
|
|
1044
|
+
var TesseractOcrProvider = class {
|
|
1045
|
+
subprocess;
|
|
1046
|
+
command;
|
|
1047
|
+
language;
|
|
1048
|
+
id = "tesseract";
|
|
1049
|
+
available = true;
|
|
1050
|
+
/**
|
|
1051
|
+
* @param subprocess - the subprocess service that runs tesseract.
|
|
1052
|
+
* @param command - tesseract executable name.
|
|
1053
|
+
* @param language - tesseract language tag.
|
|
1054
|
+
*/
|
|
1055
|
+
constructor(subprocess, command, language) {
|
|
1056
|
+
this.subprocess = subprocess;
|
|
1057
|
+
this.command = command;
|
|
1058
|
+
this.language = language;
|
|
1059
|
+
}
|
|
1060
|
+
async recognize(pngBase64, signal) {
|
|
1061
|
+
const dir = await mkdtemp(join(tmpdir(), "dsh-click-ocr-"));
|
|
1062
|
+
const imagePath = join(dir, "shot.png");
|
|
1063
|
+
try {
|
|
1064
|
+
await writeFile(imagePath, Buffer.from(pngBase64, "base64"));
|
|
1065
|
+
const handle = this.subprocess.spawn({
|
|
1066
|
+
argv: tesseractArgv(this.command, imagePath, this.language),
|
|
1067
|
+
cwd: tmpdir(),
|
|
1068
|
+
stdio: {
|
|
1069
|
+
stdin: "ignore",
|
|
1070
|
+
stdout: { maxBytes: OCR_STDOUT_BYTES },
|
|
1071
|
+
stderr: { maxBytes: 65536 }
|
|
1072
|
+
},
|
|
1073
|
+
graceMs: 5e3,
|
|
1074
|
+
...signal === void 0 ? {} : { signal }
|
|
1075
|
+
});
|
|
1076
|
+
const outcome = await handle.done;
|
|
1077
|
+
const stdout = handle.collected.stdout?.readFrom(0).text ?? "";
|
|
1078
|
+
const stderr = handle.collected.stderr?.readFrom(0).text ?? "";
|
|
1079
|
+
if (outcome.exitCode !== 0) throw new DshClickError(`tesseract exited ${String(outcome.exitCode)}: ${sanitizeText(stderr, 500)}`, "OCR_FAILED");
|
|
1080
|
+
return parseTesseractTsv(stdout);
|
|
1081
|
+
} finally {
|
|
1082
|
+
await rm(dir, {
|
|
1083
|
+
recursive: true,
|
|
1084
|
+
force: true
|
|
1085
|
+
}).catch(() => {});
|
|
1086
|
+
}
|
|
1087
|
+
}
|
|
1088
|
+
};
|
|
1089
|
+
/**
|
|
1090
|
+
* Probe tesseract through the subprocess service and return a provider.
|
|
1091
|
+
* A missing binary degrades to an unavailable provider (fail closed), never a
|
|
1092
|
+
* crash, so the zero-dependency desktop path keeps working.
|
|
1093
|
+
*
|
|
1094
|
+
* @param subprocess - the subprocess service.
|
|
1095
|
+
* @param command - tesseract executable name.
|
|
1096
|
+
* @param language - tesseract language tag.
|
|
1097
|
+
* @returns the probed OCR provider.
|
|
1098
|
+
*/
|
|
1099
|
+
async function probeTesseract(subprocess, command, language) {
|
|
1100
|
+
try {
|
|
1101
|
+
await subprocess.resolveExecutable(command);
|
|
1102
|
+
return new TesseractOcrProvider(subprocess, command, language);
|
|
1103
|
+
} catch {
|
|
1104
|
+
return new UnavailableOcrProvider("tesseract", `tesseract (${command}) is not installed`);
|
|
1105
|
+
}
|
|
1106
|
+
}
|
|
1107
|
+
//#endregion
|
|
1108
|
+
//#region src/vision/grounding.ts
|
|
1109
|
+
/**
|
|
1110
|
+
* Visual grounding seam: turning a screenshot plus a natural-language prompt
|
|
1111
|
+
* into labeled bounding boxes. Kept as an independent provider so a vision
|
|
1112
|
+
* model (or an external grounding service) can plug in later without touching
|
|
1113
|
+
* the desktop backend or the tool surface. The default is unavailable — the
|
|
1114
|
+
* OCR path (`screen_find`) already yields text targets, and a grounding
|
|
1115
|
+
* provider is optional and probed through the `dsh-click/grounding` seam.
|
|
1116
|
+
*
|
|
1117
|
+
* @module dsh-click/vision/grounding
|
|
1118
|
+
*/
|
|
1119
|
+
/** A provider that cannot run: `ground` fails closed with the given reason. */
|
|
1120
|
+
var UnavailableGroundingProvider = class {
|
|
1121
|
+
id;
|
|
1122
|
+
reason;
|
|
1123
|
+
available = false;
|
|
1124
|
+
/** @param id - provider id. @param reason - model-readable unavailability reason. */
|
|
1125
|
+
constructor(id = "unavailable", reason = "no visual grounding provider configured") {
|
|
1126
|
+
this.id = id;
|
|
1127
|
+
this.reason = reason;
|
|
1128
|
+
}
|
|
1129
|
+
get unavailableReason() {
|
|
1130
|
+
return this.reason;
|
|
1131
|
+
}
|
|
1132
|
+
ground(_pngBase64, _width, _height, _prompt, _signal) {
|
|
1133
|
+
return Promise.reject(new DshClickError(`visual grounding unavailable: ${this.reason}`, "GROUNDING_UNAVAILABLE"));
|
|
1134
|
+
}
|
|
1135
|
+
};
|
|
1136
|
+
//#endregion
|
|
1137
|
+
//#region src/vision/index.ts
|
|
1138
|
+
/**
|
|
1139
|
+
* Map one screenshot box (possibly downscaled by `maxScreenshotSide`) to its
|
|
1140
|
+
* screen-space box, centering on the window's screen rect origin.
|
|
1141
|
+
* @param rect - the window's screen rectangle.
|
|
1142
|
+
* @param imageWidth - screenshot width in pixels.
|
|
1143
|
+
* @param imageHeight - screenshot height in pixels.
|
|
1144
|
+
* @param box - the screenshot-space box.
|
|
1145
|
+
* @returns the screen-space box.
|
|
1146
|
+
*/
|
|
1147
|
+
function screenBoxFor(rect, imageWidth, imageHeight, box) {
|
|
1148
|
+
const scaleX = rect.width > 0 && imageWidth > 0 ? rect.width / imageWidth : 1;
|
|
1149
|
+
const scaleY = rect.height > 0 && imageHeight > 0 ? rect.height / imageHeight : 1;
|
|
1150
|
+
return {
|
|
1151
|
+
x: rect.x + box.x * scaleX,
|
|
1152
|
+
y: rect.y + box.y * scaleY,
|
|
1153
|
+
width: box.width * scaleX,
|
|
1154
|
+
height: box.height * scaleY
|
|
1155
|
+
};
|
|
1156
|
+
}
|
|
1157
|
+
/** Center point of a screen-space box (the coordinate `click` addresses). */
|
|
1158
|
+
function centerOf(box) {
|
|
1159
|
+
return {
|
|
1160
|
+
x: Math.round(box.x + box.width / 2),
|
|
1161
|
+
y: Math.round(box.y + box.height / 2)
|
|
1162
|
+
};
|
|
1163
|
+
}
|
|
1164
|
+
//#endregion
|
|
925
1165
|
//#region src/tools.ts
|
|
926
1166
|
/** Sanitize one snapshot into the model-visible window value. */
|
|
927
1167
|
function observedWindow(snapshot, maxTextLength) {
|
|
@@ -974,7 +1214,7 @@ function auditObservation(exec, event, auditSessionEvents = true) {
|
|
|
974
1214
|
const session = exec.agent?.session;
|
|
975
1215
|
if (session === void 0) return;
|
|
976
1216
|
try {
|
|
977
|
-
session
|
|
1217
|
+
appendAuditEvent(session, OBSERVED_EVENT, event);
|
|
978
1218
|
} catch {}
|
|
979
1219
|
}
|
|
980
1220
|
/** One-line window identity for render text. */
|
|
@@ -1291,6 +1531,177 @@ function screenReadTool(services) {
|
|
|
1291
1531
|
}
|
|
1292
1532
|
});
|
|
1293
1533
|
}
|
|
1534
|
+
/**
|
|
1535
|
+
* `screen_find` — find clickable text/visual targets in a window whose
|
|
1536
|
+
* accessibility tree is empty (no UIA elements). It captures the window,
|
|
1537
|
+
* runs OCR (when the probed provider is available) and, when a grounding
|
|
1538
|
+
* provider is mounted and a query is given, visual grounding, then returns
|
|
1539
|
+
* screen-coordinate targets plus an observationId for the coordinate-based
|
|
1540
|
+
* `click` path. Read-only: never needs approval.
|
|
1541
|
+
*
|
|
1542
|
+
* @param services - runtime services.
|
|
1543
|
+
* @returns the tool definition.
|
|
1544
|
+
*/
|
|
1545
|
+
function screenFindTool(services) {
|
|
1546
|
+
const { config, backend, observations, ocr, grounding } = services;
|
|
1547
|
+
return defineTool({
|
|
1548
|
+
name: "screen_find",
|
|
1549
|
+
description: "Find clickable text (or a described visual target) in a desktop window that exposes no accessibility tree, via OCR and optional visual grounding. Returns screen-coordinate targets plus an observationId; later `click` calls address a target by its (x, y) coordinates and cite this observationId in `basedOn`. Read-only: never needs approval.",
|
|
1550
|
+
parameters: {
|
|
1551
|
+
target: {
|
|
1552
|
+
type: "object",
|
|
1553
|
+
description: "Which window to search (windowId, windowTitle, or processId); omitted = the foreground window.",
|
|
1554
|
+
properties: {
|
|
1555
|
+
windowId: {
|
|
1556
|
+
type: "integer",
|
|
1557
|
+
description: "Native window handle from app_list."
|
|
1558
|
+
},
|
|
1559
|
+
windowTitle: {
|
|
1560
|
+
type: "string",
|
|
1561
|
+
description: "Visible window title (matched case-insensitively by substring)."
|
|
1562
|
+
},
|
|
1563
|
+
processId: {
|
|
1564
|
+
type: "integer",
|
|
1565
|
+
description: "Owning process id."
|
|
1566
|
+
}
|
|
1567
|
+
},
|
|
1568
|
+
additionalProperties: false
|
|
1569
|
+
},
|
|
1570
|
+
query: {
|
|
1571
|
+
type: "string",
|
|
1572
|
+
description: "Optional substring to filter OCR text by, or the visual target description for grounding."
|
|
1573
|
+
},
|
|
1574
|
+
maxSide: {
|
|
1575
|
+
type: "integer",
|
|
1576
|
+
description: "Longest side in pixels; larger captures are downscaled."
|
|
1577
|
+
}
|
|
1578
|
+
},
|
|
1579
|
+
output: {
|
|
1580
|
+
schema: {
|
|
1581
|
+
type: "object",
|
|
1582
|
+
properties: {
|
|
1583
|
+
ok: {
|
|
1584
|
+
type: "boolean",
|
|
1585
|
+
const: true
|
|
1586
|
+
},
|
|
1587
|
+
observationId: { type: "string" },
|
|
1588
|
+
window: {
|
|
1589
|
+
type: "object",
|
|
1590
|
+
properties: {
|
|
1591
|
+
windowId: { type: "integer" },
|
|
1592
|
+
processId: { type: "integer" },
|
|
1593
|
+
title: { type: "string" },
|
|
1594
|
+
className: { type: "string" },
|
|
1595
|
+
executablePath: { oneOf: [{ type: "string" }, { type: "null" }] },
|
|
1596
|
+
rect: {
|
|
1597
|
+
type: "object",
|
|
1598
|
+
properties: {
|
|
1599
|
+
x: { type: "integer" },
|
|
1600
|
+
y: { type: "integer" },
|
|
1601
|
+
width: { type: "integer" },
|
|
1602
|
+
height: { type: "integer" }
|
|
1603
|
+
},
|
|
1604
|
+
additionalProperties: false
|
|
1605
|
+
},
|
|
1606
|
+
foreground: { type: "boolean" }
|
|
1607
|
+
},
|
|
1608
|
+
additionalProperties: false
|
|
1609
|
+
},
|
|
1610
|
+
ocrAvailable: { type: "boolean" },
|
|
1611
|
+
targets: {
|
|
1612
|
+
type: "array",
|
|
1613
|
+
items: {
|
|
1614
|
+
type: "object",
|
|
1615
|
+
properties: {
|
|
1616
|
+
text: { type: "string" },
|
|
1617
|
+
x: { type: "integer" },
|
|
1618
|
+
y: { type: "integer" },
|
|
1619
|
+
width: { type: "integer" },
|
|
1620
|
+
height: { type: "integer" },
|
|
1621
|
+
confidence: { type: "number" },
|
|
1622
|
+
source: {
|
|
1623
|
+
type: "string",
|
|
1624
|
+
enum: ["ocr", "grounding"]
|
|
1625
|
+
}
|
|
1626
|
+
},
|
|
1627
|
+
additionalProperties: false
|
|
1628
|
+
}
|
|
1629
|
+
}
|
|
1630
|
+
},
|
|
1631
|
+
additionalProperties: false
|
|
1632
|
+
},
|
|
1633
|
+
render(_args, value) {
|
|
1634
|
+
const result = value;
|
|
1635
|
+
const lines = [
|
|
1636
|
+
`Text/visual targets in ${windowLine(result.window)}:`,
|
|
1637
|
+
`observationId: ${result.observationId}`,
|
|
1638
|
+
`OCR available: ${result.ocrAvailable}`,
|
|
1639
|
+
`${result.targets.length} target(s):`
|
|
1640
|
+
];
|
|
1641
|
+
for (const target of result.targets) lines.push(`- ${JSON.stringify(target.text)} at (${target.x}, ${target.y}) ${target.width}x${target.height} [${target.source}]`);
|
|
1642
|
+
lines.push("cite this observationId in click `basedOn` and address a target by its (x, y) coordinates.");
|
|
1643
|
+
return [{
|
|
1644
|
+
type: "text",
|
|
1645
|
+
text: lines.join("\n")
|
|
1646
|
+
}];
|
|
1647
|
+
}
|
|
1648
|
+
},
|
|
1649
|
+
timeoutMs: config.helperTimeoutMs + 3e4,
|
|
1650
|
+
async execute(args, exec) {
|
|
1651
|
+
backendGuard(services);
|
|
1652
|
+
const parsed = args;
|
|
1653
|
+
const requestedSide = parsed.maxSide;
|
|
1654
|
+
const maxSide = requestedSide === void 0 ? config.maxScreenshotSide : Math.min(requestedSide, config.maxScreenshotSide);
|
|
1655
|
+
const shot = await backend.shot(parsed.target ?? {}, maxSide, exec.signal);
|
|
1656
|
+
const sanitized = observedWindow(shot.snapshot, config.maxTextLength);
|
|
1657
|
+
const record = observations.record(shot.snapshot);
|
|
1658
|
+
const targets = [];
|
|
1659
|
+
if (ocr.available) for (const word of await ocr.recognize(shot.pngBase64, exec.signal)) {
|
|
1660
|
+
const box = screenBoxFor(shot.snapshot.rect, shot.width, shot.height, word.box);
|
|
1661
|
+
const center = centerOf(box);
|
|
1662
|
+
targets.push({
|
|
1663
|
+
text: sanitizeVisible(word.text, config.maxTextLength),
|
|
1664
|
+
x: center.x,
|
|
1665
|
+
y: center.y,
|
|
1666
|
+
width: Math.round(box.width),
|
|
1667
|
+
height: Math.round(box.height),
|
|
1668
|
+
confidence: word.confidence / 100,
|
|
1669
|
+
source: "ocr"
|
|
1670
|
+
});
|
|
1671
|
+
}
|
|
1672
|
+
const query = parsed.query?.trim() ?? "";
|
|
1673
|
+
if (grounding.available && query !== "") for (const target of await grounding.ground(shot.pngBase64, shot.width, shot.height, query, exec.signal)) {
|
|
1674
|
+
const box = screenBoxFor(shot.snapshot.rect, shot.width, shot.height, target.box);
|
|
1675
|
+
const center = centerOf(box);
|
|
1676
|
+
targets.push({
|
|
1677
|
+
text: sanitizeVisible(target.label, config.maxTextLength),
|
|
1678
|
+
x: center.x,
|
|
1679
|
+
y: center.y,
|
|
1680
|
+
width: Math.round(box.width),
|
|
1681
|
+
height: Math.round(box.height),
|
|
1682
|
+
confidence: target.confidence,
|
|
1683
|
+
source: "grounding"
|
|
1684
|
+
});
|
|
1685
|
+
}
|
|
1686
|
+
const filtered = query === "" ? targets : targets.filter((target) => target.text.toLowerCase().includes(query.toLowerCase()));
|
|
1687
|
+
auditObservation(exec, {
|
|
1688
|
+
observationId: record.id,
|
|
1689
|
+
windowId: shot.snapshot.windowId,
|
|
1690
|
+
processId: shot.snapshot.processId,
|
|
1691
|
+
executablePath: sanitized.executablePath,
|
|
1692
|
+
windowTitle: sanitized.title,
|
|
1693
|
+
elementCount: 0
|
|
1694
|
+
}, config.auditSessionEvents);
|
|
1695
|
+
return {
|
|
1696
|
+
ok: true,
|
|
1697
|
+
observationId: record.id,
|
|
1698
|
+
window: sanitized,
|
|
1699
|
+
ocrAvailable: ocr.available,
|
|
1700
|
+
targets: filtered
|
|
1701
|
+
};
|
|
1702
|
+
}
|
|
1703
|
+
});
|
|
1704
|
+
}
|
|
1294
1705
|
/** The shared window-action output schema (click/scroll/key). */
|
|
1295
1706
|
function actionOutputSchema(withRestored) {
|
|
1296
1707
|
return {
|
|
@@ -1773,6 +2184,7 @@ function allTools(services) {
|
|
|
1773
2184
|
return [
|
|
1774
2185
|
screenShotTool(services),
|
|
1775
2186
|
screenReadTool(services),
|
|
2187
|
+
screenFindTool(services),
|
|
1776
2188
|
clickTool(services),
|
|
1777
2189
|
typeTool(services),
|
|
1778
2190
|
scrollTool(services),
|
|
@@ -1787,30 +2199,35 @@ const name = "dsh-click";
|
|
|
1787
2199
|
/** Hard services: the tool registry every contribution lands in. */
|
|
1788
2200
|
const inject = ["tools"];
|
|
1789
2201
|
/**
|
|
1790
|
-
* Mount the plugin: resolve config, select the platform backend,
|
|
2202
|
+
* Mount the plugin: resolve config, select the platform backend, probe the
|
|
2203
|
+
* optional OCR provider, read the optional visual-grounding seam, and register
|
|
1791
2204
|
* every tool through `ctx.tools.register` (each registration is an effect
|
|
1792
2205
|
* whose disposer removes exactly that tool on stop/HMR).
|
|
1793
2206
|
*
|
|
1794
2207
|
* @param ctx - context carrying the tools registry.
|
|
1795
2208
|
* @param config - raw loader config; defaults applied through {@link resolveConfig}.
|
|
1796
2209
|
*/
|
|
1797
|
-
function apply(ctx, config) {
|
|
2210
|
+
async function apply(ctx, config) {
|
|
1798
2211
|
const resolved = resolveConfig(config);
|
|
1799
2212
|
const backend = ctx.get("dsh-click/backend") ?? createBackend(ctx, resolved);
|
|
1800
2213
|
const observations = new ObservationStore(resolved);
|
|
2214
|
+
const actions = new ActionExecutor({
|
|
2215
|
+
ctx,
|
|
2216
|
+
config: resolved,
|
|
2217
|
+
backend,
|
|
2218
|
+
observations
|
|
2219
|
+
});
|
|
2220
|
+
const subprocess = ctx.get("subprocess");
|
|
1801
2221
|
const services = {
|
|
1802
2222
|
ctx,
|
|
1803
2223
|
config: resolved,
|
|
1804
2224
|
backend,
|
|
1805
2225
|
observations,
|
|
1806
|
-
actions
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
backend,
|
|
1810
|
-
observations
|
|
1811
|
-
})
|
|
2226
|
+
actions,
|
|
2227
|
+
ocr: resolved.ocr.enabled && subprocess !== void 0 ? await probeTesseract(subprocess, resolved.ocr.command, resolved.ocr.language) : new UnavailableOcrProvider("ocr-disabled", resolved.ocr.enabled ? "the subprocess service is not mounted" : "OCR is disabled by config"),
|
|
2228
|
+
grounding: ctx.get("dsh-click/grounding") ?? new UnavailableGroundingProvider()
|
|
1812
2229
|
};
|
|
1813
2230
|
for (const tool of allTools(services)) ctx.effect(() => ctx.tools.register(tool), `dsh-click: ${tool.name} tool`);
|
|
1814
2231
|
}
|
|
1815
2232
|
//#endregion
|
|
1816
|
-
export { ACTION_EVENT, ActionExecutor, Config, DshClickError, HELPER_PROTOCOL_VERSION, HelperBackend, OBSERVED_EVENT, ObservationStore, UnavailableBackend, VERSION, allTools, apply, createBackend, inject, name, observationIdOf, redactSensitive, resolveConfig, sanitizePath, sanitizeText, sanitizeVisible };
|
|
2233
|
+
export { ACTION_EVENT, ActionExecutor, Config, DshClickError, HELPER_PROTOCOL_VERSION, HelperBackend, OBSERVED_EVENT, ObservationStore, TesseractOcrProvider, UnavailableBackend, UnavailableGroundingProvider, UnavailableOcrProvider, VERSION, allTools, apply, centerOf, createBackend, inject, name, observationIdOf, parseTesseractTsv, probeTesseract, redactSensitive, resolveConfig, sanitizePath, sanitizeText, sanitizeVisible, screenBoxFor, tesseractArgv };
|
package/lib/types/config.d.ts
CHANGED
|
@@ -45,6 +45,19 @@ export declare const MAX_TREE_DEPTH = 64;
|
|
|
45
45
|
export declare const DEFAULT_MAX_TEXT_LENGTH = 200;
|
|
46
46
|
/** Ceiling on the sanitization truncation length. */
|
|
47
47
|
export declare const MAX_TEXT_LENGTH = 10000;
|
|
48
|
+
/** Default tesseract executable for the optional OCR path. */
|
|
49
|
+
export declare const DEFAULT_OCR_COMMAND = "tesseract";
|
|
50
|
+
/** Default OCR language (tesseract language data). */
|
|
51
|
+
export declare const DEFAULT_OCR_LANGUAGE = "eng";
|
|
52
|
+
/** Optical character recognition (optional; probed at mount). */
|
|
53
|
+
export interface OcrConfig {
|
|
54
|
+
/** Enable OCR-based text finding when a window has no UIA tree (default true). */
|
|
55
|
+
enabled?: boolean;
|
|
56
|
+
/** tesseract executable name (default "tesseract"). */
|
|
57
|
+
command?: string;
|
|
58
|
+
/** tesseract language tag (default "eng"). */
|
|
59
|
+
language?: string;
|
|
60
|
+
}
|
|
48
61
|
/** Configuration for the dsh-click desktop-control tools. */
|
|
49
62
|
export interface Config {
|
|
50
63
|
/** Gate every mutating action (click/type/scroll/key/app_launch) behind approval (default true). */
|
|
@@ -53,11 +66,11 @@ export interface Config {
|
|
|
53
66
|
autoApproveWindows?: string[];
|
|
54
67
|
/**
|
|
55
68
|
* Whether dsh-click session audit events (`dsh-click/observed`/`dsh-click/action`)
|
|
56
|
-
* are appended to the session log (default true).
|
|
57
|
-
*
|
|
58
|
-
*
|
|
59
|
-
*
|
|
60
|
-
*
|
|
69
|
+
* are appended to the session log (default true). The adaptive audit gate
|
|
70
|
+
* already skips the append on envelope-less hosts (rc.6–rc.8, 0.1.1-rc.2,
|
|
71
|
+
* and 0.1.2-alpha.1, which fails closed on unknown types at read); set
|
|
72
|
+
* false to stop audit appends entirely, even on hosts that carry the
|
|
73
|
+
* `ignorable` envelope.
|
|
61
74
|
*/
|
|
62
75
|
auditSessionEvents?: boolean;
|
|
63
76
|
/** Whether mutating actions may bring the target window to the foreground as a fallback (default 'never'). */
|
|
@@ -84,6 +97,8 @@ export interface Config {
|
|
|
84
97
|
maxTextLength?: number;
|
|
85
98
|
/** Back up and restore control text when `type` fails (default true). */
|
|
86
99
|
rollbackEnabled?: boolean;
|
|
100
|
+
/** Optical character recognition (optional; probed at mount). */
|
|
101
|
+
ocr?: OcrConfig;
|
|
87
102
|
}
|
|
88
103
|
/** Fully resolved configuration captured at plugin load. */
|
|
89
104
|
export interface ResolvedConfig {
|
|
@@ -117,6 +132,12 @@ export interface ResolvedConfig {
|
|
|
117
132
|
maxTextLength: number;
|
|
118
133
|
/** Whether `type` backs up and restores control text on failure. */
|
|
119
134
|
rollbackEnabled: boolean;
|
|
135
|
+
/** OCR configuration (probed at mount). */
|
|
136
|
+
ocr: {
|
|
137
|
+
enabled: boolean;
|
|
138
|
+
command: string;
|
|
139
|
+
language: string;
|
|
140
|
+
};
|
|
120
141
|
}
|
|
121
142
|
/** Schemastery schema for loader-validated configuration. */
|
|
122
143
|
export declare const Config: z<Config>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/config.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,CAAC,MAAM,0BAA0B,CAAA;AAExC,+EAA+E;AAC/E,MAAM,MAAM,SAAS,GAAG,MAAM,GAAG,MAAM,CAAA;AAEvC,mFAAmF;AACnF,MAAM,MAAM,aAAa,GAAG,OAAO,GAAG,OAAO,CAAA;AAE7C,uDAAuD;AACvD,eAAO,MAAM,yBAAyB,QAAS,CAAA;AAE/C,qFAAqF;AACrF,eAAO,MAAM,qBAAqB,SAAU,CAAA;AAE5C,sFAAsF;AACtF,eAAO,MAAM,+BAA+B,QAAmB,CAAA;AAE/D,sCAAsC;AACtC,eAAO,MAAM,uBAAuB,QAAmB,CAAA;AAEvD,2FAA2F;AAC3F,eAAO,MAAM,2BAA2B,OAAO,CAAA;AAE/C,sFAAsF;AACtF,eAAO,MAAM,mBAAmB,MAAM,CAAA;AACtC,eAAO,MAAM,mBAAmB,OAAO,CAAA;AAEvC,8EAA8E;AAC9E,eAAO,MAAM,8BAA8B,QAAS,CAAA;AAEpD,sCAAsC;AACtC,eAAO,MAAM,sBAAsB,OAAQ,CAAA;AAC3C,eAAO,MAAM,sBAAsB,SAAU,CAAA;AAE7C,4EAA4E;AAC5E,eAAO,MAAM,+BAA+B,IAAI,CAAA;AAEhD,sCAAsC;AACtC,eAAO,MAAM,uBAAuB,KAAK,CAAA;AAEzC,8EAA8E;AAC9E,eAAO,MAAM,oBAAoB,MAAM,CAAA;AAEvC,kCAAkC;AAClC,eAAO,MAAM,YAAY,OAAQ,CAAA;AAEjC,4DAA4D;AAC5D,eAAO,MAAM,sBAAsB,KAAK,CAAA;AAExC,iCAAiC;AACjC,eAAO,MAAM,cAAc,KAAK,CAAA;AAEhC,qEAAqE;AACrE,eAAO,MAAM,uBAAuB,MAAM,CAAA;AAE1C,qDAAqD;AACrD,eAAO,MAAM,eAAe,QAAS,CAAA;AAErC,6DAA6D;AAC7D,MAAM,WAAW,MAAM;IACrB,oGAAoG;IACpG,eAAe,CAAC,EAAE,OAAO,CAAA;IACzB,sGAAsG;IACtG,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAA;IAC7B;;;;;;;OAOG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAA;IAC5B,8GAA8G;IAC9G,aAAa,CAAC,EAAE,aAAa,CAAA;IAC7B,iGAAiG;IACjG,SAAS,CAAC,EAAE,SAAS,CAAA;IACrB,+DAA+D;IAC/D,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,8DAA8D;IAC9D,oBAAoB,CAAC,EAAE,MAAM,CAAA;IAC7B,sEAAsE;IACtE,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B,+FAA+F;IAC/F,gBAAgB,CAAC,EAAE,OAAO,CAAA;IAC1B,4FAA4F;IAC5F,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B,8CAA8C;IAC9C,qBAAqB,CAAC,EAAE,MAAM,CAAA;IAC9B,oEAAoE;IACpE,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,iEAAiE;IACjE,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,2EAA2E;IAC3E,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,yEAAyE;IACzE,eAAe,CAAC,EAAE,OAAO,CAAA;
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/config.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,CAAC,MAAM,0BAA0B,CAAA;AAExC,+EAA+E;AAC/E,MAAM,MAAM,SAAS,GAAG,MAAM,GAAG,MAAM,CAAA;AAEvC,mFAAmF;AACnF,MAAM,MAAM,aAAa,GAAG,OAAO,GAAG,OAAO,CAAA;AAE7C,uDAAuD;AACvD,eAAO,MAAM,yBAAyB,QAAS,CAAA;AAE/C,qFAAqF;AACrF,eAAO,MAAM,qBAAqB,SAAU,CAAA;AAE5C,sFAAsF;AACtF,eAAO,MAAM,+BAA+B,QAAmB,CAAA;AAE/D,sCAAsC;AACtC,eAAO,MAAM,uBAAuB,QAAmB,CAAA;AAEvD,2FAA2F;AAC3F,eAAO,MAAM,2BAA2B,OAAO,CAAA;AAE/C,sFAAsF;AACtF,eAAO,MAAM,mBAAmB,MAAM,CAAA;AACtC,eAAO,MAAM,mBAAmB,OAAO,CAAA;AAEvC,8EAA8E;AAC9E,eAAO,MAAM,8BAA8B,QAAS,CAAA;AAEpD,sCAAsC;AACtC,eAAO,MAAM,sBAAsB,OAAQ,CAAA;AAC3C,eAAO,MAAM,sBAAsB,SAAU,CAAA;AAE7C,4EAA4E;AAC5E,eAAO,MAAM,+BAA+B,IAAI,CAAA;AAEhD,sCAAsC;AACtC,eAAO,MAAM,uBAAuB,KAAK,CAAA;AAEzC,8EAA8E;AAC9E,eAAO,MAAM,oBAAoB,MAAM,CAAA;AAEvC,kCAAkC;AAClC,eAAO,MAAM,YAAY,OAAQ,CAAA;AAEjC,4DAA4D;AAC5D,eAAO,MAAM,sBAAsB,KAAK,CAAA;AAExC,iCAAiC;AACjC,eAAO,MAAM,cAAc,KAAK,CAAA;AAEhC,qEAAqE;AACrE,eAAO,MAAM,uBAAuB,MAAM,CAAA;AAE1C,qDAAqD;AACrD,eAAO,MAAM,eAAe,QAAS,CAAA;AAErC,8DAA8D;AAC9D,eAAO,MAAM,mBAAmB,cAAc,CAAA;AAE9C,sDAAsD;AACtD,eAAO,MAAM,oBAAoB,QAAQ,CAAA;AAEzC,iEAAiE;AACjE,MAAM,WAAW,SAAS;IACxB,kFAAkF;IAClF,OAAO,CAAC,EAAE,OAAO,CAAA;IACjB,uDAAuD;IACvD,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,8CAA8C;IAC9C,QAAQ,CAAC,EAAE,MAAM,CAAA;CAClB;AAED,6DAA6D;AAC7D,MAAM,WAAW,MAAM;IACrB,oGAAoG;IACpG,eAAe,CAAC,EAAE,OAAO,CAAA;IACzB,sGAAsG;IACtG,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAA;IAC7B;;;;;;;OAOG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAA;IAC5B,8GAA8G;IAC9G,aAAa,CAAC,EAAE,aAAa,CAAA;IAC7B,iGAAiG;IACjG,SAAS,CAAC,EAAE,SAAS,CAAA;IACrB,+DAA+D;IAC/D,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,8DAA8D;IAC9D,oBAAoB,CAAC,EAAE,MAAM,CAAA;IAC7B,sEAAsE;IACtE,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B,+FAA+F;IAC/F,gBAAgB,CAAC,EAAE,OAAO,CAAA;IAC1B,4FAA4F;IAC5F,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B,8CAA8C;IAC9C,qBAAqB,CAAC,EAAE,MAAM,CAAA;IAC9B,oEAAoE;IACpE,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,iEAAiE;IACjE,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,2EAA2E;IAC3E,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,yEAAyE;IACzE,eAAe,CAAC,EAAE,OAAO,CAAA;IACzB,iEAAiE;IACjE,GAAG,CAAC,EAAE,SAAS,CAAA;CAChB;AAED,4DAA4D;AAC5D,MAAM,WAAW,cAAc;IAC7B,iDAAiD;IACjD,eAAe,EAAE,OAAO,CAAA;IACxB,qEAAqE;IACrE,mBAAmB,EAAE,aAAa,CAAC,MAAM,CAAC,CAAA;IAC1C,oEAAoE;IACpE,kBAAkB,EAAE,OAAO,CAAA;IAC3B,2CAA2C;IAC3C,aAAa,EAAE,aAAa,CAAA;IAC5B,8BAA8B;IAC9B,SAAS,EAAE,SAAS,CAAA;IACpB,+CAA+C;IAC/C,eAAe,EAAE,MAAM,CAAA;IACvB,2CAA2C;IAC3C,oBAAoB,EAAE,MAAM,CAAA;IAC5B,yCAAyC;IACzC,iBAAiB,EAAE,MAAM,CAAA;IACzB,gDAAgD;IAChD,gBAAgB,EAAE,OAAO,CAAA;IACzB,+CAA+C;IAC/C,mBAAmB,EAAE,MAAM,CAAA;IAC3B,kCAAkC;IAClC,qBAAqB,EAAE,MAAM,CAAA;IAC7B,uCAAuC;IACvC,WAAW,EAAE,MAAM,CAAA;IACnB,+BAA+B;IAC/B,YAAY,EAAE,MAAM,CAAA;IACpB,+CAA+C;IAC/C,aAAa,EAAE,MAAM,CAAA;IACrB,oEAAoE;IACpE,eAAe,EAAE,OAAO,CAAA;IACxB,2CAA2C;IAC3C,GAAG,EAAE;QAAE,OAAO,EAAE,OAAO,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAA;CAC7D;AAED,6DAA6D;AAC7D,eAAO,MAAM,MAAM,EAAE,CAAC,CAAC,MAAM,CAqB3B,CAAA;AAgBF;;;;;GAKG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,GAAG,cAAc,CAqFxE"}
|
package/lib/types/events.d.ts
CHANGED
|
@@ -1,16 +1,23 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Session audit events for dsh-click (declaration merging into the harness's
|
|
3
|
-
* `SessionEventMap`). Both events are log-only;
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
3
|
+
* `SessionEventMap`) and the adaptive append gate. Both events are log-only;
|
|
4
|
+
* tool arguments and rendered results are already logged by the tool runtime
|
|
5
|
+
* as `tool/call` + `tool/result`, and these events carry the observation and
|
|
6
|
+
* action facts that exist outside them: window identity, process identity,
|
|
7
|
+
* and the approval/outcome audit trail.
|
|
8
|
+
*
|
|
9
|
+
* The gate appends only when the host can carry the events safely:
|
|
10
|
+
* - hosts whose known-type set covers the vocabulary append plainly;
|
|
11
|
+
* - hosts with an `ignorable` append option (pre-0.1.2 master builds) append
|
|
12
|
+
* with the marker, so builds that do not know the type skip it on restore;
|
|
13
|
+
* - envelope-less hosts (0.1.0-rc.6/rc.8, 0.1.1-rc.2, and 0.1.2-alpha.1,
|
|
14
|
+
* which removed the envelope and fails closed on unknown types at read)
|
|
15
|
+
* get no append — the tool results remain the reconstructable audit trail.
|
|
10
16
|
*
|
|
11
17
|
* @module dsh-click/events
|
|
12
18
|
*/
|
|
13
19
|
import type { ImageMediaType } from '@deepseek-ai/dsh-attachment';
|
|
20
|
+
import { type Session } from '@deepseek-ai/dsh-session';
|
|
14
21
|
/** Process identity facts captured immediately before and after an action. */
|
|
15
22
|
export interface ProcessFacts {
|
|
16
23
|
/** Process id owning the target window. */
|
|
@@ -108,4 +115,14 @@ export type ActionEvent = {
|
|
|
108
115
|
restored?: boolean;
|
|
109
116
|
detail?: string;
|
|
110
117
|
};
|
|
118
|
+
/**
|
|
119
|
+
* Append one dsh-click audit event when the host can carry it safely; skip
|
|
120
|
+
* silently otherwise (the tool/call + tool/result events remain the
|
|
121
|
+
* model-visible log, so nothing model-visible is lost). See the module doc
|
|
122
|
+
* for the three host classes.
|
|
123
|
+
* @param session - the calling session.
|
|
124
|
+
* @param type - the audit event type.
|
|
125
|
+
* @param data - the audit payload.
|
|
126
|
+
*/
|
|
127
|
+
export declare function appendAuditEvent(session: Session, type: typeof OBSERVED_EVENT | typeof ACTION_EVENT, data: ObservedEvent | ActionEvent): void;
|
|
111
128
|
//# sourceMappingURL=events.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"events.d.ts","sourceRoot":"","sources":["../../src/events.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"events.d.ts","sourceRoot":"","sources":["../../src/events.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAA;AACjE,OAAO,EAA6B,KAAK,OAAO,EAAE,MAAM,0BAA0B,CAAA;AAElF,8EAA8E;AAC9E,MAAM,WAAW,YAAY;IAC3B,2CAA2C;IAC3C,GAAG,EAAE,MAAM,CAAA;IACX,gFAAgF;IAChF,cAAc,EAAE,MAAM,GAAG,IAAI,CAAA;CAC9B;AAED,kEAAkE;AAClE,MAAM,WAAW,aAAa;IAC5B,uCAAuC;IACvC,YAAY,EAAE,MAAM,CAAA;IACpB,+CAA+C;IAC/C,SAAS,EAAE,cAAc,CAAA;IACzB,2BAA2B;IAC3B,KAAK,EAAE,MAAM,CAAA;IACb,iCAAiC;IACjC,KAAK,EAAE,MAAM,CAAA;IACb,kCAAkC;IAClC,MAAM,EAAE,MAAM,CAAA;CACf;AAED,OAAO,QAAQ,gCAAgC,CAAC;IAC9C,UAAU,eAAe;QACvB;;;;;WAKG;QACH,oBAAoB,EAAE;YACpB,mDAAmD;YACnD,aAAa,EAAE,MAAM,CAAA;YACrB,8BAA8B;YAC9B,QAAQ,EAAE,MAAM,CAAA;YAChB,oCAAoC;YACpC,SAAS,EAAE,MAAM,CAAA;YACjB,0DAA0D;YAC1D,cAAc,EAAE,MAAM,GAAG,IAAI,CAAA;YAC7B,8BAA8B;YAC9B,WAAW,EAAE,MAAM,CAAA;YACnB,iDAAiD;YACjD,YAAY,EAAE,MAAM,CAAA;YACpB,8DAA8D;YAC9D,KAAK,CAAC,EAAE,aAAa,CAAA;SACtB,CAAA;QACD;;;;WAIG;QACH,kBAAkB,EAAE;YAClB,gCAAgC;YAChC,IAAI,EAAE,MAAM,CAAA;YACZ,sFAAsF;YACtF,QAAQ,EAAE,UAAU,GAAG,WAAW,GAAG,MAAM,CAAA;YAC3C,+DAA+D;YAC/D,OAAO,EAAE,IAAI,GAAG,OAAO,CAAA;YACvB,8DAA8D;YAC9D,aAAa,CAAC,EAAE,MAAM,CAAA;YACtB,gEAAgE;YAChE,QAAQ,CAAC,EAAE,MAAM,CAAA;YACjB,mDAAmD;YACnD,aAAa,CAAC,EAAE,YAAY,CAAA;YAC5B,kDAAkD;YAClD,YAAY,CAAC,EAAE,YAAY,CAAA;YAC3B,oEAAoE;YACpE,QAAQ,CAAC,EAAE,OAAO,CAAA;YAClB,kEAAkE;YAClE,MAAM,CAAC,EAAE,MAAM,CAAA;SAChB,CAAA;KACF;CACF;AAED,wCAAwC;AACxC,eAAO,MAAM,cAAc,EAAG,oBAA6B,CAAA;AAE3D,mCAAmC;AACnC,eAAO,MAAM,YAAY,EAAG,kBAA2B,CAAA;AAEvD,6CAA6C;AAC7C,MAAM,MAAM,aAAa,GAAG;IAC1B,aAAa,EAAE,MAAM,CAAA;IACrB,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAA;IAC7B,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,EAAE,MAAM,CAAA;IACpB,KAAK,CAAC,EAAE,aAAa,CAAA;CACtB,CAAA;AAED,wCAAwC;AACxC,MAAM,MAAM,WAAW,GAAG;IACxB,IAAI,EAAE,MAAM,CAAA;IACZ,QAAQ,EAAE,UAAU,GAAG,WAAW,GAAG,MAAM,CAAA;IAC3C,OAAO,EAAE,IAAI,GAAG,OAAO,CAAA;IACvB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,aAAa,CAAC,EAAE,YAAY,CAAA;IAC5B,YAAY,CAAC,EAAE,YAAY,CAAA;IAC3B,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB,CAAA;AAKD;;;;;;;;GAQG;AACH,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,OAAO,EAChB,IAAI,EAAE,OAAO,cAAc,GAAG,OAAO,YAAY,EACjD,IAAI,EAAE,aAAa,GAAG,WAAW,GAChC,IAAI,CAUN"}
|