dsh-llm-verifier 0.1.13 → 0.1.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +22 -22
- package/README.md +301 -301
- package/cordis.patch.yml +4 -4
- package/lib/client.js +176 -14
- package/lib/index.js +418 -43
- package/lib/types/client-i18n.d.ts +27 -0
- package/lib/types/client-i18n.js +18 -0
- package/lib/types/client.d.ts +15 -2
- package/lib/types/client.js +134 -9
- package/lib/types/config.d.ts +4 -0
- package/lib/types/config.js +5 -3
- package/lib/types/index.d.ts +2 -0
- package/lib/types/index.js +185 -4
- package/lib/types/plan-gate.d.ts +14 -0
- package/lib/types/plan-gate.js +70 -0
- package/lib/types/router.d.ts +10 -1
- package/lib/types/router.js +37 -7
- package/lib/types/session.js +25 -10
- package/lib/types/team-gate.d.ts +11 -0
- package/lib/types/team-gate.js +51 -0
- package/package.json +112 -112
- package/src/auto.test.ts +86 -86
- package/src/auto.ts +133 -133
- package/src/cache.ts +115 -115
- package/src/caller.test.ts +34 -34
- package/src/caller.ts +151 -151
- package/src/client-i18n.ts +18 -0
- package/src/client.tsx +385 -213
- package/src/config.ts +189 -183
- package/src/core.test.ts +71 -71
- package/src/core.ts +205 -205
- package/src/engine.test.ts +136 -136
- package/src/engine.ts +140 -140
- package/src/images.ts +33 -33
- package/src/index.ts +416 -247
- package/src/parity.test.ts +85 -85
- package/src/plan-gate.test.ts +56 -0
- package/src/plan-gate.ts +77 -0
- package/src/router.test.ts +107 -89
- package/src/router.ts +349 -309
- package/src/session.test.ts +53 -36
- package/src/session.ts +117 -103
- package/src/statistics.test.ts +80 -80
- package/src/statistics.ts +328 -328
- package/src/team-gate.test.ts +56 -0
- package/src/team-gate.ts +63 -0
- package/src/top-logprobs.test.ts +84 -84
- package/src/top-logprobs.ts +153 -153
- package/src/topic-storage.test.ts +31 -31
- package/src/topic-storage.ts +42 -42
package/lib/index.js
CHANGED
|
@@ -31,8 +31,10 @@ const Config = z.object({
|
|
|
31
31
|
autoRouteMaxInputChars: z.number().step(1).min(1e3).default(6e4),
|
|
32
32
|
autoMaxModelCallsPerTask: z.number().step(1).min(1).default(48),
|
|
33
33
|
autoMaxModelCallsPerSession: z.number().step(1).min(1).default(160),
|
|
34
|
+
autoVerifyTeamTasks: z.boolean().default(true),
|
|
35
|
+
autoVerifyPlanMode: z.boolean().default(true),
|
|
34
36
|
provider: z.string().default("deepseek-official"),
|
|
35
|
-
model: z.string().default("deepseek-
|
|
37
|
+
model: z.string().default("deepseek-flash"),
|
|
36
38
|
reasoningEffort: z.string(),
|
|
37
39
|
maxTokens: z.number().step(1).min(1).default(32768),
|
|
38
40
|
timeoutMs: z.number().step(1).min(1).default(3e5),
|
|
@@ -46,7 +48,7 @@ const Config = z.object({
|
|
|
46
48
|
});
|
|
47
49
|
function resolveConfig(config = {}) {
|
|
48
50
|
const provider = (config.provider ?? "deepseek-official").trim();
|
|
49
|
-
const model = (config.model ?? "deepseek-
|
|
51
|
+
const model = (config.model ?? "deepseek-flash").trim();
|
|
50
52
|
if (!provider) throw new Error("llm-verifier: provider must be non-empty");
|
|
51
53
|
if (!model) throw new Error("llm-verifier: model must be non-empty");
|
|
52
54
|
const autoVerifyMode = config.autoVerifyMode ?? "smart";
|
|
@@ -100,6 +102,8 @@ function resolveConfig(config = {}) {
|
|
|
100
102
|
autoRouteSemantic: config.autoRouteSemantic ?? true,
|
|
101
103
|
autoRouteMinConfidence,
|
|
102
104
|
autoTrackCompletionThreshold,
|
|
105
|
+
autoVerifyTeamTasks: config.autoVerifyTeamTasks ?? true,
|
|
106
|
+
autoVerifyPlanMode: config.autoVerifyPlanMode ?? true,
|
|
103
107
|
provider,
|
|
104
108
|
model,
|
|
105
109
|
...reasoningEffort ? { reasoningEffort } : {},
|
|
@@ -604,10 +608,17 @@ async function loadVerifierImages(inputs, signal) {
|
|
|
604
608
|
//#region src/session.ts
|
|
605
609
|
function textOf(blocks) {
|
|
606
610
|
const parts = [];
|
|
607
|
-
for (const block of blocks)
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
+
for (const block of blocks) {
|
|
612
|
+
const blockType = block.type;
|
|
613
|
+
if (blockType === "text") parts.push(block.text);
|
|
614
|
+
else if (blockType === "reasoning") parts.push("[Reasoning] " + block.text);
|
|
615
|
+
else if (blockType === "tool-call") parts.push("[Tool Call] " + block.name + " " + block.arguments);
|
|
616
|
+
else if (blockType === "tool-result") parts.push("[Tool Result] " + textOf(block.content));
|
|
617
|
+
else if (blockType === "file") {
|
|
618
|
+
const fileData = block;
|
|
619
|
+
parts.push("[File] " + (fileData.path ?? fileData.filename ?? fileData.title ?? "attachment"));
|
|
620
|
+
}
|
|
621
|
+
}
|
|
611
622
|
return parts.join("\n");
|
|
612
623
|
}
|
|
613
624
|
const DEFAULT_REDACT_PATTERNS = ["Bearer\\s+[A-Za-z0-9._~+\\/=-]+", "(?:api[_-]?key|token|password|secret)\\s*[=:]\\s*[\"']?[^\\s,\"';}]+"];
|
|
@@ -643,24 +654,35 @@ async function extractSession(agent, loadImage, options = {}) {
|
|
|
643
654
|
let problem = "";
|
|
644
655
|
const trace = [];
|
|
645
656
|
const images = [];
|
|
646
|
-
for (const
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
657
|
+
for (const rawEvent of events) {
|
|
658
|
+
const event = rawEvent;
|
|
659
|
+
if (event.type === "user/message") {
|
|
660
|
+
const sourceKind = event.data?.source?.kind;
|
|
661
|
+
if (sourceKind !== "user" && sourceKind !== "team-message") continue;
|
|
662
|
+
const text = textOf(event.data.content);
|
|
663
|
+
if (!problem && sourceKind === "user" && text.trim()) problem = text.trim();
|
|
664
|
+
for (const block of event.data.content) if (block.type === "image") images.push(await loadImage(block.attachment));
|
|
665
|
+
const tag = sourceKind === "team-message" ? "Team Message" : "User";
|
|
666
|
+
trace.push("--- " + tag + " seq " + event.seq + " ---\n" + text);
|
|
667
|
+
} else if (event.type === "assistant/message" && options.includeAssistantText !== false) trace.push("--- Assistant turn " + event.data.turn + " step " + event.data.step + " ---\n" + textOf(event.data.message.content));
|
|
668
|
+
else if (event.type === "tool/call") trace.push("--- Tool Call turn " + event.data.turn + " step " + event.data.step + " ---\n[Command] " + event.data.name + " " + event.data.arguments);
|
|
669
|
+
else if (event.type === "tool/result") trace.push("--- Tool Result turn " + event.data.turn + " step " + event.data.step + " ---\n[Output] " + textOf(event.data.message.content));
|
|
670
|
+
else if (event.type === "tool/ptc-dispatch" || event.type === "tool/code-dispatch") {
|
|
671
|
+
const data = event.data;
|
|
672
|
+
const status = data.isError ? " [Error]" : "";
|
|
673
|
+
const args = data.arguments !== void 0 ? " " + (typeof data.arguments === "string" ? data.arguments : JSON.stringify(data.arguments)) : "";
|
|
674
|
+
const content = Array.isArray(data.content) ? textOf(data.content) : "";
|
|
675
|
+
if (Array.isArray(data.content)) {
|
|
676
|
+
for (const block of data.content) if (block.type === "image") images.push(await loadImage(block.attachment));
|
|
677
|
+
}
|
|
678
|
+
const label = event.type === "tool/ptc-dispatch" ? "PTC Dispatch " : "Code Dispatch ";
|
|
679
|
+
trace.push("--- " + label + data.name + status + " seq " + event.seq + " ---\n[Command] " + data.name + args + "\n[Output] " + content);
|
|
680
|
+
} else if (event.type === "team/message/queued") {
|
|
681
|
+
const data = event.data;
|
|
682
|
+
const sender = data?.message?.senderName ?? "Teammate";
|
|
683
|
+
const content = Array.isArray(data?.message?.content) ? textOf(data.message.content) : "";
|
|
684
|
+
trace.push("--- Team Message Queued from " + sender + " seq " + event.seq + " ---\n" + content);
|
|
662
685
|
}
|
|
663
|
-
trace.push("--- Code Dispatch " + data.name + status + " seq " + event.seq + " ---\n[Command] " + data.name + args + "\n[Output] " + content);
|
|
664
686
|
}
|
|
665
687
|
const raw = redactText(trace.join("\n\n"), patterns);
|
|
666
688
|
const maxChars = options.maxChars ?? 2e5;
|
|
@@ -895,21 +917,32 @@ function buildEvidenceIndex(events) {
|
|
|
895
917
|
const calls = /* @__PURE__ */ new Map();
|
|
896
918
|
const results = /* @__PURE__ */ new Map();
|
|
897
919
|
const todos = /* @__PURE__ */ new Map();
|
|
920
|
+
const teamTasks = /* @__PURE__ */ new Map();
|
|
898
921
|
const paired = /* @__PURE__ */ new Map();
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
if (
|
|
905
|
-
|
|
906
|
-
const
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
922
|
+
const currentTeamTasks = /* @__PURE__ */ new Map();
|
|
923
|
+
for (const rawEvent of relevant) {
|
|
924
|
+
const event = rawEvent;
|
|
925
|
+
if (event.type === "tool/call") calls.set(String(event.data.callId), rawEvent);
|
|
926
|
+
else if (event.type === "tool/result" && successful(rawEvent)) results.set(String(event.data.message.source.callId), rawEvent);
|
|
927
|
+
else if (event.type === "todo/write") todos.set(event.seq, event.data.todos);
|
|
928
|
+
else if (event.type === "team/task") {
|
|
929
|
+
const data = event.data;
|
|
930
|
+
if (data?.task) {
|
|
931
|
+
currentTeamTasks.set(data.task.id, { ...data.task });
|
|
932
|
+
teamTasks.set(event.seq, [...currentTeamTasks.values()]);
|
|
933
|
+
}
|
|
934
|
+
} else if (event.type === "tool/ptc-dispatch" || event.type === "tool/code-dispatch") {
|
|
935
|
+
const data = event.data;
|
|
936
|
+
if (data.isError !== true && (!Array.isArray(data.content) || data.content.every((b) => b.isError !== true))) {
|
|
937
|
+
const subCallId = String(data.subCallId ?? "code:" + event.seq);
|
|
938
|
+
const content = Array.isArray(data.content) ? data.content : [];
|
|
939
|
+
paired.set(subCallId, {
|
|
940
|
+
name: data.name,
|
|
941
|
+
callSeq: event.seq,
|
|
942
|
+
resultSeq: event.seq,
|
|
943
|
+
text: blockText(content)
|
|
944
|
+
});
|
|
945
|
+
}
|
|
913
946
|
}
|
|
914
947
|
}
|
|
915
948
|
for (const [callId, call] of calls) {
|
|
@@ -924,7 +957,8 @@ function buildEvidenceIndex(events) {
|
|
|
924
957
|
return {
|
|
925
958
|
problemSeq: taskStartSeq,
|
|
926
959
|
calls: paired,
|
|
927
|
-
todos
|
|
960
|
+
todos,
|
|
961
|
+
teamTasks
|
|
928
962
|
};
|
|
929
963
|
}
|
|
930
964
|
function parseTrustedWorkflow(value, callId, callSeq, resultSeq, maxCandidates, maxItemChars) {
|
|
@@ -978,6 +1012,23 @@ function canonicalTodoSnapshots(index) {
|
|
|
978
1012
|
}
|
|
979
1013
|
return values;
|
|
980
1014
|
}
|
|
1015
|
+
function canonicalTeamTaskSnapshots(index) {
|
|
1016
|
+
const values = [];
|
|
1017
|
+
let previous = "";
|
|
1018
|
+
for (const [seq, tasks] of index.teamTasks) {
|
|
1019
|
+
const canonical = JSON.stringify(tasks.map((t) => ({
|
|
1020
|
+
id: t.id,
|
|
1021
|
+
status: t.status,
|
|
1022
|
+
revision: t.revision
|
|
1023
|
+
})));
|
|
1024
|
+
if (canonical !== previous) values.push({
|
|
1025
|
+
seq,
|
|
1026
|
+
tasks
|
|
1027
|
+
});
|
|
1028
|
+
previous = canonical;
|
|
1029
|
+
}
|
|
1030
|
+
return values;
|
|
1031
|
+
}
|
|
981
1032
|
function analyzeStructuredRoute(events, maxCandidates = 8, maxItemChars = 2e4) {
|
|
982
1033
|
const index = buildEvidenceIndex(events);
|
|
983
1034
|
if (!index) return void 0;
|
|
@@ -1031,13 +1082,32 @@ function analyzeStructuredRoute(events, maxCandidates = 8, maxItemChars = 2e4) {
|
|
|
1031
1082
|
evidenceSeqs: snapshots.map((snapshot) => snapshot.seq)
|
|
1032
1083
|
};
|
|
1033
1084
|
}
|
|
1085
|
+
const teamSnapshots = canonicalTeamTaskSnapshots(index);
|
|
1086
|
+
if (teamSnapshots.length >= 2) {
|
|
1087
|
+
const steps = teamSnapshots.map((snapshot) => sanitizeVerifierText("Team task checkpoint seq " + snapshot.seq + ":\n" + snapshot.tasks.map((task) => "- [" + task.status + "] " + task.subject + (task.description ? " (" + task.description + ")" : "")).join("\n"), maxItemChars));
|
|
1088
|
+
const checkpoints = steps.map((_, i) => i + 1);
|
|
1089
|
+
return {
|
|
1090
|
+
kind: "track",
|
|
1091
|
+
source: "structured",
|
|
1092
|
+
confidence: 1,
|
|
1093
|
+
reason: "changed durable team tasks",
|
|
1094
|
+
fingerprint: stableHash({
|
|
1095
|
+
kind: "track",
|
|
1096
|
+
teamSnapshots
|
|
1097
|
+
}),
|
|
1098
|
+
steps,
|
|
1099
|
+
checkpoints,
|
|
1100
|
+
evidenceSeqs: teamSnapshots.map((snapshot) => snapshot.seq)
|
|
1101
|
+
};
|
|
1102
|
+
}
|
|
1034
1103
|
}
|
|
1035
1104
|
}
|
|
1036
1105
|
function semanticRouteHint(events) {
|
|
1037
1106
|
const index = buildEvidenceIndex(events);
|
|
1038
1107
|
if (!index) return false;
|
|
1039
|
-
if ([...index.calls.values()].some((pair) => pair.name === "subagent" || pair.name === "subagent_fork" || pair.name === "workflow")) return true;
|
|
1108
|
+
if ([...index.calls.values()].some((pair) => pair.name === "subagent" || pair.name === "subagent_fork" || pair.name === "workflow" || pair.name === "exit_plan_mode")) return true;
|
|
1040
1109
|
if (canonicalTodoSnapshots(index).length >= 2) return true;
|
|
1110
|
+
if (canonicalTeamTaskSnapshots(index).length >= 2) return true;
|
|
1041
1111
|
return false;
|
|
1042
1112
|
}
|
|
1043
1113
|
function buildSemanticRoutePrompt(problem, events, maxCandidates, maxItemChars = 2e4) {
|
|
@@ -1232,6 +1302,124 @@ var AutoVerifierRouter = class {
|
|
|
1232
1302
|
}
|
|
1233
1303
|
};
|
|
1234
1304
|
//#endregion
|
|
1305
|
+
//#region src/plan-gate.ts
|
|
1306
|
+
function detectPlanExit(events, fromSeq = 0) {
|
|
1307
|
+
for (let i = events.length - 1; i >= 0; i--) {
|
|
1308
|
+
const event = events[i];
|
|
1309
|
+
if (event.seq < fromSeq) break;
|
|
1310
|
+
if (event.type === "tool/call" && event.data.name === "exit_plan_mode") {
|
|
1311
|
+
let planText = "";
|
|
1312
|
+
try {
|
|
1313
|
+
const parsed = JSON.parse(event.data.arguments);
|
|
1314
|
+
if (typeof parsed.plan === "string" && parsed.plan.trim()) planText = parsed.plan.trim();
|
|
1315
|
+
} catch {}
|
|
1316
|
+
if (!planText) for (let j = i - 1; j >= 0; j--) {
|
|
1317
|
+
const prev = events[j];
|
|
1318
|
+
if (prev.seq < fromSeq) break;
|
|
1319
|
+
if (prev.type === "assistant/message") {
|
|
1320
|
+
const msg = prev.data.message;
|
|
1321
|
+
if (Array.isArray(msg?.content)) {
|
|
1322
|
+
const texts = msg.content.filter((b) => b.type === "text").map((b) => b.text);
|
|
1323
|
+
if (texts.length > 0) {
|
|
1324
|
+
planText = texts.join("\n").trim();
|
|
1325
|
+
break;
|
|
1326
|
+
}
|
|
1327
|
+
}
|
|
1328
|
+
}
|
|
1329
|
+
}
|
|
1330
|
+
return {
|
|
1331
|
+
hasExitPlanMode: true,
|
|
1332
|
+
callSeq: event.seq,
|
|
1333
|
+
planText
|
|
1334
|
+
};
|
|
1335
|
+
}
|
|
1336
|
+
}
|
|
1337
|
+
return { hasExitPlanMode: false };
|
|
1338
|
+
}
|
|
1339
|
+
function buildPlanPreReviewPrompt(problem, planText, maxChars = 2e4) {
|
|
1340
|
+
return [
|
|
1341
|
+
"You are an expert independent technical plan verifier. A candidate implementation plan was submitted for human review.",
|
|
1342
|
+
"Evaluate whether the plan is sound, executable, and comprehensive.",
|
|
1343
|
+
"Scoring criteria:",
|
|
1344
|
+
"A: Flawless, thorough, with clear steps, rigorous verification, edge cases handled.",
|
|
1345
|
+
"B-J: Mostly sound, minor gaps or nuances missing.",
|
|
1346
|
+
"K-S: Noticeable architectural gaps, missing regression checks, or ambiguities.",
|
|
1347
|
+
"T: Fundamentally flawed, dangerous, or missing core requirements.",
|
|
1348
|
+
"",
|
|
1349
|
+
"Task requirement:",
|
|
1350
|
+
sanitizeVerifierText(problem, 4e3),
|
|
1351
|
+
"",
|
|
1352
|
+
"Proposed Plan:",
|
|
1353
|
+
sanitizeVerifierText(planText, maxChars),
|
|
1354
|
+
"",
|
|
1355
|
+
"Output format:",
|
|
1356
|
+
"Line 1: Verdict: <Single uppercase letter A-T>",
|
|
1357
|
+
"Line 2: Summary: <One sentence assessment>",
|
|
1358
|
+
"Line 3+: Key strengths, blind spots, and verification guidance."
|
|
1359
|
+
].join("\n\n");
|
|
1360
|
+
}
|
|
1361
|
+
function parsePlanReviewVerdict(text) {
|
|
1362
|
+
const match = /Verdict:\s*([A-T])/i.exec(text);
|
|
1363
|
+
const verdict = match ? match[1].toUpperCase() : "B";
|
|
1364
|
+
const index = verdict.charCodeAt(0) - 65;
|
|
1365
|
+
return {
|
|
1366
|
+
verdict,
|
|
1367
|
+
score: Math.max(0, Math.min(1, 1 - index / 19)),
|
|
1368
|
+
feedback: text.trim()
|
|
1369
|
+
};
|
|
1370
|
+
}
|
|
1371
|
+
//#endregion
|
|
1372
|
+
//#region src/team-gate.ts
|
|
1373
|
+
function inspectTeamTasks(events, fromSeq = 0) {
|
|
1374
|
+
let latestCompletedTask;
|
|
1375
|
+
let completedSeq;
|
|
1376
|
+
const taskMap = /* @__PURE__ */ new Map();
|
|
1377
|
+
for (const rawEvent of events) {
|
|
1378
|
+
const event = rawEvent;
|
|
1379
|
+
if (event.type === "team/task") {
|
|
1380
|
+
const data = event.data;
|
|
1381
|
+
if (data?.task) {
|
|
1382
|
+
taskMap.set(data.task.id, { ...data.task });
|
|
1383
|
+
if (event.seq >= fromSeq && data.task.status === "completed") {
|
|
1384
|
+
latestCompletedTask = { ...data.task };
|
|
1385
|
+
completedSeq = event.seq;
|
|
1386
|
+
}
|
|
1387
|
+
}
|
|
1388
|
+
}
|
|
1389
|
+
}
|
|
1390
|
+
return {
|
|
1391
|
+
latestCompletedTask,
|
|
1392
|
+
completedSeq,
|
|
1393
|
+
activeTasks: [...taskMap.values()].filter((t) => t.status !== "deleted"),
|
|
1394
|
+
hasRecentCompletedTask: latestCompletedTask !== void 0
|
|
1395
|
+
};
|
|
1396
|
+
}
|
|
1397
|
+
function buildTeamTaskVerificationPrompt(task, executionTrace, maxChars = 2e4) {
|
|
1398
|
+
return [
|
|
1399
|
+
"You are an expert independent technical verifier reviewing a completed Agent Teams task.",
|
|
1400
|
+
"Verify if the task goal has been satisfied by concrete evidence (code edits, test runs, successful commands).",
|
|
1401
|
+
"Task Details:",
|
|
1402
|
+
"ID: " + task.id,
|
|
1403
|
+
"Subject: " + task.subject,
|
|
1404
|
+
task.description ? "Description: " + task.description : "",
|
|
1405
|
+
"",
|
|
1406
|
+
"Execution Evidence & Trace:",
|
|
1407
|
+
sanitizeVerifierText(executionTrace, maxChars),
|
|
1408
|
+
"",
|
|
1409
|
+
"Evaluate whether the task is genuinely completed with verified proof.",
|
|
1410
|
+
"Scoring criteria:",
|
|
1411
|
+
"A: Completely fulfilled with solid execution/test evidence.",
|
|
1412
|
+
"B-J: Mostly complete, minor gaps or edge cases not fully asserted.",
|
|
1413
|
+
"K-S: Incomplete, missing key requirements or untested.",
|
|
1414
|
+
"T: Unfulfilled, failed, or contradictory evidence.",
|
|
1415
|
+
"",
|
|
1416
|
+
"Output format:",
|
|
1417
|
+
"Line 1: Verdict: <Single uppercase letter A-T>",
|
|
1418
|
+
"Line 2: Summary: <One sentence evaluation of task fulfillment>",
|
|
1419
|
+
"Line 3+: Remaining gaps or next steps for the team."
|
|
1420
|
+
].filter(Boolean).join("\n\n");
|
|
1421
|
+
}
|
|
1422
|
+
//#endregion
|
|
1235
1423
|
//#region src/statistics.ts
|
|
1236
1424
|
const VERIFIER_TOOL_NAMES = [
|
|
1237
1425
|
"verifier_route_classify",
|
|
@@ -2010,8 +2198,7 @@ function apply(ctx, config = {}) {
|
|
|
2010
2198
|
summary: "Automatic verifier routed " + decision.kind
|
|
2011
2199
|
}
|
|
2012
2200
|
});
|
|
2013
|
-
|
|
2014
|
-
if (endpoint !== "statistics") return rpcFailure("unknown llm-verifier endpoint");
|
|
2201
|
+
const handleStatisticsQuery = async (payload) => {
|
|
2015
2202
|
if (typeof payload !== "object" || payload === null) return rpcFailure("statistics payload must be an object");
|
|
2016
2203
|
const row = payload;
|
|
2017
2204
|
const sessionId = typeof row.sessionId === "string" && row.sessionId.length > 0 ? row.sessionId : void 0;
|
|
@@ -2028,7 +2215,87 @@ function apply(ctx, config = {}) {
|
|
|
2028
2215
|
} catch (error) {
|
|
2029
2216
|
return rpcFailure(error instanceof Error ? error.message : String(error));
|
|
2030
2217
|
}
|
|
2031
|
-
}
|
|
2218
|
+
};
|
|
2219
|
+
const anyConn = services.connection;
|
|
2220
|
+
if (anyConn && anyConn.fetch && typeof anyConn.fetch.register === "function") try {
|
|
2221
|
+
anyConn.fetch.register({
|
|
2222
|
+
path: "/api/llm-verifier/statistics",
|
|
2223
|
+
methods: ["POST"],
|
|
2224
|
+
requestBody: "buffered",
|
|
2225
|
+
fetch: async (request) => {
|
|
2226
|
+
let body;
|
|
2227
|
+
try {
|
|
2228
|
+
body = await request.json();
|
|
2229
|
+
} catch {
|
|
2230
|
+
return new Response("body is not JSON", { status: 400 });
|
|
2231
|
+
}
|
|
2232
|
+
const isRpcEnvelope = typeof body === "object" && body !== null && body.type === "client-request" && typeof body.rpcId === "string";
|
|
2233
|
+
const rpcId = isRpcEnvelope ? body.rpcId : "direct";
|
|
2234
|
+
const payload = isRpcEnvelope ? body.payload : body;
|
|
2235
|
+
const outcome = await handleStatisticsQuery(payload);
|
|
2236
|
+
if (isRpcEnvelope) return Response.json({
|
|
2237
|
+
type: "server-response",
|
|
2238
|
+
rpcId,
|
|
2239
|
+
result: outcome
|
|
2240
|
+
});
|
|
2241
|
+
return Response.json(outcome);
|
|
2242
|
+
}
|
|
2243
|
+
});
|
|
2244
|
+
} catch (e) {
|
|
2245
|
+
ctx.logger.warn("failed to register /api/llm-verifier/statistics fetch route: " + String(e));
|
|
2246
|
+
}
|
|
2247
|
+
const mountWebServerRoute = (ws) => {
|
|
2248
|
+
if (!ws || typeof ws.register !== "function") return;
|
|
2249
|
+
try {
|
|
2250
|
+
ws.register({
|
|
2251
|
+
kind: "prefix",
|
|
2252
|
+
path: "/llm-verifier",
|
|
2253
|
+
handler: (req, res) => {
|
|
2254
|
+
if (req.method !== "POST") {
|
|
2255
|
+
res.writeHead(405);
|
|
2256
|
+
res.end();
|
|
2257
|
+
return;
|
|
2258
|
+
}
|
|
2259
|
+
let bodyStr = "";
|
|
2260
|
+
req.on("data", (chunk) => {
|
|
2261
|
+
bodyStr += chunk.toString();
|
|
2262
|
+
});
|
|
2263
|
+
req.on("end", async () => {
|
|
2264
|
+
try {
|
|
2265
|
+
const body = JSON.parse(bodyStr);
|
|
2266
|
+
const isRpcEnvelope = typeof body === "object" && body !== null && body.type === "client-request" && typeof body.rpcId === "string";
|
|
2267
|
+
const rpcId = isRpcEnvelope ? body.rpcId : "direct";
|
|
2268
|
+
const payload = isRpcEnvelope ? body.payload : body;
|
|
2269
|
+
const outcome = await handleStatisticsQuery(payload);
|
|
2270
|
+
res.writeHead(200, { "content-type": "application/json" });
|
|
2271
|
+
if (isRpcEnvelope) res.end(JSON.stringify({
|
|
2272
|
+
type: "server-response",
|
|
2273
|
+
rpcId,
|
|
2274
|
+
result: outcome
|
|
2275
|
+
}));
|
|
2276
|
+
else res.end(JSON.stringify(outcome));
|
|
2277
|
+
} catch (err) {
|
|
2278
|
+
res.writeHead(400, { "content-type": "application/json" });
|
|
2279
|
+
res.end(JSON.stringify({ error: String(err) }));
|
|
2280
|
+
}
|
|
2281
|
+
});
|
|
2282
|
+
}
|
|
2283
|
+
});
|
|
2284
|
+
} catch {}
|
|
2285
|
+
};
|
|
2286
|
+
if (typeof ctx.inject === "function") try {
|
|
2287
|
+
ctx.inject(["webServer"], (webCtx) => {
|
|
2288
|
+
if (webCtx?.webServer) mountWebServerRoute(webCtx.webServer);
|
|
2289
|
+
});
|
|
2290
|
+
} catch {}
|
|
2291
|
+
try {
|
|
2292
|
+
services.connection?.rpc?.handle("/llm-verifier", async (endpoint, payload) => {
|
|
2293
|
+
if (endpoint !== "statistics") return rpcFailure("unknown llm-verifier endpoint");
|
|
2294
|
+
return handleStatisticsQuery(payload);
|
|
2295
|
+
}, { authority: "loopback" });
|
|
2296
|
+
} catch (e) {
|
|
2297
|
+
ctx.logger.warn("failed to register /llm-verifier rpc handler: " + String(e));
|
|
2298
|
+
}
|
|
2032
2299
|
ctx.on("agent/disposed", ({ agent }) => {
|
|
2033
2300
|
autoBudget.release(agent);
|
|
2034
2301
|
autoRouter.release(agent);
|
|
@@ -2047,6 +2314,114 @@ function apply(ctx, config = {}) {
|
|
|
2047
2314
|
const admittedLastSeq = agent.session.events.at(-1)?.seq ?? -1;
|
|
2048
2315
|
const snapshot = agent.session.events.filter((event) => event.seq <= admittedLastSeq);
|
|
2049
2316
|
const stillCurrent = () => !signal.aborted && (agent.session.events.at(-1)?.seq ?? -1) === admittedLastSeq;
|
|
2317
|
+
if (selected.autoVerifyPlanMode) {
|
|
2318
|
+
const planDetection = detectPlanExit(snapshot, evidence.taskStartSeq);
|
|
2319
|
+
if (planDetection.hasExitPlanMode && planDetection.planText) {
|
|
2320
|
+
const planFingerprint = stableHash({
|
|
2321
|
+
phase: "plan_review",
|
|
2322
|
+
callSeq: planDetection.callSeq,
|
|
2323
|
+
plan: planDetection.planText
|
|
2324
|
+
});
|
|
2325
|
+
const planReservation = autoRouter.reserve(agent, "plan_review", planFingerprint, 1, policy);
|
|
2326
|
+
if (planReservation) try {
|
|
2327
|
+
const prompt = buildPlanPreReviewPrompt((await extractTask(agent, evidence.taskStartSeq, admittedLastSeq, selected.autoVerifyMaxChars, signal)).problem, planDetection.planText, selected.autoRouteMaxInputChars);
|
|
2328
|
+
const classified = await classifyRoute(agent, prompt, signal);
|
|
2329
|
+
if (!stillCurrent()) {
|
|
2330
|
+
autoRouter.fail(agent, planReservation, false);
|
|
2331
|
+
return;
|
|
2332
|
+
}
|
|
2333
|
+
const verdict = parsePlanReviewVerdict(classified.text);
|
|
2334
|
+
if (verdict.score >= selected.autoVerifyThreshold) autoRouter.commit(agent, planReservation, admittedLastSeq);
|
|
2335
|
+
else {
|
|
2336
|
+
autoRouter.fail(agent, planReservation, selected.autoVerifyMode === "strict");
|
|
2337
|
+
agent.steer(createUserMessage({
|
|
2338
|
+
content: [{
|
|
2339
|
+
type: "text",
|
|
2340
|
+
text: "[Automatic Verifier Plan Pre-review]\nPlan pre-review scored " + (verdict.score * 100).toFixed(1) + "% (Threshold: " + (selected.autoVerifyThreshold * 100).toFixed(0) + "%).\n" + verdict.feedback + "\nAddress the identified issues or improve the plan before proceeding."
|
|
2341
|
+
}],
|
|
2342
|
+
source: {
|
|
2343
|
+
kind: "plugin",
|
|
2344
|
+
plugin: "dsh-llm-verifier",
|
|
2345
|
+
form: "notice",
|
|
2346
|
+
summary: "Plan Pre-review Feedback"
|
|
2347
|
+
}
|
|
2348
|
+
}));
|
|
2349
|
+
return;
|
|
2350
|
+
}
|
|
2351
|
+
} catch (error) {
|
|
2352
|
+
autoRouter.fail(agent, planReservation, selected.autoVerifyMode === "strict");
|
|
2353
|
+
ctx.logger.warn("llm-verifier plan pre-review failed: " + (error instanceof Error ? error.message : String(error)));
|
|
2354
|
+
if (selected.autoVerifyMode === "strict" && !signal.aborted) {
|
|
2355
|
+
agent.steer(createUserMessage({
|
|
2356
|
+
content: [{
|
|
2357
|
+
type: "text",
|
|
2358
|
+
text: "[Automatic Verifier Plan Pre-review]\nPlan pre-review failed: " + (error instanceof Error ? error.message : String(error))
|
|
2359
|
+
}],
|
|
2360
|
+
source: {
|
|
2361
|
+
kind: "plugin",
|
|
2362
|
+
plugin: "dsh-llm-verifier"
|
|
2363
|
+
}
|
|
2364
|
+
}));
|
|
2365
|
+
return;
|
|
2366
|
+
}
|
|
2367
|
+
}
|
|
2368
|
+
}
|
|
2369
|
+
}
|
|
2370
|
+
if (selected.autoVerifyTeamTasks) {
|
|
2371
|
+
const teamInspection = inspectTeamTasks(snapshot, evidence.taskStartSeq);
|
|
2372
|
+
if (teamInspection.hasRecentCompletedTask && teamInspection.latestCompletedTask) {
|
|
2373
|
+
const task = teamInspection.latestCompletedTask;
|
|
2374
|
+
const taskFingerprint = stableHash({
|
|
2375
|
+
phase: "team_task",
|
|
2376
|
+
taskId: task.id,
|
|
2377
|
+
status: task.status,
|
|
2378
|
+
seq: teamInspection.completedSeq
|
|
2379
|
+
});
|
|
2380
|
+
const taskReservation = autoRouter.reserve(agent, "team_task", taskFingerprint, 1, policy);
|
|
2381
|
+
if (taskReservation) try {
|
|
2382
|
+
const prompt = buildTeamTaskVerificationPrompt(task, (await extractTask(agent, evidence.taskStartSeq, admittedLastSeq, selected.autoVerifyMaxChars, signal)).trace, selected.autoRouteMaxInputChars);
|
|
2383
|
+
const classified = await classifyRoute(agent, prompt, signal);
|
|
2384
|
+
if (!stillCurrent()) {
|
|
2385
|
+
autoRouter.fail(agent, taskReservation, false);
|
|
2386
|
+
return;
|
|
2387
|
+
}
|
|
2388
|
+
const verdict = parsePlanReviewVerdict(classified.text);
|
|
2389
|
+
if (verdict.score >= selected.autoVerifyThreshold) autoRouter.commit(agent, taskReservation, admittedLastSeq);
|
|
2390
|
+
else {
|
|
2391
|
+
autoRouter.fail(agent, taskReservation, selected.autoVerifyMode === "strict");
|
|
2392
|
+
agent.steer(createUserMessage({
|
|
2393
|
+
content: [{
|
|
2394
|
+
type: "text",
|
|
2395
|
+
text: "[Automatic Verifier Team Task Gate]\nTask \"" + task.subject + "\" (#" + task.id + ") verification scored " + (verdict.score * 100).toFixed(1) + "% (Threshold: " + (selected.autoVerifyThreshold * 100).toFixed(0) + "%).\n" + verdict.feedback + "\nProvide verified execution evidence or resolve remaining issues before completing the task."
|
|
2396
|
+
}],
|
|
2397
|
+
source: {
|
|
2398
|
+
kind: "plugin",
|
|
2399
|
+
plugin: "dsh-llm-verifier",
|
|
2400
|
+
form: "notice",
|
|
2401
|
+
summary: "Team Task Gate Feedback"
|
|
2402
|
+
}
|
|
2403
|
+
}));
|
|
2404
|
+
return;
|
|
2405
|
+
}
|
|
2406
|
+
} catch (error) {
|
|
2407
|
+
autoRouter.fail(agent, taskReservation, selected.autoVerifyMode === "strict");
|
|
2408
|
+
ctx.logger.warn("llm-verifier team task verification failed: " + (error instanceof Error ? error.message : String(error)));
|
|
2409
|
+
if (selected.autoVerifyMode === "strict" && !signal.aborted) {
|
|
2410
|
+
agent.steer(createUserMessage({
|
|
2411
|
+
content: [{
|
|
2412
|
+
type: "text",
|
|
2413
|
+
text: "[Automatic Verifier Team Task Gate]\nTask verification failed: " + (error instanceof Error ? error.message : String(error))
|
|
2414
|
+
}],
|
|
2415
|
+
source: {
|
|
2416
|
+
kind: "plugin",
|
|
2417
|
+
plugin: "dsh-llm-verifier"
|
|
2418
|
+
}
|
|
2419
|
+
}));
|
|
2420
|
+
return;
|
|
2421
|
+
}
|
|
2422
|
+
}
|
|
2423
|
+
}
|
|
2424
|
+
}
|
|
2050
2425
|
let decision = boundDecision(analyzeStructuredRoute(snapshot, selected.autoRouteMaxCandidates, selected.autoRouteMaxItemChars), policy);
|
|
2051
2426
|
if (decision === void 0 && selected.autoRouteSemantic && (selected.autoVerifyMode === "strict" || semanticRouteHint(snapshot))) {
|
|
2052
2427
|
const fingerprint = stableHash({
|
|
@@ -2551,6 +2926,6 @@ function apply(ctx, config = {}) {
|
|
|
2551
2926
|
}));
|
|
2552
2927
|
}
|
|
2553
2928
|
//#endregion
|
|
2554
|
-
export { AutoVerificationBudget, AutoVerifierRouter, Config, DEFAULT_CRITERIA, DEFAULT_GROUND_TRUTH_NOTE, GRANULARITY, LETTERS, RequestLimiter, SCALE_DESCRIPTION, ScoreCache, SingleFlight, StatisticsStore, VERIFIER_TOOL_NAMES, VerifierEngine, accumulatePairs, analyzeAutoTask, analyzeStructuredRoute, apply, automaticFeedback, boundDecision, bradleyTerry, buildEvidenceIndex, buildPairwisePrompt, buildProgressPrompt, buildSemanticRoutePrompt, callVerifier, emptyRunStats, errorDetails, extractProgressScore, extractScore, inject, latestDirectUserSeq, mergeStatisticsOverviews, name, normalizeCriteria, normalizeScoreLetter, parseSemanticRoute, pivotRoundPairs, rankScores, resolveCacheFile, resolveStatisticsFile, resolveTopicDataDir, ringCycle, seededRandom, semanticDecision, semanticRouteHint, stableHash, topPivots };
|
|
2929
|
+
export { AutoVerificationBudget, AutoVerifierRouter, Config, DEFAULT_CRITERIA, DEFAULT_GROUND_TRUTH_NOTE, GRANULARITY, LETTERS, RequestLimiter, SCALE_DESCRIPTION, ScoreCache, SingleFlight, StatisticsStore, VERIFIER_TOOL_NAMES, VerifierEngine, accumulatePairs, analyzeAutoTask, analyzeStructuredRoute, apply, automaticFeedback, boundDecision, bradleyTerry, buildEvidenceIndex, buildPairwisePrompt, buildPlanPreReviewPrompt, buildProgressPrompt, buildSemanticRoutePrompt, buildTeamTaskVerificationPrompt, callVerifier, detectPlanExit, emptyRunStats, errorDetails, extractProgressScore, extractScore, inject, inspectTeamTasks, latestDirectUserSeq, mergeStatisticsOverviews, name, normalizeCriteria, normalizeScoreLetter, parsePlanReviewVerdict, parseSemanticRoute, pivotRoundPairs, rankScores, resolveCacheFile, resolveStatisticsFile, resolveTopicDataDir, ringCycle, seededRandom, semanticDecision, semanticRouteHint, stableHash, topPivots };
|
|
2555
2930
|
|
|
2556
2931
|
//# sourceMappingURL=index.js.map
|
|
@@ -122,6 +122,15 @@ export declare const zh: {
|
|
|
122
122
|
'models.tokens': string;
|
|
123
123
|
'models.empty': string;
|
|
124
124
|
'slot.statistics': string;
|
|
125
|
+
'slot.globalDashboard': string;
|
|
126
|
+
'global.panelTitle': string;
|
|
127
|
+
'global.panelIntro': string;
|
|
128
|
+
'guide.verifier.title': string;
|
|
129
|
+
'guide.verifier.desc': string;
|
|
130
|
+
'field.autoVerifyTeamTasks.title': string;
|
|
131
|
+
'field.autoVerifyTeamTasks.help': string;
|
|
132
|
+
'field.autoVerifyPlanMode.title': string;
|
|
133
|
+
'field.autoVerifyPlanMode.help': string;
|
|
125
134
|
};
|
|
126
135
|
export type I18nDict = typeof zh;
|
|
127
136
|
export declare const en: I18nDict;
|
|
@@ -250,6 +259,15 @@ export declare const dictionaries: {
|
|
|
250
259
|
'models.tokens': string;
|
|
251
260
|
'models.empty': string;
|
|
252
261
|
'slot.statistics': string;
|
|
262
|
+
'slot.globalDashboard': string;
|
|
263
|
+
'global.panelTitle': string;
|
|
264
|
+
'global.panelIntro': string;
|
|
265
|
+
'guide.verifier.title': string;
|
|
266
|
+
'guide.verifier.desc': string;
|
|
267
|
+
'field.autoVerifyTeamTasks.title': string;
|
|
268
|
+
'field.autoVerifyTeamTasks.help': string;
|
|
269
|
+
'field.autoVerifyPlanMode.title': string;
|
|
270
|
+
'field.autoVerifyPlanMode.help': string;
|
|
253
271
|
};
|
|
254
272
|
en: {
|
|
255
273
|
'settings.title': string;
|
|
@@ -375,6 +393,15 @@ export declare const dictionaries: {
|
|
|
375
393
|
'models.tokens': string;
|
|
376
394
|
'models.empty': string;
|
|
377
395
|
'slot.statistics': string;
|
|
396
|
+
'slot.globalDashboard': string;
|
|
397
|
+
'global.panelTitle': string;
|
|
398
|
+
'global.panelIntro': string;
|
|
399
|
+
'guide.verifier.title': string;
|
|
400
|
+
'guide.verifier.desc': string;
|
|
401
|
+
'field.autoVerifyTeamTasks.title': string;
|
|
402
|
+
'field.autoVerifyTeamTasks.help': string;
|
|
403
|
+
'field.autoVerifyPlanMode.title': string;
|
|
404
|
+
'field.autoVerifyPlanMode.help': string;
|
|
378
405
|
};
|
|
379
406
|
};
|
|
380
407
|
export declare const toolLabels: Record<'zh' | 'en', Record<string, string>>;
|
package/lib/types/client-i18n.js
CHANGED
|
@@ -137,6 +137,15 @@ export const zh = {
|
|
|
137
137
|
'models.empty': '暂无模型调用',
|
|
138
138
|
// Slot label
|
|
139
139
|
'slot.statistics': '工具统计',
|
|
140
|
+
'slot.globalDashboard': 'Verifier 看板',
|
|
141
|
+
'global.panelTitle': 'LLM Verifier 全局监控中心',
|
|
142
|
+
'global.panelIntro': '跨会话全局统计、裁判模型效能大盘与任务验收总览。',
|
|
143
|
+
'guide.verifier.title': 'Verifier 统计',
|
|
144
|
+
'guide.verifier.desc': '大模型复核工具调用、耗时与成本统计',
|
|
145
|
+
'field.autoVerifyTeamTasks.title': '自动验收 Team 任务',
|
|
146
|
+
'field.autoVerifyTeamTasks.help': '当 Agent Teams 中的任务状态变更或进入完成状态时,自动路由进度追踪与任务验收。',
|
|
147
|
+
'field.autoVerifyPlanMode.title': '计划模式自动预审',
|
|
148
|
+
'field.autoVerifyPlanMode.help': '在 Agent 调用 exit_plan_mode 提交计划交付人类评审前,自动对计划完整度与风险进行独立预审。',
|
|
140
149
|
};
|
|
141
150
|
export const en = {
|
|
142
151
|
// Settings Page
|
|
@@ -276,6 +285,15 @@ export const en = {
|
|
|
276
285
|
'models.empty': 'No model calls',
|
|
277
286
|
// Slot label
|
|
278
287
|
'slot.statistics': 'Statistics',
|
|
288
|
+
'slot.globalDashboard': 'Verifier Dashboard',
|
|
289
|
+
'global.panelTitle': 'LLM Verifier Global Dashboard',
|
|
290
|
+
'global.panelIntro': 'Cross-session statistics, judge model metrics, and task acceptance overview.',
|
|
291
|
+
'guide.verifier.title': 'Verifier Stats',
|
|
292
|
+
'guide.verifier.desc': 'LLM verifier tool calls, duration and cost statistics',
|
|
293
|
+
'field.autoVerifyTeamTasks.title': 'Verify Team Tasks',
|
|
294
|
+
'field.autoVerifyTeamTasks.help': 'Automatically route progress and verify tasks when Agent Teams task status changes or completes.',
|
|
295
|
+
'field.autoVerifyPlanMode.title': 'Plan Mode Pre-verification',
|
|
296
|
+
'field.autoVerifyPlanMode.help': 'Automatically pre-verify plan feasibility and risks before exit_plan_mode presents it for user review.',
|
|
279
297
|
};
|
|
280
298
|
export const dictionaries = { zh, en };
|
|
281
299
|
export const toolLabels = {
|