dsh-llm-verifier 0.1.13 → 0.1.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +22 -22
- package/README.md +328 -301
- package/cordis.patch.yml +4 -4
- package/lib/caller-BWwSVNZf.js.map +1 -0
- package/lib/client.js +176 -14
- package/lib/client.js.map +1 -0
- package/lib/core.js.map +1 -0
- package/lib/index.js +400 -51
- package/lib/index.js.map +1 -0
- package/lib/types/auto.d.ts.map +1 -0
- package/lib/types/auto.js +8 -2
- package/lib/types/auto.js.map +1 -0
- package/lib/types/cache.d.ts.map +1 -0
- package/lib/types/cache.js.map +1 -0
- package/lib/types/caller.d.ts.map +1 -0
- package/lib/types/caller.js.map +1 -0
- package/lib/types/client-i18n.d.ts +27 -0
- package/lib/types/client-i18n.d.ts.map +1 -0
- package/lib/types/client-i18n.js +18 -0
- package/lib/types/client-i18n.js.map +1 -0
- package/lib/types/client.d.ts +15 -2
- package/lib/types/client.d.ts.map +1 -0
- package/lib/types/client.js +137 -10
- package/lib/types/client.js.map +1 -0
- package/lib/types/config.d.ts +4 -0
- package/lib/types/config.d.ts.map +1 -0
- package/lib/types/config.js +5 -3
- package/lib/types/config.js.map +1 -0
- package/lib/types/core.d.ts.map +1 -0
- package/lib/types/core.js.map +1 -0
- package/lib/types/engine.d.ts.map +1 -0
- package/lib/types/engine.js.map +1 -0
- package/lib/types/images.d.ts.map +1 -0
- package/lib/types/images.js.map +1 -0
- package/lib/types/index.d.ts +2 -0
- package/lib/types/index.d.ts.map +1 -0
- package/lib/types/index.js +162 -9
- package/lib/types/index.js.map +1 -0
- package/lib/types/plan-gate.d.ts +27 -0
- package/lib/types/plan-gate.d.ts.map +1 -0
- package/lib/types/plan-gate.js +60 -0
- package/lib/types/plan-gate.js.map +1 -0
- package/lib/types/router.d.ts +28 -32
- package/lib/types/router.d.ts.map +1 -0
- package/lib/types/router.js +41 -9
- package/lib/types/router.js.map +1 -0
- package/lib/types/session.d.ts +11 -0
- package/lib/types/session.d.ts.map +1 -0
- package/lib/types/session.js +41 -11
- package/lib/types/session.js.map +1 -0
- package/lib/types/statistics.d.ts.map +1 -0
- package/lib/types/statistics.js.map +1 -0
- package/lib/types/team-gate.d.ts +20 -0
- package/lib/types/team-gate.d.ts.map +1 -0
- package/lib/types/team-gate.js +59 -0
- package/lib/types/team-gate.js.map +1 -0
- package/lib/types/top-logprobs.d.ts.map +1 -0
- package/lib/types/top-logprobs.js.map +1 -0
- package/lib/types/topic-storage.d.ts.map +1 -0
- package/lib/types/topic-storage.js.map +1 -0
- package/package.json +114 -112
- package/src/auto.test.ts +95 -86
- package/src/auto.ts +140 -133
- package/src/cache.ts +115 -115
- package/src/caller.test.ts +34 -34
- package/src/caller.ts +151 -151
- package/src/client-i18n.ts +18 -0
- package/src/client.tsx +383 -213
- package/src/config.ts +189 -183
- package/src/core.test.ts +71 -71
- package/src/core.ts +205 -205
- package/src/engine.test.ts +136 -136
- package/src/engine.ts +140 -140
- package/src/images.ts +33 -33
- package/src/index.ts +394 -247
- package/src/parity.test.ts +85 -85
- package/src/plan-gate.test.ts +56 -0
- package/src/plan-gate.ts +66 -0
- package/src/router.test.ts +107 -89
- package/src/router.ts +361 -309
- package/src/session.test.ts +71 -36
- package/src/session.ts +132 -103
- package/src/statistics.test.ts +80 -80
- package/src/statistics.ts +328 -328
- package/src/team-gate.test.ts +92 -0
- package/src/team-gate.ts +80 -0
- package/src/top-logprobs.test.ts +84 -84
- package/src/top-logprobs.ts +153 -153
- package/src/topic-storage.test.ts +31 -31
- package/src/topic-storage.ts +42 -42
package/lib/index.js
CHANGED
|
@@ -31,8 +31,10 @@ const Config = z.object({
|
|
|
31
31
|
autoRouteMaxInputChars: z.number().step(1).min(1e3).default(6e4),
|
|
32
32
|
autoMaxModelCallsPerTask: z.number().step(1).min(1).default(48),
|
|
33
33
|
autoMaxModelCallsPerSession: z.number().step(1).min(1).default(160),
|
|
34
|
+
autoVerifyTeamTasks: z.boolean().default(true),
|
|
35
|
+
autoVerifyPlanMode: z.boolean().default(true),
|
|
34
36
|
provider: z.string().default("deepseek-official"),
|
|
35
|
-
model: z.string().default("deepseek-
|
|
37
|
+
model: z.string().default("deepseek-flash"),
|
|
36
38
|
reasoningEffort: z.string(),
|
|
37
39
|
maxTokens: z.number().step(1).min(1).default(32768),
|
|
38
40
|
timeoutMs: z.number().step(1).min(1).default(3e5),
|
|
@@ -46,7 +48,7 @@ const Config = z.object({
|
|
|
46
48
|
});
|
|
47
49
|
function resolveConfig(config = {}) {
|
|
48
50
|
const provider = (config.provider ?? "deepseek-official").trim();
|
|
49
|
-
const model = (config.model ?? "deepseek-
|
|
51
|
+
const model = (config.model ?? "deepseek-flash").trim();
|
|
50
52
|
if (!provider) throw new Error("llm-verifier: provider must be non-empty");
|
|
51
53
|
if (!model) throw new Error("llm-verifier: model must be non-empty");
|
|
52
54
|
const autoVerifyMode = config.autoVerifyMode ?? "smart";
|
|
@@ -100,6 +102,8 @@ function resolveConfig(config = {}) {
|
|
|
100
102
|
autoRouteSemantic: config.autoRouteSemantic ?? true,
|
|
101
103
|
autoRouteMinConfidence,
|
|
102
104
|
autoTrackCompletionThreshold,
|
|
105
|
+
autoVerifyTeamTasks: config.autoVerifyTeamTasks ?? true,
|
|
106
|
+
autoVerifyPlanMode: config.autoVerifyPlanMode ?? true,
|
|
103
107
|
provider,
|
|
104
108
|
model,
|
|
105
109
|
...reasoningEffort ? { reasoningEffort } : {},
|
|
@@ -602,12 +606,33 @@ async function loadVerifierImages(inputs, signal) {
|
|
|
602
606
|
}
|
|
603
607
|
//#endregion
|
|
604
608
|
//#region src/session.ts
|
|
609
|
+
/**
|
|
610
|
+
* Read one session's event log.
|
|
611
|
+
*
|
|
612
|
+
* DSH 0.1.5 replaced the `events` array with `snapshotEvents()`; older hosts
|
|
613
|
+
* expose the array directly. Both shapes are accepted so the plugin keeps
|
|
614
|
+
* working across the versions it declares support for.
|
|
615
|
+
* @param session - Agent session, or any object exposing one of the two shapes.
|
|
616
|
+
* @returns The session's events in log order, or an empty array.
|
|
617
|
+
*/
|
|
618
|
+
function sessionEvents(session) {
|
|
619
|
+
const candidate = session;
|
|
620
|
+
if (typeof candidate?.snapshotEvents === "function") return candidate.snapshotEvents();
|
|
621
|
+
return candidate?.events ?? [];
|
|
622
|
+
}
|
|
605
623
|
function textOf(blocks) {
|
|
606
624
|
const parts = [];
|
|
607
|
-
for (const block of blocks)
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
625
|
+
for (const block of blocks) {
|
|
626
|
+
const blockType = block.type;
|
|
627
|
+
if (blockType === "text") parts.push(block.text);
|
|
628
|
+
else if (blockType === "reasoning") parts.push("[Reasoning] " + block.text);
|
|
629
|
+
else if (blockType === "tool-call") parts.push("[Tool Call] " + block.name + " " + block.arguments);
|
|
630
|
+
else if (blockType === "tool-result") parts.push("[Tool Result] " + textOf(block.content));
|
|
631
|
+
else if (blockType === "file") {
|
|
632
|
+
const fileData = block;
|
|
633
|
+
parts.push("[File] " + (fileData.path ?? fileData.filename ?? fileData.title ?? "attachment"));
|
|
634
|
+
}
|
|
635
|
+
}
|
|
611
636
|
return parts.join("\n");
|
|
612
637
|
}
|
|
613
638
|
const DEFAULT_REDACT_PATTERNS = ["Bearer\\s+[A-Za-z0-9._~+\\/=-]+", "(?:api[_-]?key|token|password|secret)\\s*[=:]\\s*[\"']?[^\\s,\"';}]+"];
|
|
@@ -635,7 +660,7 @@ function sanitizeVerifierText(text, maxChars, patterns = DEFAULT_REDACT_PATTERNS
|
|
|
635
660
|
return redacted.length <= maxChars ? redacted : redacted.slice(0, maxChars) + "\n[Truncated " + (redacted.length - maxChars) + " characters]";
|
|
636
661
|
}
|
|
637
662
|
async function extractSession(agent, loadImage, options = {}) {
|
|
638
|
-
const all = agent.session
|
|
663
|
+
const all = sessionEvents(agent.session);
|
|
639
664
|
const from = options.fromSeq ?? 0;
|
|
640
665
|
const to = options.toSeq ?? Number.MAX_SAFE_INTEGER;
|
|
641
666
|
const events = all.filter((event) => event.seq >= from && event.seq <= to);
|
|
@@ -643,24 +668,35 @@ async function extractSession(agent, loadImage, options = {}) {
|
|
|
643
668
|
let problem = "";
|
|
644
669
|
const trace = [];
|
|
645
670
|
const images = [];
|
|
646
|
-
for (const
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
671
|
+
for (const rawEvent of events) {
|
|
672
|
+
const event = rawEvent;
|
|
673
|
+
if (event.type === "user/message") {
|
|
674
|
+
const sourceKind = event.data?.source?.kind;
|
|
675
|
+
if (sourceKind !== "user" && sourceKind !== "team-message") continue;
|
|
676
|
+
const text = textOf(event.data.content);
|
|
677
|
+
if (!problem && sourceKind === "user" && text.trim()) problem = text.trim();
|
|
678
|
+
for (const block of event.data.content) if (block.type === "image") images.push(await loadImage(block.attachment));
|
|
679
|
+
const tag = sourceKind === "team-message" ? "Team Message" : "User";
|
|
680
|
+
trace.push("--- " + tag + " seq " + event.seq + " ---\n" + text);
|
|
681
|
+
} else if (event.type === "assistant/message" && options.includeAssistantText !== false) trace.push("--- Assistant turn " + event.data.turn + " step " + event.data.step + " ---\n" + textOf(event.data.message.content));
|
|
682
|
+
else if (event.type === "tool/call") trace.push("--- Tool Call turn " + event.data.turn + " step " + event.data.step + " ---\n[Command] " + event.data.name + " " + event.data.arguments);
|
|
683
|
+
else if (event.type === "tool/result") trace.push("--- Tool Result turn " + event.data.turn + " step " + event.data.step + " ---\n[Output] " + textOf(event.data.message.content));
|
|
684
|
+
else if (event.type === "tool/ptc-dispatch" || event.type === "tool/code-dispatch") {
|
|
685
|
+
const data = event.data;
|
|
686
|
+
const status = data.isError ? " [Error]" : "";
|
|
687
|
+
const args = data.arguments !== void 0 ? " " + (typeof data.arguments === "string" ? data.arguments : JSON.stringify(data.arguments)) : "";
|
|
688
|
+
const content = Array.isArray(data.content) ? textOf(data.content) : "";
|
|
689
|
+
if (Array.isArray(data.content)) {
|
|
690
|
+
for (const block of data.content) if (block.type === "image") images.push(await loadImage(block.attachment));
|
|
691
|
+
}
|
|
692
|
+
const label = event.type === "tool/ptc-dispatch" ? "PTC Dispatch " : "Code Dispatch ";
|
|
693
|
+
trace.push("--- " + label + data.name + status + " seq " + event.seq + " ---\n[Command] " + data.name + args + "\n[Output] " + content);
|
|
694
|
+
} else if (event.type === "team/message/queued") {
|
|
695
|
+
const data = event.data;
|
|
696
|
+
const sender = data?.message?.senderName ?? "Teammate";
|
|
697
|
+
const content = Array.isArray(data?.message?.content) ? textOf(data.message.content) : "";
|
|
698
|
+
trace.push("--- Team Message Queued from " + sender + " seq " + event.seq + " ---\n" + content);
|
|
662
699
|
}
|
|
663
|
-
trace.push("--- Code Dispatch " + data.name + status + " seq " + event.seq + " ---\n[Command] " + data.name + args + "\n[Output] " + content);
|
|
664
700
|
}
|
|
665
701
|
const raw = redactText(trace.join("\n\n"), patterns);
|
|
666
702
|
const maxChars = options.maxChars ?? 2e5;
|
|
@@ -726,6 +762,8 @@ function isConsequential(name) {
|
|
|
726
762
|
if (PASSIVE_TOOLS.has(name) || VERIFIER_TOOLS.has(name)) return false;
|
|
727
763
|
return /(?:edit|write|patch|apply|deploy|upload|delete|remove|kill|exec|shell|command|migration|database|tunnel|cluster)/iu.test(name);
|
|
728
764
|
}
|
|
765
|
+
/** Event names that carry one settled PTC/code dispatch, across the hosts the plugin supports. */
|
|
766
|
+
const CODE_DISPATCH_TYPES = /* @__PURE__ */ new Set(["tool/code-dispatch", "tool/ptc-dispatch"]);
|
|
729
767
|
function isSuccessfulCodeDispatch(data) {
|
|
730
768
|
if (data.isError === true) return false;
|
|
731
769
|
if (Array.isArray(data.content) && data.content.some((b) => b.isError === true)) return false;
|
|
@@ -746,7 +784,7 @@ function analyzeAutoTask(events, policy) {
|
|
|
746
784
|
const calls = relevant.filter((event) => event.type === "tool/call");
|
|
747
785
|
const successfulResults = new Set(relevant.filter((event) => event.type === "tool/result" && event.data.error === void 0 && event.data.message.content.every((block) => block.isError !== true)).map((event) => String(event.data.message.source.callId)));
|
|
748
786
|
const pairedCalls = calls.filter((event) => successfulResults.has(String(event.data.callId)));
|
|
749
|
-
const codeDispatches = relevant.filter((event) => event.type
|
|
787
|
+
const codeDispatches = relevant.filter((event) => CODE_DISPATCH_TYPES.has(event.type)).map((event) => event.data);
|
|
750
788
|
const successfulDispatches = codeDispatches.filter(isSuccessfulCodeDispatch);
|
|
751
789
|
const toolCalls = calls.filter((event) => !VERIFIER_TOOLS.has(event.data.name)).length + codeDispatches.filter((d) => !VERIFIER_TOOLS.has(d.name)).length;
|
|
752
790
|
const completedToolResults = pairedCalls.length + successfulDispatches.length;
|
|
@@ -823,7 +861,7 @@ var AutoVerificationBudget = class {
|
|
|
823
861
|
state.taskAttempts = 0;
|
|
824
862
|
state.lastEvaluatedSeq = -1;
|
|
825
863
|
}
|
|
826
|
-
const lastSeq = agent.session.
|
|
864
|
+
const lastSeq = sessionEvents(agent.session).at(-1)?.seq ?? -1;
|
|
827
865
|
if (lastSeq <= state.lastEvaluatedSeq || state.taskAttempts >= policy.maxPerTask || state.sessionAttempts >= policy.maxPerSession) {
|
|
828
866
|
this.states.set(id, state);
|
|
829
867
|
return false;
|
|
@@ -895,21 +933,32 @@ function buildEvidenceIndex(events) {
|
|
|
895
933
|
const calls = /* @__PURE__ */ new Map();
|
|
896
934
|
const results = /* @__PURE__ */ new Map();
|
|
897
935
|
const todos = /* @__PURE__ */ new Map();
|
|
936
|
+
const teamTasks = /* @__PURE__ */ new Map();
|
|
898
937
|
const paired = /* @__PURE__ */ new Map();
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
if (
|
|
905
|
-
|
|
906
|
-
const
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
938
|
+
const currentTeamTasks = /* @__PURE__ */ new Map();
|
|
939
|
+
for (const rawEvent of relevant) {
|
|
940
|
+
const event = rawEvent;
|
|
941
|
+
if (event.type === "tool/call") calls.set(String(event.data.callId), rawEvent);
|
|
942
|
+
else if (event.type === "tool/result" && successful(rawEvent)) results.set(String(event.data.message.source.callId), rawEvent);
|
|
943
|
+
else if (event.type === "todo/write") todos.set(event.seq, event.data.todos);
|
|
944
|
+
else if (event.type === "team/task") {
|
|
945
|
+
const data = event.data;
|
|
946
|
+
if (data?.task) {
|
|
947
|
+
currentTeamTasks.set(data.task.id, { ...data.task });
|
|
948
|
+
teamTasks.set(event.seq, [...currentTeamTasks.values()]);
|
|
949
|
+
}
|
|
950
|
+
} else if (event.type === "tool/ptc-dispatch" || event.type === "tool/code-dispatch") {
|
|
951
|
+
const data = event.data;
|
|
952
|
+
if (data.isError !== true && (!Array.isArray(data.content) || data.content.every((b) => b.isError !== true))) {
|
|
953
|
+
const subCallId = String(data.subCallId ?? "code:" + event.seq);
|
|
954
|
+
const content = Array.isArray(data.content) ? data.content : [];
|
|
955
|
+
paired.set(subCallId, {
|
|
956
|
+
name: data.name,
|
|
957
|
+
callSeq: event.seq,
|
|
958
|
+
resultSeq: event.seq,
|
|
959
|
+
text: blockText(content)
|
|
960
|
+
});
|
|
961
|
+
}
|
|
913
962
|
}
|
|
914
963
|
}
|
|
915
964
|
for (const [callId, call] of calls) {
|
|
@@ -924,7 +973,8 @@ function buildEvidenceIndex(events) {
|
|
|
924
973
|
return {
|
|
925
974
|
problemSeq: taskStartSeq,
|
|
926
975
|
calls: paired,
|
|
927
|
-
todos
|
|
976
|
+
todos,
|
|
977
|
+
teamTasks
|
|
928
978
|
};
|
|
929
979
|
}
|
|
930
980
|
function parseTrustedWorkflow(value, callId, callSeq, resultSeq, maxCandidates, maxItemChars) {
|
|
@@ -978,6 +1028,23 @@ function canonicalTodoSnapshots(index) {
|
|
|
978
1028
|
}
|
|
979
1029
|
return values;
|
|
980
1030
|
}
|
|
1031
|
+
function canonicalTeamTaskSnapshots(index) {
|
|
1032
|
+
const values = [];
|
|
1033
|
+
let previous = "";
|
|
1034
|
+
for (const [seq, tasks] of index.teamTasks) {
|
|
1035
|
+
const canonical = JSON.stringify(tasks.map((t) => ({
|
|
1036
|
+
id: t.id,
|
|
1037
|
+
status: t.status,
|
|
1038
|
+
revision: t.revision
|
|
1039
|
+
})));
|
|
1040
|
+
if (canonical !== previous) values.push({
|
|
1041
|
+
seq,
|
|
1042
|
+
tasks
|
|
1043
|
+
});
|
|
1044
|
+
previous = canonical;
|
|
1045
|
+
}
|
|
1046
|
+
return values;
|
|
1047
|
+
}
|
|
981
1048
|
function analyzeStructuredRoute(events, maxCandidates = 8, maxItemChars = 2e4) {
|
|
982
1049
|
const index = buildEvidenceIndex(events);
|
|
983
1050
|
if (!index) return void 0;
|
|
@@ -1031,13 +1098,32 @@ function analyzeStructuredRoute(events, maxCandidates = 8, maxItemChars = 2e4) {
|
|
|
1031
1098
|
evidenceSeqs: snapshots.map((snapshot) => snapshot.seq)
|
|
1032
1099
|
};
|
|
1033
1100
|
}
|
|
1101
|
+
const teamSnapshots = canonicalTeamTaskSnapshots(index);
|
|
1102
|
+
if (teamSnapshots.length >= 2) {
|
|
1103
|
+
const steps = teamSnapshots.map((snapshot) => sanitizeVerifierText("Team task checkpoint seq " + snapshot.seq + ":\n" + snapshot.tasks.map((task) => "- [" + task.status + "] " + task.subject + (task.description ? " (" + task.description + ")" : "")).join("\n"), maxItemChars));
|
|
1104
|
+
const checkpoints = steps.map((_, i) => i + 1);
|
|
1105
|
+
return {
|
|
1106
|
+
kind: "track",
|
|
1107
|
+
source: "structured",
|
|
1108
|
+
confidence: 1,
|
|
1109
|
+
reason: "changed durable team tasks",
|
|
1110
|
+
fingerprint: stableHash({
|
|
1111
|
+
kind: "track",
|
|
1112
|
+
teamSnapshots
|
|
1113
|
+
}),
|
|
1114
|
+
steps,
|
|
1115
|
+
checkpoints,
|
|
1116
|
+
evidenceSeqs: teamSnapshots.map((snapshot) => snapshot.seq)
|
|
1117
|
+
};
|
|
1118
|
+
}
|
|
1034
1119
|
}
|
|
1035
1120
|
}
|
|
1036
1121
|
function semanticRouteHint(events) {
|
|
1037
1122
|
const index = buildEvidenceIndex(events);
|
|
1038
1123
|
if (!index) return false;
|
|
1039
|
-
if ([...index.calls.values()].some((pair) => pair.name === "subagent" || pair.name === "subagent_fork" || pair.name === "workflow")) return true;
|
|
1124
|
+
if ([...index.calls.values()].some((pair) => pair.name === "subagent" || pair.name === "subagent_fork" || pair.name === "workflow" || pair.name === "exit_plan_mode")) return true;
|
|
1040
1125
|
if (canonicalTodoSnapshots(index).length >= 2) return true;
|
|
1126
|
+
if (canonicalTeamTaskSnapshots(index).length >= 2) return true;
|
|
1041
1127
|
return false;
|
|
1042
1128
|
}
|
|
1043
1129
|
function buildSemanticRoutePrompt(problem, events, maxCandidates, maxItemChars = 2e4) {
|
|
@@ -1161,7 +1247,7 @@ var AutoVerifierRouter = class {
|
|
|
1161
1247
|
states = /* @__PURE__ */ new Map();
|
|
1162
1248
|
serial = 0;
|
|
1163
1249
|
state(agent) {
|
|
1164
|
-
const taskStartSeq = latestDirectUserSeq(agent.session
|
|
1250
|
+
const taskStartSeq = latestDirectUserSeq(sessionEvents(agent.session));
|
|
1165
1251
|
if (taskStartSeq === void 0) return void 0;
|
|
1166
1252
|
const id = String(agent.id);
|
|
1167
1253
|
const state = this.states.get(id) ?? {
|
|
@@ -1221,6 +1307,10 @@ var AutoVerifierRouter = class {
|
|
|
1221
1307
|
state.failed.add(reservation.fingerprint);
|
|
1222
1308
|
if (strict) state.strictBlocked = true;
|
|
1223
1309
|
}
|
|
1310
|
+
/** Whether this exact fingerprint already passed within the current task. */
|
|
1311
|
+
completedFingerprint(agent, fingerprint) {
|
|
1312
|
+
return this.state(agent)?.completed.has(fingerprint) ?? false;
|
|
1313
|
+
}
|
|
1224
1314
|
finalRequired(agent) {
|
|
1225
1315
|
return this.state(agent)?.finalRequiredFromSeq;
|
|
1226
1316
|
}
|
|
@@ -1232,6 +1322,120 @@ var AutoVerifierRouter = class {
|
|
|
1232
1322
|
}
|
|
1233
1323
|
};
|
|
1234
1324
|
//#endregion
|
|
1325
|
+
//#region src/plan-gate.ts
|
|
1326
|
+
/**
|
|
1327
|
+
* Read the plan markdown out of one `exit_plan_mode` argument object.
|
|
1328
|
+
* @param args - Parsed tool arguments; anything but `{ plan: string }` yields ''.
|
|
1329
|
+
* @returns Trimmed plan markdown, or '' when the call carries no plan.
|
|
1330
|
+
*/
|
|
1331
|
+
function planFromArguments(args) {
|
|
1332
|
+
if (typeof args !== "object" || args === null) return "";
|
|
1333
|
+
const plan = args.plan;
|
|
1334
|
+
return typeof plan === "string" ? plan.trim() : "";
|
|
1335
|
+
}
|
|
1336
|
+
function buildPlanPreReviewPrompt(problem, planText, maxChars = 2e4) {
|
|
1337
|
+
return [
|
|
1338
|
+
"You are an expert independent technical plan verifier. A candidate implementation plan is about to be submitted to the human for approval.",
|
|
1339
|
+
"Evaluate whether the plan is sound, executable, and comprehensive.",
|
|
1340
|
+
"Scoring criteria:",
|
|
1341
|
+
"A: Flawless, thorough, with clear steps, rigorous verification, edge cases handled.",
|
|
1342
|
+
"B-J: Mostly sound, minor gaps or nuances missing.",
|
|
1343
|
+
"K-S: Noticeable architectural gaps, missing regression checks, or ambiguities.",
|
|
1344
|
+
"T: Fundamentally flawed, dangerous, or missing core requirements.",
|
|
1345
|
+
"",
|
|
1346
|
+
"Task requirement:",
|
|
1347
|
+
sanitizeVerifierText(problem, 4e3),
|
|
1348
|
+
"",
|
|
1349
|
+
"Proposed Plan:",
|
|
1350
|
+
sanitizeVerifierText(planText, maxChars),
|
|
1351
|
+
"",
|
|
1352
|
+
"Output format:",
|
|
1353
|
+
"The first line must be exactly \"Verdict: <single uppercase letter A-T>\" — one letter, nothing else on the line.",
|
|
1354
|
+
"Line 2: Summary: <One sentence assessment>",
|
|
1355
|
+
"Line 3+: Key strengths, blind spots, and verification guidance."
|
|
1356
|
+
].join("\n\n");
|
|
1357
|
+
}
|
|
1358
|
+
/**
|
|
1359
|
+
* Parse the `Verdict: <A-T>` line the judge prompts require.
|
|
1360
|
+
*
|
|
1361
|
+
* Markdown decoration around the required line is stripped first, so "**Verdict:
|
|
1362
|
+
* A**", "- Verdict: B." or a numbered list marker still produce a grade — a
|
|
1363
|
+
* formatting habit must never silently switch the gate off. Prose that is not a
|
|
1364
|
+
* grade is still rejected: "Verdict: Failed" and "Verdict: Approved" carry no
|
|
1365
|
+
* verdict letter, and an answer without a verdict line has no score at all —
|
|
1366
|
+
* callers must skip such a review, never treat it as a pass. A is 1.0 and T is
|
|
1367
|
+
* 0.0, matching the top-logprob A-T scale used by the judge prompts.
|
|
1368
|
+
* @param text - Raw judge model output.
|
|
1369
|
+
* @returns The parsed verdict, or undefined when no verdict line is present.
|
|
1370
|
+
*/
|
|
1371
|
+
function parseVerdictLetter(text) {
|
|
1372
|
+
for (const rawLine of text.split(/\r?\n/)) {
|
|
1373
|
+
const line = rawLine.replace(/[*_`~#>]/g, " ").replace(/^\s*(?:[-+]|\d+[.)])\s+/, " ").trim();
|
|
1374
|
+
const match = /^verdict\s*:\s*([A-Ta-t])\s*[.)]?$/i.exec(line);
|
|
1375
|
+
if (match === null) continue;
|
|
1376
|
+
const verdict = match[1].toUpperCase();
|
|
1377
|
+
return {
|
|
1378
|
+
verdict,
|
|
1379
|
+
score: Math.max(0, Math.min(1, 1 - (verdict.charCodeAt(0) - 65) / 19)),
|
|
1380
|
+
feedback: text.trim()
|
|
1381
|
+
};
|
|
1382
|
+
}
|
|
1383
|
+
}
|
|
1384
|
+
//#endregion
|
|
1385
|
+
//#region src/team-gate.ts
|
|
1386
|
+
function inspectTeamTasks(events, fromSeq = 0) {
|
|
1387
|
+
const taskMap = /* @__PURE__ */ new Map();
|
|
1388
|
+
const completionSeq = /* @__PURE__ */ new Map();
|
|
1389
|
+
for (const rawEvent of events) {
|
|
1390
|
+
const event = rawEvent;
|
|
1391
|
+
if (event.type === "team/task") {
|
|
1392
|
+
const data = event.data;
|
|
1393
|
+
if (data?.task) {
|
|
1394
|
+
taskMap.set(data.task.id, { ...data.task });
|
|
1395
|
+
if (data.task.status === "completed") completionSeq.set(data.task.id, event.seq);
|
|
1396
|
+
else completionSeq.delete(data.task.id);
|
|
1397
|
+
}
|
|
1398
|
+
}
|
|
1399
|
+
}
|
|
1400
|
+
const completedTasks = [...completionSeq.entries()].filter(([id, seq]) => seq >= fromSeq && taskMap.get(id)?.status === "completed").map(([id, seq]) => ({
|
|
1401
|
+
task: { ...taskMap.get(id) },
|
|
1402
|
+
seq
|
|
1403
|
+
})).sort((left, right) => left.seq - right.seq);
|
|
1404
|
+
const latest = completedTasks.at(-1);
|
|
1405
|
+
return {
|
|
1406
|
+
latestCompletedTask: latest?.task,
|
|
1407
|
+
completedSeq: latest?.seq,
|
|
1408
|
+
completedTasks,
|
|
1409
|
+
activeTasks: [...taskMap.values()].filter((t) => t.status !== "deleted"),
|
|
1410
|
+
hasRecentCompletedTask: latest !== void 0
|
|
1411
|
+
};
|
|
1412
|
+
}
|
|
1413
|
+
function buildTeamTaskVerificationPrompt(task, executionTrace, maxChars = 2e4) {
|
|
1414
|
+
return [
|
|
1415
|
+
"You are an expert independent technical verifier reviewing a completed Agent Teams task.",
|
|
1416
|
+
"Verify if the task goal has been satisfied by concrete evidence (code edits, test runs, successful commands).",
|
|
1417
|
+
"Task Details:",
|
|
1418
|
+
"ID: " + task.id,
|
|
1419
|
+
"Subject: " + task.subject,
|
|
1420
|
+
task.description ? "Description: " + task.description : "",
|
|
1421
|
+
"",
|
|
1422
|
+
"Execution Evidence & Trace:",
|
|
1423
|
+
sanitizeVerifierText(executionTrace, maxChars),
|
|
1424
|
+
"",
|
|
1425
|
+
"Evaluate whether the task is genuinely completed with verified proof.",
|
|
1426
|
+
"Scoring criteria:",
|
|
1427
|
+
"A: Completely fulfilled with solid execution/test evidence.",
|
|
1428
|
+
"B-J: Mostly complete, minor gaps or edge cases not fully asserted.",
|
|
1429
|
+
"K-S: Incomplete, missing key requirements or untested.",
|
|
1430
|
+
"T: Unfulfilled, failed, or contradictory evidence.",
|
|
1431
|
+
"",
|
|
1432
|
+
"Output format:",
|
|
1433
|
+
"The first line must be exactly \"Verdict: <single uppercase letter A-T>\" — one letter, nothing else on the line.",
|
|
1434
|
+
"Line 2: Summary: <One sentence evaluation of task fulfillment>",
|
|
1435
|
+
"Line 3+: Remaining gaps or next steps for the team."
|
|
1436
|
+
].filter(Boolean).join("\n\n");
|
|
1437
|
+
}
|
|
1438
|
+
//#endregion
|
|
1235
1439
|
//#region src/statistics.ts
|
|
1236
1440
|
const VERIFIER_TOOL_NAMES = [
|
|
1237
1441
|
"verifier_route_classify",
|
|
@@ -2010,8 +2214,7 @@ function apply(ctx, config = {}) {
|
|
|
2010
2214
|
summary: "Automatic verifier routed " + decision.kind
|
|
2011
2215
|
}
|
|
2012
2216
|
});
|
|
2013
|
-
|
|
2014
|
-
if (endpoint !== "statistics") return rpcFailure("unknown llm-verifier endpoint");
|
|
2217
|
+
const handleStatisticsQuery = async (payload) => {
|
|
2015
2218
|
if (typeof payload !== "object" || payload === null) return rpcFailure("statistics payload must be an object");
|
|
2016
2219
|
const row = payload;
|
|
2017
2220
|
const sessionId = typeof row.sessionId === "string" && row.sessionId.length > 0 ? row.sessionId : void 0;
|
|
@@ -2028,25 +2231,171 @@ function apply(ctx, config = {}) {
|
|
|
2028
2231
|
} catch (error) {
|
|
2029
2232
|
return rpcFailure(error instanceof Error ? error.message : String(error));
|
|
2030
2233
|
}
|
|
2031
|
-
}
|
|
2234
|
+
};
|
|
2235
|
+
const anyConn = services.connection;
|
|
2236
|
+
if (anyConn && anyConn.fetch && typeof anyConn.fetch.register === "function") try {
|
|
2237
|
+
anyConn.fetch.register({
|
|
2238
|
+
path: "/api/llm-verifier/statistics",
|
|
2239
|
+
methods: ["POST"],
|
|
2240
|
+
requestBody: "buffered",
|
|
2241
|
+
fetch: async (request) => {
|
|
2242
|
+
let body;
|
|
2243
|
+
try {
|
|
2244
|
+
body = await request.json();
|
|
2245
|
+
} catch {
|
|
2246
|
+
return new Response("body is not JSON", { status: 400 });
|
|
2247
|
+
}
|
|
2248
|
+
const isRpcEnvelope = typeof body === "object" && body !== null && body.type === "client-request" && typeof body.rpcId === "string";
|
|
2249
|
+
const rpcId = isRpcEnvelope ? body.rpcId : "direct";
|
|
2250
|
+
const payload = isRpcEnvelope ? body.payload : body;
|
|
2251
|
+
const outcome = await handleStatisticsQuery(payload);
|
|
2252
|
+
if (isRpcEnvelope) return Response.json({
|
|
2253
|
+
type: "server-response",
|
|
2254
|
+
rpcId,
|
|
2255
|
+
result: outcome
|
|
2256
|
+
});
|
|
2257
|
+
return Response.json(outcome);
|
|
2258
|
+
}
|
|
2259
|
+
});
|
|
2260
|
+
} catch (e) {
|
|
2261
|
+
ctx.logger.warn("failed to register /api/llm-verifier/statistics fetch route: " + String(e));
|
|
2262
|
+
}
|
|
2263
|
+
const legacyRpc = services.connection;
|
|
2264
|
+
if (typeof legacyRpc.rpc?.handle === "function") try {
|
|
2265
|
+
legacyRpc.rpc.handle("/llm-verifier", async (endpoint, payload) => {
|
|
2266
|
+
if (endpoint !== "statistics") return rpcFailure("unknown llm-verifier endpoint");
|
|
2267
|
+
return handleStatisticsQuery(payload);
|
|
2268
|
+
});
|
|
2269
|
+
} catch (e) {
|
|
2270
|
+
ctx.logger.warn("failed to register /llm-verifier rpc fallback: " + String(e));
|
|
2271
|
+
}
|
|
2032
2272
|
ctx.on("agent/disposed", ({ agent }) => {
|
|
2033
2273
|
autoBudget.release(agent);
|
|
2034
2274
|
autoRouter.release(agent);
|
|
2035
2275
|
topics.delete(String(agent.id));
|
|
2036
2276
|
});
|
|
2277
|
+
ctx.on("tools/pre-execute", async (exec, next) => {
|
|
2278
|
+
const selected = current();
|
|
2279
|
+
if (!selected.enabled || selected.autoVerifyMode === "manual" || !selected.autoVerifyPlanMode) return next();
|
|
2280
|
+
if (exec.name !== "exit_plan_mode" || exec.agent === void 0 || exec.signal.aborted) return next();
|
|
2281
|
+
const plan = planFromArguments(exec.arguments);
|
|
2282
|
+
if (!plan) return next();
|
|
2283
|
+
const agent = exec.agent;
|
|
2284
|
+
const policy = routePolicy(selected);
|
|
2285
|
+
const evidence = analyzeAutoTask(sessionEvents(agent.session), {
|
|
2286
|
+
mode: selected.autoVerifyMode,
|
|
2287
|
+
minToolCalls: selected.autoVerifyMinToolCalls,
|
|
2288
|
+
maxPerTask: selected.autoVerifyMaxPerTask,
|
|
2289
|
+
maxPerSession: selected.autoVerifyMaxPerSession
|
|
2290
|
+
});
|
|
2291
|
+
const planFingerprint = stableHash({
|
|
2292
|
+
phase: "plan_review",
|
|
2293
|
+
plan
|
|
2294
|
+
});
|
|
2295
|
+
const reservation = autoRouter.reserve(agent, "plan_review", planFingerprint, 1, policy);
|
|
2296
|
+
if (reservation === void 0) {
|
|
2297
|
+
if (!autoRouter.completedFingerprint(agent, planFingerprint)) ctx.logger.warn("llm-verifier plan pre-review skipped (verification in flight or auto-verification budget exhausted)");
|
|
2298
|
+
return next();
|
|
2299
|
+
}
|
|
2300
|
+
try {
|
|
2301
|
+
const toSeq = sessionEvents(agent.session).at(-1)?.seq ?? -1;
|
|
2302
|
+
const extracted = await extractTask(agent, evidence.taskStartSeq, toSeq, selected.autoVerifyMaxChars, exec.signal);
|
|
2303
|
+
const verdict = parseVerdictLetter((await classifyRoute(agent, buildPlanPreReviewPrompt(extracted.problem, plan, selected.autoRouteMaxInputChars), exec.signal)).text);
|
|
2304
|
+
if (verdict === void 0) {
|
|
2305
|
+
autoRouter.fail(agent, reservation, false);
|
|
2306
|
+
ctx.logger.warn("llm-verifier plan pre-review produced no verdict line; the plan was allowed through");
|
|
2307
|
+
return next();
|
|
2308
|
+
}
|
|
2309
|
+
if (verdict.score >= selected.autoVerifyThreshold) {
|
|
2310
|
+
autoRouter.commit(agent, reservation);
|
|
2311
|
+
return next();
|
|
2312
|
+
}
|
|
2313
|
+
autoRouter.fail(agent, reservation, selected.autoVerifyMode === "strict");
|
|
2314
|
+
return {
|
|
2315
|
+
kind: "deny",
|
|
2316
|
+
reason: "[Automatic Verifier Plan Pre-review] scored " + (verdict.score * 100).toFixed(1) + "% against a " + (selected.autoVerifyThreshold * 100).toFixed(0) + "% threshold.\n" + verdict.feedback.slice(0, 4e3) + "\nRevise the plan to address these findings, then call exit_plan_mode again."
|
|
2317
|
+
};
|
|
2318
|
+
} catch (error) {
|
|
2319
|
+
autoRouter.fail(agent, reservation, false);
|
|
2320
|
+
ctx.logger.warn("llm-verifier plan pre-review failed: " + (error instanceof Error ? error.message : String(error)));
|
|
2321
|
+
return next();
|
|
2322
|
+
}
|
|
2323
|
+
});
|
|
2037
2324
|
ctx.on("agent/turn-stopping", async ({ agent, signal }) => {
|
|
2038
2325
|
const selected = current();
|
|
2039
2326
|
if (!selected.enabled || selected.autoVerifyMode === "manual" || signal.aborted) return;
|
|
2040
2327
|
const policy = routePolicy(selected);
|
|
2041
|
-
const evidence = analyzeAutoTask(agent.session
|
|
2328
|
+
const evidence = analyzeAutoTask(sessionEvents(agent.session), {
|
|
2042
2329
|
mode: selected.autoVerifyMode,
|
|
2043
2330
|
minToolCalls: selected.autoVerifyMinToolCalls,
|
|
2044
2331
|
maxPerTask: selected.autoVerifyMaxPerTask,
|
|
2045
2332
|
maxPerSession: selected.autoVerifyMaxPerSession
|
|
2046
2333
|
});
|
|
2047
|
-
const admittedLastSeq = agent.session.
|
|
2048
|
-
const snapshot = agent.session.
|
|
2049
|
-
const stillCurrent = () => !signal.aborted && (agent.session.
|
|
2334
|
+
const admittedLastSeq = sessionEvents(agent.session).at(-1)?.seq ?? -1;
|
|
2335
|
+
const snapshot = sessionEvents(agent.session).filter((event) => event.seq <= admittedLastSeq);
|
|
2336
|
+
const stillCurrent = () => !signal.aborted && (sessionEvents(agent.session).at(-1)?.seq ?? -1) === admittedLastSeq;
|
|
2337
|
+
if (selected.autoVerifyTeamTasks) {
|
|
2338
|
+
const teamInspection = inspectTeamTasks(snapshot, evidence.taskStartSeq);
|
|
2339
|
+
for (const completed of teamInspection.completedTasks) {
|
|
2340
|
+
const task = completed.task;
|
|
2341
|
+
const taskReservation = autoRouter.reserve(agent, "team_task", stableHash({
|
|
2342
|
+
phase: "team_task",
|
|
2343
|
+
taskId: task.id,
|
|
2344
|
+
status: task.status,
|
|
2345
|
+
seq: completed.seq
|
|
2346
|
+
}), 1, policy);
|
|
2347
|
+
if (taskReservation === void 0) continue;
|
|
2348
|
+
try {
|
|
2349
|
+
const prompt = buildTeamTaskVerificationPrompt(task, (await extractTask(agent, evidence.taskStartSeq, admittedLastSeq, selected.autoVerifyMaxChars, signal)).trace, selected.autoRouteMaxInputChars);
|
|
2350
|
+
const classified = await classifyRoute(agent, prompt, signal);
|
|
2351
|
+
if (!stillCurrent()) {
|
|
2352
|
+
autoRouter.fail(agent, taskReservation, false);
|
|
2353
|
+
return;
|
|
2354
|
+
}
|
|
2355
|
+
const verdict = parseVerdictLetter(classified.text);
|
|
2356
|
+
if (verdict === void 0) {
|
|
2357
|
+
autoRouter.fail(agent, taskReservation, false);
|
|
2358
|
+
ctx.logger.warn("llm-verifier team task verification produced no verdict line; task " + task.id + " was not gated");
|
|
2359
|
+
continue;
|
|
2360
|
+
}
|
|
2361
|
+
if (verdict.score >= selected.autoVerifyThreshold) {
|
|
2362
|
+
autoRouter.commit(agent, taskReservation, admittedLastSeq);
|
|
2363
|
+
continue;
|
|
2364
|
+
}
|
|
2365
|
+
autoRouter.fail(agent, taskReservation, selected.autoVerifyMode === "strict");
|
|
2366
|
+
agent.steer(createUserMessage({
|
|
2367
|
+
content: [{
|
|
2368
|
+
type: "text",
|
|
2369
|
+
text: "[Automatic Verifier Team Task Gate]\nTask \"" + task.subject + "\" (#" + task.id + ") verification scored " + (verdict.score * 100).toFixed(1) + "% (Threshold: " + (selected.autoVerifyThreshold * 100).toFixed(0) + "%).\n" + verdict.feedback + "\nProvide verified execution evidence or resolve remaining issues before completing the task."
|
|
2370
|
+
}],
|
|
2371
|
+
source: {
|
|
2372
|
+
kind: "plugin",
|
|
2373
|
+
plugin: "dsh-llm-verifier",
|
|
2374
|
+
form: "notice",
|
|
2375
|
+
summary: "Team Task Gate Feedback"
|
|
2376
|
+
}
|
|
2377
|
+
}));
|
|
2378
|
+
return;
|
|
2379
|
+
} catch (error) {
|
|
2380
|
+
autoRouter.fail(agent, taskReservation, selected.autoVerifyMode === "strict");
|
|
2381
|
+
ctx.logger.warn("llm-verifier team task verification failed: " + (error instanceof Error ? error.message : String(error)));
|
|
2382
|
+
if (selected.autoVerifyMode === "strict" && !signal.aborted) {
|
|
2383
|
+
agent.steer(createUserMessage({
|
|
2384
|
+
content: [{
|
|
2385
|
+
type: "text",
|
|
2386
|
+
text: "[Automatic Verifier Team Task Gate]\nTask verification failed: " + (error instanceof Error ? error.message : String(error))
|
|
2387
|
+
}],
|
|
2388
|
+
source: {
|
|
2389
|
+
kind: "plugin",
|
|
2390
|
+
plugin: "dsh-llm-verifier"
|
|
2391
|
+
}
|
|
2392
|
+
}));
|
|
2393
|
+
return;
|
|
2394
|
+
}
|
|
2395
|
+
continue;
|
|
2396
|
+
}
|
|
2397
|
+
}
|
|
2398
|
+
}
|
|
2050
2399
|
let decision = boundDecision(analyzeStructuredRoute(snapshot, selected.autoRouteMaxCandidates, selected.autoRouteMaxItemChars), policy);
|
|
2051
2400
|
if (decision === void 0 && selected.autoRouteSemantic && (selected.autoVerifyMode === "strict" || semanticRouteHint(snapshot))) {
|
|
2052
2401
|
const fingerprint = stableHash({
|
|
@@ -2551,6 +2900,6 @@ function apply(ctx, config = {}) {
|
|
|
2551
2900
|
}));
|
|
2552
2901
|
}
|
|
2553
2902
|
//#endregion
|
|
2554
|
-
export { AutoVerificationBudget, AutoVerifierRouter, Config, DEFAULT_CRITERIA, DEFAULT_GROUND_TRUTH_NOTE, GRANULARITY, LETTERS, RequestLimiter, SCALE_DESCRIPTION, ScoreCache, SingleFlight, StatisticsStore, VERIFIER_TOOL_NAMES, VerifierEngine, accumulatePairs, analyzeAutoTask, analyzeStructuredRoute, apply, automaticFeedback, boundDecision, bradleyTerry, buildEvidenceIndex, buildPairwisePrompt, buildProgressPrompt, buildSemanticRoutePrompt, callVerifier, emptyRunStats, errorDetails, extractProgressScore, extractScore, inject, latestDirectUserSeq, mergeStatisticsOverviews, name, normalizeCriteria, normalizeScoreLetter, parseSemanticRoute, pivotRoundPairs, rankScores, resolveCacheFile, resolveStatisticsFile, resolveTopicDataDir, ringCycle, seededRandom, semanticDecision, semanticRouteHint, stableHash, topPivots };
|
|
2903
|
+
export { AutoVerificationBudget, AutoVerifierRouter, Config, DEFAULT_CRITERIA, DEFAULT_GROUND_TRUTH_NOTE, GRANULARITY, LETTERS, RequestLimiter, SCALE_DESCRIPTION, ScoreCache, SingleFlight, StatisticsStore, VERIFIER_TOOL_NAMES, VerifierEngine, accumulatePairs, analyzeAutoTask, analyzeStructuredRoute, apply, automaticFeedback, boundDecision, bradleyTerry, buildEvidenceIndex, buildPairwisePrompt, buildPlanPreReviewPrompt, buildProgressPrompt, buildSemanticRoutePrompt, buildTeamTaskVerificationPrompt, callVerifier, emptyRunStats, errorDetails, extractProgressScore, extractScore, inject, inspectTeamTasks, latestDirectUserSeq, mergeStatisticsOverviews, name, normalizeCriteria, normalizeScoreLetter, parseSemanticRoute, parseVerdictLetter, pivotRoundPairs, planFromArguments, rankScores, resolveCacheFile, resolveStatisticsFile, resolveTopicDataDir, ringCycle, seededRandom, semanticDecision, semanticRouteHint, stableHash, topPivots };
|
|
2555
2904
|
|
|
2556
2905
|
//# sourceMappingURL=index.js.map
|